1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
|
From dfba7de59a41bc22786d87f53b20deea14240713 Mon Sep 17 00:00:00 2001
Message-Id: <dfba7de59a41bc22786d87f53b20deea14240713.1627390028.git.maciej.zenon.borzecki@canonical.com>
From: Maciej Borzecki <maciej.zenon.borzecki@canonical.com>
Date: Tue, 27 Jul 2021 14:13:57 +0200
Subject: [PATCH] snap/squashfs: handle squashfs-tools 4.5+
Arch recently landed an update of squahfs-tools to 4.5 which broke our code. The
main difference is that the output of unsquahfs stdout does not contain the
header we expected. For instance:
google:ubuntu-20.04-64 .../mini/hello# unsquashfs -n -dest . -ll /root/foo.snap
Parallel unsquashfs: Using 1 processor
5 inodes (1 blocks) to write
drwx------ root/root 55 2021-07-27 11:31 .
-rw-r--r-- root/root 0 2021-07-27 11:31 ./data.bin
drwxr-xr-x root/root 27 2021-07-27 11:31 ./food
drwxr-xr-x root/root 27 2021-07-27 11:31 ./food/bard
drwxr-xr-x root/root 3 2021-07-27 11:31 ./food/bard/bazd
drwxr-xr-x root/root 45 2021-07-27 11:31 ./meta
drwxr-xr-x root/root 58 2021-07-27 11:31 ./meta/hooks
-rwxr-xr-x root/root 0 2021-07-27 11:31 ./meta/hooks/bar-hook
drwxr-xr-x root/root 26 2021-07-27 11:31 ./meta/hooks/dir
-rwxr-xr-x root/root 0 2021-07-27 11:31 ./meta/hooks/dir/baz
-rwxr-xr-x root/root 0 2021-07-27 11:31 ./meta/hooks/foo-hook
-rw-r--r-- root/root 9 2021-07-27 11:31 ./meta/snap.yaml
While on Arch with squashfs 4.5:
$ unsquashfs -n -dest . -ll /tmp/check-1302223697476122084/0/foo.snap
drwx------ root/root 55 2021-07-27 13:31 .
-rw-r--r-- root/root 0 2021-07-27 13:31 ./data.bin
drwxr-xr-x root/root 27 2021-07-27 13:31 ./food
drwxr-xr-x root/root 27 2021-07-27 13:31 ./food/bard
drwxr-xr-x root/root 3 2021-07-27 13:31 ./food/bard/bazd
drwxr-xr-x root/root 45 2021-07-27 13:31 ./meta
drwxr-xr-x root/root 58 2021-07-27 13:31 ./meta/hooks
-rwxr-xr-x root/root 0 2021-07-27 13:31 ./meta/hooks/bar-hook
drwxr-xr-x root/root 26 2021-07-27 13:31 ./meta/hooks/dir
-rwxr-xr-x root/root 0 2021-07-27 13:31 ./meta/hooks/dir/baz
-rwxr-xr-x root/root 0 2021-07-27 13:31 ./meta/hooks/foo-hook
-rw-r--r-- root/root 9 2021-07-27 13:31 ./meta/snap.yaml
Signed-off-by: Maciej Borzecki <maciej.zenon.borzecki@canonical.com>
---
snap/squashfs/squashfs.go | 27 ++++++---
snap/squashfs/squashfs_test.go | 101 ++++++++++++++++++++++++++++++---
2 files changed, 113 insertions(+), 15 deletions(-)
diff --git a/snap/squashfs/squashfs.go b/snap/squashfs/squashfs.go
index c656de953546f7bda4eadd95d269cc966cb4e28b..6cc9451fa9c7f8ab0f05546000404b276aa70ae4 100644
--- a/snap/squashfs/squashfs.go
+++ b/snap/squashfs/squashfs.go
@@ -294,6 +294,12 @@ func (sk skipper) Has(path string) bool {
return false
}
+// pre-4.5 unsquashfs writes a funny header like:
+// "Parallel unsquashfs: Using 1 processor"
+// "1 inodes (1 blocks) to write"
+// "" <-- empty line
+var maybeHeaderRegex = regexp.MustCompile(`^(Parallel unsquashfs: Using .* processor.*|[0-9]+ inodes .* to write)$`)
+
// Walk (part of snap.Container) is like filepath.Walk, without the ordering guarantee.
func (s *Snap) Walk(relative string, walkFn filepath.WalkFunc) error {
relative = filepath.Clean(relative)
@@ -321,16 +327,21 @@ func (s *Snap) Walk(relative string, walkFn filepath.WalkFunc) error {
defer cmd.Process.Kill()
scanner := bufio.NewScanner(stdout)
- // skip the header
- for scanner.Scan() {
- if len(scanner.Bytes()) == 0 {
- break
- }
- }
-
skipper := make(skipper)
+ seenHeader := false
for scanner.Scan() {
- st, err := fromRaw(scanner.Bytes())
+ raw := scanner.Bytes()
+ if !seenHeader {
+ // try to match the header written by older (pre-4.5)
+ // squashfs tools
+ if len(scanner.Bytes()) == 0 ||
+ maybeHeaderRegex.Match(raw) {
+ continue
+ } else {
+ seenHeader = true
+ }
+ }
+ st, err := fromRaw(raw)
if err != nil {
err = walkFn(relative, nil, err)
if err != nil {
diff --git a/snap/squashfs/squashfs_test.go b/snap/squashfs/squashfs_test.go
index f510a4466dcb1a1b7238a9365e0ce0347852df77..5618e5cc0e4f84dd05f1ea547c7c8e048c9bd796 100644
--- a/snap/squashfs/squashfs_test.go
+++ b/snap/squashfs/squashfs_test.go
@@ -20,6 +20,7 @@
package squashfs_test
import (
+ "bytes"
"errors"
"fmt"
"io/ioutil"
@@ -348,7 +349,7 @@ func (s *SquashfsTestSuite) TestListDir(c *C) {
c.Check(fileNames[2], Equals, "foo-hook")
}
-func (s *SquashfsTestSuite) TestWalk(c *C) {
+func (s *SquashfsTestSuite) TestWalkNative(c *C) {
sub := "."
sn := makeSnap(c, "name: foo", "")
sqw := map[string]os.FileInfo{}
@@ -411,6 +412,86 @@ func (s *SquashfsTestSuite) TestWalk(c *C) {
}
+func (s *SquashfsTestSuite) testWalkMockedUnsquashfs(c *C) {
+ expectingNames := []string{
+ ".",
+ "data.bin",
+ "food",
+ "meta",
+ "meta/hooks",
+ "meta/hooks/bar-hook",
+ "meta/hooks/dir",
+ "meta/hooks/dir/baz",
+ "meta/hooks/foo-hook",
+ "meta/snap.yaml",
+ }
+ sub := "."
+ sn := makeSnap(c, "name: foo", "")
+ var seen []string
+ sn.Walk(sub, func(path string, info os.FileInfo, err error) error {
+ c.Logf("got %v", path)
+ if err != nil {
+ return err
+ }
+ seen = append(seen, path)
+ if path == "food" {
+ return filepath.SkipDir
+ }
+ return nil
+ })
+ c.Assert(len(seen), Equals, len(expectingNames))
+ for idx, name := range seen {
+ c.Check(name, Equals, expectingNames[idx])
+ }
+}
+
+func (s *SquashfsTestSuite) TestWalkMockedUnsquashfs45(c *C) {
+ // mock behavior of squashfs-tools 4.5 and later
+ mockUnsquashfs := testutil.MockCommand(c, "unsquashfs", `
+cat <<EOF
+drwx------ root/root 55 2021-07-27 13:31 .
+-rw-r--r-- root/root 0 2021-07-27 13:31 ./data.bin
+drwxr-xr-x root/root 27 2021-07-27 13:31 ./food
+drwxr-xr-x root/root 27 2021-07-27 13:31 ./food/bard
+drwxr-xr-x root/root 3 2021-07-27 13:31 ./food/bard/bazd
+drwxr-xr-x root/root 45 2021-07-27 13:31 ./meta
+drwxr-xr-x root/root 58 2021-07-27 13:31 ./meta/hooks
+-rwxr-xr-x root/root 0 2021-07-27 13:31 ./meta/hooks/bar-hook
+drwxr-xr-x root/root 26 2021-07-27 13:31 ./meta/hooks/dir
+-rwxr-xr-x root/root 0 2021-07-27 13:31 ./meta/hooks/dir/baz
+-rwxr-xr-x root/root 0 2021-07-27 13:31 ./meta/hooks/foo-hook
+-rw-r--r-- root/root 9 2021-07-27 13:31 ./meta/snap.yaml
+EOF
+`)
+ defer mockUnsquashfs.Restore()
+ s.testWalkMockedUnsquashfs(c)
+}
+
+func (s *SquashfsTestSuite) TestWalkMockedUnsquashfsOld(c *C) {
+ // mock behavior of pre-4.5 squashfs-tools
+ mockUnsquashfs := testutil.MockCommand(c, "unsquashfs", `
+cat <<EOF
+Parallel unsquashfs: Using 1 processor
+5 inodes (1 blocks) to write
+
+drwx------ root/root 55 2021-07-27 13:31 .
+-rw-r--r-- root/root 0 2021-07-27 13:31 ./data.bin
+drwxr-xr-x root/root 27 2021-07-27 13:31 ./food
+drwxr-xr-x root/root 27 2021-07-27 13:31 ./food/bard
+drwxr-xr-x root/root 3 2021-07-27 13:31 ./food/bard/bazd
+drwxr-xr-x root/root 45 2021-07-27 13:31 ./meta
+drwxr-xr-x root/root 58 2021-07-27 13:31 ./meta/hooks
+-rwxr-xr-x root/root 0 2021-07-27 13:31 ./meta/hooks/bar-hook
+drwxr-xr-x root/root 26 2021-07-27 13:31 ./meta/hooks/dir
+-rwxr-xr-x root/root 0 2021-07-27 13:31 ./meta/hooks/dir/baz
+-rwxr-xr-x root/root 0 2021-07-27 13:31 ./meta/hooks/foo-hook
+-rw-r--r-- root/root 9 2021-07-27 13:31 ./meta/snap.yaml
+EOF
+`)
+ defer mockUnsquashfs.Restore()
+ s.testWalkMockedUnsquashfs(c)
+}
+
// TestUnpackGlob tests the internal unpack
func (s *SquashfsTestSuite) TestUnpackGlob(c *C) {
data := "some random data"
@@ -478,7 +559,7 @@ EOF
c.Check(err.Error(), Equals, `cannot extract "*" to "some-output-dir": failed: "Failed to write /tmp/1/modules/4.4.0-112-generic/modules.symbols, skipping", "Write on output file failed because No space left on device", "writer: failed to write data block 0", "Failed to write /tmp/1/modules/4.4.0-112-generic/modules.symbols.bin, skipping", and 15 more`)
}
-func (s *SquashfsTestSuite) TestBuild(c *C) {
+func (s *SquashfsTestSuite) TestBuildAll(c *C) {
// please keep TestBuildUsesExcludes in sync with this one so it makes sense.
buildDir := c.MkDir()
err := os.MkdirAll(filepath.Join(buildDir, "/random/dir"), 0755)
@@ -492,13 +573,16 @@ func (s *SquashfsTestSuite) TestBuild(c *C) {
err = sn.Build(buildDir, &squashfs.BuildOpts{SnapType: "app"})
c.Assert(err, IsNil)
- // unsquashfs writes a funny header like:
+ // pre-4.5 unsquashfs writes a funny header like:
// "Parallel unsquashfs: Using 1 processor"
// "1 inodes (1 blocks) to write"
outputWithHeader, err := exec.Command("unsquashfs", "-n", "-l", sn.Path()).Output()
c.Assert(err, IsNil)
- split := strings.Split(string(outputWithHeader), "\n")
- output := strings.Join(split[3:], "\n")
+ output := outputWithHeader
+ if bytes.HasPrefix(outputWithHeader, []byte(`Parallel unsquashfs: `)) {
+ split := bytes.Split(outputWithHeader, []byte("\n"))
+ output = bytes.Join(split[3:], []byte("\n"))
+ }
c.Assert(string(output), Equals, `
squashfs-root
squashfs-root/data.bin
@@ -538,8 +622,11 @@ data.bin
outputWithHeader, err := exec.Command("unsquashfs", "-n", "-l", sn.Path()).Output()
c.Assert(err, IsNil)
- split := strings.Split(string(outputWithHeader), "\n")
- output := strings.Join(split[3:], "\n")
+ output := outputWithHeader
+ if bytes.HasPrefix(outputWithHeader, []byte(`Parallel unsquashfs: `)) {
+ split := bytes.Split(outputWithHeader, []byte("\n"))
+ output = bytes.Join(split[3:], []byte("\n"))
+ }
// compare with TestBuild
c.Assert(string(output), Equals, `
squashfs-root
--
2.32.0
|