From f388a77afb31f25c387b81f60d17da1660390ca5 Mon Sep 17 00:00:00 2001 From: Giuseppe Scrivano Date: Tue, 9 Apr 2024 12:01:02 +0200 Subject: [PATCH] chunked: fix unmarshaling of file names The getString() function was used to extract string values, but it doesn't handle escaped characters. Replace it with iter.ReadString() that is slower but handles escaped characters correctly. Closes: https://github.com/containers/storage/issues/1878 Signed-off-by: Giuseppe Scrivano --- pkg/chunked/cache_linux.go | 58 +++++------------------------ pkg/chunked/cache_linux_test.go | 24 +++++++++++- pkg/chunked/internal/compression.go | 3 -- 3 files changed, 32 insertions(+), 53 deletions(-) diff --git a/pkg/chunked/cache_linux.go b/pkg/chunked/cache_linux.go index 24baf72e0e..3393af6685 100644 --- a/pkg/chunked/cache_linux.go +++ b/pkg/chunked/cache_linux.go @@ -652,45 +652,9 @@ func (c *layersCache) findChunkInOtherLayers(chunk *internal.FileMetadata) (stri } func unmarshalToc(manifest []byte) (*internal.TOC, error) { - var buf bytes.Buffer - count := 0 var toc internal.TOC iter := jsoniter.ParseBytes(jsoniter.ConfigFastest, manifest) - for field := iter.ReadObject(); field != ""; field = iter.ReadObject() { - if strings.ToLower(field) != "entries" { - iter.Skip() - continue - } - for iter.ReadArray() { - for field := iter.ReadObject(); field != ""; field = iter.ReadObject() { - switch strings.ToLower(field) { - case "type", "name", "linkname", "digest", "chunkdigest", "chunktype", "modtime", "accesstime", "changetime": - count += len(iter.ReadStringAsSlice()) - case "xattrs": - for key := iter.ReadObject(); key != ""; key = iter.ReadObject() { - count += len(iter.ReadStringAsSlice()) - } - default: - iter.Skip() - } - } - } - break - } - - buf.Grow(count) - - getString := func(b []byte) string { - from := buf.Len() - buf.Write(b) - to := buf.Len() - return byteSliceAsString(buf.Bytes()[from:to]) - } - - pool := iter.Pool() - pool.ReturnIterator(iter) - iter = pool.BorrowIterator(manifest) for field := iter.ReadObject(); field != ""; field = iter.ReadObject() { if strings.ToLower(field) == "version" { @@ -706,11 +670,11 @@ func unmarshalToc(manifest []byte) (*internal.TOC, error) { for field := iter.ReadObject(); field != ""; field = iter.ReadObject() { switch strings.ToLower(field) { case "type": - m.Type = getString(iter.ReadStringAsSlice()) + m.Type = iter.ReadString() case "name": - m.Name = getString(iter.ReadStringAsSlice()) + m.Name = iter.ReadString() case "linkname": - m.Linkname = getString(iter.ReadStringAsSlice()) + m.Linkname = iter.ReadString() case "mode": m.Mode = iter.ReadInt64() case "size": @@ -720,19 +684,19 @@ func unmarshalToc(manifest []byte) (*internal.TOC, error) { case "gid": m.GID = iter.ReadInt() case "modtime": - time, err := time.Parse(time.RFC3339, byteSliceAsString(iter.ReadStringAsSlice())) + time, err := time.Parse(time.RFC3339, iter.ReadString()) if err != nil { return nil, err } m.ModTime = &time case "accesstime": - time, err := time.Parse(time.RFC3339, byteSliceAsString(iter.ReadStringAsSlice())) + time, err := time.Parse(time.RFC3339, iter.ReadString()) if err != nil { return nil, err } m.AccessTime = &time case "changetime": - time, err := time.Parse(time.RFC3339, byteSliceAsString(iter.ReadStringAsSlice())) + time, err := time.Parse(time.RFC3339, iter.ReadString()) if err != nil { return nil, err } @@ -742,7 +706,7 @@ func unmarshalToc(manifest []byte) (*internal.TOC, error) { case "devminor": m.Devminor = iter.ReadInt64() case "digest": - m.Digest = getString(iter.ReadStringAsSlice()) + m.Digest = iter.ReadString() case "offset": m.Offset = iter.ReadInt64() case "endoffset": @@ -752,14 +716,13 @@ func unmarshalToc(manifest []byte) (*internal.TOC, error) { case "chunkoffset": m.ChunkOffset = iter.ReadInt64() case "chunkdigest": - m.ChunkDigest = getString(iter.ReadStringAsSlice()) + m.ChunkDigest = iter.ReadString() case "chunktype": - m.ChunkType = getString(iter.ReadStringAsSlice()) + m.ChunkType = iter.ReadString() case "xattrs": m.Xattrs = make(map[string]string) for key := iter.ReadObject(); key != ""; key = iter.ReadObject() { - value := iter.ReadStringAsSlice() - m.Xattrs[key] = getString(value) + m.Xattrs[key] = iter.ReadString() } default: iter.Skip() @@ -781,6 +744,5 @@ func unmarshalToc(manifest []byte) (*internal.TOC, error) { return nil, fmt.Errorf("unexpected data after manifest") } - toc.StringsBuf = buf return &toc, nil } diff --git a/pkg/chunked/cache_linux_test.go b/pkg/chunked/cache_linux_test.go index 190ddb2f39..3164e9b066 100644 --- a/pkg/chunked/cache_linux_test.go +++ b/pkg/chunked/cache_linux_test.go @@ -55,6 +55,23 @@ const jsonTOC = ` "chunkSize": 86252, "chunkOffset": 17615, "chunkDigest": "sha256:2a9d3f1b6b37abc8bb35eb8fa98b893a2a2447bcb01184c3bafc8c6b40da099d" + }, + { + "type": "reg", + "name": "usr/lib/systemd/system/system-systemd\\x2dcryptsetup.slice", + "mode": 420, + "size": 468, + "modtime": "2024-03-03T18:04:57+01:00", + "accesstime": "0001-01-01T00:00:00Z", + "changetime": "0001-01-01T00:00:00Z", + "digest": "sha256:68dc6e85631e077f2bc751352459823844911b93b7ba2afd95d96c893222bb50", + "offset": 148185424, + "endOffset": 148185753 + }, + { + "type": "reg", + "name": "usr/lib/systemd/system/system-systemd\\x2dcryptsetup-hardlink.slice", + "linkName": "usr/lib/systemd/system/system-systemd\\x2dcryptsetup.slice" } ] } @@ -65,7 +82,7 @@ func TestPrepareMetadata(t *testing.T) { if err != nil { t.Errorf("got error from prepareCacheFile: %v", err) } - if len(toc) != 2 { + if len(toc) != 4 { t.Error("prepareCacheFile returns the wrong length") } } @@ -194,7 +211,7 @@ func TestReadCache(t *testing.T) { func TestUnmarshalToc(t *testing.T) { toc, err := unmarshalToc([]byte(jsonTOC)) assert.NoError(t, err) - assert.Equal(t, 4, len(toc.Entries)) + assert.Equal(t, 6, len(toc.Entries)) _, err = unmarshalToc([]byte(jsonTOC + " \n\n\n\n ")) assert.NoError(t, err) @@ -210,4 +227,7 @@ func TestUnmarshalToc(t *testing.T) { assert.Error(t, err) _, err = unmarshalToc([]byte(jsonTOC + "123")) assert.Error(t, err) + assert.Equal(t, toc.Entries[4].Name, "usr/lib/systemd/system/system-systemd\\x2dcryptsetup.slice", "invalid name escaped") + assert.Equal(t, toc.Entries[5].Name, "usr/lib/systemd/system/system-systemd\\x2dcryptsetup-hardlink.slice", "invalid name escaped") + assert.Equal(t, toc.Entries[5].Linkname, "usr/lib/systemd/system/system-systemd\\x2dcryptsetup.slice", "invalid link name escaped") } diff --git a/pkg/chunked/internal/compression.go b/pkg/chunked/internal/compression.go index 54da17fd7f..1136d67b80 100644 --- a/pkg/chunked/internal/compression.go +++ b/pkg/chunked/internal/compression.go @@ -21,9 +21,6 @@ import ( type TOC struct { Version int `json:"version"` Entries []FileMetadata `json:"entries"` - - // internal: used by unmarshalToc - StringsBuf bytes.Buffer `json:"-"` } type FileMetadata struct {