From a17988ee1d171cc96eb71df96333b716c2049d84 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miloslav=20Trma=C4=8D?= <mitr@redhat.com> Date: Wed, 30 Oct 2024 16:14:48 +0100 Subject: [PATCH] DO NOT MERGE: Test https://github.com/containers/image/pull/2613 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Miloslav Trmač <mitr@redhat.com> --- go.mod | 4 +- go.sum | 4 +- .../containers/image/v5/copy/single.go | 2 +- .../containers/image/v5/docker/body_reader.go | 2 +- .../image/v5/pkg/compression/compression.go | 14 +++++- .../v5/pkg/compression/internal/types.go | 9 ++++ .../image/v5/storage/storage_dest.go | 44 ++++++++++++------- vendor/modules.txt | 5 ++- 8 files changed, 60 insertions(+), 24 deletions(-) diff --git a/go.mod b/go.mod index 0805cc8f91..ce53d0202e 100644 --- a/go.mod +++ b/go.mod @@ -20,7 +20,7 @@ require ( github.com/containers/libhvee v0.7.1 github.com/containers/ocicrypt v1.2.0 github.com/containers/psgo v1.9.0 - github.com/containers/storage v1.55.1-0.20241017155235-4db236377c55 + github.com/containers/storage v1.55.2-0.20241023191550-717b332e4c6c github.com/containers/winquit v1.1.0 github.com/coreos/go-systemd/v22 v22.5.1-0.20231103132048-7d375ecc2b09 github.com/coreos/stream-metadata-go v0.4.4 @@ -229,3 +229,5 @@ require ( ) replace github.com/containers/storage => github.com/mtrmac/storage v0.0.0-20241030144250-6c95f6eebe5d + +replace github.com/containers/image/v5 => github.com/mtrmac/image/v5 v5.0.0-20241030145807-090e94d1fb11 diff --git a/go.sum b/go.sum index 085af8bf5d..ee60e787d5 100644 --- a/go.sum +++ b/go.sum @@ -87,8 +87,6 @@ github.com/containers/conmon v2.0.20+incompatible h1:YbCVSFSCqFjjVwHTPINGdMX1F6J github.com/containers/conmon v2.0.20+incompatible/go.mod h1:hgwZ2mtuDrppv78a/cOBNiCm6O0UMWGx1mu7P00nu5I= github.com/containers/gvisor-tap-vsock v0.7.5 h1:bTy4u3DOmmUPwurL6me2rsgfypAFDhyeJleUcQmBR/E= github.com/containers/gvisor-tap-vsock v0.7.5/go.mod h1:GW9jOqAEEGdaS20XwTYdm6KCYDHIulOE/yEEOabkoE4= -github.com/containers/image/v5 v5.32.3-0.20241016192323-a66152c1cdf6 h1:kDsMVMhEFmWFLN6QEn0ul0MbpXCxLiIL5pqxADOqB8g= -github.com/containers/image/v5 v5.32.3-0.20241016192323-a66152c1cdf6/go.mod h1:Ulwf/jQO4757C/uOJyNiZ10dRiXRwVnyhF9wYFno3GQ= github.com/containers/libhvee v0.7.1 h1:dWGF5GLq9DZvXo3P8aDp3cNieL5eCaSell4UmeA/jY4= github.com/containers/libhvee v0.7.1/go.mod h1:fRKB3AyIqHMvq6xaeYhTpckM2cdoq0oecolyoiuLP7M= github.com/containers/libtrust v0.0.0-20230121012942-c1716e8a8d01 h1:Qzk5C6cYglewc+UyGf6lc8Mj2UaPTHy/iF2De0/77CA= @@ -382,6 +380,8 @@ github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9G github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= +github.com/mtrmac/image/v5 v5.0.0-20241030145807-090e94d1fb11 h1:VgCYZpEnS4r1BITy1TRFD49n12o8Gu/3VjlHi++ebbQ= +github.com/mtrmac/image/v5 v5.0.0-20241030145807-090e94d1fb11/go.mod h1:IjXJLPmVmdHol69Yghcis1cKU0roXiOgWySMMYwSci4= github.com/mtrmac/storage v0.0.0-20241030144250-6c95f6eebe5d h1:fYGkRzT7Nl66NPePY/8NYabz9MrRumHXRsJrw+0OMFs= github.com/mtrmac/storage v0.0.0-20241030144250-6c95f6eebe5d/go.mod h1:jlFSQ5COnXuLo1qZcKnWMAa7nYLXq2GKBwt0AVodgtE= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= diff --git a/vendor/github.com/containers/image/v5/copy/single.go b/vendor/github.com/containers/image/v5/copy/single.go index 983235158a..e008c7e86f 100644 --- a/vendor/github.com/containers/image/v5/copy/single.go +++ b/vendor/github.com/containers/image/v5/copy/single.go @@ -828,7 +828,7 @@ func (ic *imageCopier) copyLayer(ctx context.Context, srcInfo types.BlobInfo, to return false, types.BlobInfo{}, err }() if err != nil { - return types.BlobInfo{}, "", fmt.Errorf("reading blob %s: %w", srcInfo.Digest, err) + return types.BlobInfo{}, "", fmt.Errorf("partial pull of blob %s: %w", srcInfo.Digest, err) } if reused { return blobInfo, cachedDiffID, nil diff --git a/vendor/github.com/containers/image/v5/docker/body_reader.go b/vendor/github.com/containers/image/v5/docker/body_reader.go index 29d3b0420e..e69e21ef7a 100644 --- a/vendor/github.com/containers/image/v5/docker/body_reader.go +++ b/vendor/github.com/containers/image/v5/docker/body_reader.go @@ -197,7 +197,7 @@ func (br *bodyReader) Read(p []byte) (int, error) { consumedBody = true br.body = res.Body br.lastRetryOffset = br.offset - br.lastRetryTime = time.Time{} + br.lastRetryTime = time.Now() return n, nil default: diff --git a/vendor/github.com/containers/image/v5/pkg/compression/compression.go b/vendor/github.com/containers/image/v5/pkg/compression/compression.go index b83a257e4f..782c86d068 100644 --- a/vendor/github.com/containers/image/v5/pkg/compression/compression.go +++ b/vendor/github.com/containers/image/v5/pkg/compression/compression.go @@ -99,8 +99,18 @@ func CompressStream(dest io.Writer, algo Algorithm, level *int) (io.WriteCloser, return internal.AlgorithmCompressor(algo)(dest, m, level) } -// CompressStreamWithMetadata returns the compressor by its name. If the compression -// generates any metadata, it is written to the provided metadata map. +// CompressStreamWithMetadata returns the compressor by its name. +// +// Compressing a stream may create integrity data that allows consuming the compressed byte stream +// while only using subsets of the compressed data (if the compressed data is seekable and most +// of the uncompressed data is already present via other means), while still protecting integrity +// of the compressed stream against unwanted modification. (In OCI container images, this metadata +// is usually carried in manifest annotations.) +// +// Such a partial decompression is not implemented by this package; it is consumed e.g. by +// github.com/containers/storage/pkg/chunked . +// +// If the compression generates such metadata, it is written to the provided metadata map. func CompressStreamWithMetadata(dest io.Writer, metadata map[string]string, algo Algorithm, level *int) (io.WriteCloser, error) { return internal.AlgorithmCompressor(algo)(dest, metadata, level) } diff --git a/vendor/github.com/containers/image/v5/pkg/compression/internal/types.go b/vendor/github.com/containers/image/v5/pkg/compression/internal/types.go index d6f85274dd..e715705b43 100644 --- a/vendor/github.com/containers/image/v5/pkg/compression/internal/types.go +++ b/vendor/github.com/containers/image/v5/pkg/compression/internal/types.go @@ -3,6 +3,15 @@ package internal import "io" // CompressorFunc writes the compressed stream to the given writer using the specified compression level. +// +// Compressing a stream may create integrity data that allows consuming the compressed byte stream +// while only using subsets of the compressed data (if the compressed data is seekable and most +// of the uncompressed data is already present via other means), while still protecting integrity +// of the compressed stream against unwanted modification. (In OCI container images, this metadata +// is usually carried in manifest annotations.) +// +// If the compression generates such metadata, it is written to the provided metadata map. +// // The caller must call Close() on the stream (even if the input stream does not need closing!). type CompressorFunc func(io.Writer, map[string]string, *int) (io.WriteCloser, error) diff --git a/vendor/github.com/containers/image/v5/storage/storage_dest.go b/vendor/github.com/containers/image/v5/storage/storage_dest.go index c71616caca..6844eec36e 100644 --- a/vendor/github.com/containers/image/v5/storage/storage_dest.go +++ b/vendor/github.com/containers/image/v5/storage/storage_dest.go @@ -335,7 +335,7 @@ func (s *storageImageDestination) PutBlobPartial(ctx context.Context, chunkAcces out, err := s.imageRef.transport.store.PrepareStagedLayer(nil, differ) if err != nil { - return private.UploadedBlob{}, err + return private.UploadedBlob{}, fmt.Errorf("staging a partially-pulled layer: %w", err) } succeeded := false defer func() { @@ -354,23 +354,22 @@ func (s *storageImageDestination) PutBlobPartial(ctx context.Context, chunkAcces if out.UncompressedDigest != "" { s.lockProtected.indexToDiffID[options.LayerIndex] = out.UncompressedDigest if out.TOCDigest != "" { + s.lockProtected.indexToTOCDigest[options.LayerIndex] = out.TOCDigest options.Cache.RecordTOCUncompressedPair(out.TOCDigest, out.UncompressedDigest) } - // Don’t set indexToTOCDigest on this path: - // - Using UncompressedDigest allows image reuse with non-partially-pulled layers, so we want to set indexToDiffID. - // - If UncompressedDigest has been computed, that means the layer was read completely, and the TOC has been created from scratch. - // That TOC is quite unlikely to match any other TOC value. - // The computation of UncompressedDigest means the whole layer has been consumed; while doing that, chunked.GetDiffer is + // If the whole layer has been consumed, chunked.GetDiffer is // responsible for ensuring blobDigest has been validated. - if out.CompressedDigest != blobDigest { - return private.UploadedBlob{}, fmt.Errorf("internal error: PrepareStagedLayer returned CompressedDigest %q not matching expected %q", - out.CompressedDigest, blobDigest) - } - // So, record also information about blobDigest, that might benefit reuse. - // We trust PrepareStagedLayer to validate or create both values correctly. - s.lockProtected.blobDiffIDs[blobDigest] = out.UncompressedDigest - options.Cache.RecordDigestUncompressedPair(out.CompressedDigest, out.UncompressedDigest) + if out.CompressedDigest != "" { + if out.CompressedDigest != blobDigest { + return private.UploadedBlob{}, fmt.Errorf("internal error: PrepareStagedLayer returned CompressedDigest %q not matching expected %q", + out.CompressedDigest, blobDigest) + } + // So, record also information about blobDigest, that might benefit reuse. + // We trust PrepareStagedLayer to validate or create both values correctly. + s.lockProtected.blobDiffIDs[blobDigest] = out.UncompressedDigest + options.Cache.RecordDigestUncompressedPair(out.CompressedDigest, out.UncompressedDigest) + } } else { // Use diffID for layer identity if it is known. if uncompressedDigest := options.Cache.UncompressedDigestForTOC(out.TOCDigest); uncompressedDigest != "" { @@ -920,17 +919,32 @@ func (s *storageImageDestination) createNewLayer(index int, layerDigest digest.D return nil, nil } + // FIXME: Should we insist on UncompressedDigest being always set, and hard fail otherwise?? untrustedUncompressedDigest = d // While the contents of the digest are untrusted, make sure at least the _format_ is valid, // because we are going to write it to durable storage in expectedLayerDiffIDFlag . if err := untrustedUncompressedDigest.Validate(); err != nil { return nil, err } + } else { + // FIXME: Clean up. Maybe the generic code can provide us the config earlier? + // FIXME: Always enforce this for all layers??! + d, err := s.untrustedLayerDiffID(index) + if err != nil { + return nil, err + } + if d == "" { + logrus.Debugf("Skipping commit for layer %q, manifest not yet available", newLayerID) + return nil, nil + } + if diffOutput.UncompressedDigest != d { + return nil, fmt.Errorf("uncompressed digest inconsistency for layer %d: config %q vs. computed %q", index, d, diffOutput.UncompressedDigest) + } } flags := make(map[string]interface{}) if untrustedUncompressedDigest != "" { - flags[expectedLayerDiffIDFlag] = untrustedUncompressedDigest + flags[expectedLayerDiffIDFlag] = untrustedUncompressedDigest.String() logrus.Debugf("Setting uncompressed digest to %q for layer %q", untrustedUncompressedDigest, newLayerID) } diff --git a/vendor/modules.txt b/vendor/modules.txt index 9df4d62d57..9179b6d1e2 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -245,7 +245,7 @@ github.com/containers/conmon/runner/config # github.com/containers/gvisor-tap-vsock v0.7.5 ## explicit; go 1.21 github.com/containers/gvisor-tap-vsock/pkg/types -# github.com/containers/image/v5 v5.32.3-0.20241016192323-a66152c1cdf6 +# github.com/containers/image/v5 v5.32.3-0.20241016192323-a66152c1cdf6 => github.com/mtrmac/image/v5 v5.0.0-20241030145807-090e94d1fb11 ## explicit; go 1.22.6 github.com/containers/image/v5/copy github.com/containers/image/v5/directory @@ -356,7 +356,7 @@ github.com/containers/psgo/internal/dev github.com/containers/psgo/internal/host github.com/containers/psgo/internal/proc github.com/containers/psgo/internal/process -# github.com/containers/storage v1.55.1-0.20241017155235-4db236377c55 => github.com/mtrmac/storage v0.0.0-20241030144250-6c95f6eebe5d +# github.com/containers/storage v1.55.2-0.20241023191550-717b332e4c6c => github.com/mtrmac/storage v0.0.0-20241030144250-6c95f6eebe5d ## explicit; go 1.22.0 github.com/containers/storage github.com/containers/storage/drivers @@ -1387,3 +1387,4 @@ tags.cncf.io/container-device-interface/pkg/parser ## explicit; go 1.19 tags.cncf.io/container-device-interface/specs-go # github.com/containers/storage => github.com/mtrmac/storage v0.0.0-20241030144250-6c95f6eebe5d +# github.com/containers/image/v5 => github.com/mtrmac/image/v5 v5.0.0-20241030145807-090e94d1fb11