From a17988ee1d171cc96eb71df96333b716c2049d84 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Miloslav=20Trma=C4=8D?= <mitr@redhat.com>
Date: Wed, 30 Oct 2024 16:14:48 +0100
Subject: [PATCH] DO NOT MERGE: Test
 https://github.com/containers/image/pull/2613
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: Miloslav Trmač <mitr@redhat.com>
---
 go.mod                                        |  4 +-
 go.sum                                        |  4 +-
 .../containers/image/v5/copy/single.go        |  2 +-
 .../containers/image/v5/docker/body_reader.go |  2 +-
 .../image/v5/pkg/compression/compression.go   | 14 +++++-
 .../v5/pkg/compression/internal/types.go      |  9 ++++
 .../image/v5/storage/storage_dest.go          | 44 ++++++++++++-------
 vendor/modules.txt                            |  5 ++-
 8 files changed, 60 insertions(+), 24 deletions(-)

diff --git a/go.mod b/go.mod
index 0805cc8f91..ce53d0202e 100644
--- a/go.mod
+++ b/go.mod
@@ -20,7 +20,7 @@ require (
 	github.com/containers/libhvee v0.7.1
 	github.com/containers/ocicrypt v1.2.0
 	github.com/containers/psgo v1.9.0
-	github.com/containers/storage v1.55.1-0.20241017155235-4db236377c55
+	github.com/containers/storage v1.55.2-0.20241023191550-717b332e4c6c
 	github.com/containers/winquit v1.1.0
 	github.com/coreos/go-systemd/v22 v22.5.1-0.20231103132048-7d375ecc2b09
 	github.com/coreos/stream-metadata-go v0.4.4
@@ -229,3 +229,5 @@ require (
 )
 
 replace github.com/containers/storage => github.com/mtrmac/storage v0.0.0-20241030144250-6c95f6eebe5d
+
+replace github.com/containers/image/v5 => github.com/mtrmac/image/v5 v5.0.0-20241030145807-090e94d1fb11
diff --git a/go.sum b/go.sum
index 085af8bf5d..ee60e787d5 100644
--- a/go.sum
+++ b/go.sum
@@ -87,8 +87,6 @@ github.com/containers/conmon v2.0.20+incompatible h1:YbCVSFSCqFjjVwHTPINGdMX1F6J
 github.com/containers/conmon v2.0.20+incompatible/go.mod h1:hgwZ2mtuDrppv78a/cOBNiCm6O0UMWGx1mu7P00nu5I=
 github.com/containers/gvisor-tap-vsock v0.7.5 h1:bTy4u3DOmmUPwurL6me2rsgfypAFDhyeJleUcQmBR/E=
 github.com/containers/gvisor-tap-vsock v0.7.5/go.mod h1:GW9jOqAEEGdaS20XwTYdm6KCYDHIulOE/yEEOabkoE4=
-github.com/containers/image/v5 v5.32.3-0.20241016192323-a66152c1cdf6 h1:kDsMVMhEFmWFLN6QEn0ul0MbpXCxLiIL5pqxADOqB8g=
-github.com/containers/image/v5 v5.32.3-0.20241016192323-a66152c1cdf6/go.mod h1:Ulwf/jQO4757C/uOJyNiZ10dRiXRwVnyhF9wYFno3GQ=
 github.com/containers/libhvee v0.7.1 h1:dWGF5GLq9DZvXo3P8aDp3cNieL5eCaSell4UmeA/jY4=
 github.com/containers/libhvee v0.7.1/go.mod h1:fRKB3AyIqHMvq6xaeYhTpckM2cdoq0oecolyoiuLP7M=
 github.com/containers/libtrust v0.0.0-20230121012942-c1716e8a8d01 h1:Qzk5C6cYglewc+UyGf6lc8Mj2UaPTHy/iF2De0/77CA=
@@ -382,6 +380,8 @@ github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9G
 github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
 github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A=
 github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc=
+github.com/mtrmac/image/v5 v5.0.0-20241030145807-090e94d1fb11 h1:VgCYZpEnS4r1BITy1TRFD49n12o8Gu/3VjlHi++ebbQ=
+github.com/mtrmac/image/v5 v5.0.0-20241030145807-090e94d1fb11/go.mod h1:IjXJLPmVmdHol69Yghcis1cKU0roXiOgWySMMYwSci4=
 github.com/mtrmac/storage v0.0.0-20241030144250-6c95f6eebe5d h1:fYGkRzT7Nl66NPePY/8NYabz9MrRumHXRsJrw+0OMFs=
 github.com/mtrmac/storage v0.0.0-20241030144250-6c95f6eebe5d/go.mod h1:jlFSQ5COnXuLo1qZcKnWMAa7nYLXq2GKBwt0AVodgtE=
 github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
diff --git a/vendor/github.com/containers/image/v5/copy/single.go b/vendor/github.com/containers/image/v5/copy/single.go
index 983235158a..e008c7e86f 100644
--- a/vendor/github.com/containers/image/v5/copy/single.go
+++ b/vendor/github.com/containers/image/v5/copy/single.go
@@ -828,7 +828,7 @@ func (ic *imageCopier) copyLayer(ctx context.Context, srcInfo types.BlobInfo, to
 			return false, types.BlobInfo{}, err
 		}()
 		if err != nil {
-			return types.BlobInfo{}, "", fmt.Errorf("reading blob %s: %w", srcInfo.Digest, err)
+			return types.BlobInfo{}, "", fmt.Errorf("partial pull of blob %s: %w", srcInfo.Digest, err)
 		}
 		if reused {
 			return blobInfo, cachedDiffID, nil
diff --git a/vendor/github.com/containers/image/v5/docker/body_reader.go b/vendor/github.com/containers/image/v5/docker/body_reader.go
index 29d3b0420e..e69e21ef7a 100644
--- a/vendor/github.com/containers/image/v5/docker/body_reader.go
+++ b/vendor/github.com/containers/image/v5/docker/body_reader.go
@@ -197,7 +197,7 @@ func (br *bodyReader) Read(p []byte) (int, error) {
 		consumedBody = true
 		br.body = res.Body
 		br.lastRetryOffset = br.offset
-		br.lastRetryTime = time.Time{}
+		br.lastRetryTime = time.Now()
 		return n, nil
 
 	default:
diff --git a/vendor/github.com/containers/image/v5/pkg/compression/compression.go b/vendor/github.com/containers/image/v5/pkg/compression/compression.go
index b83a257e4f..782c86d068 100644
--- a/vendor/github.com/containers/image/v5/pkg/compression/compression.go
+++ b/vendor/github.com/containers/image/v5/pkg/compression/compression.go
@@ -99,8 +99,18 @@ func CompressStream(dest io.Writer, algo Algorithm, level *int) (io.WriteCloser,
 	return internal.AlgorithmCompressor(algo)(dest, m, level)
 }
 
-// CompressStreamWithMetadata returns the compressor by its name.  If the compression
-// generates any metadata, it is written to the provided metadata map.
+// CompressStreamWithMetadata returns the compressor by its name.
+//
+// Compressing a stream may create integrity data that allows consuming the compressed byte stream
+// while only using subsets of the compressed data (if the compressed data is seekable and most
+// of the uncompressed data is already present via other means), while still protecting integrity
+// of the compressed stream against unwanted modification. (In OCI container images, this metadata
+// is usually carried in manifest annotations.)
+//
+// Such a partial decompression is not implemented by this package; it is consumed e.g. by
+// github.com/containers/storage/pkg/chunked .
+//
+// If the compression generates such metadata, it is written to the provided metadata map.
 func CompressStreamWithMetadata(dest io.Writer, metadata map[string]string, algo Algorithm, level *int) (io.WriteCloser, error) {
 	return internal.AlgorithmCompressor(algo)(dest, metadata, level)
 }
diff --git a/vendor/github.com/containers/image/v5/pkg/compression/internal/types.go b/vendor/github.com/containers/image/v5/pkg/compression/internal/types.go
index d6f85274dd..e715705b43 100644
--- a/vendor/github.com/containers/image/v5/pkg/compression/internal/types.go
+++ b/vendor/github.com/containers/image/v5/pkg/compression/internal/types.go
@@ -3,6 +3,15 @@ package internal
 import "io"
 
 // CompressorFunc writes the compressed stream to the given writer using the specified compression level.
+//
+// Compressing a stream may create integrity data that allows consuming the compressed byte stream
+// while only using subsets of the compressed data (if the compressed data is seekable and most
+// of the uncompressed data is already present via other means), while still protecting integrity
+// of the compressed stream against unwanted modification. (In OCI container images, this metadata
+// is usually carried in manifest annotations.)
+//
+// If the compression generates such metadata, it is written to the provided metadata map.
+//
 // The caller must call Close() on the stream (even if the input stream does not need closing!).
 type CompressorFunc func(io.Writer, map[string]string, *int) (io.WriteCloser, error)
 
diff --git a/vendor/github.com/containers/image/v5/storage/storage_dest.go b/vendor/github.com/containers/image/v5/storage/storage_dest.go
index c71616caca..6844eec36e 100644
--- a/vendor/github.com/containers/image/v5/storage/storage_dest.go
+++ b/vendor/github.com/containers/image/v5/storage/storage_dest.go
@@ -335,7 +335,7 @@ func (s *storageImageDestination) PutBlobPartial(ctx context.Context, chunkAcces
 
 	out, err := s.imageRef.transport.store.PrepareStagedLayer(nil, differ)
 	if err != nil {
-		return private.UploadedBlob{}, err
+		return private.UploadedBlob{}, fmt.Errorf("staging a partially-pulled layer: %w", err)
 	}
 	succeeded := false
 	defer func() {
@@ -354,23 +354,22 @@ func (s *storageImageDestination) PutBlobPartial(ctx context.Context, chunkAcces
 	if out.UncompressedDigest != "" {
 		s.lockProtected.indexToDiffID[options.LayerIndex] = out.UncompressedDigest
 		if out.TOCDigest != "" {
+			s.lockProtected.indexToTOCDigest[options.LayerIndex] = out.TOCDigest
 			options.Cache.RecordTOCUncompressedPair(out.TOCDigest, out.UncompressedDigest)
 		}
-		// Don’t set indexToTOCDigest on this path:
-		// - Using UncompressedDigest allows image reuse with non-partially-pulled layers, so we want to set indexToDiffID.
-		// - If UncompressedDigest has been computed, that means the layer was read completely, and the TOC has been created from scratch.
-		//   That TOC is quite unlikely to match any other TOC value.
 
-		// The computation of UncompressedDigest means the whole layer has been consumed; while doing that, chunked.GetDiffer is
+		// If the whole layer has been consumed, chunked.GetDiffer is
 		// responsible for ensuring blobDigest has been validated.
-		if out.CompressedDigest != blobDigest {
-			return private.UploadedBlob{}, fmt.Errorf("internal error: PrepareStagedLayer returned CompressedDigest %q not matching expected %q",
-				out.CompressedDigest, blobDigest)
-		}
-		// So, record also information about blobDigest, that might benefit reuse.
-		// We trust PrepareStagedLayer to validate or create both values correctly.
-		s.lockProtected.blobDiffIDs[blobDigest] = out.UncompressedDigest
-		options.Cache.RecordDigestUncompressedPair(out.CompressedDigest, out.UncompressedDigest)
+		if out.CompressedDigest != "" {
+			if out.CompressedDigest != blobDigest {
+				return private.UploadedBlob{}, fmt.Errorf("internal error: PrepareStagedLayer returned CompressedDigest %q not matching expected %q",
+					out.CompressedDigest, blobDigest)
+			}
+			// So, record also information about blobDigest, that might benefit reuse.
+			// We trust PrepareStagedLayer to validate or create both values correctly.
+			s.lockProtected.blobDiffIDs[blobDigest] = out.UncompressedDigest
+			options.Cache.RecordDigestUncompressedPair(out.CompressedDigest, out.UncompressedDigest)
+		}
 	} else {
 		// Use diffID for layer identity if it is known.
 		if uncompressedDigest := options.Cache.UncompressedDigestForTOC(out.TOCDigest); uncompressedDigest != "" {
@@ -920,17 +919,32 @@ func (s *storageImageDestination) createNewLayer(index int, layerDigest digest.D
 				return nil, nil
 			}
 
+			// FIXME: Should we insist on UncompressedDigest being always set, and hard fail otherwise??
 			untrustedUncompressedDigest = d
 			// While the contents of the digest are untrusted, make sure at least the _format_ is valid,
 			// because we are going to write it to durable storage in expectedLayerDiffIDFlag .
 			if err := untrustedUncompressedDigest.Validate(); err != nil {
 				return nil, err
 			}
+		} else {
+			// FIXME: Clean up. Maybe the generic code can provide us the config earlier?
+			// FIXME: Always enforce this for all layers??!
+			d, err := s.untrustedLayerDiffID(index)
+			if err != nil {
+				return nil, err
+			}
+			if d == "" {
+				logrus.Debugf("Skipping commit for layer %q, manifest not yet available", newLayerID)
+				return nil, nil
+			}
+			if diffOutput.UncompressedDigest != d {
+				return nil, fmt.Errorf("uncompressed digest inconsistency for layer %d: config %q vs. computed %q", index, d, diffOutput.UncompressedDigest)
+			}
 		}
 
 		flags := make(map[string]interface{})
 		if untrustedUncompressedDigest != "" {
-			flags[expectedLayerDiffIDFlag] = untrustedUncompressedDigest
+			flags[expectedLayerDiffIDFlag] = untrustedUncompressedDigest.String()
 			logrus.Debugf("Setting uncompressed digest to %q for layer %q", untrustedUncompressedDigest, newLayerID)
 		}
 
diff --git a/vendor/modules.txt b/vendor/modules.txt
index 9df4d62d57..9179b6d1e2 100644
--- a/vendor/modules.txt
+++ b/vendor/modules.txt
@@ -245,7 +245,7 @@ github.com/containers/conmon/runner/config
 # github.com/containers/gvisor-tap-vsock v0.7.5
 ## explicit; go 1.21
 github.com/containers/gvisor-tap-vsock/pkg/types
-# github.com/containers/image/v5 v5.32.3-0.20241016192323-a66152c1cdf6
+# github.com/containers/image/v5 v5.32.3-0.20241016192323-a66152c1cdf6 => github.com/mtrmac/image/v5 v5.0.0-20241030145807-090e94d1fb11
 ## explicit; go 1.22.6
 github.com/containers/image/v5/copy
 github.com/containers/image/v5/directory
@@ -356,7 +356,7 @@ github.com/containers/psgo/internal/dev
 github.com/containers/psgo/internal/host
 github.com/containers/psgo/internal/proc
 github.com/containers/psgo/internal/process
-# github.com/containers/storage v1.55.1-0.20241017155235-4db236377c55 => github.com/mtrmac/storage v0.0.0-20241030144250-6c95f6eebe5d
+# github.com/containers/storage v1.55.2-0.20241023191550-717b332e4c6c => github.com/mtrmac/storage v0.0.0-20241030144250-6c95f6eebe5d
 ## explicit; go 1.22.0
 github.com/containers/storage
 github.com/containers/storage/drivers
@@ -1387,3 +1387,4 @@ tags.cncf.io/container-device-interface/pkg/parser
 ## explicit; go 1.19
 tags.cncf.io/container-device-interface/specs-go
 # github.com/containers/storage => github.com/mtrmac/storage v0.0.0-20241030144250-6c95f6eebe5d
+# github.com/containers/image/v5 => github.com/mtrmac/image/v5 v5.0.0-20241030145807-090e94d1fb11