From 52ab5bfec87f9cdaac5e9215bd7b8aa1ab02340b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miloslav=20Trma=C4=8D?= Date: Tue, 13 Feb 2024 21:51:10 +0100 Subject: [PATCH 1/2] Optimize digest computation for uncompressed layers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When the caller provides neither OriginalDigest nor UncompressedDigest, only digest the layer once. Signed-off-by: Miloslav Trmač --- layers.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/layers.go b/layers.go index 8dfce5e36d..1936c6433f 100644 --- a/layers.go +++ b/layers.go @@ -2286,7 +2286,7 @@ func (r *layerStore) applyDiffWithOptions(to string, layerOptions *LayerOptions, if layerOptions != nil && layerOptions.UncompressedDigest != "" && layerOptions.UncompressedDigest.Algorithm() == digest.Canonical { uncompressedDigest = layerOptions.UncompressedDigest - } else { + } else if compression != archive.Uncompressed { uncompressedDigester = digest.Canonical.Digester() } @@ -2365,6 +2365,9 @@ func (r *layerStore) applyDiffWithOptions(to string, layerOptions *LayerOptions, if uncompressedDigester != nil { uncompressedDigest = uncompressedDigester.Digest() } + if uncompressedDigest == "" && compression == archive.Uncompressed { + uncompressedDigest = compressedDigest + } updateDigestMap(&r.bycompressedsum, layer.CompressedDigest, compressedDigest, layer.ID) layer.CompressedDigest = compressedDigest From 2ca0e2772aaaa575e84b6fa181bbfa46095932f0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miloslav=20Trma=C4=8D?= Date: Tue, 13 Feb 2024 21:56:05 +0100 Subject: [PATCH 2/2] Add LayerOptions.OriginalSize MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This allows us to correctly set (CompresedDigest, CompressedSize) when copying data from another layer; in that case we don't have the compressed data, so computing the size from compressedCounter sets an incorrect value. Signed-off-by: Miloslav Trmač --- layers.go | 6 +++++- store.go | 12 +++++++++++- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/layers.go b/layers.go index 1936c6433f..76b17659d3 100644 --- a/layers.go +++ b/layers.go @@ -2371,7 +2371,11 @@ func (r *layerStore) applyDiffWithOptions(to string, layerOptions *LayerOptions, updateDigestMap(&r.bycompressedsum, layer.CompressedDigest, compressedDigest, layer.ID) layer.CompressedDigest = compressedDigest - layer.CompressedSize = compressedCounter.Count + if layerOptions != nil && layerOptions.OriginalDigest != "" && layerOptions.OriginalSize != nil { + layer.CompressedSize = *layerOptions.OriginalSize + } else { + layer.CompressedSize = compressedCounter.Count + } updateDigestMap(&r.byuncompressedsum, layer.UncompressedDigest, uncompressedDigest, layer.ID) layer.UncompressedDigest = uncompressedDigest layer.UncompressedSize = uncompressedCounter.Count diff --git a/store.go b/store.go index 3f431726e2..4a98f54e09 100644 --- a/store.go +++ b/store.go @@ -580,10 +580,19 @@ type LayerOptions struct { // initialize this layer. If set, it should be a child of the layer // which we want to use as the parent of the new layer. TemplateLayer string - // OriginalDigest specifies a digest of the tarstream (diff), if one is + // OriginalDigest specifies a digest of the (possibly-compressed) tarstream (diff), if one is // provided along with these LayerOptions, and reliably known by the caller. + // The digest might not be exactly the digest of the provided tarstream + // (e.g. the digest might be of a compressed representation, while providing + // an uncompressed one); in that case the caller is responsible for the two matching. // Use the default "" if this fields is not applicable or the value is not known. OriginalDigest digest.Digest + // OriginalSize specifies a size of the (possibly-compressed) tarstream corresponding + // to OriginalDigest. + // If the digest does not match the provided tarstream, OriginalSize must match OriginalDigest, + // not the tarstream. + // Use nil if not applicable or not known. + OriginalSize *int64 // UncompressedDigest specifies a digest of the uncompressed version (“DiffID”) // of the tarstream (diff), if one is provided along with these LayerOptions, // and reliably known by the caller. @@ -1485,6 +1494,7 @@ func (s *store) PutLayer(id, parent string, names []string, mountLabel string, w } layerOptions := LayerOptions{ OriginalDigest: options.OriginalDigest, + OriginalSize: options.OriginalSize, UncompressedDigest: options.UncompressedDigest, Flags: options.Flags, }