feat: prevent unnecessary conversions

Signed-off-by: breezeTuT <[email protected]>

pkg/converter/constant.go

@@ -20,6 +20,9 @@ const (
 	LayerAnnotationNydusBootstrap     = "containerd.io/snapshot/nydus-bootstrap"
 	LayerAnnotationNydusSourceChainID = "containerd.io/snapshot/nydus-source-chainid"
 	LayerAnnotationNydusEncryptedBlob = "containerd.io/snapshot/nydus-encrypted-blob"
+	LayerAnnotationSourceDigest       = "containerd.io/snapshot/nydus-source-digest"
+	LayerAnnotationTargetDigest       = "containerd.io/snapshot/nydus-target-digest"
+	LayerAnnotationFromCache          = "containerd.io/snapshot/nydus-from-cache"
 
 	LayerAnnotationNydusReferenceBlobIDs = "containerd.io/snapshot/nydus-reference-blob-ids"
 

pkg/converter/convert_unix.go

@@ -121,7 +121,7 @@ func unpackOciTar(ctx context.Context, dst string, reader io.Reader) error {
 
 // unpackNydusBlob unpacks a Nydus formatted tar stream into a directory.
 // unpackBlob indicates whether to unpack blob data.
-func unpackNydusBlob(bootDst, blobDst string, ra content.ReaderAt, unpackBlob bool) error {
+func UnpackNydusBlob(bootDst, blobDst string, ra content.ReaderAt, unpackBlob bool) error {
 	boot, err := os.OpenFile(bootDst, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0640)
 	if err != nil {
 		return errors.Wrapf(err, "write to bootstrap %s", bootDst)
@@ -589,6 +589,18 @@ func Merge(ctx context.Context, layers []Layer, dest io.Writer, opt MergeOption)
 				}
 				defer bootstrap.Close()
 
+				// Check if the bootstrap is already generated
+				bootCacheName := filepath.Join(opt.WorkDir, layers[idx].Digest.Hex())
+				bootCache, err := os.Open(bootCacheName)
+				if err == nil {
+					defer bootCache.Close()
+					if _, err := io.Copy(bootstrap, bootCache); err != nil {
+						return errors.Wrap(err, "copy cached bootstrap")
+					}
+					defer os.Remove(bootCacheName)
+					return nil
+				}
+
 				if _, err := UnpackEntry(layers[idx].ReaderAt, EntryBootstrap, bootstrap); err != nil {
 					return errors.Wrap(err, "unpack nydus tar")
 				}
@@ -656,7 +668,7 @@ func Unpack(ctx context.Context, ra content.ReaderAt, dest io.Writer, opt Unpack
 	defer os.RemoveAll(workDir)
 
 	bootPath, blobPath := filepath.Join(workDir, EntryBootstrap), filepath.Join(workDir, EntryBlob)
-	if err = unpackNydusBlob(bootPath, blobPath, ra, !opt.Stream); err != nil {
+	if err = UnpackNydusBlob(bootPath, blobPath, ra, !opt.Stream); err != nil {
 		return errors.Wrap(err, "unpack nydus tar")
 	}
 
@@ -771,6 +783,29 @@ func LayerConvertFunc(opt PackOption) converter.ConvertFunc {
 			return nil, nil
 		}
 
+		// Use remote cache to avoid duplicate conversion
+		info, err := cs.Info(ctx, desc.Digest)
+		if err != nil {
+			return nil, errors.Wrapf(err, "get blob info %s", desc.Digest)
+		}
+		if info.Labels[LayerAnnotationTargetDigest] != "" {
+			targetInfo, err := cs.Info(ctx, digest.Digest(info.Labels[LayerAnnotationTargetDigest]))
+			if err != nil {
+				return nil, errors.Wrapf(err, "get blob info %s", desc.Digest)
+			}
+			targetDesc := ocispec.Descriptor{
+				Digest:    targetInfo.Digest,
+				Size:      targetInfo.Size,
+				MediaType: MediaTypeNydusBlob,
+				Annotations: map[string]string{
+					LayerAnnotationUncompressed: targetInfo.Digest.String(),
+					LayerAnnotationNydusBlob:    "true",
+					LayerAnnotationFromCache:    "true",
+				},
+			}
+			return &targetDesc, nil
+		}
+
 		ra, err := cs.ReaderAt(ctx, desc)
 		if err != nil {
 			return nil, errors.Wrap(err, "get source blob reader")
@@ -825,7 +860,7 @@ func LayerConvertFunc(opt PackOption) converter.ConvertFunc {
 		}
 
 		blobDigest := digester.Digest()
-		info, err := cs.Info(ctx, blobDigest)
+		info, err = cs.Info(ctx, blobDigest)
 		if err != nil {
 			return nil, errors.Wrapf(err, "get blob info %s", blobDigest)
 		}
@@ -836,6 +871,9 @@ func LayerConvertFunc(opt PackOption) converter.ConvertFunc {
 		// diff id calculation to speed up the conversion.
 		// See: https://github.com/containerd/containerd/blob/e4fefea5544d259177abb85b64e428702ac49c97/images/diffid.go#L49
 		info.Labels[labels.LabelUncompressed] = blobDigest.String()
+		// Update layer labels in remote cache.
+		info.Labels[LayerAnnotationNydusBlob] = "true"
+		info.Labels[LayerAnnotationSourceDigest] = desc.Digest.String()
 		_, err = cs.Update(ctx, info)
 		if err != nil {
 			return nil, errors.Wrap(err, "update layer label")