From 4d4ec8af6168bb5d5c967e6034f4b36d12ef05d2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Philippe=20Vl=C3=A9rick?= Date: Sun, 20 Aug 2023 13:07:25 +0200 Subject: [PATCH] Adding support for shared blobs directory MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Philippe Vlérick --- ...2b8637b47ce96c838ba2aa0de66d14f45cedc11423 | 30 ++++++++++++++ .../delete_image_sharedblobsdir/index.json | 13 +++++++ .../delete_image_sharedblobsdir/oci-layout | 1 + ...e402ae2e97119c6007b6e52146419985ec1f0092dc | 1 + ...b77c933269586ad0226c83405776be08547e4d2a18 | 16 ++++++++ oci/layout/oci_delete.go | 39 ++++++++++++------- oci/layout/oci_delete_test.go | 31 +++++++++++++++ 7 files changed, 118 insertions(+), 13 deletions(-) create mode 100644 oci/layout/fixtures/delete_image_sharedblobsdir/blobs/sha256/a527179158cd5cebc11c152b8637b47ce96c838ba2aa0de66d14f45cedc11423 create mode 100644 oci/layout/fixtures/delete_image_sharedblobsdir/index.json create mode 100644 oci/layout/fixtures/delete_image_sharedblobsdir/oci-layout create mode 100644 oci/layout/fixtures/delete_image_sharedblobsdir/shared_blobs/sha256/0c8b263642b51b5c1dc40fe402ae2e97119c6007b6e52146419985ec1f0092dc create mode 100644 oci/layout/fixtures/delete_image_sharedblobsdir/shared_blobs/sha256/eaa95f3cfaac07c8a5153eb77c933269586ad0226c83405776be08547e4d2a18 diff --git a/oci/layout/fixtures/delete_image_sharedblobsdir/blobs/sha256/a527179158cd5cebc11c152b8637b47ce96c838ba2aa0de66d14f45cedc11423 b/oci/layout/fixtures/delete_image_sharedblobsdir/blobs/sha256/a527179158cd5cebc11c152b8637b47ce96c838ba2aa0de66d14f45cedc11423 new file mode 100644 index 0000000000..f0f06201be --- /dev/null +++ b/oci/layout/fixtures/delete_image_sharedblobsdir/blobs/sha256/a527179158cd5cebc11c152b8637b47ce96c838ba2aa0de66d14f45cedc11423 @@ -0,0 +1,30 @@ +{ + "created": "2019-08-20T20:19:55.211423266Z", + "architecture": "amd64", + "os": "linux", + "config": { + "Env": [ + "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + ], + "Cmd": [ + "/bin/sh" + ] + }, + "rootfs": { + "type": "layers", + "diff_ids": [ + "sha256:03901b4a2ea88eeaad62dbe59b072b28b6efa00491962b8741081c5df50c65e0" + ] + }, + "history": [ + { + "created": "2019-08-20T20:19:55.062606894Z", + "created_by": "/bin/sh -c #(nop) ADD file:fe64057fbb83dccb960efabbf1cd8777920ef279a7fa8dbca0a8801c651bdf7c in / " + }, + { + "created": "2019-08-20T20:19:55.211423266Z", + "created_by": "/bin/sh -c #(nop) CMD [\"/bin/sh\"]", + "empty_layer": true + } + ] +} diff --git a/oci/layout/fixtures/delete_image_sharedblobsdir/index.json b/oci/layout/fixtures/delete_image_sharedblobsdir/index.json new file mode 100644 index 0000000000..49925c19ae --- /dev/null +++ b/oci/layout/fixtures/delete_image_sharedblobsdir/index.json @@ -0,0 +1,13 @@ +{ + "schemaVersion": 2, + "manifests": [ + { + "mediaType": "application/vnd.oci.image.manifest.v1+json", + "digest": "sha256:eaa95f3cfaac07c8a5153eb77c933269586ad0226c83405776be08547e4d2a18", + "size": 405, + "annotations": { + "org.opencontainers.image.ref.name": "latest" + } + } + ] +} diff --git a/oci/layout/fixtures/delete_image_sharedblobsdir/oci-layout b/oci/layout/fixtures/delete_image_sharedblobsdir/oci-layout new file mode 100644 index 0000000000..21b1439d1c --- /dev/null +++ b/oci/layout/fixtures/delete_image_sharedblobsdir/oci-layout @@ -0,0 +1 @@ +{"imageLayoutVersion": "1.0.0"} \ No newline at end of file diff --git a/oci/layout/fixtures/delete_image_sharedblobsdir/shared_blobs/sha256/0c8b263642b51b5c1dc40fe402ae2e97119c6007b6e52146419985ec1f0092dc b/oci/layout/fixtures/delete_image_sharedblobsdir/shared_blobs/sha256/0c8b263642b51b5c1dc40fe402ae2e97119c6007b6e52146419985ec1f0092dc new file mode 100644 index 0000000000..e7e64ba41b --- /dev/null +++ b/oci/layout/fixtures/delete_image_sharedblobsdir/shared_blobs/sha256/0c8b263642b51b5c1dc40fe402ae2e97119c6007b6e52146419985ec1f0092dc @@ -0,0 +1 @@ +insert binary content here #9671 diff --git a/oci/layout/fixtures/delete_image_sharedblobsdir/shared_blobs/sha256/eaa95f3cfaac07c8a5153eb77c933269586ad0226c83405776be08547e4d2a18 b/oci/layout/fixtures/delete_image_sharedblobsdir/shared_blobs/sha256/eaa95f3cfaac07c8a5153eb77c933269586ad0226c83405776be08547e4d2a18 new file mode 100644 index 0000000000..1ff195d0f3 --- /dev/null +++ b/oci/layout/fixtures/delete_image_sharedblobsdir/shared_blobs/sha256/eaa95f3cfaac07c8a5153eb77c933269586ad0226c83405776be08547e4d2a18 @@ -0,0 +1,16 @@ +{ + "schemaVersion": 2, + "mediaType": "application/vnd.oci.image.manifest.v1+json", + "config": { + "mediaType": "application/vnd.oci.image.config.v1+json", + "digest": "sha256:a527179158cd5cebc11c152b8637b47ce96c838ba2aa0de66d14f45cedc11423", + "size": 585 + }, + "layers": [ + { + "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", + "digest": "sha256:0c8b263642b51b5c1dc40fe402ae2e97119c6007b6e52146419985ec1f0092dc", + "size": 33 + } + ] +} diff --git a/oci/layout/oci_delete.go b/oci/layout/oci_delete.go index be1f3c84d4..c284c1719a 100644 --- a/oci/layout/oci_delete.go +++ b/oci/layout/oci_delete.go @@ -17,6 +17,11 @@ import ( // DeleteImage deletes the named image from the directory, if supported. func (ref ociReference) DeleteImage(ctx context.Context, sys *types.SystemContext) error { + sharedBlobsDir := "" + if sys != nil && sys.OCISharedBlobDirPath != "" { + sharedBlobsDir = sys.OCISharedBlobDirPath + } + // Scan all the manifests in the directory: // ... collect the one that matches with the received ref // ... and store all the blobs used in all other images @@ -39,7 +44,7 @@ func (ref ociReference) DeleteImage(ctx context.Context, sys *types.SystemContex tmpDescriptionWrapper := v imageDescriptorWrapper = &tmpDescriptionWrapper } else { - otherImageManifest, err := ref.getManifest(v.descriptor) + otherImageManifest, err := ref.getManifest(v.descriptor, sharedBlobsDir) if err != nil { return err } @@ -55,7 +60,7 @@ func (ref ociReference) DeleteImage(ctx context.Context, sys *types.SystemContex return ImageNotFoundError{ref} } - manifest, err := ref.getManifest(imageDescriptorWrapper.descriptor) + manifest, err := ref.getManifest(imageDescriptorWrapper.descriptor, sharedBlobsDir) if err != nil { return err } @@ -70,15 +75,24 @@ func (ref ociReference) DeleteImage(ctx context.Context, sys *types.SystemContex } } for _, digest := range blobsToDelete.Values() { - //TODO Check if there is shared blob path? - blobPath, err := ref.blobPath(digest, "") + blobPath, err := ref.blobPath(digest, "") //Only delete in the local directory, not in the shared blobs path if err != nil { return err } - logrus.Debug("Deleting blob ", digest.Hex()) - err = os.Remove(blobPath) - if err != nil && !os.IsNotExist(err) { - return err + _, err = os.Stat(blobPath) + if err == nil { + logrus.Debug("Deleting blob ", digest.Hex()) + err = os.Remove(blobPath) + if err != nil && !os.IsNotExist(err) { + return err + } + // + } else { + if os.IsNotExist(err) { + logrus.Info("Blob ", digest.Hex(), " not found in image directory; it was either previously deleted or is in the shared blobs directory") + } else { + return err + } } } @@ -130,7 +144,7 @@ func (ref ociReference) DeleteImage(ctx context.Context, sys *types.SystemContex return err } indexNewDigest := digest.Canonical.FromBytes(buffer.Bytes()) - indexNewPath, err := ref.blobPath(indexNewDigest, "") + indexNewPath, err := ref.blobPath(indexNewDigest, sharedBlobsDir) if err != nil { return err } @@ -204,7 +218,7 @@ func (ref ociReference) getAllImageDescriptorsInDirectory() ([]descriptorWrapper wrapper := descriptorWrapper{&tmpManifestDescriptor, indexChain} descriptors = append(descriptors, wrapper) case imgspecv1.MediaTypeImageIndex: - nestedIndexBlobPath, err := ref.blobPath(manifestDescriptor.Digest, "") + nestedIndexBlobPath, err := ref.blobPath(manifestDescriptor.Digest, "") // Only scan the local directory, not the shared blobs directory if err != nil { return err } @@ -222,9 +236,8 @@ func (ref ociReference) getAllImageDescriptorsInDirectory() ([]descriptorWrapper return descriptors, err } -func (ref ociReference) getManifest(descriptor *imgspecv1.Descriptor) (*imgspecv1.Manifest, error) { - //TODO Check if there is shared blob path? - manifestPath, err := ref.blobPath(descriptor.Digest, "") +func (ref ociReference) getManifest(descriptor *imgspecv1.Descriptor, sharedBlobsDir string) (*imgspecv1.Manifest, error) { + manifestPath, err := ref.blobPath(descriptor.Digest, sharedBlobsDir) if err != nil { return nil, err } diff --git a/oci/layout/oci_delete_test.go b/oci/layout/oci_delete_test.go index 8cdf5cc9b5..4e37efdb58 100644 --- a/oci/layout/oci_delete_test.go +++ b/oci/layout/oci_delete_test.go @@ -7,6 +7,7 @@ import ( "path/filepath" "testing" + "github.com/containers/image/v5/types" imgspecv1 "github.com/opencontainers/image-spec/specs-go/v1" cp "github.com/otiai10/copy" "github.com/stretchr/testify/assert" @@ -36,6 +37,36 @@ func TestReferenceDeleteImage(t *testing.T) { require.Equal(t, 0, len(index.Manifests)) } +func TestReferenceDeleteImage_sharedBlobDir(t *testing.T) { + tmpDir := loadFixture(t, "delete_image_sharedblobsdir") + + ref, err := NewReference(tmpDir, "latest") + require.NoError(t, err) + + sys := &types.SystemContext{OCISharedBlobDirPath: filepath.Join(tmpDir, "shared_blobs")} + err = ref.DeleteImage(context.Background(), sys) + require.NoError(t, err) + + // Check that the only blob in the local directory was deleted + blobsDir := filepath.Join(tmpDir, "blobs") + files, err := os.ReadDir(filepath.Join(blobsDir, "sha256")) + require.NoError(t, err) + require.Empty(t, files) + + // Check that the blobs in the shared blob directory are still present + sharedBlobsDir := filepath.Join(tmpDir, "shared_blobs") + files, err = os.ReadDir(filepath.Join(sharedBlobsDir, "sha256")) + require.NoError(t, err) + require.Equal(t, 2, len(files)) + + // Check that the index is empty as there is only one image in the fixture + ociRef, ok := ref.(ociReference) + require.True(t, ok) + index, err := ociRef.getIndex() + require.NoError(t, err) + require.Equal(t, 0, len(index.Manifests)) +} + func TestReferenceDeleteImage_emptyImageName(t *testing.T) { tmpDir := loadFixture(t, "delete_image")