diff --git a/internal/builder/builder.go b/internal/builder/builder.go index 7b66ad891..cd6bd2e93 100644 --- a/internal/builder/builder.go +++ b/internal/builder/builder.go @@ -582,21 +582,13 @@ func (b *Builder) addExplodedModules(kind string, logger logging.Logger, tmpDir } // create tar file - layerTar, err := buildpack.ToLayerTar(modTmpDir, module) + diffID, layerTar, err := buildpack.ToLayerTar(modTmpDir, module) if err != nil { lids[i] <- modInfo{err: err} } // generate diff id - diffID, err := dist.LayerDiffID(layerTar) info := module.Descriptor().Info() - if err != nil { - lids[i] <- modInfo{err: errors.Wrapf(err, - "getting content hashes for %s %s", - kind, - style.Symbol(info.FullName()), - )} - } lids[i] <- modInfo{ info: info, layerTar: layerTar, @@ -671,6 +663,7 @@ func (b *Builder) addExplodedModules(kind string, logger logging.Logger, tmpDir func (b *Builder) addFlattenedModules(kind string, logger logging.Logger, tmpDir string, image imgutil.Image, flattenModules [][]buildpack.BuildModule, layers dist.ModuleLayers) ([]buildpack.BuildModule, error) { collectionToAdd := map[string]toAdd{} var ( + diffID v1.Hash buildModuleExcluded []buildpack.BuildModule finalTarPath string err error @@ -679,21 +672,44 @@ func (b *Builder) addFlattenedModules(kind string, logger logging.Logger, tmpDir buildModuleWriter := buildpack.NewBuildModuleWriter(logger, b.layerWriterFactory) excludedModules := buildpack.Set(b.flattenExcludeBuildpacks) + type modInfo struct { + finalTarPath string + diffID v1.Hash + err error + } + + lids := make([]chan modInfo, len(flattenModules)) + for i := range lids { + lids[i] = make(chan modInfo, 1) + } + for i, additionalModules := range flattenModules { - modFlattenTmpDir := filepath.Join(tmpDir, fmt.Sprintf("%s-%s-flatten", kind, strconv.Itoa(i))) - if err := os.MkdirAll(modFlattenTmpDir, os.ModePerm); err != nil { - return nil, errors.Wrap(err, "creating flatten temp dir") - } + go func(i int, additionalModules []buildpack.BuildModule) { + // create directory + modFlattenTmpDir := filepath.Join(tmpDir, fmt.Sprintf("%s-%s-flatten", kind, strconv.Itoa(i))) + if err := os.MkdirAll(modFlattenTmpDir, os.ModePerm); err != nil { + lids[i] <- modInfo{err: errors.Wrap(err, "creating flatten temp dir")} + } - finalTarPath, buildModuleExcluded, err = buildModuleWriter.NToLayerTar(modFlattenTmpDir, fmt.Sprintf("%s-flatten-%s", kind, strconv.Itoa(i)), additionalModules, excludedModules) - if err != nil { - return nil, errors.Wrapf(err, "writing layer %s", finalTarPath) - } + diffID, finalTarPath, buildModuleExcluded, err = buildModuleWriter.NToLayerTar(modFlattenTmpDir, fmt.Sprintf("%s-flatten-%s", kind, strconv.Itoa(i)), additionalModules, excludedModules) + if err != nil { + lids[i] <- modInfo{err: errors.Wrapf(err, "writing layer %s", finalTarPath)} + } - diffID, err := dist.LayerDiffID(finalTarPath) - if err != nil { - return nil, errors.Wrapf(err, "calculating diff layer %s", finalTarPath) + // generate diff id + lids[i] <- modInfo{ + finalTarPath: finalTarPath, + diffID: diffID, + } + }(i, additionalModules) + } + + for i, additionalModules := range flattenModules { + mi := <-lids[i] + if mi.err != nil { + return nil, mi.err } + diffID, finalTarPath := mi.diffID, mi.finalTarPath for _, module := range additionalModules { collectionToAdd[module.Descriptor().Info().FullName()] = toAdd{ diff --git a/internal/fakes/fake_package.go b/internal/fakes/fake_package.go index 4713d2ffd..a55491771 100644 --- a/internal/fakes/fake_package.go +++ b/internal/fakes/fake_package.go @@ -6,8 +6,6 @@ import ( "os" "path/filepath" - "github.com/google/go-containerregistry/pkg/v1/tarball" - "github.com/buildpacks/pack/pkg/buildpack" "github.com/buildpacks/pack/pkg/dist" ) @@ -28,17 +26,7 @@ type fakePackage struct { func NewPackage(tmpDir string, name string, buildpacks []buildpack.BuildModule) (Package, error) { processBuildpack := func(bp buildpack.BuildModule) (tarFile string, diffID string, err error) { - tarFile, err = buildpack.ToLayerTar(tmpDir, bp) - if err != nil { - return "", "", err - } - - layer, err := tarball.LayerFromFile(tarFile) - if err != nil { - return "", "", err - } - - hash, err := layer.DiffID() + hash, tarFile, err := buildpack.ToLayerTar(tmpDir, bp) if err != nil { return "", "", err } diff --git a/pkg/buildpack/builder.go b/pkg/buildpack/builder.go index 222f1e577..feb3d4236 100644 --- a/pkg/buildpack/builder.go +++ b/pkg/buildpack/builder.go @@ -184,6 +184,7 @@ func (b *PackageBuilder) finalizeImage(image WorkableImage, tmpDir string) error var ( finalTarPath string err error + diffID v1.Hash ) for i, additionalModules := range b.FlattenedModules() { modFlattenTmpDir := filepath.Join(tmpDir, fmt.Sprintf("buildpack-%s-flatten", strconv.Itoa(i))) @@ -195,16 +196,11 @@ func (b *PackageBuilder) finalizeImage(image WorkableImage, tmpDir string) error // include the buildpack itself additionalModules = append(additionalModules, b.buildpack) } - finalTarPath, individualBuildModules, err = buildModuleWriter.NToLayerTar(modFlattenTmpDir, fmt.Sprintf("buildpack-flatten-%s", strconv.Itoa(i)), additionalModules, excludedModules) + diffID, finalTarPath, individualBuildModules, err = buildModuleWriter.NToLayerTar(modFlattenTmpDir, fmt.Sprintf("buildpack-flatten-%s", strconv.Itoa(i)), additionalModules, excludedModules) if err != nil { return errors.Wrapf(err, "adding layer %s", finalTarPath) } - diffID, err := dist.LayerDiffID(finalTarPath) - if err != nil { - return errors.Wrapf(err, "calculating diffID for layer %s", finalTarPath) - } - for _, module := range additionalModules { collectionToAdd[module.Descriptor().Info().FullName()] = toAdd{ tarPath: finalTarPath, @@ -221,18 +217,11 @@ func (b *PackageBuilder) finalizeImage(image WorkableImage, tmpDir string) error // Let's create the tarball for each individual module for _, bp := range append(b.dependencies.ExplodedModules(), individualBuildModules...) { - bpLayerTar, err := ToLayerTar(tmpDir, bp) + diffID, bpLayerTar, err := ToLayerTar(tmpDir, bp) if err != nil { return err } - diffID, err := dist.LayerDiffID(bpLayerTar) - if err != nil { - return errors.Wrapf(err, - "getting content hashes for buildpack %s", - style.Symbol(bp.Descriptor().Info().FullName()), - ) - } collectionToAdd[bp.Descriptor().Info().FullName()] = toAdd{ tarPath: bpLayerTar, diffID: diffID.String(), @@ -278,19 +267,11 @@ func (b *PackageBuilder) finalizeExtensionImage(image WorkableImage, tmpDir stri } exLayers := dist.ModuleLayers{} - exLayerTar, err := ToLayerTar(tmpDir, b.extension) + diffID, exLayerTar, err := ToLayerTar(tmpDir, b.extension) if err != nil { return err } - diffID, err := dist.LayerDiffID(exLayerTar) - if err != nil { - return errors.Wrapf(err, - "getting content hashes for extension %s", - style.Symbol(b.extension.Descriptor().Info().FullName()), - ) - } - if err := image.AddLayerWithDiffID(exLayerTar, diffID.String()); err != nil { return errors.Wrapf(err, "adding layer tar for extension %s", style.Symbol(b.extension.Descriptor().Info().FullName())) } diff --git a/pkg/buildpack/buildpack.go b/pkg/buildpack/buildpack.go index 80879c10e..6a2c67af2 100644 --- a/pkg/buildpack/buildpack.go +++ b/pkg/buildpack/buildpack.go @@ -2,12 +2,16 @@ package buildpack import ( "archive/tar" + "crypto/sha256" + "encoding/hex" "fmt" "io" "os" "path" "path/filepath" + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/BurntSushi/toml" "github.com/buildpacks/lifecycle/api" "github.com/pkg/errors" @@ -304,26 +308,41 @@ func validateExtensionDescriptor(extd dist.ExtensionDescriptor) error { return nil } -func ToLayerTar(dest string, module BuildModule) (string, error) { - descriptor := module.Descriptor() +func ToLayerTar(dest string, module BuildModule) (v1.Hash, string, error) { modReader, err := module.Open() if err != nil { - return "", errors.Wrap(err, "opening blob") + return v1.Hash{}, "", errors.Wrap(err, "opening blob") } defer modReader.Close() - layerTar := filepath.Join(dest, fmt.Sprintf("%s.%s.tar", descriptor.EscapedID(), descriptor.Info().Version)) - fh, err := os.Create(layerTar) - if err != nil { - return "", errors.Wrap(err, "create file for tar") - } - defer fh.Close() + hasher := sha256.New() + var layerTar string + file, ok := modReader.(*os.File) + if ok { + layerTar = file.Name() + if _, err = io.Copy(hasher, modReader); err != nil { + return v1.Hash{}, "", errors.Wrap(err, "writing diffID") + } + } else { + descriptor := module.Descriptor() + layerTar = filepath.Join(dest, fmt.Sprintf("%s.%s.tar", descriptor.EscapedID(), descriptor.Info().Version)) + fh, err := os.Create(layerTar) + if err != nil { + return v1.Hash{}, "", errors.Wrap(err, "create file for tar") + } + defer fh.Close() - if _, err := io.Copy(fh, modReader); err != nil { - return "", errors.Wrap(err, "writing blob to tar") + writer := io.MultiWriter(fh, hasher) + + if _, err = io.Copy(writer, modReader); err != nil { + return v1.Hash{}, "", errors.Wrap(err, "writing blob to tar") + } } - return layerTar, nil + return v1.Hash{ + Algorithm: "sha256", + Hex: hex.EncodeToString(hasher.Sum(make([]byte, 0, hasher.Size()))), + }, layerTar, nil } // Set returns a set of the given string slice. diff --git a/pkg/buildpack/buildpack_tar_writer.go b/pkg/buildpack/buildpack_tar_writer.go index 1dddaaf60..27cf62745 100644 --- a/pkg/buildpack/buildpack_tar_writer.go +++ b/pkg/buildpack/buildpack_tar_writer.go @@ -2,6 +2,8 @@ package buildpack import ( "archive/tar" + "crypto/sha256" + "encoding/hex" "fmt" "io" "os" @@ -9,6 +11,8 @@ import ( "path/filepath" "strings" + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/buildpacks/lifecycle/buildpack" "github.com/pkg/errors" @@ -34,17 +38,14 @@ func NewBuildModuleWriter(logger logging.Logger, factory archive.TarWriterFactor // NToLayerTar creates a tar file containing the all the Buildpacks given, but excluding the ones which FullName() is // in the exclude list. It returns the path to the tar file, the list of Buildpacks that were excluded, and any error -func (b *BuildModuleWriter) NToLayerTar(tarPath, filename string, modules []BuildModule, exclude map[string]struct{}) (string, []BuildModule, error) { +func (b *BuildModuleWriter) NToLayerTar(tarPath, filename string, modules []BuildModule, exclude map[string]struct{}) (v1.Hash, string, []BuildModule, error) { layerTar := filepath.Join(tarPath, fmt.Sprintf("%s.tar", filename)) tarFile, err := os.Create(layerTar) b.logger.Debugf("creating file %s", style.Symbol(layerTar)) if err != nil { - return "", nil, errors.Wrap(err, "create file for tar") + return v1.Hash{}, "", nil, errors.Wrap(err, "create file for tar") } - - defer tarFile.Close() tw := b.factory.NewWriter(tarFile) - defer tw.Close() parentFolderAdded := map[string]bool{} duplicated := map[string]bool{} @@ -56,8 +57,9 @@ func (b *BuildModuleWriter) NToLayerTar(tarPath, filename string, modules []Buil duplicated[module.Descriptor().Info().FullName()] = true b.logger.Debugf("adding %s", style.Symbol(module.Descriptor().Info().FullName())) - if err := b.writeBuildModuleToTar(tw, module, &parentFolderAdded); err != nil { - return "", nil, errors.Wrapf(err, "adding %s", style.Symbol(module.Descriptor().Info().FullName())) + err := b.writeBuildModuleToTar(tw, module, &parentFolderAdded) + if err != nil { + return v1.Hash{}, "", nil, errors.Wrapf(err, "adding %s", style.Symbol(module.Descriptor().Info().FullName())) } rootPath := processRootPath(module) if !parentFolderAdded[rootPath] { @@ -73,7 +75,21 @@ func (b *BuildModuleWriter) NToLayerTar(tarPath, filename string, modules []Buil } b.logger.Debugf("%s was created successfully", style.Symbol(layerTar)) - return layerTar, buildModuleExcluded, nil + tarFile.Close() + tw.Close() + + // calculate diffID + hasher := sha256.New() + if tarFile, err = os.Open(layerTar); err != nil { + return v1.Hash{}, "", nil, errors.Wrap(err, "open tar") + } + if _, err = io.Copy(hasher, tarFile); err != nil { + return v1.Hash{}, "", nil, errors.Wrap(err, "writing diffID") + } + return v1.Hash{ + Algorithm: "sha256", + Hex: hex.EncodeToString(hasher.Sum(make([]byte, 0, hasher.Size()))), + }, layerTar, buildModuleExcluded, nil } // writeBuildModuleToTar writes the content of the given tar file into the writer, skipping the folders that were already added @@ -109,12 +125,7 @@ func (b *BuildModuleWriter) writeBuildModuleToTar(tw archive.TarWriter, module B return errors.Wrapf(err, "failed to write header for '%s'", header.Name) } - buf, err := io.ReadAll(tr) - if err != nil { - return errors.Wrapf(err, "failed to read contents of '%s'", header.Name) - } - - _, err = tw.Write(buf) + _, err = io.Copy(tw, tr) if err != nil { return errors.Wrapf(err, "failed to write contents to '%s'", header.Name) } diff --git a/pkg/buildpack/buildpack_tar_writer_test.go b/pkg/buildpack/buildpack_tar_writer_test.go index 4913c66f4..4dceb0439 100644 --- a/pkg/buildpack/buildpack_tar_writer_test.go +++ b/pkg/buildpack/buildpack_tar_writer_test.go @@ -110,7 +110,7 @@ func testBuildModuleWriter(t *testing.T, when spec.G, it spec.S) { when("there are not duplicated buildpacks", func() { it("creates a tar", func() { bpModules := []buildpack.BuildModule{bp1v1, bp2v1, bp3v1} - tarFile, bpExcluded, err := buildModuleWriter.NToLayerTar(tmpDir, "test-file-1", bpModules, nil) + _, tarFile, bpExcluded, err := buildModuleWriter.NToLayerTar(tmpDir, "test-file-1", bpModules, nil) h.AssertNil(t, err) h.AssertTrue(t, len(bpExcluded) == 0) @@ -122,7 +122,7 @@ func testBuildModuleWriter(t *testing.T, when spec.G, it spec.S) { when("there are duplicated buildpacks", func() { it("creates a tar skipping root folder from duplicated buildpacks", func() { bpModules := []buildpack.BuildModule{bp1v1, bp1v2, bp2v1, bp3v1} - tarFile, bpExcluded, err := buildModuleWriter.NToLayerTar(tmpDir, "test-file-2", bpModules, nil) + _, tarFile, bpExcluded, err := buildModuleWriter.NToLayerTar(tmpDir, "test-file-2", bpModules, nil) h.AssertNil(t, err) h.AssertTrue(t, len(bpExcluded) == 0) @@ -142,7 +142,7 @@ func testBuildModuleWriter(t *testing.T, when spec.G, it spec.S) { when("there are not duplicated buildpacks", func() { it("creates a tar skipping excluded buildpacks", func() { bpModules := []buildpack.BuildModule{bp1v1, bp2v1, bp3v1} - tarFile, bpExcluded, err := buildModuleWriter.NToLayerTar(tmpDir, "test-file-3", bpModules, exclude) + _, tarFile, bpExcluded, err := buildModuleWriter.NToLayerTar(tmpDir, "test-file-3", bpModules, exclude) h.AssertNil(t, err) h.AssertTrue(t, len(bpExcluded) == 1) h.AssertNotNil(t, tarFile) @@ -154,7 +154,7 @@ func testBuildModuleWriter(t *testing.T, when spec.G, it spec.S) { when("there are duplicated buildpacks", func() { it("creates a tar skipping excluded buildpacks and root folder from duplicated buildpacks", func() { bpModules := []buildpack.BuildModule{bp1v1, bp1v2, bp2v1, bp3v1} - tarFile, bpExcluded, err := buildModuleWriter.NToLayerTar(tmpDir, "test-file-4", bpModules, exclude) + _, tarFile, bpExcluded, err := buildModuleWriter.NToLayerTar(tmpDir, "test-file-4", bpModules, exclude) h.AssertNil(t, err) h.AssertTrue(t, len(bpExcluded) == 1) h.AssertNotNil(t, tarFile) diff --git a/pkg/dist/layers.go b/pkg/dist/layers.go index 590454980..3191f500b 100644 --- a/pkg/dist/layers.go +++ b/pkg/dist/layers.go @@ -1,9 +1,6 @@ package dist import ( - "os" - "path/filepath" - "github.com/buildpacks/lifecycle/api" v1 "github.com/google/go-containerregistry/pkg/v1" "github.com/google/go-containerregistry/pkg/v1/tarball" @@ -19,12 +16,6 @@ type Descriptor interface { } func LayerDiffID(layerTarPath string) (v1.Hash, error) { - fh, err := os.Open(filepath.Clean(layerTarPath)) - if err != nil { - return v1.Hash{}, errors.Wrap(err, "opening tar file") - } - defer fh.Close() - layer, err := tarball.LayerFromFile(layerTarPath) if err != nil { return v1.Hash{}, errors.Wrap(err, "reading layer tar")