From 7c3fdcb967d17643593acaf807e78a141afff734 Mon Sep 17 00:00:00 2001 From: Aditya R Date: Mon, 26 Jun 2023 13:39:52 +0530 Subject: [PATCH] distributedCache: allow local dir as a cache transport Allowing local directory as transport enables users to make distributed caching more flexible by making sure. * Allows end users to move cache sources and destination manually. * Allows storing cache on an `NFS` or various mount sources. * Allows end users to distribute cache without registry setup. [NO NEW TESTS NEEDED] [NO TESTS NEEDED] // Will add test in final commit Signed-off-by: Aditya R --- define/build.go | 5 ++-- imagebuildah/executor.go | 4 +-- imagebuildah/stage_executor.go | 52 +++++++++++++++++++++++++--------- pkg/cli/build.go | 5 ++-- pkg/parse/parse.go | 15 +++++----- 5 files changed, 52 insertions(+), 29 deletions(-) diff --git a/define/build.go b/define/build.go index 42c8fd72e63..4dbada60a80 100644 --- a/define/build.go +++ b/define/build.go @@ -5,7 +5,6 @@ import ( "time" nettypes "github.com/containers/common/libnetwork/types" - "github.com/containers/image/v5/docker/reference" "github.com/containers/image/v5/types" encconfig "github.com/containers/ocicrypt/config" "github.com/containers/storage/pkg/archive" @@ -141,10 +140,10 @@ type BuildOptions struct { TransientMounts []string // CacheFrom specifies any remote repository which can be treated as // potential cache source. - CacheFrom []reference.Named + CacheFrom []types.ImageReference // CacheTo specifies any remote repository which can be treated as // potential cache destination. - CacheTo []reference.Named + CacheTo []types.ImageReference // CacheTTL specifies duration, if specified using `--cache-ttl` then // cache intermediate images under this duration will be considered as // valid cache sources and images outside this duration will be ignored. diff --git a/imagebuildah/executor.go b/imagebuildah/executor.go index 5f23a5826a2..24761f80d2c 100644 --- a/imagebuildah/executor.go +++ b/imagebuildah/executor.go @@ -61,8 +61,8 @@ var builtinAllowedBuildArgs = map[string]bool{ // interface. It coordinates the entire build by using one or more // StageExecutors to handle each stage of the build. type Executor struct { - cacheFrom []reference.Named - cacheTo []reference.Named + cacheFrom []types.ImageReference + cacheTo []types.ImageReference cacheTTL time.Duration containerSuffix string logger *logrus.Logger diff --git a/imagebuildah/stage_executor.go b/imagebuildah/stage_executor.go index 7f1933e3335..7e807769ee4 100644 --- a/imagebuildah/stage_executor.go +++ b/imagebuildah/stage_executor.go @@ -24,6 +24,7 @@ import ( "github.com/containers/buildah/util" config "github.com/containers/common/pkg/config" cp "github.com/containers/image/v5/copy" + imagedirectory "github.com/containers/image/v5/directory" imagedocker "github.com/containers/image/v5/docker" "github.com/containers/image/v5/docker/reference" "github.com/containers/image/v5/manifest" @@ -985,10 +986,16 @@ func (s *StageExecutor) Execute(ctx context.Context, base string) (imgID string, } // logCachePulled produces build log for cases when `--cache-from` // is used and a valid intermediate image is pulled from remote source. - logCachePulled := func(cacheKey string, remote reference.Named) { + logCachePulled := func(cacheKey string, remote types.ImageReference) { if !s.executor.quiet { cachePullMessage := "--> Cache pulled from remote" - fmt.Fprintf(s.executor.out, "%s %s\n", cachePullMessage, fmt.Sprintf("%s:%s", remote.String(), cacheKey)) + repo := "" + if remote.Transport().Name() == imagedocker.Transport.Name() { + repo = remote.DockerReference().String() + } else { + repo = remote.StringWithinTransport() + } + fmt.Fprintf(s.executor.out, "%s %s\n", cachePullMessage, fmt.Sprintf("%s:%s", repo, cacheKey)) } } // logCachePush produces build log for cases when `--cache-to` @@ -1782,18 +1789,29 @@ func (s *StageExecutor) generateCacheKey(ctx context.Context, currNode *parser.N // cacheImageReference is internal function which generates ImageReference from Named repo sources // and a tag. -func cacheImageReferences(repos []reference.Named, cachekey string) ([]types.ImageReference, error) { +func cacheImageReferences(repos []types.ImageReference, cachekey string) ([]types.ImageReference, error) { var result []types.ImageReference for _, repo := range repos { - tagged, err := reference.WithTag(repo, cachekey) - if err != nil { - return nil, fmt.Errorf("failed generating tagged reference for %q: %w", repo, err) - } - dest, err := imagedocker.NewReference(tagged) - if err != nil { - return nil, fmt.Errorf("failed generating docker reference for %q: %w", tagged, err) + if repo.Transport().Name() == imagedocker.Transport.Name() { + ref := repo.DockerReference() + tagged, err := reference.WithTag(ref, cachekey) + if err != nil { + return nil, fmt.Errorf("failed generating tagged reference for %q: %w", repo, err) + } + dest, err := imagedocker.NewReference(tagged) + if err != nil { + return nil, fmt.Errorf("failed generating docker reference for %q: %w", tagged, err) + } + result = append(result, dest) + } else if repo.Transport().Name() == imagedirectory.Transport.Name() { + dirPath := repo.StringWithinTransport() + tagged := filepath.Join(dirPath, cachekey) + dest, err := imagedirectory.NewReference(tagged) + if err != nil { + return nil, fmt.Errorf("failed generating directory reference for %q: %w", tagged, err) + } + result = append(result, dest) } - result = append(result, dest) } return result, nil } @@ -1833,7 +1851,7 @@ func (s *StageExecutor) pushCache(ctx context.Context, src, cacheKey string) err // or a newer version of cache was found in the upstream repo. If new // image was pulled function returns image id otherwise returns empty // string "" or error if any error was encontered while pulling the cache. -func (s *StageExecutor) pullCache(ctx context.Context, cacheKey string) (reference.Named, string, error) { +func (s *StageExecutor) pullCache(ctx context.Context, cacheKey string) (types.ImageReference, string, error) { srcList, err := cacheImageReferences(s.executor.cacheFrom, cacheKey) if err != nil { return nil, "", err @@ -1851,14 +1869,20 @@ func (s *StageExecutor) pullCache(ctx context.Context, cacheKey string) (referen ReportWriter: nil, PullPolicy: define.PullIfNewer, } - id, err := buildah.Pull(ctx, src.DockerReference().String(), options) + srcString := "" + if src.Transport().Name() == imagedocker.Transport.Name() { + srcString = src.DockerReference().String() + } else if src.Transport().Name() == imagedirectory.Transport.Name() { + srcString = "dir://" + src.StringWithinTransport() + } + id, err := buildah.Pull(ctx, srcString, options) if err != nil { logrus.Debugf("failed pulling cache from source %s: %v", src, err) continue // failed pulling this one try next //return "", fmt.Errorf("failed while pulling cache from %q: %w", src, err) } logrus.Debugf("successfully pulled cache from repo %s: %s", src, id) - return src.DockerReference(), id, nil + return src, id, nil } return nil, "", fmt.Errorf("failed pulling cache from all available sources %q", srcList) } diff --git a/pkg/cli/build.go b/pkg/cli/build.go index d670cc7701b..b85fa8d2e80 100644 --- a/pkg/cli/build.go +++ b/pkg/cli/build.go @@ -17,7 +17,6 @@ import ( "github.com/containers/buildah/pkg/parse" "github.com/containers/buildah/pkg/util" "github.com/containers/common/pkg/auth" - "github.com/containers/image/v5/docker/reference" "github.com/containers/image/v5/types" "github.com/opencontainers/runtime-spec/specs-go" "github.com/sirupsen/logrus" @@ -296,8 +295,8 @@ func GenBuildOptions(c *cobra.Command, inputArgs []string, iopts BuildOptions) ( iopts.Quiet = true } } - var cacheTo []reference.Named - var cacheFrom []reference.Named + var cacheTo []types.ImageReference + var cacheFrom []types.ImageReference cacheTo = nil cacheFrom = nil if c.Flag("cache-to").Changed { diff --git a/pkg/parse/parse.go b/pkg/parse/parse.go index 720f493138a..73f5d17efe7 100644 --- a/pkg/parse/parse.go +++ b/pkg/parse/parse.go @@ -21,7 +21,7 @@ import ( "github.com/containers/buildah/pkg/sshagent" "github.com/containers/common/pkg/config" "github.com/containers/common/pkg/parse" - "github.com/containers/image/v5/docker/reference" + "github.com/containers/image/v5/transports/alltransports" "github.com/containers/image/v5/types" "github.com/containers/storage/pkg/idtools" "github.com/containers/storage/pkg/unshare" @@ -53,16 +53,17 @@ const ( ) // RepoNamesToNamedReferences parse the raw string to Named reference -func RepoNamesToNamedReferences(destList []string) ([]reference.Named, error) { - var result []reference.Named +func RepoNamesToNamedReferences(destList []string) ([]types.ImageReference, error) { + var result []types.ImageReference for _, dest := range destList { - named, err := reference.ParseNormalizedNamed(dest) + if t := alltransports.TransportFromImageName(dest); t == nil { + // assume no default transport provided in such case append docker + dest = "docker://" + dest + } + named, err := alltransports.ParseImageName(dest) if err != nil { return nil, fmt.Errorf("invalid repo %q: must contain registry and repository: %w", dest, err) } - if !reference.IsNameOnly(named) { - return nil, fmt.Errorf("repository must contain neither a tag nor digest: %v", named) - } result = append(result, named) } return result, nil