diff --git a/add.go b/add.go index f5a54a34d19..036a15958e1 100644 --- a/add.go +++ b/add.go @@ -20,12 +20,14 @@ import ( "github.com/containers/buildah/copier" "github.com/containers/buildah/define" + "github.com/containers/buildah/internal/tmpdir" "github.com/containers/buildah/pkg/chrootuser" "github.com/containers/common/pkg/retry" "github.com/containers/image/v5/pkg/tlsclientconfig" "github.com/containers/image/v5/types" "github.com/containers/storage/pkg/fileutils" "github.com/containers/storage/pkg/idtools" + "github.com/containers/storage/pkg/regexp" "github.com/docker/go-connections/tlsconfig" "github.com/hashicorp/go-multierror" digest "github.com/opencontainers/go-digest" @@ -93,9 +95,27 @@ type AddAndCopyOptions struct { RetryDelay time.Duration } -// sourceIsRemote returns true if "source" is a remote location. +// gitURLFragmentSuffix matches fragments to use as Git reference and build +// context from the Git repository e.g. +// +// github.com/containers/buildah.git +// github.com/containers/buildah.git#main +// github.com/containers/buildah.git#v1.35.0 +var gitURLFragmentSuffix = regexp.Delayed(`\.git(?:#.+)?$`) + +// sourceIsGit returns true if "source" is a git location. +func sourceIsGit(source string) bool { + return isURL(source) && gitURLFragmentSuffix.MatchString(source) +} + +func isURL(url string) bool { + return strings.HasPrefix(url, "http://") || strings.HasPrefix(url, "https://") +} + +// sourceIsRemote returns true if "source" is a remote location +// and *not* a git repo. Certain github urls such as raw.github.* are allowed. func sourceIsRemote(source string) bool { - return strings.HasPrefix(source, "http://") || strings.HasPrefix(source, "https://") + return isURL(source) && !gitURLFragmentSuffix.MatchString(source) } // getURL writes a tar archive containing the named content @@ -274,7 +294,7 @@ func (b *Builder) Add(destination string, extract bool, options AddAndCopyOption } // Figure out what sorts of sources we have. - var localSources, remoteSources []string + var localSources, remoteSources, gitSources []string for i, src := range sources { if src == "" { return errors.New("empty source location") @@ -283,12 +303,22 @@ func (b *Builder) Add(destination string, extract bool, options AddAndCopyOption remoteSources = append(remoteSources, src) continue } + if sourceIsGit(src) { + gitSources = append(gitSources, src) + continue + } if !filepath.IsAbs(src) && options.ContextDir == "" { sources[i] = filepath.Join(currentDir, src) } localSources = append(localSources, sources[i]) } + // Treat git sources as a subset of remote sources + // differentiating only in how we fetch the two later on. + if len(gitSources) > 0 { + remoteSources = append(remoteSources, gitSources...) + } + // Check how many items our local source specs matched. Each spec // should have matched at least one item, otherwise we consider it an // error. @@ -320,7 +350,7 @@ func (b *Builder) Add(destination string, extract bool, options AddAndCopyOption } numLocalSourceItems += len(localSourceStat.Globbed) } - if numLocalSourceItems+len(remoteSources) == 0 { + if numLocalSourceItems+len(remoteSources)+len(gitSources) == 0 { return fmt.Errorf("no sources %v found: %w", sources, syscall.ENOENT) } @@ -377,6 +407,9 @@ func (b *Builder) Add(destination string, extract bool, options AddAndCopyOption destCanBeFile = true } } + if len(gitSources) > 0 { + destMustBeDirectory = true + } } // We care if the destination either doesn't exist, or exists and is a @@ -448,7 +481,7 @@ func (b *Builder) Add(destination string, extract bool, options AddAndCopyOption var multiErr *multierror.Error var getErr, closeErr, renameErr, putErr error var wg sync.WaitGroup - if sourceIsRemote(src) { + if sourceIsRemote(src) || sourceIsGit(src) { pipeReader, pipeWriter := io.Pipe() var srcDigest digest.Digest if options.Checksum != "" { @@ -457,17 +490,43 @@ func (b *Builder) Add(destination string, extract bool, options AddAndCopyOption return fmt.Errorf("invalid checksum flag: %w", err) } } + wg.Add(1) - go func() { - getErr = retry.IfNecessary(context.TODO(), func() error { - return getURL(src, chownFiles, mountPoint, renameTarget, pipeWriter, chmodDirsFiles, srcDigest, options.CertPath, options.InsecureSkipTLSVerify) - }, &retry.Options{ - MaxRetry: options.MaxRetries, - Delay: options.RetryDelay, - }) - pipeWriter.Close() - wg.Done() - }() + if sourceIsGit(src) { + go func() { + var cloneDir string + cloneDir, _, getErr = define.TempDirForURL(tmpdir.GetTempDir(), "", src) + getOptions := copier.GetOptions{ + UIDMap: srcUIDMap, + GIDMap: srcGIDMap, + Excludes: options.Excludes, + ExpandArchives: extract, + ChownDirs: chownDirs, + ChmodDirs: chmodDirsFiles, + ChownFiles: chownFiles, + ChmodFiles: chmodDirsFiles, + StripSetuidBit: options.StripSetuidBit, + StripSetgidBit: options.StripSetgidBit, + StripStickyBit: options.StripStickyBit, + } + writer := io.WriteCloser(pipeWriter) + getErr = copier.Get(cloneDir, cloneDir, getOptions, []string{"."}, writer) + pipeWriter.Close() + wg.Done() + }() + } else { + go func() { + getErr = retry.IfNecessary(context.TODO(), func() error { + return getURL(src, chownFiles, mountPoint, renameTarget, pipeWriter, chmodDirsFiles, srcDigest, options.CertPath, options.InsecureSkipTLSVerify) + }, &retry.Options{ + MaxRetry: options.MaxRetries, + Delay: options.RetryDelay, + }) + pipeWriter.Close() + wg.Done() + }() + } + wg.Add(1) go func() { b.ContentDigester.Start("") diff --git a/define/types.go b/define/types.go index bfe075bbfa7..4f917b7db4a 100644 --- a/define/types.go +++ b/define/types.go @@ -254,9 +254,16 @@ func parseGitBuildContext(url string) (string, string, string) { return gitBranchPart[0], gitSubdir, gitBranch } +func isGitTag(remote, ref string) bool { + if _, err := exec.Command("git", "ls-remote", "--exit-code", remote, ref).Output(); err != nil { + return true + } + return false +} + func cloneToDirectory(url, dir string) ([]byte, string, error) { var cmd *exec.Cmd - gitRepo, gitSubdir, gitBranch := parseGitBuildContext(url) + gitRepo, gitSubdir, gitRef := parseGitBuildContext(url) // init repo cmd = exec.Command("git", "init", dir) combinedOutput, err := cmd.CombinedOutput() @@ -270,27 +277,23 @@ func cloneToDirectory(url, dir string) ([]byte, string, error) { if err != nil { return combinedOutput, gitSubdir, fmt.Errorf("failed while performing `git remote add`: %w", err) } - // fetch required branch or commit and perform checkout - // Always default to `HEAD` if nothing specified - fetch := "HEAD" - if gitBranch != "" { - fetch = gitBranch + + if gitRef != "" { + if ok := isGitTag(url, gitRef); ok { + gitRef += ":refs/tags/" + gitRef + } } - logrus.Debugf("fetching repo %q and branch (or commit ID) %q to %q", gitRepo, fetch, dir) - cmd = exec.Command("git", "fetch", "--depth=1", "origin", "--", fetch) + + logrus.Debugf("fetching repo %q and branch (or commit ID) %q to %q", gitRepo, gitRef, dir) + args := []string{"fetch", "-u", "--depth=1", "origin", "--", gitRef} + cmd = exec.Command("git", args...) cmd.Dir = dir combinedOutput, err = cmd.CombinedOutput() if err != nil { return combinedOutput, gitSubdir, fmt.Errorf("failed while performing `git fetch`: %w", err) } - if fetch == "HEAD" { - // We fetched default branch therefore - // we don't have any valid `branch` or - // `commit` name hence checkout detached - // `FETCH_HEAD` - fetch = "FETCH_HEAD" - } - cmd = exec.Command("git", "checkout", fetch) + + cmd = exec.Command("git", "checkout", "FETCH_HEAD") cmd.Dir = dir combinedOutput, err = cmd.CombinedOutput() if err != nil { diff --git a/tests/bud.bats b/tests/bud.bats index 36a271e2d8a..dea9ccb21a0 100644 --- a/tests/bud.bats +++ b/tests/bud.bats @@ -6875,3 +6875,30 @@ _EOF run_buildah 125 build --retry-delay=0.142857s --retry=14 --cert-dir ${TEST_SCRATCH_DIR} $cid ${TEST_SCRATCH_DIR} assert "$output" =~ "retrying in 142.*ms .*14/14.*" } + +@test "bud with ADD with git repository source" { + _prefetch alpine + + local contextdir=${TEST_SCRATCH_DIR}/add-git + mkdir -p $contextdir + cat > $contextdir/Dockerfile << _EOF +FROM alpine +RUN apk add git + +ADD https://github.com/containers/podman.git#v5.0 /podman-branch +ADD https://github.com/containers/podman.git#v5.0.0 /podman-tag +_EOF + + run_buildah build -f $contextdir/Dockerfile -t git-image $contextdir + run_buildah from --quiet $WITH_POLICY_JSON --name testctr git-image + + run_buildah run testctr -- sh -c 'cd podman-branch && git rev-parse HEAD' + local_head_hash=$output + run_buildah run testctr -- sh -c 'cd podman-branch && git ls-remote origin v5.0 | cut -f1' + assert "$output" = "$local_head_hash" + + run_buildah run testctr -- sh -c 'cd podman-tag && git rev-parse HEAD' + local_head_hash=$output + run_buildah run testctr -- sh -c 'cd podman-tag && git ls-remote --tags origin v5.0.0^{} | cut -f1' + assert "$output" = "$local_head_hash" +}