Merge pull request #5438 from danishprakash/add-git-ref

add: add support for git sources

add.go

@@ -20,12 +20,14 @@ import (
 
 	"github.com/containers/buildah/copier"
 	"github.com/containers/buildah/define"
+	"github.com/containers/buildah/internal/tmpdir"
 	"github.com/containers/buildah/pkg/chrootuser"
 	"github.com/containers/common/pkg/retry"
 	"github.com/containers/image/v5/pkg/tlsclientconfig"
 	"github.com/containers/image/v5/types"
 	"github.com/containers/storage/pkg/fileutils"
 	"github.com/containers/storage/pkg/idtools"
+	"github.com/containers/storage/pkg/regexp"
 	"github.com/docker/go-connections/tlsconfig"
 	"github.com/hashicorp/go-multierror"
 	digest "github.com/opencontainers/go-digest"
@@ -93,9 +95,27 @@ type AddAndCopyOptions struct {
 	RetryDelay time.Duration
 }
 
-// sourceIsRemote returns true if "source" is a remote location.
+// gitURLFragmentSuffix matches fragments to use as Git reference and build
+// context from the Git repository e.g.
+//
+//	github.com/containers/buildah.git
+//	github.com/containers/buildah.git#main
+//	github.com/containers/buildah.git#v1.35.0
+var gitURLFragmentSuffix = regexp.Delayed(`\.git(?:#.+)?$`)
+
+// sourceIsGit returns true if "source" is a git location.
+func sourceIsGit(source string) bool {
+	return isURL(source) && gitURLFragmentSuffix.MatchString(source)
+}
+
+func isURL(url string) bool {
+	return strings.HasPrefix(url, "http://") || strings.HasPrefix(url, "https://")
+}
+
+// sourceIsRemote returns true if "source" is a remote location
+// and *not* a git repo. Certain github urls such as raw.github.* are allowed.
 func sourceIsRemote(source string) bool {
-	return strings.HasPrefix(source, "http://") || strings.HasPrefix(source, "https://")
+	return isURL(source) && !gitURLFragmentSuffix.MatchString(source)
 }
 
 // getURL writes a tar archive containing the named content
@@ -274,7 +294,7 @@ func (b *Builder) Add(destination string, extract bool, options AddAndCopyOption
 	}
 
 	// Figure out what sorts of sources we have.
-	var localSources, remoteSources []string
+	var localSources, remoteSources, gitSources []string
 	for i, src := range sources {
 		if src == "" {
 			return errors.New("empty source location")
@@ -283,12 +303,22 @@ func (b *Builder) Add(destination string, extract bool, options AddAndCopyOption
 			remoteSources = append(remoteSources, src)
 			continue
 		}
+		if sourceIsGit(src) {
+			gitSources = append(gitSources, src)
+			continue
+		}
 		if !filepath.IsAbs(src) && options.ContextDir == "" {
 			sources[i] = filepath.Join(currentDir, src)
 		}
 		localSources = append(localSources, sources[i])
 	}
 
+	// Treat git sources as a subset of remote sources
+	// differentiating only in how we fetch the two later on.
+	if len(gitSources) > 0 {
+		remoteSources = append(remoteSources, gitSources...)
+	}
+
 	// Check how many items our local source specs matched.  Each spec
 	// should have matched at least one item, otherwise we consider it an
 	// error.
@@ -320,7 +350,7 @@ func (b *Builder) Add(destination string, extract bool, options AddAndCopyOption
 		}
 		numLocalSourceItems += len(localSourceStat.Globbed)
 	}
-	if numLocalSourceItems+len(remoteSources) == 0 {
+	if numLocalSourceItems+len(remoteSources)+len(gitSources) == 0 {
 		return fmt.Errorf("no sources %v found: %w", sources, syscall.ENOENT)
 	}
 
@@ -377,6 +407,9 @@ func (b *Builder) Add(destination string, extract bool, options AddAndCopyOption
 				destCanBeFile = true
 			}
 		}
+		if len(gitSources) > 0 {
+			destMustBeDirectory = true
+		}
 	}
 
 	// We care if the destination either doesn't exist, or exists and is a
@@ -448,7 +481,7 @@ func (b *Builder) Add(destination string, extract bool, options AddAndCopyOption
 		var multiErr *multierror.Error
 		var getErr, closeErr, renameErr, putErr error
 		var wg sync.WaitGroup
-		if sourceIsRemote(src) {
+		if sourceIsRemote(src) || sourceIsGit(src) {
 			pipeReader, pipeWriter := io.Pipe()
 			var srcDigest digest.Digest
 			if options.Checksum != "" {
@@ -457,17 +490,43 @@ func (b *Builder) Add(destination string, extract bool, options AddAndCopyOption
 					return fmt.Errorf("invalid checksum flag: %w", err)
 				}
 			}
+
 			wg.Add(1)
-			go func() {
-				getErr = retry.IfNecessary(context.TODO(), func() error {
-					return getURL(src, chownFiles, mountPoint, renameTarget, pipeWriter, chmodDirsFiles, srcDigest, options.CertPath, options.InsecureSkipTLSVerify)
-				}, &retry.Options{
-					MaxRetry: options.MaxRetries,
-					Delay:    options.RetryDelay,
-				})
-				pipeWriter.Close()
-				wg.Done()
-			}()
+			if sourceIsGit(src) {
+				go func() {
+					var cloneDir string
+					cloneDir, _, getErr = define.TempDirForURL(tmpdir.GetTempDir(), "", src)
+					getOptions := copier.GetOptions{
+						UIDMap:         srcUIDMap,
+						GIDMap:         srcGIDMap,
+						Excludes:       options.Excludes,
+						ExpandArchives: extract,
+						ChownDirs:      chownDirs,
+						ChmodDirs:      chmodDirsFiles,
+						ChownFiles:     chownFiles,
+						ChmodFiles:     chmodDirsFiles,
+						StripSetuidBit: options.StripSetuidBit,
+						StripSetgidBit: options.StripSetgidBit,
+						StripStickyBit: options.StripStickyBit,
+					}
+					writer := io.WriteCloser(pipeWriter)
+					getErr = copier.Get(cloneDir, cloneDir, getOptions, []string{"."}, writer)
+					pipeWriter.Close()
+					wg.Done()
+				}()
+			} else {
+				go func() {
+					getErr = retry.IfNecessary(context.TODO(), func() error {
+						return getURL(src, chownFiles, mountPoint, renameTarget, pipeWriter, chmodDirsFiles, srcDigest, options.CertPath, options.InsecureSkipTLSVerify)
+					}, &retry.Options{
+						MaxRetry: options.MaxRetries,
+						Delay:    options.RetryDelay,
+					})
+					pipeWriter.Close()
+					wg.Done()
+				}()
+			}
+
 			wg.Add(1)
 			go func() {
 				b.ContentDigester.Start("")

define/types.go

@@ -254,9 +254,16 @@ func parseGitBuildContext(url string) (string, string, string) {
 	return gitBranchPart[0], gitSubdir, gitBranch
 }
 
+func isGitTag(remote, ref string) bool {
+	if _, err := exec.Command("git", "ls-remote", "--exit-code", remote, ref).Output(); err != nil {
+		return true
+	}
+	return false
+}
+
 func cloneToDirectory(url, dir string) ([]byte, string, error) {
 	var cmd *exec.Cmd
-	gitRepo, gitSubdir, gitBranch := parseGitBuildContext(url)
+	gitRepo, gitSubdir, gitRef := parseGitBuildContext(url)
 	// init repo
 	cmd = exec.Command("git", "init", dir)
 	combinedOutput, err := cmd.CombinedOutput()
@@ -270,27 +277,23 @@ func cloneToDirectory(url, dir string) ([]byte, string, error) {
 	if err != nil {
 		return combinedOutput, gitSubdir, fmt.Errorf("failed while performing `git remote add`: %w", err)
 	}
-	// fetch required branch or commit and perform checkout
-	// Always default to `HEAD` if nothing specified
-	fetch := "HEAD"
-	if gitBranch != "" {
-		fetch = gitBranch
+
+	if gitRef != "" {
+		if ok := isGitTag(url, gitRef); ok {
+			gitRef += ":refs/tags/" + gitRef
+		}
 	}
-	logrus.Debugf("fetching repo %q and branch (or commit ID) %q to %q", gitRepo, fetch, dir)
-	cmd = exec.Command("git", "fetch", "--depth=1", "origin", "--", fetch)
+
+	logrus.Debugf("fetching repo %q and branch (or commit ID) %q to %q", gitRepo, gitRef, dir)
+	args := []string{"fetch", "-u", "--depth=1", "origin", "--", gitRef}
+	cmd = exec.Command("git", args...)
 	cmd.Dir = dir
 	combinedOutput, err = cmd.CombinedOutput()
 	if err != nil {
 		return combinedOutput, gitSubdir, fmt.Errorf("failed while performing `git fetch`: %w", err)
 	}
-	if fetch == "HEAD" {
-		// We fetched default branch therefore
-		// we don't have any valid `branch` or
-		// `commit` name hence checkout detached
-		// `FETCH_HEAD`
-		fetch = "FETCH_HEAD"
-	}
-	cmd = exec.Command("git", "checkout", fetch)
+
+	cmd = exec.Command("git", "checkout", "FETCH_HEAD")
 	cmd.Dir = dir
 	combinedOutput, err = cmd.CombinedOutput()
 	if err != nil {

tests/bud.bats

@@ -6875,3 +6875,30 @@ _EOF
   run_buildah 125 build --retry-delay=0.142857s --retry=14 --cert-dir ${TEST_SCRATCH_DIR} $cid ${TEST_SCRATCH_DIR}
   assert "$output" =~ "retrying in 142.*ms .*14/14.*"
 }
+
+@test "bud with ADD with git repository source" {
+  _prefetch alpine
+
+  local contextdir=${TEST_SCRATCH_DIR}/add-git
+  mkdir -p $contextdir
+  cat > $contextdir/Dockerfile << _EOF
+FROM alpine
+RUN apk add git
+
+ADD https://github.com/containers/podman.git#v5.0 /podman-branch
+ADD https://github.com/containers/podman.git#v5.0.0 /podman-tag
+_EOF
+
+  run_buildah build -f $contextdir/Dockerfile -t git-image $contextdir
+  run_buildah from --quiet $WITH_POLICY_JSON --name testctr git-image
+
+  run_buildah run testctr -- sh -c 'cd podman-branch && git rev-parse HEAD'
+  local_head_hash=$output
+  run_buildah run testctr -- sh -c 'cd podman-branch && git ls-remote origin v5.0 | cut -f1'
+  assert "$output" = "$local_head_hash"
+
+  run_buildah run testctr -- sh -c 'cd podman-tag && git rev-parse HEAD'
+  local_head_hash=$output
+  run_buildah run testctr -- sh -c 'cd podman-tag && git ls-remote --tags origin v5.0.0^{} | cut -f1'
+  assert "$output" = "$local_head_hash"
+}