From a9351235b831b3bb8c7820c0c3e2dd320c02108e Mon Sep 17 00:00:00 2001 From: Tom Kennedy Date: Wed, 18 Oct 2023 22:49:07 -0400 Subject: [PATCH] Add submodules to image source configuration Signed-off-by: Tom Kennedy --- cmd/build-init/main.go | 38 ++++++++++++++++------- docs/image.md | 2 ++ pkg/apis/build/v1alpha2/build_pod_test.go | 7 +++++ pkg/apis/build/v1alpha2/build_types.go | 14 ++++----- pkg/apis/build/v1alpha2/image_types.go | 14 ++++----- pkg/apis/core/v1alpha1/source_types.go | 23 +++++++++----- pkg/git/fetch.go | 31 +++++++++--------- 7 files changed, 82 insertions(+), 47 deletions(-) diff --git a/cmd/build-init/main.go b/cmd/build-init/main.go index 30dc02722..c21864e54 100644 --- a/cmd/build-init/main.go +++ b/cmd/build-init/main.go @@ -32,15 +32,16 @@ var ( imageTag = flag.String("imageTag", os.Getenv("IMAGE_TAG"), "tag of image that will get created by the lifecycle") runImage = flag.String("runImage", os.Getenv("RUN_IMAGE"), "The base image from which application images are built.") - gitURL = flag.String("git-url", os.Getenv("GIT_URL"), "The url of the Git repository to initialize.") - gitRevision = flag.String("git-revision", os.Getenv("GIT_REVISION"), "The Git revision to make the repository HEAD.") - blobURL = flag.String("blob-url", os.Getenv("BLOB_URL"), "The url of the source code blob.") - stripComponents = flag.Int("strip-components", getenvInt("BLOB_STRIP_COMPONENTS", 0), "The number of directory components to strip from the blobs content when extracting.") - registryImage = flag.String("registry-image", os.Getenv("REGISTRY_IMAGE"), "The registry location of the source code image.") - hostName = flag.String("dns-probe-hostname", os.Getenv("DNS_PROBE_HOSTNAME"), "hostname to dns poll") - sourceSubPath = flag.String("source-sub-path", os.Getenv("SOURCE_SUB_PATH"), "the subpath inside the source directory that will be the buildpack workspace") - buildChanges = flag.String("build-changes", os.Getenv("BUILD_CHANGES"), "JSON string of build changes and their reason") - descriptorPath = flag.String("project-descriptor-path", os.Getenv("PROJECT_DESCRIPTOR_PATH"), "path to project descriptor file") + gitURL = flag.String("git-url", os.Getenv("GIT_URL"), "The url of the Git repository to initialize.") + gitRevision = flag.String("git-revision", os.Getenv("GIT_REVISION"), "The Git revision to make the repository HEAD.") + gitInitializeSubmodules = flag.Bool("git-initialize-submodules", getenvBool("GIT_INITIALIZE_SUBMODULES"), "Initialize submodules during git clone") + blobURL = flag.String("blob-url", os.Getenv("BLOB_URL"), "The url of the source code blob.") + stripComponents = flag.Int("strip-components", getenvInt("BLOB_STRIP_COMPONENTS", 0), "The number of directory components to strip from the blobs content when extracting.") + registryImage = flag.String("registry-image", os.Getenv("REGISTRY_IMAGE"), "The registry location of the source code image.") + hostName = flag.String("dns-probe-hostname", os.Getenv("DNS_PROBE_HOSTNAME"), "hostname to dns poll") + sourceSubPath = flag.String("source-sub-path", os.Getenv("SOURCE_SUB_PATH"), "the subpath inside the source directory that will be the buildpack workspace") + buildChanges = flag.String("build-changes", os.Getenv("BUILD_CHANGES"), "JSON string of build changes and their reason") + descriptorPath = flag.String("project-descriptor-path", os.Getenv("PROJECT_DESCRIPTOR_PATH"), "path to project descriptor file") builderImage = flag.String("builder-image", os.Getenv("BUILDER_IMAGE"), "The builder image used to build the application") builderName = flag.String("builder-name", os.Getenv("BUILDER_NAME"), "The builder name provided during creation") @@ -207,9 +208,15 @@ func fetchSource(logger *log.Logger, keychain authn.Keychain) error { return err } + var initializeSubmodules bool + if gitInitializeSubmodules != nil { + initializeSubmodules = *gitInitializeSubmodules + } + fetcher := git.Fetcher{ - Logger: logger, - Keychain: gitKeychain, + Logger: logger, + Keychain: gitKeychain, + InitializeSubmodules: initializeSubmodules, } return fetcher.Fetch(appDir, *gitURL, *gitRevision, projectMetadataDir) case *blobURL != "": @@ -297,3 +304,12 @@ func getenvInt(key string, defaultValue int) int { } return atoi } + +func getenvBool(key string) bool { + value := os.Getenv(key) + b, err := strconv.ParseBool(value) + if err != nil { + return false + } + return b +} diff --git a/docs/image.md b/docs/image.md index ab0870418..31e0b810f 100644 --- a/docs/image.md +++ b/docs/image.md @@ -83,11 +83,13 @@ The `source` field is a composition of a source code location and a `subpath`. I git: url: "" revision: "" + initializeSubmodules: false subPath: "" ``` - `git`: (Source Code is a git repository) - `url`: The git repository url. Both https and ssh formats are supported; with ssh format requiring a [ssh secret](secrets.md#git-secrets). - `revision`: The git revision to use. This value may be a commit sha, branch name, or tag. + - `initializeSubmodules`: Initialize submodules inside repo. - `subPath`: A subdirectory within the source folder where application code resides. Can be ignored if the source code resides at the `root` level. * Blob diff --git a/pkg/apis/build/v1alpha2/build_pod_test.go b/pkg/apis/build/v1alpha2/build_pod_test.go index 6ec78d9fb..0411beca6 100644 --- a/pkg/apis/build/v1alpha2/build_pod_test.go +++ b/pkg/apis/build/v1alpha2/build_pod_test.go @@ -7,6 +7,7 @@ import ( "testing" "time" + "cuelang.org/go/pkg/strconv" "github.com/Masterminds/semver/v3" "github.com/sclevine/spec" "github.com/stretchr/testify/assert" @@ -275,6 +276,7 @@ func testBuildPod(t *testing.T, when spec.G, it spec.S) { Git: &corev1alpha1.Git{ URL: "giturl.com/git.git", Revision: "gitrev1234", + InitializeSubmodules: true, }, }, Cache: &buildapi.BuildCacheConfig{ @@ -654,6 +656,11 @@ func testBuildPod(t *testing.T, when spec.G, it spec.S) { Value: build.Spec.Source.Git.Revision, }, ) + assert.Contains(t, pod.Spec.InitContainers[0].Env, + corev1.EnvVar{ + Name: "GIT_INITIALIZE_SUBMODULES", + Value: strconv.FormatBool(build.Spec.Source.Git.InitializeSubmodules), + }) }) it("configures prepare with the blob source", func() { diff --git a/pkg/apis/build/v1alpha2/build_types.go b/pkg/apis/build/v1alpha2/build_types.go index 83b54e4cc..62816278a 100644 --- a/pkg/apis/build/v1alpha2/build_types.go +++ b/pkg/apis/build/v1alpha2/build_types.go @@ -71,13 +71,13 @@ type BuildSpec struct { Cosign *CosignConfig `json:"cosign,omitempty"` DefaultProcess string `json:"defaultProcess,omitempty"` // +listType - Tolerations []corev1.Toleration `json:"tolerations,omitempty"` - NodeSelector map[string]string `json:"nodeSelector,omitempty"` - Affinity *corev1.Affinity `json:"affinity,omitempty"` - RuntimeClassName *string `json:"runtimeClassName,omitempty"` - SchedulerName string `json:"schedulerName,omitempty"` - PriorityClassName string `json:"priorityClassName,omitempty"` - CreationTime string `json:"creationTime,omitempty"` + Tolerations []corev1.Toleration `json:"tolerations,omitempty"` + NodeSelector map[string]string `json:"nodeSelector,omitempty"` + Affinity *corev1.Affinity `json:"affinity,omitempty"` + RuntimeClassName *string `json:"runtimeClassName,omitempty"` + SchedulerName string `json:"schedulerName,omitempty"` + PriorityClassName string `json:"priorityClassName,omitempty"` + CreationTime string `json:"creationTime,omitempty"` } func (bs *BuildSpec) RegistryCacheTag() string { diff --git a/pkg/apis/build/v1alpha2/image_types.go b/pkg/apis/build/v1alpha2/image_types.go index f8553df62..623f8f006 100644 --- a/pkg/apis/build/v1alpha2/image_types.go +++ b/pkg/apis/build/v1alpha2/image_types.go @@ -72,13 +72,13 @@ type ImageBuild struct { Env []corev1.EnvVar `json:"env,omitempty"` Resources corev1.ResourceRequirements `json:"resources,omitempty"` // +listType - Tolerations []corev1.Toleration `json:"tolerations,omitempty"` - NodeSelector map[string]string `json:"nodeSelector,omitempty"` - Affinity *corev1.Affinity `json:"affinity,omitempty"` - RuntimeClassName *string `json:"runtimeClassName,omitempty"` - SchedulerName string `json:"schedulerName,omitempty"` - BuildTimeout *int64 `json:"buildTimeout,omitempty"` - CreationTime string `json:"creationTime,omitempty"` + Tolerations []corev1.Toleration `json:"tolerations,omitempty"` + NodeSelector map[string]string `json:"nodeSelector,omitempty"` + Affinity *corev1.Affinity `json:"affinity,omitempty"` + RuntimeClassName *string `json:"runtimeClassName,omitempty"` + SchedulerName string `json:"schedulerName,omitempty"` + BuildTimeout *int64 `json:"buildTimeout,omitempty"` + CreationTime string `json:"creationTime,omitempty"` } // +k8s:openapi-gen=true diff --git a/pkg/apis/core/v1alpha1/source_types.go b/pkg/apis/core/v1alpha1/source_types.go index 6bcbef216..e62fbc588 100644 --- a/pkg/apis/core/v1alpha1/source_types.go +++ b/pkg/apis/core/v1alpha1/source_types.go @@ -34,8 +34,9 @@ type Source interface { // +k8s:openapi-gen=true // +k8s:deepcopy-gen=true type Git struct { - URL string `json:"url"` - Revision string `json:"revision"` + URL string `json:"url"` + Revision string `json:"revision"` + InitializeSubmodules bool `json:"initializeSubmodules,omitempty"` } func (g *Git) BuildEnvVars() []corev1.EnvVar { @@ -48,6 +49,10 @@ func (g *Git) BuildEnvVars() []corev1.EnvVar { Name: "GIT_REVISION", Value: g.Revision, }, + { + Name: "GIT_INITIALIZE_SUBMODULES", + Value: strconv.FormatBool(g.InitializeSubmodules), + }, } } @@ -165,17 +170,19 @@ const ( // +k8s:openapi-gen=true // +k8s:deepcopy-gen=true type ResolvedGitSource struct { - URL string `json:"url"` - Revision string `json:"revision"` - SubPath string `json:"subPath,omitempty"` - Type GitSourceKind `json:"type"` + URL string `json:"url"` + Revision string `json:"revision"` + SubPath string `json:"subPath,omitempty"` + Type GitSourceKind `json:"type"` + InitializeSubmodules bool `json:"initializeSubmodules,omitempty"` } func (gs *ResolvedGitSource) SourceConfig() SourceConfig { return SourceConfig{ Git: &Git{ - URL: gs.URL, - Revision: gs.Revision, + URL: gs.URL, + Revision: gs.Revision, + InitializeSubmodules: gs.InitializeSubmodules, }, SubPath: gs.SubPath, } diff --git a/pkg/git/fetch.go b/pkg/git/fetch.go index 990f30960..8020d95d0 100644 --- a/pkg/git/fetch.go +++ b/pkg/git/fetch.go @@ -15,8 +15,9 @@ import ( ) type Fetcher struct { - Logger *log.Logger - Keychain GitKeychain + Logger *log.Logger + Keychain GitKeychain + InitializeSubmodules bool } func init() { @@ -71,20 +72,22 @@ func (f Fetcher) Fetch(dir, gitURL, gitRevision, metadataDir string) error { return errors.Wrapf(err, "checking out revision") } - submodules, err := worktree.Submodules() - if err != nil { - return errors.Wrapf(err, "getting submodules") - } - - for _, submodule := range submodules { - f.Logger.Printf("Updating submodule %v", submodule.Config().URL) - submoduleAuth, err := f.Keychain.Resolve(submodule.Config().URL) + if f.InitializeSubmodules { + submodules, err := worktree.Submodules() if err != nil { - return err + return errors.Wrapf(err, "getting submodules") } - err = submodule.Update(&gogit.SubmoduleUpdateOptions{Auth: submoduleAuth, Init: true, RecurseSubmodules: gogit.DefaultSubmoduleRecursionDepth }) - if err != nil { - return errors.Wrapf(err, "updating submodules") + + for _, submodule := range submodules { + f.Logger.Printf("Updating submodule %v", submodule.Config().URL) + submoduleAuth, err := f.Keychain.Resolve(submodule.Config().URL) + if err != nil { + return err + } + err = submodule.Update(&gogit.SubmoduleUpdateOptions{Auth: submoduleAuth, Init: true, RecurseSubmodules: gogit.DefaultSubmoduleRecursionDepth}) + if err != nil { + return errors.Wrapf(err, "updating submodules") + } } }