Merge branch 'master' into cleanup-rbac

# Conflicts:
#	charts/radix-operator/Chart.yaml

charts/radix-operator/Chart.yaml

@@ -1,7 +1,7 @@
 apiVersion: v2
 name: radix-operator
-version: 1.39.1
-appVersion: 1.59.1
+version: 1.39.2
+appVersion: 1.59.2
 kubeVersion: ">=1.24.0"
 description: Radix Operator
 keywords:

pipeline-runner/internal/jobs/build/acr_test.go

@@ -106,6 +106,7 @@ func assertACRJobSpec(t *testing.T, pushImage bool) {
 		{Name: git.GitSSHKeyVolumeName, VolumeSource: corev1.VolumeSource{Secret: &corev1.SecretVolumeSource{SecretName: git.GitSSHKeyVolumeName, DefaultMode: pointers.Ptr[int32](256)}}},
 		{Name: defaults.AzureACRServicePrincipleSecretName, VolumeSource: corev1.VolumeSource{Secret: &corev1.SecretVolumeSource{SecretName: defaults.AzureACRServicePrincipleSecretName}}},
 		{Name: "radix-image-builder-home", VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{SizeLimit: resource.NewScaledQuantity(5, resource.Mega)}}},
+		{Name: git.CloneRepoHomeVolumeName, VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{SizeLimit: resource.NewScaledQuantity(5, resource.Mega)}}},
 	}
 	for _, image := range componentImages {
 		expectedVolumes = append(expectedVolumes,
@@ -119,11 +120,12 @@ func assertACRJobSpec(t *testing.T, pushImage bool) {
 	assert.ElementsMatch(t, []string{"internal-nslookup", "clone", "internal-chmod"}, slice.Map(job.Spec.Template.Spec.InitContainers, func(c corev1.Container) string { return c.Name }))
 	cloneContainer, _ := slice.FindFirst(job.Spec.Template.Spec.InitContainers, func(c corev1.Container) bool { return c.Name == "clone" })
 	assert.Equal(t, args.GitCloneGitImage, cloneContainer.Image)
-	assert.Equal(t, []string{"git", "clone", "--recurse-submodules", cloneURL, "-b", args.Branch, "--verbose", "--progress", git.Workspace}, cloneContainer.Command)
+	assert.Equal(t, []string{"sh", "-c", "git config --global --add safe.directory /workspace && git clone --recurse-submodules anycloneurl -b anybranch --verbose --progress /workspace && cd /workspace && if [ -n \"$(git lfs ls-files 2>/dev/null)\" ]; then git lfs install && echo 'Pulling large files...' && git lfs pull && echo 'Done'; fi && cd -"}, cloneContainer.Command)
 	assert.Empty(t, cloneContainer.Args)
 	expectedCloneVolumeMounts := []corev1.VolumeMount{
 		{Name: git.BuildContextVolumeName, MountPath: git.Workspace},
 		{Name: git.GitSSHKeyVolumeName, MountPath: "/.ssh", ReadOnly: true},
+		{Name: git.CloneRepoHomeVolumeName, MountPath: git.CloneRepoHomeVolumePath},
 	}
 	assert.ElementsMatch(t, expectedCloneVolumeMounts, cloneContainer.VolumeMounts)
 

pipeline-runner/internal/jobs/build/buildkit_test.go

@@ -133,6 +133,7 @@ func assertBuildKitJobSpec(t *testing.T, useCache, pushImage bool, buildSecrets
 				{Name: "radix-image-builder-home", VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{SizeLimit: resource.NewScaledQuantity(5, resource.Mega)}}},
 				{Name: "build-kit-run", VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{SizeLimit: resource.NewScaledQuantity(100, resource.Giga)}}},
 				{Name: "build-kit-root", VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{SizeLimit: resource.NewScaledQuantity(100, resource.Giga)}}},
+				{Name: git.CloneRepoHomeVolumeName, VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{SizeLimit: resource.NewScaledQuantity(5, resource.Mega)}}},
 			}
 			if len(args.ExternalContainerRegistryDefaultAuthSecret) > 0 {
 				expectedVolumes = append(expectedVolumes, corev1.Volume{
@@ -150,11 +151,12 @@ func assertBuildKitJobSpec(t *testing.T, useCache, pushImage bool, buildSecrets
 			assert.ElementsMatch(t, []string{"internal-nslookup", "clone", "internal-chmod"}, slice.Map(job.Spec.Template.Spec.InitContainers, func(c corev1.Container) string { return c.Name }))
 			cloneContainer, _ := slice.FindFirst(job.Spec.Template.Spec.InitContainers, func(c corev1.Container) bool { return c.Name == "clone" })
 			assert.Equal(t, args.GitCloneGitImage, cloneContainer.Image)
-			assert.Equal(t, []string{"git", "clone", "--recurse-submodules", cloneURL, "-b", args.Branch, "--verbose", "--progress", git.Workspace}, cloneContainer.Command)
+			assert.Equal(t, []string{"sh", "-c", "git config --global --add safe.directory /workspace && git clone --recurse-submodules anycloneurl -b anybranch --verbose --progress /workspace && cd /workspace && if [ -n \"$(git lfs ls-files 2>/dev/null)\" ]; then git lfs install && echo 'Pulling large files...' && git lfs pull && echo 'Done'; fi && cd -"}, cloneContainer.Command)
 			assert.Empty(t, cloneContainer.Args)
 			expectedCloneVolumeMounts := []corev1.VolumeMount{
 				{Name: git.BuildContextVolumeName, MountPath: git.Workspace},
 				{Name: git.GitSSHKeyVolumeName, MountPath: "/.ssh", ReadOnly: true},
+				{Name: git.CloneRepoHomeVolumeName, MountPath: git.CloneRepoHomeVolumePath},
 			}
 			assert.ElementsMatch(t, expectedCloneVolumeMounts, cloneContainer.VolumeMounts)
 

pipeline-runner/internal/jobs/build/common.go

@@ -78,6 +78,14 @@ func getCommonPodVolumes(componentImages []pipeline.BuildComponentImage) []corev
 				},
 			},
 		},
+		{
+			Name: git.CloneRepoHomeVolumeName,
+			VolumeSource: corev1.VolumeSource{
+				EmptyDir: &corev1.EmptyDirVolumeSource{
+					SizeLimit: resource.NewScaledQuantity(5, resource.Mega),
+				},
+			},
+		},
 	}
 
 	for _, image := range componentImages {

pipeline-runner/internal/tekton/tekton.go

@@ -97,6 +97,9 @@ func getJobVolumes() []corev1.Volume {
 		{
 			Name: git.BuildContextVolumeName,
 		},
+		{
+			Name: git.CloneRepoHomeVolumeName,
+		},
 		{
 			Name: git.GitSSHKeyVolumeName,
 			VolumeSource: corev1.VolumeSource{

pipeline-runner/steps/preparepipeline/step.go

@@ -200,7 +200,7 @@ func (cli *PreparePipelinesStepImplementation) getPreparePipelinesJobConfig(pipe
 			Value: strings.Join(pipelineInfo.PipelineArguments.DNSConfig.ReservedDNSAliases, ","),
 		},
 	}
-	initContainers := git.CloneInitContainersWithContainerName(registration.Spec.CloneURL, configBranch, git.CloneConfigContainerName, internalgit.CloneConfigFromPipelineArgs(pipelineInfo.PipelineArguments))
+	initContainers := git.CloneInitContainersWithContainerName(registration.Spec.CloneURL, configBranch, git.CloneConfigContainerName, internalgit.CloneConfigFromPipelineArgs(pipelineInfo.PipelineArguments), false)
 	return internaltekton.CreateActionPipelineJob(defaults.RadixPipelineJobPreparePipelinesContainerName, action, pipelineInfo, appName, initContainers, &envVars)
 }
 

pkg/apis/defaults/environment_variables.go

@@ -223,10 +223,10 @@ const (
 	// RadixGitCloneGitImageEnvironmentVariable The container image containing git, used in pipeline git clone init containers
 	RadixGitCloneGitImageEnvironmentVariable = "RADIX_PIPELINE_GIT_CLONE_GIT_IMAGE"
 
-	// RadixGitCloneGitImageEnvironmentVariable The container image containing bash, used in pipeline git clone init containers
+	// RadixGitCloneBashImageEnvironmentVariable The container image containing bash, used in pipeline git clone init containers
 	RadixGitCloneBashImageEnvironmentVariable = "RADIX_PIPELINE_GIT_CLONE_BASH_IMAGE"
 
-	// Name of the secret containing default credentials for external container registries.
+	// RadixExternalRegistryDefaultAuthEnvironmentVariable Name of the secret containing default credentials for external container registries.
 	// Used when pulling images for components and jobs and for pulling images in Dockerfiles when building with buildah.
 	RadixExternalRegistryDefaultAuthEnvironmentVariable = "RADIX_EXTERNAL_REGISTRY_DEFAULT_AUTH_SECRET"
 )

pkg/apis/utils/git/clone.go

@@ -6,29 +6,12 @@ import (
 
 	"github.com/equinor/radix-common/utils/pointers"
 	"github.com/equinor/radix-operator/pkg/apis/securitycontext"
+	"github.com/equinor/radix-operator/pkg/apis/utils"
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/resource"
 )
 
 const (
-	// InternalContainerPrefix To indicate that this is not for user interest
-	InternalContainerPrefix = "internal-"
-
-	// CloneConfigContainerName Name of container for clone in the outer pipeline
-	CloneConfigContainerName = "clone-config"
-
-	// CloneContainerName Name of container
-	CloneContainerName = "clone"
-
-	// GitSSHKeyVolumeName Deploy key + known_hosts
-	GitSSHKeyVolumeName = "git-ssh-keys"
-
-	// BuildContextVolumeName Name of volume to hold build context
-	BuildContextVolumeName = "build-context"
-
-	// Workspace Folder to hold the code to build
-	Workspace = "/workspace"
-
 	// The script to ensure that github responds before cloning. It breaks after max attempts
 	waitForGithubToRespond = "n=1;max=10;delay=2;while true; do if [ \"$n\" -lt \"$max\" ]; then nslookup github.com && break; n=$((n+1)); sleep $(($delay*$n)); else echo \"The command has failed after $n attempts.\"; break; fi done"
 )
@@ -41,25 +24,29 @@ type CloneConfig struct {
 
 // CloneInitContainers The sidecars for cloning repo
 func CloneInitContainers(sshURL, branch string, config CloneConfig) []corev1.Container {
-	return CloneInitContainersWithContainerName(sshURL, branch, CloneContainerName, config)
+	return CloneInitContainersWithContainerName(sshURL, branch, CloneContainerName, config, true)
 }
 
 // CloneInitContainersWithContainerName The sidecars for cloning repo
-func CloneInitContainersWithContainerName(sshURL, branch, cloneContainerName string, config CloneConfig) []corev1.Container {
-	gitCloneCmd := []string{"git", "clone", "--recurse-submodules", sshURL, "-b", branch, "--verbose", "--progress", Workspace}
+func CloneInitContainersWithContainerName(sshURL, branch, cloneContainerName string, config CloneConfig, useLfs bool) []corev1.Container {
+	gitConfigCommand := fmt.Sprintf("git config --global --add safe.directory %s", Workspace)
+	gitCloneCommand := fmt.Sprintf("git clone --recurse-submodules %s -b %s --verbose --progress %s", sshURL, branch, Workspace)
+	getLfsFilesCommands := fmt.Sprintf("cd %s && if [ -n \"$(git lfs ls-files 2>/dev/null)\" ]; then git lfs install && echo 'Pulling large files...' && git lfs pull && echo 'Done'; fi && cd -", Workspace)
+	gitCloneCmd := []string{"sh", "-c", fmt.Sprintf("%s && %s %s", gitConfigCommand, gitCloneCommand,
+		utils.TernaryString(useLfs, fmt.Sprintf("&& %s", getLfsFilesCommands), ""))}
 	containers := []corev1.Container{
 		{
-			Name:            fmt.Sprintf("%snslookup", InternalContainerPrefix),
-			Image:           config.NSlookupImage,
-			ImagePullPolicy: corev1.PullIfNotPresent,
-			Args:            []string{waitForGithubToRespond},
-			Command:         []string{"/bin/sh", "-c"},
+			Name:    fmt.Sprintf("%snslookup", InternalContainerPrefix),
+			Image:   config.NSlookupImage,
+			Command: []string{"/bin/sh", "-c"},
+			Args:    []string{waitForGithubToRespond},
 			Resources: corev1.ResourceRequirements{
 				Requests: corev1.ResourceList{
 					corev1.ResourceCPU:    *resource.NewScaledQuantity(10, resource.Milli),
 					corev1.ResourceMemory: *resource.NewScaledQuantity(1, resource.Mega),
 				},
 			},
+			ImagePullPolicy: corev1.PullIfNotPresent,
 			SecurityContext: securitycontext.Container(
 				securitycontext.WithContainerRunAsUser(1000), // Any user will probably do
 				securitycontext.WithContainerRunAsGroup(1000),
@@ -73,6 +60,12 @@ func CloneInitContainersWithContainerName(sshURL, branch, cloneContainerName str
 			Image:           config.GitImage,
 			ImagePullPolicy: corev1.PullIfNotPresent,
 			Command:         gitCloneCmd,
+			Env: []corev1.EnvVar{
+				{
+					Name:  "HOME",
+					Value: CloneRepoHomeVolumePath,
+				},
+			},
 			VolumeMounts: []corev1.VolumeMount{
 				{
 					Name:      BuildContextVolumeName,
@@ -83,6 +76,11 @@ func CloneInitContainersWithContainerName(sshURL, branch, cloneContainerName str
 					MountPath: "/.ssh",
 					ReadOnly:  true,
 				},
+				{
+					Name:      CloneRepoHomeVolumeName,
+					MountPath: CloneRepoHomeVolumePath,
+					ReadOnly:  false,
+				},
 			},
 			SecurityContext: securitycontext.Container(
 				securitycontext.WithContainerRunAsUser(65534), // Must be this user when running as non root

pkg/apis/utils/git/clone_test.go

@@ -33,7 +33,7 @@ func Test_CloneInitContainersWithContainerName_CustomImages(t *testing.T) {
 	}
 	cloneName := "anyclonename"
 	cfg := git.CloneConfig{NSlookupImage: "anynslookup:any", GitImage: "anygit:any", BashImage: "anybash:any"}
-	containers := git.CloneInitContainersWithContainerName("anysshurl", "anybranch", cloneName, cfg)
+	containers := git.CloneInitContainersWithContainerName("anysshurl", "anybranch", cloneName, cfg, true)
 	actual := slice.Map(containers, func(c v1.Container) containerInfo { return containerInfo{name: c.Name, image: c.Image} })
 	expected := []containerInfo{
 		{name: fmt.Sprintf("%snslookup", git.InternalContainerPrefix), image: cfg.NSlookupImage},

pkg/apis/utils/git/defaults.go

@@ -0,0 +1,27 @@
+package git
+
+const (
+	// InternalContainerPrefix To indicate that this is not for user interest
+	InternalContainerPrefix = "internal-"
+
+	// CloneConfigContainerName Name of container for clone in the outer pipeline
+	CloneConfigContainerName = "clone-config"
+
+	// CloneContainerName Name of container
+	CloneContainerName = "clone"
+
+	// GitSSHKeyVolumeName Deploy key + known_hosts
+	GitSSHKeyVolumeName = "git-ssh-keys"
+
+	// BuildContextVolumeName Name of volume to hold build context
+	BuildContextVolumeName = "build-context"
+
+	// CloneRepoHomeVolumeName Name of volume to hold clone repo home folder
+	CloneRepoHomeVolumeName = "builder-home"
+
+	// CloneRepoHomeVolumePath Name of home volume path
+	CloneRepoHomeVolumePath = "/home/clone"
+
+	// Workspace Folder to hold the code to build
+	Workspace = "/workspace"
+)