Merge branch 'jenkinsci:master' into NS-3176

Makefile

@@ -89,11 +89,18 @@ test: ## Runs the go tests
 	@RUNNING_TESTS=1 go test -tags "$(BUILDTAGS) cgo" $(PACKAGES_FOR_UNIT_TESTS)
 
 .PHONY: e2e
-e2e: deepcopy-gen manifests ## Runs e2e tests, you can use EXTRA_ARGS
+e2e: deepcopy-gen manifests backup-kind-load ## Runs e2e tests, you can use EXTRA_ARGS
 	@echo "+ $@"
 	RUNNING_TESTS=1 go test -parallel=1 "./test/e2e/" -ginkgo.v -tags "$(BUILDTAGS) cgo" -v -timeout 60m -run "$(E2E_TEST_SELECTOR)" \
 		-jenkins-api-hostname=$(JENKINS_API_HOSTNAME) -jenkins-api-port=$(JENKINS_API_PORT) -jenkins-api-use-nodeport=$(JENKINS_API_USE_NODEPORT) $(E2E_TEST_ARGS)
 
+## Backup Section
+
+.PHONY: backup-kind-load
+backup-kind-load: ## Load latest backup image in the cluster
+	@echo "+ $@"
+	make -C backup/pvc backup-kind-load
+
 ## HELM Section
 
 .PHONY: helm
@@ -374,12 +381,12 @@ kind-clean: ## Delete kind cluster
 IMAGE_NAME := quay.io/$(QUAY_ORGANIZATION)/$(QUAY_REGISTRY):$(GITCOMMIT)-amd64
 BUILD_PRESENT := $(shell docker images |grep -q ${IMAGE_NAME})
 ifndef BUILD_PRESENT
-bats-tests: container-runtime-build-amd64 ## Run bats tests
+bats-tests: backup-kind-load container-runtime-build-amd64 ## Run bats tests
 	@echo "+ $@"
 	kind load docker-image ${IMAGE_NAME} --name $(KIND_CLUSTER_NAME)
 	OPERATOR_IMAGE="${IMAGE_NAME}" TERM=xterm bats -T -p test/bats
 else
-bats-tests: ## Run bats tests
+bats-tests: backup-kind-load
 	@echo "+ $@"
 	OPERATOR_IMAGE="${IMAGE_NAME}" TERM=xterm bats -T -p test/bats
 endif

backup/pvc/Makefile

@@ -118,6 +118,12 @@ docker-release-latest: docker-build ## Release image with latest tags (in additi
 docker-release: docker-release-version docker-release-latest ## Release image with version and latest tags (in addition to build tag)
 	@echo "+ $@"
 
+.PHONY: backup-kind-load
+backup-kind-load: docker-build ## Build and load backup img in kind with e2e-test tag
+	@echo "+ $@"
+	docker tag quay.io/$(QUAY_ORGANIZATION)/$(QUAY_REGISTRY)-$(NAME):$(GITCOMMIT) quay.io/$(QUAY_ORGANIZATION)/$(QUAY_REGISTRY)-$(NAME):e2e-test
+	kind load docker-image quay.io/$(QUAY_ORGANIZATION)/$(QUAY_REGISTRY)-$(NAME):e2e-test --name $(KIND_CLUSTER_NAME)
+
 # if this session isn't interactive, then we don't want to allocate a
 # TTY, which would fail, but if it is interactive, we do want to attach
 # so that the user can send e.g. ^C through.
@@ -127,7 +133,7 @@ ifeq ($(INTERACTIVE), 1)
 endif
 
 .PHONY: docker-run
-docker-run: docker-build ## Run the container in docker, you can use EXTRA_ARGS
+docker-run: docker-build
 	@echo "+ $@"
 	docker run --rm -i $(DOCKER_FLAGS) \
 		quay.io/$(QUAY_ORGANIZATION)/$(QUAY_REGISTRY)-$(NAME):$(GITCOMMIT) $(ARGS)

backup/pvc/VERSION.txt

@@ -1 +1 @@
-v0.2.5
+v0.3.0

backup/pvc/bin/get-latest.sh

@@ -2,9 +2,38 @@
 
 set -eo pipefail
 
-[[ -z "${BACKUP_DIR}" ]] && echo "Required 'BACKUP_DIR' env not set" && exit 1
+is_backup_not_exist() {
+    local backup_dir="$1"
+    # Save the current value of 'set -e'
+    local previous_e
+    previous_e=$(set +e; :; echo $?)
+
+    # Temporarily turn off 'set -e'
+    set +e
+
+    # Run ls command to check if any files matching the pattern exist
+    ls "${backup_dir}"/*.tar.* 1> /dev/null 2>&1
+
+    # Store the exit status of the ls command
+    local ls_exit_status=$?
+
+    # Restore the previous value of 'set -e'
+    [ "$previous_e" = "0" ] && set -e
+
+    # Return true if ls command succeeded (no files found), otherwise return false
+    [ $ls_exit_status -ne 0 ]
+}
+
+[[ -z "${BACKUP_DIR}" ]] && { echo "Required 'BACKUP_DIR' env not set"; exit 1; }
+
+# Check if we have any backup
+if is_backup_not_exist "${BACKUP_DIR}"; then
+  echo "-1"
+  exit 0
+fi
+
 # Search for all the tar.* inside the backup dir to support the migration between gzip vs zstd
-latest=$(find ${BACKUP_DIR} -name '*.tar.*' -exec basename {} \; | sort -g | tail -n 1)
+latest=$(find "${BACKUP_DIR}"/*.tar.* -maxdepth 0 -exec basename {} \; | sort -g | tail -n 1)
 
 if [[ "${latest}" == "" ]]; then
   echo "-1"

backup/pvc/bin/run.sh

@@ -2,15 +2,70 @@
 
 set -eo pipefail
 
-[[ -z "${BACKUP_DIR}" ]] && echo "Required 'BACKUP_DIR' env not set" && exit 1;
-[[ -z "${JENKINS_HOME}" ]] && echo "Required 'JENKINS_HOME' env not set" && exit 1;
+# Use 60 as default in case BACKUP_CLEANUP_INTERVAL did not set
+BACKUP_CLEANUP_INTERVAL=${BACKUP_CLEANUP_INTERVAL:=60}
+
+# Ensure required environment variables are set
+check_env_var() {
+    if [[ -z "${!1}" ]]; then
+        echo "Required '$1' environment variable is not set"
+        exit 1
+    fi
+}
+
+is_backup_not_exist() {
+    local backup_dir="$1"
+    # Save the current value of 'set -e'
+    local previous_e
+    previous_e=$(set +e; :; echo $?)
+
+    # Temporarily turn off 'set -e'
+    set +e
+
+    # Run ls command to check if any files matching the pattern exist
+    ls "${backup_dir}"/*.tar.* 1> /dev/null 2>&1
+
+    # Store the exit status of the ls command
+    local ls_exit_status=$?
+
+    # Restore the previous value of 'set -e'
+    [ "$previous_e" = "0" ] && set -e
+
+    # Return true if ls command succeeded (no files found), otherwise return false
+    [ $ls_exit_status -ne 0 ]
+}
+
+# Function to find exceeding backups
+find_exceeding_backups() {
+    local backup_dir="$1"
+    local backup_count="$2"
+    # Check if we have any backup
+    if is_backup_not_exist "${backup_dir}"; then
+        echo "backups not found in ${backup_dir}" >&2
+        return
+    fi
+    find "${backup_dir}"/*.tar.zstd -maxdepth 0 -exec basename {} \; | sort -gr | tail -n +$((backup_count +1))
+}
+
+check_env_var "BACKUP_DIR"
+check_env_var "JENKINS_HOME"
+
+if [[ -z "${BACKUP_COUNT}" ]]; then
+    echo "ATTENTION! No BACKUP_COUNT set, it means you MUST delete old backups manually or by custom script"
+else
+    echo "Retaining only the ${BACKUP_COUNT} most recent backups, cleanup occurs every ${BACKUP_CLEANUP_INTERVAL} seconds"
+fi
 
 while true;
 do
-    sleep 10
-    if [[ ! -z "${BACKUP_COUNT}" ]]; then
-        echo "Trimming to only ${BACKUP_COUNT} recent backups in preparation for new backup"
-        #TODO: add the list of exceeding backup before delete
-        find ${BACKUP_DIR} -maxdepth 1 -name '*.tar.zstd' -exec basename {} \; | sort -gr | tail -n +$((BACKUP_COUNT +1)) | xargs -I '{}' rm ${BACKUP_DIR}/'{}'
+    sleep "$BACKUP_CLEANUP_INTERVAL"
+    if [[ -n "${BACKUP_COUNT}" ]]; then
+        exceeding_backups=$(find_exceeding_backups "${BACKUP_DIR}" "${BACKUP_COUNT}")
+        if [[ -n "$exceeding_backups" ]]; then
+            echo "Removing backups: $(echo "$exceeding_backups" | tr '\n' ', ' | sed 's/,$//')"
+            echo "$exceeding_backups" | while read -r file; do
+                rm "${BACKUP_DIR}/${file}"
+            done
+        fi
     fi
 done

backup/pvc/config.env

@@ -4,3 +4,4 @@ QUAY_ORGANIZATION=jenkins-kubernetes-operator
 QUAY_REGISTRY=backup
 UID=1000
 GID=1000
+KIND_CLUSTER_NAME=jenkins

backup/pvc/e2e/get-latest/test.sh

@@ -30,6 +30,9 @@ touch ${BACKUP_DIR}/8.tar.zstd
 touch ${BACKUP_DIR}/9.tar.zstd
 touch ${BACKUP_DIR}/10.tar.zstd
 touch ${BACKUP_DIR}/11.tar.zstd
+# Emulate backup creation
+BACKUP_TMP_DIR=$(mktemp -d --tmpdir="${BACKUP_DIR}")
+touch ${BACKUP_TMP_DIR}/12.tar.zstd
 
 # Create an instance of the container under testing
 cid="$(docker run -e JENKINS_HOME=${JENKINS_HOME} -v ${JENKINS_HOME}:${JENKINS_HOME}:ro -e BACKUP_DIR=${BACKUP_DIR} -v ${BACKUP_DIR}:${BACKUP_DIR}:rw -d ${docker_image})"
@@ -38,10 +41,20 @@ echo "Docker container ID '${cid}'"
 # Remove test directory and container afterwards
 trap "docker rm -vf $cid > /dev/null;rm -rf ${BACKUP_DIR};rm -rf ${JENKINS_HOME}" EXIT
 
+echo "Try to get latest against 11 backups and one in progress"
 latest=$(docker exec ${cid} /bin/bash -c "JENKINS_HOME=${RESTORE_FOLDER};/home/user/bin/get-latest.sh")
+
 rm ${BACKUP_DIR}/*.tar.zstd
+echo "Try to get latest against one in progress"
 empty_latest=$(docker exec ${cid} /bin/bash -c "JENKINS_HOME=${RESTORE_FOLDER};/home/user/bin/get-latest.sh")
 
+rmdir ${BACKUP_DIR}/lost+found
+rm ${BACKUP_TMP_DIR}/*.tar.zstd
+rmdir ${BACKUP_TMP_DIR}
+echo "Try to get latest against empty dir"
+empty_dir_latest=$(docker exec ${cid} /bin/bash -c "JENKINS_HOME=${RESTORE_FOLDER};/home/user/bin/get-latest.sh")
+
+
 if [[ "${DEBUG}" ]]; then
     docker logs ${cid}
     ls -la ${BACKUP_DIR}
@@ -55,5 +68,9 @@ if [[ ! "${empty_latest}" == "-1" ]]; then
     echo "Latest backup number should be '-1' but is '${empty_latest}'"
     exit 1
 fi
+if [[ ! "${empty_dir_latest}" == "-1" ]]; then
+    echo "Latest backup number should be '-1' but is '${empty_dir_latest}'"
+    exit 1
+fi
 
 echo PASS

backup/pvc/e2e/limit_backup_count/test.sh

@@ -1,6 +1,8 @@
 #!/bin/bash
 set -eo pipefail
 
+echo "Running limit_backup_count e2e test..."
+
 [[ "${DEBUG}" ]] && set -x
 
 # set current working directory to the directory of the script
@@ -30,17 +32,20 @@ touch ${BACKUP_DIR}/8.tar.zstd
 touch ${BACKUP_DIR}/9.tar.zstd
 touch ${BACKUP_DIR}/10.tar.zstd
 touch ${BACKUP_DIR}/11.tar.zstd
+# Emulate backup creation
+BACKUP_TMP_DIR=$(mktemp -d --tmpdir="${BACKUP_DIR}")
+touch ${BACKUP_TMP_DIR}/12.tar.zstd
 
 # Create an instance of the container under testing
-cid="$(docker run -e BACKUP_COUNT=2 -e JENKINS_HOME=${JENKINS_HOME} -v ${JENKINS_HOME}:${JENKINS_HOME}:ro -e BACKUP_DIR=${BACKUP_DIR} -v ${BACKUP_DIR}:${BACKUP_DIR}:rw -d ${docker_image})"
+cid="$(docker run -e BACKUP_CLEANUP_INTERVAL=1 -e BACKUP_COUNT=2 -e JENKINS_HOME=${JENKINS_HOME} -v ${JENKINS_HOME}:${JENKINS_HOME}:ro -e BACKUP_DIR=${BACKUP_DIR} -v ${BACKUP_DIR}:${BACKUP_DIR}:rw -d ${docker_image})"
 echo "Docker container ID '${cid}'"
 
 # Remove test directory and container afterwards
 trap "docker rm -vf $cid > /dev/null;rm -rf ${BACKUP_DIR};rm -rf ${JENKINS_HOME}" EXIT
 
-sleep 11
-touch ${BACKUP_DIR}/12.tar.zstd
-sleep 11
+sleep 2
+mv ${BACKUP_TMP_DIR}/12.tar.zstd ${BACKUP_DIR}/
+sleep 2
 
 if [[ "${DEBUG}" ]]; then
     docker logs ${cid}

backup/pvc/e2e/limit_backup_count_no_backups/test.sh

@@ -1,6 +1,8 @@
 #!/bin/bash
 set -eo pipefail
 
+echo "Running limit_backup_count_no_backups e2e test..."
+
 [[ "${DEBUG}" ]] && set -x
 
 # set current working directory to the directory of the script
@@ -19,16 +21,24 @@ mkdir -p ${BACKUP_DIR}
 mkdir -p ${JENKINS_HOME}
 
 # Create an instance of the container under testing
-cid="$(docker run -e BACKUP_COUNT=2 -e JENKINS_HOME=${JENKINS_HOME} -v ${JENKINS_HOME}:${JENKINS_HOME}:ro -e BACKUP_DIR=${BACKUP_DIR} -v ${BACKUP_DIR}:${BACKUP_DIR}:rw -d ${docker_image})"
+cid="$(docker run -e BACKUP_CLEANUP_INTERVAL=1 -e BACKUP_COUNT=2 -e JENKINS_HOME=${JENKINS_HOME} -v ${JENKINS_HOME}:${JENKINS_HOME}:ro -e BACKUP_DIR=${BACKUP_DIR} -v ${BACKUP_DIR}:${BACKUP_DIR}:rw -d ${docker_image})"
 echo "Docker container ID '${cid}'"
 
 # Remove test directory and container afterwards
 trap "docker rm -vf $cid > /dev/null;rm -rf ${BACKUP_DIR};rm -rf ${JENKINS_HOME}" EXIT
 
 # container should be running
 echo 'Checking if container is running'
-sleep 11
+sleep 3
+set +e
 docker exec ${cid} echo
+exit_code=$?
+set -e
+if [ $exit_code -ne 0 ]; then
+    echo "container terminated with following logs:"
+    docker logs "${cid}"
+    exit 1
+fi
 echo 'Container is running'
 
 echo PASS

backup/pvc/e2e/tmp_dir_clean_after_backup_creation/test.sh

@@ -1,6 +1,8 @@
 #!/bin/bash
 set -eo pipefail
 
+echo "Running tmp_dir_clean_after_backup_creation e2e test..."
+
 [[ "${DEBUG}" ]] && set -x
 
 # set current working directory to the directory of the script
@@ -18,7 +20,7 @@ BACKUP_DIR="$(pwd)/backup"
 mkdir -p ${BACKUP_DIR}
 
 # Create an instance of the container under testing
-cid="$(docker run -e JENKINS_HOME=${JENKINS_HOME} -v ${JENKINS_HOME}:${JENKINS_HOME}:ro -e BACKUP_DIR=${BACKUP_DIR} -v ${BACKUP_DIR}:${BACKUP_DIR}:rw -d ${docker_image})"
+cid="$(docker run -e BACKUP_CLEANUP_INTERVAL=1 -e JENKINS_HOME=${JENKINS_HOME} -v ${JENKINS_HOME}:${JENKINS_HOME}:ro -e BACKUP_DIR=${BACKUP_DIR} -v ${BACKUP_DIR}:${BACKUP_DIR}:rw -d ${docker_image})"
 echo "Docker container ID '${cid}'"
 
 # Remove test directory and container afterwards
@@ -28,6 +30,8 @@ backup_number=1
 docker exec ${cid} /home/user/bin/backup.sh ${backup_number}
 
 [ "$(docker exec ${cid} ls /tmp | grep 'tmp')" ] && echo "tmp directory not empty" && exit 1;
+# We should also check backup directory, since after #1000 we create temp directory at backup filesystem
+[ "$(docker exec ${cid} ls ${BACKUP_DIR} | grep 'tmp')" ] && echo "backup dir consists temp directory" && exit 1;
 
 backup_file="${BACKUP_DIR}/${backup_number}.tar.zstd"
 [[ ! -f ${backup_file} ]] && echo "Backup file ${backup_file} not found" && exit 1;

chart/jenkins-operator/README.md

@@ -54,7 +54,7 @@ Kubernetes native operator which fully manages Jenkins on Kubernetes
 | jenkins.enabled | bool | `true` |  |
 | jenkins.env | list | `[]` |  |
 | jenkins.hostAliases | object | `{}` |  |
-| jenkins.image | string | `"jenkins/jenkins:2.452.1-lts"` |  |
+| jenkins.image | string | `"jenkins/jenkins:2.452.2-lts"` |  |
 | jenkins.imagePullPolicy | string | `"Always"` |  |
 | jenkins.imagePullSecrets | list | `[]` |  |
 | jenkins.labels | object | `{}` |  |

chart/jenkins-operator/values.yaml

@@ -36,7 +36,7 @@ jenkins:
   # image is the name (and tag) of the Jenkins instance
   # Default: jenkins/jenkins:lts
   # It's recommended to use LTS (tag: "lts") version
-  image: jenkins/jenkins:2.452.1-lts
+  image: jenkins/jenkins:2.452.2-lts
 
   # env contains jenkins container environment variables
   env: []
@@ -94,7 +94,7 @@ jenkins:
   #
   # basePlugins:
   # - name: kubernetes
-  #   version: 4238.v41b_3ef14a_5d8
+  #   version: 4246.v5a_12b_1fe120e
   # - name: workflow-job
   #   version: 1400.v7fd111b_ec82f
   # - name: workflow-aggregator
@@ -139,7 +139,7 @@ jenkins:
   #    repositoryUrl: https://github.com/jenkinsci/kubernetes-operator.git
   seedJobs: []
 
-  # SeedJobAgentImage defines the image that will be used by the seed job agent. If not defined jenkins/inbound-agent:3206.vb_15dcf73f6a_9-3 will be used.
+  # SeedJobAgentImage defines the image that will be used by the seed job agent. If not defined jenkins/inbound-agent:3248.v65ecb_254c298-6 will be used.
   seedJobAgentImage: ""
 
   # Resource limit/request for Jenkins

config.base.env

@@ -7,7 +7,7 @@ GO_VERSION="1.15.6"
 HELM_VERSION="3.12.3"
 IMAGE_PULL_MODE="local"
 KIND_CLUSTER_NAME="jenkins"
-LATEST_LTS_VERSION="2.452.1"
+LATEST_LTS_VERSION="2.452.2"
 NAME="kubernetes-operator"
 NAMESPACE="default"
 OPERATOR_SDK_VERSION="1.3.0"

pkg/configuration/user/seedjobs/seedjobs.go

@@ -46,7 +46,7 @@ const (
 	AgentName = "seed-job-agent"
 
 	// DefaultAgentImage is the default image used for the seed-job agent
-	defaultAgentImage = "jenkins/inbound-agent:3248.v65ecb_254c298-4"
+	defaultAgentImage = "jenkins/inbound-agent:3248.v65ecb_254c298-6"
 
 	creatingGroovyScriptName = "seed-job-groovy-script.groovy"
 

pkg/configuration/user/seedjobs/seedjobs_test.go

@@ -113,7 +113,7 @@ func TestEnsureSeedJobs(t *testing.T) {
 		var agentDeployment appsv1.Deployment
 		err = fakeClient.Get(ctx, types.NamespacedName{Namespace: jenkins.Namespace, Name: agentDeploymentName(*jenkins, AgentName)}, &agentDeployment)
 		assert.NoError(t, err)
-		assert.Equal(t, "jenkins/inbound-agent:3248.v65ecb_254c298-4", agentDeployment.Spec.Template.Spec.Containers[0].Image)
+		assert.Equal(t, "jenkins/inbound-agent:3248.v65ecb_254c298-6", agentDeployment.Spec.Template.Spec.Containers[0].Image)
 		assert.Equal(t, "JENKINS_WEB_SOCKET", agentDeployment.Spec.Template.Spec.Containers[0].Env[0].Name)
 		assert.Equal(t, "true", agentDeployment.Spec.Template.Spec.Containers[0].Env[0].Value)
 	})

pkg/constants/constants.go

@@ -8,7 +8,7 @@ const (
 	// SeedJobSuffix is a suffix added for all seed jobs
 	SeedJobSuffix = "job-dsl-seed"
 	// DefaultJenkinsMasterImage is the default Jenkins master docker image
-	DefaultJenkinsMasterImage = "jenkins/jenkins:2.452.1-lts"
+	DefaultJenkinsMasterImage = "jenkins/jenkins:2.452.2-lts"
 	// DefaultHTTPPortInt32 is the default Jenkins HTTP port
 	DefaultHTTPPortInt32 = int32(8080)
 	// DefaultSlavePortInt32 is the default Jenkins port for slaves