Skip to content

Commit

Permalink
feat(RELEASE-1252): make run-file-updates task idempotent
Browse files Browse the repository at this point in the history
Signed-off-by: Jing Qi <[email protected]>

The PR is to make the task idempotent by ensuring that a new MR isn't
 created unnecessarily when one already exists with the same content
in the seed and replacements of paths parameter.
And it added some logic to check the keys in the paths parameter. If the
 keys exist but their content does not require any replacement, return
 success.
  • Loading branch information
jinqi7 committed Dec 13, 2024
1 parent 6bd8cf8 commit 8ba7160
Show file tree
Hide file tree
Showing 4 changed files with 191 additions and 4 deletions.
3 changes: 3 additions & 0 deletions internal/tasks/process-file-updates-task/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,9 @@ replacements to a yaml file that already exists. It will attempt to create a Mer
| tempDir | temp dir for cloning and updates | Yes | /tmp/$(context.taskRun.uid)/file-updates |
| internalRequestPipelineRunName | name of the PipelineRun that called this task | No | - |

## Changes in 0.1.0
* make run-file-updates task idempotent

## Changes in 0.0.2
* add new `internalRequestPipelineRunName` parameter and result
- Tekton only supports passing task results as pipeline results,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ kind: Task
metadata:
name: process-file-updates-task
labels:
app.kubernetes.io/version: "0.0.2"
app.kubernetes.io/version: "0.1.0"
annotations:
tekton.dev/pipelines.minVersion: "0.12.1"
tekton.dev/tags: release
Expand Down Expand Up @@ -160,6 +160,7 @@ spec:
exit 1
fi
keyNotFound=false
for (( REPLACEMENT_INDEX=0; REPLACEMENT_INDEX < REPLACEMENTS_LENGTH; REPLACEMENT_INDEX++ )); do
echo "REPLACEMENT: #${REPLACEMENT_INDEX}"
key="$(jq -cr ".[${PATH_INDEX}].replacements[${REPLACEMENT_INDEX}].key" "${updatePathsTmpfile}")"
Expand All @@ -173,6 +174,7 @@ spec:
foundAt=$(head -n 1 "${TEMP}/found.txt")
if (( foundAt == 0 )); then
echo "NOT FOUND"
keyNotFound=true
continue
fi
echo "FOUND"
Expand Down Expand Up @@ -236,14 +238,70 @@ spec:
error="\"no replacements were performed\"" \
yq -o json --null-input '.str = strenv(error), .error = strenv(error)' \
| tee "$(results.fileUpdatesInfo.path)"
echo -n "Failed" |tee "$(results.fileUpdatesState.path)"
if [[ "$keyNotFound" == true ]]; then
echo -n "Failed" |tee "$(results.fileUpdatesState.path)"
else
echo -n "Success" |tee "$(results.fileUpdatesState.path)"
fi
# it should exit 0 otherwise the task does not set the results
# this way the InternalRequest can see what was wrong
exit 0
fi
echo -e "\n*** START LOCAL CHANGES ***\n"
git diff
git diff | tee "${TEMP}"/tempMRFile.diff
if [[ -s "${TEMP}"/tempMRFile.diff ]]; then
# replacements exist
# It's to deal with the lines like "@@ -N1,N2 +N3,N4 @@",
# but it's possible to meet a line like "@@ -4,7 +4,7 @@ $schema: /app/app-settings.yml".
awk '/^@@/ {match($0, /@@ ([^@]+) @@/, arr); print arr[1]}' "${TEMP}"/tempMRFile.diff |\
tee "${TEMP}"/lineFile
fi
openMRList=$(glab mr list -R "${UPSTREAM_REPO}" --search "Konflux release" |grep "^!"| cut -f1 | tr -d '!')
for mrNum in ${openMRList} ; do
foundMR=false
glab mr diff "${mrNum}" --repo "${UPSTREAM_REPO}" > "${TEMP}"/oneMR.diff
if [[ ! -s "${TEMP}"/lineFile ]]; then
# only seed files
grep '^[+][^+]' "${TEMP}"/oneMR.diff | sed -E 's/^[+]//' | tee "${TEMP}/mrFile"
if diff -q "${TEMP}"/mrFile "${targetFile}" > /dev/null; then
grep "^diff --git" "${TEMP}"/oneMR.diff | tee "${TEMP}"/tmpFile
while read -r line ; do
changedFileName=$(echo "$line" | awk '{sub(/^b\//, "", $NF); print $NF}')
if [[ "${changedFileName}" != "${targetFile}" ]]; then
continue
fi
foundMR=true
break
done < "${TEMP}"/tmpFile
fi
if [[ "$foundMR" == false ]]; then
continue
fi
else
# replacements exist
grep '^[-+@]' "${TEMP}"/oneMR.diff | sed -E 's/^[@]//; s/@@.*//' | tee "${TEMP}/mrFile"
grep '^[-+@]' "${TEMP}"/tempMRFile.diff | sed -E 's/^[@]//; s/@@.*//' | tee "${TEMP}/tmpFile"
if diff -q "${TEMP}"/mrFile "${TEMP}"/tmpFile > /dev/null; then
foundMR=true
fi
fi
# if all the lines are matched, the MR is the same one
# it should exit 0 if the same MR exists
if [[ "$foundMR" == true ]]; then
echo "there is an existing MR with the same updates in the repo"
echo "{\"merge_request\":\"${UPSTREAM_REPO}/-/merge_requests/${mrNum}\"}" \
| tee -a "$(results.fileUpdatesInfo.path)"
echo -n "Success" | tee "$(results.fileUpdatesState.path)"
exit 0
fi
done
echo -e "\n*** END LOCAL CHANGES ***\n"
WORKING_BRANCH=$(uuidgen |awk '{print substr($1, 1, 8)}')
Expand All @@ -253,7 +311,7 @@ spec:
GITLAB_MR_MSG="[Konflux release] $(params.application): fileUpdates changes ${WORKING_BRANCH}"
gitlab_create_mr --head "$WORKING_BRANCH" --target-branch "$REVISION" --title "${GITLAB_MR_MSG}" \
--description "${GITLAB_MR_MSG}" --upstream-repo "${UPSTREAM_REPO}" | jq '. | tostring' \
|tee -a "$(results.fileUpdatesInfo.path)"
| tee -a "$(results.fileUpdatesInfo.path)"
echo -n "Success" |tee "$(results.fileUpdatesState.path)"
Expand Down
22 changes: 22 additions & 0 deletions internal/tasks/process-file-updates-task/tests/mocks.sh
Original file line number Diff line number Diff line change
Expand Up @@ -28,11 +28,33 @@ function git() {
if [[ "$*" == "config"* ]]; then
/usr/bin/git "$@"
fi
if [[ "$*" == "diff"* ]]; then
/usr/bin/git "$@"
fi
}

function glab() {
if [[ "$*" == *"mr create"* ]]; then
gitRepo=$(echo "$*" | cut -f5 -d/ | cut -f1 -d.)
echo "/merge_request/1"
elif [[ "$*" == *"mr list"* ]]; then
echo '!1'
elif [[ "$*" == *"mr diff"* ]]; then
gitRepo=$(echo "$*" | cut -f5 -d/ | cut -f1 -d.)
if [[ "${gitRepo}" == "replace-idempotent" ]]; then
echo "diff --git a/addons/my-addon2.yaml b/addons/my-addon2.yaml
--- a/addons/my-addon2.yaml
+++ b/addons/my-addon2.yaml
@@ -1,2 +1,2 @@
-indexImage:
+indexImage: Jack
"
else
echo "diff --git a/test/one-update.yaml b/test/one-update.yaml
+++ b/test/one-update.yaml
@@ -1,2 +1,2 @@
+indexImage: Jack
"
fi
fi
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,104 @@
---
apiVersion: tekton.dev/v1
kind: Pipeline
metadata:
name: test-process-file-updates-replacements-idempotent
spec:
description: |
Run the process-file-updates task with replacements. No commit
is created if there is a mocked MR with the same content.
workspaces:
- name: tests-workspace
tasks:
- name: setup
workspaces:
- name: pipeline
workspace: tests-workspace
taskSpec:
workspaces:
- name: pipeline
steps:
- name: setup-values
image: quay.io/konflux-ci/release-service-utils:e633d51cd41d73e4b3310face21bb980af7a662f
script: |
#!/usr/bin/env bash
set -eux
mkdir -p "$(workspaces.pipeline.path)/$(context.pipelineRun.uid)/file-updates"
cd "$(workspaces.pipeline.path)/$(context.pipelineRun.uid)/file-updates"
mkdir replace-idempotent
cd replace-idempotent
git config --global init.defaultBranch main
git init .
git config --global user.email "[email protected]"
git config --global user.name "tester"
mkdir addons
cat > "addons/my-addon2.yaml" << EOF
indexImage:
name: test
EOF
git add addons/my-addon2.yaml
git commit -m "prior commit"
- name: run-task
taskRef:
name: process-file-updates-task
params:
- name: upstream_repo
value: "https://some.gitlab/test/replace-idempotent"
- name: repo
value: "https://some.gitlab/test/replace-idempotent"
- name: ref
value: "main"
- name: paths
value: >-
[{"path":"addons/my-addon2.yaml","replacements":[{"key":".indexImage",
"replacement":"|indexImage.*|indexImage: Jack|"}]}]
- name: application
value: "scott"
- name: file_updates_secret
value: "file-updates-secret"
- name: tempDir
value: "$(workspaces.pipeline.path)/$(context.pipelineRun.uid)/file-updates"
- name: internalRequestPipelineRunName
value: $(context.pipelineRun.name)
workspaces:
- name: pipeline
workspace: tests-workspace
runAfter:
- setup
- name: check-result
runAfter:
- run-task
params:
- name: fileUpdatesInfo
value: $(tasks.run-task.results.fileUpdatesInfo)
- name: fileUpdatesState
value: $(tasks.run-task.results.fileUpdatesState)
- name: tempDir
value: "$(workspaces.pipeline.path)/$(context.pipelineRun.uid)/file-updates"
taskSpec:
params:
- name: fileUpdatesInfo
type: string
- name: fileUpdatesState
type: string
- name: tempDir
type: string
steps:
- name: check-result
image: quay.io/konflux-ci/release-service-utils:e633d51cd41d73e4b3310face21bb980af7a662f
script: |
#!/usr/bin/env bash
set -eux
cd "$(params.tempDir)/replace-idempotent"
commits=$(git log --oneline | wc -l)

echo "Test no more commit created except the one in setup step"
test "${commits}" == "1"
rm -rf "$(params.tempDir)"

workspaces:
- name: pipeline
workspace: tests-workspace

0 comments on commit 8ba7160

Please sign in to comment.