From 067054fbbcbe077fcdbfd98dedf498671bc3aeb5 Mon Sep 17 00:00:00 2001 From: Scott Hebert Date: Wed, 17 Apr 2024 11:38:09 -0400 Subject: [PATCH 1/3] prefetch: add git auth support and logging - needed for private repos - add ability to turn on cachi2 debug logging fixes KFLUXBUGS-1215 Signed-off-by: Scott Hebert --- task/prefetch-dependencies/0.1/README.md | 10 +-- .../0.1/prefetch-dependencies.yaml | 64 ++++++++++++++++++- 2 files changed, 67 insertions(+), 7 deletions(-) diff --git a/task/prefetch-dependencies/0.1/README.md b/task/prefetch-dependencies/0.1/README.md index 076b1f338b..1e8c6b4c87 100644 --- a/task/prefetch-dependencies/0.1/README.md +++ b/task/prefetch-dependencies/0.1/README.md @@ -4,12 +4,14 @@ Task that uses Cachi2 to prefetch build dependencies. See docs at https://github.com/containerbuildsystem/cachi2#basic-usage. ## Parameters -|name|description|default value|required| -|---|---|---|---| -|input|Configures project packages that will have their dependencies prefetched.||true| -|dev-package-managers|Enable in-development package managers. WARNING: the behavior may change at any time without notice. Use at your own risk. |false|false| +|name| description |default value|required| +|---|-----------------------------------------------------------------------------------------------------------------------------------------------------|---|---| +|input| Configures project packages that will have their dependencies prefetched. ||true| +|dev-package-managers| Enable in-development package managers. WARNING: the behavior may change at any time without notice. Use at your own risk. |false|false| +|enable-debug-logging| Enable debug logging with cachi2 |false|false| ## Workspaces |name|description|optional| |---|---|---| |source|Workspace with the source code, cachi2 artifacts will be stored on the workspace as well|false| +|basic-auth|A Workspace containing a .gitconfig and .git-credentials file or username and password. These will be copied to the user's home before any git commands are run. Any other files in this Workspace are ignored. It is strongly recommended to use ssh-directory over basic-auth whenever possible and to bind a Secret to this Workspace over other volume types. |true| diff --git a/task/prefetch-dependencies/0.1/prefetch-dependencies.yaml b/task/prefetch-dependencies/0.1/prefetch-dependencies.yaml index a290248a15..460bc6dc94 100644 --- a/task/prefetch-dependencies/0.1/prefetch-dependencies.yaml +++ b/task/prefetch-dependencies/0.1/prefetch-dependencies.yaml @@ -19,6 +19,10 @@ spec: notice. Use at your own risk. name: dev-package-managers default: "false" + - description: > + Enable cachi2 debug logging + name: enable-debug-logging + default: "false" - name: caTrustConfigMapName type: string description: The name of the ConfigMap to read CA bundle data from. @@ -27,6 +31,11 @@ spec: type: string description: The name of the key in the ConfigMap that contains the CA bundle data. default: ca-bundle.crt + - default: /tekton/home + description: | + Absolute path to the user's home directory. Set this explicitly if you are running the image as a non-root user. + name: userHome + type: string steps: - image: quay.io/redhat-appstudio/cachi2:0.7.0@sha256:1fc772aa3636fd0b43d62120d832e5913843e028e8cac42814b487c3a0a32bd8 # per https://kubernetes.io/docs/concepts/containers/images/#imagepullpolicy-defaulting @@ -38,6 +47,14 @@ spec: value: $(params.input) - name: DEV_PACKAGE_MANAGERS value: $(params.dev-package-managers) + - name: ENABLE_DEBUG + value: $(params.enable-debug-logging) + - name: WORKSPACE_BASIC_AUTH_DIRECTORY_BOUND + value: $(workspaces.basic-auth.bound) + - name: WORKSPACE_BASIC_AUTH_DIRECTORY_PATH + value: $(workspaces.basic-auth.path) + - name: PARAM_USER_HOME + value: $(params.userHome) volumeMounts: - name: trusted-ca mountPath: /mnt/trusted-ca @@ -55,6 +72,31 @@ spec: dev_pacman_flag="" fi + if [ "$ENABLE_DEBUG" = "true" ]; then + debug_log_flag="--log-level=debug" + else + debug_log_flag="" + fi + + if [ "${WORKSPACE_BASIC_AUTH_DIRECTORY_BOUND}" = "true" ] ; then + if [ -f "${WORKSPACE_BASIC_AUTH_DIRECTORY_PATH}/.git-credentials" ] && [ -f "${WORKSPACE_BASIC_AUTH_DIRECTORY_PATH}/.gitconfig" ]; then + cp "${WORKSPACE_BASIC_AUTH_DIRECTORY_PATH}/.git-credentials" "${PARAM_USER_HOME}/.git-credentials" + cp "${WORKSPACE_BASIC_AUTH_DIRECTORY_PATH}/.gitconfig" "${PARAM_USER_HOME}/.gitconfig" + # Compatibility with kubernetes.io/basic-auth secrets + elif [ -f "${WORKSPACE_BASIC_AUTH_DIRECTORY_PATH}/username" ] && [ -f "${WORKSPACE_BASIC_AUTH_DIRECTORY_PATH}/password" ]; then + HOSTNAME=$(echo $PARAM_URL | awk -F/ '{print $3}') + echo "https://$(cat ${WORKSPACE_BASIC_AUTH_DIRECTORY_PATH}/username):$(cat ${WORKSPACE_BASIC_AUTH_DIRECTORY_PATH}/password)@$HOSTNAME" > "${PARAM_USER_HOME}/.git-credentials" + echo -e "[credential \"https://$HOSTNAME\"]\n helper = store" > "${PARAM_USER_HOME}/.gitconfig" + else + echo "Unknown basic-auth workspace format" + exit 1 + fi + chmod 400 "${PARAM_USER_HOME}/.git-credentials" + chmod 400 "${PARAM_USER_HOME}/.gitconfig" + # needed or else you'll see "could not read Username for 'https://gitlab.com':" + cd $(workspaces.source.path)/source && git config remote.origin.url $(cat "${PARAM_USER_HOME}/.git-credentials") + fi + ca_bundle=/mnt/trusted-ca/ca-bundle.crt if [ -f "$ca_bundle" ]; then echo "INFO: Using mounted CA bundle: $ca_bundle" @@ -62,22 +104,38 @@ spec: update-ca-trust fi - cachi2 fetch-deps \ + cachi2 \ + $debug_log_flag \ + fetch-deps \ $dev_pacman_flag \ --source=$(workspaces.source.path)/source \ --output=$(workspaces.source.path)/cachi2/output \ "${INPUT}" - cachi2 generate-env $(workspaces.source.path)/cachi2/output \ + cachi2 \ + $debug_log_flag \ + generate-env \ + $(workspaces.source.path)/cachi2/output \ --format env \ --for-output-dir=/cachi2/output \ --output $(workspaces.source.path)/cachi2/cachi2.env - cachi2 inject-files $(workspaces.source.path)/cachi2/output \ + cachi2 \ + $debug_log_flag \ + inject-files \ + $(workspaces.source.path)/cachi2/output \ --for-output-dir=/cachi2/output workspaces: - name: source description: Workspace with the source code, cachi2 artifacts will be stored on the workspace as well + - description: | + A Workspace containing a .gitconfig and .git-credentials file or username and password. + These will be copied to the user's home before any git commands are run. Any + other files in this Workspace are ignored. It is strongly recommended + to use ssh-directory over basic-auth whenever possible and to bind a + Secret to this Workspace over other volume types. + name: basic-auth + optional: true volumes: - name: trusted-ca configMap: From f29639aeeab8d89ef5ffd4a768f560a9654bf97a Mon Sep 17 00:00:00 2001 From: Scott Hebert Date: Thu, 18 Apr 2024 14:38:06 -0400 Subject: [PATCH 2/3] use log level for logging Signed-off-by: Scott Hebert --- task/prefetch-dependencies/0.1/README.md | 2 +- .../0.1/prefetch-dependencies.yaml | 24 ++++++++----------- 2 files changed, 11 insertions(+), 15 deletions(-) diff --git a/task/prefetch-dependencies/0.1/README.md b/task/prefetch-dependencies/0.1/README.md index 1e8c6b4c87..eabf51b5bf 100644 --- a/task/prefetch-dependencies/0.1/README.md +++ b/task/prefetch-dependencies/0.1/README.md @@ -8,7 +8,7 @@ See docs at https://github.com/containerbuildsystem/cachi2#basic-usage. |---|-----------------------------------------------------------------------------------------------------------------------------------------------------|---|---| |input| Configures project packages that will have their dependencies prefetched. ||true| |dev-package-managers| Enable in-development package managers. WARNING: the behavior may change at any time without notice. Use at your own risk. |false|false| -|enable-debug-logging| Enable debug logging with cachi2 |false|false| +|log-level| Set cachi2 log level |info|false| ## Workspaces |name|description|optional| diff --git a/task/prefetch-dependencies/0.1/prefetch-dependencies.yaml b/task/prefetch-dependencies/0.1/prefetch-dependencies.yaml index 460bc6dc94..6e444a97e8 100644 --- a/task/prefetch-dependencies/0.1/prefetch-dependencies.yaml +++ b/task/prefetch-dependencies/0.1/prefetch-dependencies.yaml @@ -20,9 +20,9 @@ spec: name: dev-package-managers default: "false" - description: > - Enable cachi2 debug logging - name: enable-debug-logging - default: "false" + Set cachi2 log level + name: log-level + default: "info" - name: caTrustConfigMapName type: string description: The name of the ConfigMap to read CA bundle data from. @@ -47,8 +47,8 @@ spec: value: $(params.input) - name: DEV_PACKAGE_MANAGERS value: $(params.dev-package-managers) - - name: ENABLE_DEBUG - value: $(params.enable-debug-logging) + - name: LOG_LEVEL + value: $(params.log-level) - name: WORKSPACE_BASIC_AUTH_DIRECTORY_BOUND value: $(workspaces.basic-auth.bound) - name: WORKSPACE_BASIC_AUTH_DIRECTORY_PATH @@ -72,12 +72,6 @@ spec: dev_pacman_flag="" fi - if [ "$ENABLE_DEBUG" = "true" ]; then - debug_log_flag="--log-level=debug" - else - debug_log_flag="" - fi - if [ "${WORKSPACE_BASIC_AUTH_DIRECTORY_BOUND}" = "true" ] ; then if [ -f "${WORKSPACE_BASIC_AUTH_DIRECTORY_PATH}/.git-credentials" ] && [ -f "${WORKSPACE_BASIC_AUTH_DIRECTORY_PATH}/.gitconfig" ]; then cp "${WORKSPACE_BASIC_AUTH_DIRECTORY_PATH}/.git-credentials" "${PARAM_USER_HOME}/.git-credentials" @@ -104,8 +98,10 @@ spec: update-ca-trust fi + log_level_flag="--log-level=${LOG_LEVEL}" + cachi2 \ - $debug_log_flag \ + $log_level_flag \ fetch-deps \ $dev_pacman_flag \ --source=$(workspaces.source.path)/source \ @@ -113,7 +109,7 @@ spec: "${INPUT}" cachi2 \ - $debug_log_flag \ + $log_level_flag \ generate-env \ $(workspaces.source.path)/cachi2/output \ --format env \ @@ -121,7 +117,7 @@ spec: --output $(workspaces.source.path)/cachi2/cachi2.env cachi2 \ - $debug_log_flag \ + $log_level_flag \ inject-files \ $(workspaces.source.path)/cachi2/output \ --for-output-dir=/cachi2/output From 04a33a575bad7f80bb13d169bfcf9b9a74703a34 Mon Sep 17 00:00:00 2001 From: Scott Hebert Date: Thu, 18 Apr 2024 16:21:28 -0400 Subject: [PATCH 3/3] improve handling of gitconfig and gitcredentials Signed-off-by: Scott Hebert --- .../0.1/prefetch-dependencies.yaml | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/task/prefetch-dependencies/0.1/prefetch-dependencies.yaml b/task/prefetch-dependencies/0.1/prefetch-dependencies.yaml index 6e444a97e8..c4aba18076 100644 --- a/task/prefetch-dependencies/0.1/prefetch-dependencies.yaml +++ b/task/prefetch-dependencies/0.1/prefetch-dependencies.yaml @@ -76,19 +76,25 @@ spec: if [ -f "${WORKSPACE_BASIC_AUTH_DIRECTORY_PATH}/.git-credentials" ] && [ -f "${WORKSPACE_BASIC_AUTH_DIRECTORY_PATH}/.gitconfig" ]; then cp "${WORKSPACE_BASIC_AUTH_DIRECTORY_PATH}/.git-credentials" "${PARAM_USER_HOME}/.git-credentials" cp "${WORKSPACE_BASIC_AUTH_DIRECTORY_PATH}/.gitconfig" "${PARAM_USER_HOME}/.gitconfig" + # + # needed or else you'll see "could not read Username for 'https://gitlab.com':" + # See https://wahlnetwork.com/2020/08/11/using-private-git-repositories-as-terraform-modules/ + # (1) add path to creds in gitconfig file + sed -i "s#store#store --file ${PARAM_USER_HOME}/.git-credentials#g" "${PARAM_USER_HOME}/.gitconfig" + # (2) add this snippet to the cloned repo's config + cat "${PARAM_USER_HOME}/.gitconfig" >> $(workspaces.source.path)/source/.git/config + # # Compatibility with kubernetes.io/basic-auth secrets elif [ -f "${WORKSPACE_BASIC_AUTH_DIRECTORY_PATH}/username" ] && [ -f "${WORKSPACE_BASIC_AUTH_DIRECTORY_PATH}/password" ]; then HOSTNAME=$(echo $PARAM_URL | awk -F/ '{print $3}') echo "https://$(cat ${WORKSPACE_BASIC_AUTH_DIRECTORY_PATH}/username):$(cat ${WORKSPACE_BASIC_AUTH_DIRECTORY_PATH}/password)@$HOSTNAME" > "${PARAM_USER_HOME}/.git-credentials" - echo -e "[credential \"https://$HOSTNAME\"]\n helper = store" > "${PARAM_USER_HOME}/.gitconfig" + echo -e "[credential \"https://$HOSTNAME\"]\n helper = store --file ${PARAM_USER_HOME}/.git-credentials" > "${PARAM_USER_HOME}/.gitconfig" else echo "Unknown basic-auth workspace format" exit 1 fi chmod 400 "${PARAM_USER_HOME}/.git-credentials" chmod 400 "${PARAM_USER_HOME}/.gitconfig" - # needed or else you'll see "could not read Username for 'https://gitlab.com':" - cd $(workspaces.source.path)/source && git config remote.origin.url $(cat "${PARAM_USER_HOME}/.git-credentials") fi ca_bundle=/mnt/trusted-ca/ca-bundle.crt