From 277cf531418eb8d9c684699c4937bd5b13c2c038 Mon Sep 17 00:00:00 2001 From: Chuntao Han Date: Wed, 11 Sep 2024 22:54:48 +0800 Subject: [PATCH] sast-unicode-check: add version 0.1 task Resolves: https://issues.redhat.com/browse/OSH-739 Signed-off-by: Chuntao Han --- .tekton/pull-request.yaml | 10 +- task/sast-unicode-check/0.1/README.md | 29 ++ .../0.1/sast-unicode-check.yaml | 265 ++++++++++++++++++ task/sast-unicode-check/OWNERS | 5 + 4 files changed, 308 insertions(+), 1 deletion(-) create mode 100644 task/sast-unicode-check/0.1/README.md create mode 100644 task/sast-unicode-check/0.1/sast-unicode-check.yaml create mode 100644 task/sast-unicode-check/OWNERS diff --git a/.tekton/pull-request.yaml b/.tekton/pull-request.yaml index d7172aeae9..d39ca0541f 100644 --- a/.tekton/pull-request.yaml +++ b/.tekton/pull-request.yaml @@ -5,7 +5,7 @@ metadata: name: build-definitions-pull-request annotations: pipelinesascode.tekton.dev/on-cel-expression: (event == "pull_request" && target_branch == "main" && ( !has(body.pull_request) || !body.pull_request.draft) ) || (event == "push" && target_branch.startsWith("gh-readonly-queue/main/")) - pipelinesascode.tekton.dev/task: "[task/git-clone/0.1/git-clone.yaml, .tekton/tasks/buildah.yaml, .tekton/tasks/task-lint.yaml, .tekton/tasks/e2e-test.yaml, task/sast-snyk-check/0.2/sast-snyk-check.yaml]" + pipelinesascode.tekton.dev/task: "[task/git-clone/0.1/git-clone.yaml, .tekton/tasks/buildah.yaml, .tekton/tasks/task-lint.yaml, .tekton/tasks/e2e-test.yaml, task/sast-snyk-check/0.2/sast-snyk-check.yaml, task/sast-unicode-check.yaml/0.1/sast-unicode-check.yaml]" pipelinesascode.tekton.dev/max-keep-runs: "5" spec: params: @@ -52,6 +52,14 @@ spec: workspaces: - name: workspace workspace: workspace + - name: sast-unicode-check + runAfter: + - build-image-index + taskRef: + name: sast-unicode-check + workspaces: + - name: workspace + workspace: workspace - name: build-container runAfter: - task-lint-check diff --git a/task/sast-unicode-check/0.1/README.md b/task/sast-unicode-check/0.1/README.md new file mode 100644 index 0000000000..c70699d845 --- /dev/null +++ b/task/sast-unicode-check/0.1/README.md @@ -0,0 +1,29 @@ +# sast-unicode-check task + +## Description: + +The sast-unicode-check task uses [find-unicode-control](https://github.com/siddhesh/find-unicode-control.git) tool to perform Static Application Security Testing (SAST) to look for non-printable unicode characters in all text files in a source tree. + +## Parameters: + +| name | description | +|------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------| +| FIND_UNICODE_CONTROL_GIT_URL | URL from repository to find unicode control | +| KFP_GIT_URL | Known False Positives git URL, optionally taking a revision delimited by #; If empty, filtering of known false positives is disabled. | +| PROJECT_NVR | Name-Version-Release (NVR) of the scanned project. It is used to find path exclusions (it is optional) | +| RECORD_EXCLUDED | Whether to record the excluded findings (defaults to false). If `true`, the the excluded findings will be stored in `excluded-findings.json`. | + +## Results: + +| name | description | +|---------------|------------------------------------------| +| TEST_OUTPUT | Tekton task test output. | + +## Source repository for image: + +https://github.com/konflux-ci/konflux-test + +## Additional links: + +* https://github.com/siddhesh/find-unicode-control.git +* https://gitlab.cee.redhat.com/osh/known-false-positives.git diff --git a/task/sast-unicode-check/0.1/sast-unicode-check.yaml b/task/sast-unicode-check/0.1/sast-unicode-check.yaml new file mode 100644 index 0000000000..8108ee11d7 --- /dev/null +++ b/task/sast-unicode-check/0.1/sast-unicode-check.yaml @@ -0,0 +1,265 @@ +apiVersion: tekton.dev/v1 +kind: Task +metadata: + labels: + app.kubernetes.io/version: "0.1" + annotations: + tekton.dev/pipelines.minVersion: "0.12.1" + tekton.dev/tags: "konflux" + name: sast-unicode-check +spec: + description: >- + Scans source code for non-printable unicode characters in all text files. + results: + - description: Tekton task test output. + name: TEST_OUTPUT + params: + - description: Image URL. + name: image-url + type: string + default: "" + - description: Image digest to report findings for. + name: image-digest + type: string + default: "" + - name: FIND_UNICODE_CONTROL_GIT_URL + type: string + description: URL from repository to find unicode control + default: "https://github.com/siddhesh/find-unicode-control.git#c2accbfbba7553a8bc1ebd97089ae08ad8347e58" + - name: KFP_GIT_URL + type: string + description: URL from repository to download known false positives files + default: "https://gitlab.cee.redhat.com/osh/known-false-positives.git" + - default: "" + description: | + Name-Version-Release (NVR) of the scanned project. + It is used to find path exclusions (it is optional) + name: PROJECT_NVR + type: string + - default: "false" + description: | + Whether to record the excluded findings (defaults to false). + If `true`, the the excluded findings will be stored in `excluded-findings.json`. + name: RECORD_EXCLUDED + type: string + volumes: + steps: + - name: sast-unicode-check + image: quay.io/redhat-appstudio/konflux-test:v1.4.6@sha256:5f298d8d990dfa82023e50029b71b08e19c3c9cedb181dfc4bc86c9ecad8700c + # per https://kubernetes.io/docs/concepts/containers/images/#imagepullpolicy-defaulting + # the cluster will set imagePullPolicy to IfNotPresent + workingDir: $(workspaces.workspace.path)/hacbs/$(context.task.name) + volumeMounts: + env: + - name: KFP_GIT_URL + value: $(params.KFP_GIT_URL) + - name: FIND_UNICODE_CONTROL_GIT_URL + value: $(params.FIND_UNICODE_CONTROL_GIT_URL) + - name: PROJECT_NVR + value: $(params.PROJECT_NVR) + - name: RECORD_EXCLUDED + value: $(params.RECORD_EXCLUDED) + - name: SOURCE_CODE_DIR + value: $(workspaces.workspace.path) + script: | + #!/usr/bin/env bash + set -exuo pipefail + . /utils.sh + trap 'handle_error $(results.TEST_OUTPUT.path)' EXIT + + SCAN_PROP="" + + # Installation of Red Hat certificates for cloning Red Hat internal repositories + curl -sS https://certs.corp.redhat.com/certs/2015-IT-Root-CA.pem > /etc/pki/ca-trust/source/anchors/2015-RH-IT-Root-CA.crt + curl -sS https://certs.corp.redhat.com/certs/2022-IT-Root-CA.pem > /etc/pki/ca-trust/source/anchors/2022-IT-Root-CA.pem + update-ca-trust + + # Clone the source code from upstream repo + clone_unicontrol() + { + GIT_URL=$(echo ${FIND_UNICODE_CONTROL_GIT_URL} | awk -F'#' '{print $1}') + REV=$(echo ${FIND_UNICODE_CONTROL_GIT_URL} | awk -F'#' '{print $2}') + + # Clone find-unicode-control repository + git clone --depth=1 "${GIT_URL}" cloned >&2 + status=$? + if [ "${status}" -ne 0 ]; then + echo "Error: Failed to clone the repository: ${GIT_URL}" >&2 + return 1 + else + echo "Message: Succeed to clone the repository: ${GIT_URL}" >&2 + fi + + if [[ -n "${REV}" ]]; then + # Get git url suffix + git_url_suffix=$(git -C ./cloned/ rev-parse HEAD) + if [[ "${git_url_suffix}" != "${REV}" ]]; then + (cd cloned && git checkout --quiet "${REV}") + fi + SCAN_PROP="find-unicode-control-git-url:${FIND_UNICODE_CONTROL_GIT_URL}" + else + SCAN_PROP="find-unicode-control-git-url:${FIND_UNICODE_CONTROL_GIT_URL}#${git_url_suffix}" + fi + + # Check usage of find-unicode-control to confirm the tool can work + ./cloned/find_unicode_control.py --help >&2 + status=$? + if [ "${status}" -ne 0 ]; then + echo "Error: Failed to check usage of find-unicode-control" >&2 + return 1 + else + echo "Message: Succeed to check usage of find-unicode-control" >&2 + fi + } + + # Find unicode control + unicontrol_scan() + { + LANG=en_US.utf8 ./cloned/find_unicode_control.py -p bidi -v -d -t "${SOURCE_CODE_DIR}/source" \ + >raw_sast_unicode_check_out.txt \ + 2>raw_sast_unicode_check_out.log \ + || FUC_EXIT_CODE=$? + + if [[ -z "${FUC_EXIT_CODE}" || "${FUC_EXIT_CODE}" -eq 1 ]]; then + # No non-printable unicode characters found, or successfully found + echo "Message: Succeed to run find-unicode-control command" >&2 + else + # Failed to run the command + echo "Error: failed to run find unicode control command" >&2 + return 1 + fi + + # Translate the output format + sed -i raw_sast_unicode_check_out.txt -E -e 's|(.*:[0-9]+)(.*)|\1: warning:\2|' -e 's|^|Error: UNICONTROL_WARNING:\n|' + status=$? + if [ "${status}" -ne 0 ] && [ "${status}" -ne 1 ]; then + echo "Error: failed to translate the unicontrol output format" >&2 + return 1 + else + echo "Message: Succeed to translate the unicontrol output format" >&2 + fi + } + + # Process all results as configured with CSGERP_OPTS + process_all_results() + { + CSGERP_OPTS=( + --mode=json + --event='error|warning' + --remove-duplicates + --embed-context=3 + --set-scan-prop="${SCAN_PROP}" + ) + # In order to generate csdiff/v1, we need to add the whole path of the source code as Snyk only provides an URI to embed the context + csgrep --mode=json --prepend-path-prefix="${SOURCE_CODE_DIR}"/ raw_sast_unicode_check_out.txt \ + | csgrep "${CSGERP_OPTS[@]}" \ + | csgrep --mode=json --strip-path-prefix="${SOURCE_CODE_DIR}"/source/ \ + > processed_sast_unicode_check_out.json + + csgrep --mode=evtstat processed_sast_unicode_check_out.json + } + + # Filter known false positive + filter_kfp() + { + # We check if the KFP_GIT_URL variable is set to apply the filters or not + if [[ -z "${KFP_GIT_URL}" ]]; then + mv processed_sast_unicode_check_out.json sast_unicode_check_out.json + echo "Error: KFP_GIT_URL variable not defined. False positives won't be filtered" >&2 + return 0 + else + echo "Message: Filtering false positives in results files using csfilter-kfp..." >&2 + CMD=( + csfilter-kfp + --verbose + --kfp-git-url="${KFP_GIT_URL}" + ) + + if [[ -n "${PROJECT_NVR}" ]] && [[ "${RECORD_EXCLUDED}" == "true" ]]; then + CMD+=( + --project-nvr="${PROJECT_NVR}" + --record-excluded=raw-excluded-findings.json + ) + fi + + "${CMD[@]}" processed_sast_unicode_check_out.json > sast_unicode_check_out.json + status=$? + if [ "$status" -ne 0 ]; then + echo "Error: failed to filter known false positives" >&2 + return 1 + else + echo "Message: Succeed to filter known false positives" >&2 + fi + + if [ -f "raw-excluded-findings.json" ]; then + csgrep --mode=json --strip-path-prefix="${SOURCE_CODE_DIR}"/source/ raw-excluded-findings.json > excluded-findings.json + fi + fi + } + + show_output() + { + csgrep --mode=sarif sast_unicode_check_out.json > sast_unicode_check_out.sarif + if [[ -z "${FUC_EXIT_CODE}" ]]; then + note="Task $(context.task.name) success: No finding was detected" + ERROR_OUTPUT=$(make_result_json -r SUCCESS -t "$note") + elif [[ "${FUC_EXIT_CODE}" -eq 1 ]] && [[ ! -s sast_unicode_check_out.sarif ]]; then + note="Task $(context.task.name) success: Some findings were detected, but filtered by known false positive" + ERROR_OUTPUT=$(make_result_json -r SUCCESS -t "$note") + else + echo "sast-unicode-check test failed because of the following issues:" + cat processed_sast_unicode_check_out.json + TEST_OUTPUT= + parse_test_output $(context.task.name) sarif sast_unicode_check_out.sarif || true + note="Task $(context.task.name) failed: For details, check Tekton task log." + ERROR_OUTPUT=$(make_result_json -r ERROR -t "$note") + fi + echo "${TEST_OUTPUT:-${ERROR_OUTPUT}}" | tee $(results.TEST_OUTPUT.path) + } + + main() + { + clone_unicontrol + unicontrol_scan + process_all_results + filter_kfp + show_output + } + + main + - name: upload + image: quay.io/konflux-ci/oras:latest@sha256:f4b891ee3038a5f13cd92ff4f473faad5601c2434d1c6b9bccdfc134d9d5f820 + workingDir: $(workspaces.workspace.path)/hacbs/$(context.task.name) + env: + - name: IMAGE_URL + value: $(params.image-url) + - name: IMAGE_DIGEST + value: $(params.image-digest) + script: | + #!/usr/bin/env bash + + if [ -z "${IMAGE_URL}" ] || [ -z "${IMAGE_DIGEST}" ]; then + echo 'No image-url or image-digest param provided. Skipping upload.' + exit 0; + fi + + UPLOAD_FILES="sast_unicode_check_out.sarif excluded-findings.json" + for UPLOAD_FILE in ${UPLOAD_FILES}; do + if [ ! -f "${UPLOAD_FILE}" ]; then + echo "No ${UPLOAD_FILE} exists. Skipping upload." + exit 0; + fi + + if [ "${UPLOAD_FILES}" == "excluded-findings.json" ]; then + MEDIA_TYPE=application/json + else + MEDIA_TYPE=application/sarif+json + fi + + echo "Selecting auth" + select-oci-auth "${IMAGE_URL}" > "${HOME}/auth.json" + echo "Attaching to ${IMAGE_URL}" + oras attach --no-tty --registry-config "$HOME/auth.json" --artifact-type "${MEDIA_TYPE}" "${IMAGE_URL}" "${UPLOAD_FILE}:${MEDIA_TYPE}" + done + workspaces: + - name: workspace diff --git a/task/sast-unicode-check/OWNERS b/task/sast-unicode-check/OWNERS new file mode 100644 index 0000000000..27203edec2 --- /dev/null +++ b/task/sast-unicode-check/OWNERS @@ -0,0 +1,5 @@ +# See the OWNERS docs: https://go.k8s.io/owners +approvers: + - integration-team +reviewers: + - integration-team