From 346ea59caf4ee1310af6a937f894392f6675acbe Mon Sep 17 00:00:00 2001 From: Chuntao Han Date: Wed, 11 Sep 2024 22:54:48 +0800 Subject: [PATCH] Add sast-find-unicode-control task for testing Signed-off-by: Chuntao Han --- .tekton/pull-request.yaml | 10 +- task/sast-find-unicode-control/0.1/README.md | 30 +++ .../0.1/sast-find-unicode-control.yaml | 232 ++++++++++++++++++ task/sast-find-unicode-control/OWNERS | 5 + 4 files changed, 276 insertions(+), 1 deletion(-) create mode 100644 task/sast-find-unicode-control/0.1/README.md create mode 100644 task/sast-find-unicode-control/0.1/sast-find-unicode-control.yaml create mode 100644 task/sast-find-unicode-control/OWNERS diff --git a/.tekton/pull-request.yaml b/.tekton/pull-request.yaml index b08092c7db..d25f096eb4 100644 --- a/.tekton/pull-request.yaml +++ b/.tekton/pull-request.yaml @@ -5,7 +5,7 @@ metadata: name: build-definitions-pull-request annotations: pipelinesascode.tekton.dev/on-cel-expression: (event == "pull_request" && target_branch == "main" && ( !has(body.pull_request) || !body.pull_request.draft) ) || (event == "push" && target_branch.startsWith("gh-readonly-queue/main/")) - pipelinesascode.tekton.dev/task: "[task/git-clone/0.1/git-clone.yaml, .tekton/tasks/buildah.yaml, .tekton/tasks/yaml-lint.yaml, .tekton/tasks/e2e-test.yaml, task/sast-snyk-check/0.2/sast-snyk-check.yaml]" + pipelinesascode.tekton.dev/task: "[task/git-clone/0.1/git-clone.yaml, .tekton/tasks/buildah.yaml, .tekton/tasks/yaml-lint.yaml, .tekton/tasks/e2e-test.yaml, task/sast-snyk-check/0.2/sast-snyk-check.yaml, task/sast-find-unicode-control/0.1/sast-find-unicode-control.yaml]" pipelinesascode.tekton.dev/task-2: "yaml-lint" pipelinesascode.tekton.dev/max-keep-runs: "5" spec: @@ -59,6 +59,14 @@ spec: workspaces: - name: workspace workspace: workspace + - name: sast-find-unicode-control + runAfter: + - fetch-repository + taskRef: + name: sast-find-unicode-control + workspaces: + - name: workspace + workspace: workspace - name: build-container runAfter: - yaml-lint-check diff --git a/task/sast-find-unicode-control/0.1/README.md b/task/sast-find-unicode-control/0.1/README.md new file mode 100644 index 0000000000..7497276d7c --- /dev/null +++ b/task/sast-find-unicode-control/0.1/README.md @@ -0,0 +1,30 @@ +# sast-snyk-check task + +## Description: + +The sast-find-unicode-control task uses [find-unicode-control](https://github.com/siddhesh/find-unicode-control.git) tool to perform Static Application Security Testing (SAST) to look for non-printable unicode characters in all text files in a source tree. + +## Parameters: + +| name | description | +|------------------------------|--------------------------------------------------------------------------------------------------| +| FIND_UNICODE_CONTROL_GIT_URL | URL from repository to find unicode control | +| KFP_GIT_URL | Link to the known-false-positives repository. If left blank, results won't be filtered | +| PROJECT_NVR | Name-Version-Release (NVR) of the scanned project, used to find path exclusions (it is optional) | +| RECORD_EXCLUDED | File to store all excluded findings to (it is optional) | + + +## Results: + +| name | description | +|-------------|--------------------------| +| TEST_OUTPUT | Tekton task test output. | + +## Source repository for image: + +https://github.com/konflux-ci/konflux-test + +## Additional links: + +* https://github.com/siddhesh/find-unicode-control.git +* https://gitlab.cee.redhat.com/osh/known-false-positives.git diff --git a/task/sast-find-unicode-control/0.1/sast-find-unicode-control.yaml b/task/sast-find-unicode-control/0.1/sast-find-unicode-control.yaml new file mode 100644 index 0000000000..83e927a834 --- /dev/null +++ b/task/sast-find-unicode-control/0.1/sast-find-unicode-control.yaml @@ -0,0 +1,232 @@ +apiVersion: tekton.dev/v1 +kind: Task +metadata: + labels: + app.kubernetes.io/version: "0.1" + annotations: + tekton.dev/pipelines.minVersion: "0.12.1" + tekton.dev/tags: "konflux" + name: sast-find-unicode-control +spec: + description: >- + Scans source code for non-printable unicode characters in all text files. + results: + - description: Tekton task test output. + name: TEST_OUTPUT + params: + - description: Image URL. + name: image-url + type: string + # In a future 0.4 version of the task, drop the default to make this required + default: "" + - description: Image digest to report findings for. + name: image-digest + type: string + # In a future 0.4 version of the task, drop the default to make this required + default: "" + - name: FIND_UNICODE_CONTROL_GIT_URL + type: string + description: URL from repository to find unicode control + default: "https://github.com/siddhesh/find-unicode-control.git" + - name: KFP_GIT_URL + type: string + description: URL from repository to download known false positives files + default: "https://gitlab.cee.redhat.com/osh/known-false-positives.git" + - default: "" + description: Name-Version-Release (NVR) of the scanned project, used to find path exclusions (it is optional) + name: PROJECT_NVR + type: string + - default: "" + description: File to store all excluded findings to (it is optional) + name: RECORD_EXCLUDED + type: string + volumes: + steps: + - name: sast-find-unicode-control + image: quay.io/redhat-appstudio/konflux-test:v1.4.6@sha256:5f298d8d990dfa82023e50029b71b08e19c3c9cedb181dfc4bc86c9ecad8700c + # per https://kubernetes.io/docs/concepts/containers/images/#imagepullpolicy-defaulting + # the cluster will set imagePullPolicy to IfNotPresent + workingDir: $(workspaces.workspace.path)/hacbs/$(context.task.name) + volumeMounts: + env: + - name: KFP_GIT_URL + value: $(params.KFP_GIT_URL) + - name: FIND_UNICODE_CONTROL_GIT_URL + value: $(params.FIND_UNICODE_CONTROL_GIT_URL) + - name: PROJECT_NVR + value: $(params.PROJECT_NVR) + - name: RECORD_EXCLUDED + value: $(params.RECORD_EXCLUDED) + - name: SOURCE_CODE_DIR + value: $(workspaces.workspace.path) + script: | + #!/usr/bin/env bash + set -euo pipefail + . /utils.sh + trap 'handle_error $(results.TEST_OUTPUT.path)' EXIT + + SCAN_RESULT="" + + # Installation of Red Hat certificates for cloning Red Hat internal repositories + curl -sS https://certs.corp.redhat.com/certs/2015-IT-Root-CA.pem > /etc/pki/ca-trust/source/anchors/2015-RH-IT-Root-CA.crt + curl -sS https://certs.corp.redhat.com/certs/2022-IT-Root-CA.pem > /etc/pki/ca-trust/source/anchors/2022-IT-Root-CA.pem + update-ca-trust + + + # Clone the source code from upstream repo + clone_unicontrol() + { + # Clone find-unicode-control repository + git clone --depth=1 "${FIND_UNICODE_CONTROL_GIT_URL}" cloned + status=$? + if [ ${status} -ne 0 ]; then + echo "Error: Failed to clone the repository: ${FIND_UNICODE_CONTROL_GIT_URL}" >&2 + show_error + return 1 + else + echo "Message: Successfully cloned the repository: ${FIND_UNICODE_CONTROL_GIT_URL}" >&2 + fi + + # Check usage of find-unicode-control to confirm the tool can work + ./cloned/find_unicode_control.py --help >&2 + status=$? + if [ ${status} -ne 0 ]; then + echo "Error: Failed to check usage of find-unicode-control" >&2 + show_error + return 1 + else + echo "Message: Succeed to check usage of find-unicode-control" >&2 + fi + } + + # Find unicode control + unicontrol_scan() + { + LANG=en_US.utf8 ./cloned/find_unicode_control.py -d -v -p bidi --notests "$SOURCE_CODE_DIR" >raw_sast_find_unicode_control_check_out.txt 2>raw_sast_find_unicode_control_check_out.log + status=$? + if [ "$status" -ne 0 ] && [ "$status" -ne 1 ]; then + echo "Error: sast-find-unicode-control test failed because of the following issues:" >&2 + cat raw_sast_find_unicode_control_check_out.log + show_error + return 1 + else + echo "Message: Succeed to run find-unicode-control" >&2 + fi + + # translate the output format + sed -i raw_sast_find_unicode_control_check_out.txt -E -e 's|(.*:[0-9]+)(.*)|\1: warning:\2|' -e 's|^|Error: UNICONTROL_WARNING:\n|' + status=$? + if [ "$status" -ne 0 ] && [ "$status" -ne 1 ]; then + echo "Error: failed to translate the unicontrol output format" >&2 + show_error + return 1 + else + echo "Message: Succeed to translate the unicontrol output format\n" >&2 + SCAN_RESULT="raw_sast_find_unicode_control_check_out.txt" + fi + } + + # Filter known false positive + filter_kfp() + { + # We check if the KFP_GIT_URL variable is set to apply the filters or not + if [[ -z "${KFP_GIT_URL}" ]]; then + echo "KFP_GIT_URL variable not defined. False positives won't be filtered" >&2 + mv sast_snyk_check_out_all_findings.json filtered_sast_snyk_check_out.json + return 0 + else + echo "Filtering false positives in results files using csfilter-kfp..." >&2 + cmd=（ + csfilter-kfp + --verbose + --kfp-git-url=$KFP_GIT_URL" + ） + + if [[ -z "${PROJECT_NVR}" ]]; then + cmd+=("--project-nvr=$PROJECT_NVR") + if + + if [[ -z "${RECORD_EXCLUDED}" ]]; then + cmd+=("--record-excluded=RECORD_EXCLUDED") + if + + "${cmd[@]}" raw_sast_find_unicode_control_check_out.txt > filtered_sast_find_unicode_control_check_out.json + status=$? + if [ "$status" -ne 0 ]; then + echo "Error: failed to filter known false positives" >&2 + show_error + return 1 + else + echo "Message: Succeed to filter known false positives" >&2 + SCAN_RESULT="filtered_sast_find_unicode_control_check_out.json" + fi + fi + } + + # Process all results as configured with CSGERP_OPTS + process_all_results() + { + CSGERP_OPTS=( + --mode=json + --event='error|warning' + --remove-duplicates + --embed-context=3 + ) + # In order to generate csdiff/v1, we need to add the whole path of the source code as Snyk only provides an URI to embed the context + set -x + csgrep --mode=json --prepend-path-prefix="$SOURCE_CODE_DIR"/ "$SCAN_RESULT" | csgrep "${CSGERP_OPTS[@]}" | csgrep --mode=json --strip-path-prefix="$SOURCE_CODE_DIR"/source/ > processed_sast_find_unicode_control_check_out.json + set +x + + csgrep --mode=evtstat processed_sast_find_unicode_control_check_out.json + csgrep --mode=sarif processed_sast_find_unicode_control_check_out.json > sast_find_unicode_control_check_out.sarif + } + + show_error () + { + note="Task $(context.task.name) failed: For details, check Tekton task log." + ERROR_OUTPUT=$(make_result_json -r ERROR -t "$note") + echo "${ERROR_OUTPUT}" | tee $(results.TEST_OUTPUT.path) + } + + main() + { + cd ${SOURCE_CODE_DIR} + clone_unicontrol + unicontrol_scan + filter_kfp + process_all_results + } + + main + - name: upload + image: quay.io/konflux-ci/oras:latest@sha256:f4b891ee3038a5f13cd92ff4f473faad5601c2434d1c6b9bccdfc134d9d5f820 + workingDir: $(workspaces.workspace.path)/hacbs/$(context.task.name) + env: + - name: IMAGE_URL + value: $(params.image-url) + - name: IMAGE_DIGEST + value: $(params.image-digest) + script: | + #!/usr/bin/env bash + + UPLOAD_FILE=sast_find_unicode_control_check_out.sarif + MEDIA_TYPE=application/sarif+json + + if [ -z "${IMAGE_URL}" ] || [ -z "${IMAGE_DIGEST}" ]; then + echo 'No image-url or image-digest param provided. Skipping upload.' + exit 0; + fi + + if [ ! -f "${UPLOAD_FILE}" ]; then + echo "No ${UPLOAD_FILE} exists. Skipping upload." + exit 0; + fi + + echo "Selecting auth" + select-oci-auth $IMAGE_URL > $HOME/auth.json + echo "Attaching to ${IMAGE_URL} via the OCI 1.1 Referrers API" + oras attach --no-tty --registry-config "$HOME/auth.json" --distribution-spec v1.1-referrers-api --artifact-type "${MEDIA_TYPE}" "${IMAGE_URL}" "${UPLOAD_FILE}:${MEDIA_TYPE}" + echo "Attaching to ${IMAGE_URL} via the OCI 1.1 Referrers Tag" + oras attach --no-tty --registry-config "$HOME/auth.json" --distribution-spec v1.1-referrers-tag --artifact-type "${MEDIA_TYPE}" "${IMAGE_URL}" "${UPLOAD_FILE}:${MEDIA_TYPE}" + workspaces: + - name: workspace diff --git a/task/sast-find-unicode-control/OWNERS b/task/sast-find-unicode-control/OWNERS new file mode 100644 index 0000000000..27203edec2 --- /dev/null +++ b/task/sast-find-unicode-control/OWNERS @@ -0,0 +1,5 @@ +# See the OWNERS docs: https://go.k8s.io/owners +approvers: + - integration-team +reviewers: + - integration-team