From 586f9407fb9c9b73f27fe5b95eb831d4835c1536 Mon Sep 17 00:00:00 2001 From: Adam Date: Tue, 28 Nov 2023 08:47:39 +0100 Subject: [PATCH 1/5] ci: introduce a lightweight workflow which utilizes composite actions Jira issue: ADDON-66448 --- .github/actions/appinspect-api/action.yml | 40 +++ .github/actions/appinspect-cli/action.yml | 44 +++ .github/actions/artifact-registry/action.yml | 94 ++++++ .github/actions/build/action.yml | 179 ++++++++++ .../actions/compliance-copyrights/action.yml | 14 + .github/actions/fossa-scan/action.yml | 30 ++ .github/actions/fossa-test/action.yml | 23 ++ .github/actions/lint/action.yml | 19 ++ .github/actions/meta/action.yml | 95 ++++++ .../requirements-unit-tests/action.yml | 27 ++ .github/actions/review-secrets/action.yml | 27 ++ .github/actions/semgrep/action.yml | 19 ++ .github/actions/test-inventory/action.yml | 44 +++ .github/actions/unit-tests/action.yml | 54 +++ .github/actions/validate-pr-title/action.yml | 19 ++ .github/actions/virustotal/action.yml | 23 ++ .github/workflows/reusable-lightweight.yml | 307 ++++++++++++++++++ 17 files changed, 1058 insertions(+) create mode 100644 .github/actions/appinspect-api/action.yml create mode 100644 .github/actions/appinspect-cli/action.yml create mode 100644 .github/actions/artifact-registry/action.yml create mode 100644 .github/actions/build/action.yml create mode 100644 .github/actions/compliance-copyrights/action.yml create mode 100644 .github/actions/fossa-scan/action.yml create mode 100644 .github/actions/fossa-test/action.yml create mode 100644 .github/actions/lint/action.yml create mode 100644 .github/actions/meta/action.yml create mode 100644 .github/actions/requirements-unit-tests/action.yml create mode 100644 .github/actions/review-secrets/action.yml create mode 100644 .github/actions/semgrep/action.yml create mode 100644 .github/actions/test-inventory/action.yml create mode 100644 .github/actions/unit-tests/action.yml create mode 100644 .github/actions/validate-pr-title/action.yml create mode 100644 .github/actions/virustotal/action.yml create mode 100644 .github/workflows/reusable-lightweight.yml diff --git a/.github/actions/appinspect-api/action.yml b/.github/actions/appinspect-api/action.yml new file mode 100644 index 000000000..2ed50520c --- /dev/null +++ b/.github/actions/appinspect-api/action.yml @@ -0,0 +1,40 @@ +name: AppInspect + +description: > + Performs validation checks on your Splunk app package against a set of standardized criteria to evaluate + the app structure, features, security, and adherence to Splunk Cloud Platform requirements. + Uses AppInspect API. + +inputs: + matrix_tags: + required: true + SPL_COM_USER: + required: true + SPL_COM_PASSWORD: + required: true + +runs: + using: composite + steps: + - name: Checkout repository + uses: actions/checkout@v3 + + - name: Download artifact + uses: actions/download-artifact@v3 + with: + name: package-splunkbase + path: build/package/ + + - name: AppInspect API + uses: splunk/appinspect-api-action@v3.0 + with: + username: ${{ inputs.SPL_COM_USER }} + password: ${{ inputs.SPL_COM_PASSWORD }} + app_path: build/package/ + included_tags: ${{ inputs.matrix_tags }} + + - uses: actions/upload-artifact@v3 + if: always() + with: + name: appinspect-api-html-report-${{ inputs.matrix_tags }} + path: AppInspect_response.html \ No newline at end of file diff --git a/.github/actions/appinspect-cli/action.yml b/.github/actions/appinspect-cli/action.yml new file mode 100644 index 000000000..97e139a57 --- /dev/null +++ b/.github/actions/appinspect-cli/action.yml @@ -0,0 +1,44 @@ +name: AppInspect + +description: > + Performs validation checks on your Splunk app package against a set of standardized criteria to evaluate + the app structure, features, security, and adherence to Splunk Cloud Platform requirements. + Uses AppInspect CLI. + +inputs: + matrix_tags: + required: true + +runs: + using: composite + steps: + - name: Checkout repository + uses: actions/checkout@v3 + + - name: Download artifact + uses: actions/download-artifact@v3 + with: + name: package-splunkbase + path: build/package/ + + - name: Scan + uses: splunk/appinspect-cli-action@v1.9 + with: + app_path: build/package/ + included_tags: ${{ inputs.matrix_tags }} + result_file: appinspect_result_${{ inputs.matrix_tags }}.json + + - name: Upload AppInspect report + if: ${{ !cancelled() }} + uses: actions/upload-artifact@v3 + with: + name: appinspect_${{ inputs.matrix_tags }}_checks.json + path: appinspect_result_${{ inputs.matrix_tags }}.json + + - name: Upload Markdown + if: inputs.matrix_tags == 'manual' + uses: actions/upload-artifact@v3 + with: + name: check_markdown + path: | + *_markdown.txt \ No newline at end of file diff --git a/.github/actions/artifact-registry/action.yml b/.github/actions/artifact-registry/action.yml new file mode 100644 index 000000000..544ab3728 --- /dev/null +++ b/.github/actions/artifact-registry/action.yml @@ -0,0 +1,94 @@ +name: Artifact registry + +description: Uploads the generated addon build to ghcr.io + +inputs: + sc4s: + required: true + +outputs: + artifact: + value: ${{ steps.artifactid.outputs.result }} + +runs: + using: composite + steps: + - name: Checkout repository + uses: actions/checkout@v3 + + - name: Download artifact + uses: actions/download-artifact@v3 + with: + name: package-splunkbase + path: build/package/splunkbase + + - name: Get app ID + id: getappid + shell: bash + run: | + appid=$(jq -r '.info.id.name' package/app.manifest) + echo appid="$appid" + echo "result=$appid" >> "$GITHUB_OUTPUT" + + - name: Download ORAS + shell: bash + run: | + curl -LO https://github.com/oras-project/oras/releases/download/v0.12.0/oras_0.12.0_linux_amd64.tar.gz + mkdir -p oras-install/ + tar -zxf oras_0.12.0_*.tar.gz -C oras-install/ + mv oras-install/oras /usr/local/bin/ + rm -rf oras_0.12.0_*.tar.gz oras-install/ + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + + - name: Login to GitHub Packages Docker Registry + uses: docker/login-action@v2.2.0 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ github.token }} + + - name: Docker meta + id: meta + uses: docker/metadata-action@v4.6.0 + with: + images: ghcr.io/${{ github.repository }} + tags: | + type=semver,pattern=v{{major}}.{{minor}},prefix=${{ steps.getappid.outputs.result }}- + type=semver,pattern=v{{major}},prefix=${{ steps.getappid.outputs.result }}- + type=semver,pattern=v{{version}},prefix=${{ steps.getappid.outputs.result }}- + type=semver,pattern={{major}}.{{minor}},prefix=${{ steps.getappid.outputs.result }}- + type=semver,pattern={{major}},prefix=${{ steps.getappid.outputs.result }}- + type=semver,pattern={{version}},prefix=${{ steps.getappid.outputs.result }}- + type=ref,event=branch,prefix=${{ steps.getappid.outputs.result }}- + type=ref,event=pr,prefix=${{ steps.getappid.outputs.result }}- + type=sha,prefix=${{ steps.getappid.outputs.result }}- + type=sha,format=long,prefix=${{ steps.getappid.outputs.result }}- + + - name: Upload artifacts + shell: bash + run: | + tee /tmp/tags &>/dev/null <>$line<<" + oras push \ + --manifest-config /dev/null:application/vnd.splunk.ent.package.v1.tar+gzip \ + "$line" \ + "${{ steps.getappid.outputs.result }}".spl + echo " complete" + done < /tmp/tags + popd + + - name: Output artifact locator + id: artifactid + shell: bash + run: | + echo "result= ${{ inputs.sc4s }}" >> "$GITHUB_OUTPUT" diff --git a/.github/actions/build/action.yml b/.github/actions/build/action.yml new file mode 100644 index 000000000..94bbe56ef --- /dev/null +++ b/.github/actions/build/action.yml @@ -0,0 +1,179 @@ +name: Build + +description: > + Creates the UCC build for the addon, generates the SPL file using slim + and uploads it to the Github registry + +inputs: + python_version: + required: true + SA_GH_USER_NAME: + required: true + SA_GH_USER_EMAIL: + required: true + SA_GPG_PRIVATE_KEY: + required: true + SA_GPG_PASSPHRASE: + required: true + AWS_ACCESS_KEY_ID: + required: true + AWS_DEFAULT_REGION: + required: true + AWS_SECRET_ACCESS_KEY: + required: true + ucc_modinput_functional: + required: true + modinput_functional: + required: true + +outputs: + buildname: + value: ${{ steps.buildupload.outputs.name }} + +runs: + using: composite + steps: + - name: Checkout repository + uses: actions/checkout@v3 + with: + # Very Important: semantic-release won't trigger a tagged + # build if this is not set to false + persist-credentials: false + + - name: Setup python + uses: actions/setup-python@v4 + with: + python-version: 3.7 + + - name: Create requirements file for pip + shell: bash + run: | + if [ -f "poetry.lock" ] + then + echo " poetry.lock found " + sudo pip3 install poetry==1.5.1 poetry-plugin-export==1.4.0 + poetry lock --check + poetry export --without-hashes -o requirements.txt + if [ "$(grep -cve '^\s*$' requirements.txt)" -ne 0 ] + then + echo "Prod dependencies were found, creating package/lib folder" + mkdir -p package/lib || true + mv requirements.txt package/lib + else + echo "No prod dependencies were found" + rm requirements.txt + fi + poetry export --without-hashes --dev -o requirements_dev.txt + cat requirements_dev.txt + fi + + - name: Get pip cache dir + id: pip-cache + shell: bash + run: | + echo "dir=$(pip cache dir)" >> "$GITHUB_OUTPUT" + + - name: Run Check there are libraries to scan + id: checklibs + shell: bash + run: if [ -f requirements_dev.txt ]; then echo "ENABLED=true" >> "$GITHUB_OUTPUT"; fi + + - name: Run pip cache + if: ${{ steps.checklibs.outputs.ENABLED == 'true' }} + uses: actions/cache@v3 + with: + path: ${{ steps.pip-cache.outputs.dir }} + key: ${{ runner.os }}-pip-${{ hashFiles('requirements_dev.txt') }} + restore-keys: | + ${{ runner.os }}-pip- + + - name: Install deps + if: ${{ steps.checklibs.outputs.ENABLED == 'true' }} + shell: bash + run: pip install -r requirements_dev.txt + + - name: Semantic Release Get Next + id: semantic + if: github.event_name != 'pull_request' + uses: splunk/semantic-release-action@v1.3 + with: + dry_run: true + git_committer_name: ${{ inputs.SA_GH_USER_NAME }} + git_committer_email: ${{ inputs.SA_GH_USER_EMAIL }} + gpg_private_key: ${{ inputs.SA_GPG_PRIVATE_KEY }} + passphrase: ${{ inputs.SA_GPG_PASSPHRASE }} + env: + GITHUB_TOKEN: ${{ github.token }} + + - name: Determine the version to build + id: BuildVersion + uses: splunk/addonfactory-get-splunk-package-version-action@v1 + with: + SemVer: ${{ steps.semantic.outputs.new_release_version }} + PrNumber: ${{ github.event.number }} + + - name: Download THIRDPARTY + if: ${{ inputs.python_version }} == '3.7' && github.event_name != 'pull_request' && github.event_name != 'schedule' + uses: actions/download-artifact@v3 + with: + name: THIRDPARTY + + - name: Download THIRDPARTY (Optional for PR and schedule) + if: ${{ inputs.python_version }} == '3.7' && github.event_name == 'pull_request' || github.event_name == 'schedule' + continue-on-error: true + uses: actions/download-artifact@v3 + with: + name: THIRDPARTY + + - name: Update Notices + if: ${{ inputs.python_version }} == '3.7' + shell: bash + run: | + cp -f THIRDPARTY package/THIRDPARTY || echo "THIRDPARTY file not found (allowed for PR and schedule)" + + - name: Build Package + id: uccgen + uses: splunk/addonfactory-ucc-generator-action@v2 + with: + version: ${{ steps.BuildVersion.outputs.VERSION }} + + - name: Slim Package + if: always() && ${{ inputs.python_version }} == '3.7' + id: slim + uses: splunk/addonfactory-packaging-toolkit-action@v1 + with: + source: ${{ steps.uccgen.outputs.OUTPUT }} + + - name: Artifact OpenAPI + if: ${{ inputs.python_version }} == '3.7' && ${{ !cancelled() && inputs.ucc_modinput_functional == 'true' && inputs.modinput_functional == 'true' }} + uses: actions/upload-artifact@v3 + with: + name: artifact-openapi + path: ${{ github.workspace }}/${{ steps.uccgen.outputs.OUTPUT }}/static/openapi.json + + - name: Artifact Splunkbase + if: ${{ !cancelled() }} && ${{ inputs.python_version }} == '3.7' + uses: actions/upload-artifact@v3 + with: + name: package-splunkbase + path: ${{ steps.slim.outputs.OUTPUT }} + + - name: Upload build to S3 + if: ${{ inputs.python_version }} == '3.7' + id: buildupload + shell: bash + env: + AWS_ACCESS_KEY_ID: ${{ inputs.AWS_ACCESS_KEY_ID }} + AWS_DEFAULT_REGION: ${{ inputs.AWS_DEFAULT_REGION }} + AWS_SECRET_ACCESS_KEY: ${{ inputs.AWS_SECRET_ACCESS_KEY }} + run: | + echo "name=$(basename "${{ steps.slim.outputs.OUTPUT }}")" >> "$GITHUB_OUTPUT" + basename "${{ steps.slim.outputs.OUTPUT }}" + aws s3 cp "${{ steps.slim.outputs.OUTPUT }}" s3://ta-production-artifacts/ta-apps/ + + - name: Artifact Splunk parts + if: ${{ !cancelled() }} && ${{ inputs.python_version }} == '3.7' + uses: actions/upload-artifact@v3 + with: + name: package-deployment + path: build/package/deployment** diff --git a/.github/actions/compliance-copyrights/action.yml b/.github/actions/compliance-copyrights/action.yml new file mode 100644 index 000000000..0319acfeb --- /dev/null +++ b/.github/actions/compliance-copyrights/action.yml @@ -0,0 +1,14 @@ +name: Compliance copyrights + +description: > + Reuse compliance check. Analyze third-party dependencies, generate a report, + and upload the report as an artifact for further inspection or use. + +runs: + using: composite + steps: + - name: Checkout repository + uses: actions/checkout@v3 + + - name: REUSE Compliance Check + uses: fsfe/reuse-action@v1.1 \ No newline at end of file diff --git a/.github/actions/fossa-scan/action.yml b/.github/actions/fossa-scan/action.yml new file mode 100644 index 000000000..061a8070f --- /dev/null +++ b/.github/actions/fossa-scan/action.yml @@ -0,0 +1,30 @@ +name: Fossa scan + +description: > + Analyzes third-party dependencies, generate a report, and upload + the report as an artifact for further inspection or use. + +inputs: + FOSSA_API_KEY: + required: true + +runs: + using: composite + steps: + - name: Checkout repository + uses: actions/checkout@v3 + + - name: Run fossa anlysis and create report + shell: bash + run: | + curl -H 'Cache-Control: no-cache' https://raw.githubusercontent.com/fossas/fossa-cli/master/install-latest.sh | bash + fossa analyze --debug + fossa report attribution --format text --timeout 600 > /tmp/THIRDPARTY + env: + FOSSA_API_KEY: ${{ inputs.FOSSA_API_KEY }} + + - name: Upload THIRDPARTY file + uses: actions/upload-artifact@v3 + with: + name: THIRDPARTY + path: /tmp/THIRDPARTY \ No newline at end of file diff --git a/.github/actions/fossa-test/action.yml b/.github/actions/fossa-test/action.yml new file mode 100644 index 000000000..5098e8573 --- /dev/null +++ b/.github/actions/fossa-test/action.yml @@ -0,0 +1,23 @@ +name: Fossa test + +description: > + Checks report created in fossa-scan job. This action checks license compliance + and vulnerabilities. The job uses .fossa.yml configuration file + +inputs: + FOSSA_API_KEY: + required: true + +runs: + using: composite + steps: + - name: Checkout repository + uses: actions/checkout@v3 + + - name: Run fossa test + shell: bash + run: | + curl -H 'Cache-Control: no-cache' https://raw.githubusercontent.com/fossas/fossa-cli/master/install-latest.sh | bash + fossa test --debug + env: + FOSSA_API_KEY: ${{ inputs.FOSSA_API_KEY }} \ No newline at end of file diff --git a/.github/actions/lint/action.yml b/.github/actions/lint/action.yml new file mode 100644 index 000000000..55e55ac27 --- /dev/null +++ b/.github/actions/lint/action.yml @@ -0,0 +1,19 @@ +name: Lint + +description: 'Run black, flake8 and other linting checks' + +runs-on: ubuntu-latest + +runs: + using: composite + steps: + - name: Checkout repository + uses: actions/checkout@v3 + + - name: Set up python + uses: actions/setup-python@v4 + with: + python-version: "3.7" + + - name: Run pre-commit + uses: pre-commit/action@v3.0.0 diff --git a/.github/actions/meta/action.yml b/.github/actions/meta/action.yml new file mode 100644 index 000000000..f0753887b --- /dev/null +++ b/.github/actions/meta/action.yml @@ -0,0 +1,95 @@ +name: Prepare metadata + +description: > + Focuses on preparing metadata and information necessary for subsequent steps + in the GitHub Actions workflow. It handles versioning (dry run of the release process), + Docker image metadata, matrix setup, and Python version information + +inputs: + SA_GH_USER_NAME: + required: true + SA_GH_USER_EMAIL: + required: true + SA_GPG_PRIVATE_KEY: + required: true + SA_GPG_PASSPHRASE: + required: true + +outputs: + sc4s: + # using fromJSON as below gave an error `error reading JToken from JsonReader` - hence using jq + # value: ghcr.io/${{ github.repository }}/container:${{ fromJSON(steps.docker_action_meta.outputs.json).labels['org.opencontainers.image.version'] }} + # https://github.com/splunk/splunk-add-on-for-microsoft-cloud-services/actions/runs/7019443167/job/19104471863#step:7:4 + value: ghcr.io/${{ github.repository }}/container:$(echo "${{ steps.docker_action_meta.outputs.json }}" | jq -r '.labels["org.opencontainers.image.version"]') + container_tags: + value: ${{ steps.docker_action_meta.outputs.tags }} + container_labels: + value: ${{ steps.docker_action_meta.outputs.labels }} + container_buildtime: + value: $(echo "${{ steps.docker_action_meta.outputs.json }}" | jq -r '.labels["org.opencontainers.image.created"]') + container_version: + value: $(echo "${{ steps.docker_action_meta.outputs.json }}" | jq -r '.labels["org.opencontainers.image.version"]') + container_revision: + value: $(echo "${{ steps.docker_action_meta.outputs.json }}" | jq -r '.labels["org.opencontainers.image.revision"]') + container_base: + value: $(echo "${{ steps.docker_action_meta.outputs.json }}" | jq -r '.tags[0]') + matrix_supportedSplunk: + value: ${{ steps.matrix.outputs.supportedSplunk }} + matrix_latestSplunk: + value: ${{ steps.matrix.outputs.latestSplunk }} + matrix_supportedSC4S: + value: ${{ steps.matrix.outputs.supportedSC4S }} + matrix_supportedModinputFunctionalVendors: + value: ${{ steps.matrix.outputs.supportedModinputFunctionalVendors }} + matrix_supportedUIVendors: + value: ${{ steps.matrix.outputs.supportedUIVendors }} + python39_splunk: + value: ${{steps.python39_splunk.outputs.splunk}} + python39_sc4s: + value: ${{steps.python39_splunk.outputs.sc4s}} + +runs: + using: composite + steps: + - name: Checkout repository + uses: actions/checkout@v3 + with: + submodules: false + persist-credentials: false + + - name: Semantic Release + id: version + uses: splunk/semantic-release-action@v1.3 + with: + dry_run: true + git_committer_name: ${{ inputs.SA_GH_USER_NAME }} + git_committer_email: ${{ inputs.SA_GH_USER_EMAIL }} + gpg_private_key: ${{ inputs.SA_GPG_PRIVATE_KEY }} + passphrase: ${{ inputs.SA_GPG_PASSPHRASE }} + env: + GITHUB_TOKEN: ${{ github.token }} + + - name: Docker meta + id: docker_action_meta + uses: docker/metadata-action@v4.6.0 + with: + images: ghcr.io/${{ github.repository }}/container + tags: | + type=sha,format=long + type=sha + type=semver,pattern={{version}},value=${{ steps.version.outputs.new_release_version }} + type=semver,pattern={{major}},value=${{ steps.version.outputs.new_release_version }} + type=semver,pattern={{major}}.{{minor}},value=${{ steps.version.outputs.new_release_version }} + type=ref,event=branch + type=ref,event=pr + + - name: matrix + id: matrix + uses: splunk/addonfactory-test-matrix-action@v1.10 + + - name: python39_Splunk + id: python39_splunk + shell: bash + run: | + echo "splunk={\"version\":\"unreleased-python3_9-a076ce4c50aa\", \"build\":\"a076ce4c50aa\", \"islatest\":false, \"isoldest\":false}" >> "$GITHUB_OUTPUT" + echo "sc4s={\"version\":\"2.49.5\", \"docker_registry\":\"ghcr.io/splunk/splunk-connect-for-syslog/container2\"}" >> "$GITHUB_OUTPUT" diff --git a/.github/actions/requirements-unit-tests/action.yml b/.github/actions/requirements-unit-tests/action.yml new file mode 100644 index 000000000..b4b2cb915 --- /dev/null +++ b/.github/actions/requirements-unit-tests/action.yml @@ -0,0 +1,27 @@ +name: Requirements unit test + +description: Checks whether project's requirement files meet the specified unit test criteria. + +runs: + using: composite + steps: + - name: Checkout repository + uses: actions/checkout@v3 + + - name: Install Python 3 + uses: actions/setup-python@v4 + with: + python-version: 3.7 + + - name: Run tests + uses: splunk/addonfactory-workflow-requirement-files-unit-tests@v1.4 + with: + input-files: tests/requirement_test/logs + + - name: Archive production artifacts + if: ${{ !cancelled() }} + uses: actions/upload-artifact@v3 + with: + name: test-results + path: | + test_*.txt \ No newline at end of file diff --git a/.github/actions/review-secrets/action.yml b/.github/actions/review-secrets/action.yml new file mode 100644 index 000000000..33b1fc444 --- /dev/null +++ b/.github/actions/review-secrets/action.yml @@ -0,0 +1,27 @@ +name: Review secrets + +description: > + Checks for addition/deletion of any secret/sensitive data in last 50 commits of the repository + +runs: + using: composite + steps: + - name: Checkout repository + if: github.event_name != 'pull_request' + uses: actions/checkout@v3 + with: + submodules: false + fetch-depth: "0" + + - name: Checkout repo for PR + if: github.event_name == 'pull_request' + uses: actions/checkout@v3 + with: + submodules: false + fetch-depth: "0" + ref: ${{ github.head_ref }} + + - name: Trufflehog Actions Scan + uses: edplato/trufflehog-actions-scan@v0.9l-beta + with: + scanArguments: "--max_dept 5 -x .github/workflows/exclude-patterns.txt --allow .github/workflows/trufflehog-false-positive.json" diff --git a/.github/actions/semgrep/action.yml b/.github/actions/semgrep/action.yml new file mode 100644 index 000000000..f1c5bf1f7 --- /dev/null +++ b/.github/actions/semgrep/action.yml @@ -0,0 +1,19 @@ +name: Semgrep security check + +description: Analyzes the codebase for security vulnerabilities and issues using Semgrep + +inputs: + SEMGREP_PUBLISH_TOKEN: + required: true + +runs: + using: composite + steps: + - name: Checkout repository + uses: actions/checkout@v3 + + - name: Semgrep + id: semgrep + uses: semgrep/semgrep-action@v1 + with: + publishToken: ${{ inputs.SEMGREP_PUBLISH_TOKEN }} \ No newline at end of file diff --git a/.github/actions/test-inventory/action.yml b/.github/actions/test-inventory/action.yml new file mode 100644 index 000000000..4b2a28771 --- /dev/null +++ b/.github/actions/test-inventory/action.yml @@ -0,0 +1,44 @@ +name: Test inventory + +description: Detects number and version of test-types present in the addons tests folder + +outputs: + unit: + value: ${{ steps.testset.outputs.unit }} + knowledge: + value: ${{ steps.testset.outputs.knowledge }} + ui: + value: ${{ steps.testset.outputs.ui }} + modinput_functional: + value: ${{ steps.testset.outputs.modinput_functional }} + requirement_test: + value: ${{ steps.testset.outputs.requirement_test }} + scripted_inputs: + value: ${{ steps.testset.outputs.scripted_inputs }} + escu: + value: ${{ steps.testset.outputs.escu }} + ucc_modinput_functional: + value: ${{ steps.modinput-version.outputs.ucc_modinput_tests }} + +runs: + using: composite + steps: + - name: Checkout repository + uses: actions/checkout@v3 + + - name: Check available test types + id: testset + shell: bash + run: | + find tests -type d -maxdepth 1 -mindepth 1 | sed 's|^tests/||g' | while read -r TESTSET; do echo "$TESTSET=true" >> "$GITHUB_OUTPUT"; echo "$TESTSET::true"; done + + - name: Check modinput tests version + id: modinput-version + shell: bash + run: | + CENTAURS_MODINPUT_TESTS_CHECK_DIR="tests/modinput_functional/centaurs" + ucc_modinput_tests="true" + if [ -d "$CENTAURS_MODINPUT_TESTS_CHECK_DIR" ]; then + ucc_modinput_tests="false" + fi + echo "ucc_modinput_tests=$ucc_modinput_tests" >> "$GITHUB_OUTPUT" diff --git a/.github/actions/unit-tests/action.yml b/.github/actions/unit-tests/action.yml new file mode 100644 index 000000000..ea36bd874 --- /dev/null +++ b/.github/actions/unit-tests/action.yml @@ -0,0 +1,54 @@ +name: Run unit tests + +inputs: + python_version: + required: true + +runs: + using: composite + steps: + - name: Checkout repository + uses: actions/checkout@v3 + + - name: debug input + shell: bash + run: echo '${{ inputs.python_version }}' + + - name: Setup python + uses: actions/setup-python@v4 + with: + python-version: ${{ inputs.python_version }} + + - name: Setup addon + shell: bash + run: | + if [ -f "poetry.lock" ] + then + mkdir -p package/lib || true + pip install poetry==1.5.1 poetry-plugin-export==1.4.0 + poetry export --without-hashes -o package/lib/requirements.txt + poetry export --without-hashes --dev -o requirements_dev.txt + fi + if [ ! -f requirements_dev.txt ]; then echo no requirements;exit 0 ;fi + pip install -r requirements_dev.txt + + - name: Create directories + shell: bash + run: | + mkdir -p /opt/splunk/var/log/splunk + chmod -R 777 /opt/splunk/var/log/splunk + + - name: Copy pytest ini + shell: bash + run: cp tests/unit/pytest-ci.ini pytest.ini + + - name: Run Pytest with coverage + shell: bash + run: pytest --cov=./ --cov-report=xml --junitxml=test-results/junit.xml tests/unit + + - name: Upload artifact + uses: actions/upload-artifact@v3 + if: success() || failure() + with: + name: test-results-unit-python_${{ inputs.python_version }} + path: test-results/* \ No newline at end of file diff --git a/.github/actions/validate-pr-title/action.yml b/.github/actions/validate-pr-title/action.yml new file mode 100644 index 000000000..8b98ec9cb --- /dev/null +++ b/.github/actions/validate-pr-title/action.yml @@ -0,0 +1,19 @@ +name: Validate PR title + +description: Check if PR titles match the Conventional Commits spec (https://www.conventionalcommits.org/en/v1.0.0/) + +inputs: + GITHUB_TOKEN: + required: true + +if: github.event_name != 'pull_request' + +runs: + using: composite + steps: + - uses: amannn/action-semantic-pull-request@v5.2.0 + with: + wip: true + validateSingleCommit: true + env: + GITHUB_TOKEN: ${{ inputs.GITHUB_TOKEN }} \ No newline at end of file diff --git a/.github/actions/virustotal/action.yml b/.github/actions/virustotal/action.yml new file mode 100644 index 000000000..eaec2a11a --- /dev/null +++ b/.github/actions/virustotal/action.yml @@ -0,0 +1,23 @@ +name: Virustotal security check + +description: Performs a VirusTotal scan on the artifacts generated by a "build" job + +inputs: + VT_API_KEY: + required: true + +runs: + using: composite + steps: + - name: Download artifact + uses: actions/download-artifact@v3 + with: + name: package-splunkbase + path: build/package/ + + - name: VirusTotal Scan + uses: crazy-max/ghaction-virustotal@v3 + with: + vt_api_key: ${{ inputs.VT_API_KEY }} + files: | + build/package/* \ No newline at end of file diff --git a/.github/workflows/reusable-lightweight.yml b/.github/workflows/reusable-lightweight.yml new file mode 100644 index 000000000..42b0fb381 --- /dev/null +++ b/.github/workflows/reusable-lightweight.yml @@ -0,0 +1,307 @@ +name: lightweight-check + +on: + workflow_call: + secrets: + SA_GH_USER_NAME: + description: GPG signature username + required: true + SA_GH_USER_EMAIL: + description: GPG signature user email + required: true + SA_GPG_PRIVATE_KEY: + description: GPG signature private key + required: true + SA_GPG_PASSPHRASE: + description: GPG signature passphrase + required: true + SEMGREP_PUBLISH_TOKEN: + description: Semgrep token + required: true + AWS_ACCESS_KEY_ID: + description: AWS access key id + required: true + AWS_DEFAULT_REGION: + description: AWS default region + required: true + AWS_SECRET_ACCESS_KEY: + description: AWS secret access key + required: true + VT_API_KEY: + description: Virustotal api key + required: true + SPL_COM_USER: + description: username to splunk.com + required: true + SPL_COM_PASSWORD: + description: password to splunk.com + required: true + FOSSA_API_KEY: + description: API token for FOSSA app + required: true + +permissions: + contents: read + packages: read + +concurrency: + group: ${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +jobs: + validate-pr-title: + name: Validate PR title + runs-on: ubuntu-latest + if: ${{ github.event_name == 'pull_request' }} + permissions: + contents: read + packages: read + pull-requests: read + statuses: write + steps: + - name: Run title validation + # uses: ./.github/actions/validate-pr-title + uses: splunk/addonfactory-workflow-addon-release/.github/actions/validate-pr-title@ci/lightweight-workflow-ADDON-66448 + with: + GITHUB_TOKEN: ${{ github.token }} + + meta: + name: Prepare metadata + runs-on: ubuntu-latest + outputs: + sc4s: ${{ steps.meta.outputs.sc4s }} + steps: + - name: Run meta preparation + id: meta + # uses: ./.github/actions/meta + uses: splunk/addonfactory-workflow-addon-release/.github/actions/meta@ci/lightweight-workflow-ADDON-66448 + with: + SA_GH_USER_NAME: ${{ secrets.SA_GH_USER_NAME }} + SA_GH_USER_EMAIL: ${{ secrets.SA_GH_USER_EMAIL }} + SA_GPG_PRIVATE_KEY: ${{ secrets.SA_GPG_PRIVATE_KEY }} + SA_GPG_PASSPHRASE: ${{ secrets.SA_GPG_PASSPHRASE }} + + fossa-scan: + name: FOSSA scan + runs-on: ubuntu-latest + steps: + - name: Run FOSSA scan + # uses: ./.github/actions/fossa-scan + uses: splunk/addonfactory-workflow-addon-release/.github/actions/fossa-scan@ci/lightweight-workflow-ADDON-66448 + with: + FOSSA_API_KEY: ${{ secrets.FOSSA_API_KEY }} + + fossa-test: + name: FOSSA test + runs-on: ubuntu-latest + needs: + - fossa-scan + steps: + - name: Run FOSSA test + # uses: ./.github/actions/fossa-test + uses: splunk/addonfactory-workflow-addon-release/.github/actions/fossa-test@ci/lightweight-workflow-ADDON-66448 + with: + FOSSA_API_KEY: ${{ secrets.FOSSA_API_KEY }} + + compliance-copyrights: + name: Compliance copyrights + runs-on: ubuntu-latest + steps: + - name: Run compliance copyrights + # uses: ./.github/actions/compliance-copyrights + uses: splunk/addonfactory-workflow-addon-release/.github/actions/compliance-copyrights@ci/lightweight-workflow-ADDON-66448 + + lint: + name: Lint + runs-on: ubuntu-latest + steps: + - name: Run linting checks + # uses: ./.github/actions/lint + uses: splunk/addonfactory-workflow-addon-release/.github/actions/lint@ci/lightweight-workflow-ADDON-66448 + + review-secrets: + name: Review secrets + runs-on: ubuntu-latest + steps: + - name: Run secrets review + # uses: ./.github/actions/review-secrets + uses: splunk/addonfactory-workflow-addon-release/.github/actions/review-secrets@ci/lightweight-workflow-ADDON-66448 + + semgrep: + name: Semgrep security check + runs-on: ubuntu-latest + steps: + - name: Run semgrep + # uses: ./.github/actions/semgrep + uses: splunk/addonfactory-workflow-addon-release/.github/actions/semgrep@ci/lightweight-workflow-ADDON-66448 + with: + SEMGREP_PUBLISH_TOKEN: ${{ secrets.SEMGREP_PUBLISH_TOKEN }} + + test-inventory: + name: Test inventory + runs-on: ubuntu-latest + outputs: + unit: ${{ steps.test-inventory.outputs.unit }} + ucc_modinput_functional: ${{ steps.test-inventory.outputs.ucc_modinput_functional}} + modinput_functional: ${{ steps.test-inventory.outputs.modinput_functional}} + requirement_test: ${{ steps.test-inventory.outputs.requirement_test }} + steps: + - name: Run test inventory check + id: test-inventory + # uses: ./.github/actions/test-inventory + uses: splunk/addonfactory-workflow-addon-release/.github/actions/test-inventory@ci/lightweight-workflow-ADDON-66448 + + # Two separate unit test jobs needed as jobs that depend on unit-test success can't proceed + # if any matrix job fails. Currently python 3.9 always fails as it's not supported. + # TODO: group these jobs into the matrix once python 3.9 is supported + + run-unit-tests-3_7: + name: Unit tests 3.7 + if: ${{ needs.test-inventory.outputs.unit == 'true' }} + runs-on: ubuntu-latest + needs: + - test-inventory + permissions: + actions: read + deployments: read + contents: read + packages: read + statuses: read + checks: write + steps: + - name: Run unit tests for python 3.7 + id: unit-tests-3_7 + # uses: ./.github/actions/unit-tests + uses: splunk/addonfactory-workflow-addon-release/.github/actions/unit-tests@ci/lightweight-workflow-ADDON-66448 + with: + python_version: '3.7' + + run-unit-tests-3_9: + name: Unit tests 3.9 + if: ${{ needs.test-inventory.outputs.unit == 'true' }} + runs-on: ubuntu-latest + continue-on-error: true + needs: + - test-inventory + permissions: + actions: read + deployments: read + contents: read + packages: read + statuses: read + checks: write + steps: + - name: Run unit tests for python 3.9 + id: unit-tests-3_9 + # uses: ./.github/actions/unit-tests + uses: splunk/addonfactory-workflow-addon-release/.github/actions/unit-tests@ci/lightweight-workflow-ADDON-66448 + with: + python_version: '3.9' + + build: + name: Build python-${{ matrix.python-version }} + runs-on: ubuntu-latest + needs: + - test-inventory + - meta + - compliance-copyrights + - lint + - review-secrets + - semgrep + - run-unit-tests-3_7 + strategy: + fail-fast: false + matrix: + python-version: + - "3.7" + - "3.9" + if: ${{ !cancelled() && (needs.run-unit-tests-3_7.result == 'success' || needs.run-unit-tests-3_7.result == 'skipped') }} + permissions: + contents: write + packages: read + steps: + - name: Run build + # uses: ./.github/actions/build + uses: splunk/addonfactory-workflow-addon-release/.github/actions/build@ci/lightweight-workflow-ADDON-66448 + with: + python_version: ${{ matrix.python-version }} + SA_GH_USER_NAME: ${{ secrets.SA_GH_USER_NAME }} + SA_GH_USER_EMAIL: ${{ secrets.SA_GH_USER_EMAIL }} + SA_GPG_PRIVATE_KEY: ${{ secrets.SA_GPG_PRIVATE_KEY }} + SA_GPG_PASSPHRASE: ${{ secrets.SA_GPG_PASSPHRASE }} + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + ucc_modinput_functional: ${{ needs.test-inventory.outputs.ucc_modinput_functional}} + modinput_functional: ${{ needs.test-inventory.outputs.modinput_functional}} + + virustotal: + runs-on: ubuntu-latest + needs: build + if: ${{ !cancelled() && needs.build.result == 'success' }} + steps: + - name: Run VirusTotal check + # uses: ./.github/actions/virustotal + uses: splunk/addonfactory-workflow-addon-release/.github/actions/virustotal@ci/lightweight-workflow-ADDON-66448 + with: + VT_API_KEY: ${{ secrets.VT_API_KEY }} + + run-requirements-unit-tests: + name: Requirements unit tests + runs-on: ubuntu-latest + needs: + - build + - test-inventory + if: ${{ !cancelled() && needs.build.result == 'success' && needs.test-inventory.outputs.requirement_test == 'true' }} + permissions: + actions: read + deployments: read + contents: read + packages: read + statuses: read + checks: write + steps: + - name: Run requirements unit tests + # uses: ./.github/actions/requirements-unit-tests + uses: splunk/addonfactory-workflow-addon-release/.github/actions/requirements-unit-tests@ci/lightweight-workflow-ADDON-66448 + + appinspect-cli: + name: AppInspect CLI ${{ matrix.tags }} + runs-on: ubuntu-latest + needs: build + if: ${{ !cancelled() && needs.build.result == 'success' }} + strategy: + fail-fast: false + matrix: + tags: + - "cloud" + - "appapproval" + - "deprecated_feature" + - "developer_guidance" + - "future" + - "self-service" + - "splunk_appinspect" + - "manual" + steps: + - name: Run appinspect CLI + # uses: ./.github/actions/appinspect-cli + uses: splunk/addonfactory-workflow-addon-release/.github/actions/appinspect-cli@ci/lightweight-workflow-ADDON-66448 + with: + matrix_tags: ${{ matrix.tags }} + + artifact-registry: + name: Artifact registry + runs-on: ubuntu-latest + needs: + - virustotal + - meta + if: ${{ !cancelled() && needs.virustotal.result == 'success' && needs.meta.result == 'success' }} + permissions: + contents: read + packages: write + steps: + - name: Run artifact registry + # uses: ./.github/actions/artifact-registry + uses: splunk/addonfactory-workflow-addon-release/.github/actions/artifact-registry@ci/lightweight-workflow-ADDON-66448 + with: + sc4s: ${{ needs.meta.outputs.sc4s }} From 159d34a2220bb767f909a87146f99ff21a882681 Mon Sep 17 00:00:00 2001 From: Adam Date: Tue, 5 Dec 2023 17:11:57 +0100 Subject: [PATCH 2/5] ci: use composite actions in main reusable workflow Jira issue: ADDON-66448 --- .../workflows/reusable-build-test-release.yml | 710 +++++------------- 1 file changed, 174 insertions(+), 536 deletions(-) diff --git a/.github/workflows/reusable-build-test-release.yml b/.github/workflows/reusable-build-test-release.yml index ca9fbf276..49fccb5d6 100644 --- a/.github/workflows/reusable-build-test-release.yml +++ b/.github/workflows/reusable-build-test-release.yml @@ -195,224 +195,157 @@ jobs: validate-pr-title: name: Validate PR title + runs-on: ubuntu-latest needs: - - setup-workflow + - setup-workflow if: ${{ needs.setup-workflow.outputs.skip-workflow != 'Yes' && github.event_name == 'pull_request' }} - runs-on: ubuntu-latest permissions: contents: read packages: read pull-requests: read statuses: write steps: - - uses: amannn/action-semantic-pull-request@v5.2.0 + - name: Run title validation + # uses: ./.github/actions/validate-pr-title + uses: splunk/addonfactory-workflow-addon-release/.github/actions/validate-pr-title@ci/lightweight-workflow-ADDON-66448 with: - wip: true - validateSingleCommit: true - env: GITHUB_TOKEN: ${{ github.token }} meta: + name: Prepare metadata runs-on: ubuntu-latest - needs: + needs: - setup-workflow if: ${{ needs.setup-workflow.outputs.skip-workflow != 'Yes' }} outputs: - sc4s: ghcr.io/${{ github.repository }}/container:${{ fromJSON(steps.docker_action_meta.outputs.json).labels['org.opencontainers.image.version'] }} - container_tags: ${{ steps.docker_action_meta.outputs.tags }} - container_labels: ${{ steps.docker_action_meta.outputs.labels }} - container_buildtime: ${{ fromJSON(steps.docker_action_meta.outputs.json).labels['org.opencontainers.image.created'] }} - container_version: ${{ fromJSON(steps.docker_action_meta.outputs.json).labels['org.opencontainers.image.version'] }} - container_revision: ${{ fromJSON(steps.docker_action_meta.outputs.json).labels['org.opencontainers.image.revision'] }} - container_base: ${{ fromJSON(steps.docker_action_meta.outputs.json).tags[0] }} - matrix_supportedSplunk: ${{ steps.matrix.outputs.supportedSplunk }} - matrix_latestSplunk: ${{ steps.matrix.outputs.latestSplunk }} - matrix_supportedSC4S: ${{ steps.matrix.outputs.supportedSC4S }} - matrix_supportedModinputFunctionalVendors: ${{ steps.matrix.outputs.supportedModinputFunctionalVendors }} - matrix_supportedUIVendors: ${{ steps.matrix.outputs.supportedUIVendors }} - python39_splunk: ${{steps.python39_splunk.outputs.splunk}} - python39_sc4s: ${{steps.python39_splunk.outputs.sc4s}} - permissions: - contents: write - packages: read + sc4s: ${{ steps.meta.outputs.sc4s }} + container_tags: ${{ steps.meta.outputs.container_tags }} + container_labels: ${{ steps.meta.outputs.container_labels }} + container_buildtime: ${{ steps.meta.outputs.container_buildtime }} + container_version: ${{ steps.meta.outputs.container_version }} + container_revision: ${{ steps.meta.outputs.container_revision }} + container_base: ${{ steps.meta.outputs.container_base }} + matrix_supportedSplunk: ${{ steps.meta.outputs.matrix_supportedSplunk }} + matrix_latestSplunk: ${{ steps.meta.outputs.matrix_latestSplunk }} + matrix_supportedSC4S: ${{ steps.meta.outputs.matrix_supportedSC4S }} + matrix_supportedModinputFunctionalVendors: ${{ steps.meta.outputs.matrix_supportedModinputFunctionalVendors }} + matrix_supportedUIVendors: ${{ steps.meta.outputs.matrix_supportedUIVendors }} + python39_splunk: ${{ steps.meta.outputs.python39_splunk }} + python39_sc4s: ${{ steps.meta.outputs.python39_sc4s }} steps: - - name: Checkout - uses: actions/checkout@v3 - with: - submodules: false - persist-credentials: false - - name: Semantic Release - id: version - uses: splunk/semantic-release-action@v1.3 + - name: Run meta preparation + id: meta + # uses: ./.github/actions/meta + uses: splunk/addonfactory-workflow-addon-release/.github/actions/meta@ci/lightweight-workflow-ADDON-66448 with: - dry_run: true - git_committer_name: ${{ secrets.SA_GH_USER_NAME }} - git_committer_email: ${{ secrets.SA_GH_USER_EMAIL }} - gpg_private_key: ${{ secrets.SA_GPG_PRIVATE_KEY }} - passphrase: ${{ secrets.SA_GPG_PASSPHRASE }} - env: - GITHUB_TOKEN: ${{ github.token }} - - name: Docker meta - id: docker_action_meta - uses: docker/metadata-action@v4.6.0 - with: - images: ghcr.io/${{ github.repository }}/container - tags: | - type=sha,format=long - type=sha - type=semver,pattern={{version}},value=${{ steps.version.outputs.new_release_version }} - type=semver,pattern={{major}},value=${{ steps.version.outputs.new_release_version }} - type=semver,pattern={{major}}.{{minor}},value=${{ steps.version.outputs.new_release_version }} - type=ref,event=branch - type=ref,event=pr - - name: matrix - id: matrix - uses: splunk/addonfactory-test-matrix-action@v1.10 - - name: python39_Splunk - id: python39_splunk - run: | - echo "splunk={\"version\":\"unreleased-python3_9-a076ce4c50aa\", \"build\":\"a076ce4c50aa\", \"islatest\":false, \"isoldest\":false}" >> "$GITHUB_OUTPUT" - echo "sc4s={\"version\":\"2.49.5\", \"docker_registry\":\"ghcr.io/splunk/splunk-connect-for-syslog/container2\"}" >> "$GITHUB_OUTPUT" + SA_GH_USER_NAME: ${{ secrets.SA_GH_USER_NAME }} + SA_GH_USER_EMAIL: ${{ secrets.SA_GH_USER_EMAIL }} + SA_GPG_PRIVATE_KEY: ${{ secrets.SA_GPG_PRIVATE_KEY }} + SA_GPG_PASSPHRASE: ${{ secrets.SA_GPG_PASSPHRASE }} fossa-scan: + name: FOSSA scan runs-on: ubuntu-latest needs: - setup-workflow if: ${{ needs.setup-workflow.outputs.skip-workflow != 'Yes' }} steps: - - uses: actions/checkout@v3 - - name: run fossa anlyze and create report - run: | - curl -H 'Cache-Control: no-cache' https://raw.githubusercontent.com/fossas/fossa-cli/master/install-latest.sh | bash - fossa analyze --debug - fossa report attribution --format text --timeout 600 > /tmp/THIRDPARTY - env: - FOSSA_API_KEY: ${{ secrets.FOSSA_API_KEY }} - - name: upload THIRDPARTY file - uses: actions/upload-artifact@v3 + - name: Run FOSSA scan + # uses: ./.github/actions/fossa-scan + uses: splunk/addonfactory-workflow-addon-release/.github/actions/fossa-scan@ci/lightweight-workflow-ADDON-66448 with: - name: THIRDPARTY - path: /tmp/THIRDPARTY + FOSSA_API_KEY: ${{ secrets.FOSSA_API_KEY }} fossa-test: continue-on-error: true + name: FOSSA test runs-on: ubuntu-latest needs: - fossa-scan - if: ${{ needs.setup-workflow.outputs.skip-workflow != 'Yes' }} steps: - - uses: actions/checkout@v3 - - name: run fossa test - run: | - curl -H 'Cache-Control: no-cache' https://raw.githubusercontent.com/fossas/fossa-cli/master/install-latest.sh | bash - fossa test --debug - env: + - name: Run FOSSA test + # uses: ./.github/actions/fossa-test + uses: splunk/addonfactory-workflow-addon-release/.github/actions/fossa-test@ci/lightweight-workflow-ADDON-66448 + with: FOSSA_API_KEY: ${{ secrets.FOSSA_API_KEY }} compliance-copyrights: - name: compliance-copyrights + name: Compliance copyrights runs-on: ubuntu-latest needs: - setup-workflow if: ${{ needs.setup-workflow.outputs.skip-workflow != 'Yes' }} steps: - - name: Checkout - uses: actions/checkout@v3 - - name: REUSE Compliance Check - uses: fsfe/reuse-action@v1.1 + - name: Run compliance copyrights + # uses: ./.github/actions/compliance-copyrights + uses: splunk/addonfactory-workflow-addon-release/.github/actions/compliance-copyrights@ci/lightweight-workflow-ADDON-66448 lint: + name: Lint runs-on: ubuntu-latest needs: - setup-workflow if: ${{ needs.setup-workflow.outputs.skip-workflow != 'Yes' }} steps: - - uses: actions/checkout@v3 - - uses: actions/setup-python@v4 - with: - python-version: "3.7" - - uses: pre-commit/action@v3.0.0 + - name: Run linting checks + # uses: ./.github/actions/lint + uses: splunk/addonfactory-workflow-addon-release/.github/actions/lint@ci/lightweight-workflow-ADDON-66448 - review_secrets: - name: security-detect-secrets + review-secrets: + name: Review secrets runs-on: ubuntu-latest needs: - setup-workflow if: ${{ needs.setup-workflow.outputs.skip-workflow != 'Yes' }} steps: - - name: Checkout - if: github.event_name != 'pull_request' - uses: actions/checkout@v3 - with: - submodules: false - fetch-depth: "0" - - name: Checkout for PR - if: github.event_name == 'pull_request' - uses: actions/checkout@v3 - with: - submodules: false - fetch-depth: "0" - ref: ${{ github.head_ref }} - - name: Trufflehog Actions Scan - uses: edplato/trufflehog-actions-scan@v0.9l-beta - with: - scanArguments: "--max_dept 5 -x .github/workflows/exclude-patterns.txt --allow .github/workflows/trufflehog-false-positive.json" + - name: Run secrets review + # uses: ./.github/actions/review-secrets + uses: splunk/addonfactory-workflow-addon-release/.github/actions/review-secrets@ci/lightweight-workflow-ADDON-66448 semgrep: + name: Semgrep security check runs-on: ubuntu-latest - name: security-sast-semgrep needs: - setup-workflow - if: ${{ needs.setup-workflow.outputs.skip-workflow != 'Yes' && github.actor != 'dependabot[bot]' }} + if: ${{ needs.setup-workflow.outputs.skip-workflow != 'Yes' }} steps: - - uses: actions/checkout@v3 - - name: Semgrep - id: semgrep - uses: semgrep/semgrep-action@v1 + - name: Run semgrep + # uses: ./.github/actions/semgrep + uses: splunk/addonfactory-workflow-addon-release/.github/actions/semgrep@ci/lightweight-workflow-ADDON-66448 with: - publishToken: ${{ secrets.SEMGREP_PUBLISH_TOKEN }} + SEMGREP_PUBLISH_TOKEN: ${{ secrets.SEMGREP_PUBLISH_TOKEN }} test-inventory: + name: Test inventory runs-on: ubuntu-latest - needs: setup-workflow + needs: + - setup-workflow if: ${{ needs.setup-workflow.outputs.skip-workflow != 'Yes' }} - # Map a step output to a job output outputs: - unit: ${{ steps.testset.outputs.unit }} - knowledge: ${{ steps.testset.outputs.knowledge }} - ui: ${{ steps.testset.outputs.ui }} - modinput_functional: ${{ steps.testset.outputs.modinput_functional }} - requirement_test: ${{ steps.testset.outputs.requirement_test }} - scripted_inputs: ${{ steps.testset.outputs.scripted_inputs }} - escu: ${{ steps.testset.outputs.escu }} - ucc_modinput_functional: ${{ steps.modinput-version.outputs.ucc_modinput_tests }} + unit: ${{ steps.test-inventory.outputs.unit }} + ucc_modinput_functional: ${{ steps.test-inventory.outputs.ucc_modinput_functional}} + modinput_functional: ${{ steps.test-inventory.outputs.modinput_functional}} + requirement_test: ${{ steps.test-inventory.outputs.requirement_test }} + knowledge: ${{ steps.test-inventory.outputs.knowledge }} + ui: ${{ steps.test-inventory.outputs.ui }} + scripted_inputs: ${{ steps.test-inventory.outputs.scripted_inputs }} + escu: ${{ steps.test-inventory.outputs.escu }} steps: - - uses: actions/checkout@v3 - - id: testset - name: Check available test types - run: | - find tests -type d -maxdepth 1 -mindepth 1 | sed 's|^tests/||g' | while read -r TESTSET; do echo "$TESTSET=true" >> "$GITHUB_OUTPUT"; echo "$TESTSET::true"; done - - id: modinput-version - name: Check modinput tests version - run: | - CENTAURS_MODINPUT_TESTS_CHECK_DIR="tests/modinput_functional/centaurs" - ucc_modinput_tests="true" - if [ -d "$CENTAURS_MODINPUT_TESTS_CHECK_DIR" ]; then - ucc_modinput_tests="false" - fi - echo "ucc_modinput_tests=$ucc_modinput_tests" >> "$GITHUB_OUTPUT" + - name: Run test inventory check + id: test-inventory + # uses: ./.github/actions/test-inventory + uses: splunk/addonfactory-workflow-addon-release/.github/actions/test-inventory@ci/lightweight-workflow-ADDON-66448 - run-unit-tests: - name: test-unit-python3-${{ matrix.python-version }} + # Two separate unit test jobs needed as jobs that depend on unit-test success can't proceed + # if any matrix job fails. Currently python 3.9 always fails as it's not supported. + # TODO: group these jobs into the matrix once python 3.9 is supported + + run-unit-tests-3_7: + name: Unit tests if: ${{ needs.test-inventory.outputs.unit == 'true' }} runs-on: ubuntu-latest needs: - test-inventory - strategy: - fail-fast: false - matrix: - python-version: - - "3.7" permissions: actions: read deployments: read @@ -421,46 +354,20 @@ jobs: statuses: read checks: write steps: - - uses: actions/checkout@v3 - - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python-version }} - - name: Setup addon - run: | - if [ -f "poetry.lock" ] - then - mkdir -p package/lib || true - pip install poetry==1.5.1 poetry-plugin-export==1.4.0 - poetry export --without-hashes -o package/lib/requirements.txt - poetry export --without-hashes --dev -o requirements_dev.txt - fi - if [ ! -f requirements_dev.txt ]; then echo no requirements;exit 0 ;fi - pip install -r requirements_dev.txt - - name: Create directories - run: | - mkdir -p /opt/splunk/var/log/splunk - chmod -R 777 /opt/splunk/var/log/splunk - - name: Copy pytest ini - run: cp tests/unit/pytest-ci.ini pytest.ini - - name: Run Pytest with coverage - run: pytest --cov=./ --cov-report=xml --junitxml=test-results/junit.xml tests/unit - - uses: actions/upload-artifact@v3 - if: success() || failure() + - name: Run unit tests for python 3.7 + id: unit-tests-3_7 + # uses: ./.github/actions/unit-tests + uses: splunk/addonfactory-workflow-addon-release/.github/actions/unit-tests@ci/lightweight-workflow-ADDON-66448 with: - name: test-results-unit-python_${{ matrix.python-version }} - path: test-results/* + python_version: '3.7' run-unit-tests-3_9: - name: test-unit-python3-${{ matrix.python-version }} + name: Unit tests if: ${{ needs.test-inventory.outputs.unit == 'true' }} runs-on: ubuntu-latest + continue-on-error: true needs: - test-inventory - strategy: - fail-fast: false - matrix: - python-version: - - "3.9" permissions: actions: read deployments: read @@ -469,271 +376,103 @@ jobs: statuses: read checks: write steps: - - uses: actions/checkout@v3 - - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python-version }} - - name: Setup addon - run: | - if [ -f "poetry.lock" ] - then - mkdir -p package/lib || true - pip install poetry==1.5.1 poetry-plugin-export==1.4.0 - poetry export --without-hashes -o package/lib/requirements.txt - poetry export --without-hashes --dev -o requirements_dev.txt - fi - if [ ! -f requirements_dev.txt ]; then echo no requirements;exit 0 ;fi - pip install -r requirements_dev.txt - - name: Create directories - run: | - mkdir -p /opt/splunk/var/log/splunk - chmod -R 777 /opt/splunk/var/log/splunk - - name: Copy pytest ini - run: cp tests/unit/pytest-ci.ini pytest.ini - - name: Run Pytest with coverage - run: pytest --cov=./ --cov-report=xml --junitxml=test-results/junit.xml tests/unit - - uses: actions/upload-artifact@v3 - if: success() || failure() + - name: Run unit tests for python 3.9 + id: unit-tests-3_9 + # uses: ./.github/actions/unit-tests + uses: splunk/addonfactory-workflow-addon-release/.github/actions/unit-tests@ci/lightweight-workflow-ADDON-66448 with: - name: test-results-unit-python_${{ matrix.python-version }} - path: test-results/* + python_version: '3.9' build: + name: Build python 3.7 runs-on: ubuntu-latest needs: - - setup-workflow - test-inventory - meta - compliance-copyrights - lint - - review_secrets + - review-secrets - semgrep - - run-unit-tests - - fossa-scan - if: ${{ !cancelled() && (needs.run-unit-tests.result == 'success' || needs.run-unit-tests.result == 'skipped') }} + - run-unit-tests-3_7 + # strategy: + # fail-fast: false + # matrix: + # python-version: + # - "3.7" + # - "3.9" + if: ${{ !cancelled() && (needs.run-unit-tests-3_7.result == 'success' || needs.run-unit-tests-3_7.result == 'skipped') }} outputs: - buildname: ${{ steps.buildupload.outputs.name }} + buildname: ${{ steps.build.outputs.buildname }} permissions: contents: write packages: read steps: - - uses: actions/checkout@v3 - with: - # Very Important semantic-release won't trigger a tagged - # build if this is not set false - persist-credentials: false - - name: Setup python - uses: actions/setup-python@v4 - with: - python-version: 3.7 - - name: create requirements file for pip - run: | - if [ -f "poetry.lock" ] - then - echo " poetry.lock found " - sudo pip3 install poetry==1.5.1 poetry-plugin-export==1.4.0 - poetry lock --check - poetry export --without-hashes -o requirements.txt - if [ "$(grep -cve '^\s*$' requirements.txt)" -ne 0 ] - then - echo "Prod dependencies were found, creating package/lib folder" - mkdir -p package/lib || true - mv requirements.txt package/lib - else - echo "No prod dependencies were found" - rm requirements.txt - fi - poetry export --without-hashes --dev -o requirements_dev.txt - cat requirements_dev.txt - fi - - name: Get pip cache dir - id: pip-cache - run: | - echo "dir=$(pip cache dir)" >> "$GITHUB_OUTPUT" - - name: Run Check there are libraries to scan - id: checklibs - run: if [ -f requirements_dev.txt ]; then echo "ENABLED=true" >> "$GITHUB_OUTPUT"; fi - - name: pip cache - if: ${{ steps.checklibs.outputs.ENABLED == 'true' }} - uses: actions/cache@v3 - with: - path: ${{ steps.pip-cache.outputs.dir }} - key: ${{ runner.os }}-pip-${{ hashFiles('requirements_dev.txt') }} - restore-keys: | - ${{ runner.os }}-pip- - - name: Install deps - if: ${{ steps.checklibs.outputs.ENABLED == 'true' }} - run: pip install -r requirements_dev.txt - - name: Semantic Release Get Next - id: semantic - if: github.event_name != 'pull_request' - uses: splunk/semantic-release-action@v1.3 - with: - dry_run: true - git_committer_name: ${{ secrets.SA_GH_USER_NAME }} - git_committer_email: ${{ secrets.SA_GH_USER_EMAIL }} - gpg_private_key: ${{ secrets.SA_GPG_PRIVATE_KEY }} - passphrase: ${{ secrets.SA_GPG_PASSPHRASE }} - env: - GITHUB_TOKEN: ${{ github.token }} - - name: Determine the version to build - id: BuildVersion - uses: splunk/addonfactory-get-splunk-package-version-action@v1 - with: - SemVer: ${{ steps.semantic.outputs.new_release_version }} - PrNumber: ${{ github.event.number }} - - name: Download THIRDPARTY - if: github.event_name != 'pull_request' && github.event_name != 'schedule' - uses: actions/download-artifact@v3 - with: - name: THIRDPARTY - - name: Download THIRDPARTY (Optional for PR and schedule) - if: github.event_name == 'pull_request' || github.event_name == 'schedule' - continue-on-error: true - uses: actions/download-artifact@v3 - with: - name: THIRDPARTY - - name: Update Notices - run: | - cp -f THIRDPARTY package/THIRDPARTY || echo "THIRDPARTY file not found (allowed for PR and schedule)" - - name: Build Package - id: uccgen - uses: splunk/addonfactory-ucc-generator-action@v2 - with: - version: ${{ steps.BuildVersion.outputs.VERSION }} - - name: Slim Package - id: slim - uses: splunk/addonfactory-packaging-toolkit-action@v1 - with: - source: ${{ steps.uccgen.outputs.OUTPUT }} - if: always() - - name: artifact-openapi - uses: actions/upload-artifact@v3 - with: - name: artifact-openapi - path: ${{ github.workspace }}/${{ steps.uccgen.outputs.OUTPUT }}/static/openapi.json - if: ${{ !cancelled() && needs.test-inventory.outputs.ucc_modinput_functional == 'true' && needs.test-inventory.outputs.modinput_functional == 'true' }} - - name: artifact-splunk-base - uses: actions/upload-artifact@v3 - with: - name: package-splunkbase - path: ${{ steps.slim.outputs.OUTPUT }} - if: ${{ !cancelled() }} - - name: upload-build-to-s3 - id: buildupload - env: + - name: Run build 3.7 + id: build + # uses: ./.github/actions/build + uses: splunk/addonfactory-workflow-addon-release/.github/actions/build@ci/lightweight-workflow-ADDON-66448 + with: + python_version: "3.7" + SA_GH_USER_NAME: ${{ secrets.SA_GH_USER_NAME }} + SA_GH_USER_EMAIL: ${{ secrets.SA_GH_USER_EMAIL }} + SA_GPG_PRIVATE_KEY: ${{ secrets.SA_GPG_PRIVATE_KEY }} + SA_GPG_PASSPHRASE: ${{ secrets.SA_GPG_PASSPHRASE }} AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - run: | - echo "name=$(basename "${{ steps.slim.outputs.OUTPUT }}")" >> "$GITHUB_OUTPUT" - basename "${{ steps.slim.outputs.OUTPUT }}" - aws s3 cp "${{ steps.slim.outputs.OUTPUT }}" s3://ta-production-artifacts/ta-apps/ - - name: artifact-splunk-parts - uses: actions/upload-artifact@v3 - with: - name: package-deployment - path: build/package/deployment** - if: ${{ !cancelled() }} + ucc_modinput_functional: ${{ needs.test-inventory.outputs.ucc_modinput_functional}} + modinput_functional: ${{ needs.test-inventory.outputs.modinput_functional}} build-3_9: + name: Build python 3.9 runs-on: ubuntu-latest needs: - - setup-workflow - test-inventory - meta - compliance-copyrights - lint - - review_secrets + - review-secrets - semgrep - - run-unit-tests-3_9 - - fossa-scan - if: | - always() && - (needs.run-unit-tests-3_9.result == 'success' || needs.run-unit-tests-3_9.result == 'skipped') + - run-unit-tests-3_7 + # strategy: + # fail-fast: false + # matrix: + # python-version: + # - "3.7" + # - "3.9" + if: ${{ !cancelled() && (needs.run-unit-tests-3_7.result == 'success' || needs.run-unit-tests-3_7.result == 'skipped') }} permissions: contents: write packages: read steps: - - uses: actions/checkout@v3 - with: - # Very Important semantic-release won't trigger a tagged - # build if this is not set false - persist-credentials: false - - name: Setup python - uses: actions/setup-python@v4 - with: - python-version: 3.9 - - name: create requirements file for pip - run: | - if [ -f "poetry.lock" ] - then - echo " poetry.lock found " - sudo pip3 install poetry==1.5.1 poetry-plugin-export==1.4.0 - poetry export --without-hashes -o requirements.txt - if [ "$(grep -cve '^\s*$' requirements.txt)" -ne 0 ] - then - echo "Prod dependencies were found, creating package/lib folder" - mkdir -p package/lib || true - mv requirements.txt package/lib - else - echo "No prod dependencies were found" - rm requirements.txt - fi - poetry export --without-hashes --dev -o requirements_dev.txt - cat requirements_dev.txt - fi - - id: pip-cache - run: | - echo "dir=$(pip cache dir)" >> "$GITHUB_OUTPUT" - - name: pip cache - uses: actions/cache@v3 - with: - path: ${{ steps.pip-cache.outputs.dir }} - key: ${{ runner.os }}-pip-python3_9-${{ hashFiles('requirements_dev.txt') }} - restore-keys: | - ${{ runner.os }}-pip-python3_9 - - run: pip install -r requirements_dev.txt - - id: semantic - if: github.event_name != 'pull_request' - uses: splunk/semantic-release-action@v1.3 - with: - dry_run: true - git_committer_name: ${{ secrets.SA_GH_USER_NAME }} - git_committer_email: ${{ secrets.SA_GH_USER_EMAIL }} - gpg_private_key: ${{ secrets.SA_GPG_PRIVATE_KEY }} - passphrase: ${{ secrets.SA_GPG_PASSPHRASE }} - env: - GITHUB_TOKEN: ${{ github.token }} - - id: BuildVersion - uses: splunk/addonfactory-get-splunk-package-version-action@v1 - with: - SemVer: ${{ steps.semantic.outputs.new_release_version }} - PrNumber: ${{ github.event.number }} - - id: uccgen - uses: splunk/addonfactory-ucc-generator-action@v2 - with: - version: ${{ steps.BuildVersion.outputs.VERSION }} + - name: Run build 3.9 + # uses: ./.github/actions/build + uses: splunk/addonfactory-workflow-addon-release/.github/actions/build@ci/lightweight-workflow-ADDON-66448 + with: + python_version: "3.9" + SA_GH_USER_NAME: ${{ secrets.SA_GH_USER_NAME }} + SA_GH_USER_EMAIL: ${{ secrets.SA_GH_USER_EMAIL }} + SA_GPG_PRIVATE_KEY: ${{ secrets.SA_GPG_PRIVATE_KEY }} + SA_GPG_PASSPHRASE: ${{ secrets.SA_GPG_PASSPHRASE }} + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + ucc_modinput_functional: ${{ needs.test-inventory.outputs.ucc_modinput_functional}} + modinput_functional: ${{ needs.test-inventory.outputs.modinput_functional}} - security-virustotal: - continue-on-error: true - name: security-virustotal + virustotal: + runs-on: ubuntu-latest needs: build if: ${{ !cancelled() && needs.build.result == 'success' }} - runs-on: ubuntu-latest steps: - - uses: actions/download-artifact@v3 + - name: Run VirusTotal check + # uses: ./.github/actions/virustotal + uses: splunk/addonfactory-workflow-addon-release/.github/actions/virustotal@ci/lightweight-workflow-ADDON-66448 with: - name: package-splunkbase - path: build/package/ - - name: VirusTotal Scan - uses: crazy-max/ghaction-virustotal@v3 - with: - vt_api_key: ${{ secrets.VT_API_KEY }} - files: | - build/package/* + VT_API_KEY: ${{ secrets.VT_API_KEY }} run-requirements-unit-tests: + name: Requirements unit tests runs-on: ubuntu-latest needs: - build @@ -747,28 +486,15 @@ jobs: statuses: read checks: write steps: - - uses: actions/checkout@v3 - - name: Install Python 3 - uses: actions/setup-python@v4 - with: - python-version: 3.7 - - name: run-tests - uses: splunk/addonfactory-workflow-requirement-files-unit-tests@v1.4 - with: - input-files: tests/requirement_test/logs - - name: Archive production artifacts - if: ${{ !cancelled() }} - uses: actions/upload-artifact@v3 - with: - name: test-results - path: | - test_*.txt + - name: Run requirements unit tests + # uses: ./.github/actions/requirements-unit-tests + uses: splunk/addonfactory-workflow-addon-release/.github/actions/requirements-unit-tests@ci/lightweight-workflow-ADDON-66448 - appinspect: - name: quality-appinspect-${{ matrix.tags }} + appinspect-cli: + name: AppInspect CLI ${{ matrix.tags }} + runs-on: ubuntu-latest needs: build if: ${{ !cancelled() && needs.build.result == 'success' }} - runs-on: ubuntu-latest strategy: fail-fast: false matrix: @@ -782,39 +508,20 @@ jobs: - "splunk_appinspect" - "manual" steps: - - uses: actions/checkout@v3 - - uses: actions/download-artifact@v3 - with: - name: package-splunkbase - path: build/package/ - - name: Scan - uses: splunk/appinspect-cli-action@v1.12 - with: - app_path: build/package/ - included_tags: ${{ matrix.tags }} - result_file: appinspect_result_${{ matrix.tags }}.json - - name: upload-appinspect-report - if: ${{ !cancelled() }} - uses: actions/upload-artifact@v3 + - name: Run appinspect CLI + # uses: ./.github/actions/appinspect-cli + uses: splunk/addonfactory-workflow-addon-release/.github/actions/appinspect-cli@ci/lightweight-workflow-ADDON-66448 with: - name: appinspect_${{ matrix.tags }}_checks.json - path: appinspect_result_${{ matrix.tags }}.json - - name: upload-markdown - if: matrix.tags == 'manual' - uses: actions/upload-artifact@v3 - with: - name: check_markdown - path: | - *_markdown.txt + matrix_tags: ${{ matrix.tags }} appinspect-api: - name: appinspect api ${{ matrix.tags }} + name: AppInspect API ${{ matrix.tags }} + runs-on: ubuntu-latest needs: build if: | !cancelled() && needs.build.result == 'success' && ( github.base_ref == 'main' || github.ref_name == 'main' ) - runs-on: ubuntu-latest strategy: fail-fast: false matrix: @@ -823,99 +530,30 @@ jobs: - "self-service" - "" steps: - - uses: actions/checkout@v3 - - uses: actions/download-artifact@v3 - with: - name: package-splunkbase - path: build/package - - name: AppInspect API - uses: splunk/appinspect-api-action@v3.0 - with: - username: ${{ secrets.SPL_COM_USER }} - password: ${{ secrets.SPL_COM_PASSWORD }} - app_path: build/package/ - included_tags: ${{ matrix.tags }} - - uses: actions/upload-artifact@v3 - if: always() + - name: Run appinspect API + # uses: ./.github/actions/appinspect-api + uses: splunk/addonfactory-workflow-addon-release/.github/actions/appinspect-api@ci/lightweight-workflow-ADDON-66448 with: - name: appinspect-api-html-report-${{ matrix.tags }} - path: AppInspect_response.html + matrix_tags: ${{ matrix.tags }} + SPL_COM_USER: ${{ secrets.SPL_COM_USER }} + SPL_COM_PASSWORD: ${{ secrets.SPL_COM_PASSWORD }} artifact-registry: + name: Artifact registry runs-on: ubuntu-latest needs: - - security-virustotal + - virustotal - meta - if: ${{ !cancelled() && needs.security-virustotal.result == 'success' && needs.meta.result == 'success' }} - outputs: - artifact: ${{ steps.artifactid.outputs.result }} + if: ${{ !cancelled() && needs.virustotal.result == 'success' && needs.meta.result == 'success' }} permissions: contents: read packages: write steps: - - uses: actions/checkout@v3 - - uses: actions/download-artifact@v3 + - name: Run artifact registry + # uses: ./.github/actions/artifact-registry + uses: splunk/addonfactory-workflow-addon-release/.github/actions/artifact-registry@ci/lightweight-workflow-ADDON-66448 with: - name: package-splunkbase - path: build/package/splunkbase - - id: getappid - run: | - appid=$(jq -r '.info.id.name' package/app.manifest) - echo appid="$appid" - echo "result=$appid" >> "$GITHUB_OUTPUT" - - run: | - curl -LO https://github.com/oras-project/oras/releases/download/v0.12.0/oras_0.12.0_linux_amd64.tar.gz - mkdir -p oras-install/ - tar -zxf oras_0.12.0_*.tar.gz -C oras-install/ - mv oras-install/oras /usr/local/bin/ - rm -rf oras_0.12.0_*.tar.gz oras-install/ - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 - - name: Login to GitHub Packages Docker Registry - uses: docker/login-action@v2.2.0 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ github.token }} - - name: Docker meta - id: meta - uses: docker/metadata-action@v4.6.0 - with: - images: ghcr.io/${{ github.repository }} - tags: | - type=semver,pattern=v{{major}}.{{minor}},prefix=${{ steps.getappid.outputs.result }}- - type=semver,pattern=v{{major}},prefix=${{ steps.getappid.outputs.result }}- - type=semver,pattern=v{{version}},prefix=${{ steps.getappid.outputs.result }}- - type=semver,pattern={{major}}.{{minor}},prefix=${{ steps.getappid.outputs.result }}- - type=semver,pattern={{major}},prefix=${{ steps.getappid.outputs.result }}- - type=semver,pattern={{version}},prefix=${{ steps.getappid.outputs.result }}- - type=ref,event=branch,prefix=${{ steps.getappid.outputs.result }}- - type=ref,event=pr,prefix=${{ steps.getappid.outputs.result }}- - type=sha,prefix=${{ steps.getappid.outputs.result }}- - type=sha,format=long,prefix=${{ steps.getappid.outputs.result }}- - - name: Upload artifacts - run: | - tee /tmp/tags &>/dev/null <>$line<<" - oras push \ - --manifest-config /dev/null:application/vnd.splunk.ent.package.v1.tar+gzip \ - "$line" \ - "${{ steps.getappid.outputs.result }}".spl - echo " complete" - done < /tmp/tags - popd - - name: Output artifact locator - id: artifactid - run: | - echo "result= ${{ needs.meta.outputs.sc4s }}" >> "$GITHUB_OUTPUT" + sc4s: ${{ needs.meta.outputs.sc4s }} setup: needs: @@ -2671,13 +2309,13 @@ jobs: - meta - compliance-copyrights - lint - - review_secrets + - review-secrets - semgrep - build - - security-virustotal + - virustotal - test-inventory - - run-unit-tests - - appinspect + - run-unit-tests-3_7 + - appinspect-cli - setup - run-knowledge-tests - run-modinput-tests From e0a05dcbfd93298891609a1ca51e86693611c87a Mon Sep 17 00:00:00 2001 From: Adam Date: Wed, 6 Dec 2023 14:07:42 +0100 Subject: [PATCH 3/5] ci: use local paths to composite actions --- .../workflows/reusable-build-test-release.yml | 69 ++++++------------- .github/workflows/reusable-lightweight.yml | 50 +++++--------- 2 files changed, 37 insertions(+), 82 deletions(-) diff --git a/.github/workflows/reusable-build-test-release.yml b/.github/workflows/reusable-build-test-release.yml index 49fccb5d6..5f281b684 100644 --- a/.github/workflows/reusable-build-test-release.yml +++ b/.github/workflows/reusable-build-test-release.yml @@ -206,8 +206,7 @@ jobs: statuses: write steps: - name: Run title validation - # uses: ./.github/actions/validate-pr-title - uses: splunk/addonfactory-workflow-addon-release/.github/actions/validate-pr-title@ci/lightweight-workflow-ADDON-66448 + uses: ./.github/actions/validate-pr-title with: GITHUB_TOKEN: ${{ github.token }} @@ -235,8 +234,7 @@ jobs: steps: - name: Run meta preparation id: meta - # uses: ./.github/actions/meta - uses: splunk/addonfactory-workflow-addon-release/.github/actions/meta@ci/lightweight-workflow-ADDON-66448 + uses: ./.github/actions/meta with: SA_GH_USER_NAME: ${{ secrets.SA_GH_USER_NAME }} SA_GH_USER_EMAIL: ${{ secrets.SA_GH_USER_EMAIL }} @@ -251,8 +249,7 @@ jobs: if: ${{ needs.setup-workflow.outputs.skip-workflow != 'Yes' }} steps: - name: Run FOSSA scan - # uses: ./.github/actions/fossa-scan - uses: splunk/addonfactory-workflow-addon-release/.github/actions/fossa-scan@ci/lightweight-workflow-ADDON-66448 + uses: ./.github/actions/fossa-scan with: FOSSA_API_KEY: ${{ secrets.FOSSA_API_KEY }} @@ -264,8 +261,7 @@ jobs: - fossa-scan steps: - name: Run FOSSA test - # uses: ./.github/actions/fossa-test - uses: splunk/addonfactory-workflow-addon-release/.github/actions/fossa-test@ci/lightweight-workflow-ADDON-66448 + uses: ./.github/actions/fossa-test with: FOSSA_API_KEY: ${{ secrets.FOSSA_API_KEY }} @@ -277,8 +273,7 @@ jobs: if: ${{ needs.setup-workflow.outputs.skip-workflow != 'Yes' }} steps: - name: Run compliance copyrights - # uses: ./.github/actions/compliance-copyrights - uses: splunk/addonfactory-workflow-addon-release/.github/actions/compliance-copyrights@ci/lightweight-workflow-ADDON-66448 + uses: ./.github/actions/compliance-copyrights lint: name: Lint @@ -288,8 +283,7 @@ jobs: if: ${{ needs.setup-workflow.outputs.skip-workflow != 'Yes' }} steps: - name: Run linting checks - # uses: ./.github/actions/lint - uses: splunk/addonfactory-workflow-addon-release/.github/actions/lint@ci/lightweight-workflow-ADDON-66448 + uses: ./.github/actions/lint review-secrets: name: Review secrets @@ -299,8 +293,7 @@ jobs: if: ${{ needs.setup-workflow.outputs.skip-workflow != 'Yes' }} steps: - name: Run secrets review - # uses: ./.github/actions/review-secrets - uses: splunk/addonfactory-workflow-addon-release/.github/actions/review-secrets@ci/lightweight-workflow-ADDON-66448 + uses: ./.github/actions/review-secrets semgrep: name: Semgrep security check @@ -310,8 +303,7 @@ jobs: if: ${{ needs.setup-workflow.outputs.skip-workflow != 'Yes' }} steps: - name: Run semgrep - # uses: ./.github/actions/semgrep - uses: splunk/addonfactory-workflow-addon-release/.github/actions/semgrep@ci/lightweight-workflow-ADDON-66448 + uses: ./.github/actions/semgrep with: SEMGREP_PUBLISH_TOKEN: ${{ secrets.SEMGREP_PUBLISH_TOKEN }} @@ -333,11 +325,10 @@ jobs: steps: - name: Run test inventory check id: test-inventory - # uses: ./.github/actions/test-inventory - uses: splunk/addonfactory-workflow-addon-release/.github/actions/test-inventory@ci/lightweight-workflow-ADDON-66448 + uses: ./.github/actions/test-inventory # Two separate unit test jobs needed as jobs that depend on unit-test success can't proceed - # if any matrix job fails. Currently python 3.9 always fails as it's not supported. + # if any matrix job fails. Currently python 3.9 may fail as it's not supported in all TAs. # TODO: group these jobs into the matrix once python 3.9 is supported run-unit-tests-3_7: @@ -356,8 +347,7 @@ jobs: steps: - name: Run unit tests for python 3.7 id: unit-tests-3_7 - # uses: ./.github/actions/unit-tests - uses: splunk/addonfactory-workflow-addon-release/.github/actions/unit-tests@ci/lightweight-workflow-ADDON-66448 + uses: ./.github/actions/unit-tests with: python_version: '3.7' @@ -378,8 +368,7 @@ jobs: steps: - name: Run unit tests for python 3.9 id: unit-tests-3_9 - # uses: ./.github/actions/unit-tests - uses: splunk/addonfactory-workflow-addon-release/.github/actions/unit-tests@ci/lightweight-workflow-ADDON-66448 + uses: ./.github/actions/unit-tests with: python_version: '3.9' @@ -394,12 +383,6 @@ jobs: - review-secrets - semgrep - run-unit-tests-3_7 - # strategy: - # fail-fast: false - # matrix: - # python-version: - # - "3.7" - # - "3.9" if: ${{ !cancelled() && (needs.run-unit-tests-3_7.result == 'success' || needs.run-unit-tests-3_7.result == 'skipped') }} outputs: buildname: ${{ steps.build.outputs.buildname }} @@ -409,8 +392,7 @@ jobs: steps: - name: Run build 3.7 id: build - # uses: ./.github/actions/build - uses: splunk/addonfactory-workflow-addon-release/.github/actions/build@ci/lightweight-workflow-ADDON-66448 + uses: ./.github/actions/build with: python_version: "3.7" SA_GH_USER_NAME: ${{ secrets.SA_GH_USER_NAME }} @@ -434,20 +416,13 @@ jobs: - review-secrets - semgrep - run-unit-tests-3_7 - # strategy: - # fail-fast: false - # matrix: - # python-version: - # - "3.7" - # - "3.9" if: ${{ !cancelled() && (needs.run-unit-tests-3_7.result == 'success' || needs.run-unit-tests-3_7.result == 'skipped') }} permissions: contents: write packages: read steps: - name: Run build 3.9 - # uses: ./.github/actions/build - uses: splunk/addonfactory-workflow-addon-release/.github/actions/build@ci/lightweight-workflow-ADDON-66448 + uses: ./.github/actions/build with: python_version: "3.9" SA_GH_USER_NAME: ${{ secrets.SA_GH_USER_NAME }} @@ -461,13 +436,13 @@ jobs: modinput_functional: ${{ needs.test-inventory.outputs.modinput_functional}} virustotal: + continue-on-error: true runs-on: ubuntu-latest needs: build if: ${{ !cancelled() && needs.build.result == 'success' }} steps: - name: Run VirusTotal check - # uses: ./.github/actions/virustotal - uses: splunk/addonfactory-workflow-addon-release/.github/actions/virustotal@ci/lightweight-workflow-ADDON-66448 + uses: ./.github/actions/virustotal with: VT_API_KEY: ${{ secrets.VT_API_KEY }} @@ -487,8 +462,7 @@ jobs: checks: write steps: - name: Run requirements unit tests - # uses: ./.github/actions/requirements-unit-tests - uses: splunk/addonfactory-workflow-addon-release/.github/actions/requirements-unit-tests@ci/lightweight-workflow-ADDON-66448 + uses: ./.github/actions/requirements-unit-tests appinspect-cli: name: AppInspect CLI ${{ matrix.tags }} @@ -509,8 +483,7 @@ jobs: - "manual" steps: - name: Run appinspect CLI - # uses: ./.github/actions/appinspect-cli - uses: splunk/addonfactory-workflow-addon-release/.github/actions/appinspect-cli@ci/lightweight-workflow-ADDON-66448 + uses: ./.github/actions/appinspect-cli with: matrix_tags: ${{ matrix.tags }} @@ -531,8 +504,7 @@ jobs: - "" steps: - name: Run appinspect API - # uses: ./.github/actions/appinspect-api - uses: splunk/addonfactory-workflow-addon-release/.github/actions/appinspect-api@ci/lightweight-workflow-ADDON-66448 + uses: ./.github/actions/appinspect-api with: matrix_tags: ${{ matrix.tags }} SPL_COM_USER: ${{ secrets.SPL_COM_USER }} @@ -550,8 +522,7 @@ jobs: packages: write steps: - name: Run artifact registry - # uses: ./.github/actions/artifact-registry - uses: splunk/addonfactory-workflow-addon-release/.github/actions/artifact-registry@ci/lightweight-workflow-ADDON-66448 + uses: ./.github/actions/artifact-registry with: sc4s: ${{ needs.meta.outputs.sc4s }} diff --git a/.github/workflows/reusable-lightweight.yml b/.github/workflows/reusable-lightweight.yml index 42b0fb381..537da1083 100644 --- a/.github/workflows/reusable-lightweight.yml +++ b/.github/workflows/reusable-lightweight.yml @@ -60,8 +60,7 @@ jobs: statuses: write steps: - name: Run title validation - # uses: ./.github/actions/validate-pr-title - uses: splunk/addonfactory-workflow-addon-release/.github/actions/validate-pr-title@ci/lightweight-workflow-ADDON-66448 + uses: ./.github/actions/validate-pr-title with: GITHUB_TOKEN: ${{ github.token }} @@ -73,8 +72,7 @@ jobs: steps: - name: Run meta preparation id: meta - # uses: ./.github/actions/meta - uses: splunk/addonfactory-workflow-addon-release/.github/actions/meta@ci/lightweight-workflow-ADDON-66448 + uses: ./.github/actions/meta with: SA_GH_USER_NAME: ${{ secrets.SA_GH_USER_NAME }} SA_GH_USER_EMAIL: ${{ secrets.SA_GH_USER_EMAIL }} @@ -86,8 +84,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Run FOSSA scan - # uses: ./.github/actions/fossa-scan - uses: splunk/addonfactory-workflow-addon-release/.github/actions/fossa-scan@ci/lightweight-workflow-ADDON-66448 + uses: ./.github/actions/fossa-scan with: FOSSA_API_KEY: ${{ secrets.FOSSA_API_KEY }} @@ -98,8 +95,7 @@ jobs: - fossa-scan steps: - name: Run FOSSA test - # uses: ./.github/actions/fossa-test - uses: splunk/addonfactory-workflow-addon-release/.github/actions/fossa-test@ci/lightweight-workflow-ADDON-66448 + uses: ./.github/actions/fossa-test with: FOSSA_API_KEY: ${{ secrets.FOSSA_API_KEY }} @@ -108,32 +104,28 @@ jobs: runs-on: ubuntu-latest steps: - name: Run compliance copyrights - # uses: ./.github/actions/compliance-copyrights - uses: splunk/addonfactory-workflow-addon-release/.github/actions/compliance-copyrights@ci/lightweight-workflow-ADDON-66448 + uses: ./.github/actions/compliance-copyrights lint: name: Lint runs-on: ubuntu-latest steps: - name: Run linting checks - # uses: ./.github/actions/lint - uses: splunk/addonfactory-workflow-addon-release/.github/actions/lint@ci/lightweight-workflow-ADDON-66448 + uses: ./.github/actions/lint review-secrets: name: Review secrets runs-on: ubuntu-latest steps: - name: Run secrets review - # uses: ./.github/actions/review-secrets - uses: splunk/addonfactory-workflow-addon-release/.github/actions/review-secrets@ci/lightweight-workflow-ADDON-66448 + uses: ./.github/actions/review-secrets semgrep: name: Semgrep security check runs-on: ubuntu-latest steps: - name: Run semgrep - # uses: ./.github/actions/semgrep - uses: splunk/addonfactory-workflow-addon-release/.github/actions/semgrep@ci/lightweight-workflow-ADDON-66448 + uses: ./.github/actions/semgrep with: SEMGREP_PUBLISH_TOKEN: ${{ secrets.SEMGREP_PUBLISH_TOKEN }} @@ -148,11 +140,10 @@ jobs: steps: - name: Run test inventory check id: test-inventory - # uses: ./.github/actions/test-inventory - uses: splunk/addonfactory-workflow-addon-release/.github/actions/test-inventory@ci/lightweight-workflow-ADDON-66448 + uses: ./.github/actions/test-inventory # Two separate unit test jobs needed as jobs that depend on unit-test success can't proceed - # if any matrix job fails. Currently python 3.9 always fails as it's not supported. + # if any matrix job fails. Currently python 3.9 may fail as it's not supported in all TAs. # TODO: group these jobs into the matrix once python 3.9 is supported run-unit-tests-3_7: @@ -171,8 +162,7 @@ jobs: steps: - name: Run unit tests for python 3.7 id: unit-tests-3_7 - # uses: ./.github/actions/unit-tests - uses: splunk/addonfactory-workflow-addon-release/.github/actions/unit-tests@ci/lightweight-workflow-ADDON-66448 + uses: ./.github/actions/unit-tests with: python_version: '3.7' @@ -193,8 +183,7 @@ jobs: steps: - name: Run unit tests for python 3.9 id: unit-tests-3_9 - # uses: ./.github/actions/unit-tests - uses: splunk/addonfactory-workflow-addon-release/.github/actions/unit-tests@ci/lightweight-workflow-ADDON-66448 + uses: ./.github/actions/unit-tests with: python_version: '3.9' @@ -221,8 +210,7 @@ jobs: packages: read steps: - name: Run build - # uses: ./.github/actions/build - uses: splunk/addonfactory-workflow-addon-release/.github/actions/build@ci/lightweight-workflow-ADDON-66448 + uses: ./.github/actions/build with: python_version: ${{ matrix.python-version }} SA_GH_USER_NAME: ${{ secrets.SA_GH_USER_NAME }} @@ -241,8 +229,7 @@ jobs: if: ${{ !cancelled() && needs.build.result == 'success' }} steps: - name: Run VirusTotal check - # uses: ./.github/actions/virustotal - uses: splunk/addonfactory-workflow-addon-release/.github/actions/virustotal@ci/lightweight-workflow-ADDON-66448 + uses: ./.github/actions/virustotal with: VT_API_KEY: ${{ secrets.VT_API_KEY }} @@ -262,8 +249,7 @@ jobs: checks: write steps: - name: Run requirements unit tests - # uses: ./.github/actions/requirements-unit-tests - uses: splunk/addonfactory-workflow-addon-release/.github/actions/requirements-unit-tests@ci/lightweight-workflow-ADDON-66448 + uses: ./.github/actions/requirements-unit-tests appinspect-cli: name: AppInspect CLI ${{ matrix.tags }} @@ -284,8 +270,7 @@ jobs: - "manual" steps: - name: Run appinspect CLI - # uses: ./.github/actions/appinspect-cli - uses: splunk/addonfactory-workflow-addon-release/.github/actions/appinspect-cli@ci/lightweight-workflow-ADDON-66448 + uses: ./.github/actions/appinspect-cli with: matrix_tags: ${{ matrix.tags }} @@ -301,7 +286,6 @@ jobs: packages: write steps: - name: Run artifact registry - # uses: ./.github/actions/artifact-registry - uses: splunk/addonfactory-workflow-addon-release/.github/actions/artifact-registry@ci/lightweight-workflow-ADDON-66448 + uses: ./.github/actions/artifact-registry with: sc4s: ${{ needs.meta.outputs.sc4s }} From a689ad5dac0475a3818acf171dd1bc19e6bf776e Mon Sep 17 00:00:00 2001 From: Adam Date: Fri, 8 Dec 2023 13:42:30 +0100 Subject: [PATCH 4/5] ci(ADDON-66730): add HTTPS with PAT git config for installing internal repos as dependencies --- .github/actions/unit-tests/action.yml | 5 +++++ .github/workflows/reusable-build-test-release.yml | 9 ++++++++- .github/workflows/reusable-lightweight.yml | 4 ++++ 3 files changed, 17 insertions(+), 1 deletion(-) diff --git a/.github/actions/unit-tests/action.yml b/.github/actions/unit-tests/action.yml index ea36bd874..2b2161fed 100644 --- a/.github/actions/unit-tests/action.yml +++ b/.github/actions/unit-tests/action.yml @@ -3,6 +3,10 @@ name: Run unit tests inputs: python_version: required: true + GH_USER_ADMIN: + required: true + GH_TOKEN_ADMIN: + required: true runs: using: composite @@ -30,6 +34,7 @@ runs: poetry export --without-hashes --dev -o requirements_dev.txt fi if [ ! -f requirements_dev.txt ]; then echo no requirements;exit 0 ;fi + git config --global url."https://${{ inputs.GH_TOKEN_ADMIN }}@github.com".insteadOf https://github.com pip install -r requirements_dev.txt - name: Create directories diff --git a/.github/workflows/reusable-build-test-release.yml b/.github/workflows/reusable-build-test-release.yml index 5f281b684..89e340d4c 100644 --- a/.github/workflows/reusable-build-test-release.yml +++ b/.github/workflows/reusable-build-test-release.yml @@ -12,6 +12,9 @@ on: GH_TOKEN_ADMIN: description: Github admin token required: true + GH_USER_ADMIN: + description: Github admin user + required: true SEMGREP_PUBLISH_TOKEN: description: Semgrep token required: true @@ -350,6 +353,8 @@ jobs: uses: ./.github/actions/unit-tests with: python_version: '3.7' + GH_USER_ADMIN: ${{ secrets.GH_USER_ADMIN }} + GH_TOKEN_ADMIN: ${{ secrets.GH_TOKEN_ADMIN }} run-unit-tests-3_9: name: Unit tests @@ -368,9 +373,11 @@ jobs: steps: - name: Run unit tests for python 3.9 id: unit-tests-3_9 - uses: ./.github/actions/unit-tests + uses: ./.github/actions/unit-tests with: python_version: '3.9' + GH_USER_ADMIN: ${{ secrets.GH_USER_ADMIN }} + GH_TOKEN_ADMIN: ${{ secrets.GH_TOKEN_ADMIN }} build: name: Build python 3.7 diff --git a/.github/workflows/reusable-lightweight.yml b/.github/workflows/reusable-lightweight.yml index 537da1083..9f6fc68ab 100644 --- a/.github/workflows/reusable-lightweight.yml +++ b/.github/workflows/reusable-lightweight.yml @@ -165,6 +165,8 @@ jobs: uses: ./.github/actions/unit-tests with: python_version: '3.7' + GH_USER_ADMIN: ${{ secrets.GH_USER_ADMIN }} + GH_TOKEN_ADMIN: ${{ secrets.GH_TOKEN_ADMIN }} run-unit-tests-3_9: name: Unit tests 3.9 @@ -186,6 +188,8 @@ jobs: uses: ./.github/actions/unit-tests with: python_version: '3.9' + GH_USER_ADMIN: ${{ secrets.GH_USER_ADMIN }} + GH_TOKEN_ADMIN: ${{ secrets.GH_TOKEN_ADMIN }} build: name: Build python-${{ matrix.python-version }} From 6c311de41bc52c959a69303ba9422e4e7c2f44fe Mon Sep 17 00:00:00 2001 From: Adam Date: Fri, 8 Dec 2023 16:41:39 +0100 Subject: [PATCH 5/5] ci: update appinspect CLI action version --- .github/actions/appinspect-cli/action.yml | 2 +- .github/actions/unit-tests/action.yml | 2 -- .github/workflows/reusable-build-test-release.yml | 11 +++-------- .github/workflows/reusable-lightweight.yml | 2 -- 4 files changed, 4 insertions(+), 13 deletions(-) diff --git a/.github/actions/appinspect-cli/action.yml b/.github/actions/appinspect-cli/action.yml index 97e139a57..fe5e394e8 100644 --- a/.github/actions/appinspect-cli/action.yml +++ b/.github/actions/appinspect-cli/action.yml @@ -22,7 +22,7 @@ runs: path: build/package/ - name: Scan - uses: splunk/appinspect-cli-action@v1.9 + uses: splunk/appinspect-cli-action@v1.12 with: app_path: build/package/ included_tags: ${{ inputs.matrix_tags }} diff --git a/.github/actions/unit-tests/action.yml b/.github/actions/unit-tests/action.yml index 2b2161fed..a4c29e12a 100644 --- a/.github/actions/unit-tests/action.yml +++ b/.github/actions/unit-tests/action.yml @@ -3,8 +3,6 @@ name: Run unit tests inputs: python_version: required: true - GH_USER_ADMIN: - required: true GH_TOKEN_ADMIN: required: true diff --git a/.github/workflows/reusable-build-test-release.yml b/.github/workflows/reusable-build-test-release.yml index 89e340d4c..66f6d8be4 100644 --- a/.github/workflows/reusable-build-test-release.yml +++ b/.github/workflows/reusable-build-test-release.yml @@ -12,9 +12,6 @@ on: GH_TOKEN_ADMIN: description: Github admin token required: true - GH_USER_ADMIN: - description: Github admin user - required: true SEMGREP_PUBLISH_TOKEN: description: Semgrep token required: true @@ -335,7 +332,7 @@ jobs: # TODO: group these jobs into the matrix once python 3.9 is supported run-unit-tests-3_7: - name: Unit tests + name: Unit tests python 3.7 if: ${{ needs.test-inventory.outputs.unit == 'true' }} runs-on: ubuntu-latest needs: @@ -353,11 +350,10 @@ jobs: uses: ./.github/actions/unit-tests with: python_version: '3.7' - GH_USER_ADMIN: ${{ secrets.GH_USER_ADMIN }} GH_TOKEN_ADMIN: ${{ secrets.GH_TOKEN_ADMIN }} run-unit-tests-3_9: - name: Unit tests + name: Unit tests python 3.9 if: ${{ needs.test-inventory.outputs.unit == 'true' }} runs-on: ubuntu-latest continue-on-error: true @@ -373,10 +369,9 @@ jobs: steps: - name: Run unit tests for python 3.9 id: unit-tests-3_9 - uses: ./.github/actions/unit-tests + uses: ./.github/actions/unit-tests with: python_version: '3.9' - GH_USER_ADMIN: ${{ secrets.GH_USER_ADMIN }} GH_TOKEN_ADMIN: ${{ secrets.GH_TOKEN_ADMIN }} build: diff --git a/.github/workflows/reusable-lightweight.yml b/.github/workflows/reusable-lightweight.yml index 9f6fc68ab..7c7692fdb 100644 --- a/.github/workflows/reusable-lightweight.yml +++ b/.github/workflows/reusable-lightweight.yml @@ -165,7 +165,6 @@ jobs: uses: ./.github/actions/unit-tests with: python_version: '3.7' - GH_USER_ADMIN: ${{ secrets.GH_USER_ADMIN }} GH_TOKEN_ADMIN: ${{ secrets.GH_TOKEN_ADMIN }} run-unit-tests-3_9: @@ -188,7 +187,6 @@ jobs: uses: ./.github/actions/unit-tests with: python_version: '3.9' - GH_USER_ADMIN: ${{ secrets.GH_USER_ADMIN }} GH_TOKEN_ADMIN: ${{ secrets.GH_TOKEN_ADMIN }} build: