ci.yml

name: CI

on:
  merge_group:
  workflow_dispatch:
  pull_request:
  push:
    branches:
      - main

permissions:
  contents: read
  issues: write

jobs:
  sanity:
    name: Sanity
    runs-on: ubuntu-20.04

    steps:
      - name: Harden Runner
        uses: step-security/harden-runner@91182cccc01eb5e619899d80e4e971d6181294a7 # v2.10.1
        with:
          egress-policy: audit

      - name: Checkout
        uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7

      - name: Install Python
        uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5.1.0
        with:
          python-version: '3.11'

      - name: Run pre-commit checks
        uses: pre-commit/action@2c7b3805fd2a0fd8c1884dcaebf91fc102a13ecd # v3.0.1
        env:
          SETUPTOOLS_USE_DISTUTILS: stdlib

      - name: Check clowdapp manifest
        run: bash .github/scripts/check_clowdapp.sh

  smokes-labeler:
    name: Smoke Test Label
    runs-on: ubuntu-20.04

    permissions:
      contents: read
      issues: write
      pull-requests: write

    steps:
      - name: Harden Runner
        uses: step-security/harden-runner@91182cccc01eb5e619899d80e4e971d6181294a7 # v2.10.1
        with:
          egress-policy: audit

      - name: Checkout
        uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
        with:
          fetch-depth: 0

      - name: get Docker image files
        id: docker-files
        run: .github/scripts/files_require_smokes.sh >> docker-files.txt

      - name: add other required files
        id: add-files
        run: |
          echo .dockerignore >> docker-files.txt;
          echo Dockerfile >> docker-files.txt;
          echo pr_check.sh >> docker-files.txt;
          echo deploy/clowdapp.yaml >> docker-files.txt;

      - name: Show Dockerfiles
        run: cat docker-files.txt

      - name: Get changed files
        id: changed-files
        uses: tj-actions/changed-files@d6babd6899969df1a11d14c368283ea4436bca78 # v44.5.2
        with:
          files_from_source_file: docker-files.txt

      - name: Set whether to run tests
        id: check-files
        run: |
          if [ ! -z "${{ steps.changed-files.outputs.all_changed_and_modified_files }}" ]; then
            echo "RUN_TESTS=true" >> $GITHUB_ENV
          fi

      - name: Set smokes-required label
        if: env.RUN_TESTS == 'true'
        uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1
        continue-on-error: true
        with:
          script: |
            github.rest.issues.addLabels({
              issue_number: context.issue.number,
              owner: context.repo.owner,
              repo: context.repo.repo,
              labels: [ 'smokes-required' ]
            })

            const response = await github.rest.issues.listLabelsOnIssue({
                issue_number: context.issue.number,
                owner: context.repo.owner,
                repo: context.repo.repo,
              })

            const labels = response.data.map(({name}) => name)

            if (labels.includes('ok-to-skip-smokes')) {
              github.rest.issues.removeLabel({
                issue_number: context.issue.number,
                owner: context.repo.owner,
                repo: context.repo.repo,
                name: 'ok-to-skip-smokes'
              })
            }

      - name: Set ok-to-skip-smokes label
        if: env.RUN_TESTS != 'true'
        uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1
        continue-on-error: true
        with:
          script: |
            github.rest.issues.addLabels({
              issue_number: context.issue.number,
              owner: context.repo.owner,
              repo: context.repo.repo,
              labels: [ 'ok-to-skip-smokes' ]
            })

            const response = await github.rest.issues.listLabelsOnIssue({
                issue_number: context.issue.number,
                owner: context.repo.owner,
                repo: context.repo.repo,
              })

            const labels = response.data.map(({name}) => name)

            if (labels.includes('smokes-required')) {
              github.rest.issues.removeLabel({
                issue_number: context.issue.number,
                owner: context.repo.owner,
                repo: context.repo.repo,
                name: 'smokes-required'
              })
            }

  changed-files:
    name: Detect changed files
    runs-on: ubuntu-20.04
    outputs:
      run_tests: ${{ steps.check-files-or-fork.outputs.run_tests }}

    steps:
      - name: Harden Runner
        uses: step-security/harden-runner@91182cccc01eb5e619899d80e4e971d6181294a7 # v2.10.1
        with:
          egress-policy: audit

      - name: Checkout
        uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
        with:
          fetch-depth: 0

      - name: Get changed files
        id: changed-files
        uses: tj-actions/changed-files@d6babd6899969df1a11d14c368283ea4436bca78 # v44.5.2
        with:
          files: |
            db_functions/**
            koku/**
            .github/postgres/**
            .github/scripts/check_migrations.sh
            .github/workflows/ci.yml
            Pipfile.lock

      - name: Check files or fork
        id: check-files-or-fork
        run: |
          if [ "${{ steps.changed-files.outputs.any_modified }}" == "true" ] || [ "${{ github.event.pull_request.head.repo.full_name }}" != "project-koku/koku" ]; then
            echo "run_tests=true" >> $GITHUB_OUTPUT
          fi

      - name: Show changed files
        run: |
          echo "Changed files:"
          for file in ${{ steps.changed-files.outputs.all_changed_and_modified_files }}; do
            echo "  $file"
          done

  units:
    name: Units - ${{ matrix.python-version }}
    needs: changed-files
    runs-on: ubuntu-20.04
    strategy:
      fail-fast: false
      max-parallel: 4
      matrix:
        python-version:
          - '3.11'
    env:
      COMPOSE_FILE: .github/postgres/docker-compose.yaml

    steps:
      - name: Harden Runner
        uses: step-security/harden-runner@91182cccc01eb5e619899d80e4e971d6181294a7 # v2.10.1
        with:
          egress-policy: audit

      - name: Checkout
        if: needs.changed-files.outputs.run_tests == 'true'
        uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
        with:
          fetch-depth: 0

      - name: Display build environment
        if: needs.changed-files.outputs.run_tests == 'true'
        run: printenv

      - name: Start the postgres DB
        if: needs.changed-files.outputs.run_tests == 'true'
        run: docker compose up -d db

      - name: Set up Python ${{ matrix.python-version }}
        if: needs.changed-files.outputs.run_tests == 'true'
        uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5.1.0
        with:
          python-version: ${{ matrix.python-version }}

      - name: Install pipenv
        if: needs.changed-files.outputs.run_tests == 'true'
        run: sudo python3 -m pip install pipenv

      - name: Cache dependencies
        if: needs.changed-files.outputs.run_tests == 'true'
        id: cache-dependencies
        uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4.0.2
        with:
          path: |
            ~/.cache/pipenv
            ~/.local/share/virtualenvs
          key: ${{ runner.os }}-env-${{ matrix.python-version }}-${{ hashFiles('**/Pipfile.lock') }}-${{ secrets.ACTIONS_CACHE_KEY_UUID }}

      - name: Install dependencies
        if: needs.changed-files.outputs.run_tests == 'true' && steps.cache-dependencies.outputs.cache-hit != 'true'
        run: |
          pipenv install --dev --deploy --python ${{ matrix.python-version }}

      - name: Check migrations
        if: |
          needs.changed-files.outputs.run_tests == 'true'
          && !(contains(github.event.pull_request.labels.*.name, 'skip-migration-check'))
        run: bash .github/scripts/check_migrations.sh
        env:
          DATABASE_SERVICE_NAME: POSTGRES_SQL
          DATABASE_ENGINE: postgresql
          DATABASE_NAME: postgres
          DATABASE_USER: postgres
          DATABASE_PASSWORD: postgres
          POSTGRES_SQL_SERVICE_HOST: localhost
          POSTGRES_SQL_SERVICE_PORT: ${{ job.services.postgres.ports[5432] }}
          PROMETHEUS_MULTIPROC_DIR: /tmp

      - name: Run unit tests
        if: needs.changed-files.outputs.run_tests == 'true'
        id: unit_tests_run
        run: pipenv run coverage run ./koku/manage.py test --noinput --verbosity 2 ./koku/
        env:
          DATABASE_SERVICE_NAME: POSTGRES_SQL
          DATABASE_ENGINE: postgresql
          DATABASE_NAME: postgres
          DATABASE_USER: postgres
          DATABASE_PASSWORD: postgres
          POSTGRES_SQL_SERVICE_HOST: localhost
          POSTGRES_SQL_SERVICE_PORT: ${{ job.services.postgres.ports[5432] }}
          HIVE_DATABASE_PASSWORD: hivedbpw
          ACCOUNT_ENHANCED_METRICS: True
          PROMETHEUS_MULTIPROC_DIR: /tmp
          TRINO_DATE_STEP: 31
          MIDDLEWARE_TIME_TO_LIVE: 0
          ENHANCED_ORG_ADMIN: True
          ENABLE_S3_ARCHIVING: True

      - name: Show test status
        if: needs.changed-files.outputs.run_tests == 'true'
        id: show_test_status
        run: echo "Unit tests and migration check were '${{ steps.unit_tests_run.outcome }}'"

      - name: Convert coverage report to XML
        if: steps.unit_tests_run.outcome == 'success'
        run: pipenv run coverage xml

      - name: Upload test coverage file
        if: steps.unit_tests_run.outcome == 'success'
        uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3
        with:
          name: coverage
          path: coverage.xml
          overwrite: true

  coverage:
    name: Coverage
    needs: [ changed-files,units ]
    runs-on: ubuntu-20.04
    permissions:
      contents: write
      statuses: write

    steps:

      - name: Harden Runner
        uses: step-security/harden-runner@91182cccc01eb5e619899d80e4e971d6181294a7 # v2.10.1
        with:
          egress-policy: audit

      - name: Checkout
        # this checkout is required for the coverage report. If we don't do this, then
        # the uploaded report is invalid and codecov doesn't know how to process it.
        if: needs.changed-files.outputs.run_tests == 'true'
        uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
        with:
          fetch-depth: 0

      - name: Download coverage result from units
        if: needs.changed-files.outputs.run_tests == 'true'
        uses: actions/download-artifact@65a9edc5881444af0b9093a5e628f2fe47ea3b2e # v4.1.7
        with:
          name: coverage

      - name: Upload coverage to Codecov
        if: needs.changed-files.outputs.run_tests == 'true'
        uses: codecov/codecov-action@125fc84a9a348dbcf27191600683ec096ec9021c # v4.4.1
        env:
          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
        with:
          file: ./coverage.xml
          flags: unittests
          name: Python-${{ matrix.python-version}}
          fail_ci_if_error: true
          plugin: pycoverage  # Only run one plugin even though we don't want any to run.

      - name: Set Codecov job status on skipped tests
        if: needs.changed-files.outputs.run_tests != 'true'
        env:
          GITHUB_TOKEN: "${{ github.token }}"
        run: |
          gh api "/repos/project-koku/koku/statuses/${{ github.event.pull_request.head.sha }}" -f description="codecov skipped because unit tests were not required" -f context="codecov/patch" -f state="success"
          gh api "/repos/project-koku/koku/statuses/${{ github.event.pull_request.head.sha }}" -f description="codecov skipped because unit tests were not required" -f context="codecov/project" -f state="success"