ci.yml

name: CI

on:
  merge_group:
  workflow_dispatch:
  pull_request:
  push:
    branches:
      - main

jobs:
  sanity:
    name: Sanity
    runs-on: ubuntu-20.04

    steps:
      - name: Checkout
        uses: actions/checkout@v3.1.0

      - name: Install Python
        uses: actions/setup-python@v4.3.0
        with:
          python-version: '3.9'

      - name: Run pre-commit checks
        uses: pre-commit/action@v3.0.0
        env:
          SETUPTOOLS_USE_DISTUTILS: stdlib

      - name: Check clowdapp manifest
        run: bash .github/scripts/check_clowdapp.sh

  smokes-labeler:
    name: Smoke Test Label
    runs-on: ubuntu-20.04

    steps:
      - name: Checkout
        uses: actions/checkout@v3.1.0
        with:
          fetch-depth: 0

      - name: get Docker image files
        id: docker-files
        run: .github/scripts/files_require_smokes.sh >> docker-files.txt

      - name: add other required files
        id: add-files
        run: |
          echo .dockerignore >> docker-files.txt;
          echo Dockerfile >> docker-files.txt;
          echo pr_check.sh >> docker-files.txt;
          echo deploy/clowdapp.yaml >> docker-files.txt;

      - name: Show Dockerfiles
        run: cat docker-files.txt

      - name: Get changed files
        id: changed-files
        uses: tj-actions/changed-files@v41.0.0
        with:
          files_from_source_file: docker-files.txt

      - name: Set whether to run tests
        id: check-files
        run: |
          if [ ! -z "${{ steps.changed-files.outputs.all_changed_and_modified_files }}" ]; then
            echo "RUN_TESTS=true" >> $GITHUB_ENV
          fi

      - name: Setting smokes-required label
        if: env.RUN_TESTS == 'true'
        uses: actions/github-script@v6.3.3
        continue-on-error: true
        with:
          script: |
            github.rest.issues.addLabels({
              issue_number: context.issue.number,
              owner: context.repo.owner,
              repo: context.repo.repo,
              labels: [ 'smokes-required' ]
            })
            github.rest.issues.removeLabel({
              issue_number: context.issue.number,
              owner: context.repo.owner,
              repo: context.repo.repo,
              name: 'ok-to-skip-smokes'
            })

      - name: Remove smokes-required label
        if: env.RUN_TESTS != 'true'
        uses: actions/github-script@v6.3.3
        continue-on-error: true
        with:
          script: |
            github.rest.issues.addLabels({
              issue_number: context.issue.number,
              owner: context.repo.owner,
              repo: context.repo.repo,
              labels: [ 'ok-to-skip-smokes' ]
            })
            github.rest.issues.removeLabel({
              issue_number: context.issue.number,
              owner: context.repo.owner,
              repo: context.repo.repo,
              name: 'smokes-required'
            })

  changed-files:
    name: Detect changed files
    runs-on: ubuntu-20.04
    outputs:
      run_tests: ${{ steps.check-files-or-fork.outputs.run_tests }}

    steps:
      - name: Checkout
        uses: actions/checkout@v3.1.0
        with:
          fetch-depth: 0

      - name: Get changed files
        id: changed-files
        uses: tj-actions/changed-files@v41.0.0
        with:
          files: |
            db_functions/**
            koku/**
            .github/postgres/**
            .github/scripts/check_migrations.sh
            .github/workflows/ci.yml
            Pipfile.lock

      - name: Check files or fork
        id: check-files-or-fork
        run: |
          if [ "${{ steps.changed-files.outputs.any_modified }}" == "true" ] || [ "${{ github.event.pull_request.head.repo.full_name }}" != "project-koku/koku" ]; then
            echo "run_tests=true" >> $GITHUB_OUTPUT
          fi

      - name: Show changed files
        run: |
          echo "Changed files:"
          for file in ${{ steps.changed-files.outputs.all_changed_and_modified_files }}; do
            echo "  $file"
          done

  units:
    name: Units - ${{ matrix.python-version }}
    needs: changed-files
    runs-on: ubuntu-20.04
    strategy:
      fail-fast: false
      max-parallel: 4
      matrix:
        python-version:
          - '3.9'
    env:
      COMPOSE_FILE: .github/postgres/docker-compose.yaml

    steps:
      - name: Checkout
        if: needs.changed-files.outputs.run_tests == 'true'
        uses: actions/checkout@v3.1.0
        with:
          fetch-depth: 0

      - name: Display build environment
        if: needs.changed-files.outputs.run_tests == 'true'
        run: printenv

      - name: Start the postgres DB
        if: needs.changed-files.outputs.run_tests == 'true'
        run: docker-compose up -d db

      - name: Set up Python ${{ matrix.python-version }}
        if: needs.changed-files.outputs.run_tests == 'true'
        uses: actions/setup-python@v4.3.0
        with:
          python-version: ${{ matrix.python-version }}

      - name: Install pipenv
        if: needs.changed-files.outputs.run_tests == 'true'
        run: sudo python3 -m pip install pipenv

      - name: Cache dependencies
        if: needs.changed-files.outputs.run_tests == 'true'
        id: cache-dependencies
        uses: actions/cache@v3.0.11
        with:
          path: |
            ~/.cache/pipenv
            ~/.local/share/virtualenvs
          key: ${{ runner.os }}-env-${{ matrix.python-version }}-${{ hashFiles('**/Pipfile.lock') }}-${{ secrets.ACTIONS_CACHE_KEY_UUID }}

      - name: Install dependencies
        if: needs.changed-files.outputs.run_tests == 'true' && steps.cache-dependencies.outputs.cache-hit != 'true'
        run: |
          pipenv install --dev --ignore-pipfile --python ${{ matrix.python-version }}

      - name: Check migrations
        if: needs.changed-files.outputs.run_tests == 'true'
        run: bash .github/scripts/check_migrations.sh
        env:
          DATABASE_SERVICE_NAME: POSTGRES_SQL
          DATABASE_ENGINE: postgresql
          DATABASE_NAME: postgres
          DATABASE_USER: postgres
          DATABASE_PASSWORD: postgres
          POSTGRES_SQL_SERVICE_HOST: localhost
          POSTGRES_SQL_SERVICE_PORT: ${{ job.services.postgres.ports[5432] }}
          PROMETHEUS_MULTIPROC_DIR: /tmp

      - name: Run unit tests
        if: needs.changed-files.outputs.run_tests == 'true'
        id: unit_tests_run
        run: pipenv run coverage run ./koku/manage.py test --noinput --verbosity 2 ./koku/
        env:
          DATABASE_SERVICE_NAME: POSTGRES_SQL
          DATABASE_ENGINE: postgresql
          DATABASE_NAME: postgres
          DATABASE_USER: postgres
          DATABASE_PASSWORD: postgres
          POSTGRES_SQL_SERVICE_HOST: localhost
          POSTGRES_SQL_SERVICE_PORT: ${{ job.services.postgres.ports[5432] }}
          HIVE_DATABASE_PASSWORD: hivedbpw
          ACCOUNT_ENHANCED_METRICS: True
          PROMETHEUS_MULTIPROC_DIR: /tmp
          TRINO_DATE_STEP: 31
          MIDDLEWARE_TIME_TO_LIVE: 0
          ENHANCED_ORG_ADMIN: True
          ENABLE_S3_ARCHIVING: True

      - name: Show test status
        if: needs.changed-files.outputs.run_tests == 'true'
        id: show_test_status
        run: echo "Unit tests and migration check were '${{ steps.unit_tests_run.outcome }}'"

      - name: Convert coverage report to XML
        if: steps.unit_tests_run.outcome == 'success'
        run: pipenv run coverage xml

      - name: Upload test coverage file
        if: steps.unit_tests_run.outcome == 'success'
        uses: actions/upload-artifact@v3
        with:
          name: coverage
          path: coverage.xml

  coverage:
    name: Coverage
    needs: [changed-files,units]
    runs-on: ubuntu-20.04
    steps:

      - name: Checkout
        # this checkout is required for the coverage report. If we don't do this, then
        # the uploaded report is invalid and codecov doesn't know how to process it.
        if: needs.changed-files.outputs.run_tests == 'true'
        uses: actions/checkout@v3.1.0
        with:
          fetch-depth: 0

      - name: Download coverage result from units
        if: needs.changed-files.outputs.run_tests == 'true'
        uses: actions/download-artifact@v3
        with:
          name: coverage

      - name: Upload coverage to Codecov
        if: needs.changed-files.outputs.run_tests == 'true'
        uses: codecov/codecov-action@v4.2.0
        env:
          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
        with:
          file: ./coverage.xml
          flags: unittests
          name: Python-${{ matrix.python-version}}
          fail_ci_if_error: true
          plugin: pycoverage  # Only run one plugin even though we don't want any to run.

      - name: Set Codecov job status on skipped tests
        if: needs.changed-files.outputs.run_tests != 'true'
        env:
          GITHUB_TOKEN: "${{ github.token }}"
        run: |
          gh api "/repos/project-koku/koku/statuses/${{ github.event.pull_request.head.sha }}" -f description="codecov skipped because unit tests were not required" -f context="codecov/patch" -f state="success"
          gh api "/repos/project-koku/koku/statuses/${{ github.event.pull_request.head.sha }}" -f description="codecov skipped because unit tests were not required" -f context="codecov/project" -f state="success"