From 95da61676e418535414d20178f50d31a9414a727 Mon Sep 17 00:00:00 2001 From: mata Date: Tue, 6 Aug 2024 16:00:13 +0200 Subject: [PATCH 1/2] Update workflows --- .github/CODEOWNERS | 2 +- .github/workflows/black.yaml | 57 +++++++++++++++++++++++ .github/workflows/codestyle.yaml | 35 --------------- .github/workflows/deploy_pypi.yaml | 57 +++++++++++++++++++++++ .github/workflows/dev-release.yaml | 56 +++++++++++++++++++++++ .github/workflows/prepare_release.yaml | 18 -------- .github/workflows/pylint.yaml | 56 +++++++++++++++++++++++ .github/workflows/release.yaml | 55 ++++------------------- .github/workflows/test.yaml | 62 ++++++++++++++++++++++++++ .github/workflows/validate.yaml | 37 --------------- 10 files changed, 297 insertions(+), 138 deletions(-) create mode 100644 .github/workflows/black.yaml delete mode 100644 .github/workflows/codestyle.yaml create mode 100644 .github/workflows/deploy_pypi.yaml create mode 100644 .github/workflows/dev-release.yaml delete mode 100644 .github/workflows/prepare_release.yaml create mode 100644 .github/workflows/pylint.yaml create mode 100644 .github/workflows/test.yaml delete mode 100644 .github/workflows/validate.yaml diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 29c798d..899bb7b 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -4,4 +4,4 @@ # These owners will be the default owners for everything in # the repo. Unless a later match takes precedence, these people will be requests a review # review when someone opens a pull request. -* @matt035343 +* @or_scientists diff --git a/.github/workflows/black.yaml b/.github/workflows/black.yaml new file mode 100644 index 0000000..a615842 --- /dev/null +++ b/.github/workflows/black.yaml @@ -0,0 +1,57 @@ +name: Validate code format + +on: + pull_request: + branches: [ main ] + +jobs: + validate_code: + name: Validate code format + runs-on: ubuntu-latest + + permissions: + contents: read + id-token: write + pull-requests: write + + if: ${{ github.ref != 'refs/heads/main' }} + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + - uses: actions/setup-python@v4 + with: + python-version: '3.11.x' + architecture: 'x64' + + - name: Import Secrets + uses: hashicorp/vault-action@v2.7.3 + with: + url: https://hashicorp-vault.awsp.sneaksanddata.com/ + role: github + method: jwt + secrets: | + /secret/data/common/package-publishing/aws-ca/production/artifact-user access_key ; + /secret/data/common/package-publishing/aws-ca/production/artifact-user access_key_id ; + - name: Setup AWS CA + uses: SneaksAndData/github-actions/setup_aws_ca@v0.1.9 + with: + aws_access_key: ${{ env.ACCESS_KEY }} + aws_access_key_id: ${{ env.ACCESS_KEY_ID }} + mode: read + aws_ca_domain: esd-code-artifact-production + aws_ca_domain_owner: 497010676583 + aws_ca_repository: esd-artifacts + id: setup_aws_ca + - name: Install Poetry and dependencies + uses: SneaksAndData/github-actions/install_poetry@v0.1.9 + with: + pypi_repo_url: ${{ steps.setup_aws_ca.outputs.url }} + pypi_token_username: ${{ steps.setup_aws_ca.outputs.user }} + pypi_token: ${{ steps.setup_aws_ca.outputs.token }} + install_extras: "all" + - name: Black + shell: bash + run: | + set -e + poetry run black . --check --diff diff --git a/.github/workflows/codestyle.yaml b/.github/workflows/codestyle.yaml deleted file mode 100644 index 17398ce..0000000 --- a/.github/workflows/codestyle.yaml +++ /dev/null @@ -1,35 +0,0 @@ -name: Code Quality - -on: - push: - branches: [ main ] - pull_request: - branches: [ main ] - -jobs: - codestyle: - name: codestyle - runs-on: ubuntu-latest - if: github.ref != 'refs/heads/main' - steps: - - uses: actions/checkout@v3 - - uses: actions/setup-python@v4 - with: - python-version: '3.11.x' - architecture: 'x64' - - - name: Install Poetry and dependencies - uses: SneaksAndData/github-actions/install_poetry@v0.1.0 - with: - pypi_repo_url: ${{ secrets.AZOPS_PYPI_REPO_URL }} - pypi_token_username: ${{ secrets.AZOPS_PAT_USER }} - pypi_token: ${{ secrets.AZOPS_PAT }} - - name: Black - shell: bash - run: | - set -e - poetry run black . --check --diff - - name: Lint - run: | - set -e - find ./anti_clustering -type f -name "*.py" | xargs poetry run pylint diff --git a/.github/workflows/deploy_pypi.yaml b/.github/workflows/deploy_pypi.yaml new file mode 100644 index 0000000..4b32739 --- /dev/null +++ b/.github/workflows/deploy_pypi.yaml @@ -0,0 +1,57 @@ +name: Release a new version + +on: workflow_dispatch +jobs: + release_to_pypi: + name: Release distribution to PyPi + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + - uses: actions/setup-python@v4 + with: + python-version: '3.11.x' + architecture: 'x64' + - name: Install Poetry + uses: SneaksAndData/github-actions/install_poetry@v0.1.0 + with: + pypi_repo_url: ${{ secrets.AZOPS_PYPI_REPO_URL }} + pypi_token_username: ${{ secrets.AZOPS_PAT_USER }} + pypi_token: ${{ secrets.AZOPS_PAT }} + install_extras: "all" + skip_dependencies: true + - name: Prepare version + run: | + set -e + + version=$(git describe --tags --abbrev=7) + sed -i "s/version = \"0.0.0\"/version = \"${version:1}\"/" pyproject.toml + echo "__version__ = '${version:1}'" > ./anti_clustering/_version.py + + - name: Configure Test PyPi + if: ${{ github.ref == 'refs/heads/main' }} + env: + PYPI_TEST_TOKEN: ${{ secrets.PYPI_TEST_API_TOKEN }} + run: | + set -e + + poetry config repositories.test-pypi https://test.pypi.org/legacy/ + poetry config pypi-token.test-pypi $PYPI_TEST_TOKEN + + - name: Publish distribution 📦 to test PyPI + if: ${{ github.ref == 'refs/heads/main' }} + run: | + set -e + + poetry build && poetry publish -r test-pypi + + - name: Publish distribution 📦 to PyPI + env: + POETRY_PYPI_TOKEN_PYPI: ${{ secrets.PYPI_API_TOKEN }} + if: ${{ startsWith(github.ref, 'refs/tags') }} + run: | + set -e + + poetry build && poetry publish diff --git a/.github/workflows/dev-release.yaml b/.github/workflows/dev-release.yaml new file mode 100644 index 0000000..184f8e0 --- /dev/null +++ b/.github/workflows/dev-release.yaml @@ -0,0 +1,56 @@ +on: issue_comment + +jobs: + pr_commented: + # This job only runs for pull request comments + name: PR comment + + permissions: + contents: read + id-token: write + pull-requests: write + + runs-on: ubuntu-latest + if: ${{ github.event.issue.pull_request && github.event.comment.body == 'create_package' && github.event.issue.state == 'open' }} + steps: + - uses: actions/setup-python@v4 + with: + python-version: '3.9.x' + architecture: 'x64' + - uses: actions/checkout@v3 + with: + ref: refs/pull/${{github.event.issue.number}}/merge + fetch-depth: 0 + - name: Import Secrets + uses: hashicorp/vault-action@v2.4.2 + with: + url: https://hashicorp-vault.production.sneaksanddata.com/ + role: github + method: jwt + secrets: | + /secret/data/common/package-publishing/aws-ca/production/artifact-user access_key ; + /secret/data/common/package-publishing/aws-ca/production/artifact-user access_key_id ; + - name: Setup AWS CA + uses: SneaksAndData/github-actions/setup_aws_ca@v0.1.9 + with: + aws_access_key: ${{ env.ACCESS_KEY }} + aws_access_key_id: ${{ env.ACCESS_KEY_ID }} + mode: publish + aws_ca_domain: esd-code-artifact-production + aws_ca_domain_owner: 497010676583 + aws_ca_repository: esd-artifacts + id: setup_aws_ca + - name: Install Poetry and dependencies + uses: SneaksAndData/github-actions/install_poetry@v0.1.9 + with: + pypi_repo_url: ${{ steps.setup_aws_ca.outputs.url }} + pypi_token_username: ${{ steps.setup_aws_ca.outputs.user }} + pypi_token: ${{ steps.setup_aws_ca.outputs.token }} + install_extras: all + - name: Create package + uses: SneaksAndData/github-actions/create_package@v0.1.9 + with: + repo_url: ${{ steps.setup_aws_ca.outputs.url }} + repo_token_username: ${{ steps.setup_aws_ca.outputs.user }} + repo_token: ${{ steps.setup_aws_ca.outputs.token }} + package_name: anti_clustering diff --git a/.github/workflows/prepare_release.yaml b/.github/workflows/prepare_release.yaml deleted file mode 100644 index eb5c98c..0000000 --- a/.github/workflows/prepare_release.yaml +++ /dev/null @@ -1,18 +0,0 @@ -name: Prepare GH Release - -on: workflow_dispatch - -jobs: - create_release: - name: Create Release - runs-on: ubuntu-latest - if: ${{ github.ref == 'refs/heads/main' }} - - steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - uses: SneaksAndData/github-actions/semver_release@v0.1.0 - with: - major_v: 0 - minor_v: 4 diff --git a/.github/workflows/pylint.yaml b/.github/workflows/pylint.yaml new file mode 100644 index 0000000..1ab7fac --- /dev/null +++ b/.github/workflows/pylint.yaml @@ -0,0 +1,56 @@ +name: Validate code linting + +on: + pull_request: + branches: [ main ] + +jobs: + validate_code: + name: Validate code linting + runs-on: ubuntu-latest + + permissions: + contents: read + id-token: write + pull-requests: write + + if: ${{ github.ref != 'refs/heads/main' }} + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + - uses: actions/setup-python@v4 + with: + python-version: '3.11.x' + architecture: 'x64' + + - name: Import Secrets + uses: hashicorp/vault-action@v2.7.3 + with: + url: https://hashicorp-vault.awsp.sneaksanddata.com/ + role: github + method: jwt + secrets: | + /secret/data/common/package-publishing/aws-ca/production/artifact-user access_key ; + /secret/data/common/package-publishing/aws-ca/production/artifact-user access_key_id ; + - name: Setup AWS CA + uses: SneaksAndData/github-actions/setup_aws_ca@v0.1.9 + with: + aws_access_key: ${{ env.ACCESS_KEY }} + aws_access_key_id: ${{ env.ACCESS_KEY_ID }} + mode: read + aws_ca_domain: esd-code-artifact-production + aws_ca_domain_owner: 497010676583 + aws_ca_repository: esd-artifacts + id: setup_aws_ca + - name: Install Poetry and dependencies + uses: SneaksAndData/github-actions/install_poetry@v0.1.9 + with: + pypi_repo_url: ${{ steps.setup_aws_ca.outputs.url }} + pypi_token_username: ${{ steps.setup_aws_ca.outputs.user }} + pypi_token: ${{ steps.setup_aws_ca.outputs.token }} + install_extras: "all" + - name: Lint code + run: | + set -e + find ./anti_clustering -type f -name "*.py" | xargs poetry run pylint diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 4b32739..77bdc5f 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -1,57 +1,18 @@ name: Release a new version on: workflow_dispatch + jobs: - release_to_pypi: - name: Release distribution to PyPi + create_release: + name: Create release runs-on: ubuntu-latest + if: ${{ github.ref == 'refs/heads/master' }} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v2 with: fetch-depth: 0 - - uses: actions/setup-python@v4 - with: - python-version: '3.11.x' - architecture: 'x64' - - name: Install Poetry - uses: SneaksAndData/github-actions/install_poetry@v0.1.0 + - uses: SneaksAndData/github-actions/semver_release@v0.1.9 with: - pypi_repo_url: ${{ secrets.AZOPS_PYPI_REPO_URL }} - pypi_token_username: ${{ secrets.AZOPS_PAT_USER }} - pypi_token: ${{ secrets.AZOPS_PAT }} - install_extras: "all" - skip_dependencies: true - - name: Prepare version - run: | - set -e - - version=$(git describe --tags --abbrev=7) - sed -i "s/version = \"0.0.0\"/version = \"${version:1}\"/" pyproject.toml - echo "__version__ = '${version:1}'" > ./anti_clustering/_version.py - - - name: Configure Test PyPi - if: ${{ github.ref == 'refs/heads/main' }} - env: - PYPI_TEST_TOKEN: ${{ secrets.PYPI_TEST_API_TOKEN }} - run: | - set -e - - poetry config repositories.test-pypi https://test.pypi.org/legacy/ - poetry config pypi-token.test-pypi $PYPI_TEST_TOKEN - - - name: Publish distribution 📦 to test PyPI - if: ${{ github.ref == 'refs/heads/main' }} - run: | - set -e - - poetry build && poetry publish -r test-pypi - - - name: Publish distribution 📦 to PyPI - env: - POETRY_PYPI_TOKEN_PYPI: ${{ secrets.PYPI_API_TOKEN }} - if: ${{ startsWith(github.ref, 'refs/tags') }} - run: | - set -e - - poetry build && poetry publish + major_v: 0 + minor_v: 4 diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml new file mode 100644 index 0000000..19f32b8 --- /dev/null +++ b/.github/workflows/test.yaml @@ -0,0 +1,62 @@ +name: Code unit tests + +on: + pull_request: + branches: [ main ] + +jobs: + validate_code: + name: Code unit tests + runs-on: ubuntu-latest + + permissions: + contents: read + id-token: write + pull-requests: write + + if: ${{ github.ref != 'refs/heads/main' }} + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + - uses: actions/setup-python@v4 + with: + python-version: '3.11.x' + architecture: 'x64' + + - name: Import Secrets + uses: hashicorp/vault-action@v2.7.3 + with: + url: https://hashicorp-vault.awsp.sneaksanddata.com/ + role: github + method: jwt + secrets: | + /secret/data/common/package-publishing/aws-ca/production/artifact-user access_key ; + /secret/data/common/package-publishing/aws-ca/production/artifact-user access_key_id ; + - name: Setup AWS CA + uses: SneaksAndData/github-actions/setup_aws_ca@v0.1.9 + with: + aws_access_key: ${{ env.ACCESS_KEY }} + aws_access_key_id: ${{ env.ACCESS_KEY_ID }} + mode: read + aws_ca_domain: esd-code-artifact-production + aws_ca_domain_owner: 497010676583 + aws_ca_repository: esd-artifacts + id: setup_aws_ca + - name: Install Poetry and dependencies + uses: SneaksAndData/github-actions/install_poetry@v0.1.9 + with: + pypi_repo_url: ${{ steps.setup_aws_ca.outputs.url }} + pypi_token_username: ${{ steps.setup_aws_ca.outputs.user }} + pypi_token: ${{ steps.setup_aws_ca.outputs.token }} + install_extras: "all" + + - name: Run test + run: | + set -euo pipefail + + poetry run pytest ./tests --cov-config=.coveragerc --doctest-modules --junitxml=junit/test-results.xml --cov=. --cov-report=term-missing:skip-covered | tee pytest-coverage.txt + - name: Publish Code Coverage + uses: MishaKav/pytest-coverage-comment@main + with: + pytest-coverage-path: ./pytest-coverage.txt diff --git a/.github/workflows/validate.yaml b/.github/workflows/validate.yaml deleted file mode 100644 index 767ba93..0000000 --- a/.github/workflows/validate.yaml +++ /dev/null @@ -1,37 +0,0 @@ -name: Validate code - -on: - push: - branches: [ main ] - pull_request: - branches: [ main ] - -jobs: - validate_code: - name: Validate code - runs-on: ubuntu-latest - if: ${{ github.ref != 'refs/heads/main' }} - steps: - - uses: actions/checkout@v3 - - uses: actions/setup-python@v4 - with: - python-version: '3.11.x' - architecture: 'x64' - - name: Install Poetry and dependencies - uses: SneaksAndData/github-actions/install_poetry@v0.1.0 - with: - pypi_repo_url: ${{ secrets.AZOPS_PYPI_REPO_URL }} - pypi_token_username: ${{ secrets.AZOPS_PAT_USER }} - pypi_token: ${{ secrets.AZOPS_PAT }} - install_extras: "all" - - name: Unit test - run: | - set -e - pypath=$(pwd) - export PYTHONPATH="$pypath:$PYTHONPATH" - poetry run pytest ./tests --cov-config=.coveragerc --doctest-modules --junitxml=junit/test-results.xml --cov=. --cov-report=term-missing:skip-covered | tee pytest-coverage.txt - - name: Publish Code Coverage - uses: MishaKav/pytest-coverage-comment@main - with: - pytest-coverage-path: ./pytest-coverage.txt - junitxml-path: ./junit/test-results.xml From f353d84f038f2417c304a2e74979620c893feb1b Mon Sep 17 00:00:00 2001 From: Matthias Als Date: Wed, 7 Aug 2024 09:32:56 +0200 Subject: [PATCH 2/2] Update CODEOWNERS --- .github/CODEOWNERS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 899bb7b..766c403 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -4,4 +4,4 @@ # These owners will be the default owners for everything in # the repo. Unless a later match takes precedence, these people will be requests a review # review when someone opens a pull request. -* @or_scientists +* @SneaksAndData/or-scientists