From 9ca7aa4b3620279bf4e0c0e14c49cf9ed3f9df72 Mon Sep 17 00:00:00 2001 From: jctanner Date: Thu, 29 Feb 2024 10:30:35 -0500 Subject: [PATCH] update plugin template files (#2087) * Update plugin template files. * Debug? * Debug? * Fix commit check task. * Fix yamllint errors on some workflow files. * Fix more workflow files. * This is the only way to make this work. * Twine insists on having a long_description now. * Add new required file? * Need to cd to the parent first. * Fix tdd script indexerror. No-Issue Signed-off-by: James Tanner --- .bumpversion.cfg | 21 ++ .ci/ansible/Containerfile.j2 | 22 +- .ci/scripts/check_release.py | 13 +- .ci/scripts/collect_changes.py | 8 +- .github/post-job-template.yml.j2 | 7 +- .github/pre-job-template.yml.j2 | 2 + .github/template_gitref | 2 +- .github/workflows/build.yml | 44 +++ .github/workflows/build_publish.yml | 126 ++++---- .github/workflows/changelog.yml | 32 +-- .github/workflows/ci.yml | 186 +++--------- .github/workflows/ci_tdd.yml | 2 +- .github/workflows/codeql-analysis.yml | 16 +- .github/workflows/create-branch.yml | 34 ++- .../ci_standalone-certified-sync.yml | 2 +- .../ci_standalone-iqe-rbac-tests.yml | 2 +- .../ci_standalone-rbac-on-repos.yml | 2 +- .../deactivated/ci_standalone-rbac-roles.yml | 2 +- .../ci_standalone-x-repo-search.yml | 2 +- .github/workflows/i18n.yml | 12 +- .github/workflows/lint.yml | 52 ++++ .github/workflows/nightly.yml | 268 ++---------------- .github/workflows/nightly_latest.yml | 12 +- .github/workflows/pr_checks.yml | 62 ++++ .github/workflows/publish.yml | 131 +++++++++ .github/workflows/release.yml | 209 ++------------ .github/workflows/scripts/before_install.sh | 14 +- .github/workflows/scripts/check_commit.sh | 11 +- .github/workflows/scripts/install.sh | 30 +- .../scripts/install_python_client.sh | 68 +++-- .../workflows/scripts/install_ruby_client.sh | 45 ++- .../workflows/scripts/publish_client_gem.sh | 25 +- .../workflows/scripts/publish_client_pypi.sh | 24 +- .github/workflows/scripts/publish_docs.sh | 31 +- .../workflows/scripts/publish_plugin_pypi.sh | 28 +- .../scripts/push_branch_and_tag_to_github.sh | 25 +- .github/workflows/scripts/release.sh | 26 ++ .github/workflows/scripts/script.sh | 148 ++++++---- .../scripts/update_backport_labels.py | 8 +- .github/workflows/test.yml | 137 +++++++++ .github/workflows/update-labels.yml | 22 +- .github/workflows/update_ci.yml | 48 ++-- CHANGES/.TEMPLATE.rst | 12 +- dev/common/tdd.py | 2 +- functest_requirements.txt | 2 +- lint_requirements.txt | 2 +- setup.py | 1 + template_config.yml | 35 +-- unittest_requirements.txt | 1 + 49 files changed, 1011 insertions(+), 1005 deletions(-) create mode 100644 .bumpversion.cfg create mode 100644 .github/workflows/build.yml create mode 100644 .github/workflows/lint.yml create mode 100644 .github/workflows/pr_checks.yml create mode 100644 .github/workflows/publish.yml create mode 100755 .github/workflows/scripts/release.sh create mode 100644 .github/workflows/test.yml diff --git a/.bumpversion.cfg b/.bumpversion.cfg new file mode 100644 index 0000000000..9e78927f02 --- /dev/null +++ b/.bumpversion.cfg @@ -0,0 +1,21 @@ +[bumpversion] +current_version = 4.10.0dev +commit = False +tag = False +parse = (?P\d+)\.(?P\d+)\.(?P\d+)(\.(?P[a-z]+))? +serialize = + {major}.{minor}.{patch}.{release} + {major}.{minor}.{patch} + +[bumpversion:part:release] +optional_value = prod +first_value = dev +values = + dev + prod + +[bumpversion:file:./galaxy_ng/app/__init__.py] + +[bumpversion:file:./setup.py] + +# [bumpversion:file:./docs/conf.py] diff --git a/.ci/ansible/Containerfile.j2 b/.ci/ansible/Containerfile.j2 index 0eb5d5ccb0..338ff69741 100644 --- a/.ci/ansible/Containerfile.j2 +++ b/.ci/ansible/Containerfile.j2 @@ -1,28 +1,22 @@ -FROM {{ ci_base | default("ghcr.io/pulp/pulp-ci-centos:" + pulp_container_tag) }} +FROM {{ ci_base | default(pulp_default_container) }} # Add source directories to container {% for item in plugins %} -{% if item.source.startswith("./") or item.ci_requirements | default(false) %} ADD ./{{ item.name }} ./{{ item.name }} -{% endif %} {% endfor %} # Install python packages +# S3 botocore needs to be patched to handle responses from minio during 0-byte uploads +# Hacking botocore (https://github.com/boto/botocore/pull/1990) RUN pip3 install -{%- for item in plugins -%} -{%- if item.name == "pulp-certguard" -%} -{{ " " }}python-dateutil rhsm -{%- endif -%} -{{ " " }}{{ item.source }} -{%- if item.name == "pulpcore" -%} {%- if s3_test | default(false) -%} -[s3] -{%- elif azure_test | default(false) -%} -[azure] -{%- elif gcp_test | default(false) -%} -[google] +{{ " " }}git+https://github.com/gerrod3/botocore.git@fix-100-continue {%- endif -%} +{%- for item in plugins -%} +{{ " " }}{{ item.source }} +{%- if item.lowerbounds | default(false) -%} +{{ " " }}-c ./{{ item.name }}/lowerbounds_constraints.txt {%- endif -%} {%- if item.ci_requirements | default(false) -%} {{ " " }}-r ./{{ item.name }}/ci_requirements.txt diff --git a/.ci/scripts/check_release.py b/.ci/scripts/check_release.py index bf1e6bc02a..c2a78dbbf0 100755 --- a/.ci/scripts/check_release.py +++ b/.ci/scripts/check_release.py @@ -29,7 +29,7 @@ def main(): "--branches", default="supported", help="A comma separated list of branches to check for releases. Can also use keyword: " - "'supported'. Defaults to 'supported', see `ci_update_branches` in " + "'supported'. Defaults to 'supported', see `supported_release_branches` in " "`plugin_template.yml`.", ) opts = parser.parse_args() @@ -46,12 +46,15 @@ def main(): if branches == "supported": with open(f"{d}/template_config.yml", mode="r") as f: tc = yaml.safe_load(f) - branches = tc["ci_update_branches"] - branches.append(DEFAULT_BRANCH) + branches = set(tc["supported_release_branches"]) + latest_release_branch = tc["latest_release_branch"] + if latest_release_branch is not None: + branches.add(latest_release_branch) + branches.add(DEFAULT_BRANCH) else: - branches = branches.split(",") + branches = set(branches.split(",")) - if diff := set(branches) - set(available_branches): + if diff := branches - set(available_branches): print(f"Supplied branches contains non-existent branches! {diff}") exit(1) diff --git a/.ci/scripts/collect_changes.py b/.ci/scripts/collect_changes.py index d0658c8622..3508fa54b4 100755 --- a/.ci/scripts/collect_changes.py +++ b/.ci/scripts/collect_changes.py @@ -19,9 +19,11 @@ CHANGELOG_FILE = tc_settings.get("filename", "NEWS.rst") START_STRING = tc_settings.get( "start_string", - "\n" - if CHANGELOG_FILE.endswith(".md") - else ".. towncrier release notes start\n", + ( + "\n" + if CHANGELOG_FILE.endswith(".md") + else ".. towncrier release notes start\n" + ), ) TITLE_FORMAT = tc_settings.get("title_format", "{name} {version} ({project_date})") diff --git a/.github/post-job-template.yml.j2 b/.github/post-job-template.yml.j2 index 1f8397ba0f..ba5c40c4e4 100644 --- a/.github/post-job-template.yml.j2 +++ b/.github/post-job-template.yml.j2 @@ -1,5 +1,4 @@ - - update_manifest: +update_manifest: runs-on: ubuntu-latest needs: test steps: @@ -15,5 +14,7 @@ env: GITHUB_PULL_REQUEST: {{ "${{ github.event.number }}" }} MANIFEST_PASSPHRASE: {{ "${{ secrets.MANIFEST_PASSPHRASE }}" }} - run: .github/workflows/scripts/update_manifest.sh shell: bash + run: | + cd .. + .github/workflows/scripts/update_manifest.sh diff --git a/.github/pre-job-template.yml.j2 b/.github/pre-job-template.yml.j2 index a61b4b42be..b6ee54f3c4 100644 --- a/.github/pre-job-template.yml.j2 +++ b/.github/pre-job-template.yml.j2 @@ -15,6 +15,7 @@ check_commit: START_COMMIT: {{ "${{ github.event.before }}" }} END_COMMIT: {{ "${{ github.event.after }}" }} run: | + cd .. python .ci/scripts/validate_commit_message_custom.py lint_po: @@ -24,5 +25,6 @@ check_commit: uses: actions/checkout@v2 - run: | + cd .. pip install lint-po lint-po ./galaxy_ng/locale/*/LC_MESSAGES/*.po diff --git a/.github/template_gitref b/.github/template_gitref index 256a0012e0..e456a48e1d 100644 --- a/.github/template_gitref +++ b/.github/template_gitref @@ -1 +1 @@ -2021.08.26-251-g45d399d +2021.08.26-316-g69ef1f6 diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 0000000000..3a6f576419 --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,44 @@ +# WARNING: DO NOT EDIT! +# +# This file was generated by plugin_template, and is managed by it. Please use +# './plugin-template --github galaxy_ng' to update this file. +# +# For more info visit https://github.com/pulp/plugin_template + +--- +name: "Build" +on: + workflow_call: + +defaults: + run: + working-directory: "galaxy_ng" + +jobs: + build: + runs-on: "ubuntu-latest" + + steps: + - uses: "actions/checkout@v4" + with: + fetch-depth: 1 + path: "galaxy_ng" + - uses: "actions/setup-python@v4" + with: + python-version: "3.11" + - name: "Install python dependencies" + run: | + echo ::group::PYDEPS + pip install packaging twine wheel + echo ::endgroup:: + - name: "Build package" + run: | + python3 setup.py sdist bdist_wheel --python-tag py3 + twine check dist/* + - name: "Upload Package whl" + uses: "actions/upload-artifact@v3" + with: + name: "plugin_package" + path: "galaxy_ng/dist/" + if-no-files-found: "error" + retention-days: 5 diff --git a/.github/workflows/build_publish.yml b/.github/workflows/build_publish.yml index 938534c0fa..ae4dc7a08c 100644 --- a/.github/workflows/build_publish.yml +++ b/.github/workflows/build_publish.yml @@ -15,66 +15,66 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 - - - name: Set up Python - uses: actions/setup-python@v4 - - - name: Get galaxy_ng package version - shell: bash - run: echo "galaxy_ng_version=$(python3 setup.py --version)" >> $GITHUB_ENV - - - name: Validate tag and galaxy_ng version match - shell: bash - if: env.galaxy_ng_version != github.ref_name - run: | - echo "::error::Tag ${{ github.ref_name }} and galaxy_ng version ${{ env.galaxy_ng_version }} doesn't match." - exit 1 - - - name: Update apt - run: sudo apt -y update - - - name: Install LDAP requirements - run: sudo apt-get install -y libsasl2-dev libldap2-dev libssl-dev build-essential gettext python-setuptools - - - name: Upgrade pip - run: pip install pip --upgrade - - - name: Install galaxy_ng editable - run: pip install -e . - - # set variable for the next step, so only versions higher than 4.4 gets translated - # 4.2 and 4.3 don't support translations - - name: Check galaxy_ng version for translations - shell: bash - id: version_check - run: | - is_translatable=$(python -c 'from pkg_resources import parse_version; cmp_res = parse_version("${{ env.galaxy_ng_version }}") >= parse_version("4.4"); print(cmp_res)') - echo "is_translatable=$is_translatable" >> $GITHUB_OUTPUT - - - name: Build API translations - shell: bash - run: "django-admin compilemessages" - if: steps.version_check.outputs.is_translatable == 'True' - - - name: Check if UI has corresponding tag - shell: bash - run: | - status_code=$(curl -LI https://api.github.com/repos/ansible/ansible-hub-ui/releases/tags/${{ env.galaxy_ng_version }} -o /dev/null -w '%{http_code}\n' -s) - if [ $status_code == "404" ] - then - echo "::error::Corresponding tag ${{ env.galaxy_ng_version }} for ansible-hub-ui not found." - exit 1 - fi - - - name: Build galaxy_ng python packages - run: "python3 setup.py sdist bdist_wheel" - - - name: Install twine - run: "pip install twine" - - - name: Publish galaxy_ng to PyPI - run: "python3 -m twine upload dist/*" - env: - TWINE_USERNAME: __token__ - TWINE_PASSWORD: ${{ secrets.PYPI_RELEASE_TOKEN }} + - uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v4 + + - name: Get galaxy_ng package version + shell: bash + run: echo "galaxy_ng_version=$(python3 setup.py --version)" >> $GITHUB_ENV + + - name: Validate tag and galaxy_ng version match + shell: bash + if: env.galaxy_ng_version != github.ref_name + run: | + echo "::error::Tag ${{ github.ref_name }} and galaxy_ng version ${{ env.galaxy_ng_version }} doesn't match." + exit 1 + + - name: Update apt + run: sudo apt -y update + + - name: Install LDAP requirements + run: sudo apt-get install -y libsasl2-dev libldap2-dev libssl-dev build-essential gettext python-setuptools + + - name: Upgrade pip + run: pip install pip --upgrade + + - name: Install galaxy_ng editable + run: pip install -e . + + # set variable for the next step, so only versions higher than 4.4 gets translated + # 4.2 and 4.3 don't support translations + - name: Check galaxy_ng version for translations + shell: bash + id: version_check + run: | + is_translatable=$(python -c 'from pkg_resources import parse_version; cmp_res = parse_version("${{ env.galaxy_ng_version }}") >= parse_version("4.4"); print(cmp_res)') + echo "is_translatable=$is_translatable" >> $GITHUB_OUTPUT + + - name: Build API translations + shell: bash + run: "django-admin compilemessages" + if: steps.version_check.outputs.is_translatable == 'True' + + - name: Check if UI has corresponding tag + shell: bash + run: | + status_code=$(curl -LI https://api.github.com/repos/ansible/ansible-hub-ui/releases/tags/${{ env.galaxy_ng_version }} -o /dev/null -w '%{http_code}\n' -s) + if [ $status_code == "404" ] + then + echo "::error::Corresponding tag ${{ env.galaxy_ng_version }} for ansible-hub-ui not found." + exit 1 + fi + + - name: Build galaxy_ng python packages + run: "python3 setup.py sdist bdist_wheel" + + - name: Install twine + run: "pip install twine" + + - name: Publish galaxy_ng to PyPI + run: "python3 -m twine upload dist/*" + env: + TWINE_USERNAME: __token__ + TWINE_PASSWORD: ${{ secrets.PYPI_RELEASE_TOKEN }} diff --git a/.github/workflows/changelog.yml b/.github/workflows/changelog.yml index 17c35ad77a..d64787ecea 100644 --- a/.github/workflows/changelog.yml +++ b/.github/workflows/changelog.yml @@ -6,53 +6,53 @@ # For more info visit https://github.com/pulp/plugin_template --- -name: Galaxy changelog update +name: "Galaxy changelog update" on: push: branches: - - master + - "master" paths: - - CHANGES.rst - - CHANGES.md + - "CHANGES.rst" + - "CHANGES.md" workflow_dispatch: jobs: update-changelog: - runs-on: ubuntu-latest + runs-on: "ubuntu-latest" strategy: fail-fast: false steps: - - uses: actions/checkout@v4 + - uses: "actions/checkout@v4" with: fetch-depth: 1 - - uses: actions/setup-python@v4 + - uses: "actions/setup-python@v4" with: - python-version: "3.9" + python-version: "3.11" - - name: Install python dependencies + - name: "Install python dependencies" run: | echo ::group::PYDEPS pip install -r doc_requirements.txt echo ::endgroup:: - - name: Fake api schema + - name: "Fake api schema" run: | mkdir -p docs/_build/html echo "{}" > docs/_build/html/api.json mkdir -p docs/_static echo "{}" > docs/_static/api.json - - name: + - name: "Build Docs" run: | - pip install "Jinja2<3.1" make diagrams html - working-directory: ./docs + working-directory: "./docs" env: PULP_CONTENT_ORIGIN: "http://localhost/" - - name: Publish changlog to pulpproject.org - run: .github/workflows/scripts/publish_docs.sh changelog ${GITHUB_REF##*/} + - name: "Publish changlog to pulpproject.org" + run: | + .github/workflows/scripts/publish_docs.sh changelog ${GITHUB_REF##*/} env: - PULP_DOCS_KEY: ${{ secrets.PULP_DOCS_KEY }} + PULP_DOCS_KEY: "${{ secrets.PULP_DOCS_KEY }}" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 705c338ac2..43dd462ef9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -6,13 +6,17 @@ # For more info visit https://github.com/pulp/plugin_template --- -name: Galaxy CI +name: "Galaxy CI" on: {pull_request: {branches: ['**']}, push: {branches: ['**']}} concurrency: group: ${{ github.ref_name }}-${{ github.workflow }} cancel-in-progress: true +defaults: + run: + working-directory: "galaxy_ng" + jobs: check_commit: @@ -26,19 +30,16 @@ jobs: - name: Run script to validate commits for both pull request and a push env: - PY_COLORS: '1' - ANSIBLE_FORCE_COLOR: '1' - GITHUB_PULL_REQUEST: ${{ github.event.number }} - GITHUB_PULL_REQUEST_BODY: ${{ github.event.pull_request.body }} - GITHUB_BRANCH: ${{ github.head_ref }} - GITHUB_REPO_SLUG: ${{ github.repository }} - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - GITHUB_CONTEXT: ${{ github.event.pull_request.commits_url }} - GITHUB_USER: ${{ github.event.pull_request.user.login }} + PY_COLORS: "1" + ANSIBLE_FORCE_COLOR: "1" + GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" + GITHUB_CONTEXT: "${{ github.event.pull_request.commits_url }}" + GITHUB_USER: "${{ github.event.pull_request.user.login }}" GITHUB_PR_COMMITS_URL: ${{ github.event.pull_request.commits_url }} START_COMMIT: ${{ github.event.before }} END_COMMIT: ${{ github.event.after }} run: | + cd .. python .ci/scripts/validate_commit_message_custom.py lint_po: @@ -48,157 +49,36 @@ jobs: uses: actions/checkout@v2 - run: | + cd .. pip install lint-po lint-po ./galaxy_ng/locale/*/LC_MESSAGES/*.po lint: - runs-on: ubuntu-latest needs: check_commit + uses: "./.github/workflows/lint.yml" - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 1 - - uses: actions/setup-python@v4 - with: - python-version: "3.9" - # lint_requirements contains tools needed for flake8, etc. - - name: Install requirements - run: pip3 install -r lint_requirements.txt - - - - - - - name: Run extra lint checks - run: "[ ! -x .ci/scripts/extra_linting.sh ] || .ci/scripts/extra_linting.sh" - - # check for any files unintentionally left out of MANIFEST.in - - name: Check manifest - run: check-manifest - - - name: Check for pulpcore imports outside of pulpcore.plugin - run: sh .ci/scripts/check_pulpcore_imports.sh - - - name: Check for gettext problems - run: sh .ci/scripts/check_gettext.sh + build: + needs: "lint" + uses: "./.github/workflows/build.yml" test: - runs-on: ubuntu-latest - # run only after lint finishes - needs: lint - strategy: - fail-fast: false - matrix: - env: - - TEST: pulp - - TEST: azure - - TEST: s3 - outputs: - deprecations-pulp: ${{ steps.deprecations.outputs.deprecations-pulp }} - deprecations-azure: ${{ steps.deprecations.outputs.deprecations-azure }} - deprecations-s3: ${{ steps.deprecations.outputs.deprecations-s3 }} - + needs: "build" + uses: "./.github/workflows/test.yml" + + ready-to-ship: + # This is a dummy dependent task to have a single entry for the branch protection rules. + runs-on: "ubuntu-latest" + needs: + - "lint" + - "test" + if: "always()" steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 1 - - - uses: actions/setup-python@v4 - with: - python-version: "3.9" - - - name: Install httpie - run: | - echo ::group::HTTPIE - pip install httpie - echo ::endgroup:: - echo "HTTPIE_CONFIG_DIR=$GITHUB_WORKSPACE/.ci/assets/httpie/" >> $GITHUB_ENV - - - name: Set environment variables - run: | - echo "TEST=${{ matrix.env.TEST }}" >> $GITHUB_ENV - - - name: Before Install - run: .github/workflows/scripts/before_install.sh - shell: bash - env: - PY_COLORS: '1' - ANSIBLE_FORCE_COLOR: '1' - GITHUB_PULL_REQUEST: ${{ github.event.number }} - GITHUB_PULL_REQUEST_BODY: ${{ github.event.pull_request.body }} - GITHUB_BRANCH: ${{ github.head_ref }} - GITHUB_REPO_SLUG: ${{ github.repository }} - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - GITHUB_CONTEXT: ${{ github.event.pull_request.commits_url }} - GITHUB_USER: ${{ github.event.pull_request.user.login }} - - - name: Install - run: .github/workflows/scripts/install.sh - shell: bash - env: - PY_COLORS: '1' - ANSIBLE_FORCE_COLOR: '1' - GITHUB_PULL_REQUEST: ${{ github.event.number }} - GITHUB_PULL_REQUEST_BODY: ${{ github.event.pull_request.body }} - GITHUB_BRANCH: ${{ github.head_ref }} - GITHUB_REPO_SLUG: ${{ github.repository }} - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - GITHUB_CONTEXT: ${{ github.event.pull_request.commits_url }} - GITHUB_USER: ${{ github.event.pull_request.user.login }} - - - name: Before Script - run: .github/workflows/scripts/before_script.sh - shell: bash - env: - PY_COLORS: '1' - ANSIBLE_FORCE_COLOR: '1' - GITHUB_PULL_REQUEST: ${{ github.event.number }} - GITHUB_PULL_REQUEST_BODY: ${{ github.event.pull_request.body }} - GITHUB_BRANCH: ${{ github.head_ref }} - GITHUB_REPO_SLUG: ${{ github.repository }} - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - GITHUB_CONTEXT: ${{ github.event.pull_request.commits_url }} - GITHUB_USER: ${{ github.event.pull_request.user.login }} - REDIS_DISABLED: ${{ contains('', matrix.env.TEST) }} - - - name: Setting secrets - if: github.event_name != 'pull_request' - run: python3 .github/workflows/scripts/secrets.py "$SECRETS_CONTEXT" - env: - SECRETS_CONTEXT: ${{ toJson(secrets) }} - - - name: Script - run: .github/workflows/scripts/script.sh - shell: bash - env: - PY_COLORS: '1' - ANSIBLE_FORCE_COLOR: '1' - GITHUB_PULL_REQUEST: ${{ github.event.number }} - GITHUB_PULL_REQUEST_BODY: ${{ github.event.pull_request.body }} - GITHUB_BRANCH: ${{ github.head_ref }} - GITHUB_REPO_SLUG: ${{ github.repository }} - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - GITHUB_CONTEXT: ${{ github.event.pull_request.commits_url }} - GITHUB_USER: ${{ github.event.pull_request.user.login }} - - - name: Extract Deprecations from Logs - id: deprecations - run: echo deprecations-${{ matrix.env.TEST }}=$(docker logs pulp 2>&1 | grep -i pulpcore.deprecation | base64 -w 0) >> $GITHUB_OUTPUT - - - name: Logs - if: always() + - name: "Collect needed jobs results" + working-directory: "." run: | - echo "Need to debug? Please check: https://github.com/marketplace/actions/debugging-with-tmate" - http --timeout 30 --check-status --pretty format --print hb "https://pulp${PULP_API_ROOT}api/v3/status/" || true - docker images || true - docker ps -a || true - docker logs pulp || true - docker exec pulp ls -latr /etc/yum.repos.d/ || true - docker exec pulp cat /etc/yum.repos.d/* || true - docker exec pulp bash -c "pip3 list && pip3 install pipdeptree && pipdeptree" - - + echo '${{toJson(needs)}}' | jq -r 'to_entries[]|select(.value.result!="success")|.key + ": " + .value.result' + echo '${{toJson(needs)}}' | jq -e 'to_entries|map(select(.value.result!="success"))|length == 0' + echo "CI says: Looks good!" update_manifest: runs-on: ubuntu-latest needs: test @@ -215,5 +95,7 @@ jobs: env: GITHUB_PULL_REQUEST: ${{ github.event.number }} MANIFEST_PASSPHRASE: ${{ secrets.MANIFEST_PASSPHRASE }} - run: .github/workflows/scripts/update_manifest.sh shell: bash + run: | + cd .. + .github/workflows/scripts/update_manifest.sh diff --git a/.github/workflows/ci_tdd.yml b/.github/workflows/ci_tdd.yml index d25113b4a4..83fb490563 100644 --- a/.github/workflows/ci_tdd.yml +++ b/.github/workflows/ci_tdd.yml @@ -1,6 +1,6 @@ --- name: TDD -on: +on: pull_request: branches: - '*' diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 36f662ae19..62187cb1e7 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -30,13 +30,13 @@ jobs: language: [ 'python' ] steps: - - name: Checkout repository - uses: actions/checkout@v3 + - name: Checkout repository + uses: actions/checkout@v3 - - name: Initialize CodeQL - uses: github/codeql-action/init@v2 - with: - languages: ${{ matrix.language }} + - name: Initialize CodeQL + uses: github/codeql-action/init@v2 + with: + languages: ${{ matrix.language }} - - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v2 + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v2 diff --git a/.github/workflows/create-branch.yml b/.github/workflows/create-branch.yml index b2cd863a04..07a8045c7c 100644 --- a/.github/workflows/create-branch.yml +++ b/.github/workflows/create-branch.yml @@ -21,26 +21,27 @@ jobs: fail-fast: false steps: - - uses: actions/checkout@v4 + - uses: "actions/checkout@v4" with: fetch-depth: 0 - path: galaxy_ng + path: "galaxy_ng" - - uses: actions/setup-python@v4 + - uses: "actions/setup-python@v4" with: - python-version: "3.8" + python-version: "3.11" - - name: Install python dependencies + - name: "Install python dependencies" run: | echo ::group::PYDEPS - pip install bump2version jinja2 pyyaml + pip install bump2version jinja2 pyyaml packaging echo ::endgroup:: - - name: Setting secrets - working-directory: galaxy_ng - run: python3 .github/workflows/scripts/secrets.py "$SECRETS_CONTEXT" + - name: "Setting secrets" + working-directory: "galaxy_ng" + run: | + python3 .github/workflows/scripts/secrets.py "$SECRETS_CONTEXT" env: - SECRETS_CONTEXT: ${{ toJson(secrets) }} + SECRETS_CONTEXT: "${{ toJson(secrets) }}" - name: Determine new branch name working-directory: galaxy_ng @@ -70,6 +71,19 @@ jobs: run: | find CHANGES -type f -regex ".*\.\(bugfix\|doc\|feature\|misc\|deprecation\|removal\)" -exec git rm {} + + - name: Checkout plugin template + uses: actions/checkout@v3 + with: + repository: pulp/plugin_template + path: plugin_template + fetch-depth: 0 + + - name: Update CI branches in template_config + working-directory: plugin_template + run: | + python3 ./plugin-template galaxy_ng --github --latest-release-branch "${NEW_BRANCH}" + git add -A + - name: Make a PR with version bump and without CHANGES/* uses: peter-evans/create-pull-request@v4 with: diff --git a/.github/workflows/deactivated/ci_standalone-certified-sync.yml b/.github/workflows/deactivated/ci_standalone-certified-sync.yml index c141316393..f52ffbb344 100644 --- a/.github/workflows/deactivated/ci_standalone-certified-sync.yml +++ b/.github/workflows/deactivated/ci_standalone-certified-sync.yml @@ -1,6 +1,6 @@ --- name: Local Standalone Sync Against Local Insights -on: +on: pull_request: branches: - '**' diff --git a/.github/workflows/deactivated/ci_standalone-iqe-rbac-tests.yml b/.github/workflows/deactivated/ci_standalone-iqe-rbac-tests.yml index 1b13a38eb7..33449ab8a7 100644 --- a/.github/workflows/deactivated/ci_standalone-iqe-rbac-tests.yml +++ b/.github/workflows/deactivated/ci_standalone-iqe-rbac-tests.yml @@ -1,6 +1,6 @@ --- name: Standalone IQE RBAC tests -on: +on: pull_request: branches: - '**' diff --git a/.github/workflows/deactivated/ci_standalone-rbac-on-repos.yml b/.github/workflows/deactivated/ci_standalone-rbac-on-repos.yml index f0a535d1ff..eae646efa8 100644 --- a/.github/workflows/deactivated/ci_standalone-rbac-on-repos.yml +++ b/.github/workflows/deactivated/ci_standalone-rbac-on-repos.yml @@ -1,6 +1,6 @@ --- name: Standalone RBAC on Repositories tests -on: +on: pull_request: branches: - '**' diff --git a/.github/workflows/deactivated/ci_standalone-rbac-roles.yml b/.github/workflows/deactivated/ci_standalone-rbac-roles.yml index ea1df9f9c8..e9a8b65973 100644 --- a/.github/workflows/deactivated/ci_standalone-rbac-roles.yml +++ b/.github/workflows/deactivated/ci_standalone-rbac-roles.yml @@ -1,6 +1,6 @@ --- name: Standalone RBAC Roles -on: +on: pull_request: branches: - '**' diff --git a/.github/workflows/deactivated/ci_standalone-x-repo-search.yml b/.github/workflows/deactivated/ci_standalone-x-repo-search.yml index 62e60d0abd..0fc751c92e 100644 --- a/.github/workflows/deactivated/ci_standalone-x-repo-search.yml +++ b/.github/workflows/deactivated/ci_standalone-x-repo-search.yml @@ -1,6 +1,6 @@ --- name: Standalone Cross Repository Search tests -on: +on: pull_request: branches: - '**' diff --git a/.github/workflows/i18n.yml b/.github/workflows/i18n.yml index 1c370799e3..230c0df0ea 100644 --- a/.github/workflows/i18n.yml +++ b/.github/workflows/i18n.yml @@ -6,7 +6,7 @@ on: workflow_dispatch: # run weekly schedule: - - cron: '30 5 1 * *' + - cron: '30 5 1 * *' jobs: i18n: @@ -15,11 +15,11 @@ jobs: strategy: matrix: branch: - - 'master' - - 'stable-4.6' - - 'stable-4.7' - - 'stable-4.8' - - 'stable-4.9' + - 'master' + - 'stable-4.6' + - 'stable-4.7' + - 'stable-4.8' + - 'stable-4.9' steps: - uses: actions/checkout@v4 diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml new file mode 100644 index 0000000000..eabc5e1a41 --- /dev/null +++ b/.github/workflows/lint.yml @@ -0,0 +1,52 @@ +# WARNING: DO NOT EDIT! +# +# This file was generated by plugin_template, and is managed by it. Please use +# './plugin-template --github galaxy_ng' to update this file. +# +# For more info visit https://github.com/pulp/plugin_template + +--- +name: "Lint" +on: + workflow_call: + +defaults: + run: + working-directory: "galaxy_ng" + +jobs: + lint: + runs-on: ubuntu-latest + + steps: + - uses: "actions/checkout@v4" + with: + fetch-depth: 1 + path: "galaxy_ng" + + - uses: "actions/setup-python@v4" + with: + python-version: "3.11" + + - name: "Install python dependencies" + run: | + echo ::group::PYDEPS + pip install -r lint_requirements.txt + echo ::endgroup:: + + - name: Lint workflow files + run: | + yamllint -s -d '{extends: relaxed, rules: {line-length: disable}}' .github/workflows + + - name: Run extra lint checks + run: "[ ! -x .ci/scripts/extra_linting.sh ] || .ci/scripts/extra_linting.sh" + + # check for any files unintentionally left out of MANIFEST.in + - name: Check manifest + run: check-manifest + + - name: Check for pulpcore imports outside of pulpcore.plugin + run: sh .ci/scripts/check_pulpcore_imports.sh + + - name: Check for gettext problems + run: sh .ci/scripts/check_gettext.sh diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index b8df6ecab9..ea10379c49 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -6,7 +6,7 @@ # For more info visit https://github.com/pulp/plugin_template --- -name: Galaxy Nightly CI +name: "Galaxy Nightly CI" on: schedule: # * is a special character in YAML so you have to quote this string @@ -14,170 +14,41 @@ on: - cron: '00 3 * * *' workflow_dispatch: +defaults: + run: + working-directory: "galaxy_ng" + concurrency: - group: ${{ github.ref_name }}-${{ github.workflow }} + group: "${{ github.ref_name }}-${{ github.workflow }}" cancel-in-progress: true jobs: - test: - runs-on: ubuntu-latest - - strategy: - fail-fast: false - matrix: - env: - - TEST: pulp - - TEST: azure - - TEST: s3 - - - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 1 - - - uses: actions/setup-python@v4 - with: - python-version: "3.9" - - - name: Install httpie - run: | - echo ::group::HTTPIE - pip install httpie - echo ::endgroup:: - echo "HTTPIE_CONFIG_DIR=$GITHUB_WORKSPACE/.ci/assets/httpie/" >> $GITHUB_ENV - - - name: Set environment variables - run: | - echo "TEST=${{ matrix.env.TEST }}" >> $GITHUB_ENV - - - name: Before Install - run: .github/workflows/scripts/before_install.sh - shell: bash - env: - PY_COLORS: '1' - ANSIBLE_FORCE_COLOR: '1' - GITHUB_PULL_REQUEST: ${{ github.event.number }} - GITHUB_PULL_REQUEST_BODY: ${{ github.event.pull_request.body }} - GITHUB_BRANCH: ${{ github.head_ref }} - GITHUB_REPO_SLUG: ${{ github.repository }} - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - GITHUB_CONTEXT: ${{ github.event.pull_request.commits_url }} - GITHUB_USER: ${{ github.event.pull_request.user.login }} - - - uses: ruby/setup-ruby@v1 - if: ${{ env.TEST == 'bindings' || env.TEST == 'generate-bindings' }} - with: - ruby-version: "2.6" - - - name: Install - run: .github/workflows/scripts/install.sh - shell: bash - env: - PY_COLORS: '1' - ANSIBLE_FORCE_COLOR: '1' - GITHUB_PULL_REQUEST: ${{ github.event.number }} - GITHUB_PULL_REQUEST_BODY: ${{ github.event.pull_request.body }} - GITHUB_BRANCH: ${{ github.head_ref }} - GITHUB_REPO_SLUG: ${{ github.repository }} - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - GITHUB_CONTEXT: ${{ github.event.pull_request.commits_url }} - GITHUB_USER: ${{ github.event.pull_request.user.login }} - - - name: Before Script - run: .github/workflows/scripts/before_script.sh - shell: bash - env: - PY_COLORS: '1' - ANSIBLE_FORCE_COLOR: '1' - GITHUB_PULL_REQUEST: ${{ github.event.number }} - GITHUB_PULL_REQUEST_BODY: ${{ github.event.pull_request.body }} - GITHUB_BRANCH: ${{ github.head_ref }} - GITHUB_REPO_SLUG: ${{ github.repository }} - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - GITHUB_CONTEXT: ${{ github.event.pull_request.commits_url }} - GITHUB_USER: ${{ github.event.pull_request.user.login }} - REDIS_DISABLED: ${{ contains('', matrix.env.TEST) }} - - - name: Setting secrets - run: python3 .github/workflows/scripts/secrets.py "$SECRETS_CONTEXT" - env: - SECRETS_CONTEXT: ${{ toJson(secrets) }} - - - name: Install Python client - run: .github/workflows/scripts/install_python_client.sh - shell: bash - - - name: Install Ruby client - if: ${{ env.TEST == 'bindings' || env.TEST == 'generate-bindings' }} - run: .github/workflows/scripts/install_ruby_client.sh - shell: bash - - - name: Script - run: .github/workflows/scripts/script.sh - shell: bash - env: - PY_COLORS: '1' - ANSIBLE_FORCE_COLOR: '1' - GITHUB_PULL_REQUEST: ${{ github.event.number }} - GITHUB_PULL_REQUEST_BODY: ${{ github.event.pull_request.body }} - GITHUB_BRANCH: ${{ github.head_ref }} - GITHUB_REPO_SLUG: ${{ github.repository }} - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - GITHUB_CONTEXT: ${{ github.event.pull_request.commits_url }} - GITHUB_USER: ${{ github.event.pull_request.user.login }} + build: + uses: "./.github/workflows/build.yml" - - name: Upload python client packages - if: ${{ env.TEST == 'bindings' || env.TEST == 'generate-bindings' }} - uses: actions/upload-artifact@v3 - with: - name: python-client.tar - path: python-client.tar - - - name: Upload python client docs - if: ${{ env.TEST == 'bindings' || env.TEST == 'generate-bindings' }} - uses: actions/upload-artifact@v3 - with: - name: python-client-docs.tar - path: python-client-docs.tar - - - name: Upload ruby client packages - if: ${{ env.TEST == 'bindings' || env.TEST == 'generate-bindings' }} - uses: actions/upload-artifact@v3 - with: - name: ruby-client.tar - path: ruby-client.tar - - - name: Logs - if: always() - run: | - echo "Need to debug? Please check: https://github.com/marketplace/actions/debugging-with-tmate" - http --timeout 30 --check-status --pretty format --print hb "https://pulp${PULP_API_ROOT}api/v3/status/" || true - docker images || true - docker ps -a || true - docker logs pulp || true - docker exec pulp ls -latr /etc/yum.repos.d/ || true - docker exec pulp cat /etc/yum.repos.d/* || true - docker exec pulp bash -c "pip3 list && pip3 install pipdeptree && pipdeptree" + test: + needs: "build" + uses: "./.github/workflows/test.yml" changelog: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: "actions/checkout@v4" with: fetch-depth: 0 + path: "galaxy_ng" - - uses: actions/setup-python@v4 + - uses: "actions/setup-python@v4" with: python-version: "3.11" - - name: Install python dependencies + - name: "Install python dependencies" run: | echo ::group::PYDEPS pip install gitpython toml echo ::endgroup:: - - name: Configure Git with ansible name and email + - name: "Configure Git with ansible name and email" run: | git config --global user.name 'ansible' git config --global user.email 'ansible-infra@redhat.com' @@ -189,114 +60,37 @@ jobs: uses: peter-evans/create-pull-request@v4 with: token: ${{ secrets.RELEASE_TOKEN }} - title: 'Update Changelog' - body: '' - branch: 'changelog/update' + title: "Update Changelog" + body: "" + branch: "changelog/update" delete-branch: true + path: "galaxy_ng" publish: runs-on: ubuntu-latest needs: test - env: - TEST: publish - steps: - - uses: actions/checkout@v4 + - uses: "actions/checkout@v4" with: fetch-depth: 1 + path: "galaxy_ng" - - uses: actions/setup-python@v4 + - uses: actions/download-artifact@v3 with: - python-version: "3.9" + name: "plugin_package" + path: "galaxy_ng/dist/" - - uses: ruby/setup-ruby@v1 + - uses: "actions/setup-python@v4" with: - ruby-version: "2.6" - - - name: Install httpie - run: | - echo ::group::HTTPIE - pip install httpie - echo ::endgroup:: - echo "HTTPIE_CONFIG_DIR=$GITHUB_WORKSPACE/.ci/assets/httpie/" >> $GITHUB_ENV - - - name: Set environment variables - run: | - echo "TEST=${{ matrix.env.TEST }}" >> $GITHUB_ENV + python-version: "3.11" - - name: Install python dependencies + - name: "Install python dependencies" run: | echo ::group::PYDEPS - pip install wheel + pip install requests 'packaging~=21.3' mkdocs pymdown-extensions 'Jinja2<3.1' echo ::endgroup:: - - name: Before Install - run: .github/workflows/scripts/before_install.sh - shell: bash - env: - PY_COLORS: '1' - ANSIBLE_FORCE_COLOR: '1' - GITHUB_PULL_REQUEST: ${{ github.event.number }} - GITHUB_PULL_REQUEST_BODY: ${{ github.event.pull_request.body }} - GITHUB_BRANCH: ${{ github.head_ref }} - GITHUB_REPO_SLUG: ${{ github.repository }} - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - GITHUB_CONTEXT: ${{ github.event.pull_request.commits_url }} - GITHUB_USER: ${{ github.event.pull_request.user.login }} - - - name: Install - run: .github/workflows/scripts/install.sh - shell: bash - env: - PY_COLORS: '1' - ANSIBLE_FORCE_COLOR: '1' - GITHUB_PULL_REQUEST: ${{ github.event.number }} - GITHUB_PULL_REQUEST_BODY: ${{ github.event.pull_request.body }} - GITHUB_BRANCH: ${{ github.head_ref }} - GITHUB_REPO_SLUG: ${{ github.repository }} - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - GITHUB_CONTEXT: ${{ github.event.pull_request.commits_url }} - GITHUB_USER: ${{ github.event.pull_request.user.login }} - - - name: Install Python client - run: .github/workflows/scripts/install_python_client.sh - shell: bash - - - name: Install Ruby client - run: .github/workflows/scripts/install_ruby_client.sh - shell: bash - - - name: Before Script - run: .github/workflows/scripts/before_script.sh - shell: bash - env: - PY_COLORS: '1' - ANSIBLE_FORCE_COLOR: '1' - GITHUB_PULL_REQUEST: ${{ github.event.number }} - GITHUB_PULL_REQUEST_BODY: ${{ github.event.pull_request.body }} - GITHUB_BRANCH: ${{ github.head_ref }} - GITHUB_REPO_SLUG: ${{ github.repository }} - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - GITHUB_CONTEXT: ${{ github.event.pull_request.commits_url }} - GITHUB_USER: ${{ github.event.pull_request.user.login }} - REDIS_DISABLED: ${{ contains('', matrix.env.TEST) }} - - - name: Setting secrets - run: python3 .github/workflows/scripts/secrets.py "$SECRETS_CONTEXT" - env: - SECRETS_CONTEXT: ${{ toJson(secrets) }} - - - - - name: Logs - if: always() + - name: "Set environment variables" run: | - echo "Need to debug? Please check: https://github.com/marketplace/actions/debugging-with-tmate" - http --timeout 30 --check-status --pretty format --print hb "https://pulp${PULP_API_ROOT}api/v3/status/" || true - docker images || true - docker ps -a || true - docker logs pulp || true - docker exec pulp ls -latr /etc/yum.repos.d/ || true - docker exec pulp cat /etc/yum.repos.d/* || true - docker exec pulp bash -c "pip3 list && pip3 install pipdeptree && pipdeptree" + echo "TEST=${{ matrix.env.TEST }}" >> $GITHUB_ENV diff --git a/.github/workflows/nightly_latest.yml b/.github/workflows/nightly_latest.yml index 297cae02e9..bd7b2cb890 100644 --- a/.github/workflows/nightly_latest.yml +++ b/.github/workflows/nightly_latest.yml @@ -46,7 +46,7 @@ jobs: echo "HTTPIE_CONFIG_DIR=$GITHUB_WORKSPACE/.ci/assets/httpie/" >> $GITHUB_ENV - name: Before Install - + run: .github/workflows/scripts/before_install.sh shell: bash env: @@ -65,7 +65,7 @@ jobs: ruby-version: "2.6" - name: Install - + run: .github/workflows/scripts/install.sh shell: bash env: @@ -79,7 +79,7 @@ jobs: GITHUB_CONTEXT: ${{ github.event.pull_request.commits_url }} - name: Before Script - + run: .github/workflows/scripts/before_script.sh shell: bash env: @@ -93,13 +93,13 @@ jobs: GITHUB_CONTEXT: ${{ github.event.pull_request.commits_url }} - name: Setting secrets - + run: python3 .github/workflows/scripts/secrets.py "$SECRETS_CONTEXT" env: SECRETS_CONTEXT: ${{ toJson(secrets) }} - name: Install Python client - + run: .github/workflows/scripts/install_python_client.sh shell: bash @@ -152,4 +152,4 @@ jobs: run: | echo "${{ needs.test.outputs.deprecations-pulp }}" | base64 -d echo "${{ needs.test.outputs.deprecations-azure }}" | base64 -d - echo "${{ needs.test.outputs.deprecations-s3 }}" | base64 -d \ No newline at end of file + echo "${{ needs.test.outputs.deprecations-s3 }}" | base64 -d diff --git a/.github/workflows/pr_checks.yml b/.github/workflows/pr_checks.yml new file mode 100644 index 0000000000..9b368affe6 --- /dev/null +++ b/.github/workflows/pr_checks.yml @@ -0,0 +1,62 @@ +# WARNING: DO NOT EDIT! +# +# This file was generated by plugin_template, and is managed by it. Please use +# './plugin-template --github galaxy_ng' to update this file. +# +# For more info visit https://github.com/pulp/plugin_template + +--- +name: Galaxy PR static checks +on: + pull_request_target: + types: [opened, synchronize, reopened] + +# This workflow runs with elevated permissions. +# Do not even think about running a single bit of code from the PR. +# Static analysis should be fine however. + +concurrency: + group: ${{ github.event.pull_request.number }}-${{ github.workflow }} + cancel-in-progress: true + +jobs: + single_commit: + runs-on: ubuntu-latest + name: Label multiple commit PR + permissions: + pull-requests: write + steps: + - uses: "actions/checkout@v4" + with: + fetch-depth: 0 + - name: Commit Count Check + run: | + git fetch origin ${{ github.event.pull_request.head.sha }} + echo "COMMIT_COUNT=$(git log --oneline --no-merges origin/${{ github.base_ref }}..${{ github.event.pull_request.head.sha }} | wc -l)" >> "$GITHUB_ENV" + - uses: actions/github-script@v7 + with: + script: | + const labelName = "multi-commit"; + const { COMMIT_COUNT } = process.env; + + if (COMMIT_COUNT == 1) + { + try { + await github.rest.issues.removeLabel({ + issue_number: context.issue.number, + owner: context.repo.owner, + repo: context.repo.repo, + name: labelName, + }); + } catch(err) { + } + } + else + { + await github.rest.issues.addLabels({ + issue_number: context.issue.number, + owner: context.repo.owner, + repo: context.repo.repo, + labels: [labelName], + }); + } diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml new file mode 100644 index 0000000000..76c0260395 --- /dev/null +++ b/.github/workflows/publish.yml @@ -0,0 +1,131 @@ +# WARNING: DO NOT EDIT! +# +# This file was generated by plugin_template, and is managed by it. Please use +# './plugin-template --github galaxy_ng' to update this file. +# +# For more info visit https://github.com/pulp/plugin_template + +--- +name: "Galaxy Publish Release" +on: + push: + tags: + - "[0-9]+.[0-9]+.[0-9]+" + +defaults: + run: + working-directory: "galaxy_ng" + +jobs: + build: + uses: "./.github/workflows/build.yml" + + build-bindings-docs: + needs: + - "build" + runs-on: "ubuntu-latest" + # Install scripts expect TEST to be set, 'docs' is most appropriate even though we don't run tests + env: + TEST: "docs" + steps: + - uses: "actions/checkout@v4" + with: + fetch-depth: 1 + path: "galaxy_ng" + + - uses: "actions/checkout@v4" + with: + fetch-depth: 1 + repository: "pulp/pulp-openapi-generator" + path: "pulp-openapi-generator" + + - uses: "actions/setup-python@v4" + with: + python-version: "3.11" + + - uses: "actions/download-artifact@v3" + with: + name: "plugin_package" + path: "galaxy_ng/dist/" + + - name: "Install python dependencies" + run: | + echo ::group::PYDEPS + pip install towncrier twine wheel httpie docker netaddr boto3 ansible mkdocs + echo "HTTPIE_CONFIG_DIR=$GITHUB_WORKSPACE/galaxy_ng/.ci/assets/httpie/" >> $GITHUB_ENV + echo ::endgroup:: + + # Building the bindings and docs requires accessing the OpenAPI specs endpoint, so we need to + # setup the Pulp instance. + - name: "Before Install" + run: | + .github/workflows/scripts/before_install.sh + shell: "bash" + env: + PY_COLORS: "1" + ANSIBLE_FORCE_COLOR: "1" + GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" + GITHUB_CONTEXT: "${{ github.event.pull_request.commits_url }}" + GITHUB_USER: "${{ github.event.pull_request.user.login }}" + + - name: "Install" + run: | + .github/workflows/scripts/install.sh + shell: "bash" + env: + PY_COLORS: "1" + ANSIBLE_FORCE_COLOR: "1" + GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" + GITHUB_CONTEXT: "${{ github.event.pull_request.commits_url }}" + GITHUB_USER: "${{ github.event.pull_request.user.login }}" + + - name: "Install Python client" + run: | + .github/workflows/scripts/install_python_client.sh + shell: "bash" + - name: "Upload python client packages" + uses: "actions/upload-artifact@v3" + with: + name: "python-client.tar" + path: "galaxy_ng/galaxy-python-client.tar" + if-no-files-found: "error" + + - name: "Upload python client docs" + uses: "actions/upload-artifact@v3" + with: + name: "python-client-docs.tar" + path: "galaxy_ng/galaxy-python-client-docs.tar" + if-no-files-found: "error" + + - name: "Logs" + if: always() + run: | + echo "Need to debug? Please check: https://github.com/marketplace/actions/debugging-with-tmate" + http --timeout 30 --check-status --pretty format --print hb "https://pulp${PULP_API_ROOT}api/v3/status/" || true + docker images || true + docker ps -a || true + docker logs pulp || true + docker exec pulp ls -latr /etc/yum.repos.d/ || true + docker exec pulp cat /etc/yum.repos.d/* || true + docker exec pulp bash -c "pip3 list && pip3 install pipdeptree && pipdeptree" + + create-gh-release: + runs-on: "ubuntu-latest" + needs: + - "build-bindings-docs" + + steps: + - name: "Create release on GitHub" + uses: "actions/github-script@v7" + env: + TAG_NAME: "${{ github.ref_name }}" + with: + script: | + const { TAG_NAME } = process.env; + + await github.rest.repos.createRelease({ + owner: context.repo.owner, + repo: context.repo.repo, + tag_name: TAG_NAME, + make_latest: "legacy", + }); diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index e4140d04fd..841b32be42 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -9,216 +9,53 @@ name: Galaxy Release Pipeline on: workflow_dispatch: - inputs: - release: - description: "Release tag (e.g. 3.2.1)" - before_script: - description: | - Bash code to run before bindings and docs are built. This should only be used when re-running - a workflow to correct some aspect of the docs. e.g.: git checkout origin/3.14 CHANGES.rst - required: false -env: - RELEASE_WORKFLOW: true +defaults: + run: + working-directory: "galaxy_ng" jobs: build-artifacts: - runs-on: ubuntu-latest + runs-on: "ubuntu-latest" strategy: fail-fast: false steps: - - uses: actions/checkout@v4 + - uses: "actions/checkout@v4" with: fetch-depth: 0 + path: "galaxy_ng" + token: ${{ secrets.RELEASE_TOKEN }} - - uses: actions/setup-python@v4 + - uses: "actions/setup-python@v4" with: - python-version: "3.8" + python-version: "3.11" - - name: Install python dependencies + - name: "Install python dependencies" run: | echo ::group::PYDEPS - pip install packaging~=21.3 bump2version gitpython towncrier==19.9.0 wheel requests + pip install bump2version towncrier echo ::endgroup:: - - name: Configure Git with ansible name and email + - name: "Configure Git with ansible name and email" run: | git config --global user.name 'ansible' git config --global user.email 'ansible-infra@redhat.com' - - name: Setting secrets - run: python3 .github/workflows/scripts/secrets.py "$SECRETS_CONTEXT" - env: - SECRETS_CONTEXT: ${{ toJson(secrets) }} - - - name: Create the release commit, tag it, create a post-release commit, and build plugin package - run: python .github/workflows/scripts/release.py ${{ github.event.inputs.release }} - - - name: 'Tar files' - run: tar -cvf galaxy_ng.tar . - - - name: 'Upload Artifact' - uses: actions/upload-artifact@v3 - with: - name: galaxy_ng.tar - path: galaxy_ng.tar - - - build-bindings-docs: - needs: build-artifacts - runs-on: ubuntu-latest - # Install scripts expect TEST to be set, 'docs' is most appropriate even though we don't run tests - env: - TEST: docs - - steps: - - uses: actions/download-artifact@v3 - with: - name: galaxy_ng.tar - - - uses: actions/setup-python@v4 - with: - python-version: "3.9" - - - name: Untar repository + - name: "Setting secrets" run: | - tar -xf galaxy_ng.tar - - # update to the branch's latest ci files rather than the ones from the release tag. this is - # helpful when there was a problem with the ci files during the release which needs to be - # fixed after the release tag has been created - - name: Update ci files - run: | - git checkout "origin/${GITHUB_REF##*/}" -- .ci - git checkout "origin/${GITHUB_REF##*/}" -- .github - - - name: Install httpie - run: | - echo ::group::HTTPIE - pip install httpie - echo ::endgroup:: - echo "HTTPIE_CONFIG_DIR=$GITHUB_WORKSPACE/.ci/assets/httpie/" >> $GITHUB_ENV - - # Building the bindings and docs requires accessing the OpenAPI specs endpoint, so we need to - # setup the Pulp instance. - - name: Before Install - run: .github/workflows/scripts/before_install.sh - shell: bash + python3 .github/workflows/scripts/secrets.py "$SECRETS_CONTEXT" env: - PY_COLORS: '1' - ANSIBLE_FORCE_COLOR: '1' - GITHUB_PULL_REQUEST: ${{ github.event.number }} - GITHUB_PULL_REQUEST_BODY: ${{ github.event.pull_request.body }} - GITHUB_BRANCH: ${{ github.head_ref }} - GITHUB_REPO_SLUG: ${{ github.repository }} - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - GITHUB_CONTEXT: ${{ github.event.pull_request.commits_url }} - GITHUB_USER: ${{ github.event.pull_request.user.login }} - - - name: Install - run: | - export PLUGIN_VERSION=${{ github.event.inputs.release }} - .github/workflows/scripts/install.sh - env: - PY_COLORS: '1' - ANSIBLE_FORCE_COLOR: '1' - GITHUB_PULL_REQUEST: ${{ github.event.number }} - GITHUB_PULL_REQUEST_BODY: ${{ github.event.pull_request.body }} - GITHUB_BRANCH: ${{ github.head_ref }} - GITHUB_REPO_SLUG: ${{ github.repository }} - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - GITHUB_CONTEXT: ${{ github.event.pull_request.commits_url }} - GITHUB_USER: ${{ github.event.pull_request.user.login }} - shell: bash - - - name: Additional before_script - run: ${{ github.event.inputs.before_script }} - shell: bash - - - name: Install Python client - run: .github/workflows/scripts/install_python_client.sh - shell: bash - - - name: Upload python client packages - uses: actions/upload-artifact@v3 - with: - name: python-client.tar - path: python-client.tar - - - name: Upload python client docs - uses: actions/upload-artifact@v3 - with: - name: python-client-docs.tar - path: python-client-docs.tar - - - name: Logs - if: always() - run: | - echo "Need to debug? Please check: https://github.com/marketplace/actions/debugging-with-tmate" - http --timeout 30 --check-status --pretty format --print hb "https://pulp${PULP_API_ROOT}api/v3/status/" || true - docker images || true - docker ps -a || true - docker logs pulp || true - docker exec pulp ls -latr /etc/yum.repos.d/ || true - docker exec pulp cat /etc/yum.repos.d/* || true - docker exec pulp bash -c "pip3 list && pip3 install pipdeptree && pipdeptree" - - - publish: - runs-on: ubuntu-latest - needs: build-bindings-docs - - env: - TEST: publish - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - steps: - - uses: actions/download-artifact@v3 - with: - name: galaxy_ng.tar - - - uses: actions/setup-python@v4 - with: - python-version: "3.8" + SECRETS_CONTEXT: "${{ toJson(secrets) }}" - - uses: ruby/setup-ruby@v1 - with: - ruby-version: "2.6" - - - name: Configure Git with ansible name and email + - name: "Tag the release" run: | - git config --global user.name 'ansible' - git config --global user.email 'ansible-infra@redhat.com' - - - name: Untar repository - run: | - tar -xf galaxy_ng.tar - - # update to the branch's latest ci files rather than the ones from the release tag. this is - # helpful when there was a problem with the ci files during the release which needs to be - # fixed after the release tag has been created - - name: Update ci files - run: | - git checkout "origin/${GITHUB_REF##*/}" -- .ci - git checkout "origin/${GITHUB_REF##*/}" -- .github - - - name: Setting secrets - run: python3 .github/workflows/scripts/secrets.py "$SECRETS_CONTEXT" + .github/workflows/scripts/release.sh + shell: "bash" env: - SECRETS_CONTEXT: ${{ toJson(secrets) }} - - - name: Install python dependencies - run: | - echo ::group::PYDEPS - pip install gitpython requests packaging~=21.3 tweepy - echo ::endgroup:: - - - name: Push branch and tag to GitHub - run: bash .github/workflows/scripts/push_branch_and_tag_to_github.sh ${{ github.event.inputs.release }} - - - - - name: Create release on GitHub - run: bash .github/workflows/scripts/create_release_from_tag.sh ${{ github.event.inputs.release }} + PY_COLORS: "1" + ANSIBLE_FORCE_COLOR: "1" + GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" + GITHUB_CONTEXT: "${{ github.event.pull_request.commits_url }}" + GITHUB_USER: "${{ github.event.pull_request.user.login }}" diff --git a/.github/workflows/scripts/before_install.sh b/.github/workflows/scripts/before_install.sh index 20929d7f28..20f4d8ab59 100755 --- a/.github/workflows/scripts/before_install.sh +++ b/.github/workflows/scripts/before_install.sh @@ -55,14 +55,6 @@ then echo $COMMIT_MSG | sed -n -e 's/.*CI Base Image:\s*\([-_/[:alnum:]]*:[-_[:alnum:]]*\).*/ci_base: "\1"/p' >> .ci/ansible/vars/main.yaml fi - -cd .. - -git clone --depth=1 https://github.com/pulp/pulp-openapi-generator.git - -# Intall requirements for ansible playbooks -pip install docker netaddr boto3 ansible - for i in {1..3} do ansible-galaxy collection install "amazon.aws:1.5.0" && s=0 && break || s=$? && sleep 3 @@ -73,11 +65,9 @@ then exit $s fi -cd galaxy_ng - if [[ "$TEST" = "lowerbounds" ]]; then - python3 .ci/scripts/calc_deps_lowerbounds.py > lowerbounds_requirements.txt - mv lowerbounds_requirements.txt requirements.txt + python3 .ci/scripts/calc_deps_lowerbounds.py > lowerbounds_constraints.txt + sed -i 's/\[.*\]//g' lowerbounds_constraints.txt fi if [ -f $POST_BEFORE_INSTALL ]; then diff --git a/.github/workflows/scripts/check_commit.sh b/.github/workflows/scripts/check_commit.sh index db76b23248..920970913b 100755 --- a/.github/workflows/scripts/check_commit.sh +++ b/.github/workflows/scripts/check_commit.sh @@ -8,18 +8,13 @@ # For more info visit https://github.com/pulp/plugin_template # make sure this script runs at the repo root -cd "$(dirname "$(realpath -e "$0")")"/../../.. +cd "$(dirname "$(realpath -e "$0")")/../../.." set -euv -echo ::group::REQUESTS -pip3 install requests - -echo ::endgroup:: - -for sha in $(curl -H "Authorization: token $GITHUB_TOKEN" $GITHUB_CONTEXT | jq '.[].sha' | sed 's/"//g') +for SHA in $(curl -H "Authorization: token $GITHUB_TOKEN" "$GITHUB_CONTEXT" | jq -r '.[].sha') do - python3 .ci/scripts/validate_commit_message.py $sha + python3 .ci/scripts/validate_commit_message.py "$SHA" VALUE=$? if [ "$VALUE" -gt 0 ]; then exit $VALUE diff --git a/.github/workflows/scripts/install.sh b/.github/workflows/scripts/install.sh index 5c9f3cb338..ca976daf67 100755 --- a/.github/workflows/scripts/install.sh +++ b/.github/workflows/scripts/install.sh @@ -15,6 +15,9 @@ set -euv source .github/workflows/scripts/utils.sh +PLUGIN_VERSION="$(sed -n -e 's/^\s*current_version\s*=\s*//p' .bumpversion.cfg | python -c 'from packaging.version import Version; print(Version(input()))')" +PLUGIN_NAME="./galaxy_ng/dist/galaxy_ng-${PLUGIN_VERSION}-py3-none-any.whl" + export PULP_API_ROOT="/api/galaxy/pulp/" PIP_REQUIREMENTS=("pulp-cli") @@ -28,19 +31,21 @@ pip install ${PIP_REQUIREMENTS[*]} cd .ci/ansible/ - -if [[ "${RELEASE_WORKFLOW:-false}" == "true" ]]; then - PLUGIN_NAME=./galaxy_ng/dist/galaxy_ng-$PLUGIN_VERSION-py3-none-any.whl -else - PLUGIN_NAME=./galaxy_ng +PLUGIN_SOURCE="${PLUGIN_NAME}" +if [ "$TEST" = "s3" ]; then + PLUGIN_SOURCE="${PLUGIN_SOURCE} pulpcore[s3]" +fi +if [ "$TEST" = "azure" ]; then + PLUGIN_SOURCE="${PLUGIN_SOURCE} pulpcore[azure]" fi + cat >> vars/main.yaml << VARSYAML image: name: pulp tag: "ci_build" plugins: - name: galaxy_ng - source: "${PLUGIN_NAME}" + source: "${PLUGIN_SOURCE}" VARSYAML if [[ -f ../../ci_requirements.txt ]]; then cat >> vars/main.yaml << VARSYAML @@ -66,9 +71,7 @@ cat >> vars/main.yaml << VARSYAML pulp_env: {} pulp_settings: {"allowed_export_paths": "/tmp", "allowed_import_paths": "/tmp", "galaxy_api_default_distribution_base_path": "published", "galaxy_enable_api_access_log": true, "galaxy_require_content_approval": false, "rh_entitlement_required": "insights"} pulp_scheme: https - -pulp_container_tag: "latest" - +pulp_default_container: ghcr.io/pulp/pulp-ci-centos9:latest VARSYAML if [ "$TEST" = "s3" ]; then @@ -145,10 +148,17 @@ cat "$CERTIFI" | sudo tee -a "$CERT" > /dev/null sudo update-ca-certificates echo ::endgroup:: +# Add our azcert.crt certificate to the container image along with the certificates from certifi +# so that we can use HTTPS with our fake Azure CI. certifi is self-contained and doesn't allow +# extension or modification of the trust store, so we do a weird and hacky thing (above) where we just +# overwrite or append to certifi's trust store behind it's back. +# +# We do this for both the CI host and the CI image. if [[ "$TEST" = "azure" ]]; then AZCERTIFI=$(/opt/az/bin/python3 -c 'import certifi; print(certifi.where())') + PULPCERTIFI=$(cmd_prefix python3 -c 'import certifi; print(certifi.where())') cat /usr/local/share/ca-certificates/azcert.crt >> $AZCERTIFI - cat /usr/local/share/ca-certificates/azcert.crt | cmd_stdin_prefix tee -a /usr/local/lib/python3.8/site-packages/certifi/cacert.pem > /dev/null + cat /usr/local/share/ca-certificates/azcert.crt | cmd_stdin_prefix tee -a "$PULPCERTIFI" > /dev/null cat /usr/local/share/ca-certificates/azcert.crt | cmd_stdin_prefix tee -a /etc/pki/tls/cert.pem > /dev/null AZURE_STORAGE_CONNECTION_STRING='DefaultEndpointsProtocol=https;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=https://ci-azurite:10000/devstoreaccount1;' az storage container create --name pulp-test --connection-string $AZURE_STORAGE_CONNECTION_STRING diff --git a/.github/workflows/scripts/install_python_client.sh b/.github/workflows/scripts/install_python_client.sh index 1868916875..f204b29964 100755 --- a/.github/workflows/scripts/install_python_client.sh +++ b/.github/workflows/scripts/install_python_client.sh @@ -9,45 +9,61 @@ set -mveuo pipefail -export PULP_URL="${PULP_URL:-https://pulp}" - # make sure this script runs at the repo root cd "$(dirname "$(realpath -e "$0")")"/../../.. -pip install twine wheel +source .github/workflows/scripts/utils.sh -export REPORTED_VERSION=$(http $PULP_URL/pulp/api/v3/status/ | jq --arg plugin galaxy --arg legacy_plugin galaxy_ng -r '.versions[] | select(.component == $plugin or .component == $legacy_plugin) | .version') -export DESCRIPTION="$(git describe --all --exact-match `git rev-parse HEAD`)" -if [[ $DESCRIPTION == 'tags/'$REPORTED_VERSION ]]; then - export VERSION=${REPORTED_VERSION} -else - export EPOCH="$(date +%s)" - export VERSION=${REPORTED_VERSION}${EPOCH} -fi +PULP_URL="${PULP_URL:-https://pulp}" +export PULP_URL +PULP_API_ROOT="${PULP_API_ROOT:-/pulp/}" +export PULP_API_ROOT -export response=$(curl --write-out %{http_code} --silent --output /dev/null https://pypi.org/project/galaxy-ng-client/$VERSION/) +REPORTED_STATUS="$(pulp status)" +REPORTED_VERSION="$(echo "$REPORTED_STATUS" | jq --arg plugin "galaxy" -r '.versions[] | select(.component == $plugin) | .version')" +VERSION="$(echo "$REPORTED_VERSION" | python -c 'from packaging.version import Version; print(Version(input()))')" -if [ "$response" == "200" ]; +pushd ../pulp-openapi-generator +rm -rf galaxy_ng-client + +if pulp debug has-plugin --name "core" --specifier ">=3.44.0.dev" then - echo "galaxy_ng client $VERSION has already been released. Installing from PyPI." - docker exec pulp pip3 install galaxy-ng-client==$VERSION - mkdir -p dist - tar cvf python-client.tar ./dist - exit + curl --fail-with-body -k -o api.json "${PULP_URL}${PULP_API_ROOT}api/v3/docs/api.json?bindings&component=galaxy" + USE_LOCAL_API_JSON=1 ./generate.sh galaxy_ng python "$VERSION" +else + ./generate.sh galaxy_ng python "$VERSION" fi -cd ../pulp-openapi-generator -rm -rf galaxy_ng-client -./generate.sh galaxy_ng python $VERSION -cd galaxy_ng-client +pushd galaxy_ng-client python setup.py sdist bdist_wheel --python-tag py3 -find . -name "*.whl" -exec docker exec pulp pip3 install /root/pulp-openapi-generator/galaxy_ng-client/{} \; -tar cvf ../../galaxy_ng/python-client.tar ./dist + +twine check "dist/galaxy_ng_client-$VERSION-py3-none-any.whl" +twine check "dist/galaxy_ng-client-$VERSION.tar.gz" + +cmd_prefix pip3 install "/root/pulp-openapi-generator/galaxy_ng-client/dist/galaxy_ng_client-${VERSION}-py3-none-any.whl" +tar cvf ../../galaxy_ng/galaxy-python-client.tar ./dist find ./docs/* -exec sed -i 's/Back to README/Back to HOME/g' {} \; find ./docs/* -exec sed -i 's/README//g' {} \; cp README.md docs/index.md sed -i 's/docs\///g' docs/index.md find ./docs/* -exec sed -i 's/\.md//g' {} \; -tar cvf ../../galaxy_ng/python-client-docs.tar ./docs -exit $? + +cat >> mkdocs.yml << DOCSYAML +--- +site_name: GalaxyNg Client +site_description: Galaxy bindings +site_author: Pulp Team +site_url: https://docs.pulpproject.org/galaxy_ng_client/ +repo_name: pulp/galaxy_ng +repo_url: https://github.com/pulp/galaxy_ng +theme: readthedocs +DOCSYAML + +# Building the bindings docs +mkdocs build + +# Pack the built site. +tar cvf ../../galaxy_ng/galaxy-python-client-docs.tar ./site +popd +popd diff --git a/.github/workflows/scripts/install_ruby_client.sh b/.github/workflows/scripts/install_ruby_client.sh index 6bbf16d76c..561da4dfcd 100755 --- a/.github/workflows/scripts/install_ruby_client.sh +++ b/.github/workflows/scripts/install_ruby_client.sh @@ -7,37 +7,36 @@ # # For more info visit https://github.com/pulp/plugin_template -set -euv +set -mveuo pipefail # make sure this script runs at the repo root cd "$(dirname "$(realpath -e "$0")")"/../../.. -export PULP_URL="${PULP_URL:-https://pulp}" +source .github/workflows/scripts/utils.sh -export REPORTED_VERSION=$(http $PULP_URL/pulp/api/v3/status/ | jq --arg plugin galaxy --arg legacy_plugin galaxy_ng -r '.versions[] | select(.component == $plugin or .component == $legacy_plugin) | .version') -export DESCRIPTION="$(git describe --all --exact-match `git rev-parse HEAD`)" -if [[ $DESCRIPTION == 'tags/'$REPORTED_VERSION ]]; then - export VERSION=${REPORTED_VERSION} -else - export EPOCH="$(date +%s)" - export VERSION=${REPORTED_VERSION}${EPOCH} -fi +PULP_URL="${PULP_URL:-https://pulp}" +export PULP_URL +PULP_API_ROOT="${PULP_API_ROOT:-/pulp/}" +export PULP_API_ROOT + +REPORTED_STATUS="$(pulp status)" +REPORTED_VERSION="$(echo "$REPORTED_STATUS" | jq --arg plugin "galaxy" -r '.versions[] | select(.component == $plugin) | .version')" +VERSION="$(echo "$REPORTED_VERSION" | python -c 'from packaging.version import Version; print(Version(input()))')" -export response=$(curl --write-out %{http_code} --silent --output /dev/null https://rubygems.org/gems/galaxy_ng_client/versions/$VERSION) +pushd ../pulp-openapi-generator +rm -rf galaxy_ng-client -if [ "$response" == "200" ]; +if pulp debug has-plugin --name "core" --specifier ">=3.44.0.dev" then - echo "galaxy_ng client $VERSION has already been released. Installing from RubyGems.org." - gem install galaxy_ng_client -v $VERSION - touch galaxy_ng_client-$VERSION.gem - tar cvf ruby-client.tar ./galaxy_ng_client-$VERSION.gem - exit + curl --fail-with-body -k -o api.json "${PULP_URL}${PULP_API_ROOT}api/v3/docs/api.json?bindings&component=galaxy" + USE_LOCAL_API_JSON=1 ./generate.sh galaxy_ng ruby "$VERSION" +else + ./generate.sh galaxy_ng ruby "$VERSION" fi -cd ../pulp-openapi-generator -rm -rf galaxy_ng-client -./generate.sh galaxy_ng ruby $VERSION -cd galaxy_ng-client +pushd galaxy_ng-client gem build galaxy_ng_client -gem install --both ./galaxy_ng_client-$VERSION.gem -tar cvf ../../galaxy_ng/ruby-client.tar ./galaxy_ng_client-$VERSION.gem +gem install --both "./galaxy_ng_client-$VERSION.gem" +tar cvf ../../galaxy_ng/galaxy-ruby-client.tar "./galaxy_ng_client-$VERSION.gem" +popd +popd diff --git a/.github/workflows/scripts/publish_client_gem.sh b/.github/workflows/scripts/publish_client_gem.sh index 2d35f051f6..ae76f8fa88 100755 --- a/.github/workflows/scripts/publish_client_gem.sh +++ b/.github/workflows/scripts/publish_client_gem.sh @@ -12,27 +12,24 @@ set -euv # make sure this script runs at the repo root cd "$(dirname "$(realpath -e "$0")")"/../../.. - -mkdir ~/.gem || true -touch ~/.gem/credentials -echo "--- -:rubygems_api_key: $RUBYGEMS_API_KEY" > ~/.gem/credentials -sudo chmod 600 ~/.gem/credentials - -export VERSION=$(ls galaxy_ng_client* | sed -rn 's/galaxy_ng_client-(.*)\.gem/\1/p') +VERSION="$1" if [[ -z "$VERSION" ]]; then - echo "No client package found." - exit + echo "No version specified." + exit 1 fi -export response=$(curl --write-out %{http_code} --silent --output /dev/null https://rubygems.org/gems/galaxy_ng_client/versions/$VERSION) +RESPONSE="$(curl --write-out '%{http_code}' --silent --output /dev/null "https://rubygems.org/gems/galaxy_ng_client/versions/$VERSION")" -if [ "$response" == "200" ]; +if [ "$RESPONSE" == "200" ]; then echo "galaxy_ng client $VERSION has already been released. Skipping." exit fi -GEM_FILE="$(ls galaxy_ng_client-*)" -gem push ${GEM_FILE} +mkdir -p ~/.gem +touch ~/.gem/credentials +echo "--- +:rubygems_api_key: $RUBYGEMS_API_KEY" > ~/.gem/credentials +sudo chmod 600 ~/.gem/credentials +gem push "galaxy_ng_client-${VERSION}.gem" diff --git a/.github/workflows/scripts/publish_client_pypi.sh b/.github/workflows/scripts/publish_client_pypi.sh index eaa85a2742..331e53a3c0 100755 --- a/.github/workflows/scripts/publish_client_pypi.sh +++ b/.github/workflows/scripts/publish_client_pypi.sh @@ -10,28 +10,24 @@ set -euv # make sure this script runs at the repo root -cd "$(dirname "$(realpath -e "$0")")"/../../.. +cd "$(dirname "$(realpath -e "$0")")/../../.." -pip install twine - -export VERSION=$(ls dist | sed -rn 's/galaxy_ng-client-(.*)\.tar.gz/\1/p') +VERSION="$1" if [[ -z "$VERSION" ]]; then - echo "No client package found." - exit + echo "No version specified." + exit 1 fi -export response=$(curl --write-out %{http_code} --silent --output /dev/null https://pypi.org/project/galaxy-ng-client/$VERSION/) +RESPONSE="$(curl --write-out '%{http_code}' --silent --output /dev/null "https://pypi.org/project/galaxy-ng-client/$VERSION/")" -if [ "$response" == "200" ]; +if [ "$RESPONSE" == "200" ]; then echo "galaxy_ng client $VERSION has already been released. Skipping." exit fi -twine check dist/galaxy_ng_client-$VERSION-py3-none-any.whl || exit 1 -twine check dist/galaxy_ng-client-$VERSION.tar.gz || exit 1 -twine upload dist/galaxy_ng_client-$VERSION-py3-none-any.whl -u pulp -p $PYPI_PASSWORD -twine upload dist/galaxy_ng-client-$VERSION.tar.gz -u pulp -p $PYPI_PASSWORD - -exit $? +twine upload -u __token__ -p "$PYPI_API_TOKEN" \ +"dist/galaxy_ng_client-$VERSION-py3-none-any.whl" \ +"dist/galaxy_ng-client-$VERSION.tar.gz" \ +; diff --git a/.github/workflows/scripts/publish_docs.sh b/.github/workflows/scripts/publish_docs.sh index 6b2648f58c..573782b9a6 100755 --- a/.github/workflows/scripts/publish_docs.sh +++ b/.github/workflows/scripts/publish_docs.sh @@ -10,7 +10,7 @@ set -euv # make sure this script runs at the repo root -cd "$(dirname "$(realpath -e "$0")")"/../../.. +cd "$(dirname "$(realpath -e "$0")")/../../.." mkdir ~/.ssh touch ~/.ssh/pulp-infra @@ -20,44 +20,29 @@ echo "$PULP_DOCS_KEY" > ~/.ssh/pulp-infra echo "docs.pulpproject.org,8.43.85.236 ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBGXG+8vjSQvnAkq33i0XWgpSrbco3rRqNZr0SfVeiqFI7RN/VznwXMioDDhc+hQtgVhd6TYBOrV07IMcKj+FAzg=" >> ~/.ssh/known_hosts chmod 644 ~/.ssh/known_hosts -pip3 install packaging - export PYTHONUNBUFFERED=1 export DJANGO_SETTINGS_MODULE=pulpcore.app.settings export PULP_SETTINGS=$PWD/.ci/ansible/settings/settings.py export WORKSPACE=$PWD -eval "$(ssh-agent -s)" #start the ssh agent +# start the ssh agent +eval "$(ssh-agent -s)" ssh-add ~/.ssh/pulp-infra -python3 .github/workflows/scripts/docs-publisher.py --build-type $1 --branch $2 +python3 .github/workflows/scripts/docs-publisher.py --build-type "$1" --branch "$2" if [[ "$GITHUB_WORKFLOW" == "Galaxy changelog update" ]]; then # Do not build bindings docs on changelog update exit fi -pip install mkdocs pymdown-extensions "Jinja2<3.1" - -mkdir -p ../bindings -tar -xvf python-client-docs.tar --directory ../bindings -cd ../bindings -cat >> mkdocs.yml << DOCSYAML ---- -site_name: GalaxyNG Client -site_description: Galaxy bindings -site_author: Pulp Team -site_url: https://docs.pulpproject.org/galaxy_ng_client/ -repo_name: pulp/galaxy_ng -repo_url: https://github.com/pulp/galaxy_ng -theme: readthedocs -DOCSYAML - -# Building the bindings docs -mkdocs build +mkdir -p ../galaxy-bindings +tar -xvf galaxy-python-client-docs.tar --directory ../galaxy-bindings +pushd ../galaxy-bindings # publish to docs.pulpproject.org/galaxy_ng_client rsync -avzh site/ doc_builder_galaxy_ng@docs.pulpproject.org:/var/www/docs.pulpproject.org/galaxy_ng_client/ # publish to docs.pulpproject.org/galaxy_ng_client/en/{release} rsync -avzh site/ doc_builder_galaxy_ng@docs.pulpproject.org:/var/www/docs.pulpproject.org/galaxy_ng_client/en/"$2" +popd diff --git a/.github/workflows/scripts/publish_plugin_pypi.sh b/.github/workflows/scripts/publish_plugin_pypi.sh index 7c17815123..46ecabd8a1 100755 --- a/.github/workflows/scripts/publish_plugin_pypi.sh +++ b/.github/workflows/scripts/publish_plugin_pypi.sh @@ -7,23 +7,27 @@ # # For more info visit https://github.com/pulp/plugin_template +set -euv + # make sure this script runs at the repo root cd "$(dirname "$(realpath -e "$0")")"/../../.. -set -euv +VERSION="$1" -export response=$(curl --write-out %{http_code} --silent --output /dev/null https://pypi.org/project/galaxy-ng/$1/) -if [ "$response" == "200" ]; -then - echo "galaxy_ng $1 has already been released. Skipping." - exit +if [[ -z "$VERSION" ]]; then + echo "No version specified." + exit 1 fi -pip install twine +RESPONSE="$(curl --write-out '%{http_code}' --silent --output /dev/null "https://pypi.org/project/galaxy-ng/$VERSION/")" -twine check dist/galaxy_ng-$1-py3-none-any.whl || exit 1 -twine check dist/galaxy-ng-$1.tar.gz || exit 1 -twine upload dist/galaxy_ng-$1-py3-none-any.whl -u pulp -p $PYPI_PASSWORD -twine upload dist/galaxy-ng-$1.tar.gz -u pulp -p $PYPI_PASSWORD +if [ "$RESPONSE" == "200" ]; +then + echo "galaxy_ng $VERSION has already been released. Skipping." + exit +fi -exit $? +twine upload -u __token__ -p "$PYPI_API_TOKEN" \ +"dist/galaxy_ng-$VERSION-py3-none-any.whl" \ +"dist/galaxy-ng-$VERSION.tar.gz" \ +; diff --git a/.github/workflows/scripts/push_branch_and_tag_to_github.sh b/.github/workflows/scripts/push_branch_and_tag_to_github.sh index 4944afec8a..24494c5b43 100755 --- a/.github/workflows/scripts/push_branch_and_tag_to_github.sh +++ b/.github/workflows/scripts/push_branch_and_tag_to_github.sh @@ -1,21 +1,16 @@ #!/bin/sh -set -e -BRANCH_NAME=$(echo $GITHUB_REF | sed -rn 's/refs\/heads\/(.*)/\1/p') +# WARNING: DO NOT EDIT! +# +# This file was generated by plugin_template, and is managed by it. Please use +# './plugin-template --github galaxy_ng' to update this file. +# +# For more info visit https://github.com/pulp/plugin_template -ref_string=$(git show-ref --tags | grep refs/tags/$1) +set -eu -SHA=${ref_string:0:40} +BRANCH_NAME="$(echo "$GITHUB_REF" | sed -rn 's/refs\/heads\/(.*)/\1/p')" -remote_repo=https://ansible:${RELEASE_TOKEN}@github.com/${GITHUB_REPOSITORY}.git +remote_repo="https://ansible:${RELEASE_TOKEN}@github.com/${GITHUB_REPOSITORY}.git" -git push "${remote_repo}" $BRANCH_NAME - -curl -s -X POST https://api.github.com/repos/$GITHUB_REPOSITORY/git/refs \ --H "Authorization: token $RELEASE_TOKEN" \ --d @- << EOF -{ - "ref": "refs/tags/$1", - "sha": "$SHA" -} -EOF +git push "${remote_repo}" "$BRANCH_NAME" "$1" diff --git a/.github/workflows/scripts/release.sh b/.github/workflows/scripts/release.sh new file mode 100755 index 0000000000..9525f229d9 --- /dev/null +++ b/.github/workflows/scripts/release.sh @@ -0,0 +1,26 @@ +#!/bin/bash + +set -eu -o pipefail + +BRANCH=$(git branch --show-current) + +if ! [[ "${BRANCH}" =~ ^[0-9]+\.[0-9]+$ ]] +then + echo ERROR: This is not a release branch! + exit 1 +fi + +NEW_VERSION="$(bump2version --dry-run --list release | sed -ne 's/^new_version=//p')" +echo "Release ${NEW_VERSION}" + +if ! [[ "${NEW_VERSION}" == "${BRANCH}"* ]] +then + echo ERROR: Version does not match release branch + exit 1 +fi + +towncrier build --yes --version "${NEW_VERSION}" +bump2version release --commit --message "Release {new_version}" --tag --tag-name "{new_version}" --tag-message "Release {new_version}" --allow-dirty +bump2version patch --commit + +git push origin "${BRANCH}" "${NEW_VERSION}" diff --git a/.github/workflows/scripts/script.sh b/.github/workflows/scripts/script.sh index b85717ce6f..4f3664b758 100755 --- a/.github/workflows/scripts/script.sh +++ b/.github/workflows/scripts/script.sh @@ -12,7 +12,6 @@ set -mveuo pipefail # make sure this script runs at the repo root cd "$(dirname "$(realpath -e "$0")")"/../../.. -REPO_ROOT="$PWD" source .github/workflows/scripts/utils.sh @@ -30,108 +29,131 @@ export PULP_URL="https://pulp" if [[ "$TEST" = "docs" ]]; then if [[ "$GITHUB_WORKFLOW" == "Galaxy CI" ]]; then - pip install towncrier==19.9.0 - towncrier --yes --version 4.0.0.ci + towncrier build --yes --version 4.0.0.ci fi cd docs make PULP_URL="$PULP_URL" diagrams html tar -cvf docs.tar ./_build cd .. - if [ -f $POST_DOCS_TEST ]; then - source $POST_DOCS_TEST + if [ -f "$POST_DOCS_TEST" ]; then + source "$POST_DOCS_TEST" fi exit fi REPORTED_STATUS="$(pulp status)" -if [[ "${RELEASE_WORKFLOW:-false}" == "true" ]]; then - REPORTED_VERSION="$(echo $REPORTED_STATUS | jq --arg plugin galaxy --arg legacy_plugin galaxy_ng -r '.versions[] | select(.component == $plugin or .component == $legacy_plugin) | .version')" - response=$(curl --write-out %{http_code} --silent --output /dev/null https://pypi.org/project/galaxy-ng/$REPORTED_VERSION/) - if [ "$response" == "200" ]; - then - echo "galaxy_ng $REPORTED_VERSION has already been released. Skipping running tests." - exit - fi -fi - echo "machine pulp login admin password password " | cmd_user_stdin_prefix bash -c "cat >> ~pulp/.netrc" # Some commands like ansible-galaxy specifically require 600 -cmd_user_stdin_prefix bash -c "chmod 600 ~pulp/.netrc" +cmd_prefix bash -c "chmod 600 ~pulp/.netrc" -cd ../pulp-openapi-generator -if [ "$(echo "$REPORTED_STATUS" | jq -r '.versions[0].package')" = "null" ] +# Generate and install binding +pushd ../pulp-openapi-generator +if pulp debug has-plugin --name "core" --specifier ">=3.44.0.dev" then - # We are on an old version of pulpcore without package in the status report - for app_label in $(echo "$REPORTED_STATUS" | jq -r '.versions[].component') - do - if [ "$app_label" = "core" ] - then - item=pulpcore - else - item="pulp_${app_label}" - fi - ./generate.sh "${item}" python - cmd_prefix pip3 install "/root/pulp-openapi-generator/${item}-client" - sudo rm -rf "./${item}-client" - done + # Use app_label to generate api.json and package to produce the proper package name. + + if [ "$(jq -r '.domain_enabled' <<<"$REPORTED_STATUS")" = "true" ] + then + # Workaround: Domains are not supported by the published bindings. + # Generate new bindings for all packages. + for item in $(jq -r '.versions[] | tojson' <<<"$REPORTED_STATUS") + do + echo $item + COMPONENT="$(jq -r '.component' <<<"$item")" + VERSION="$(jq -r '.version' <<<"$item")" + MODULE="$(jq -r '.module' <<<"$item")" + PACKAGE="${MODULE%%.*}" + curl --fail-with-body -k -o api.json "${PULP_URL}${PULP_API_ROOT}api/v3/docs/api.json?bindings&component=$COMPONENT" + USE_LOCAL_API_JSON=1 ./generate.sh "${PACKAGE}" python "${VERSION}" + cmd_prefix pip3 install "/root/pulp-openapi-generator/${PACKAGE}-client" + sudo rm -rf "./${PACKAGE}-client" + done + else + # Sadly: Different pulpcore-versions aren't either... + for item in $(jq -r '.versions[]| select(.component!="galaxy")| tojson' <<<"$REPORTED_STATUS") + do + echo $item + COMPONENT="$(jq -r '.component' <<<"$item")" + VERSION="$(jq -r '.version' <<<"$item")" + MODULE="$(jq -r '.module' <<<"$item")" + PACKAGE="${MODULE%%.*}" + curl --fail-with-body -k -o api.json "${PULP_URL}${PULP_API_ROOT}api/v3/docs/api.json?bindings&component=$COMPONENT" + USE_LOCAL_API_JSON=1 ./generate.sh "${PACKAGE}" python "${VERSION}" + cmd_prefix pip3 install "/root/pulp-openapi-generator/${PACKAGE}-client" + sudo rm -rf "./${PACKAGE}-client" + done + fi else - for item in $(echo "$REPORTED_STATUS" | jq -r '.versions[].package|sub("-"; "_")') - do - ./generate.sh "${item}" python - cmd_prefix pip3 install "/root/pulp-openapi-generator/${item}-client" - sudo rm -rf "./${item}-client" - done -fi + # Infer the client name from the package name by replacing "-" with "_". + # Use the component to infer the package name on older versions of pulpcore. -cd $REPO_ROOT + if [ "$(echo "$REPORTED_STATUS" | jq -r '.domain_enabled')" = "true" ] + then + # Workaround: Domains are not supported by the published bindings. + # Generate new bindings for all packages. + for item in $(echo "$REPORTED_STATUS" | jq -r '.versions[]|(.package // ("pulp_" + .component)|sub("pulp_core"; "pulpcore"))|sub("-"; "_")') + do + ./generate.sh "${item}" python + cmd_prefix pip3 install "/root/pulp-openapi-generator/${item}-client" + sudo rm -rf "./${item}-client" + done + else + # Sadly: Different pulpcore-versions aren't either... + for item in $(echo "$REPORTED_STATUS" | jq -r '.versions[]|select(.component!="galaxy")|(.package // ("pulp_" + .component)|sub("pulp_core"; "pulpcore"))|sub("-"; "_")') + do + ./generate.sh "${item}" python + cmd_prefix pip3 install "/root/pulp-openapi-generator/${item}-client" + sudo rm -rf "./${item}-client" + done + fi +fi +popd -cat unittest_requirements.txt | cmd_stdin_prefix bash -c "cat > /tmp/unittest_requirements.txt" -cat functest_requirements.txt | cmd_stdin_prefix bash -c "cat > /tmp/functest_requirements.txt" -cmd_prefix pip3 install -r /tmp/unittest_requirements.txt -r /tmp/functest_requirements.txt +# At this point, this is a safeguard only, so let's not make too much fuzz about the old status format. +echo "$REPORTED_STATUS" | jq -r '.versions[]|select(.package)|(.package|sub("_"; "-")) + "-client==" + .version' > bindings_requirements.txt +cmd_stdin_prefix bash -c "cat > /tmp/unittest_requirements.txt" < unittest_requirements.txt +cmd_stdin_prefix bash -c "cat > /tmp/functest_requirements.txt" < functest_requirements.txt +cmd_stdin_prefix bash -c "cat > /tmp/bindings_requirements.txt" < bindings_requirements.txt +cmd_prefix pip3 install -r /tmp/unittest_requirements.txt -r /tmp/functest_requirements.txt -r /tmp/bindings_requirements.txt CERTIFI=$(cmd_prefix python3 -c 'import certifi; print(certifi.where())') -cmd_prefix bash -c "cat /etc/pulp/certs/pulp_webserver.crt | tee -a "$CERTIFI" > /dev/null" +cmd_prefix bash -c "cat /etc/pulp/certs/pulp_webserver.crt >> '$CERTIFI'" # check for any uncommitted migrations echo "Checking for uncommitted migrations..." cmd_user_prefix bash -c "django-admin makemigrations galaxy --check --dry-run" # Run unit tests. -cmd_user_prefix bash -c "PULP_DATABASES__default__USER=postgres pytest -v -r sx --color=yes -p no:pulpcore --pyargs galaxy_ng.tests.unit" +cmd_user_prefix bash -c "PULP_DATABASES__default__USER=postgres pytest -v -r sx --color=yes --suppress-no-test-exit-code -p no:pulpcore --pyargs galaxy_ng.tests.unit" # Run functional tests if [[ "$TEST" == "performance" ]]; then if [[ -z ${PERFORMANCE_TEST+x} ]]; then - cmd_user_prefix bash -c "pytest -vv -r sx --color=yes --pyargs --capture=no --durations=0 galaxy_ng.tests.performance" + cmd_user_prefix bash -c "pytest -vv -r sx --color=yes --suppress-no-test-exit-code --capture=no --durations=0 --pyargs galaxy_ng.tests.performance" else - cmd_user_prefix bash -c "pytest -vv -r sx --color=yes --pyargs --capture=no --durations=0 galaxy_ng.tests.performance.test_$PERFORMANCE_TEST" + cmd_user_prefix bash -c "pytest -vv -r sx --color=yes --suppress-no-test-exit-code --capture=no --durations=0 --pyargs galaxy_ng.tests.performance.test_${PERFORMANCE_TEST}" fi exit fi -if [ -f $FUNC_TEST_SCRIPT ]; then - source $FUNC_TEST_SCRIPT +if [ -f "$FUNC_TEST_SCRIPT" ]; then + source "$FUNC_TEST_SCRIPT" else - - if [[ "$GITHUB_WORKFLOW" == "Galaxy Nightly CI/CD" ]] || [[ "${RELEASE_WORKFLOW:-false}" == "true" ]]; then - cmd_user_prefix bash -c "pytest -v -r sx --color=yes --suppress-no-test-exit-code --pyargs galaxy_ng.tests.functional -m parallel -n 8 --nightly" - cmd_user_prefix bash -c "pytest -v -r sx --color=yes --pyargs galaxy_ng.tests.functional -m 'not parallel' --nightly" - - - else - cmd_user_prefix bash -c "pytest -v -r sx --color=yes --suppress-no-test-exit-code --pyargs galaxy_ng.tests.functional -m parallel -n 8" - cmd_user_prefix bash -c "pytest -v -r sx --color=yes --pyargs galaxy_ng.tests.functional -m 'not parallel'" - - - fi - + if [[ "$GITHUB_WORKFLOW" =~ "Nightly" ]] + then + cmd_user_prefix bash -c "pytest -v -r sx --color=yes --suppress-no-test-exit-code --pyargs galaxy_ng.tests.functional -m parallel -n 8 --nightly" + cmd_user_prefix bash -c "pytest -v -r sx --color=yes --suppress-no-test-exit-code --pyargs galaxy_ng.tests.functional -m 'not parallel' --nightly" + else + cmd_user_prefix bash -c "pytest -v -r sx --color=yes --suppress-no-test-exit-code --pyargs galaxy_ng.tests.functional -m parallel -n 8" + cmd_user_prefix bash -c "pytest -v -r sx --color=yes --suppress-no-test-exit-code --pyargs galaxy_ng.tests.functional -m 'not parallel'" + fi fi -if [ -f $POST_SCRIPT ]; then - source $POST_SCRIPT +if [ -f "$POST_SCRIPT" ]; then + source "$POST_SCRIPT" fi diff --git a/.github/workflows/scripts/update_backport_labels.py b/.github/workflows/scripts/update_backport_labels.py index e03c20e792..e8a2b860c2 100755 --- a/.github/workflows/scripts/update_backport_labels.py +++ b/.github/workflows/scripts/update_backport_labels.py @@ -32,10 +32,14 @@ def random_color(): assert response.status_code == 200 old_labels = set([x["name"] for x in response.json() if x["name"].startswith("backport-")]) -# get ci_update_branches from template_config.yml +# get list of branches from template_config.yml with open("./template_config.yml", "r") as f: plugin_template = yaml.safe_load(f) -new_labels = set(["backport-" + x for x in plugin_template["ci_update_branches"]]) +branches = set(plugin_template["supported_release_branches"]) +latest_release_branch = plugin_template["latest_release_branch"] +if latest_release_branch is not None: + branches.add(latest_release_branch) +new_labels = {"backport-" + x for x in branches} # delete old labels that are not in new labels for label in old_labels.difference(new_labels): diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000000..70ed306e47 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,137 @@ +# WARNING: DO NOT EDIT! +# +# This file was generated by plugin_template, and is managed by it. Please use +# './plugin-template --github galaxy_ng' to update this file. +# +# For more info visit https://github.com/pulp/plugin_template + +--- +name: "Test" +on: + workflow_call: + +defaults: + run: + working-directory: "galaxy_ng" + +jobs: + test: + runs-on: "ubuntu-latest" + strategy: + fail-fast: false + matrix: + env: + - TEST: pulp + - TEST: azure + - TEST: s3 + + steps: + - uses: "actions/checkout@v4" + with: + fetch-depth: 1 + path: "galaxy_ng" + + - uses: "actions/checkout@v4" + with: + fetch-depth: 1 + repository: "pulp/pulp-openapi-generator" + path: "pulp-openapi-generator" + + - uses: "actions/setup-python@v4" + with: + python-version: "3.11" + + - uses: "actions/download-artifact@v3" + with: + name: "plugin_package" + path: "galaxy_ng/dist/" + + - name: "Install python dependencies" + run: | + echo ::group::PYDEPS + pip install towncrier twine wheel httpie docker netaddr boto3 ansible mkdocs + echo "HTTPIE_CONFIG_DIR=$GITHUB_WORKSPACE/galaxy_ng/.ci/assets/httpie/" >> $GITHUB_ENV + echo ::endgroup:: + + - name: "Set environment variables" + run: | + echo "TEST=${{ matrix.env.TEST }}" >> $GITHUB_ENV + + - name: "Before Install" + run: | + .github/workflows/scripts/before_install.sh + shell: "bash" + env: + PY_COLORS: "1" + ANSIBLE_FORCE_COLOR: "1" + GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" + GITHUB_CONTEXT: "${{ github.event.pull_request.commits_url }}" + GITHUB_USER: "${{ github.event.pull_request.user.login }}" + + - name: "Install" + run: | + .github/workflows/scripts/install.sh + shell: "bash" + env: + PY_COLORS: "1" + ANSIBLE_FORCE_COLOR: "1" + GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" + GITHUB_CONTEXT: "${{ github.event.pull_request.commits_url }}" + GITHUB_USER: "${{ github.event.pull_request.user.login }}" + + - name: "Before Script" + run: | + .github/workflows/scripts/before_script.sh + shell: "bash" + env: + PY_COLORS: "1" + ANSIBLE_FORCE_COLOR: "1" + GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" + GITHUB_CONTEXT: "${{ github.event.pull_request.commits_url }}" + GITHUB_USER: "${{ github.event.pull_request.user.login }}" + REDIS_DISABLED: "${{ contains('', matrix.env.TEST) }}" + + - name: "Install Python client" + run: | + .github/workflows/scripts/install_python_client.sh + shell: "bash" + + - name: "Script" + run: | + .github/workflows/scripts/script.sh + shell: "bash" + env: + PY_COLORS: "1" + ANSIBLE_FORCE_COLOR: "1" + GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" + GITHUB_CONTEXT: "${{ github.event.pull_request.commits_url }}" + GITHUB_USER: "${{ github.event.pull_request.user.login }}" + - name: Upload python client packages + if: ${{ env.TEST == 'pulp' }} + uses: actions/upload-artifact@v3 + with: + name: "python-client.tar" + path: "galaxy_ng/galaxy-python-client.tar" + if-no-files-found: "error" + retention-days: 5 + + - name: Upload python client docs + if: ${{ env.TEST == 'pulp' }} + uses: actions/upload-artifact@v3 + with: + name: "python-client-docs.tar" + path: "galaxy_ng/galaxy-python-client-docs.tar" + if-no-files-found: "error" + retention-days: 5 + + - name: "Logs" + if: always() + run: | + echo "Need to debug? Please check: https://github.com/marketplace/actions/debugging-with-tmate" + http --timeout 30 --check-status --pretty format --print hb "https://pulp${PULP_API_ROOT}api/v3/status/" || true + docker images || true + docker ps -a || true + docker logs pulp || true + docker exec pulp ls -latr /etc/yum.repos.d/ || true + docker exec pulp cat /etc/yum.repos.d/* || true + docker exec pulp bash -c "pip3 list && pip3 install pipdeptree && pipdeptree" diff --git a/.github/workflows/update-labels.yml b/.github/workflows/update-labels.yml index 269ecdb7d3..91454891d6 100644 --- a/.github/workflows/update-labels.yml +++ b/.github/workflows/update-labels.yml @@ -7,33 +7,33 @@ --- -name: Galaxy Update Labels +name: "Galaxy Update Labels" on: push: branches: - - main + - "main" paths: - - 'template_config.yml' + - "template_config.yml" jobs: update_backport_labels: - runs-on: ubuntu-latest + runs-on: "ubuntu-latest" steps: - - uses: actions/setup-python@v4 + - uses: "actions/setup-python@v4" with: - python-version: "3.9" - - name: Configure Git with ansible name and email + python-version: "3.11" + - name: "Configure Git with ansible name and email" run: | git config --global user.name 'ansible' git config --global user.email 'ansible-infra@redhat.com' - - name: Install python dependencies + - name: "Install python dependencies" run: | echo ::group::PYDEPS pip install requests pyyaml echo ::endgroup:: - - uses: actions/checkout@v3 - - name: Update labels + - uses: "actions/checkout@v3" + - name: "Update labels" run: | python3 .github/workflows/scripts/update_backport_labels.py env: - GITHUB_TOKEN: ${{ secrets.RELEASE_TOKEN }} + GITHUB_TOKEN: "${{ secrets.RELEASE_TOKEN }}" diff --git a/.github/workflows/update_ci.yml b/.github/workflows/update_ci.yml index 450611e1a8..40126b4aba 100644 --- a/.github/workflows/update_ci.yml +++ b/.github/workflows/update_ci.yml @@ -7,60 +7,60 @@ --- -name: Galaxy CI Update +name: "Galaxy CI Update" on: workflow_dispatch: jobs: update: - runs-on: ubuntu-latest + runs-on: "ubuntu-latest" strategy: fail-fast: false steps: - - uses: actions/checkout@v3 + - uses: "actions/checkout@v4" with: - repository: pulp/plugin_template - path: plugin_template fetch-depth: 0 + repository: "pulp/plugin_template" + path: "plugin_template" - - uses: actions/setup-python@v4 + - uses: "actions/setup-python@v4" with: - python-version: "3.9" + python-version: "3.11" - - name: Install python dependencies + - name: "Install python dependencies" run: | echo ::group::PYDEPS pip install gitpython requests packaging jinja2 pyyaml echo ::endgroup:: - - name: Configure Git with ansible name and email + - name: "Configure Git with ansible name and email" run: | git config --global user.name 'ansible' git config --global user.email 'ansible-infra@redhat.com' - - uses: actions/checkout@v3 + - uses: "actions/checkout@v4" with: - path: galaxy_ng - ref: 'master' fetch-depth: 0 + path: "galaxy_ng" + ref: "master" - - name: Run update - working-directory: galaxy_ng + - name: "Run update" + working-directory: "galaxy_ng" run: | ../plugin_template/scripts/update_ci.sh - - name: Create Pull Request for CI files - uses: peter-evans/create-pull-request@v4 + - name: "Create Pull Request for CI files" + uses: "peter-evans/create-pull-request@v4" with: - token: ${{ secrets.RELEASE_TOKEN }} - path: galaxy_ng - committer: ansible - author: ansible - title: 'Update CI files for branch master' - body: 'No-Issue' - branch: 'update-ci/master' - base: 'master' + token: "${{ secrets.RELEASE_TOKEN }}" + path: "galaxy_ng" + committer: "ansible " + author: "ansible " + title: "Update CI files for branch master" + body: "" + branch: "update-ci/master" + base: "master" commit-message: | Update CI files diff --git a/CHANGES/.TEMPLATE.rst b/CHANGES/.TEMPLATE.rst index a5b57d6962..49c2305d7b 100644 --- a/CHANGES/.TEMPLATE.rst +++ b/CHANGES/.TEMPLATE.rst @@ -1,4 +1,12 @@ -{# TOWNCRIER TEMPLATE #} +{% if render_title %} +{% if versiondata.name %} +{{ versiondata.name }} {{ versiondata.version }} ({{ versiondata.date }}) +{{ top_underline * ((versiondata.name + versiondata.version + versiondata.date)|length + 4)}} +{% else %} +{{ versiondata.version }} ({{ versiondata.date }}) +{{ top_underline * ((versiondata.version + versiondata.date)|length + 3)}} +{% endif %} +{% endif %} {% for section, _ in sections.items() %} {% set underline = underlines[0] %}{% if section %}{{section}} {{ underline * section|length }}{% set underline = underlines[1] %} @@ -35,3 +43,5 @@ No significant changes. {% endfor %} ---- + + diff --git a/dev/common/tdd.py b/dev/common/tdd.py index 89b6baa778..3c27ecfaaf 100644 --- a/dev/common/tdd.py +++ b/dev/common/tdd.py @@ -31,7 +31,7 @@ def get_changed_files(pr_branch, target_branch="master"): raw = f.read() filenames = raw.split('\n') filenames = [x for x in filenames if x.startswith('---') or x.startswith('+++')] - filenames = [x.split(None, 1)[1] for x in filenames] + filenames = [x.split(None, 1)[1] for x in filenames if len(x.split(None, 1)) > 1] filenames = [x for x in filenames if not x.startswith('/dev/null')] filenames = [x.lstrip('a/') for x in filenames] filenames = [x.lstrip('b/') for x in filenames] diff --git a/functest_requirements.txt b/functest_requirements.txt index 5d9ad3336a..71c9289c9b 100644 --- a/functest_requirements.txt +++ b/functest_requirements.txt @@ -1,3 +1,3 @@ git+https://github.com/pulp/pulp-smash.git#egg=pulp-smash -pytest +pytest<8 orionutils diff --git a/lint_requirements.txt b/lint_requirements.txt index bcc93f361a..987eeb1fc2 100644 --- a/lint_requirements.txt +++ b/lint_requirements.txt @@ -8,4 +8,4 @@ # python packages handy for developers, but not required by pulp check-manifest flake8 - +yamllint diff --git a/setup.py b/setup.py index 9255f70ccf..c2fbfbcee2 100644 --- a/setup.py +++ b/setup.py @@ -166,6 +166,7 @@ def strip_package_name(spec): name=package_name, version=version, description="galaxy-ng plugin for the Pulp Project", + long_description="galaxy-ng plugin for the Pulp Project", license="GPLv2+", author="Red Hat, Inc.", author_email="info@ansible.com", diff --git a/template_config.yml b/template_config.yml index 535479595d..384e8d1058 100644 --- a/template_config.yml +++ b/template_config.yml @@ -1,33 +1,19 @@ # This config represents the latest values used when running the plugin-template. Any settings that # were not present before running plugin-template have been added with their default values. -# generated with plugin_template@2021.08.26-251-g45d399d +# generated with plugin_template@2021.08.26-316-g69ef1f6 -additional_repos: -- bindings: true - branch: 0.17.1 - name: pulp_ansible - org: pulp -- bindings: true - branch: 2.14.5 - name: pulp_container - org: pulp -- bindings: false - branch: v0.4.11 - name: galaxy-importer - org: ansible api_root: /api/galaxy/pulp/ black: false check_commit_message: false check_gettext: true check_manifest: true check_stray_pulpcore_imports: true +ci_base_image: ghcr.io/pulp/pulp-ci-centos9 ci_env: GITHUB_USER: ${{ github.event.pull_request.user.login }} ci_trigger: '{pull_request: {branches: [''**'']}, push: {branches: [''**'']}}' -ci_update_branches: [] ci_update_docs: false -ci_update_release_behavior: null cli_package: pulp-cli cli_repo: https://github.com/pulp/pulp-cli.git core_import_allowed: @@ -49,19 +35,16 @@ flake8_ignore: [] github_org: ansible issue_tracker: null kanban: false +latest_release_branch: null lint_requirements: false noissue_marker: No-Issue parallel_test_workers: 8 plugin_app_label: galaxy -plugin_camel: GalaxyNG -plugin_camel_short: Galaxy -plugin_caps: GALAXY_NG -plugin_caps_short: GALAXY -plugin_dash: galaxy-ng -plugin_dash_short: galaxy plugin_default_branch: master plugin_name: galaxy_ng -plugin_snake: galaxy_ng +plugins: +- app_label: galaxy + name: galaxy_ng post_job_template: name: update_manifest path: galaxy_ng/.github/post-job-template.yml.j2 @@ -86,16 +69,13 @@ pulp_settings_gcp: null pulp_settings_s3: null pulpprojectdotorg_key_id: null pydocstyle: true -pypi_username: null -python_version: '3.9' release_email: ansible-infra@redhat.com release_user: ansible -run_pulpcore_tests_for_plugins: false -single_commit_check: false stalebot: true stalebot_days_until_close: 30 stalebot_days_until_stale: 90 stalebot_limit_to_pulls: true +supported_release_branches: [] sync_ci: false test_azure: true test_cli: false @@ -105,6 +85,5 @@ test_lowerbounds: false test_performance: false test_reroute: false test_s3: true -update_github: false use_issue_template: false diff --git a/unittest_requirements.txt b/unittest_requirements.txt index 51c7f8078b..861b52c0b8 100644 --- a/unittest_requirements.txt +++ b/unittest_requirements.txt @@ -1,3 +1,4 @@ mock orionutils pytest-django +pytest<8