From fde28e49750f1532ad5d66a31d837e2339b58e39 Mon Sep 17 00:00:00 2001 From: Andrew Crosby Date: Thu, 12 May 2022 11:24:00 -0400 Subject: [PATCH] Update CI via plugin_template (#1262) No-Issue --- .ci/ansible/Containerfile.j2 | 5 ++++- .ci/ansible/inventory.yaml | 1 + .ci/ansible/settings.py.j2 | 20 ++++++++++++++++---- .ci/ansible/start_container.yaml | 7 +++++++ .ci/scripts/update_ci_branches.py | 2 +- .ci/scripts/upper_bound.py | 8 +++++--- .github/template_gitref | 2 +- .github/workflows/ci.yml | 2 +- .github/workflows/nightly.yml | 4 ++-- .github/workflows/release.yml | 2 +- .github/workflows/scripts/before_script.sh | 3 ++- .github/workflows/scripts/install.sh | 21 ++++++++++++--------- .github/workflows/scripts/script.sh | 17 ++++++----------- .github/workflows/update_ci.yml | 7 ++++++- template_config.yml | 10 ++++++---- 15 files changed, 71 insertions(+), 40 deletions(-) diff --git a/.ci/ansible/Containerfile.j2 b/.ci/ansible/Containerfile.j2 index b35d6b18b1..d61722b070 100644 --- a/.ci/ansible/Containerfile.j2 +++ b/.ci/ansible/Containerfile.j2 @@ -11,11 +11,14 @@ ADD {{ item.source }} {{ item.source }} # Hacking botocore (https://github.com/boto/botocore/pull/1990) RUN pip3 install \ +{%- if stream_test | default(false) -%} +{{ " " }}django-storages[sftp] +{%- endif -%} {%- if s3_test | default(false) -%} {{ " " }}django-storages[boto3] git+https://github.com/fabricio-aguiar/botocore.git@fix-100-continue {%- endif -%} {%- if azure_test | default(false) -%} -{{ " " }}django-storages[azure]>=1.12.2 "azure-storage-blob<12.10.0" +{{ " " }}django-storages[azure]>=1.12.2 {%- endif -%} {%- for item in plugins -%} {%- if item.name == "pulp-certguard" -%} diff --git a/.ci/ansible/inventory.yaml b/.ci/ansible/inventory.yaml index c1c9fb6cfb..3b85c5f405 100644 --- a/.ci/ansible/inventory.yaml +++ b/.ci/ansible/inventory.yaml @@ -6,6 +6,7 @@ all: pulp: pulp-fixtures: minio: + ci-sftp: vars: ansible_connection: docker ... diff --git a/.ci/ansible/settings.py.j2 b/.ci/ansible/settings.py.j2 index 8b86569fe3..1a76b5d55b 100644 --- a/.ci/ansible/settings.py.j2 +++ b/.ci/ansible/settings.py.j2 @@ -19,7 +19,21 @@ API_ROOT = {{ api_root | repr }} {% endfor %} {% endif %} +{% if stream_test | default(false) -%} +REDIRECT_TO_OBJECT_STORAGE = False +DEFAULT_FILE_STORAGE = "pulpcore.app.models.storage.PulpSFTPStorage" +MEDIA_ROOT = "" +SFTP_STORAGE_HOST = "ci-sftp" +SFTP_STORAGE_ROOT = "/storage/" +SFTP_STORAGE_PARAMS = { + "username": "foo", + "key_filename": "/keys/id_ed25519", +} +{%- endif %} + {% if s3_test | default(false) %} +DEFAULT_FILE_STORAGE = "storages.backends.s3boto3.S3Boto3Storage" +MEDIA_ROOT = "" AWS_ACCESS_KEY_ID = "{{ minio_access_key }}" AWS_SECRET_ACCESS_KEY = "{{ minio_secret_key }}" AWS_S3_REGION_NAME = "eu-central-1" @@ -28,12 +42,12 @@ S3_USE_SIGV4 = True AWS_S3_SIGNATURE_VERSION = "s3v4" AWS_STORAGE_BUCKET_NAME = "pulp3" AWS_S3_ENDPOINT_URL = "http://minio:9000" -DEFAULT_FILE_STORAGE = "storages.backends.s3boto3.S3Boto3Storage" AWS_DEFAULT_ACL = "@none None" -MEDIA_ROOT = "" {% endif %} {% if azure_test | default(false) %} +DEFAULT_FILE_STORAGE = "storages.backends.azure_storage.AzureStorage" +MEDIA_ROOT = "" AZURE_ACCOUNT_KEY = "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==" AZURE_ACCOUNT_NAME = "devstoreaccount1" AZURE_CONTAINER = "pulp-test" @@ -41,6 +55,4 @@ AZURE_LOCATION = "pulp3" AZURE_OVERWRITE_FILES = True AZURE_URL_EXPIRATION_SECS = 120 AZURE_CONNECTION_STRING = 'DefaultEndpointsProtocol={{ pulp_scheme }};AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint={{ pulp_scheme }}://ci-azurite:10000/devstoreaccount1;' -DEFAULT_FILE_STORAGE = "storages.backends.azure_storage.AzureStorage" -MEDIA_ROOT = "" {% endif %} diff --git a/.ci/ansible/start_container.yaml b/.ci/ansible/start_container.yaml index 9dedd90d56..09da135581 100644 --- a/.ci/ansible/start_container.yaml +++ b/.ci/ansible/start_container.yaml @@ -12,8 +12,15 @@ mode: "0755" loop: - settings + - ssh - ~/.config/pulp_smash + - name: Generate an OpenSSH keypair + community.crypto.openssh_keypair: + path: ssh/id_ed25519 + type: ed25519 + when: stream_test | default(false) + - name: "Generate Pulp Settings" template: src: settings.py.j2 diff --git a/.ci/scripts/update_ci_branches.py b/.ci/scripts/update_ci_branches.py index 68a0315d8d..44ac04d96f 100755 --- a/.ci/scripts/update_ci_branches.py +++ b/.ci/scripts/update_ci_branches.py @@ -21,7 +21,7 @@ } if not initial_branch or initial_branch not in branches: - starting = -2 + exit("Initial branch not found") else: starting = branches.index(initial_branch) diff --git a/.ci/scripts/upper_bound.py b/.ci/scripts/upper_bound.py index 58e91ddeb0..e9d87fb2bd 100755 --- a/.ci/scripts/upper_bound.py +++ b/.ci/scripts/upper_bound.py @@ -3,9 +3,11 @@ packages = [] with open("requirements.txt", "r") as fd: for line in fd.readlines(): - req = Requirement.parse(line) - if len(req.specs) < 2 and "~=" not in str(req.specs) and "==" not in str(req.specs): - packages.append(req.name) + if not line.startswith("#"): + req = Requirement.parse(line) + spec = str(req.specs) + if len(req.specs) < 2 and "~=" not in spec and "==" not in spec and "<" not in spec: + packages.append(req.name) if packages: raise RuntimeError( "The following packages are missing upper bound: {}".format(", ".join(packages)) diff --git a/.github/template_gitref b/.github/template_gitref index 99fed76df4..e77179c6cd 100644 --- a/.github/template_gitref +++ b/.github/template_gitref @@ -1 +1 @@ -2021.08.26-115-gf0baa22 +2021.08.26-129-gf780fda diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5fecfbcdf6..b4f2edb28e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -210,7 +210,7 @@ jobs: if: always() run: | echo "Need to debug? Please check: https://github.com/marketplace/actions/debugging-with-tmate" - http --timeout 30 --check-status --pretty format --print hb https://pulp/pulp/api/v3/status/ || true + http --timeout 30 --check-status --pretty format --print hb "https://pulp${PULP_API_ROOT}api/v3/status/" || true docker images || true docker ps -a || true docker logs pulp || true diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 25b26690d0..ddd8e135be 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -155,7 +155,7 @@ jobs: if: always() run: | echo "Need to debug? Please check: https://github.com/marketplace/actions/debugging-with-tmate" - http --timeout 30 --check-status --pretty format --print hb https://pulp/pulp/api/v3/status/ || true + http --timeout 30 --check-status --pretty format --print hb "https://pulp${PULP_API_ROOT}api/v3/status/" || true docker images || true docker ps -a || true docker logs pulp || true @@ -318,7 +318,7 @@ jobs: if: always() run: | echo "Need to debug? Please check: https://github.com/marketplace/actions/debugging-with-tmate" - http --timeout 30 --check-status --pretty format --print hb https://pulp/pulp/api/v3/status/ || true + http --timeout 30 --check-status --pretty format --print hb "https://pulp${PULP_API_ROOT}api/v3/status/" || true docker images || true docker ps -a || true docker logs pulp || true diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index f1cbf183aa..e0cbfd41fa 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -225,7 +225,7 @@ jobs: if: always() run: | echo "Need to debug? Please check: https://github.com/marketplace/actions/debugging-with-tmate" - http --timeout 30 --check-status --pretty format --print hb https://pulp/pulp/api/v3/status/ || true + http --timeout 30 --check-status --pretty format --print hb "https://pulp${PULP_API_ROOT}api/v3/status/" || true docker images || true docker ps -a || true docker logs pulp || true diff --git a/.github/workflows/scripts/before_script.sh b/.github/workflows/scripts/before_script.sh index 1c8a9518cd..1ab0a78d03 100755 --- a/.github/workflows/scripts/before_script.sh +++ b/.github/workflows/scripts/before_script.sh @@ -29,7 +29,8 @@ tail -v -n +1 .ci/ansible/vars/main.yaml echo "PULP CONFIG:" tail -v -n +1 .ci/ansible/settings/settings.* ~/.config/pulp_smash/settings.json -if [[ "$TEST" == 'pulp' || "$TEST" == 'performance' || "$TEST" == 'upgrade' || "$TEST" == 's3' || "$TEST" == 'azure' || "$TEST" == "plugin-from-pypi" || "$TEST" == "generate-bindings" ]]; then +SCENARIOS=("pulp" "performance" "upgrade" "azure" "s3" "stream" "plugin-from-pypi" "generate-bindings") +if [[ " ${SCENARIOS[*]} " =~ " ${TEST} " ]]; then # Many functional tests require these cmd_prefix dnf install -yq lsof which dnf-plugins-core fi diff --git a/.github/workflows/scripts/install.sh b/.github/workflows/scripts/install.sh index 6876dceae1..fa9fa05ea6 100755 --- a/.github/workflows/scripts/install.sh +++ b/.github/workflows/scripts/install.sh @@ -15,11 +15,14 @@ set -euv source .github/workflows/scripts/utils.sh +export PULP_API_ROOT="/pulp/" + if [[ "$TEST" = "docs" || "$TEST" = "publish" ]]; then pip install -r ../pulpcore/doc_requirements.txt pip install -r doc_requirements.txt fi +pip install -e ../pulpcore pip install -r functest_requirements.txt cd .ci/ansible/ @@ -68,11 +71,6 @@ plugins: source: pulp_container - name: galaxy-importer source: galaxy-importer -services: - - name: pulp - image: "pulp:${TAG}" - volumes: - - ./settings:/etc/pulp VARSYAML else cat >> vars/main.yaml << VARSYAML @@ -90,13 +88,17 @@ plugins: source: $GALAXY_IMPORTER - name: pulpcore source: ./pulpcore +VARSYAML +fi + +cat >> vars/main.yaml << VARSYAML services: - name: pulp image: "pulp:${TAG}" volumes: - ./settings:/etc/pulp + - ./ssh:/keys/ VARSYAML -fi cat >> vars/main.yaml << VARSYAML pulp_settings: {"allowed_export_paths": "/tmp", "allowed_import_paths": "/tmp", "galaxy_api_default_distribution_base_path": "published", "galaxy_enable_api_access_log": true, "galaxy_require_content_approval": false, "rh_entitlement_required": "insights"} @@ -123,7 +125,6 @@ if [ "$TEST" = "s3" ]; then sed -i -e '$a s3_test: true\ minio_access_key: "'$MINIO_ACCESS_KEY'"\ minio_secret_key: "'$MINIO_SECRET_KEY'"' vars/main.yaml - echo "PULP_API_ROOT=/rerouted/djnd/" >> "$GITHUB_ENV" export PULP_API_ROOT="/rerouted/djnd/" fi @@ -143,6 +144,8 @@ if [ "$TEST" = "azure" ]; then sed -i -e '$a azure_test: true' vars/main.yaml fi +echo "PULP_API_ROOT=${PULP_API_ROOT}" >> "$GITHUB_ENV" + if [ "${PULP_API_ROOT:-}" ]; then sed -i -e '$a api_root: "'"$PULP_API_ROOT"'"' vars/main.yaml fi @@ -156,7 +159,7 @@ sudo docker cp pulp:/etc/pulp/certs/pulp_webserver.crt /usr/local/share/ca-certi # Hack: adding pulp CA to certifi.where() CERTIFI=$(python -c 'import certifi; print(certifi.where())') cat /usr/local/share/ca-certificates/pulp_webserver.crt | sudo tee -a "$CERTIFI" > /dev/null -if [ "$TEST" = "azure" ]; then +if [[ "$TEST" = "azure" ]]; then cat /usr/local/share/ca-certificates/azcert.crt | sudo tee -a "$CERTIFI" > /dev/null fi @@ -168,7 +171,7 @@ cat "$CERTIFI" | sudo tee -a "$CERT" > /dev/null sudo update-ca-certificates echo ::endgroup:: -if [ "$TEST" = "azure" ]; then +if [[ "$TEST" = "azure" ]]; then AZCERTIFI=$(/opt/az/bin/python3 -c 'import certifi; print(certifi.where())') cat /usr/local/share/ca-certificates/azcert.crt >> $AZCERTIFI cat /usr/local/share/ca-certificates/azcert.crt | cmd_stdin_prefix tee -a /usr/local/lib/python3.8/site-packages/certifi/cacert.pem > /dev/null diff --git a/.github/workflows/scripts/script.sh b/.github/workflows/scripts/script.sh index 0c6dfdbd25..4bea4da748 100755 --- a/.github/workflows/scripts/script.sh +++ b/.github/workflows/scripts/script.sh @@ -38,11 +38,6 @@ if [[ "$TEST" = "docs" ]]; then tar -cvf docs.tar ./_build cd .. - echo "Validating OpenAPI schema..." - cat $PWD/.ci/scripts/schema.py | cmd_stdin_prefix bash -c "cat > /tmp/schema.py" - cmd_prefix bash -c "python3 /tmp/schema.py" - cmd_prefix bash -c "pulpcore-manager spectacular --file pulp_schema.yml --validate" - if [ -f $POST_DOCS_TEST ]; then source $POST_DOCS_TEST fi @@ -50,10 +45,7 @@ if [[ "$TEST" = "docs" ]]; then fi if [[ "${RELEASE_WORKFLOW:-false}" == "true" ]]; then - STATUS_ENDPOINT="${PULP_URL}/pulp/api/v3/status/" - if [ "${PULP_API_ROOT:-}" ]; then - STATUS_ENDPOINT="${PULP_URL}${PULP_API_ROOT}api/v3/status/" - fi + STATUS_ENDPOINT="${PULP_URL}${PULP_API_ROOT}api/v3/status/" echo $STATUS_ENDPOINT REPORTED_VERSION=$(http $STATUS_ENDPOINT | jq --arg plugin galaxy --arg legacy_plugin galaxy_ng -r '.versions[] | select(.component == $plugin or .component == $legacy_plugin) | .version') response=$(curl --write-out %{http_code} --silent --output /dev/null https://pypi.org/project/galaxy-ng/$REPORTED_VERSION/) @@ -120,14 +112,13 @@ cmd_prefix bash -c "django-admin makemigrations --check --dry-run" if [[ "$TEST" != "upgrade" ]]; then # Run unit tests. - cmd_prefix bash -c "PULP_DATABASES__default__USER=postgres pytest -v -r sx --color=yes --pyargs galaxy_ng.tests.unit" + cmd_prefix bash -c "PULP_DATABASES__default__USER=postgres pytest -v -r sx --color=yes -p no:pulpcore --pyargs galaxy_ng.tests.unit" fi # Run functional tests export PYTHONPATH=$REPO_ROOT/../pulp_ansible${PYTHONPATH:+:${PYTHONPATH}} export PYTHONPATH=$REPO_ROOT/../pulp_container${PYTHONPATH:+:${PYTHONPATH}} export PYTHONPATH=$REPO_ROOT/../galaxy-importer${PYTHONPATH:+:${PYTHONPATH}} -export PYTHONPATH=$REPO_ROOT/../pulpcore${PYTHONPATH:+:${PYTHONPATH}} export PYTHONPATH=$REPO_ROOT${PYTHONPATH:+:${PYTHONPATH}} @@ -148,9 +139,13 @@ else if [[ "$GITHUB_WORKFLOW" == "Galaxy Nightly CI/CD" ]]; then pytest -v -r sx --color=yes --suppress-no-test-exit-code --pyargs galaxy_ng.tests.functional -m parallel -n 8 pytest -v -r sx --color=yes --pyargs galaxy_ng.tests.functional -m "not parallel" + + else pytest -v -r sx --color=yes --suppress-no-test-exit-code --pyargs galaxy_ng.tests.functional -m "parallel and not nightly" -n 8 pytest -v -r sx --color=yes --pyargs galaxy_ng.tests.functional -m "not parallel and not nightly" + + fi fi diff --git a/.github/workflows/update_ci.yml b/.github/workflows/update_ci.yml index 670ff8c7e4..376ca84e14 100644 --- a/.github/workflows/update_ci.yml +++ b/.github/workflows/update_ci.yml @@ -10,6 +10,11 @@ name: CI Update on: workflow_dispatch: + inputs: + all_branches: + description: "Run on all branches" + default: 'no' + required: false jobs: update: @@ -45,7 +50,7 @@ jobs: run: echo ::set-output name=short_ref::${GITHUB_REF#refs/*/} - name: Dispatching - if: github.event_name == 'schedule' + if: github.event_name == 'schedule' || github.event.inputs.all_branches == 'yes' run: python .ci/scripts/update_ci_branches.py None env: GITHUB_TOKEN: ${{ secrets.RELEASE_TOKEN }} diff --git a/template_config.yml b/template_config.yml index 04858d0784..937a756038 100644 --- a/template_config.yml +++ b/template_config.yml @@ -1,7 +1,7 @@ # This config represents the latest values used when running the plugin-template. Any settings that # were not present before running plugin-template have been added with their default values. -# generated with plugin_template@2021.08.26-50-g7c31f64 +# generated with plugin_template@2021.08.26-129-gf780fda additional_repos: - bindings: true @@ -17,6 +17,7 @@ additional_repos: name: galaxy-importer org: ansible aiohttp_fixtures_origin: 172.18.0.1 +api_root: /pulp/ black: false check_commit_message: false check_gettext: true @@ -24,9 +25,8 @@ check_manifest: true check_openapi_schema: true check_stray_pulpcore_imports: true cherry_pick_automation: false -ci_env: { - GITHUB_USER: '${{ github.event.pull_request.user.login }}' -} +ci_env: + GITHUB_USER: ${{ github.event.pull_request.user.login }} ci_trigger: '{pull_request: {branches: [''*'']}, push: {branches: [''*'']}}' core_import_allowed: - pulpcore.app.*viewsets @@ -84,6 +84,7 @@ python_version: '3.8' redmine_project: null release_email: ansible-infra@redhat.com release_user: ansible +run_pulpcore_tests_for_plugins: false single_commit_check: false stable_branch: stable sync_ci: false @@ -96,6 +97,7 @@ test_performance: false test_released_plugin_with_next_pulpcore_release: false test_reroute: true test_s3: true +test_stream: false travis_addtl_services: [] travis_notifications: null update_github: false