From 92cd9cb6bea325531d5c552a9079f52a9f9c0930 Mon Sep 17 00:00:00 2001 From: Fabricio Aguiar Date: Thu, 23 Sep 2021 17:09:41 -0300 Subject: [PATCH] Run pulp-ansible functional tests (#937) Issue: AAH-873 --- .ci/ansible/Containerfile.j2 | 3 ++ .ci/ansible/settings.py.j2 | 13 ++++++ .ci/ansible/smash-config.json | 8 ++-- .ci/assets/nginx/nginx.conf | 10 ++++ .github/template_gitref | 2 +- .github/workflows/ci.yml | 3 +- .github/workflows/nightly.yml | 5 +- .github/workflows/release.yml | 3 +- .github/workflows/scripts/before_script.sh | 2 +- .github/workflows/scripts/func_test_script.sh | 13 ++++++ .github/workflows/scripts/install.sh | 46 ++++++++++++++++++- .../scripts/install_python_client.sh | 2 +- .../workflows/scripts/install_ruby_client.sh | 2 +- .../workflows/scripts/post_before_script.sh | 9 +++- .../workflows/scripts/pre_before_install.sh | 10 ++++ .github/workflows/scripts/script.sh | 2 +- CHANGES/873.misc | 1 + galaxy_ng/app/api/v3/viewsets/collection.py | 7 +++ galaxy_ng/tests/functional/utils.py | 2 +- template_config.yml | 6 ++- 20 files changed, 130 insertions(+), 19 deletions(-) create mode 100644 .ci/assets/nginx/nginx.conf create mode 100755 .github/workflows/scripts/func_test_script.sh create mode 100755 .github/workflows/scripts/pre_before_install.sh create mode 100644 CHANGES/873.misc diff --git a/.ci/ansible/Containerfile.j2 b/.ci/ansible/Containerfile.j2 index eaff66a86e..4483eab8a0 100644 --- a/.ci/ansible/Containerfile.j2 +++ b/.ci/ansible/Containerfile.j2 @@ -14,6 +14,9 @@ RUN pip3 install \ {%- if s3_test | default(false) -%} {{ " " }}django-storages[boto3] git+https://github.com/fabricio-aguiar/botocore.git@fix-100-continue {%- endif -%} +{%- if azure_test | default(false) -%} +{{ " " }}django-storages[azure] +{%- endif -%} {%- for item in plugins -%} {%- if item.name == "pulp-certguard" -%} {{ " " }}python-dateutil rhsm diff --git a/.ci/ansible/settings.py.j2 b/.ci/ansible/settings.py.j2 index e0e040a7ac..9e62d935bb 100644 --- a/.ci/ansible/settings.py.j2 +++ b/.ci/ansible/settings.py.j2 @@ -23,4 +23,17 @@ AWS_STORAGE_BUCKET_NAME = "pulp3" AWS_S3_ENDPOINT_URL = "http://minio:9000" DEFAULT_FILE_STORAGE = "storages.backends.s3boto3.S3Boto3Storage" AWS_DEFAULT_ACL = "@none None" +MEDIA_ROOT = "" +{% endif %} + +{% if azure_test | default(false) %} +AZURE_ACCOUNT_KEY = "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==" +AZURE_ACCOUNT_NAME = "devstoreaccount1" +AZURE_CONTAINER = "pulp-test" +AZURE_LOCATION = "pulp3" +AZURE_OVERWRITE_FILES = True +AZURE_URL_EXPIRATION_SECS = 120 +AZURE_CONNECTION_STRING = 'DefaultEndpointsProtocol={{ pulp_scheme }};AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint={{ pulp_scheme }}://pulp-azurite:10000/devstoreaccount1;' +DEFAULT_FILE_STORAGE = "storages.backends.azure_storage.AzureStorage" +MEDIA_ROOT = "" {% endif %} diff --git a/.ci/ansible/smash-config.json b/.ci/ansible/smash-config.json index 9af43820bf..308fe38b9d 100644 --- a/.ci/ansible/smash-config.json +++ b/.ci/ansible/smash-config.json @@ -12,13 +12,13 @@ "hostname": "pulp", "roles": { "api": { - "port": 80, - "scheme": "http", + "port": 443, + "scheme": "https", "service": "nginx" }, "content": { - "port": 80, - "scheme": "http", + "port": 443, + "scheme": "https", "service": "pulp_content_app" }, "pulp resource manager": {}, diff --git a/.ci/assets/nginx/nginx.conf b/.ci/assets/nginx/nginx.conf new file mode 100644 index 0000000000..05f6e57647 --- /dev/null +++ b/.ci/assets/nginx/nginx.conf @@ -0,0 +1,10 @@ +location /ui/ { + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header Host $http_host; + rewrite /ui* /static/galaxy_ng/index.html break; + # we don't want nginx trying to do something clever with + # redirects, we set the Host: header above already. + proxy_redirect off; + proxy_pass http://pulp-api/static/galaxy_ng/index.html; +} diff --git a/.github/template_gitref b/.github/template_gitref index 9327dade35..484651eeab 100644 --- a/.github/template_gitref +++ b/.github/template_gitref @@ -1 +1 @@ -2021.08.26-15-g314a3c1 +2021.08.26-19-g88bcdd3 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7075e147c0..71a2ceab4a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -82,6 +82,7 @@ jobs: matrix: env: - TEST: pulp + - TEST: azure - TEST: s3 steps: @@ -180,7 +181,7 @@ jobs: if: failure() run: | echo "Need to debug? Please check: https://github.com/marketplace/actions/debugging-with-tmate" - http --timeout 30 --check-status --pretty format --print hb http://pulp/pulp/api/v3/status/ || true + http --timeout 30 --check-status --pretty format --print hb https://pulp/pulp/api/v3/status/ || true docker images || true docker ps -a || true docker logs pulp || true diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 46981e19d6..03c66bbac1 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -22,6 +22,7 @@ jobs: matrix: env: - TEST: pulp + - TEST: azure - TEST: s3 steps: @@ -146,7 +147,7 @@ jobs: if: failure() run: | echo "Need to debug? Please check: https://github.com/marketplace/actions/debugging-with-tmate" - http --timeout 30 --check-status --pretty format --print hb http://pulp/pulp/api/v3/status/ || true + http --timeout 30 --check-status --pretty format --print hb https://pulp/pulp/api/v3/status/ || true docker images || true docker ps -a || true docker logs pulp || true @@ -260,7 +261,7 @@ jobs: if: failure() run: | echo "Need to debug? Please check: https://github.com/marketplace/actions/debugging-with-tmate" - http --timeout 30 --check-status --pretty format --print hb http://pulp/pulp/api/v3/status/ || true + http --timeout 30 --check-status --pretty format --print hb https://pulp/pulp/api/v3/status/ || true docker images || true docker ps -a || true docker logs pulp || true diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 6a6c5e4c4c..7ef8e3d931 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -78,6 +78,7 @@ jobs: matrix: env: - TEST: pulp + - TEST: azure - TEST: s3 steps: @@ -214,7 +215,7 @@ jobs: if: failure() run: | echo "Need to debug? Please check: https://github.com/marketplace/actions/debugging-with-tmate" - http --timeout 30 --check-status --pretty format --print hb http://pulp/pulp/api/v3/status/ || true + http --timeout 30 --check-status --pretty format --print hb https://pulp/pulp/api/v3/status/ || true docker images || true docker ps -a || true docker logs pulp || true diff --git a/.github/workflows/scripts/before_script.sh b/.github/workflows/scripts/before_script.sh index 95cef0d743..60e19333cd 100755 --- a/.github/workflows/scripts/before_script.sh +++ b/.github/workflows/scripts/before_script.sh @@ -29,7 +29,7 @@ tail -v -n +1 .ci/ansible/vars/main.yaml echo "PULP CONFIG:" tail -v -n +1 .ci/ansible/settings/settings.* ~/.config/pulp_smash/settings.json -if [[ "$TEST" == 'pulp' || "$TEST" == 'performance' || "$TEST" == 'upgrade' || "$TEST" == 's3' || "$TEST" == "plugin-from-pypi" ]]; then +if [[ "$TEST" == 'pulp' || "$TEST" == 'performance' || "$TEST" == 'upgrade' || "$TEST" == 's3' || "$TEST" == 'azure' || "$TEST" == "plugin-from-pypi" ]]; then # Many functional tests require these cmd_prefix dnf install -yq lsof which dnf-plugins-core fi diff --git a/.github/workflows/scripts/func_test_script.sh b/.github/workflows/scripts/func_test_script.sh new file mode 100755 index 0000000000..6a619985d6 --- /dev/null +++ b/.github/workflows/scripts/func_test_script.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +# coding=utf-8 + +set -mveuo pipefail + +pip install ../pulp_ansible +pip install -r ../pulp_ansible/functest_requirements.txt + +pytest -v -r sx --color=yes --pyargs galaxy_ng.tests.functional + +cd ../pulp_ansible + +pytest -v -r sx --color=yes --pyargs pulp_ansible.tests.functional.cli diff --git a/.github/workflows/scripts/install.sh b/.github/workflows/scripts/install.sh index ceae2d2b40..a0bf2011fb 100755 --- a/.github/workflows/scripts/install.sh +++ b/.github/workflows/scripts/install.sh @@ -100,9 +100,9 @@ fi cat >> vars/main.yaml << VARSYAML pulp_settings: {"allowed_export_paths": "/tmp", "allowed_import_paths": "/tmp", "rh_entitlement_required": "insights"} -pulp_scheme: http +pulp_scheme: https -pulp_container_tag: latest +pulp_container_tag: https VARSYAML @@ -121,8 +121,50 @@ minio_access_key: "'$MINIO_ACCESS_KEY'"\ minio_secret_key: "'$MINIO_SECRET_KEY'"' vars/main.yaml fi +if [ "$TEST" = "azure" ]; then + mkdir -p azurite + cd azurite + openssl req -newkey rsa:2048 -x509 -nodes -keyout azkey.pem -new -out azcert.pem -sha256 -days 365 -addext "subjectAltName=DNS:pulp-azurite" -subj "/C=CO/ST=ST/L=LO/O=OR/OU=OU/CN=CN" + sudo cp azcert.pem /usr/local/share/ca-certificates/azcert.crt + sudo dpkg-reconfigure ca-certificates + cd .. + sed -i -e '/^services:/a \ + - name: pulp-azurite\ + image: mcr.microsoft.com/azure-storage/azurite\ + volumes:\ + - ./azurite:/etc/pulp\ + command: "azurite-blob --blobHost 0.0.0.0 --cert /etc/pulp/azcert.pem --key /etc/pulp/azkey.pem"' vars/main.yaml + sed -i -e '$a azure_test: true' vars/main.yaml +fi + ansible-playbook build_container.yaml ansible-playbook start_container.yaml +echo ::group::SSL +# Copy pulp CA +sudo docker cp pulp:/etc/pulp/certs/pulp_webserver.crt /usr/local/share/ca-certificates/pulp_webserver.crt + +# Hack: adding pulp CA to certifi.where() +CERTIFI=$(python -c 'import certifi; print(certifi.where())') +cat /usr/local/share/ca-certificates/pulp_webserver.crt | sudo tee -a $CERTIFI +if [ "$TEST" = "azure" ]; then + cat /usr/local/share/ca-certificates/azcert.crt | sudo tee -a $CERTIFI +fi + +# Hack: adding pulp CA to default CA file +CERT=$(python -c 'import ssl; print(ssl.get_default_verify_paths().openssl_cafile)') +cat $CERTIFI | sudo tee -a $CERT + +# Updating certs +sudo update-ca-certificates +echo ::endgroup:: + +if [ "$TEST" = "azure" ]; then + cat /usr/local/share/ca-certificates/azcert.crt >> /opt/az/lib/python3.6/site-packages/certifi/cacert.pem + cat /usr/local/share/ca-certificates/azcert.crt | docker exec -i pulp bash -c "cat >> /usr/local/lib/python3.8/site-packages/certifi/cacert.pem" + cat /usr/local/share/ca-certificates/azcert.crt | docker exec -i pulp bash -c "cat >> /etc/pki/tls/cert.pem" + AZURE_STORAGE_CONNECTION_STRING='DefaultEndpointsProtocol=https;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=https://pulp-azurite:10000/devstoreaccount1;' + az storage container create --name pulp-test --connection-string $AZURE_STORAGE_CONNECTION_STRING +fi echo ::group::PIP_LIST cmd_prefix bash -c "pip3 list && pip3 install pipdeptree && pipdeptree" diff --git a/.github/workflows/scripts/install_python_client.sh b/.github/workflows/scripts/install_python_client.sh index 202f84e112..6c765cfa93 100755 --- a/.github/workflows/scripts/install_python_client.sh +++ b/.github/workflows/scripts/install_python_client.sh @@ -9,7 +9,7 @@ set -euv -export PULP_URL="${PULP_URL:-http://pulp}" +export PULP_URL="${PULP_URL:-https://pulp}" # make sure this script runs at the repo root cd "$(dirname "$(realpath -e "$0")")"/../../.. diff --git a/.github/workflows/scripts/install_ruby_client.sh b/.github/workflows/scripts/install_ruby_client.sh index 11923dcda9..6bbf16d76c 100755 --- a/.github/workflows/scripts/install_ruby_client.sh +++ b/.github/workflows/scripts/install_ruby_client.sh @@ -12,7 +12,7 @@ set -euv # make sure this script runs at the repo root cd "$(dirname "$(realpath -e "$0")")"/../../.. -export PULP_URL="${PULP_URL:-http://pulp}" +export PULP_URL="${PULP_URL:-https://pulp}" export REPORTED_VERSION=$(http $PULP_URL/pulp/api/v3/status/ | jq --arg plugin galaxy --arg legacy_plugin galaxy_ng -r '.versions[] | select(.component == $plugin or .component == $legacy_plugin) | .version') export DESCRIPTION="$(git describe --all --exact-match `git rev-parse HEAD`)" diff --git a/.github/workflows/scripts/post_before_script.sh b/.github/workflows/scripts/post_before_script.sh index 3cf0f8e8e6..4e1848e613 100644 --- a/.github/workflows/scripts/post_before_script.sh +++ b/.github/workflows/scripts/post_before_script.sh @@ -2,4 +2,11 @@ set -mveuo pipefail source .github/workflows/scripts/utils.sh -cmd_prefix bash -c "django-admin compilemessages" \ No newline at end of file +cmd_prefix bash -c "django-admin compilemessages" + +echo "machine pulp +login admin +password password +" > ~/.netrc + +chmod og-rw ~/.netrc diff --git a/.github/workflows/scripts/pre_before_install.sh b/.github/workflows/scripts/pre_before_install.sh new file mode 100755 index 0000000000..524fd2567c --- /dev/null +++ b/.github/workflows/scripts/pre_before_install.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash + +# make sure this script runs at the repo root +cd "$(dirname "$(realpath -e "$0")")"/../../.. + +set -mveuo pipefail + +# Replacing ngix conf +rm galaxy_ng/app/webserver_snippets/nginx.conf +mv .ci/assets/nginx/nginx.conf galaxy_ng/app/webserver_snippets/nginx.conf diff --git a/.github/workflows/scripts/script.sh b/.github/workflows/scripts/script.sh index 70b3acc676..cdac942bae 100755 --- a/.github/workflows/scripts/script.sh +++ b/.github/workflows/scripts/script.sh @@ -26,7 +26,7 @@ export FUNC_TEST_SCRIPT=$PWD/.github/workflows/scripts/func_test_script.sh export DJANGO_SETTINGS_MODULE=pulpcore.app.settings export PULP_SETTINGS=$PWD/.ci/ansible/settings/settings.py -export PULP_URL="http://pulp" +export PULP_URL="https://pulp" if [[ "$TEST" = "docs" ]]; then cd docs diff --git a/CHANGES/873.misc b/CHANGES/873.misc new file mode 100644 index 0000000000..dd2e529664 --- /dev/null +++ b/CHANGES/873.misc @@ -0,0 +1 @@ +Run pulp-ansible functional tests diff --git a/galaxy_ng/app/api/v3/viewsets/collection.py b/galaxy_ng/app/api/v3/viewsets/collection.py index 79315ea1e5..ee86aa8fd5 100644 --- a/galaxy_ng/app/api/v3/viewsets/collection.py +++ b/galaxy_ng/app/api/v3/viewsets/collection.py @@ -291,6 +291,13 @@ def _check_path_matches_expected_repo(path, filename_ns): % INBOUND_REPO_NAME_FORMAT.format(namespace_name=filename_ns) ) + @extend_schema( + description="Create an artifact and trigger an asynchronous task to create " + "Collection content from it.", + summary="Upload a collection", + request=CollectionUploadSerializer, + responses={202: AsyncOperationResponseSerializer}, + ) def create(self, request, *args, **kwargs): data = self._get_data(request) filename = data['filename'] diff --git a/galaxy_ng/tests/functional/utils.py b/galaxy_ng/tests/functional/utils.py index 00062c8cb4..251b0f6277 100644 --- a/galaxy_ng/tests/functional/utils.py +++ b/galaxy_ng/tests/functional/utils.py @@ -43,7 +43,7 @@ def set_up_module(): """Skip tests Pulp 3 isn't under test or if galaxy_ng isn't installed.""" require_pulp_3(SkipTest) - require_pulp_plugins({"galaxy_ng"}, SkipTest) + require_pulp_plugins({"galaxy"}, SkipTest) def gen_galaxy_client(): diff --git a/template_config.yml b/template_config.yml index 576129887f..7ee451688b 100644 --- a/template_config.yml +++ b/template_config.yml @@ -1,7 +1,7 @@ # This config represents the latest values used when running the plugin-template. Any settings that # were not present before running plugin-template have been added with their default values. -# generated with plugin_template@2021.04.08-99-gd8b54c6 +# generated with plugin_template@2021.08.26-19-g88bcdd3 additional_repos: - bindings: true @@ -58,7 +58,7 @@ pre_job_template: name: check_commit path: galaxy_ng/.github/pre-job-template.yml.j2 publish_docs_to_pulpprojectdotorg: false -pulp_scheme: http +pulp_scheme: https pulp_settings: allowed_export_paths: /tmp allowed_import_paths: /tmp @@ -76,6 +76,8 @@ release_user: ansible single_commit_check: false stable_branch: stable sync_ci: true +tasking_allow_async_unsafe: true +test_azure: true test_bindings: false test_cli: false test_performance: false