Run pulp-ansible functional tests (#937)

Issue: AAH-873
ansible · Sep 23, 2021 · 92cd9cb · 92cd9cb
1 parent 076d526
commit 92cd9cb
Show file tree

Hide file tree

Showing 20 changed files with 130 additions and 19 deletions.
diff --git a/.ci/ansible/Containerfile.j2 b/.ci/ansible/Containerfile.j2
@@ -14,6 +14,9 @@ RUN pip3 install \
 {%- if s3_test | default(false) -%}
 {{ " " }}django-storages[boto3] git+https://github.com/fabricio-aguiar/botocore.git@fix-100-continue
 {%- endif -%}
+{%- if azure_test | default(false) -%}
+{{ " " }}django-storages[azure]
+{%- endif -%}
 {%- for item in plugins -%}
 {%- if item.name == "pulp-certguard" -%}
 {{ " " }}python-dateutil rhsm

diff --git a/.ci/ansible/settings.py.j2 b/.ci/ansible/settings.py.j2
@@ -23,4 +23,17 @@ AWS_STORAGE_BUCKET_NAME = "pulp3"
 AWS_S3_ENDPOINT_URL = "http://minio:9000"
 DEFAULT_FILE_STORAGE = "storages.backends.s3boto3.S3Boto3Storage"
 AWS_DEFAULT_ACL = "@none None"
+MEDIA_ROOT = ""
+{% endif %}
+
+{% if azure_test | default(false) %}
+AZURE_ACCOUNT_KEY = "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw=="
+AZURE_ACCOUNT_NAME = "devstoreaccount1"
+AZURE_CONTAINER =  "pulp-test"
+AZURE_LOCATION = "pulp3"
+AZURE_OVERWRITE_FILES = True
+AZURE_URL_EXPIRATION_SECS = 120
+AZURE_CONNECTION_STRING = 'DefaultEndpointsProtocol={{ pulp_scheme }};AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint={{ pulp_scheme }}://pulp-azurite:10000/devstoreaccount1;'
+DEFAULT_FILE_STORAGE = "storages.backends.azure_storage.AzureStorage"
+MEDIA_ROOT = ""
 {% endif %}
diff --git a/.ci/ansible/smash-config.json b/.ci/ansible/smash-config.json
@@ -12,13 +12,13 @@
       "hostname": "pulp",
       "roles": {
         "api": {
-          "port": 80,
-          "scheme": "http",
+          "port": 443,
+          "scheme": "https",
           "service": "nginx"
         },
         "content": {
-          "port": 80,
-          "scheme": "http",
+          "port": 443,
+          "scheme": "https",
           "service": "pulp_content_app"
         },
         "pulp resource manager": {},

diff --git a/.ci/assets/nginx/nginx.conf b/.ci/assets/nginx/nginx.conf
@@ -0,0 +1,10 @@
+location /ui/ {
+    proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+    proxy_set_header X-Forwarded-Proto $scheme;
+    proxy_set_header Host $http_host;
+    rewrite /ui* /static/galaxy_ng/index.html break;
+    # we don't want nginx trying to do something clever with
+    # redirects, we set the Host: header above already.
+    proxy_redirect off;
+    proxy_pass http://pulp-api/static/galaxy_ng/index.html;
+}
diff --git a/.github/template_gitref b/.github/template_gitref
@@ -1 +1 @@
-2021.08.26-15-g314a3c1
+2021.08.26-19-g88bcdd3
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -82,6 +82,7 @@ jobs:
       matrix:
         env:
           - TEST: pulp
+          - TEST: azure
           - TEST: s3
 
     steps:
@@ -180,7 +181,7 @@ jobs:
         if: failure()
         run: |
           echo "Need to debug? Please check: https://github.com/marketplace/actions/debugging-with-tmate"
-          http --timeout 30 --check-status --pretty format --print hb http://pulp/pulp/api/v3/status/ || true
+          http --timeout 30 --check-status --pretty format --print hb https://pulp/pulp/api/v3/status/ || true
           docker images || true
           docker ps -a || true
           docker logs pulp || true

diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml
@@ -22,6 +22,7 @@ jobs:
       matrix:
         env:
           - TEST: pulp
+          - TEST: azure
           - TEST: s3
 
     steps:
@@ -146,7 +147,7 @@ jobs:
         if: failure()
         run: |
           echo "Need to debug? Please check: https://github.com/marketplace/actions/debugging-with-tmate"
-          http --timeout 30 --check-status --pretty format --print hb http://pulp/pulp/api/v3/status/ || true
+          http --timeout 30 --check-status --pretty format --print hb https://pulp/pulp/api/v3/status/ || true
           docker images || true
           docker ps -a || true
           docker logs pulp || true
@@ -260,7 +261,7 @@ jobs:
         if: failure()
         run: |
           echo "Need to debug? Please check: https://github.com/marketplace/actions/debugging-with-tmate"
-          http --timeout 30 --check-status --pretty format --print hb http://pulp/pulp/api/v3/status/ || true
+          http --timeout 30 --check-status --pretty format --print hb https://pulp/pulp/api/v3/status/ || true
           docker images || true
           docker ps -a || true
           docker logs pulp || true

diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -78,6 +78,7 @@ jobs:
       matrix:
         env:
           - TEST: pulp
+          - TEST: azure
           - TEST: s3
 
     steps:
@@ -214,7 +215,7 @@ jobs:
         if: failure()
         run: |
           echo "Need to debug? Please check: https://github.com/marketplace/actions/debugging-with-tmate"
-          http --timeout 30 --check-status --pretty format --print hb http://pulp/pulp/api/v3/status/ || true
+          http --timeout 30 --check-status --pretty format --print hb https://pulp/pulp/api/v3/status/ || true
           docker images || true
           docker ps -a || true
           docker logs pulp || true

diff --git a/.github/workflows/scripts/before_script.sh b/.github/workflows/scripts/before_script.sh
@@ -29,7 +29,7 @@ tail -v -n +1 .ci/ansible/vars/main.yaml
 echo "PULP CONFIG:"
 tail -v -n +1 .ci/ansible/settings/settings.* ~/.config/pulp_smash/settings.json
 
-if [[ "$TEST" == 'pulp' || "$TEST" == 'performance' || "$TEST" == 'upgrade' || "$TEST" == 's3' || "$TEST" == "plugin-from-pypi" ]]; then
+if [[ "$TEST" == 'pulp' || "$TEST" == 'performance' || "$TEST" == 'upgrade' || "$TEST" == 's3' || "$TEST" == 'azure' || "$TEST" == "plugin-from-pypi" ]]; then
   # Many functional tests require these
   cmd_prefix dnf install -yq lsof which dnf-plugins-core
 fi

diff --git a/.github/workflows/scripts/func_test_script.sh b/.github/workflows/scripts/func_test_script.sh
@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+# coding=utf-8
+
+set -mveuo pipefail
+
+pip install ../pulp_ansible
+pip install -r ../pulp_ansible/functest_requirements.txt
+
+pytest -v -r sx --color=yes --pyargs galaxy_ng.tests.functional
+
+cd ../pulp_ansible
+
+pytest -v -r sx --color=yes --pyargs pulp_ansible.tests.functional.cli
diff --git a/.github/workflows/scripts/install.sh b/.github/workflows/scripts/install.sh
@@ -100,9 +100,9 @@ fi
 
 cat >> vars/main.yaml << VARSYAML
 pulp_settings: {"allowed_export_paths": "/tmp", "allowed_import_paths": "/tmp", "rh_entitlement_required": "insights"}
-pulp_scheme: http
+pulp_scheme: https
 
-pulp_container_tag: latest
+pulp_container_tag: https
 
 VARSYAML
 
@@ -121,8 +121,50 @@ minio_access_key: "'$MINIO_ACCESS_KEY'"\
 minio_secret_key: "'$MINIO_SECRET_KEY'"' vars/main.yaml
 fi
 
+if [ "$TEST" = "azure" ]; then
+  mkdir -p azurite
+  cd azurite
+  openssl req -newkey rsa:2048 -x509 -nodes -keyout azkey.pem -new -out azcert.pem -sha256 -days 365 -addext "subjectAltName=DNS:pulp-azurite" -subj "/C=CO/ST=ST/L=LO/O=OR/OU=OU/CN=CN"
+  sudo cp azcert.pem /usr/local/share/ca-certificates/azcert.crt
+  sudo dpkg-reconfigure ca-certificates
+  cd ..
+  sed -i -e '/^services:/a \
+  - name: pulp-azurite\
+    image: mcr.microsoft.com/azure-storage/azurite\
+    volumes:\
+      - ./azurite:/etc/pulp\
+    command: "azurite-blob --blobHost 0.0.0.0 --cert /etc/pulp/azcert.pem --key /etc/pulp/azkey.pem"' vars/main.yaml
+  sed -i -e '$a azure_test: true' vars/main.yaml
+fi
+
 ansible-playbook build_container.yaml
 ansible-playbook start_container.yaml
+echo ::group::SSL
+# Copy pulp CA
+sudo docker cp pulp:/etc/pulp/certs/pulp_webserver.crt /usr/local/share/ca-certificates/pulp_webserver.crt
+
+# Hack: adding pulp CA to certifi.where()
+CERTIFI=$(python -c 'import certifi; print(certifi.where())')
+cat /usr/local/share/ca-certificates/pulp_webserver.crt | sudo tee -a $CERTIFI
+if [ "$TEST" = "azure" ]; then
+  cat /usr/local/share/ca-certificates/azcert.crt | sudo tee -a $CERTIFI
+fi
+
+# Hack: adding pulp CA to default CA file
+CERT=$(python -c 'import ssl; print(ssl.get_default_verify_paths().openssl_cafile)')
+cat $CERTIFI | sudo tee -a $CERT
+
+# Updating certs
+sudo update-ca-certificates
+echo ::endgroup::
+
+if [ "$TEST" = "azure" ]; then
+  cat /usr/local/share/ca-certificates/azcert.crt >> /opt/az/lib/python3.6/site-packages/certifi/cacert.pem
+  cat /usr/local/share/ca-certificates/azcert.crt | docker exec -i pulp bash -c "cat >> /usr/local/lib/python3.8/site-packages/certifi/cacert.pem"
+  cat /usr/local/share/ca-certificates/azcert.crt | docker exec -i pulp bash -c "cat >> /etc/pki/tls/cert.pem"
+  AZURE_STORAGE_CONNECTION_STRING='DefaultEndpointsProtocol=https;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=https://pulp-azurite:10000/devstoreaccount1;'
+  az storage container create --name pulp-test --connection-string $AZURE_STORAGE_CONNECTION_STRING
+fi
 
 echo ::group::PIP_LIST
 cmd_prefix bash -c "pip3 list && pip3 install pipdeptree && pipdeptree"

diff --git a/.github/workflows/scripts/install_python_client.sh b/.github/workflows/scripts/install_python_client.sh
@@ -9,7 +9,7 @@
 
 set -euv
 
-export PULP_URL="${PULP_URL:-http://pulp}"
+export PULP_URL="${PULP_URL:-https://pulp}"
 
 # make sure this script runs at the repo root
 cd "$(dirname "$(realpath -e "$0")")"/../../..

diff --git a/.github/workflows/scripts/install_ruby_client.sh b/.github/workflows/scripts/install_ruby_client.sh
@@ -12,7 +12,7 @@ set -euv
 # make sure this script runs at the repo root
 cd "$(dirname "$(realpath -e "$0")")"/../../..
 
-export PULP_URL="${PULP_URL:-http://pulp}"
+export PULP_URL="${PULP_URL:-https://pulp}"
 
 export REPORTED_VERSION=$(http $PULP_URL/pulp/api/v3/status/ | jq --arg plugin galaxy --arg legacy_plugin galaxy_ng -r '.versions[] | select(.component == $plugin or .component == $legacy_plugin) | .version')
 export DESCRIPTION="$(git describe --all --exact-match `git rev-parse HEAD`)"

diff --git a/.github/workflows/scripts/post_before_script.sh b/.github/workflows/scripts/post_before_script.sh
@@ -2,4 +2,11 @@
 
 set -mveuo pipefail
 source .github/workflows/scripts/utils.sh
-cmd_prefix bash -c "django-admin compilemessages"
+cmd_prefix bash -c "django-admin compilemessages"
+
+echo "machine pulp
+login admin
+password password
+" > ~/.netrc
+
+chmod og-rw ~/.netrc
diff --git a/.github/workflows/scripts/pre_before_install.sh b/.github/workflows/scripts/pre_before_install.sh
@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+
+# make sure this script runs at the repo root
+cd "$(dirname "$(realpath -e "$0")")"/../../..
+
+set -mveuo pipefail
+
+# Replacing ngix conf
+rm galaxy_ng/app/webserver_snippets/nginx.conf
+mv .ci/assets/nginx/nginx.conf galaxy_ng/app/webserver_snippets/nginx.conf
diff --git a/.github/workflows/scripts/script.sh b/.github/workflows/scripts/script.sh
@@ -26,7 +26,7 @@ export FUNC_TEST_SCRIPT=$PWD/.github/workflows/scripts/func_test_script.sh
 export DJANGO_SETTINGS_MODULE=pulpcore.app.settings
 export PULP_SETTINGS=$PWD/.ci/ansible/settings/settings.py
 
-export PULP_URL="http://pulp"
+export PULP_URL="https://pulp"
 
 if [[ "$TEST" = "docs" ]]; then
   cd docs

diff --git a/CHANGES/873.misc b/CHANGES/873.misc
@@ -0,0 +1 @@
+Run pulp-ansible functional tests
diff --git a/galaxy_ng/app/api/v3/viewsets/collection.py b/galaxy_ng/app/api/v3/viewsets/collection.py
@@ -291,6 +291,13 @@ def _check_path_matches_expected_repo(path, filename_ns):
             % INBOUND_REPO_NAME_FORMAT.format(namespace_name=filename_ns)
         )
 
+    @extend_schema(
+        description="Create an artifact and trigger an asynchronous task to create "
+        "Collection content from it.",
+        summary="Upload a collection",
+        request=CollectionUploadSerializer,
+        responses={202: AsyncOperationResponseSerializer},
+    )
     def create(self, request, *args, **kwargs):
         data = self._get_data(request)
         filename = data['filename']

diff --git a/galaxy_ng/tests/functional/utils.py b/galaxy_ng/tests/functional/utils.py
@@ -43,7 +43,7 @@
 def set_up_module():
     """Skip tests Pulp 3 isn't under test or if galaxy_ng isn't installed."""
     require_pulp_3(SkipTest)
-    require_pulp_plugins({"galaxy_ng"}, SkipTest)
+    require_pulp_plugins({"galaxy"}, SkipTest)
 
 
 def gen_galaxy_client():

diff --git a/template_config.yml b/template_config.yml
@@ -1,7 +1,7 @@
 # This config represents the latest values used when running the plugin-template. Any settings that
 # were not present before running plugin-template have been added with their default values.
 
-# generated with plugin_template@2021.04.08-99-gd8b54c6
+# generated with plugin_template@2021.08.26-19-g88bcdd3
 
 additional_repos:
 - bindings: true
@@ -58,7 +58,7 @@ pre_job_template:
   name: check_commit
   path: galaxy_ng/.github/pre-job-template.yml.j2
 publish_docs_to_pulpprojectdotorg: false
-pulp_scheme: http
+pulp_scheme: https
 pulp_settings:
   allowed_export_paths: /tmp
   allowed_import_paths: /tmp
@@ -76,6 +76,8 @@ release_user: ansible
 single_commit_check: false
 stable_branch: stable
 sync_ci: true
+tasking_allow_async_unsafe: true
+test_azure: true
 test_bindings: false
 test_cli: false
 test_performance: false