Update CI via plugin_template (#1262)

No-Issue

.ci/ansible/Containerfile.j2

@@ -11,11 +11,14 @@ ADD {{ item.source }} {{ item.source }}
 # Hacking botocore (https://github.com/boto/botocore/pull/1990)
 
 RUN pip3 install \
+{%- if stream_test | default(false) -%}
+{{ " " }}django-storages[sftp]
+{%- endif -%}
 {%- if s3_test | default(false) -%}
 {{ " " }}django-storages[boto3] git+https://github.com/fabricio-aguiar/botocore.git@fix-100-continue
 {%- endif -%}
 {%- if azure_test | default(false) -%}
-{{ " " }}django-storages[azure]>=1.12.2 "azure-storage-blob<12.10.0"
+{{ " " }}django-storages[azure]>=1.12.2
 {%- endif -%}
 {%- for item in plugins -%}
 {%- if item.name == "pulp-certguard" -%}

.ci/ansible/inventory.yaml

@@ -6,6 +6,7 @@ all:
         pulp:
         pulp-fixtures:
         minio:
+        ci-sftp:
       vars:
         ansible_connection: docker
 ...

.ci/ansible/settings.py.j2

@@ -19,7 +19,21 @@ API_ROOT = {{ api_root | repr }}
 {% endfor %}
 {% endif %}
 
+{% if stream_test | default(false) -%}
+REDIRECT_TO_OBJECT_STORAGE = False
+DEFAULT_FILE_STORAGE = "pulpcore.app.models.storage.PulpSFTPStorage"
+MEDIA_ROOT = ""
+SFTP_STORAGE_HOST = "ci-sftp"
+SFTP_STORAGE_ROOT = "/storage/"
+SFTP_STORAGE_PARAMS = {
+    "username": "foo",
+    "key_filename": "/keys/id_ed25519",
+}
+{%- endif %}
+
 {% if s3_test | default(false) %}
+DEFAULT_FILE_STORAGE = "storages.backends.s3boto3.S3Boto3Storage"
+MEDIA_ROOT = ""
 AWS_ACCESS_KEY_ID = "{{ minio_access_key }}"
 AWS_SECRET_ACCESS_KEY = "{{ minio_secret_key }}"
 AWS_S3_REGION_NAME = "eu-central-1"
@@ -28,19 +42,17 @@ S3_USE_SIGV4 = True
 AWS_S3_SIGNATURE_VERSION = "s3v4"
 AWS_STORAGE_BUCKET_NAME = "pulp3"
 AWS_S3_ENDPOINT_URL = "http://minio:9000"
-DEFAULT_FILE_STORAGE = "storages.backends.s3boto3.S3Boto3Storage"
 AWS_DEFAULT_ACL = "@none None"
-MEDIA_ROOT = ""
 {% endif %}
 
 {% if azure_test | default(false) %}
+DEFAULT_FILE_STORAGE = "storages.backends.azure_storage.AzureStorage"
+MEDIA_ROOT = ""
 AZURE_ACCOUNT_KEY = "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw=="
 AZURE_ACCOUNT_NAME = "devstoreaccount1"
 AZURE_CONTAINER =  "pulp-test"
 AZURE_LOCATION = "pulp3"
 AZURE_OVERWRITE_FILES = True
 AZURE_URL_EXPIRATION_SECS = 120
 AZURE_CONNECTION_STRING = 'DefaultEndpointsProtocol={{ pulp_scheme }};AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint={{ pulp_scheme }}://ci-azurite:10000/devstoreaccount1;'
-DEFAULT_FILE_STORAGE = "storages.backends.azure_storage.AzureStorage"
-MEDIA_ROOT = ""
 {% endif %}

.ci/ansible/start_container.yaml

@@ -12,8 +12,15 @@
         mode: "0755"
       loop:
         - settings
+        - ssh
         - ~/.config/pulp_smash
 
+    - name: Generate an OpenSSH keypair
+      community.crypto.openssh_keypair:
+        path: ssh/id_ed25519
+        type: ed25519
+      when: stream_test | default(false)
+
     - name: "Generate Pulp Settings"
       template:
         src: settings.py.j2

.ci/scripts/update_ci_branches.py

@@ -21,7 +21,7 @@
 }
 
 if not initial_branch or initial_branch not in branches:
-    starting = -2
+    exit("Initial branch not found")
 else:
     starting = branches.index(initial_branch)
 

.ci/scripts/upper_bound.py

@@ -3,9 +3,11 @@
 packages = []
 with open("requirements.txt", "r") as fd:
     for line in fd.readlines():
-        req = Requirement.parse(line)
-        if len(req.specs) < 2 and "~=" not in str(req.specs) and "==" not in str(req.specs):
-            packages.append(req.name)
+        if not line.startswith("#"):
+            req = Requirement.parse(line)
+            spec = str(req.specs)
+            if len(req.specs) < 2 and "~=" not in spec and "==" not in spec and "<" not in spec:
+                packages.append(req.name)
 if packages:
     raise RuntimeError(
         "The following packages are missing upper bound: {}".format(", ".join(packages))

.github/template_gitref

@@ -1 +1 @@
-2021.08.26-115-gf0baa22
+2021.08.26-129-gf780fda

.github/workflows/ci.yml

@@ -210,7 +210,7 @@ jobs:
         if: always()
         run: |
           echo "Need to debug? Please check: https://github.com/marketplace/actions/debugging-with-tmate"
-          http --timeout 30 --check-status --pretty format --print hb https://pulp/pulp/api/v3/status/ || true
+          http --timeout 30 --check-status --pretty format --print hb "https://pulp${PULP_API_ROOT}api/v3/status/" || true
           docker images || true
           docker ps -a || true
           docker logs pulp || true

.github/workflows/nightly.yml

@@ -155,7 +155,7 @@ jobs:
         if: always()
         run: |
           echo "Need to debug? Please check: https://github.com/marketplace/actions/debugging-with-tmate"
-          http --timeout 30 --check-status --pretty format --print hb https://pulp/pulp/api/v3/status/ || true
+          http --timeout 30 --check-status --pretty format --print hb "https://pulp${PULP_API_ROOT}api/v3/status/" || true
           docker images || true
           docker ps -a || true
           docker logs pulp || true
@@ -318,7 +318,7 @@ jobs:
         if: always()
         run: |
           echo "Need to debug? Please check: https://github.com/marketplace/actions/debugging-with-tmate"
-          http --timeout 30 --check-status --pretty format --print hb https://pulp/pulp/api/v3/status/ || true
+          http --timeout 30 --check-status --pretty format --print hb "https://pulp${PULP_API_ROOT}api/v3/status/" || true
           docker images || true
           docker ps -a || true
           docker logs pulp || true

.github/workflows/release.yml

@@ -225,7 +225,7 @@ jobs:
         if: always()
         run: |
           echo "Need to debug? Please check: https://github.com/marketplace/actions/debugging-with-tmate"
-          http --timeout 30 --check-status --pretty format --print hb https://pulp/pulp/api/v3/status/ || true
+          http --timeout 30 --check-status --pretty format --print hb "https://pulp${PULP_API_ROOT}api/v3/status/" || true
           docker images || true
           docker ps -a || true
           docker logs pulp || true

.github/workflows/scripts/before_script.sh

@@ -29,7 +29,8 @@ tail -v -n +1 .ci/ansible/vars/main.yaml
 echo "PULP CONFIG:"
 tail -v -n +1 .ci/ansible/settings/settings.* ~/.config/pulp_smash/settings.json
 
-if [[ "$TEST" == 'pulp' || "$TEST" == 'performance' || "$TEST" == 'upgrade' || "$TEST" == 's3' || "$TEST" == 'azure' || "$TEST" == "plugin-from-pypi" || "$TEST" == "generate-bindings" ]]; then
+SCENARIOS=("pulp" "performance" "upgrade" "azure" "s3" "stream" "plugin-from-pypi" "generate-bindings")
+if [[ " ${SCENARIOS[*]} " =~ " ${TEST} " ]]; then
   # Many functional tests require these
   cmd_prefix dnf install -yq lsof which dnf-plugins-core
 fi

.github/workflows/scripts/install.sh

@@ -15,11 +15,14 @@ set -euv
 
 source .github/workflows/scripts/utils.sh
 
+export PULP_API_ROOT="/pulp/"
+
 if [[ "$TEST" = "docs" || "$TEST" = "publish" ]]; then
   pip install -r ../pulpcore/doc_requirements.txt
   pip install -r doc_requirements.txt
 fi
 
+pip install -e ../pulpcore
 pip install -r functest_requirements.txt
 
 cd .ci/ansible/
@@ -68,11 +71,6 @@ plugins:
     source: pulp_container
   - name: galaxy-importer
     source: galaxy-importer
-services:
-  - name: pulp
-    image: "pulp:${TAG}"
-    volumes:
-      - ./settings:/etc/pulp
 VARSYAML
 else
   cat >> vars/main.yaml << VARSYAML
@@ -90,13 +88,17 @@ plugins:
     source: $GALAXY_IMPORTER
   - name: pulpcore
     source: ./pulpcore
+VARSYAML
+fi
+
+cat >> vars/main.yaml << VARSYAML
 services:
   - name: pulp
     image: "pulp:${TAG}"
     volumes:
       - ./settings:/etc/pulp
+      - ./ssh:/keys/
 VARSYAML
-fi
 
 cat >> vars/main.yaml << VARSYAML
 pulp_settings: {"allowed_export_paths": "/tmp", "allowed_import_paths": "/tmp", "galaxy_api_default_distribution_base_path": "published", "galaxy_enable_api_access_log": true, "galaxy_require_content_approval": false, "rh_entitlement_required": "insights"}
@@ -123,7 +125,6 @@ if [ "$TEST" = "s3" ]; then
   sed -i -e '$a s3_test: true\
 minio_access_key: "'$MINIO_ACCESS_KEY'"\
 minio_secret_key: "'$MINIO_SECRET_KEY'"' vars/main.yaml
-  echo "PULP_API_ROOT=/rerouted/djnd/" >> "$GITHUB_ENV"
   export PULP_API_ROOT="/rerouted/djnd/"
 fi
 
@@ -143,6 +144,8 @@ if [ "$TEST" = "azure" ]; then
   sed -i -e '$a azure_test: true' vars/main.yaml
 fi
 
+echo "PULP_API_ROOT=${PULP_API_ROOT}" >> "$GITHUB_ENV"
+
 if [ "${PULP_API_ROOT:-}" ]; then
   sed -i -e '$a api_root: "'"$PULP_API_ROOT"'"' vars/main.yaml
 fi
@@ -156,7 +159,7 @@ sudo docker cp pulp:/etc/pulp/certs/pulp_webserver.crt /usr/local/share/ca-certi
 # Hack: adding pulp CA to certifi.where()
 CERTIFI=$(python -c 'import certifi; print(certifi.where())')
 cat /usr/local/share/ca-certificates/pulp_webserver.crt | sudo tee -a "$CERTIFI" > /dev/null
-if [ "$TEST" = "azure" ]; then
+if [[ "$TEST" = "azure" ]]; then
   cat /usr/local/share/ca-certificates/azcert.crt | sudo tee -a "$CERTIFI" > /dev/null
 fi
 
@@ -168,7 +171,7 @@ cat "$CERTIFI" | sudo tee -a "$CERT" > /dev/null
 sudo update-ca-certificates
 echo ::endgroup::
 
-if [ "$TEST" = "azure" ]; then
+if [[ "$TEST" = "azure" ]]; then
   AZCERTIFI=$(/opt/az/bin/python3 -c 'import certifi; print(certifi.where())')
   cat /usr/local/share/ca-certificates/azcert.crt >> $AZCERTIFI
   cat /usr/local/share/ca-certificates/azcert.crt | cmd_stdin_prefix tee -a /usr/local/lib/python3.8/site-packages/certifi/cacert.pem > /dev/null

.github/workflows/scripts/script.sh

@@ -38,22 +38,14 @@ if [[ "$TEST" = "docs" ]]; then
   tar -cvf docs.tar ./_build
   cd ..
 
-  echo "Validating OpenAPI schema..."
-  cat $PWD/.ci/scripts/schema.py | cmd_stdin_prefix bash -c "cat > /tmp/schema.py"
-  cmd_prefix bash -c "python3 /tmp/schema.py"
-  cmd_prefix bash -c "pulpcore-manager spectacular --file pulp_schema.yml --validate"
-
   if [ -f $POST_DOCS_TEST ]; then
     source $POST_DOCS_TEST
   fi
   exit
 fi
 
 if [[ "${RELEASE_WORKFLOW:-false}" == "true" ]]; then
-  STATUS_ENDPOINT="${PULP_URL}/pulp/api/v3/status/"
-  if [ "${PULP_API_ROOT:-}" ]; then
-    STATUS_ENDPOINT="${PULP_URL}${PULP_API_ROOT}api/v3/status/"
-  fi
+  STATUS_ENDPOINT="${PULP_URL}${PULP_API_ROOT}api/v3/status/"
   echo $STATUS_ENDPOINT
   REPORTED_VERSION=$(http $STATUS_ENDPOINT | jq --arg plugin galaxy --arg legacy_plugin galaxy_ng -r '.versions[] | select(.component == $plugin or .component == $legacy_plugin) | .version')
   response=$(curl --write-out %{http_code} --silent --output /dev/null https://pypi.org/project/galaxy-ng/$REPORTED_VERSION/)
@@ -120,14 +112,13 @@ cmd_prefix bash -c "django-admin makemigrations --check --dry-run"
 
 if [[ "$TEST" != "upgrade" ]]; then
   # Run unit tests.
-  cmd_prefix bash -c "PULP_DATABASES__default__USER=postgres pytest -v -r sx --color=yes --pyargs galaxy_ng.tests.unit"
+  cmd_prefix bash -c "PULP_DATABASES__default__USER=postgres pytest -v -r sx --color=yes -p no:pulpcore --pyargs galaxy_ng.tests.unit"
 fi
 
 # Run functional tests
 export PYTHONPATH=$REPO_ROOT/../pulp_ansible${PYTHONPATH:+:${PYTHONPATH}}
 export PYTHONPATH=$REPO_ROOT/../pulp_container${PYTHONPATH:+:${PYTHONPATH}}
 export PYTHONPATH=$REPO_ROOT/../galaxy-importer${PYTHONPATH:+:${PYTHONPATH}}
-export PYTHONPATH=$REPO_ROOT/../pulpcore${PYTHONPATH:+:${PYTHONPATH}}
 export PYTHONPATH=$REPO_ROOT${PYTHONPATH:+:${PYTHONPATH}}
 
 
@@ -148,9 +139,13 @@ else
     if [[ "$GITHUB_WORKFLOW" == "Galaxy Nightly CI/CD" ]]; then
         pytest -v -r sx --color=yes --suppress-no-test-exit-code --pyargs galaxy_ng.tests.functional -m parallel -n 8
         pytest -v -r sx --color=yes --pyargs galaxy_ng.tests.functional -m "not parallel"
+
+
     else
         pytest -v -r sx --color=yes --suppress-no-test-exit-code --pyargs galaxy_ng.tests.functional -m "parallel and not nightly" -n 8
         pytest -v -r sx --color=yes --pyargs galaxy_ng.tests.functional -m "not parallel and not nightly"
+
+
     fi
 
 fi

.github/workflows/update_ci.yml

@@ -10,6 +10,11 @@
 name: CI Update
 on:
   workflow_dispatch:
+    inputs:
+      all_branches:
+        description: "Run on all branches"
+        default: 'no'
+        required: false
 
 jobs:
   update:
@@ -45,7 +50,7 @@ jobs:
         run: echo ::set-output name=short_ref::${GITHUB_REF#refs/*/}
 
       - name: Dispatching
-        if: github.event_name == 'schedule'
+        if: github.event_name == 'schedule' || github.event.inputs.all_branches == 'yes'
         run: python .ci/scripts/update_ci_branches.py None
         env:
           GITHUB_TOKEN: ${{ secrets.RELEASE_TOKEN }}

template_config.yml

@@ -1,7 +1,7 @@
 # This config represents the latest values used when running the plugin-template. Any settings that
 # were not present before running plugin-template have been added with their default values.
 
-# generated with [email protected]50-g7c31f64
+# generated with [email protected]129-gf780fda
 
 additional_repos:
 - bindings: true
@@ -17,16 +17,16 @@ additional_repos:
   name: galaxy-importer
   org: ansible
 aiohttp_fixtures_origin: 172.18.0.1
+api_root: /pulp/
 black: false
 check_commit_message: false
 check_gettext: true
 check_manifest: true
 check_openapi_schema: true
 check_stray_pulpcore_imports: true
 cherry_pick_automation: false
-ci_env: {
-  GITHUB_USER: '${{ github.event.pull_request.user.login }}'
-}
+ci_env:
+  GITHUB_USER: ${{ github.event.pull_request.user.login }}
 ci_trigger: '{pull_request: {branches: [''*'']}, push: {branches: [''*'']}}'
 core_import_allowed:
 - pulpcore.app.*viewsets
@@ -84,6 +84,7 @@ python_version: '3.8'
 redmine_project: null
 release_email: [email protected]
 release_user: ansible
+run_pulpcore_tests_for_plugins: false
 single_commit_check: false
 stable_branch: stable
 sync_ci: false
@@ -96,6 +97,7 @@ test_performance: false
 test_released_plugin_with_next_pulpcore_release: false
 test_reroute: true
 test_s3: true
+test_stream: false
 travis_addtl_services: []
 travis_notifications: null
 update_github: false