Community tests in docker compose CI (#2373)

* init ci docker compose integration file with community profile
* dont run other CI for now
* dont run sonar as well
* add django browsable api and fix test
* fix tests with browsable api
* run commands inside docker compose profile
* run less CI
* fix browsable api mistake
* remove comment
* first checkin
* fix test_role_import_overrides
* cleanup
* remove unused and revert
* run ruff and flake
* dont run community in oci env

.github/workflows/ci-docker-compose-integration.yml

@@ -0,0 +1,74 @@
+---
+name: Docker Compose Integration
+on:
+  pull_request:
+    branches:
+      - '**'
+  push:
+    branches:
+      - '**'
+  workflow_dispatch:
+
+jobs:
+  integration:
+    strategy:
+      fail-fast: false
+      matrix:
+        env:
+#          - TEST_PROFILE: ldap
+#          - TEST_PROFILE: keycloak
+#          - TEST_PROFILE: standalone
+#          # - TEST_PROFILE: rbac
+#          - TEST_PROFILE: rbac_parallel_group_1
+#          - TEST_PROFILE: rbac_parallel_group_2
+#          - TEST_PROFILE: certified-sync
+#          - TEST_PROFILE: insights
+#          - TEST_PROFILE: iqe_rbac
+#          - TEST_PROFILE: x_repo_search
+          - TEST_PROFILE: community
+#          - TEST_PROFILE: dab_jwt
+
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-python@v4
+        with:
+          python-version: "3.10"
+
+      - name: (Linux) Install docker compose
+        run: |
+          curl -L -o /tmp/docker-compose https://github.com/docker/compose/releases/download/v2.29.1/docker-compose-linux-x86_64
+          install /tmp/docker-compose /usr/local/bin/
+
+      - name: Spin up dev/compose/${{ matrix.env.TEST_PROFILE }}.yaml
+        run: |
+          docker compose -f dev/compose/community.yaml up --detach
+
+      - name: Export environment variables to host
+        run: |
+          docker compose -f dev/compose/${{ matrix.env.TEST_PROFILE }}.yaml exec manager /bin/bash -c 'env | grep -v -w "HOME"' >> $GITHUB_ENV
+          cat $GITHUB_ENV
+
+      - name: Wait for API online status
+        run: |
+          max_runs=10
+          for i in $(seq 1 $max_runs); do
+            echo "$i: checking api status..."
+            RESPONSE=$(curl -s -o /dev/null -w "%{http_code}" -u "${{ env.DJANGO_SUPERUSER_USERNAME }}:${{ env.DJANGO_SUPERUSER_PASSWORD }}" -LI http://localhost:5001/api/galaxy/pulp/api/v3/status/)
+            if [ "$RESPONSE" -ne 200 ]; then
+              echo "API is down. Retrying in 10 seconds..."
+              sleep 10
+            else
+              echo "API online."
+              exit 0
+            fi
+          done
+
+      - name: Install integration requirements
+        run: |
+          pip install -r integration_requirements.txt
+
+      - name: Run integration tests
+        run: |
+          pytest -v -r sx --color=yes -m '${{ env.HUB_TEST_MARKS }}' galaxy_ng/tests/integration

.github/workflows/ci_oci-env-integration.yml

@@ -25,7 +25,7 @@ jobs:
           - TEST_PROFILE: insights
           - TEST_PROFILE: iqe_rbac
           - TEST_PROFILE: x_repo_search
-          - TEST_PROFILE: community
+          # - TEST_PROFILE: community
           - TEST_PROFILE: dab_jwt
     runs-on: ubuntu-latest
     steps:

.gitignore

@@ -110,6 +110,8 @@ gng_unit_testing/
 
 # Galaxy NG
 db_snapshots/
+temp_roles/*
+.gitconfig
 
 # oci-env profile files
 .compiled/

dev/compose/community.yaml

@@ -30,9 +30,9 @@ x-common-env: &common-env
   # Hostname and prefix has to be correct
   PULP_GALAXY_API_PATH_PREFIX: '/api/'
   PULP_CONTENT_PATH_PREFIX: '/pulp/content/'
-  PULP_ANSIBLE_API_HOSTNAME: 'https://localhost'
-  PULP_ANSIBLE_CONTENT_HOSTNAME: "https://localhost"
-  PULP_CONTENT_ORIGIN: "https://localhost"
+  PULP_ANSIBLE_API_HOSTNAME: 'http://localhost:5001'
+  PULP_ANSIBLE_CONTENT_HOSTNAME: "http://localhost:5001"
+  PULP_CONTENT_ORIGIN: "http://localhost:5001"
   PULP_CSRF_TRUSTED_ORIGINS: "['https://localhost']"
 
   # auth ...
@@ -59,7 +59,7 @@ x-common-env: &common-env
   PULP_GALAXY_AUTO_SIGN_COLLECTIONS: 'false'
 
   # disable user/group modifications
-  PULP_ALLOW_LOCAL_RESOURCE_MANAGEMENT: 'false'
+  PULP_ALLOW_LOCAL_RESOURCE_MANAGEMENT: 'true'
 
   # role content workaround
   PULP_ANSIBLE_BASE_ROLES_REQUIRE_VIEW: 'false'
@@ -74,8 +74,9 @@ x-common-env: &common-env
   # PULP_RESOURCE_SERVER__VALIDATE_HTTPS='false'
 
   # Integration test settings
-  HUB_TEST_AUTHENTICATION_BACKEND: "community"
-  HUB_TEST_MARKS: "deployment_community"
+  HUB_TEST_AUTHENTICATION_BACKEND: 'community'
+  HUB_TEST_MARKS: 'deployment_community'
+  HUB_API_ROOT: 'http://localhost:5001/api/'
 
   # Unpin dependencies on setup.py if set to 0
   LOCK_REQUIREMENTS: 0

docker/etc/settings.py

@@ -39,7 +39,10 @@
 REDIS_HOST = os.environ.get('PULP_REDIS_HOST')
 REDIS_PORT = os.environ.get('PULP_REDIS_PORT')
 
-REST_FRAMEWORK__DEFAULT_RENDERER_CLASSES = ['rest_framework.renderers.JSONRenderer']
+REST_FRAMEWORK__DEFAULT_RENDERER_CLASSES = [
+    'rest_framework.renderers.JSONRenderer',
+    'galaxy_ng.app.renderers.CustomBrowsableAPIRenderer'
+]
 
 _enabled_handlers = ['console']
 _extra_handlers = {}

galaxy_ng/tests/integration/api/test_ui_paths.py

@@ -1,6 +1,5 @@
 import random
 import json
-import subprocess
 from http import HTTPStatus
 
 import pytest
@@ -874,7 +873,7 @@ def build_upload_wait(tags):
 
 # /api/automation-hub/_ui/v1/tags/roles/
 @pytest.mark.deployment_community
-def test_api_ui_v1_tags_roles(ansible_config):
+def test_api_ui_v1_tags_roles(ansible_config, docker_compose_exec):
     """Test endpoint's sorting and filtering"""
 
     def _sync_role(github_user, role_name):
@@ -886,11 +885,7 @@ def _sync_role(github_user, role_name):
         wait_for_v1_task(resp=resp, api_client=api_admin_client)
 
     def _populate_tags_cmd():
-        proc = subprocess.run(
-            "django-admin populate-role-tags",
-            shell=True,
-            capture_output=True,
-        )
+        proc = docker_compose_exec('django-admin populate-role-tags')
         assert proc.returncode == 0
 
     config = ansible_config("basic_user")

galaxy_ng/tests/integration/cli/test_community_sync.py

@@ -4,7 +4,6 @@
 import json
 import pytest
 import requests
-import subprocess
 
 from ansible.galaxy.api import GalaxyError
 
@@ -15,7 +14,7 @@
 
 
 @pytest.mark.deployment_community
-def test_community_collection_download_count_sync(ansible_config):
+def test_community_collection_download_count_sync(ansible_config, docker_compose_exec):
     """ Test collection download count sync command """
 
     # FIXME(jctanner): once beta switches over, this test is no longer needed.
@@ -83,11 +82,7 @@ def test_community_collection_download_count_sync(ansible_config):
     wait_for_task(api_client, sync_task)
 
     # run the django command
-    pid = subprocess.run(
-        'pulpcore-manager sync-collection-download-counts',
-        shell=True,
-        capture_output=True,
-    )
+    pid = docker_compose_exec('pulpcore-manager sync-collection-download-counts')
     assert pid.returncode == 0
 
     # check the counter in the api

galaxy_ng/tests/integration/community/test_community_api.py

@@ -813,10 +813,10 @@ def test_v1_role_versions(ansible_config):
     id = resp["results"][0]["id"]
     versions = resp["results"][0]["summary_fields"]["versions"]
 
-    resp = api_client(f'/api/v1/roles/{id}/versions')
+    resp = api_client(f'/api/v1/roles/{id}/versions/')
     assert resp["count"] >= len(versions)
 
     with pytest.raises(AnsibleError) as html:
-        api_client(f"v1/roles/{id}/versions", headers={"Accept": "text/html"})
+        api_client(f"v1/roles/{id}/versions/", headers={"Accept": "text/html"})
     assert not isinstance(html.value, dict)
     assert "results" in str(html.value)

galaxy_ng/tests/integration/community/test_role_import_overrides.py

@@ -39,11 +39,10 @@
             'alternate_namespace_name': None,
             'alternate_role_name': None,
         }
-
     ]
 )
 @pytest.mark.deployment_community
-def test_role_import_overrides(ansible_config, spec):
+def test_role_import_overrides(ansible_config, spec, docker_compose_exec):
     """" Validate setting namespace in meta/main.yml does the right thing """
 
     admin_config = ansible_config("admin")
@@ -83,10 +82,12 @@ def test_role_import_overrides(ansible_config, spec):
         'meta_namespace': spec['meta_namespace'],
         'meta_name': spec['meta_name'],
     }
-    lr = LegacyRoleGitRepoBuilder(**builder_kwargs)
+    lr = LegacyRoleGitRepoBuilder(**builder_kwargs, docker_compose_exec=docker_compose_exec)
+
+    local_tmp_dir = lr.role_cont_dir
 
     # run the import
-    payload = {'alternate_clone_url': lr.role_dir}
+    payload = {'alternate_clone_url': local_tmp_dir}
     for key in ['github_user', 'github_repo', 'alternate_namespace_name', 'alternate_role_name']:
         if spec.get(key):
             payload[key] = spec[key]
@@ -102,3 +103,6 @@ def test_role_import_overrides(ansible_config, spec):
     assert roles_search['results'][0]['name'] == spec['name']
     assert roles_search['results'][0]['github_user'] == spec['github_user']
     assert roles_search['results'][0]['github_repo'] == spec['github_repo']
+
+    # cleanup
+    lr.local_roles_cleanup()

galaxy_ng/tests/integration/community/test_v1_api.py

@@ -133,7 +133,7 @@ def test_custom_browsable_format(ansible_config):
         require_auth=True,
     )
 
-    resp = api_client("v1/namespaces")
+    resp = api_client("v1/namespaces/")
     assert isinstance(resp, dict)
     assert "results" in resp
 
@@ -142,7 +142,7 @@ def test_custom_browsable_format(ansible_config):
     assert "results" in resp
 
     with pytest.raises(AnsibleError) as html:
-        api_client("v1/namespaces", headers={"Accept": "text/html"})
+        api_client("v1/namespaces/", headers={"Accept": "text/html"})
     assert not isinstance(html.value, dict)
     assert "results" in str(html.value)
 
@@ -154,7 +154,7 @@ def test_custom_browsable_format(ansible_config):
         require_auth=True,
     )
 
-    resp = api_client("v1/namespaces")
+    resp = api_client("v1/namespaces/")
     assert isinstance(resp, dict)
     assert "results" in resp
 
@@ -163,6 +163,6 @@ def test_custom_browsable_format(ansible_config):
     assert "results" in resp
 
     with pytest.raises(AnsibleError) as html:
-        api_client("v1/namespaces", headers={"Accept": "text/html"})
+        api_client("v1/namespaces/", headers={"Accept": "text/html"})
     assert not isinstance(html.value, dict)
     assert "results" in str(html.value)

galaxy_ng/tests/integration/conftest.py

@@ -3,6 +3,7 @@
 import os
 import shutil
 import yaml
+import subprocess
 
 import pytest
 from orionutils.utils import increment_version
@@ -726,3 +727,20 @@ def skip_if_require_signature_for_approval():
 def skip_if_not_require_signature_for_approval():
     if not require_signature_for_approval():
         pytest.skip("This test needs refactoring to work with signatures required on move.")
+
+
+@pytest.fixture
+def docker_compose_exec():
+    def _exec(cmd: str, cwd=None):
+        cd = ''
+        if cwd is not None:
+            cd = f'cd {cwd};'
+
+        proc = subprocess.run(
+            f"docker exec compose-manager-1 /bin/bash -c '{cd}{cmd}'",
+            shell=True,
+            capture_output=True,
+        )
+        return proc
+
+    return _exec

galaxy_ng/tests/integration/utils/legacy.py

@@ -234,32 +234,51 @@ def __init__(
         namespace=None,
         name=None,
         meta_namespace=None,
-        meta_name=None
+        meta_name=None,
+        docker_compose_exec=None
     ):
         self.namespace = namespace
         self.name = name
         self.meta_namespace = meta_namespace
         self.meta_name = meta_name
 
-        self.workdir = tempfile.mkdtemp(prefix='gitrepo_')
+        self.docker_compose_exec = docker_compose_exec
+
+        # local tmp dir for roles
+        self.temp_roles = 'temp_roles/'
+        self.local_roles_cleanup()
+        if not os.path.exists(self.temp_roles):
+            os.makedirs(self.temp_roles)
+
+        self.workdir = tempfile.mkdtemp(prefix='gitrepo_', dir=self.temp_roles)
+        path_parts = self.workdir.partition(self.temp_roles)
+
+        # should be equal to HOME=/app
+        # TODO(jjerabek): better way to get env var from container?
+        pid = self.docker_compose_exec('printenv HOME')
+        home = pid.stdout.decode('utf-8').strip() or '/app'
+
+        self.workdir_cont = os.path.join(home, path_parts[1], path_parts[2])
+
         self.role_dir = None
+        self.role_cont_dir = None
 
         self.role_init()
         self.role_edit()
         self.git_init()
         self.git_commit()
 
-        self.fix_perms()
-
-    def fix_perms(self):
-        subprocess.run(f'chown -R pulp:pulp {self.workdir}', shell=True)
-
     def role_init(self):
         cmd = f'ansible-galaxy role init {self.namespace}.{self.name}'
         self.role_dir = os.path.join(self.workdir, self.namespace + '.' + self.name)
-        pid = subprocess.run(cmd, shell=True, cwd=self.workdir)
+        self.role_cont_dir = os.path.join(self.workdir_cont, self.namespace + '.' + self.name)
+
+        pid = subprocess.run(cmd, shell=True, cwd=self.workdir, capture_output=True)
+
         assert pid.returncode == 0
         assert os.path.exists(self.role_dir)
+        ag_init_stdout = f'- Role {self.namespace}.{self.name} was created successfully'
+        assert pid.stdout.decode('utf-8').strip() == ag_init_stdout
 
     def role_edit(self):
         if self.meta_namespace or self.meta_name:
@@ -277,12 +296,17 @@ def role_edit(self):
                 f.write(yaml.dump(meta))
 
     def git_init(self):
-        subprocess.run('git init', shell=True, cwd=self.role_dir)
+        self.docker_compose_exec('git init', cwd=self.role_cont_dir)
+
+        # hack to make git inside git dir work
+        self.docker_compose_exec(f'git config --global --add safe.directory {self.role_cont_dir}')
 
     def git_commit(self):
+        self.docker_compose_exec('git config --global user.email "root@localhost"')
+        self.docker_compose_exec('git config --global user.name "root at localhost"')
 
-        subprocess.run('git config --global user.email "root@localhost"', shell=True)
-        subprocess.run('git config --global user.name "root at localhost"', shell=True)
+        self.docker_compose_exec('git add *', cwd=self.role_cont_dir)
+        self.docker_compose_exec('git commit -m "first checkin"', cwd=self.role_cont_dir)
 
-        subprocess.run('git add *', shell=True, cwd=self.role_dir)
-        subprocess.run('git commit -m "first checkin"', shell=True, cwd=self.role_dir)
+    def local_roles_cleanup(self):
+        self.docker_compose_exec(f'rm -rf {self.temp_roles}')