Wrote first api performance tests (#2218)

* Wrote first api performance tests
* Created files for github actions
* Issue-AAH-27837 cleaned files from unused imports
* Issue-AAH27837s removed unnecessary lines
* Issue: AAP-27837 cleaned codes
* Issue: AAP-27837 ,cleaned codes
* Added performance test in oci-env
* Alter modification from ci_standalone-community
* Add subclass for performance tests
* Fixed linting errors
* Added performance testing to other profiles
* Revert unnecessary changes
* Revert standalone changes
* Removed unnecesary lines
* Fixed Sonarcloud failure
* Fixed Sonarcloud errors

No Issue

Signed-off-by: Stevenson Michel <[email protected]>

dev/common/RUN_PERFORMANCE.sh

@@ -0,0 +1,61 @@
+#!/bin/bash
+
+# Expected to be called by:
+# - GitHub Actions ci_standalone.yml for DeploymentMode.STANDALONE
+# - Developer Env makefile commands for DeploymentMode.STANDALONE
+# - TODO: Ephemeral Env pr_check.sh (merge smoke_test.sh into this) for DeploymentMode.INSIGHTS
+
+set -e
+
+if [[ -z $HUB_LOCAL ]]; then
+    export NAMESPACE="ephemeral-1riioj"
+    export HUB_API_ROOT="https://front-end-aggregator-${NAMESPACE}.apps.c-rh-c-eph.8p0c.p1.openshiftapps.com/api/automation-hub/"
+    export HUB_AUTH_URL="https://mocks-keycloak-${NAMESPACE}.apps.c-rh-c-eph.8p0c.p1.openshiftapps.com/auth/realms/redhat-external/protocol/openid-connect/token"
+    export HUB_USE_MOVE_ENDPOINT="true"
+    unset HUB_TOKEN
+else
+    unset NAMESPACE
+    unset HUB_AUTH_URL
+    export HUB_USE_MOVE_ENDPOINT="true"
+fi
+
+# which virtualenv || pip install --user virtualenv
+
+VENVPATH=/tmp/gng_testing
+PIP=${VENVPATH}/bin/pip
+
+if [[ ! -d $VENVPATH ]]; then
+    #virtualenv $VENVPATH
+    python3.10 -m venv $VENVPATH
+    # $PIP install --retries=0 --verbose --upgrade pip wheel
+fi
+source "$VENVPATH/bin/activate"
+echo "PYTHON: $(which python)"
+
+$VENVPATH/bin/pip install -r integration_requirements.txt
+$VENVPATH/bin/pip show epdb || pip install epdb
+
+CONTAINER_API=$(docker ps --filter="name=galaxy_ng" --format="table {{.Names}}" | grep -F api)
+CONTAINER_WORKER=$(docker ps --filter="name=galaxy_ng" --format="table {{.Names}}" | grep -F worker)
+
+# when running user can specify extra pytest arguments such as
+# export HUB_LOCAL=1
+# dev/common/RUN_INTEGRATION.sh --pdb -sv --log-cli-level=DEBUG "-m standalone_only" -k mytest
+if [[ -z $HUB_LOCAL ]]; then
+    $VENVPATH/bin/pytest --capture=no -m "not standalone_only and not community_only and not rbac_roles and not iqe_rbac_test and not sync and not rm_sync and not x_repo_search and not rbac_repos" $@ -v galaxy_ng/tests/performance
+    RC=$?
+else
+    $VENVPATH/bin/pytest --capture=no -m "not cloud_only and not community_only and not rbac_roles and not iqe_rbac_test and not sync and not rm_sync and not x_repo_search and not rbac_repos" -v $@ galaxy_ng/tests/performance
+    RC=$?
+
+    if [[ $RC != 0 ]]; then
+        # dump the api logs
+        docker logs "${CONTAINER_API}"
+
+        # dump the worker logs
+        docker logs "${CONTAINER_WORKER}"
+    fi
+
+fi
+
+exit $RC

dev/common/RUN_PERFORMANCE_STAGE.sh

@@ -0,0 +1,31 @@
+#!/bin/bash
+
+set -e
+
+export HTTPS_PROXY="http://squid.corp.redhat.com:3128"
+export HUB_UPLOAD_SIGNATURES=true
+export IQE_VAULT_GITHUB_TOKEN=${IQE_VAULT_GITHUB_TOKEN}
+export IQE_VAULT_ROLE_ID=${IQE_VAULT_ROLE_ID}
+export IQE_VAULT_SECRET_ID=${IQE_VAULT_SECRET_ID}
+export TESTS_AGAINST_STAGE=true
+export HUB_USE_MOVE_ENDPOINT=true
+export HUB_API_ROOT="https://console.stage.redhat.com/api/automation-hub/"
+export HUB_AUTH_URL="https://sso.stage.redhat.com/auth/realms/redhat-external/protocol/openid-connect/token/"
+
+which virtualenv || pip3 install virtualenv
+
+VENVPATH=/tmp/gng_testing
+PIP=${VENVPATH}/bin/pip
+
+if [[ ! -d $VENVPATH ]]; then
+    virtualenv $VENVPATH
+    $PIP install --retries=0 --verbose --upgrade pip wheel
+fi
+source $VENVPATH/bin/activate
+echo "PYTHON: $(which python)"
+
+pip3 install --upgrade pip wheel
+
+pip3 install -r integration_requirements.txt
+
+pytest --log-cli-level=DEBUG -m "(deployment_cloud or all) and not slow_in_cloud" --junitxml=galaxy_ng-results.xml -v galaxy_ng/tests/performance

dev/oci_env_integration/actions/action_lib.py

@@ -110,6 +110,9 @@ def exec_cmd(self, env, cmd):
 
         assert rc == 0
 
+    def get_test_script(self, env):
+        return "/src/galaxy_ng/profiles/base/run_integration.sh"
+
     def run_test(self):
         for env in self.envs:
             pytest_flags = self.envs[env].get("pytest_flags")
@@ -121,6 +124,7 @@ def run_test(self):
                 time.sleep(wait_time)
 
             if self.envs[env]["run_tests"]:
+                script = self.get_test_script(env)
                 if self.envs[env].get("test_script"):
                     self.exec_cmd(
                         env,
@@ -130,7 +134,7 @@ def run_test(self):
                 else:
                     self.exec_cmd(
                         env,
-                        "exec bash /src/galaxy_ng/profiles/base/run_integration.sh"
+                        f"exec bash {script}"
                         f" {pytest_flags} {self.flags}"
                     )
 
@@ -146,3 +150,11 @@ def teardown(self):
             return
         for env in self.envs:
             self.exec_cmd(env, "compose down -v")
+
+
+class OCIEnvPerformanceTest(OCIEnvIntegrationTest):
+    def __init__(self, envs):
+        super().__init__(envs)
+
+    def get_test_script(self, env):
+        return "/src/galaxy_ng/profiles/base/run_performance.sh"

dev/oci_env_integration/actions/community.py

@@ -9,3 +9,14 @@
         }
     ]
 )
+
+
+env_performance = action_lib.OCIEnvPerformanceTest(
+    envs=[
+        {
+            "env_file": "community.compose.env",
+            "run_tests": True,
+            "db_restore": None,
+        }
+    ]
+)

galaxy_ng/tests/performance/conftest.py

@@ -0,0 +1,76 @@
+import logging
+import pytest
+from ..integration.utils.iqe_utils import (
+    get_ansible_config, get_galaxy_client
+)
+
+MARKER_CONFIG = """
+qa: Mark tests to run in the vortex job.
+galaxyapi_smoke: Smoke tests for galaxy-api backend.
+deployment_standalone: Tests that should not run against the Insights version of Hub.
+deployment_cloud: Tests that should not run against the standalone version of Hub.
+deployment_community: Tests relevant to the community deployment profile.
+cli: Tests that shell out to the real ansible-galaxy cli.
+ui: Tests that navigate the UI via selenium.
+ui_standalone: UI tests that only work in standalone mode.
+smoke: Same as galaxyapi_smoke?
+prod_status: For checking prod?
+busted: Doesn't work yet.
+stage_health: For checking stage?
+namespace: Tests that manipulate namespaces.
+certification: Related to certification.
+collection_move: Related to moving collection versions.
+collection_delete: Deletes collections.
+collection_version_delete: Deletes collections versions.
+collection_signing: Related to collection signing.
+delete: Tests for deleting objects.
+move: Related to the move api.
+synclist: Related to synclist object and synclist repo.
+openapi: Checks the openapi schema and routes.
+openapi_generate_bindings: Verifies pulp client bindings generator
+package: tests for the pip packaging
+api_ui: tests for the _ui v1 api
+importer: tests related checks in galaxy-importer
+pulp_api: tests related to the pulp api endpoints
+ldap: tests related to the ldap integration
+role: Related to RBAC Roles
+rbac_roles: Tests checking Role permissions
+group: Related to Groups
+slow_in_cloud: tests that take too long to be run against stage
+max_hub_version: This marker takes an argument that indicates the maximum hub version
+min_hub_version: This marker takes an argument that indicates the minimum hub version
+iqe_rbac_test: imported iqe tests checking role permissions
+iqe_ldap: imported iqe tests checking ldap integration
+sync: sync tests against stage
+certified_sync: sync tests container against container
+auto_approve: run tests that require AUTO_APPROVE to be set to true
+private_repos: run tests verifying private repositories
+rbac_repos: tests verifying rbac roles on custom repositories
+x_repo_search: tests verifying cross-repo search endpoint
+repositories: tests verifying custom repositories
+all: tests that are unmarked and should pass in all deployment modes
+galaxy_stage_ansible: tests that run against galaxy-stage.ansible.com
+installer_smoke_test: smoke tests to validate AAP installation (VM)
+load_data: tests that load data that will be verified after upgrade or backup/restore
+verify_data: tests that verify the data previously loaded by load_data test
+skip_in_gw: tests that need to be skipped if hub is behind the gateway (temporary)
+"""
+
+logger = logging.getLogger(__name__)
+
+
+def pytest_configure(config):
+    for line in MARKER_CONFIG.split('\n'):
+        if not line:
+            continue
+        config.addinivalue_line('markers', line)
+
+
+@pytest.fixture(scope="session")
+def ansible_config():
+    return get_ansible_config()
+
+
+@pytest.fixture(scope="session")
+def galaxy_client(ansible_config):
+    return get_galaxy_client(ansible_config)

galaxy_ng/tests/performance/constants.py

@@ -0,0 +1,59 @@
+URLS = {
+    '_ui/v1/settings/' : {
+        'baseline' : 0.5,
+        'method' : 'GET'
+    },
+    '_ui/v1/namespaces/' : {
+        'baseline' : 0.5,
+        'method' : 'GET'
+    },
+    '_ui/v1/auth/login/' : {
+        'baseline' : 0.5,
+        'method' : 'GET'
+    },
+    '_ui/v1/collection-versions/' : {
+        'baseline' : 0.5,
+        'method' : 'GET'
+    },
+    '_ui/v1/distributions/' : {
+        'baseline' : 0.5,
+        'method' : 'GET'
+    },
+    '_ui/v1/imports/collections/' : {
+        'baseline' : 0.5,
+        'method' : 'GET'
+    },
+    '_ui/v1/me/' : {
+        'baseline' : 0.5,
+        'method' : 'GET'
+    },
+    '_ui/v1/remotes/' : {
+        'baseline' : 0.5,
+        'method' : 'GET'
+    },
+    '_ui/v1/search/' : {
+        'baseline' : 0.5,
+        'method' : 'GET'
+    },
+    '_ui/v1/users/': {
+        'baseline' : 0.5,
+        'method' : 'GET'
+    },
+    '_ui/v1/tags/': {
+        'baseline' : 0.5,
+        'method' : 'GET'
+    },
+    '_ui/v1/tags/collections/': {
+        'baseline' : 0.5,
+        'method' : 'GET'
+    },
+    '_ui/v1/feature-flags/': {
+        'baseline' : 0.5,
+        'method' : 'GET'
+    },
+    '_ui/v1/tags/roles/': {
+        'baseline' : 0.5,
+        'method' : 'GET'
+    },
+    # more API calls with POST requests will be added
+}

galaxy_ng/tests/performance/test_performance.py

@@ -0,0 +1,34 @@
+import time
+import pytest
+
+from galaxy_ng.tests.performance.constants import URLS
+from ..integration.utils import (
+    UIClient,
+    get_client
+)
+
+
+@pytest.fixture(scope="function")
+def api_client(ansible_config):
+    config = ansible_config("admin")
+    api_client = get_client(
+        config=config,
+        request_token=False,
+        require_auth=False
+    )
+    return api_client
+
+
+@pytest.mark.deployment_community
+@pytest.mark.parametrize("url,info", URLS.items())
+def test_api_performance(ansible_config, api_client, url, info):
+    threshold = 0.25
+    results = []
+    cfg = ansible_config('basic_user')
+    with UIClient(config=cfg) as uclient:
+        start_time = time.time()
+        uclient.get(url)
+        elapsed_time = time.time() - start_time
+        difference = (elapsed_time - info['baseline']) / info['baseline']
+        results.append({url: difference})
+        assert difference < threshold

profiles/base/run_performance.sh

@@ -0,0 +1,56 @@
+set -e
+
+VENVPATH=/tmp/gng_testing
+PIP=${VENVPATH}/bin/pip
+source $VENVPATH/bin/activate
+
+cd /src/galaxy_ng/
+
+django-admin shell < ./dev/common/setup_test_data.py
+cd galaxy_ng
+django-admin makemessages --all
+
+cd /src/galaxy_ng/
+
+# Mark taxonomy:
+
+# would be good to be able to just get rid of these entirely
+# - `auth_ldap` -> tests specific to ldap authentication
+# - `auth_keycloak` -> tests specific to keycloak authentication
+# - `auth_standalone` -> tests specific to standalone
+
+# - `deployment_all` -> run test in all environments
+# - `deployment_private_hub` -> tests specific to private hub (EEs, private features)
+# - `deployment_insights` -> tests specific to insights mode (synclists, RH auth)
+# - `deployment_community` -> tests specific to community (GH auth, legacy roles, legacy namespaces)
+# - `rbac` -> rbac roles tests
+# - `sync` -> requires external service to sync from
+
+
+
+# check the environment to see if the test fixtures are set up. If they aren't,
+# initialize them
+
+# if [[ $COMPOSE_PROFILE =~ "galaxy_ng/ldap" ]]; then
+#     MARKS="all or private_hub or auth_ldap"
+# elif [[ $COMPOSE_PROFILE =~ "galaxy_ng/keycloak" ]]; then
+#     MARKS="all or private_hub or auth_keycloak"
+# elif [[ $COMPOSE_PROFILE =~ "galaxy_ng/community" ]]; then
+#     MARKS="all or community"
+# elif [[ $COMPOSE_PROFILE =~ "galaxy_ng/insights" ]]; then
+#     MARKS="all or insights"
+# else 
+#     MARKS="all or private_hub or auth_standalone"
+# fi
+
+# echo $MARKS
+
+
+
+# TODO: fix marks
+set -x
+
+$VENVPATH/bin/pytest -v -r sx --color=yes -m "$HUB_TEST_MARKS" "$@" galaxy_ng/tests/performance
+RC=$?
+
+exit $RC