From 969c1a7cb00bacb23aa3594224df802bba631b29 Mon Sep 17 00:00:00 2001 From: jctanner Date: Wed, 18 Sep 2024 15:45:39 -0400 Subject: [PATCH] Make a simple/self-contained docker-compose to use for AAP dev. (#2266) * Make a simple docker-compose to use for AAP dev. * Don't bind postgres to the host. * Don't add the compose to the manifest. * Fix some issues. * Fix the file storage setting. * Disable signing and auto-signing for now. No-Issue Signed-off-by: James Tanner --- MANIFEST.in | 1 + aap_compose_dev.yaml | 231 +++++++++++++++++++++++++++++++++ dev/nginx/nginx.conf | 92 +++++++++++++ profiles/dab/make_test_data.py | 9 +- 4 files changed, 332 insertions(+), 1 deletion(-) create mode 100644 aap_compose_dev.yaml create mode 100644 dev/nginx/nginx.conf diff --git a/MANIFEST.in b/MANIFEST.in index 42bdb40fb9..a53d5df20e 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -24,3 +24,4 @@ include galaxy_ng/tests/integration/utils/gpg/qe-sign-priv.gpg include galaxy-operator/bin/readyz.py exclude .coveragerc exclude .sonarcloud.properties +exclude aap_compose_dev.yaml diff --git a/aap_compose_dev.yaml b/aap_compose_dev.yaml new file mode 100644 index 0000000000..8eadce5e8a --- /dev/null +++ b/aap_compose_dev.yaml @@ -0,0 +1,231 @@ +x-common-env: &common-env + + DJANGO_SUPERUSER_USERNAME: admin + DJANGO_SUPERUSER_EMAIL: admin@example.com + DJANGO_SUPERUSER_PASSWORD: admin + + POSTGRES_USER: galaxy_ng + POSTGRES_PASSWORD: galaxy_ng + POSTGRES_DB: galaxy_ng + + # no spying + PULP_ANALYTICS: 'false' + + # normally goes into settings.py ... + PULP_DATABASES__default__ENGINE: django.db.backends.postgresql + PULP_DATABASES__default__NAME: galaxy_ng + PULP_DATABASES__default__USER: galaxy_ng + PULP_DATABASES__default__PASSWORD: galaxy_ng + PULP_DATABASES__default__HOST: postgres + PULP_DATABASES__default__PORT: 5432 + + PULP_DEBUG: 1 + PULP_GALAXY_DEPLOYMENT_MODE: 'standalone' + PULP_DEFAULT_FILE_STORAGE: "pulpcore.app.models.storage.FileSystem" + + # Hostname and prefix has to be correct + PULP_GALAXY_API_PATH_PREFIX: '/api/galaxy/' + PULP_ANSIBLE_API_HOSTNAME: 'https://localhost' + PULP_ANSIBLE_CONTENT_HOSTNAME: "https://localhost/pulp/content" + PULP_CONTENT_ORIGIN: "https://localhost" + PULP_CSRF_TRUSTED_ORIGINS: "['https://localhost']" + + # signing ... + ENABLE_SIGNING: 0 + PULP_GALAXY_AUTO_SIGN_COLLECTIONS: 'false' + PULP_GALAXY_REQUIRE_CONTENT_APPROVAL: 'true' + PULP_GALAXY_REQUIRE_SIGNATURE_FOR_APPROVAL: 'false' + PULP_GALAXY_COLLECTION_SIGNING_SERVICE: 'ansible-default' + PULP_GALAXY_CONTAINER_SIGNING_SERVICE: 'container-default' + + # pulp container ... + PULP_TOKEN_AUTH_DISABLED: 'false' + PULP_TOKEN_SERVER: 'https://localhost/token/' + PULP_TOKEN_SIGNATURE_ALGORITHM: 'ES256' + PULP_PUBLIC_KEY_PATH: '/src/galaxy_ng/dev/common/container_auth_public_key.pem' + PULP_PRIVATE_KEY_PATH: '/src/galaxy_ng/dev/common/container_auth_private_key.pem' + + # auth ... + PULP_GALAXY_AUTHENTICATION_CLASSES: "['galaxy_ng.app.auth.session.SessionAuthentication', 'ansible_base.jwt_consumer.hub.auth.HubJWTAuth', 'rest_framework.authentication.TokenAuthentication', 'rest_framework.authentication.BasicAuthentication']" + PULP_ANSIBLE_BASE_JWT_VALIDATE_CERT: 'false' + PULP_ANSIBLE_BASE_JWT_KEY: 'https://localhost' + PULP_GALAXY_FEATURE_FLAGS__external_authentication: 'true' + + # disable user/group modifications + PULP_ALLOW_LOCAL_RESOURCE_MANAGEMENT: 'false' + + # role content workaround .. + PULP_ANSIBLE_BASE_ROLES_REQUIRE_VIEW: 'false' + + +services: + base_img: + build: + context: . + dockerfile: Dockerfile + #image: galaxy_base_image:latest + image: "localhost/galaxy_ng/galaxy_ng:base" + + redis: + image: "redis:5" + ports: + - "6379:6379" + + postgres: + image: "postgres:13" + #ports: + # - "5432:5432" + environment: + <<: *common-env + healthcheck: + test: ["CMD", "pg_isready", "-U", "galaxy_ng"] + interval: 10s + retries: 5 + + helper: + image: quay.io/centos/centos:stream9 + environment: + <<: *common-env + depends_on: + - postgres + volumes: + - "etc_pulp_certs:/etc/pulp/certs" + - "var_lib_pulp:/var/lib/pulp" + command: | + bash -c " + if [[ ! -e /etc/pulp/certs/database_fields.symmetric.key ]] || [[ -s /etc/pulp/certs/database_fields.symmetric.key ]]; then + mkdir -p /etc/pulp/certs/; + echo 'check openssl and install ...'; + rpm -q openssl || dnf -y install openssl; + echo 'generate key ...'; + openssl rand -base64 32 > /etc/pulp/certs/database_fields.symmetric.key; + echo 'chown key ...'; + chmod 640 /etc/pulp/certs/database_fields.symmetric.key; + else + echo 'symmetric key exists' + fi; + find /etc/pulp ; + echo '# KEY >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>'; + cat /etc/pulp/certs/database_fields.symmetric.key; + echo '<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<'; + echo "DONE!"; + " + + migrations: + image: "localhost/galaxy_ng/galaxy_ng:base" + depends_on: + - base_img + - postgres + - helper + volumes: + - "etc_pulp_certs:/etc/pulp/certs" + - "var_lib_pulp:/var/lib/pulp" + - ".:/src/galaxy_ng" + environment: + <<: *common-env + user: root + command: | + bash -c " + set -e; + while [[ ! -f /etc/pulp/certs/database_fields.symmetric.key ]]; do + echo 'Waiting for key'; + sleep 2; + done; + pulpcore-manager check --database default; + pulpcore-manager migrate; + touch /var/lib/pulp/.migrated; + pulpcore-manager shell < /src/galaxy_ng/dev/common/setup_test_data.py; + pulpcore-manager createsuperuser --noinput; + " + + api: + image: "localhost/galaxy_ng/galaxy_ng:base" + depends_on: + - base_img + - postgres + - helper + - migrations + volumes: + - "etc_pulp_certs:/etc/pulp/certs" + - "var_lib_pulp:/var/lib/pulp" + - ".:/src/galaxy_ng" + environment: + <<: *common-env + extra_hosts: + localhost: "host-gateway" + command: | + bash -c " + while [[ ! -f /var/lib/pulp/.migrated ]]; do + echo 'Waiting for migrations ...'; + sleep 2; + done && exec pulpcore-api --reload; + " + + content: + image: "localhost/galaxy_ng/galaxy_ng:base" + depends_on: + - base_img + - postgres + - helper + - migrations + volumes: + - "etc_pulp_certs:/etc/pulp/certs" + - "var_lib_pulp:/var/lib/pulp" + - ".:/src/galaxy_ng" + environment: + <<: *common-env + extra_hosts: + localhost: "host-gateway" + command: | + bash -c " + while [[ ! -f /var/lib/pulp/.migrated ]]; do + echo 'Waiting for migrations ...'; + sleep 2; + done && exec pulpcore-content --reload; + " + + worker: + image: "localhost/galaxy_ng/galaxy_ng:base" + depends_on: + - base_img + - postgres + - helper + - migrations + volumes: + - "etc_pulp_certs:/etc/pulp/certs" + - "var_lib_pulp:/var/lib/pulp" + - ".:/src/galaxy_ng" + environment: + <<: *common-env + command: | + bash -c " + while [[ ! -f /var/lib/pulp/.migrated ]]; do + echo 'Waiting for migrations ...'; + sleep 2; + done && exec pulpcore-worker; + " + + nginx: + image: "nginx:latest" + depends_on: + - base_img + - postgres + - helper + - migrations + - api + - content + ports: + - '5001:5001' + volumes: + - './dev/nginx/nginx.conf:/etc/nginx/nginx.conf:ro' + +volumes: + var_lib_pulp: + name: var_lib_pulp + etc_pulp_certs: + name: etc_pulp_certs + +#networks: +# dbnet: +# service-mesh: +# name: service-mesh diff --git a/dev/nginx/nginx.conf b/dev/nginx/nginx.conf new file mode 100644 index 0000000000..73f91b11d4 --- /dev/null +++ b/dev/nginx/nginx.conf @@ -0,0 +1,92 @@ +# TODO: Support IPv6. +# TODO: Maybe serve multiple `location`s, not just one. + +# The "nginx" package on fedora creates this user and group. +# user nginx nginx; + +# Gunicorn docs suggest this value. +worker_processes 1; +# daemon off; +events { + worker_connections 1024; # increase if you have lots of clients + accept_mutex off; # set to 'on' if nginx worker_processes > 1 +} + +http { + include mime.types; + # fallback in case we can't determine a type + default_type application/octet-stream; + sendfile on; + + # If left at the default of 1024, nginx emits a warning about being unable + # to build optimal hash types. + types_hash_max_size 4096; + + upstream pulp-content { + server content:24816; + } + + upstream pulp-api { + server api:24817; + } + + server { + # Gunicorn docs suggest the use of the "deferred" directive on Linux. + listen 5001 default_server deferred; + server_name $hostname; + + # The default client_max_body_size is 1m. Clients uploading + # files larger than this will need to chunk said files. + client_max_body_size 10m; + + # Gunicorn docs suggest this value. + keepalive_timeout 5; + + location /pulp/content/ { + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header Host $http_host; + # we don't want nginx trying to do something clever with + # redirects, we set the Host: header above already. + proxy_redirect off; + proxy_pass http://pulp-content; + } + + location /api/galaxy/api/v3/ { + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header Host $http_host; + # we don't want nginx trying to do something clever with + # redirects, we set the Host: header above already. + proxy_redirect off; + proxy_pass http://pulp-api; + client_max_body_size 0; + } + + location /auth/login/ { + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header Host $http_host; + # we don't want nginx trying to do something clever with + # redirects, we set the Host: header above already. + proxy_redirect off; + proxy_pass http://pulp-api; + } + + # include pulp/*.conf; + + location / { + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header Host $http_host; + # we don't want nginx trying to do something clever with + # redirects, we set the Host: header above already. + proxy_redirect off; + proxy_pass http://pulp-api; + # most pulp static files are served through whitenoise + # http://whitenoise.evans.io/en/stable/ + } + + } +} + diff --git a/profiles/dab/make_test_data.py b/profiles/dab/make_test_data.py index 8e8dd1b5ab..3a1f260e48 100755 --- a/profiles/dab/make_test_data.py +++ b/profiles/dab/make_test_data.py @@ -1,5 +1,6 @@ #!/usr/bin/env python +import os import requests import warnings @@ -9,7 +10,13 @@ HUB_API_ROOT = "https://localhost/api/galaxy/" GW_ROOT_URL = "https://localhost" -ADMIN_AUTH = ('admin', 'redhat1234') + +# 26 export AAP_GATEWAY_ADMIN_USERNAME=admin +# 27 export AAP_GATEWAY_ADMIN_PASSWORD=admin +ADMIN_AUTH = ( + os.environ.get('AAP_GATEWAY_ADMIN_USERNAME', 'admin'), + os.environ.get('AAP_GATEWAY_ADMIN_PASSWORD', 'redhat1234') +) NAMESPACES = ("autohubtest2", "autohubtest3", "signing") USERS = ("notifications_admin", "iqe_normal_user", "jdoe", "org-admin", "iqe_admin", "ee_admin")