Make a simple/self-contained docker-compose to use for AAP dev. (#2266)

* Make a simple docker-compose to use for AAP dev. * Don't bind postgres to the host. * Don't add the compose to the manifest. * Fix some issues. * Fix the file storage setting. * Disable signing and auto-signing for now. No-Issue Signed-off-by: James Tanner <[email protected]>
ansible · Sep 18, 2024 · 969c1a7 · 969c1a7
1 parent 32fe176
commit 969c1a7
Show file tree

Hide file tree

Showing 4 changed files with 332 additions and 1 deletion.
diff --git a/MANIFEST.in b/MANIFEST.in
@@ -24,3 +24,4 @@ include galaxy_ng/tests/integration/utils/gpg/qe-sign-priv.gpg
 include galaxy-operator/bin/readyz.py
 exclude .coveragerc
 exclude .sonarcloud.properties
+exclude aap_compose_dev.yaml
diff --git a/aap_compose_dev.yaml b/aap_compose_dev.yaml
@@ -0,0 +1,231 @@
+x-common-env: &common-env
+
+  DJANGO_SUPERUSER_USERNAME: admin
+  DJANGO_SUPERUSER_EMAIL: [email protected]
+  DJANGO_SUPERUSER_PASSWORD: admin
+
+  POSTGRES_USER: galaxy_ng
+  POSTGRES_PASSWORD: galaxy_ng
+  POSTGRES_DB: galaxy_ng
+
+  # no spying
+  PULP_ANALYTICS: 'false'
+
+  # normally goes into settings.py ...
+  PULP_DATABASES__default__ENGINE: django.db.backends.postgresql
+  PULP_DATABASES__default__NAME: galaxy_ng
+  PULP_DATABASES__default__USER: galaxy_ng
+  PULP_DATABASES__default__PASSWORD: galaxy_ng
+  PULP_DATABASES__default__HOST: postgres
+  PULP_DATABASES__default__PORT: 5432
+
+  PULP_DEBUG: 1
+  PULP_GALAXY_DEPLOYMENT_MODE: 'standalone'
+  PULP_DEFAULT_FILE_STORAGE: "pulpcore.app.models.storage.FileSystem"
+
+  # Hostname and prefix has to be correct
+  PULP_GALAXY_API_PATH_PREFIX: '/api/galaxy/'
+  PULP_ANSIBLE_API_HOSTNAME: 'https://localhost'
+  PULP_ANSIBLE_CONTENT_HOSTNAME: "https://localhost/pulp/content"
+  PULP_CONTENT_ORIGIN: "https://localhost"
+  PULP_CSRF_TRUSTED_ORIGINS: "['https://localhost']"
+
+  # signing ...
+  ENABLE_SIGNING: 0
+  PULP_GALAXY_AUTO_SIGN_COLLECTIONS: 'false'
+  PULP_GALAXY_REQUIRE_CONTENT_APPROVAL: 'true'
+  PULP_GALAXY_REQUIRE_SIGNATURE_FOR_APPROVAL: 'false'
+  PULP_GALAXY_COLLECTION_SIGNING_SERVICE: 'ansible-default'
+  PULP_GALAXY_CONTAINER_SIGNING_SERVICE: 'container-default'
+
+  # pulp container ...
+  PULP_TOKEN_AUTH_DISABLED: 'false'
+  PULP_TOKEN_SERVER: 'https://localhost/token/'
+  PULP_TOKEN_SIGNATURE_ALGORITHM: 'ES256'
+  PULP_PUBLIC_KEY_PATH: '/src/galaxy_ng/dev/common/container_auth_public_key.pem'
+  PULP_PRIVATE_KEY_PATH: '/src/galaxy_ng/dev/common/container_auth_private_key.pem'
+
+  # auth ...
+  PULP_GALAXY_AUTHENTICATION_CLASSES: "['galaxy_ng.app.auth.session.SessionAuthentication', 'ansible_base.jwt_consumer.hub.auth.HubJWTAuth', 'rest_framework.authentication.TokenAuthentication', 'rest_framework.authentication.BasicAuthentication']"
+  PULP_ANSIBLE_BASE_JWT_VALIDATE_CERT: 'false'
+  PULP_ANSIBLE_BASE_JWT_KEY: 'https://localhost'
+  PULP_GALAXY_FEATURE_FLAGS__external_authentication: 'true'
+
+  # disable user/group modifications
+  PULP_ALLOW_LOCAL_RESOURCE_MANAGEMENT: 'false'
+
+  # role content workaround ..
+  PULP_ANSIBLE_BASE_ROLES_REQUIRE_VIEW: 'false'
+
+
+services:
+  base_img:
+    build:
+      context: .
+      dockerfile: Dockerfile
+    #image: galaxy_base_image:latest
+    image: "localhost/galaxy_ng/galaxy_ng:base"
+
+  redis:
+    image: "redis:5"
+    ports:
+      - "6379:6379"
+
+  postgres:
+    image: "postgres:13"
+    #ports:
+    #  - "5432:5432"
+    environment:
+      <<: *common-env
+    healthcheck:
+      test: ["CMD", "pg_isready", "-U", "galaxy_ng"]
+      interval: 10s
+      retries: 5
+
+  helper:
+    image: quay.io/centos/centos:stream9
+    environment:
+      <<: *common-env
+    depends_on:
+      - postgres
+    volumes:
+      - "etc_pulp_certs:/etc/pulp/certs"
+      - "var_lib_pulp:/var/lib/pulp"
+    command: |
+      bash -c "
+        if [[ ! -e /etc/pulp/certs/database_fields.symmetric.key ]] || [[ -s /etc/pulp/certs/database_fields.symmetric.key ]]; then
+          mkdir -p /etc/pulp/certs/;
+          echo 'check openssl and install ...';
+          rpm -q openssl || dnf -y install openssl;
+          echo 'generate key ...';
+          openssl rand -base64 32 > /etc/pulp/certs/database_fields.symmetric.key;
+          echo 'chown key ...';
+          chmod 640 /etc/pulp/certs/database_fields.symmetric.key;
+        else
+          echo 'symmetric key exists'
+        fi;
+        find /etc/pulp ;
+        echo '# KEY >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>';
+        cat /etc/pulp/certs/database_fields.symmetric.key;
+        echo '<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<';
+        echo "DONE!";
+      "
+
+  migrations:
+    image: "localhost/galaxy_ng/galaxy_ng:base"
+    depends_on:
+      - base_img
+      - postgres
+      - helper
+    volumes:
+      - "etc_pulp_certs:/etc/pulp/certs"
+      - "var_lib_pulp:/var/lib/pulp"
+      - ".:/src/galaxy_ng"
+    environment:
+      <<: *common-env
+    user: root
+    command: |
+      bash -c "
+        set -e;
+        while [[ ! -f /etc/pulp/certs/database_fields.symmetric.key ]]; do
+          echo 'Waiting for key';
+          sleep 2;
+        done;
+        pulpcore-manager check --database default;
+        pulpcore-manager migrate;
+        touch /var/lib/pulp/.migrated;
+        pulpcore-manager shell < /src/galaxy_ng/dev/common/setup_test_data.py;
+        pulpcore-manager createsuperuser --noinput;
+      "
+
+  api:
+    image: "localhost/galaxy_ng/galaxy_ng:base"
+    depends_on:
+      - base_img
+      - postgres
+      - helper
+      - migrations
+    volumes:
+      - "etc_pulp_certs:/etc/pulp/certs"
+      - "var_lib_pulp:/var/lib/pulp"
+      - ".:/src/galaxy_ng"
+    environment:
+      <<: *common-env
+    extra_hosts:
+      localhost: "host-gateway"
+    command: |
+      bash -c "
+        while [[ ! -f /var/lib/pulp/.migrated ]]; do
+          echo 'Waiting for migrations ...';
+          sleep 2;
+        done && exec pulpcore-api --reload;
+      "
+
+  content:
+    image: "localhost/galaxy_ng/galaxy_ng:base"
+    depends_on:
+      - base_img
+      - postgres
+      - helper
+      - migrations
+    volumes:
+      - "etc_pulp_certs:/etc/pulp/certs"
+      - "var_lib_pulp:/var/lib/pulp"
+      - ".:/src/galaxy_ng"
+    environment:
+      <<: *common-env
+    extra_hosts:
+      localhost: "host-gateway"
+    command: |
+      bash -c "
+        while [[ ! -f /var/lib/pulp/.migrated ]]; do
+          echo 'Waiting for migrations ...';
+          sleep 2;
+        done && exec pulpcore-content --reload;
+      "
+
+  worker:
+    image: "localhost/galaxy_ng/galaxy_ng:base"
+    depends_on:
+      - base_img
+      - postgres
+      - helper
+      - migrations
+    volumes:
+      - "etc_pulp_certs:/etc/pulp/certs"
+      - "var_lib_pulp:/var/lib/pulp"
+      - ".:/src/galaxy_ng"
+    environment:
+      <<: *common-env
+    command: |
+      bash -c "
+        while [[ ! -f /var/lib/pulp/.migrated ]]; do
+          echo 'Waiting for migrations ...';
+          sleep 2;
+        done && exec pulpcore-worker;
+      "
+
+  nginx:
+    image: "nginx:latest"
+    depends_on:
+      - base_img
+      - postgres
+      - helper
+      - migrations
+      - api
+      - content
+    ports:
+      - '5001:5001'
+    volumes:
+      - './dev/nginx/nginx.conf:/etc/nginx/nginx.conf:ro'
+
+volumes:
+  var_lib_pulp:
+    name: var_lib_pulp 
+  etc_pulp_certs:
+    name: etc_pulp_certs 
+
+#networks:
+#  dbnet:
+#  service-mesh:
+#    name: service-mesh
diff --git a/dev/nginx/nginx.conf b/dev/nginx/nginx.conf
@@ -0,0 +1,92 @@
+# TODO: Support IPv6.
+# TODO: Maybe serve multiple `location`s, not just one.
+
+# The "nginx" package on fedora creates this user and group.
+# user nginx nginx;
+
+# Gunicorn docs suggest this value.
+worker_processes 1;
+# daemon off;
+events {
+    worker_connections 1024;  # increase if you have lots of clients
+    accept_mutex off;  # set to 'on' if nginx worker_processes > 1
+}
+
+http {
+    include mime.types;
+    # fallback in case we can't determine a type
+    default_type application/octet-stream;
+    sendfile on;
+
+    # If left at the default of 1024, nginx emits a warning about being unable
+    # to build optimal hash types.
+    types_hash_max_size 4096;
+
+    upstream pulp-content {
+         server content:24816;
+    }
+
+    upstream pulp-api {
+         server api:24817;
+    }
+
+    server {
+        # Gunicorn docs suggest the use of the "deferred" directive on Linux.
+        listen 5001 default_server deferred;
+        server_name $hostname;
+
+        # The default client_max_body_size is 1m. Clients uploading
+        # files larger than this will need to chunk said files.
+        client_max_body_size 10m;
+
+        # Gunicorn docs suggest this value.
+        keepalive_timeout 5;
+
+        location /pulp/content/ {
+            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+            proxy_set_header X-Forwarded-Proto $scheme;
+            proxy_set_header Host $http_host;
+            # we don't want nginx trying to do something clever with
+            # redirects, we set the Host: header above already.
+            proxy_redirect off;
+            proxy_pass http://pulp-content;
+        }
+
+        location /api/galaxy/api/v3/ {
+            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+            proxy_set_header X-Forwarded-Proto $scheme;
+            proxy_set_header Host $http_host;
+            # we don't want nginx trying to do something clever with
+            # redirects, we set the Host: header above already.
+            proxy_redirect off;
+            proxy_pass http://pulp-api;
+            client_max_body_size 0;
+        }
+
+        location /auth/login/ {
+            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+            proxy_set_header X-Forwarded-Proto $scheme;
+            proxy_set_header Host $http_host;
+            # we don't want nginx trying to do something clever with
+            # redirects, we set the Host: header above already.
+            proxy_redirect off;
+            proxy_pass http://pulp-api;
+        }
+
+        # include pulp/*.conf;
+
+        location / {
+            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+            proxy_set_header X-Forwarded-Proto $scheme;
+            proxy_set_header Host $http_host;
+            # we don't want nginx trying to do something clever with
+            # redirects, we set the Host: header above already.
+            proxy_redirect off;
+            proxy_pass http://pulp-api;
+            # most pulp static files are served through whitenoise
+            # http://whitenoise.evans.io/en/stable/
+        }
+
+    }
+}
+
diff --git a/profiles/dab/make_test_data.py b/profiles/dab/make_test_data.py
@@ -1,5 +1,6 @@
 #!/usr/bin/env python
 
+import os
 import requests
 import warnings
 
@@ -9,7 +10,13 @@
 
 HUB_API_ROOT = "https://localhost/api/galaxy/"
 GW_ROOT_URL = "https://localhost"
-ADMIN_AUTH = ('admin', 'redhat1234')
+
+# 26 export AAP_GATEWAY_ADMIN_USERNAME=admin
+# 27 export AAP_GATEWAY_ADMIN_PASSWORD=admin
+ADMIN_AUTH = (
+    os.environ.get('AAP_GATEWAY_ADMIN_USERNAME', 'admin'),
+    os.environ.get('AAP_GATEWAY_ADMIN_PASSWORD', 'redhat1234')
+)
 
 NAMESPACES = ("autohubtest2", "autohubtest3", "signing")
 USERS = ("notifications_admin", "iqe_normal_user", "jdoe", "org-admin", "iqe_admin", "ee_admin")