From ac7fae95008a5d21382a335b19a30ef62953c9be Mon Sep 17 00:00:00 2001 From: Theresa Kamerman Date: Tue, 8 Aug 2023 13:24:04 -0700 Subject: [PATCH] Restrict worker pool to 2 connections - add test docker compose with 12 total workers --- e2e-tests/docker-compose-many-workers.yml | 209 ++++++++++++++++++ .../merlin/worker/MerlinWorkerAppDriver.java | 1 + .../worker/SchedulerWorkerAppDriver.java | 1 + 3 files changed, 211 insertions(+) create mode 100644 e2e-tests/docker-compose-many-workers.yml diff --git a/e2e-tests/docker-compose-many-workers.yml b/e2e-tests/docker-compose-many-workers.yml new file mode 100644 index 0000000000..9c47c39abb --- /dev/null +++ b/e2e-tests/docker-compose-many-workers.yml @@ -0,0 +1,209 @@ +version: "3.7" +services: + aerie_gateway: + container_name: aerie_gateway + depends_on: ["postgres"] + environment: + AUTH_TYPE: none + AUTH_URL: https://atb-ocio-12b.jpl.nasa.gov:8443/cam-api + GQL_API_URL: http://localhost:8080/v1/graphql + HASURA_GRAPHQL_JWT_SECRET: "${HASURA_GRAPHQL_JWT_SECRET}" + LOG_FILE: console + LOG_LEVEL: debug + NODE_TLS_REJECT_UNAUTHORIZED: "0" + PORT: 9000 + POSTGRES_AERIE_MERLIN_DB: aerie_merlin + POSTGRES_HOST: postgres + POSTGRES_PASSWORD: "${AERIE_PASSWORD}" + POSTGRES_PORT: 5432 + POSTGRES_USER: "${AERIE_USERNAME}" + image: "ghcr.io/nasa-ammos/aerie-gateway:develop" + ports: ["9000:9000"] + restart: always + volumes: + - aerie_file_store:/app/files + aerie_merlin: + build: + context: ../merlin-server + dockerfile: Dockerfile + container_name: aerie_merlin + depends_on: ["postgres"] + environment: + HASURA_GRAPHQL_ADMIN_SECRET: "${HASURA_GRAPHQL_ADMIN_SECRET}" + HASURA_GRAPHQL_URL: http://hasura:8080/v1/graphql + MERLIN_DB: "aerie_merlin" + MERLIN_DB_PASSWORD: "${AERIE_PASSWORD}" + MERLIN_DB_PORT: 5432 + MERLIN_DB_SERVER: postgres + MERLIN_DB_USER: "${AERIE_USERNAME}" + MERLIN_LOCAL_STORE: /usr/src/app/merlin_file_store + MERLIN_PORT: 27183 + JAVA_OPTS: > + -agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=*:5005 + -Dorg.slf4j.simpleLogger.defaultLogLevel=DEBUG + -Dorg.slf4j.simpleLogger.log.com.zaxxer.hikari=INFO + -Dorg.slf4j.simpleLogger.logFile=System.err + UNTRUE_PLAN_START: "2000-01-01T11:58:55.816Z" + image: aerie_merlin + ports: ["27183:27183", "5005:5005"] + restart: always + volumes: + - aerie_file_store:/usr/src/app/merlin_file_store + aerie_scheduler: + build: + context: ../scheduler-server + dockerfile: Dockerfile + container_name: aerie_scheduler + depends_on: ["aerie_merlin", "postgres"] + environment: + HASURA_GRAPHQL_ADMIN_SECRET: "${HASURA_GRAPHQL_ADMIN_SECRET}" + MERLIN_GRAPHQL_URL: http://hasura:8080/v1/graphql + SCHEDULER_DB: "aerie_scheduler" + SCHEDULER_DB_PASSWORD: "${AERIE_PASSWORD}" + SCHEDULER_DB_PORT: 5432 + SCHEDULER_DB_SERVER: postgres + SCHEDULER_DB_USER: "${AERIE_USERNAME}" + SCHEDULER_PORT: 27185 + JAVA_OPTS: > + -agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=*:5005 + -Dorg.slf4j.simpleLogger.defaultLogLevel=DEBUG + -Dorg.slf4j.simpleLogger.log.com.zaxxer.hikari=INFO + -Dorg.slf4j.simpleLogger.logFile=System.err + image: aerie_scheduler + ports: ["27185:27185", "5006:5005"] + restart: always + volumes: + - aerie_file_store:/usr/src/app/merlin_file_store + aerie_sequencing: + build: + context: ../sequencing-server + dockerfile: Dockerfile.dev + container_name: aerie_sequencing + depends_on: ["postgres"] + environment: + HASURA_GRAPHQL_ADMIN_SECRET: "${HASURA_GRAPHQL_ADMIN_SECRET}" + LOG_FILE: console + LOG_LEVEL: debug + MERLIN_GRAPHQL_URL: http://hasura:8080/v1/graphql + SEQUENCING_SERVER_PORT: 27184 + SEQUENCING_DB: aerie_sequencing + SEQUENCING_DB_PASSWORD: "${AERIE_PASSWORD}" + SEQUENCING_DB_PORT: 5432 + SEQUENCING_DB_SERVER: postgres + SEQUENCING_DB_USER: "${AERIE_USERNAME}" + SEQUENCING_LOCAL_STORE: /usr/src/app/sequencing_file_store + image: aerie_sequencing + ports: ["27184:27184"] + restart: always + volumes: + - ./sequencing-server:/app:cached + - aerie_file_store:/usr/src/app/sequencing_file_store + aerie_ui: + container_name: aerie_ui + depends_on: ["postgres"] + environment: + NODE_TLS_REJECT_UNAUTHORIZED: "0" + PUBLIC_LOGIN_PAGE: "enabled" + ORIGIN: http://localhost + PUBLIC_GATEWAY_CLIENT_URL: http://localhost:9000 + PUBLIC_GATEWAY_SERVER_URL: http://aerie_gateway:9000 + PUBLIC_HASURA_CLIENT_URL: http://localhost:8080/v1/graphql + PUBLIC_HASURA_SERVER_URL: http://hasura:8080/v1/graphql + PUBLIC_HASURA_WEB_SOCKET_URL: ws://localhost:8080/v1/graphql + image: "ghcr.io/nasa-ammos/aerie-ui:develop" + ports: ["80:80"] + restart: always + aerie_merlin_workers: + build: + context: ../merlin-worker + dockerfile: Dockerfile + deploy: + replicas: 8 + depends_on: [ "postgres" ] + environment: + MERLIN_WORKER_DB: "aerie_merlin" + MERLIN_WORKER_DB_PASSWORD: "${AERIE_PASSWORD}" + MERLIN_WORKER_DB_PORT: 5432 + MERLIN_WORKER_DB_SERVER: postgres + MERLIN_WORKER_DB_USER: "${AERIE_USERNAME}" + MERLIN_WORKER_LOCAL_STORE: /usr/src/app/merlin_file_store + SIMULATION_PROGRESS_POLL_PERIOD_MILLIS: 2000 + JAVA_OPTS: > + -Dorg.slf4j.simpleLogger.defaultLogLevel=DEBUG + -Dorg.slf4j.simpleLogger.log.com.zaxxer.hikari=INFO + -Dorg.slf4j.simpleLogger.logFile=System.err + UNTRUE_PLAN_START: "2000-01-01T11:58:55.816Z" + image: "aerie_merlin_worker_1" + restart: always + volumes: + - aerie_file_store:/usr/src/app/merlin_file_store:ro + aerie_scheduler_workers: + build: + context: ../scheduler-worker + dockerfile: Dockerfile + deploy: + replicas: 8 + depends_on: ["postgres"] + environment: + HASURA_GRAPHQL_ADMIN_SECRET: "${HASURA_GRAPHQL_ADMIN_SECRET}" + MERLIN_GRAPHQL_URL: http://hasura:8080/v1/graphql + SCHEDULER_WORKER_DB: "aerie_scheduler" + SCHEDULER_WORKER_DB_PASSWORD: "${AERIE_PASSWORD}" + SCHEDULER_WORKER_DB_PORT: 5432 + SCHEDULER_WORKER_DB_SERVER: postgres + SCHEDULER_WORKER_DB_USER: "${AERIE_USERNAME}" + SCHEDULER_OUTPUT_MODE: UpdateInputPlanWithNewActivities + MERLIN_LOCAL_STORE: /usr/src/app/merlin_file_store + SCHEDULER_RULES_JAR: /usr/src/app/merlin_file_store/scheduler_rules.jar + JAVA_OPTS: > + -Dorg.slf4j.simpleLogger.defaultLogLevel=DEBUG + -Dorg.slf4j.simpleLogger.log.com.zaxxer.hikari=INFO + -Dorg.slf4j.simpleLogger.logFile=System.err + image: "aerie_scheduler_worker_1" + restart: always + volumes: + - aerie_file_store:/usr/src/app/merlin_file_store:ro + hasura: + container_name: aerie_hasura + depends_on: ["postgres"] + environment: + AERIE_MERLIN_DATABASE_URL: "postgres://${AERIE_USERNAME}:${AERIE_PASSWORD}@postgres:5432/aerie_merlin" + AERIE_MERLIN_URL: "http://aerie_merlin:27183" + AERIE_SCHEDULER_DATABASE_URL: "postgres://${AERIE_USERNAME}:${AERIE_PASSWORD}@postgres:5432/aerie_scheduler" + AERIE_SCHEDULER_URL: "http://aerie_scheduler:27185" + AERIE_SEQUENCING_DATABASE_URL: "postgres://${AERIE_USERNAME}:${AERIE_PASSWORD}@postgres:5432/aerie_sequencing" + AERIE_SEQUENCING_URL: "http://aerie_sequencing:27184" + AERIE_UI_DATABASE_URL: "postgres://${AERIE_USERNAME}:${AERIE_PASSWORD}@postgres:5432/aerie_ui" + HASURA_GRAPHQL_ADMIN_SECRET: "${HASURA_GRAPHQL_ADMIN_SECRET}" + HASURA_GRAPHQL_DEV_MODE: "true" + HASURA_GRAPHQL_ENABLE_CONSOLE: "true" + HASURA_GRAPHQL_ENABLED_LOG_TYPES: startup, http-log, webhook-log, websocket-log, query-log + HASURA_GRAPHQL_INFER_FUNCTION_PERMISSIONS: false + HASURA_GRAPHQL_JWT_SECRET: "${HASURA_GRAPHQL_JWT_SECRET}" + HASURA_GRAPHQL_LOG_LEVEL: info + HASURA_GRAPHQL_METADATA_DATABASE_URL: "postgres://${AERIE_USERNAME}:${AERIE_PASSWORD}@postgres:5432/aerie_hasura" + HASURA_GRAPHQL_METADATA_DIR: /hasura-metadata + image: "hasura/graphql-engine:v2.12.1.cli-migrations-v3" + ports: ["8080:8080"] + restart: always + volumes: + - ./deployment/hasura/metadata:/hasura-metadata + postgres: + container_name: aerie_postgres + environment: + POSTGRES_DB: postgres + POSTGRES_PASSWORD: "${POSTGRES_PASSWORD}" + POSTGRES_USER: "${POSTGRES_USER}" + AERIE_USERNAME: "${AERIE_USERNAME}" + AERIE_PASSWORD: "${AERIE_PASSWORD}" + image: postgres:14.8 + ports: ["5432:5432"] + restart: always + volumes: + - postgres_data:/var/lib/postgresql/data + - ./deployment/postgres-init-db:/docker-entrypoint-initdb.d + +volumes: + aerie_file_store: + mission_file_store: + postgres_data: diff --git a/merlin-worker/src/main/java/gov/nasa/jpl/aerie/merlin/worker/MerlinWorkerAppDriver.java b/merlin-worker/src/main/java/gov/nasa/jpl/aerie/merlin/worker/MerlinWorkerAppDriver.java index 1283fd8365..7a24565db2 100644 --- a/merlin-worker/src/main/java/gov/nasa/jpl/aerie/merlin/worker/MerlinWorkerAppDriver.java +++ b/merlin-worker/src/main/java/gov/nasa/jpl/aerie/merlin/worker/MerlinWorkerAppDriver.java @@ -38,6 +38,7 @@ public static void main(String[] args) throws InterruptedException { hikariConfig.addDataSourceProperty("applicationName", "Merlin Server"); hikariConfig.setUsername(postgresStore.user()); hikariConfig.setPassword(postgresStore.password()); + hikariConfig.setMaximumPoolSize(2); hikariConfig.setConnectionInitSql("set time zone 'UTC'"); diff --git a/scheduler-worker/src/main/java/gov/nasa/jpl/aerie/scheduler/worker/SchedulerWorkerAppDriver.java b/scheduler-worker/src/main/java/gov/nasa/jpl/aerie/scheduler/worker/SchedulerWorkerAppDriver.java index 25a2ad59a4..0cc7cbd3df 100644 --- a/scheduler-worker/src/main/java/gov/nasa/jpl/aerie/scheduler/worker/SchedulerWorkerAppDriver.java +++ b/scheduler-worker/src/main/java/gov/nasa/jpl/aerie/scheduler/worker/SchedulerWorkerAppDriver.java @@ -51,6 +51,7 @@ public static void main(String[] args) throws Exception { hikariConfig.addDataSourceProperty("applicationName", "Scheduler Worker"); hikariConfig.setUsername(postgresStore.user()); hikariConfig.setPassword(postgresStore.password()); + hikariConfig.setMaximumPoolSize(2); hikariConfig.setConnectionInitSql("set time zone 'UTC'");