diff --git a/installers/esg/.env b/installers/esg/.env new file mode 100644 index 000000000..a45164c25 --- /dev/null +++ b/installers/esg/.env @@ -0,0 +1,68 @@ +# Compose +COMPOSE_PROJECT_NAME=legend + +# MongoDB +MONGODB_IMAGE_VERSION=4.2 +MONGODB_NAME=legend +MONGODB_USERNAME=admin +MONGODB_PASSWORD=password +MONGODB_PORT=27017 +MONGODB_URI=mongodb://admin:password@legend-mongodb:27017 + +# Session management +MONGO_SESSION_ENABLED=true + +# Gitlab OAuth configuration +GITLAB_HOST=gitlab.com +GITLAB_APP_ID=${GITLAB_APP_ID:?err} +GITLAB_APP_SECRET=${GITLAB_APP_SECRET:?err} + +# Common +LEGEND_EXTERNAL_BASE_URL=https://d7nifimeibcde.cloudfront.net + +# Engine +ENGINE_IMAGE_VERSION=4.33.0 +ENGINE_MAVEN_VERSION=3.15.3 +ENGINE_HOST=localhost +ENGINE_PORT=6300 +METADATA_PURE_HOST=localhost +METADATA_PURE_PORT=6301 +METADATA_ALLOY_HOST=localhost +METADATA_ALLOY_PORT=6200 +TEMP_DB_PORT=6302 + +# SDLC +SDLC_IMAGE_VERSION=0.142.0 +SDLC_MAVEN_VERSION=0.91.1 +SDLC_HOST=localhost +SDLC_PORT=6100 +SDLC_ADMIN_PORT=6101 +SDLC_REDIRECT_URI=http://localhost:6100/api/auth/callback +SDLC_PROJECT_TAG=legend +SDLC_PROJECT_ID_PREFIX=^org\.demo\.legend\..+ + +# Studio +STUDIO_IMAGE_VERSION=10.23.0 +STUDIO_HOST=localhost +STUDIO_PORT=9000 + +# Depot store +DEPOT_STORE_IMAGE_VERSION=1.5.3 +DEPOT_STORE_HOST=localhost +DEPOT_STORE_PORT=6201 +DEPOT_STORE_ADMIN_USER=${DEPOT_STORE_ADMIN_USER:-nobody} + +# Depot +DEPOT_IMAGE_VERSION=1.5.3 +DEPOT_HOST=localhost +DEPOT_PORT=6200 + +# Query +QUERY_IMAGE_VERSION=8.38.0 +QUERY_HOST=localhost +QUERY_PORT=9001 + +# Postgres +POSTGRES_IMAGE_VERSION=alpine3.17 + +SETUP_DIR=$PWD/setup diff --git a/installers/esg/README.md b/installers/esg/README.md new file mode 100644 index 000000000..c402b3b76 --- /dev/null +++ b/installers/esg/README.md @@ -0,0 +1,145 @@ +# Overview + +This directory contains a Docker Compose specification that can be used to spin up an instance of Legend Studio and Legend Query. + +# Assumptions + +## Localhost + +This installer assumes that localhost resolves to where the various Legend JVMs are running and localhost can be reached from a browser. + +If this is not the case, for e.g, Legend containers are being run on a machine accessible only by an IP or other names, all references to localhost, both in the .env and Gitlab OAuth configuration has to be changed. + +## Gitlab.com + +Out of the box, Legend Studio uses Gitlab for model version control. The installer uses the public gitlab.com instance. + +If you do not wish to use gitlab.com, you can use any Gitlab instance. Make sure to change all the GITHUB variables in .env to point to your Gitlab instance. + +# Usage Instructions + +## Create a Gitlab.com account + +Legend uses Gitlab as the identity provider. Create a user account at https://gitlab.com + +## Create a Gitlab OAuth application + +Create an OAuth application as described here https://docs.gitlab.com/ee/integration/oauth_provider.html + +The OAuth application should be configured as follows : + +- Redirect URI: + +``` +http://localhost:6300/callback +http://localhost:6100/api/auth/callback +http://localhost:6100/api/pac4j/login/callback +http://localhost:6201/depot-store/callback +http://localhost:6200/depot/callback +http://localhost:9000/studio/log.in/callback +http://localhost:9001/query/log.in/callback +``` + +- Enable the "Confidential" check box +- Enable these scopes: openid, profile, api + +Save the application and record the application id and secret. + +## Set the app id and secret of your Gitlab application + +``` +export GITLAB_APP_ID= +export GITLAB_APP_SECRET= +``` + +## Studio + +Start Studio as follows. + +``` +./docker-compose.sh --profile studio up -d +``` + +After a few minutes, the containers should pass their health checks and be marked as healthy. + +``` + +./docker-compose.sh ps + Name Command State Ports +--------------------------------------------------------------------------------------------- +legend-engine /bin/sh -c java -cp /app/b ... Up (healthy) 0.0.0.0:6300->6300/tcp +legend-mongodb docker-entrypoint.sh --auth Up 0.0.0.0:27017->27017/tcp +legend-sdlc /bin/sh -c java -cp /app/b ... Up (healthy) 0.0.0.0:6100->6100/tcp +legend-studio /bin/sh -c java -cp /app/b ... Up (healthy) 0.0.0.0:9000->9000/tcp +setup /setup/setup.sh Exit 0``` +``` + +### Use Studio + +Once all containers are running, you should be able to access Studio at `http://localhost:9000/studio` + +When accessing Studio for the first time, you will see a URL redirect, redirecting you to Gitlab.com to authorize the Legend OAuth application. + +Once authorized, you should be able to start using Studio. + + +### Open a sample project + +In the opening page of Studio, select the "Legend Installer Demo" project or navigate directly to the project using this link http://localhost:9000/studio/setup/40061958 + +## Query + +Run Query as follows : +``` +./docker-compose.sh --profile query up -d +``` + +After a few minutes, the containers should pass their health checks and be marked as healthy. + +``` +./docker-compose.sh ps + Name Command State Ports +--------------------------------------------------------------------------------------------- +legend-depot /app/entrypoint.sh Up (healthy) 0.0.0.0:6200->6200/tcp +legend-depot-store /app/entrypoint.sh Up (healthy) 0.0.0.0:6201->6201/tcp +legend-engine /bin/sh -c java -cp /app/b ... Up (healthy) 0.0.0.0:6300->6300/tcp +legend-mongodb docker-entrypoint.sh --auth Up 0.0.0.0:27017->27017/tcp +legend-query /bin/sh -c java -cp /app/b ... Up (healthy) 0.0.0.0:9001->9001/tcp +legend-sdlc /bin/sh -c java -cp /app/b ... Up (healthy) 0.0.0.0:6100->6100/tcp +legend-studio /bin/sh -c java -cp /app/b ... Up (healthy) 0.0.0.0:9000->9000/tcp +setup /setup/setup.sh Exit 0 + +``` + +### Use Query + +Once all containers are running, you should be able to access Studio at `http://localhost:9001/query` + +When accessing Studio for the first time, you will see a URL redirect, redirecting you to Gitlab.com to authorize the Legend OAuth application. + +Once authorized, you should be able to start using Query. + +### Index Projects + +Query allows you to access projects that have been indexed by the depot store server. + +We have automatically indexed a project into the depot store. If this project is not visible in query, execute the following command. + +``` +curl -v -X GET "http://localhost:6201/depot-store/api/queue/PROD-1234/org.finos.legend.demo/legend-query/1.0.2?maxRetries=5" -H "accept: text/plain" +``` + +# Swagger + +Each component exposes a Swagger endpoint that can be used to explore the component's API. + +| Component | Endpoint | +| ------ | ---------| +| Engine | http://localhost:6300/api/swagger# | +| SDLC Server | http://localhost:6100/api/swagger# | +| Depot Store Server | http://localhost:6201/depot-store/api/swagger# | +| Depot Server | http://localhost:6200/depot/api/swagger# | + +# Known Issues / Gotchas + +In some cases, navigating to the Studio/Query web page can return an "unauthorized" error. This is usually because of stale cookies. Clear browser cookies for localhost (or domain name/IP that you are using) and try again. diff --git a/installers/esg/depot-store/config/authorisedIdentities.json b/installers/esg/depot-store/config/authorisedIdentities.json new file mode 100644 index 000000000..1eeb1f20e --- /dev/null +++ b/installers/esg/depot-store/config/authorisedIdentities.json @@ -0,0 +1,7 @@ +{ + "Artifacts": ["${DEPOT_STORE_ADMIN_USER}"], + "Store Administration": ["${DEPOT_STORE_ADMIN_USER}"], + "Projects": ["${DEPOT_STORE_ADMIN_USER}"], + "Notifications": ["${DEPOT_STORE_ADMIN_USER}"], + "Entities": ["${DEPOT_STORE_ADMIN_USER}"] +} diff --git a/installers/esg/depot-store/config/config.json b/installers/esg/depot-store/config/config.json new file mode 100644 index 000000000..48a3071ea --- /dev/null +++ b/installers/esg/depot-store/config/config.json @@ -0,0 +1,109 @@ +{ + "applicationName": "Depot Store Manager API", + "deployment": "DEV", + "sessionCookie": "LEGEND_DEPOT_STORE_JSESSIONID", + "urlPattern": "/depot-store/api/*", + "server": { + "type": "simple", + "applicationContextPath": "/", + "adminContextPath": "/admin", + "connector": { + "type": "http", + "port": ${DEPOT_STORE_PORT}, + "maxRequestHeaderSize": "32KiB" + }, + "gzip": { + "includedMethods": [ + "GET", + "POST" + ] + }, + "requestLog": { + "appenders": [ + { + "type": "console", + "filterFactories": [ + { + "type": "healthcheck-filter-factory" + } + ] + } + ] + } + }, + "mongo": { + "url": "${MONGODB_URI}", + "database": "depot" + }, + "openTracing": { + "openTracingUri": "your URL here", + "serviceName": "legend-depot-store", + "enabled": false + }, + "logging": { + "level": "INFO", + "loggers": { + "org.jboss.shrinkwrap.resolver": "off", + "org.eclipse.aether": "off", + "Legend Depot Store Manager": { + "level": "info", + "appenders": [ + { + "type": "console", + "logFormat": "%msg\r\n" + } + ] + } + } + }, + "swagger": { + "resourcePackage": "org.finos.legend.depot.store", + "title": "Legend Depot Store Manager", + "uriPrefix": "/depot-store/api" + }, + "pac4j": { + "callbackPrefix": "/depot-store", + "bypassPaths": [ + "/depot-store/api/info" + ], + "bypassBranches": [ + "/depot-store/api/queue" + ], + "clients": [ + { + "org.finos.legend.server.pac4j.gitlab.GitlabClient": { + "name": "gitlab", + "clientId": "${GITLAB_APP_ID}", + "secret": "${GITLAB_APP_SECRET}", + "discoveryUri": "https://${GITLAB_HOST}/.well-known/openid-configuration", + "scope": "openid profile api" + } + } + ], + "mongoUri": "${MONGODB_URI}", + "mongoDb": "${MONGODB_NAME}", + "mongoSession": { + "enabled": "${MONGO_SESSION_ENABLED}", + "collection": "userSessions" + } + }, + "filterPriorities": { + "Username": 1, + "OpenTracingFilter": 2, + "org.pac4j.j2e.filter.SecurityFilter": 3, + "org.pac4j.j2e.filter.CallbackFilter": 4, + "CORS": 5 + }, + "artifactRepositoryProviderConfiguration": { + + "org.finos.legend.depot.artifacts.repository.maven.impl.MavenArtifactRepositoryConfiguration": { + "settingsLocation": "/config/settings.xml", + "versionsUpdateIntervalInMillis": 3600000, + "latestUpdateIntervalInMillis": 3600000 + } + }, + "includeProjectPropertiesConfiguration": { + "properties": [] + }, + "queue-interval": 30 +} diff --git a/installers/esg/depot-store/config/settings.xml b/installers/esg/depot-store/config/settings.xml new file mode 100644 index 000000000..0a2ddbf31 --- /dev/null +++ b/installers/esg/depot-store/config/settings.xml @@ -0,0 +1,75 @@ + + + + + target/.m2/repository + + + false + false + + + + true + + + + 13480401 + https://gitlab.com/api/v4/groups/13480401/-/packages/maven + + true + never + + + true + always + + + + 12998827 + https://gitlab.com/api/v4/groups/12998827/-/packages/maven + + true + never + + + true + always + + + + + + + + 12998827 + + + ... + + + 13480401 + + ... + + + + diff --git a/installers/esg/depot-store/setup/setup.sh b/installers/esg/depot-store/setup/setup.sh new file mode 100755 index 000000000..350fa03c1 --- /dev/null +++ b/installers/esg/depot-store/setup/setup.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +sleep 5 + +curl -v -X GET "http://legend-depot-store:${DEPOT_STORE_PORT}/depot-store/api/queue/PROD-1234/org.finos.legend.demo/legend-query/1.0.2?maxRetries=5" -H "accept: text/plain" diff --git a/installers/esg/docker-compose.sh b/installers/esg/docker-compose.sh new file mode 100755 index 000000000..b1b0816cc --- /dev/null +++ b/installers/esg/docker-compose.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +U_ID=`id -u` G_ID=`id -g` docker-compose $* diff --git a/installers/esg/docker-compose.yml b/installers/esg/docker-compose.yml new file mode 100644 index 000000000..fe26e974a --- /dev/null +++ b/installers/esg/docker-compose.yml @@ -0,0 +1,308 @@ +version: "3.9" +services: + # Run a container that runs the setup script and generates various config files needed by the other containers + # The current dir on the host is mounted into the container as a writable volume. Any writes by the setup script are now visible on the host + # To avoid the files generated inside the container from being owned as root, the container is started with the user/group id of the current user (on the host) + setup: + profiles: ["setup", "engine", "sdlc", "depot", "query", "studio", "postgres"] + container_name: "setup" + image: finos/legend-engine-server:${ENGINE_IMAGE_VERSION} + user: ${U_ID}:${G_ID} + volumes: + - .:/setup:rw + - /etc/group:/etc/group:ro + - /etc/passwd:/etc/passwd:ro + - /etc/shadow:/etc/shadow:ro + entrypoint: /setup/setup.sh + env_file: .env + environment: + - GITLAB_HOST=${GITLAB_HOST} + - GITLAB_APP_ID=${GITLAB_APP_ID} + - GITLAB_APP_SECRET=${GITLAB_APP_SECRET} + - ENGINE_HOST=${ENGINE_HOST} + - MONGODB_URI=${MONGODB_URI} + - MONGODB_NAME=${MONGODB_NAME} + - MONGO_SESSION_ENABLED=${MONGO_SESSION_ENABLED} + - METADATA_PURE_HOST=${METADATA_PURE_HOST} + - METADATA_PURE_PORT=${METADATA_PURE_PORT} + - METADATA_ALLOY_HOST=${METADATA_ALLOY_HOST} + - METADATA_ALLOY_PORT=${METADATA_ALLOY_PORT} + - DEPOT_STORE_HOST=${DEPOT_STORE_HOST} + - DEPOT_STORE_PORT=${DEPOT_STORE_PORT} + mongodb: + profiles: ["engine", "sdlc", "depot", "query", "studio"] + container_name: "legend-mongodb" + image: mongo:${MONGODB_IMAGE_VERSION} + ports: + - ${MONGODB_PORT}:${MONGODB_PORT} + networks: + - legend + environment: + MONGO_INITDB_ROOT_PASSWORD: ${MONGODB_PASSWORD} + MONGO_INITDB_ROOT_USERNAME: ${MONGODB_USERNAME} + env_file: .env + command: --auth + # Servers + legend-engine: + profiles: ["engine", "query", "studio"] + depends_on: + - setup + - mongodb + container_name: "legend-engine" + image: blacksteed232/legend-engine-server:gic-pat-v2 + env_file: .env + environment: + - GITLAB_HOST=${GITLAB_HOST} + - GITLAB_APP_ID=${GITLAB_APP_ID} + - GITLAB_APP_SECRET=${GITLAB_APP_SECRET} + - ENGINE_HOST=${ENGINE_HOST} + - MONGODB_URI=${MONGODB_URI} + - MONGODB_NAME=${MONGODB_NAME} + - MONGO_SESSION_ENABLED=${MONGO_SESSION_ENABLED} + - METADATA_PURE_HOST=legend-depot + - METADATA_PURE_PORT=${METADATA_PURE_PORT} + - METADATA_ALLOY_HOST=legend-depot + - METADATA_ALLOY_PORT=${METADATA_ALLOY_PORT} + ports: + - ${ENGINE_PORT}:${ENGINE_PORT} + volumes: + - ./z_generated/engine/config:/config + networks: + - legend + healthcheck: + test: + [ + "CMD", + "curl", + "-f", + "http://${ENGINE_HOST}:${ENGINE_PORT}/engine/api/server/v1/info", + ] + start_period: 20s + interval: 5s + timeout: 10s + retries: 5 + legend-sdlc: + profiles: ["sdlc", "query", "studio"] + depends_on: + - setup + - mongodb + container_name: "legend-sdlc" + image: finos/legend-sdlc-server:${SDLC_IMAGE_VERSION} + env_file: .env + environment: + - GITLAB_HOST=${GITLAB_HOST} + - GITLAB_APP_ID=${GITLAB_APP_ID} + - GITLAB_APP_SECRET=${GITLAB_APP_SECRET} + - SDLC_PORT=${SDLC_PORT} + - MONGODB_URI=${MONGODB_URI} + - MONGODB_NAME=${MONGODB_NAME} + - MONGO_SESSION_ENABLED=${MONGO_SESSION_ENABLED} + - SDLC_MAVEN_VERSION=${SDLC_MAVEN_VERSION} + - ENGINE_MAVEN_VERSION=${ENGINE_MAVEN_VERSION} + - LEGEND_EXTERNAL_BASE_URL=$LEGEND_EXTERNAL_BASE_URL + ports: + - ${SDLC_PORT}:${SDLC_PORT} + volumes: + - ./z_generated/sdlc/config:/config + networks: + - legend + healthcheck: + test: ["CMD", "curl", "-f", "http://${SDLC_HOST}:${SDLC_PORT}/api/info"] + start_period: 20s + interval: 5s + timeout: 10s + retries: 5 + legend-depot-store: + profiles: ["depot", "query"] + depends_on: + - setup + - mongodb + container_name: "legend-depot-store" + image: finos/legend-depot-store-server:${DEPOT_STORE_IMAGE_VERSION} + env_file: .env + environment: + - GITLAB_HOST=$GITLAB_HOST + - GITLAB_APP_ID=$GITLAB_APP_ID + - GITLAB_APP_SECRET=$GITLAB_APP_SECRET + - DEPOT_STORE_HOST=${DEPOT_STORE_HOST} + - DEPOT_STORE_PORT=${DEPOT_STORE_PORT} + - DEPOT_STORE_ADMIN_USER=${DEPOT_STORE_ADMIN_USER} + - MONGODB_URI=${MONGODB_URI} + - MONGODB_NAME=${MONGODB_NAME} + - MONGO_SESSION_ENABLED=${MONGO_SESSION_ENABLED} + ports: + - ${DEPOT_STORE_PORT}:${DEPOT_STORE_PORT} + volumes: + - ./z_generated/depot-store/config:/config + networks: + - legend + healthcheck: + test: + [ + "CMD", + "curl", + "-f", + "http://${DEPOT_STORE_HOST}:${DEPOT_STORE_PORT}/depot-store/api/info", + ] + start_period: 20s + interval: 5s + legend-depot-store-setup: + profiles: ["depot", "query"] + depends_on: + - setup + - legend-depot-store + container_name: "legend-depot-store-setup" + image: finos/legend-engine-server:${ENGINE_IMAGE_VERSION} + volumes: + - ./depot-store/setup:/setup:ro + entrypoint: /setup/setup.sh + env_file: .env + networks: + - legend + legend-depot: + profiles: ["depot", "query"] + depends_on: + - setup + - mongodb + container_name: "legend-depot" + image: finos/legend-depot-server:${DEPOT_IMAGE_VERSION} + env_file: .env + environment: + - GITLAB_HOST=$GITLAB_HOST + - GITLAB_APP_ID=$GITLAB_APP_ID + - GITLAB_APP_SECRET=$GITLAB_APP_SECRET + - DEPOT_HOST=${DEPOT_HOST} + - DEPOT_PORT=${DEPOT_PORT} + - MONGODB_URI=${MONGODB_URI} + - MONGODB_NAME=${MONGODB_NAME} + - MONGO_SESSION_ENABLED=${MONGO_SESSION_ENABLED} + ports: + - ${DEPOT_PORT}:${DEPOT_PORT} + networks: + - legend + healthcheck: + test: + [ + "CMD", + "curl", + "-f", + "http://${DEPOT_HOST}:${DEPOT_PORT}/depot/api/info", + ] + start_period: 20s + interval: 5s + # UI Clients + legend-studio: + profiles: ["studio", "query"] + depends_on: + - legend-engine + - legend-sdlc + container_name: legend-studio + image: finos/legend-studio:${STUDIO_IMAGE_VERSION} + entrypoint: java -cp /app/bin/webapp-content:/app/bin/* org.finos.legend.server.shared.staticserver.Server server /config/server-config.yml + env_file: .env + environment: + - GITLAB_HOST=${GITLAB_HOST} + - GITLAB_APP_ID=${GITLAB_APP_ID} + - GITLAB_APP_SECRET=${GITLAB_APP_SECRET} + - ENGINE_HOST=${ENGINE_HOST} + - ENGINE_PORT=${ENGINE_PORT} + - SDLC_HOST=${SDLC_HOST} + - SDLC_PORT=${SDLC_PORT} + - STUDIO_PORT=${STUDIO_PORT} + - DEPOT_HOST=${DEPOT_HOST} + - DEPOT_PORT=${DEPOT_PORT} + - MONGODB_URI=${MONGODB_URI} + - MONGODB_NAME=${MONGODB_NAME} + - MONGO_SESSION_ENABLED=${MONGO_SESSION_ENABLED} + ports: + - ${STUDIO_PORT}:${STUDIO_PORT} + volumes: + - ./z_generated/studio/config:/config + networks: + - legend + healthcheck: + test: + [ + "CMD", + "curl", + "-f", + "http://${STUDIO_HOST}:${STUDIO_PORT}/studio/config.json", + ] + start_period: 20s + interval: 5s + legend-query: + profiles: ["query"] + depends_on: + - setup + - mongodb + - legend-engine + - legend-studio + - legend-depot + container_name: "legend-query" + image: finos/legend-query:${QUERY_IMAGE_VERSION} + env_file: .env + environment: + - GITLAB_HOST=$GITLAB_HOST + - GITLAB_APP_ID=$GITLAB_APP_ID + - GITLAB_APP_SECRET=$GITLAB_APP_SECRET + - QUERY_HOST=${QUERY_HOST} + - QUERY_PORT=${QUERY_PORT} + - ENGINE_HOST=${ENGINE_HOST} + - ENGINE_PORT=${ENGINE_PORT} + - STUDIO_HOST=${STUDIO_HOST} + - STUDIO_PORT=${STUDIO_PORT} + - DEPOT_HOST=${DEPOT_HOST} + - DEPOT_PORT=${DEPOT_PORT} + - MONGODB_URI=${MONGODB_URI} + - MONGODB_NAME=${MONGODB_NAME} + - MONGO_SESSION_ENABLED=${MONGO_SESSION_ENABLED} + ports: + - ${QUERY_PORT}:${QUERY_PORT} + networks: + - legend + healthcheck: + test: + [ + "CMD", + "curl", + "-f", + "http://${QUERY_HOST}:${QUERY_PORT}/query/config.json", + ] + start_period: 20s + interval: 5s + postgres: + profiles: ["studio", "query", "postgres"] + depends_on: + - setup + container_name: "postgres" + image: postgres:${POSTGRES_IMAGE_VERSION} + env_file: .env + environment: + - POSTGRES_USER=postgres + - POSTGRES_PASSWORD=postgres + - POSTGRES_HOST_AUTH_METHOD=password + ports: + - 5432:5432 + volumes: + - ./postgres/create_tables.sql:/docker-entrypoint-initdb.d/create_tables.sql + networks: + - legend + healthcheck: + test: + [ + "CMD", + "pg_isready" + ] + start_period: 20s + interval: 5s + apache: + profiles: ["graphiql"] + image: httpd:latest + container_name: legend-httpd + ports: + - 9099:80 + volumes: + - ./graphiql:/usr/local/apache2/htdocs + +networks: + legend: {} diff --git a/installers/esg/engine/config/config.json b/installers/esg/engine/config/config.json new file mode 100644 index 000000000..20fdf849f --- /dev/null +++ b/installers/esg/engine/config/config.json @@ -0,0 +1,124 @@ + +{ + "deployment": { + "mode": "TEST_IGNORE_FUNCTION_MATCH" + }, + "logging": { + "level": "INFO", + "appenders": [ + { + "type": "console", + "logFormat": "%msg\r\n" + } + ] + }, + "pac4j": { + "bypassPaths": ["/engine/api/server/v1/info"], + "callbackPrefix": "/engine/api/pac4j", + "mongoUri": "${MONGODB_URI}", + "mongoDb": "${MONGODB_NAME}", + "clients": [ + { + "org.finos.legend.server.pac4j.gitlab.GitlabClient": { + "name": "gitlab", + "clientId": "${GITLAB_APP_ID}", + "secret": "${GITLAB_APP_SECRET}", + "discoveryUri": "https://${GITLAB_HOST}/.well-known/openid-configuration", + "scope": "openid profile api" + } + }, + { + "org.finos.legend.server.pac4j.gitlab.GitlabPersonalAccessTokenClient": { + "headerTokenName": "legend-test-pat", + "scheme": "https", + "gitlabHost": "gitlab.com", + "gitlabApiVersion": "v4" + } + } + ], + "mongoSession": { + "enabled": "${MONGO_SESSION_ENABLED}", + "collection": "userSessions" + } + }, + "opentracing": { + "elastic": "", + "zipkin": "", + "uri": "", + "authenticator": { + "principal": "", + "keytab": "" + } + }, + "swagger": { + "title": "Legend Engine", + "resourcePackage": "org.finos.legend" + }, + "server": { + "rootPath": "/engine/api", + "applicationConnectors": [ + { + "maxRequestHeaderSize": "32KiB", + "type": "http", + "port": "${ENGINE_PORT}" + } + ], + "requestLog": { + "appenders": [] + } + }, + "cors": { + "allowedHeaders": [ + "X-Requested-With", + "Content-Type", + "Accept", + "Origin", + "Access-Control-Allow-Credentials", + "x-b3-parentspanid", + "x-b3-sampled", + "x-b3-spanid", + "x-b3-traceid", + "legend-test-pat" + ] + }, + "metadataserver": { + "pure": { + "host": "${METADATA_PURE_HOST}", + "port": ${METADATA_PURE_PORT} + }, + "alloy": { + "host": "${METADATA_ALLOY_HOST}", + "port": ${METADATA_ALLOY_PORT}, + "prefix": "/depot/api" + }, + "sdlc": { + "host": "legend-sdlc", + "port": ${SDLC_PORT}, + "prefix": "/sdlc", + "pac4j": { + "_type": "privateAccessToken", + "accessTokenHeaderName": "legend-test-pat" + } + } + }, + "temporarytestdb": { + "port": ${TEMP_DB_PORT} + }, + "relationalexecution": { + "temporarytestdb": { + "port": ${TEMP_DB_PORT} + }, + "tempPath": "/tmp/", + "flowProviderClass": "org.finos.legend.engine.authentication.LegendDefaultDatabaseAuthenticationFlowProvider", + "flowProviderConfiguration": { + "_type": "legendDefault" + } + }, + "vaults": [ + { + "_type":"property", + "location": "/config/vault.properties" + } + ] +} + diff --git a/installers/esg/engine/config/vault.properties b/installers/esg/engine/config/vault.properties new file mode 100644 index 000000000..ad6decc62 --- /dev/null +++ b/installers/esg/engine/config/vault.properties @@ -0,0 +1,6 @@ +query.mongo.database=querydb +query.mongo.collection.query=queries +query.mongo.collection.queryEvent=collections +query.mongo.connectionString=${MONGODB_URI} +postgres_username_ref=postgres +postgres_password_ref=postgres diff --git a/installers/esg/postgres/create_tables.sql b/installers/esg/postgres/create_tables.sql new file mode 100644 index 000000000..c2ddcb536 --- /dev/null +++ b/installers/esg/postgres/create_tables.sql @@ -0,0 +1,15 @@ +create table TEST(a int); +insert into TEST values(1); +insert into TEST values(2); + +drop table if exists FIRM; +create table FIRM(ID INTEGER, LEGAL_NAME VARCHAR(200)); +insert into FIRM(ID, LEGAL_NAME) values(100, 'ACME Corp.'); +insert into FIRM(ID, LEGAL_NAME) values(200, 'Monsters Inc.'); + +drop table if exists PERSON; +create table PERSON(ID INTEGER, FIRMID INTEGER, FIRST_NAME VARCHAR(200), LAST_NAME VARCHAR(200)); +insert into PERSON(ID, FIRMID, FIRST_NAME, LAST_NAME) values(1, 100, 'Road', 'Runner'); +insert into PERSON(ID, FIRMID, FIRST_NAME, LAST_NAME) values(2, 100, 'Wile', 'Coyote'); +insert into PERSON(ID, FIRMID, FIRST_NAME, LAST_NAME) values(3, 200, 'Jake', 'Sullivan'); +insert into PERSON(ID, FIRMID, FIRST_NAME, LAST_NAME) values(4, 200, 'Mike', 'Wazwoski'); diff --git a/installers/esg/script.txt b/installers/esg/script.txt new file mode 100644 index 000000000..8efc9350b --- /dev/null +++ b/installers/esg/script.txt @@ -0,0 +1,22 @@ + +curl --location 'https://d7nifimeibcde.cloudfront.net/engine/api/sql/v1/execution/executeQueryString?serializationFormat=DEFAULT&client_name=gitlabPAToken' \ +--header 'legend-test-pat: PUT_YOUR_PAT_HERE' \ +--header 'Content-Type: text/plain' \ +--data 'SELECT + "root"."entityId" AS "entityId", + "root"."fiscalYear" AS "fiscalYear", + "root"."env.climate.emissions.ghgScope1Absolute" AS "env.climate.emissions.ghgScope1Absolute", + "root"."env.climate.emissions.ghgScope1Absolute_unit" AS "env.climate.emissions.ghgScope1Absolute_unit", + "root"."env.climate.emissions.ghgScope1Absolute.pcafScore" AS "env.climate.emissions.ghgScope1Absolute.pcafScore", + "root"."env.climate.emissions.ghgScope1Absolute.sourceDescription" AS "env.climate.emissions.ghgScope1Absolute.sourceDescription", + "root"."env.climate.emissions.ghgScope2LocAbsolute" AS "env.climate.emissions.ghgScope2LocAbsolute", + "root"."env.climate.emissions.ghgScope2LocAbsolute_unit" AS "env.climate.emissions.ghgScope2LocAbsolute_unit", + "root"."env.climate.emissions.ghgScope2LocAbsolute.pcafScore" AS "env.climate.emissions.ghgScope2LocAbsolute.pcafScore", + "root"."env.climate.emissions.ghgScope2LocAbsolute.sourceDescription" AS "env.climate.emissions.ghgScope2LocAbsolute.sourceDescription" +FROM + service( + pattern => '\''/finos/esg/env/climate/getEmissionData'\'', + project => '\''OMNIBUS-51104062'\'', + groupWorkspace => '\''GWS8'\'' + + ) AS "root"' diff --git a/installers/esg/sdlc/config/config.json b/installers/esg/sdlc/config/config.json new file mode 100644 index 000000000..7910ba932 --- /dev/null +++ b/installers/esg/sdlc/config/config.json @@ -0,0 +1,104 @@ +server: + adminContextPath: /admin + applicationConnectors: + - type: http + port: ${SDLC_PORT} + maxRequestHeaderSize: 128KiB + adminConnectors: + - type: http + port: ${SDLC_ADMIN_PORT} + gzip: + includedMethods: + - GET + - POST + requestLog: + type: classic + level: "OFF" + appenders: + - type: console + logFormat: "OFF" + rootPath: /sdlc/api +cors: + allowedHeaders: + - X-Requested-With + - Content-Type + - Accept + - Origin + - Access-Control-Allow-Credentials + - x-b3-parentspanid + - x-b3-sampled + - x-b3-spanid + - x-b3-traceid + - legend-test-pat +sessionCookie: LEGEND_SDLC_JSESSIONID +logging: + level: INFO + appenders: + - type: console + logFormat: "%d{yyyy-MM-dd HH:mm:ss.SSS} %-5p [%thread] %c - %m%n" +swagger: + resourcePackage: org.finos.legend.sdlc.server.resources + title: Legend SDLC + version: local-snapshot + schemes: [] +pac4j: + callbackPrefix: /sdlc/api/pac4j + mongoUri: ${MONGODB_URI} + mongoDb: ${MONGODB_NAME} + mongoSession: + enabled: ${MONGO_SESSION_ENABLED} + collection: userSessions + clients: + - org.finos.legend.server.pac4j.gitlab.GitlabClient: + name: gitlab + clientId: ${GITLAB_APP_ID} + secret: ${GITLAB_APP_SECRET} + discoveryUri: https://gitlab.com/.well-known/openid-configuration + scope: openid profile api + - org.finos.legend.server.pac4j.gitlab.GitlabPersonalAccessTokenClient: + name: gitlab-pat + headerTokenName: legend-test-pat + scheme: https + gitlabHost: gitlab.com + gitlabApiVersion: v4 + bypassPaths: + - /sdlc/api/info + - /sdlc/api/server/info + - /sdlc/api/server/platforms + - /sdlc/api/auth/authorized + +applicationName: Legend SDLC +filterPriorities: + GitLab: 1 + org.pac4j.j2e.filter.CallbackFilter: 2 + org.pac4j.j2e.filter.SecurityFilter: 3 + CORS: 4 +features: + canCreateProject: true + canCreateVersion: true +projectStructure: + projectCreation: {} + extensionProvider: + org.finos.legend.sdlc.server.gitlab.finos.FinosGitlabProjectStructureExtensionProvider: + {} + platforms: + legend-engine: + groupId: org.finos.legend.engine + platformVersion: + version: ${ENGINE_IMAGE_VERSION} + legend-sdlc: + groupId: org.finos.legend.sdlc + platformVersion: + version: ${SDLC_IMAGE_VERSION} +gitLab: + newProjectVisibility: public + projectTag: legend + projectIdPrefix: OMNIBUS + server: + scheme: https + host: gitlab.com + port: 443 + app: + id: ${GITLAB_APP_ID} + secret: ${GITLAB_APP_SECRET} + redirectURI: ${LEGEND_EXTERNAL_BASE_URL}/sdlc/api/auth/callback diff --git a/installers/esg/setup.sh b/installers/esg/setup.sh new file mode 100755 index 000000000..ec5e62aad --- /dev/null +++ b/installers/esg/setup.sh @@ -0,0 +1,42 @@ +#!/bin/bash + +BASE=/setup +. $BASE/.env + +GEN=$BASE/z_generated +mkdir -p $GEN + +## Engine +SRC=$BASE/engine/config +DEST=$GEN/engine/config +mkdir -p $DEST +cp -r $SRC/* $DEST +sed -i 's~${MONGODB_URI}~'${MONGODB_URI}'~g' $DEST/vault.properties + +## Sdlc +SRC=$BASE/sdlc/config +DEST=$GEN/sdlc/config +mkdir -p $DEST +cp -r $SRC/* $DEST + +## Studio +SRC=$BASE/studio/config +DEST=$GEN/studio/config +mkdir -p $DEST +cp -r $SRC/* $DEST +sed -i 's~__LEGEND_EXTERNAL_BASE_URL__~'${LEGEND_EXTERNAL_BASE_URL}'~g' $DEST/config.json + +## Depot Store +SRC=$BASE/depot-store/config +DEST=$GEN/depot-store/config +mkdir -p $DEST +cp -r $SRC/* $DEST +sed -i 's/${DEPOT_STORE_ADMIN_USER}/'$DEPOT_STORE_ADMIN_USER'/g' $DEST/authorisedIdentities.json + +SRC=$BASE/depot-store/setup +DEST=$GEN/depot-store/setup +mkdir -p $DEST +cp -r $SRC/* $DEST + +sed -i 's/${DEPOT_STORE_HOST}/'$DEPOT_STORE_HOST'/g' $DEST/setup.sh +sed -i 's/${DEPOT_STORE_PORT}/'$DEPOT_STORE_PORT'/g' $DEST/setup.sh diff --git a/installers/esg/studio/config/config.json b/installers/esg/studio/config/config.json new file mode 100644 index 000000000..8c486a702 --- /dev/null +++ b/installers/esg/studio/config/config.json @@ -0,0 +1,24 @@ +{ + "appName": "studio", + "env": "prod", + "sdlc": { + "url": "__LEGEND_EXTERNAL_BASE_URL__/sdlc/api" + }, + "engine": { + "url": "__LEGEND_EXTERNAL_BASE_URL__/engine/api" + }, + "depot": { + "url": "__LEGEND_EXTERNAL_BASE_URL__/depot/api" + }, + "documentation": { + "url": "https://legend.finos.org" + }, + "extensions": { + "core": { + "TEMPORARY__disableSDLCProjectCreation": false, + "TEMPORARY__disableServiceRegistration": true, + "TEMPORARY__enableFunctionActivatorSupport": true, + "TEMPORARY__enableRawSQLExecutor": true + } + } +} diff --git a/installers/esg/studio/config/server-config.yml b/installers/esg/studio/config/server-config.yml new file mode 100644 index 000000000..8d1e1f67f --- /dev/null +++ b/installers/esg/studio/config/server-config.yml @@ -0,0 +1,44 @@ +server: + type: simple + applicationContextPath: / + adminContextPath: /studio/admin + connector: + type: http + port: ${STUDIO_PORT} +sessionCookie: LEGEND_STUDIO_JSESSIONID +logging: + level: INFO + loggers: + root: + level: INFO + org.pac4j: + level: INFO + appenders: + - type: console + +pac4j: + callbackPrefix: /studio/api/pac4j + clients: + - org.finos.legend.server.pac4j.gitlab.GitlabClient: + name: gitlab + clientId: ${GITLAB_APP_ID} + secret: ${GITLAB_APP_SECRET} + discoveryUri: https://gitlab.com/.well-known/openid-configuration + scope: openid profile api + bypassPaths: + - /studio/admin/healthcheck + - /studio/config.json + +html5Router: true +routerExemptPaths: + - /editor.worker.js + - /json.worker.js + - /editor.worker.js.map + - /json.worker.js.map + - /version.json + - /config.json + - /favicon.ico + - /static +localAssetPaths: + /studio/config.json: /config/config.json +uiPath: /studio diff --git a/installers/esg/studio/run-studio.sh b/installers/esg/studio/run-studio.sh new file mode 100755 index 000000000..895acedd7 --- /dev/null +++ b/installers/esg/studio/run-studio.sh @@ -0,0 +1,24 @@ +#!/bin/bash + +PROTOCOL=${PROTOCOL:-http} +ENGINE_HOST=${ENGINE_HOST:-localhost} +ENGINE_PORT=${ENGINE_PORT:-6300} +SDLC_HOST=${SDLC_HOST:-localhost} +SDLC_PORT=${SDLC_PORT:-6100} +DEPOT_HOST=${DEPOT_HOST:-localhost} +DEPOT_PORT=${DEPOT_PORT:-7000} +TAXONOMY_HOST=${TAXONOMY_HOST:-localhost} +TAXONOMY_PORT=${TAXONOMY_PORT:-8000} + +sed -i 's~${PROTOCOL}~'$PROTOCOL'~g' $STUDIO_HOME/config/config.json +sed -i 's~${ENGINE_HOST}~'$ENGINE_HOST'~g' $STUDIO_HOME/config/config.json +sed -i 's~${ENGINE_PORT}~'$ENGINE_PORT'~g' $STUDIO_HOME/config/config.json +sed -i 's~${SDLC_HOST}~'$SDLC_HOST'~g' $STUDIO_HOME/config/config.json +sed -i 's~${SDLC_PORT}~'$SDLC_PORT'~g' $STUDIO_HOME/config/config.json +sed -i 's~${DEPOT_HOST}~'$DEPOT_HOST'~g' $STUDIO_HOME/config/config.json +sed -i 's~${DEPOT_PORT}~'$DEPOT_PORT'~g' $STUDIO_HOME/config/config.json +sed -i 's~${TAXONOMY_HOST}~'$TAXONOMY_HOST'~g' $STUDIO_HOME/config/config.json +sed -i 's~${TAXONOMY_PORT}~'$TAXONOMY_PORT'~g' $STUDIO_HOME/config/config.json + +java -cp $STUDIO_HOME/webapp-content:$STUDIO_HOME/* org.finos.legend.server.shared.staticserver.Server server $STUDIO_HOME/config/server-config.json +