From 56896794b311027d8b68602564fb874199ae3ed0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Diego=20Nogu=C3=AAs?= <49420+diegocn@users.noreply.github.com> Date: Sun, 17 Dec 2023 13:43:06 -0300 Subject: [PATCH] feat: organizing infrastructure_files folder and adds new envs (#1235) This PR aims to organize a little the files within `infrastructure_files` folder and adds some new ENV vars to the process. 1. It creates the `artifacts` folder within the `infrastructure_files` folder, the idea behind it is to split templates from artifacts created after running `./configure.sh`. It makes it easier to cp/rsync only `artifacts` content to the final server/destination. 2. Creates `NETBIRD_TURN_DOMAIN` and `TURN_DOMAIN` ENV vars. The idea behind it is to make it possible to split the management/signal server from TURN server. If `NETBIRD_TURN_DOMAIN` is not set, then, `TURN_DOMAIN` will be set as `NETBIRD_DOMAIN`. 3. Creates `*_TAG` ENVs for each component. The idea behind it is to give the users the choice to use `latest` tag as default or tie it to specific versions of each component in the stack. --- .../workflows/test-infrastructure-files.yml | 12 +++---- .gitignore | 10 +++--- infrastructure_files/artifacts/.gitkeep | 0 infrastructure_files/base.setup.env | 18 +++++++++- infrastructure_files/configure.sh | 33 ++++++++++--------- infrastructure_files/docker-compose.yml.tmpl | 10 +++--- .../docker-compose.yml.tmpl.traefik | 10 +++--- infrastructure_files/management.json.tmpl | 4 +-- infrastructure_files/setup.env.example | 14 +++++++- 9 files changed, 71 insertions(+), 40 deletions(-) create mode 100644 infrastructure_files/artifacts/.gitkeep diff --git a/.github/workflows/test-infrastructure-files.yml b/.github/workflows/test-infrastructure-files.yml index 6482b716fe3..e01844c3349 100644 --- a/.github/workflows/test-infrastructure-files.yml +++ b/.github/workflows/test-infrastructure-files.yml @@ -62,7 +62,7 @@ jobs: CI_NETBIRD_MGMT_IDP_SIGNKEY_REFRESH: false - name: check values - working-directory: infrastructure_files + working-directory: infrastructure_files/artifacts env: CI_NETBIRD_DOMAIN: localhost CI_NETBIRD_AUTH_CLIENT_ID: testing.client.id @@ -107,7 +107,7 @@ jobs: grep Engine management.json | grep "$CI_NETBIRD_STORE_CONFIG_ENGINE" grep IdpSignKeyRefreshEnabled management.json | grep "$CI_NETBIRD_MGMT_IDP_SIGNKEY_REFRESH" grep UseIDToken management.json | grep false - grep -A 1 IdpManagerConfig management.json | grep ManagerType | grep $CI_NETBIRD_MGMT_IDP + grep -A 1 IdpManagerConfig management.json | grep ManagerType | grep $CI_NETBIRD_MGMT_IDP grep -A 3 IdpManagerConfig management.json | grep -A 1 ClientConfig | grep Issuer | grep $CI_NETBIRD_AUTH_AUTHORITY grep -A 4 IdpManagerConfig management.json | grep -A 2 ClientConfig | grep TokenEndpoint | grep $CI_NETBIRD_AUTH_TOKEN_ENDPOINT grep -A 5 IdpManagerConfig management.json | grep -A 3 ClientConfig | grep ClientID | grep $CI_NETBIRD_IDP_MGMT_CLIENT_ID @@ -143,7 +143,7 @@ jobs: docker build -t netbirdio/signal:latest . - name: run docker compose up - working-directory: infrastructure_files + working-directory: infrastructure_files/artifacts run: | docker-compose up -d sleep 5 @@ -152,9 +152,9 @@ jobs: - name: test running containers run: | - count=$(docker compose ps --format json | jq '. | select(.Name | contains("infrastructure_files")) | .State' | grep -c running) + count=$(docker compose ps --format json | jq '. | select(.Name | contains("artifacts")) | .State' | grep -c running) test $count -eq 4 - working-directory: infrastructure_files + working-directory: infrastructure_files/artifacts test-getting-started-script: runs-on: ubuntu-latest @@ -179,4 +179,4 @@ jobs: - name: test zitadel.env file gen run: test -f zitadel.env - name: test dashboard.env file gen - run: test -f dashboard.env \ No newline at end of file + run: test -f dashboard.env diff --git a/.gitignore b/.gitignore index 7f7f53ce86f..d8fafb2c812 100644 --- a/.gitignore +++ b/.gitignore @@ -6,11 +6,11 @@ bin/ .env conf.json http-cmds.sh -infrastructure_files/management.json -infrastructure_files/management-*.json -infrastructure_files/docker-compose.yml -infrastructure_files/openid-configuration.json -infrastructure_files/turnserver.conf +infrastructure_files/artifacts/management.json +infrastructure_files/artifacts/management-*.json +infrastructure_files/artifacts/docker-compose.yml +infrastructure_files/artifacts/openid-configuration.json +infrastructure_files/artifacts/turnserver.conf management/management client/client client/client.exe diff --git a/infrastructure_files/artifacts/.gitkeep b/infrastructure_files/artifacts/.gitkeep new file mode 100644 index 00000000000..e69de29bb2d diff --git a/infrastructure_files/base.setup.env b/infrastructure_files/base.setup.env index e254aa6f323..7dd456745bb 100644 --- a/infrastructure_files/base.setup.env +++ b/infrastructure_files/base.setup.env @@ -20,6 +20,9 @@ NETBIRD_MGMT_IDP_SIGNKEY_REFRESH=${NETBIRD_MGMT_IDP_SIGNKEY_REFRESH:-false} NETBIRD_SIGNAL_PROTOCOL="http" NETBIRD_SIGNAL_PORT=${NETBIRD_SIGNAL_PORT:-10000} +# Turn +TURN_DOMAIN=${NETBIRD_TURN_DOMAIN:-$NETBIRD_DOMAIN} + # Turn credentials # User TURN_USER=self @@ -59,8 +62,16 @@ NETBIRD_DASH_AUTH_AUDIENCE=$NETBIRD_AUTH_AUDIENCE # Store config NETBIRD_STORE_CONFIG_ENGINE=${NETBIRD_STORE_CONFIG_ENGINE:-"jsonfile"} +# Image tags +NETBIRD_DASHBOARD_TAG=${NETBIRD_DASHBOARD_TAG:-"latest"} +NETBIRD_SIGNAL_TAG=${NETBIRD_SIGNAL_TAG:-"latest"} +NETBIRD_MANAGEMENT_TAG=${NETBIRD_MANAGEMENT_TAG:-"latest"} +COTURN_TAG=${COTURN_TAG:-"latest"} + + # exports export NETBIRD_DOMAIN +export NETBIRD_TURN_DOMAIN export NETBIRD_AUTH_CLIENT_ID export NETBIRD_AUTH_CLIENT_SECRET export NETBIRD_AUTH_AUDIENCE @@ -79,6 +90,7 @@ export NETBIRD_AUTH_DEVICE_AUTH_CLIENT_ID export NETBIRD_AUTH_OIDC_CONFIGURATION_ENDPOINT export NETBIRD_AUTH_REDIRECT_URI export NETBIRD_AUTH_SILENT_REDIRECT_URI +export TURN_DOMAIN export TURN_USER export TURN_PASSWORD export TURN_MIN_PORT @@ -103,4 +115,8 @@ export NETBIRD_AUTH_PKCE_USE_ID_TOKEN export NETBIRD_AUTH_PKCE_AUDIENCE export NETBIRD_DASH_AUTH_USE_AUDIENCE export NETBIRD_DASH_AUTH_AUDIENCE -export NETBIRD_STORE_CONFIG_ENGINE \ No newline at end of file +export NETBIRD_STORE_CONFIG_ENGINE +export NETBIRD_DASHBOARD_TAG +export NETBIRD_SIGNAL_TAG +export NETBIRD_MANAGEMENT_TAG +export COTURN_TAG diff --git a/infrastructure_files/configure.sh b/infrastructure_files/configure.sh index 1f70584f3e7..22ddcb07b42 100755 --- a/infrastructure_files/configure.sh +++ b/infrastructure_files/configure.sh @@ -54,6 +54,9 @@ if [[ "x-$TURN_PASSWORD" == "x-" ]]; then export TURN_PASSWORD=$(openssl rand -base64 32 | sed 's/=//g') fi +artifacts_path="./artifacts" +mkdir -p $artifacts_path + MGMT_VOLUMENAME="${VOLUME_PREFIX}${MGMT_VOLUMESUFFIX}" SIGNAL_VOLUMENAME="${VOLUME_PREFIX}${SIGNAL_VOLUMESUFFIX}" LETSENCRYPT_VOLUMENAME="${VOLUME_PREFIX}${LETSENCRYPT_VOLUMESUFFIX}" @@ -94,13 +97,13 @@ if [[ -z "${NETBIRD_AUTH_OIDC_CONFIGURATION_ENDPOINT}" ]]; then fi echo "loading OpenID configuration from ${NETBIRD_AUTH_OIDC_CONFIGURATION_ENDPOINT} to the openid-configuration.json file" -curl "${NETBIRD_AUTH_OIDC_CONFIGURATION_ENDPOINT}" -q -o openid-configuration.json +curl "${NETBIRD_AUTH_OIDC_CONFIGURATION_ENDPOINT}" -q -o ${artifacts_path}/openid-configuration.json -export NETBIRD_AUTH_AUTHORITY=$(jq -r '.issuer' openid-configuration.json) -export NETBIRD_AUTH_JWT_CERTS=$(jq -r '.jwks_uri' openid-configuration.json) -export NETBIRD_AUTH_TOKEN_ENDPOINT=$(jq -r '.token_endpoint' openid-configuration.json) -export NETBIRD_AUTH_DEVICE_AUTH_ENDPOINT=$(jq -r '.device_authorization_endpoint' openid-configuration.json) -export NETBIRD_AUTH_PKCE_AUTHORIZATION_ENDPOINT=$(jq -r '.authorization_endpoint' openid-configuration.json) +export NETBIRD_AUTH_AUTHORITY=$(jq -r '.issuer' ${artifacts_path}/openid-configuration.json) +export NETBIRD_AUTH_JWT_CERTS=$(jq -r '.jwks_uri' ${artifacts_path}/openid-configuration.json) +export NETBIRD_AUTH_TOKEN_ENDPOINT=$(jq -r '.token_endpoint' ${artifacts_path}/openid-configuration.json) +export NETBIRD_AUTH_DEVICE_AUTH_ENDPOINT=$(jq -r '.device_authorization_endpoint' ${artifacts_path}/openid-configuration.json) +export NETBIRD_AUTH_PKCE_AUTHORIZATION_ENDPOINT=$(jq -r '.authorization_endpoint' ${artifacts_path}/openid-configuration.json) if [[ ! -z "${NETBIRD_AUTH_DEVICE_AUTH_CLIENT_ID}" ]]; then # user enabled Device Authorization Grant feature @@ -185,17 +188,17 @@ fi env | grep NETBIRD bkp_postfix="$(date +%s)" -if test -f 'docker-compose.yml'; then - cp docker-compose.yml "docker-compose.yml.bkp.${bkp_postfix}" +if test -f "${artifacts_path}/docker-compose.yml"; then + cp $artifacts_path/docker-compose.yml "${artifacts_path}/docker-compose.yml.bkp.${bkp_postfix}" fi -if test -f 'management.json'; then - cp management.json "management.json.bkp.${bkp_postfix}" +if test -f "${artifacts_path}/management.json"; then + cp $artifacts_path/management.json "${artifacts_path}/management.json.bkp.${bkp_postfix}" fi -if test -f 'turnserver.conf'; then - cp turnserver.conf "turnserver.conf.bpk.${bkp_postfix}" +if test -f "${artifacts_path}/turnserver.conf"; then + cp ${artifacts_path}/turnserver.conf "${artifacts_path}/turnserver.conf.bpk.${bkp_postfix}" fi -envsubst docker-compose.yml -envsubst management.json -envsubst turnserver.conf +envsubst $artifacts_path/docker-compose.yml +envsubst $artifacts_path/management.json +envsubst $artifacts_path/turnserver.conf diff --git a/infrastructure_files/docker-compose.yml.tmpl b/infrastructure_files/docker-compose.yml.tmpl index c5ea3ae56b0..fa18515fc1f 100644 --- a/infrastructure_files/docker-compose.yml.tmpl +++ b/infrastructure_files/docker-compose.yml.tmpl @@ -2,7 +2,7 @@ version: "3" services: #UI dashboard dashboard: - image: wiretrustee/dashboard:latest + image: wiretrustee/dashboard:$NETBIRD_DASHBOARD_TAG restart: unless-stopped ports: - 80:80 @@ -31,7 +31,7 @@ services: # Signal signal: - image: netbirdio/signal:latest + image: netbirdio/signal:$NETBIRD_SIGNAL_TAG restart: unless-stopped volumes: - $SIGNAL_VOLUMENAME:/var/lib/netbird @@ -43,7 +43,7 @@ services: # Management management: - image: netbirdio/management:latest + image: netbirdio/management:$NETBIRD_MANAGEMENT_TAG restart: unless-stopped depends_on: - dashboard @@ -65,9 +65,9 @@ services: # Coturn coturn: - image: coturn/coturn + image: coturn/coturn:$COTURN_TAG restart: unless-stopped - domainname: $NETBIRD_DOMAIN + domainname: $TURN_DOMAIN volumes: - ./turnserver.conf:/etc/turnserver.conf:ro # - ./privkey.pem:/etc/coturn/private/privkey.pem:ro diff --git a/infrastructure_files/docker-compose.yml.tmpl.traefik b/infrastructure_files/docker-compose.yml.tmpl.traefik index cab471df646..fd194a042bf 100644 --- a/infrastructure_files/docker-compose.yml.tmpl.traefik +++ b/infrastructure_files/docker-compose.yml.tmpl.traefik @@ -2,7 +2,7 @@ version: "3" services: #UI dashboard dashboard: - image: wiretrustee/dashboard:latest + image: wiretrustee/dashboard:$NETBIRD_DASHBOARD_TAG restart: unless-stopped #ports: # - 80:80 @@ -35,7 +35,7 @@ services: # Signal signal: - image: netbirdio/signal:latest + image: netbirdio/signal:$NETBIRD_SIGNAL_TAG restart: unless-stopped volumes: - $SIGNAL_VOLUMENAME:/var/lib/netbird @@ -52,7 +52,7 @@ services: # Management management: - image: netbirdio/management:latest + image: netbirdio/management:$NETBIRD_MANAGEMENT_TAG restart: unless-stopped depends_on: - dashboard @@ -84,9 +84,9 @@ services: # Coturn coturn: - image: coturn/coturn + image: coturn/coturn:$COTURN_TAG restart: unless-stopped - domainname: $NETBIRD_DOMAIN + domainname: $TURN_DOMAIN volumes: - ./turnserver.conf:/etc/turnserver.conf:ro # - ./privkey.pem:/etc/coturn/private/privkey.pem:ro diff --git a/infrastructure_files/management.json.tmpl b/infrastructure_files/management.json.tmpl index 64c2d081694..0b607245f2f 100644 --- a/infrastructure_files/management.json.tmpl +++ b/infrastructure_files/management.json.tmpl @@ -2,7 +2,7 @@ "Stuns": [ { "Proto": "udp", - "URI": "stun:$NETBIRD_DOMAIN:3478", + "URI": "stun:$TURN_DOMAIN:3478", "Username": "", "Password": null } @@ -11,7 +11,7 @@ "Turns": [ { "Proto": "udp", - "URI": "turn:$NETBIRD_DOMAIN:3478", + "URI": "turn:$TURN_DOMAIN:3478", "Username": "$TURN_USER", "Password": "$TURN_PASSWORD" } diff --git a/infrastructure_files/setup.env.example b/infrastructure_files/setup.env.example index 00c0c07f9d1..f00aca2b470 100644 --- a/infrastructure_files/setup.env.example +++ b/infrastructure_files/setup.env.example @@ -1,8 +1,20 @@ ## example file, you can copy this file to setup.env and update its values ## + +# Image tags +# you can force specific tags for each component; will be set to latest if empty +NETBIRD_DASHBOARD_TAG="" +NETBIRD_SIGNAL_TAG="" +NETBIRD_MANAGEMENT_TAG="" +COTURN_TAG="" + # Dashboard domain. e.g. app.mydomain.com NETBIRD_DOMAIN="" +# TURN server domain. e.g. turn.mydomain.com +# if not specified it will assume NETBIRD_DOMAIN +NETBIRD_TURN_DOMAIN="" + # ------------------------------------------- # OIDC # e.g., https://example.eu.auth0.com/.well-known/openid-configuration @@ -70,4 +82,4 @@ NETBIRD_LETSENCRYPT_EMAIL="" # Disable anonymous metrics collection, see more information at https://netbird.io/docs/FAQ/metrics-collection NETBIRD_DISABLE_ANONYMOUS_METRICS=false # DNS DOMAIN configures the domain name used for peer resolution. By default it is netbird.selfhosted -NETBIRD_MGMT_DNS_DOMAIN=netbird.selfhosted \ No newline at end of file +NETBIRD_MGMT_DNS_DOMAIN=netbird.selfhosted