From 516f93fe59d0edc94d966b8e929aecaaeb116da0 Mon Sep 17 00:00:00 2001 From: Dusan Borovcanin Date: Fri, 12 Jul 2024 14:44:44 +0200 Subject: [PATCH] Restructure docker directory Signed-off-by: Dusan Borovcanin --- .github/workflows/tests.yml | 41 +--- docker/Dockerfile | 2 +- docker/addons/bootstrap/docker-compose.yml | 83 ------- docker/addons/certs/docker-compose.yml | 90 ------- docker/addons/journal/docker-compose.yml | 67 ----- .../addons/postgres-reader/docker-compose.yml | 80 ------ docker/addons/postgres-writer/config.toml | 19 -- .../addons/postgres-writer/docker-compose.yml | 63 ----- docker/addons/provision/configs/config.toml | 74 ------ docker/addons/provision/docker-compose.yml | 45 ---- .../timescale-reader/docker-compose.yml | 80 ------ docker/addons/timescale-writer/config.toml | 8 - .../timescale-writer/docker-compose.yml | 65 ----- docker/addons/vault/.gitignore | 5 - docker/addons/vault/README.md | 170 ------------- docker/addons/vault/config.hcl | 10 - docker/addons/vault/docker-compose.yml | 39 --- docker/addons/vault/entrypoint.sh | 25 -- ...magistrala_things_certs_issue.template.hcl | 32 --- docker/addons/vault/vault_cmd.sh | 24 -- docker/addons/vault/vault_copy_certs.sh | 53 ---- docker/addons/vault/vault_copy_env.sh | 24 -- docker/addons/vault/vault_create_approle.sh | 97 -------- docker/addons/vault/vault_init.sh | 24 -- docker/addons/vault/vault_set_pki.sh | 229 ------------------ docker/addons/vault/vault_unseal.sh | 24 -- .../cassandra-reader/docker-compose.yml | 0 .../{addons => }/cassandra-writer/config.toml | 0 .../cassandra-writer/docker-compose.yml | 0 docker/{addons => }/cassandra-writer/init.sh | 0 .../influxdb-reader/docker-compose.yml | 0 .../{addons => }/influxdb-writer/config.toml | 0 .../influxdb-writer/docker-compose.yml | 0 .../lora-adapter/docker-compose.yml | 0 .../mongodb-reader/docker-compose.yml | 0 .../{addons => }/mongodb-writer/config.toml | 0 .../mongodb-writer/docker-compose.yml | 0 .../opcua-adapter/docker-compose.yml | 0 .../prometheus/docker-compose.yml | 0 .../prometheus/grafana/dashboard.yml | 0 .../prometheus/grafana/datasource.yml | 0 .../prometheus/grafana/example-dashboard.json | 0 .../prometheus/metrics/prometheus.yml | 0 docker/{addons => }/smpp-notifier/config.toml | 0 .../smpp-notifier/docker-compose.yml | 0 docker/{addons => }/smtp-notifier/config.toml | 0 .../smtp-notifier/docker-compose.yml | 0 docker/{addons => }/twins/docker-compose.yml | 0 48 files changed, 2 insertions(+), 1471 deletions(-) delete mode 100644 docker/addons/bootstrap/docker-compose.yml delete mode 100644 docker/addons/certs/docker-compose.yml delete mode 100644 docker/addons/journal/docker-compose.yml delete mode 100644 docker/addons/postgres-reader/docker-compose.yml delete mode 100644 docker/addons/postgres-writer/config.toml delete mode 100644 docker/addons/postgres-writer/docker-compose.yml delete mode 100644 docker/addons/provision/configs/config.toml delete mode 100644 docker/addons/provision/docker-compose.yml delete mode 100644 docker/addons/timescale-reader/docker-compose.yml delete mode 100644 docker/addons/timescale-writer/config.toml delete mode 100644 docker/addons/timescale-writer/docker-compose.yml delete mode 100644 docker/addons/vault/.gitignore delete mode 100644 docker/addons/vault/README.md delete mode 100644 docker/addons/vault/config.hcl delete mode 100644 docker/addons/vault/docker-compose.yml delete mode 100644 docker/addons/vault/entrypoint.sh delete mode 100644 docker/addons/vault/magistrala_things_certs_issue.template.hcl delete mode 100644 docker/addons/vault/vault_cmd.sh delete mode 100755 docker/addons/vault/vault_copy_certs.sh delete mode 100755 docker/addons/vault/vault_copy_env.sh delete mode 100755 docker/addons/vault/vault_create_approle.sh delete mode 100755 docker/addons/vault/vault_init.sh delete mode 100755 docker/addons/vault/vault_set_pki.sh delete mode 100755 docker/addons/vault/vault_unseal.sh rename docker/{addons => }/cassandra-reader/docker-compose.yml (100%) rename docker/{addons => }/cassandra-writer/config.toml (100%) rename docker/{addons => }/cassandra-writer/docker-compose.yml (100%) rename docker/{addons => }/cassandra-writer/init.sh (100%) rename docker/{addons => }/influxdb-reader/docker-compose.yml (100%) rename docker/{addons => }/influxdb-writer/config.toml (100%) rename docker/{addons => }/influxdb-writer/docker-compose.yml (100%) rename docker/{addons => }/lora-adapter/docker-compose.yml (100%) rename docker/{addons => }/mongodb-reader/docker-compose.yml (100%) rename docker/{addons => }/mongodb-writer/config.toml (100%) rename docker/{addons => }/mongodb-writer/docker-compose.yml (100%) rename docker/{addons => }/opcua-adapter/docker-compose.yml (100%) rename docker/{addons => }/prometheus/docker-compose.yml (100%) rename docker/{addons => }/prometheus/grafana/dashboard.yml (100%) rename docker/{addons => }/prometheus/grafana/datasource.yml (100%) rename docker/{addons => }/prometheus/grafana/example-dashboard.json (100%) rename docker/{addons => }/prometheus/metrics/prometheus.yml (100%) rename docker/{addons => }/smpp-notifier/config.toml (100%) rename docker/{addons => }/smpp-notifier/docker-compose.yml (100%) rename docker/{addons => }/smtp-notifier/config.toml (100%) rename docker/{addons => }/smtp-notifier/docker-compose.yml (100%) rename docker/{addons => }/twins/docker-compose.yml (100%) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 7cd8231..017bf21 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -58,14 +58,6 @@ jobs: workflow: - ".github/workflows/tests.yml" - certs: - - "certs/**" - - "cmd/certs/**" - - "auth.pb.go" - - "auth_grpc.pb.go" - - "auth/**" - - "pkg/sdk/**" - consumers: - "consumers/**" - "cmd/cassandra-writer/**" @@ -75,55 +67,29 @@ jobs: - "cmd/timescale-writer/**" - "cmd/smpp-notifier/**" - "cmd/smtp-notifier/**" - - "auth.pb.go" - - "auth_grpc.pb.go" - - "auth/**" - - "pkg/ulid/**" - - "pkg/uuid/**" - - "pkg/messaging/**" lora: - "lora/**" - "cmd/lora/**" - - "pkg/messaging/**" - + opcua: - "opcua/**" - "cmd/opcua/**" - - "logger/**" readers: - "readers/**" - "cmd/cassandra-reader/**" - "cmd/influxdb-reader/**" - "cmd/mongodb-reader/**" - - "cmd/postgres-reader/**" - - "cmd/timescale-reader/**" - - "auth.pb.go" - - "auth_grpc.pb.go" - - "things/**" - - "auth/**" twins: - "twins/**" - "cmd/twins/**" - - "auth.pb.go" - - "auth_grpc.pb.go" - - "auth/**" - - "pkg/messaging/**" - - "pkg/ulid/**" - - "pkg/uuid/**" - - "logger/**" - name: Create coverage directory run: | mkdir coverage - - name: Run certs tests - if: steps.changes.outputs.certs == 'true' || steps.changes.outputs.workflow == 'true' - run: | - go test --race -v -count=1 -coverprofile=coverage/certs.out ./certs/... - - name: Run consumers tests if: steps.changes.outputs.consumers == 'true' || steps.changes.outputs.workflow == 'true' run: | @@ -139,11 +105,6 @@ jobs: run: | go test --race -v -count=1 -coverprofile=coverage/opcua.out ./opcua/... - - name: Run provision tests - if: steps.changes.outputs.provision == 'true' || steps.changes.outputs.workflow == 'true' - run: | - go test --race -v -count=1 -coverprofile=coverage/provision.out ./provision/... - - name: Run readers tests if: steps.changes.outputs.readers == 'true' || steps.changes.outputs.workflow == 'true' run: | diff --git a/docker/Dockerfile b/docker/Dockerfile index 9557117..8e15a22 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -9,7 +9,7 @@ ARG VERSION ARG COMMIT ARG TIME -WORKDIR /go/src/github.com/absmach/magistrala +WORKDIR /go/src/github.com/mg-contirb COPY . . RUN apk update \ && apk add make\ diff --git a/docker/addons/bootstrap/docker-compose.yml b/docker/addons/bootstrap/docker-compose.yml deleted file mode 100644 index 9792972..0000000 --- a/docker/addons/bootstrap/docker-compose.yml +++ /dev/null @@ -1,83 +0,0 @@ -# Copyright (c) Abstract Machines -# SPDX-License-Identifier: Apache-2.0 - -# This docker-compose file contains optional bootstrap services. Since it's optional, this file is -# dependent of docker-compose file from /docker. In order to run this services, execute command: -# docker compose -f docker/docker-compose.yml -f docker/addons/bootstrap/docker-compose.yml up -# from project root. - -networks: - magistrala-base-net: - -volumes: - magistrala-bootstrap-db-volume: - - -services: - bootstrap-db: - image: postgres:16.2-alpine - container_name: magistrala-bootstrap-db - restart: on-failure - environment: - POSTGRES_USER: ${MG_BOOTSTRAP_DB_USER} - POSTGRES_PASSWORD: ${MG_BOOTSTRAP_DB_PASS} - POSTGRES_DB: ${MG_BOOTSTRAP_DB_NAME} - networks: - - magistrala-base-net - volumes: - - magistrala-bootstrap-db-volume:/var/lib/postgresql/data - - bootstrap: - image: magistrala/bootstrap:${MG_RELEASE_TAG} - container_name: magistrala-bootstrap - depends_on: - - bootstrap-db - restart: on-failure - ports: - - ${MG_BOOTSTRAP_HTTP_PORT}:${MG_BOOTSTRAP_HTTP_PORT} - environment: - MG_BOOTSTRAP_LOG_LEVEL: ${MG_BOOTSTRAP_LOG_LEVEL} - MG_BOOTSTRAP_ENCRYPT_KEY: ${MG_BOOTSTRAP_ENCRYPT_KEY} - MG_BOOTSTRAP_EVENT_CONSUMER: ${MG_BOOTSTRAP_EVENT_CONSUMER} - MG_ES_URL: ${MG_ES_URL} - MG_BOOTSTRAP_HTTP_HOST: ${MG_BOOTSTRAP_HTTP_HOST} - MG_BOOTSTRAP_HTTP_PORT: ${MG_BOOTSTRAP_HTTP_PORT} - MG_BOOTSTRAP_HTTP_SERVER_CERT: ${MG_BOOTSTRAP_HTTP_SERVER_CERT} - MG_BOOTSTRAP_HTTP_SERVER_KEY: ${MG_BOOTSTRAP_HTTP_SERVER_KEY} - MG_BOOTSTRAP_DB_HOST: ${MG_BOOTSTRAP_DB_HOST} - MG_BOOTSTRAP_DB_PORT: ${MG_BOOTSTRAP_DB_PORT} - MG_BOOTSTRAP_DB_USER: ${MG_BOOTSTRAP_DB_USER} - MG_BOOTSTRAP_DB_PASS: ${MG_BOOTSTRAP_DB_PASS} - MG_BOOTSTRAP_DB_NAME: ${MG_BOOTSTRAP_DB_NAME} - MG_BOOTSTRAP_DB_SSL_MODE: ${MG_BOOTSTRAP_DB_SSL_MODE} - MG_BOOTSTRAP_DB_SSL_CERT: ${MG_BOOTSTRAP_DB_SSL_CERT} - MG_BOOTSTRAP_DB_SSL_KEY: ${MG_BOOTSTRAP_DB_SSL_KEY} - MG_BOOTSTRAP_DB_SSL_ROOT_CERT: ${MG_BOOTSTRAP_DB_SSL_ROOT_CERT} - MG_AUTH_GRPC_URL: ${MG_AUTH_GRPC_URL} - MG_AUTH_GRPC_TIMEOUT: ${MG_AUTH_GRPC_TIMEOUT} - MG_AUTH_GRPC_CLIENT_CERT: ${MG_AUTH_GRPC_CLIENT_CERT:+/auth-grpc-client.crt} - MG_AUTH_GRPC_CLIENT_KEY: ${MG_AUTH_GRPC_CLIENT_KEY:+/auth-grpc-client.key} - MG_AUTH_GRPC_SERVER_CA_CERTS: ${MG_AUTH_GRPC_SERVER_CA_CERTS:+/auth-grpc-server-ca.crt} - MG_THINGS_URL: ${MG_THINGS_URL} - MG_JAEGER_URL: ${MG_JAEGER_URL} - MG_JAEGER_TRACE_RATIO: ${MG_JAEGER_TRACE_RATIO} - MG_SEND_TELEMETRY: ${MG_SEND_TELEMETRY} - MG_BOOTSTRAP_INSTANCE_ID: ${MG_BOOTSTRAP_INSTANCE_ID} - networks: - - magistrala-base-net - volumes: - - type: bind - source: ${MG_ADDONS_CERTS_PATH_PREFIX}${MG_AUTH_GRPC_CLIENT_CERT:-./ssl/certs/dummy/client_cert} - target: /auth-grpc-client${MG_AUTH_GRPC_CLIENT_CERT:+.crt} - bind: - create_host_path: true - - type: bind - source: ${MG_ADDONS_CERTS_PATH_PREFIX}${MG_AUTH_GRPC_CLIENT_KEY:-./ssl/certs/dummy/client_key} - target: /auth-grpc-client${MG_AUTH_GRPC_CLIENT_KEY:+.key} - bind: - create_host_path: true - - type: bind - source: ${MG_ADDONS_CERTS_PATH_PREFIX}${MG_AUTH_GRPC_SERVER_CA_CERTS:-./ssl/certs/dummy/server_ca} - target: /auth-grpc-server-ca${MG_AUTH_GRPC_SERVER_CA_CERTS:+.crt} - bind: - create_host_path: true diff --git a/docker/addons/certs/docker-compose.yml b/docker/addons/certs/docker-compose.yml deleted file mode 100644 index 69cc9e0..0000000 --- a/docker/addons/certs/docker-compose.yml +++ /dev/null @@ -1,90 +0,0 @@ -# Copyright (c) Abstract Machines -# SPDX-License-Identifier: Apache-2.0 - -# This docker-compose file contains optional certs services. Since it's optional, this file is -# dependent of docker-compose file from /docker. In order to run this services, execute command: -# docker compose -f docker/docker-compose.yml -f docker/addons/certs/docker-compose.yml up -# from project root. - -networks: - magistrala-base-net: - -volumes: - magistrala-certs-db-volume: - - -services: - certs-db: - image: postgres:16.2-alpine - container_name: magistrala-certs-db - restart: on-failure - environment: - POSTGRES_USER: ${MG_CERTS_DB_USER} - POSTGRES_PASSWORD: ${MG_CERTS_DB_PASS} - POSTGRES_DB: ${MG_CERTS_DB_NAME} - networks: - - magistrala-base-net - volumes: - - magistrala-certs-db-volume:/var/lib/postgresql/data - - certs: - image: magistrala/certs:${MG_RELEASE_TAG} - container_name: magistrala-certs - depends_on: - - certs-db - restart: on-failure - networks: - - magistrala-base-net - ports: - - ${MG_CERTS_HTTP_PORT}:${MG_CERTS_HTTP_PORT} - environment: - MG_CERTS_LOG_LEVEL: ${MG_CERTS_LOG_LEVEL} - MG_CERTS_SIGN_CA_PATH: ${MG_CERTS_SIGN_CA_PATH} - MG_CERTS_SIGN_CA_KEY_PATH: ${MG_CERTS_SIGN_CA_KEY_PATH} - MG_CERTS_VAULT_HOST: ${MG_CERTS_VAULT_HOST} - MG_CERTS_VAULT_NAMESPACE: ${MG_CERTS_VAULT_NAMESPACE} - MG_CERTS_VAULT_APPROLE_ROLEID: ${MG_CERTS_VAULT_APPROLE_ROLEID} - MG_CERTS_VAULT_APPROLE_SECRET: ${MG_CERTS_VAULT_APPROLE_SECRET} - MG_CERTS_VAULT_THINGS_CERTS_PKI_PATH: ${MG_CERTS_VAULT_THINGS_CERTS_PKI_PATH} - MG_CERTS_VAULT_THINGS_CERTS_PKI_ROLE_NAME: ${MG_CERTS_VAULT_THINGS_CERTS_PKI_ROLE_NAME} - MG_CERTS_HTTP_HOST: ${MG_CERTS_HTTP_HOST} - MG_CERTS_HTTP_PORT: ${MG_CERTS_HTTP_PORT} - MG_CERTS_HTTP_SERVER_CERT: ${MG_CERTS_HTTP_SERVER_CERT} - MG_CERTS_HTTP_SERVER_KEY: ${MG_CERTS_HTTP_SERVER_KEY} - MG_CERTS_DB_HOST: ${MG_CERTS_DB_HOST} - MG_CERTS_DB_PORT: ${MG_CERTS_DB_PORT} - MG_CERTS_DB_PASS: ${MG_CERTS_DB_PASS} - MG_CERTS_DB_USER: ${MG_CERTS_DB_USER} - MG_CERTS_DB_NAME: ${MG_CERTS_DB_NAME} - MG_CERTS_DB_SSL_MODE: ${MG_CERTS_DB_SSL_MODE} - MG_CERTS_DB_SSL_CERT: ${MG_CERTS_DB_SSL_CERT} - MG_CERTS_DB_SSL_KEY: ${MG_CERTS_DB_SSL_KEY} - MG_CERTS_DB_SSL_ROOT_CERT: ${MG_CERTS_DB_SSL_ROOT_CERT} - MG_AUTH_GRPC_URL: ${MG_AUTH_GRPC_URL} - MG_AUTH_GRPC_TIMEOUT: ${MG_AUTH_GRPC_TIMEOUT} - MG_AUTH_GRPC_CLIENT_CERT: ${MG_AUTH_GRPC_CLIENT_CERT:+/auth-grpc-client.crt} - MG_AUTH_GRPC_CLIENT_KEY: ${MG_AUTH_GRPC_CLIENT_KEY:+/auth-grpc-client.key} - MG_AUTH_GRPC_SERVER_CA_CERTS: ${MG_AUTH_GRPC_SERVER_CA_CERTS:+/auth-grpc-server-ca.crt} - MG_THINGS_URL: ${MG_THINGS_URL} - MG_JAEGER_URL: ${MG_JAEGER_URL} - MG_JAEGER_TRACE_RATIO: ${MG_JAEGER_TRACE_RATIO} - MG_SEND_TELEMETRY: ${MG_SEND_TELEMETRY} - MG_CERTS_INSTANCE_ID: ${MG_CERTS_INSTANCE_ID} - volumes: - - ../../ssl/certs/ca.key:/etc/ssl/certs/ca.key - - ../../ssl/certs/ca.crt:/etc/ssl/certs/ca.crt - - type: bind - source: ${MG_ADDONS_CERTS_PATH_PREFIX}${MG_AUTH_GRPC_CLIENT_CERT:-./ssl/certs/dummy/client_cert} - target: /auth-grpc-client${MG_AUTH_GRPC_CLIENT_CERT:+.crt} - bind: - create_host_path: true - - type: bind - source: ${MG_ADDONS_CERTS_PATH_PREFIX}${MG_AUTH_GRPC_CLIENT_KEY:-./ssl/certs/dummy/client_key} - target: /auth-grpc-client${MG_AUTH_GRPC_CLIENT_KEY:+.key} - bind: - create_host_path: true - - type: bind - source: ${MG_ADDONS_CERTS_PATH_PREFIX}${MG_AUTH_GRPC_SERVER_CA_CERTS:-./ssl/certs/dummy/server_ca} - target: /auth-grpc-server-ca${MG_AUTH_GRPC_SERVER_CA_CERTS:+.crt} - bind: - create_host_path: true diff --git a/docker/addons/journal/docker-compose.yml b/docker/addons/journal/docker-compose.yml deleted file mode 100644 index fa51df0..0000000 --- a/docker/addons/journal/docker-compose.yml +++ /dev/null @@ -1,67 +0,0 @@ -# Copyright (c) Abstract Machines -# SPDX-License-Identifier: Apache-2.0 - -# This docker-compose file contains optional Postgres and journal services -# for Magistrala platform. Since these are optional, this file is dependent of docker-compose file -# from /docker. In order to run these services, execute command: -# docker-compose -f docker/docker-compose.yml -f docker/addons/journal/docker-compose.yml up -# from project root. PostgreSQL default port (5432) is exposed, so you can use various tools for database -# inspection and data visualization. - -networks: - magistrala-base-net: - -volumes: - magistrala-journal-volume: - -services: - journal-db: - image: postgres:16.2-alpine - container_name: magistrala-journal-db - restart: on-failure - command: postgres -c "max_connections=${MG_POSTGRES_MAX_CONNECTIONS}" - environment: - POSTGRES_USER: ${MG_JOURNAL_USER} - POSTGRES_PASSWORD: ${MG_JOURNAL_PASS} - POSTGRES_DB: ${MG_JOURNAL_NAME} - MG_POSTGRES_MAX_CONNECTIONS: ${MG_POSTGRES_MAX_CONNECTIONS} - networks: - - magistrala-base-net - volumes: - - magistrala-journal-volume:/var/lib/postgresql/data - - journal: - image: magistrala/journal:${MG_RELEASE_TAG} - container_name: magistrala-journal - depends_on: - - journal-db - restart: on-failure - environment: - MG_JOURNAL_LOG_LEVEL: ${MG_JOURNAL_LOG_LEVEL} - MG_JOURNAL_HTTP_HOST: ${MG_JOURNAL_HTTP_HOST} - MG_JOURNAL_HTTP_PORT: ${MG_JOURNAL_HTTP_PORT} - MG_JOURNAL_HTTP_SERVER_CERT: ${MG_JOURNAL_HTTP_SERVER_CERT} - MG_JOURNAL_HTTP_SERVER_KEY: ${MG_JOURNAL_HTTP_SERVER_KEY} - MG_JOURNAL_HOST: ${MG_JOURNAL_HOST} - MG_JOURNAL_PORT: ${MG_JOURNAL_PORT} - MG_JOURNAL_USER: ${MG_JOURNAL_USER} - MG_JOURNAL_PASS: ${MG_JOURNAL_PASS} - MG_JOURNAL_NAME: ${MG_JOURNAL_NAME} - MG_JOURNAL_SSL_MODE: ${MG_JOURNAL_SSL_MODE} - MG_JOURNAL_SSL_CERT: ${MG_JOURNAL_SSL_CERT} - MG_JOURNAL_SSL_KEY: ${MG_JOURNAL_SSL_KEY} - MG_JOURNAL_SSL_ROOT_CERT: ${MG_JOURNAL_SSL_ROOT_CERT} - MG_AUTH_GRPC_URL: ${MG_AUTH_GRPC_URL} - MG_AUTH_GRPC_TIMEOUT: ${MG_AUTH_GRPC_TIMEOUT} - MG_AUTH_GRPC_CLIENT_CERT: ${MG_AUTH_GRPC_CLIENT_CERT:+/auth-grpc-client.crt} - MG_AUTH_GRPC_CLIENT_KEY: ${MG_AUTH_GRPC_CLIENT_KEY:+/auth-grpc-client.key} - MG_AUTH_GRPC_SERVER_CA_CERTS: ${MG_AUTH_GRPC_SERVER_CA_CERTS:+/auth-grpc-server-ca.crt} - MG_ES_URL: ${MG_ES_URL} - MG_JAEGER_URL: ${MG_JAEGER_URL} - MG_JAEGER_TRACE_RATIO: ${MG_JAEGER_TRACE_RATIO} - MG_SEND_TELEMETRY: ${MG_SEND_TELEMETRY} - MG_JOURNAL_INSTANCE_ID: ${MG_JOURNAL_INSTANCE_ID} - ports: - - ${MG_JOURNAL_HTTP_PORT}:${MG_JOURNAL_HTTP_PORT} - networks: - - magistrala-base-net diff --git a/docker/addons/postgres-reader/docker-compose.yml b/docker/addons/postgres-reader/docker-compose.yml deleted file mode 100644 index 3b84d6c..0000000 --- a/docker/addons/postgres-reader/docker-compose.yml +++ /dev/null @@ -1,80 +0,0 @@ -# Copyright (c) Abstract Machines -# SPDX-License-Identifier: Apache-2.0 - -# This docker-compose file contains optional Postgres-reader service for Magistrala platform. -# Since this service is optional, this file is dependent of docker-compose.yml file -# from /docker. In order to run this service, execute command: -# docker compose -f docker/docker-compose.yml -f docker/addons/postgres-reader/docker-compose.yml up -# from project root. - -networks: - magistrala-base-net: - -services: - postgres-reader: - image: magistrala/postgres-reader:${MG_RELEASE_TAG} - container_name: magistrala-postgres-reader - restart: on-failure - environment: - MG_POSTGRES_READER_LOG_LEVEL: ${MG_POSTGRES_READER_LOG_LEVEL} - MG_POSTGRES_READER_HTTP_HOST: ${MG_POSTGRES_READER_HTTP_HOST} - MG_POSTGRES_READER_HTTP_PORT: ${MG_POSTGRES_READER_HTTP_PORT} - MG_POSTGRES_READER_HTTP_SERVER_CERT: ${MG_POSTGRES_READER_HTTP_SERVER_CERT} - MG_POSTGRES_READER_HTTP_SERVER_KEY: ${MG_POSTGRES_READER_HTTP_SERVER_KEY} - MG_POSTGRES_HOST: ${MG_POSTGRES_HOST} - MG_POSTGRES_PORT: ${MG_POSTGRES_PORT} - MG_POSTGRES_USER: ${MG_POSTGRES_USER} - MG_POSTGRES_PASS: ${MG_POSTGRES_PASS} - MG_POSTGRES_NAME: ${MG_POSTGRES_NAME} - MG_POSTGRES_SSL_MODE: ${MG_POSTGRES_SSL_MODE} - MG_POSTGRES_SSL_CERT: ${MG_POSTGRES_SSL_CERT} - MG_POSTGRES_SSL_KEY: ${MG_POSTGRES_SSL_KEY} - MG_POSTGRES_SSL_ROOT_CERT: ${MG_POSTGRES_SSL_ROOT_CERT} - MG_THINGS_AUTH_GRPC_URL: ${MG_THINGS_AUTH_GRPC_URL} - MG_THINGS_AUTH_GRPC_TIMEOUT: ${MG_THINGS_AUTH_GRPC_TIMEOUT} - MG_THINGS_AUTH_GRPC_CLIENT_CERT: ${MG_THINGS_AUTH_GRPC_CLIENT_CERT:+/things-grpc-client.crt} - MG_THINGS_AUTH_GRPC_CLIENT_KEY: ${MG_THINGS_AUTH_GRPC_CLIENT_KEY:+/things-grpc-client.key} - MG_THINGS_AUTH_GRPC_SERVER_CA_CERTS: ${MG_THINGS_AUTH_GRPC_SERVER_CA_CERTS:+/things-grpc-server-ca.crt} - MG_AUTH_GRPC_URL: ${MG_AUTH_GRPC_URL} - MG_AUTH_GRPC_TIMEOUT: ${MG_AUTH_GRPC_TIMEOUT} - MG_AUTH_GRPC_CLIENT_CERT: ${MG_AUTH_GRPC_CLIENT_CERT:+/auth-grpc-client.crt} - MG_AUTH_GRPC_CLIENT_KEY: ${MG_AUTH_GRPC_CLIENT_KEY:+/auth-grpc-client.key} - MG_AUTH_GRPC_SERVER_CA_CERTS: ${MG_AUTH_GRPC_SERVER_CA_CERTS:+/auth-grpc-server-ca.crt} - MG_SEND_TELEMETRY: ${MG_SEND_TELEMETRY} - MG_POSTGRES_READER_INSTANCE_ID: ${MG_POSTGRES_READER_INSTANCE_ID} - ports: - - ${MG_POSTGRES_READER_HTTP_PORT}:${MG_POSTGRES_READER_HTTP_PORT} - networks: - - magistrala-base-net - volumes: - - type: bind - source: ${MG_ADDONS_CERTS_PATH_PREFIX}${MG_AUTH_GRPC_CLIENT_CERT:-./ssl/certs/dummy/client_cert} - target: /auth-grpc-client${MG_AUTH_GRPC_CLIENT_CERT:+.crt} - bind: - create_host_path: true - - type: bind - source: ${MG_ADDONS_CERTS_PATH_PREFIX}${MG_AUTH_GRPC_CLIENT_KEY:-./ssl/certs/dummy/client_key} - target: /auth-grpc-client${MG_AUTH_GRPC_CLIENT_KEY:+.key} - bind: - create_host_path: true - - type: bind - source: ${MG_ADDONS_CERTS_PATH_PREFIX}${MG_AUTH_GRPC_SERVER_CA_CERTS:-./ssl/certs/dummy/server_ca} - target: /auth-grpc-server-ca${MG_AUTH_GRPC_SERVER_CA_CERTS:+.crt} - bind: - create_host_path: true - # Things gRPC mTLS client certificates - - type: bind - source: ${MG_ADDONS_CERTS_PATH_PREFIX}${MG_THINGS_AUTH_GRPC_CLIENT_CERT:-ssl/certs/dummy/client_cert} - target: /things-grpc-client${MG_THINGS_AUTH_GRPC_CLIENT_CERT:+.crt} - bind: - create_host_path: true - - type: bind - source: ${MG_ADDONS_CERTS_PATH_PREFIX}${MG_THINGS_AUTH_GRPC_CLIENT_KEY:-ssl/certs/dummy/client_key} - target: /things-grpc-client${MG_THINGS_AUTH_GRPC_CLIENT_KEY:+.key} - bind: - create_host_path: true - - type: bind - source: ${MG_ADDONS_CERTS_PATH_PREFIX}${MG_THINGS_AUTH_GRPC_SERVER_CA_CERTS:-ssl/certs/dummy/server_ca} - target: /things-grpc-server-ca${MG_THINGS_AUTH_GRPC_SERVER_CA_CERTS:+.crt} - bind: - create_host_path: true diff --git a/docker/addons/postgres-writer/config.toml b/docker/addons/postgres-writer/config.toml deleted file mode 100644 index b04ce56..0000000 --- a/docker/addons/postgres-writer/config.toml +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright (c) Abstract Machines -# SPDX-License-Identifier: Apache-2.0 - -# To listen all messsage broker subjects use default value "channels.>". -# To subscribe to specific subjects use values starting by "channels." and -# followed by a subtopic (e.g ["channels..sub.topic.x", ...]). -[subscriber] -subjects = ["channels.>"] - -[transformer] -# SenML or JSON -format = "senml" -# Used if format is SenML -content_type = "application/senml+json" -# Used as timestamp fields if format is JSON -time_fields = [{ field_name = "seconds_key", field_format = "unix", location = "UTC"}, - { field_name = "millis_key", field_format = "unix_ms", location = "UTC"}, - { field_name = "micros_key", field_format = "unix_us", location = "UTC"}, - { field_name = "nanos_key", field_format = "unix_ns", location = "UTC"}] diff --git a/docker/addons/postgres-writer/docker-compose.yml b/docker/addons/postgres-writer/docker-compose.yml deleted file mode 100644 index c5e1964..0000000 --- a/docker/addons/postgres-writer/docker-compose.yml +++ /dev/null @@ -1,63 +0,0 @@ -# Copyright (c) Abstract Machines -# SPDX-License-Identifier: Apache-2.0 - -# This docker-compose file contains optional Postgres and Postgres-writer services -# for Magistrala platform. Since these are optional, this file is dependent of docker-compose file -# from /docker. In order to run these services, execute command: -# docker compose -f docker/docker-compose.yml -f docker/addons/postgres-writer/docker-compose.yml up -# from project root. PostgreSQL default port (5432) is exposed, so you can use various tools for database -# inspection and data visualization. - -networks: - magistrala-base-net: - -volumes: - magistrala-postgres-writer-volume: - -services: - postgres: - image: postgres:16.2-alpine - container_name: magistrala-postgres - restart: on-failure - environment: - POSTGRES_USER: ${MG_POSTGRES_USER} - POSTGRES_PASSWORD: ${MG_POSTGRES_PASS} - POSTGRES_DB: ${MG_POSTGRES_NAME} - networks: - - magistrala-base-net - volumes: - - magistrala-postgres-writer-volume:/var/lib/postgresql/data - - postgres-writer: - image: magistrala/postgres-writer:${MG_RELEASE_TAG} - container_name: magistrala-postgres-writer - depends_on: - - postgres - restart: on-failure - environment: - MG_POSTGRES_WRITER_LOG_LEVEL: ${MG_POSTGRES_WRITER_LOG_LEVEL} - MG_POSTGRES_WRITER_CONFIG_PATH: ${MG_POSTGRES_WRITER_CONFIG_PATH} - MG_POSTGRES_WRITER_HTTP_HOST: ${MG_POSTGRES_WRITER_HTTP_HOST} - MG_POSTGRES_WRITER_HTTP_PORT: ${MG_POSTGRES_WRITER_HTTP_PORT} - MG_POSTGRES_WRITER_HTTP_SERVER_CERT: ${MG_POSTGRES_WRITER_HTTP_SERVER_CERT} - MG_POSTGRES_WRITER_HTTP_SERVER_KEY: ${MG_POSTGRES_WRITER_HTTP_SERVER_KEY} - MG_POSTGRES_HOST: ${MG_POSTGRES_HOST} - MG_POSTGRES_PORT: ${MG_POSTGRES_PORT} - MG_POSTGRES_USER: ${MG_POSTGRES_USER} - MG_POSTGRES_PASS: ${MG_POSTGRES_PASS} - MG_POSTGRES_NAME: ${MG_POSTGRES_NAME} - MG_POSTGRES_SSL_MODE: ${MG_POSTGRES_SSL_MODE} - MG_POSTGRES_SSL_CERT: ${MG_POSTGRES_SSL_CERT} - MG_POSTGRES_SSL_KEY: ${MG_POSTGRES_SSL_KEY} - MG_POSTGRES_SSL_ROOT_CERT: ${MG_POSTGRES_SSL_ROOT_CERT} - MG_MESSAGE_BROKER_URL: ${MG_MESSAGE_BROKER_URL} - MG_JAEGER_URL: ${MG_JAEGER_URL} - MG_JAEGER_TRACE_RATIO: ${MG_JAEGER_TRACE_RATIO} - MG_SEND_TELEMETRY: ${MG_SEND_TELEMETRY} - MG_POSTGRES_WRITER_INSTANCE_ID: ${MG_POSTGRES_WRITER_INSTANCE_ID} - ports: - - ${MG_POSTGRES_WRITER_HTTP_PORT}:${MG_POSTGRES_WRITER_HTTP_PORT} - networks: - - magistrala-base-net - volumes: - - ./config.toml:/config.toml diff --git a/docker/addons/provision/configs/config.toml b/docker/addons/provision/configs/config.toml deleted file mode 100644 index ec1ee38..0000000 --- a/docker/addons/provision/configs/config.toml +++ /dev/null @@ -1,74 +0,0 @@ -# Copyright (c) Abstract Machines -# SPDX-License-Identifier: Apache-2.0 - -[bootstrap] - [bootstrap.content] - [bootstrap.content.agent.edgex] - url = "http://localhost:48090/api/v1/" - - [bootstrap.content.agent.log] - level = "info" - - [bootstrap.content.agent.mqtt] - mtls = false - qos = 0 - retain = false - skip_tls_ver = true - url = "localhost:1883" - - [bootstrap.content.agent.server] - nats_url = "localhost:4222" - port = "9000" - - [bootstrap.content.agent.heartbeat] - interval = "30s" - - [bootstrap.content.agent.terminal] - session_timeout = "30s" - - - [bootstrap.content.export.exp] - log_level = "debug" - nats = "nats://localhost:4222" - port = "8172" - cache_url = "localhost:6379" - cache_pass = "" - cache_db = "0" - - [bootstrap.content.export.mqtt] - ca_path = "ca.crt" - cert_path = "thing.crt" - channel = "" - host = "tcp://localhost:1883" - mtls = false - password = "" - priv_key_path = "thing.key" - qos = 0 - retain = false - skip_tls_ver = false - username = "" - - [[bootstrap.content.export.routes]] - mqtt_topic = "" - nats_topic = ">" - subtopic = "" - type = "plain" - workers = 10 - -[[things]] - name = "thing" - - [things.metadata] - external_id = "xxxxxx" - -[[channels]] - name = "control-channel" - - [channels.metadata] - type = "control" - -[[channels]] - name = "data-channel" - - [channels.metadata] - type = "data" diff --git a/docker/addons/provision/docker-compose.yml b/docker/addons/provision/docker-compose.yml deleted file mode 100644 index 7709f40..0000000 --- a/docker/addons/provision/docker-compose.yml +++ /dev/null @@ -1,45 +0,0 @@ -# Copyright (c) Abstract Machines -# SPDX-License-Identifier: Apache-2.0 - -# This docker-compose file contains optional provision services. Since it's optional, this file is -# dependent of docker-compose file from /docker. In order to run this services, execute command: -# docker compose -f docker/docker-compose.yml -f docker/addons/provision/docker-compose.yml up -# from project root. - -networks: - magistrala-base-net: - -services: - provision: - image: magistrala/provision:${MG_RELEASE_TAG} - container_name: magistrala-provision - restart: on-failure - networks: - - magistrala-base-net - ports: - - ${MG_PROVISION_HTTP_PORT}:${MG_PROVISION_HTTP_PORT} - environment: - MG_PROVISION_LOG_LEVEL: ${MG_PROVISION_LOG_LEVEL} - MG_PROVISION_HTTP_PORT: ${MG_PROVISION_HTTP_PORT} - MG_PROVISION_CONFIG_FILE: ${MG_PROVISION_CONFIG_FILE} - MG_PROVISION_ENV_CLIENTS_TLS: ${MG_PROVISION_ENV_CLIENTS_TLS} - MG_PROVISION_SERVER_CERT: ${MG_PROVISION_SERVER_CERT} - MG_PROVISION_SERVER_KEY: ${MG_PROVISION_SERVER_KEY} - MG_PROVISION_USERS_LOCATION: ${MG_PROVISION_USERS_LOCATION} - MG_PROVISION_THINGS_LOCATION: ${MG_PROVISION_THINGS_LOCATION} - MG_PROVISION_USER: ${MG_PROVISION_USER} - MG_PROVISION_PASS: ${MG_PROVISION_PASS} - MG_PROVISION_API_KEY: ${MG_PROVISION_API_KEY} - MG_PROVISION_CERTS_SVC_URL: ${MG_PROVISION_CERTS_SVC_URL} - MG_PROVISION_X509_PROVISIONING: ${MG_PROVISION_X509_PROVISIONING} - MG_PROVISION_BS_SVC_URL: ${MG_PROVISION_BS_SVC_URL} - MG_PROVISION_BS_CONFIG_PROVISIONING: ${MG_PROVISION_BS_CONFIG_PROVISIONING} - MG_PROVISION_BS_AUTO_WHITELIST: ${MG_PROVISION_BS_AUTO_WHITELIST} - MG_PROVISION_BS_CONTENT: ${MG_PROVISION_BS_CONTENT} - MG_PROVISION_CERTS_HOURS_VALID: ${MG_PROVISION_CERTS_HOURS_VALID} - MG_SEND_TELEMETRY: ${MG_SEND_TELEMETRY} - MG_PROVISION_INSTANCE_ID: ${MG_PROVISION_INSTANCE_ID} - volumes: - - ./configs:/configs - - ../../ssl/certs/ca.key:/etc/ssl/certs/ca.key - - ../../ssl/certs/ca.crt:/etc/ssl/certs/ca.crt diff --git a/docker/addons/timescale-reader/docker-compose.yml b/docker/addons/timescale-reader/docker-compose.yml deleted file mode 100644 index 269e1c6..0000000 --- a/docker/addons/timescale-reader/docker-compose.yml +++ /dev/null @@ -1,80 +0,0 @@ -# Copyright (c) Abstract Machines -# SPDX-License-Identifier: Apache-2.0 - -# This docker-compose file contains optional Timescale-reader service for Magistrala platform. -# Since this service is optional, this file is dependent of docker-compose.yml file -# from /docker. In order to run this service, execute command: -# docker compose -f docker/docker-compose.yml -f docker/addons/timescale-reader/docker-compose.yml up -# from project root. - -networks: - magistrala-base-net: - -services: - timescale-reader: - image: magistrala/timescale-reader:${MG_RELEASE_TAG} - container_name: magistrala-timescale-reader - restart: on-failure - environment: - MG_TIMESCALE_READER_LOG_LEVEL: ${MG_TIMESCALE_READER_LOG_LEVEL} - MG_TIMESCALE_READER_HTTP_HOST: ${MG_TIMESCALE_READER_HTTP_HOST} - MG_TIMESCALE_READER_HTTP_PORT: ${MG_TIMESCALE_READER_HTTP_PORT} - MG_TIMESCALE_READER_HTTP_SERVER_CERT: ${MG_TIMESCALE_READER_HTTP_SERVER_CERT} - MG_TIMESCALE_READER_HTTP_SERVER_KEY: ${MG_TIMESCALE_READER_HTTP_SERVER_KEY} - MG_TIMESCALE_HOST: ${MG_TIMESCALE_HOST} - MG_TIMESCALE_PORT: ${MG_TIMESCALE_PORT} - MG_TIMESCALE_USER: ${MG_TIMESCALE_USER} - MG_TIMESCALE_PASS: ${MG_TIMESCALE_PASS} - MG_TIMESCALE_NAME: ${MG_TIMESCALE_NAME} - MG_TIMESCALE_SSL_MODE: ${MG_TIMESCALE_SSL_MODE} - MG_TIMESCALE_SSL_CERT: ${MG_TIMESCALE_SSL_CERT} - MG_TIMESCALE_SSL_KEY: ${MG_TIMESCALE_SSL_KEY} - MG_TIMESCALE_SSL_ROOT_CERT: ${MG_TIMESCALE_SSL_ROOT_CERT} - MG_THINGS_AUTH_GRPC_URL: ${MG_THINGS_AUTH_GRPC_URL} - MG_THINGS_AUTH_GRPC_TIMEOUT: ${MG_THINGS_AUTH_GRPC_TIMEOUT} - MG_THINGS_AUTH_GRPC_CLIENT_CERT: ${MG_THINGS_AUTH_GRPC_CLIENT_CERT:+/things-grpc-client.crt} - MG_THINGS_AUTH_GRPC_CLIENT_KEY: ${MG_THINGS_AUTH_GRPC_CLIENT_KEY:+/things-grpc-client.key} - MG_THINGS_AUTH_GRPC_SERVER_CA_CERTS: ${MG_THINGS_AUTH_GRPC_SERVER_CA_CERTS:+/things-grpc-server-ca.crt} - MG_AUTH_GRPC_URL: ${MG_AUTH_GRPC_URL} - MG_AUTH_GRPC_TIMEOUT: ${MG_AUTH_GRPC_TIMEOUT} - MG_AUTH_GRPC_CLIENT_CERT: ${MG_AUTH_GRPC_CLIENT_CERT:+/auth-grpc-client.crt} - MG_AUTH_GRPC_CLIENT_KEY: ${MG_AUTH_GRPC_CLIENT_KEY:+/auth-grpc-client.key} - MG_AUTH_GRPC_SERVER_CA_CERTS: ${MG_AUTH_GRPC_SERVER_CA_CERTS:+/auth-grpc-server-ca.crt} - MG_SEND_TELEMETRY: ${MG_SEND_TELEMETRY} - MG_TIMESCALE_READER_INSTANCE_ID: ${MG_TIMESCALE_READER_INSTANCE_ID} - ports: - - ${MG_TIMESCALE_READER_HTTP_PORT}:${MG_TIMESCALE_READER_HTTP_PORT} - networks: - - magistrala-base-net - volumes: - - type: bind - source: ${MG_ADDONS_CERTS_PATH_PREFIX}${MG_AUTH_GRPC_CLIENT_CERT:-./ssl/certs/dummy/client_cert} - target: /auth-grpc-client${MG_AUTH_GRPC_CLIENT_CERT:+.crt} - bind: - create_host_path: true - - type: bind - source: ${MG_ADDONS_CERTS_PATH_PREFIX}${MG_AUTH_GRPC_CLIENT_KEY:-./ssl/certs/dummy/client_key} - target: /auth-grpc-client${MG_AUTH_GRPC_CLIENT_KEY:+.key} - bind: - create_host_path: true - - type: bind - source: ${MG_ADDONS_CERTS_PATH_PREFIX}${MG_AUTH_GRPC_SERVER_CA_CERTS:-./ssl/certs/dummy/server_ca} - target: /auth-grpc-server-ca${MG_AUTH_GRPC_SERVER_CA_CERTS:+.crt} - bind: - create_host_path: true - # Things gRPC mTLS client certificates - - type: bind - source: ${MG_ADDONS_CERTS_PATH_PREFIX}${MG_THINGS_AUTH_GRPC_CLIENT_CERT:-ssl/certs/dummy/client_cert} - target: /things-grpc-client${MG_THINGS_AUTH_GRPC_CLIENT_CERT:+.crt} - bind: - create_host_path: true - - type: bind - source: ${MG_ADDONS_CERTS_PATH_PREFIX}${MG_THINGS_AUTH_GRPC_CLIENT_KEY:-ssl/certs/dummy/client_key} - target: /things-grpc-client${MG_THINGS_AUTH_GRPC_CLIENT_KEY:+.key} - bind: - create_host_path: true - - type: bind - source: ${MG_ADDONS_CERTS_PATH_PREFIX}${MG_THINGS_AUTH_GRPC_SERVER_CA_CERTS:-ssl/certs/dummy/server_ca} - target: /things-grpc-server-ca${MG_THINGS_AUTH_GRPC_SERVER_CA_CERTS:+.crt} - bind: - create_host_path: true diff --git a/docker/addons/timescale-writer/config.toml b/docker/addons/timescale-writer/config.toml deleted file mode 100644 index f3ad91d..0000000 --- a/docker/addons/timescale-writer/config.toml +++ /dev/null @@ -1,8 +0,0 @@ -# Copyright (c) Abstract Machines -# SPDX-License-Identifier: Apache-2.0 - -# To listen all messsage broker subjects use default value "channels.>". -# To subscribe to specific subjects use values starting by "channels." and -# followed by a subtopic (e.g ["channels..sub.topic.x", ...]). -[subjects] -filter = ["channels.>"] diff --git a/docker/addons/timescale-writer/docker-compose.yml b/docker/addons/timescale-writer/docker-compose.yml deleted file mode 100644 index 125315a..0000000 --- a/docker/addons/timescale-writer/docker-compose.yml +++ /dev/null @@ -1,65 +0,0 @@ -# Copyright (c) Abstract Machines -# SPDX-License-Identifier: Apache-2.0 - -# This docker-compose file contains optional Timescale and Timescale-writer services -# for Magistrala platform. Since these are optional, this file is dependent of docker-compose file -# from /docker. In order to run these services, execute command: -# docker compose -f docker/docker-compose.yml -f docker/addons/timescale-writer/docker-compose.yml up -# from project root. PostgreSQL default port (5432) is exposed, so you can use various tools for database -# inspection and data visualization. - -networks: - magistrala-base-net: - -volumes: - magistrala-timescale-writer-volume: - -services: - timescale: - image: timescale/timescaledb:2.13.1-pg16 - container_name: magistrala-timescale - restart: on-failure - environment: - POSTGRES_PASSWORD: ${MG_TIMESCALE_PASS} - POSTGRES_USER: ${MG_TIMESCALE_USER} - POSTGRES_DB: ${MG_TIMESCALE_NAME} - ports: - - 5433:5432 - networks: - - magistrala-base-net - volumes: - - magistrala-timescale-writer-volume:/var/lib/timescalesql/data - - timescale-writer: - image: magistrala/timescale-writer:${MG_RELEASE_TAG} - container_name: magistrala-timescale-writer - depends_on: - - timescale - restart: on-failure - environment: - MG_TIMESCALE_WRITER_LOG_LEVEL: ${MG_TIMESCALE_WRITER_LOG_LEVEL} - MG_TIMESCALE_WRITER_CONFIG_PATH: ${MG_TIMESCALE_WRITER_CONFIG_PATH} - MG_TIMESCALE_WRITER_HTTP_HOST: ${MG_TIMESCALE_WRITER_HTTP_HOST} - MG_TIMESCALE_WRITER_HTTP_PORT: ${MG_TIMESCALE_WRITER_HTTP_PORT} - MG_TIMESCALE_WRITER_HTTP_SERVER_CERT: ${MG_TIMESCALE_WRITER_HTTP_SERVER_CERT} - MG_TIMESCALE_WRITER_HTTP_SERVER_KEY: ${MG_TIMESCALE_WRITER_HTTP_SERVER_KEY} - MG_TIMESCALE_HOST: ${MG_TIMESCALE_HOST} - MG_TIMESCALE_PORT: ${MG_TIMESCALE_PORT} - MG_TIMESCALE_USER: ${MG_TIMESCALE_USER} - MG_TIMESCALE_PASS: ${MG_TIMESCALE_PASS} - MG_TIMESCALE_NAME: ${MG_TIMESCALE_NAME} - MG_TIMESCALE_SSL_MODE: ${MG_TIMESCALE_SSL_MODE} - MG_TIMESCALE_SSL_CERT: ${MG_TIMESCALE_SSL_CERT} - MG_TIMESCALE_SSL_KEY: ${MG_TIMESCALE_SSL_KEY} - MG_TIMESCALE_SSL_ROOT_CERT: ${MG_TIMESCALE_SSL_ROOT_CERT} - MG_MESSAGE_BROKER_URL: ${MG_MESSAGE_BROKER_URL} - MG_JAEGER_URL: ${MG_JAEGER_URL} - MG_JAEGER_TRACE_RATIO: ${MG_JAEGER_TRACE_RATIO} - MG_SEND_TELEMETRY: ${MG_SEND_TELEMETRY} - MG_TIMESCALE_WRITER_INSTANCE_ID: ${MG_TIMESCALE_WRITER_INSTANCE_ID} - ports: - - ${MG_TIMESCALE_WRITER_HTTP_PORT}:${MG_TIMESCALE_WRITER_HTTP_PORT} - networks: - - magistrala-base-net - volumes: - - ./config.toml:/config.toml diff --git a/docker/addons/vault/.gitignore b/docker/addons/vault/.gitignore deleted file mode 100644 index 4f14d39..0000000 --- a/docker/addons/vault/.gitignore +++ /dev/null @@ -1,5 +0,0 @@ -# Copyright (c) Abstract Machines -# SPDX-License-Identifier: Apache-2.0 - -data -magistrala_things_certs_issue.hcl diff --git a/docker/addons/vault/README.md b/docker/addons/vault/README.md deleted file mode 100644 index 1ac1136..0000000 --- a/docker/addons/vault/README.md +++ /dev/null @@ -1,170 +0,0 @@ -# Vault - -This is Vault service deployment to be used with Magistrala. - -When the Vault service is started, some initialization steps need to be done to set things up. - -## Configuration - -| Variable | Description | Default | -| :-------------------------------------- | ----------------------------------------------------------------------------- | ------------------------------------- | -| MG_VAULT_ADDR | Vault Address | http://vault:8200 | -| MG_VAULT_UNSEAL_KEY_1 | Vault unseal key | "" | -| MG_VAULT_UNSEAL_KEY_2 | Vault unseal key | "" | -| MG_VAULT_UNSEAL_KEY_3 | Vault unseal key | "" | -| MG_VAULT_TOKEN | Vault cli access token | "" | -| MG_VAULT_PKI_PATH | Vault secrets engine path for Root CA | pki | -| MG_VAULT_PKI_ROLE_NAME | Vault Root CA role name to issue intermediate CA | magistrala_int_ca | -| MG_VAULT_PKI_FILE_NAME | Root CA Certificates name used by`vault_set_pki.sh` | mg_root | -| MG_VAULT_PKI_CA_CN | Common name used for Root CA creation by`vault_set_pki.sh` | Magistrala Root Certificate Authority | -| MG_VAULT_PKI_CA_OU | Organization unit used for Root CA creation by`vault_set_pki.sh` | Magistrala | -| MG_VAULT_PKI_CA_O | Organization used for Root CA creation by`vault_set_pki.sh` | Magistrala | -| MG_VAULT_PKI_CA_C | Country used for Root CA creation by`vault_set_pki.sh` | FRANCE | -| MG_VAULT_PKI_CA_L | Location used for Root CA creation by`vault_set_pki.sh` | PARIS | -| MG_VAULT_PKI_CA_ST | State or Provisions used for Root CA creation by`vault_set_pki.sh` | PARIS | -| MG_VAULT_PKI_CA_ADDR | Address used for Root CA creation by`vault_set_pki.sh` | 5 Av. Anatole | -| MG_VAULT_PKI_CA_PO | Postal code used for Root CA creation by`vault_set_pki.sh` | 75007 | -| MG_VAULT_PKI_CLUSTER_PATH | Vault Root CA Cluster Path | http://localhost | -| MG_VAULT_PKI_CLUSTER_AIA_PATH | Vault Root CA Cluster AIA Path | http://localhost | -| MG_VAULT_PKI_INT_PATH | Vault secrets engine path for Intermediate CA | pki_int | -| MG_VAULT_PKI_INT_SERVER_CERTS_ROLE_NAME | Vault Intermediate CA role name to issue server certificate | magistrala_server_certs | -| MG_VAULT_PKI_INT_THINGS_CERTS_ROLE_NAME | Vault Intermediate CA role name to issue Things certificates | magistrala_things_certs | -| MG_VAULT_PKI_INT_FILE_NAME | Intermediate CA Certificates name used by`vault_set_pki.sh` | mg_root | -| MG_VAULT_PKI_INT_CA_CN | Common name used for Intermediate CA creation by`vault_set_pki.sh` | Magistrala Root Certificate Authority | -| MG_VAULT_PKI_INT_CA_OU | Organization unit used for Root CA creation by`vault_set_pki.sh` | Magistrala | -| MG_VAULT_PKI_INT_CA_O | Organization used for Intermediate CA creation by`vault_set_pki.sh` | Magistrala | -| MG_VAULT_PKI_INT_CA_C | Country used for Intermediate CA creation by`vault_set_pki.sh` | FRANCE | -| MG_VAULT_PKI_INT_CA_L | Location used for Intermediate CA creation by`vault_set_pki.sh` | PARIS | -| MG_VAULT_PKI_INT_CA_ST | State or Provisions used for Intermediate CA creation by`vault_set_pki.sh` | PARIS | -| MG_VAULT_PKI_INT_CA_ADDR | Address used for Intermediate CA creation by`vault_set_pki.sh` | 5 Av. Anatole | -| MG_VAULT_PKI_INT_CA_PO | Postal code used for Intermediate CA creation by`vault_set_pki.sh` | 75007 | -| MG_VAULT_PKI_INT_CLUSTER_PATH | Vault Intermediate CA Cluster Path | http://localhost | -| MG_VAULT_PKI_INT_CLUSTER_AIA_PATH | Vault Intermediate CA Cluster AIA Path | http://localhost | -| MG_VAULT_THINGS_CERTS_ISSUER_ROLEID | Vault Intermediate CA Things Certificate issuer AppRole authentication RoleID | magistrala | -| MG_VAULT_THINGS_CERTS_ISSUER_SECRET | Vault Intermediate CA Things Certificate issuer AppRole authentication Secret | magistrala | - -## Setup - -The following scripts are provided, which work on the running Vault service in Docker. - -### 1. `vault_init.sh` - -Calls `vault operator init` to perform the initial vault initialization and generates a `docker/addons/vault/data/secrets` file which contains the Vault unseal keys and root tokens. - -Example contents for `data/secrets`: - -```bash -Unseal Key 1: Ay0YZecYJ2HVtNtXfPootXK5LtF+JZoDmBb7IbbYdLBI -Unseal Key 2: P6hb7x2cglv0p61jdLyNE3+d44cJUOFaDt9jHFDfr8Df -Unseal Key 3: zSBfDHzUiWoOzXKY1pnnBqKO8UD2MDLuy8DNTxNtEBFy -Unseal Key 4: 5oJuDDuMI0I8snaw/n4VLNpvndvvKi6JlkgOxuWXqMSz -Unseal Key 5: ZhsUkk2tXBYEcWgz4WUCHH9rocoW6qZoiARWlkE5Epi5 - -Initial Root Token: s.V2hdd00P4bHtUQnoWZK2hSaS - -Vault initialized with 5 key shares and a key threshold of 3. Please securely -distribute the key shares printed above. When the Vault is re-sealed, -restarted, or stopped, you must supply at least 3 of these keys to unseal it -before it can start servicing requests. - -Vault does not store the generated master key. Without at least 3 key to -reconstruct the master key, Vault will remain permanently sealed! - -It is possible to generate new unseal keys, provided you have a quorum of -existing unseal keys shares. See "vault operator rekey" for more information. -bash-4.4 - -Use 3 out of five keys presented and put it into .env file and than start the composition again Vault should be in unsealed state ( take a note that this is not recommended in terms of security, this is deployment for development) A real production deployment can use Vault auto unseal mode where vault gets unseal keys from some 3rd party KMS ( on AWS for example) -``` - -### 2. `vault_copy_env.sh` - -After first step, the corresponding Vault environment variables (`MG_VAULT_TOKEN`, `MG_VAULT_UNSEAL_KEY_1`, `MG_VAULT_UNSEAL_KEY_2`, `MG_VAULT_UNSEAL_KEY_3`) should be updated in `.env` file. - -`vault_copy_env.sh` scripts copies values from `docker/addons/vault/data/secrets` file and update environmental variables `MG_VAULT_TOKEN`, `MG_VAULT_UNSEAL_KEY_1`, `MG_VAULT_UNSEAL_KEY_2`, `MG_VAULT_UNSEAL_KEY_3` present in `.env` file. - -### 3. `vault_unseal.sh` - -This can be run after the initialization to unseal Vault, which is necessary for it to be used to store and/or get secrets. - -This can be used if you don't want to restart the service. - -The unseal environment variables need to be set in `.env` for the script to work (`MG_VAULT_TOKEN`,`MG_VAULT_UNSEAL_KEY_1`, `MG_VAULT_UNSEAL_KEY_2`, `MG_VAULT_UNSEAL_KEY_3`). - -This script should not be necessary to run after the initial setup, since the Vault service unseals itself when starting the container. - -### 4. `vault_set_pki.sh` - -This script is used to generate the root certificate, intermediate certificate and HTTPS server certificate. -All generate certificates, keys and CSR by `vault_set_pki.sh` will be present at `docker/addons/vault/data`. - -The parameters required for generating certificate are obtained from the environment variables which are loaded from `docker/.env`. - -Environmental variables starting with `MG_VAULT_PKI` in `docker/.env` file are used by `vault_set_pki.sh` to generate root CA. -Environmental variables starting with`MG_VAULT_PKI_INT` in `docker/.env` file are used by `vault_set_pki.sh` to generate intermediate CA. - -Passing command line args `--skip-server-cert` to `vault_set_pki.sh` will skip server certificate role & process of generation of server certificate & key. - -### 5. `vault_create_approle.sh` - -This script is used to enable app role authorization in Vault. Certs service used the approle credentials to issue, revoke things certificate from vault intermedate CA. - -`vault_create_approle.sh` script by default tries to enable auth approle. -If approle is already enabled in vault, then use args `--skip-enable-approle` to skip enable auth approle step. -To skip enable auth approle step use the following `vault_create_approle.sh --skip-enable-approle` - -### 6. `vault_copy_certs.sh` - -This scripts copies the necessary certificates and keys from `docker/addons/vault/data` to the `docker/ssl/certs` folder. - -## Hashicorp Cloud Platform (HCP) Vault - -To have the same PKI setup can done in Hashicorp Cloud Platform (HCP) Vault follow the below steps: -Requirement: [VAULT CLI](https://developer.hashicorp.com/vault/tutorials/getting-started/getting-started-install) - -- Replace the environmental variable `MG_VAULT_ADDR` in `docker/.env` with HCP Vault address. -- Replace the environmental variable `MG_VAULT_TOKEN` in `docker/.env` with HCP Vault Admin token. -- Run script `vault_set_pki.sh` and `vault_create_approle.sh`. -- Optional step, run script `vault_copy_certs.sh` to copy certificates to magistrala default path. - -## Vault CLI - -It can also be useful to run the Vault CLI for inspection and administration work. - -```bash -Usage: vault [args] - -Common commands: - read Read data and retrieves secrets - write Write data, configuration, and secrets - delete Delete secrets and configuration - list List data or secrets - login Authenticate locally - agent Start a Vault agent - server Start a Vault server - status Print seal and HA status - unwrap Unwrap a wrapped secret - -Other commands: - audit Interact with audit devices - auth Interact with auth methods - debug Runs the debug command - kv Interact with Vault's Key-Value storage - lease Interact with leases - monitor Stream log messages from a Vault server - namespace Interact with namespaces - operator Perform operator-specific tasks - path-help Retrieve API help for paths - plugin Interact with Vault plugins and catalog - policy Interact with policies - print Prints runtime configurations - secrets Interact with secrets engines - ssh Initiate an SSH session - token Interact with tokens -``` - -If the Vault is setup through `docker/addons/vault`, then Vault CLI can be run directly using the Vault image in Docker: `docker run -it magistrala/vault:latest vault` - -## Vault Web UI - -If the Vault is setup through `docker/addons/vault`, Then Vault Web UI is accessible by default on `http://localhost:8200/ui`. diff --git a/docker/addons/vault/config.hcl b/docker/addons/vault/config.hcl deleted file mode 100644 index 192dd5a..0000000 --- a/docker/addons/vault/config.hcl +++ /dev/null @@ -1,10 +0,0 @@ -storage "file" { - path = "/vault/file" -} - -listener "tcp" { - address = "0.0.0.0:8200" - tls_disable = 1 -} - -ui = true diff --git a/docker/addons/vault/docker-compose.yml b/docker/addons/vault/docker-compose.yml deleted file mode 100644 index 8f380b4..0000000 --- a/docker/addons/vault/docker-compose.yml +++ /dev/null @@ -1,39 +0,0 @@ -# Copyright (c) Abstract Machines -# SPDX-License-Identifier: Apache-2.0 - -# This docker-compose file contains optional Vault service for Magistrala platform. -# Since this is optional, this file is dependent of docker-compose file -# from /docker. In order to run these services, execute command: -# docker compose -f docker/docker-compose.yml -f docker/addons/vault/docker-compose.yml up -# from project root. Vault default port (8200) is exposed, so you can use Vault CLI tool for -# vault inspection and administration, as well as access the UI. - -networks: - magistrala-base-net: - -volumes: - magistrala-vault-volume: - -services: - vault: - image: hashicorp/vault:1.15.4 - container_name: magistrala-vault - ports: - - ${MG_VAULT_PORT}:8200 - networks: - - magistrala-base-net - volumes: - - magistrala-vault-volume:/vault/file - - magistrala-vault-volume:/vault/logs - - ./config.hcl:/vault/config/config.hcl - - ./entrypoint.sh:/entrypoint.sh - environment: - VAULT_ADDR: http://127.0.0.1:${MG_VAULT_PORT} - MG_VAULT_PORT: ${MG_VAULT_PORT} - MG_VAULT_UNSEAL_KEY_1: ${MG_VAULT_UNSEAL_KEY_1} - MG_VAULT_UNSEAL_KEY_2: ${MG_VAULT_UNSEAL_KEY_2} - MG_VAULT_UNSEAL_KEY_3: ${MG_VAULT_UNSEAL_KEY_3} - entrypoint: /bin/sh - command: /entrypoint.sh - cap_add: - - IPC_LOCK diff --git a/docker/addons/vault/entrypoint.sh b/docker/addons/vault/entrypoint.sh deleted file mode 100644 index efc6f5a..0000000 --- a/docker/addons/vault/entrypoint.sh +++ /dev/null @@ -1,25 +0,0 @@ -#!/usr/bin/dumb-init /bin/sh -# Copyright (c) Abstract Machines -# SPDX-License-Identifier: Apache-2.0 - -VAULT_CONFIG_DIR=/vault/config - -docker-entrypoint.sh server & -VAULT_PID=$! - -sleep 2 - -echo $MG_VAULT_UNSEAL_KEY_1 -echo $MG_VAULT_UNSEAL_KEY_2 -echo $MG_VAULT_UNSEAL_KEY_3 - -if [[ ! -z "${MG_VAULT_UNSEAL_KEY_1}" ]] && - [[ ! -z "${MG_VAULT_UNSEAL_KEY_2}" ]] && - [[ ! -z "${MG_VAULT_UNSEAL_KEY_3}" ]]; then - echo "Unsealing Vault" - vault operator unseal ${MG_VAULT_UNSEAL_KEY_1} - vault operator unseal ${MG_VAULT_UNSEAL_KEY_2} - vault operator unseal ${MG_VAULT_UNSEAL_KEY_3} -fi - -wait $VAULT_PID \ No newline at end of file diff --git a/docker/addons/vault/magistrala_things_certs_issue.template.hcl b/docker/addons/vault/magistrala_things_certs_issue.template.hcl deleted file mode 100644 index 1b13f6d..0000000 --- a/docker/addons/vault/magistrala_things_certs_issue.template.hcl +++ /dev/null @@ -1,32 +0,0 @@ - -# Allow issue certificate with role with default issuer from Intermediate PKI -path "${MG_VAULT_PKI_INT_PATH}/issue/${MG_VAULT_PKI_INT_THINGS_CERTS_ROLE_NAME}" { - capabilities = ["create", "update"] -} - -## Revole certificate from Intermediate PKI -path "${MG_VAULT_PKI_INT_PATH}/revoke" { - capabilities = ["create", "update"] -} - -## List Revoked Certificates from Intermediate PKI -path "${MG_VAULT_PKI_INT_PATH}/certs/revoked" { - capabilities = ["list"] -} - - -## List Certificates from Intermediate PKI -path "${MG_VAULT_PKI_INT_PATH}/certs" { - capabilities = ["list"] -} - -## Read Certificate from Intermediate PKI -path "${MG_VAULT_PKI_INT_PATH}/cert/+" { - capabilities = ["read"] -} -path "${MG_VAULT_PKI_INT_PATH}/cert/+/raw" { - capabilities = ["read"] -} -path "${MG_VAULT_PKI_INT_PATH}/cert/+/raw/pem" { - capabilities = ["read"] -} diff --git a/docker/addons/vault/vault_cmd.sh b/docker/addons/vault/vault_cmd.sh deleted file mode 100644 index 97a8cc9..0000000 --- a/docker/addons/vault/vault_cmd.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!/usr/bin/bash -# Copyright (c) Abstract Machines -# SPDX-License-Identifier: Apache-2.0 - -vault() { - if is_container_running "magistrala-vault"; then - docker exec -it magistrala-vault vault "$@" - else - if which vault &> /dev/null; then - $(which vault) "$@" - else - echo "magistrala-vault container or vault command not found. Please refer to the documentation: https://github.com/absmach/magistrala/blob/main/docker/addons/vault/README.md" - fi - fi -} - -is_container_running() { - local container_name="$1" - if [ "$(docker inspect --format '{{.State.Running}}' "$container_name" 2>/dev/null)" = "true" ]; then - return 0 - else - return 1 - fi -} diff --git a/docker/addons/vault/vault_copy_certs.sh b/docker/addons/vault/vault_copy_certs.sh deleted file mode 100755 index c4656df..0000000 --- a/docker/addons/vault/vault_copy_certs.sh +++ /dev/null @@ -1,53 +0,0 @@ -#!/usr/bin/bash -# Copyright (c) Abstract Machines -# SPDX-License-Identifier: Apache-2.0 - -set -euo pipefail - -scriptdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" -export MAGISTRALA_DIR=$scriptdir/../../../ - -cd $scriptdir - -readDotEnv() { - set -o allexport - source $MAGISTRALA_DIR/docker/.env - set +o allexport -} - -readDotEnv - -server_name="localhost" - -# Check if MG_NGINX_SERVER_NAME is set or not empty -if [ -n "${MG_NGINX_SERVER_NAME:-}" ]; then - server_name="$MG_NGINX_SERVER_NAME" -fi - -echo "Copying certificate files" - -if [ -e "data/${server_name}.crt" ]; then - cp -v data/${server_name}.crt ${MAGISTRALA_DIR}/docker/ssl/certs/magistrala-server.crt -else - echo "${server_name}.crt file not available" -fi - -if [ -e "data/${server_name}.key" ]; then - cp -v data/${server_name}.key ${MAGISTRALA_DIR}/docker/ssl/certs/magistrala-server.key -else - echo "${server_name}.key file not available" -fi - -if [ -e "data/${MG_VAULT_PKI_INT_FILE_NAME}.key" ]; then - cp -v data/${MG_VAULT_PKI_INT_FILE_NAME}.key ${MAGISTRALA_DIR}/docker/ssl/certs/ca.key -else - echo "data/${MG_VAULT_PKI_INT_FILE_NAME}.key file not available" -fi - -if [ -e "data/${MG_VAULT_PKI_INT_FILE_NAME}_bundle.crt" ]; then - cp -v data/${MG_VAULT_PKI_INT_FILE_NAME}_bundle.crt ${MAGISTRALA_DIR}/docker/ssl/certs/ca.crt -else - echo "data/${MG_VAULT_PKI_INT_FILE_NAME}_bundle.crt file not available" -fi - -exit 0 diff --git a/docker/addons/vault/vault_copy_env.sh b/docker/addons/vault/vault_copy_env.sh deleted file mode 100755 index dbb5fe4..0000000 --- a/docker/addons/vault/vault_copy_env.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!/usr/bin/bash -# Copyright (c) Abstract Machines -# SPDX-License-Identifier: Apache-2.0 - -set -euo pipefail - -scriptdir="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" -export MAGISTRALA_DIR=$scriptdir/../../../ - -cd $scriptdir - -write_env() { - if [ -e "data/secrets" ]; then - sed -i "s,MG_VAULT_UNSEAL_KEY_1=.*,MG_VAULT_UNSEAL_KEY_1=$(awk -F ": " '$1 == "Unseal Key 1" {print $2}' data/secrets)," $MAGISTRALA_DIR/docker/.env - sed -i "s,MG_VAULT_UNSEAL_KEY_2=.*,MG_VAULT_UNSEAL_KEY_2=$(awk -F ": " '$1 == "Unseal Key 2" {print $2}' data/secrets)," $MAGISTRALA_DIR/docker/.env - sed -i "s,MG_VAULT_UNSEAL_KEY_3=.*,MG_VAULT_UNSEAL_KEY_3=$(awk -F ": " '$1 == "Unseal Key 3" {print $2}' data/secrets)," $MAGISTRALA_DIR/docker/.env - sed -i "s,MG_VAULT_TOKEN=.*,MG_VAULT_TOKEN=$(awk -F ": " '$1 == "Initial Root Token" {print $2}' data/secrets)," $MAGISTRALA_DIR/docker/.env - echo "Vault environment varaibles are set successfully in docker/.env" - else - echo "Error: Source file 'data/secrets' not found." - fi -} - -write_env diff --git a/docker/addons/vault/vault_create_approle.sh b/docker/addons/vault/vault_create_approle.sh deleted file mode 100755 index 614f8dc..0000000 --- a/docker/addons/vault/vault_create_approle.sh +++ /dev/null @@ -1,97 +0,0 @@ -#!/usr/bin/bash -# Copyright (c) Abstract Machines -# SPDX-License-Identifier: Apache-2.0 - -set -euo pipefail - -scriptdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" -export MAGISTRALA_DIR=$scriptdir/../../../ - -cd $scriptdir - -SKIP_ENABLE_APP_ROLE=${1:-} - -readDotEnv() { - set -o allexport - source $MAGISTRALA_DIR/docker/.env - set +o allexport -} - -source vault_cmd.sh - -vaultCreatePolicyFile() { - envsubst ' - ${MG_VAULT_PKI_INT_PATH} - ${MG_VAULT_PKI_INT_THINGS_CERTS_ROLE_NAME} - ' < magistrala_things_certs_issue.template.hcl > magistrala_things_certs_issue.hcl -} -vaultCreatePolicy() { - echo "Creating new policy for AppRole" - if is_container_running "magistrala-vault"; then - docker cp magistrala_things_certs_issue.hcl magistrala-vault:/vault/magistrala_things_certs_issue.hcl - vault policy write -namespace=${MG_VAULT_NAMESPACE} -address=${MG_VAULT_ADDR} magistrala_things_certs_issue /vault/magistrala_things_certs_issue.hcl - else - vault policy write -namespace=${MG_VAULT_NAMESPACE} -address=${MG_VAULT_ADDR} magistrala_things_certs_issue magistrala_things_certs_issue.hcl - fi -} - -vaultEnableAppRole() { - if [ "$SKIP_ENABLE_APP_ROLE" == "--skip-enable-approle" ]; then - echo "Skipping Enable AppRole" - else - echo "Enabling AppRole" - vault auth enable -namespace=${MG_VAULT_NAMESPACE} -address=${MG_VAULT_ADDR} approle - fi -} - -vaultDeleteRole() { - echo "Deleteing old AppRole" - vault delete -namespace=${MG_VAULT_NAMESPACE} -address=${MG_VAULT_ADDR} auth/approle/role/magistrala_things_certs_issuer -} - -vaultCreateRole() { - echo "Creating new AppRole" - vault write -namespace=${MG_VAULT_NAMESPACE} -address=${MG_VAULT_ADDR} auth/approle/role/magistrala_things_certs_issuer \ - token_policies=magistrala_things_certs_issue secret_id_num_uses=0 \ - secret_id_ttl=0 token_ttl=1h token_max_ttl=3h token_num_uses=0 -} - -vaultWriteCustomRoleID(){ - echo "Writing custom role id" - vault read -namespace=${MG_VAULT_NAMESPACE} -address=${MG_VAULT_ADDR} auth/approle/role/magistrala_things_certs_issuer/role-id - vault write -namespace=${MG_VAULT_NAMESPACE} -address=${MG_VAULT_ADDR} auth/approle/role/magistrala_things_certs_issuer/role-id role_id=${MG_VAULT_THINGS_CERTS_ISSUER_ROLEID} -} - -vaultWriteCustomSecret() { - echo "Writing custom secret" - vault write -namespace=${MG_VAULT_NAMESPACE} -address=${MG_VAULT_ADDR} -f auth/approle/role/magistrala_things_certs_issuer/secret-id - vault write -namespace=${MG_VAULT_NAMESPACE} -address=${MG_VAULT_ADDR} auth/approle/role/magistrala_things_certs_issuer/custom-secret-id secret_id=${MG_VAULT_THINGS_CERTS_ISSUER_SECRET} num_uses=0 ttl=0 -} - -vaultTestRoleLogin() { - echo "Testing custom roleid secret by logging in" - vault write -namespace=${MG_VAULT_NAMESPACE} -address=${MG_VAULT_ADDR} auth/approle/login \ - role_id=${MG_VAULT_THINGS_CERTS_ISSUER_ROLEID} \ - secret_id=${MG_VAULT_THINGS_CERTS_ISSUER_SECRET} - -} -if ! command -v jq &> /dev/null -then - echo "jq command could not be found, please install it and try again." - exit -fi - -readDotEnv - -vault login -namespace=${MG_VAULT_NAMESPACE} -address=${MG_VAULT_ADDR} ${MG_VAULT_TOKEN} - -vaultCreatePolicyFile -vaultCreatePolicy -vaultEnableAppRole -vaultDeleteRole -vaultCreateRole -vaultWriteCustomRoleID -vaultWriteCustomSecret -vaultTestRoleLogin - -exit 0 diff --git a/docker/addons/vault/vault_init.sh b/docker/addons/vault/vault_init.sh deleted file mode 100755 index bd1e05f..0000000 --- a/docker/addons/vault/vault_init.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!/usr/bin/bash -# Copyright (c) Abstract Machines -# SPDX-License-Identifier: Apache-2.0 - -set -euo pipefail - -scriptdir="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" -export MAGISTRALA_DIR=$scriptdir/../../../ - -cd $scriptdir - -readDotEnv() { - set -o allexport - source $MAGISTRALA_DIR/docker/.env - set +o allexport -} - -source vault_cmd.sh - -readDotEnv - -mkdir -p data - -vault operator init -address=$MG_VAULT_ADDR 2>&1 | tee >(sed -r 's/\x1b\[[0-9;]*m//g' > data/secrets) diff --git a/docker/addons/vault/vault_set_pki.sh b/docker/addons/vault/vault_set_pki.sh deleted file mode 100755 index 6f8ebdc..0000000 --- a/docker/addons/vault/vault_set_pki.sh +++ /dev/null @@ -1,229 +0,0 @@ -#!/usr/bin/bash -# Copyright (c) Abstract Machines -# SPDX-License-Identifier: Apache-2.0 - -set -euo pipefail - -scriptdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" -export MAGISTRALA_DIR=$scriptdir/../../../ - -SKIP_SERVER_CERT=${1:-} - -cd $scriptdir - -readDotEnv() { - set -o allexport - source $MAGISTRALA_DIR/docker/.env - set +o allexport -} - -server_name="localhost" - -# Check if MG_NGINX_SERVER_NAME is set or not empty -if [ -n "${MG_NGINX_SERVER_NAME:-}" ]; then - server_name="$MG_NGINX_SERVER_NAME" -fi - -source vault_cmd.sh - -vaultEnablePKI() { - vault secrets enable -namespace=${MG_VAULT_NAMESPACE} -address=${MG_VAULT_ADDR} -path ${MG_VAULT_PKI_PATH} pki - vault secrets tune -namespace=${MG_VAULT_NAMESPACE} -address=${MG_VAULT_ADDR} -max-lease-ttl=87600h ${MG_VAULT_PKI_PATH} -} - -vaultConfigPKIClusterPath() { - vault write -namespace=${MG_VAULT_NAMESPACE} -address=${MG_VAULT_ADDR} ${MG_VAULT_PKI_PATH}/config/cluster aia_path=${MG_VAULT_PKI_CLUSTER_AIA_PATH} path=${MG_VAULT_PKI_CLUSTER_PATH} -} - -vaultConfigPKICrl() { - vault write -namespace=${MG_VAULT_NAMESPACE} -address=${MG_VAULT_ADDR} ${MG_VAULT_PKI_PATH}/config/crl expiry="5m" ocsp_disable=false ocsp_expiry=0 auto_rebuild=true auto_rebuild_grace_period="2m" enable_delta=true delta_rebuild_interval="1m" -} - -vaultAddRoleToSecret() { - vault write -namespace=${MG_VAULT_NAMESPACE} -address=${MG_VAULT_ADDR} ${MG_VAULT_PKI_PATH}/roles/${MG_VAULT_PKI_ROLE_NAME} \ - allow_any_name=true \ - max_ttl="8760h" \ - default_ttl="8760h" \ - generate_lease=true -} - -vaultGenerateRootCACertificate() { - echo "Generate root CA certificate" - vault write -namespace=${MG_VAULT_NAMESPACE} -address=${MG_VAULT_ADDR} -format=json ${MG_VAULT_PKI_PATH}/root/generate/exported \ - common_name="\"$MG_VAULT_PKI_CA_CN\"" \ - ou="\"$MG_VAULT_PKI_CA_OU\"" \ - organization="\"$MG_VAULT_PKI_CA_O\"" \ - country="\"$MG_VAULT_PKI_CA_C\"" \ - locality="\"$MG_VAULT_PKI_CA_L\"" \ - province="\"$MG_VAULT_PKI_CA_ST\"" \ - street_address="\"$MG_VAULT_PKI_CA_ADDR\"" \ - postal_code="\"$MG_VAULT_PKI_CA_PO\"" \ - ttl=87600h | tee >(jq -r .data.certificate >data/${MG_VAULT_PKI_FILE_NAME}_ca.crt) \ - >(jq -r .data.issuing_ca >data/${MG_VAULT_PKI_FILE_NAME}_issuing_ca.crt) \ - >(jq -r .data.private_key >data/${MG_VAULT_PKI_FILE_NAME}_ca.key) -} - -vaultSetupRootCAIssuingURLs() { - echo "Setup URLs for CRL and issuing" - vault write -namespace=${MG_VAULT_NAMESPACE} -address=${MG_VAULT_ADDR} ${MG_VAULT_PKI_PATH}/config/urls \ - issuing_certificates="{{cluster_aia_path}}/v1/${MG_VAULT_PKI_PATH}/ca" \ - crl_distribution_points="{{cluster_aia_path}}/v1/${MG_VAULT_PKI_PATH}/crl" \ - ocsp_servers="{{cluster_aia_path}}/v1/${MG_VAULT_PKI_PATH}/ocsp" \ - enable_templating=true -} - -vaultGenerateIntermediateCAPKI() { - echo "Generate Intermediate CA PKI" - vault secrets enable -namespace=${MG_VAULT_NAMESPACE} -address=${MG_VAULT_ADDR} -path=${MG_VAULT_PKI_INT_PATH} pki - vault secrets tune -namespace=${MG_VAULT_NAMESPACE} -address=${MG_VAULT_ADDR} -max-lease-ttl=43800h ${MG_VAULT_PKI_INT_PATH} -} - -vaultConfigIntermediatePKIClusterPath() { - vault write -namespace=${MG_VAULT_NAMESPACE} -address=${MG_VAULT_ADDR} ${MG_VAULT_PKI_INT_PATH}/config/cluster aia_path=${MG_VAULT_PKI_INT_CLUSTER_AIA_PATH} path=${MG_VAULT_PKI_INT_CLUSTER_PATH} -} - -vaultConfigIntermediatePKICrl() { - vault write -namespace=${MG_VAULT_NAMESPACE} -address=${MG_VAULT_ADDR} ${MG_VAULT_PKI_INT_PATH}/config/crl expiry="5m" ocsp_disable=false ocsp_expiry=0 auto_rebuild=true auto_rebuild_grace_period="2m" enable_delta=true delta_rebuild_interval="1m" -} - -vaultGenerateIntermediateCSR() { - echo "Generate intermediate CSR" - vault write -namespace=${MG_VAULT_NAMESPACE} -address=${MG_VAULT_ADDR} -format=json ${MG_VAULT_PKI_INT_PATH}/intermediate/generate/exported \ - common_name="\"$MG_VAULT_PKI_INT_CA_CN\"" \ - ou="\"$MG_VAULT_PKI_INT_CA_OU\""\ - organization="\"$MG_VAULT_PKI_INT_CA_O\"" \ - country="\"$MG_VAULT_PKI_INT_CA_C\"" \ - locality="\"$MG_VAULT_PKI_INT_CA_L\"" \ - province="\"$MG_VAULT_PKI_INT_CA_ST\"" \ - street_address="\"$MG_VAULT_PKI_INT_CA_ADDR\"" \ - postal_code="\"$MG_VAULT_PKI_INT_CA_PO\"" \ - | tee >(jq -r .data.csr >data/${MG_VAULT_PKI_INT_FILE_NAME}.csr) \ - >(jq -r .data.private_key >data/${MG_VAULT_PKI_INT_FILE_NAME}.key) -} - -vaultSignIntermediateCSR() { - echo "Sign intermediate CSR" - if is_container_running "magistrala-vault"; then - docker cp data/${MG_VAULT_PKI_INT_FILE_NAME}.csr magistrala-vault:/vault/${MG_VAULT_PKI_INT_FILE_NAME}.csr - vault write -namespace=${MG_VAULT_NAMESPACE} -address=${MG_VAULT_ADDR} -format=json ${MG_VAULT_PKI_PATH}/root/sign-intermediate \ - csr=@/vault/${MG_VAULT_PKI_INT_FILE_NAME}.csr ttl="8760h" \ - ou="\"$MG_VAULT_PKI_INT_CA_OU\""\ - organization="\"$MG_VAULT_PKI_INT_CA_O\"" \ - country="\"$MG_VAULT_PKI_INT_CA_C\"" \ - locality="\"$MG_VAULT_PKI_INT_CA_L\"" \ - province="\"$MG_VAULT_PKI_INT_CA_ST\"" \ - street_address="\"$MG_VAULT_PKI_INT_CA_ADDR\"" \ - postal_code="\"$MG_VAULT_PKI_INT_CA_PO\"" \ - | tee >(jq -r .data.certificate >data/${MG_VAULT_PKI_INT_FILE_NAME}.crt) \ - >(jq -r .data.issuing_ca >data/${MG_VAULT_PKI_INT_FILE_NAME}_issuing_ca.crt) - else - vault write -namespace=${MG_VAULT_NAMESPACE} -address=${MG_VAULT_ADDR} -format=json ${MG_VAULT_PKI_PATH}/root/sign-intermediate \ - csr=@data/${MG_VAULT_PKI_INT_FILE_NAME}.csr ttl="8760h" \ - ou="\"$MG_VAULT_PKI_INT_CA_OU\""\ - organization="\"$MG_VAULT_PKI_INT_CA_O\"" \ - country="\"$MG_VAULT_PKI_INT_CA_C\"" \ - locality="\"$MG_VAULT_PKI_INT_CA_L\"" \ - province="\"$MG_VAULT_PKI_INT_CA_ST\"" \ - street_address="\"$MG_VAULT_PKI_INT_CA_ADDR\"" \ - postal_code="\"$MG_VAULT_PKI_INT_CA_PO\"" \ - | tee >(jq -r .data.certificate >data/${MG_VAULT_PKI_INT_FILE_NAME}.crt) \ - >(jq -r .data.issuing_ca >data/${MG_VAULT_PKI_INT_FILE_NAME}_issuing_ca.crt) - fi - -} - -vaultInjectIntermediateCertificate() { - echo "Inject Intermediate Certificate" - if is_container_running "magistrala-vault"; then - docker cp data/${MG_VAULT_PKI_INT_FILE_NAME}.crt magistrala-vault:/vault/${MG_VAULT_PKI_INT_FILE_NAME}.crt - vault write -namespace=${MG_VAULT_NAMESPACE} -address=${MG_VAULT_ADDR} ${MG_VAULT_PKI_INT_PATH}/intermediate/set-signed certificate=@/vault/${MG_VAULT_PKI_INT_FILE_NAME}.crt - else - vault write -namespace=${MG_VAULT_NAMESPACE} -address=${MG_VAULT_ADDR} ${MG_VAULT_PKI_INT_PATH}/intermediate/set-signed certificate=@data/${MG_VAULT_PKI_INT_FILE_NAME}.crt - fi -} - -vaultGenerateIntermediateCertificateBundle() { - echo "Generate intermediate certificate bundle" - cat data/${MG_VAULT_PKI_INT_FILE_NAME}.crt data/${MG_VAULT_PKI_FILE_NAME}_ca.crt \ - > data/${MG_VAULT_PKI_INT_FILE_NAME}_bundle.crt -} - -vaultSetupIntermediateIssuingURLs() { - echo "Setup URLs for CRL and issuing" - vault write -namespace=${MG_VAULT_NAMESPACE} -address=${MG_VAULT_ADDR} ${MG_VAULT_PKI_INT_PATH}/config/urls \ - issuing_certificates="{{cluster_aia_path}}/v1/${MG_VAULT_PKI_INT_PATH}/ca" \ - crl_distribution_points="{{cluster_aia_path}}/v1/${MG_VAULT_PKI_INT_PATH}/crl" \ - ocsp_servers="{{cluster_aia_path}}/v1/${MG_VAULT_PKI_INT_PATH}/ocsp" \ - enable_templating=true -} - -vaultSetupServerCertsRole() { - if [ "$SKIP_SERVER_CERT" == "--skip-server-cert" ]; then - echo "Skipping server certificate role" - else - echo "Setup Server certificate role" - vault write -namespace=${MG_VAULT_NAMESPACE} -address=${MG_VAULT_ADDR} ${MG_VAULT_PKI_INT_PATH}/roles/${MG_VAULT_PKI_INT_SERVER_CERTS_ROLE_NAME} \ - allow_subdomains=true \ - max_ttl="4320h" - fi -} - -vaultGenerateServerCertificate() { - if [ "$SKIP_SERVER_CERT" == "--skip-server-cert" ]; then - echo "Skipping generate server certificate" - else - echo "Generate server certificate" - vault write -namespace=${MG_VAULT_NAMESPACE} -address=${MG_VAULT_ADDR} -format=json ${MG_VAULT_PKI_INT_PATH}/issue/${MG_VAULT_PKI_INT_SERVER_CERTS_ROLE_NAME} \ - common_name="$server_name" ttl="4320h" \ - | tee >(jq -r .data.certificate >data/${server_name}.crt) \ - >(jq -r .data.private_key >data/${server_name}.key) - fi - -} - -vaultSetupThingCertsRole() { - echo "Setup Thing Certs role" - vault write -namespace=${MG_VAULT_NAMESPACE} -address=${MG_VAULT_ADDR} ${MG_VAULT_PKI_INT_PATH}/roles/${MG_VAULT_PKI_INT_THINGS_CERTS_ROLE_NAME} \ - allow_subdomains=true \ - allow_any_name=true \ - max_ttl="2160h" -} - -vaultCleanupFiles() { - if is_container_running "magistrala-vault"; then - docker exec magistrala-vault sh -c 'rm -rf /vault/*.{crt,csr}' - fi -} - -if ! command -v jq &> /dev/null -then - echo "jq command could not be found, please install it and try again." - exit -fi - -readDotEnv - -mkdir -p data - -vault login -namespace=${MG_VAULT_NAMESPACE} -address=${MG_VAULT_ADDR} ${MG_VAULT_TOKEN} - -vaultEnablePKI -vaultConfigPKIClusterPath -vaultConfigPKICrl -vaultAddRoleToSecret -vaultGenerateRootCACertificate -vaultSetupRootCAIssuingURLs -vaultGenerateIntermediateCAPKI -vaultConfigIntermediatePKIClusterPath -vaultConfigIntermediatePKICrl -vaultGenerateIntermediateCSR -vaultSignIntermediateCSR -vaultInjectIntermediateCertificate -vaultGenerateIntermediateCertificateBundle -vaultSetupIntermediateIssuingURLs -vaultSetupServerCertsRole -vaultGenerateServerCertificate -vaultSetupThingCertsRole -vaultCleanupFiles - -exit 0 diff --git a/docker/addons/vault/vault_unseal.sh b/docker/addons/vault/vault_unseal.sh deleted file mode 100755 index b80b6ee..0000000 --- a/docker/addons/vault/vault_unseal.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!/usr/bin/bash -# Copyright (c) Abstract Machines -# SPDX-License-Identifier: Apache-2.0 - -set -euo pipefail - -scriptdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" -export MAGISTRALA_DIR=$scriptdir/../../../ - -cd $scriptdir - -readDotEnv() { - set -o allexport - source $MAGISTRALA_DIR/docker/.env - set +o allexport -} - -source vault_cmd.sh - -readDotEnv - -vault operator unseal -address=${MG_VAULT_ADDR} ${MG_VAULT_UNSEAL_KEY_1} -vault operator unseal -address=${MG_VAULT_ADDR} ${MG_VAULT_UNSEAL_KEY_2} -vault operator unseal -address=${MG_VAULT_ADDR} ${MG_VAULT_UNSEAL_KEY_3} diff --git a/docker/addons/cassandra-reader/docker-compose.yml b/docker/cassandra-reader/docker-compose.yml similarity index 100% rename from docker/addons/cassandra-reader/docker-compose.yml rename to docker/cassandra-reader/docker-compose.yml diff --git a/docker/addons/cassandra-writer/config.toml b/docker/cassandra-writer/config.toml similarity index 100% rename from docker/addons/cassandra-writer/config.toml rename to docker/cassandra-writer/config.toml diff --git a/docker/addons/cassandra-writer/docker-compose.yml b/docker/cassandra-writer/docker-compose.yml similarity index 100% rename from docker/addons/cassandra-writer/docker-compose.yml rename to docker/cassandra-writer/docker-compose.yml diff --git a/docker/addons/cassandra-writer/init.sh b/docker/cassandra-writer/init.sh similarity index 100% rename from docker/addons/cassandra-writer/init.sh rename to docker/cassandra-writer/init.sh diff --git a/docker/addons/influxdb-reader/docker-compose.yml b/docker/influxdb-reader/docker-compose.yml similarity index 100% rename from docker/addons/influxdb-reader/docker-compose.yml rename to docker/influxdb-reader/docker-compose.yml diff --git a/docker/addons/influxdb-writer/config.toml b/docker/influxdb-writer/config.toml similarity index 100% rename from docker/addons/influxdb-writer/config.toml rename to docker/influxdb-writer/config.toml diff --git a/docker/addons/influxdb-writer/docker-compose.yml b/docker/influxdb-writer/docker-compose.yml similarity index 100% rename from docker/addons/influxdb-writer/docker-compose.yml rename to docker/influxdb-writer/docker-compose.yml diff --git a/docker/addons/lora-adapter/docker-compose.yml b/docker/lora-adapter/docker-compose.yml similarity index 100% rename from docker/addons/lora-adapter/docker-compose.yml rename to docker/lora-adapter/docker-compose.yml diff --git a/docker/addons/mongodb-reader/docker-compose.yml b/docker/mongodb-reader/docker-compose.yml similarity index 100% rename from docker/addons/mongodb-reader/docker-compose.yml rename to docker/mongodb-reader/docker-compose.yml diff --git a/docker/addons/mongodb-writer/config.toml b/docker/mongodb-writer/config.toml similarity index 100% rename from docker/addons/mongodb-writer/config.toml rename to docker/mongodb-writer/config.toml diff --git a/docker/addons/mongodb-writer/docker-compose.yml b/docker/mongodb-writer/docker-compose.yml similarity index 100% rename from docker/addons/mongodb-writer/docker-compose.yml rename to docker/mongodb-writer/docker-compose.yml diff --git a/docker/addons/opcua-adapter/docker-compose.yml b/docker/opcua-adapter/docker-compose.yml similarity index 100% rename from docker/addons/opcua-adapter/docker-compose.yml rename to docker/opcua-adapter/docker-compose.yml diff --git a/docker/addons/prometheus/docker-compose.yml b/docker/prometheus/docker-compose.yml similarity index 100% rename from docker/addons/prometheus/docker-compose.yml rename to docker/prometheus/docker-compose.yml diff --git a/docker/addons/prometheus/grafana/dashboard.yml b/docker/prometheus/grafana/dashboard.yml similarity index 100% rename from docker/addons/prometheus/grafana/dashboard.yml rename to docker/prometheus/grafana/dashboard.yml diff --git a/docker/addons/prometheus/grafana/datasource.yml b/docker/prometheus/grafana/datasource.yml similarity index 100% rename from docker/addons/prometheus/grafana/datasource.yml rename to docker/prometheus/grafana/datasource.yml diff --git a/docker/addons/prometheus/grafana/example-dashboard.json b/docker/prometheus/grafana/example-dashboard.json similarity index 100% rename from docker/addons/prometheus/grafana/example-dashboard.json rename to docker/prometheus/grafana/example-dashboard.json diff --git a/docker/addons/prometheus/metrics/prometheus.yml b/docker/prometheus/metrics/prometheus.yml similarity index 100% rename from docker/addons/prometheus/metrics/prometheus.yml rename to docker/prometheus/metrics/prometheus.yml diff --git a/docker/addons/smpp-notifier/config.toml b/docker/smpp-notifier/config.toml similarity index 100% rename from docker/addons/smpp-notifier/config.toml rename to docker/smpp-notifier/config.toml diff --git a/docker/addons/smpp-notifier/docker-compose.yml b/docker/smpp-notifier/docker-compose.yml similarity index 100% rename from docker/addons/smpp-notifier/docker-compose.yml rename to docker/smpp-notifier/docker-compose.yml diff --git a/docker/addons/smtp-notifier/config.toml b/docker/smtp-notifier/config.toml similarity index 100% rename from docker/addons/smtp-notifier/config.toml rename to docker/smtp-notifier/config.toml diff --git a/docker/addons/smtp-notifier/docker-compose.yml b/docker/smtp-notifier/docker-compose.yml similarity index 100% rename from docker/addons/smtp-notifier/docker-compose.yml rename to docker/smtp-notifier/docker-compose.yml diff --git a/docker/addons/twins/docker-compose.yml b/docker/twins/docker-compose.yml similarity index 100% rename from docker/addons/twins/docker-compose.yml rename to docker/twins/docker-compose.yml