fix: move secrets from env to files

Makefile

@@ -7,7 +7,8 @@ BUILD_DIR=build
 SECRETS_DIR=${BUILD_DIR}/secrets
 DB_PASSWORD_PATH=${SECRETS_DIR}/db_password
 PGPASS_PATH=${SECRETS_DIR}/.pgpass
-SECRET_KEY_PATH=${SECRETS_DIR}/secret_key
+SECRET_KEY_PATH=${SECRETS_DIR}/django_secret_key
+EXT_SECRETS=mailgun_api_key hcaptcha_secret github_client_secret orcid_client_secret discourse_api_key discourse_sso_secret mail_api_key
 GENERATED_SECRETS=$(DB_PASSWORD_PATH) $(PGPASS_PATH) $(SECRET_KEY_PATH)
 
 ENVREPLACE := deploy/scripts/envreplace
@@ -73,8 +74,8 @@ release-version: .env
 	if [ ! -f .env ]; then \
 		cp $(ENV_TEMPLATE) .env; \
 	fi; \
-	$(ENVREPLACE) DB_PASSWORD $$(cat $(DB_PASSWORD_PATH)) .env; \
-	$(ENVREPLACE) SECRET_KEY $$(cat $(SECRET_KEY_PATH)) .env; \
+	# $(ENVREPLACE) DB_PASSWORD $$(cat $(DB_PASSWORD_PATH)) .env; \
+	# $(ENVREPLACE) SECRET_KEY $$(cat $(SECRET_KEY_PATH)) .env; \
 	$(ENVREPLACE) TEST_BASIC_AUTH_PASSWORD $$(openssl rand -base64 42) .env
 
 .PHONY: docker-compose.yml
@@ -91,9 +92,12 @@ set-db-password: $(DB_PASSWORD_PATH) .env
 
 .PHONY: secrets
 secrets: $(SECRETS_DIR) $(GENERATED_SECRETS)
+	for secret_path in $(EXT_SECRETS); do \
+		touch ${SECRETS_DIR}/$$secret_path; \
+	done
 
 .PHONY: deploy
-deploy: build
+deploy: build .env
 	docker compose pull db redis elasticsearch
 ifneq ($(DEPLOY_ENVIRONMENT),dev)
 	docker compose pull nginx
@@ -125,6 +129,10 @@ clean:
 	@echo "Backing up generated files to /tmp directory"
 	mv .env config.mk docker-compose.yml $(shell mktemp -d)
 
+.PHONY: clean_deploy
+clean_deploy: clean
+	+@$(MAKE) deploy
+
 .PHONY: test
 test: build
 	docker compose run --rm server /code/deploy/test.sh

base.yml

@@ -33,7 +33,9 @@ services:
       - ./deploy/elasticsearch.conf.d/log4j2.properties:/usr/share/elasticsearch/config/log4j2.properties
       - esdata:/usr/share/elasticsearch/data
   db:
-    image: postgis/postgis:15-3.3
+    image: postgis/postgis:15-3.4
+    secrets:
+      - db_password
     volumes:
       - ./docker/pgdata:/var/lib/postgresql/data
       - ./build/secrets/db_password:/run/secrets/db_password
@@ -56,8 +58,16 @@ services:
   server:
     build: django
     image: comses/server
+    secrets:
+      - db_password
+      - discourse_api_key
+      - discourse_sso_secret
+      - django_secret_key
+      - github_client_secret
+      - orcid_client_secret
+      - hcaptcha_secret
+      - mail_api_key
     volumes:
-      - ./build/secrets:/run/secrets
       - ./deploy/elasticsearch.conf.d:/etc/elasticsearch
       - ./docker/shared:/shared
     depends_on:
@@ -71,6 +81,25 @@ services:
         condition: service_started
     env_file:
       - .env
+
+secrets:
+  db_password:
+    file: ./build/secrets/db_password
+  discourse_api_key:
+    file: ./build/secrets/discourse_api_key
+  discourse_sso_secret:
+    file: ./build/secrets/discourse_sso_secret
+  django_secret_key:
+    file: ./build/secrets/django_secret_key
+  github_client_secret:
+    file: ./build/secrets/github_client_secret
+  hcaptcha_secret:
+    file: ./build/secrets/hcaptcha_secret
+  mail_api_key:
+    file: ./build/secrets/mail_api_key
+  orcid_client_secret:
+    file: ./build/secrets/orcid_client_secret
+
 volumes:
   esdata:
     driver: local

deploy/conf/.env.template

@@ -12,37 +12,35 @@ DB_PASSWORD=
 CLEAN_DATABASE="false"  # allowed values: "true" or "false"
 
 # elastic search
-ES_VERSION=7.17.10
+ES_VERSION=7.17.18
 
 # captcha
-RECAPTCHA_PUBLIC_KEY=
-RECAPTCHA_PRIVATE_KEY=
 HCAPTCHA_SITEKEY=
-HCAPTCHA_SECRET=
+# HCAPTCHA_SECRET=
 
 # discourse
 DISCOURSE_BASE_URL=
 DISCOURSE_API_USERNAME=
 
 # email
 EMAIL_SUBJECT_PREFIX=[CoMSES Net]
-MAILGUN_API_KEY=
+# MAILGUN_API_KEY=
 MAILGUN_SENDER_DOMAIN=
 
 # logging
 LOG_DIRECTORY=/shared/logs
 SENTRY_DSN=
 
 # secrets
-DISCOURSE_SSO_SECRET=
-DISCOURSE_API_KEY=
+# DISCOURSE_SSO_SECRET=
+# DISCOURSE_API_KEY=
+# django secret key
+# SECRET_KEY=
 
-SECRET_KEY=
-
-ORCID_CLIENT_ID=
-ORCID_CLIENT_SECRET=
 GITHUB_CLIENT_ID=
-GITHUB_CLIENT_SECRET=
+# GITHUB_CLIENT_SECRET=
+ORCID_CLIENT_ID=
+# ORCID_CLIENT_SECRET=
 
 # storage
 DATA_ROOT=/shared

django/core/settings/defaults.py

@@ -11,9 +11,18 @@
 
 import os
 from enum import Enum
+from pathlib import Path
 
 from django.contrib.messages import constants as messages
 
+def read_secret(file, fallback=''):
+    secrets_file_path = Path('/run/secrets', file)
+    if secrets_file_path.is_file():
+        return secrets_file_path.read_text().strip()
+    else:
+        return fallback
+
+
 
 class Environment(Enum):
     DEVELOPMENT = "http://localhost:8000"
@@ -251,7 +260,7 @@ def is_test(self):
 # sentry DSN
 SENTRY_DSN = os.getenv("SENTRY_DSN", "https://sentry.example.com/2")
 
-SECRET_KEY = os.getenv("SECRET_KEY")
+SECRET_KEY = read_secret('django_secret_key', os.getenv("SECRET_KEY"))
 
 # regular settings
 
@@ -269,7 +278,7 @@ def is_test(self):
         "ENGINE": "django.db.backends.postgresql",
         "NAME": os.getenv("DB_NAME"),
         "USER": os.getenv("DB_USER"),
-        "PASSWORD": os.getenv("DB_PASSWORD"),
+        "PASSWORD": read_secret('db_password', os.getenv("DB_PASSWORD")),
         "HOST": os.getenv("DB_HOST"),
         "PORT": os.getenv("DB_PORT"),
     }
@@ -474,10 +483,11 @@ def is_test(self):
 ACCOUNT_CHANGE_EMAIL = True
 
 ORCID_CLIENT_ID = os.getenv("ORCID_CLIENT_ID", "")
-ORCID_CLIENT_SECRET = os.getenv("ORCID_CLIENT_SECRET", "")
+
+ORCID_CLIENT_SECRET = read_secret('orcid_client_secret')
 
 GITHUB_CLIENT_ID = os.getenv("GITHUB_CLIENT_ID", "")
-GITHUB_CLIENT_SECRET = os.getenv("GITHUB_CLIENT_SECRET", "")
+GITHUB_CLIENT_SECRET = read_secret('github_client_secret')
 
 TEST_BASIC_AUTH_PASSWORD = os.getenv("TEST_BASIC_AUTH_PASSWORD", "test password")
 TEST_USER_ID = os.getenv("TEST_USER_ID", 1000000)
@@ -503,8 +513,8 @@ def is_test(self):
 DISCOURSE_BASE_URL = os.getenv(
     "DISCOURSE_BASE_URL", "https://staging-discourse.comses.net"
 )
-DISCOURSE_SSO_SECRET = os.getenv("DISCOURSE_SSO_SECRET", "unconfigured")
-DISCOURSE_API_KEY = os.getenv("DISCOURSE_API_KEY", "unconfigured")
+DISCOURSE_SSO_SECRET = read_secret('discourse_sso_secret', "unconfigured")
+DISCOURSE_API_KEY = read_secret('discourse_api_key', "unconfigured")
 DISCOURSE_API_USERNAME = os.getenv("DISCOURSE_API_USERNAME", "unconfigured")
 
 # https://docs.djangoproject.com/en/4.2/ref/settings/#templates

django/core/settings/staging.py

@@ -21,7 +21,7 @@
 # EMAIL_FILE_PATH = '/shared/logs/mail.log'
 EMAIL_BACKEND = "anymail.backends.mailgun.EmailBackend"
 
-MAILGUN_API_KEY = os.getenv("MAILGUN_API_KEY")
+MAILGUN_API_KEY = read_secret('mail_api_key')
 MAILGUN_SENDER_DOMAIN = os.getenv("MAILGUN_SENDER_DOMAIN", "mg.comses.net")
 EMAIL_SUBJECT_PREFIX = os.getenv("EMAIL_SUBJECT_PREFIX", "[staging.comses.net]")
 EMAIL_USE_TLS = True
@@ -104,7 +104,7 @@
 
 # hcaptcha config
 HCAPTCHA_SITEKEY = os.getenv("HCAPTCHA_SITEKEY", "")
-HCAPTCHA_SECRET = os.getenv("HCAPTCHA_SECRET", "")
+HCAPTCHA_SECRET = read_secret('hcaptcha_secret', 'unconfigured')
 
 WSGI_APPLICATION = "core.wsgi.application"