fix: add DB keepalives

IATI · Jan 7, 2025 · d73f41a · d73f41a
1 parent f438ddf
commit d73f41a
Show file tree

Hide file tree

Showing 7 changed files with 87 additions and 6 deletions.
diff --git a/.env b/.env
diff --git a/.env-example b/.env-example
@@ -0,0 +1,31 @@
+# Used for local development, when running in docker or on command line
+
+DB_USER=
+DB_PASS=
+DB_HOST=
+DB_NAME=
+DB_PORT=
+DB_SSL_MODE=
+DB_KEEPALIVE_IDLE=60
+DB_KEEPALIVE_INTERVAL=15
+DB_KEEPALIVE_COUNT=5
+
+AZURE_STORAGE_CONNECTION_STRING=
+AZURE_STORAGE_CONTAINER_SOURCE=source
+AZURE_STORAGE_CONTAINER_CLEAN=clean
+ACTIVITIES_LAKE_CONTAINER_NAME=lake
+
+SCHEMA_VALIDATION_API_URL=
+SCHEMA_VALIDATION_KEY_NAME=x-functions-key
+SCHEMA_VALIDATION_KEY_VALUE=
+
+VALIDATOR_API_URL=
+VALIDATOR_API_KEY_NAME=x-functions-key
+VALIDATOR_API_KEY_VALUE=
+
+SOLR_API_URL=
+SOLR_USER=
+SOLR_PASSWORD=
+LOG_LEVEL=
+SOLR_PARALLEL_PROCESSES=
+SOLR_500_SLEEP=
diff --git a/.github/workflows/develop.yml b/.github/workflows/develop.yml
@@ -34,6 +34,9 @@ env:
   DB_HOST: ${{ secrets.DB_HOST }}
   DB_NAME: ${{ secrets.DB_NAME }}
   DB_PORT: ${{ secrets.DB_PORT }}
+  DB_KEEPALIVE_IDLE: ${{ vars.DEV_DB_KEEPALIVE_IDLE }}
+  DB_KEEPALIVE_INTERVAL: ${{ vars.DEV_DB_KEEPALIVE_INTERVAL }}
+  DB_KEEPALIVE_COUNT: ${{ vars.DEV_DB_KEEPALIVE_COUNT }}
   LOG_WORKSPACE_ID: ${{ secrets.DEV_LOG_WORKSPACE_ID }}
   LOG_WORKSPACE_KEY: ${{ secrets.DEV_LOG_WORKSPACE_KEY }}
   COMMSHUB_URL: ${{ secrets.DEV_COMMSHUB_URL }}
@@ -115,6 +118,9 @@ jobs:
           sed -i ''s^#DB_USER#^$DB_USER^g'' ./deployment/deployment.yml
           sed -i ''s^#DB_PASS#^$DB_PASS^g'' ./deployment/deployment.yml
           sed -i ''s^#DB_NAME#^$DB_NAME^g'' ./deployment/deployment.yml
+          sed -i ''s^#DB_KEEPALIVE_IDLE#^$DB_KEEPALIVE_IDLE^g'' ./deployment/deployment.yml
+          sed -i ''s^#DB_KEEPALIVE_INTERVAL#^$DB_KEEPALIVE_INTERVAL^g'' ./deployment/deployment.yml
+          sed -i ''s^#DB_KEEPALIVE_COUNT#^$DB_KEEPALIVE_COUNT^g'' ./deployment/deployment.yml
 
           sed -i ''s^#SOLR_API_URL#^$SOLR_API_URL^g'' ./deployment/deployment.yml
           sed -i ''s^#SOLR_USER#^$SOLR_USER^g'' ./deployment/deployment.yml

diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
@@ -31,6 +31,9 @@ env:
   DB_HOST: ${{ secrets.PROD_DB_HOST }}
   DB_NAME: ${{ secrets.PROD_DB_NAME }}
   DB_PORT: ${{ secrets.PROD_DB_PORT }}
+  DB_KEEPALIVE_IDLE: ${{ vars.PROD_DB_KEEPALIVE_IDLE }}
+  DB_KEEPALIVE_INTERVAL: ${{ vars.PROD_DB_KEEPALIVE_INTERVAL }}
+  DB_KEEPALIVE_COUNT: ${{ vars.PROD_DB_KEEPALIVE_COUNT }}
   LOG_WORKSPACE_ID: ${{ secrets.PROD_LOG_WORKSPACE_ID }}
   LOG_WORKSPACE_KEY: ${{ secrets.PROD_LOG_WORKSPACE_KEY }}
   COMMSHUB_URL: ${{ secrets.PROD_COMMSHUB_URL }}
@@ -111,6 +114,9 @@ jobs:
           sed -i ''s^#DB_USER#^$DB_USER^g'' ./deployment/deployment.yml
           sed -i ''s^#DB_PASS#^$DB_PASS^g'' ./deployment/deployment.yml
           sed -i ''s^#DB_NAME#^$DB_NAME^g'' ./deployment/deployment.yml
+          sed -i ''s^#DB_KEEPALIVE_IDLE#^$DB_KEEPALIVE_IDLE^g'' ./deployment/deployment.yml
+          sed -i ''s^#DB_KEEPALIVE_INTERVAL#^$DB_KEEPALIVE_INTERVAL^g'' ./deployment/deployment.yml
+          sed -i ''s^#DB_KEEPALIVE_COUNT#^$DB_KEEPALIVE_COUNT^g'' ./deployment/deployment.yml
 
           sed -i ''s^#SOLR_API_URL#^$SOLR_API_URL^g'' ./deployment/deployment.yml
           sed -i ''s^#SOLR_USER#^$SOLR_USER^g'' ./deployment/deployment.yml

diff --git a/deployment/deployment.yml b/deployment/deployment.yml
@@ -45,6 +45,12 @@ properties: # Properties of container group
             secureValue: '#DB_PASS#'
           - name: DB_NAME
             secureValue: '#DB_NAME#'
+          - name: DB_KEEPALIVE_IDLE
+            value: '#DB_KEEPALIVE_IDLE#'
+          - name: DB_KEEPALIVE_INTERVAL
+            value: '#DB_KEEPALIVE_INTERVAL#'
+          - name: DB_KEEPALIVE_COUNT
+            value: '#DB_KEEPALIVE_COUNT#'
           - name: SOLR_API_URL
             secureValue: '#SOLR_API_URL#'
           - name: SOLR_USER
@@ -84,6 +90,12 @@ properties: # Properties of container group
             secureValue: '#DB_PASS#'
           - name: DB_NAME
             secureValue: '#DB_NAME#'
+          - name: DB_KEEPALIVE_IDLE
+            value: '#DB_KEEPALIVE_IDLE#'
+          - name: DB_KEEPALIVE_INTERVAL
+            value: '#DB_KEEPALIVE_INTERVAL#'
+          - name: DB_KEEPALIVE_COUNT
+            value: '#DB_KEEPALIVE_COUNT#'
           - name: SOLR_API_URL
             secureValue: '#SOLR_API_URL#'
           - name: SOLR_USER
@@ -137,6 +149,12 @@ properties: # Properties of container group
             secureValue: '#DB_PASS#'
           - name: DB_NAME
             secureValue: '#DB_NAME#'
+          - name: DB_KEEPALIVE_IDLE
+            value: '#DB_KEEPALIVE_IDLE#'
+          - name: DB_KEEPALIVE_INTERVAL
+            value: '#DB_KEEPALIVE_INTERVAL#'
+          - name: DB_KEEPALIVE_COUNT
+            value: '#DB_KEEPALIVE_COUNT#'
           - name: SOLR_API_URL
             secureValue: '#SOLR_API_URL#'
           - name: SOLR_USER
@@ -190,6 +208,12 @@ properties: # Properties of container group
             secureValue: '#DB_PASS#'
           - name: DB_NAME
             secureValue: '#DB_NAME#'
+          - name: DB_KEEPALIVE_IDLE
+            value: '#DB_KEEPALIVE_IDLE#'
+          - name: DB_KEEPALIVE_INTERVAL
+            value: '#DB_KEEPALIVE_INTERVAL#'
+          - name: DB_KEEPALIVE_COUNT
+            value: '#DB_KEEPALIVE_COUNT#'
           - name: SOLR_API_URL
             secureValue: '#SOLR_API_URL#'
           - name: SOLR_USER
@@ -239,6 +263,12 @@ properties: # Properties of container group
             secureValue: '#DB_PASS#'
           - name: DB_NAME
             secureValue: '#DB_NAME#'
+          - name: DB_KEEPALIVE_IDLE
+            value: '#DB_KEEPALIVE_IDLE#'
+          - name: DB_KEEPALIVE_INTERVAL
+            value: '#DB_KEEPALIVE_INTERVAL#'
+          - name: DB_KEEPALIVE_COUNT
+            value: '#DB_KEEPALIVE_COUNT#'
           - name: SOLR_API_URL
             secureValue: '#SOLR_API_URL#'
           - name: COMMSHUB_URL
@@ -280,6 +310,12 @@ properties: # Properties of container group
             secureValue: '#DB_PASS#'
           - name: DB_NAME
             secureValue: '#DB_NAME#'
+          - name: DB_KEEPALIVE_IDLE
+            value: '#DB_KEEPALIVE_IDLE#'
+          - name: DB_KEEPALIVE_INTERVAL
+            value: '#DB_KEEPALIVE_INTERVAL#'
+          - name: DB_KEEPALIVE_COUNT
+            value: '#DB_KEEPALIVE_COUNT#'
           - name: SOLR_API_URL
             secureValue: '#SOLR_API_URL#'
           - name: SOLR_USER

diff --git a/src/constants/config.py b/src/constants/config.py
@@ -12,11 +12,14 @@
     DB_PORT=os.getenv("DB_PORT"),
     DB_NAME=os.getenv("DB_NAME"),
     DB_SSL_MODE=os.getenv("DB_SSL_MODE") or "require",
-    # Database retry/timeout constants
+    # Database retry/timeout/keepalive
     DB_CONN_RETRY_LIMIT=8,
     DB_CONN_SLEEP_START=5,
     DB_CONN_SLEEP_MAX=60,
     DB_CONN_TIMEOUT=5,
+    DB_KEEPALIVE_IDLE=int(os.getenv("DB_KEEPALIVE_IDLE", default=60)),
+    DB_KEEPALIVE_INTERVAL=int(os.getenv("DB_KEEPALIVE_INTERVAL", default=15)),
+    DB_KEEPALIVE_COUNT=int(os.getenv("DB_KEEPALIVE_COUNT", default=5)),
     # Azure Storage Account Connection String
     # This can be found in the Azure Portal > Storage Account > Access Keys
     STORAGE_CONNECTION_STR=os.getenv("AZURE_STORAGE_CONNECTION_STRING"),

diff --git a/src/library/db.py b/src/library/db.py
@@ -22,6 +22,10 @@ def getDirectConnection(retry_counter=0):
             port=config["DB_PORT"],
             sslmode=config["DB_SSL_MODE"],
             connect_timeout=config["DB_CONN_TIMEOUT"],
+            keepalives=1,
+            keepalives_idle=config["DB_KEEPALIVE_IDLE"],
+            keepalives_interval=config["DB_KEEPALIVE_INTERVAL"],
+            keepalives_count=config["DB_KEEPALIVE_COUNT"],
         )
         retry_counter = 0
         return connection