Feature/2729 remote migrations (#2779)

* add apply remote migrations script

* add missing required env

* rm migrate from gunicorn start

* add required lib

* add migrate command to pipeline

* fix deps

* rm message

* tom's workaround

* re-lock deps

* remove commented migration lines

* rm graphviz

* catch exit status

* add failing test migration

* delete test migration

* testing migration from local

* rm test migration

---------

Co-authored-by: Alexandra Pennington <[email protected]>
Co-authored-by: raftmsohani <[email protected]>

.circleci/deployment/commands.yml

@@ -97,6 +97,10 @@
             sudo apt update
             sudo add-apt-repository ppa:rmescandon/yq
             sudo apt-get install yq
+      - run:
+          name: Apply database migrations
+          command: |
+            bash ./scripts/apply-remote-migrations.sh <<parameters.backend-appname>>
       - run:
           name: Deploy backend application
           command: |

.gitattributes

@@ -44,4 +44,4 @@
 # Annoying product-updates zip files
 product-updates/knowledge-center/FTANF_2009.zip binary eol=lf
 product-updates/knowledge-center/SSPMOE_2009.zip binary eol=lf
-product-updates/knowledge-center/ftanf.zip binary eol=lf
+product-updates/knowledge-center/ftanf.zip binary eol=lf

.gitignore

@@ -21,6 +21,7 @@ tdrs-backend/coverage.xml
 tdrs-backend/htmlcov/*
 tdrs-backend/.env
 tdrs-backend/.env.production
+tdrs-backend/.env.ci
 tdrs-backend/ADS*
 tdrs-backend/temp_key_file
 tdrs-backend/test

scripts/apply-remote-migrations.sh

@@ -0,0 +1,68 @@
+#!/bin/bash
+
+app=${1}
+
+cd ./tdrs-backend
+
+echo "Install dependencies..."
+sudo apt install -y gcc
+sudo apt install -y libpq-dev python3-dev
+
+python -m venv ./env
+source ./env/bin/activate
+pip install --upgrade pip pipenv
+pipenv install --dev --system --deploy
+echo "Done."
+
+echo "Getting credentials..."
+guid=$(cf app --guid $app)
+app_vars=$(cf curl /v2/apps/$guid/env)
+
+db_creds=$(echo $app_vars | jq -r '.system_env_json.VCAP_SERVICES."aws-rds"[0].credentials')
+connection_str=$(echo $db_creds | jq -r '[.host, .port]' | jq -r 'join(":")')
+echo "Done."
+
+echo "Starting tunnel..."
+cf ssh -N -L 5432:$connection_str $app &
+sleep 5
+echo "Done."
+
+echo "Setting up environment..."
+cp ./.env.example ./.env.ci
+
+vcap_services=$(echo $app_vars | jq -r '.system_env_json.VCAP_SERVICES')
+vcap_application=$(echo $app_vars | jq -rc '.application_env_json.VCAP_APPLICATION')
+
+# replace host env var
+fixed_vcap_services=$(echo $vcap_services | jq -rc '."aws-rds"[0].credentials.host="localhost"')
+
+echo "VCAP_SERVICES='$fixed_vcap_services'" >> .env.ci
+echo "VCAP_APPLICATION='$vcap_application'" >> .env.ci
+
+set -a
+source .env.ci
+export DJANGO_CONFIGURATION=Development
+export DJANGO_SETTINGS_MODULE=tdpservice.settings.cloudgov
+set +a
+echo "Done."
+
+echo "Applying migrations..."
+python manage.py migrate
+status=$?
+echo "Done."
+
+echo "Cleaning up..."
+deactivate
+kill $!
+rm ./.env.ci
+cd ..
+echo "Done."
+
+if [ $status -eq 0 ]
+then
+    echo "Migrations applied successfully."
+    exit 0
+else
+    echo "Migrations failed."
+    exit $status
+fi

tdrs-backend/.env.example

@@ -86,3 +86,6 @@ ELASTIC_HOST=elastic:9200
 
 # testing
 CYPRESS_TOKEN=local-cypress-token
+
+# sftp
+ACFTITAN_SFTP_PYTEST=local-acftitan-key

tdrs-backend/Dockerfile

@@ -17,6 +17,7 @@ RUN apt-get -y upgrade
 # Install a new package:
 RUN apt-get install -y gcc && apt-get install -y graphviz && apt-get install -y graphviz-dev
 RUN apt-get install postgresql-client -y
+RUN apt-get install -y libpq-dev python3-dev
 # Install pipenv
 RUN pip install --upgrade pip pipenv
 RUN pipenv install --dev --system --deploy

tdrs-backend/Pipfile

@@ -42,7 +42,7 @@ gunicorn = "==20.1.0"
 ipdb = "==0.13.9"
 jwcrypto = "==1.3.1"
 markdown = "==3.3.7"
-psycopg2-binary = "==2.9.3"
+psycopg2 = "==2.9.9"
 pyjwt = "==2.4.0"
 requests = "==2.27.1"
 wait-for-it = "==2.2.0"

tdrs-backend/gunicorn_start.sh

@@ -1,5 +1,4 @@
 #!/usr/bin/env bash
-# Apply database migrations
 set -e
 
 echo "REDIS_SERVER"
@@ -12,11 +11,6 @@ else
     ( cd  /home/vcap/deps/0/bin/; ./redis-server /home/vcap/app/redis.conf &)
 fi
 
-#
-echo "Applying database migrations"
-python manage.py migrate
-#python manage.py populate_stts
-
 # Collect static files. This is needed for swagger to work in local environment
 if [[ $DISABLE_COLLECTSTATIC ]]; then 
     echo "DISABLE_COLLECTSTATIC is set to true, skipping collectstatic"
@@ -25,7 +19,6 @@ else
     python manage.py collectstatic --noinput
 fi
 
-#python manage.py collectstatic --noinput
 
 celery -A tdpservice.settings worker -c 1 &
 sleep 5