diff --git a/scripts/deploy-backend.sh b/scripts/deploy-backend.sh index 24bef90d9..ebbce8243 100755 --- a/scripts/deploy-backend.sh +++ b/scripts/deploy-backend.sh @@ -1,7 +1,7 @@ #!/bin/bash ############################## -# Global Variable Decls +# Global Variable Decls ############################## # The deployment strategy you wish to employ ( rolling update or setting up a new environment) @@ -77,7 +77,7 @@ set_cf_envs() else cf_cmd="cf set-env $CGAPPNAME_BACKEND $var_name ${!var_name}" fi - + echo "Setting var : $var_name" $cf_cmd done @@ -85,7 +85,7 @@ set_cf_envs() } # Helper method to generate JWT cert and keys for new environment -generate_jwt_cert() +generate_jwt_cert() { echo "regenerating JWT cert/key" yes 'XX' | openssl req -x509 -newkey rsa:4096 -keyout key.pem -out cert.pem -days 365 -nodes -sha256 @@ -94,7 +94,7 @@ generate_jwt_cert() } update_kibana() -{ +{ # Add network policy allowing Kibana to talk to the proxy and to allow the backend to talk to Kibana cf add-network-policy "$CGAPPNAME_BACKEND" "$CGAPPNAME_KIBANA" --protocol tcp --port 5601 cf add-network-policy "$CGAPPNAME_FRONTEND" "$CGAPPNAME_KIBANA" --protocol tcp --port 5601 @@ -105,12 +105,16 @@ update_backend() { cd tdrs-backend || exit cf unset-env "$CGAPPNAME_BACKEND" "AV_SCAN_URL" - + if [ "$CF_SPACE" = "tanf-prod" ]; then cf set-env "$CGAPPNAME_BACKEND" AV_SCAN_URL "http://tanf-prod-clamav-rest.apps.internal:9000/scan" else # Add environment varilables for clamav cf set-env "$CGAPPNAME_BACKEND" AV_SCAN_URL "http://tdp-clamav-nginx-$env.apps.internal:9000/scan" + + # Add variable for dev/staging apps to know their DB name. Prod uses default AWS name. + cf unset-env "$CGAPPNAME_BACKEND" "APP_DB_NAME" + cf set-env "$CGAPPNAME_BACKEND" "APP_DB_NAME" "tdp_db_$backend_app_name" fi if [ "$1" = "rolling" ] ; then @@ -129,12 +133,12 @@ update_backend() fi set_cf_envs - + cf map-route "$CGAPPNAME_BACKEND" apps.internal --hostname "$CGAPPNAME_BACKEND" # Add network policy to allow frontend to access backend cf add-network-policy "$CGAPPNAME_FRONTEND" "$CGAPPNAME_BACKEND" --protocol tcp --port 8080 - + if [ "$CF_SPACE" = "tanf-prod" ]; then # Add network policy to allow backend to access tanf-prod services cf add-network-policy "$CGAPPNAME_BACKEND" clamav-rest --protocol tcp --port 9000 @@ -149,7 +153,7 @@ bind_backend_to_services() { echo "Binding services to app: $CGAPPNAME_BACKEND" if [ "$CGAPPNAME_BACKEND" = "tdp-backend-develop" ]; then - # TODO: this is technical debt, we should either make staging mimic tanf-dev + # TODO: this is technical debt, we should either make staging mimic tanf-dev # or make unique services for all apps but we have a services limit # Introducing technical debt for release 3.0.0 specifically. env="develop" @@ -158,10 +162,10 @@ bind_backend_to_services() { cf bind-service "$CGAPPNAME_BACKEND" "tdp-staticfiles-${env}" cf bind-service "$CGAPPNAME_BACKEND" "tdp-datafiles-${env}" cf bind-service "$CGAPPNAME_BACKEND" "tdp-db-${env}" - + # Setting up the ElasticSearch service cf bind-service "$CGAPPNAME_BACKEND" "es-${env}" - + set_cf_envs echo "Restarting app: $CGAPPNAME_BACKEND" diff --git a/tdrs-backend/tdpservice/scheduling/management/db_backup.py b/tdrs-backend/tdpservice/scheduling/management/db_backup.py index 2ee42c14a..11beceaed 100644 --- a/tdrs-backend/tdpservice/scheduling/management/db_backup.py +++ b/tdrs-backend/tdpservice/scheduling/management/db_backup.py @@ -57,28 +57,16 @@ def get_system_values(): sys_values['S3_SECRET_ACCESS_KEY'] = sys_values['S3_CREDENTIALS']['secret_access_key'] sys_values['S3_BUCKET'] = sys_values['S3_CREDENTIALS']['bucket'] sys_values['S3_REGION'] = sys_values['S3_CREDENTIALS']['region'] - sys_values['DATABASE_URI'] = OS_ENV['DATABASE_URL'] + # Set AWS credentials in env, Boto3 uses the env variables for connection os.environ["AWS_ACCESS_KEY_ID"] = sys_values['S3_ACCESS_KEY_ID'] os.environ["AWS_SECRET_ACCESS_KEY"] = sys_values['S3_SECRET_ACCESS_KEY'] # Set Database connection info - AWS_RDS_SERVICE_JSON = json.loads(OS_ENV['VCAP_SERVICES'])['aws-rds'][0]['credentials'] - sys_values['DATABASE_PORT'] = AWS_RDS_SERVICE_JSON['port'] - sys_values['DATABASE_PASSWORD'] = AWS_RDS_SERVICE_JSON['password'] - sys_values['DATABASE_DB_NAME'] = AWS_RDS_SERVICE_JSON['db_name'] - sys_values['DATABASE_HOST'] = AWS_RDS_SERVICE_JSON['host'] - sys_values['DATABASE_USERNAME'] = AWS_RDS_SERVICE_JSON['username'] - - # write .pgpass - with open('/home/vcap/.pgpass', 'w') as f: - f.write(sys_values['DATABASE_HOST'] + ":" - + sys_values['DATABASE_PORT'] + ":" - + settings.DATABASES['default']['NAME'] + ":" - + sys_values['DATABASE_USERNAME'] + ":" - + sys_values['DATABASE_PASSWORD']) - os.environ['PGPASSFILE'] = '/home/vcap/.pgpass' - os.system('chmod 0600 /home/vcap/.pgpass') + AWS_RDS_SERVICE_JSON = json.loads(OS_ENV['VCAP_SERVICES'])['aws-rds'][0] + sys_values['DATABASE_URI'] = AWS_RDS_SERVICE_JSON['credentials']['uri'].rsplit('/', 1)[0] + sys_values['DATABASE_DB_NAME'] = AWS_RDS_SERVICE_JSON['credentials']['db_name'] + return sys_values @@ -94,19 +82,11 @@ def backup_database(file_name, pg_dump -F c --no-acl --no-owner -f backup.pg postgresql://${USERNAME}:${PASSWORD}@${HOST}:${PORT}/${NAME} """ try: - # TODO: This is a bandaid until the correct logic is determined for the system values with respect to the - # correct database name. - # cmd = postgres_client + "pg_dump -Fc --no-acl -f " + file_name + " -d " + database_uri - db_host = settings.DATABASES['default']['HOST'] - db_port = settings.DATABASES['default']['PORT'] - db_name = settings.DATABASES['default']['NAME'] - db_user = settings.DATABASES['default']['USER'] - - export_password = f"export PGPASSWORD={settings.DATABASES['default']['PASSWORD']}" - cmd = (f"{export_password} && {postgres_client}pg_dump -h {db_host} -p {db_port} -d {db_name} -U {db_user} " - f"-F c --no-password --no-acl --no-owner -f {file_name}") + cmd = f"{postgres_client}pg_dump -Fc --no-acl -f {file_name} -d {database_uri}" logger.info(f"Executing backup command: {cmd}") - os.system(cmd) + code = os.system(cmd) + if code != 0: + raise Exception("pg_dump command failed with a non zero exit code.") msg = "Successfully executed backup. Wrote pg dumpfile to {}".format(file_name) logger.info(msg) LogEntry.objects.log_action( @@ -268,28 +248,47 @@ def get_database_credentials(database_uri): database_name = database_uri return [username, password, host, port, database_name] - -def main(argv, sys_values, system_user): - """Handle commandline args.""" - arg_file = "/tmp/backup.pg" - arg_database = sys_values['DATABASE_URI'] +def get_opts(argv, db_name): + """Parse command line options.""" + arg_file = f"/tmp/{db_name}_backup.pg" arg_to_restore = False arg_to_backup = False + restore_db_name = None + opts, args = getopt.getopt(argv, "hbrf:n:", ["help", "backup", "restore", "file=", "restore_db_name="]) + for opt, arg in opts: + if "backup" in opt or "-b" in opt: + arg_to_backup = True + elif "restore" in opt or "-r" in opt: + arg_to_restore = True + if "file" in opt or "-f" in opt and arg: + arg_file = arg if arg[0] == "/" else "/tmp/" + arg + if "restore_db_name" in opt or "-n" in opt and arg: + restore_db_name = arg + + if arg_to_restore and not restore_db_name: + raise ValueError("You must pass a `-n ` when trying to restore a DB.") + + return arg_file, arg_to_backup, arg_to_restore, restore_db_name + +def get_db_name(sys_values): + """ + Get the correct database name. - try: - opts, args = getopt.getopt(argv, "hbrf:d:", ["help", "backup", "restore", "file=", "database=", ]) - for opt, arg in opts: - if "backup" in opt or "-b" in opt: - arg_to_backup = True - elif "restore" in opt or "-r" in opt: - arg_to_restore = True - if "file" in opt or "-f" in opt and arg: - arg_file = arg if arg[0] == "/" else "/tmp/" + arg - if "database" in opt or "-d" in opt: - arg_database = arg + In prod we use the default database name that AWS creates. In the Dev and Staging environments the databases are + named based off of their app; i.e. tdp_db_raft. The deploy script sets the APP_DB_NAME environment variable for all + apps except prod. + """ + env_db_name = os.getenv("APP_DB_NAME", None) + if env_db_name is None: + return sys_values['DATABASE_DB_NAME'] + return env_db_name - except Exception as e: - raise e +def main(argv, sys_values, system_user): + """Handle commandline args.""" + db_base_uri = sys_values['DATABASE_URI'] + + db_name = get_db_name(sys_values) + arg_file, arg_to_backup, arg_to_restore, restore_db_name = get_opts(argv, db_name) if arg_to_backup: LogEntry.objects.log_action( @@ -303,7 +302,7 @@ def main(argv, sys_values, system_user): # back up database backup_database(file_name=arg_file, postgres_client=sys_values['POSTGRES_CLIENT_DIR'], - database_uri=arg_database, + database_uri=f"{db_base_uri}/{db_name}", system_user=system_user) # upload backup file @@ -348,7 +347,7 @@ def main(argv, sys_values, system_user): # restore database restore_database(file_name=arg_file, postgres_client=sys_values['POSTGRES_CLIENT_DIR'], - database_uri=arg_database, + database_uri=f"{db_base_uri}/{restore_db_name}", system_user=system_user) LogEntry.objects.log_action(