From a886f701b2bc9af1839666a7bae37e556b4c7f8d Mon Sep 17 00:00:00 2001 From: Andrew <84722778+andrew-jameson@users.noreply.github.com> Date: Thu, 2 Nov 2023 09:44:43 -0400 Subject: [PATCH 1/5] Update cloudgov.py (#2730) * Update cloudgov.py * hijack develop workflow to test deployments. * syntax typo. * I can't believe this typo. * linting whitespace * removed extra space * Adding in staging jwt_key due to recent deployment failure. * Removing self-reference branch filter for mergability --------- Co-authored-by: andrew-jameson Co-authored-by: George Hudson --- scripts/deploy-backend.sh | 6 +++++- tdrs-backend/tdpservice/settings/cloudgov.py | 6 +++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/scripts/deploy-backend.sh b/scripts/deploy-backend.sh index f50152891..7fb1fb657 100755 --- a/scripts/deploy-backend.sh +++ b/scripts/deploy-backend.sh @@ -51,6 +51,8 @@ set_cf_envs() "FRONTEND_BASE_URL" "LOGGING_LEVEL" "REDIS_URI" + "JWT_KEY" + "STAGING_JWT_KEY" ) echo "Setting environment variables for $CGAPPNAME_BACKEND" @@ -62,6 +64,8 @@ set_cf_envs() cf_cmd="cf unset-env $CGAPPNAME_BACKEND $var_name ${!var_name}" $cf_cmd continue + elif [[ ("$var_name" =~ "STAGING_*") && ("$CF_SPACE" = "tanf-staging") ]]; then + var_name=$(echo "$var_name" | sed -e 's@STAGING_@@g') fi cf_cmd="cf set-env $CGAPPNAME_BACKEND $var_name ${!var_name}" @@ -128,7 +132,7 @@ update_backend() bind_backend_to_services() { echo "Binding services to app: $CGAPPNAME_BACKEND" - if [ "$CFAPPNAME_BACKEND" = "tdp-backend-develop" ]; then + if [ "$CGAPPNAME_BACKEND" = "tdp-backend-develop" ]; then # TODO: this is technical debt, we should either make staging mimic tanf-dev # or make unique services for all apps but we have a services limit # Introducing technical debt for release 3.0.0 specifically. diff --git a/tdrs-backend/tdpservice/settings/cloudgov.py b/tdrs-backend/tdpservice/settings/cloudgov.py index 6f7c7342b..b7def9383 100644 --- a/tdrs-backend/tdpservice/settings/cloudgov.py +++ b/tdrs-backend/tdpservice/settings/cloudgov.py @@ -70,7 +70,11 @@ class CloudGov(Common): # env_based_db_name = f'tdp_db_{cloudgov_space_suffix}_{cloudgov_name}' - db_name = database_creds['db_name'] if (cloudgov_space_suffix in ["prod", "staging"]) else env_based_db_name + logger.debug("css: " + cloudgov_space_suffix) + if (cloudgov_space_suffix in ["prod", "staging"]): + db_name = database_creds['db_name'] + else: + db_name = env_based_db_name DATABASES = { 'default': { From 84f357d9ac2fb485dab632e72b5d6a6aa07034aa Mon Sep 17 00:00:00 2001 From: Smithh-Co <121890311+Smithh-Co@users.noreply.github.com> Date: Thu, 2 Nov 2023 08:01:55 -0700 Subject: [PATCH 2/5] Create sprint-83-summary.md (#2728) Sprint summary Co-authored-by: Andrew <84722778+andrew-jameson@users.noreply.github.com> --- docs/Sprint-Review/sprint-83-summary.md | 49 +++++++++++++++++++++++++ 1 file changed, 49 insertions(+) create mode 100644 docs/Sprint-Review/sprint-83-summary.md diff --git a/docs/Sprint-Review/sprint-83-summary.md b/docs/Sprint-Review/sprint-83-summary.md new file mode 100644 index 000000000..cf47de6cf --- /dev/null +++ b/docs/Sprint-Review/sprint-83-summary.md @@ -0,0 +1,49 @@ + +# Sprint 83 Summary + +09/30/23 - 10/11/23 + +Velocity: Dev (18) + +## Sprint Goal +* Complete parsing engine development for TANF Section (04) and begin SSP (01), close out subsmission history and metadata workflows (1613/12/10). +* UX to continue regional staff and in-app messaging research, errors audit approach, and bridge onboarding to >95% of total users +* DevOps to investigate singluar ClamAV (2429), resolve utlity images for CircleCI and evaluate CI/CD pipeline. + + +## Tickets +### Completed/Merged +* [#1612 Detailed case level metadata](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/1612) +* [#1610 As a user, I need information about the acceptance of my data and a link for the error report](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/1610) +* [#1111 TANF (04) Parsing and Validation](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/1111) + +### Ready to Merge +* N/A + +### Submitted (QASP Review, OCIO Review) +* N/A + +### Closed (not merged) +* N/A + +## Moved to Next Sprint (Blocked, Raft Review, In Progress, Current Sprint Backlog) +### In Progress +* [#2536 [spike] Cat 4 validation](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2536) +* [#2709 SSP (Section 1) validation](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2709) +* [#2663 Investigate OWASP NightlyScan findings](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2663) + +### Blocked +* N/A + +### Raft Review +* [#2429 Singular ClamAV scanner](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2429) +* [#2664 (bug) file extension](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2664) +* [#2695 space-filled values update](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2695) +* [#2411 As system admin, I need to view metadata on parsed datafiles](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2411) + +### Demo +* Internal: + * 1111, 1610, 1612 +* External: + * 1111, 1610, 1612 + From d6784ad521f04650b045a08ce73f3ff52164307f Mon Sep 17 00:00:00 2001 From: Miles Reiter Date: Fri, 3 Nov 2023 10:00:33 -0400 Subject: [PATCH 3/5] Sprint 84 summary (#2737) * Create sprint-84-summary.md * Update sprint-84-summary.md --------- Co-authored-by: Andrew <84722778+andrew-jameson@users.noreply.github.com> --- docs/Sprint-Review/sprint-84-summary.md | 61 +++++++++++++++++++++++++ 1 file changed, 61 insertions(+) create mode 100644 docs/Sprint-Review/sprint-84-summary.md diff --git a/docs/Sprint-Review/sprint-84-summary.md b/docs/Sprint-Review/sprint-84-summary.md new file mode 100644 index 000000000..a22dfd912 --- /dev/null +++ b/docs/Sprint-Review/sprint-84-summary.md @@ -0,0 +1,61 @@ +# Sprint 84 Summary +10/10/23 - 10/24/23 + +Velocity: Dev (10) + +### Sprint Goal +* Dev: + * Continue parsing engine development + * Complete SSP Sec (01) and SSP Sec (02) + * Resolve deployment blocker + * Coordinate w/ OFA and draft dev contingency plan for future gov shutdown +* DevOps: + * 2429 - Singular Clam AV + * 2722 - Singular deployment workflow +* UX: Resume regional staff research, synthesize in-app messaging research, continue supporting onboarding/utilization +* Prod: Find path forward on Sendgrid + +## Tickets +### Completed/Merged +* [#2411 As system admin, I want to view metadata on parsed datafiles](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2411) +* [#2429 Singular ClamAV Scanner](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2429) +* [#2664 (bug) file extension](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2664) + + + +### Ready to Merge +* [#2695 space-filled values update](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2695) +* [#2725 file input render issue](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2725) + + +### Submitted (QASP Review, OCIO Review) +* [#2701 FETCH_STTS Infinite Request](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2701) +* [#2709 SSP (Section 1) validation](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2709) + +### Closed (not merged) +* N/A + +## Moved to Next Sprint (Blocked, Raft Review, In Progress, Current Sprint Backlog) +### In Progress +* [#2536 [spike] Cat 4 validation](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2536) +* [#1119 SSP Aggregate (03) Parsing](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/1119) +* [#2592 Deploy celery as a separate cloud.gov app](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2592) +* [#2599 Readability enhancements for error reports](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2599) +* [#2683 ZAP result - CORS config issue](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2683) +* [#2722 simplify workflows and de-bloat pipeline code](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2722) + + +### Blocked +* N/A + +### Raft Review +* [#1118 SSP Closed Data (02) Parsing](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/1118) +* [#1120 SSP Stratum (04) Parsing](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/1120) +* [#2116 Container Registry creation](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2116) +* [Spike - Investigate OWASP nightly scan findings](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2663) + +### Demo +* N/A + + + From 96452dcd6fb7343532fa7e86c4d3ac7f0c04179f Mon Sep 17 00:00:00 2001 From: Andrew <84722778+andrew-jameson@users.noreply.github.com> Date: Fri, 3 Nov 2023 12:44:47 -0400 Subject: [PATCH 4/5] Debug/develop deployment failures (#2743) * Update cloudgov.py * hijack develop workflow to test deployments. * syntax typo. * I can't believe this typo. * linting whitespace * removed extra space * Adding in staging jwt_key due to recent deployment failure. * Removing self-reference branch filter for mergability * Updated elif logic based on CI failures --------- Co-authored-by: andrew-jameson Co-authored-by: George Hudson --- scripts/deploy-backend.sh | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/scripts/deploy-backend.sh b/scripts/deploy-backend.sh index 7fb1fb657..ec372396a 100755 --- a/scripts/deploy-backend.sh +++ b/scripts/deploy-backend.sh @@ -64,11 +64,13 @@ set_cf_envs() cf_cmd="cf unset-env $CGAPPNAME_BACKEND $var_name ${!var_name}" $cf_cmd continue - elif [[ ("$var_name" =~ "STAGING_*") && ("$CF_SPACE" = "tanf-staging") ]]; then - var_name=$(echo "$var_name" | sed -e 's@STAGING_@@g') + elif [[ ("$var_name" =~ "STAGING_") && ("$CF_SPACE" = "tanf-staging") ]]; then + sed_var_name=$(echo "$var_name" | sed -e 's@STAGING_@@g') + cf_cmd="cf set-env $CGAPPNAME_BACKEND $sed_var_name ${!var_name}" + else + cf_cmd="cf set-env $CGAPPNAME_BACKEND $var_name ${!var_name}" fi - - cf_cmd="cf set-env $CGAPPNAME_BACKEND $var_name ${!var_name}" + echo "Setting var : $var_name" $cf_cmd done From 7115532cd187955fe81766b9908f9461a655ccf4 Mon Sep 17 00:00:00 2001 From: Eric Lipe <125676261+elipe17@users.noreply.github.com> Date: Fri, 3 Nov 2023 12:24:39 -0600 Subject: [PATCH 5/5] File Input Render Issue (#2725) * - small grammar fix * - Fix lint suggestions * - updated message * - Fixed lint errors * - Added correct extension to datafiles without one * - Adding cherry picks for file extension error handling * - Updated regex * - updating to keep file in dropbox in event of error to help user correct their mistake. * - Fix icon rendering incorrectly * update test file extensions * - making timeout longer * - Resolved issue causing test failure - resetting timeout * - passing param * - updated nginx conf --------- Co-authored-by: Alex P <63075587+ADPennington@users.noreply.github.com> Co-authored-by: Miles Reiter Co-authored-by: Jan Timpe Co-authored-by: Andrew <84722778+andrew-jameson@users.noreply.github.com> --- tdrs-frontend/nginx/cloud.gov/buildpack.nginx.conf | 8 ++++---- tdrs-frontend/nginx/local/default.conf.template | 6 +++--- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/tdrs-frontend/nginx/cloud.gov/buildpack.nginx.conf b/tdrs-frontend/nginx/cloud.gov/buildpack.nginx.conf index 1ab4677bb..4ed6804f9 100644 --- a/tdrs-frontend/nginx/cloud.gov/buildpack.nginx.conf +++ b/tdrs-frontend/nginx/cloud.gov/buildpack.nginx.conf @@ -21,9 +21,9 @@ http { log_format compression '$remote_addr - $remote_user [$time_local] ' '"proxy_host and upstream_addr": $proxy_host $upstream_addr, ' ' "request": $request, ' - '"body_bytes_sent" : $body_bytes_sent, ' + '"body_bytes_sent" : $body_bytes_sent, ' '"request_body": $request_body, ' - '"http_x_forwarded_for": $http_x_forwarded_for, ' + '"http_x_forwarded_for": $http_x_forwarded_for, ' '"host": $host, ' ' "status": $status, ' '"proxy_add_x_forwarded_for": $proxy_add_x_forwarded_for, ' @@ -47,7 +47,7 @@ http { } client_max_body_size 100m; - + # Block all requests except ones listed in whitelist; disabled for local # First have to correct the source IP address using real_ip_header, otherwise # the IP address will be the internal IP address of the router @@ -63,7 +63,7 @@ http { set $CSP "default-src 'self';"; set $CSP "${CSP}script-src 'self';"; set $CSP "${CSP}script-src-elem 'self';"; - set $CSP "${CSP}script-src-attr 'self';"; + set $CSP "${CSP}script-src-attr 'self' 'unsafe-inline';"; set $CSP "${CSP}img-src 'self' data:;"; set $CSP "${CSP}font-src 'self';"; set $CSP "${CSP}connect-src 'self' ${CONNECT_SRC};"; diff --git a/tdrs-frontend/nginx/local/default.conf.template b/tdrs-frontend/nginx/local/default.conf.template index 2243c720b..c4d306340 100644 --- a/tdrs-frontend/nginx/local/default.conf.template +++ b/tdrs-frontend/nginx/local/default.conf.template @@ -82,7 +82,7 @@ http { set $CSP "${CSP}prefetch-src 'none';"; set $CSP "${CSP}form-action *;"; set $CSP "${CSP}script-src-elem 'self' http://localhost:* http://www.w3.org;"; - set $CSP "${CSP}script-src-attr 'self';"; + set $CSP "${CSP}script-src-attr 'self' 'unsafe-inline';"; set $CSP "${CSP}style-src-elem 'self' 'unsafe-inline';"; set $CSP "${CSP}style-src-attr 'self';"; set $CSP "${CSP}worker-src 'none';"; @@ -104,7 +104,7 @@ http { access_log /dev/stdout compression; #access_log stderr compression; - + # Content caching # saves cached fies in /tmp # cache zone name = tdp_cache @@ -126,7 +126,7 @@ http { set $CSP "default-src 'self';"; set $CSP "${CSP}script-src 'self';"; set $CSP "${CSP}script-src-elem 'self';"; - set $CSP "${CSP}script-src-attr 'self';"; + set $CSP "${CSP}script-src-attr 'self' 'unsafe-inline';"; set $CSP "${CSP}img-src 'self' data:;"; set $CSP "${CSP}font-src 'self';"; set $CSP "${CSP}manifest-src 'self';";