Merge branch 'develop' into feature/2709-ssp-section-1-validation

.circleci/deployment/commands.yml

@@ -37,16 +37,40 @@
           backend-appname: <<parameters.backend-appname>>
           frontend-appname: <<parameters.frontend-appname>>
           cf-space: <<parameters.cf-space>>
-      - deploy-clamav:
-          backend-appname: <<parameters.backend-appname>>
-          cf-org: <<parameters.cf-org>>
-          cf-space: <<parameters.cf-space>>
       - deploy-frontend:
           environment: <<parameters.environment>>
           backend-appname: <<parameters.backend-appname>>
           frontend-appname: <<parameters.frontend-appname>>
           cf-space: <<parameters.cf-space>>
 
+  clamav-cloud-dot-gov:
+    parameters:
+      backend-appname:
+        default: tdp-backend
+        type: string
+      cf-password:
+        default: CF_PASSWORD_DEV
+        type: env_var_name
+      cf-org:
+        default: CF_ORG
+        type: env_var_name
+      cf-space:
+        default: tanf-dev
+        type: string
+      cf-username:
+        default: CF_USERNAME_DEV
+        type: env_var_name
+    steps:
+      - checkout
+      - sudo-check
+      - cf-check
+      - login-cloud-dot-gov:
+          cf-password: <<parameters.cf-password>>
+          cf-org: <<parameters.cf-org>>
+          cf-space: <<parameters.cf-space>>
+          cf-username: <<parameters.cf-username>>
+      - deploy-clamav
+
   deploy-backend:
     parameters:
       backend-appname:
@@ -71,30 +95,12 @@
               <<parameters.cf-space>>
 
   deploy-clamav:
-    parameters:
-      backend-appname:
-        default: tdp-backend
-        type: string
-      cf-org:
-        default: CF_ORG
-        type: env_var_name
-      cf-space:
-        default: tanf-dev
-        type: string
     steps:
       - run:
           name: Deploy ClamAV REST application
           command: |
             cf push clamav-rest -f tdrs-backend/manifest.clamav.yml \
-              --var cf-space=<<parameters.cf-space>> \
-      - run:
-          name: Enable internal route between backend and clamav-rest app
-          command: |
-            cf add-network-policy <<parameters.backend-appname>> clamav-rest \
-              -s <<parameters.cf-space>> \
-              -o ${<<parameters.cf-org>>} \
-              --protocol tcp \
-              --port 9000
+              --var cf-space=tanf-prod
 
   deploy-frontend:
     parameters:

.circleci/deployment/jobs.yml

@@ -151,3 +151,12 @@
           cf-password: CF_PASSWORD_PROD
           cf-space: tanf-prod
           cf-username: CF_USERNAME_PROD
+  prod-deploy-clamav:
+    executor: docker-executor
+    working_directory: ~/tdp-deploy
+    steps:
+      - clamav-cloud-dot-gov:
+          backend-appname: tdp-backend-prod
+          cf-password: CF_PASSWORD_PROD
+          cf-space: tanf-prod
+          cf-username: CF_USERNAME_PROD

.circleci/deployment/workflows.yml

@@ -118,6 +118,13 @@
             branches:
               only:
                 - master
+      - prod-deploy-clamav:
+          requires:
+            - deploy-infrastructure-production
+          filters:
+            branches:
+              only:
+                - master
       - make_erd: # from ../util folder
           filters:
             branches:

docs/Sprint-Review/sprint-82-summary.md

@@ -0,0 +1,49 @@
+
+# Sprint 82 Summary
+
+09/13/23 - 09/26/23
+
+Velocity: Dev (13)
+
+## Sprint Goal
+* Continue parsing engine development for TANF Section (04) and SSP (01), close out subsmission history and metadata workflows (1613/12/10).
+* UX to continue regional staff and in-app messaging research, errors audit approach, and bridge onboarding to >95% of total users
+* DevOps to investigate singluar ClamAV (2429), resolve utlity images for CircleCI and evaluate CI/CD pipeline.
+
+## Tickets
+### Completed/Merged
+* [#1613 As a developer, I need parsed file meta data (TANF Section 1)](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/board)
+* [#2700 Deployment/migration issue](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2700)
+
+### Ready to Merge
+* N/A
+
+### Submitted (QASP Review, OCIO Review)
+* [#1612 Detailed case level metadata](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/1612)
+
+### Closed (not merged)
+* N/A
+
+## Moved to Next Sprint (Blocked, Raft Review, In Progress, Current Sprint Backlog)
+### In Progress
+
+* [#2429 Singular ClamAV scanner](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2429)
+* [#2695 space-filled values update](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2695)
+* [#2411 As system admin, I need to view metadata on parsed datafiles](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2411)
+* [#2536 [spike] Cat 4 validation](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2536)
+* [#2709 SSP (Section 1) validation](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2709)
+
+### Blocked
+* N/A
+
+### Raft Review
+* [#1610 As a user, I need information about the acceptance of my data and a link for the error report](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/1610)
+* [#1111 TANF (04) Parsing and Validation](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/1111)
+* [#2664 (bug) file extension](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2664)
+
+### Demo
+* Internal:
+    * 1613
+
+
+

docs/Technical-Documentation/README.md

@@ -12,6 +12,7 @@ This directory contains system and architecture documentation including diagrams
 - [buildpack-changelog.md](./buildpack-changelog.md) : A running log of updates to our Cloud.gov buildpacks in use.
 - [circle-ci-audit-template.md](./circle-ci-audit-template.md) : This is a checklist document used during audits of our continuous intergration pipeline tool.
 - [circle-ci.md](./circle-ci.md) : Overview of our CI/CD platform jobs.
+- [clamav.md](./clamav.md) : How to access ClamAV from different apps/spaces.
 - [cypress-integration-tests.md](./cypress-integration-tests.md) : Shows how we use Cypress to manage our end to end integration testing.
 - [data-file-downloads.md](./data-file-downloads.md) : Provides an architecture-level view of data file storage and downloading.
 - [django-admin-logging.md](./django-admin-logging.md) : Outlines sections of the Django Administrator Console and details what should be logged.

docs/Technical-Documentation/clamav.md

@@ -0,0 +1,48 @@
+# CLAMAV 
+
+In order to have one CLAMAV instance, existing in prod, the Nginx router is created
+for CLAMAV to forward the traffic from 'dev' and 'staging' spaces into
+prod space, where the CLAMAV service exists.
+
+## Deploy Nginx instance
+To route the clamav traffic to clamav in prod, each space needs to have one instance of _Nginx Router_ which routes traffic to clamav.
+
+In order to deploy the nginx router instance, change your directory to `tdrs-backend/clamav-router/` and run the following command while logged into the target space:
+
+
+>`cf push tdp-clamav-nginx-${cf-shortened-space} -f manifest.yml --no-route`
+
+, where _cf-shortened-space_ can be : _dev_, _staging_, or _prod_. 
+
+The instance name then will be set as an environment variable to redirect each instance traffic. This will deploy the nginx instance to the target environment.
+
+## Further communication configurations
+
+### Setup Individual Instances
+
+First, set the environment variable __AV_SCAN_URL__ as follows:
+```
+Environment variable name: AV_SCAN_URL
+Environment variable value: http://{nginx_instance}.apps.internal:9000/scan
+```
+
+### Add network policy from _{tdp-clamav-nginx}_ to clamav in prod
+To enable traffic between the "__nginx instance__" and "__clamav instance in production__", we need to add the network policiy and route between the two:
+
+>`cf add-network-policy {nginx_instance} "clamav-rest" -s "tanf-prod" --protocol tcp --port 9000`
+e.g: `{nginx_instance_name} = tdp-clamav-nginx-dev`
+
+### Add network policy from _{backend_instance}_ to _tdp-clamav-nginx_
+
+>`cf add-network-policy {backend_instance} {nginx_instance} --protocol tcp --port 9000`
+
+where e.g: `backend_instance = tdp-backend-develop`
+
+### Add route for _tdp-clamav-nginx_
+
+ Note: Make sure to delete (if existing) routes that are not being used. In some rare cases, a mal-assigned network policy can interfere with outgoing traffic. As an example, a policy like 
+ >`cf delete-route app.cloud.gov --hostname tdp-frontend-staging`
+
+Add route:
+
+>`cf map-route {nginx_instance} apps.internal --hostname {nginx_instance}`

scripts/deploy-backend.sh

@@ -40,7 +40,6 @@ set_cf_envs()
   "AMS_CLIENT_ID"
   "AMS_CLIENT_SECRET"
   "AMS_CONFIGURATION_ENDPOINT"
-  "AV_SCAN_URL"
   "BASE_URL"
   "CLAMAV_NEEDED"
   "CYPRESS_TOKEN"
@@ -84,6 +83,15 @@ generate_jwt_cert()
 update_backend()
 {
     cd tdrs-backend || exit
+    cf unset-env "$CGAPPNAME_BACKEND" "AV_SCAN_URL"
+
+    if ["$CF_SPACE" = "tanf-prod" ]; then
+      cf set-env "$CGAPPNAME_BACKEND" AV_SCAN_URL "http://tanf-prod-clamav-rest.apps.internal:9000/scan"
+    else
+      # Add environment varilables for clamav
+      cf set-env "$CGAPPNAME_BACKEND" AV_SCAN_URL "http://tdp-clamav-nginx-$env.apps.internal:9000/scan"
+    fi
+
     if [ "$1" = "rolling" ] ; then
         set_cf_envs
 
@@ -101,11 +109,18 @@ update_backend()
     fi
 
     set_cf_envs
-
+    
     cf map-route "$CGAPPNAME_BACKEND" apps.internal --hostname "$CGAPPNAME_BACKEND"
 
     # Add network policy to allow frontend to access backend
     cf add-network-policy "$CGAPPNAME_FRONTEND" "$CGAPPNAME_BACKEND" --protocol tcp --port 8080
+
+    if ["$CF_SPACE" = "tanf-prod" ]; then
+      # Add network policy to allow backend to access tanf-prod services
+      cf add-network-policy "$CGAPPNAME_BACKEND" clamav-rest --protocol tcp --port 9000
+    else
+      cf add-network-policy "$CGAPPNAME_BACKEND" tdp-clamav-nginx-$env --protocol tcp --port 9000
+    fi
 
     cd ..
 }

tdrs-backend/clamav-router/manifest.yml

@@ -0,0 +1,9 @@
+version: 1
+applications:
+- name: tdp-clamav-nginx
+  buildpacks:
+    - https://github.com/cloudfoundry/nginx-buildpack.git#v1.2.6
+  memory: 32M
+  instances: 1
+  disk_quota: 64M
+  timeout: 180

tdrs-backend/clamav-router/nginx.conf

@@ -0,0 +1,20 @@
+events { worker_connections 1024;
+}
+
+# This opens a route to clamav prod
+http{ 
+    server {
+        listen {{port}};
+        location /scan {
+            proxy_pass                      http://tanf-prod-clamav-rest.apps.internal:9000/scan;
+            proxy_pass_request_headers      on;
+        }
+    }
+    server {
+        listen 9000;
+        location /scan {
+            proxy_pass                      http://tanf-prod-clamav-rest.apps.internal:9000/scan;
+            proxy_pass_request_headers      on;
+        }
+    }
+}

tdrs-backend/manifest.clamav.yml

@@ -9,4 +9,4 @@ applications:
   env:
     MAX_FILE_SIZE: 200M
   routes:
-    - route: ((cf-space))-clamav-rest.apps.internal
+    - route: tanf-prod-clamav-rest.apps.internal

tdrs-backend/manifest.yml

@@ -6,5 +6,3 @@ applications:
   disk_quota: 2G
   docker:
     image: ((docker-backend))
-  env:
-    AV_SCAN_URL: http://((cf-space))-clamav-rest.apps.internal:9000/scan

tdrs-backend/tdpservice/data_files/admin.py

@@ -3,19 +3,73 @@
 
 from ..core.utils import ReadOnlyAdminMixin
 from .models import DataFile, LegacyFileTransfer
+from tdpservice.parsers.models import DataFileSummary, ParserError
+from django.conf import settings
+from django.utils.html import format_html
 
+DOMAIN = settings.FRONTEND_BASE_URL
+
+class DataFileSummaryPrgTypeFilter(admin.SimpleListFilter):
+    """Admin class filter for Program Type on datafile model."""
+
+    title = 'Program Type'
+    parameter_name = 'program_type'
+
+    def lookups(self, request, model_admin):
+        """Return a list of tuples."""
+        return [
+            ('TAN', 'TAN'),
+            ('SSP', 'SSP'),
+        ]
+
+    def queryset(self, request, queryset):
+        """Return a queryset."""
+        if self.value():
+            query_set_ids = [df.id for df in queryset if df.prog_type == self.value()]
+            return queryset.filter(id__in=query_set_ids)
+        else:
+            return queryset
 
 @admin.register(DataFile)
 class DataFileAdmin(ReadOnlyAdminMixin, admin.ModelAdmin):
     """Admin class for DataFile models."""
 
+    def status(self, obj):
+        """Return the status of the data file summary."""
+        return DataFileSummary.objects.get(datafile=obj).status
+
+    def case_totals(self, obj):
+        """Return the case totals."""
+        return DataFileSummary.objects.get(datafile=obj).case_aggregates
+
+    def error_report_link(self, obj):
+        """Return the link to the error report."""
+        pe_len = ParserError.objects.filter(file=obj).count()
+
+        filtered_parserror_list_url = f'{DOMAIN}/admin/parsers/parsererror/?file=' + str(obj.id)
+        # have to find the error id from obj
+        return format_html("<a href='{url}'>{field}</a>",
+                           field="Parser Errors: " + str(pe_len),
+                           url=filtered_parserror_list_url)
+
+    error_report_link.allow_tags = True
+
+    def data_file_summary(self, obj):
+        """Return the data file summary."""
+        df = DataFileSummary.objects.get(datafile=obj)
+        return format_html("<a href='{url}'>{field}</a>",
+                           field=f'{df.id}' + ":" + df.get_status(),
+                           url=f"{DOMAIN}/admin/parsers/datafilesummary/{df.id}/change/")
+
     list_display = [
         'id',
         'stt',
         'year',
         'quarter',
         'section',
         'version',
+        'data_file_summary',
+        'error_report_link',
     ]
 
     list_filter = [
@@ -25,6 +79,8 @@ class DataFileAdmin(ReadOnlyAdminMixin, admin.ModelAdmin):
         'user',
         'year',
         'version',
+        'summary__status',
+        DataFileSummaryPrgTypeFilter
     ]
 
 @admin.register(LegacyFileTransfer)