Merge branch 'develop' into 2592/separate-celery

docs/Security-Compliance/boundary-diagram.md

@@ -6,7 +6,11 @@
 
 ### Data flow
 
-Users with `OFA Admin` and (STT) `Data Analyst` roles can upload data on upload data files locally into the web application which will store the files in cloud.gov AWS S3 buckets only after the files are successfully scanned for viruses via [ClamAV](../Technical-Documentation/Architecture-Decision-Record/012-antivirus-strategy.md). Developers will deploy new code through GitHub, initiating the continuous integration process through Circle CI.
+Users with `OFA Admin` and (STT) `Data Analyst` roles can upload data on upload data files locally into the web application which will store the files in cloud.gov AWS S3 buckets only after the files are successfully scanned for viruses via [ClamAV](../Technical-Documentation/Architecture-Decision-Record/012-antivirus-strategy.md). For lower environments, we use an NGINX server to function as a proxy, routing to the ClamAV-rest server in the production space. The NGINX server also functions as a gatekeeper, allowing documents for scanning to only come from backend servers, and only able to route them directly to the ClamAV-rest server.
+
+### Code Repository and CI Pipeline
+
+Developers will deploy new code through GitHub, initiating the continuous integration process through Circle CI.
 
 ### Environments/Spaces
 

scripts/deploy-backend.sh

@@ -88,7 +88,7 @@ update_backend() {
     cd tdrs-backend || exit
     cf unset-env "$CGAPPNAME_BACKEND" "AV_SCAN_URL"
 
-    if ["$CF_SPACE" = "tanf-prod" ]; then
+    if [ "$CF_SPACE" = "tanf-prod" ]; then
       cf set-env "$CGAPPNAME_BACKEND" AV_SCAN_URL "http://tanf-prod-clamav-rest.apps.internal:9000/scan"
     else
       # Add environment varilables for clamav
@@ -97,7 +97,6 @@ update_backend() {
 
     if [ "$1" = "rolling" ] ; then
         set_cf_envs
-
         # Do a zero downtime deploy.  This requires enough memory for
         # two apps to exist in the org/space at one time.
         cf push "$CGAPPNAME_BACKEND" --no-route -f manifest.buildpack.yml -t 180 --strategy rolling || exit 1
@@ -120,7 +119,7 @@ update_backend() {
     # Add network policy to allow frontend to access backend
     cf add-network-policy "$CGAPPNAME_FRONTEND" "$CGAPPNAME_BACKEND" --protocol tcp --port 8080
 
-    if ["$CF_SPACE" = "tanf-prod" ]; then
+    if [ "$CF_SPACE" = "tanf-prod" ]; then
       # Add network policy to allow backend to access tanf-prod services
       cf add-network-policy "$CGAPPNAME_BACKEND" clamav-rest --protocol tcp --port 9000
     else

scripts/zap-scanner.sh

@@ -40,7 +40,7 @@ cd "$TARGET_DIR" || exit 2
 
 
 if [[ $(docker network inspect external-net 2>&1 | grep -c Scope) == 0 ]]; then 
-    docker network create external-net
+  docker network create external-net
 fi
 
 # Ensure the APP_URL is reachable from the zaproxy container
@@ -112,10 +112,6 @@ ZAP_CLI_OPTIONS="\
   -config globalexcludeurl.url_list.url\(14\).description='Site - FontAwesome.com' \
   -config globalexcludeurl.url_list.url\(14\).enabled=true \
 
-  -config globalexcludeurl.url_list.url\(15\).regex='^https:\/\/.*\.cloud.gov\/.*$' \
-  -config globalexcludeurl.url_list.url\(15\).description='Site - Cloud.gov' \
-  -config globalexcludeurl.url_list.url\(15\).enabled=true \
-
   -config globalexcludeurl.url_list.url\(16\).regex='^https:\/\/.*\.googletagmanager.com\/.*$' \
   -config globalexcludeurl.url_list.url\(16\).description='Site - googletagmanager.com' \
   -config globalexcludeurl.url_list.url\(16\).enabled=true \
@@ -140,7 +136,6 @@ ZAP_CLI_OPTIONS="\
   -config globalexcludeurl.url_list.url\(21\).description='Site - IdentitySandbox.gov' \
   -config globalexcludeurl.url_list.url\(21\).enabled=true \
   -config spider.postform=true"
-
 # How long ZAP will crawl the app with the spider process
 ZAP_SPIDER_MINS=10
 

tdrs-backend/clamav-router/nginx.conf

@@ -4,14 +4,18 @@ events { worker_connections 1024;
 # This opens a route to clamav prod
 http{ 
     server {
+        client_max_body_size 100m;
         listen {{port}};
+        client_max_body_size 100m;
         location /scan {
             proxy_pass                      http://tanf-prod-clamav-rest.apps.internal:9000/scan;
             proxy_pass_request_headers      on;
         }
     }
     server {
+        client_max_body_size 100m;
         listen 9000;
+        client_max_body_size 100m;
         location /scan {
             proxy_pass                      http://tanf-prod-clamav-rest.apps.internal:9000/scan;
             proxy_pass_request_headers      on;

tdrs-backend/tdpservice/parsers/fields.py

@@ -1,19 +1,10 @@
 """Datafile field representations."""
 
 import logging
+from .validators import value_is_empty
 
 logger = logging.getLogger(__name__)
 
-def value_is_empty(value, length):
-    """Handle 'empty' values as field inputs."""
-    empty_values = [
-        ' '*length,  # '     '
-        '#'*length,  # '#####'
-        '_'*length,  # '_____'
-    ]
-
-    return value is None or value in empty_values
-
 
 class Field:
     """Provides a mapping between a field name and its position."""
@@ -38,8 +29,9 @@ def __repr__(self):
     def parse_value(self, line):
         """Parse the value for a field given a line, startIndex, endIndex, and field type."""
         value = line[self.startIndex:self.endIndex]
+        value_length = self.endIndex-self.startIndex
 
-        if value_is_empty(value, self.endIndex-self.startIndex):
+        if len(value) < value_length or value_is_empty(value, value_length):
             logger.debug(f"Field: '{self.name}' at position: [{self.startIndex}, {self.endIndex}) is empty.")
             return None
 

tdrs-backend/tdpservice/parsers/row_schema.py

@@ -1,6 +1,7 @@
 """Row schema for datafile."""
 from .models import ParserErrorCategoryChoices
-from .fields import Field,  value_is_empty
+from .fields import Field
+from .validators import value_is_empty
 import logging
 
 logger = logging.getLogger(__name__)

tdrs-backend/tdpservice/parsers/test/test_util.py

@@ -1,7 +1,7 @@
 """Test the methods of RowSchema to ensure parsing and validation work in all individual cases."""
 
 import pytest
-from ..fields import Field, value_is_empty
+from ..fields import Field
 from ..row_schema import RowSchema
 from ..util import SchemaManager
 
@@ -224,6 +224,7 @@ class TestModel:
 
 
 @pytest.mark.parametrize('first,second', [
+    ('', ''),
     (' ', '  '),
     ('#', '##'),
     (None, None),
@@ -308,33 +309,6 @@ def test_run_postparsing_validators_returns_invalid_and_errors():
     assert errors == ['Value is not valid.']
 
 
-@pytest.mark.parametrize("value,length", [
-    (None, 0),
-    (None, 10),
-    ('     ', 5),
-    ('###', 3)
-])
-def test_value_is_empty_returns_true(value, length):
-    """Test value_is_empty returns valid."""
-    result = value_is_empty(value, length)
-    assert result is True
-
-
-@pytest.mark.parametrize("value,length", [
-    (0, 1),
-    (1, 1),
-    (10, 2),
-    ('0', 1),
-    ('0000', 4),
-    ('1    ', 5),
-    ('##3', 3)
-])
-def test_value_is_empty_returns_false(value, length):
-    """Test value_is_empty returns invalid."""
-    result = value_is_empty(value, length)
-    assert result is False
-
-
 def test_multi_record_schema_parses_and_validates():
     """Test SchemaManager parse_and_validate."""
     line = '12345'

tdrs-backend/tdpservice/parsers/test/test_validators.py

@@ -6,6 +6,35 @@
 from tdpservice.parsers.test.factories import SSPM5Factory
 
 
+@pytest.mark.parametrize("value,length", [
+    (None, 0),
+    (None, 10),
+    ('     ', 5),
+    ('###', 3),
+    ('', 0),
+    ('', 10),
+])
+def test_value_is_empty_returns_true(value, length):
+    """Test value_is_empty returns valid."""
+    result = validators.value_is_empty(value, length)
+    assert result is True
+
+
+@pytest.mark.parametrize("value,length", [
+    (0, 1),
+    (1, 1),
+    (10, 2),
+    ('0', 1),
+    ('0000', 4),
+    ('1    ', 5),
+    ('##3', 3),
+])
+def test_value_is_empty_returns_false(value, length):
+    """Test value_is_empty returns invalid."""
+    result = validators.value_is_empty(value, length)
+    assert result is False
+
+
 def test_or_validators():
     """Test `or_validators` gives a valid result."""
     value = "2"
@@ -295,6 +324,17 @@ def test_notEmpty_returns_invalid_substring():
     assert is_valid is False
     assert error == "111  333 contains blanks between positions 3 and 5."
 
+
+def test_notEmpty_returns_nonexistent_substring():
+    """Test `notEmpty` gives an invalid result for a nonexistent substring."""
+    value = '111  333'
+
+    validator = validators.notEmpty(start=10, end=12)
+    is_valid, error = validator(value)
+
+    assert is_valid is False
+    assert error == "111  333 contains blanks between positions 10 and 12."
+
 @pytest.mark.usefixtures('db')
 class TestCat3ValidatorsBase:
     """A base test class for tests that evaluate category three validators."""

tdrs-backend/tdpservice/parsers/validators.py

@@ -6,6 +6,18 @@
 
 logger = logging.getLogger(__name__)
 
+
+def value_is_empty(value, length):
+    """Handle 'empty' values as field inputs."""
+    empty_values = [
+        '',
+        ' '*length,  # '     '
+        '#'*length,  # '#####'
+        '_'*length,  # '_____'
+    ]
+
+    return value is None or value in empty_values
+
 # higher order validator func
 
 def make_validator(validator_func, error_func):
@@ -191,17 +203,25 @@ def isStringLargerThan(val):
         lambda value: f'{value} is not larger than {val}.'
     )
 
+
+def _is_empty(value, start, end):
+    end = end if end else len(str(value))
+    vlen = end - start
+    subv = str(value)[start:end]
+    return value_is_empty(subv, vlen) or len(subv) < vlen
+
+
 def notEmpty(start=0, end=None):
     """Validate that string value isn't only blanks."""
     return make_validator(
-        lambda value: not str(value)[start:end if end else len(str(value))].isspace(),
+        lambda value: not _is_empty(value, start, end),
         lambda value: f'{str(value)} contains blanks between positions {start} and {end if end else len(str(value))}.'
     )
 
 def isEmpty(start=0, end=None):
     """Validate that string value is only blanks."""
     return make_validator(
-        lambda value: value[start:end if end else len(value)].isspace(),
+        lambda value: _is_empty(value, start, end),
         lambda value: f'{value} is not blank between positions {start} and {end if end else len(value)}.'
     )
 

tdrs-backend/tdpservice/settings/cloudgov.py

@@ -164,7 +164,14 @@ class Development(CloudGov):
 
     # https://docs.djangoproject.com/en/2.0/ref/settings/#allowed-hosts
     ALLOWED_HOSTS = ['.app.cloud.gov']
-
+    CORS_ORIGIN_ALLOW_ALL = False
+    CORS_ALLOWED_ORIGINS = ['https://*.app.cloud.gov']
+    CORS_ALLOW_CREDENTIALS = True
+    CORS_ALLOW_METHODS = (
+        "GET",
+        "PATCH",
+        "POST",
+    )
 
 class Staging(CloudGov):
     """Settings for applications deployed in the Cloud.gov staging space."""
@@ -173,7 +180,14 @@ class Staging(CloudGov):
         'tdp-frontend-staging.acf.hhs.gov',
         'tdp-frontend-develop.acf.hhs.gov'
         ]
-
+    CORS_ALLOWED_ORIGINS = ['https://*.acf.hhs.gov']
+    CORS_ORIGIN_ALLOW_ALL = False
+    CORS_ALLOW_CREDENTIALS = True
+    CORS_ALLOW_METHODS = (
+        "GET",
+        "PATCH",
+        "POST",
+    )
     LOGIN_GOV_CLIENT_ID = os.getenv(
         'OIDC_RP_CLIENT_ID',
         'urn:gov:gsa:openidconnect.profiles:sp:sso:hhs:tanf-proto-staging'
@@ -198,3 +212,10 @@ class Production(CloudGov):
 
     # CORS allowed origins
     CORS_ALLOWED_ORIGINS = ['https://tanfdata.acf.hhs.gov']
+    CORS_ORIGIN_ALLOW_ALL = False
+    CORS_ALLOW_CREDENTIALS = True
+    CORS_ALLOW_METHODS = (
+        "GET",
+        "PATCH",
+        "POST",
+    )

tdrs-frontend/nginx/cloud.gov/locations.conf

@@ -16,9 +16,16 @@ location ~ ^/(v1|admin|static/admin|swagger|redocs) {
         proxy_buffer_size     4k;
         proxy_temp_file_write_size 64k;
 
+        limit_except GET HEAD POST { deny all;
+        }
+
         add_header Access-Control-Allow-Origin 's3-us-gov-west-1.amazonaws.com';
 }
 
+if ($request_method ~ ^(PATCH|TRACE|OPTION)$) { 
+			return 405; 
+}
+
 location = /profile {
     index index.html index.htm;
     try_files $uri $uri/ /index.html;