Merge branch 'develop' into 2884-preparser-errors-multi-record-rows-w…

…rong-length-missing-space-filled-M3-Tribal-T3

docs/Technical-Documentation/create-elastic-kibana.md

@@ -0,0 +1,10 @@
+The elastic router and Kibana app are created using ```create-es-kibana.sh``` script. This script creates one elastic router and one kibana app per cloud space.
+The Kibana app will connect to the ES router, which connects to the ES service (created by terraform) at the space level.
+
+## Running create-es-kibana.sh
+This script accepts six inputs:
+1: Deploy strategy      e.g.: rolling, initial, ...
+2: Space name           e.g.: dev, staging, prod
+
+an example of deploying kibana and elastic for dev env:
+```./create-es-kibana.sh initial dev```

scripts/create-es-kibana.sh

@@ -0,0 +1,91 @@
+#!/bin/bash
+
+##############################
+# Global Variable Decls 
+##############################
+
+# The deployment strategy you wish to employ ( rolling update or setting up a new environment)
+DEPLOY_STRATEGY=${1}
+CF_SPACE=${2}             # dev, staging, prod
+
+
+#The application name  defined via the manifest yml for the frontend
+CGAPPNAME_KIBANA="tdp-kibana"
+CGAPPNAME_PROXY="tdp-elastic-proxy"
+
+
+strip() {
+    # Usage: strip "string" "pattern"
+    printf '%s\n' "${1##$2}"
+}
+# The cloud.gov space defined via environment variable (e.g., "tanf-dev", "tanf-staging")
+env=$(strip $CF_SPACE "tanf-")
+
+# Update the Kibana and Elastic proxy names to include the environment
+CGAPPNAME_KIBANA="${CGAPPNAME_KIBANA}-${env}" # tdp-kibana-staging, tdp-kibana-prod, tdp-kibana-dev
+CGAPPNAME_PROXY="${CGAPPNAME_PROXY}-${env}"   # tdp-elastic-proxy-staging, tdp-elastic-proxy-prod, tdp-elastic-proxy-dev
+
+echo DEPLOY_STRATEGY: "$DEPLOY_STRATEGY"
+echo KIBANA_HOST: "$CGAPPNAME_KIBANA"
+echo ELASTIC_PROXY_HOST: "$CGAPPNAME_PROXY"
+echo CF_SPACE: "$CF_SPACE"
+echo env: "$env"
+
+
+update_kibana()
+{
+  cd ../tdrs-backend || exit
+
+  # Run template evaluation on manifest
+  # 2814: need to update this and set it to env instaead of app name
+  yq eval -i ".applications[0].services[0] = \"es-${env}\""  manifest.proxy.yml
+  yq eval -i ".applications[0].env.CGAPPNAME_PROXY = \"${CGAPPNAME_PROXY}\""  manifest.kibana.yml
+
+  if [ "$1" = "rolling" ] ; then
+      # Do a zero downtime deploy.  This requires enough memory for
+      # two apps to exist in the org/space at one time.
+      cf push "$CGAPPNAME_PROXY" --no-route -f manifest.proxy.yml -t 180 --strategy rolling || exit 1
+      cf push "$CGAPPNAME_KIBANA" --no-route -f manifest.kibana.yml -t 180 --strategy rolling || exit 1
+  else
+      cf push "$CGAPPNAME_PROXY" --no-route -f manifest.proxy.yml -t 180
+      cf push "$CGAPPNAME_KIBANA" --no-route -f manifest.kibana.yml -t 180
+  fi
+
+  cf map-route "$CGAPPNAME_PROXY" apps.internal --hostname "$CGAPPNAME_PROXY"
+  cf map-route "$CGAPPNAME_KIBANA" apps.internal --hostname "$CGAPPNAME_KIBANA"
+
+  # Add network policy allowing Kibana to talk to the proxy and to allow the backend to talk to Kibana
+  cf add-network-policy "$CGAPPNAME_KIBANA" "$CGAPPNAME_PROXY" --protocol tcp --port 8080
+  cd ..
+}
+
+##############################
+# Main script body
+##############################
+
+echo "Deploying Kibana and Elastic Proxy to $CF_SPACE"
+echo "Deploy strategy: $DEPLOY_STRATEGY"
+
+if [ "$DEPLOY_STRATEGY" = "rolling" ] ; then
+    # Perform a rolling update for the backend and frontend deployments if
+    # specified, otherwise perform a normal deployment
+    update_kibana 'rolling'
+elif [ "$DEPLOY_STRATEGY" = "bind" ] ; then
+    # Bind the services the application depends on and restage the app.
+    echo "Deploying Kibana and Elastic Proxy to $CF_SPACE"
+elif [ "$DEPLOY_STRATEGY" = "initial" ]; then
+    # There is no app with this name, and the services need to be bound to it
+    # for it to work. the app will fail to start once, have the services bind,
+    # and then get restaged.
+    update_kibana
+elif [ "$DEPLOY_STRATEGY" = "rebuild" ]; then
+    # You want to redeploy the instance under the same name
+    # Delete the existing app (with out deleting the services)
+    # and perform the initial deployment strategy.
+    cf delete "$CGAPPNAME_KIBANA" -r -f
+    cf delete "$CGAPPNAME_PROXY" -r -f
+    update_kibana
+else
+    # No changes to deployment config, just deploy the changes and restart
+    update_kibana
+fi

scripts/deploy-backend.sh

@@ -23,8 +23,8 @@ env=$(strip $CF_SPACE "tanf-")
 backend_app_name=$(echo $CGAPPNAME_BACKEND | cut -d"-" -f3)
 
 # Update the Kibana and Elastic proxy names to include the environment
-CGAPPNAME_KIBANA="${CGAPPNAME_KIBANA}-${backend_app_name}"
-CGAPPNAME_PROXY="${CGAPPNAME_PROXY}-${backend_app_name}"
+CGAPPNAME_KIBANA="${CGAPPNAME_KIBANA}-${env}"
+CGAPPNAME_PROXY="${CGAPPNAME_PROXY}-${env}"
 
 echo DEPLOY_STRATEGY: "$DEPLOY_STRATEGY"
 echo BACKEND_HOST: "$CGAPPNAME_BACKEND"
@@ -96,33 +96,11 @@ generate_jwt_cert()
 }
 
 update_kibana()
-{
-  cd tdrs-backend || exit
-
-  # Run template evaluation on manifest
-  yq eval -i ".applications[0].services[0] = \"es-${backend_app_name}\""  manifest.proxy.yml
-  yq eval -i ".applications[0].env.CGAPPNAME_PROXY = \"${CGAPPNAME_PROXY}\""  manifest.kibana.yml
-
-  if [ "$1" = "rolling" ] ; then
-        # Do a zero downtime deploy.  This requires enough memory for
-        # two apps to exist in the org/space at one time.
-        cf push "$CGAPPNAME_PROXY" --no-route -f manifest.proxy.yml -t 180 --strategy rolling || exit 1
-        cf push "$CGAPPNAME_KIBANA" --no-route -f manifest.kibana.yml -t 180 --strategy rolling || exit 1
-    else
-        cf push "$CGAPPNAME_PROXY" --no-route -f manifest.proxy.yml -t 180
-        cf push "$CGAPPNAME_KIBANA" --no-route -f manifest.kibana.yml -t 180
-    fi
-
-    cf map-route "$CGAPPNAME_PROXY" apps.internal --hostname "$CGAPPNAME_PROXY"
-    cf map-route "$CGAPPNAME_KIBANA" apps.internal --hostname "$CGAPPNAME_KIBANA"
-
-    # Add network policy allowing Kibana to talk to the proxy and to allow the backend to talk to Kibana
-    cf add-network-policy "$CGAPPNAME_KIBANA" "$CGAPPNAME_PROXY" --protocol tcp --port 8080
-    cf add-network-policy "$CGAPPNAME_BACKEND" "$CGAPPNAME_KIBANA" --protocol tcp --port 5601
-    cf add-network-policy "$CGAPPNAME_FRONTEND" "$CGAPPNAME_KIBANA" --protocol tcp --port 5601
-    cf add-network-policy "$CGAPPNAME_KIBANA" "$CGAPPNAME_FRONTEND" --protocol tcp --port 80
-
-    cd ..
+{  
+  # Add network policy allowing Kibana to talk to the proxy and to allow the backend to talk to Kibana
+  cf add-network-policy "$CGAPPNAME_BACKEND" "$CGAPPNAME_KIBANA" --protocol tcp --port 5601
+  cf add-network-policy "$CGAPPNAME_FRONTEND" "$CGAPPNAME_KIBANA" --protocol tcp --port 5601
+  cf add-network-policy "$CGAPPNAME_KIBANA" "$CGAPPNAME_FRONTEND" --protocol tcp --port 80
 }
 
 update_backend()
@@ -183,8 +161,8 @@ bind_backend_to_services() {
     cf bind-service "$CGAPPNAME_BACKEND" "tdp-datafiles-${env}"
     cf bind-service "$CGAPPNAME_BACKEND" "tdp-db-${env}"
 
-    # The below command is different because they cannot be shared like the 3 above services
-    cf bind-service "$CGAPPNAME_BACKEND" "es-${backend_app_name}"
+    # Setting up the ElasticSearch service
+    cf bind-service "$CGAPPNAME_BACKEND" "es-${env}"
 
     set_cf_envs
 
@@ -265,8 +243,6 @@ elif [ "$DEPLOY_STRATEGY" = "rebuild" ]; then
     # Delete the existing app (with out deleting the services)
     # and perform the initial deployment strategy.
     cf delete "$CGAPPNAME_BACKEND" -r -f
-    cf delete "$CGAPPNAME_KIBANA" -r -f
-    cf delete "$CGAPPNAME_PROXY" -r -f
     update_backend
     update_kibana
     bind_backend_to_services

scripts/deploy-frontend.sh

@@ -12,10 +12,10 @@ CGAPPNAME_KIBANA=${4}
 CF_SPACE=${5}
 ENVIRONMENT=${6}
 
-backend_app_name=$(echo $CGHOSTNAME_BACKEND | cut -d"-" -f3)
+env=${CF_SPACE#"tanf-"}
 
 # Update the Kibana name to include the environment
-KIBANA_BASE_URL="${CGAPPNAME_KIBANA}-${backend_app_name}.apps.internal"
+KIBANA_BASE_URL="${CGAPPNAME_KIBANA}-${env}.apps.internal"
 
 update_frontend()
 {

tdrs-backend/docker-compose.yml

@@ -3,7 +3,7 @@ version: "3.4"
 
 services:
   zaproxy:
-    image: owasp/zap2docker-stable:2.13.0
+    image: softwaresecurityproject/zap-stable:2.13.0
     command: sleep 3600
     depends_on:
       - web
@@ -101,6 +101,7 @@ services:
       - CYPRESS_TOKEN
       - DJANGO_DEBUG
       - SENDGRID_API_KEY
+      - GENERATE_TRAILER_ERRORS=True
       - BYPASS_KIBANA_AUTH
     volumes:
       - .:/tdpapp

tdrs-backend/tdpservice/data_files/test/test_api.py

@@ -32,7 +32,11 @@ def user(self):
     @pytest.fixture
     def test_datafile(self, stt_user, stt):
         """Fixture for small_incorrect_file_cross_validator."""
-        return util.create_test_datafile('small_incorrect_file_cross_validator.txt', stt_user, stt)
+        test_datafile = util.create_test_datafile('small_incorrect_file_cross_validator.txt', stt_user, stt)
+        test_datafile.year = 2021
+        test_datafile.quarter = 'Q1'
+        test_datafile.save()
+        return test_datafile
 
     @pytest.fixture
     def test_ssp_datafile(self, stt_user, stt):
@@ -99,7 +103,7 @@ def assert_error_report_tanf_file_content_matches_with_friendly_names(response):
         assert ws.cell(row=1, column=1).value == "Error reporting in TDP is still in development.We'll" \
             + " be in touch when it's ready to use!For now please refer to the reports you receive via email"
         assert ws.cell(row=4, column=COL_ERROR_MESSAGE).value == "if cash amount :873 validator1 passed" \
-            + " then number of months 0 is not larger than 0."
+            + " then number of months T1: 0 is not larger than 0."
 
     @staticmethod
     def assert_error_report_ssp_file_content_matches_with_friendly_names(response):
@@ -110,7 +114,8 @@ def assert_error_report_ssp_file_content_matches_with_friendly_names(response):
 
         assert ws.cell(row=1, column=1).value == "Error reporting in TDP is still in development.We'll" \
             + " be in touch when it's ready to use!For now please refer to the reports you receive via email"
-        assert ws.cell(row=4, column=COL_ERROR_MESSAGE).value == "Trailer length is 15 but must be 23 characters."
+        assert ws.cell(row=4, column=COL_ERROR_MESSAGE).value == "TRAILER record length is 15 characters " + \
+            "but must be 23."
 
     @staticmethod
     def assert_error_report_file_content_matches_without_friendly_names(response):
@@ -129,8 +134,8 @@ def assert_error_report_file_content_matches_without_friendly_names(response):
 
         assert ws.cell(row=1, column=1).value == "Error reporting in TDP is still in development.We'll" \
             + " be in touch when it's ready to use!For now please refer to the reports you receive via email"
-        assert ws.cell(row=4, column=COL_ERROR_MESSAGE).value == "if CASH_AMOUNT :873 validator1 passed" \
-            + " then NBR_MONTHS 0 is not larger than 0."
+        assert ws.cell(row=4, column=COL_ERROR_MESSAGE).value == ("if CASH_AMOUNT :873 validator1 passed then "
+                                                                  "NBR_MONTHS T1: 0 is not larger than 0.")
 
     @staticmethod
     def assert_data_file_exists(data_file_data, version, user):

tdrs-backend/tdpservice/parsers/parse.py

@@ -1,6 +1,7 @@
 """Convert raw uploaded Datafile into a parsed model, and accumulate/return any errors."""
 
 
+from django.conf import settings
 from django.db import DatabaseError
 from django.contrib.admin.models import LogEntry, ADDITION
 from django.contrib.contenttypes.models import ContentType
@@ -103,8 +104,22 @@ def bulk_create_records(unsaved_records, line_number, header_count, datafile, df
             for document, records in unsaved_records.items():
                 num_expected_db_records += len(records)
                 created_objs = document.Django.model.objects.bulk_create(records)
-                num_elastic_records_created += document.update(created_objs)[0]
                 num_db_records_created += len(created_objs)
+
+                try:
+                    num_elastic_records_created += document.update(created_objs)[0]
+                except BulkIndexError as e:
+                    logger.error(f"Encountered error while indexing datafile documents: {e}")
+                    LogEntry.objects.log_action(
+                        user_id=datafile.user.pk,
+                        content_type_id=ContentType.objects.get_for_model(DataFile).pk,
+                        object_id=datafile,
+                        object_repr=f"Datafile id: {datafile.pk}; year: {datafile.year}, quarter: {datafile.quarter}",
+                        action_flag=ADDITION,
+                        change_message=f"Encountered error while indexing datafile documents: {e}",
+                    )
+                    continue
+
             dfs.total_number_of_records_created += num_db_records_created
             if num_db_records_created != num_expected_db_records:
                 logger.error(f"Bulk Django record creation only created {num_db_records_created}/" +
@@ -114,22 +129,10 @@ def bulk_create_records(unsaved_records, line_number, header_count, datafile, df
                              f"{num_expected_db_records}!")
             else:
                 logger.info(f"Created {num_db_records_created}/{num_expected_db_records} records.")
-            return num_db_records_created == num_expected_db_records and \
-                num_elastic_records_created == num_expected_db_records, {}
+            return num_db_records_created == num_expected_db_records, {}
         except DatabaseError as e:
             logger.error(f"Encountered error while creating datafile records: {e}")
             return False, unsaved_records
-        except BulkIndexError as e:
-            logger.error(f"Encountered error while indexing datafile documents: {e}")
-            LogEntry.objects.log_action(
-                user_id=datafile.user.pk,
-                content_type_id=ContentType.objects.get_for_model(DataFile).pk,
-                object_id=datafile,
-                object_repr=f"Datafile id: {datafile.pk}; year: {datafile.year}, quarter: {datafile.quarter}",
-                action_flag=ADDITION,
-                change_message=f"Encountered error while indexing datafile documents: {e}",
-            )
-            return False, unsaved_records
     return True, unsaved_records
 
 def bulk_create_errors(unsaved_parser_errors, num_errors, batch_size=5000, flush=False):
@@ -174,6 +177,14 @@ def rollback_parser_errors(datafile):
     num_deleted, models = ParserError.objects.filter(file=datafile).delete()
     logger.debug(f"Deleted {num_deleted} {ParserError}.")
 
+def generate_trailer_errors(trailer_errors, errors, unsaved_parser_errors, num_errors):
+    """Generate trailer errors if we care to see them."""
+    if settings.GENERATE_TRAILER_ERRORS:
+        errors['trailer'] = trailer_errors
+        unsaved_parser_errors.update({"trailer": trailer_errors})
+        num_errors += len(trailer_errors)
+    return errors, unsaved_parser_errors, num_errors
+
 def create_no_records_created_pre_check_error(datafile, dfs):
     """Generate a precheck error if no records were created."""
     errors = {}
@@ -224,9 +235,10 @@ def parse_datafile_lines(datafile, dfs, program_type, section, is_encrypted):
         if trailer_errors is not None:
             logger.debug(f"{len(trailer_errors)} trailer error(s) detected for file " +
                          f"'{datafile.original_filename}' on line {line_number}.")
-            errors['trailer'] = trailer_errors
-            unsaved_parser_errors.update({"trailer": trailer_errors})
-            num_errors += len(trailer_errors)
+            errors, unsaved_parser_errors, num_errors = generate_trailer_errors(trailer_errors,
+                                                                                errors,
+                                                                                unsaved_parser_errors,
+                                                                                num_errors)
 
         generate_error = util.make_generate_parser_error(datafile, line_number)
 

tdrs-backend/tdpservice/parsers/row_schema.py

@@ -11,7 +11,8 @@ class RowSchema:
 
     def __init__(
             self,
-            document,
+            record_type="T1",
+            document=None,
             preparsing_validators=[],
             postparsing_validators=[],
             fields=[],
@@ -22,6 +23,7 @@ def __init__(
         self.postparsing_validators = postparsing_validators
         self.fields = fields
         self.quiet_preparser_errors = quiet_preparser_errors
+        self.record_type = record_type
         self.datafile = None
 
     def _add_field(self, item, name, length, start, end, type):
@@ -78,7 +80,7 @@ def run_preparsing_validators(self, line, generate_error):
         errors = []
 
         for validator in self.preparsing_validators:
-            validator_is_valid, validator_error = validator(line, self)
+            validator_is_valid, validator_error = validator(line, self, "record type", "0")
             is_valid = False if not validator_is_valid else is_valid
 
             if validator_error and not self.quiet_preparser_errors:
@@ -125,7 +127,7 @@ def run_field_validators(self, instance, generate_error):
             should_validate = not field.required and not is_empty
             if (field.required and not is_empty) or should_validate:
                 for validator in field.validators:
-                    validator_is_valid, validator_error = validator(value)
+                    validator_is_valid, validator_error = validator(value, self, field.friendly_name, field.item)
                     is_valid = False if not validator_is_valid else is_valid
                     if validator_error:
                         errors.append(
@@ -157,7 +159,7 @@ def run_postparsing_validators(self, instance, generate_error):
         errors = []
 
         for validator in self.postparsing_validators:
-            validator_is_valid, validator_error, field_names = validator(instance)
+            validator_is_valid, validator_error, field_names = validator(instance, self)
             is_valid = False if not validator_is_valid else is_valid
             if validator_error:
                 # get field from field name