Proof of Concept TDP Based Kibana Auth

tdrs-backend/.env.example

@@ -2,6 +2,8 @@
 # Copy this file to `.env` and replace variables as needed
 # 
 
+#REACT_APP_DEVAUTH=1
+
 # ##
 # Required environment variables
 # These must be defined or the application will encounter fatal errors

tdrs-backend/Pipfile

@@ -58,7 +58,7 @@ django-elasticsearch-dsl = "==7.3"
 django-elasticsearch-dsl-drf = "==0.22.5"
 requests-aws4auth = "==1.1.2"
 cerberus = "==1.3.4"
-xlsxwriter = "==3.0.1"
+xlsxwriter = "==3.1.9"
 sendgrid = "==6.10.0"
 
 [requires]

tdrs-backend/docker-compose.yml

@@ -45,12 +45,25 @@ services:
       # Copy in the Localstack setup script to configure any buckets needed
       - ../scripts/localstack-setup.sh:/docker-entrypoint-initaws.d/localstack-setup.sh
 
+  kibana:
+    image: elastic/kibana:7.17.10
+    ports:
+      - 5601:5601
+    environment:
+      - xpack.security.encryptionKey="something_at_least_32_characters"
+      - xpack.security.session.idleTimeout="1h"
+      - xpack.security.session.lifespan="30d"
+    volumes:
+      - ./kibana.yml:/usr/share/kibana/config/kibana.yml
+    depends_on:
+      - elastic
+
   elastic:
     image: elasticsearch:7.17.6
     environment:
       - discovery.type=single-node
-      - xpack.security.enabled=false
       - logger.discovery.level=debug
+      - xpack.security.enabled=false
     ports:
       - 9200:9200
       - 9300:9300
@@ -88,6 +101,7 @@ services:
       - CYPRESS_TOKEN
       - DJANGO_DEBUG
       - SENDGRID_API_KEY
+      - BYPASS_KIBANA_AUTH
     volumes:
       - .:/tdpapp
     image: tdp

tdrs-backend/kibana.yml

@@ -0,0 +1,2 @@
+elasticsearch.hosts: ["http://elastic:9200"]
+server.host: kibana

tdrs-backend/tdpservice/data_files/views.py

@@ -54,13 +54,15 @@ class DataFileViewSet(ModelViewSet):
 
     def create(self, request, *args, **kwargs):
         """Override create to upload in case of successful scan."""
+        logger.debug(f"{self.__class__.__name__}: {request}")
         response = super().create(request, *args, **kwargs)
 
         # only if file is passed the virus scan and created successfully will we perform side-effects:
         # * Send to parsing
         # * Upload to ACF-TITAN
         # * Send email to user
 
+        logger.debug(f"{self.__class__.__name__}: status: {response.status_code}")
         if response.status_code == status.HTTP_201_CREATED or response.status_code == status.HTTP_200_OK:
             user = request.user
             data_file_id = response.data.get('id')
@@ -109,6 +111,7 @@ def create(self, request, *args, **kwargs):
             if len(recipients) > 0:
                 send_data_submitted_email(list(recipients), data_file, email_context, subject)
 
+        logger.debug(f"{self.__class__.__name__}: return val: {response}")
         return response
 
     def get_s3_versioning_id(self, file_name, prefix):

tdrs-backend/tdpservice/parsers/fields.py

@@ -9,9 +9,20 @@
 class Field:
     """Provides a mapping between a field name and its position."""
 
-    def __init__(self, item, name, type, startIndex, endIndex, required=True, validators=[]):
+    def __init__(
+        self,
+        item,
+        name,
+        friendly_name,
+        type,
+        startIndex,
+        endIndex,
+        required=True,
+        validators=[],
+    ):
         self.item = item
         self.name = name
+        self.friendly_name = friendly_name
         self.type = type
         self.startIndex = startIndex
         self.endIndex = endIndex
@@ -36,25 +47,34 @@ def parse_value(self, line):
             return None
 
         match self.type:
-            case 'number':
+            case "number":
                 try:
                     value = int(value)
                     return value
                 except ValueError:
                     logger.error(f"Error parsing field value: {value} to integer.")
                     return None
-            case 'string':
+            case "string":
                 return value
             case _:
                 logger.warn(f"Unknown field type: {self.type}.")
                 return None
 
+
 class TransformField(Field):
     """Represents a field that requires some transformation before serializing."""
 
-    def __init__(self, transform_func, item, name, type, startIndex, endIndex, required=True,
+    def __init__(self, transform_func, item, name, friendly_name, type, startIndex, endIndex, required=True,
                  validators=[], **kwargs):
-        super().__init__(item, name, type, startIndex, endIndex, required, validators)
+        super().__init__(
+            item=item,
+            name=name,
+            type=type,
+            friendly_name=friendly_name,
+            startIndex=startIndex,
+            endIndex=endIndex,
+            required=required,
+            validators=validators)
         self.transform_func = transform_func
         self.kwargs = kwargs
 

tdrs-backend/tdpservice/parsers/models.py

@@ -93,7 +93,6 @@ class Status(models.TextChoices):
     def get_status(self):
         """Set and return the status field based on errors and models associated with datafile."""
         errors = ParserError.objects.filter(file=self.datafile)
-        [print(error) for error in errors]
 
         # excluding row-level pre-checks and trailer pre-checks.
         precheck_errors = errors.filter(error_type=ParserErrorCategoryChoices.PRE_CHECK)\

tdrs-backend/tdpservice/parsers/parse.py

@@ -55,22 +55,27 @@ def parse_datafile(datafile):
 
     return errors
 
-
 def bulk_create_records(unsaved_records, line_number, header_count, batch_size=10000, flush=False):
     """Bulk create passed in records."""
     if (line_number % batch_size == 0 and header_count > 0) or flush:
         logger.debug("Bulk creating records.")
         try:
-            num_created = 0
-            num_expected = 0
-            for model, records in unsaved_records.items():
-                num_expected += len(records)
-                num_created += len(model.objects.bulk_create(records))
-            if num_created != num_expected:
-                logger.error(f"Bulk create only created {num_created}/{num_expected}!")
+            num_records_created = 0
+            num_expected_records = 0
+            num_documents_created = 0
+            for document, records in unsaved_records.items():
+                num_expected_records += len(records)
+                created_objs = document.Django.model.objects.bulk_create(records)
+                num_documents_created += document.update(created_objs)[0]
+                num_records_created += len(created_objs)
+            if num_records_created != num_expected_records:
+                logger.error(f"Bulk Django record creation only created {num_records_created}/{num_expected_records}!")
+            elif num_documents_created != num_expected_records:
+                logger.error(f"Bulk Elastic document creation only created {num_documents_created}/" +
+                             f"{num_expected_records}!")
             else:
-                logger.info(f"Created {num_created}/{num_expected} records.")
-            return num_created == num_expected, {}
+                logger.info(f"Created {num_records_created}/{num_expected_records} records.")
+            return num_records_created == num_expected_records and num_documents_created == num_expected_records, {}
         except DatabaseError as e:
             logger.error(f"Encountered error while creating datafile records: {e}")
             return False, unsaved_records
@@ -107,7 +112,8 @@ def evaluate_trailer(datafile, trailer_count, multiple_trailer_errors, is_last_l
 def rollback_records(unsaved_records, datafile):
     """Delete created records in the event of a failure."""
     logger.info("Rolling back created records.")
-    for model in unsaved_records:
+    for document in unsaved_records:
+        model = document.Django.model
         num_deleted, models = model.objects.filter(datafile=datafile).delete()
         logger.debug(f"Deleted {num_deleted} records of type: {model}.")
 
@@ -198,7 +204,7 @@ def parse_datafile_lines(datafile, program_type, section, is_encrypted):
             if record:
                 s = schema_manager.schemas[i]
                 record.datafile = datafile
-                unsaved_records.setdefault(s.model, []).append(record)
+                unsaved_records.setdefault(s.document, []).append(record)
 
         all_created, unsaved_records = bulk_create_records(unsaved_records, line_number, header_count,)
         unsaved_parser_errors, num_errors = bulk_create_errors(unsaved_parser_errors, num_errors)

tdrs-backend/tdpservice/parsers/row_schema.py

@@ -12,13 +12,13 @@ class RowSchema:
 
     def __init__(
             self,
-            model=dict,
+            document,
             preparsing_validators=[],
             postparsing_validators=[],
             fields=[],
-            quiet_preparser_errors=False
+            quiet_preparser_errors=False,
             ):
-        self.model = model
+        self.document = document
         self.preparsing_validators = preparsing_validators
         self.postparsing_validators = postparsing_validators
         self.fields = fields
@@ -90,7 +90,7 @@ def run_preparsing_validators(self, line, generate_error):
 
     def parse_line(self, line):
         """Create a model for the line based on the schema."""
-        record = self.model()
+        record = self.document.Django.model() if self.document is not None else dict()
 
         for field in self.fields:
             value = field.parse_value(line)
@@ -151,16 +151,18 @@ def run_postparsing_validators(self, instance, generate_error):
         errors = []
 
         for validator in self.postparsing_validators:
-            validator_is_valid, validator_error = validator(instance)
+            validator_is_valid, validator_error, field_names = validator(instance)
             is_valid = False if not validator_is_valid else is_valid
             if validator_error:
+                # get field from field name
+                fields = [self.get_field_by_name(name) for name in field_names]
                 errors.append(
                     generate_error(
                         schema=self,
                         error_category=ParserErrorCategoryChoices.VALUE_CONSISTENCY,
                         error_message=validator_error,
                         record=instance,
-                        field=None
+                        field=fields,
                     )
                 )