diff --git a/.gitignore b/.gitignore index a230915c5..f8701dc6d 100644 --- a/.gitignore +++ b/.gitignore @@ -22,6 +22,9 @@ tdrs-backend/coverage.xml tdrs-backend/htmlcov/* tdrs-backend/.env tdrs-backend/.env.production +tdrs-backend/ADS* +tdrs-backend/temp_key_file +tdrs-backend/test *.pyc /backend/db.sqlite3 .DS_Store diff --git a/tdrs-backend/docker-compose.local.yml b/tdrs-backend/docker-compose.local.yml index 190739e05..a1411421a 100644 --- a/tdrs-backend/docker-compose.local.yml +++ b/tdrs-backend/docker-compose.local.yml @@ -58,7 +58,7 @@ services: - ACFTITAN_HOST - ACFTITAN_KEY - ACFTITAN_USERNAME - - REDIS_URI + - REDIS_URI=redis://redis-server:6379 - REDIS_SERVER_LOCAL=TRUE - ACFTITAN_SFTP_PYTEST volumes: diff --git a/tdrs-backend/docker-compose.yml b/tdrs-backend/docker-compose.yml index bd78d098f..6ef8f1603 100644 --- a/tdrs-backend/docker-compose.yml +++ b/tdrs-backend/docker-compose.yml @@ -70,7 +70,7 @@ services: - ACFTITAN_HOST - ACFTITAN_KEY - ACFTITAN_USERNAME - - REDIS_URI + - REDIS_URI=redis://redis-server:6379 - REDIS_SERVER_LOCAL=TRUE - ACFTITAN_SFTP_PYTEST volumes: diff --git a/tdrs-backend/manifest.buildpack.yml b/tdrs-backend/manifest.buildpack.yml index 462efd07e..cdb458afd 100755 --- a/tdrs-backend/manifest.buildpack.yml +++ b/tdrs-backend/manifest.buildpack.yml @@ -4,6 +4,8 @@ applications: memory: 512M instances: 1 disk_quota: 2G + env: + REDIS_URI: redis://localhost:6379 buildpacks: - https://github.com/cloudfoundry/apt-buildpack - https://github.com/cloudfoundry/python-buildpack.git#v1.7.55 diff --git a/tdrs-backend/tdpservice/conftest.py b/tdrs-backend/tdpservice/conftest.py index cde08baf5..2b9902616 100644 --- a/tdrs-backend/tdpservice/conftest.py +++ b/tdrs-backend/tdpservice/conftest.py @@ -218,7 +218,8 @@ def base_data_file_data(fake_file_name, data_analyst): "user": str(data_analyst.id), "quarter": "Q1", "year": 2020, - "stt": int(data_analyst.stt.id) + "stt": int(data_analyst.stt.id), + "ssp": False, } diff --git a/tdrs-backend/tdpservice/data_files/models.py b/tdrs-backend/tdpservice/data_files/models.py index 1e7534bac..35d6fdfb3 100644 --- a/tdrs-backend/tdpservice/data_files/models.py +++ b/tdrs-backend/tdpservice/data_files/models.py @@ -149,12 +149,7 @@ class Meta: @property def filename(self): """Return the correct filename for this data file.""" - if str(self.stt.type).lower() == 'tribe': - return self.stt.filenames.get( - ('Tribal ' if 'Tribal' not in self.section else '') + self.section, - None) - else: - return self.stt.filenames.get(self.section, None) + return self.stt.filenames.get(self.section, None) @property def fiscal_year(self): @@ -165,11 +160,11 @@ def fiscal_year(self): case DataFile.Quarter.Q1: quarter_month_str = "(Oct - Dec)" case DataFile.Quarter.Q2: - quarter_month_str = "(Jul - Sep)" + quarter_month_str = "(Jan - Mar)" case DataFile.Quarter.Q3: quarter_month_str = "(Apr - Jun)" case DataFile.Quarter.Q4: - quarter_month_str = "(Jan - Mar)" + quarter_month_str = "(Jul - Sep)" return f"{self.year} - {self.quarter} {quarter_month_str}" diff --git a/tdrs-backend/tdpservice/data_files/serializers.py b/tdrs-backend/tdpservice/data_files/serializers.py index 26331832d..1f9a6f7f3 100644 --- a/tdrs-backend/tdpservice/data_files/serializers.py +++ b/tdrs-backend/tdpservice/data_files/serializers.py @@ -19,6 +19,7 @@ class DataFileSerializer(serializers.ModelSerializer): file = serializers.FileField(write_only=True) stt = serializers.PrimaryKeyRelatedField(queryset=STT.objects.all()) user = serializers.PrimaryKeyRelatedField(queryset=User.objects.all()) + ssp = serializers.BooleanField(write_only=True) class Meta: """Metadata.""" @@ -35,13 +36,18 @@ class Meta: "year", "quarter", "section", - "created_at" + "created_at", + "ssp", ] def create(self, validated_data): """Create a new entry with a new version number.""" + ssp = validated_data.pop('ssp') + if ssp: + validated_data['section'] = 'SSP ' + validated_data['section'] + if validated_data.get('stt').type == 'tribe': + validated_data['section'] = 'Tribal ' + validated_data['section'] data_file = DataFile.create_new_version(validated_data) - # Determine the matching ClamAVFileScan for this DataFile. av_scan = ClamAVFileScan.objects.filter( file_name=data_file.original_filename, diff --git a/tdrs-backend/tdpservice/data_files/test/test_api.py b/tdrs-backend/tdpservice/data_files/test/test_api.py index 9a05f5464..36099bb56 100644 --- a/tdrs-backend/tdpservice/data_files/test/test_api.py +++ b/tdrs-backend/tdpservice/data_files/test/test_api.py @@ -1,5 +1,6 @@ """Tests for DataFiles Application.""" from unittest.mock import ANY, patch + from rest_framework import status import pytest @@ -206,6 +207,35 @@ def test_download_data_file_file_rejected_for_other_stt( assert response.status_code == status.HTTP_403_FORBIDDEN + def test_data_files_data_upload_ssp( + self, api_client, data_file_data, + ): + """Test that when Data Analysts upload file with ssp true the section name is updated.""" + data_file_data['ssp'] = True + + response = self.post_data_file_file(api_client, data_file_data) + assert response.data['section'] == 'SSP Active Case Data' + + def test_data_file_data_upload_tribe( + self, api_client, data_file_data, stt + ): + """Test that when we upload a file for Tribe the section name is updated.""" + stt.type = 'tribe' + stt.save() + response = self.post_data_file_file(api_client, data_file_data) + assert 'Tribal Active Case Data' == response.data['section'] + stt.type = '' + stt.save() + + def test_data_files_data_upload_tanf( + self, api_client, data_file_data, + ): + """Test that when Data Analysts upload file with ssp true the section name is updated.""" + data_file_data['ssp'] = False + + response = self.post_data_file_file(api_client, data_file_data) + assert response.data['section'] == 'Active Case Data' + def test_data_analyst_gets_email_when_user_uploads_report_for_their_stt( self, api_client, data_file_data, user ): diff --git a/tdrs-backend/tdpservice/data_files/test/test_models.py b/tdrs-backend/tdpservice/data_files/test/test_models.py index 42e1acbd4..2783759c8 100644 --- a/tdrs-backend/tdpservice/data_files/test/test_models.py +++ b/tdrs-backend/tdpservice/data_files/test/test_models.py @@ -81,7 +81,4 @@ def test_data_files_filename_is_expected(user): "user": user, "stt": stt }) - if stt.type == 'tribe': - assert new_data_file.filename == stt.filenames['Tribal ' if 'Tribal' not in section else '' + section] - else: - assert new_data_file.filename == stt.filenames[section] + assert new_data_file.filename == stt.filenames[section] diff --git a/tdrs-backend/tdpservice/data_files/views.py b/tdrs-backend/tdpservice/data_files/views.py index b10521d7e..8580ba8dd 100644 --- a/tdrs-backend/tdpservice/data_files/views.py +++ b/tdrs-backend/tdpservice/data_files/views.py @@ -93,10 +93,22 @@ def create(self, request, *args, **kwargs): return response + def get_queryset(self): + """Apply custom queryset filters.""" + queryset = super().get_queryset() + + if self.request.query_params.get('file_type') == 'ssp-moe': + queryset = queryset.filter(section__contains='SSP') + else: + queryset = queryset.exclude(section__contains='SSP') + + return queryset + def filter_queryset(self, queryset): """Only apply filters to the list action.""" if self.action != 'list': self.filterset_class = None + return super().filter_queryset(queryset) def get_serializer_context(self): diff --git a/tdrs-backend/tdpservice/stts/management/commands/populate_stts.py b/tdrs-backend/tdpservice/stts/management/commands/populate_stts.py index d4a82d960..5c7713468 100644 --- a/tdrs-backend/tdpservice/stts/management/commands/populate_stts.py +++ b/tdrs-backend/tdpservice/stts/management/commands/populate_stts.py @@ -21,54 +21,31 @@ def _populate_regions(): Region.objects.get_or_create(id=row["Id"]) Region.objects.get_or_create(id=1000) - -def _get_states(): - with open(DATA_DIR / "states.csv") as csvfile: +def _load_csv(filename, entity): + with open(DATA_DIR / filename) as csvfile: reader = csv.DictReader(csvfile) - return [ - STT( - code=row["Code"], - name=row["Name"], - region_id=row["Region"], - type=STT.EntityType.STATE, - filenames=json.loads(row["filenames"].replace('\'', '"')), - stt_code=row["STT_CODE"], - ) - for row in reader - ] - - -def _get_territories(): - with open(DATA_DIR / "territories.csv") as csvfile: - reader = csv.DictReader(csvfile) - return [ - STT( - code=row["Code"], - name=row["Name"], - region_id=row["Region"], - type=STT.EntityType.TERRITORY, - filenames=json.loads(row["filenames"].replace('\'', '"')), - stt_code=row["STT_CODE"], - ) - for row in reader - ] - -def _populate_tribes(): - with open(DATA_DIR / "tribes.csv") as csvfile: - reader = csv.DictReader(csvfile) - stts = [ - STT( - name=row["Name"], - region_id=row["Region"], - state=STT.objects.get(code=row["Code"]), - type=STT.EntityType.TRIBE, - filenames=json.loads(row["filenames"].replace('\'', '"')), - stt_code=row["STT_CODE"], - ) - for row in reader - ] - STT.objects.bulk_create(stts, ignore_conflicts=True) + for row in reader: + stt, stt_created = STT.objects.get_or_create(name=row["Name"]) + if stt_created: # These lines are spammy, should remove before merge + logger.debug("Created new entry for " + row["Name"]) + else: + logger.debug("Found STT " + row["Name"] + ", will sync with data csv.") + + stt.code = row["Code"] + stt.region_id = row["Region"] + if filename == "tribes.csv": + stt.state = STT.objects.get(code=row["Code"], type=STT.EntityType.STATE) + + stt.type = entity + stt.filenames = json.loads(row["filenames"].replace('\'', '"')) + stt.stt_code = row["STT_CODE"] + stt.ssp = row["SSP"] + # TODO: Was seeing lots of references to STT.objects.filter(pk=... + # We could probably one-line this but we'd miss .save() signals + # https://stackoverflow.com/questions/41744096/ + # TODO: we should finish the last columns from the csvs: Sample, SSN_Encrypted + stt.save() class Command(BaseCommand): @@ -79,8 +56,14 @@ class Command(BaseCommand): def handle(self, *args, **options): """Populate the various regions, states, territories, and tribes.""" _populate_regions() - stts = _get_states() - stts.extend(_get_territories()) - STT.objects.bulk_create(stts, ignore_conflicts=True) - _populate_tribes() + + stt_map = [ + ("states.csv", STT.EntityType.STATE), + ("territories.csv", STT.EntityType.TERRITORY), + ("tribes.csv", STT.EntityType.TRIBE) + ] + + for datafile, entity in stt_map: + _load_csv(datafile, entity) + logger.info("STT import executed by Admin at %s", timezone.now()) diff --git a/tdrs-backend/tdpservice/stts/migrations/0006_alter_stt_filenames.py b/tdrs-backend/tdpservice/stts/migrations/0006_alter_stt_filenames.py index af1dbe26e..6639d3039 100644 --- a/tdrs-backend/tdpservice/stts/migrations/0006_alter_stt_filenames.py +++ b/tdrs-backend/tdpservice/stts/migrations/0006_alter_stt_filenames.py @@ -1,49 +1,9 @@ # Generated by Django 3.2.13 on 2022-06-08 14:43 -import csv -import json -from pathlib import Path - from django.core.management import call_command from django.db import migrations, models -def add_filenames(apps, schema_editor): - call_command("populate_stts") - data_dir = Path(__file__).resolve().parent.parent /"management" / "commands" / "data" - STT = apps.get_model('stts','STT') - - fieldnames = ["Code", "Name", "Region", "STT_CODE", "Sample", "SSP", "SSN_Encrypted", "filenames"] - with open(data_dir / "states.csv", "r") as csvfile: - reader = csv.DictReader(csvfile) - rows=[] - for row in reader: - filenames = row["filenames"] = row["filenames"].replace('\'', '"') - rows.append(row) - state = STT.objects.get(name=row["Name"]) - state.filenames = json.loads(filenames) - state.save() - - with open(data_dir / "tribes.csv", "r") as csvfile: - reader = csv.DictReader(csvfile) - rows=[] - for row in reader: - filenames = row["filenames"] = row["filenames"].replace('\'', '"') - rows.append(row) - tribe = STT.objects.get(name=row["Name"]) - tribe.filenames = json.loads(filenames) - tribe.save() - - with open(data_dir / "territories.csv", "r") as csvfile: - reader = csv.DictReader(csvfile) - rows=[] - for row in reader: - filenames = row["filenames"] = row["filenames"].replace('\'', '"') - rows.append(row) - territory = STT.objects.get(name=row["Name"]) - territory.filenames = json.loads(filenames) - territory.save() - class Migration(migrations.Migration): dependencies = [ @@ -56,6 +16,4 @@ class Migration(migrations.Migration): name='filenames', field=models.JSONField(blank=True, max_length=512, null=True), ), - migrations.RunPython(add_filenames) ] - diff --git a/tdrs-backend/tdpservice/stts/migrations/0007_stt_ssp.py b/tdrs-backend/tdpservice/stts/migrations/0007_stt_ssp.py new file mode 100644 index 000000000..57002735b --- /dev/null +++ b/tdrs-backend/tdpservice/stts/migrations/0007_stt_ssp.py @@ -0,0 +1,18 @@ +# Generated by Django 3.2.15 on 2022-10-28 20:00 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('stts', '0006_alter_stt_filenames'), + ] + + operations = [ + migrations.AddField( + model_name='stt', + name='ssp', + field=models.BooleanField(default=False, null=True), + ), + ] diff --git a/tdrs-backend/tdpservice/stts/models.py b/tdrs-backend/tdpservice/stts/models.py index f35840f6e..1f950fdde 100644 --- a/tdrs-backend/tdpservice/stts/models.py +++ b/tdrs-backend/tdpservice/stts/models.py @@ -37,6 +37,7 @@ class EntityType(models.TextChoices): stt_code = models.PositiveIntegerField(blank=True, null=True) # Tribes have a state, which we need to store. state = models.ForeignKey("self", on_delete=models.CASCADE, blank=True, null=True) + ssp = models.BooleanField(default=False, null=True) class Meta: """Metadata.""" diff --git a/tdrs-backend/tdpservice/stts/serializers.py b/tdrs-backend/tdpservice/stts/serializers.py index 06a37fe6f..41c8ebceb 100644 --- a/tdrs-backend/tdpservice/stts/serializers.py +++ b/tdrs-backend/tdpservice/stts/serializers.py @@ -14,7 +14,7 @@ class Meta: """Metadata.""" model = STT - fields = ["id", "type", "code", "name", "region"] + fields = ["id", "type", "code", "name", "region", "ssp"] def get_code(self, obj): """Return the state code.""" diff --git a/tdrs-backend/tdpservice/users/test/test_api/test_set_profile.py b/tdrs-backend/tdpservice/users/test/test_api/test_set_profile.py index 39ef4c02f..32d01b60a 100644 --- a/tdrs-backend/tdpservice/users/test/test_api/test_set_profile.py +++ b/tdrs-backend/tdpservice/users/test/test_api/test_set_profile.py @@ -38,30 +38,6 @@ def test_set_profile_data(api_client, user): assert user.last_name == "Bloggs" -@pytest.mark.django_db -def test_user_can_request_access(api_client, user, stt): - """Test `access_request` endpoint updates the `account_approval_status` field to `Access Request`.""" - api_client.login(username=user.username, password="test_password") - - response = api_client.patch( - "/v1/users/request_access/", - {"first_name": "Joe", "last_name": "Bloggs", "stt": stt.id, "email": user.username}, - format="json", - ) - assert response.data == { - "id": str(user.id), - "email": user.username, - "first_name": "Joe", - "last_name": "Bloggs", - "access_request": False, # old value no longer touched - "account_approval_status": "Access request", # new value updated - "stt": {"id": stt.id, "type": stt.type, "code": stt.code, "name": stt.name, "region": stt.region.id}, - "region": None, - "roles": [], - } - - # TODO: In the future, we would like to test that users can be activated and their roles are correctly assigned. - @pytest.mark.django_db def test_cannot_set_account_approval_status_through_api(api_client, user): """Test that the `account_approval_status` field cannot be updated through an api call to `set_profile`.""" diff --git a/tdrs-frontend/src/actions/reports.js b/tdrs-frontend/src/actions/reports.js index 9d4e0cbab..cd9885298 100644 --- a/tdrs-frontend/src/actions/reports.js +++ b/tdrs-frontend/src/actions/reports.js @@ -43,14 +43,14 @@ export const clearError = if the download button should be present. */ export const getAvailableFileList = - ({ quarter = 'Q1', stt, year }) => + ({ quarter = 'Q1', stt, year, file_type }) => async (dispatch) => { dispatch({ type: FETCH_FILE_LIST, }) try { const response = await axios.get( - `${BACKEND_URL}/data_files/?year=${year}&quarter=${quarter}&stt=${stt.id}`, + `${BACKEND_URL}/data_files/?year=${year}&quarter=${quarter}&stt=${stt.id}&file_type=${file_type}`, { responseType: 'json', } @@ -68,6 +68,7 @@ export const getAvailableFileList = error, year, quarter, + file_type, }, }) } @@ -148,6 +149,7 @@ export const submit = uploadedFiles, user, year, + ssp, }) => async (dispatch) => { const submissionRequests = uploadedFiles.map((file) => { @@ -161,6 +163,7 @@ export const submit = year, stt, quarter, + ssp, } for (const [key, value] of Object.entries(dataFile)) { formData.append(key, value) @@ -219,6 +222,7 @@ export const submit = export const SET_SELECTED_STT = 'SET_SELECTED_STT' export const SET_SELECTED_YEAR = 'SET_SELECTED_YEAR' export const SET_SELECTED_QUARTER = 'SET_SELECTED_QUARTER' +export const SET_FILE_TYPE = 'SET_FILE_TYPE' export const setStt = (stt) => (dispatch) => { dispatch({ type: SET_SELECTED_STT, payload: { stt } }) @@ -230,3 +234,7 @@ export const setYear = (year) => (dispatch) => { export const setQuarter = (quarter) => (dispatch) => { dispatch({ type: SET_SELECTED_QUARTER, payload: { quarter } }) } + +export const setFileType = (fileType) => (dispatch) => { + dispatch({ type: SET_FILE_TYPE, payload: { fileType } }) +} diff --git a/tdrs-frontend/src/components/FileUpload/FileUpload.jsx b/tdrs-frontend/src/components/FileUpload/FileUpload.jsx index 0a478103b..0911fcc68 100644 --- a/tdrs-frontend/src/components/FileUpload/FileUpload.jsx +++ b/tdrs-frontend/src/components/FileUpload/FileUpload.jsx @@ -28,10 +28,10 @@ function FileUpload({ section, setLocalAlertState }) { const [sectionNumber, sectionName] = section.split(' - ') const hasFile = files?.some( - (file) => file.section === sectionName && file.uuid + (file) => file.section.includes(sectionName) && file.uuid ) - const selectedFile = files.find((file) => sectionName === file.section) + const selectedFile = files.find((file) => file.section.includes(sectionName)) const formattedSectionName = selectedFile.section .split(' ') diff --git a/tdrs-frontend/src/components/Modal/Modal.test.js b/tdrs-frontend/src/components/Modal/Modal.test.js index 74460c75f..14b316064 100644 --- a/tdrs-frontend/src/components/Modal/Modal.test.js +++ b/tdrs-frontend/src/components/Modal/Modal.test.js @@ -62,7 +62,7 @@ describe('Modal tests', () => { describe('Acessability trap', () => { it('should focus modal header when displayed', async () => { - const { queryByText } = await setup() + await setup() expect(document.activeElement).toHaveTextContent('Test') }) diff --git a/tdrs-frontend/src/components/Reports/Reports.jsx b/tdrs-frontend/src/components/Reports/Reports.jsx index 67ba7988a..741465d02 100644 --- a/tdrs-frontend/src/components/Reports/Reports.jsx +++ b/tdrs-frontend/src/components/Reports/Reports.jsx @@ -9,6 +9,7 @@ import { setStt, setQuarter, getAvailableFileList, + setFileType, } from '../../actions/reports' import UploadReport from '../UploadReport' import STTComboBox from '../STTComboBox' @@ -60,6 +61,9 @@ function Reports() { const stt = sttList?.find((stt) => stt?.name === currentStt) + const selectedFileType = useSelector((state) => state.reports.fileType) + const [fileTypeInputValue, setFileTypeInputValue] = useState(selectedFileType) + const errorsCount = formValidation.errors const missingStt = !isOFAAdmin && !currentStt @@ -70,6 +74,7 @@ function Reports() { setQuarterInputValue(selectedQuarter || '') setYearInputValue(selectedYear || '') setSttInputValue(selectedStt || '') + setFileTypeInputValue(selectedFileType || 'tanf') } const handleSearch = () => { @@ -96,13 +101,15 @@ function Reports() { dispatch(setYear(yearInputValue)) dispatch(setQuarter(quarterInputValue)) dispatch(setStt(sttInputValue)) + dispatch(setFileType(fileTypeInputValue)) - // Retrieve the files matching the selected year and quarter. + // Retrieve the files matching the selected year, quarter, and ssp. dispatch( getAvailableFileList({ - quarter: selectedQuarter, - year: selectedYear, + quarter: quarterInputValue, + year: yearInputValue, stt, + file_type: fileTypeInputValue, }) ) @@ -227,7 +234,40 @@ function Reports() { /> )} - + {(stt?.ssp ? stt.ssp : false) && ( +