From 6c83c235fad251c2c4a1e9e37f84aa78ab204197 Mon Sep 17 00:00:00 2001 From: srh-sloan Date: Wed, 20 Nov 2024 11:48:41 +0000 Subject: [PATCH] Switching to ruff formatting (#374) * adding ruff config * adding ruff config * line length * reformatting for ruff --------- Co-authored-by: srh-sloan --- .pre-commit-config.yaml | 42 +++---- api/routes/application/routes.py | 111 ++++++++++-------- api/routes/queues/routes.py | 10 +- app.py | 10 +- config/envs/default.py | 6 +- config/envs/dev.py | 4 +- config/envs/development.py | 4 +- config/envs/production.py | 4 +- config/envs/test.py | 4 +- config/envs/unit_testing.py | 1 + .../cof25_eoi_key_report_mapping.py | 4 +- .../cof_eoi_key_report_mapping.py | 4 +- .../cof_key_report_mapping.py | 10 +- .../cof_r2_key_report_mapping.py | 4 +- .../cof_r3w2_key_report_mapping.py | 10 +- .../cyp_r1_key_report_mapping.py | 3 +- .../dpif_r2_key_report_mapping.py | 3 +- config/key_report_mappings/mappings.py | 1 - config/key_report_mappings/model.py | 3 +- db/migrations/versions/5bf853808dfb_.py | 1 + db/migrations/versions/d37dca03b539_.py | 2 +- db/migrations/versions/e86cbf275e45_.py | 1 + db/migrations/versions/eb61bc354c57_.py | 2 +- db/migrations/versions/f3332786bb2e_.py | 1 + db/migrations/versions/f524201314e9_.py | 1 + db/models/__init__.py | 3 +- db/models/application/applications.py | 11 +- db/models/eligibility/eligibility.py | 7 +- db/models/eligibility/eligibility_trail.py | 2 +- .../feedback/end_of_application_survey.py | 2 +- db/models/feedback/feedback.py | 7 +- db/models/forms/forms.py | 5 +- db/models/research/research.py | 2 +- db/queries/__init__.py | 38 +++--- db/queries/application/queries.py | 49 ++++---- db/queries/feedback/queries.py | 11 +- db/queries/reporting/queries.py | 8 +- db/queries/updating/queries.py | 23 ++-- db/schemas/application.py | 15 +-- db/schemas/eligibility.py | 5 +- db/schemas/end_of_application_survey.py | 4 +- db/schemas/feedback.py | 7 +- db/schemas/form.py | 6 +- external_services/aws.py | 1 + external_services/data.py | 26 ++-- external_services/http_methods.py | 15 ++- external_services/models/fund.py | 3 +- external_services/models/notification.py | 15 ++- openapi/utils.py | 1 + pyproject.toml | 23 +++- scripts/seed_db_test_data.py | 16 +-- scripts/send_application_on_closure.py | 54 ++++++--- scripts/send_application_reminder.py | 39 +++--- tasks.py | 5 +- testing.py | 2 +- tests/conftest.py | 21 ++-- tests/helpers.py | 7 +- tests/seed_data/seed_db.py | 3 +- tests/test_all_feedbacks.py | 6 +- tests/test_application_status.py | 23 ++-- tests/test_aws.py | 1 + tests/test_forms.py | 1 - tests/test_notification.py | 7 +- tests/test_queries.py | 16 +-- tests/test_reports.py | 7 +- tests/test_routes.py | 32 ++--- tests/test_seed_db.py | 12 +- tests/test_send_app_on_closure.py | 1 + 68 files changed, 441 insertions(+), 347 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index fde31675..8652d75d 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,36 +1,22 @@ repos: -- repo: https://github.com/psf/black - rev: 23.12.1 - hooks: - - id: black - language_version: python3 - args: - - --experimental-string-processing -- repo: https://github.com/PyCQA/flake8 - rev: 7.1.1 - hooks: - - id: flake8 - additional_dependencies: [Flake8-pyproject] -- repo: https://github.com/pre-commit/pre-commit-hooks + - repo: https://github.com/pre-commit/pre-commit-hooks rev: v4.6.0 hooks: - id: trailing-whitespace - id: end-of-file-fixer - id: check-ast -- repo: https://github.com/asottile/reorder-python-imports - rev: v3.14.0 + - repo: https://github.com/astral-sh/ruff-pre-commit + # Ruff version. If bumping this, please also bump requirements-dev.in + rev: v0.6.7 hooks: - - id: reorder-python-imports - name: Reorder Python imports (src, tests) - args: ["--application-directories", "src"] -- repo: https://github.com/asottile/pyupgrade - rev: v3.19.0 + # Run the linter. + - id: ruff + args: [--fix] + # Run the formatter. + - id: ruff-format + - repo: https://github.com/Yelp/detect-secrets + rev: v1.4.0 hooks: - - id: pyupgrade - args: ["--py39-plus"] -- repo: https://github.com/Yelp/detect-secrets - rev: v1.5.0 - hooks: - - id: detect-secrets - args: ['--disable-plugin', 'HexHighEntropyString'] - exclude: .env.development + - id: detect-secrets + args: ['--disable-plugin', 'HexHighEntropyString'] + exclude: .env.development diff --git a/api/routes/application/routes.py b/api/routes/application/routes.py index 82aa3f0a..e3adc4a5 100644 --- a/api/routes/application/routes.py +++ b/api/routes/application/routes.py @@ -3,51 +3,50 @@ from typing import Optional from uuid import uuid4 -from _helpers import get_blank_forms -from _helpers import order_applications +from flask import current_app, jsonify, request, send_file +from flask.views import MethodView +from fsd_utils import Decision, evaluate_response +from fsd_utils.config.notify_constants import NotifyConstants +from sqlalchemy.orm.exc import NoResultFound + +from _helpers import get_blank_forms, order_applications from config import Config from config.key_report_mappings.mappings import ROUND_ID_TO_KEY_REPORT_MAPPING from db.models.application.enums import Status -from db.queries import add_new_forms -from db.queries import create_application -from db.queries import export_json_to_csv -from db.queries import export_json_to_excel -from db.queries import get_application -from db.queries import get_feedback -from db.queries import get_fund_id -from db.queries import get_general_status_applications_report -from db.queries import get_key_report_field_headers -from db.queries import get_report_for_applications -from db.queries import search_applications -from db.queries import submit_application -from db.queries import update_form -from db.queries import upsert_feedback +from db.queries import ( + add_new_forms, + create_application, + export_json_to_csv, + export_json_to_excel, + get_application, + get_feedback, + get_fund_id, + get_general_status_applications_report, + get_key_report_field_headers, + get_report_for_applications, + search_applications, + submit_application, + update_form, + upsert_feedback, +) from db.queries.application import create_qa_base64file -from db.queries.feedback import retrieve_all_feedbacks_and_surveys -from db.queries.feedback import retrieve_end_of_application_survey_data -from db.queries.feedback import upsert_end_of_application_survey_data -from db.queries.reporting.queries import export_application_statuses_to_csv -from db.queries.reporting.queries import map_application_key_fields -from db.queries.research import retrieve_research_survey_data -from db.queries.research import upsert_research_survey_data -from db.queries.statuses import check_is_fund_round_open -from db.queries.statuses import update_statuses -from external_services import get_account -from external_services import get_fund -from external_services import get_round -from external_services import get_round_eoi_schema -from external_services.exceptions import NotificationError -from external_services.exceptions import SubmitError +from db.queries.feedback import ( + retrieve_all_feedbacks_and_surveys, + retrieve_end_of_application_survey_data, + upsert_end_of_application_survey_data, +) +from db.queries.reporting.queries import ( + export_application_statuses_to_csv, + map_application_key_fields, +) +from db.queries.research import ( + retrieve_research_survey_data, + upsert_research_survey_data, +) +from db.queries.statuses import check_is_fund_round_open, update_statuses +from external_services import get_account, get_fund, get_round, get_round_eoi_schema +from external_services.exceptions import NotificationError, SubmitError from external_services.models.notification import Notification -from flask import current_app -from flask import jsonify -from flask import request -from flask import send_file -from flask.views import MethodView -from fsd_utils import Decision -from fsd_utils import evaluate_response -from fsd_utils.config.notify_constants import NotifyConstants -from sqlalchemy.orm.exc import NoResultFound class ApplicationsView(MethodView): @@ -105,8 +104,8 @@ def get_key_application_data_report(self, application_id): def get_applications_statuses_report( self, - round_id: Optional[list] = [], - fund_id: Optional[list] = [], + round_id: Optional[list] = None, + fund_id: Optional[list] = None, format: Optional[str] = "csv", ): try: @@ -232,7 +231,10 @@ def submit(self, application_id): full_name.title() if full_name else None, contents, ) - current_app.logger.info(f"Message added to the queue msg_id: [{message_id}]") + current_app.logger.info( + "Message added to the queue msg_id: [{message_id}]", + extra=dict(message_id=message_id), + ) return { "id": application_id, "reference": application_with_form_json["reference"], @@ -241,19 +243,27 @@ def submit(self, application_id): }, 201 except KeyError as e: current_app.logger.exception( - f"Key error on processing application submissionfor application: '{application_id}'" + "Key error on processing application submissionfor application: '{application_id}'", + extra=dict(application_id=application_id), ) return str(e), 500, {"x-error": "key error"} except NotificationError as e: current_app.logger.exception( - f"Notification error on sending SUBMIT notification for application {application_id}" + "Notification error on sending SUBMIT notification for application {application_id}", + extra=dict(application_id=application_id), ) return str(e), 500, {"x-error": "notification error"} except SubmitError as e: - current_app.logger.exception(f"Submit error on sending SUBMIT application {application_id}") + current_app.logger.exception( + "Submit error on sending SUBMIT application {application_id}", + extra=dict(application_id=application_id), + ) return str(e), 500, {"x-error": "Submit error"} except Exception as e: - current_app.logger.exception(f"Error on sending SUBMIT notification for application {application_id}") + current_app.logger.exception( + "Error on sending SUBMIT notification for application {application_id}", + extra=dict(application_id=application_id), + ) return str(e), 500, {"x-error": "Error"} def _send_submit_queue(self, application_id, application_with_form_json): @@ -276,11 +286,14 @@ def _send_submit_queue(self, application_id, application_with_form_json): message_deduplication_id=str(uuid4()), # ensures message uniqueness extra_attributes=application_attributes, ) - current_app.logger.info(f"Message sent to SQS queue and message id is [{message_id}]") + current_app.logger.info( + "Message sent to SQS queue and message id is [{message_id}]", + extra=dict(message_id=message_id), + ) except Exception as e: current_app.logger.error("An error occurred while sending message") current_app.logger.error(e) - raise SubmitError(message="Sorry, cannot submit the message") + raise SubmitError(message="Sorry, cannot submit the message") from e def post_feedback(self): args = request.get_json() diff --git a/api/routes/queues/routes.py b/api/routes/queues/routes.py index 90d8f095..c3a3f2bd 100644 --- a/api/routes/queues/routes.py +++ b/api/routes/queues/routes.py @@ -2,11 +2,12 @@ import json from uuid import uuid4 -from config import Config -from db.queries import get_application from flask import current_app from flask.views import MethodView +from config import Config +from db.queries import get_application + class QueueView(MethodView): def post_submitted_application_to_assessment(self, application_id=None): @@ -36,7 +37,10 @@ def post_submitted_application_to_assessment(self, application_id=None): message_deduplication_id=str(uuid4()), # ensures message uniqueness extra_attributes=application_attributes, ) - current_app.logger.info(f"Message sent to SQS queue and message id is [{message_id}]") + current_app.logger.info( + "Message sent to SQS queue and message id is [{message_id}]", + extra=dict(message_id=message_id), + ) return f"Message queued, message_id is: {message_id}.", 201 except Exception as e: current_app.logger.error("An error occurred while sending message") diff --git a/app.py b/app.py index c1c04fbc..9460be39 100644 --- a/app.py +++ b/app.py @@ -1,18 +1,18 @@ from os import getenv import connexion -from api.routes.application.routes import ApplicationsView # noqa -from config import Config from connexion import FlaskApp from connexion.resolver import MethodResolver -from db.exceptions.application import ApplicationError from flask import jsonify from fsd_utils import init_sentry -from fsd_utils.healthchecks.checkers import DbChecker -from fsd_utils.healthchecks.checkers import FlaskRunningChecker +from fsd_utils.healthchecks.checkers import DbChecker, FlaskRunningChecker from fsd_utils.healthchecks.healthcheck import Healthcheck from fsd_utils.logging import logging from fsd_utils.services.aws_extended_client import SQSExtendedClient + +from api.routes.application.routes import ApplicationsView # noqa +from config import Config +from db.exceptions.application import ApplicationError from openapi.utils import get_bundled_specs diff --git a/config/envs/default.py b/config/envs/default.py index a64e9903..53343505 100644 --- a/config/envs/default.py +++ b/config/envs/default.py @@ -1,12 +1,12 @@ """Flask configuration.""" + import logging import os +from distutils.util import strtobool from os import environ from pathlib import Path -from distutils.util import strtobool -from fsd_utils import CommonConfig -from fsd_utils import configclass +from fsd_utils import CommonConfig, configclass @configclass diff --git a/config/envs/dev.py b/config/envs/dev.py index eed2cfb7..17c31e38 100644 --- a/config/envs/dev.py +++ b/config/envs/dev.py @@ -1,10 +1,12 @@ """Flask configuration.""" + import logging from os import environ -from config.envs.default import DefaultConfig from fsd_utils import configclass +from config.envs.default import DefaultConfig + @configclass class DevConfig(DefaultConfig): diff --git a/config/envs/development.py b/config/envs/development.py index 80f6ea50..6190cb56 100644 --- a/config/envs/development.py +++ b/config/envs/development.py @@ -1,9 +1,11 @@ """Flask configuration.""" + import logging -from config.envs.default import DefaultConfig from fsd_utils import configclass +from config.envs.default import DefaultConfig + @configclass class DevelopmentConfig(DefaultConfig): diff --git a/config/envs/production.py b/config/envs/production.py index d6ded768..852c0836 100644 --- a/config/envs/production.py +++ b/config/envs/production.py @@ -1,9 +1,11 @@ """Flask configuration.""" + from os import environ -from config.envs.default import DefaultConfig from fsd_utils import configclass +from config.envs.default import DefaultConfig + @configclass class ProductionConfig(DefaultConfig): diff --git a/config/envs/test.py b/config/envs/test.py index 70ebb371..ad9c47e3 100644 --- a/config/envs/test.py +++ b/config/envs/test.py @@ -1,9 +1,11 @@ """Flask configuration.""" + from os import environ -from config.envs.default import DefaultConfig from fsd_utils import configclass +from config.envs.default import DefaultConfig + @configclass class TestConfig(DefaultConfig): diff --git a/config/envs/unit_testing.py b/config/envs/unit_testing.py index a11ce681..1cbf8490 100644 --- a/config/envs/unit_testing.py +++ b/config/envs/unit_testing.py @@ -1,5 +1,6 @@ # flake8 : noqa """Flask Unit Testing Environment Configuration.""" + from os import environ from config.envs.default import DefaultConfig diff --git a/config/key_report_mappings/cof25_eoi_key_report_mapping.py b/config/key_report_mappings/cof25_eoi_key_report_mapping.py index 65b45579..3d0bca27 100644 --- a/config/key_report_mappings/cof25_eoi_key_report_mapping.py +++ b/config/key_report_mappings/cof25_eoi_key_report_mapping.py @@ -1,6 +1,4 @@ -from config.key_report_mappings.model import extract_postcode -from config.key_report_mappings.model import FormMappingItem -from config.key_report_mappings.model import KeyReportMapping +from config.key_report_mappings.model import FormMappingItem, KeyReportMapping, extract_postcode COF25_EOI_KEY_REPORT_MAPPING = KeyReportMapping( round_id="9104d809-0fb0-4144-b514-55e81cc2b6fa", diff --git a/config/key_report_mappings/cof_eoi_key_report_mapping.py b/config/key_report_mappings/cof_eoi_key_report_mapping.py index 5f36ba64..b267e37f 100644 --- a/config/key_report_mappings/cof_eoi_key_report_mapping.py +++ b/config/key_report_mappings/cof_eoi_key_report_mapping.py @@ -1,6 +1,4 @@ -from config.key_report_mappings.model import extract_postcode -from config.key_report_mappings.model import FormMappingItem -from config.key_report_mappings.model import KeyReportMapping +from config.key_report_mappings.model import FormMappingItem, KeyReportMapping, extract_postcode COF_EOI_KEY_REPORT_MAPPING = KeyReportMapping( round_id="6a47c649-7bac-4583-baed-9c4e7a35c8b3", diff --git a/config/key_report_mappings/cof_key_report_mapping.py b/config/key_report_mappings/cof_key_report_mapping.py index d3b99125..130a0430 100644 --- a/config/key_report_mappings/cof_key_report_mapping.py +++ b/config/key_report_mappings/cof_key_report_mapping.py @@ -1,7 +1,9 @@ -from config.key_report_mappings.model import ApplicationColumnMappingItem -from config.key_report_mappings.model import extract_postcode -from config.key_report_mappings.model import FormMappingItem -from config.key_report_mappings.model import KeyReportMapping +from config.key_report_mappings.model import ( + ApplicationColumnMappingItem, + FormMappingItem, + KeyReportMapping, + extract_postcode, +) COF_KEY_REPORT_MAPPING = KeyReportMapping( round_id=["4efc3263-aefe-4071-b5f4-0910abec12d2", "33726b63-efce-4749-b149-20351346c76e"], diff --git a/config/key_report_mappings/cof_r2_key_report_mapping.py b/config/key_report_mappings/cof_r2_key_report_mapping.py index 668c60e6..be54123f 100644 --- a/config/key_report_mappings/cof_r2_key_report_mapping.py +++ b/config/key_report_mappings/cof_r2_key_report_mapping.py @@ -1,6 +1,4 @@ -from config.key_report_mappings.model import extract_postcode -from config.key_report_mappings.model import FormMappingItem -from config.key_report_mappings.model import KeyReportMapping +from config.key_report_mappings.model import FormMappingItem, KeyReportMapping, extract_postcode COF_R2_KEY_REPORT_MAPPING = KeyReportMapping( round_id="c603d114-5364-4474-a0c4-c41cbf4d3bbd", diff --git a/config/key_report_mappings/cof_r3w2_key_report_mapping.py b/config/key_report_mappings/cof_r3w2_key_report_mapping.py index 130d3c8a..b1f7da02 100644 --- a/config/key_report_mappings/cof_r3w2_key_report_mapping.py +++ b/config/key_report_mappings/cof_r3w2_key_report_mapping.py @@ -1,7 +1,9 @@ -from config.key_report_mappings.model import ApplicationColumnMappingItem -from config.key_report_mappings.model import extract_postcode -from config.key_report_mappings.model import FormMappingItem -from config.key_report_mappings.model import KeyReportMapping +from config.key_report_mappings.model import ( + ApplicationColumnMappingItem, + FormMappingItem, + KeyReportMapping, + extract_postcode, +) COF_R3W2_KEY_REPORT_MAPPING = KeyReportMapping( round_id="6af19a5e-9cae-4f00-9194-cf10d2d7c8a7", diff --git a/config/key_report_mappings/cyp_r1_key_report_mapping.py b/config/key_report_mappings/cyp_r1_key_report_mapping.py index 31f48ac9..4ff26af6 100644 --- a/config/key_report_mappings/cyp_r1_key_report_mapping.py +++ b/config/key_report_mappings/cyp_r1_key_report_mapping.py @@ -1,5 +1,4 @@ -from config.key_report_mappings.model import FormMappingItem -from config.key_report_mappings.model import KeyReportMapping +from config.key_report_mappings.model import FormMappingItem, KeyReportMapping CYP_R1_KEY_REPORT_MAPPING = KeyReportMapping( round_id="888aae3d-7e2c-4523-b9c1-95952b3d1644", diff --git a/config/key_report_mappings/dpif_r2_key_report_mapping.py b/config/key_report_mappings/dpif_r2_key_report_mapping.py index e70b2ce1..ee5790c7 100644 --- a/config/key_report_mappings/dpif_r2_key_report_mapping.py +++ b/config/key_report_mappings/dpif_r2_key_report_mapping.py @@ -1,5 +1,4 @@ -from config.key_report_mappings.model import FormMappingItem -from config.key_report_mappings.model import KeyReportMapping +from config.key_report_mappings.model import FormMappingItem, KeyReportMapping DPIF_R2_KEY_REPORT_MAPPING = KeyReportMapping( round_id="0059aad4-5eb5-11ee-8c99-0242ac120002", diff --git a/config/key_report_mappings/mappings.py b/config/key_report_mappings/mappings.py index 12fb60b1..68c24ae5 100644 --- a/config/key_report_mappings/mappings.py +++ b/config/key_report_mappings/mappings.py @@ -22,7 +22,6 @@ DPIF_R2_KEY_REPORT_MAPPING, ) - ROUND_ID_TO_KEY_REPORT_MAPPING = defaultdict( lambda: COF_R2_KEY_REPORT_MAPPING.mapping, { diff --git a/config/key_report_mappings/model.py b/config/key_report_mappings/model.py index 153d4626..c30d3e7a 100644 --- a/config/key_report_mappings/model.py +++ b/config/key_report_mappings/model.py @@ -1,7 +1,6 @@ import re from dataclasses import dataclass -from typing import Any -from typing import Callable +from typing import Any, Callable @dataclass diff --git a/db/migrations/versions/5bf853808dfb_.py b/db/migrations/versions/5bf853808dfb_.py index 6d9033be..640c572d 100644 --- a/db/migrations/versions/5bf853808dfb_.py +++ b/db/migrations/versions/5bf853808dfb_.py @@ -5,6 +5,7 @@ Create Date: 2022-09-27 12:34:43.514139 """ + import sqlalchemy as sa import sqlalchemy_utils from alembic import op diff --git a/db/migrations/versions/d37dca03b539_.py b/db/migrations/versions/d37dca03b539_.py index 8024875a..747fb83a 100644 --- a/db/migrations/versions/d37dca03b539_.py +++ b/db/migrations/versions/d37dca03b539_.py @@ -5,10 +5,10 @@ Create Date: 2024-01-29 10:28:40.018763 """ + import sqlalchemy as sa from alembic import op - # revision identifiers, used by Alembic. revision = "d37dca03b539" down_revision = "f524201314e9" diff --git a/db/migrations/versions/e86cbf275e45_.py b/db/migrations/versions/e86cbf275e45_.py index 99670cdf..d2f70377 100644 --- a/db/migrations/versions/e86cbf275e45_.py +++ b/db/migrations/versions/e86cbf275e45_.py @@ -5,6 +5,7 @@ Create Date: 2023-08-18 12:47:57.470108 """ + import sqlalchemy as sa from alembic import op from sqlalchemy.dialects import postgresql diff --git a/db/migrations/versions/eb61bc354c57_.py b/db/migrations/versions/eb61bc354c57_.py index dab6197d..a986757e 100644 --- a/db/migrations/versions/eb61bc354c57_.py +++ b/db/migrations/versions/eb61bc354c57_.py @@ -5,10 +5,10 @@ Create Date: 2024-05-30 14:49:25.748523 """ + import sqlalchemy as sa from alembic import op - # revision identifiers, used by Alembic. revision = "eb61bc354c57" down_revision = "d37dca03b539" diff --git a/db/migrations/versions/f3332786bb2e_.py b/db/migrations/versions/f3332786bb2e_.py index 0b30c331..9d2435b1 100644 --- a/db/migrations/versions/f3332786bb2e_.py +++ b/db/migrations/versions/f3332786bb2e_.py @@ -5,6 +5,7 @@ Create Date: 2022-11-10 11:47:03.141413 """ + import sqlalchemy as sa from alembic import op from sqlalchemy.dialects import postgresql diff --git a/db/migrations/versions/f524201314e9_.py b/db/migrations/versions/f524201314e9_.py index 8901a0bd..1cba24b1 100644 --- a/db/migrations/versions/f524201314e9_.py +++ b/db/migrations/versions/f524201314e9_.py @@ -5,6 +5,7 @@ Create Date: 2023-08-25 20:25:36.554088 """ + import sqlalchemy as sa from alembic import op from sqlalchemy.dialects import postgresql diff --git a/db/models/__init__.py b/db/models/__init__.py index 57fc4baf..544c8de2 100644 --- a/db/models/__init__.py +++ b/db/models/__init__.py @@ -1,7 +1,6 @@ from .application import Applications from .eligibility import Eligibility -from .feedback import EndOfApplicationSurveyFeedback -from .feedback import Feedback +from .feedback import EndOfApplicationSurveyFeedback, Feedback from .forms import Forms from .research import ResearchSurvey diff --git a/db/models/application/applications.py b/db/models/application/applications.py index 83fd531b..f507056c 100644 --- a/db/models/application/applications.py +++ b/db/models/application/applications.py @@ -1,16 +1,13 @@ import uuid -from db import db -from db.models.application.enums import Language -from db.models.application.enums import Status from flask_sqlalchemy.model import DefaultMeta -from sqlalchemy import Column -from sqlalchemy import DateTime -from sqlalchemy.dialects.postgresql import ENUM -from sqlalchemy.dialects.postgresql import UUID +from sqlalchemy import Column, DateTime +from sqlalchemy.dialects.postgresql import ENUM, UUID from sqlalchemy.orm import relationship from sqlalchemy.sql import func +from db import db +from db.models.application.enums import Language, Status BaseModel: DefaultMeta = db.Model diff --git a/db/models/eligibility/eligibility.py b/db/models/eligibility/eligibility.py index d4b3aaf5..648a65ed 100644 --- a/db/models/eligibility/eligibility.py +++ b/db/models/eligibility/eligibility.py @@ -1,13 +1,12 @@ import uuid -from db import db -from db.models.application.applications import Applications from flask_sqlalchemy.model import DefaultMeta -from sqlalchemy import Column -from sqlalchemy import DateTime +from sqlalchemy import Column, DateTime from sqlalchemy.dialects.postgresql import UUID from sqlalchemy_json import NestedMutableJson +from db import db +from db.models.application.applications import Applications BaseModel: DefaultMeta = db.Model diff --git a/db/models/eligibility/eligibility_trail.py b/db/models/eligibility/eligibility_trail.py index 47462175..4c9dc1df 100644 --- a/db/models/eligibility/eligibility_trail.py +++ b/db/models/eligibility/eligibility_trail.py @@ -1,11 +1,11 @@ from uuid import uuid4 -from db import db from flask_sqlalchemy.model import DefaultMeta from sqlalchemy import Column from sqlalchemy.dialects.postgresql import UUID from sqlalchemy.sql import func +from db import db BaseModel: DefaultMeta = db.Model diff --git a/db/models/feedback/end_of_application_survey.py b/db/models/feedback/end_of_application_survey.py index 407d5c89..84aaf302 100644 --- a/db/models/feedback/end_of_application_survey.py +++ b/db/models/feedback/end_of_application_survey.py @@ -1,9 +1,9 @@ -from db import db from flask_sqlalchemy.model import DefaultMeta from sqlalchemy import DateTime from sqlalchemy.dialects.postgresql import UUID from sqlalchemy.sql import func +from db import db BaseModel: DefaultMeta = db.Model diff --git a/db/models/feedback/feedback.py b/db/models/feedback/feedback.py index 68b0e80a..b11138d8 100644 --- a/db/models/feedback/feedback.py +++ b/db/models/feedback/feedback.py @@ -1,14 +1,15 @@ import uuid -from db import db -from db.models.application.applications import Applications -from db.models.application.enums import Status from flask_sqlalchemy.model import DefaultMeta from sqlalchemy import DateTime from sqlalchemy.dialects.postgresql import UUID from sqlalchemy.sql import func from sqlalchemy_json import NestedMutableJson +from db import db +from db.models.application.applications import Applications +from db.models.application.enums import Status + BaseModel: DefaultMeta = db.Model diff --git a/db/models/forms/forms.py b/db/models/forms/forms.py index 2cea96b4..69313647 100644 --- a/db/models/forms/forms.py +++ b/db/models/forms/forms.py @@ -1,11 +1,12 @@ import uuid -from db import db -from db.models.application.applications import Applications from flask_sqlalchemy.model import DefaultMeta from sqlalchemy_json import NestedMutableJson from sqlalchemy_utils.types import UUIDType +from db import db +from db.models.application.applications import Applications + from .enums import Status BaseModel: DefaultMeta = db.Model diff --git a/db/models/research/research.py b/db/models/research/research.py index 1a4e58bf..b0fce00a 100644 --- a/db/models/research/research.py +++ b/db/models/research/research.py @@ -1,9 +1,9 @@ -from db import db from flask_sqlalchemy.model import DefaultMeta from sqlalchemy import DateTime from sqlalchemy.dialects.postgresql import UUID from sqlalchemy.sql import func +from db import db BaseModel: DefaultMeta = db.Model diff --git a/db/queries/__init__.py b/db/queries/__init__.py index 1edcb66c..c33d9adf 100644 --- a/db/queries/__init__.py +++ b/db/queries/__init__.py @@ -1,22 +1,22 @@ -from .application import create_application -from .application import get_application -from .application import get_applications -from .application import get_count_by_status -from .application import get_fund_id -from .application import search_applications -from .application import submit_application -from .feedback import get_feedback -from .feedback import upsert_feedback -from .form import add_new_forms -from .form import get_form -from .form import get_forms_by_app_id -from .reporting import export_json_to_csv -from .reporting import export_json_to_excel -from .reporting import get_general_status_applications_report -from .reporting import get_key_report_field_headers -from .reporting import get_report_for_applications -from .updating import update_application_and_related_form -from .updating import update_form +from .application import ( + create_application, + get_application, + get_applications, + get_count_by_status, + get_fund_id, + search_applications, + submit_application, +) +from .feedback import get_feedback, upsert_feedback +from .form import add_new_forms, get_form, get_forms_by_app_id +from .reporting import ( + export_json_to_csv, + export_json_to_excel, + get_general_status_applications_report, + get_key_report_field_headers, + get_report_for_applications, +) +from .updating import update_application_and_related_form, update_form __all__ = [ create_application, diff --git a/db/queries/application/queries.py b/db/queries/application/queries.py index 00ca0a67..0ef75c11 100644 --- a/db/queries/application/queries.py +++ b/db/queries/application/queries.py @@ -1,31 +1,26 @@ import base64 import random import string -from datetime import datetime -from datetime import timezone +from datetime import datetime, timezone from io import BytesIO from itertools import groupby from typing import Optional +from flask import current_app +from fsd_utils import extract_questions_and_answers, generate_text_of_application +from sqlalchemy import func, select +from sqlalchemy.exc import IntegrityError +from sqlalchemy.orm import joinedload, noload +from sqlalchemy.sql.expression import Select + from config import Config from db import db from db.exceptions import ApplicationError from db.models import Applications from db.models.application.enums import Status as ApplicationStatus from db.schemas import ApplicationSchema -from external_services import get_fund -from external_services import get_round -from external_services.aws import FileData -from external_services.aws import list_files_by_prefix -from flask import current_app -from fsd_utils import extract_questions_and_answers -from fsd_utils import generate_text_of_application -from sqlalchemy import func -from sqlalchemy import select -from sqlalchemy.exc import IntegrityError -from sqlalchemy.orm import joinedload -from sqlalchemy.orm import noload -from sqlalchemy.sql.expression import Select +from external_services import get_fund, get_round +from external_services.aws import FileData, list_files_by_prefix def get_application(app_id, include_forms=False, as_json=False) -> dict | Applications: @@ -97,9 +92,10 @@ def _create_application_try(account_id, fund_id, round_id, key, language, refere except IntegrityError: db.session.remove() current_app.logger.error( - f"Failed {attempt} attempt(s) to create application with" - f" application reference {reference}, for fund_id" - f" {fund_id} and round_id {round_id}" + "Failed {attempt} attempt(s) to create application with" + " application reference {reference}, for fund_id" + " {fund_id} and round_id {round_id}", + extra=dict(attempt=attempt, reference=reference, fund_id=fund_id, round_id=round_id), ) @@ -145,7 +141,7 @@ def get_all_applications() -> list: return application_list -def get_count_by_status(round_ids: Optional[list] = [], fund_ids: Optional[list] = []) -> dict[str, int]: +def get_count_by_status(round_ids: Optional[list] = None, fund_ids: Optional[list] = None) -> dict[str, int]: query = db.session.query( Applications.fund_id, Applications.round_id, @@ -193,7 +189,10 @@ def create_qa_base64file(application_data: dict, with_questions_file: bool): fund_details = get_fund(application_data["fund_id"]) q_and_a = extract_questions_and_answers(application_data["forms"], application_data["language"]) contents = BytesIO( - bytes(generate_text_of_application(q_and_a, fund_details.name, application_data["language"]), "utf-8") + bytes( + generate_text_of_application(q_and_a, fund_details.name, application_data["language"]), + "utf-8", + ) ).read() if len(contents) > Config.DOCUMENT_UPLOAD_SIZE_LIMIT: raise ValueError("File is larger than 2MB") @@ -239,7 +238,10 @@ def search_applications(**params): def submit_application(application_id) -> Applications: - current_app.logger.info(f"Processing database submission for application_id: '{application_id}.") + current_app.logger.info( + "Processing database submission for application_id: '{application_id}.", + extra=dict(application_id=application_id), + ) application = get_application(application_id) application.date_submitted = datetime.now(timezone.utc).isoformat() @@ -284,7 +286,10 @@ def get_fund_id(application_id): else: return None except Exception: - current_app.logger.error(f"Incorrect application id: {application_id}") + current_app.logger.error( + "Incorrect application id: {application_id}", + extra=dict(application_id=application_id), + ) return None diff --git a/db/queries/feedback/queries.py b/db/queries/feedback/queries.py index 8e6496d5..9887d268 100644 --- a/db/queries/feedback/queries.py +++ b/db/queries/feedback/queries.py @@ -1,16 +1,16 @@ from datetime import datetime +from flask import current_app + from config.key_report_mappings.mappings import get_report_mapping_for_round from db import db -from db.models import Applications -from db.models import Feedback +from db.models import Applications, Feedback from db.models.feedback import EndOfApplicationSurveyFeedback from db.queries.application.queries import get_applications from db.queries.reporting.queries import map_application_key_fields from db.schemas.application import ApplicationSchema from db.schemas.end_of_application_survey import EndOfApplicationSurveyFeedbackSchema from external_services.data import get_application_sections -from flask import current_app def upsert_feedback(application_id, fund_id, round_id, section_id, feedback_json, status): @@ -125,7 +125,10 @@ def retrieve_all_feedbacks_and_surveys(fund_id, round_id, status): applicant_email = result["applicant_email"] applicant_organisation = result["organisation_name"] except Exception as e: - current_app.logger.error(f"Coudn't extract applicant email & organisation. Exception :{e}") + current_app.logger.error( + "Coudn't extract applicant email & organisation. Exception :{error}", + extra=dict(error=e), + ) applicant_email = "" applicant_organisation = "" diff --git a/db/queries/reporting/queries.py b/db/queries/reporting/queries.py index d05e9545..c4a39e44 100644 --- a/db/queries/reporting/queries.py +++ b/db/queries/reporting/queries.py @@ -1,13 +1,11 @@ import csv import io -from typing import Any -from typing import Optional +from typing import Any, Optional import pandas as pd + from config.key_report_mappings.mappings import ROUND_ID_TO_KEY_REPORT_MAPPING -from config.key_report_mappings.model import ApplicationColumnMappingItem -from config.key_report_mappings.model import FormMappingItem -from config.key_report_mappings.model import MappingItem +from config.key_report_mappings.model import ApplicationColumnMappingItem, FormMappingItem, MappingItem from db.models import Applications from db.queries import get_applications from db.queries.application import get_count_by_status diff --git a/db/queries/updating/queries.py b/db/queries/updating/queries.py index 0209f0dc..ddce1b03 100644 --- a/db/queries/updating/queries.py +++ b/db/queries/updating/queries.py @@ -1,20 +1,23 @@ import sqlalchemy +from flask import abort, current_app +from sqlalchemy import func + from db import db from db.models.application.enums import Status as ApplicationStatus -from db.queries.application import attempt_to_find_and_update_project_name -from db.queries.application import get_application +from db.queries.application import ( + attempt_to_find_and_update_project_name, + get_application, +) from db.queries.form import get_form from db.queries.statuses import update_statuses -from flask import abort -from flask import current_app -from sqlalchemy import func def update_application_and_related_form(application_id, question_json, form_name, is_summary_page_submit): application = get_application(application_id) if application.status == ApplicationStatus.SUBMITTED: current_app.logger.error( - f"Not allowed. Attempted to PUT data into a SUBMITTED application with an application_id: {application_id}." + "Not allowed. Attempted to PUT data into a SUBMITTED application with an application_id: {application_id}.", + extra=dict(application_id=application_id), ) abort(400, "Not allowed to edit a submitted application.") @@ -25,7 +28,10 @@ def update_application_and_related_form(application_id, question_json, form_name form_sql_row.json = question_json update_statuses(application_id, form_name, is_summary_page_submit) db.session.commit() - current_app.logger.info(f"Application updated for application_id: '{application_id}.") + current_app.logger.info( + "Application updated for application_id: '{application_id}.", + extra=dict(application_id=application_id), + ) def update_form(application_id, form_name, question_json, is_summary_page_submit): @@ -42,7 +48,8 @@ def update_form(application_id, form_name, question_json, is_summary_page_submit # Removing all data in the form (should not be allowed) elif form_sql_row.json and not question_json: current_app.logger.error( - f"Application update aborted for application_id: '{application_id}. Invalid data supplied" + "Application update aborted for application_id: '{application_id}. Invalid data supplied", + extra=dict(application_id=application_id), ) raise Exception("ABORTING UPDATE, INVALID DATA GIVEN") # Updating form subsequent times diff --git a/db/schemas/application.py b/db/schemas/application.py index b1343eb0..2597abec 100644 --- a/db/schemas/application.py +++ b/db/schemas/application.py @@ -1,15 +1,12 @@ -from db.models import Applications -from db.models.application.enums import Language -from db.models.application.enums import Status -from external_services import get_round_name from marshmallow import post_dump -from marshmallow.fields import DateTime -from marshmallow.fields import Enum -from marshmallow.fields import Method -from marshmallow_sqlalchemy import auto_field -from marshmallow_sqlalchemy import SQLAlchemyAutoSchema +from marshmallow.fields import DateTime, Enum, Method +from marshmallow_sqlalchemy import SQLAlchemyAutoSchema, auto_field from marshmallow_sqlalchemy.fields import Nested +from db.models import Applications +from db.models.application.enums import Language, Status +from external_services import get_round_name + from .form import FormsRunnerSchema diff --git a/db/schemas/eligibility.py b/db/schemas/eligibility.py index 33a00e15..81328018 100644 --- a/db/schemas/eligibility.py +++ b/db/schemas/eligibility.py @@ -1,8 +1,9 @@ -from db.models import Eligibility -from db.models.eligibility.eligibility_trail import EligibilityUpdate from marshmallow import post_dump from marshmallow_sqlalchemy import SQLAlchemyAutoSchema +from db.models import Eligibility +from db.models.eligibility.eligibility_trail import EligibilityUpdate + class EligibilitySchema(SQLAlchemyAutoSchema): class Meta: diff --git a/db/schemas/end_of_application_survey.py b/db/schemas/end_of_application_survey.py index 3f87c04b..d4e6e2d6 100644 --- a/db/schemas/end_of_application_survey.py +++ b/db/schemas/end_of_application_survey.py @@ -1,6 +1,6 @@ +from marshmallow_sqlalchemy import SQLAlchemySchema, auto_field + from db.models import EndOfApplicationSurveyFeedback -from marshmallow_sqlalchemy import auto_field -from marshmallow_sqlalchemy import SQLAlchemySchema class EndOfApplicationSurveyFeedbackSchema(SQLAlchemySchema): diff --git a/db/schemas/feedback.py b/db/schemas/feedback.py index d81faefb..3df9e119 100644 --- a/db/schemas/feedback.py +++ b/db/schemas/feedback.py @@ -1,9 +1,8 @@ +from marshmallow.fields import DateTime, Enum +from marshmallow_sqlalchemy import SQLAlchemySchema, auto_field + from db.models import Feedback from db.models.feedback.enums import Status -from marshmallow.fields import DateTime -from marshmallow.fields import Enum -from marshmallow_sqlalchemy import auto_field -from marshmallow_sqlalchemy import SQLAlchemySchema class FeedbackSchema(SQLAlchemySchema): diff --git a/db/schemas/form.py b/db/schemas/form.py index 41c60b08..67a255b7 100644 --- a/db/schemas/form.py +++ b/db/schemas/form.py @@ -1,8 +1,8 @@ +from marshmallow.fields import Enum +from marshmallow_sqlalchemy import SQLAlchemySchema, auto_field + from db.models import Forms from db.models.forms.enums import Status -from marshmallow.fields import Enum -from marshmallow_sqlalchemy import auto_field -from marshmallow_sqlalchemy import SQLAlchemySchema class FormsRunnerSchema(SQLAlchemySchema): diff --git a/external_services/aws.py b/external_services/aws.py index cfdad814..4f40929b 100644 --- a/external_services/aws.py +++ b/external_services/aws.py @@ -2,6 +2,7 @@ from os import getenv import boto3 + from config import Config _KEY_PARTS = ("application_id", "form", "path", "component_id", "filename") diff --git a/external_services/data.py b/external_services/data.py index 64e133b0..5d832ff1 100644 --- a/external_services/data.py +++ b/external_services/data.py @@ -5,9 +5,9 @@ from urllib.parse import urlencode import requests +from flask import abort, current_app + from config import Config -from flask import abort -from flask import current_app from .models.account import Account from .models.fund import Fund @@ -27,13 +27,22 @@ def get_data(endpoint: str, params: Optional[dict] = None): """ if Config.USE_LOCAL_DATA: - current_app.logger.info(f"Fetching local data from '{endpoint}'" + f" with params {params}.") + current_app.logger.info( + "Fetching local data from '{endpoint}' with params {params}.", + extra=dict(endpoint=endpoint, params=params), + ) data = get_local_data(endpoint, params) else: - current_app.logger.info(f"Fetching data from '{endpoint}'" + f" with params {params}.") + current_app.logger.info( + "Fetching data from '{endpoint}' with params {params}.", + extra=dict(endpoint=endpoint, params=params), + ) data = get_remote_data(endpoint, params) if data is None: - current_app.logger.error(f"Data request failed, unable to recover: {endpoint}") + current_app.logger.error( + "Data request failed, unable to recover: {endpoint}", + extra=dict(endpoint=endpoint), + ) return abort(500) return data @@ -51,7 +60,10 @@ def get_remote_data(endpoint, params: Optional[dict] = None): data = response.json() return data else: - current_app.logger.warn(f"GET remote data call was unsuccessful with status code: {response.status_code}.") + current_app.logger.warning( + "GET remote data call was unsuccessful with status code: {status_code}.", + extra=dict(status_code=response.status_code), + ) return None @@ -93,7 +105,7 @@ def get_funds() -> list[Fund] | None: def get_fund(fund_id: str) -> Fund | None: endpoint = Config.FUND_STORE_API_HOST + Config.FUND_ENDPOINT.format(fund_id=fund_id) - current_app.logger.info(f"Request made to {endpoint}") + current_app.logger.info("Request made to {endpoint}", extra=dict(endpoint=endpoint)) response = get_data(endpoint) if response is None: current_app.logger.info("Request to fund store returned None") diff --git a/external_services/http_methods.py b/external_services/http_methods.py index 496b619d..a5c4bdbb 100644 --- a/external_services/http_methods.py +++ b/external_services/http_methods.py @@ -3,24 +3,31 @@ from typing import Optional import requests +from flask import current_app + from config import Config from external_services.exceptions import NotificationError -from flask import current_app def post_data(endpoint: str, json_payload: Optional[dict] = None) -> dict: if Config.USE_LOCAL_DATA: - current_app.logger.info(f"Posting to local dummy endpoint: {endpoint}") + current_app.logger.info("Posting to local dummy endpoint: {endpoint}", extra=dict(endpoint=endpoint)) response = post_local_data(endpoint) else: if json_payload: json_payload = {k: v for k, v in json_payload.items() if v is not None} - current_app.logger.info(f"Attempting POST to the following endpoint: '{endpoint}'.") + current_app.logger.info( + "Attempting POST to the following endpoint: '{endpoint}'.", + extra=dict(endpoint=endpoint), + ) response = requests.post(endpoint, json=json_payload) if response.status_code in [200, 201]: - current_app.logger.info(f"Post successfully sent to {endpoint} with response code: '{response.status_code}'.") + current_app.logger.info( + "Post successfully sent to {endpoint} with response code: '{status_code}'.", + extra=dict(endpoint=endpoint, status_code=response.status_code), + ) return response.json() diff --git a/external_services/models/fund.py b/external_services/models/fund.py index 9835f10c..8a9d0642 100644 --- a/external_services/models/fund.py +++ b/external_services/models/fund.py @@ -1,9 +1,10 @@ from dataclasses import dataclass from typing import Optional -from external_services.models.round import Round from flask import current_app +from external_services.models.round import Round + @dataclass class Fund: diff --git a/external_services/models/notification.py b/external_services/models/notification.py index f1bc7b1a..db7bf991 100644 --- a/external_services/models/notification.py +++ b/external_services/models/notification.py @@ -1,9 +1,10 @@ import json from uuid import uuid4 +from flask import current_app + from config import Config from external_services.exceptions import NotificationError -from flask import current_app NOTIFICATION_CONST = "notification" NOTIFICATION_S3_KEY_CONST = "application/notification" @@ -33,7 +34,10 @@ def send(template_type: str, to_email: str, full_name: str, content: dict): "full_name": full_name, "content": content, } - current_app.logger.info(f" json payload '{template_type}' to '{to_email}'.") + current_app.logger.info( + " json payload '{template_type}' to '{to_email}'.", + extra=dict(template_type=template_type, to_email=to_email), + ) try: sqs_extended_client = Notification._get_sqs_client() message_id = sqs_extended_client.submit_single_message( @@ -48,12 +52,15 @@ def send(template_type: str, to_email: str, full_name: str, content: dict): }, }, ) - current_app.logger.info(f"Message sent to SQS queue and message id is [{message_id}]") + current_app.logger.info( + "Message sent to SQS queue and message id is [{message_id}]", + extra=dict(message_id=message_id), + ) return message_id except Exception as e: current_app.logger.error("An error occurred while sending message") current_app.logger.error(e) - raise NotificationError(message="Sorry, the notification could not be sent") + raise NotificationError(message="Sorry, the notification could not be sent") from e @staticmethod def _get_sqs_client(): diff --git a/openapi/utils.py b/openapi/utils.py index d9bfbdc8..c85a2e99 100644 --- a/openapi/utils.py +++ b/openapi/utils.py @@ -1,6 +1,7 @@ from typing import Any import prance + from config import Config diff --git a/pyproject.toml b/pyproject.toml index 60619e61..04dca281 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -34,12 +34,27 @@ dependencies = [ "uvicorn==0.30.3", ] -[tool.black] +[tool.ruff] line-length = 120 -[tool.flake8] -max-line-length = 120 -count = true +[tool.ruff.lint] +select = [ + "E", # pycodestyle + "W", # pycodestyle + "F", # pyflakes + "I", # isort + "B", # flake8-bugbear + "C90", # mccabe cyclomatic complexity + "G", # flake8-logging-format +] +ignore = [] +exclude = [ + "db/migrations/versions/", + "venv*", + ".venv*", + "__pycache__", +] +mccabe.max-complexity = 12 [tool.uv] diff --git a/scripts/seed_db_test_data.py b/scripts/seed_db_test_data.py index 551f9658..1ca57009 100644 --- a/scripts/seed_db_test_data.py +++ b/scripts/seed_db_test_data.py @@ -4,17 +4,17 @@ sys.path.insert(1, ".") +from fsd_test_utils.test_config.useful_config import UsefulConfig # noqa: E402 + from app import app # noqa: E402 from db.models.application.applications import Status # noqa: E402 -from fsd_test_utils.test_config.useful_config import UsefulConfig # noqa: E402 -from tests.seed_data.seed_db import seed_in_progress_application # noqa: E402 from tests.seed_data.seed_db import ( # noqa: E402 - seed_not_started_application, seed_completed_application, + seed_in_progress_application, # noqa: E402 + seed_not_started_application, seed_submitted_application, ) # noqa: E402 - FUND_CONFIG = { "COF": { "id": UsefulConfig.COF_FUND_ID, @@ -93,7 +93,7 @@ def seed_applications(fund_short_code, round_short_code, account_id, status, cou round_config = fund_config["rounds"][round_short_code] match status: case Status.NOT_STARTED.name: - for i in range(count): + for _i in range(count): app = seed_not_started_application( fund_config=fund_config, round_config=round_config, @@ -102,7 +102,7 @@ def seed_applications(fund_short_code, round_short_code, account_id, status, cou ) print(f"{app.id} - {app.reference} - {app.status.name}") case Status.IN_PROGRESS.name: - for i in range(count): + for _i in range(count): app = seed_in_progress_application( fund_config=fund_config, round_config=round_config, @@ -111,7 +111,7 @@ def seed_applications(fund_short_code, round_short_code, account_id, status, cou ) print(f"{app.id} - {app.reference} - {app.status.name}") case Status.COMPLETED.name: - for i in range(count): + for _i in range(count): app = seed_completed_application( fund_config=fund_config, round_config=round_config, @@ -120,7 +120,7 @@ def seed_applications(fund_short_code, round_short_code, account_id, status, cou ) print(f"{app.id} - {app.reference} - {app.status.name}") case Status.SUBMITTED.name: - for i in range(count): + for _i in range(count): app = seed_submitted_application( fund_config=fund_config, round_config=round_config, diff --git a/scripts/send_application_on_closure.py b/scripts/send_application_on_closure.py index 5009996d..72aff3be 100755 --- a/scripts/send_application_on_closure.py +++ b/scripts/send_application_on_closure.py @@ -2,20 +2,22 @@ import argparse import sys from datetime import datetime - from distutils.util import strtobool sys.path.insert(1, ".") +from flask import current_app # noqa: E402 + import external_services # noqa: E402 from app import app # noqa: E402 from config import Config # noqa: E402 -from db.queries import search_applications # noqa: E402 -from db.queries import get_forms_by_app_id # noqa: E402 +from db.queries import ( + get_forms_by_app_id, # noqa: E402 + search_applications, # noqa: E402 +) from db.queries.application import create_qa_base64file # noqa: E402 -from external_services.models.notification import Notification # noqa: E402 from external_services.data import get_fund # noqa: E402 -from flask import current_app # noqa: E402 +from external_services.models.notification import Notification # noqa: E402 def send_incomplete_applications_after_deadline( @@ -78,21 +80,36 @@ def send_incomplete_applications_after_deadline( ) current_app.logger.info( - f"Found {len(matching_applications)} applications with matching" - f" statuses. Retrieved all data for {len(applications_to_send)} of" - " them." + "Found {matching_app_count} applications with matching" + " statuses. Retrieved all data for {apps_to_send_count} of" + " them.", + extra=dict( + matching_app_count=len(matching_applications), + apps_to_send_count=len(applications_to_send), + ), ) if send_email: total_applications = len(applications_to_send) current_app.logger.info( - "Send email set to true, will now send" - f" {total_applications} {'emails' if total_applications > 1 else 'email'}." + "Send email set to true, will now send" " {total_applications} {emails}.", + extra=dict( + total_applications=total_applications, + emails="emails" if total_applications > 1 else "email", + ), ) if total_applications > 0: for count, application in enumerate(applications_to_send, start=1): - email = {"email": application.get("account_email") for application in application.values()} + email = { + "email": application.get("account_email") # noqa: B035 needs refactor + for application in application.values() + } current_app.logger.info( - f"Sending application {count} of {total_applications} to {email.get('email')}" + "Sending application {count} of {total_applications} to {email}", + extra=dict( + count=count, + total_applications=total_applications, + email=email.get("email"), + ), ) application["contact_help_email"] = fund_rounds.get("contact_email") message_id = Notification.send( @@ -104,8 +121,14 @@ def send_incomplete_applications_after_deadline( "contact_help_email": application["contact_help_email"], }, ) - current_app.logger.info(f"Message added to the queue msg_id: [{message_id}]") - current_app.logger.info(f"Sent {count} {'emails' if count > 1 else 'email'}") + current_app.logger.info( + "Message added to the queue msg_id: [{message_id}]", + extra=dict(message_id=message_id), + ) + current_app.logger.info( + "Sent {count} {emails}", + extra=dict(count=count, emails="emails" if count > 1 else "email"), + ) return count else: current_app.logger.warning("There are no applications to be sent.") @@ -113,7 +136,8 @@ def send_incomplete_applications_after_deadline( else: count = len(applications_to_send) current_app.logger.warning( - f"Send email set to false, will not send {count} {'emails' if count > 1 else 'email'}." + "Send email set to false, will not send {count} {emails}.", + extra=dict(count=count, emails="emails" if count > 1 else "email"), ) return len(applications_to_send) else: diff --git a/scripts/send_application_reminder.py b/scripts/send_application_reminder.py index 7951d7db..79b169d9 100755 --- a/scripts/send_application_reminder.py +++ b/scripts/send_application_reminder.py @@ -3,20 +3,20 @@ sys.path.insert(1, ".") -from external_services.exceptions import NotificationError # noqa: E402 -import external_services # noqa: E402 -from config import Config # noqa: E402 -from external_services.models.notification import Notification # noqa: E402 -from flask import current_app # noqa: E402 -from db.queries import search_applications # noqa: E402 - from datetime import datetime # noqa: E402 -import requests # noqa: E402 import pytz # noqa: E402 +import requests # noqa: E402 +from flask import current_app # noqa: E402 + +import external_services # noqa: E402 +from config import Config # noqa: E402 +from db.queries import search_applications # noqa: E402 +from external_services.exceptions import NotificationError # noqa: E402 +from external_services.models.notification import Notification # noqa: E402 -def application_deadline_reminder(flask_app): +def application_deadline_reminder(flask_app): # noqa:C901 from before ruff with flask_app.app_context(): uk_timezone = pytz.timezone("Europe/London") current_datetime = datetime.now(uk_timezone).replace(tzinfo=None) @@ -33,7 +33,10 @@ def application_deadline_reminder(flask_app): reminder_date_str = round.get("reminder_date") if not reminder_date_str: - current_app.logger.info(f"No reminder is set for the round {round.get('title')}") + current_app.logger.info( + "No reminder is set for the round {round_title}", + extra=round.get("title"), + ) continue application_reminder_sent = round.get("application_reminder_sent") @@ -79,7 +82,10 @@ def application_deadline_reminder(flask_app): for count, application in enumerate(unique_application_email_addresses, start=1): email = {"email": application["application"]["account_email"]} - current_app.logger.info(f"Sending reminder {count} of {len(unique_email_account)}") + current_app.logger.info( + "Sending reminder {count} of {total}", + extra=dict(count=count, total=len(unique_email_account)), + ) try: message_id = Notification.send( @@ -87,7 +93,10 @@ def application_deadline_reminder(flask_app): to_email=email.get("email"), content=application, ) - current_app.logger.info(f"Message added to the queue msg_id: [{message_id}]") + current_app.logger.info( + "Message added to the queue msg_id: [{message_id}]", + extra=dict(message_id=message_id), + ) if len(unique_application_email_addresses) == count: try: application_reminder_endpoint = ( @@ -99,13 +108,15 @@ def application_deadline_reminder(flask_app): current_app.logger.info( "The application reminder has been" " sent successfully for round_id" - f" {round_id}" + " {round_id}", + extra=dict(found_id=round_id), ) except Exception as e: current_app.logger.info( "There was an issue updating the" " application_reminder_sent column in the" - f" Round store for {round_id}. Errro {e}" + " Round store for {round_id}. Errro {errno}", + extra=dict(round_id=round_id, errno=e), ) except NotificationError as e: diff --git a/tasks.py b/tasks.py index d802fd72..a54247d6 100644 --- a/tasks.py +++ b/tasks.py @@ -2,12 +2,9 @@ import venv from pathlib import Path -from colored import attr -from colored import fg -from colored import stylize +from colored import attr, fg, stylize from invoke import task - ECHO_STYLE = fg("light_gray") + attr("bold") diff --git a/testing.py b/testing.py index cb9d544a..70bc353e 100644 --- a/testing.py +++ b/testing.py @@ -12,7 +12,7 @@ def print_data(): executor = ThreadPoolExecutor(max_workers=5, thread_name_prefix="TestingExecutor") -for x in range(4): +for _x in range(4): executor.submit(print_data) print_data() diff --git a/tests/conftest.py b/tests/conftest.py index 6b39c515..73a61d5e 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,20 +1,21 @@ -from datetime import datetime -from datetime import timedelta +from datetime import datetime, timedelta from uuid import uuid4 import pytest +from flask import Response + from app import create_app from db.models.application.applications import Applications from db.queries.application import create_application from db.queries.form import add_new_forms -from external_services.models.fund import Fund -from external_services.models.fund import Round -from flask import Response -from tests.helpers import APPLICATION_DISPLAY_CONFIG -from tests.helpers import local_api_call -from tests.helpers import test_application_data -from tests.helpers import test_question_data -from tests.helpers import test_question_data_cy +from external_services.models.fund import Fund, Round +from tests.helpers import ( + APPLICATION_DISPLAY_CONFIG, + local_api_call, + test_application_data, + test_question_data, + test_question_data_cy, +) # Make the utils fixtures available, used in seed_application_records pytest_plugins = ["fsd_test_utils.fixtures.db_fixtures"] diff --git a/tests/helpers.py b/tests/helpers.py index f826b2e8..d62947bc 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -4,9 +4,10 @@ import urllib from datetime import datetime +from deepdiff import DeepDiff + from config import Config from db.models.application.enums import Language -from deepdiff import DeepDiff def get_row_by_pk(table, primary_key): @@ -317,14 +318,14 @@ def post_test_applications(client): def key_list_to_regex( - exclude_keys: list[str] = [ + exclude_keys: list[str] = ( "id", "reference", "started_at", "project_name", "last_edited", "date_submitted", - ] + ), ): exclude_regex_path_strings = [rf"root\[\d+\]\['{key}'\]" for key in exclude_keys] diff --git a/tests/seed_data/seed_db.py b/tests/seed_data/seed_db.py index b71734f1..d86cd68a 100644 --- a/tests/seed_data/seed_db.py +++ b/tests/seed_data/seed_db.py @@ -3,8 +3,7 @@ from _helpers import get_blank_forms from db.models.application import Applications from db.queries import add_new_forms -from db.queries.application import create_application -from db.queries.application import submit_application +from db.queries.application import create_application, submit_application from db.queries.updating.queries import update_form diff --git a/tests/test_all_feedbacks.py b/tests/test_all_feedbacks.py index 9cf2dd5c..87f89951 100644 --- a/tests/test_all_feedbacks.py +++ b/tests/test_all_feedbacks.py @@ -1,11 +1,9 @@ import pytest + from config.key_report_mappings.cof_r3w2_key_report_mapping import ( COF_R3W2_KEY_REPORT_MAPPING, ) -from db.models import Applications -from db.models import EndOfApplicationSurveyFeedback -from db.models import Feedback -from db.models import Forms +from db.models import Applications, EndOfApplicationSurveyFeedback, Feedback, Forms from db.queries.feedback import retrieve_all_feedbacks_and_surveys app_sections = [ diff --git a/tests/test_application_status.py b/tests/test_application_status.py index e0430e14..a2e55f95 100644 --- a/tests/test_application_status.py +++ b/tests/test_application_status.py @@ -1,14 +1,17 @@ from unittest.mock import MagicMock import pytest -from db.queries.statuses.queries import _determine_question_page_status_from_answers -from db.queries.statuses.queries import _is_all_sections_feedback_complete -from db.queries.statuses.queries import _is_feedback_survey_complete -from db.queries.statuses.queries import _is_field_answered -from db.queries.statuses.queries import _is_research_survey_complete -from db.queries.statuses.queries import update_application_status -from db.queries.statuses.queries import update_form_status -from db.queries.statuses.queries import update_question_page_statuses + +from db.queries.statuses.queries import ( + _determine_question_page_status_from_answers, + _is_all_sections_feedback_complete, + _is_feedback_survey_complete, + _is_field_answered, + _is_research_survey_complete, + update_application_status, + update_form_status, + update_question_page_statuses, +) from external_services.models.round import FeedbackSurveyConfig @@ -57,8 +60,8 @@ def test_update_question_statuses_with_mocks(mocker): assert test_json[0]["status"] == "NOT_STARTED" assert test_json[1]["status"] == "NOT_STARTED" - mock_question_status.call_count == 2 - mock_answer_status.call_count == 2 + assert mock_question_status.call_count == 2 + assert mock_answer_status.call_count == 2 @pytest.mark.parametrize( diff --git a/tests/test_aws.py b/tests/test_aws.py index 71fd52e2..ca401f2f 100644 --- a/tests/test_aws.py +++ b/tests/test_aws.py @@ -1,4 +1,5 @@ import pytest + from external_services.aws import list_files_by_prefix diff --git a/tests/test_forms.py b/tests/test_forms.py index b68ac8ce..c34edf50 100644 --- a/tests/test_forms.py +++ b/tests/test_forms.py @@ -1,6 +1,5 @@ from _helpers.form import get_forms_from_sections - section_config = [ { "form_name": None, diff --git a/tests/test_notification.py b/tests/test_notification.py index c783730d..9060b8f9 100644 --- a/tests/test_notification.py +++ b/tests/test_notification.py @@ -4,13 +4,14 @@ import boto3 import pytest -from config import Config -from external_services.exceptions import NotificationError -from external_services.models.notification import Notification from fsd_utils import NotifyConstants from fsd_utils.services.aws_extended_client import SQSExtendedClient from moto import mock_aws +from config import Config +from external_services.exceptions import NotificationError +from external_services.models.notification import Notification + class NotificationTest(unittest.TestCase): @mock_aws diff --git a/tests/test_queries.py b/tests/test_queries.py index ef709ea1..bb3d13e9 100644 --- a/tests/test_queries.py +++ b/tests/test_queries.py @@ -3,6 +3,7 @@ from uuid import uuid4 import pytest + from config.key_report_mappings.cof_eoi_key_report_mapping import COF_EOI_KEY_REPORT_MAPPING from config.key_report_mappings.cof_key_report_mapping import COF_KEY_REPORT_MAPPING from config.key_report_mappings.cof_r2_key_report_mapping import ( @@ -12,15 +13,10 @@ COF_R3W2_KEY_REPORT_MAPPING, ) from config.key_report_mappings.mappings import ROUND_ID_TO_KEY_REPORT_MAPPING -from config.key_report_mappings.model import extract_postcode -from config.key_report_mappings.model import KeyReportMapping -from db.models import Applications -from db.models import Forms -from db.queries.application import create_application -from db.queries.application import create_qa_base64file -from db.queries.application import process_files -from db.queries.reporting.queries import export_application_statuses_to_csv -from db.queries.reporting.queries import map_application_key_fields +from config.key_report_mappings.model import KeyReportMapping, extract_postcode +from db.models import Applications, Forms +from db.queries.application import create_application, create_qa_base64file, process_files +from db.queries.reporting.queries import export_application_statuses_to_csv, map_application_key_fields from external_services.aws import FileData from external_services.models.fund import Fund from tests.seed_data.application_data import expected_application_json @@ -196,7 +192,7 @@ def test_process_files(application, all_application_files, expected): THEN the application object is expected to be updated with the relevant file information """ result = process_files(application, all_application_files) - for form, expected_form in zip(result.forms, expected.forms): + for form, expected_form in zip(result.forms, expected.forms, strict=False): assert form.json == pytest.approx(expected_form.json) diff --git a/tests/test_reports.py b/tests/test_reports.py index 085d5b9f..51c17d5f 100644 --- a/tests/test_reports.py +++ b/tests/test_reports.py @@ -1,8 +1,8 @@ import pytest + from db.models import Applications from db.models.application.enums import Status -from tests.helpers import get_row_by_pk -from tests.helpers import test_application_data +from tests.helpers import get_row_by_pk, test_application_data @pytest.mark.apps_to_insert(test_application_data) @@ -244,8 +244,7 @@ def test_get_applications_report_query_param(flask_test_client, seed_data_multip lines = [line.decode("utf-8") for line in response.content.splitlines()] assert ( - lines[0] - == "eoi_reference,organisation_name,organisation_type,asset_type," + lines[0] == "eoi_reference,organisation_name,organisation_type,asset_type," "geography,capital,revenue,organisation_name_nstf" ) diff --git a/tests/test_routes.py b/tests/test_routes.py index e394309f..ad78ecd7 100644 --- a/tests/test_routes.py +++ b/tests/test_routes.py @@ -1,31 +1,31 @@ import json -from datetime import datetime -from datetime import timedelta +from datetime import datetime, timedelta from unittest import mock -from unittest.mock import ANY -from unittest.mock import MagicMock +from unittest.mock import ANY, MagicMock from uuid import uuid4 import boto3 import pytest +from fsd_utils.services.aws_extended_client import SQSExtendedClient +from moto import mock_aws + from config import Config from db import db -from db.models import Applications -from db.models import ResearchSurvey +from db.models import Applications, ResearchSurvey from db.queries.application import get_all_applications from db.schemas import ApplicationSchema from external_services.models.fund import Fund from external_services.models.round import Round -from fsd_utils.services.aws_extended_client import SQSExtendedClient -from moto import mock_aws -from tests.helpers import application_expected_data -from tests.helpers import count_fund_applications -from tests.helpers import expected_data_within_response -from tests.helpers import get_row_by_pk -from tests.helpers import key_list_to_regex -from tests.helpers import post_data -from tests.helpers import test_application_data -from tests.helpers import test_question_data +from tests.helpers import ( + application_expected_data, + count_fund_applications, + expected_data_within_response, + get_row_by_pk, + key_list_to_regex, + post_data, + test_application_data, + test_question_data, +) @pytest.mark.unique_fund_round(True) diff --git a/tests/test_seed_db.py b/tests/test_seed_db.py index dd8a6f36..f6cf90b6 100644 --- a/tests/test_seed_db.py +++ b/tests/test_seed_db.py @@ -3,16 +3,18 @@ from uuid import uuid4 import pytest + from config import Config from db.models.application.applications import Status from db.queries.application import get_application_status from db.queries.form import get_forms_by_app_id from scripts.seed_db_test_data import FUND_CONFIG -from tests.seed_data.seed_db import seed_completed_application -from tests.seed_data.seed_db import seed_in_progress_application -from tests.seed_data.seed_db import seed_not_started_application -from tests.seed_data.seed_db import seed_submitted_application - +from tests.seed_data.seed_db import ( + seed_completed_application, + seed_in_progress_application, + seed_not_started_application, + seed_submitted_application, +) LANG_EN = "en" COF = FUND_CONFIG["COF"] diff --git a/tests/test_send_app_on_closure.py b/tests/test_send_app_on_closure.py index 7ebd382a..521ca997 100644 --- a/tests/test_send_app_on_closure.py +++ b/tests/test_send_app_on_closure.py @@ -1,5 +1,6 @@ import pytest from pytest import raises + from scripts.send_application_on_closure import ( send_incomplete_applications_after_deadline, )