Merge pull request #313 from communitiesuk/FS-3431-use-sqs-from-utils

FS-3431 refactored to use sqs funcs from fsd-utils
communitiesuk · Oct 31, 2023 · e478fe9 · e478fe9
2 parents bbdf7d0 + 7646765
commit e478fe9
Show file tree

Hide file tree

Showing 9 changed files with 81 additions and 246 deletions.
diff --git a/.env.development b/.env.development
@@ -10,6 +10,8 @@ AWS_ACCESS_KEY_ID=FSDIOSFODNN7EXAMPLE
 AWS_SECRET_ACCESS_KEY=fsdlrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY
 AWS_ENDPOINT_OVERRIDE=http://localhost:4566
 AWS_REGION=eu-west-2
-AWS_SQS_QUEUE_NAME=fsd-queue
-AWS_DLQ_QUEUE_NAME=fsd-dlq
+AWS_SQS_IMPORT_APP_QUEUE_NAME=import-queue
+AWS_DLQ_IMPORT_APP_QUEUE_NAME=import-dlq
 AWS_DLQ_MAX_RECIEVE_COUNT=3
+AWS_SQS_IMPORT_APP_PRIMARY_QUEUE_URL=http://localhost:4566/000000000000/import-queue
+AWS_SQS_IMPORT_APP_SECONDARY_QUEUE_URL=http://localhost:4566/000000000000/import-queue-dlq
diff --git a/_helpers/import_application.py b/_helpers/import_application.py
@@ -8,17 +8,18 @@
 )
 from db.queries import bulk_insert_application_record
 from flask import current_app
-from services import _SQS_QUEUE_URL
-from services import delete_messages
-from services import receive_messages
+from services import _SQS_CLIENT
 
 
 def import_applications_from_queue():
     batch_size = Config.SQS_BATCH_SIZE
     visibility_time = Config.SQS_VISIBILITY_TIME
     wait_time = Config.SQS_WAIT_TIME
-    application_messages = receive_messages(
-        _SQS_QUEUE_URL, batch_size, visibility_time, wait_time
+    application_messages = _SQS_CLIENT.receive_messages(
+        Config.AWS_SQS_IMPORT_APP_PRIMARY_QUEUE_URL,
+        batch_size,
+        visibility_time,
+        wait_time,
     )
 
     if not application_messages:
@@ -61,7 +62,10 @@ def import_applications_from_queue():
             ):
                 reciept_handles_to_delete.append(message["ReceiptHandle"])
         if reciept_handles_to_delete:
-            delete_messages(_SQS_QUEUE_URL, reciept_handles_to_delete)
+            _SQS_CLIENT.delete_messages(
+                Config.AWS_SQS_IMPORT_APP_PRIMARY_QUEUE_URL,
+                reciept_handles_to_delete,
+            )
 
         for message in application_messages:
             receive_count = int(

diff --git a/config/envs/default.py b/config/envs/default.py
@@ -50,23 +50,33 @@ class DefaultConfig:
     # ---------------
     if "PRIMARY_QUEUE_URL" in environ:
         AWS_REGION = environ.get("AWS_REGION")
-        AWS_PRIMARY_QUEUE_URL = environ.get("PRIMARY_QUEUE_URL")
-        AWS_SECONDARY_QUEUE_URL = environ.get("DEAD_LETTER_QUEUE_URL")
+        AWS_SQS_IMPORT_APP_PRIMARY_QUEUE_URL = environ.get("PRIMARY_QUEUE_URL")
+        AWS_SQS_IMPORT_APP_SECONDARY_QUEUE_URL = environ.get(
+            "DEAD_LETTER_QUEUE_URL"
+        )
     elif "VCAP_SERVICES" in environ:
         vcap_services = json.loads(environ["VCAP_SERVICES"])
         if "aws-sqs-queue" in vcap_services:
             sqs_credentials = vcap_services["aws-sqs-queue"][0]["credentials"]
             AWS_REGION = sqs_credentials["aws_region"]
             AWS_ACCESS_KEY_ID = sqs_credentials["aws_access_key_id"]
             AWS_SECRET_ACCESS_KEY = sqs_credentials["aws_secret_access_key"]
-            AWS_PRIMARY_QUEUE_URL = sqs_credentials["primary_queue_url"]
-            AWS_SECONDARY_QUEUE_URL = sqs_credentials["secondary_queue_url"]
+            AWS_SQS_IMPORT_APP_PRIMARY_QUEUE_URL = sqs_credentials[
+                "primary_queue_url"
+            ]
+            AWS_SQS_IMPORT_APP_SECONDARY_QUEUE_URL = sqs_credentials[
+                "secondary_queue_url"
+            ]
     else:
         AWS_ACCESS_KEY_ID = environ.get("AWS_ACCESS_KEY_ID")
         AWS_SECRET_ACCESS_KEY = environ.get("AWS_SECRET_ACCESS_KEY")
         AWS_REGION = environ.get("AWS_REGION")
-        AWS_PRIMARY_QUEUE_URL = ""
-        AWS_SECONDARY_QUEUE_URL = ""
+        AWS_SQS_IMPORT_APP_PRIMARY_QUEUE_URL = environ.get(
+            "AWS_SQS_IMPORT_APP_PRIMARY_QUEUE_URL"
+        )
+        AWS_SQS_IMPORT_APP_SECONDARY_QUEUE_URL = environ.get(
+            "AWS_SQS_IMPORT_APP_SECONDARY_QUEUE_URL"
+        )
     AWS_DLQ_MAX_RECIEVE_COUNT = int(
         environ.get("AWS_DLQ_MAX_RECIEVE_COUNT", 3)
     )

diff --git a/config/envs/unit_testing.py b/config/envs/unit_testing.py
@@ -29,5 +29,5 @@ class UnitTestingConfig(DefaultConfig):
     AWS_ACCESS_KEY_ID = "test_access_id"
     AWS_SECRET_ACCESS_KEY = "test_secret_key"  # pragma: allowlist secret
     AWS_REGION = "eu-west-2"
-    AWS_PRIMARY_QUEUE_URL = "test_primary_url"
-    AWS_SECONDARY_QUEUE_URL = "test_secondary_url"
+    AWS_SQS_IMPORT_APP_PRIMARY_QUEUE_URL = "test_primary_url"
+    AWS_SQS_IMPORT_APP_SECONDARY_QUEUE_URL = "test_secondary_url"
diff --git a/requirements-dev.txt b/requirements-dev.txt
@@ -30,6 +30,10 @@ babel==2.11.0
     #   flask-babel
 bandit==1.7.4
     # via -r requirements-dev.in
+beautifulsoup4==4.12.2
+    # via
+    #   -r requirements.txt
+    #   funding-service-design-utils
 black==22.10.0
     # via -r requirements-dev.in
 blinker==1.5
@@ -81,13 +85,6 @@ clickclick==20.10.2
     # via
     #   -r requirements.txt
     #   connexion
-colorama==0.4.6
-    # via
-    #   -r requirements.txt
-    #   bandit
-    #   build
-    #   click
-    #   pytest
 colored==1.4.4
     # via -r requirements-dev.in
 commonmark==0.9.1
@@ -112,6 +109,8 @@ deepdiff==6.2.3
     # via -r requirements-dev.in
 distlib==0.3.6
     # via virtualenv
+exceptiongroup==1.1.3
+    # via pytest
 filelock==3.8.0
     # via virtualenv
 flask==2.2.3
@@ -153,13 +152,13 @@ flupy==1.2.0
     # via
     #   -r requirements.txt
     #   alembic-utils
-funding-service-design-utils==2.0.1
+funding-service-design-utils==2.0.30
     # via -r requirements.txt
 gitdb==4.0.9
     # via gitpython
 gitpython==3.1.29
     # via bandit
-greenlet==2.0.2
+greenlet==3.0.1
     # via
     #   -r requirements.txt
     #   sqlalchemy
@@ -229,6 +228,7 @@ markupsafe==2.1.1
     #   -r requirements.txt
     #   jinja2
     #   mako
+    #   sentry-sdk
     #   werkzeug
 marshmallow==3.19.0
     # via
@@ -397,6 +397,10 @@ ruamel-yaml==0.17.21
     # via
     #   -r requirements.txt
     #   prance
+ruamel-yaml-clib==0.2.8
+    # via
+    #   -r requirements.txt
+    #   ruamel-yaml
 s3transfer==0.6.0
     # via
     #   -r requirements.txt
@@ -405,7 +409,7 @@ semver==2.13.0
     # via
     #   -r requirements.txt
     #   prance
-sentry-sdk[flask]==1.11.1
+sentry-sdk[flask]==1.31.0
     # via
     #   -r requirements.txt
     #   funding-service-design-utils
@@ -421,6 +425,10 @@ six==1.16.0
     #   thrift
 smmap==5.0.0
     # via gitdb
+soupsieve==2.5
+    # via
+    #   -r requirements.txt
+    #   beautifulsoup4
 sqlalchemy[mypy]==1.4.48
     # via
     #   -r requirements.txt
@@ -451,6 +459,14 @@ thrift==0.16.0
     # via
     #   -r requirements.txt
     #   flipper-client
+tomli==2.0.1
+    # via
+    #   -r requirements.txt
+    #   black
+    #   build
+    #   mypy
+    #   pep517
+    #   pytest
 typing-extensions==4.4.0
     # via
     #   -r requirements.txt
@@ -460,10 +476,6 @@ typing-extensions==4.4.0
     #   mypy
     #   sqlalchemy-stubs
     #   sqlalchemy2-stubs
-tzdata==2023.3
-    # via
-    #   -r requirements.txt
-    #   tzlocal
 tzlocal==5.0.1
     # via
     #   -r requirements.txt

diff --git a/requirements.txt b/requirements.txt
@@ -86,8 +86,10 @@ flipper-client==1.3.1
     # via funding-service-design-utils
 flupy==1.2.0
     # via alembic-utils
-funding-service-design-utils==2.0.26
-    # via -r requirements.in
+funding-service-design-utils==2.0.30
+    # via -r requirements.txt
+greenlet==3.0.1
+    # via sqlalchemy
 gunicorn==20.1.0
     # via funding-service-design-utils
 idna==3.4

diff --git a/services/__init__.py b/services/__init__.py
@@ -1,4 +1 @@
 from .aws import _SQS_CLIENT  # noqa
-from .aws import _SQS_QUEUE_URL  # noqa
-from .aws import delete_messages  # noqa
-from .aws import receive_messages  # noqa