BFD-3433: Update Python3 versions to 3.11 (#2347)

Co-authored-by: Michael J Burling <[email protected]>

apps/utils/locust_tests/common/stats/aggregated_stats.py

@@ -4,7 +4,7 @@
 import hashlib
 import time
 from dataclasses import dataclass
-from enum import Enum
+from enum import StrEnum
 from typing import Any, Dict, List, Optional, Union
 
 from locust.env import Environment
@@ -168,7 +168,7 @@ def __get_percentiles_dict(cls, stats_entry: StatsEntry) -> ResponseTimePercenti
         }
 
 
-class FinalCompareResult(str, Enum):
+class FinalCompareResult(StrEnum):
     """Enum that indicates the _overall_ result of a given AggregatedStats' comparison with a
     baseline AggregatedStats. Used to filter out runs that did not pass comparison against a
     baseline, but may want to be stored for future analysis"""

apps/utils/locust_tests/common/stats/stats_compare.py

@@ -6,7 +6,7 @@
 import json
 import logging
 from dataclasses import asdict, dataclass
-from enum import Enum
+from enum import StrEnum
 from typing import Any, Dict, List, Optional, Tuple, Union
 
 from locust.env import Environment
@@ -16,14 +16,14 @@
 from common.stats.stats_loaders import StatsLoader
 
 
-class StatBetterIf(str, Enum):
+class StatBetterIf(StrEnum):
     """Enum representing whether a given stat is better if its value is smaller or larger"""
 
     SMALLER = "SMALLER"
     LARGER = "LARGER"
 
 
-class StatCompareResult(str, Enum):
+class StatCompareResult(StrEnum):
     """Enum representing the result of a particular stat comparison"""
 
     PASS = "PASS"

apps/utils/locust_tests/common/stats/stats_config.py

@@ -2,21 +2,21 @@
 import re
 from argparse import Namespace
 from dataclasses import dataclass, field
-from enum import Enum
+from enum import StrEnum
 from typing import Any, Dict, List, Optional
 
 from locust.argument_parser import LocustArgumentParser
 
 
-class StatsStorageType(str, Enum):
+class StatsStorageType(StrEnum):
     """Enumeration for each available type of storage for JSON stats"""
 
     FILE = "file"
     """Indicates that aggregated statistics will be stored to a local file"""
     S3 = "s3"
     """Indicates that aggregated statistics will be stored to an S3 bucket"""
 
-class StatsComparisonType(str, Enum):
+class StatsComparisonType(StrEnum):
     """Enumeration for each possible type of stats comparison"""
 
     PREVIOUS = "previous"

apps/utils/locust_tests/common/validation.py

@@ -1,7 +1,7 @@
 """Validate tests against target SLIs"""
 import logging
 import time
-from enum import Enum
+from enum import StrEnum, Enum
 from typing import Optional
 
 import gevent
@@ -14,7 +14,7 @@
 """The SLA goals against which to measure the test run's results"""
 
 
-class ValidationResult(str, Enum):
+class ValidationResult(StrEnum):
     """Enum representing the result of failure ratio and SLA validation"""
 
     NOT_APPLICABLE = "NOT_APPLICABLE"

apps/utils/locust_tests/lambda/server-regression/app.py

@@ -3,7 +3,7 @@
 import subprocess
 import urllib.parse
 from dataclasses import asdict, dataclass
-from enum import Enum
+from enum import StrEnum
 from typing import List
 
 import boto3
@@ -40,7 +40,7 @@ class PipelineSignalMessage:
     log_group_name: str
 
 
-class TestResult(str, Enum):
+class TestResult(StrEnum):
     SUCCESS = "SUCCESS"
     FAILURE = "FAILURE"
 

apps/utils/locust_tests/requirements.txt

@@ -1,5 +1,5 @@
 #
-# This file is autogenerated by pip-compile with Python 3.9
+# This file is autogenerated by pip-compile with Python 3.11
 # by the following command:
 #
 #    pip-compile --allow-unsafe --generate-hashes --output-file=requirements.txt requirements.in
@@ -444,10 +444,6 @@ idna==3.7 \
     # via
     #   -r requirements.in
     #   requests
-importlib-metadata==7.1.0 \
-    --hash=sha256:30962b96c0c223483ed6cc7280e7f0199feb01a0e40cfae4d4450fc6fab1f570 \
-    --hash=sha256:b78938b926ee8d5f020fc4772d487045805a55ddbad2ecf21c6d60938dc7fcd2
-    # via flask
 itsdangerous==2.1.2 \
     --hash=sha256:2c2349112351b88699d8d4b6b075022c0808887cb7ad10069318a8b0bc88db44 \
     --hash=sha256:5dbbc68b317e5e42f327f9021763545dc3fc3bfe22e6deb96aaf1fc38874156a
@@ -809,10 +805,6 @@ werkzeug==3.0.1 \
     #   -r requirements.in
     #   flask
     #   locust
-zipp==3.18.1 \
-    --hash=sha256:206f5a15f2af3dbaee80769fb7dc6f249695e940acca08dfb2a4769fe61e538b \
-    --hash=sha256:2884ed22e7d8961de1c9a05142eb69a247f120291bc0206a00a7642f09b5b715
-    # via importlib-metadata
 zope-event==5.0 \
     --hash=sha256:2832e95014f4db26c47a13fdaef84cef2f4df37e66b59d8f1f4a8f319a632c26 \
     --hash=sha256:bac440d8d9891b4068e2b5a2c5e2c9765a9df762944bda6955f96bb9b91e67cd

ops/Dockerfile

@@ -26,7 +26,7 @@ RUN echo flyway.placeholders.type.int4=int4 > /flyway/conf/flyway.conf \
     && echo flyway.postgresql.transactional.lock=false >> /flyway/conf/flyway.conf
 
 FROM amazonlinux:2 as al2
-ARG PYTHON3_TAR_SOURCE="https://www.python.org/ftp/python/3.9.16/Python-3.9.16.tgz"
+ARG PYTHON3_TAR_SOURCE="https://www.python.org/ftp/python/3.11.9/Python-3.11.9.tgz"
 LABEL org.opencontainers.image.source=https://github.com/CMSgov/beneficiary-fhir-data
 
 # Getting systemd to run correctly inside Docker is very tricky. Need to
@@ -76,8 +76,8 @@ RUN curl -o python3.tgz $PYTHON3_TAR_SOURCE && \
     cd $(basename $PYTHON3_TAR_SOURCE .tgz) && \
     ./configure && make && make altinstall
 
-RUN ln -fs /usr/local/bin/python3.9 /usr/bin/python3 && \
-    ln -fs /usr/local/bin/pip3.9 /usr/bin/pip3
+RUN ln -fs /usr/local/bin/python3.11 /usr/bin/python3 && \
+    ln -fs /usr/local/bin/pip3.11 /usr/bin/pip3
 
 RUN /usr/bin/python3 -m pip install --upgrade pip
 

ops/ansible/playbooks-ccs/build_bfd-platinum.yml

@@ -10,7 +10,7 @@
   gather_facts: false
   vars:
     ansible_ssh_pipelining: false
-    python3_version: '3.9.16'
+    python3_version: '3.11.9'
 
   tasks:
     - name: Apply Platinum Role

ops/jenkins/bfd-build-server-load-controller/Dockerfile

@@ -1,4 +1,4 @@
-FROM python:3.9-slim as base
+FROM python:3.11-slim as base
 
 COPY . /app
 WORKDIR /app

ops/jenkins/bfd-build-server-load-node/Dockerfile

@@ -1,6 +1,6 @@
-FROM public.ecr.aws/lambda/python:3.9 as base
+FROM public.ecr.aws/lambda/python:3.11 as base
 
-FROM public.ecr.aws/lambda/python:3.9 as prereqs
+FROM public.ecr.aws/lambda/python:3.11 as prereqs
 RUN yum install git gcc python3-devel -y
 
 # Build required python packages
@@ -10,11 +10,11 @@ RUN pip3 install -r requirements.txt
 
 # Copy function code
 FROM base as node
-COPY --from=python /var/lang/lib/python3.9/site-packages /var/lang/lib/python3.9/site-packages
+COPY --from=python /var/lang/lib/python3.11/site-packages /var/lang/lib/python3.11/site-packages
 COPY --from=python /var/lang/bin /var/lang/bin
 COPY . "${LAMBDA_TASK_ROOT}/locust_tests"
 RUN mv $LAMBDA_TASK_ROOT/locust_tests/services/server-load/* $LAMBDA_TASK_ROOT/
-ENV PATH="/var/lang/lib/python3.9/site-packages:${PATH}"
+ENV PATH="/var/lang/lib/python3.11/site-packages:${PATH}"
 ENV NODE_LOCUST_TESTS_DIR="${LAMBDA_TASK_ROOT}/locust_tests"
 CMD [ "node/node.handler" ]
 

ops/jenkins/bfd-cbc-build/Dockerfile

@@ -28,8 +28,8 @@ RUN curl -o python3.tgz $PYTHON3_TAR_SOURCE && \
     cd $(basename $PYTHON3_TAR_SOURCE .tgz) && \
     ./configure && make && make altinstall
 
-RUN ln -fs /usr/local/bin/python3.9 /usr/bin/python3 && \
-    ln -fs /usr/local/bin/pip3.9 /usr/bin/pip3
+RUN ln -fs /usr/local/bin/python3.11 /usr/bin/python3 && \
+    ln -fs /usr/local/bin/pip3.11 /usr/bin/pip3
 
 RUN /usr/bin/python3 -m pip install --upgrade pip
 

ops/jenkins/bfd-cbc-build/build-push.sh

@@ -16,7 +16,7 @@ docker build "$BUILD_ROOT_DIR" \
   --build-arg PACKER_VERSION="${CBC_PACKER_VERSION:-1.6.6}" \
   --build-arg TFENV_REPO_HASH="${CBC_TFENV_REPO_HASH:-c05c364}" \
   --build-arg TFENV_VERSIONS="${CBC_TFENV_VERSIONS:-1.5.0}" \
-  --build-arg PYTHON3_TAR_SOURCE="${CBC_PYTHON3_TAR_SOURCE:-https://www.python.org/ftp/python/3.9.16/Python-3.9.16.tgz}" \
+  --build-arg PYTHON3_TAR_SOURCE="${CBC_PYTHON3_TAR_SOURCE:-https://www.python.org/ftp/python/3.11.9/Python-3.11.9.tgz}" \
   --build-arg KOTLINC_ZIP_SOURCE="${CBC_KOTLINC_ZIP_SOURCE:-https://github.com/JetBrains/kotlin/releases/download/v1.9.10/kotlin-compiler-1.9.10.zip}" \
   --build-arg CIPHER_SCRIPT="${CIPHER_SCRIPT}" \
   --build-arg YQ_VERSION="${CBC_YQ_VERSION:-4}" \

ops/jenkins/bfd-server-regression-build/Dockerfile

@@ -1,12 +1,12 @@
-FROM public.ecr.aws/lambda/python:3.9 as base
+FROM public.ecr.aws/lambda/python:3.11 as base
 RUN  yum install git gcc python3-devel -y
 
 FROM base as python
 # Install the function's dependencies using file requirements.txt
 # from your project folder.
 COPY requirements.txt .
 RUN  pip3 install -r requirements.txt
-ENV  PATH="/var/lang/lib/python3.9/site-packages:${PATH}"
+ENV  PATH="/var/lang/lib/python3.11/site-packages:${PATH}"
 
 FROM python as app
 # Copy function code

ops/terraform/services/common/modules/bfd_common_cw_alarms_slack_notifier/main.tf

@@ -170,7 +170,7 @@ resource "aws_lambda_function" "this" {
   handler          = "cw_alarms_slack_notifier.handler"
   memory_size      = 128
   package_type     = "Zip"
-  runtime          = "python3.9"
+  runtime          = "python3.11"
   timeout          = local.lambda_timeout_seconds
   environment {
     variables = {

ops/terraform/services/common/terraform.tf

@@ -17,7 +17,7 @@ terraform {
   required_providers {
     aws = {
       source  = "hashicorp/aws"
-      version = "~> 4.22"
+      version = "~> 5.53.0"
     }
   }
 }

ops/terraform/services/eft/lambda_src/sftp_outbound_transfer/s3.py

@@ -1,7 +1,7 @@
-from enum import Enum
+from enum import StrEnum
 
 
-class S3EventType(str, Enum):
+class S3EventType(StrEnum):
     """Represents the types of S3 events that this Lambda is invoked by and supports. The value of
     each Enum is a substring that is matched for on the "eventName" property of an invocation
     event"""

ops/terraform/services/eft/modules/bfd_eft_outbound_o11y/lambda_src/outbound_slack_notifier/dev-requirements.txt

@@ -1,5 +1,5 @@
 #
-# This file is autogenerated by pip-compile with Python 3.10
+# This file is autogenerated by pip-compile with Python 3.11
 # by the following command:
 #
 #    pip-compile --allow-unsafe --constraint=requirements.txt --generate-hashes --output-file=dev-requirements.txt dev-requirements.in
@@ -72,23 +72,10 @@ pylint==3.1.0 \
     --hash=sha256:507a5b60953874766d8a366e8e8c7af63e058b26345cfcb5f91f89d987fd6b74 \
     --hash=sha256:6a69beb4a6f63debebaab0a3477ecd0f559aa726af4954fc948c51f7a2549e23
     # via -r dev-requirements.in
-tomli==2.0.1 \
-    --hash=sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc \
-    --hash=sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f
-    # via
-    #   black
-    #   pylint
 tomlkit==0.12.4 \
     --hash=sha256:5cd82d48a3dd89dee1f9d64420aa20ae65cfbd00668d6f094d7578a78efbb77b \
     --hash=sha256:7ca1cfc12232806517a8515047ba66a19369e71edf2439d0f5824f91032b6cc3
     # via pylint
-typing-extensions==4.10.0 \
-    --hash=sha256:69b1a937c3a517342112fb4c6df7e72fc39a38e7891a5730ed4985b5214b5475 \
-    --hash=sha256:b0abd7c89e8fb96f98db18d86106ff1d90ab692004eb746cf6eda2682f91b3cb
-    # via
-    #   -c requirements.txt
-    #   astroid
-    #   black
 
 # The following packages are considered to be unsafe in a requirements file:
 setuptools==69.1.1 \

ops/terraform/services/eft/modules/bfd_eft_outbound_o11y/lambda_src/outbound_slack_notifier/outbound_slack_notifier.py

@@ -1,7 +1,7 @@
 import json
 import os
 from dataclasses import asdict, dataclass
-from enum import Enum
+from enum import StrEnum
 from typing import Any
 from urllib.request import Request, urlopen
 
@@ -15,7 +15,7 @@
 logger = Logger()
 
 
-class NotificationType(str, Enum):
+class NotificationType(StrEnum):
     FILE_DISCOVERED = "FILE_DISCOVERED"
     TRANSFER_SUCCESS = "TRANSFER_SUCCESS"
     TRANSFER_FAILED = "TRANSFER_FAILED"

ops/terraform/services/eft/modules/bfd_eft_outbound_o11y/lambda_src/outbound_slack_notifier/requirements.txt

@@ -1,5 +1,5 @@
 #
-# This file is autogenerated by pip-compile with Python 3.10
+# This file is autogenerated by pip-compile with Python 3.11
 # by the following command:
 #
 #    pip-compile --allow-unsafe --generate-hashes --output-file=requirements.txt requirements.in

ops/terraform/services/eft/modules/bfd_eft_outbound_o11y/main.tf

@@ -142,7 +142,7 @@ resource "aws_lambda_function" "slack_notifier" {
   handler          = "${local.slack_notifier_lambda_src}.handler"
   memory_size      = 128
   package_type     = "Zip"
-  runtime          = "python3.10"
+  runtime          = "python3.11"
   timeout          = 300
 
   tags = {

ops/terraform/services/eft/terraform.tf

@@ -17,7 +17,7 @@ terraform {
   required_providers {
     aws = {
       source  = "hashicorp/aws"
-      version = "~> 4.22"
+      version = "~> 5.53.0"
     }
   }
 }

ops/terraform/services/pipeline/modules/bfd_pipeline_scheduler/lambda_src/pipeline_scheduler.py

@@ -6,7 +6,7 @@
 import sys
 from dataclasses import dataclass
 from datetime import datetime, timedelta
-from enum import Enum
+from enum import StrEnum
 from typing import Any, Optional
 from urllib.parse import unquote
 
@@ -41,7 +41,7 @@
     sys.exit(0)
 
 
-class S3EventType(str, Enum):
+class S3EventType(StrEnum):
     """Represents the types of S3 events that this Lambda is invoked by and supports. The value of
     each Enum is a substring that is matched for on the "eventName" property of an invocation
     event"""
@@ -50,7 +50,7 @@ class S3EventType(str, Enum):
     OBJECT_REMOVED = "ObjectRemoved"
 
 
-class PipelineLoadType(str, Enum):
+class PipelineLoadType(StrEnum):
     """Represents the possible types of data loads: either the data load is non-synthetic, meaning
     that it contains production data and was placed within the root-level Incoming/Done folders of
     the ETL bucket, or it is synthetic, meaning that it contains non-production, testing data and
@@ -62,7 +62,7 @@ class PipelineLoadType(str, Enum):
     SYNTHETIC = "Synthetic"
 
 
-class PipelineDataStatus(str, Enum):
+class PipelineDataStatus(StrEnum):
     """Represents the possible states of data: either data is available to load, or has been loaded
     by the ETL pipeline. The value of each enum is the parent directory of the incoming file,
     indicating status"""
@@ -75,7 +75,7 @@ def match_str(cls) -> str:
         return "|".join([e.value for e in cls])
 
 
-class RifFileType(str, Enum):
+class RifFileType(StrEnum):
     """Represents all of the possible RIF file types that can be loaded by the BFD ETL Pipeline. The
     value of each enum is a specific substring that is used to match on each type of file
     """

ops/terraform/services/pipeline/modules/bfd_pipeline_scheduler/main.tf

@@ -42,7 +42,7 @@ resource "aws_lambda_function" "this" {
   handler          = "pipeline_scheduler.handler"
   memory_size      = 128
   package_type     = "Zip"
-  runtime          = "python3.9"
+  runtime          = "python3.11"
   timeout          = 300
 
   environment {

ops/terraform/services/pipeline/modules/bfd_pipeline_slis/lambda_src/common.py

@@ -1,11 +1,11 @@
 import os
 from dataclasses import dataclass
-from enum import Enum
+from enum import StrEnum, Enum
 
 METRICS_NAMESPACE = os.environ.get("METRICS_NAMESPACE", "")
 
 
-class RifFileType(str, Enum):
+class RifFileType(StrEnum):
     """Represents all of the possible RIF file types that can be loaded by the BFD ETL Pipeline. The
     value of each enum is a specific substring that is used to match on each type of file"""