Test with Python 3.13, drop support for Python 3.8

.github/workflows/ci_workflow.yml

@@ -5,51 +5,36 @@ name: Test tap-dynamodb
 
 on: [push]
 
+env:
+  FORCE_COLOR: 1
+
 jobs:
   linting:
     runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        # Only lint using the primary version used for dev
-        python-version: ["3.12"]
     steps:
     - uses: actions/checkout@v4
-    - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v5
+    - uses: actions/setup-python@v5
       with:
-        python-version: ${{ matrix.python-version }}
-    - name: Install pipx and Poetry
-      run: |
-        pip install pipx poetry
-    - name: Run lint command from tox.ini
-      run: |
-        pipx run tox -e lint
+        python-version: 3.x
+    - run: pipx install tox
+    - run: tox -e lint
 
   pytest:
     runs-on: ubuntu-latest
-    env:
-      GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}}
     strategy:
       fail-fast: false
       matrix:
         python-version:
-        - "3.8"
         - "3.9"
         - "3.10"
         - "3.11"
         - "3.12"
+        - "3.13"
     steps:
     - uses: actions/checkout@v4
-    - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v5
+    - uses: actions/setup-python@v5
       with:
         python-version: ${{ matrix.python-version }}
-    - name: Install Poetry
-      run: |
-        pip install poetry
-    - name: Install dependencies
-      run: |
-        poetry install
-    - name: Test with pytest
-      run: |
-        poetry run pytest
+        allow-prereleases: true
+    - run: pipx install tox
+    - run: tox -e py

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "meltanolabs-tap-dynamodb"
-version = "0.3.0"
+version = "0.4.0"
 description = "`tap-dynamodb` is a Singer tap for DynamoDB, built with the Meltano Singer SDK."
 readme = "README.md"
 authors = ["Pat Nadolny"]
@@ -14,38 +14,56 @@ packages = [
 ]
 
 [tool.poetry.dependencies]
-python = ">=3.8"
-singer-sdk = { version="^0.41.0" }
-fs-s3fs = { version = "^1.1.1", optional = true }
-boto3 = "^1.34.136"
-genson = "^1.3.0"
-orjson = "^3.10.5"
+python = ">=3.9"
+boto3 = "~=1.35.35"
+fs-s3fs = { version = "~=1.1.1", optional = true }
+genson = "~=1.3.0"
+orjson = "~=3.10.5"
+
+[tool.poetry.dependencies.singer-sdk]
+version = "~=0.41.0"
 
 [tool.poetry.group.dev.dependencies]
-pytest = "^8.2.2"
-flake8 = "^5.0.4"
-darglint = { version = "^1.8.1", "python" = "<4" }
-black = "^24.4.2"
-pyupgrade = "^3.3.1"
-mypy = "^1.10.1"
-isort = "^5.11.5"
-singer-sdk = { version="^0.41.0", extras = ["testing"] }
-moto = "^5.0.10"
-coverage = "^7.5.4"
-pydocstyle = "^6.3.0"
+boto3-stubs = {extras = ["dynamodb", "sts"], version = "~=1.35.35"}
+coverage = ">=7.5.4"
+moto = ">=5.0.10"
+mypy = ">=1.11.2"
+pytest = ">=8.2.2"
+
+[tool.poetry.group.dev.dependencies.singer-sdk]
+version="~=0.41.0"
+extras = ["testing"]
 
 [tool.poetry.extras]
 s3 = ["fs-s3fs"]
 
-[tool.isort]
-profile = "black"
-multi_line_output = 3 # Vertical Hanging Indent
-src_paths = "tap_dynamodb"
-
 [tool.mypy]
-python_version = "3.9"
+python_version = "3.12"
 warn_unused_configs = true
 
+[tool.ruff]
+target-version = "py39"
+
+[tool.ruff.lint]
+select = [
+  "F",    # pyflakes
+  "E",    # pycodestyle (errors)
+  "I",    # isort
+  "D",    # pydocstyle
+  "UP",   # pyupgrade
+  "TCH",  # flake8-type-checking
+]
+
+[tool.ruff.lint.per-file-ignores]
+"tests/*" = ["D"]
+
+[tool.ruff.lint.pydocstyle]
+convention = "google"
+
+[tool.coverage]
+source = ["tap_dynamodb/"]
+omit = ["*/.tox/*", "*/test/*"]
+
 [build-system]
 requires = ["poetry-core>=1.0.8"]
 build-backend = "poetry.core.masonry.api"

tap_dynamodb/connectors/aws_boto_connector.py

@@ -1,7 +1,13 @@
+"""AWS Boto Connector class for Singer SDK."""
+
 import logging
 import os
-from typing import Union
+import typing as t
 
+import boto3.session
+from boto3.resources.base import ServiceResource
+from boto3.session import Session
+from botocore.client import BaseClient
 from singer_sdk import typing as th  # JSON schema typing helpers
 
 try:
@@ -12,6 +18,10 @@
         "Please install it with `poetry add boto3`."
     )
 
+if t.TYPE_CHECKING:
+    from mypy_boto3_sts import STSClient
+
+
 AWS_AUTH_CONFIG = th.PropertiesList(
     th.Property(
         "aws_access_key_id",
@@ -66,7 +76,14 @@
 ).to_dict()
 
 
-class AWSBotoConnector:
+_T = t.TypeVar("_T", bound=t.Union[ServiceResource, BaseClient])
+_R = t.TypeVar("_R", bound=ServiceResource)
+_C = t.TypeVar("_C", bound=BaseClient)
+
+
+class AWSBotoConnector(t.Generic[_R, _C]):
+    """AWS Boto Connector class for Singer SDK."""
+
     def __init__(
         self,
         config: dict,
@@ -80,8 +97,8 @@ def __init__(
         """
         self._service_name = service_name
         self._config = config
-        self._client = None
-        self._resource = None
+        self._client: _C | None = None
+        self._resource: _R | None = None
         # config for use environment variables
         if config.get("use_aws_env_vars"):
             self.aws_access_key_id = os.environ.get("AWS_ACCESS_KEY_ID")
@@ -118,7 +135,7 @@ def logger(self) -> logging.Logger:
         return logging.getLogger("aws_boto_connector")
 
     @property
-    def client(self) -> boto3.client:
+    def client(self) -> _C:
         """Return the boto3 client for the service.
 
         Returns:
@@ -128,11 +145,11 @@ def client(self) -> boto3.client:
             return self._client
         else:
             session = self.get_session()
-            self._client = self.get_client(session, self._service_name)
-            return self._client
+            self._client = self.get_client(session, self._service_name)  # type: ignore[assignment]
+            return self._client  # type: ignore[return-value]
 
     @property
-    def resource(self) -> boto3.resource:
+    def resource(self) -> _R:
         """Return the boto3 resource for the service.
 
         Returns:
@@ -142,10 +159,10 @@ def resource(self) -> boto3.resource:
             return self._resource
         else:
             session = self.get_session()
-            self._resource = self.get_resource(session, self._service_name)
-            return self._resource
+            self._resource = self.get_resource(session, self._service_name)  # type: ignore[assignment]
+            return self._resource  # type: ignore[return-value]
 
-    def get_session(self) -> boto3.session:
+    def get_session(self) -> Session:
         """Return the boto3 session.
 
         Returns:
@@ -168,10 +185,8 @@ def get_session(self) -> boto3.session:
                 region_name=self.aws_default_region,
             )
             self.logger.info(
-                (
-                    "Authenticating using access key id, secret access key, and "
-                    "session token."
-                )
+                "Authenticating using access key id, secret access key, and "
+                "session token."
             )
         elif (
             self.aws_access_key_id
@@ -199,9 +214,7 @@ def get_session(self) -> boto3.session:
             session = self._assume_role(session, self.aws_assume_role_arn)
         return session
 
-    def _factory(
-        self, aws_obj, service_name: str
-    ) -> Union[boto3.resource, boto3.client]:
+    def _factory(self, aws_obj: t.Callable[..., _T], service_name: str) -> _T:
         if self.aws_endpoint_url:
             return aws_obj(
                 service_name,
@@ -212,7 +225,7 @@ def _factory(
                 service_name,
             )
 
-    def get_resource(self, session: boto3.session, service_name: str) -> boto3.resource:
+    def get_resource(self, session: Session, service_name: str) -> ServiceResource:
         """Return the boto3 resource for the service.
 
         Args:
@@ -224,9 +237,7 @@ def get_resource(self, session: boto3.session, service_name: str) -> boto3.resou
         """
         return self._factory(session.resource, service_name)
 
-    def get_client(
-        self, session: boto3.session.Session, service_name: str
-    ) -> boto3.client:
+    def get_client(self, session: Session, service_name: str) -> BaseClient:
         """Return the boto3 client for the service.
 
         Args:
@@ -238,13 +249,12 @@ def get_client(
         """
         return self._factory(session.client, service_name)
 
-    def _assume_role(
-        self, session: boto3.session.Session, role_arn: str
-    ) -> boto3.session.Session:
+    def _assume_role(self, session: Session, role_arn: str) -> Session:
         # TODO: use for auto refresh https://github.com/benkehoe/aws-assume-role-lib
-        sts_client = self.get_client(session, "sts")
+        sts_client: STSClient = self.get_client(session, "sts")  # type: ignore[assignment]
         response = sts_client.assume_role(
-            RoleArn=role_arn, RoleSessionName="tap-dynamodb"
+            RoleArn=role_arn,
+            RoleSessionName="tap-dynamodb",
         )
         return boto3.Session(
             aws_access_key_id=response["Credentials"]["AccessKeyId"],

tap_dynamodb/dynamodb_connector.py

@@ -1,12 +1,15 @@
+"""DynamoDB connector class."""
+
 import genson
 import orjson
 from botocore.exceptions import ClientError
+from mypy_boto3_dynamodb import DynamoDBClient, DynamoDBServiceResource
 
 from tap_dynamodb.connectors.aws_boto_connector import AWSBotoConnector
 from tap_dynamodb.exception import EmptyTableException
 
 
-class DynamoDbConnector(AWSBotoConnector):
+class DynamoDbConnector(AWSBotoConnector[DynamoDBServiceResource, DynamoDBClient]):
     """DynamoDB connector class."""
 
     def __init__(
@@ -45,6 +48,7 @@ def _recursively_drop_required(self, schema: dict) -> None:
                     self._recursively_drop_required(schema["properties"][prop])
 
     def list_tables(self, include=None):
+        """List tables in DynamoDB."""
         try:
             tables = []
             for table in self.resource.tables.all():
@@ -61,6 +65,7 @@ def list_tables(self, include=None):
             return tables
 
     def get_items_iter(self, table_name: str, scan_kwargs_override: dict):
+        """Get items from a table in DynamoDB."""
         scan_kwargs = scan_kwargs_override.copy()
         if "ConsistentRead" not in scan_kwargs:
             scan_kwargs["ConsistentRead"] = True
@@ -106,6 +111,7 @@ def _get_sample_records(
     def get_table_json_schema(
         self, table_name: str, sample_size, scan_kwargs: dict, strategy: str = "infer"
     ) -> dict:
+        """Get the JSON schema for a table in DynamoDB."""
         sample_records = self._get_sample_records(table_name, sample_size, scan_kwargs)
 
         if not sample_records:
@@ -127,5 +133,6 @@ def get_table_json_schema(
         return schema
 
     def get_table_key_properties(self, table_name):
+        """Get the key properties for a table in DynamoDB."""
         key_schema = self.resource.Table(table_name).key_schema
         return [key.get("AttributeName") for key in key_schema]