catch s3 upload in CI to mock

.github/workflows/docker-build-test.yml

@@ -96,8 +96,9 @@ jobs:
           echo "${{ secrets.AWS_DEFAULT_REGION }}" > ./secrets/aws_default_region.txt
 
 
-      - name: Run Docker Compose
-        run: docker-compose --env-file .env up
-
-      - name: Tear down Docker Compose
-        run: docker-compose down
+      - name: Build and Test
+        env:
+          CI: true
+        run: |
+          docker-compose up --build
+          docker-compose down

src/sr2silo/s3.py

@@ -3,11 +3,14 @@
 from __future__ import annotations
 
 import bz2
+import logging
+import os
 import shutil
 from pathlib import Path
 
 import boto3
 from botocore.exceptions import NoCredentialsError
+from moto import mock_aws
 
 
 def compress_bz2(input_fp: Path, output_fp: Path) -> None:
@@ -68,6 +71,15 @@ def upload_file_to_s3(file_name, bucket, object_name=None, client=None):
     if object_name is None:
         object_name = file_name
 
+    # If running in CI, mock the S3 upload
+    if os.getenv("CI"):
+        logging.info("Running in CI environment, mocking S3 upload with moto.")
+        with mock_aws():
+            s3_client = boto3.client("s3", region_name="us-east-1")
+            s3_client.create_bucket(Bucket=bucket)
+            s3_client.upload_file(file_name, bucket, object_name or file_name)
+            return True
+
     # If client was given, use it; otherwise, get the s3 client
     s3_client = client if client else get_s3_client()