test: add mock s3 clients / mack CI uploads mock

.github/workflows/docker-build-test.yml

@@ -96,8 +96,9 @@ jobs:
           echo "${{ secrets.AWS_DEFAULT_REGION }}" > ./secrets/aws_default_region.txt
 
 
-      - name: Run Docker Compose
-        run: docker-compose --env-file .env up
-
-      - name: Tear down Docker Compose
-        run: docker-compose down
+      - name: Build and Test
+        env:
+          CI: true
+        run: |
+          docker-compose up --build
+          docker-compose down

src/sr2silo/s3.py

@@ -3,11 +3,14 @@
 from __future__ import annotations
 
 import bz2
+import logging
+import os
 import shutil
 from pathlib import Path
 
 import boto3
 from botocore.exceptions import NoCredentialsError
+from moto import mock_aws
 
 
 def compress_bz2(input_fp: Path, output_fp: Path) -> None:
@@ -62,14 +65,23 @@ def get_s3_client():
     return s3_client
 
 
-def upload_file_to_s3(file_name, bucket, object_name=None):
+def upload_file_to_s3(file_name, bucket, object_name=None, client=None):
     """Upload a file to an S3 bucket"""
     # If S3 object_name was not specified, use file_name
     if object_name is None:
         object_name = file_name
 
-    # get the s3 client
-    s3_client = get_s3_client()
+    # If running in CI, mock the S3 upload
+    if os.getenv("CI"):
+        logging.info("Running in CI environment, mocking S3 upload with moto.")
+        with mock_aws():
+            s3_client = boto3.client("s3", region_name="us-east-1")
+            s3_client.create_bucket(Bucket=bucket)
+            s3_client.upload_file(file_name, bucket, object_name or file_name)
+            return True
+
+    # If client was given, use it; otherwise, get the s3 client
+    s3_client = client if client else get_s3_client()
 
     try:
         s3_client.upload_file(file_name, bucket, object_name)

tests/test_s3.py

@@ -2,7 +2,22 @@
 
 from __future__ import annotations
 
-from sr2silo.s3 import compress_bz2
+import boto3
+import pytest
+from moto import mock_aws
+
+from sr2silo.s3 import compress_bz2, upload_file_to_s3
+
+
+@pytest.fixture
+def s3_client():
+    """Mocked S3 client."""
+    with mock_aws():
+        # Initialize the mock S3 service
+        s3 = boto3.client("s3", region_name="us-east-1")
+        # Create a mock bucket
+        s3.create_bucket(Bucket="test-bucket")
+        yield s3
 
 
 def test_compress_bz2(tmp_path):
@@ -11,10 +26,10 @@ def test_compress_bz2(tmp_path):
     test_file = tmp_path / "test.txt"
     test_file.write_text(
         """
-                        This is a test file – it should compress well.
-                        This is a test file – it should compress well.
-                        This is a test file – it should compress well.
-                         """
+        This is a test file – it should compress well.
+        This is a test file – it should compress well.
+        This is a test file – it should compress well.
+        """
     )
 
     # Compress the test file
@@ -25,3 +40,18 @@ def test_compress_bz2(tmp_path):
     assert compressed_file.exists()
     # Check that the compressed file is smaller than the original file
     assert compressed_file.stat().st_size < test_file.stat().st_size
+
+
+def test_upload_file_to_s3(tmp_path, s3_client):
+    """Test the upload_file_to_s3 function."""
+    # Create a test file
+    test_file = tmp_path / "test.txt"
+    test_file.write_text("This is a test file.")
+
+    # Upload the test file to the mock S3 bucket
+    upload_file_to_s3(str(test_file), "test-bucket", "test.txt", client=s3_client)
+
+    # Check that the file exists in the mock S3 bucket
+    response = s3_client.list_objects_v2(Bucket="test-bucket")
+    assert "Contents" in response
+    assert any(obj["Key"] == "test.txt" for obj in response["Contents"])