From 4230a4d0b7c660f3f01e1b5d6bfdec2d622877bf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gordon=20J=2E=20K=C3=B6hn?= Date: Fri, 20 Dec 2024 16:52:15 +0100 Subject: [PATCH] catch s3 upload in CI to mock --- .github/workflows/docker-build-test.yml | 11 ++++++----- src/sr2silo/s3.py | 12 ++++++++++++ 2 files changed, 18 insertions(+), 5 deletions(-) diff --git a/.github/workflows/docker-build-test.yml b/.github/workflows/docker-build-test.yml index f637486..c6b9de9 100644 --- a/.github/workflows/docker-build-test.yml +++ b/.github/workflows/docker-build-test.yml @@ -96,8 +96,9 @@ jobs: echo "${{ secrets.AWS_DEFAULT_REGION }}" > ./secrets/aws_default_region.txt - - name: Run Docker Compose - run: docker-compose --env-file .env up - - - name: Tear down Docker Compose - run: docker-compose down + - name: Build and Test + env: + CI: true + run: | + docker-compose up --build + docker-compose down diff --git a/src/sr2silo/s3.py b/src/sr2silo/s3.py index 25aa225..edb82c2 100644 --- a/src/sr2silo/s3.py +++ b/src/sr2silo/s3.py @@ -3,11 +3,14 @@ from __future__ import annotations import bz2 +import logging +import os import shutil from pathlib import Path import boto3 from botocore.exceptions import NoCredentialsError +from moto import mock_aws def compress_bz2(input_fp: Path, output_fp: Path) -> None: @@ -68,6 +71,15 @@ def upload_file_to_s3(file_name, bucket, object_name=None, client=None): if object_name is None: object_name = file_name + # If running in CI, mock the S3 upload + if os.getenv("CI"): + logging.info("Running in CI environment, mocking S3 upload with moto.") + with mock_aws(): + s3_client = boto3.client("s3", region_name="us-east-1") + s3_client.create_bucket(Bucket=bucket) + s3_client.upload_file(file_name, bucket, object_name or file_name) + return True + # If client was given, use it; otherwise, get the s3 client s3_client = client if client else get_s3_client()