diff --git a/docker-compose.env.example b/.env.example similarity index 66% rename from docker-compose.env.example rename to .env.example index 95de5df..ee07172 100644 --- a/docker-compose.env.example +++ b/.env.example @@ -5,6 +5,3 @@ TIMELINE_FILE=./../../../data/sr2silo/demo_real/timeline.tsv PRIMER_FILE=./tests/data/samples_large/primers.yaml NEXTCLADE_REFERENCE=sars-cov2 RESULTS_DIR=./../results/A1_10_2024_09_30/20241018_AAG55WNM5/1000/ -AWS_ACCESS_KEY_ID="XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX" -AWS_SECRET_ACCESS_KEY="XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX" -AWS_DEFAULT_REGION="eu-central-1" diff --git a/.github/workflows/docker-build-test.yml b/.github/workflows/docker-build-test.yml index fb324a8..852d4b7 100644 --- a/.github/workflows/docker-build-test.yml +++ b/.github/workflows/docker-build-test.yml @@ -42,10 +42,62 @@ jobs: username: ${{ secrets.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_PASSWORD }} + - name: Create .env file + run: | + echo SAMPLE_DIR=${{ secrets.SAMPLE_DIR }} >> .env + echo SAMPLE_ID=${{ secrets.SAMPLE_ID }} >> .env + echo BATCH_ID=${{ secrets.BATCH_ID }} >> .env + echo TIMELINE_FILE=${{ secrets.TIMELINE_FILE }} >> .env + echo PRIMER_FILE=${{ secrets.PRIMER_FILE }} >> .env + echo NEXTCLADE_REFERENCE=${{ secrets.NEXTCLADE_REFERENCE }} >> .env + echo RESULTS_DIR=${{ secrets.RESULTS_DIR }} >> .env + echo AWS_ACCESS_KEY_ID=${{ secrets.AWS_ACCESS_KEY_ID }} >> .env + echo AWS_SECRET_ACCESS_KEY=${{ secrets.AWS_SECRET_ACCESS_KEY }} >> .env + echo AWS_DEFAULT_REGION=${{ secrets.AWS_DEFAULT_REGION }} >> .env + - name: Build Docker image - run: docker-compose --env-file docker-compose.env build + run: docker-compose --env-file .env build - name: Push to DockerHub if: github.ref == 'refs/heads/main' run: | docker-compose push + + test: + needs: build + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v3 + + - name: Set up Docker Compose + run: | + sudo curl -L "https://github.com/docker/compose/releases/download/1.29.2/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose + sudo chmod +x /usr/local/bin/docker-compose + docker-compose --version + + - name: Create .env file + run: | + echo SAMPLE_DIR=${{ secrets.SAMPLE_DIR }} >> .env + echo SAMPLE_ID=${{ secrets.SAMPLE_ID }} >> .env + echo BATCH_ID=${{ secrets.BATCH_ID }} >> .env + echo TIMELINE_FILE=${{ secrets.TIMELINE_FILE }} >> .env + echo PRIMER_FILE=${{ secrets.PRIMER_FILE }} >> .env + echo NEXTCLADE_REFERENCE=${{ secrets.NEXTCLADE_REFERENCE }} >> .env + echo RESULTS_DIR=${{ secrets.RESULTS_DIR }} >> .env + + + - name: Create Docker secrets files + run: | + mkdir -p ./secrets + echo "${{ secrets.AWS_ACCESS_KEY_ID }}" > ./secrets/aws_access_key_id.txt + echo "${{ secrets.AWS_SECRET_ACCESS_KEY }}" > ./secrets/aws_secret_access_key.txt + echo "${{ secrets.AWS_DEFAULT_REGION }}" > ./secrets/aws_default_region.txt + + + - name: Run Docker Compose + run: docker-compose --env-file .env up + + - name: Tear down Docker Compose + run: docker-compose down diff --git a/README.md b/README.md index 34ac11d..2bb1d67 100644 --- a/README.md +++ b/README.md @@ -71,7 +71,48 @@ $ poetry shell $ pytest ``` -#### Tool Sections +### [WIP]: Run V-Pipe to SILO Transformation +This is currently implemented as script and under heavy development. +To run, we recommend a build as a docker compose as it relies on other RUST components. + +#### Configuration + +Edit the `docker-compose.env` file in the `docker-compose` directory with the following paths: + +```env +SAMPLE_DIR=../../../data/sr2silo/daemon_test/samples/A1_05_2024_10_08/20241024_2411515907/alignments/ +SAMPLE_ID=A1_05_2024_10_08 +BATCH_ID=20241024_2411515907 +TIMELINE_FILE=../../../data/sr2silo/daemon_test/timeline.tsv +NEXTCLADE_REFERENCE=sars-cov2 +RESULTS_DIR=./results +``` + + +#### Docker Secrets +To upload the processed outputs S3 storage is required. + +For sensitive information like AWS credentials, use Docker secrets. Create the following files in the secrets directory: + +- secrets/aws_access_key_id.txt: +```YourAWSAccessKeyId```` + +- secrets/aws_secret_access_key.txt: +```YourAWSSecretAccessKey```` + +- secrets/aws_default_region.txt: +```YourAWSRegion``` + +#### Run Transformation + +To process a single sample, run the following command: + +```sh +docker-compose --env-file .env up --build +``` + + +### Tool Sections The code quality checks run on GitHub can be seen in - ``.github/workflows/test.yml`` for the python package CI/CD, diff --git a/docker-compose.env b/docker-compose.env deleted file mode 100644 index 9a71e1b..0000000 --- a/docker-compose.env +++ /dev/null @@ -1,7 +0,0 @@ -SAMPLE_DIR=./tests/data/samples_large/A1_05_2024_10_08/20241024_2411515907/alignments -SAMPLE_ID=A1_05_2024_10_08 -BATCH_ID=20241024_2411515907 -TIMELINE_FILE=./tests/data/samples_large/timeline_A1_05_2024_10_08.tsv -PRIMER_FILE=./tests/data/samples_large/primers.yaml -NEXTCLADE_REFERENCE=sars-cov2 -RESULTS_DIR=./results diff --git a/docker-compose.yml b/docker-compose.yml index 6819e53..cf59daa 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,3 +1,5 @@ +version: '3.8' + services: sr2silo: build: . @@ -7,7 +9,7 @@ services: - ${PRIMER_FILE}:/app/primers.yaml - ${RESULTS_DIR}:/app/results - ./scripts/database_config.yaml:/app/scripts/database_config.yaml - - ./scripts/reference_genomes.json:/app/scripts/- ./scripts/database_config.yaml:/app/scripts/reference_genomes.json + - ./scripts/reference_genomes.json:/app/scripts/reference_genomes.json environment: - PYTHONUNBUFFERED=1 - SAMPLE_DIR=${SAMPLE_DIR} @@ -16,9 +18,18 @@ services: - TIMELINE_FILE=${TIMELINE_FILE} - PRIMER_FILE=${PRIMER_FILE} - RESULTS_DIR=${RESULTS_DIR} - - AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID} - - AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY} - - AWS_DEFAULT_REGION=${AWS_DEFAULT_REGION} + secrets: + - aws_access_key_id + - aws_secret_access_key + - aws_default_region + +secrets: + aws_access_key_id: + file: ./secrets/aws_access_key_id.txt + aws_secret_access_key: + file: ./secrets/aws_secret_access_key.txt + aws_default_region: + file: ./secrets/aws_default_region.txt volumes: results: diff --git a/scripts/README.md b/scripts/README.md deleted file mode 100644 index 16f8224..0000000 --- a/scripts/README.md +++ /dev/null @@ -1,22 +0,0 @@ -# Docker: Process a Single BAM File - -## Configuration - -Edit the `docker-compose.env` file in the `docker-compose` directory with the following paths: - -```env -SAMPLE_DIR=../../../data/sr2silo/daemon_test/samples/A1_05_2024_10_08/20241024_2411515907/alignments/ -SAMPLE_ID=A1_05_2024_10_08 -BATCH_ID=20241024_2411515907 -TIMELINE_FILE=../../../data/sr2silo/daemon_test/timeline.tsv -NEXTCLADE_REFERENCE=sars-cov2 -RESULTS_DIR=./results -``` - -## Processing - -To process a single sample, run the following command: - -```sh -docker-compose --env-file docker-compose.env up --build -``` diff --git a/src/sr2silo/s3.py b/src/sr2silo/s3.py index 99965de..2353330 100644 --- a/src/sr2silo/s3.py +++ b/src/sr2silo/s3.py @@ -3,7 +3,6 @@ from __future__ import annotations import bz2 -import os import shutil from pathlib import Path @@ -23,13 +22,34 @@ def compress_bz2(input_fp: Path, output_fp: Path) -> None: shutil.copyfileobj(f_in, f_out) +def get_aws_credentials(): + """Get AWS credentials from Docker secrets. + + Returns: + Tuple[str, str, str]: AWS access key ID, AWS secret access key, + and AWS default region. + + Raises: + RuntimeError: If any of the required secrets are missing. + """ + try: + with open("/run/secrets/aws_access_key_id") as f: + aws_access_key_id = f.read().strip() + with open("/run/secrets/aws_secret_access_key") as f: + aws_secret_access_key = f.read().strip() + with open("/run/secrets/aws_default_region") as f: + aws_default_region = f.read().strip() + except FileNotFoundError as e: + raise RuntimeError("Required secret is missing: " + str(e)) + + return aws_access_key_id, aws_secret_access_key, aws_default_region + + def get_s3_client(): """Get an S3 client using AWS credentials from environment variables.""" # Get AWS credentials from environment variables - aws_access_key_id = os.getenv("AWS_ACCESS_KEY_ID") - aws_secret_access_key = os.getenv("AWS_SECRET_ACCESS_KEY") - aws_default_region = os.getenv("AWS_DEFAULT_REGION") + aws_access_key_id, aws_secret_access_key, aws_default_region = get_aws_credentials() # Create an S3 client s3_client = boto3.client(