Fetch and ingest #111
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Fetch and ingest | |
| on: | |
| schedule: | |
| # Note times are in UTC, which is 1 or 2 hours behind CET depending on daylight savings. | |
| # | |
| # Note the actual runs might be late. | |
| # Numerous people were confused, about that, including me: | |
| # - https://github.community/t/scheduled-action-running-consistently-late/138025/11 | |
| # - https://github.com/github/docs/issues/3059 | |
| # | |
| # Note, '*' is a special character in YAML, so you have to quote this string. | |
| # | |
| # Docs: | |
| # - https://docs.github.com/en/actions/learn-github-actions/events-that-trigger-workflows#schedule | |
| # | |
| # Tool that deciphers this particular format of crontab string: | |
| # - https://crontab.guru/ | |
| # | |
| # Runs at 4pm UTC (12pm EDT) since curation by NCBI happens on the East Coast. | |
| - cron: '0 16 * * 2' | |
| repository_dispatch: | |
| types: | |
| - fetch-and-ingest | |
| # Manually triggered using GitHub's UI | |
| workflow_dispatch: | |
| inputs: | |
| trial_name: | |
| description: 'Short name for a trial run. WARNING: without this we will overwrite files in s3://nextstrain-data/files/workflows/rsv' | |
| jobs: | |
| set_config_overrides: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - id: s3_dst | |
| run: | | |
| S3_DST=s3://nextstrain-data/files/workflows/rsv | |
| if [[ -n "$TRIAL_NAME" ]]; then | |
| S3_DST+=/trial/"$TRIAL_NAME" | |
| fi | |
| echo "s3_dst=$S3_DST" >> "$GITHUB_OUTPUT" | |
| env: | |
| TRIAL_NAME: ${{ inputs.trial_name }} | |
| - id: trigger_rebuild | |
| run: | | |
| TRIGGER_REBUILD=true | |
| if [[ -n "$TRIAL_NAME" ]]; then | |
| TRIGGER_REBUILD=false | |
| fi | |
| echo "trigger_rebuild=$TRIGGER_REBUILD" >> "$GITHUB_OUTPUT" | |
| outputs: | |
| s3_dst: ${{ steps.s3_dst.outputs.s3_dst }} | |
| trigger_rebuild: ${{ steps.trigger_rebuild.outputs.trigger_rebuild }} | |
| fetch-and-ingest: | |
| needs: [set_config_overrides] | |
| permissions: | |
| id-token: write | |
| uses: nextstrain/.github/.github/workflows/pathogen-repo-build.yaml@master | |
| secrets: inherit | |
| with: | |
| runtime: aws-batch | |
| run: | | |
| nextstrain build \ | |
| --aws-batch \ | |
| --detach \ | |
| --no-download \ | |
| --cpus 8 \ | |
| --memory 32gib \ | |
| --env AWS_ACCESS_KEY_ID \ | |
| --env AWS_SECRET_ACCESS_KEY \ | |
| --env PAT_GITHUB_DISPATCH="$GH_TOKEN_NEXTSTRAIN_BOT_WORKFLOW_DISPATCH" \ | |
| --env S3_DST \ | |
| --env TRIGGER_REBUILD \ | |
| ingest \ | |
| --configfiles config/config.yaml config/optional.yaml \ | |
| --config s3_dst="$S3_DST" trigger_rebuild="$TRIGGER_REBUILD" \ | |
| --printshellcmds | |
| env: | | |
| S3_DST: ${{ needs.set_config_overrides.outputs.s3_dst }} | |
| TRIGGER_REBUILD: ${{ needs.set_config_overrides.outputs.trigger_rebuild }} |