Skip to content

Daily dataset updates #479

Daily dataset updates

Daily dataset updates #479

Workflow file for this run

name: Daily dataset updates
on:
workflow_dispatch: # allow manual triggering
schedule:
- cron: "0 0 * * *" # Every day at midnight
jobs:
update_dataset:
strategy:
matrix:
datasource: ${{ fromJson(vars.DAILY_DATASOURCES) }}
uses: ./.github/workflows/fetch-dataset.yml
with:
datasource: ${{ matrix.datasource }}
coda_token: ${{ inputs.coda_token }}
airtable_api_key: ${{ inputs.airtable_api_key }}
agisf_airtable_base_id: ${{ inputs.agisf_airtable_base_id }}
agisf_airtable_table_id: ${{ inputs.agisf_airtable_table_id }}
youtube_api_key: ${{ inputs.youtube_api_key }}
db_user: ${{ inputs.db_user }}
db_password: ${{ inputs.db_password }}
db_host: ${{ inputs.db_host }}
secrets: inherit
update_indexes:
needs: update_dataset
strategy:
matrix:
datasource: ${{ fromJson(vars.DAILY_DATASOURCES) }}
uses: ./.github/workflows/update-pinecone.yml
with:
datasource: ${{ matrix.datasource }}
db_user: ${{ inputs.db_user }}
db_password: ${{ inputs.db_password }}
db_host: ${{ inputs.db_host }}
openai_api_key: ${{ inputs.openai_api_key }}
pinecone_api_key: ${{ inputs.pinecone_api_key }}
pinecone_environment: ${{ inputs.pinecone_environment }}
secrets: inherit