fetch-dataset.yml

name: Fetch Alignment Research Dataset

on:
  workflow_call:
    inputs:
      datasource:
        type: string
        required: true
      coda_token:
        type: string
        required: true
      airtable_api_key:
        type: string
        required: true
      agisf_airtable_base_id:
        type: string
        required: true
      agisf_airtable_table_id:
        type: string
        required: true
      youtube_api_key:
        type: string
        required: true
      db_user:
        type: string
        required: true
      db_password:
        type: string
        required: true
      db_host:
        type: string
        required: true
  workflow_dispatch: # allow manual triggering
    inputs:
      datasource:
        description: 'The datasource to process'
        type: choice
        options:
          - agentmodels
          - agisf
          - aisafety.info
          - alignment_newsletter
          - alignmentforum
          - arbital
          - arxiv
          - blogs
          - distill
          - eaforum
          - indices
          - lesswrong
          - special_docs
          - youtube

jobs:
  build-dataset:
    runs-on: ubuntu-latest

    steps:
    - name: Checkout repository
      uses: actions/checkout@v2

    - name: Setup Python environment
      uses: actions/setup-python@v2
      with:
        python-version: '3.11'

    - name: Install Pandoc
      run: |
        if [ "${{ inputs.datasource }}" = "gdocs" ]; then
          sudo apt-get update
          sudo apt-get -y install pandoc
        fi

    - name: Install dependencies
      run: pip install -r requirements.txt

    - name: Process dataset
      env:
        CODA_TOKEN: ${{ secrets.CODA_TOKEN || inputs.coda_token }}
        AIRTABLE_API_KEY: ${{ secrets.AIRTABLE_API_KEY || inputs.airtable_api_key }}
        AGISF_AIRTABLE_BASE_ID: ${{ secrets.AGISF_AIRTABLE_BASE_ID || inputs.agisf_airtable_base_id }}
        AGISF_AIRTABLE_TABLE_ID: ${{ secrets.AGISF_AIRTABLE_TABLE_ID || inputs.agisf_airtable_table_id }}
        YOUTUBE_API_KEY: ${{ secrets.YOUTUBE_API_KEY || inputs.youtube_api_key }}
        OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY || inputs.openai_api_key }}
        ARD_DB_USER: ${{ secrets.ARD_DB_USER || inputs.db_user }}
        ARD_DB_PASSWORD: ${{ secrets.ARD_DB_PASSWORD || inputs.db_password }}
        ARD_DB_HOST: ${{ secrets.ARD_DB_HOST || inputs.db_host }}
        ARD_DB_NAME: alignment_research_dataset
      run: python main.py fetch ${{ inputs.datasource }}