run_ingestion_pipeline.yaml

name: Data Ingestion Pipeline

on:
    workflow_dispatch:
    schedule:
        - cron: "25 * * * *"

jobs:
    build:
        runs-on: ubuntu-latest
        steps:
            - uses: actions/checkout@v4
            - uses: actions/setup-python@v5
              with:
                python-version: "3.11.9"
            - name: Install dependencies
              run: |
                python -m pip install --upgrade pip
                pip install requests pandas fake-useragent
           
            - name: Run pipeline for most recent results
              run: python -m src.ingestion.run_ingestion
        
            - name: Commit files and push changes
              run: |
                git config user.email "github-actions[bot]@users.noreply.github.com"
                git config user.name "github-actions[bot]"
                git add -A
                timestamp=$(date -u)
                git commit -m "Latest data: ${timestamp}" || exit 0
                git push