Skip to content

Data Ingestion Pipeline #1338

Data Ingestion Pipeline

Data Ingestion Pipeline #1338

name: Data Ingestion Pipeline
on:
workflow_dispatch:
schedule:
- cron: "25 * * * *"
jobs:
build:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: actions/setup-python@v5
with:
python-version: "3.11.9"
- name: Install dependencies
run: |
python -m pip install --upgrade pip
pip install requests pandas fake-useragent
- name: Run pipeline for most recent results
run: python -m src.ingestion.run_ingestion
- name: Commit files and push changes
run: |
git config user.email "github-actions[bot]@users.noreply.github.com"
git config user.name "github-actions[bot]"
git add -A
timestamp=$(date -u)
git commit -m "Latest data: ${timestamp}" || exit 0
git push