terraform-pyspark.yml

name: Deploy and Run PySpark Job on Dataproc

on:
  push:
    branches:
      - main

jobs:
  deploy:
    runs-on: ubuntu-latest

    steps:
    - name: Checkout repository
      uses: actions/checkout@v2

    # Authenticate using the correct auth action
    - name: Authenticate with Google Cloud
      uses: google-github-actions/auth@v1
      with:
        credentials_json: ${{ secrets.GCP_SA_KEY }}  

    # Set up Google Cloud SDK
    - name: Set up Google Cloud SDK
      uses: google-github-actions/setup-gcloud@v1
      with:
        project_id: "liqour-store-etl"

    - name: Setup Terraform
      uses: hashicorp/setup-terraform@v1
      with:
        terraform_version: 1.5.7

    - name: Terraform init and validate
      run: |
        terraform init
        terraform validate

    - name: Terraform apply
      run: |
        terraform apply -auto-approve

    # Step to upload the PySpark job script from GitHub repo to the GCS bucket
    - name: Upload PySpark job to GCS
      run: |
        gsutil cp main.py gs://liquor-store-bucket/main.py

    # Instantiate the Dataproc workflow template
    - name: Run Dataproc Workflow Template
      run: |
        gcloud dataproc workflow-templates instantiate liquor-store-etl-workflow \
          --region us-central1