tf conditional apply

.github/workflows/terraform-pyspark.yml

@@ -17,7 +17,7 @@ jobs:
     - name: Authenticate with Google Cloud
       uses: google-github-actions/auth@v1
       with:
-        credentials_json: ${{ secrets.GCP_SA_KEY }}  
+        credentials_json: ${{ secrets.GCP_SA_KEY }}
 
     # Set up Google Cloud SDK
     - name: Set up Google Cloud SDK
@@ -35,17 +35,31 @@ jobs:
         terraform init
         terraform validate
 
-    - name: Terraform apply
+    - name: Terraform Plan
+      id: plan
+      run: terraform plan -out=plan.out
+      continue-on-error: true # Ensure the job continues even if no changes
+
+    # Conditional Terraform apply step based on the plan output
+    - name: Check if changes need to be applied
+      if: ${{ steps.plan.outcome == 'success' && steps.plan.conclusion == 'success' }}
       run: |
-        terraform apply -auto-approve
+        terraform apply -auto-approve plan.out
+
+    # Skip apply if no changes (plan success with no diffs)
+    - name: Skip apply if no changes
+      if: ${{ steps.plan.outcome != 'success' }}
+      run: echo "No changes to apply."
 
     # Step to upload the PySpark job script from GitHub repo to the GCS bucket
     - name: Upload PySpark job to GCS
+      if: ${{ steps.plan.conclusion == 'success' }}
       run: |
         gsutil cp main.py gs://liquor-store-bucket/main.py
 
     # Instantiate the Dataproc workflow template
     - name: Run Dataproc Workflow Template
+      if: ${{ steps.plan.conclusion == 'success' }}
       run: |
         gcloud dataproc workflow-templates instantiate liquor-store-etl-workflow \
           --region us-central1

README.md

@@ -1,13 +1,13 @@
 # liqour-sales-spark-etl
 
 Develop to do: 
+- clean up repo
+- describe in readme file
 
+- add a linter to yml
 - add biq query to tf, github actions and script 
 - big query analyse results examples
 
-- clean up repo
-- describe in readme file
-
 
 take data from: 
 https://console.cloud.google.com/bigquery?_ga=2.8611638.288326788.1726990215-459272757.1699721835&project=mybookdashboard&ws=!1m5!1m4!4m3!1sbigquery-public-data!2siowa_liquor_sales!3ssales
@@ -34,6 +34,9 @@ pip install -r requirements.txt
 
 
 
+
+
+# 
 gcloud dataproc workflow-templates set-managed-cluster liquor-etl-workflow \
   --region us-central1 \
   --cluster-name liquor-etl-workflow-jobs \

main.py

@@ -85,12 +85,20 @@ def main():
 
 
     df_transformed = transform(df)
-
+    try:
+        df_transformed = transform(df)
+
+        print('Dataframe transformed')
+
+        df_transformed.show()
+    except Exception as e:
+
+        print("Transformation failed", e)
+
+        raise
     print('Dataframe transformed')
 
-    # load to big query
-    df_transformed.show()
-
+    # load to big query.....
 
     spark.stop()
 

main.tf

@@ -13,6 +13,14 @@ resource "google_storage_bucket" "data_bucket" {
   }
 }
 
+# remote state management
+terraform {
+  backend "gcs" {
+    bucket = "liquor-store-terraform-state"
+    prefix = "terraform/state"
+  }
+}
+
 # Create a Dataproc workflow template
 resource "google_dataproc_workflow_template" "template" {
   name     = "liquor-store-etl-workflow"