Improve Action Worflow

deepset-ai · Oct 15, 2024 · 169e2d1 · 169e2d1
1 parent 4ea15f3
commit 169e2d1
Show file tree

Hide file tree

Showing 2 changed files with 156 additions and 50 deletions.
diff --git a/.github/workflows/deploy-dev.yml b/.github/workflows/deploy-dev.yml
@@ -5,6 +5,10 @@ on:
     branches:
       - dev
 
+env:
+  DEEPSET_CLOUD_API_KEY: ${{ secrets.DEEPSET_CLOUD_API_KEY }}
+  DEEPSET_CLOUD_WORKSPACE_NAME: "your-dev-workspace-name"
+
 jobs:
   deploy:
     runs-on: ubuntu-latest
@@ -20,10 +24,19 @@ jobs:
           sudo wget https://github.com/mikefarah/yq/releases/latest/download/yq_linux_amd64 -O /usr/bin/yq
           sudo chmod +x /usr/bin/yq
 
+      - name: Check if workspace exists
+        run: |
+          response=$(curl --silent --show-error --fail \
+            --request GET \
+            --url "https://api.cloud.deepset.ai/api/v1/workspaces/${DEEPSET_CLOUD_WORKSPACE_NAME}" \
+            --header "Authorization: Bearer ${DEEPSET_CLOUD_API_KEY}")
+          
+          if [[ $response == *"error"* ]]; then
+            echo "Workspace ${DEEPSET_CLOUD_WORKSPACE_NAME} does not exist"
+            exit 1
+          fi
+
       - name: Create or Update Pipelines
-        env:
-          DEEPSET_CLOUD_API_KEY: ${{ secrets.DEEPSET_CLOUD_API_KEY }}
-          DEEPSET_CLOUD_WORKSPACE_NAME: "your-dev-workspace-name"
         run: |
           for pipeline_dir in pipelines/*/; do
             pipeline_name=$(basename "$pipeline_dir")
@@ -35,40 +48,68 @@ jobs:
               exit 1
             fi
             
-            indexing_content=$(yq eval -o=json "$indexing_yaml")
-            query_content=$(yq eval -o=json "$query_yaml")
-            
-            payload=$(jq -n \
-              --arg name "$pipeline_name" \
-              --arg indexing "$indexing_content" \
-              --arg query "$query_content" \
-              '{
-                "name": $name,
-                "yaml": null,
-                "indexing_yaml": $indexing,
-                "query_yaml": $query,
-                "deepset_cloud_version": "v2"
-              }')
+            # Check if pipeline exists
+            pipeline_exists=$(curl --silent --show-error --fail \
+              --request GET \
+              --url "https://api.cloud.deepset.ai/api/v1/workspaces/${DEEPSET_CLOUD_WORKSPACE_NAME}/pipelines/${pipeline_name}" \
+              --header "Authorization: Bearer ${DEEPSET_CLOUD_API_KEY}")
             
-            response=$(curl --silent --show-error --fail \
-              --request POST \
-              --url "https://api.cloud.deepset.ai/api/v1/workspaces/${DEEPSET_CLOUD_WORKSPACE_NAME}/pipelines" \
-              --header "Authorization: Bearer ${DEEPSET_CLOUD_API_KEY}" \
-              --header 'Content-Type: application/json' \
-              --data "$payload")
-            
-            echo "Pipeline creation/update response for $pipeline_name: $response"
+            if [[ $pipeline_exists == *"error"* ]]; then
+              # Pipeline doesn't exist, create new pipeline
+              indexing_content=$(yq eval -o=json "$indexing_yaml")
+              query_content=$(yq eval -o=json "$query_yaml")
+              
+              payload=$(jq -n \
+                --arg name "$pipeline_name" \
+                --arg indexing "$indexing_content" \
+                --arg query "$query_content" \
+                '{
+                  "name": $name,
+                  "yaml": null,
+                  "indexing_yaml": $indexing,
+                  "query_yaml": $query,
+                  "deepset_cloud_version": "v2"
+                }')
+              
+              response=$(curl --silent --show-error --fail \
+                --request POST \
+                --url "https://api.cloud.deepset.ai/api/v1/workspaces/${DEEPSET_CLOUD_WORKSPACE_NAME}/pipelines" \
+                --header "Authorization: Bearer ${DEEPSET_CLOUD_API_KEY}" \
+                --header 'Content-Type: application/json' \
+                --data "$payload")
+              
+              echo "Pipeline creation response for $pipeline_name: $response"
+            else
+              # Pipeline exists, update only query pipeline
+              query_content=$(yq eval -o=json "$query_yaml")
+              
+              payload=$(jq -n \
+                --arg name "$pipeline_name" \
+                --arg query "$query_content" \
+                '{
+                  "name": $name,
+                  "yaml": null,
+                  "query_yaml": $query,
+                  "deepset_cloud_version": "v2"
+                }')
+              
+              response=$(curl --silent --show-error --fail \
+                --request PATCH \
+                --url "https://api.cloud.deepset.ai/api/v1/workspaces/${DEEPSET_CLOUD_WORKSPACE_NAME}/pipelines/${pipeline_name}" \
+                --header "Authorization: Bearer ${DEEPSET_CLOUD_API_KEY}" \
+                --header 'Content-Type: application/json' \
+                --data "$payload")
+              
+              echo "Pipeline update response for $pipeline_name: $response"
+            fi
             
             if [[ $response == *"error"* ]]; then
-              echo "Pipeline creation/update failed for $pipeline_name"
+              echo "Pipeline operation failed for $pipeline_name"
               exit 1
             fi
           done
 
       - name: Validate Pipelines
-        env:
-          DEEPSET_CLOUD_API_KEY: ${{ secrets.DEEPSET_CLOUD_API_KEY }}
-          DEEPSET_CLOUD_WORKSPACE_NAME: "your-dev-workspace-name"
         run: |
           validation_response=$(curl --silent --show-error --fail \
             --request POST \

diff --git a/.github/workflows/deploy-prod.yml b/.github/workflows/deploy-prod.yml
@@ -6,6 +6,10 @@ on:
       - main
   workflow_dispatch:
 
+env:
+  DEEPSET_CLOUD_API_KEY: ${{ secrets.DEEPSET_CLOUD_API_KEY }}
+  DEEPSET_CLOUD_WORKSPACE_NAME: "your-prod-workspace-name"
+
 jobs:
   deploy:
     runs-on: ubuntu-latest
@@ -14,38 +18,99 @@ jobs:
       - name: Checkout code
         uses: actions/checkout@v3
 
-      - name: Install jq
-        run: sudo apt-get install jq
+      - name: Install jq and yq
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y jq
+          sudo wget https://github.com/mikefarah/yq/releases/latest/download/yq_linux_amd64 -O /usr/bin/yq
+          sudo chmod +x /usr/bin/yq
+
+      - name: Check if workspace exists
+        run: |
+          response=$(curl --silent --show-error --fail \
+            --request GET \
+            --url "https://api.cloud.deepset.ai/api/v1/workspaces/${DEEPSET_CLOUD_WORKSPACE_NAME}" \
+            --header "Authorization: Bearer ${DEEPSET_CLOUD_API_KEY}")
+          
+          if [[ $response == *"error"* ]]; then
+            echo "Workspace ${DEEPSET_CLOUD_WORKSPACE_NAME} does not exist"
+            exit 1
+          fi
 
       - name: Create or Update Pipelines
-        env:
-          DEEPSET_CLOUD_API_KEY: ${{ secrets.DEEPSET_CLOUD_API_KEY }}
-          DEEPSET_CLOUD_WORKSPACE_NAME: "your-prod-workspace-name"
         run: |
-          for pipeline in pipelines/*.yaml; do
-            pipeline_content=$(cat "$pipeline" | jq -sR .)
-            response=$(curl --silent --show-error --fail \
-              --request POST \
-              --url "https://api.cloud.deepset.ai/api/v1/workspaces/${DEEPSET_CLOUD_WORKSPACE_NAME}/pipelines?dry_run=false" \
-              --header "Authorization: Bearer ${DEEPSET_CLOUD_API_KEY}" \
-              --header 'Content-Type: application/json' \
-              --data "{
-                \"yaml_content\": ${pipeline_content},
-                \"deepset_cloud_version\": \"v2\"
-              }")
+          for pipeline_dir in pipelines/*/; do
+            pipeline_name=$(basename "$pipeline_dir")
+            indexing_yaml="${pipeline_dir}indexing.yaml"
+            query_yaml="${pipeline_dir}query.yaml"
             
-            echo "Pipeline creation/update response for $pipeline: $response"
+            if [[ ! -f "$indexing_yaml" || ! -f "$query_yaml" ]]; then
+              echo "Error: Both indexing.yaml and query.yaml must exist in $pipeline_dir"
+              exit 1
+            fi
+            
+            # Check if pipeline exists
+            pipeline_exists=$(curl --silent --show-error --fail \
+              --request GET \
+              --url "https://api.cloud.deepset.ai/api/v1/workspaces/${DEEPSET_CLOUD_WORKSPACE_NAME}/pipelines/${pipeline_name}" \
+              --header "Authorization: Bearer ${DEEPSET_CLOUD_API_KEY}")
+            
+            if [[ $pipeline_exists == *"error"* ]]; then
+              # Pipeline doesn't exist, create new pipeline
+              indexing_content=$(yq eval -o=json "$indexing_yaml")
+              query_content=$(yq eval -o=json "$query_yaml")
+              
+              payload=$(jq -n \
+                --arg name "$pipeline_name" \
+                --arg indexing "$indexing_content" \
+                --arg query "$query_content" \
+                '{
+                  "name": $name,
+                  "yaml": null,
+                  "indexing_yaml": $indexing,
+                  "query_yaml": $query,
+                  "deepset_cloud_version": "v2"
+                }')
+              
+              response=$(curl --silent --show-error --fail \
+                --request POST \
+                --url "https://api.cloud.deepset.ai/api/v1/workspaces/${DEEPSET_CLOUD_WORKSPACE_NAME}/pipelines" \
+                --header "Authorization: Bearer ${DEEPSET_CLOUD_API_KEY}" \
+                --header 'Content-Type: application/json' \
+                --data "$payload")
+              
+              echo "Pipeline creation response for $pipeline_name: $response"
+            else
+              # Pipeline exists, update only query pipeline
+              query_content=$(yq eval -o=json "$query_yaml")
+              
+              payload=$(jq -n \
+                --arg name "$pipeline_name" \
+                --arg query "$query_content" \
+                '{
+                  "name": $name,
+                  "yaml": null,
+                  "query_yaml": $query,
+                  "deepset_cloud_version": "v2"
+                }')
+              
+              response=$(curl --silent --show-error --fail \
+                --request PATCH \
+                --url "https://api.cloud.deepset.ai/api/v1/workspaces/${DEEPSET_CLOUD_WORKSPACE_NAME}/pipelines/${pipeline_name}" \
+                --header "Authorization: Bearer ${DEEPSET_CLOUD_API_KEY}" \
+                --header 'Content-Type: application/json' \
+                --data "$payload")
+              
+              echo "Pipeline update response for $pipeline_name: $response"
+            fi
             
             if [[ $response == *"error"* ]]; then
-              echo "Pipeline creation/update failed for $pipeline"
+              echo "Pipeline operation failed for $pipeline_name"
               exit 1
             fi
           done
 
       - name: Validate Pipelines
-        env:
-          DEEPSET_CLOUD_API_KEY: ${{ secrets.DEEPSET_CLOUD_API_KEY }}
-          DEEPSET_CLOUD_WORKSPACE_NAME: "your-prod-workspace-name"
         run: |
           validation_response=$(curl --silent --show-error --fail \
             --request POST \