Initial commit

.github/workflows/deploy-dev.yml

@@ -0,0 +1,85 @@
+name: Deploy to Dev Workspace
+
+on:
+  push:
+    branches:
+      - dev
+
+jobs:
+  deploy:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+
+      - name: Install jq and yq
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y jq
+          sudo wget https://github.com/mikefarah/yq/releases/latest/download/yq_linux_amd64 -O /usr/bin/yq
+          sudo chmod +x /usr/bin/yq
+
+      - name: Create or Update Pipelines
+        env:
+          DEEPSET_CLOUD_API_KEY: ${{ secrets.DEEPSET_CLOUD_API_KEY }}
+          DEEPSET_CLOUD_WORKSPACE_NAME: "your-dev-workspace-name"
+        run: |
+          for pipeline_dir in pipelines/*/; do
+            pipeline_name=$(basename "$pipeline_dir")
+            indexing_yaml="${pipeline_dir}indexing.yaml"
+            query_yaml="${pipeline_dir}query.yaml"
+            
+            if [[ ! -f "$indexing_yaml" || ! -f "$query_yaml" ]]; then
+              echo "Error: Both indexing.yaml and query.yaml must exist in $pipeline_dir"
+              exit 1
+            fi
+            
+            indexing_content=$(yq eval -o=json "$indexing_yaml")
+            query_content=$(yq eval -o=json "$query_yaml")
+            
+            payload=$(jq -n \
+              --arg name "$pipeline_name" \
+              --arg indexing "$indexing_content" \
+              --arg query "$query_content" \
+              '{
+                "name": $name,
+                "yaml": null,
+                "indexing_yaml": $indexing,
+                "query_yaml": $query,
+                "deepset_cloud_version": "v2"
+              }')
+            
+            response=$(curl --silent --show-error --fail \
+              --request POST \
+              --url "https://api.cloud.deepset.ai/api/v1/workspaces/${DEEPSET_CLOUD_WORKSPACE_NAME}/pipelines" \
+              --header "Authorization: Bearer ${DEEPSET_CLOUD_API_KEY}" \
+              --header 'Content-Type: application/json' \
+              --data "$payload")
+            
+            echo "Pipeline creation/update response for $pipeline_name: $response"
+            
+            if [[ $response == *"error"* ]]; then
+              echo "Pipeline creation/update failed for $pipeline_name"
+              exit 1
+            fi
+          done
+
+      - name: Validate Pipelines
+        env:
+          DEEPSET_CLOUD_API_KEY: ${{ secrets.DEEPSET_CLOUD_API_KEY }}
+          DEEPSET_CLOUD_WORKSPACE_NAME: "your-dev-workspace-name"
+        run: |
+          validation_response=$(curl --silent --show-error --fail \
+            --request POST \
+            --url "https://api.cloud.deepset.ai/api/v1/workspaces/${DEEPSET_CLOUD_WORKSPACE_NAME}/pipeline_validations" \
+            --header "Authorization: Bearer ${DEEPSET_CLOUD_API_KEY}" \
+            --header 'content-type: application/json' \
+            --data '{"deepset_cloud_version": "v2"}')
+          
+          echo "Validation response: $validation_response"
+          
+          if [[ $validation_response == *"error"* ]]; then
+            echo "Pipeline validation failed"
+            exit 1
+          fi

.github/workflows/deploy-prod.yml

@@ -0,0 +1,74 @@
+name: Deploy to Prod Workspace
+
+on:
+  push:
+    branches:
+      - main
+  workflow_dispatch:
+
+jobs:
+  deploy:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+
+      - name: Install jq
+        run: sudo apt-get install jq
+
+      - name: Create or Update Pipelines
+        env:
+          DEEPSET_CLOUD_API_KEY: ${{ secrets.DEEPSET_CLOUD_API_KEY }}
+          DEEPSET_CLOUD_WORKSPACE_NAME: "your-prod-workspace-name"
+        run: |
+          for pipeline in pipelines/*.yaml; do
+            pipeline_content=$(cat "$pipeline" | jq -sR .)
+            response=$(curl --silent --show-error --fail \
+              --request POST \
+              --url "https://api.cloud.deepset.ai/api/v1/workspaces/${DEEPSET_CLOUD_WORKSPACE_NAME}/pipelines?dry_run=false" \
+              --header "Authorization: Bearer ${DEEPSET_CLOUD_API_KEY}" \
+              --header 'Content-Type: application/json' \
+              --data "{
+                \"yaml_content\": ${pipeline_content},
+                \"deepset_cloud_version\": \"v2\"
+              }")
+            
+            echo "Pipeline creation/update response for $pipeline: $response"
+            
+            if [[ $response == *"error"* ]]; then
+              echo "Pipeline creation/update failed for $pipeline"
+              exit 1
+            fi
+          done
+
+      - name: Validate Pipelines
+        env:
+          DEEPSET_CLOUD_API_KEY: ${{ secrets.DEEPSET_CLOUD_API_KEY }}
+          DEEPSET_CLOUD_WORKSPACE_NAME: "your-prod-workspace-name"
+        run: |
+          validation_response=$(curl --silent --show-error --fail \
+            --request POST \
+            --url "https://api.cloud.deepset.ai/api/v1/workspaces/${DEEPSET_CLOUD_WORKSPACE_NAME}/pipeline_validations" \
+            --header "Authorization: Bearer ${DEEPSET_CLOUD_API_KEY}" \
+            --header 'content-type: application/json' \
+            --data '{"deepset_cloud_version": "v2"}')
+          
+          echo "Validation response: $validation_response"
+          
+          if [[ $validation_response == *"error"* ]]; then
+            echo "Pipeline validation failed"
+            exit 1
+          fi
+
+      - name: Deploy Pipelines to Prod Workspace
+        env:
+          DEEPSET_CLOUD_API_KEY: ${{ secrets.DEEPSET_CLOUD_API_KEY }}
+          DEEPSET_CLOUD_WORKSPACE_NAME: "your-prod-workspace-name"
+        run: |
+          for pipeline in pipelines/*.yaml; do
+            deepset-cloud pipelines upload \
+              --api-key "$DEEPSET_CLOUD_API_KEY" \
+              --workspace-name "$DEEPSET_CLOUD_WORKSPACE_NAME" \
+              --file "$pipeline"
+          done

.gitignore

@@ -0,0 +1,49 @@
+# Python-related files
+__pycache__/
+*.py[cod]
+*$py.class
+*.so
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+
+# Virtual environments
+venv/
+env/
+ENV/
+
+# IDEs and editors
+.vscode/
+.idea/
+*.swp
+*.swo
+*~
+
+# Operating system files
+.DS_Store
+Thumbs.db
+
+# Logs
+*.log
+
+# Local configuration files
+*.env
+config.local.yaml
+
+# Temporary files
+*.tmp
+*.bak
+*.swp