From 97a3db378023a225156888abe72227bc71ca7116 Mon Sep 17 00:00:00 2001
From: oryx1729 <78848855+oryx1729@users.noreply.github.com>
Date: Wed, 16 Oct 2024 14:42:31 +0200
Subject: [PATCH] Improve Action Workflows

---
 .github/workflows/deploy-dev.yml  |  62 ++++++--------
 .github/workflows/deploy-prod.yml |  60 ++++++-------
 README.md                         | 136 +++++-------------------------
 3 files changed, 78 insertions(+), 180 deletions(-)

diff --git a/.github/workflows/deploy-dev.yml b/.github/workflows/deploy-dev.yml
index c946ff8..364657c 100644
--- a/.github/workflows/deploy-dev.yml
+++ b/.github/workflows/deploy-dev.yml
@@ -7,7 +7,7 @@ on:
 
 env:
   DEEPSET_CLOUD_API_KEY: ${{ secrets.DEEPSET_CLOUD_API_KEY }}
-  DEEPSET_CLOUD_WORKSPACE_NAME: "your-dev-workspace-name"
+  DEEPSET_CLOUD_WORKSPACE_NAME: "YOUR_DEV_WORKSPACE_NAME"
 
 jobs:
   deploy:
@@ -71,54 +71,48 @@ jobs:
             
             echo "Pipeline existence check status code: $pipeline_exists"
             
+            indexing_content=$(cat "$indexing_yaml")
+            query_content=$(cat "$query_yaml")
+            
             if [[ $pipeline_exists == "404" ]]; then
               echo "Pipeline $pipeline_name doesn't exist. Creating new pipeline..."
-              indexing_content=$(cat "$indexing_yaml")
-              query_content=$(cat "$query_yaml")
-              
-              combined_yaml=$(cat <<EOF
-          indexing:
-          $indexing_content
-          ---
-          query:
-          $query_content
-          EOF
-          )
               
-              echo "Combined YAML for pipeline creation:"
-              echo "$combined_yaml"
+              payload=$(jq -n \
+                --arg name "$pipeline_name" \
+                --arg indexing "$indexing_content" \
+                --arg query "$query_content" \
+                '{
+                  "name": $name,
+                  "deepset_cloud_version": "v2",
+                  "indexing_yaml": $indexing,
+                  "query_yaml": $query
+                }')
               
               response=$(curl --silent --show-error \
                 --write-out "\n%{http_code}" \
                 --request POST \
                 --url "https://api.cloud.deepset.ai/api/v1/workspaces/${DEEPSET_CLOUD_WORKSPACE_NAME}/pipelines" \
                 --header "Authorization: Bearer ${DEEPSET_CLOUD_API_KEY}" \
-                --header 'Content-Type: application/x-yaml' \
-                --data-binary "$combined_yaml")
+                --header 'Content-Type: application/json' \
+                --data "$payload")
             else
               echo "Pipeline $pipeline_name exists. Updating pipeline..."
-              indexing_content=$(cat "$indexing_yaml")
-              query_content=$(cat "$query_yaml")
               
-              combined_yaml=$(cat <<EOF
-          indexing:
-          $indexing_content
-          ---
-          query:
-          $query_content
-          EOF
-          )
-              
-              echo "Combined YAML for pipeline update:"
-              echo "$combined_yaml"
+              payload=$(jq -n \
+                --arg indexing "$indexing_content" \
+                --arg query "$query_content" \
+                '{
+                  "indexing_yaml": $indexing,
+                  "query_yaml": $query
+                }')
               
               response=$(curl --silent --show-error \
                 --write-out "\n%{http_code}" \
                 --request PUT \
                 --url "https://api.cloud.deepset.ai/api/v1/workspaces/${DEEPSET_CLOUD_WORKSPACE_NAME}/pipelines/${pipeline_name}/yaml" \
                 --header "Authorization: Bearer ${DEEPSET_CLOUD_API_KEY}" \
-                --header 'Content-Type: application/x-yaml' \
-                --data-binary "$combined_yaml")
+                --header 'Content-Type: application/json' \
+                --data "$payload")
             fi
             
             status_code=$(echo "$response" | tail -n1)
@@ -133,7 +127,7 @@ jobs:
               exit 1
             fi
           done
-
+          
       - name: Validate Pipelines
         run: |
           echo "Validating pipelines..."
@@ -151,7 +145,7 @@ jobs:
           echo "Validation status code: $status_code"
           echo "Validation response body: $response_body"
           
-          if [[ $status_code != "200" ]]; then
+          if [[ $status_code != "204" ]]; then
             echo "Pipeline validation failed"
             exit 1
-          fi
+          fi
\ No newline at end of file
diff --git a/.github/workflows/deploy-prod.yml b/.github/workflows/deploy-prod.yml
index 9961cf6..2603a2f 100644
--- a/.github/workflows/deploy-prod.yml
+++ b/.github/workflows/deploy-prod.yml
@@ -8,7 +8,7 @@ on:
 
 env:
   DEEPSET_CLOUD_API_KEY: ${{ secrets.DEEPSET_CLOUD_API_KEY }}
-  DEEPSET_CLOUD_WORKSPACE_NAME: "your-prod-workspace-name"
+  DEEPSET_CLOUD_WORKSPACE_NAME: "YOUR_PROD_WORKSPACE_NAME"
 
 jobs:
   deploy:
@@ -72,54 +72,48 @@ jobs:
             
             echo "Pipeline existence check status code: $pipeline_exists"
             
+            indexing_content=$(cat "$indexing_yaml")
+            query_content=$(cat "$query_yaml")
+            
             if [[ $pipeline_exists == "404" ]]; then
               echo "Pipeline $pipeline_name doesn't exist. Creating new pipeline..."
-              indexing_content=$(cat "$indexing_yaml")
-              query_content=$(cat "$query_yaml")
-              
-              combined_yaml=$(cat <<EOF
-          indexing:
-          $indexing_content
-          ---
-          query:
-          $query_content
-          EOF
-          )
               
-              echo "Combined YAML for pipeline creation:"
-              echo "$combined_yaml"
+              payload=$(jq -n \
+                --arg name "$pipeline_name" \
+                --arg indexing "$indexing_content" \
+                --arg query "$query_content" \
+                '{
+                  "name": $name,
+                  "deepset_cloud_version": "v2",
+                  "indexing_yaml": $indexing,
+                  "query_yaml": $query
+                }')
               
               response=$(curl --silent --show-error \
                 --write-out "\n%{http_code}" \
                 --request POST \
                 --url "https://api.cloud.deepset.ai/api/v1/workspaces/${DEEPSET_CLOUD_WORKSPACE_NAME}/pipelines" \
                 --header "Authorization: Bearer ${DEEPSET_CLOUD_API_KEY}" \
-                --header 'Content-Type: application/x-yaml' \
-                --data-binary "$combined_yaml")
+                --header 'Content-Type: application/json' \
+                --data "$payload")
             else
               echo "Pipeline $pipeline_name exists. Updating pipeline..."
-              indexing_content=$(cat "$indexing_yaml")
-              query_content=$(cat "$query_yaml")
               
-              combined_yaml=$(cat <<EOF
-          indexing:
-          $indexing_content
-          ---
-          query:
-          $query_content
-          EOF
-          )
-              
-              echo "Combined YAML for pipeline update:"
-              echo "$combined_yaml"
+              payload=$(jq -n \
+                --arg indexing "$indexing_content" \
+                --arg query "$query_content" \
+                '{
+                  "indexing_yaml": $indexing,
+                  "query_yaml": $query
+                }')
               
               response=$(curl --silent --show-error \
                 --write-out "\n%{http_code}" \
                 --request PUT \
                 --url "https://api.cloud.deepset.ai/api/v1/workspaces/${DEEPSET_CLOUD_WORKSPACE_NAME}/pipelines/${pipeline_name}/yaml" \
                 --header "Authorization: Bearer ${DEEPSET_CLOUD_API_KEY}" \
-                --header 'Content-Type: application/x-yaml' \
-                --data-binary "$combined_yaml")
+                --header 'Content-Type: application/json' \
+                --data "$payload")
             fi
             
             status_code=$(echo "$response" | tail -n1)
@@ -152,7 +146,7 @@ jobs:
           echo "Validation status code: $status_code"
           echo "Validation response body: $response_body"
           
-          if [[ $status_code != "200" ]]; then
+          if [[ $status_code != "204" ]]; then
             echo "Pipeline validation failed"
             exit 1
           fi
@@ -160,7 +154,7 @@ jobs:
       - name: Deploy Pipelines to Prod Workspace
         env:
           DEEPSET_CLOUD_API_KEY: ${{ secrets.DEEPSET_CLOUD_API_KEY }}
-          DEEPSET_CLOUD_WORKSPACE_NAME: "your-prod-workspace-name"
+          DEEPSET_CLOUD_WORKSPACE_NAME: "YOUR_PROD_WORKSPACE_NAME"
         run: |
           for pipeline in pipelines/*.yaml; do
             deepset-cloud pipelines upload \
diff --git a/README.md b/README.md
index bcc5dba..c7eeca3 100644
--- a/README.md
+++ b/README.md
@@ -1,6 +1,8 @@
 # CI/CD Template with GitHub Actions for deepset Cloud
 
-This repository serves as a template for using **GitHub Actions** as a CI/CD pipeline to manage and deploy pipelines to **deepset Cloud**. It demonstrates how to work with two separate workspaces: **dev** and **prod**.
+This repository serves as a sample template example to showcase how you can implement CI/CD with GitHub Actions by leveraging the deepset Cloud REST APIs. It demonstrates how to work with two separate workspaces: **dev** and **prod**.
+
+The purpose of this template is to illustrate best practices and provide a starting point for setting up automated pipeline management and deployment in deepset Cloud environments. While it's designed to be functional, you may need to adapt and extend it to fit your specific use case and requirements.
 
 ## Table of Contents
 
@@ -35,8 +37,12 @@ your-repo/
 │       ├── deploy-dev.yml
 │       └── deploy-prod.yml
 ├── pipelines/
-│   ├── pipeline1.yaml
-│   └── pipeline2.yaml
+│   ├── pipeline1/
+│   │   ├── indexing.yaml
+│   │   └── query.yaml
+│   └── pipeline2/
+│       ├── indexing.yaml
+│       └── query.yaml
 └── README.md
 ```
 
@@ -67,155 +73,59 @@ your-repo/
 
    - Update the workspace names in the workflow files:
 
-     - In `.github/workflows/deploy-dev.yml`, replace `"your-dev-workspace-name"` with your actual **dev** workspace name.
-     - In `.github/workflows/deploy-prod.yml`, replace `"your-prod-workspace-name"` with your actual **prod** workspace name.
-
-4. **Install Dependencies (Optional)**:
-
-   - If you plan to run the deepset Cloud CLI locally, install it with:
-
-     ```bash
-     pip install deepset-cloud
-     ```
+     - In `.github/workflows/deploy-dev.yml`, replace `"YOUR_DEV_WORKSPACE_NAME"` with your actual **dev** workspace name.
+     - In `.github/workflows/deploy-prod.yml`, replace `"YOUR_PROD_WORKSPACE_NAME"` with your actual **prod** workspace name.
 
 ## Usage
 
 ### Branching Strategy
 
-This template uses a simple branching strategy to manage deployments to different environments:
-
-- **Development Branch (`dev`)**:
-
-  - Used for integrating and testing new changes.
-  - Automatically deploys to the **dev** workspace upon push.
-
-- **Production Branch (`main`)**:
-
-  - Contains stable and reviewed code.
-  - Deployment to the **prod** workspace is triggered on push or manually via GitHub Actions.
+- **Development Branch (`dev`)**: Used for integrating and testing new changes. Automatically deploys to the **dev** workspace upon push.
+- **Production Branch (`main`)**: Contains stable and reviewed code. Deployment to the **prod** workspace is triggered on push or manually via GitHub Actions.
 
 ### Adding or Updating Pipelines
 
 1. **Add or Modify Pipeline Files**:
-
-   - Pipeline configuration files are located in the `pipelines/` directory.
-   - Create new pipeline YAML files or modify existing ones.
-
-   **Example**:
-
-   ```yaml:pipelines/pipeline1.yaml
-   version: "1.0"
-   components:
-     - name: Retriever
-       type: BM25Retriever
-       params:
-         document_store: MyDocumentStore
-
-     - name: Reader
-       type: TransformersReader
-       params:
-         model_name_or_path: deepset/roberta-base-squad2
-
-   pipelines:
-     - name: question-answering
-       nodes:
-         - name: Retriever
-           inputs: [Query]
-         - name: Reader
-           inputs: [Retriever]
-   ```
+   - Create a new directory under `pipelines/` for each pipeline.
+   - Each pipeline directory should contain two files: `indexing.yaml` and `query.yaml`.
 
 2. **Commit Changes**:
-
    ```bash
-   git add pipelines/your-pipeline.yaml
-   git commit -m "Add new pipeline"
+   git add pipelines/your-pipeline-name/
+   git commit -m "Add/Update pipeline: your-pipeline-name"
    ```
 
 3. **Push to the Appropriate Branch**:
-
-   - For development:
-
-     ```bash
-     git push origin dev
-     ```
-
-   - For production:
-
-     ```bash
-     git push origin main
-     ```
+   - For development: `git push origin dev`
+   - For production: `git push origin main`
 
 ### Triggering Deployments
 
 - **Automatic Deployment**:
-
   - **Dev Workspace**: Pushing to the `dev` branch triggers the `deploy-dev.yml` workflow.
   - **Prod Workspace**: Pushing to the `main` branch triggers the `deploy-prod.yml` workflow.
 
 - **Manual Deployment**:
-
   - For production, you can manually trigger the workflow:
-
     1. Go to the **Actions** tab in your GitHub repository.
     2. Select **"Deploy to Prod Workspace"** workflow.
     3. Click **"Run workflow"**.
 
 ### Rollback Procedures
 
-If you need to revert to a previous pipeline version:
+To revert to a previous pipeline version:
 
 1. **Revert Changes in Git**:
-
    ```bash
    git revert <commit-hash>
    git push origin dev  # or main, depending on the branch
    ```
 
-2. **Deployment**:
-
-   - The GitHub Actions workflow will redeploy the pipelines based on the reverted code.
-
-## Security Considerations
-
-- **API Keys**:
-
-  - Store sensitive information like API keys in **GitHub Secrets**.
-  - **Do not** commit sensitive data to the repository.
+2. The GitHub Actions workflow will automatically redeploy the pipelines based on the reverted code.
 
-- **Workspace Names**:
-
-  - Workspace names are specified in the workflow files. Ensure they are correct to avoid deploying to unintended workspaces.
-
-- **Secret Rotation**:
-
-  - Regularly rotate your API keys and update the GitHub Secrets accordingly.
 
 ## Extending the Template
 
-- **Add More Environments**:
-
-  - To add environments like staging, copy one of the existing workflows and modify the workspace name and trigger conditions.
+- **Add More Environments**: Copy and modify existing workflows for additional environments like staging.
+- **Notifications**: Add steps to send deployment notifications via email, Slack, etc.
 
-- **Validation and Testing**:
-
-  - Incorporate pipeline validation steps before deployment.
-
-    ```yaml
-    - name: Validate Pipelines
-      run: |
-        for pipeline in pipelines/*.yaml; do
-          deepset-cloud pipelines validate --file "$pipeline"
-        done
-    ```
-
-- **Notifications**:
-
-  - Add steps to send deployment notifications via email, Slack, etc.
-
-## Support and Contributions
-
-- **Issues**: If you encounter problems, please open an issue.
-- **Contributions**: Contributions are welcome! Feel free to fork the repository and submit a pull request.
-
----