Add more logging for Actions #3
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Deploy to Prod Workspace | |
| on: | |
| push: | |
| branches: | |
| - main | |
| workflow_dispatch: | |
| env: | |
| DEEPSET_CLOUD_API_KEY: ${{ secrets.DEEPSET_CLOUD_API_KEY }} | |
| DEEPSET_CLOUD_WORKSPACE_NAME: "your-prod-workspace-name" | |
| jobs: | |
| deploy: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v3 | |
| - name: Install jq and yq | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install -y jq | |
| sudo wget https://github.com/mikefarah/yq/releases/latest/download/yq_linux_amd64 -O /usr/bin/yq | |
| sudo chmod +x /usr/bin/yq | |
| - name: Check if workspace exists | |
| run: | | |
| echo "Checking if workspace ${DEEPSET_CLOUD_WORKSPACE_NAME} exists..." | |
| response=$(curl --silent --show-error \ | |
| --write-out "\n%{http_code}" \ | |
| --request GET \ | |
| --url "https://api.cloud.deepset.ai/api/v1/workspaces/${DEEPSET_CLOUD_WORKSPACE_NAME}" \ | |
| --header "Authorization: Bearer ${DEEPSET_CLOUD_API_KEY}") | |
| status_code=$(echo "$response" | tail -n1) | |
| response_body=$(echo "$response" | sed '$d') | |
| echo "Workspace check status code: $status_code" | |
| echo "Workspace check response body: $response_body" | |
| if [[ $status_code != "200" ]]; then | |
| echo "Error: Workspace ${DEEPSET_CLOUD_WORKSPACE_NAME} does not exist or cannot be accessed" | |
| exit 1 | |
| fi | |
| - name: Create or Update Pipelines | |
| run: | | |
| for pipeline_dir in pipelines/*/; do | |
| pipeline_name=$(basename "$pipeline_dir") | |
| indexing_yaml="${pipeline_dir}indexing.yaml" | |
| query_yaml="${pipeline_dir}query.yaml" | |
| echo "Processing pipeline: $pipeline_name" | |
| echo "Indexing YAML: $indexing_yaml" | |
| echo "Query YAML: $query_yaml" | |
| if [[ ! -f "$indexing_yaml" || ! -f "$query_yaml" ]]; then | |
| echo "Error: Both indexing.yaml and query.yaml must exist in $pipeline_dir" | |
| exit 1 | |
| fi | |
| # Check if pipeline exists | |
| echo "Checking if pipeline $pipeline_name exists..." | |
| pipeline_exists=$(curl --silent --show-error \ | |
| --write-out "%{http_code}" \ | |
| --output /dev/null \ | |
| --request GET \ | |
| --url "https://api.cloud.deepset.ai/api/v1/workspaces/${DEEPSET_CLOUD_WORKSPACE_NAME}/pipelines/${pipeline_name}" \ | |
| --header "Authorization: Bearer ${DEEPSET_CLOUD_API_KEY}") | |
| echo "Pipeline existence check status code: $pipeline_exists" | |
| if [[ $pipeline_exists == "404" ]]; then | |
| echo "Pipeline $pipeline_name doesn't exist. Creating new pipeline..." | |
| indexing_content=$(yq eval -o=json "$indexing_yaml") | |
| query_content=$(yq eval -o=json "$query_yaml") | |
| payload=$(jq -n \ | |
| --arg name "$pipeline_name" \ | |
| --arg indexing "$indexing_content" \ | |
| --arg query "$query_content" \ | |
| '{ | |
| "name": $name, | |
| "yaml": null, | |
| "indexing_yaml": $indexing, | |
| "query_yaml": $query, | |
| "deepset_cloud_version": "v2" | |
| }') | |
| echo "Payload for pipeline creation:" | |
| echo "$payload" | jq . | |
| response=$(curl --silent --show-error \ | |
| --write-out "\n%{http_code}" \ | |
| --request POST \ | |
| --url "https://api.cloud.deepset.ai/api/v1/workspaces/${DEEPSET_CLOUD_WORKSPACE_NAME}/pipelines" \ | |
| --header "Authorization: Bearer ${DEEPSET_CLOUD_API_KEY}" \ | |
| --header 'Content-Type: application/json' \ | |
| --data "$payload") | |
| status_code=$(echo "$response" | tail -n1) | |
| response_body=$(echo "$response" | sed '$d') | |
| echo "Pipeline creation response for $pipeline_name:" | |
| echo "Status code: $status_code" | |
| echo "Response body: $response_body" | |
| if [[ $status_code != "201" ]]; then | |
| echo "Pipeline creation failed for $pipeline_name" | |
| exit 1 | |
| fi | |
| else | |
| echo "Pipeline $pipeline_name exists. Updating query pipeline..." | |
| query_content=$(yq eval -o=json "$query_yaml") | |
| payload=$(jq -n \ | |
| --arg name "$pipeline_name" \ | |
| --arg query "$query_content" \ | |
| '{ | |
| "name": $name, | |
| "yaml": null, | |
| "query_yaml": $query, | |
| "deepset_cloud_version": "v2" | |
| }') | |
| echo "Payload for pipeline update:" | |
| echo "$payload" | jq . | |
| response=$(curl --silent --show-error \ | |
| --write-out "\n%{http_code}" \ | |
| --request PATCH \ | |
| --url "https://api.cloud.deepset.ai/api/v1/workspaces/${DEEPSET_CLOUD_WORKSPACE_NAME}/pipelines/${pipeline_name}" \ | |
| --header "Authorization: Bearer ${DEEPSET_CLOUD_API_KEY}" \ | |
| --header 'Content-Type: application/json' \ | |
| --data "$payload") | |
| status_code=$(echo "$response" | tail -n1) | |
| response_body=$(echo "$response" | sed '$d') | |
| echo "Pipeline update response for $pipeline_name:" | |
| echo "Status code: $status_code" | |
| echo "Response body: $response_body" | |
| if [[ $status_code != "200" ]]; then | |
| echo "Pipeline update failed for $pipeline_name" | |
| exit 1 | |
| fi | |
| fi | |
| done | |
| - name: Validate Pipelines | |
| run: | | |
| echo "Validating pipelines..." | |
| validation_response=$(curl --silent --show-error \ | |
| --write-out "\n%{http_code}" \ | |
| --request POST \ | |
| --url "https://api.cloud.deepset.ai/api/v1/workspaces/${DEEPSET_CLOUD_WORKSPACE_NAME}/pipeline_validations" \ | |
| --header "Authorization: Bearer ${DEEPSET_CLOUD_API_KEY}" \ | |
| --header 'content-type: application/json' \ | |
| --data '{"deepset_cloud_version": "v2"}') | |
| status_code=$(echo "$validation_response" | tail -n1) | |
| response_body=$(echo "$validation_response" | sed '$d') | |
| echo "Validation status code: $status_code" | |
| echo "Validation response body: $response_body" | |
| if [[ $status_code != "200" ]]; then | |
| echo "Pipeline validation failed" | |
| exit 1 | |
| fi | |
| - name: Deploy Pipelines to Prod Workspace | |
| env: | |
| DEEPSET_CLOUD_API_KEY: ${{ secrets.DEEPSET_CLOUD_API_KEY }} | |
| DEEPSET_CLOUD_WORKSPACE_NAME: "your-prod-workspace-name" | |
| run: | | |
| for pipeline in pipelines/*.yaml; do | |
| deepset-cloud pipelines upload \ | |
| --api-key "$DEEPSET_CLOUD_API_KEY" \ | |
| --workspace-name "$DEEPSET_CLOUD_WORKSPACE_NAME" \ | |
| --file "$pipeline" | |
| done |