From 4c86af498c4990697b915514928444894d50c667 Mon Sep 17 00:00:00 2001 From: Uday Beswal Date: Mon, 4 Nov 2024 00:07:20 -0800 Subject: [PATCH] Update Zulip Bot to manage image creation run alerts Signed-off-by: Uday Beswal --- .github/workflows/zulip_notification_bot.yml | 534 ++++++++++++++----- 1 file changed, 396 insertions(+), 138 deletions(-) diff --git a/.github/workflows/zulip_notification_bot.yml b/.github/workflows/zulip_notification_bot.yml index e3a3009711..7153290325 100644 --- a/.github/workflows/zulip_notification_bot.yml +++ b/.github/workflows/zulip_notification_bot.yml @@ -9,153 +9,411 @@ on: permissions: read-all jobs: - ib_pipelines_tracker_alerts: - name: IB Pipelines Tracker Alerts - runs-on: ubuntu-latest - environment: actions-cicd + # ib_pipelines_tracker_alerts: + # name: IB Pipelines Tracker Alerts + # runs-on: ubuntu-latest + # environment: actions-cicd - steps: - - name: Checkout Repository - uses: actions/checkout@v4 - - - name: Get Latest Completed Run ID - id: get_run_id - uses: actions/github-script@v7 - with: - github-token: ${{ secrets.GITHUB_TOKEN }} - script: | - const workflow_id = 'ib_pipelines_check.yml'; - const { data } = await github.rest.actions.listWorkflowRuns({ - owner: context.repo.owner, - repo: context.repo.repo, - workflow_id, - status: 'completed', - per_page: 1 - }); - if (data.workflow_runs.length === 0) { - core.setFailed('No completed workflow runs found.'); - } else { - const run = data.workflow_runs[0]; - core.setOutput('run_id', run.id); - core.setOutput('conclusion', run.conclusion || 'unknown'); - } + # steps: + # - name: Checkout Repository + # uses: actions/checkout@v4 - - name: Download Logs from Latest Run - run: | - run_id=${{ steps.get_run_id.outputs.run_id }} - auth_header="Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}" - api_url="https://api.github.com/repos/${{ github.repository }}/actions/runs/$run_id/logs" - curl -H "$auth_header" -L "$api_url" -o logs.zip - - - name: Unzip Logs - run: | - unzip logs.zip -d logs - - - name: Extract and Process Log - id: extract_summary - shell: bash {0} - run: | - # Set the step name - step_name="Check Pipeline Status" - - # Find the log file for the required step - step_log=$(find logs -type f -name "*_${step_name}.txt" -print -quit) - if [ -z "$step_log" ]; then - echo "Step log file for '$step_name' not found." - echo "Available log files:" - find logs -type f -name '*.txt' - exit 1 - fi + # - name: Get Latest Completed Run ID + # id: get_run_id + # uses: actions/github-script@v7 + # with: + # github-token: ${{ secrets.GITHUB_TOKEN }} + # script: | + # const workflow_id = 'ib_pipelines_check.yml'; + # const { data } = await github.rest.actions.listWorkflowRuns({ + # owner: context.repo.owner, + # repo: context.repo.repo, + # workflow_id, + # status: 'completed', + # per_page: 1 + # }); + # if (data.workflow_runs.length === 0) { + # core.setFailed('No completed workflow runs found.'); + # } else { + # const run = data.workflow_runs[0]; + # core.setOutput('run_id', run.id); + # core.setOutput('conclusion', run.conclusion || 'unknown'); + # } + + # - name: Download Logs from Latest Run + # run: | + # run_id=${{ steps.get_run_id.outputs.run_id }} + # auth_header="Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}" + # api_url="https://api.github.com/repos/${{ github.repository }}/actions/runs/$run_id/logs" + # curl -H "$auth_header" -L "$api_url" -o logs.zip + + # - name: Unzip Logs + # run: | + # unzip logs.zip -d logs + + # - name: Extract and Process Log + # id: extract_summary + # shell: bash {0} + # run: | + # # Set the step name + # step_name="Check Pipeline Status" + + # # Find the log file for the required step + # step_log=$(find logs -type f -name "*_${step_name}.txt" -print -quit) + # if [ -z "$step_log" ]; then + # echo "Step log file for '$step_name' not found." + # echo "Available log files:" + # find logs -type f -name '*.txt' + # exit 1 + # fi - # Extract the content starting from Summary of Pipelines - summary=$(sed -n '/Summary of Pipelines:/,$p' "$step_log") + # # Extract the content starting from Summary of Pipelines + # summary=$(sed -n '/Summary of Pipelines:/,$p' "$step_log") - # If summary is empty, use the entire log - if [ -z "$summary" ]; then - echo "Warning: 'Summary of Pipelines:' not found. Using entire log as summary." - summary=$(cat "$step_log") - fi + # # If summary is empty, use the entire log + # if [ -z "$summary" ]; then + # echo "Warning: 'Summary of Pipelines:' not found. Using entire log as summary." + # summary=$(cat "$step_log") + # fi - # Remove unwanted lines (timestamps and whitespaces) - summary=$(echo "$summary" | sed -E 's/^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9:\.]+Z\s+//') + # # Remove unwanted lines (timestamps and whitespaces) + # summary=$(echo "$summary" | sed -E 's/^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9:\.]+Z\s+//') - # Get the workflow conclusion - workflow_conclusion="${{ steps.get_run_id.outputs.conclusion }}" - - # Determine emoji based on status - if [ "$workflow_conclusion" = "success" ]; then - status_emoji="✅" - elif [ "$workflow_conclusion" = "failure" ]; then - status_emoji="❌" - else - status_emoji="⚠️" - fi + # # Get the workflow conclusion + # workflow_conclusion="${{ steps.get_run_id.outputs.conclusion }}" + + # # Determine emoji based on status + # if [ "$workflow_conclusion" = "success" ]; then + # status_emoji="✅" + # elif [ "$workflow_conclusion" = "failure" ]; then + # status_emoji="❌" + # else + # status_emoji="⚠️" + # fi - # Construct the pipeline run URL - pipeline_url="${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ steps.get_run_id.outputs.run_id }}" + # # Construct the pipeline run URL + # pipeline_url="${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ steps.get_run_id.outputs.run_id }}" - # Format the summary as a code block - formatted_summary=$(printf '```\n%s\n```' "$summary") + # # Format the summary as a code block + # formatted_summary=$(printf '```\n%s\n```' "$summary") - # Create the Message - message=$(cat < summary.txt + # # Save the message to summary.txt + # echo "$message" > summary.txt - # Split the message into chunks of up to 9999 characters - max_length=9999 # Zulip limit is 10,000 characters - total_length=${#message} - start=0 - chunk_index=0 - mkdir -p chunks - while [ $start -lt $total_length ]; do - chunk="${message:$start:$max_length}" - # Save each chunk to a separate file - echo "$chunk" > "chunks/chunk_${chunk_index}.txt" - start=$((start + max_length)) - chunk_index=$((chunk_index + 1)) - done + # # Split the message into chunks of up to 9999 characters + # max_length=9999 # Zulip limit is 10,000 characters + # total_length=${#message} + # start=0 + # chunk_index=0 + # mkdir -p chunks + # while [ $start -lt $total_length ]; do + # chunk="${message:$start:$max_length}" + # # Save each chunk to a separate file + # echo "$chunk" > "chunks/chunk_${chunk_index}.txt" + # start=$((start + max_length)) + # chunk_index=$((chunk_index + 1)) + # done - # Save the number of chunks as an output - echo "chunks_count=$chunk_index" >> $GITHUB_OUTPUT - - - name: Send to Zulip - if: ${{ always() }} - shell: bash - env: - ZULIP_SERVER_URL: ${{ secrets.ZULIP_SERVER_URL }} - ZULIP_BOT_EMAIL: ${{ secrets.ZULIP_BOT_EMAIL }} - ZULIP_BOT_API_KEY: ${{ secrets.ZULIP_BOT_API_KEY }} - ZULIP_STREAM_NAME: 'community images' - ZULIP_TOPIC_NAME: 'IB Pipeline Tracker 🤖' - run: | - chunks_count=${{ steps.extract_summary.outputs.chunks_count }} - for ((i=0; i> $GITHUB_OUTPUT + + # - name: Send to Zulip + # if: ${{ always() }} + # shell: bash + # env: + # ZULIP_SERVER_URL: ${{ secrets.ZULIP_SERVER_URL }} + # ZULIP_BOT_EMAIL: ${{ secrets.ZULIP_BOT_EMAIL }} + # ZULIP_BOT_API_KEY: ${{ secrets.ZULIP_BOT_API_KEY }} + # ZULIP_STREAM_NAME: 'community images' + # ZULIP_TOPIC_NAME: 'IB Pipeline Tracker 🤖' + # run: | + # chunks_count=${{ steps.extract_summary.outputs.chunks_count }} + # for ((i=0; i wf.name === workflowName + ); + if (!workflow) { + core.setFailed(`Workflow "${workflowName}" not found.`); + return; + } + const workflow_id = workflow.id; + + // Get the last two workflow runs + const runsResponse = await github.rest.actions.listWorkflowRuns({ + owner, + repo, + workflow_id, + per_page: 2, + }); + const runs = runsResponse.data.workflow_runs; + if (runs.length < 2) { + core.setFailed('Not enough workflow runs found.'); + return; + } + + // Output the run IDs, conclusions, URLs, and dates + core.setOutput('run_id1', runs[0].id.toString()); + core.setOutput('run_id2', runs[1].id.toString()); + core.setOutput('conclusion1', runs[0].conclusion || 'unknown'); + core.setOutput('url1', runs[0].html_url); + core.setOutput('url2', runs[1].html_url); + + - name: Get failed ironbank images from the last two runs + id: get_failed_jobs + uses: actions/github-script@v7 + env: + RUN_ID1: ${{ steps.get_runs.outputs.run_id1 }} + RUN_ID2: ${{ steps.get_runs.outputs.run_id2 }} + with: + script: | + const owner = context.repo.owner; + const repo = context.repo.repo; + + // Function to get all jobs with pagination + async function getAllJobs(run_id) { + let jobs = []; + let page = 1; + let moreJobs = true; + + while (moreJobs) { + const response = await github.rest.actions.listJobsForWorkflowRun({ + owner, + repo, + run_id: run_id, + per_page: 100, + page: page, + }); + + if (response.data.jobs.length > 0) { + jobs = jobs.concat(response.data.jobs); + page++; + } else { + moreJobs = false; + } + } + + return jobs; + } + + // Function to get failed jobs ending with "-ib" + async function getFailedJobs(run_id) { + const jobs = await getAllJobs(run_id); + const failedJobs = jobs.filter((job) => { + return job.conclusion === 'failure' && job.name.endsWith('-ib'); + }); + return failedJobs.map((job) => job.name); + } + + const run_id1 = process.env.RUN_ID1; + const run_id2 = process.env.RUN_ID2; + + const failedJobs1 = await getFailedJobs(run_id1); + const failedJobs2 = await getFailedJobs(run_id2); + + // Find jobs that failed in both runs + const failedInBoth = failedJobs1.filter((job) => failedJobs2.includes(job)); + + core.setOutput('failed_jobs', failedInBoth.join(',')); + + - name: Get failed non-ironbank images from the last two runs + id: get_failed_jobs_no_ib + uses: actions/github-script@v7 + env: + RUN_ID1: ${{ steps.get_runs.outputs.run_id1 }} + RUN_ID2: ${{ steps.get_runs.outputs.run_id2 }} + with: + script: | + const owner = context.repo.owner; + const repo = context.repo.repo; + + // Function to get all jobs with pagination + async function getAllJobs(run_id) { + let jobs = []; + let page = 1; + let moreJobs = true; + + while (moreJobs) { + const response = await github.rest.actions.listJobsForWorkflowRun({ + owner, + repo, + run_id: run_id, + per_page: 100, + page: page, + }); + + if (response.data.jobs.length > 0) { + jobs = jobs.concat(response.data.jobs); + page++; + } else { + moreJobs = false; + } + } + + return jobs; + } + + // Function to get failed jobs NOT ending with "-ib" + async function getFailedJobs(run_id) { + const jobs = await getAllJobs(run_id); + const failedJobs = jobs.filter((job) => { + return job.conclusion === 'failure' && !job.name.endsWith('-ib'); + }); + return failedJobs.map((job) => job.name); + } + + const run_id1 = process.env.RUN_ID1; + const run_id2 = process.env.RUN_ID2; + + const failedJobs1 = await getFailedJobs(run_id1); + const failedJobs2 = await getFailedJobs(run_id2); + + // Find jobs that failed in both runs + const failedInBoth = failedJobs1.filter((job) => failedJobs2.includes(job)); + + core.setOutput('failed_jobs', failedInBoth.join(',')); + + - name: Extract and Process Log + id: extract_summary + shell: bash {0} + run: | + # Get the failed jobs from outputs + failed_jobs_ib="${{ steps.get_failed_jobs.outputs.failed_jobs }}" + failed_jobs_no_ib="${{ steps.get_failed_jobs_no_ib.outputs.failed_jobs }}" + + # Get the conclusions and URLs + conclusion1="${{ steps.get_runs.outputs.conclusion1 }}" + url1="${{ steps.get_runs.outputs.url1 }}" + url2="${{ steps.get_runs.outputs.url2 }}" + + # Function to get status emoji + get_status_emoji() { + local conclusion="$1" + if [ "$conclusion" = "success" ]; then + echo "✅" + elif [ "$conclusion" = "failure" ]; then + echo "❌" + else + echo "⚠️" + fi + } + + emoji1=$(get_status_emoji "$conclusion1") + + # Initialize message + message="" + + # Add run details + message+="**Date:** $(date -u +"%b %d %Y")\n" + message+="**Status:** $emoji1 $conclusion1\n\n" + + message+="Pipeline Run 1: [View Run]($url1)\n" + message+="Pipeline Run 2: [View Run]($url2)\n\n" + + # Process jobs with '-ib' suffix + if [[ -n "$failed_jobs_ib" ]]; then + message+="**Ironbank Images failed consecutively in last two runs**:\n" + count=1 + IFS=',' read -ra JOBS_IB <<< "$failed_jobs_ib" + for job in "${JOBS_IB[@]}"; do + message+="$count. $job\n" + count=$((count + 1)) + done + fi + + # Process jobs without '-ib' suffix + if [[ -n "$failed_jobs_no_ib" ]]; then + message+="\n**Non-Ironbank Images failed consecutively in last two runs**:\n" + count=1 + IFS=',' read -ra JOBS_NO_IB <<< "$failed_jobs_no_ib" + for job in "${JOBS_NO_IB[@]}"; do + message+="$count. $job\n" + count=$((count + 1)) + done + fi + + # If no failed jobs, set message accordingly + if [[ -z "$failed_jobs_ib" && -z "$failed_jobs_no_ib" ]]; then + message+="All images passed in the last two runs." + fi + + # Save message to summary.txt + echo -e "$message" > summary.txt + + - name: Send to Zulip + if: ${{ always() }} + shell: bash + env: + ZULIP_SERVER_URL: ${{ secrets.ZULIP_SERVER_URL }} + ZULIP_BOT_EMAIL: ${{ secrets.ZULIP_BOT_EMAIL }} + ZULIP_BOT_API_KEY: ${{ secrets.ZULIP_BOT_API_KEY }} + ZULIP_STREAM_NAME: 'community images' + ZULIP_TOPIC_NAME: 'Image Creation Run Alerts 🤖' + run: | + message=$(cat summary.txt) + # Send the message to Zulip + response=$(curl -sSf -X POST "$ZULIP_SERVER_URL/api/v1/messages" \ + -u "$ZULIP_BOT_EMAIL:$ZULIP_BOT_API_KEY" \ + -d type="stream" \ + -d to="$ZULIP_STREAM_NAME" \ + -d topic="$ZULIP_TOPIC_NAME" \ + --data-urlencode content="$message") + + # Check if the curl command was successful + if [ $? -ne 0 ]; then + echo "Failed to send message to Zulip." + echo "Response: $response" + exit 1 + fi