Merge branch 'main' into vdelev_auto_release_part_2

datavisyn · Nov 18, 2024 · c40e50c · c40e50c
2 parents a42f0bd + 6077d00
commit c40e50c
Show file tree

Hide file tree

Showing 2 changed files with 65 additions and 13 deletions.
diff --git a/.github/actions/get-ecr-scan-result/action.yml b/.github/actions/get-ecr-scan-result/action.yml
@@ -53,8 +53,18 @@ runs:
     - name: Get AWS ECR Scan results
       id: get-scan-results
       run: |
-        aws ecr wait image-scan-complete --repository-name $ECR_REPOSITORY --image-id imageTag=$IMAGE_TAG
-        if [ $(echo $?) -eq 0 ]; then
+        # As the image scan itself may not be started yet, we have to wait (and retry) until it is actually available
+        max_retries=5
+        retries=0
+        scan_complete=1
+        until [ $retries -eq $max_retries ]; do
+            aws ecr wait image-scan-complete --repository-name $ECR_REPOSITORY --image-id imageTag=$IMAGE_TAG && scan_complete=0 && break
+            sleep 5
+            retries=$((retries + 1))
+            echo "Retry $retries/$max_retries: Waiting for image scan to start..."
+        done
+
+        if [ $scan_complete -eq 0 ]; then
           scan_findings=$(aws ecr describe-image-scan-findings --repository-name $ECR_REPOSITORY --image-id imageTag=$IMAGE_TAG | jq '.imageScanFindings.findingSeverityCounts')
           critical=$(echo $scan_findings | jq '.CRITICAL')
           high=$(echo $scan_findings | jq '.HIGH')

diff --git a/.github/workflows/build-node-python.yml b/.github/workflows/build-node-python.yml
@@ -215,23 +215,43 @@ jobs:
         ports:
           # will assign a random free host port
           - 5432/tcp
-
+      redis:
+        image: redis:6
+        options: >-
+          --health-cmd "redis-cli ping"
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+          --name redis_${{ github.job }}_${{ inputs.deduplication_id }}_${{ github.run_id }}_${{ github.run_attempt }}
+        ports:
+          - 6379/tcp
+
     steps:
       - name: Set self-hosted env variable to github env
         run: echo "GH_ACTIONS_SELF_HOSTED_NETWORK_NAME=${GH_ACTIONS_SELF_HOSTED_NETWORK_NAME}" >> "$GITHUB_ENV"
-      - name: Set github token, hostname, port and docker network for self-hosted runner
+      - name: Set up custom postgres and redis hostname, port and docker network for self-hosted runner
         if: env.GH_ACTIONS_SELF_HOSTED_NETWORK_NAME != ''
-        run: |
+        env:
+          REDIS_HOSTNAME: redis_${{ github.job }}_${{ inputs.deduplication_id }}_${{ github.run_id }}_${{ github.run_attempt }}
+          REDIS_PORT: 6379
+        run: | 
           {
             echo "POSTGRES_HOSTNAME=postgres_${{ github.job }}_${{ inputs.deduplication_id }}_${{ github.run_id }}_${{ github.run_attempt }}"
             echo "POSTGRES_PORT=5432"
+            echo "REDIS_HOSTNAME=redis_${{ github.job }}_${{ inputs.deduplication_id }}_${{ github.run_id }}_${{ github.run_attempt }}"
+            echo "REDIS_PORT=6379"
           } >> "$GITHUB_ENV"
-          docker network connect ${{ env.GH_ACTIONS_SELF_HOSTED_NETWORK_NAME }} "postgres_${{ github.job }}_${{ inputs.deduplication_id }}_${{ github.run_id }}_${{ github.run_attempt }}"
-      - name: Set postgres connection details to hosted runner
+          docker network connect ${{ env.GH_ACTIONS_SELF_HOSTED_NETWORK_NAME }} ${{ env.POSTGRES_HOSTNAME }}
+          docker network connect ${{ env.GH_ACTIONS_SELF_HOSTED_NETWORK_NAME }} ${{ env.REDIS_HOSTNAME }}
+      - name: Set service connection details to hosted runner
         if: env.GH_ACTIONS_SELF_HOSTED_NETWORK_NAME == ''
         run: |
-          echo "POSTGRES_HOSTNAME=localhost" >> "$GITHUB_ENV"
-          echo "POSTGRES_PORT=${{ job.services.postgres.ports['5432'] }}" >> "$GITHUB_ENV"
+          {
+            echo "POSTGRES_HOSTNAME=localhost"
+            echo "POSTGRES_PORT=${{ job.services.postgres.ports['5432'] }}" 
+            echo "REDIS_HOSTNAME=localhost"
+            echo "REDIS_PORT=${{ job.services.redis.ports['6379'] }}" 
+          } >> "$GITHUB_ENV"
       - name: Checkout source repository
         uses: actions/checkout@v4
         with:
@@ -334,23 +354,45 @@ jobs:
         ports:
           # will assign a random free host port
           - 5432/tcp
+
+      redis:
+        image: redis:6
+        options: >-
+          --health-cmd "redis-cli ping"
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+          --name redis_${{ github.job }}_${{ inputs.deduplication_id }}_${{ github.run_id }}_${{ github.run_attempt }}
+        ports:
+          - 6379/tcp
 
     steps:
       - name: Set system env variable to github env
         run: echo "GH_ACTIONS_SELF_HOSTED_NETWORK_NAME=${GH_ACTIONS_SELF_HOSTED_NETWORK_NAME}" >> "$GITHUB_ENV"
-      - name: Set github token, hostname, port and docker network for self-hosted runner
+      - name: Set up custom postgres and redis hostname, port and docker network for self-hosted runner
         if: env.GH_ACTIONS_SELF_HOSTED_NETWORK_NAME != ''
+        env:
+          REDIS_HOSTNAME: redis_${{ github.job }}_${{ inputs.deduplication_id }}_${{ github.run_id }}_${{ github.run_attempt }}
+          REDIS_PORT: 6379
         run: | 
           {
             echo "POSTGRES_HOSTNAME=postgres_${{ github.job }}_${{ inputs.deduplication_id }}_${{ github.run_id }}_${{ github.run_attempt }}"
             echo "POSTGRES_PORT=5432"
+            echo "REDIS_HOSTNAME=redis_${{ github.job }}_${{ inputs.deduplication_id }}_${{ github.run_id }}_${{ github.run_attempt }}"
+            echo "REDIS_PORT=6379"
           } >> "$GITHUB_ENV"
           docker network connect ${{ env.GH_ACTIONS_SELF_HOSTED_NETWORK_NAME }} ${{ env.POSTGRES_HOSTNAME }}
-      - name: Set postgres connection details to hosted runner
+          docker network connect ${{ env.GH_ACTIONS_SELF_HOSTED_NETWORK_NAME }} ${{ env.REDIS_HOSTNAME }}
+
+      - name: Set service connection details to hosted runner
         if: env.GH_ACTIONS_SELF_HOSTED_NETWORK_NAME == ''
         run: |
-          echo "POSTGRES_HOSTNAME=localhost" >> "$GITHUB_ENV"
-          echo "POSTGRES_PORT=${{ job.services.postgres.ports['5432'] }}" >> "$GITHUB_ENV"
+          {
+            echo "POSTGRES_HOSTNAME=localhost"
+            echo "POSTGRES_PORT=${{ job.services.postgres.ports['5432'] }}" 
+            echo "REDIS_HOSTNAME=localhost"
+            echo "REDIS_PORT=${{ job.services.redis.ports['6379'] }}" 
+          } >> "$GITHUB_ENV"
       - name: Checkout source repository
         uses: actions/checkout@v4
         with: