From 88ede087bdbdde3c3cb57aeb42bb9119f49f2d86 Mon Sep 17 00:00:00 2001
From: jctanner <tanner.jc@gmail.com>
Date: Mon, 11 Sep 2023 12:43:24 -0400
Subject: [PATCH] Catch & retry more possible errors in the insights proxy
 (#1876)

No-Issue

Signed-off-by: James Tanner <tanner.jc@gmail.com>
---
 .github/workflows/ci_oci-env-integration.yml  |  2 +
 dev/oci_env_integration/actions/action_lib.py | 12 ++---
 profiles/insights/proxy/main.go               | 44 ++++++++++++++++++-
 3 files changed, 52 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/ci_oci-env-integration.yml b/.github/workflows/ci_oci-env-integration.yml
index 9785e94fb0..bb02ce4a45 100644
--- a/.github/workflows/ci_oci-env-integration.yml
+++ b/.github/workflows/ci_oci-env-integration.yml
@@ -37,6 +37,8 @@ jobs:
         run: |
           echo "OCI_ENV_PATH=${HOME}/work/galaxy_ng/oci_env" >> $GITHUB_ENV
           echo "COMPOSE_INTERACTIVE_NO_CLI=1" >> $GITHUB_ENV
+          echo "OCI_VERBOSE=1" >> $GITHUB_ENV
+          echo "GH_DUMP_LOGS=1" >> $GITHUB_ENV
 
       - name: Update apt
         run: sudo apt -y update
diff --git a/dev/oci_env_integration/actions/action_lib.py b/dev/oci_env_integration/actions/action_lib.py
index f44c9c80b2..8be2df9dca 100644
--- a/dev/oci_env_integration/actions/action_lib.py
+++ b/dev/oci_env_integration/actions/action_lib.py
@@ -48,6 +48,7 @@ class OCIEnvIntegrationTest:
     do_teardown = True
     flags = ""
     envs = {}
+    failed = None
 
     def __init__(self, envs):
         for env in envs:
@@ -59,18 +60,18 @@ def __init__(self, envs):
         self.do_teardown = os.environ.get("GH_TEARDOWN", "1") == "1"
         self.flags = os.environ.get("GH_FLAGS", "")
 
-        failed = False
+        self.failed = False
         try:
             self.set_up_env()
             self.run_test()
         except Exception as e:
             print(e)
-            failed = True
+            self.failed = True
         finally:
             self.dump_logs()
             self.teardown()
 
-        if failed:
+        if self.failed:
             exit(1)
 
     def set_up_env(self):
@@ -129,8 +130,9 @@ def run_test(self):
     def dump_logs(self):
         if not self.do_dump_logs:
             return
-        for env in self.envs:
-            self.exec_cmd(env, "compose logs")
+        if self.failed:
+            for env in self.envs:
+                self.exec_cmd(env, "compose logs")
 
     def teardown(self):
         if not self.do_teardown:
diff --git a/profiles/insights/proxy/main.go b/profiles/insights/proxy/main.go
index 89418a57e2..6844dd49f3 100644
--- a/profiles/insights/proxy/main.go
+++ b/profiles/insights/proxy/main.go
@@ -4,6 +4,7 @@ import (
 	"encoding/base64"
 	"encoding/json"
 	"fmt"
+	"io"
 	"io/ioutil"
 	"log"
 	"math/rand"
@@ -223,6 +224,25 @@ func isConnectionResetByPeer(err error) bool {
 	return false
 }
 
+func isUseOfClosedNetworkConnection(err error) bool {
+	return strings.Contains(err.Error(), "use of closed network connection")
+}
+
+func isWriteBrokenPipe(err error) bool {
+	return strings.Contains(err.Error(), "write: broken pipe")
+}
+
+func isInvalidReadOnClosedBody(err error) bool {
+	return strings.Contains(err.Error(), "invalid Read on closed Body")
+}
+
+func isEOFerror(err error) bool {
+	if err == io.EOF {
+		return true
+	}
+	return false
+}
+
 func retryHTTPRequest(client *http.Client, req *http.Request, maxRetries int) (*http.Response, error) {
 	for retry := 0; retry < maxRetries; retry++ {
 		resp, err := client.Do(req)
@@ -237,6 +257,26 @@ func retryHTTPRequest(client *http.Client, req *http.Request, maxRetries int) (*
 				time.Sleep(1 * time.Second) // Wait before retrying
 				continue
 			}
+			if isUseOfClosedNetworkConnection(err) {
+				fmt.Printf("Retry attempt %d: use of closed network connection\n", retry+1)
+				time.Sleep(1 * time.Second) // Wait before retrying
+				continue
+			}
+			if isWriteBrokenPipe(err) {
+				fmt.Printf("Retry attempt %d: write broken pipe\n", retry+1)
+				time.Sleep(1 * time.Second) // Wait before retrying
+				continue
+			}
+			if isInvalidReadOnClosedBody(err) {
+				fmt.Printf("Retry attempt %d: invalid read on closed body\n", retry+1)
+				time.Sleep(1 * time.Second) // Wait before retrying
+				continue
+			}
+			if isEOFerror(err) {
+				fmt.Printf("Retry attempt %d: EOF error\n", retry+1)
+				time.Sleep(1 * time.Second) // Wait before retrying
+				continue
+			}
 			return nil, err
 		}
 
@@ -247,7 +287,7 @@ func retryHTTPRequest(client *http.Client, req *http.Request, maxRetries int) (*
 }
 
 func main() {
-	fmt.Println("Staring insights proxy.")
+	fmt.Println("Starting insights proxy.")
 
 	// define origin server URL
 	urlToProxyTo, err := url.Parse(getEnv("UPSTREAM_URL", "http://localhost:5001"))
@@ -326,6 +366,8 @@ func main() {
 		data, _ := ioutil.ReadAll(upstreamServerResponse.Body)
 		modified := downloadUrlReg.ReplaceAll(data, replacementURL)
 
+		fmt.Printf("MODIFIED DATA: %s\n", modified)
+
 		// Write the response
 		rw.WriteHeader(upstreamServerResponse.StatusCode)
 		rw.Write(modified)