Better node chooser

packages/cosmos/src/client/node_chooser.rs

@@ -32,14 +32,27 @@ impl NodeChooser {
     }
 
     pub(super) fn choose_node(&self) -> Result<&Node, ConnectionError> {
-        let primary_health = self.primary.is_healthy(self.allowed_error_count);
+        // First we try to find a node that has had no issues at all.
+        // If that fails, then we go for the allowed error count.
+        // Motivation: we previously had issues where we would retry the primary
+        // node multiple times, exhausting our retry counts, and never fall
+        // back to another node.
+        self.choose_node_with_allowed(0)
+            .or_else(|_| self.choose_node_with_allowed(self.allowed_error_count))
+    }
+
+    fn choose_node_with_allowed(
+        &self,
+        allowed_error_count: usize,
+    ) -> Result<&Node, ConnectionError> {
+        let primary_health = self.primary.is_healthy(allowed_error_count);
         if primary_health.is_healthy() {
             Ok(&self.primary)
         } else {
             let fallbacks = self
                 .fallbacks
                 .iter()
-                .filter(|node| node.is_healthy(self.allowed_error_count).is_healthy())
+                .filter(|node| node.is_healthy(allowed_error_count).is_healthy())
                 .collect::<Vec<_>>();
 
             let mut rng = rand::thread_rng();

packages/cosmos/src/error.rs

@@ -412,6 +412,8 @@ pub enum QueryErrorDetails {
     RateLimited { source: tonic::Status },
     #[error("The gRPC server is returning a 'forbidden' response: {source:?}")]
     Forbidden { source: tonic::Status },
+    #[error("Server returned response that does not look like valid gRPC: {source:?}")]
+    NotGrpc { source: tonic::Status },
 }
 
 /// Different known Cosmos SDK error codes
@@ -540,6 +542,7 @@ impl QueryErrorDetails {
             QueryErrorDetails::AccountSequenceMismatch { .. } => ConnectionIsFine,
             QueryErrorDetails::RateLimited { .. } => NetworkIssue,
             QueryErrorDetails::Forbidden { .. } => NetworkIssue,
+            QueryErrorDetails::NotGrpc { .. } => NetworkIssue,
         }
     }
 
@@ -605,6 +608,14 @@ impl QueryErrorDetails {
             };
         }
 
+        if err.message().contains("status: 405") {
+            return QueryErrorDetails::NotGrpc { source: err };
+        }
+
+        if err.message().contains("invalid compression flag") {
+            return QueryErrorDetails::NotGrpc { source: err };
+        }
+
         QueryErrorDetails::Unknown(err)
     }
 
@@ -623,7 +634,8 @@ impl QueryErrorDetails {
             | QueryErrorDetails::TransportError { .. }
             | QueryErrorDetails::BlocksLagDetected { .. }
             | QueryErrorDetails::NoNewBlockFound { .. }
-            | QueryErrorDetails::AccountSequenceMismatch(_) => false,
+            | QueryErrorDetails::AccountSequenceMismatch(_)
+            | QueryErrorDetails::NotGrpc { .. } => false,
             QueryErrorDetails::RateLimited { .. } | QueryErrorDetails::Forbidden { .. } => true,
         }
     }