chore: Try to detect too many network issues and reset network

Instead of being stuck in the non-working setup.

Ticket: MEN-7555
Changelog: none
Signed-off-by: Vratislav Podzimek <[email protected]>

cmake/CMake_defaults.txt

@@ -20,3 +20,4 @@ set(CONFIG_MENDER_FULL_PARSE_ARTIFACT ON CACHE BOOL "Full parse artifact")
 set(CONFIG_MENDER_PROVIDES_DEPENDS ON CACHE BOOL "Provides depends")
 set(CONFIG_MENDER_COMMIT_REQUIRE_AUTH ON CACHE BOOL "Authentication required for update commit")
 set(CONFIG_MENDER_ALL_WARNINGS_AS_ERRORS ON CACHE BOOL "All warnings as errors")
+set(CONFIG_MENDER_ERRORS_THRESHOLD_NET 10 CACHE STRING "Network errors threshold")

cmake/mender_mcu_sources.txt

@@ -126,9 +126,10 @@ file(GLOB MENDER_MCU_SOURCES
     "${MENDER_MCU_ROOT}/core/src/mender-artifact.c"
     "${MENDER_MCU_ROOT}/core/src/mender-artifact-download.c"
     "${MENDER_MCU_ROOT}/core/src/mender-client.c"
+    "${MENDER_MCU_ROOT}/core/src/mender-deployment-data.c"
+    "${MENDER_MCU_ROOT}/core/src/mender-error-counters.c"
     "${MENDER_MCU_ROOT}/core/src/mender-update-module.c"
     "${MENDER_MCU_ROOT}/core/src/mender-utils.c"
-    "${MENDER_MCU_ROOT}/core/src/mender-deployment-data.c"
     "${MENDER_MCU_ROOT}/platform/log/${CONFIG_MENDER_PLATFORM_LOG_TYPE}/src/mender-log.c"
     "${MENDER_MCU_ROOT}/platform/net/${CONFIG_MENDER_PLATFORM_NET_TYPE}/src/mender-http.c"
     "${MENDER_MCU_ROOT}/platform/scheduler/${CONFIG_MENDER_PLATFORM_SCHEDULER_TYPE}/src/mender-scheduler.c"

core/src/mender-api.c

@@ -23,6 +23,7 @@
 
 #include "mender-api.h"
 #include "mender-artifact.h"
+#include "mender-error-counters.h"
 #include "mender-scheduler.h"
 #include "mender-storage.h"
 #include "mender-http.h"
@@ -230,6 +231,7 @@ perform_authentication(void) {
                                       (void *)&response,
                                       &status))) {
         mender_log_error("Unable to perform HTTP request");
+        mender_err_count_net_inc();
         goto END;
     }
 
@@ -295,6 +297,7 @@ authenticated_http_perform(char *path, mender_http_method_t method, char *payloa
     }
     if (MENDER_OK != ret) {
         /* HTTP errors already logged. */
+        mender_err_count_net_inc();
         return ret;
     }
 
@@ -309,6 +312,10 @@ authenticated_http_perform(char *path, mender_http_method_t method, char *payloa
                 mender_log_error("Unable to release the authentication lock");
                 return MENDER_FAIL;
             }
+            if (MENDER_OK != ret) {
+                /* HTTP errors already logged. */
+                mender_err_count_net_inc();
+            }
         } else if (MENDER_LOCK_FAILED != ret) {
             if (MENDER_OK != mender_scheduler_mutex_give(auth_lock)) {
                 mender_log_error("Unable to release the authentication lock");

core/src/mender-artifact-download.c

@@ -22,6 +22,7 @@
 #include "mender-artifact.h"
 #include "mender-artifact-download.h"
 #include "mender-artifact-download-data.h"
+#include "mender-error-counters.h"
 #include "mender-http.h"
 #include "mender-log.h"
 
@@ -48,6 +49,7 @@ mender_download_artifact(const char *uri, mender_deployment_data_t *deployment_d
     /* Perform HTTP request */
     if (MENDER_OK != (ret = mender_http_artifact_download(uri, &dl_data, &status))) {
         mender_log_error("Unable to perform HTTP request");
+        mender_err_count_net_inc();
         return ret;
     }
 

core/src/mender-client.c

@@ -29,6 +29,7 @@
 #include "mender-update-module.h"
 #include "mender-utils.h"
 #include "mender-deployment-data.h"
+#include "mender-error-counters.h"
 
 #ifdef CONFIG_MENDER_CLIENT_INVENTORY
 #include "mender-inventory.h"
@@ -433,6 +434,7 @@ mender_client_exit(void) {
 
 static mender_err_t
 mender_client_work_function(void) {
+    mender_err_t ret;
     mender_log_debug("Inside work function [state: %d]", mender_client_state);
 
     switch (mender_client_state) {
@@ -448,7 +450,15 @@ mender_client_work_function(void) {
             mender_client_state = MENDER_CLIENT_STATE_OPERATIONAL;
             /* fallthrough */
         case MENDER_CLIENT_STATE_OPERATIONAL:
-            return mender_client_update_work_function();
+            ret = mender_client_update_work_function();
+            if (MENDER_FAIL == ret) {
+                if (MENDER_FAIL == mender_err_count_net_check()) {
+                    /* Try to release network so that it gets set up again next
+                       time. */
+                    mender_client_network_release();
+                }
+            }
+            return ret;
     }
 
     /* This should never be reached, all the cases should be covered in the

core/src/mender-error-counters.c

@@ -0,0 +1,59 @@
+/**
+ * @file      mender-error-counters.c
+ * @brief     Mender Error counters implementation
+ *
+ * Copyright Northern.tech AS
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#if CONFIG_MENDER_ERRORS_THRESHOLD_NET > 0
+
+#include <stdint.h>
+
+#include "mender-error-counters.h"
+#include "mender-log.h"
+#include "mender-utils.h"
+
+static uint8_t net_errors = 0;
+#if CONFIG_MENDER_ERRORS_THRESHOLD_NET > UINT8_MAX
+#error "CONFIG_MENDER_ERRORS_THRESHOLD_NET must be <= UINT8_MAX"
+#endif
+
+mender_err_t
+mender_err_count_net_inc(void) {
+    if (net_errors < UINT8_MAX) {
+        net_errors++;
+    }
+    if (net_errors > CONFIG_MENDER_ERRORS_THRESHOLD_NET) {
+        mender_log_warning("Network errors limit exceeded");
+        return MENDER_FAIL;
+    }
+    return MENDER_OK;
+}
+
+mender_err_t
+mender_err_count_net_check(void) {
+    if (net_errors > CONFIG_MENDER_ERRORS_THRESHOLD_NET) {
+        mender_log_warning("Network errors limit exceeded");
+        return MENDER_FAIL;
+    }
+    return MENDER_OK;
+}
+
+mender_err_t
+mender_err_count_net_reset(void) {
+    net_errors = 0;
+    return MENDER_OK;
+}
+#endif /* CONFIG_MENDER_ERRORS_THRESHOLD_NET > 0 */

include/mender-error-counters.h

@@ -0,0 +1,75 @@
+/**
+ * @file      mender-error-counters.h
+ * @brief     Mender Error Counters interface
+ *
+ * Copyright Northern.tech AS
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef __MENDER_ERROR_COUNTERS_H__
+#define __MENDER_ERROR_COUNTERS_H__
+
+#ifdef __cplusplus
+extern "C" {
+#endif /* __cplusplus */
+
+#include <mender-utils.h>
+
+#ifndef CONFIG_MENDER_ERRORS_THRESHOLD_NET
+#define CONFIG_MENDER_ERRORS_THRESHOLD_NET 0
+#endif
+
+#if CONFIG_MENDER_ERRORS_THRESHOLD_NET > 0
+
+/**
+ * @brief Increment the network errors counter
+ * @return MENDER_OK if not too many errors, MENDER_FAIL if too many errors
+ */
+mender_err_t mender_err_count_net_inc(void);
+
+/**
+ * @brief Check the network errors counter
+ * @return MENDER_OK if not too many errors, MENDER_FAIL if too many errors
+ */
+mender_err_t mender_err_count_net_check(void);
+
+/**
+ * @brief Reset the network errors counter
+ * @return MENDER_OK if successful, error otherwise
+ */
+mender_err_t mender_err_count_net_reset(void);
+
+#else
+
+/* Define the functions as inline noops so that the compiler can simply rule them out. */
+inline mender_err_t
+mender_err_count_net_inc(void) {
+    return MENDER_OK;
+}
+inline mender_err_t
+mender_err_count_net_check(void) {
+    return MENDER_OK;
+}
+inline mender_err_t
+mender_err_count_net_reset(void) {
+    return MENDER_OK;
+}
+
+#endif /* CONFIG_MENDER_ERRORS_THRESHOLD_NET > 0 */
+
+#ifdef __cplusplus
+}
+#endif /* __cplusplus */
+
+#endif /* __MENDER_ERROR_COUNTERS_H__ */

target/posix/CMakeLists.txt

@@ -74,6 +74,10 @@ endif()
 if (CONFIG_MENDER_COMMIT_REQUIRE_AUTH)
     target_compile_definitions(mender-mcu-client PUBLIC CONFIG_MENDER_COMMIT_REQUIRE_AUTH)
 endif()
+if (CONFIG_MENDER_ERRORS_THRESHOLD_NET)
+    message("Applying network errors threshold ${CONFIG_MENDER_ERRORS_THRESHOLD_NET}" )
+    target_compile_definitions(mender-mcu-client PUBLIC CONFIG_MENDER_ERRORS_THRESHOLD_NET=${CONFIG_MENDER_ERRORS_THRESHOLD_NET})
+endif()
 
 find_package(PkgConfig REQUIRED)
 

target/zephyr/Kconfig

@@ -336,6 +336,24 @@ if MENDER_MCU_CLIENT
                 Whether a successful authentication to the Mender server is a requirement for an update to be considered OK.
                 WARNING: Disabling this can leave devices disconnected from the Mender forever! A similar check must be performed in the update module being used!
 
+        menu "Error detection and recovery"
+
+            config MENDER_DETECT_NETWORK_ERRORS
+                bool "Try to detect network errors and reset network to fix them"
+                default y
+                help
+                    Whether Mender should try to detect network errors and attempt to fix them when there are too many.
+
+            config MENDER_ERRORS_THRESHOLD_NET
+                int "Network errors threshold"
+                range 1 255
+                default 10
+                depends on MENDER_DETECT_NETWORK_ERRORS
+                help
+                    The number of errors triggering network reset.
+
+        endmenu
+
     endmenu
 
 endif