fix(kbuild): Implement command retry to firmware clone

kernelci/kbuild.py

@@ -1,4 +1,4 @@
 # SPDX-License-Identifier: LGPL-2.1-or-later
 #
 # Copyright (C) 2023 Collabora Limited
 # Author: Denys Fedoryshchenko <[email protected]>
@@ -40,14 +40,14 @@
 
 
 CROS_CONFIG_URL = \
     "https://chromium.googlesource.com/chromiumos/third_party/kernel/+archive/refs/heads/{branch}/chromeos/config.tar.gz"  # noqa
 LEGACY_CONFIG = [
     'config/core/build-configs.yaml',
     '/etc/kernelci/core/build-configs.yaml',
 ]
 FW_GIT = "https://git.kernel.org/pub/scm/linux/kernel/git/firmware/linux-firmware.git"  # noqa
 
 # TODO: find a way to automatically fetch this information
 LATEST_LTS_MAJOR = 6
 LATEST_LTS_MINOR = 6
 
@@ -110,10 +110,10 @@
     except requests.exceptions.RequestException as e:
         print(f"[_download_file_inner] Error: {e}")
         return False
     except requests.exceptions.Timeout as e:
         print(f"[_download_file_inner] Timeout: {e}")
         return False
     except requests.exceptions.ConnectionError as e:
         print(f"[_download_file_inner] Connection error: {e}")
         return False
     except Exception as e:
@@ -381,6 +381,21 @@
         cmd = f'echo jobsts:{jobname}=$(date +%s) >> {self._af_dir}/state.txt'
         self._steps.append(cmd)
 
+    def addcmdretry(self, cmd, retries=10):
+        '''
+        Retry command if it fails up to retries times
+        '''
+        self.addcomment(f"Retry command {cmd} {retries} times")
+        self.addcmd("for i in $(seq 1 " + str(retries) + "); do")
+        self.addcmd("  echo \"Attempt $i\"")
+        self.addcmd("  sleep 1")
+        self.addcmd("  " + cmd + " && break")
+        self.addcmd("done")
+        self.addcmd("if [ $? -ne 0 ]; then")
+        self.addcmd("  echo \"Failed to run " + cmd + "\"")
+        self.addcmd("  exit 1")
+        self.addcmd("fi")
+
     def addcmd(self, cmd, critical=True):
         '''
         critical - if True, check return code and exit, as command is critical
@@ -397,7 +412,8 @@
         TODO: Implement firmware commit id meta/settings
         '''
         self.startjob("fetch_firmware")
-        self.addcmd(f"git clone {FW_GIT} --depth 1", False)
+        # self.addcmd(f"git clone {FW_GIT} --depth 1", False)
+        self.addcmdretry(f"git clone {FW_GIT} --depth 1", 10)
         self.addcmd("cd linux-firmware")
         self.addcmd("./copy-firmware.sh " + self._firmware_dir)
         self.addcmd("cd ..")
@@ -494,8 +510,8 @@
             sys.exit(1)
 
         print(f"Searching for fragment {fragname} in {cfg_path}")
         if 'fragments' in yml:
             frag = yml['fragments']
         else:
             print("No fragments section in config file")
             self.submit_failure("No fragments section in config file")
@@ -521,7 +537,7 @@
             elif fragment.startswith("CONFIG_"):
                 content = fragment + '\n'
             else:
                 # TODO: implement 'path' option properly
                 content = self.add_legacy_fragment(fragment)
 
             fragfile = os.path.join(self._fragments_dir, f"{num}.config")
@@ -570,8 +586,8 @@
         for i in range(0, fragnum):
             self.addcmd("./scripts/kconfig/merge_config.sh" +
                         f" -m .config {self._fragments_dir}/{i}.config")
         # TODO: olddefconfig should be optional/configurable
         # TODO: log all warnings/errors of olddefconfig to separate file
         self.addcmd("make olddefconfig")
         self.addcmd(f"cp .config {self._af_dir}/")
         self.addcmd("cd ..")
@@ -579,12 +595,12 @@
 
     def _generate_script(self):
         """ Generate shell script for complete build """
         # TODO(nuclearcat): Fetch firmware only if needed
         print("Generating shell script")
         fragnum = self._parse_fragments(firmware=True)
         self._merge_frags(fragnum)
         if not self._dtbs_check:
             # TODO: verify if CONFIG_EXTRA_FIRMWARE have any files
             # We can check that if fragments have CONFIG_EXTRA_FIRMWARE
             self._fetch_firmware()
             self._build_kernel()
@@ -683,7 +699,7 @@
         """ Add kernel image packaging steps """
         self.startjob("package_kimage")
         self.addcmd("cd " + self._srcdir)
         # TODO(nuclearcat): Not all images might be present
         for img in KERNEL_IMAGE_NAMES[self._arch]:
             self.addcmd("cp arch/" + self._arch + "/boot/" + img + " ../artifacts", False)
             # add image to artifacts relative to artifacts dir
@@ -752,7 +768,7 @@
 
     def serialize(self, filename):
         """ Serialize class to json """
         # TODO(nuclearcat): Implement to_json method?
         data = json.dumps(self, default=lambda o: o.__dict__,
                           sort_keys=True, indent=4)
         with open(filename, 'w') as f:
@@ -897,7 +913,7 @@
             api.node.update(node)
         except requests.exceptions.HTTPError as err:
             err_msg = json.loads(err.response.content).get("detail", [])
             self.log.error(err_msg)
         return
 
     def submit(self, retcode, dry_run=False):