fix(kbuild): Implement file download retry

kernelci/kbuild.py

@@ -1,4 +1,4 @@
 # SPDX-License-Identifier: LGPL-2.1-or-later
 #
 # Copyright (C) 2023 Collabora Limited
 # Author: Denys Fedoryshchenko <[email protected]>
@@ -30,6 +30,7 @@
 import tarfile
 import json
 import requests
+import time
 import yaml
 import kernelci.api
 import kernelci.api.helper
@@ -39,14 +40,14 @@
 
 
 CROS_CONFIG_URL = \
     "https://chromium.googlesource.com/chromiumos/third_party/kernel/+archive/refs/heads/{branch}/chromeos/config.tar.gz"  # noqa
 LEGACY_CONFIG = [
     'config/core/build-configs.yaml',
     '/etc/kernelci/core/build-configs.yaml',
 ]
 FW_GIT = "https://git.kernel.org/pub/scm/linux/kernel/git/firmware/linux-firmware.git"  # noqa
 
 # TODO: find a way to automatically fetch this information
 LATEST_LTS_MAJOR = 6
 LATEST_LTS_MINOR = 6
 
@@ -103,18 +104,40 @@
 REDIR = ' > >(tee {}) 2> >(tee {} >&1)'
 
 
+def _download_file_inner(url, file_path):
+    try:
+        r = requests.get(url, stream=True, timeout=60)
+    except requests.exceptions.RequestException as e:
+        print(f"[_download_file_inner] Error: {e}")
+        return False
+    except requests.exceptions.Timeout as e:
+        print(f"[_download_file_inner] Timeout: {e}")
+        return False
+    except requests.exceptions.ConnectionError as e:
+        print(f"[_download_file_inner] Connection error: {e}")
+        return False
+    except Exception as e:
+        print(f"[_download_file_inner] Exception: {e}")
+        return False
+    if r.status_code == 200:
+        with open(file_path, 'wb') as f:
+            for chunk in r:
+                f.write(chunk)
+        return True
+
+
 def _download_file(url, file_path):
     '''
     Download file to file_path
     TODO(nuclearcat): Do we have anything generic in KernelCI for this?
     '''
     print(f"[_download_file] Downloading {url} to {file_path}")
-    r = requests.get(url, stream=True, timeout=60)
-    if r.status_code == 200:
-        with open(file_path, 'wb') as f:
-            for chunk in r:
-                f.write(chunk)
-        return True
+    retries = 10
+    while retries > 0:
+        if _download_file_inner(url, file_path):
+            return True
+        time.sleep(5)
+        retries -= 1
     return False
 
 
@@ -422,6 +445,25 @@
 
         return (buffer, fragment)
 
+    def _verify_network(self):
+        """ Verify network connectivity """
+        # TBD: Different URL? pool of urls?
+        retries = 10
+        while retries > 0:
+            try:
+                r = requests.get("https://google.com")
+            except Exception as e:
+                print(f"[_verify_network] Error: {e}")
+                time.sleep(5)
+                retries -= 1
+                continue
+            if r.status_code == 200:
+                return
+            time.sleep(5)
+
+        print("[_verify_network] Network is not available")
+        sys.exit(1)
+
     def extract_config(self, frag):
         """ Extract config fragments from legacy config file """
         txt = ''
@@ -452,8 +494,8 @@
             sys.exit(1)
 
         print(f"Searching for fragment {fragname} in {cfg_path}")
         if 'fragments' in yml:
             frag = yml['fragments']
         else:
             print("No fragments section in config file")
             self.submit_failure("No fragments section in config file")
@@ -479,7 +521,7 @@
             elif fragment.startswith("CONFIG_"):
                 content = fragment + '\n'
             else:
                 # TODO: implement 'path' option properly
                 content = self.add_legacy_fragment(fragment)
 
             fragfile = os.path.join(self._fragments_dir, f"{num}.config")
@@ -528,8 +570,8 @@
         for i in range(0, fragnum):
             self.addcmd("./scripts/kconfig/merge_config.sh" +
                         f" -m .config {self._fragments_dir}/{i}.config")
         # TODO: olddefconfig should be optional/configurable
         # TODO: log all warnings/errors of olddefconfig to separate file
         self.addcmd("make olddefconfig")
         self.addcmd(f"cp .config {self._af_dir}/")
         self.addcmd("cd ..")
@@ -537,12 +579,12 @@
 
     def _generate_script(self):
         """ Generate shell script for complete build """
         # TODO(nuclearcat): Fetch firmware only if needed
         print("Generating shell script")
         fragnum = self._parse_fragments(firmware=True)
         self._merge_frags(fragnum)
         if not self._dtbs_check:
             # TODO: verify if CONFIG_EXTRA_FIRMWARE have any files
             # We can check that if fragments have CONFIG_EXTRA_FIRMWARE
             self._fetch_firmware()
             self._build_kernel()
@@ -570,6 +612,7 @@
 
     def write_script(self, filename):
         """ Write shell compile script to file """
+        self._verify_network()
         self._generate_script()
         with open(filename, 'w') as f:
             for step in self._steps:
@@ -640,7 +683,7 @@
         """ Add kernel image packaging steps """
         self.startjob("package_kimage")
         self.addcmd("cd " + self._srcdir)
         # TODO(nuclearcat): Not all images might be present
         for img in KERNEL_IMAGE_NAMES[self._arch]:
             self.addcmd("cp arch/" + self._arch + "/boot/" + img + " ../artifacts", False)
             # add image to artifacts relative to artifacts dir
@@ -709,7 +752,7 @@
 
     def serialize(self, filename):
         """ Serialize class to json """
         # TODO(nuclearcat): Implement to_json method?
         data = json.dumps(self, default=lambda o: o.__dict__,
                           sort_keys=True, indent=4)
         with open(filename, 'w') as f:
@@ -854,7 +897,7 @@
             api.node.update(node)
         except requests.exceptions.HTTPError as err:
             err_msg = json.loads(err.response.content).get("detail", [])
             self.log.error(err_msg)
         return
 
     def submit(self, retcode, dry_run=False):