From a2c8973f6a133947f6ef50c8daca1c0376ddb0f6 Mon Sep 17 00:00:00 2001 From: emilsvennesson Date: Tue, 30 May 2023 13:57:31 +0000 Subject: [PATCH] [script.module.inputstreamhelper] 0.6.1+matrix.1 --- script.module.inputstreamhelper/README.md | 4 + script.module.inputstreamhelper/addon.xml | 12 ++- .../lib/inputstreamhelper/config.py | 59 +++--------- .../lib/inputstreamhelper/utils.py | 31 ++++--- .../lib/inputstreamhelper/widevine/arm.py | 29 +++--- .../widevine/arm_chromeos.py | 91 +++++++++++++++++-- .../resource.language.en_gb/strings.po | 12 +++ 7 files changed, 150 insertions(+), 88 deletions(-) diff --git a/script.module.inputstreamhelper/README.md b/script.module.inputstreamhelper/README.md index 50e9fcadc..93c5e8eb6 100644 --- a/script.module.inputstreamhelper/README.md +++ b/script.module.inputstreamhelper/README.md @@ -91,6 +91,10 @@ Please report any issues or bug reports on the [GitHub Issues](https://github.co This module is licensed under the **The MIT License**. Please see the [LICENSE.txt](LICENSE.txt) file for details. ## Releases +### v0.6.1 (2023-05-30) +- Performance improvements on Linux ARM (@horstle) +- This will be the last release for Python 2 i.e. Kodi 18 (Leia) and below. The next release will require Python 3 and Kodi 19 (Matrix) or higher. + ### v0.6.0 (2023-05-03) - Initial support for AARCH64 Linux (@horstle) - Initial support for AARCH64 Macs (@mediaminister) diff --git a/script.module.inputstreamhelper/addon.xml b/script.module.inputstreamhelper/addon.xml index 1e7cb745a..610b2a195 100644 --- a/script.module.inputstreamhelper/addon.xml +++ b/script.module.inputstreamhelper/addon.xml @@ -1,5 +1,5 @@ - + @@ -25,10 +25,14 @@ Jednostavan Kodi modul koji olakšava razvijanje dodataka koji se temelje na InputStream dodatku i reprodukciji DRM zaštićenog sadržaja. Простой модуль для Kodi, который облегчает жизнь разработчикам дополнений, с использованием InputStream дополнений и воспроизведения DRM контента. +v0.6.1 (2023-05-30) +- Performance improvements on Linux ARM +- This will be the last release for Python 2 i.e. Kodi 18 (Leia) and below. The next release will require Python 3 and Kodi 19 (Matrix) or higher. + v0.6.0 (2023-05-03) -- Initial support for AARCH64 Linux (@horstle) -- Initial support for AARCH64 Macs (@mediaminister) -- New option to install a specific version on most platforms (@horstle) +- Initial support for AARCH64 Linux +- Initial support for AARCH64 Macs +- New option to install a specific version on most platforms v0.5.10 (2022-04-18) - Fix automatic submission of release diff --git a/script.module.inputstreamhelper/lib/inputstreamhelper/config.py b/script.module.inputstreamhelper/lib/inputstreamhelper/config.py index fae104c86..110e60223 100644 --- a/script.module.inputstreamhelper/lib/inputstreamhelper/config.py +++ b/script.module.inputstreamhelper/lib/inputstreamhelper/config.py @@ -84,53 +84,22 @@ # https://www.chromium.org/chromium-os/developer-information-for-chrome-os-devices # https://chromiumdash.appspot.com/serving-builds?deviceCategory=Chrome%20OS # Last updated: 2023-03-24 -CHROMEOS_RECOVERY_ARM_HWIDS = [ - 'BOB', - 'BURNET', - 'COACHZ', - 'COZMO', - 'DAMU', - 'DOJO-EJPG', - 'DRUWL', - 'DUMO', - 'ELM', - 'ESCHE', - 'FENNEL', - 'FENNEL14', - 'HANA', - 'HAYATO-YLRO', - 'HOMESTAR-MBLE', - 'JUNIPER-HVPU', - 'KAKADU-WFIQ', - 'KAPPA', - 'KAPPA-EWFK', - 'KATSU', - 'KENZO-IGRW', - 'KEVIN', - 'KODAMA', - 'KRANE-ZDKS', - 'MAKOMO-UTTX', - 'PICO-EXEM', - 'QUACKINGSTICK', - 'SCARLET', - 'SPHERION', - 'TOMATO-LYVN', - 'WILLOW-TFIY', - 'WORMDINGLER-JQAO', +CHROMEOS_RECOVERY_ARM_BNAMES = [ + 'asurada', + 'bob', + 'cherry', + 'elm', + 'hana', + 'jacuzzi', + 'kevin', + 'kukui', + 'scarlet', + 'strongbad', ] -CHROMEOS_RECOVERY_ARM64_HWIDS = [ - 'KINGOFTOWN-KDDA', - 'LAZOR', - 'LIMOZEEN', - 'MAGNETON-LCKC', - 'PAZQUEL-HGNV', - 'PAZQUEL-OPNA', - 'POMPOM-MZVS', - 'RUSTY-ZNCE', - 'STEELIX-VZSZ', - 'TENTACOOL-ZLJE', - 'TENTACRUEL-VAFH', +CHROMEOS_RECOVERY_ARM64_BNAMES = [ + 'corsola', + 'trogdor', ] MINIMUM_INPUTSTREAM_VERSION_ARM64 = { diff --git a/script.module.inputstreamhelper/lib/inputstreamhelper/utils.py b/script.module.inputstreamhelper/lib/inputstreamhelper/utils.py index 0ab01a174..b4b4452fd 100644 --- a/script.module.inputstreamhelper/lib/inputstreamhelper/utils.py +++ b/script.module.inputstreamhelper/lib/inputstreamhelper/utils.py @@ -164,21 +164,26 @@ def http_download(url, message=None, checksum=None, hash_alg='sha1', dl_size=Non progress.update(percent, prog_message) - if checksum and calc_checksum.hexdigest() != checksum: - progress.close() - req.close() - log(4, 'Download failed, checksums do not match!') - return False - - if dl_size and stat_file(dl_path).st_size() != dl_size: - progress.close() - req.close() - free_space = sizeof_fmt(diskspace()) - log(4, 'Download failed, filesize does not match! Filesystem full? Remaining diskspace in temp: {}.'.format(free_space)) - return False - progress.close() req.close() + + checksum_ok = (not checksum or calc_checksum.hexdigest() == checksum) + size_ok = (not dl_size or stat_file(dl_path).st_size() == dl_size) + + if not all((checksum_ok, size_ok)): + free_space = sizeof_fmt(diskspace()) + log(4, 'Something may be wrong with the downloaded file.') + if not checksum_ok: + log(4, 'Provided checksum: {}\nCalculated checksum: {}'.format(checksum, calc_checksum.hexdigest())) + if not size_ok: + free_space = sizeof_fmt(diskspace()) + log(4, 'Expected filesize: {}\nReal filesize: {}\nRemaining diskspace: {}'.format(dl_size, stat_file(dl_path).st_size(), free_space)) + + if yesno_dialog(localize(30003), localize(30070, filename=filename)): # file maybe broken. Continue anyway? + log(4, 'Continuing despite possibly corrupt file!') + else: + return False + store('download_path', dl_path) return True diff --git a/script.module.inputstreamhelper/lib/inputstreamhelper/widevine/arm.py b/script.module.inputstreamhelper/lib/inputstreamhelper/widevine/arm.py index 714e10905..3cd7bd0e0 100644 --- a/script.module.inputstreamhelper/lib/inputstreamhelper/widevine/arm.py +++ b/script.module.inputstreamhelper/lib/inputstreamhelper/widevine/arm.py @@ -17,47 +17,44 @@ def select_best_chromeos_image(devices): log(0, 'Find best ARM image to use from the Chrome OS recovery.json') if userspace64(): - arm_hwids = config.CHROMEOS_RECOVERY_ARM64_HWIDS + arm_bnames = config.CHROMEOS_RECOVERY_ARM64_BNAMES else: - arm_hwids = config.CHROMEOS_RECOVERY_ARM_HWIDS + arm_bnames = config.CHROMEOS_RECOVERY_ARM_BNAMES - arm_hwids = [h for arm_hwid in arm_hwids for h in ['^{} '.format(arm_hwid), '^{}-.*'.format(arm_hwid), '^{}.*'.format(arm_hwid)]] best = None for device in devices: # Select ARM hardware only - for arm_hwid in arm_hwids: - if arm_hwid in device['hwidmatch']: - hwid = arm_hwid + for arm_bname in arm_bnames: + if arm_bname == device['file'].split('_')[2]: + device['boardname'] = arm_bname # add this new entry to avoid extracting it from the filename all the time break # We found an ARM device, rejoice ! else: continue # Not ARM, skip this device - device['hwid'] = hwid - # Select the first ARM device if best is None: best = device continue # Go to the next device - # Skip identical hwid - if hwid == best['hwid']: + # Skip identical boardname + if device['boardname'] == best['boardname']: continue # Select the newest version if parse_version(device['version']) > parse_version(best['version']): - log(0, '{device[hwid]} ({device[version]}) is newer than {best[hwid]} ({best[version]})', + log(0, '{device[boardname]} ({device[version]}) is newer than {best[boardname]} ({best[version]})', device=device, best=best) best = device # Select the smallest image (disk space requirement) elif parse_version(device['version']) == parse_version(best['version']): - if int(device['filesize']) + int(device['zipfilesize']) < int(best['filesize']) + int(best['zipfilesize']): - log(0, '{device[hwid]} ({device_size}) is smaller than {best[hwid]} ({best_size})', + if int(device['zipfilesize']) < int(best['zipfilesize']): + log(0, '{device[boardname]} ({device_size}) is smaller than {best[boardname]} ({best_size})', device=device, + device_size=int(device['zipfilesize']), best=best, - device_size=int(device['filesize']) + int(device['zipfilesize']), - best_size=int(best['filesize']) + int(best['zipfilesize'])) + best_size=int(best['zipfilesize'])) best = device return best @@ -96,7 +93,7 @@ def install_widevine_arm(backup_path): localize(30018, diskspace=sizeof_fmt(required_diskspace))) return False - log(2, 'Downloading ChromeOS image for Widevine: {hwid} ({version})'.format(**arm_device)) + log(2, 'Downloading ChromeOS image for Widevine: {boardname} ({version})'.format(**arm_device)) url = arm_device['url'] extracted = dl_extract_widevine(url, backup_path, arm_device) diff --git a/script.module.inputstreamhelper/lib/inputstreamhelper/widevine/arm_chromeos.py b/script.module.inputstreamhelper/lib/inputstreamhelper/widevine/arm_chromeos.py index 120713d39..9a334d102 100644 --- a/script.module.inputstreamhelper/lib/inputstreamhelper/widevine/arm_chromeos.py +++ b/script.module.inputstreamhelper/lib/inputstreamhelper/widevine/arm_chromeos.py @@ -73,12 +73,67 @@ def chromeos_offset(self): return offset - def find_file(self, filename, path_to_file=("opt", "google", "chrome", "WidevineCdm", "_platform_specific", "cros_arm")): + def _find_file_in_chunk(self, bfname, chunk): """ - Finds a file at a given path, or search upwards if not found. + Checks if the filename is found in the given chunk. + Then makes some plausibility checks, to see if it is in fact a proper dentry. + + returns the dentry if found, else False. + """ + + if bfname in chunk: + i_index_pos = chunk.index(bfname) - 8 # the filename is the last element of the dentry, the elements before are 8 bytes total + file_entry = self.dir_entry(chunk[i_index_pos:i_index_pos + len(bfname) + 8]) # 8 because see above + if file_entry['inode'] < self.sb_dict['s_inodes_count'] and file_entry['name_len'] == len(bfname): + return file_entry + + log(0, 'Found filename, but checks did not pass:') + log(0, 'inode number: {inode} < {count}, name_len: {name_len} == {len_fname}'.format(inode=file_entry['inode'], + count=self.sb_dict['s_inodes_count'], + name_len=file_entry['name_len'], + len_fname=len(bfname))) + return False + + def _find_file_naive(self, fname): + """ + Finds a file by basically searching for the filename as bytes in the bytestream. + Searches through the whole image only once, making it fast, but may be unreliable at times. + Returns a directory entry. + """ + + fname_alt = fname + '#new' # Sometimes the filename has "#new" at the end + bfname = fname.encode('ascii') + bfname_alt = fname_alt.encode('ascii') + chunksize = 4 * 1024**2 + chunk1 = self.read_stream(chunksize) + while True: + chunk2 = self.read_stream(chunksize) + if not chunk2: + raise ChromeOSError('File {fname} not found in the ChromeOS image'.format(fname=fname)) + + chunk = chunk1 + chunk2 + + file_entry = self._find_file_in_chunk(bfname, chunk) + if file_entry: + break + + file_entry = self._find_file_in_chunk(bfname_alt, chunk) + if file_entry: + break + + chunk1 = chunk2 + + return file_entry + + def _find_file_properly(self, filename, path_to_file=("opt", "google", "chrome", "WidevineCdm", "_platform_specific", "cros_arm")): + """ + Finds a file at a given path, or searches upwards if not found. Assumes the path is roughly correct, else it might take long. + It also might take long for ZIP files, since it might have to jump back and forth while traversing down the given path. + + Returns a directory entry. """ root_inode_pos = self._calc_inode_pos(2) root_inode_dict = self._inode_table(root_inode_pos) @@ -107,10 +162,9 @@ def find_file(self, filename, path_to_file=("opt", "google", "chrome", "Widevine def _find_file_in_dir(self, filename, dentries): """ Finds a file in a directory or recursively in its subdirectories. - Returns a directory entry. - Can take long for deep searches. - Returns the first result. + + Returns the first result as a directory entry. """ try: return dentries[filename] @@ -128,6 +182,23 @@ def _find_file_in_dir(self, filename, dentries): return None + def find_file(self, filename, path_to_file=None): + """ + Finds a file. Supplying a path could take longer for ZIP files! + + Returns a directory entry. + """ + + if path_to_file: + return self._find_file_properly(filename, path_to_file) + + try: + return self._find_file_naive(filename) + except ChromeOSError: + if self.progress: + self.progress.update(5, localize(30071)) # Could not find file, doing proper search + return self._find_file_properly(filename) + def _calc_inode_pos(self, inode_num): """Calculate the byte position of an inode from its index""" blk_group_num = (inode_num - 1) // self.sb_dict['s_inodes_per_group'] @@ -223,10 +294,10 @@ def dir_entry(chunk): dir_names = ('inode', 'rec_len', 'name_len', 'file_type', 'name') dir_fmt = '