From 8b7c9733afdbdb9017d4fd3beb17e1888b0b5fa0 Mon Sep 17 00:00:00 2001 From: Ognyan Moore Date: Thu, 15 Feb 2024 11:40:43 -0500 Subject: [PATCH] Addresses a few issues brought by msmith * doppkit-cli not works again * default logging level of doppkit is set to logging.info * clearer logging messages * slightly more robust error handling when GRiD returns an error * better download progress calculation --- src/doppkit/__init__.py | 2 +- src/doppkit/cache.py | 11 +++++++---- src/doppkit/cli/list.py | 10 +++------- src/doppkit/cli/sync.py | 29 ++++++++-------------------- src/doppkit/grid.py | 28 ++++++++++++++++++++++++--- src/doppkit/gui/__main__.py | 3 ++- src/doppkit/gui/window.py | 38 ++++++++++++++++++------------------- 7 files changed, 64 insertions(+), 57 deletions(-) diff --git a/src/doppkit/__init__.py b/src/doppkit/__init__.py index 19eac42..3eef31c 100644 --- a/src/doppkit/__init__.py +++ b/src/doppkit/__init__.py @@ -1,4 +1,4 @@ -__version__ = "0.3.2" +__version__ = "0.3.3rc0" from . import cache from . import grid diff --git a/src/doppkit/cache.py b/src/doppkit/cache.py index 001604f..e259112 100644 --- a/src/doppkit/cache.py +++ b/src/doppkit/cache.py @@ -20,7 +20,8 @@ class DownloadUrl(NamedTuple): url: str - storage_path: str = "." + name: str = "" + save_path: str = "." total: int = 1 @@ -92,7 +93,7 @@ async def cache( app: 'Application', urls: Iterable[DownloadUrl], headers: dict[str, str], - progress: Optional[Progress]=None + progress: Optional[Progress] = None ) -> Iterable[Union[Content, Exception, httpx.Response]]: limits = httpx.Limits( max_keepalive_connections=app.threads, max_connections=app.threads @@ -118,7 +119,7 @@ async def cache( ], return_exceptions=True ) - logger.debug("Cache operation complete.") + logger.info(f"Cache operation complete for {len(files)} files.") return files @@ -131,6 +132,8 @@ async def cache_url( ) -> Union[Content, httpx.Response]: limit = args.limit async with limit: + if url.name: + logger.info(f"Getting {url.name}...") request = client.build_request("GET", url.url, headers=headers, timeout=None) response = await client.send(request, stream=True) if response.is_error: @@ -150,7 +153,7 @@ async def cache_url( response = await client.send(request, stream=True) total = max(total, int(response.headers.get("Content-length", 0))) if filename is not None: # we are not saving to BytesIO - filename = pathlib.Path(url.storage_path.lstrip("/")) + filename = pathlib.Path(url.save_path.lstrip("/")) c = Content( response.headers, filename=filename, diff --git a/src/doppkit/cli/list.py b/src/doppkit/cli/list.py index dfb9aac..7f6b238 100644 --- a/src/doppkit/cli/list.py +++ b/src/doppkit/cli/list.py @@ -36,20 +36,16 @@ def listExports(args, id_): table = Table(title=f"Exports for {aoi['name']} – {id_}") table.add_column("Export ID") - table.add_column("Item ID") table.add_column("Name") - table.add_column("Type") table.add_column("Size") if aoi.get('exports'): for export in aoi['exports']: export_id = export['id'] - exports = asyncio.run(api.get_exports(export_id)) + exports = asyncio.run(api.get_exports(export_id)) for e in exports: table.add_row( str(export_id), - str(e['id']), - e['name'], - e['datatype'], - str(e['filesize']) + e.name, + str(e.total) ) console.print(table) diff --git a/src/doppkit/cli/sync.py b/src/doppkit/cli/sync.py index 7730a5d..bf5db8c 100644 --- a/src/doppkit/cli/sync.py +++ b/src/doppkit/cli/sync.py @@ -28,41 +28,28 @@ async def sync(args: 'Application', id_: str) -> Iterable[Content]: if args.filter: logger.debug(f'Filtering AOIs with "{args.filter}"') aois = [aoi for aoi in aois if args.filter in aoi["notes"]] - exportfiles = [] + files_to_download = [] for aoi in aois: for export in aoi["exports"]: logger.debug(f"export: {export}") files = await api.get_exports(export["id"]) - exportfiles.extend(files) + files_to_download.extend(files) - total_downloads = len(exportfiles) - count = 0 + total_downloads = len(files_to_download) urls = [] logger.debug(f"{total_downloads} files found, downloading to dir: {download_dir}") - for exportfile in sorted(exportfiles, key=lambda x: int(x.get("id"))): - id_ = exportfile.get("id") - if id_ < int(args.start_id): - logger.info(f"Skipping file {id_}") - total_downloads -= 1 - logger.info(f"{count} of {total_downloads} downloads complete") - continue - filename = exportfile["name"] - download_url = exportfile["url"] - download_destination = download_dir.joinpath(filename) + for file_ in files_to_download: + download_url = file_.url + download_destination = download_dir.joinpath(file_.save_path) logger.debug( - f"Exportfile ID {id_} downloading from {download_url} to {download_destination}" + f"File {file_.name} downloading from {download_url} to {download_destination}" ) - # Skip this file if we've already downloaded it if not args.override and download_destination.exists(): logger.debug(f"File already exists, skipping: {download_destination}") else: urls.append( - DownloadUrl( - download_url, - f"{exportfile.get('storage_path', '.')}/{exportfile['name']}", - exportfile["filesize"] - ) + file_ ) headers = {"Authorization": f"Bearer {args.token}"} logger.debug(urls, headers) diff --git a/src/doppkit/grid.py b/src/doppkit/grid.py index ddbf217..c2599e8 100644 --- a/src/doppkit/grid.py +++ b/src/doppkit/grid.py @@ -43,6 +43,11 @@ class Auxfile(TypedDict): url: str strage_path: str +class Licensefile(TypedDict): + filesize: int + url: str + name: str + storage_path: str class VectorProduct(TypedDict): id: int @@ -74,6 +79,7 @@ class Export(TypedDict): datatype: str export_type: str exportfiles: Union[list[Exportfile], bool] + licensefiles: list[Licensefile] export_total_size: int auxfile_total_size: int complete_size: int @@ -112,6 +118,8 @@ class AOI(TypedDict): vector_intersects: list[VectorProduct] + + class Grid: def __init__(self, args): self.args = args @@ -233,7 +241,7 @@ async def check_task(self, task_id: Optional[str] = None) -> list[Task]: raise RuntimeError(f"GRiD Task Endpoint Returned Error {r.status_code}") return output - async def get_exports(self, export_id: int) -> list[Union[Exportfile, Auxfile]]: + async def get_exports(self, export_id: int) -> list[DownloadUrl]: """ Parameters ---------- @@ -305,9 +313,23 @@ async def get_exports(self, export_id: int) -> list[Union[Exportfile, Auxfile]]: f"./{exportfile['datatype']}" f"{exportfile['storage_name'].rpartition(str(export_id))[-1].rpartition('/')[0]}" ) - files.append(exportfile) + files.append( + DownloadUrl( + url=exportfile["url"], + save_path=f"{item['name']}/{exportfile['storage_path'].strip('/')}/{exportfile['name']}", + total=exportfile["filesize"], + name=exportfile["name"] + ) + ) for supplemental_file in itertools.chain(item["auxfiles"], item.get('licensefiles', [])): if supplemental_file["url"] not in supplemental_urls: - files.append(supplemental_file) + files.append( + DownloadUrl( + url=supplemental_file["url"], + save_path=f"{item['name']}/{supplemental_file['storage_path'].strip('/')}/{supplemental_file['name']}", + total=supplemental_file["filesize"], + name=supplemental_file["name"] + ) + ) supplemental_urls.add(supplemental_file["url"]) return files diff --git a/src/doppkit/gui/__main__.py b/src/doppkit/gui/__main__.py index cce41fc..a3c4d2a 100644 --- a/src/doppkit/gui/__main__.py +++ b/src/doppkit/gui/__main__.py @@ -63,11 +63,12 @@ def close(*args): await future return True + def main(): app = Application( token=None, url="https://grid.nga.mil/grid", - log_level=logging.DEBUG, + # log_level=logging.DEBUG, # override for custom messaging threads=5, run_method="GUI", progress=True, diff --git a/src/doppkit/gui/window.py b/src/doppkit/gui/window.py index ace65c9..24fb1ce 100644 --- a/src/doppkit/gui/window.py +++ b/src/doppkit/gui/window.py @@ -75,6 +75,7 @@ def create_task(self, name: str, url: str, total: int): def update(self, name: str, url: str, completed: int) -> None: export_id = self.urls_to_export_id[url] + old_progress = self.export_files[export_id][url] self.export_files[export_id][url] = ExportFileProgress( url, @@ -85,7 +86,6 @@ def update(self, name: str, url: str, completed: int) -> None: ) export_progress = self.export_progress[export_id] - export_downloaded = sum(file_progress.current for file_progress in self.export_files[export_id].values()) export_progress.update(export_downloaded) self.taskUpdated.emit(export_progress) @@ -421,31 +421,23 @@ async def downloadExports(self): urls = [] for aoi in self.AOIs: for export in aoi["exports"]: - # need to check for export_files, if not present, we need to populate - if isinstance(export["exportfiles"], bool): - # we need to grab the list of exportfiles... - export["exportfiles"] = await api.get_exports(export["id"]) - elif 'auxfiles' in export.keys(): - export["exportfiles"].extend(export["auxfiles"]) + files = await api.get_exports(export["id"]) download_size = 0 - for export_file in export["exportfiles"]: - filename = export_file["name"] - download_destination = download_dir.joinpath(filename) + for download_file in files: + filename = download_file.name + download_destination = download_dir.joinpath(download_file.save_path) # TODO: compare filesizes, not just if it exists if not self.doppkit.override and download_destination.exists(): logger.debug(f"File already exists, skipping {filename}") else: urls.append( - DownloadUrl( - export_file["url"], - f"{export_file['storage_path'].strip('/')}/{export_file['name']}", - export_file["filesize"] - ) + download_file ) - download_size += export_file["filesize"] - self.progressInterconnect.urls_to_export_id[export_file["url"]] = export["id"] + print(f"{download_file.save_path} = {download_file.total} bytes") + download_size += download_file.total + self.progressInterconnect.urls_to_export_id[download_file.url] = export["id"] progress_tracker = ProgressTracking( export["id"], export_name=export["name"], @@ -478,9 +470,15 @@ def ratio(self) -> float: ratio = self.current / self.total except ZeroDivisionError: ratio = 0.0 - if ratio >= 1.0: - logger.warning("Completed download ratio calculated to be > 1.0, likely incorrect...") - return min([ratio, 1.0]) + else: + if math.floor(100 * ratio) > 100: + logger.warning( + f"Completed download ratio for {self.export_name} calculated " + f"to be {self.current=}/{self.total=}={ratio} (> 1.0), " + "likely incorrect..." + ) + return 1.0 + return ratio def percentage(self) -> int: return math.floor(100 * self.ratio())