Merge pull request #64 from j9ac9k/address-mike-suggestions

Minor tweaks and fixes to doppkit-cli

src/doppkit/__init__.py

@@ -1,4 +1,4 @@
-__version__ = "0.3.2"
+__version__ = "0.3.3rc0"
 
 from . import cache
 from . import grid

src/doppkit/cache.py

@@ -20,7 +20,8 @@
 
 class DownloadUrl(NamedTuple):
     url: str
-    storage_path: str = "."
+    name: str = ""
+    save_path: str = "."
     total: int = 1
 
 
@@ -92,7 +93,7 @@ async def cache(
         app: 'Application',
         urls: Iterable[DownloadUrl],
         headers: dict[str, str],
-        progress: Optional[Progress]=None
+        progress: Optional[Progress] = None
 ) -> Iterable[Union[Content, Exception, httpx.Response]]:
     limits = httpx.Limits(
         max_keepalive_connections=app.threads, max_connections=app.threads
@@ -118,7 +119,7 @@ async def cache(
             ],
             return_exceptions=True
         )
-    logger.debug("Cache operation complete.")
+    logger.info(f"Cache operation complete for {len(files)} files.")
     return files
 
 
@@ -131,6 +132,8 @@ async def cache_url(
 ) -> Union[Content, httpx.Response]:
     limit = args.limit
     async with limit:
+        if url.name:
+            logger.info(f"Getting {url.name}...")
         request = client.build_request("GET", url.url, headers=headers, timeout=None)
         response = await client.send(request, stream=True)
         if response.is_error:
@@ -150,7 +153,7 @@ async def cache_url(
             response = await client.send(request, stream=True)
             total = max(total, int(response.headers.get("Content-length", 0)))
         if filename is not None:  # we are not saving to BytesIO
-            filename = pathlib.Path(url.storage_path.lstrip("/"))
+            filename = pathlib.Path(url.save_path.lstrip("/"))
         c = Content(
             response.headers,
             filename=filename,

src/doppkit/cli/list.py

@@ -36,20 +36,16 @@ def listExports(args, id_):
     table = Table(title=f"Exports for {aoi['name']} – {id_}")
 
     table.add_column("Export ID")
-    table.add_column("Item ID")
     table.add_column("Name")
-    table.add_column("Type")
     table.add_column("Size")
     if aoi.get('exports'):
         for export in aoi['exports']:
             export_id = export['id']
-            exports =  asyncio.run(api.get_exports(export_id))
+            exports = asyncio.run(api.get_exports(export_id))
             for e in exports:
                 table.add_row(
                     str(export_id),
-                    str(e['id']),
-                    e['name'],
-                    e['datatype'],
-                    str(e['filesize'])
+                    e.name,
+                    str(e.total)
                 )
     console.print(table)

src/doppkit/cli/sync.py

@@ -28,41 +28,28 @@ async def sync(args: 'Application', id_: str) -> Iterable[Content]:
     if args.filter:
         logger.debug(f'Filtering AOIs with "{args.filter}"')
         aois = [aoi for aoi in aois if args.filter in aoi["notes"]]
-    exportfiles = []
+    files_to_download = []
     for aoi in aois:
         for export in aoi["exports"]:
             logger.debug(f"export: {export}")
             files = await api.get_exports(export["id"])
-            exportfiles.extend(files)
+            files_to_download.extend(files)
 
-    total_downloads = len(exportfiles)
-    count = 0
+    total_downloads = len(files_to_download)
     urls = []
     logger.debug(f"{total_downloads} files found, downloading to dir: {download_dir}")
-    for exportfile in sorted(exportfiles, key=lambda x: int(x.get("id"))):
-        id_ = exportfile.get("id")
-        if id_ < int(args.start_id):
-            logger.info(f"Skipping file {id_}")
-            total_downloads -= 1
-            logger.info(f"{count} of {total_downloads} downloads complete")
-            continue
-        filename = exportfile["name"]
-        download_url = exportfile["url"]
-        download_destination = download_dir.joinpath(filename)
+    for file_ in files_to_download:
+        download_url = file_.url
+        download_destination = download_dir.joinpath(file_.save_path)
         logger.debug(
-            f"Exportfile ID {id_} downloading from {download_url} to {download_destination}"
+            f"File {file_.name} downloading from {download_url} to {download_destination}"
         )
-
         # Skip this file if we've already downloaded it
         if not args.override and download_destination.exists():
             logger.debug(f"File already exists, skipping: {download_destination}")
         else:
             urls.append(
-                DownloadUrl(
-                    download_url,
-                    f"{exportfile.get('storage_path', '.')}/{exportfile['name']}",
-                    exportfile["filesize"]
-                )
+                file_
             )
     headers = {"Authorization": f"Bearer {args.token}"}
     logger.debug(urls, headers)

src/doppkit/grid.py

@@ -43,6 +43,11 @@ class Auxfile(TypedDict):
     url: str
     strage_path: str
 
+class Licensefile(TypedDict):
+    filesize: int
+    url: str
+    name: str
+    storage_path: str
 
 class VectorProduct(TypedDict):
     id: int
@@ -74,6 +79,7 @@ class Export(TypedDict):
     datatype: str
     export_type: str
     exportfiles: Union[list[Exportfile], bool]
+    licensefiles: list[Licensefile]
     export_total_size: int
     auxfile_total_size: int
     complete_size: int
@@ -112,6 +118,8 @@ class AOI(TypedDict):
     vector_intersects: list[VectorProduct]
 
 
+
+
 class Grid:
     def __init__(self, args):
         self.args = args
@@ -233,7 +241,7 @@ async def check_task(self, task_id: Optional[str] = None) -> list[Task]:
             raise RuntimeError(f"GRiD Task Endpoint Returned Error {r.status_code}")
         return output
 
-    async def get_exports(self, export_id: int) -> list[Union[Exportfile, Auxfile]]:
+    async def get_exports(self, export_id: int) -> list[DownloadUrl]:
         """
         Parameters
         ----------
@@ -305,9 +313,23 @@ async def get_exports(self, export_id: int) -> list[Union[Exportfile, Auxfile]]:
                             f"./{exportfile['datatype']}"
                             f"{exportfile['storage_name'].rpartition(str(export_id))[-1].rpartition('/')[0]}"
                         )
-                    files.append(exportfile)
+                    files.append(
+                        DownloadUrl(
+                            url=exportfile["url"],
+                            save_path=f"{item['name']}/{exportfile['storage_path'].strip('/')}/{exportfile['name']}",
+                            total=exportfile["filesize"],
+                            name=exportfile["name"]
+                        )
+                    )
                 for supplemental_file in itertools.chain(item["auxfiles"], item.get('licensefiles', [])):
                     if supplemental_file["url"] not in supplemental_urls:
-                        files.append(supplemental_file)
+                        files.append(
+                            DownloadUrl(
+                                url=supplemental_file["url"],
+                                save_path=f"{item['name']}/{supplemental_file['storage_path'].strip('/')}/{supplemental_file['name']}",
+                                total=supplemental_file["filesize"],
+                                name=supplemental_file["name"]
+                            )
+                        )
                         supplemental_urls.add(supplemental_file["url"])
         return files

src/doppkit/gui/__main__.py

@@ -63,11 +63,12 @@ def close(*args):
         await future
     return True
 
+
 def main():
     app = Application(
         token=None,
         url="https://grid.nga.mil/grid",
-        log_level=logging.DEBUG,
+        # log_level=logging.DEBUG,  # override for custom messaging
         threads=5,
         run_method="GUI",
         progress=True,

src/doppkit/gui/window.py

@@ -75,6 +75,7 @@ def create_task(self, name: str, url: str, total: int):
 
     def update(self, name: str, url: str, completed: int) -> None:
         export_id = self.urls_to_export_id[url]
+
         old_progress = self.export_files[export_id][url]
         self.export_files[export_id][url] = ExportFileProgress(
             url,
@@ -85,7 +86,6 @@ def update(self, name: str, url: str, completed: int) -> None:
         )
 
         export_progress = self.export_progress[export_id]
-
         export_downloaded = sum(file_progress.current for file_progress in self.export_files[export_id].values())
         export_progress.update(export_downloaded)
         self.taskUpdated.emit(export_progress)
@@ -421,31 +421,23 @@ async def downloadExports(self):
         urls = []
         for aoi in self.AOIs:
             for export in aoi["exports"]:
-                # need to check for export_files, if not present, we need to populate
-                if isinstance(export["exportfiles"], bool):
-                    # we need to grab the list of exportfiles...
-                    export["exportfiles"] = await api.get_exports(export["id"])
-                elif 'auxfiles' in export.keys():
-                    export["exportfiles"].extend(export["auxfiles"])
+                files = await api.get_exports(export["id"])
 
                 download_size = 0
-                for export_file in export["exportfiles"]:
-                    filename = export_file["name"]
-                    download_destination = download_dir.joinpath(filename)
+                for download_file in files:
+                    filename = download_file.name
+                    download_destination = download_dir.joinpath(download_file.save_path)
 
                     # TODO: compare filesizes, not just if it exists
                     if not self.doppkit.override and download_destination.exists():
                         logger.debug(f"File already exists, skipping {filename}")
                     else:
                         urls.append(
-                            DownloadUrl(
-                                export_file["url"],
-                                f"{export_file['storage_path'].strip('/')}/{export_file['name']}",
-                                export_file["filesize"]
-                            )
+                            download_file
                         )
-                        download_size += export_file["filesize"]
-                        self.progressInterconnect.urls_to_export_id[export_file["url"]] = export["id"]
+                        print(f"{download_file.save_path} = {download_file.total} bytes")
+                        download_size += download_file.total
+                        self.progressInterconnect.urls_to_export_id[download_file.url] = export["id"]
                 progress_tracker = ProgressTracking(
                     export["id"],
                     export_name=export["name"],
@@ -478,9 +470,15 @@ def ratio(self) -> float:
             ratio = self.current / self.total
         except ZeroDivisionError:
             ratio = 0.0
-        if ratio >= 1.0:
-            logger.warning("Completed download ratio calculated to be > 1.0, likely incorrect...")
-        return min([ratio, 1.0])
+        else:
+            if math.floor(100 * ratio) > 100:
+                logger.warning(
+                    f"Completed download ratio for {self.export_name} calculated "
+                    f"to be {self.current=}/{self.total=}={ratio} (> 1.0), "
+                    "likely incorrect..."
+                )
+                return 1.0
+        return ratio
 
     def percentage(self) -> int:
         return math.floor(100 * self.ratio())