Add upload functionality (#71)

Add GRiD upload capability to doppkit API and basic capability to the GUI

src/doppkit/__init__.py

@@ -1,4 +1,4 @@
-__version__ = "0.3.4"
+__version__ = "0.4.0rc0"
 
 from . import cache
 from . import grid

src/doppkit/cache.py

@@ -28,13 +28,13 @@ class DownloadUrl(NamedTuple):
 
 class Progress(Protocol):
 
-    def update(self, name: str, url: str, completed: int) -> None:
+    def update(self, name: str, source: str, completed: int) -> None:
         ...
 
-    def create_task(self, name: str, url: str, total: int) -> None:
+    def create_task(self, name: str, source: str, total: int) -> None:
         ...
 
-    def complete_task(self, name: str, url: str) -> None:
+    def complete_task(self, name: str, source: str) -> None:
         ...
 
 

src/doppkit/cli/cache.py

@@ -21,14 +21,14 @@ def __init__(self, context_manager: Progress):
         self.context_manager = context_manager
         self.tasks: dict[str, TaskID] = {}
 
-    def create_task(self, name: str, url: str, total: int):
+    def create_task(self, name: str, source: str, total: int):
         self.tasks[name] = self.context_manager.add_task(name, total=total)
 
-    def update(self, name: str, url: str, completed: int):
+    def update(self, name: str, source: str, completed: int):
         task = self.tasks[name]
         self.context_manager.update(task, completed=completed)
 
-    def complete_task(self, name: str, url: str):
+    def complete_task(self, name: str, source: str):
         task = self.tasks.pop(name)
         self.context_manager.update(task, visible=False)
 

src/doppkit/grid.py

@@ -4,19 +4,27 @@
 import json
 import warnings
 import logging
+import pathlib
+import math
 
 import httpx
-from typing import Optional, Iterable, TypedDict, Union
+from typing import Optional, Iterable, TypedDict, Union, TYPE_CHECKING
 
-from .cache import cache, DownloadUrl
+from .cache import cache, DownloadUrl, Progress
+from .upload import upload
+
+if TYPE_CHECKING:
+    from .upload import ETagDict
 
 logger = logging.getLogger(__name__)
 
 API_VERSION = "v4"
+MULTIPART_BYTES_PER_CHUNK = 10_000_000  # ~ 6mb
 
 aoi_endpoint_ext = f"/api/{API_VERSION}/aois"
 export_endpoint_ext = f"/api/{API_VERSION}/exports"
 task_endpoint_ext = f"/api/{API_VERSION}/tasks"
+upload_endpoint_ext = f"/api/{API_VERSION}/upload"
 
 
 class ExportStarted(TypedDict):
@@ -167,6 +175,79 @@ async def get_aois(self, id: Optional[int]=None) -> list[AOI]:
         return response["aois"]
 
 
+    async def upload_asset(
+            self,
+            filepath: pathlib.Path,
+            bytes_per_chunk=MULTIPART_BYTES_PER_CHUNK,
+            progress: Optional[Progress]=None
+    ):
+        logger.info(f"Starting upload of {filepath}")
+        source_size = filepath.stat().st_size
+        chunks_count = int(math.ceil(source_size / float(bytes_per_chunk)))
+
+        key = f"test-ogi/upload/{filepath.name}"
+        upload_endpoint_url = f"{self.args.url}{upload_endpoint_ext}"
+
+        headers = {"Authorization": f"Bearer {self.args.token}"}
+        async with httpx.AsyncClient(verify=not self.args.disable_ssl_verification) as client:
+            params = {"key": key}
+            response_upload_id = await client.get(
+                f"{upload_endpoint_url}/open/",
+                params=params,
+                headers=headers
+            )
+            logger.debug(f"Upload open call returned {response_upload_id}")
+
+            try:
+                upload_id = response_upload_id.json()["upload_id"]
+            except KeyError as e:
+                if "error" in response_upload_id.json():
+                    raise ConnectionError(response_upload_id.json()["error"]) from e
+                else:
+                    raise ConnectionError(response_upload_id.json()) from e
+
+            params.update(upload_id=upload_id, nparts=str(chunks_count))
+            response_urls = await client.get(
+                f"{upload_endpoint_url}/get_urls/",
+                params=params,
+                headers=headers
+            )
+
+            # want to make sure URLs are in order of part
+            urls = [
+                part['url'] 
+                for part in sorted(
+                    response_urls.json()["parts"],
+                    key=lambda part: part['part']
+                )
+            ]
+
+            part_info = await upload(
+                app=self.args,
+                filepath=filepath,
+                urls=urls,
+                bytes_per_chunk=bytes_per_chunk,
+                auth_header=headers,
+                progress=progress
+            )
+
+            data = {
+                "upload_id": upload_id,
+                "key": key,
+                "upload_info": part_info
+            }
+
+            response_finish = await client.put(
+                f"{upload_endpoint_url}/close/",
+                json=data,
+                headers=headers
+            )
+            logger.debug(f"Upload close call returned {response_finish}")
+            logger.info(f"Finished upload of {filepath}")
+
+        return None
+
+
     async def make_exports(
             self,
             aoi: AOI,
@@ -200,7 +281,7 @@ async def make_exports(
                 product_ids.extend([entry["id"] for entry in aoi["vector_intersects"]])
             else:
                 warnings.warn(
-                    f"Unknown intersect type {intersection}, needs to be one of "
+                    f"Unknown intersect type {intersection}, needs to be one of " +
                     "raster, mesh, pointcloud, or vector.  Ignoring.",
                     stacklevel=2
                 )
@@ -286,7 +367,7 @@ async def get_exports(self, export_id: int) -> list[DownloadUrl]:
         else:
             if "error" in response:
                 logger.warning(
-                    f"Attempting to access {export_id=} resulted in the following error "
+                    f"Attempting to access {export_id=} resulted in the following error " +
                     f"from GRiD: {response['error']}"
                 )
                 return []

src/doppkit/gui/MenuBar.py

@@ -1,9 +1,10 @@
+import os
 import typing
 
 
-from qtpy.QtCore import Slot
+from qtpy.QtCore import Slot, QDir
 from qtpy.QtGui import QKeySequence
-from qtpy.QtWidgets import QAction, QApplication, QMenu, QMenuBar, QMainWindow
+from qtpy.QtWidgets import QAction, QApplication, QFileDialog, QMenu, QMenuBar, QMainWindow
 
 from .SettingsDialog import SettingsDialog
 
@@ -39,6 +40,8 @@ def __init__(self, title: typing.Optional[str] = None, parent: MenuBar = None) -
         self.setTitle("File")
         self.settingsDialog: typing.Optional['SettingsDialog'] = None
         self._settingsAction()
+        self._uploadFileAction()
+        self._uploadDirectoryAction()
         self._quitAction()
 
     def _settingsAction(self) -> None:
@@ -54,6 +57,61 @@ def invokeSettings(self):
             self.settingsDialog = SettingsDialog(self)
             self.settingsDialog.rejected.connect(self._resetSettingsDialog)
             self.settingsDialog.show()
+
+    @Slot()
+    def uploadFileDialog(self):
+        filename, filter_ = QFileDialog.getOpenFileName(
+            self,
+            "Upload File",
+            QDir.home().absolutePath(),
+            options=QFileDialog.Option.ReadOnly
+        )
+
+        if filename is None:
+            # user cancelled, abort
+            return None
+
+        for widget in QApplication.instance().topLevelWidgets():
+            if isinstance(widget, QMainWindow):
+                break
+        else:
+            raise RuntimeError("Main Window not found, how did this happen?")
+
+        if hasattr(widget, 'uploadFiles'):
+            widget.uploadFiles([filename])
+
+
+    @Slot()
+    def uploadDirectoryDialog(self):
+        directory = QFileDialog.getExistingDirectory(
+            self,
+            "Upload Directory",
+            QDir.home().absolutePath(),
+            options=QFileDialog.Option.ReadOnly
+        )
+
+        if directory is None:
+            # user cancelled, abort
+            return None
+
+        files_to_upload = []
+        for root, dirs, files in os.walk(directory):
+            files_to_upload.extend(
+                [
+                    os.path.join(root, file_)
+                    for file_ in files
+                    if not file_.startswith(".")
+                ]
+            )
+
+        for widget in QApplication.instance().topLevelWidgets():
+            if isinstance(widget, QMainWindow):
+                break
+        else:
+            raise RuntimeError("Main Window not found, how did this happen?")
+
+        if hasattr(widget, 'uploadFiles'):
+            widget.uploadFiles(files_to_upload)
 
     @Slot()
     def _resetSettingsDialog(self) -> None:
@@ -68,6 +126,17 @@ def _quitAction(self) -> None:
         self.addAction(quitAction)
 
 
+    def _uploadFileAction(self):
+        uploadFileAction = QAction("Upload File", self)
+        uploadFileAction.setStatusTip("Upload File")
+        uploadFileAction.triggered.connect(self.uploadFileDialog)
+        self.addAction(uploadFileAction)
+
+    def _uploadDirectoryAction(self):
+        uploadDirectoryAction = QAction("Upload Directory Contents", self)
+        uploadDirectoryAction.setStatusTip("Upload Directory Contents")
+        uploadDirectoryAction.triggered.connect(self.uploadDirectoryDialog)
+        self.addAction(uploadDirectoryAction)
 
 
 class ViewMenu(QMenu):