Merge pull request #50 from cloudfactory/import-image-from-bucket

Introduce method to Import image from bucket

.github/workflows/python-package.yml

@@ -15,7 +15,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: [3.7]
+        python-version: [3.9]
 
     steps:
     - uses: actions/checkout@v2

hasty/__init__.py

@@ -21,7 +21,7 @@ def int_or_str(value):
         return value
 
 
-__version__ = '0.3.9'
+__version__ = '0.3.10'
 VERSION = tuple(map(int_or_str, __version__.split('.')))
 
 __all__ = [

hasty/bucket.py

@@ -0,0 +1,106 @@
+from collections import OrderedDict
+from dataclasses import dataclass
+from typing import Union, Protocol
+
+from .constants import BucketProviders
+from .hasty_object import HastyObject
+
+@dataclass
+class Credentials(Protocol):
+    def get_credentials(self):
+        raise NotImplementedError
+
+    def cloud_provider(self):
+        raise NotImplementedError
+
+@dataclass
+class DummyCreds(Credentials):
+    secret: str
+
+    def get_credentials(self):
+        return {"secret": self.secret, "cloud_provider": BucketProviders.DUMMY}
+
+    def cloud_provider(self):
+        return BucketProviders.DUMMY
+
+@dataclass
+class GCSCreds(Credentials):
+    bucket: str
+    key_json: str
+
+    def get_credentials(self):
+        return {"bucket_gcs": self.bucket, "key_json": self.key_json, "cloud_provider": BucketProviders.GCS}
+
+    def cloud_provider(self):
+        return BucketProviders.GCS
+
+@dataclass
+class S3Creds(Credentials):
+    bucket: str
+    role: str
+
+    def get_credentials(self):
+        return {"bucket_s3": self.bucket, "role": self.role, "cloud_provider": BucketProviders.S3}
+
+    def cloud_provider(self):
+        return BucketProviders.S3
+
+@dataclass
+class AZCreds(Credentials):
+    account_name: str
+    secret_access_key: str
+    container: str
+
+    def get_credentials(self):
+        return {"account_name": self.account_name, "secret_access_key": self.secret_access_key,
+                "container": self.container, "cloud_provider": BucketProviders.AZ}
+
+    def cloud_provider(self):
+        return BucketProviders.AZ
+
+class Bucket(HastyObject):
+    """Class that contains some basic requests and features for bucket management"""
+    endpoint = '/v1/buckets/{workspace_id}/credentials'
+
+    def __repr__(self):
+        return self.get__repr__(OrderedDict({"id": self._id, "name": self._name, "cloud_provider": self._cloud_provider}))
+
+    @property
+    def id(self):
+        """
+        :type: string
+        """
+        return self._id
+
+    @property
+    def name(self):
+        """
+        :type: string
+        """
+        return self._name
+
+    @property
+    def cloud_provider(self):
+        """
+        :type: string
+        """
+        return self._cloud_provider
+
+    def _init_properties(self):
+        self._id = None
+        self._name = None
+        self._cloud_provider = None
+
+    def _set_prop_values(self, data):
+        if "credential_id" in data:
+            self._id = data["credential_id"]
+        if "description" in data:
+            self._name = data["description"]
+        if "cloud_provider" in data:
+            self._cloud_provider = data["cloud_provider"]
+
+    @staticmethod
+    def _create_bucket(requester, workspace_id, name, credentials: Union[DummyCreds, GCSCreds, S3Creds, AZCreds]):
+        json = {"description": name, "cloud_provider": credentials.cloud_provider(), **credentials.get_credentials()}
+        data = requester.post(Bucket.endpoint.format(workspace_id=workspace_id), json_data=json)
+        return Bucket(requester, data)

hasty/constants.py

@@ -41,6 +41,13 @@ class SemanticOrder:
     CLASS_ORDER = "class_order"
 
 
+class BucketProviders:
+    GCS = "gcs"
+    S3 = "s3"
+    AZ = "az"
+    DUMMY = "dummy"
+
+
 WAIT_INTERVAL_SEC = 10
 
 VALID_STATUSES = [ImageStatus.New, ImageStatus.Done, ImageStatus.Skipped, ImageStatus.InProgress, ImageStatus.ToReview,

hasty/image.py

@@ -156,6 +156,18 @@ def _upload_from_url(requester, project_id, dataset_id, filename, url, copy_orig
         return Image(requester, res, {"project_id": project_id,
                                       "dataset_id": dataset_id})
 
+    @staticmethod
+    def _upload_from_bucket(requester, project_id, dataset_id, filename, path, bucket_id, copy_original=False,
+                            external_id: Optional[str] = None):
+        res = requester.post(Image.endpoint.format(project_id=project_id),
+                             json_data={"dataset_id": dataset_id,
+                                        "url": path,
+                                        "filename": filename,
+                                        "bucket_id": bucket_id,
+                                        "copy_original": copy_original,
+                                        "external_id": external_id})
+        return Image(requester, res, {"project_id": project_id,
+                                      "dataset_id": dataset_id})
     def get_labels(self):
         """
         Returns image labels (list of `~hasty.Label` objects)

hasty/project.py

@@ -215,6 +215,25 @@ def upload_from_url(self, dataset: Union[Dataset, str], filename: str, url: str,
         return Image._upload_from_url(self._requester, self._id, dataset_id, filename, url, copy_original=copy_original,
                                       external_id=external_id)
 
+    def upload_from_bucket(self, dataset: Union[Dataset, str], filename: str, path: str, bucket_id: str, copy_original: Optional[bool] = False,
+                            external_id: Optional[str] = None):
+        """
+        Uploads image from the given bucket
+
+        Args:
+            dataset (`~hasty.Dataset`, str): Dataset object or id that the image should belongs to
+            filename (str): Filename of the image
+            path (str): Path in the bucket
+            bucket_id (str): Bucket ID (format: UUID)
+            copy_original (str): If True Hasty makes a copy of the image. Default False.
+            external_id (str): External ID (optional)
+        """
+        dataset_id = dataset
+        if isinstance(dataset, Dataset):
+            dataset_id = dataset.id
+        return Image._upload_from_bucket(self._requester, self._id, dataset_id, filename, path, bucket_id, copy_original=copy_original,
+                                      external_id=external_id)
+
     def get_label_classes(self):
         """
         Get label classes, list of :py:class:`~hasty.LabelClass` objects.

hasty/workspace.py

@@ -1,6 +1,9 @@
+from typing import Union
 from collections import OrderedDict
 
+
 from .hasty_object import HastyObject
+from .bucket import DummyCreds, GCSCreds, S3Creds, AZCreds, Bucket
 
 
 class Workspace(HastyObject):
@@ -34,3 +37,13 @@ def _set_prop_values(self, data):
             self._id = data["id"]
         if "name" in data:
             self._name = data["name"]
+
+    def create_bucket(self, name: str, credentials: Union[DummyCreds, GCSCreds, S3Creds, AZCreds]):
+        """
+        Create a new bucket in the workspace.
+
+        Args:
+            name (str): Name of the bucket.
+            credentials (Credentials): Credentials object.
+        """
+        return Bucket._create_bucket(self._requester, self._id, name, credentials)

setup.py

@@ -30,7 +30,7 @@
         "numpy>=1.16",
     ],
     install_requires=["numpy>=1.16", 'requests >= 2.23.0', 'retrying==1.3.3'],
-    python_requires=">=3.6",
+    python_requires=">=3.9",
     classifiers=[
         "Development Status :: 4 - Beta",
         "License :: OSI Approved :: MIT License",

tests/test_bucket.py

@@ -0,0 +1,40 @@
+import unittest
+
+from tests.utils import get_client
+
+from hasty.bucket import S3Creds
+
+
+class TestBucketManagement(unittest.TestCase):
+    def setUp(self):
+        self.h = get_client()
+        self.workspace = self.h.get_workspaces()[0]
+        self.project = self.h.create_project(self.workspace, "Test Project 1")
+
+    def tearDown(self):
+        self.project.delete()
+
+    def test_bucket_creation(self):
+        ws = self.h.get_workspaces()[0]
+        res = ws.create_bucket("test_bucket", S3Creds(bucket="hasty-public-bucket-mounter", role="arn:aws:iam::045521589961:role/hasty-public-bucket-mounter"))
+        self.assertIsNotNone(res.id)
+        self.assertEqual("test_bucket", res.name)
+        self.assertEqual("s3", res.cloud_provider)
+
+    def test_import_image(self):
+        # create a bucket
+        bucket = self.workspace.create_bucket("test_bucket", S3Creds(bucket="hasty-public-bucket-mounter", role="arn:aws:iam::045521589961:role/hasty-public-bucket-mounter"))
+
+        # Import an image from the bucket
+        dataset = self.project.create_dataset("ds2")
+        img = self.project.upload_from_bucket(dataset, "1645001880-075718046bb2fbf9b8c35d6e88571cd7f91ca1a1.png",
+                                              "dummy/1645001880-075718046bb2fbf9b8c35d6e88571cd7f91ca1a1.png", bucket.id)
+        self.assertEqual("1645001880-075718046bb2fbf9b8c35d6e88571cd7f91ca1a1.png", img.name)
+        self.assertEqual("ds2", img.dataset_name)
+        self.assertIsNotNone(img.id)
+        self.assertEqual(1280, img.width)
+        self.assertEqual(720, img.height)
+
+
+if __name__ == '__main__':
+    unittest.main()