NXDRIVE-1999: Direct Transfer let user select document type (#3695)

* NXDRIVE-1999: Direct Transfer let user select document type

docs/changes/5.3.0.md

@@ -1,18 +1,21 @@
 # 5.3.0
 
-Release date: `2022-xx-xx`
+Release date: `2022-12-xx`
 
 ## Core
 
-- [NXDRIVE-2779](https://jira.nuxeo.com/browse/NXDRIVE-2779):
+- [NXDRIVE-2779](https://jira.nuxeo.com/browse/NXDRIVE-2779): Investigate the login issue with SSL and without legacy authentication check i.e. oauth2
 
 ### Direct Edit
 
 - [NXDRIVE-2](https://jira.nuxeo.com/browse/NXDRIVE-2):
 
 ### Direct Transfer
 
-- [NXDRIVE-2](https://jira.nuxeo.com/browse/NXDRIVE-2):
+- [NXDRIVE-2667](https://jira.nuxeo.com/browse/NXDRIVE-2667): [Direct Transfer] Allow to select folderish type when using the new remote folder capability
+- [NXDRIVE-2669](https://jira.nuxeo.com/browse/NXDRIVE-2667): [Direct Transfer] Allow to select folderish type
+- [NXDRIVE-2774](https://jira.nuxeo.com/browse/NXDRIVE-2774): [Direct Transfer] Make Document type selection a feature
+- [NXDRIVE-2775](https://jira.nuxeo.com/browse/NXDRIVE-2765): [Direct Transfer] Update Migration DB
 
 ## GUI
 
@@ -33,6 +36,8 @@ Release date: `2022-xx-xx`
 ## Minor Changes
 
 - Added `charset-normalizer` 2.1.1
+- Added `packaging` 21.0
+- Added `pyparsing` 2.4.7
 - Upgraded `authlib` from 0.15.4 to 1.1.0
 - Upgraded `boto3` from 1.20.19 to 1.25.0
 - Upgraded `botocore` from 1.23.19 to 1.28.0

nxdrive/client/remote_client.py

@@ -58,7 +58,13 @@
 )
 from ..metrics.poll_metrics import CustomPollMetrics
 from ..metrics.utils import current_os, user_agent
-from ..objects import Download, Metrics, NuxeoDocumentInfo, RemoteFileInfo
+from ..objects import (
+    Download,
+    Metrics,
+    NuxeoDocumentInfo,
+    RemoteFileInfo,
+    SubTypeEnricher,
+)
 from ..options import Options
 from ..qt.imports import QApplication
 from ..utils import (
@@ -288,7 +294,7 @@ def execute(self, /, **kwargs: Any) -> Any:
             return self.operations.execute(ssl_verify=Options.ssl_no_verify, **kwargs)
         except HTTPError as e:
             if e.status == requests.codes.not_found:
-                raise NotFound()
+                raise NotFound("Response code not found")
             raise e
 
     @staticmethod
@@ -575,6 +581,19 @@ def upload_folder(
         res: Dict[str, Any] = self.execute(**kwargs)
         return res
 
+    def upload_folder_type(
+        self, parent: str, params: Dict[str, str], /, *, headers: Dict[str, Any] = None
+    ) -> Dict[str, Any]:
+        """Create a folder using REST api."""
+        resp = self.client.request(
+            "POST",
+            f"{self.client.api_path}/path{parent}",
+            headers=headers,
+            data=params,
+            ssl_verify=self.verification_needed,
+        )
+        return resp
+
     def cancel_batch(self, batch_details: Dict[str, Any], /) -> None:
         """Cancel an uploaded Batch."""
         batch = Batch(service=self.uploads, **batch_details)
@@ -976,7 +995,56 @@ def get_server_configuration(self) -> Dict[str, Any]:
             log.warning(f"Error getting server configuration: {exc}")
             return {}
 
+    def get_config_types(self) -> Dict[str, Any]:
+        try:
+            resp = self.client.request(
+                "GET",
+                f"{self.client.api_path}/config/types",
+                ssl_verify=self.verification_needed,
+            ).json()
+
+            return json.loads(json.dumps(resp))
+
+        except Exception as exc:
+            log.warning(f"Error getting server configuration: {exc}")
+            return {}
+
     def _get_trash_condition(self) -> str:
         if version_lt(self.client.server_version, "10.2"):
             return "AND ecm:currentLifeCycleState != 'deleted'"
         return "AND ecm:isTrashed = 0"
+
+    def get_doc_enricher(
+        self, parent: str, enricherType: str = "subtypes", isFolderish: bool = True
+    ) -> SubTypeEnricher:
+
+        headers: Dict[str, str] = {}
+        headers = {"enrichers.document": enricherType}
+
+        enricherList = SubTypeEnricher.from_dict(
+            self.fetch(parent, headers=headers, enrichers=[enricherType]),
+            isFolderish=isFolderish,
+        )
+
+        docTypeFiletList = self.filter_schema(enricherList)
+        if isFolderish:
+            return enricherList.facets
+        else:
+            return [x for x in enricherList.facets if x in docTypeFiletList]
+
+    def get_doc_enricher_list(
+        self, parent: str, enricherType: str = "subtypes", isFolderish: bool = True
+    ) -> List[str]:
+
+        doc_enricher = self.get_doc_enricher(parent, enricherType, isFolderish)
+        return doc_enricher
+
+    def filter_schema(self, enricherList: SubTypeEnricher) -> SubTypeEnricher:
+
+        configTypes = self.get_config_types()
+        docTypeList = []
+        for docType in configTypes["doctypes"]:
+            if "file" in configTypes["doctypes"][docType]["schemas"]:
+                docTypeList.append(str(docType))
+
+        return docTypeList

nxdrive/client/uploader/__init__.py

@@ -24,6 +24,7 @@
 from ...objects import Upload
 from ...options import Options
 from ...qt.imports import QApplication
+from ...utils import get_verify
 
 if TYPE_CHECKING:
     from ..remote_client import Remote  # noqa
@@ -43,6 +44,7 @@ class BaseUploader:
     def __init__(self, remote: "Remote", /) -> None:
         self.remote = remote
         self.dao = remote.dao
+        self.verification_needed = get_verify()
 
     @abstractmethod
     def get_upload(
@@ -448,16 +450,56 @@ def link_blob_to_doc(
         else:
             kwargs["headers"] = headers
         try:
-            res: Dict[str, Any] = self.remote.execute(
-                command=command,
-                input_obj=blob,
-                timeout=kwargs.pop("timeout", TX_TIMEOUT),
-                **kwargs,
-            )
+            doc_type = kwargs.get("doc_type", "")
+            if transfer.is_direct_transfer and doc_type and doc_type != "":
+                res = self._transfer_docType_file(transfer, headers, doc_type)
+            else:
+                res = self._transfer_autoType_file(command, blob, kwargs)
+
             return res
+        except Exception as exc:
+            err = f"Error while linking blob to doc: {exc!r}"
+            log.warning(err)
         finally:
             action.finish_action()
 
+    def _transfer_autoType_file(self, command, blob, kwargs):
+        res: Dict[str, Any] = self.remote.execute(
+            command=command,
+            input_obj=blob,
+            timeout=kwargs.pop("timeout", TX_TIMEOUT),
+            **kwargs,
+        )
+
+        return res
+
+    def _transfer_docType_file(self, transfer, headers, doc_type):
+        content = {
+            "entity-type": "document",
+            "name": transfer.name,
+            "type": doc_type,
+            "properties": {
+                "dc:title": transfer.name,
+                "file:content": {
+                    "upload-batch": transfer.batch_obj.uid,
+                    "upload-fileId": "0",
+                },
+            },
+        }
+
+        self.remote.client.request(
+            "POST",
+            f"{self.remote.client.api_path}/path{transfer.remote_parent_path}",
+            headers=headers,
+            data=content,
+            ssl_verify=self.verification_needed,
+        )
+        res = self.remote.fetch(
+            f"{self.remote.client.api_path}/path{transfer.remote_parent_path}",
+            headers=headers,
+        )
+        return res
+
     @staticmethod
     def _complete_upload(transfer: Upload, blob: FileBlob, /) -> None:
         """Helper to complete an upload."""

nxdrive/client/uploader/direct_transfer.py

@@ -8,6 +8,8 @@
 
 from nuxeo.utils import guess_mimetype
 
+from nxdrive.exceptions import NotFound
+
 from ...engine.activity import LinkingAction, UploadAction
 from ...metrics.constants import (
     DT_DUPLICATE_BEHAVIOR,
@@ -64,7 +66,6 @@ def upload(
             - create a new operation `Document.GetOrCreate` that ensures atomicity.
         """
         doc_pair: DocPair = kwargs.pop("doc_pair")
-
         log.info(
             f"Direct Transfer of {file_path!r} into {doc_pair.remote_parent_path!r} ({doc_pair.remote_parent_ref!r})"
         )
@@ -77,11 +78,31 @@ def upload(
             return {}
 
         if doc_pair.folderish:
-            item = self.remote.upload_folder(
-                doc_pair.remote_parent_path,
-                {"title": doc_pair.local_name},
-                headers={DT_SESSION_NUMBER: doc_pair.session},
-            )
+            if not doc_pair.doc_type:
+                item = self.remote.upload_folder(
+                    doc_pair.remote_parent_path,
+                    {"title": doc_pair.local_name},
+                    headers={DT_SESSION_NUMBER: str(doc_pair.session)},
+                )
+            else:
+                try:
+                    payload = {
+                        "entity-type": "document",
+                        "name": doc_pair.local_name,
+                        "type": doc_pair.doc_type,
+                        "properties{'dc:title'}": doc_pair.local_name,
+                    }
+                    item = self.remote.upload_folder_type(
+                        doc_pair.remote_parent_path,
+                        payload,
+                        headers={DT_SESSION_NUMBER: str(doc_pair.session)},
+                    )
+                    filepath = f"{doc_pair.remote_parent_path}/{doc_pair.local_name}"
+                    item = self.remote.fetch(filepath)
+                except NotFound:
+                    raise NotFound(
+                        f"Could not find {filepath!r} on {self.remote.client.host}"
+                    )
             self.dao.update_remote_parent_path_dt(file_path, item["path"], item["uid"])
         else:
             # Only replace the document if the user wants to
@@ -98,6 +119,7 @@ def upload(
                 remote_parent_path=doc_pair.remote_parent_path,
                 remote_parent_ref=doc_pair.remote_parent_ref,
                 doc_pair=doc_pair.id,
+                doc_type=doc_pair.doc_type,
                 headers={
                     REQUEST_METRICS: json.dumps(
                         {

nxdrive/dao/__init__.py

@@ -12,4 +12,5 @@
 # To check and update on each beta release !!!
 versions_history = {
     "5.2.8": 21,
+    "5.3.0": 22,
 }

nxdrive/dao/engine.py

@@ -528,6 +528,9 @@ def _migrate_db_old(self, cursor: Cursor, version: int, /) -> None:
         if version < 21:
             self.store_int(SCHEMA_VERSION, 21)
             self.set_schema_version(cursor, 21)
+        if version < 22:
+            self.store_int(SCHEMA_VERSION, 22)
+            self.set_schema_version(cursor, 22)
 
     def _create_table(
         self, cursor: Cursor, name: str, /, *, force: bool = False
@@ -848,9 +851,9 @@ def plan_many_direct_transfer_items(
             query = (
                 "INSERT INTO States "
                 "(local_path, local_parent_path, local_name, folderish, size, "
-                "remote_parent_path, remote_parent_ref, duplicate_behavior, "
+                "remote_parent_path, remote_parent_ref, doc_type, duplicate_behavior, "
                 "local_state, remote_state, pair_state, session)"
-                f"VALUES (?, ?, ?, ?, ?, ?, ?, ?, 'direct', ?, 'direct_transfer', {session})"
+                f"VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, 'direct', ?, 'direct_transfer', {session})"
             )
             c.executemany(query, items)
             return current_max_row_id
@@ -2197,7 +2200,6 @@ def decrease_session_counts(self, uid: int, /) -> Optional[Session]:
 
     def save_session_item(self, session_id: int, item: Dict[str, Any]) -> None:
         """Save the session uploaded item data into the SessionItems table."""
-
         with self.lock:
             c = self._get_write_connection().cursor()
             c.execute(

nxdrive/dao/migrations/engine/0022_initial_migration.py

@@ -0,0 +1,32 @@
+from sqlite3 import Cursor
+
+from ..migration import MigrationInterface
+
+
+class MigrationInitial(MigrationInterface):
+    def upgrade(self, cursor: Cursor) -> None:
+        """
+        Update the States table.
+        """
+        self._update_state_table(cursor)
+
+    def downgrade(self, cursor: Cursor) -> None:
+        """Update the States table."""
+        # Drop Column doc_type to States table
+        cursor.execute("ALTER TABLE States DROP COLUMN doc_type")
+
+    @property
+    def version(self) -> int:
+        return 22
+
+    @property
+    def previous_version(self) -> int:
+        return 21
+
+    @staticmethod
+    def _update_state_table(cursor: Cursor) -> None:
+        """Update the States table."""
+        cursor.execute("ALTER TABLE States ADD doc_type VARCHAR")
+
+
+migration = MigrationInitial()

nxdrive/dao/migrations/engine/__init__.py

@@ -1,7 +1,7 @@
 import importlib
 from typing import Any, Dict
 
-__migrations_list = ["0021_initial_migration"]  # Keep sorted
+__migrations_list = ["0021_initial_migration", "0022_initial_migration"]  # Keep sorted
 
 
 def import_migrations() -> Dict[str, Any]:

nxdrive/data/i18n/i18n.json

@@ -47,6 +47,7 @@
     "CONNECTION_REFUSED": "Connection refused",
     "CONNECTION_SUCCESS": "Server successfully connected",
     "CONNECTION_UNKNOWN": "Unknown error occurred while trying to connect",
+    "CONTAINER_TYPE": "Folder Type",
     "CONTEXT_MENU_1": "Access online",
     "CONTEXT_MENU_2": "Copy share-link",
     "CONTEXT_MENU_3": "Edit metadata",
@@ -114,6 +115,7 @@
     "DIRECT_TRANSFER_WINDOW_TITLE": "Direct Transfer - %1",
     "DISCONNECT": "Remove account",
     "DISCONNECTING": "Disconnecting …",
+    "DOCUMENT_TYPE": "Document Type",
     "DRIVE_ROOT_DELETED": "The synchronization folder \"%1\" of %2 has been deleted. Would you like to disconnect from the server or do you want to recreate it?",
     "DRIVE_ROOT_DELETED_HEADER": "Deletion of the synchronization folder detected.",
     "DRIVE_ROOT_DISCONNECT": "Disconnect",
@@ -161,6 +163,7 @@
     "FATAL_ERROR_UPDATE_BTN": "Download latest version",
     "FATAL_ERROR_UPDATE_TOOLTIP": "We advise you to update to the latest version of %1",
     "FEATURE_AUTO_UPDATE": "Notify when new update is available.",
+    "FEATURE_DOCUMENT_TYPE_SELECTION": "Select the file and/or folder type(s) of your documents when using Direct Transfer.",
     "FEATURE_DIRECT_TRANSFER": "One-time transfer of content between the Nuxeo Server and the desktop.",
     "FEATURE_DIRECT_EDIT": "Edit any of document’s content from their Summary tab even if they are not synchronized.",
     "FEATURE_S3": "Improve the speed of your uploads by leveraging the AWS infrastructure. Even enabled, this feature will be effective only for Nuxeo Cloud customers and for Nuxeo servers with S3 Direct Upload addon installed and up-to-date.",

nxdrive/data/qml/FeaturesTab.qml

@@ -41,7 +41,7 @@ Rectangle {
 
                         text: item[0] + (beta_features.includes(item[1]) ? sup_tag: "")
                         checked: manager.get_feature_state(item[1])
-                        enabled: !disabled_features.includes(item[1])
+                        enabled: (item[1] == "document_type_selection") ? feat_direct_transfer.enabled && isFrozen: !disabled_features.includes(item[1])
                         onClicked: manager.set_feature_state(item[1], checked)
                         Layout.leftMargin: -5
                     }