fix: import notion zip file as workspace (#868)

* chore: insert content length in header

* chore: custom header

* chore: workspace name

* chore: file name

* chore: handle file name that is not utf8 encode

* chore: clippy

* fix: end of file when unzip file

* chore: docs

Cargo.toml

@@ -293,13 +293,13 @@ debug = true
 [patch.crates-io]
 # It's diffcult to resovle different version with the same crate used in AppFlowy Frontend and the Client-API crate.
 # So using patch to workaround this issue.
-collab = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "f882a1720f7b35a874ea276e604434f0b5b04698" }
-collab-entity = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "f882a1720f7b35a874ea276e604434f0b5b04698" }
-collab-folder = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "f882a1720f7b35a874ea276e604434f0b5b04698" }
-collab-document = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "f882a1720f7b35a874ea276e604434f0b5b04698" }
-collab-user = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "f882a1720f7b35a874ea276e604434f0b5b04698" }
-collab-database = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "f882a1720f7b35a874ea276e604434f0b5b04698" }
-collab-importer = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "f882a1720f7b35a874ea276e604434f0b5b04698" }
+collab = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "2934851d337c7d322c5dd7a944c05cb7ffb930d4" }
+collab-entity = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "2934851d337c7d322c5dd7a944c05cb7ffb930d4" }
+collab-folder = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "2934851d337c7d322c5dd7a944c05cb7ffb930d4" }
+collab-document = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "2934851d337c7d322c5dd7a944c05cb7ffb930d4" }
+collab-user = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "2934851d337c7d322c5dd7a944c05cb7ffb930d4" }
+collab-database = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "2934851d337c7d322c5dd7a944c05cb7ffb930d4" }
+collab-importer = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "2934851d337c7d322c5dd7a944c05cb7ffb930d4" }
 
 [features]
 history = []

libs/client-api/src/native/http_native.rs

@@ -298,10 +298,38 @@ impl Client {
     AppResponse::<()>::from_response(resp).await?.into_error()
   }
 
+  /// Sends a POST request to import a file to the server.
+  ///
+  /// This function streams the contents of a file located at the provided `file_path`
+  /// as part of a multipart form data request to the server's `/api/import` endpoint.
+  ///
+  /// ### HTTP Request Details:
+  ///
+  /// - **Method:** POST
+  /// - **URL:** `{base_url}/api/import`
+  ///   - The `base_url` is dynamically provided and appended with `/api/import`.
+  ///
+  /// - **Headers:**
+  ///   - `X-Host`: The value of the `base_url` is sent as the host header.
+  ///   - `X-Content-Length`: The size of the file, in bytes, is provided from the file's metadata.
+  ///
+  /// - **Multipart Form:**
+  ///   - The file is sent as a multipart form part:
+  ///     - **Field Name:** The file name derived from the file path or a UUID if unavailable.
+  ///     - **File Content:** The file's content is streamed using `reqwest::Body::wrap_stream`.
+  ///     - **MIME Type:** Guessed from the file's extension using the `mime_guess` crate,
+  ///       defaulting to `application/octet-stream` if undetermined.
+  ///
+  /// ### Parameters:
+  /// - `file_path`: The path to the file to be uploaded.
+  ///   - The file is opened asynchronously and its metadata (like size) is extracted.
+  /// - The MIME type is automatically determined based on the file extension using `mime_guess`.
+  ///
   pub async fn import_file(&self, file_path: &Path) -> Result<(), AppResponseError> {
     let file = File::open(&file_path).await?;
+    let metadata = file.metadata().await?;
     let file_name = file_path
-      .file_name()
+      .file_stem()
       .map(|s| s.to_string_lossy().to_string())
       .unwrap_or_else(|| uuid::Uuid::new_v4().to_string());
 
@@ -311,18 +339,20 @@ impl Client {
       .to_string();
 
     let file_part = multipart::Part::stream(reqwest::Body::wrap_stream(stream))
-      .file_name(file_name)
+      .file_name(file_name.clone())
       .mime_str(&mime)?;
 
-    let form = multipart::Form::new().part("file", file_part);
+    let form = multipart::Form::new().part(file_name, file_part);
     let url = format!("{}/api/import", self.base_url);
     let mut builder = self
       .http_client_with_auth(Method::POST, &url)
       .await?
       .multipart(form);
 
     // set the host header
-    builder = builder.header("X-Host", self.base_url.clone());
+    builder = builder
+      .header("X-Host", self.base_url.clone())
+      .header("X-Content-Length", metadata.len());
     let resp = builder.send().await?;
 
     AppResponse::<()>::from_response(resp).await?.into_error()

libs/workspace-template/src/database/database_collab.rs

@@ -12,8 +12,8 @@ pub async fn create_database_collab(
     collab_service,
     notifier: Default::default(),
   };
-  Database::create_with_view(params, context)
-    .await?
+  let database = Database::create_with_view(params, context).await?;
+  database
     .encode_database_collabs()
     .await
     .map_err(|e| anyhow::anyhow!("Failed to encode database collabs: {:?}", e))

libs/workspace-template/src/document/getting_started.rs

@@ -1,10 +1,9 @@
 use std::collections::HashMap;
-use std::sync::Arc;
 
 use anyhow::Error;
 use async_trait::async_trait;
 use collab::core::origin::CollabOrigin;
-use collab::lock::RwLock;
+
 use collab::preclude::Collab;
 use collab_database::database::{timestamp, DatabaseData};
 use collab_database::entity::CreateDatabaseParams;
@@ -66,12 +65,7 @@ impl GettingStartedTemplate {
     create_database_params: CreateDatabaseParams,
   ) -> anyhow::Result<Vec<TemplateData>> {
     let database_id = create_database_params.database_id.clone();
-    let encoded_database = tokio::task::spawn_blocking({
-      let create_database_params = create_database_params.clone();
-      move || create_database_collab(create_database_params)
-    })
-    .await?
-    .await?;
+    let encoded_database = create_database_collab(create_database_params).await?;
 
     let encoded_database_collab = encoded_database
       .encoded_database_collab
@@ -236,7 +230,7 @@ impl WorkspaceTemplate for GettingStartedTemplate {
   async fn create_workspace_view(
     &self,
     _uid: i64,
-    workspace_view_builder: Arc<RwLock<WorkspaceViewBuilder>>,
+    workspace_view_builder: &mut WorkspaceViewBuilder,
   ) -> anyhow::Result<Vec<TemplateData>> {
     let general_view_uuid = gen_view_id().to_string();
     let shared_view_uuid = gen_view_id().to_string();
@@ -263,12 +257,10 @@ impl WorkspaceTemplate for GettingStartedTemplate {
       )
       .await?;
 
-    let mut builder = workspace_view_builder.write().await;
-
     // Create general space with 2 built-in views: Getting started, To-dos
     //    The Getting started view is a document view, and the To-dos view is a board view
     //    The Getting started view contains 2 sub views: Desktop guide, Mobile guide
-    builder
+    workspace_view_builder
       .with_view_builder(|view_builder| async {
         let created_at = timestamp();
         let mut view_builder = view_builder
@@ -305,7 +297,7 @@ impl WorkspaceTemplate for GettingStartedTemplate {
       .await;
 
     // Create shared space without any built-in views
-    builder
+    workspace_view_builder
       .with_view_builder(|view_builder| async {
         let created_at = timestamp();
         let view_builder = view_builder
@@ -371,12 +363,11 @@ impl WorkspaceTemplate for DocumentTemplate {
   async fn create_workspace_view(
     &self,
     _uid: i64,
-    workspace_view_builder: Arc<RwLock<WorkspaceViewBuilder>>,
+    workspace_view_builder: &mut WorkspaceViewBuilder,
   ) -> anyhow::Result<Vec<TemplateData>> {
     let view_id = gen_view_id().to_string();
 
-    let mut builder = workspace_view_builder.write().await;
-    builder
+    workspace_view_builder
       .with_view_builder(|view_builder| async {
         view_builder
           .with_name("Getting started")

libs/workspace-template/src/lib.rs

@@ -5,7 +5,7 @@ pub use anyhow::Result;
 use async_trait::async_trait;
 use collab::core::origin::CollabOrigin;
 use collab::entity::EncodedCollab;
-use collab::lock::RwLock;
+
 use collab::preclude::Collab;
 use collab_entity::CollabType;
 use collab_folder::{
@@ -30,7 +30,7 @@ pub trait WorkspaceTemplate {
   async fn create_workspace_view(
     &self,
     uid: i64,
-    workspace_view_builder: Arc<RwLock<WorkspaceViewBuilder>>,
+    workspace_view_builder: &mut WorkspaceViewBuilder,
   ) -> Result<Vec<TemplateData>>;
 }
 
@@ -91,22 +91,18 @@ impl WorkspaceTemplateBuilder {
   }
 
   pub async fn build(&self) -> Result<Vec<TemplateData>> {
-    let workspace_view_builder = Arc::new(RwLock::from(WorkspaceViewBuilder::new(
-      self.workspace_id.clone(),
-      self.uid,
-    )));
-
+    let mut workspace_view_builder = WorkspaceViewBuilder::new(self.workspace_id.clone(), self.uid);
     let mut templates: Vec<TemplateData> = vec![];
     for handler in self.handlers.values() {
       if let Ok(template) = handler
-        .create_workspace_view(self.uid, workspace_view_builder.clone())
+        .create_workspace_view(self.uid, &mut workspace_view_builder)
         .await
       {
         templates.extend(template);
       }
     }
 
-    let views = workspace_view_builder.write().await.build();
+    let views = workspace_view_builder.build();
     // Safe to unwrap because we have at least one view.
     let first_view = views.first().unwrap().parent_view.clone();
     let first_level_views = views

libs/workspace-template/src/tests/getting_started_tests.rs

@@ -1,7 +1,5 @@
 use std::collections::HashMap;
-use std::sync::Arc;
 
-use collab::lock::RwLock;
 use collab::preclude::uuid_v4;
 use collab_database::database::DatabaseData;
 use collab_database::entity::CreateDatabaseParams;
@@ -95,18 +93,17 @@ mod tests {
   #[tokio::test]
   async fn create_workspace_view_with_getting_started_template_test() {
     let template = GettingStartedTemplate;
-    let workspace_view_builder = Arc::new(RwLock::new(WorkspaceViewBuilder::new(generate_id(), 1)));
+    let mut workspace_view_builder = WorkspaceViewBuilder::new(generate_id(), 1);
 
     let result = template
-      .create_workspace_view(1, workspace_view_builder.clone())
+      .create_workspace_view(1, &mut workspace_view_builder)
       .await
       .unwrap();
 
     // 2 spaces + 3 documents + 1 database + 5 database rows
     assert_eq!(result.len(), 11);
 
-    let mut builder = workspace_view_builder.write().await;
-    let views = builder.build();
+    let views = workspace_view_builder.build();
 
     // check the number of spaces
     assert_eq!(views.len(), 2);

services/appflowy-worker/Cargo.toml

@@ -46,3 +46,4 @@ mime_guess = "2.0"
 bytes.workspace = true
 uuid.workspace = true
 
+

services/appflowy-worker/src/application.rs

@@ -97,7 +97,7 @@ pub async fn create_app(listener: TcpListener, config: Config) -> Result<(), Err
     Arc::new(state.s3_client.clone()),
     Arc::new(email_notifier),
     "import_task_stream",
-    30,
+    10,
   ));
 
   let app = Router::new().with_state(state);