diff --git a/.github/workflows/integration_test.yml b/.github/workflows/integration_test.yml index 53ba1ff44..079111e96 100644 --- a/.github/workflows/integration_test.yml +++ b/.github/workflows/integration_test.yml @@ -24,6 +24,7 @@ env: LOCALHOST_URL: http://localhost LOCALHOST_WS: ws://localhost/ws/v1 APPFLOWY_REDIS_URI: redis://redis:6379 + APPFLOWY_AI_REDIS_URL: redis://redis:6379 LOCALHOST_GOTRUE: http://localhost/gotrue POSTGRES_PASSWORD: password DATABASE_URL: postgres://postgres:password@localhost:5432/postgres @@ -44,20 +45,17 @@ jobs: - name: Build Docker Images run: | export DOCKER_DEFAULT_PLATFORM=linux/amd64 - docker compose build appflowy_cloud appflowy_history appflowy_worker admin_frontend + docker compose build appflowy_cloud appflowy_worker admin_frontend - name: Push docker images to docker hub run: | docker tag appflowyinc/appflowy_cloud appflowyinc/appflowy_cloud:${GITHUB_SHA} - docker tag appflowyinc/appflowy_history appflowyinc/appflowy_history:${GITHUB_SHA} docker tag appflowyinc/appflowy_worker appflowyinc/appflowy_worker:${GITHUB_SHA} docker tag appflowyinc/admin_frontend appflowyinc/admin_frontend:${GITHUB_SHA} echo ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} | docker login --username appflowyinc --password-stdin docker push appflowyinc/appflowy_cloud:${GITHUB_SHA} - docker push appflowyinc/appflowy_history:${GITHUB_SHA} docker push appflowyinc/appflowy_worker:${GITHUB_SHA} docker push appflowyinc/admin_frontend:${GITHUB_SHA} - APPFLOWY_HISTORY_VERSION=${GITHUB_SHA} APPFLOWY_WORKER_VERSION=${GITHUB_SHA} APPFLOWY_CLOUD_VERSION=${GITHUB_SHA} APPFLOWY_ADMIN_FRONTEND_VERSION=${GITHUB_SHA} @@ -71,8 +69,6 @@ jobs: include: - test_service: "appflowy_cloud" test_cmd: "--workspace --exclude appflowy-history --exclude appflowy-ai-client --features ai-test-enabled" - - test_service: "appflowy_history" - test_cmd: "-p appflowy-history" - test_service: "appflowy_worker" test_cmd: "-p appflowy-worker" - test_service: "admin_frontend" @@ -110,38 +106,34 @@ jobs: # the wasm-pack headless tests will run on random ports, so we need to allow all origins run: sed -i 's/http:\/\/127\.0\.0\.1:8000/http:\/\/127.0.0.1/g' nginx/nginx.conf + - name: Log in to Docker Hub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKER_HUB_USERNAME }} + password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} + - name: Run Docker-Compose run: | - export APPFLOWY_HISTORY_VERSION=${GITHUB_SHA} export APPFLOWY_WORKER_VERSION=${GITHUB_SHA} export APPFLOWY_CLOUD_VERSION=${GITHUB_SHA} export APPFLOWY_ADMIN_FRONTEND_VERSION=${GITHUB_SHA} docker compose -f docker-compose-ci.yml up -d docker ps -a - container_id=$(docker ps --filter name=appflowy-cloud-ai-1 -q) - if [ -n "$container_id" ]; then - echo "Displaying logs for the AppFlowy-AI container..." - docker logs "$container_id" - else - echo "No running container found to display logs." - fi - - name: Install prerequisites run: | sudo apt-get update - sudo apt-get install protobuf-compiler + sudo apt-get install -y protobuf-compiler - name: Run Tests run: | echo "Running tests for ${{ matrix.test_service }} with flags: ${{ matrix.test_cmd }}" RUST_LOG="info" DISABLE_CI_TEST_LOG="true" cargo test ${{ matrix.test_cmd }} - - name: Run Tests from main branch + - name: Docker Logs + if: always() run: | - git fetch origin main - git checkout main - RUST_LOG="info" DISABLE_CI_TEST_LOG="true" cargo test ${{ matrix.test_cmd }} + docker logs appflowy-cloud-ai-1 cleanup: name: Cleanup Docker Images diff --git a/.github/workflows/push_latest_docker.yml b/.github/workflows/push_latest_docker.yml index 666caabb6..361867969 100644 --- a/.github/workflows/push_latest_docker.yml +++ b/.github/workflows/push_latest_docker.yml @@ -95,6 +95,7 @@ jobs: labels: ${{ steps.meta.outputs.labels }} provenance: false build-args: | + PROFILE=release FEATURES= - name: Logout from Docker Hub @@ -238,102 +239,6 @@ jobs: if: always() run: docker logout - appflowy_history_image: - runs-on: ubuntu-22.04 - env: - IMAGE_NAME: ${{ secrets.DOCKER_HUB_USERNAME }}/appflowy_history - strategy: - fail-fast: false - matrix: - job: - - { name: "amd64", docker_platform: "linux/amd64" } - - { name: "arm64v8", docker_platform: "linux/arm64" } - - steps: - - name: Check out the repository - uses: actions/checkout@v2 - with: - fetch-depth: 1 - - - name: Set up QEMU - uses: docker/setup-qemu-action@v3 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - - name: Log in to Docker Hub - uses: docker/login-action@v3 - with: - username: ${{ secrets.DOCKER_HUB_USERNAME }} - password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} - - - name: Get git tag - id: vars - run: | - T=${GITHUB_REF#refs/*/} # Remove "refs/*/" prefix from GITHUB_REF - echo "GIT_TAG=$T" >> $GITHUB_ENV - - - name: Extract metadata - id: meta - uses: docker/metadata-action@v4 - with: - images: registry.hub.docker.com/${{ env.IMAGE_NAME }} - - - name: Build and push ${{ matrix.job.image_name }}:${{ env.GIT_TAG }} - uses: docker/build-push-action@v5 - with: - platforms: ${{ matrix.job.docker_platform }} - file: ./services/appflowy-history/Dockerfile - push: true - tags: | - ${{ env.IMAGE_NAME }}:${{ env.LATEST_TAG }}-${{ matrix.job.name }} - ${{ env.IMAGE_NAME }}:${{ env.GIT_TAG }}-${{ matrix.job.name }} - labels: ${{ steps.meta.outputs.labels }} - provenance: false - - - name: Logout from Docker Hub - if: always() - run: docker logout - - appflowy_history_manifest: - runs-on: ubuntu-22.04 - needs: [ appflowy_history_image ] - strategy: - fail-fast: false - matrix: - job: - - { image_name: "appflowy_history" } - - steps: - - name: Log in to Docker Hub - uses: docker/login-action@v3 - with: - username: ${{ secrets.DOCKER_HUB_USERNAME }} - password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} - - - name: Get git tag - id: vars - run: | - T=${GITHUB_REF#refs/*/} # Remove "refs/*/" prefix from GITHUB_REF - echo "GIT_TAG=$T" >> $GITHUB_ENV - - - name: Create and push manifest for ${{ matrix.job.image_name }}:version - uses: Noelware/docker-manifest-action@master - with: - inputs: ${{ secrets.DOCKER_HUB_USERNAME }}/${{ matrix.job.image_name }}:${{ env.GIT_TAG }} - images: ${{ secrets.DOCKER_HUB_USERNAME }}/${{ matrix.job.image_name }}:${{ env.GIT_TAG }}-amd64,${{ secrets.DOCKER_HUB_USERNAME }}/${{ matrix.job.image_name }}:${{ env.GIT_TAG }}-arm64v8 - push: true - - - name: Create and push manifest for ${{ matrix.job.image_name }}:latest - uses: Noelware/docker-manifest-action@master - with: - inputs: ${{ secrets.DOCKER_HUB_USERNAME }}/${{ matrix.job.image_name }}:${{ env.LATEST_TAG }} - images: ${{ secrets.DOCKER_HUB_USERNAME }}/${{ matrix.job.image_name }}:${{ env.LATEST_TAG }}-amd64,${{ secrets.DOCKER_HUB_USERNAME }}/${{ matrix.job.image_name }}:${{ env.LATEST_TAG }}-arm64v8 - push: true - - - name: Logout from Docker Hub - if: always() - run: docker logout appflowy_worker_image: runs-on: ubuntu-22.04 diff --git a/.sqlx/query-12dcf313d0e4c0c0da2569f3326e49e1a78fa54537cb8826a20bef2769d04dd1.json b/.sqlx/query-15613595695e2e722c45712931ce0eb8d2a3deb1bb665d1f091f354a3ad96b92.json similarity index 72% rename from .sqlx/query-12dcf313d0e4c0c0da2569f3326e49e1a78fa54537cb8826a20bef2769d04dd1.json rename to .sqlx/query-15613595695e2e722c45712931ce0eb8d2a3deb1bb665d1f091f354a3ad96b92.json index 4b670cffd..6525089b9 100644 --- a/.sqlx/query-12dcf313d0e4c0c0da2569f3326e49e1a78fa54537cb8826a20bef2769d04dd1.json +++ b/.sqlx/query-15613595695e2e722c45712931ce0eb8d2a3deb1bb665d1f091f354a3ad96b92.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n SELECT EXISTS(\n SELECT 1\n FROM af_published_collab\n WHERE workspace_id = $1\n AND publish_name = $2\n )\n ", + "query": "\n SELECT EXISTS(\n SELECT 1\n FROM af_published_collab\n WHERE workspace_id = $1\n AND publish_name = $2\n AND unpublished_at IS NULL\n )\n ", "describe": { "columns": [ { @@ -19,5 +19,5 @@ null ] }, - "hash": "12dcf313d0e4c0c0da2569f3326e49e1a78fa54537cb8826a20bef2769d04dd1" + "hash": "15613595695e2e722c45712931ce0eb8d2a3deb1bb665d1f091f354a3ad96b92" } diff --git a/.sqlx/query-6a1722f63a88debb617c20f91d2adfe4049234258ff2c8429db85206a96c53c1.json b/.sqlx/query-6a1722f63a88debb617c20f91d2adfe4049234258ff2c8429db85206a96c53c1.json deleted file mode 100644 index 7c848d9aa..000000000 --- a/.sqlx/query-6a1722f63a88debb617c20f91d2adfe4049234258ff2c8429db85206a96c53c1.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "DELETE FROM af_collab_embeddings WHERE fragment_id IN (SELECT unnest($1::text[]))", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "TextArray" - ] - }, - "nullable": [] - }, - "hash": "6a1722f63a88debb617c20f91d2adfe4049234258ff2c8429db85206a96c53c1" -} diff --git a/.sqlx/query-2ba7fdabdd71d2e3ac6dc1b67af17e8dcb59d13b644bc336da5351184ec4c9e1.json b/.sqlx/query-a98cb855107a0641979d3a7fecaf01df960e3fe5abd841752c05782c7203ff12.json similarity index 60% rename from .sqlx/query-2ba7fdabdd71d2e3ac6dc1b67af17e8dcb59d13b644bc336da5351184ec4c9e1.json rename to .sqlx/query-a98cb855107a0641979d3a7fecaf01df960e3fe5abd841752c05782c7203ff12.json index 926c6e56a..f6ce671ae 100644 --- a/.sqlx/query-2ba7fdabdd71d2e3ac6dc1b67af17e8dcb59d13b644bc336da5351184ec4c9e1.json +++ b/.sqlx/query-a98cb855107a0641979d3a7fecaf01df960e3fe5abd841752c05782c7203ff12.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n UPDATE af_published_collab\n SET publish_name = $1\n WHERE workspace_id = $2\n AND view_id = $3\n ", + "query": "\n UPDATE af_published_collab\n SET publish_name = $1\n WHERE workspace_id = $2\n AND view_id = $3\n ", "describe": { "columns": [], "parameters": { @@ -12,5 +12,5 @@ }, "nullable": [] }, - "hash": "2ba7fdabdd71d2e3ac6dc1b67af17e8dcb59d13b644bc336da5351184ec4c9e1" + "hash": "a98cb855107a0641979d3a7fecaf01df960e3fe5abd841752c05782c7203ff12" } diff --git a/.sqlx/query-aa75996ca6aa12f0bcaa5fb092ac279f8a94aadcc29d0e2b652dc420506835e7.json b/.sqlx/query-aa75996ca6aa12f0bcaa5fb092ac279f8a94aadcc29d0e2b652dc420506835e7.json new file mode 100644 index 000000000..443398544 --- /dev/null +++ b/.sqlx/query-aa75996ca6aa12f0bcaa5fb092ac279f8a94aadcc29d0e2b652dc420506835e7.json @@ -0,0 +1,29 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT workspace_id, role_id\n FROM af_workspace_member\n WHERE workspace_id = ANY($1)\n AND uid = (SELECT uid FROM public.af_user WHERE uuid = $2)\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "workspace_id", + "type_info": "Uuid" + }, + { + "ordinal": 1, + "name": "role_id", + "type_info": "Int4" + } + ], + "parameters": { + "Left": [ + "UuidArray", + "Uuid" + ] + }, + "nullable": [ + false, + false + ] + }, + "hash": "aa75996ca6aa12f0bcaa5fb092ac279f8a94aadcc29d0e2b652dc420506835e7" +} diff --git a/.sqlx/query-b7d2c2d32d4b221ed8c74d6549a9aa0e03922fcdddddb1c473eb9496b7bb9721.json b/.sqlx/query-b7d2c2d32d4b221ed8c74d6549a9aa0e03922fcdddddb1c473eb9496b7bb9721.json new file mode 100644 index 000000000..2a50aa455 --- /dev/null +++ b/.sqlx/query-b7d2c2d32d4b221ed8c74d6549a9aa0e03922fcdddddb1c473eb9496b7bb9721.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM af_collab_embeddings WHERE oid = $1", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [] + }, + "hash": "b7d2c2d32d4b221ed8c74d6549a9aa0e03922fcdddddb1c473eb9496b7bb9721" +} diff --git a/.sqlx/query-dbc31936b3e79632f9c8bae449182274d9d75766bd9a5c383b96bd60e9c5c866.json b/.sqlx/query-dbc31936b3e79632f9c8bae449182274d9d75766bd9a5c383b96bd60e9c5c866.json new file mode 100644 index 000000000..bd35ee2c0 --- /dev/null +++ b/.sqlx/query-dbc31936b3e79632f9c8bae449182274d9d75766bd9a5c383b96bd60e9c5c866.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT rag_ids\n FROM af_chat\n WHERE chat_id = $1 AND deleted_at IS NULL\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "rag_ids", + "type_info": "Jsonb" + } + ], + "parameters": { + "Left": [ + "Uuid" + ] + }, + "nullable": [ + false + ] + }, + "hash": "dbc31936b3e79632f9c8bae449182274d9d75766bd9a5c383b96bd60e9c5c866" +} diff --git a/Cargo.lock b/Cargo.lock index 7824ccceb..1e9082fe6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -178,7 +178,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "13d324164c51f63867b57e73ba5936ea151b8a41a1d23d1031eeb9f70d0236f8" dependencies = [ "bytestring", - "cfg-if 1.0.0", + "cfg-if", "http 0.2.12", "regex", "regex-lite", @@ -292,7 +292,7 @@ dependencies = [ "ahash 0.8.11", "bytes", "bytestring", - "cfg-if 1.0.0", + "cfg-if", "cookie 0.16.2", "derive_more", "encoding_rs", @@ -425,7 +425,7 @@ version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "cipher", "cpufeatures", ] @@ -472,7 +472,7 @@ version = "0.8.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e89da841a80418a9b391ebaea17f5c112ffaaa96f621d2c285b5174da76b9011" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "const-random", "getrandom 0.2.15", "once_cell", @@ -747,57 +747,20 @@ dependencies = [ "shared-entity", "sqlx", "thiserror", + "tiktoken-rs", "tokio", "tokio-stream", "tokio-util", "tracing", "tracing-subscriber", + "unicode-normalization", + "unicode-segmentation", "uuid", "validator", "workspace-template", "yrs", ] -[[package]] -name = "appflowy-history" -version = "0.1.0" -dependencies = [ - "anyhow", - "arc-swap", - "assert-json-diff", - "axum 0.7.5", - "bincode", - "chrono", - "collab", - "collab-entity", - "collab-stream", - "dashmap 5.5.3", - "database", - "dotenvy", - "futures", - "infra", - "log", - "prost", - "rand 0.8.5", - "redis 0.25.4", - "serde", - "serde_json", - "serde_repr", - "serial_test", - "sqlx", - "thiserror", - "tokio", - "tokio-stream", - "tonic", - "tonic-proto", - "tower", - "tower-http", - "tower-service", - "tracing", - "tracing-subscriber", - "uuid", -] - [[package]] name = "appflowy-worker" version = "0.1.0" @@ -1588,7 +1551,7 @@ checksum = "5cc23269a4f8976d0a4d2e7109211a419fe30e8d88d677cd60b6bc79c5732e0a" dependencies = [ "addr2line", "cc", - "cfg-if 1.0.0", + "cfg-if", "libc", "miniz_oxide 0.7.4", "object", @@ -1960,12 +1923,6 @@ dependencies = [ "libc", ] -[[package]] -name = "cfg-if" -version = "0.1.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822" - [[package]] name = "cfg-if" version = "1.0.0" @@ -2194,32 +2151,6 @@ dependencies = [ "web-sys", ] -[[package]] -name = "client-api-wasm" -version = "0.1.0" -dependencies = [ - "bytes", - "client-api", - "collab-entity", - "collab-rt-entity", - "console_error_panic_hook", - "database-entity", - "lazy_static", - "serde", - "serde-wasm-bindgen", - "serde_json", - "serde_repr", - "tracing", - "tracing-core", - "tracing-wasm", - "tsify", - "uuid", - "wasm-bindgen", - "wasm-bindgen-futures", - "wasm-bindgen-test", - "wee_alloc", -] - [[package]] name = "client-websocket" version = "0.1.0" @@ -2518,7 +2449,7 @@ version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a06aeb73f470f66dcdbf7223caeebb85984942f22f1adb2a088cf9668146bbbc" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "wasm-bindgen", ] @@ -2689,7 +2620,7 @@ version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a97769d94ddab943e4510d138150169a2758b5ef3eb191a9ee688de3e23ef7b3" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", ] [[package]] @@ -2869,7 +2800,7 @@ version = "4.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "97fb8b7c4503de7d6ae7b42ab72a5a59857b4c937ec27a3d4539dba95b5ab2be" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "cpufeatures", "curve25519-dalek-derive", "fiat-crypto", @@ -2930,7 +2861,7 @@ version = "5.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "978747c1d849a7d2ee5e8adc0159961c48fb7e5db2f06af6723b80123bb53856" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "hashbrown 0.14.5", "lock_api", "once_cell", @@ -2943,7 +2874,7 @@ version = "6.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "804c8821570c3f8b70230c2ba75ffa5c0f9a4189b9a432b6656c536712acae28" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "crossbeam-utils", "hashbrown 0.14.5", "lock_api", @@ -3198,7 +3129,7 @@ version = "0.8.34" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b45de904aa0b010bce2ab45264d0631681847fa7b6f2eaa7dab7619943bc4f59" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", ] [[package]] @@ -3250,7 +3181,7 @@ version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "136d1b5283a1ab77bd9257427ffd09d8667ced0570b6f938942bc7568ed5b943" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "home", "windows-sys 0.48.0", ] @@ -3412,9 +3343,9 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eac8f7d7865dcb88bd4373ab671c8cf4508703796caa2b1985a9ca867b3fcb78" +checksum = "2dff15bf788c671c1934e366d07e30c1814a8ef514e1af724a602e8a2fbe1b10" dependencies = [ "futures-core", "futures-sink", @@ -3422,9 +3353,9 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dfc6580bb841c5a68e9ef15c77ccc837b40a7504914d52e47b8b0e9bbda25a1d" +checksum = "05f29059c0c2090612e8d742178b0580d2dc940c837851ad723096f87af6663e" [[package]] name = "futures-executor" @@ -3450,9 +3381,9 @@ dependencies = [ [[package]] name = "futures-io" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a44623e20b9681a318efdd71c299b6b222ed6f231972bfe2f224ebad6311f0c1" +checksum = "9e5c1b78ca4aae1ac06c48a526a655760685149f0d465d21f37abfe57ce075c6" [[package]] name = "futures-lite" @@ -3469,9 +3400,9 @@ dependencies = [ [[package]] name = "futures-macro" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" +checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" dependencies = [ "proc-macro2", "quote", @@ -3480,15 +3411,15 @@ dependencies = [ [[package]] name = "futures-sink" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9fb8e00e87438d937621c1c6269e53f536c14d3fbd6a042bb24879e57d474fb5" +checksum = "e575fab7d1e0dcb8d0c7bcf9a63ee213816ab51902e6d244a95819acacf1d4f7" [[package]] name = "futures-task" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38d84fa142264698cdce1a9f9172cf383a0c82de1bddcf3092901442c4097004" +checksum = "f90f7dce0722e95104fcb095585910c0977252f286e354b5e3bd38902cd99988" [[package]] name = "futures-timer" @@ -3498,9 +3429,9 @@ checksum = "f288b0a4f20f9a56b5d1da57e2227c661b7b16168e2f72365f57b63326e29b24" [[package]] name = "futures-util" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d6401deb83407ab3da39eba7e33987a73c3df0c82b4bb5813ee871c19c41d48" +checksum = "9fa08315bb612088cc391249efdc3bc77536f16c91f6cf495e6fbe85b20a4a81" dependencies = [ "futures-channel", "futures-core", @@ -3558,7 +3489,7 @@ version = "0.1.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8fc3cb4d91f53b50155bdcfd23f6a4c39ae1969c2ae85982b135750cccaf5fce" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "libc", "wasi 0.9.0+wasi-snapshot-preview1", ] @@ -3569,7 +3500,7 @@ version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "js-sys", "libc", "wasi 0.11.0+wasi-snapshot-preview1", @@ -3656,7 +3587,7 @@ version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "68a7f542ee6b35af73b06abc0dad1c1bae89964e4e253bc4b587b91c9637867b" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "dashmap 5.5.3", "futures", "futures-timer", @@ -3725,7 +3656,7 @@ version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6dd08c532ae367adf81c312a4580bc67f1d0fe8bc9c460520283f4c0ff277888" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "crunchy", ] @@ -3828,7 +3759,7 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f9c7c7c8ac16c798734b8a24560c1362120597c40d5e1459f09498f8f6c8f2ba" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "libc", "windows", ] @@ -4217,7 +4148,7 @@ version = "0.1.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e0242819d153cba4b4b05a5a8f2a7e9bbf97b6055b2a002b395c96b5ff3c0222" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", ] [[package]] @@ -4512,7 +4443,7 @@ version = "0.10.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "digest", ] @@ -4528,12 +4459,6 @@ version = "2.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" -[[package]] -name = "memory_units" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8452105ba047068f40ff7093dd1d9da90898e63dd61736462e9cdda6a90ad3c3" - [[package]] name = "mime" version = "0.3.17" @@ -4830,7 +4755,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9529f4786b70a3e8c61e11179af17ab6188ad8d0ded78c5529441ed39d4bd9c1" dependencies = [ "bitflags 2.6.0", - "cfg-if 1.0.0", + "cfg-if", "foreign-types", "libc", "once_cell", @@ -4933,7 +4858,7 @@ version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "60a2cfe6f0ad2bfc16aefa463b497d5c7a5ecd44a23efa72aa342d90177356dc" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "instant", "libc", "redox_syscall 0.2.16", @@ -4947,7 +4872,7 @@ version = "0.9.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e401f977ab385c9e4e3ab30627d6f26d00e2c73eef317493c4ec6d468726cf8" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "libc", "redox_syscall 0.5.3", "smallvec", @@ -5302,7 +5227,7 @@ version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9d1fe60d06143b2430aa532c94cfe9e29783047f06c0d7fd359a9a51b729fa25" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "cpufeatures", "opaque-debug", "universal-hash", @@ -6063,7 +5988,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c17fa4cb658e3583423e915b9f3acc01cceaee1860e33d59ebae66adc3a2dc0d" dependencies = [ "cc", - "cfg-if 1.0.0", + "cfg-if", "getrandom 0.2.15", "libc", "spin 0.9.8", @@ -6314,15 +6239,6 @@ dependencies = [ "regex", ] -[[package]] -name = "scc" -version = "2.1.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05ccfb12511cdb770157ace92d7dda771e498445b78f9886e8cdbc5140a4eced" -dependencies = [ - "sdd", -] - [[package]] name = "schannel" version = "0.1.23" @@ -6377,12 +6293,6 @@ dependencies = [ "untrusted 0.9.0", ] -[[package]] -name = "sdd" -version = "2.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "177258b64c0faaa9ffd3c65cd3262c2bc7e2588dbbd9c1641d0346145c1bbda8" - [[package]] name = "seahash" version = "4.1.0" @@ -6473,17 +6383,6 @@ dependencies = [ "serde_derive", ] -[[package]] -name = "serde-wasm-bindgen" -version = "0.6.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8302e169f0eddcc139c70f139d19d6467353af16f9fce27e8c30158036a1e16b" -dependencies = [ - "js-sys", - "serde", - "wasm-bindgen", -] - [[package]] name = "serde_derive" version = "1.0.204" @@ -6560,31 +6459,6 @@ dependencies = [ "serde", ] -[[package]] -name = "serial_test" -version = "3.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b4b487fe2acf240a021cf57c6b2b4903b1e78ca0ecd862a71b71d2a51fed77d" -dependencies = [ - "futures", - "log", - "once_cell", - "parking_lot 0.12.3", - "scc", - "serial_test_derive", -] - -[[package]] -name = "serial_test_derive" -version = "3.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "82fe9db325bcef1fbcde82e078a5cc4efdf787e96b3b9cf45b50b529f2083d67" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.72", -] - [[package]] name = "servo_arc" version = "0.3.0" @@ -6600,7 +6474,7 @@ version = "0.10.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "cpufeatures", "digest", ] @@ -6617,7 +6491,7 @@ version = "0.10.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "793db75ad2bcafc3ffa7c68b215fee268f537982cd901d132f89c6343f3a3dc8" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "cpufeatures", "digest", ] @@ -7054,7 +6928,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c886bd4480155fd3ef527d45e9ac8dd7118a898a46530b7b94c3e21866259fce" dependencies = [ "cc", - "cfg-if 1.0.0", + "cfg-if", "libc", "psm", "winapi", @@ -7231,7 +7105,7 @@ version = "3.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "85b77fafb263dd9d05cbeac119526425676db3784113aa9295c88498cbf8bff1" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "fastrand", "rustix", "windows-sys 0.52.0", @@ -7283,7 +7157,7 @@ version = "1.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b9ef9bad013ada3808854ceac7b46812a6465ba368859a37e2100283d2d719c" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "once_cell", ] @@ -7298,6 +7172,22 @@ dependencies = [ "weezl", ] +[[package]] +name = "tiktoken-rs" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44075987ee2486402f0808505dd65692163d243a337fc54363d49afac41087f6" +dependencies = [ + "anyhow", + "base64 0.21.7", + "bstr", + "fancy-regex 0.13.0", + "lazy_static", + "parking_lot 0.12.3", + "regex", + "rustc-hash", +] + [[package]] name = "time" version = "0.3.36" @@ -7729,17 +7619,6 @@ dependencies = [ "tracing-serde", ] -[[package]] -name = "tracing-wasm" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4575c663a174420fa2d78f4108ff68f65bf2fbb7dd89f33749b6e826b3626e07" -dependencies = [ - "tracing", - "tracing-subscriber", - "wasm-bindgen", -] - [[package]] name = "triomphe" version = "0.1.13" @@ -8041,7 +7920,7 @@ version = "0.2.92" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4be2531df63900aeb2bca0daaaddec08491ee64ceecbee5076636a3b026795a8" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "wasm-bindgen-macro", ] @@ -8066,7 +7945,7 @@ version = "0.4.42" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "76bc14366121efc8dbb487ab05bcc9d346b3b5ec0eaa76e46594cabbe51762c0" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "js-sys", "wasm-bindgen", "web-sys", @@ -8190,18 +8069,6 @@ dependencies = [ "rustls-pki-types", ] -[[package]] -name = "wee_alloc" -version = "0.4.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dbb3b5a6b2bb17cb6ad44a2e68a43e8d2722c997da10e928665c72ec6c0a0b8e" -dependencies = [ - "cfg-if 0.1.10", - "libc", - "memory_units", - "winapi", -] - [[package]] name = "weezl" version = "0.1.8" @@ -8422,7 +8289,7 @@ version = "0.50.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "524e57b2c537c0f9b1e69f1965311ec12182b4122e45035b1508cd24d2adadb1" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "windows-sys 0.48.0", ] @@ -8432,7 +8299,7 @@ version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a277a57398d4bfa075df44f501a17cfdf8542d224f0d36095a2adc7aee4ef0a5" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "windows-sys 0.48.0", ] diff --git a/Cargo.toml b/Cargo.toml index 14e8f8bda..1a1d72ed7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -211,11 +211,10 @@ members = [ "libs/client-websocket", "libs/client-api-test", "libs/wasm-test", - "libs/client-api-wasm", "libs/appflowy-ai-client", "libs/client-api-entity", # services - "services/appflowy-history", + #"services/appflowy-history", "services/appflowy-collaborate", "services/appflowy-worker", # xtask @@ -302,6 +301,11 @@ codegen-units = 1 inherits = "release" debug = true +[profile.ci] +inherits = "release" +opt-level = 2 +lto = false # Disable Link-Time Optimization + [patch.crates-io] # It's diffcult to resovle different version with the same crate used in AppFlowy Frontend and the Client-API crate. # So using patch to workaround this issue. @@ -315,4 +319,5 @@ collab-importer = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev [features] history = [] +# Some AI test features are not available for self-hosted AppFlowy Cloud. Therefore, AI testing is disabled by default. ai-test-enabled = ["client-api-test/ai-test-enabled"] diff --git a/Dockerfile b/Dockerfile index de8b72726..e31bc4ee1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -16,16 +16,19 @@ RUN apt update && apt install -y protobuf-compiler lld clang # Specify a default value for FEATURES; it could be an empty string if no features are enabled by default ARG FEATURES="" +ARG PROFILE="release" COPY --from=planner /app/recipe.json recipe.json # Build our project dependencies +ENV CARGO_BUILD_JOBS=4 RUN cargo chef cook --release --recipe-path recipe.json + COPY . . ENV SQLX_OFFLINE true # Build the project -RUN echo "Building with features: ${FEATURES}" -RUN cargo build --profile=release --features "${FEATURES}" --bin appflowy_cloud +RUN echo "Building with profile: ${PROFILE}, features: ${FEATURES}, " +RUN cargo build --profile=${PROFILE} --features "${FEATURES}" --bin appflowy_cloud FROM debian:bookworm-slim AS runtime WORKDIR /app diff --git a/admin_frontend/src/web_api.rs b/admin_frontend/src/web_api.rs index a6a2a4a75..a8fd09212 100644 --- a/admin_frontend/src/web_api.rs +++ b/admin_frontend/src/web_api.rs @@ -124,7 +124,7 @@ async fn open_app_handler( session.token.refresh_token, session.token.token_type, ); - Ok(Redirect::to(&app_sign_in_url).into_response()) + Ok(htmx_redirect(&app_sign_in_url).into_response()) } /// Delete the user account and all associated data. diff --git a/deny.toml b/deny.toml index 852fe786b..9aaada8c7 100644 --- a/deny.toml +++ b/deny.toml @@ -1,2 +1,2 @@ [advisories] -ignore = ["RUSTSEC-2024-0370"] +ignore = ["RUSTSEC-2024-0370", "RUSTSEC-2024-0384"] diff --git a/deploy.env b/deploy.env index c8b80dd33..afae1ebfa 100644 --- a/deploy.env +++ b/deploy.env @@ -113,6 +113,7 @@ APPFLOWY_S3_BUCKET=appflowy APPFLOWY_MAILER_SMTP_HOST=smtp.gmail.com APPFLOWY_MAILER_SMTP_PORT=465 APPFLOWY_MAILER_SMTP_USERNAME=email_sender@some_company.com +APPFLOWY_MAILER_SMTP_EMAIL=email_sender@some_company.com APPFLOWY_MAILER_SMTP_PASSWORD=email_sender_password # Log level for the appflowy-cloud service @@ -145,13 +146,9 @@ APPFLOWY_AI_OPENAI_API_KEY= APPFLOWY_AI_SERVER_PORT=5001 APPFLOWY_AI_SERVER_HOST=ai APPFLOWY_AI_DATABASE_URL=postgresql+psycopg://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${POSTGRES_HOST}:${POSTGRES_PORT}/${POSTGRES_DB} +APPFLOWY_AI_REDIS_URL=redis://${REDIS_HOST}:${REDIS_PORT} APPFLOWY_LOCAL_AI_TEST_ENABLED=false -# AppFlowy History -APPFLOWY_GRPC_HISTORY_ADDRS=http://localhost:50051 -APPFLOWY_HISTORY_REDIS_URL=redis://${REDIS_HOST}:${REDIS_PORT} -APPFLOWY_HISTORY_DATABASE_URL=postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${POSTGRES_HOST}:${POSTGRES_PORT}/${POSTGRES_DB} - # AppFlowy Indexer APPFLOWY_INDEXER_ENABLED=true APPFLOWY_INDEXER_DATABASE_URL=postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${POSTGRES_HOST}:${POSTGRES_PORT}/${POSTGRES_DB} diff --git a/dev.env b/dev.env index b514f507f..57c52fcf2 100644 --- a/dev.env +++ b/dev.env @@ -4,6 +4,7 @@ APPFLOWY_DATABASE_URL=postgres://postgres:password@localhost:5432/postgres APPFLOWY_ACCESS_CONTROL=true APPFLOWY_WEBSOCKET_MAILBOX_SIZE=6000 APPFLOWY_DATABASE_MAX_CONNECTIONS=40 +APPFLOWY_DOCUMENT_CONTENT_SPLIT_LEN=8000 # This file is used to set the environment variables for local development # Copy this file to .env and change the values as needed @@ -88,6 +89,7 @@ APPFLOWY_S3_BUCKET=appflowy # Note that smtps (TLS) is always required, even for ports other than 465 APPFLOWY_MAILER_SMTP_HOST=smtp.gmail.com APPFLOWY_MAILER_SMTP_USERNAME=notify@appflowy.io +APPFLOWY_MAILER_SMTP_EMAIL=notify@appflowy.io APPFLOWY_MAILER_SMTP_PASSWORD=email_sender_password RUST_LOG=info @@ -111,13 +113,9 @@ APPFLOWY_AI_OPENAI_API_KEY= APPFLOWY_AI_SERVER_PORT=5001 APPFLOWY_AI_SERVER_HOST=localhost APPFLOWY_AI_DATABASE_URL=postgresql+psycopg://postgres:password@postgres:5432/postgres +APPFLOWY_AI_REDIS_URL=redis://redis:6379 APPFLOWY_LOCAL_AI_TEST_ENABLED=false -# AppFlowy History -APPFLOWY_GRPC_HISTORY_ADDRS=http://localhost:50051 -APPFLOWY_HISTORY_REDIS_URL=redis://redis:6379 -APPFLOWY_HISTORY_DATABASE_URL=postgres://postgres:password@postgres:5432/postgres - # AppFlowy Indexer APPFLOWY_INDEXER_ENABLED=true APPFLOWY_INDEXER_DATABASE_URL=postgres://postgres:password@postgres:5432/postgres diff --git a/docker-compose-ci.yml b/docker-compose-ci.yml index 498a7076f..b56851b98 100644 --- a/docker-compose-ci.yml +++ b/docker-compose-ci.yml @@ -120,6 +120,7 @@ services: dockerfile: Dockerfile args: FEATURES: "" + PROFILE: ci image: appflowyinc/appflowy_cloud:${APPFLOWY_CLOUD_VERSION:-latest} admin_frontend: @@ -138,7 +139,7 @@ services: ai: restart: on-failure - image: appflowyinc/appflowy_ai:${APPFLOWY_AI_VERSION:-latest} + image: appflowyinc/appflowy_ai_premium:${APPFLOWY_AI_VERSION:-latest} ports: - "5001:5001" environment: @@ -147,20 +148,7 @@ services: - LOCAL_AI_AWS_SECRET_ACCESS_KEY=${LOCAL_AI_AWS_SECRET_ACCESS_KEY} - APPFLOWY_AI_SERVER_PORT=${APPFLOWY_AI_SERVER_PORT} - APPFLOWY_AI_DATABASE_URL=${APPFLOWY_AI_DATABASE_URL} - - appflowy_history: - restart: on-failure - build: - context: . - dockerfile: ./services/appflowy-history/Dockerfile - image: appflowyinc/appflowy_history:${APPFLOWY_HISTORY_VERSION:-latest} - ports: - - "50051:50051" - environment: - - RUST_LOG=${RUST_LOG:-info} - - APPFLOWY_HISTORY_REDIS_URL=redis://redis:6379 - - APPFLOWY_HISTORY_ENVIRONMENT=production - - APPFLOWY_HISTORY_DATABASE_URL=${APPFLOWY_HISTORY_DATABASE_URL} + - APPFLOWY_AI_REDIS_URL=${APPFLOWY_AI_REDIS_URL} appflowy_worker: restart: on-failure diff --git a/docker-compose-dev.yml b/docker-compose-dev.yml index 33dd18f41..1f314aae4 100644 --- a/docker-compose-dev.yml +++ b/docker-compose-dev.yml @@ -122,6 +122,7 @@ services: - OPENAI_API_KEY=${APPFLOWY_AI_OPENAI_API_KEY} - APPFLOWY_AI_SERVER_PORT=${APPFLOWY_AI_SERVER_PORT} - APPFLOWY_AI_DATABASE_URL=${APPFLOWY_AI_DATABASE_URL} + - APPFLOWY_AI_REDIS_URL=${APPFLOWY_AI_REDIS_URL} volumes: postgres_data: diff --git a/docker-compose.yml b/docker-compose.yml index 7bcab45e4..b39056cd8 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -140,18 +140,7 @@ services: - OPENAI_API_KEY=${APPFLOWY_AI_OPENAI_API_KEY} - APPFLOWY_AI_SERVER_PORT=${APPFLOWY_AI_SERVER_PORT} - APPFLOWY_AI_DATABASE_URL=${APPFLOWY_AI_DATABASE_URL} - - appflowy_history: - restart: on-failure - image: appflowyinc/appflowy_history:${APPFLOWY_HISTORY_VERSION:-latest} - build: - context: . - dockerfile: ./services/appflowy-history/Dockerfile - environment: - - RUST_LOG=${RUST_LOG:-info} - - APPFLOWY_HISTORY_REDIS_URL=redis://redis:6379 - - APPFLOWY_HISTORY_ENVIRONMENT=production - - APPFLOWY_HISTORY_DATABASE_URL=${APPFLOWY_HISTORY_DATABASE_URL} + - APPFLOWY_AI_REDIS_URL=${APPFLOWY_AI_REDIS_URL} appflowy_worker: restart: on-failure diff --git a/libs/access-control/src/casbin/access.rs b/libs/access-control/src/casbin/access.rs index 5c1b56276..6b42ecdac 100644 --- a/libs/access-control/src/casbin/access.rs +++ b/libs/access-control/src/casbin/access.rs @@ -1,7 +1,7 @@ use super::adapter::PgAdapter; -use super::enforcer::{AFEnforcer, NoEnforceGroup}; +use super::enforcer::AFEnforcer; use crate::act::{Action, ActionVariant, Acts}; -use crate::entity::ObjectType; +use crate::entity::{ObjectType, SubjectType}; use crate::metrics::{tick_metric, AccessControlMetrics}; use anyhow::anyhow; @@ -14,15 +14,8 @@ use database_entity::dto::{AFAccessLevel, AFRole}; use sqlx::PgPool; use std::sync::Arc; -use tokio::sync::broadcast; use tracing::trace; -#[derive(Debug, Clone)] -pub enum AccessControlChange { - UpdatePolicy { uid: i64, oid: String }, - RemovePolicy { uid: i64, oid: String }, -} - /// Manages access control. /// /// Stores access control policies in the form `subject, object, role` @@ -37,10 +30,9 @@ pub enum AccessControlChange { /// according to the model defined. #[derive(Clone)] pub struct AccessControl { - enforcer: Arc>, + enforcer: Arc, #[allow(dead_code)] access_control_metrics: Arc, - change_tx: broadcast::Sender, } impl AccessControl { @@ -55,41 +47,33 @@ impl AccessControl { })?; enforcer.add_function("cmpRoleOrLevel", OperatorFunction::Arg2(cmp_role_or_level)); - let enforcer = Arc::new(AFEnforcer::new(enforcer, NoEnforceGroup).await?); + let enforcer = Arc::new(AFEnforcer::new(enforcer).await?); tick_metric( enforcer.metrics_state.clone(), access_control_metrics.clone(), ); - let (change_tx, _) = broadcast::channel(1000); Ok(Self { enforcer, access_control_metrics, - change_tx, }) } - pub fn subscribe_change(&self) -> broadcast::Receiver { - self.change_tx.subscribe() - } - pub async fn update_policy( &self, - uid: &i64, + sub: SubjectType, obj: ObjectType<'_>, act: ActionVariant<'_>, ) -> Result<(), AppError> { - let access_control_change = self.enforcer.update_policy(uid, obj, act).await?; - if let Some(change) = access_control_change { - let _ = self.change_tx.send(change); - } + self.enforcer.update_policy(sub, obj, act).await?; Ok(()) } - pub async fn remove_policy(&self, uid: &i64, obj: &ObjectType<'_>) -> Result<(), AppError> { - let access_control_change = self.enforcer.remove_policy(uid, obj).await?; - if let Some(change) = access_control_change { - let _ = self.change_tx.send(change); - } + pub async fn remove_policy( + &self, + sub: &SubjectType, + obj: &ObjectType<'_>, + ) -> Result<(), AppError> { + self.enforcer.remove_policy(sub, obj).await?; Ok(()) } @@ -169,7 +153,7 @@ r = sub, obj, act p = sub, obj, act [role_definition] -g = _, _ # role and access level rule +g = _, _ # grouping rule [policy_effect] e = some(where (p.eft == allow)) diff --git a/libs/access-control/src/casbin/adapter.rs b/libs/access-control/src/casbin/adapter.rs index 629c80802..72026639f 100644 --- a/libs/access-control/src/casbin/adapter.rs +++ b/libs/access-control/src/casbin/adapter.rs @@ -7,8 +7,6 @@ use casbin::Filter; use casbin::Model; use casbin::Result; -use database::collab::select_collab_member_access_level; -use database::pg_row::AFCollabMemberAccessLevelRow; use database::pg_row::AFWorkspaceMemberPermRow; use database::workspace::select_workspace_member_perm_stream; @@ -35,28 +33,6 @@ impl PgAdapter { } } -async fn load_collab_policies( - mut stream: BoxStream<'_, sqlx::Result>, -) -> Result>> { - let mut policies: Vec> = Vec::new(); - - while let Some(Ok(member_access_lv)) = stream.next().await { - let uid = member_access_lv.uid; - let object_type = ObjectType::Collab(&member_access_lv.oid); - for act in member_access_lv.access_level.policy_acts() { - let policy = [ - uid.to_string(), - object_type.policy_object(), - act.to_string(), - ] - .to_vec(); - policies.push(policy); - } - } - - Ok(policies) -} - /// Loads workspace policies from a given stream of workspace member permissions. /// /// This function iterates over the stream of member permissions, constructing and accumulating @@ -128,12 +104,6 @@ impl Adapter for PgAdapter { // Policy definition `p` of type `p`. See `model.conf` model.add_policies("p", "p", workspace_policies); - let collab_member_access_lv_stream = select_collab_member_access_level(&self.pg_pool); - let collab_policies = load_collab_policies(collab_member_access_lv_stream).await?; - - // Policy definition `p` of type `p`. See `model.conf` - model.add_policies("p", "p", collab_policies); - self .access_control_metrics .record_load_all_policies_in_ms(start.elapsed().as_millis() as u64); diff --git a/libs/access-control/src/casbin/collab.rs b/libs/access-control/src/casbin/collab.rs index 7646599c4..f64289a46 100644 --- a/libs/access-control/src/casbin/collab.rs +++ b/libs/access-control/src/casbin/collab.rs @@ -6,7 +6,7 @@ use tracing::instrument; use crate::{ act::{Action, ActionVariant}, collab::{CollabAccessControl, RealtimeAccessControl}, - entity::ObjectType, + entity::{ObjectType, SubjectType}, }; use super::access::AccessControl; @@ -70,7 +70,7 @@ impl CollabAccessControl for CollabAccessControlImpl { self .access_control .update_policy( - uid, + SubjectType::User(*uid), ObjectType::Collab(oid), ActionVariant::FromAccessLevel(&level), ) @@ -83,7 +83,7 @@ impl CollabAccessControl for CollabAccessControlImpl { async fn remove_access_level(&self, uid: &i64, oid: &str) -> Result<(), AppError> { self .access_control - .remove_policy(uid, &ObjectType::Collab(oid)) + .remove_policy(&SubjectType::User(*uid), &ObjectType::Collab(oid)) .await?; Ok(()) } @@ -96,20 +96,6 @@ pub struct RealtimeCollabAccessControlImpl { impl RealtimeCollabAccessControlImpl { pub fn new(access_control: AccessControl) -> Self { - // let action_by_oid = Arc::new(DashMap::new()); - // let mut sub = access_control.subscribe_change(); - // let weak_action_by_oid = Arc::downgrade(&action_by_oid); - // tokio::spawn(async move { - // while let Ok(change) = sub.recv().await { - // match weak_action_by_oid.upgrade() { - // None => break, - // Some(action_by_oid) => match change { - // AccessControlChange::UpdatePolicy { uid, oid } => {}, - // AccessControlChange::RemovePolicy { uid, oid } => {}, - // }, - // } - // } - // }); Self { access_control } } diff --git a/libs/access-control/src/casbin/enforcer.rs b/libs/access-control/src/casbin/enforcer.rs index 8b9d92729..ef7b1c9e4 100644 --- a/libs/access-control/src/casbin/enforcer.rs +++ b/libs/access-control/src/casbin/enforcer.rs @@ -1,41 +1,26 @@ -use super::access::{ - load_group_policies, AccessControlChange, POLICY_FIELD_INDEX_OBJECT, POLICY_FIELD_INDEX_SUBJECT, -}; +use super::access::{load_group_policies, POLICY_FIELD_INDEX_OBJECT, POLICY_FIELD_INDEX_SUBJECT}; use crate::act::ActionVariant; -use crate::entity::ObjectType; +use crate::entity::{ObjectType, SubjectType}; use crate::metrics::MetricsCalState; -use crate::request::{GroupPolicyRequest, PolicyRequest, WorkspacePolicyRequest}; +use crate::request::{PolicyRequest, WorkspacePolicyRequest}; use anyhow::anyhow; use app_error::AppError; -use async_trait::async_trait; use casbin::{CoreApi, Enforcer, MgmtApi}; use std::sync::atomic::Ordering; use tokio::sync::RwLock; use tracing::{event, instrument, trace}; -#[async_trait] -pub trait EnforcerGroup { - /// Get the group id of the user. - /// User might belong to multiple groups. So return the highest permission group id. - async fn get_enforce_group_id(&self, uid: &i64) -> Option; -} - -pub struct AFEnforcer { +pub struct AFEnforcer { enforcer: RwLock, pub(crate) metrics_state: MetricsCalState, - enforce_group: T, } -impl AFEnforcer -where - T: EnforcerGroup, -{ - pub async fn new(mut enforcer: Enforcer, enforce_group: T) -> Result { +impl AFEnforcer { + pub async fn new(mut enforcer: Enforcer) -> Result { load_group_policies(&mut enforcer).await?; Ok(Self { enforcer: RwLock::new(enforcer), metrics_state: MetricsCalState::new(), - enforce_group, }) } @@ -47,18 +32,17 @@ where #[instrument(level = "debug", skip_all, err)] pub async fn update_policy( &self, - uid: &i64, + sub: SubjectType, obj: ObjectType<'_>, act: ActionVariant<'_>, - ) -> Result, AppError> { + ) -> Result<(), AppError> { validate_obj_action(&obj, &act)?; let policies = act .policy_acts() .into_iter() - .map(|act| vec![uid.to_string(), obj.policy_object(), act.to_string()]) + .map(|act| vec![sub.policy_subject(), obj.policy_object(), act.to_string()]) .collect::>>(); - let number_of_updated_policies = policies.len(); trace!("[access control]: add policy:{:?}", policies); self @@ -69,35 +53,40 @@ where .await .map_err(|e| AppError::Internal(anyhow!("fail to add policy: {e:?}")))?; - if number_of_updated_policies > 0 { - Ok(Some(AccessControlChange::UpdatePolicy { - uid: *uid, - oid: obj.object_id().to_string(), - })) - } else { - Ok(None) - } + Ok(()) } /// Returns policies that match the filter. pub async fn remove_policy( &self, - uid: &i64, + sub: &SubjectType, object_type: &ObjectType<'_>, - ) -> Result, AppError> { + ) -> Result<(), AppError> { let mut enforcer = self.enforcer.write().await; self - .remove_with_enforcer(uid, object_type, &mut enforcer) + .remove_with_enforcer(sub, object_type, &mut enforcer) .await } + /// Add a grouping policy. + #[allow(dead_code)] + pub async fn add_grouping_policy( + &self, + sub: &SubjectType, + group_sub: &SubjectType, + ) -> Result<(), AppError> { + let mut enforcer = self.enforcer.write().await; + enforcer + .add_grouping_policy(vec![sub.policy_subject(), group_sub.policy_subject()]) + .await + .map_err(|e| AppError::Internal(anyhow!("fail to add grouping policy: {e:?}")))?; + Ok(()) + } + /// 1. **Workspace Policy**: Initially, it checks if the user has permission at the workspace level. If the user /// has permission to perform the action on the workspace, the function returns `true` without further checks. /// - /// 2. **Group Policy**: (If applicable) If the workspace policy check fails (`false`), the function will then - /// evaluate group-level policies. - /// - /// 3. **Object-Specific Policy**: If both previous checks fail, the function finally evaluates the policy + /// 2. **Object-Specific Policy**: If workspace policy check fail, the function evaluates the policy /// specific to the object itself. /// /// ## Parameters: @@ -134,20 +123,7 @@ where .enforce(policy) .map_err(|e| AppError::Internal(anyhow!("enforce: {e:?}")))?; - // 2. Fallback to group policy if workspace-level check fails. - if !result { - if let Some(guid) = self.enforce_group.get_enforce_group_id(uid).await { - let policy_request = GroupPolicyRequest::new(&guid, &obj, &act); - result = self - .enforcer - .read() - .await - .enforce(policy_request.to_policy()) - .map_err(|e| AppError::Internal(anyhow!("enforce: {e:?}")))?; - } - } - - // 3. Finally, enforce object-specific policy if previous checks fail. + // 2. Finally, enforce object-specific policy if previous checks fail. if !result { let policy_request = PolicyRequest::new(*uid, &obj, &act); let policy = policy_request.to_policy(); @@ -162,29 +138,27 @@ where if result { Ok(()) } else { - Err(AppError::NotEnoughPermissions) + Err(AppError::NotEnoughPermissions { + user: uid.to_string(), + workspace_id: workspace_id.to_string(), + }) } } #[inline] async fn remove_with_enforcer( &self, - uid: &i64, + sub: &SubjectType, object_type: &ObjectType<'_>, enforcer: &mut Enforcer, - ) -> Result, AppError> { + ) -> Result<(), AppError> { let policies_for_user_on_object = - policies_for_subject_with_given_object(uid, object_type, enforcer).await; - - // if there are no policies for the user on the object, return early. - if policies_for_user_on_object.is_empty() { - return Ok(None); - } + policies_for_subject_with_given_object(sub, object_type, enforcer).await; event!( tracing::Level::INFO, - "[access control]: remove policy:user={}, object={}, policies={:?}", - uid, + "[access control]: remove policy:subject={}, object={}, policies={:?}", + sub.policy_subject(), object_type.policy_object(), policies_for_user_on_object ); @@ -194,10 +168,7 @@ where .await .map_err(|e| AppError::Internal(anyhow!("error enforce: {e:?}")))?; - Ok(Some(AccessControlChange::RemovePolicy { - uid: *uid, - oid: object_type.object_id().to_string(), - })) + Ok(()) } } @@ -213,111 +184,55 @@ fn validate_obj_action(obj: &ObjectType<'_>, act: &ActionVariant) -> Result<(), } } #[inline] -async fn policies_for_subject_with_given_object( - subject: T, +async fn policies_for_subject_with_given_object( + subject: &SubjectType, object_type: &ObjectType<'_>, enforcer: &Enforcer, ) -> Vec> { - let subject = subject.to_string(); + let subject_id = subject.policy_subject(); let object_type_id = object_type.policy_object(); let policies_related_to_object = enforcer.get_filtered_policy(POLICY_FIELD_INDEX_OBJECT, vec![object_type_id]); policies_related_to_object .into_iter() - .filter(|p| p[POLICY_FIELD_INDEX_SUBJECT] == subject) + .filter(|p| p[POLICY_FIELD_INDEX_SUBJECT] == subject_id) .collect::>() } -pub struct NoEnforceGroup; -#[async_trait] -impl EnforcerGroup for NoEnforceGroup { - async fn get_enforce_group_id(&self, _uid: &i64) -> Option { - None - } -} - #[cfg(test)] mod tests { use crate::{ act::{Action, ActionVariant}, - casbin::{ - access::{casbin_model, cmp_role_or_level}, - enforcer::NoEnforceGroup, - }, - entity::ObjectType, + casbin::access::{casbin_model, cmp_role_or_level}, + entity::{ObjectType, SubjectType}, }; use app_error::ErrorCode; - use async_trait::async_trait; use casbin::{function_map::OperatorFunction, prelude::*}; use database_entity::dto::{AFAccessLevel, AFRole}; - use super::{AFEnforcer, EnforcerGroup}; + use super::AFEnforcer; - pub struct TestEnforceGroup { - guid: String, - } - #[async_trait] - impl EnforcerGroup for TestEnforceGroup { - async fn get_enforce_group_id(&self, _uid: &i64) -> Option { - Some(self.guid.clone()) - } - } - - async fn test_enforcer(enforce_group: T) -> AFEnforcer - where - T: EnforcerGroup, - { + async fn test_enforcer() -> AFEnforcer { let model = casbin_model().await.unwrap(); let mut enforcer = casbin::Enforcer::new(model, MemoryAdapter::default()) .await .unwrap(); enforcer.add_function("cmpRoleOrLevel", OperatorFunction::Arg2(cmp_role_or_level)); - AFEnforcer::new(enforcer, enforce_group).await.unwrap() - } - #[tokio::test] - async fn collab_group_test() { - let enforcer = test_enforcer(NoEnforceGroup).await; - - let uid = 1; - let workspace_id = "w1"; - let object_1 = "o1"; - - // add user as a member of the collab - enforcer - .update_policy( - &uid, - ObjectType::Collab(object_1), - ActionVariant::FromAccessLevel(&AFAccessLevel::FullAccess), - ) - .await - .unwrap(); - - // when the user is the owner of the collab, then the user should have access to the collab - for action in [Action::Write, Action::Read] { - let result = enforcer - .enforce_policy( - workspace_id, - &uid, - ObjectType::Collab(object_1), - ActionVariant::FromAction(&action), - ) - .await; - assert!(result.is_ok()); - } + AFEnforcer::new(enforcer).await.unwrap() } #[tokio::test] async fn workspace_group_policy_test() { - let enforcer = test_enforcer(NoEnforceGroup).await; + let enforcer = test_enforcer().await; let uid = 1; let workspace_id = "w1"; // add user as a member of the workspace enforcer .update_policy( - &uid, + SubjectType::User(uid), ObjectType::Workspace(workspace_id), ActionVariant::FromRole(&AFRole::Member), ) @@ -340,7 +255,7 @@ mod tests { #[tokio::test] async fn workspace_owner_and_try_to_full_access_collab_test() { - let enforcer = test_enforcer(NoEnforceGroup).await; + let enforcer = test_enforcer().await; let uid = 1; let workspace_id = "w1"; @@ -349,7 +264,7 @@ mod tests { // add user as a member of the workspace enforcer .update_policy( - &uid, + SubjectType::User(uid), ObjectType::Workspace(workspace_id), ActionVariant::FromRole(&AFRole::Owner), ) @@ -371,7 +286,7 @@ mod tests { #[tokio::test] async fn workspace_member_collab_owner_try_to_full_access_collab_test() { - let enforcer = test_enforcer(NoEnforceGroup).await; + let enforcer = test_enforcer().await; let uid = 1; let workspace_id = "w1"; @@ -380,7 +295,7 @@ mod tests { // add user as a member of the workspace enforcer .update_policy( - &uid, + SubjectType::User(uid), ObjectType::Workspace(workspace_id), ActionVariant::FromRole(&AFRole::Member), ) @@ -389,7 +304,7 @@ mod tests { enforcer .update_policy( - &uid, + SubjectType::User(uid), ObjectType::Collab(object_1), ActionVariant::FromAccessLevel(&AFAccessLevel::FullAccess), ) @@ -411,7 +326,7 @@ mod tests { #[tokio::test] async fn workspace_owner_collab_member_try_to_full_access_collab_test() { - let enforcer = test_enforcer(NoEnforceGroup).await; + let enforcer = test_enforcer().await; let uid = 1; let workspace_id = "w1"; @@ -420,7 +335,7 @@ mod tests { // add user as a member of the workspace enforcer .update_policy( - &uid, + SubjectType::User(uid), ObjectType::Workspace(workspace_id), ActionVariant::FromRole(&AFRole::Owner), ) @@ -429,7 +344,7 @@ mod tests { enforcer .update_policy( - &uid, + SubjectType::User(uid), ObjectType::Collab(object_1), ActionVariant::FromAccessLevel(&AFAccessLevel::ReadAndWrite), ) @@ -451,7 +366,7 @@ mod tests { #[tokio::test] async fn workspace_member_collab_member_try_to_full_access_collab_test() { - let enforcer = test_enforcer(NoEnforceGroup).await; + let enforcer = test_enforcer().await; let uid = 1; let workspace_id = "w1"; @@ -460,7 +375,7 @@ mod tests { // add user as a member of the workspace enforcer .update_policy( - &uid, + SubjectType::User(uid), ObjectType::Workspace(workspace_id), ActionVariant::FromRole(&AFRole::Member), ) @@ -469,7 +384,7 @@ mod tests { enforcer .update_policy( - &uid, + SubjectType::User(uid), ObjectType::Collab(object_1), ActionVariant::FromAccessLevel(&AFAccessLevel::ReadAndWrite), ) @@ -503,7 +418,7 @@ mod tests { #[tokio::test] async fn workspace_member_but_not_collab_member_and_try_full_access_collab_test() { - let enforcer = test_enforcer(NoEnforceGroup).await; + let enforcer = test_enforcer().await; let uid = 1; let workspace_id = "w1"; @@ -512,7 +427,7 @@ mod tests { // add user as a member of the workspace enforcer .update_policy( - &uid, + SubjectType::User(uid), ObjectType::Workspace(workspace_id), ActionVariant::FromRole(&AFRole::Member), ) @@ -549,14 +464,14 @@ mod tests { #[tokio::test] async fn not_workspace_member_but_collab_owner_try_full_access_collab_test() { - let enforcer = test_enforcer(NoEnforceGroup).await; + let enforcer = test_enforcer().await; let uid = 1; let workspace_id = "w1"; let object_1 = "o1"; enforcer .update_policy( - &uid, + SubjectType::User(uid), ObjectType::Collab(object_1), ActionVariant::FromAccessLevel(&AFAccessLevel::FullAccess), ) @@ -578,7 +493,7 @@ mod tests { #[tokio::test] async fn not_workspace_member_not_collab_member_and_try_full_access_collab_test() { - let enforcer = test_enforcer(NoEnforceGroup).await; + let enforcer = test_enforcer().await; let uid = 1; let workspace_id = "w1"; let object_1 = "o1"; @@ -606,7 +521,7 @@ mod tests { #[tokio::test] async fn cmp_owner_role_test() { - let enforcer = test_enforcer(NoEnforceGroup).await; + let enforcer = test_enforcer().await; let uid = 1; let workspace_id = "w1"; let object_1 = "o1"; @@ -614,7 +529,7 @@ mod tests { // add user as a member of the workspace enforcer .update_policy( - &uid, + SubjectType::User(uid), ObjectType::Workspace(workspace_id), ActionVariant::FromRole(&AFRole::Owner), ) @@ -645,7 +560,7 @@ mod tests { #[tokio::test] async fn cmp_member_role_test() { - let enforcer = test_enforcer(NoEnforceGroup).await; + let enforcer = test_enforcer().await; let uid = 1; let workspace_id = "w1"; let object_1 = "o1"; @@ -653,7 +568,7 @@ mod tests { // add user as a member of the workspace enforcer .update_policy( - &uid, + SubjectType::User(uid), ObjectType::Workspace(workspace_id), ActionVariant::FromRole(&AFRole::Member), ) @@ -710,7 +625,7 @@ mod tests { #[tokio::test] async fn cmp_guest_role_test() { - let enforcer = test_enforcer(NoEnforceGroup).await; + let enforcer = test_enforcer().await; let uid = 1; let workspace_id = "w1"; let object_1 = "o1"; @@ -718,7 +633,7 @@ mod tests { // add user as a member of the workspace enforcer .update_policy( - &uid, + SubjectType::User(uid), ObjectType::Workspace(workspace_id), ActionVariant::FromRole(&AFRole::Guest), ) @@ -754,14 +669,14 @@ mod tests { #[tokio::test] async fn cmp_full_access_level_test() { - let enforcer = test_enforcer(NoEnforceGroup).await; + let enforcer = test_enforcer().await; let uid = 1; let workspace_id = "w1"; let object_1 = "o1"; enforcer .update_policy( - &uid, + SubjectType::User(uid), ObjectType::Collab(object_1), ActionVariant::FromAccessLevel(&AFAccessLevel::FullAccess), ) @@ -787,14 +702,14 @@ mod tests { #[tokio::test] async fn cmp_read_only_level_test() { - let enforcer = test_enforcer(NoEnforceGroup).await; + let enforcer = test_enforcer().await; let uid = 1; let workspace_id = "w1"; let object_1 = "o1"; enforcer .update_policy( - &uid, + SubjectType::User(uid), ObjectType::Collab(object_1), ActionVariant::FromAccessLevel(&AFAccessLevel::ReadOnly), ) diff --git a/libs/access-control/src/casbin/mod.rs b/libs/access-control/src/casbin/mod.rs index 22e3bc5e1..af149de31 100644 --- a/libs/access-control/src/casbin/mod.rs +++ b/libs/access-control/src/casbin/mod.rs @@ -2,5 +2,4 @@ pub mod access; mod adapter; pub mod collab; mod enforcer; -pub mod notification; pub mod workspace; diff --git a/libs/access-control/src/casbin/notification.rs b/libs/access-control/src/casbin/notification.rs deleted file mode 100644 index 1f87fa83e..000000000 --- a/libs/access-control/src/casbin/notification.rs +++ /dev/null @@ -1,80 +0,0 @@ -use super::access::AccessControl; -use crate::act::ActionVariant; -use crate::entity::ObjectType; -use database_entity::dto::AFRole; -use serde::Deserialize; -use tokio::sync::broadcast; -use tracing::error; -use tracing::log::warn; -use uuid::Uuid; - -pub fn spawn_listen_on_workspace_member_change( - mut listener: broadcast::Receiver, - access_control: AccessControl, -) { - tokio::spawn(async move { - while let Ok(change) = listener.recv().await { - match change.action_type { - WorkspaceMemberAction::INSERT | WorkspaceMemberAction::UPDATE => match change.new { - None => { - warn!("The workspace member change can't be None when the action is INSERT or UPDATE") - }, - Some(member_row) => { - if let Err(err) = access_control - .update_policy( - &member_row.uid, - ObjectType::Workspace(&member_row.workspace_id.to_string()), - ActionVariant::FromRole(&AFRole::from(member_row.role_id as i32)), - ) - .await - { - error!( - "Failed to update the user:{} workspace:{} access control, error: {}", - member_row.uid, member_row.workspace_id, err - ); - } - }, - }, - WorkspaceMemberAction::DELETE => match change.old { - None => warn!("The workspace member change can't be None when the action is DELETE"), - Some(member_row) => { - if let Err(err) = access_control - .remove_policy( - &member_row.uid, - &ObjectType::Workspace(&member_row.workspace_id.to_string()), - ) - .await - { - error!( - "Failed to remove the user:{} workspace: {} access control, error: {}", - member_row.uid, member_row.workspace_id, err - ); - } - }, - }, - } - } - }); -} - -#[allow(clippy::upper_case_acronyms)] -#[derive(Deserialize, Clone, Debug)] -pub enum WorkspaceMemberAction { - INSERT, - UPDATE, - DELETE, -} - -#[derive(Deserialize, Debug, Clone)] -pub struct WorkspaceMemberNotification { - pub old: Option, - pub new: Option, - pub action_type: WorkspaceMemberAction, -} - -#[derive(Deserialize, Debug, Clone)] -pub struct WorkspaceMemberRow { - pub uid: i64, - pub role_id: i64, - pub workspace_id: Uuid, -} diff --git a/libs/access-control/src/casbin/workspace.rs b/libs/access-control/src/casbin/workspace.rs index 882105fe7..d15557910 100644 --- a/libs/access-control/src/casbin/workspace.rs +++ b/libs/access-control/src/casbin/workspace.rs @@ -4,7 +4,7 @@ use uuid::Uuid; use super::access::AccessControl; use crate::act::{Action, ActionVariant}; -use crate::entity::ObjectType; +use crate::entity::{ObjectType, SubjectType}; use crate::workspace::WorkspaceAccessControl; use app_error::AppError; use database_entity::dto::AFRole; @@ -66,7 +66,7 @@ impl WorkspaceAccessControl for WorkspaceAccessControlImpl { self .access_control .update_policy( - uid, + SubjectType::User(*uid), ObjectType::Workspace(&workspace_id.to_string()), ActionVariant::FromRole(&role), ) @@ -82,12 +82,18 @@ impl WorkspaceAccessControl for WorkspaceAccessControlImpl { ) -> Result<(), AppError> { self .access_control - .remove_policy(uid, &ObjectType::Workspace(&workspace_id.to_string())) + .remove_policy( + &SubjectType::User(*uid), + &ObjectType::Workspace(&workspace_id.to_string()), + ) .await?; self .access_control - .remove_policy(uid, &ObjectType::Collab(&workspace_id.to_string())) + .remove_policy( + &SubjectType::User(*uid), + &ObjectType::Collab(&workspace_id.to_string()), + ) .await?; Ok(()) } diff --git a/libs/access-control/src/entity.rs b/libs/access-control/src/entity.rs index 1bbfa3822..f020a4714 100644 --- a/libs/access-control/src/entity.rs +++ b/libs/access-control/src/entity.rs @@ -1,3 +1,18 @@ +#[derive(Debug)] +pub enum SubjectType { + User(i64), + Group(String), +} + +impl SubjectType { + pub fn policy_subject(&self) -> String { + match self { + SubjectType::User(i) => i.to_string(), + SubjectType::Group(s) => s.clone(), + } + } +} + /// Represents the object type that is stored in the access control policy. #[derive(Debug)] pub enum ObjectType<'id> { diff --git a/libs/access-control/src/request.rs b/libs/access-control/src/request.rs index c4d0f522c..7ec329f08 100644 --- a/libs/access-control/src/request.rs +++ b/libs/access-control/src/request.rs @@ -1,33 +1,6 @@ use crate::act::ActionVariant; use crate::entity::ObjectType; -pub struct GroupPolicyRequest<'a> { - pub guid: &'a str, - pub object_type: &'a ObjectType<'a>, - pub action: &'a ActionVariant<'a>, -} - -impl GroupPolicyRequest<'_> { - pub fn new<'a>( - guid: &'a str, - object_type: &'a ObjectType<'a>, - action: &'a ActionVariant<'a>, - ) -> GroupPolicyRequest<'a> { - GroupPolicyRequest { - guid, - object_type, - action, - } - } - pub fn to_policy(&self) -> Vec { - vec![ - self.guid.to_string(), - self.object_type.policy_object(), - self.action.to_enforce_act().to_string(), - ] - } -} - pub struct WorkspacePolicyRequest<'a> { workspace_id: &'a str, uid: &'a i64, diff --git a/libs/app-error/src/lib.rs b/libs/app-error/src/lib.rs index 5ee189979..7aa803585 100644 --- a/libs/app-error/src/lib.rs +++ b/libs/app-error/src/lib.rs @@ -3,7 +3,6 @@ pub mod gotrue; #[cfg(feature = "gotrue_error")] use crate::gotrue::GoTrueError; -use std::error::Error as StdError; use std::string::FromUtf8Error; #[cfg(feature = "appflowy_ai_error")] @@ -64,8 +63,10 @@ pub enum AppError { #[error("Not Logged In:{0}")] NotLoggedIn(String), - #[error("User does not have permissions to execute this action")] - NotEnoughPermissions, + #[error( + "User:{user} does not have permissions to execute this action in workspace:{workspace_id}" + )] + NotEnoughPermissions { user: String, workspace_id: String }, #[error("s3 response error:{0}")] S3ResponseError(String), @@ -90,7 +91,7 @@ pub enum AppError { #[error("{desc}: {err}")] SqlxArgEncodingError { desc: String, - err: Box, + err: Box, }, #[cfg(feature = "validation_error")] @@ -173,6 +174,9 @@ pub enum AppError { #[error("There is an invalid character in the publish namespace: {character}")] CustomNamespaceInvalidCharacter { character: char }, + + #[error("{0}")] + ServiceTemporaryUnavailable(String), } impl AppError { @@ -249,6 +253,7 @@ impl AppError { AppError::CustomNamespaceInvalidCharacter { .. } => { ErrorCode::CustomNamespaceInvalidCharacter }, + AppError::ServiceTemporaryUnavailable(_) => ErrorCode::ServiceTemporaryUnavailable, } } } @@ -388,6 +393,7 @@ pub enum ErrorCode { PublishNameInvalidCharacter = 1051, PublishNameTooLong = 1052, CustomNamespaceInvalidCharacter = 1053, + ServiceTemporaryUnavailable = 1054, } impl ErrorCode { diff --git a/libs/appflowy-ai-client/src/client.rs b/libs/appflowy-ai-client/src/client.rs index 7c57425de..e90c008e1 100644 --- a/libs/appflowy-ai-client/src/client.rs +++ b/libs/appflowy-ai-client/src/client.rs @@ -1,8 +1,9 @@ use crate::dto::{ - AIModel, ChatAnswer, ChatQuestion, CompleteTextResponse, CompletionType, CreateChatContext, - CustomPrompt, Document, EmbeddingRequest, EmbeddingResponse, LocalAIConfig, MessageData, - RepeatedLocalAIPackage, RepeatedRelatedQuestion, SearchDocumentsRequest, SummarizeRowResponse, - TranslateRowData, TranslateRowResponse, + AIModel, CalculateSimilarityParams, ChatAnswer, ChatQuestion, CompleteTextResponse, + CompletionType, CreateChatContext, CustomPrompt, Document, EmbeddingRequest, EmbeddingResponse, + LocalAIConfig, MessageData, RepeatedLocalAIPackage, RepeatedRelatedQuestion, + SearchDocumentsRequest, SimilarityResponse, SummarizeRowResponse, TranslateRowData, + TranslateRowResponse, }; use crate::error::AIError; @@ -202,6 +203,7 @@ impl AppFlowyAIClient { pub async fn send_question( &self, chat_id: &str, + question_id: i64, content: &str, model: &AIModel, metadata: Option, @@ -211,6 +213,8 @@ impl AppFlowyAIClient { data: MessageData { content: content.to_string(), metadata, + rag_ids: vec![], + message_id: Some(question_id.to_string()), }, }; let url = format!("{}/chat/message", self.url); @@ -230,6 +234,7 @@ impl AppFlowyAIClient { chat_id: &str, content: &str, metadata: Option, + rag_ids: Vec, model: &AIModel, ) -> Result>, AIError> { let json = ChatQuestion { @@ -237,6 +242,8 @@ impl AppFlowyAIClient { data: MessageData { content: content.to_string(), metadata, + rag_ids, + message_id: None, }, }; let url = format!("{}/chat/message/stream", self.url); @@ -253,8 +260,10 @@ impl AppFlowyAIClient { pub async fn stream_question_v2( &self, chat_id: &str, + question_id: i64, content: &str, metadata: Option, + rag_ids: Vec, model: &AIModel, ) -> Result>, AIError> { let json = ChatQuestion { @@ -262,6 +271,8 @@ impl AppFlowyAIClient { data: MessageData { content: content.to_string(), metadata, + rag_ids, + message_id: Some(question_id.to_string()), }, }; let url = format!("{}/v2/chat/message/stream", self.url); @@ -323,6 +334,21 @@ impl AppFlowyAIClient { .into_data() } + pub async fn calculate_similarity( + &self, + params: CalculateSimilarityParams, + ) -> Result { + let url = format!("{}/similarity", self.url); + let resp = self + .http_client(Method::POST, &url)? + .json(¶ms) + .send() + .await?; + AIResponse::::from_response(resp) + .await? + .into_data() + } + fn http_client(&self, method: Method, url: &str) -> Result { let request_builder = self.client.request(method, url); Ok(request_builder) diff --git a/libs/appflowy-ai-client/src/dto.rs b/libs/appflowy-ai-client/src/dto.rs index 19b1dbbcd..eed948546 100644 --- a/libs/appflowy-ai-client/src/dto.rs +++ b/libs/appflowy-ai-client/src/dto.rs @@ -23,6 +23,10 @@ pub struct MessageData { pub content: String, #[serde(skip_serializing_if = "Option::is_none")] pub metadata: Option, + #[serde(default)] + pub rag_ids: Vec, + #[serde(default)] + pub message_id: Option, } #[derive(Clone, Debug, Serialize, Deserialize)] @@ -182,7 +186,7 @@ pub struct EmbeddingRequest { } #[derive(Serialize, Deserialize, Debug, Clone, PartialEq)] -pub enum EmbeddingsModel { +pub enum EmbeddingModel { #[serde(rename = "text-embedding-3-small")] TextEmbedding3Small, #[serde(rename = "text-embedding-3-large")] @@ -191,12 +195,55 @@ pub enum EmbeddingsModel { TextEmbeddingAda002, } -impl Display for EmbeddingsModel { +impl EmbeddingModel { + pub fn supported_models() -> &'static [&'static str] { + &[ + "text-embedding-ada-002", + "text-embedding-3-small", + "text-embedding-3-large", + ] + } + + pub fn max_token(&self) -> usize { + match self { + EmbeddingModel::TextEmbeddingAda002 => 8191, + EmbeddingModel::TextEmbedding3Large => 8191, + EmbeddingModel::TextEmbedding3Small => 8191, + } + } + + pub fn default_dimensions(&self) -> i32 { + match self { + EmbeddingModel::TextEmbeddingAda002 => 1536, + EmbeddingModel::TextEmbedding3Large => 3072, + EmbeddingModel::TextEmbedding3Small => 1536, + } + } + + pub fn name(&self) -> &'static str { + match self { + EmbeddingModel::TextEmbeddingAda002 => "text-embedding-ada-002", + EmbeddingModel::TextEmbedding3Large => "text-embedding-3-large", + EmbeddingModel::TextEmbedding3Small => "text-embedding-3-small", + } + } + + pub fn from_name(name: &str) -> Option { + match name { + "text-embedding-ada-002" => Some(EmbeddingModel::TextEmbeddingAda002), + "text-embedding-3-large" => Some(EmbeddingModel::TextEmbedding3Large), + "text-embedding-3-small" => Some(EmbeddingModel::TextEmbedding3Small), + _ => None, + } + } +} + +impl Display for EmbeddingModel { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { match self { - EmbeddingsModel::TextEmbedding3Small => write!(f, "text-embedding-3-small"), - EmbeddingsModel::TextEmbedding3Large => write!(f, "text-embedding-3-large"), - EmbeddingsModel::TextEmbeddingAda002 => write!(f, "text-embedding-ada-002"), + EmbeddingModel::TextEmbedding3Small => write!(f, "text-embedding-3-small"), + EmbeddingModel::TextEmbedding3Large => write!(f, "text-embedding-3-large"), + EmbeddingModel::TextEmbeddingAda002 => write!(f, "text-embedding-ada-002"), } } } @@ -320,3 +367,15 @@ pub struct CustomPrompt { pub system: String, pub user: Option, } + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct CalculateSimilarityParams { + pub workspace_id: String, + pub input: String, + pub expected: String, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct SimilarityResponse { + pub score: f64, +} diff --git a/libs/appflowy-ai-client/tests/chat_test/context_test.rs b/libs/appflowy-ai-client/tests/chat_test/context_test.rs index d9a265725..79cfceb0f 100644 --- a/libs/appflowy-ai-client/tests/chat_test/context_test.rs +++ b/libs/appflowy-ai-client/tests/chat_test/context_test.rs @@ -14,7 +14,7 @@ async fn create_chat_context_test() { }; client.create_chat_text_context(context).await.unwrap(); let resp = client - .send_question(&chat_id, "Where I live?", &AIModel::GPT4oMini, None) + .send_question(&chat_id, 1, "Where I live?", &AIModel::GPT4oMini, None) .await .unwrap(); // response will be something like: diff --git a/libs/appflowy-ai-client/tests/chat_test/embedding_test.rs b/libs/appflowy-ai-client/tests/chat_test/embedding_test.rs index 1536a58b6..20f9aaaf7 100644 --- a/libs/appflowy-ai-client/tests/chat_test/embedding_test.rs +++ b/libs/appflowy-ai-client/tests/chat_test/embedding_test.rs @@ -1,7 +1,7 @@ use crate::appflowy_ai_client; use appflowy_ai_client::dto::{ - EmbeddingEncodingFormat, EmbeddingInput, EmbeddingRequest, EmbeddingsModel, + EmbeddingEncodingFormat, EmbeddingInput, EmbeddingModel, EmbeddingRequest, }; #[tokio::test] @@ -9,10 +9,10 @@ async fn embedding_test() { let client = appflowy_ai_client(); let request = EmbeddingRequest { input: EmbeddingInput::String("hello world".to_string()), - model: EmbeddingsModel::TextEmbedding3Small.to_string(), + model: EmbeddingModel::TextEmbedding3Small.to_string(), chunk_size: 1000, encoding_format: EmbeddingEncodingFormat::Float, - dimensions: 1536, + dimensions: EmbeddingModel::TextEmbedding3Small.default_dimensions(), }; let result = client.embeddings(request).await.unwrap(); assert!(result.total_tokens > 0); diff --git a/libs/appflowy-ai-client/tests/chat_test/qa_test.rs b/libs/appflowy-ai-client/tests/chat_test/qa_test.rs index f0f7fabf1..2aac663ae 100644 --- a/libs/appflowy-ai-client/tests/chat_test/qa_test.rs +++ b/libs/appflowy-ai-client/tests/chat_test/qa_test.rs @@ -11,7 +11,7 @@ async fn qa_test() { client.health_check().await.unwrap(); let chat_id = uuid::Uuid::new_v4().to_string(); let resp = client - .send_question(&chat_id, "I feel hungry", &AIModel::GPT4o, None) + .send_question(&chat_id, 1, "I feel hungry", &AIModel::GPT4o, None) .await .unwrap(); assert!(!resp.content.is_empty()); @@ -30,7 +30,7 @@ async fn stop_stream_test() { client.health_check().await.unwrap(); let chat_id = uuid::Uuid::new_v4().to_string(); let mut stream = client - .stream_question(&chat_id, "I feel hungry", None, &AIModel::GPT4oMini) + .stream_question(&chat_id, "I feel hungry", None, vec![], &AIModel::GPT4oMini) .await .unwrap(); @@ -52,7 +52,14 @@ async fn stream_test() { client.health_check().await.expect("Health check failed"); let chat_id = uuid::Uuid::new_v4().to_string(); let stream = client - .stream_question_v2(&chat_id, "I feel hungry", None, &AIModel::GPT4oMini) + .stream_question_v2( + &chat_id, + 1, + "I feel hungry", + None, + vec![], + &AIModel::GPT4oMini, + ) .await .expect("Failed to initiate question stream"); diff --git a/libs/client-api-test/src/test_client.rs b/libs/client-api-test/src/test_client.rs index 4149cc9ee..1970d2a65 100644 --- a/libs/client-api-test/src/test_client.rs +++ b/libs/client-api-test/src/test_client.rs @@ -31,7 +31,10 @@ use uuid::Uuid; #[cfg(feature = "collab-sync")] use client_api::collab_sync::{SinkConfig, SyncObject, SyncPlugin}; use client_api::entity::id::user_awareness_object_id; -use client_api::entity::{PublishCollabItem, PublishCollabMetadata, QueryWorkspaceMember}; +use client_api::entity::{ + PublishCollabItem, PublishCollabMetadata, QueryWorkspaceMember, QuestionStream, + QuestionStreamValue, +}; use client_api::ws::{WSClient, WSClientConfig}; use database_entity::dto::{ AFAccessLevel, AFRole, AFSnapshotMeta, AFSnapshotMetas, AFUserProfile, AFUserWorkspaceInfo, @@ -845,24 +848,21 @@ impl TestClient { #[allow(unused_variables)] pub async fn create_collab_with_data( &mut self, - object_id: String, workspace_id: &str, + object_id: &str, collab_type: CollabType, - encoded_collab_v1: Option, + encoded_collab_v1: EncodedCollab, ) -> Result<(), AppResponseError> { // Subscribe to object let origin = CollabOrigin::Client(CollabClient::new(self.uid().await, self.device_id.clone())); - let collab = match encoded_collab_v1 { - None => Collab::new_with_origin(origin.clone(), &object_id, vec![], false), - Some(data) => Collab::new_with_source( - origin.clone(), - &object_id, - DataSource::DocStateV1(data.doc_state.to_vec()), - vec![], - false, - ) - .unwrap(), - }; + let collab = Collab::new_with_source( + origin.clone(), + object_id, + DataSource::DocStateV1(encoded_collab_v1.doc_state.to_vec()), + vec![], + false, + ) + .unwrap(); let encoded_collab_v1 = collab .encode_collab_v1(|collab| collab_type.validate_require_data(collab)) @@ -873,7 +873,7 @@ impl TestClient { self .api_client .create_collab(CreateCollabParams { - object_id: object_id.clone(), + object_id: object_id.to_string(), encoded_collab_v1, collab_type: collab_type.clone(), workspace_id: workspace_id.to_string(), @@ -1167,3 +1167,16 @@ pub async fn get_collab_json_from_server( .unwrap() .to_json_value() } + +pub async fn collect_answer(mut stream: QuestionStream) -> String { + let mut answer = String::new(); + while let Some(value) = stream.next().await { + match value.unwrap() { + QuestionStreamValue::Answer { value } => { + answer.push_str(&value); + }, + QuestionStreamValue::Metadata { .. } => {}, + } + } + answer +} diff --git a/libs/client-api-wasm/.gitignore b/libs/client-api-wasm/.gitignore deleted file mode 100644 index 4e301317e..000000000 --- a/libs/client-api-wasm/.gitignore +++ /dev/null @@ -1,6 +0,0 @@ -/target -**/*.rs.bk -Cargo.lock -bin/ -pkg/ -wasm-pack.log diff --git a/libs/client-api-wasm/Cargo.toml b/libs/client-api-wasm/Cargo.toml deleted file mode 100644 index 0a7913bf3..000000000 --- a/libs/client-api-wasm/Cargo.toml +++ /dev/null @@ -1,39 +0,0 @@ -[package] -name = "client-api-wasm" -version = "0.1.0" -authors = ["Admin"] -edition = "2018" - -[lib] -crate-type = ["cdylib", "rlib"] - -[dependencies] -wasm-bindgen = "0.2.90" - -# The `console_error_panic_hook` crate provides better debugging of panics by -# logging them with `console.error`. This is great for development, but requires -# all the `std::fmt` and `std::panicking` infrastructure, so isn't great for -# code size when deploying. -console_error_panic_hook = { version = "0.1.7" } -serde.workspace = true -serde_json.workspace = true -client-api = { path = "../client-api" } -lazy_static = "1.4.0" -wasm-bindgen-futures = "0.4.40" -tsify = "0.4.5" -tracing.workspace = true -bytes.workspace = true -tracing-core = { version = "0.1.32" } -tracing-wasm = "0.2.1" -uuid.workspace = true -database-entity.workspace = true -collab-rt-entity.workspace = true -collab-entity.workspace = true -serde_repr = "0.1.19" -wee_alloc = { version = "0.4.5" } -serde-wasm-bindgen = "0.6.5" -[dev-dependencies] -wasm-bindgen-test = "0.3.34" - -[features] -default = [] diff --git a/libs/client-api-wasm/README.md b/libs/client-api-wasm/README.md deleted file mode 100644 index 30cb5dcfc..000000000 --- a/libs/client-api-wasm/README.md +++ /dev/null @@ -1,65 +0,0 @@ -
- -

Client API WASM

- - Client-API to WebAssembly Compiler - -
- -## 🚴 Usage - -### 🐑 Prepare - -```bash -# Clone the repository (if you haven't already) -git clone https://github.com/AppFlowy-IO/AppFlowy-Cloud.git - -# Navigate to the client-for-wasm directory -cd libs/client-api-wasm - -# Install the dependencies (if you haven't already) -cargo install wasm-pack -``` - -### 🛠️ Build with `wasm-pack build` - -``` -wasm-pack build -``` - -### 🔬 Test in Headless Browsers with `wasm-pack test` - -```bash -# Ensure you have geckodriver installed -wasm-pack test --headless --firefox - -# or -# Ensure you have chromedriver installed -# https://googlechromelabs.github.io/chrome-for-testing/ -# Example (Linux): -# 1. wget https://storage.googleapis.com/chrome-for-testing-public/123.0.6312.86/linux64/chromedriver-linux64.zip -# 2. unzip chromedriver-linux64.zip -# 3. sudo mv chromedriver /usr/local/bin -# 4. chromedriver -v -# If you see the version, then you have successfully installed chromedriver -# Note: the version of chromedriver should match the version of chrome installed on your system -wasm-pack test --headless --chrome -``` - -### 🎁 Publish to NPM with ~~`wasm-pack publish`~~ - -##### Don't publish in local development, only publish in github actions - -``` -wasm-pack publish -``` - -### 📦 Use your package as a dependency - -``` -npm install --save @appflowy/client-api-for-wasm -``` - -### 📝 How to use the package in development? - -See the [README.md](https://github.com/AppFlowy-IO/AppFlowy/tree/main/frontend/appflowy_web_app/README.md) in the AppFlowy Repository. diff --git a/libs/client-api-wasm/src/entities.rs b/libs/client-api-wasm/src/entities.rs deleted file mode 100644 index c5e09b88f..000000000 --- a/libs/client-api-wasm/src/entities.rs +++ /dev/null @@ -1,279 +0,0 @@ -use client_api::entity::{AFUserProfile, AuthProvider}; -use client_api::error::{AppResponseError, ErrorCode}; -use collab_entity::{CollabType, EncodedCollab}; -use database_entity::dto::{ - AFUserWorkspaceInfo, AFWorkspace, BatchQueryCollabResult, QueryCollab, QueryCollabParams, - QueryCollabResult, -}; -use serde::{Deserialize, Serialize}; -use serde_repr::{Deserialize_repr, Serialize_repr}; -use std::collections::HashMap; -use tsify::Tsify; -use wasm_bindgen::JsValue; - -macro_rules! from_struct_for_jsvalue { - ($type:ty) => { - impl From<$type> for JsValue { - fn from(value: $type) -> Self { - match serde_wasm_bindgen::to_value(&value) { - Ok(js_value) => js_value, - Err(err) => { - tracing::error!("Failed to convert User to JsValue: {:?}", err); - JsValue::NULL - }, - } - } - } - }; -} - -#[derive(Tsify, Serialize, Deserialize, Default, Debug)] -#[tsify(into_wasm_abi, from_wasm_abi)] -pub struct Configuration { - pub compression_quality: u32, - pub compression_buffer_size: usize, -} - -#[derive(Tsify, Serialize, Deserialize, Default, Debug)] -#[tsify(into_wasm_abi, from_wasm_abi)] -pub struct ClientAPIConfig { - pub base_url: String, - pub ws_addr: String, - pub gotrue_url: String, - pub device_id: String, - pub configuration: Option, - pub client_id: String, -} - -#[derive(Tsify, Serialize, Deserialize, Default, Debug)] -#[tsify(into_wasm_abi, from_wasm_abi)] -pub struct ClientResponse { - pub code: ErrorCode, - pub message: String, -} - -from_struct_for_jsvalue!(ClientResponse); -impl From for ClientResponse { - fn from(err: AppResponseError) -> Self { - ClientResponse { - code: err.code, - message: err.message.to_string(), - } - } -} - -#[derive(Tsify, Serialize, Deserialize)] -#[tsify(into_wasm_abi, from_wasm_abi)] -pub struct User { - pub uid: String, - pub uuid: String, - pub email: Option, - pub name: Option, - pub latest_workspace_id: String, - pub icon_url: Option, -} - -from_struct_for_jsvalue!(User); -impl From for User { - fn from(profile: AFUserProfile) -> Self { - User { - uid: profile.uid.to_string(), - uuid: profile.uuid.to_string(), - email: profile.email, - name: profile.name, - latest_workspace_id: profile.latest_workspace_id.to_string(), - icon_url: None, - } - } -} - -#[derive(Tsify, Serialize, Deserialize)] -#[tsify(into_wasm_abi, from_wasm_abi)] -pub struct UserWorkspace { - pub user: User, - pub visiting_workspace_id: String, - pub workspaces: Vec, -} - -from_struct_for_jsvalue!(UserWorkspace); - -impl From for UserWorkspace { - fn from(info: AFUserWorkspaceInfo) -> Self { - UserWorkspace { - user: User::from(info.user_profile), - visiting_workspace_id: info.visiting_workspace.workspace_id.to_string(), - workspaces: info.workspaces.into_iter().map(Workspace::from).collect(), - } - } -} - -#[derive(Tsify, Serialize, Deserialize)] -#[tsify(into_wasm_abi, from_wasm_abi)] -pub struct Workspace { - pub workspace_id: String, - pub database_storage_id: String, - pub owner_uid: String, - pub owner_name: String, - pub workspace_type: i32, - pub workspace_name: String, - pub created_at: String, - pub icon: String, -} - -from_struct_for_jsvalue!(Workspace); - -impl From for Workspace { - fn from(workspace: AFWorkspace) -> Self { - Workspace { - workspace_id: workspace.workspace_id.to_string(), - database_storage_id: workspace.database_storage_id.to_string(), - owner_uid: workspace.owner_uid.to_string(), - owner_name: workspace.owner_name, - workspace_type: workspace.workspace_type, - workspace_name: workspace.workspace_name, - created_at: workspace.created_at.timestamp().to_string(), - icon: workspace.icon, - } - } -} - -#[derive(Tsify, Serialize, Deserialize, Default, Debug)] -#[tsify(into_wasm_abi, from_wasm_abi)] -pub struct ClientQueryCollabParams { - pub workspace_id: String, - pub object_id: String, - #[tsify(type = "0 | 1 | 2 | 3 | 4 | 5")] - pub collab_type: i32, -} - -impl From for QueryCollabParams { - fn from(value: ClientQueryCollabParams) -> QueryCollabParams { - QueryCollabParams { - workspace_id: value.workspace_id, - inner: QueryCollab { - collab_type: CollabType::from(value.collab_type), - object_id: value.object_id, - }, - } - } -} - -#[derive(Tsify, Serialize, Deserialize, Default)] -#[tsify(into_wasm_abi, from_wasm_abi)] -pub struct ClientEncodeCollab { - pub state_vector: Vec, - pub doc_state: Vec, - #[serde(default)] - pub version: ClientEncoderVersion, -} - -#[derive(Tsify, Default, Serialize_repr, Deserialize_repr)] -#[repr(u8)] -pub enum ClientEncoderVersion { - #[default] - V1 = 0, - V2 = 1, -} - -from_struct_for_jsvalue!(ClientEncodeCollab); - -impl From for ClientEncodeCollab { - fn from(collab: EncodedCollab) -> Self { - ClientEncodeCollab { - state_vector: collab.state_vector.to_vec(), - doc_state: collab.doc_state.to_vec(), - version: ClientEncoderVersion::V1, - } - } -} - -#[derive(Tsify, Serialize, Deserialize, Default, Debug)] -#[tsify(into_wasm_abi, from_wasm_abi)] -pub struct BatchClientQueryCollab(pub Vec); -#[derive(Tsify, Serialize, Deserialize, Default, Debug)] -#[tsify(into_wasm_abi, from_wasm_abi)] -pub struct ClientQueryCollab { - pub object_id: String, - #[tsify(type = "0 | 1 | 2 | 3 | 4 | 5")] - pub collab_type: i32, -} - -from_struct_for_jsvalue!(ClientQueryCollab); - -impl From for QueryCollab { - fn from(value: ClientQueryCollab) -> QueryCollab { - QueryCollab { - collab_type: CollabType::from(value.collab_type), - object_id: value.object_id, - } - } -} - -#[derive(Tsify, Serialize, Deserialize, Default)] -#[tsify(into_wasm_abi, from_wasm_abi)] -pub struct BatchClientEncodeCollab(pub HashMap); - -from_struct_for_jsvalue!(BatchClientEncodeCollab); - -impl From for BatchClientEncodeCollab { - fn from(result: BatchQueryCollabResult) -> Self { - let mut hash_map = HashMap::new(); - - result.0.into_iter().for_each(|(k, v)| match v { - QueryCollabResult::Success { encode_collab_v1 } => { - EncodedCollab::decode_from_bytes(&encode_collab_v1) - .map(|collab| { - hash_map.insert(k, ClientEncodeCollab::from(collab)); - }) - .unwrap_or_else(|err| { - tracing::error!("Failed to decode collab: {:?}", err); - }); - }, - QueryCollabResult::Failed { .. } => { - tracing::error!("Failed to get collab: {:?}", k); - }, - }); - - BatchClientEncodeCollab(hash_map) - } -} - -#[derive(Tsify, Serialize, Deserialize, Default, Debug)] -#[tsify(into_wasm_abi, from_wasm_abi)] -pub struct PublishViewMeta { - pub data: String, -} - -#[derive(Tsify, Serialize, Deserialize, Default, Debug)] -#[tsify(into_wasm_abi, from_wasm_abi)] -pub struct PublishViewPayload { - pub meta: PublishViewMeta, - /// The doc_state of the encoded collab. - pub data: Vec, -} -#[derive(Tsify, Serialize, Deserialize, Default, Debug)] -#[tsify(into_wasm_abi, from_wasm_abi)] -pub struct PublishInfo { - pub namespace: Option, - pub publish_name: String, -} -from_struct_for_jsvalue!(PublishViewMeta); -from_struct_for_jsvalue!(PublishViewPayload); -from_struct_for_jsvalue!(PublishInfo); - -pub fn parse_provider(provider: &str) -> AuthProvider { - match provider { - "google" => AuthProvider::Google, - "github" => AuthProvider::Github, - "discord" => AuthProvider::Discord, - _ => AuthProvider::Google, - } -} - -#[derive(Tsify, Serialize, Deserialize, Default, Debug)] -#[tsify(into_wasm_abi, from_wasm_abi)] -pub struct OAuthURLResponse { - pub url: String, -} - -from_struct_for_jsvalue!(OAuthURLResponse); diff --git a/libs/client-api-wasm/src/lib.rs b/libs/client-api-wasm/src/lib.rs deleted file mode 100644 index d8ac1465a..000000000 --- a/libs/client-api-wasm/src/lib.rs +++ /dev/null @@ -1,265 +0,0 @@ -pub mod entities; - -use crate::entities::*; - -use client_api::notify::TokenState; -use client_api::{Client, ClientConfiguration}; -use std::sync::Arc; -use uuid::Uuid; - -use client_api::error::ErrorCode; - -use database_entity::dto::QueryCollab; -use wasm_bindgen::prelude::*; - -#[global_allocator] -static ALLOC: wee_alloc::WeeAlloc = wee_alloc::WeeAlloc::INIT; - -#[wasm_bindgen] -extern "C" { - #[wasm_bindgen(js_namespace = console)] - fn log(msg: &str); - - #[wasm_bindgen(js_namespace = console)] - fn error(msg: &str); - - #[wasm_bindgen(js_namespace = console)] - fn info(msg: &str); - - #[wasm_bindgen(js_namespace = console)] - fn debug(msg: &str); - - #[wasm_bindgen(js_namespace = console)] - fn warn(msg: &str); - - #[wasm_bindgen(js_namespace = console)] - fn trace(msg: &str); - - #[wasm_bindgen(js_namespace = window)] - fn refresh_token(token: &str); - - #[wasm_bindgen(js_namespace = window)] - fn invalid_token(); -} - -#[wasm_bindgen] -pub struct ClientAPI { - client: Arc, -} - -#[wasm_bindgen] -impl ClientAPI { - pub fn new(config: ClientAPIConfig) -> ClientAPI { - tracing_wasm::set_as_global_default(); - console_error_panic_hook::set_once(); - let configuration = ClientConfiguration::default(); - - if let Some(compression) = &config.configuration { - configuration - .to_owned() - .with_compression_buffer_size(compression.compression_buffer_size) - .with_compression_quality(compression.compression_quality); - } - - let client = Client::new( - config.base_url.as_str(), - config.ws_addr.as_str(), - config.gotrue_url.as_str(), - config.device_id.as_str(), - configuration, - config.client_id.as_str(), - ); - - tracing::debug!("Client API initialized, config: {:?}", config); - ClientAPI { - client: Arc::new(client), - } - } - - pub fn subscribe(&self) { - let mut rx = self.client.subscribe_token_state(); - let client = self.client.clone(); - - wasm_bindgen_futures::spawn_local(async move { - while let Ok(state) = rx.recv().await { - match state { - TokenState::Refresh => { - if let Ok(token) = client.get_token() { - refresh_token(token.as_str()); - } else { - invalid_token(); - } - }, - TokenState::Invalid => { - invalid_token(); - }, - } - } - }); - } - pub async fn login(&self, email: &str, password: &str) -> Result<(), ClientResponse> { - match self.client.sign_in_password(email, password).await { - Ok(_) => Ok(()), - Err(err) => Err(ClientResponse::from(err)), - } - } - - pub async fn sign_up(&self, email: &str, password: &str) -> Result<(), ClientResponse> { - match self.client.sign_up(email, password).await { - Ok(_) => Ok(()), - Err(err) => Err(ClientResponse::from(err)), - } - } - - pub async fn logout(&self) -> Result<(), ClientResponse> { - match self.client.sign_out().await { - Ok(_) => Ok(()), - Err(err) => Err(ClientResponse::from(err)), - } - } - - pub async fn sign_in_with_magic_link( - &self, - email: &str, - redirect_to: &str, - ) -> Result<(), ClientResponse> { - match self - .client - .sign_in_with_magic_link(email, Some(redirect_to.to_string())) - .await - { - Ok(_) => Ok(()), - Err(err) => Err(ClientResponse::from(err)), - } - } - - pub async fn generate_oauth_url_with_provider( - &self, - provider: &str, - redirect_to: &str, - ) -> Result { - if provider.is_empty() { - return Err(ClientResponse { - code: ErrorCode::OAuthError, - message: "Provider is empty".to_string(), - }); - } - - let provider = parse_provider(provider); - - match self - .client - .generate_url_with_provider_and_redirect_to(&provider, Some(redirect_to.to_string())) - .await - { - Ok(url) => Ok(OAuthURLResponse { url }), - Err(err) => Err(ClientResponse::from(err)), - } - } - - pub async fn sign_in_with_url(&self, url: &str) -> Result<(), ClientResponse> { - match self.client.sign_in_with_url(url).await { - Ok(_) => Ok(()), - Err(err) => Err(ClientResponse::from(err)), - } - } - - pub async fn get_user(&self) -> Result { - match self.client.get_profile().await { - Ok(profile) => Ok(User::from(profile)), - Err(err) => Err(ClientResponse::from(err)), - } - } - - pub fn restore_token(&self, token: &str) -> Result<(), ClientResponse> { - match self.client.restore_token(token) { - Ok(_) => Ok(()), - Err(err) => Err(ClientResponse::from(err)), - } - } - - pub async fn get_collab( - &self, - params: ClientQueryCollabParams, - ) -> Result { - tracing::debug!("get_collab: {:?}", params); - match self.client.get_collab(params.into()).await { - Ok(data) => Ok(ClientEncodeCollab::from(data.encode_collab)), - Err(err) => Err(ClientResponse::from(err)), - } - } - - pub async fn batch_get_collab( - &self, - workspace_id: String, - params: BatchClientQueryCollab, - ) -> Result { - tracing::debug!("batch_get_collab: {:?}", params); - let workspace_id = workspace_id.as_str(); - let params: Vec = params.0.into_iter().map(|p| p.into()).collect(); - match self.client.batch_post_collab(workspace_id, params).await { - Ok(data) => Ok(BatchClientEncodeCollab::from(data)), - Err(err) => Err(ClientResponse::from(err)), - } - } - - pub async fn get_user_workspace(&self) -> Result { - match self.client.get_user_workspace_info().await { - Ok(workspace_info) => Ok(UserWorkspace::from(workspace_info)), - Err(err) => Err(ClientResponse::from(err)), - } - } - - pub async fn get_publish_view_meta( - &self, - publish_namespace: String, - publish_name: String, - ) -> Result { - match self - .client - .get_published_collab::(publish_namespace.as_str(), publish_name.as_str()) - .await - { - Ok(data) => Ok(PublishViewMeta { - data: data.to_string(), - }), - - Err(err) => Err(ClientResponse::from(err)), - } - } - - pub async fn get_publish_view( - &self, - publish_namespace: String, - publish_name: String, - ) -> Result { - let meta = self - .get_publish_view_meta(publish_namespace.clone(), publish_name.clone()) - .await?; - - let data = self - .client - .get_published_collab_blob(publish_namespace.as_str(), publish_name.as_str()) - .await - .map_err(ClientResponse::from)?; - - Ok(PublishViewPayload { - meta, - data: data.to_vec(), - }) - } - - pub async fn get_publish_info(&self, view_id: String) -> Result { - let view_id = Uuid::parse_str(view_id.as_str()).map_err(|err| ClientResponse { - code: ErrorCode::UuidError, - message: format!("Failed to parse view_id: {}", err), - })?; - match self.client.get_published_collab_info(&view_id).await { - Ok(info) => Ok(PublishInfo { - namespace: Some(info.namespace), - publish_name: info.publish_name, - }), - Err(err) => Err(ClientResponse::from(err)), - } - } -} diff --git a/libs/client-api/src/collab_sync/collab_sink.rs b/libs/client-api/src/collab_sync/collab_sink.rs index ceb83f911..c379e4010 100644 --- a/libs/client-api/src/collab_sync/collab_sink.rs +++ b/libs/client-api/src/collab_sync/collab_sink.rs @@ -13,11 +13,9 @@ use tokio::sync::{broadcast, watch}; use tokio::time::{interval, sleep}; use tracing::{error, trace, warn}; -use collab_rt_entity::{ClientCollabMessage, MsgId, ServerCollabMessage, SinkMessage}; - -use crate::af_spawn; use crate::collab_sync::collab_stream::SeqNumCounter; use crate::collab_sync::{SinkConfig, SyncError, SyncObject}; +use collab_rt_entity::{ClientCollabMessage, MsgId, ServerCollabMessage, SinkMessage}; pub(crate) const SEND_INTERVAL: Duration = Duration::from_secs(8); pub const COLLAB_SINK_DELAY_MILLIS: u64 = 500; @@ -81,7 +79,7 @@ where let cloned_state = state.clone(); let weak_notifier = Arc::downgrade(¬ifier); - af_spawn(async move { + tokio::spawn(async move { // Initial delay to make sure the first tick waits for SEND_INTERVAL sleep(SEND_INTERVAL).await; loop { diff --git a/libs/client-api/src/collab_sync/collab_stream.rs b/libs/client-api/src/collab_sync/collab_stream.rs index a3094509c..526dd8123 100644 --- a/libs/client-api/src/collab_sync/collab_stream.rs +++ b/libs/client-api/src/collab_sync/collab_stream.rs @@ -23,7 +23,6 @@ use collab_rt_protocol::{ ClientSyncProtocol, CollabSyncProtocol, Message, MessageReader, SyncMessage, }; -use crate::af_spawn; use crate::collab_sync::{ start_sync, CollabSink, MissUpdateReason, SyncError, SyncObject, SyncReason, }; @@ -72,7 +71,7 @@ where if let Some(interval) = periodic_sync_interval { tracing::trace!("setting periodic sync step 1 for {}", object_id); - af_spawn(ObserveCollab::::periodic_sync_step_1( + tokio::spawn(ObserveCollab::::periodic_sync_step_1( origin.clone(), sink.clone(), cloned_weak_collab.clone(), @@ -80,7 +79,7 @@ where object_id.clone(), )); } - af_spawn(ObserveCollab::::observer_collab_message( + tokio::spawn(ObserveCollab::::observer_collab_message( origin, arc_object, stream, diff --git a/libs/client-api/src/collab_sync/plugin.rs b/libs/client-api/src/collab_sync/plugin.rs index d908e7847..6f4f51496 100644 --- a/libs/client-api/src/collab_sync/plugin.rs +++ b/libs/client-api/src/collab_sync/plugin.rs @@ -21,7 +21,6 @@ use client_api_entity::{CollabObject, CollabType}; use collab_rt_entity::{ClientCollabMessage, ServerCollabMessage, UpdateSync}; use collab_rt_protocol::{Message, SyncMessage}; -use crate::af_spawn; use crate::collab_sync::collab_stream::CollabRef; use crate::collab_sync::{CollabSyncState, SinkConfig, SyncControl, SyncReason}; use crate::ws::{ConnectState, WSConnectStateReceiver}; @@ -79,7 +78,7 @@ where let mut sync_state_stream = sync_queue.subscribe_sync_state(); let sync_state_collab = collab.clone(); - af_spawn(async move { + tokio::spawn(async move { while let Ok(sink_state) = sync_state_stream.recv().await { if let Some(collab) = sync_state_collab.upgrade() { let sync_state = match sink_state { @@ -97,7 +96,7 @@ where let sync_queue = Arc::new(sync_queue); let weak_local_collab = collab.clone(); let weak_sync_queue = Arc::downgrade(&sync_queue); - af_spawn(async move { + tokio::spawn(async move { while let Ok(connect_state) = ws_connect_state.recv().await { match connect_state { ConnectState::Connected => { diff --git a/libs/client-api/src/collab_sync/sync_control.rs b/libs/client-api/src/collab_sync/sync_control.rs index 99437da04..07510841c 100644 --- a/libs/client-api/src/collab_sync/sync_control.rs +++ b/libs/client-api/src/collab_sync/sync_control.rs @@ -16,7 +16,6 @@ use yrs::{ReadTxn, StateVector}; use collab_rt_entity::{ClientCollabMessage, InitSync, ServerCollabMessage, UpdateSync}; use collab_rt_protocol::{ClientSyncProtocol, CollabSyncProtocol, Message, SyncMessage}; -use crate::af_spawn; use crate::collab_sync::collab_stream::{CollabRef, ObserveCollab}; use crate::collab_sync::{ CollabSink, CollabSinkRunner, CollabSyncState, MissUpdateReason, SinkSignal, SyncError, @@ -76,7 +75,7 @@ where sync_state_tx.clone(), sink_config, )); - af_spawn(CollabSinkRunner::run(Arc::downgrade(&sink), notifier_rx)); + tokio::spawn(CollabSinkRunner::run(Arc::downgrade(&sink), notifier_rx)); // Create the observe collab stream. let _cloned_protocol = protocol.clone(); diff --git a/libs/client-api/src/http.rs b/libs/client-api/src/http.rs index f65dee591..732ca0399 100644 --- a/libs/client-api/src/http.rs +++ b/libs/client-api/src/http.rs @@ -23,6 +23,7 @@ use parking_lot::RwLock; use reqwest::Method; use reqwest::RequestBuilder; +use anyhow::anyhow; use client_api_entity::{ AFSnapshotMeta, AFSnapshotMetas, AFUserProfile, AFUserWorkspaceInfo, AFWorkspace, QuerySnapshotParams, SnapshotData, @@ -32,12 +33,15 @@ use shared_entity::dto::auth_dto::SignInTokenResponse; use shared_entity::dto::auth_dto::UpdateUserParams; use shared_entity::dto::workspace_dto::WorkspaceSpaceUsage; use shared_entity::response::{AppResponse, AppResponseError}; -use std::sync::atomic::AtomicBool; +use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::Arc; use std::time::Duration; -use tracing::{error, event, info, instrument, trace, warn}; +use tokio_retry::strategy::FixedInterval; +use tokio_retry::RetryIf; +use tracing::{debug, error, event, info, instrument, trace, warn}; use url::Url; +use crate::retry::{RefreshTokenAction, RefreshTokenRetryCondition}; use crate::ws::ConnectInfo; use client_api_entity::SignUpResponse::{Authenticated, NotAuthenticated}; use client_api_entity::{GotrueTokenResponse, UpdateGotrueUserParams, User}; @@ -990,6 +994,61 @@ impl Client { } } + /// Refreshes the access token using the stored refresh token. + /// + /// This function attempts to refresh the access token by sending a request to the authentication server + /// using the stored refresh token. If successful, it updates the stored access token with the new one + /// received from the server. + #[instrument(level = "debug", skip_all, err)] + pub async fn refresh_token(&self, reason: &str) -> Result<(), AppResponseError> { + let (tx, rx) = tokio::sync::oneshot::channel(); + self.refresh_ret_txs.write().push(tx); + + if !self.is_refreshing_token.load(Ordering::SeqCst) { + self.is_refreshing_token.store(true, Ordering::SeqCst); + + info!("refresh token reason:{}", reason); + let result = self.inner_refresh_token().await; + let txs = std::mem::take(&mut *self.refresh_ret_txs.write()); + for tx in txs { + let _ = tx.send(result.clone()); + } + self.is_refreshing_token.store(false, Ordering::SeqCst); + } else { + debug!("refresh token is already in progress"); + } + + // Wait for the result of the refresh token request. + match tokio::time::timeout(Duration::from_secs(60), rx).await { + Ok(Ok(result)) => result, + Ok(Err(err)) => Err(AppError::Internal(anyhow!("refresh token error: {}", err)).into()), + Err(_) => { + self.is_refreshing_token.store(false, Ordering::SeqCst); + Err(AppError::RequestTimeout("refresh token timeout".to_string()).into()) + }, + } + } + + async fn inner_refresh_token(&self) -> Result<(), AppResponseError> { + let retry_strategy = FixedInterval::new(Duration::from_secs(2)).take(4); + let action = RefreshTokenAction::new(self.token.clone(), self.gotrue_client.clone()); + match RetryIf::spawn(retry_strategy, action, RefreshTokenRetryCondition).await { + Ok(_) => { + event!(tracing::Level::INFO, "refresh token success"); + Ok(()) + }, + Err(err) => { + let err = AppError::from(err); + event!(tracing::Level::ERROR, "refresh token failed: {}", err); + // If the error is an OAuth error, unset the token. + if err.is_unauthorized() { + self.token.write().unset(); + } + Err(err.into()) + }, + } + } + // Refresh token if given timestamp is close to the token expiration time pub async fn refresh_if_expired(&self, ts: i64, reason: &str) -> Result<(), AppResponseError> { let expires_at = self.token_expires_at()?; diff --git a/libs/client-api/src/http_ai.rs b/libs/client-api/src/http_ai.rs index 73f913c59..a5c314d01 100644 --- a/libs/client-api/src/http_ai.rs +++ b/libs/client-api/src/http_ai.rs @@ -1,5 +1,7 @@ use crate::http::log_request_id; use crate::Client; +use bytes::Bytes; +use futures_core::Stream; use reqwest::Method; use shared_entity::dto::ai_dto::{ CompleteTextParams, CompleteTextResponse, LocalAIConfig, SummarizeRowParams, @@ -10,6 +12,22 @@ use std::time::Duration; use tracing::instrument; impl Client { + pub async fn stream_completion_text( + &self, + workspace_id: &str, + params: CompleteTextParams, + ) -> Result>, AppResponseError> { + let url = format!("{}/api/ai/{}/complete/stream", self.base_url, workspace_id); + let resp = self + .http_client_with_auth(Method::POST, &url) + .await? + .json(¶ms) + .send() + .await?; + log_request_id(&resp); + AppResponse::<()>::answer_response_stream(resp).await + } + #[instrument(level = "info", skip_all)] pub async fn summarize_row( &self, diff --git a/libs/client-api/src/http_chat.rs b/libs/client-api/src/http_chat.rs index 06a410d86..6f020c632 100644 --- a/libs/client-api/src/http_chat.rs +++ b/libs/client-api/src/http_chat.rs @@ -9,7 +9,10 @@ use futures_core::{ready, Stream}; use pin_project::pin_project; use reqwest::Method; use serde_json::Value; -use shared_entity::dto::ai_dto::{RepeatedRelatedQuestion, STREAM_ANSWER_KEY, STREAM_METADATA_KEY}; +use shared_entity::dto::ai_dto::{ + CalculateSimilarityParams, RepeatedRelatedQuestion, SimilarityResponse, STREAM_ANSWER_KEY, + STREAM_METADATA_KEY, +}; use shared_entity::response::{AppResponse, AppResponseError}; use std::pin::Pin; use std::task::{Context, Poll}; @@ -215,6 +218,26 @@ impl Client { .await? .into_data() } + + pub async fn calculate_similarity( + &self, + params: CalculateSimilarityParams, + ) -> Result { + let url = format!( + "{}/api/ai/{}/calculate_similarity", + self.base_url, ¶ms.workspace_id + ); + let resp = self + .http_client_with_auth(Method::POST, &url) + .await? + .json(¶ms) + .send() + .await?; + log_request_id(&resp); + AppResponse::::from_response(resp) + .await? + .into_data() + } } #[pin_project] diff --git a/libs/client-api/src/http_collab.rs b/libs/client-api/src/http_collab.rs index cf4a240ea..76ed47b4a 100644 --- a/libs/client-api/src/http_collab.rs +++ b/libs/client-api/src/http_collab.rs @@ -1,14 +1,28 @@ use crate::http::log_request_id; -use crate::{blocking_brotli_compress, Client}; +use crate::{blocking_brotli_compress, brotli_compress, Client}; use app_error::AppError; +use bytes::Bytes; use client_api_entity::workspace_dto::{AFDatabase, ListDatabaseParam}; use client_api_entity::{ - BatchQueryCollabParams, BatchQueryCollabResult, CreateCollabParams, DeleteCollabParams, - QueryCollab, UpdateCollabWebParams, + BatchQueryCollabParams, BatchQueryCollabResult, CollabParams, CreateCollabParams, + DeleteCollabParams, PublishCollabItem, QueryCollab, QueryCollabParams, UpdateCollabWebParams, }; -use reqwest::Method; +use collab_rt_entity::HttpRealtimeMessage; +use futures::Stream; +use futures_util::stream; +use prost::Message; +use rayon::prelude::*; +use reqwest::{Body, Method}; +use serde::Serialize; +use shared_entity::dto::workspace_dto::{CollabResponse, CollabTypeParam}; use shared_entity::response::{AppResponse, AppResponseError}; -use tracing::instrument; +use std::future::Future; +use std::pin::Pin; +use std::task::{Context, Poll}; +use std::time::Duration; +use tokio_retry::strategy::ExponentialBackoff; +use tokio_retry::{Action, Condition, RetryIf}; +use tracing::{event, instrument}; impl Client { #[instrument(level = "info", skip_all, err)] @@ -157,4 +171,216 @@ impl Client { log_request_id(&resp); AppResponse::from_response(resp).await?.into_data() } + + #[instrument(level = "debug", skip_all, err)] + pub async fn post_realtime_msg( + &self, + device_id: &str, + msg: client_websocket::Message, + ) -> Result<(), AppResponseError> { + let device_id = device_id.to_string(); + let payload = + blocking_brotli_compress(msg.into_data(), 6, self.config.compression_buffer_size).await?; + + let msg = HttpRealtimeMessage { device_id, payload }.encode_to_vec(); + let body = Body::wrap_stream(stream::iter(vec![Ok::<_, reqwest::Error>(msg)])); + let url = format!("{}/api/realtime/post/stream", self.base_url); + let resp = self + .http_client_with_auth_compress(Method::POST, &url) + .await? + .body(body) + .send() + .await?; + crate::http::log_request_id(&resp); + AppResponse::<()>::from_response(resp).await?.into_error() + } + + #[instrument(level = "debug", skip_all, err)] + pub async fn create_collab_list( + &self, + workspace_id: &str, + params_list: Vec, + ) -> Result<(), AppResponseError> { + let url = self.batch_create_collab_url(workspace_id); + + let compression_tasks = params_list + .into_par_iter() + .filter_map(|params| { + let data = params.to_bytes().ok()?; + brotli_compress( + data, + self.config.compression_quality, + self.config.compression_buffer_size, + ) + .ok() + }) + .collect::>(); + + let mut framed_data = Vec::new(); + let mut size_count = 0; + for compressed in compression_tasks { + // The length of a u32 in bytes is 4. The server uses a u32 to read the size of each data frame, + // hence the frame size header is always 4 bytes. It's crucial not to alter this size value, + // as the server's logic for frame size reading is based on this fixed 4-byte length. + // note: + // the size of a u32 is a constant 4 bytes across all platforms that Rust supports. + let size = compressed.len() as u32; + framed_data.extend_from_slice(&size.to_be_bytes()); + framed_data.extend_from_slice(&compressed); + size_count += size; + } + event!( + tracing::Level::INFO, + "create batch collab with size: {}", + size_count + ); + let body = Body::wrap_stream(stream::once(async { Ok::<_, AppError>(framed_data) })); + let resp = self + .http_client_with_auth_compress(Method::POST, &url) + .await? + .timeout(Duration::from_secs(60)) + .body(body) + .send() + .await?; + + log_request_id(&resp); + AppResponse::<()>::from_response(resp).await?.into_error() + } + + #[instrument(level = "debug", skip_all)] + pub async fn get_collab( + &self, + params: QueryCollabParams, + ) -> Result { + // 2 seconds, 4 seconds, 8 seconds + let retry_strategy = ExponentialBackoff::from_millis(2).factor(1000).take(3); + let action = GetCollabAction::new(self.clone(), params); + RetryIf::spawn(retry_strategy, action, RetryGetCollabCondition).await + } + + pub async fn publish_collabs( + &self, + workspace_id: &str, + items: Vec>, + ) -> Result<(), AppResponseError> + where + Metadata: serde::Serialize + Send + 'static + Unpin, + Data: AsRef<[u8]> + Send + 'static + Unpin, + { + let publish_collab_stream = PublishCollabItemStream::new(items); + let url = format!("{}/api/workspace/{}/publish", self.base_url, workspace_id,); + let resp = self + .http_client_with_auth(Method::POST, &url) + .await? + .body(Body::wrap_stream(publish_collab_stream)) + .send() + .await?; + AppResponse::<()>::from_response(resp).await?.into_error() + } +} + +struct RetryGetCollabCondition; +impl Condition for RetryGetCollabCondition { + fn should_retry(&mut self, error: &AppResponseError) -> bool { + !error.is_record_not_found() + } +} + +pub struct PublishCollabItemStream { + items: Vec>, + idx: usize, + done: bool, +} + +impl PublishCollabItemStream { + pub fn new(publish_collab_items: Vec>) -> Self { + PublishCollabItemStream { + items: publish_collab_items, + idx: 0, + done: false, + } + } +} + +impl Stream for PublishCollabItemStream +where + Metadata: Serialize + Send + 'static + Unpin, + Data: AsRef<[u8]> + Send + 'static + Unpin, +{ + type Item = Result; + + fn poll_next(mut self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll> { + let mut self_mut = self.as_mut(); + + if self_mut.idx >= self_mut.items.len() { + if !self_mut.done { + self_mut.done = true; + return Poll::Ready(Some(Ok((0_u32).to_le_bytes().to_vec().into()))); + } + return Poll::Ready(None); + } + + let item = &self_mut.items[self_mut.idx]; + match serialize_metadata_data(&item.meta, item.data.as_ref()) { + Err(e) => Poll::Ready(Some(Err(e))), + Ok(chunk) => { + self_mut.idx += 1; + Poll::Ready(Some(Ok::(chunk))) + }, + } + } +} + +fn serialize_metadata_data(m: Metadata, d: &[u8]) -> Result +where + Metadata: Serialize, +{ + let meta = serde_json::to_vec(&m)?; + + let mut chunk = Vec::with_capacity(8 + meta.len() + d.len()); + chunk.extend_from_slice(&(meta.len() as u32).to_le_bytes()); // Encode metadata length + chunk.extend_from_slice(&meta); + chunk.extend_from_slice(&(d.len() as u32).to_le_bytes()); // Encode data length + chunk.extend_from_slice(d); + + Ok(Bytes::from(chunk)) +} + +pub(crate) struct GetCollabAction { + client: Client, + params: QueryCollabParams, +} + +impl GetCollabAction { + pub fn new(client: Client, params: QueryCollabParams) -> Self { + Self { client, params } + } +} + +impl Action for GetCollabAction { + type Future = Pin> + Send + Sync>>; + type Item = CollabResponse; + type Error = AppResponseError; + + fn run(&mut self) -> Self::Future { + let client = self.client.clone(); + let params = self.params.clone(); + let collab_type = self.params.collab_type.clone(); + + Box::pin(async move { + let url = format!( + "{}/api/workspace/v1/{}/collab/{}", + client.base_url, ¶ms.workspace_id, ¶ms.object_id + ); + let resp = client + .http_client_with_auth(Method::GET, &url) + .await? + .query(&CollabTypeParam { collab_type }) + .send() + .await?; + log_request_id(&resp); + let resp = AppResponse::::from_response(resp).await?; + resp.into_data() + }) + } } diff --git a/libs/client-api/src/native/http_native.rs b/libs/client-api/src/http_file.rs similarity index 51% rename from libs/client-api/src/native/http_native.rs rename to libs/client-api/src/http_file.rs index c637c4955..ad4fe5546 100644 --- a/libs/client-api/src/native/http_native.rs +++ b/libs/client-api/src/http_file.rs @@ -1,68 +1,30 @@ use crate::http::log_request_id; -use crate::native::GetCollabAction; use crate::ws::{ConnectInfo, WSClientConnectURLProvider, WSClientHttpSender, WSError}; -use crate::{blocking_brotli_compress, brotli_compress, Client}; -use crate::{RefreshTokenAction, RefreshTokenRetryCondition}; -use anyhow::anyhow; +use crate::Client; + use app_error::AppError; use async_trait::async_trait; -use rayon::iter::ParallelIterator; use std::fs::metadata; -use bytes::Bytes; -use client_api_entity::{ - CollabParams, CreateImportTask, CreateImportTaskResponse, PublishCollabItem, QueryCollabParams, -}; use client_api_entity::{ CompleteUploadRequest, CreateUploadRequest, CreateUploadResponse, UploadPartResponse, }; -use collab_rt_entity::HttpRealtimeMessage; -use futures::Stream; -use futures_util::stream; +use client_api_entity::{CreateImportTask, CreateImportTaskResponse}; + use percent_encoding::{utf8_percent_encode, NON_ALPHANUMERIC}; -use prost::Message; use reqwest::{multipart, Body, Method}; -use serde::Serialize; -use shared_entity::dto::workspace_dto::CollabResponse; use shared_entity::response::{AppResponse, AppResponseError}; -use std::future::Future; use std::path::Path; -use std::pin::Pin; -use std::sync::atomic::Ordering; use base64::engine::general_purpose::STANDARD; use base64::Engine; -pub use infra::file_util::ChunkedBytes; -use rayon::prelude::IntoParallelIterator; -use shared_entity::dto::ai_dto::CompleteTextParams; use shared_entity::dto::import_dto::UserImportTask; -use std::task::{Context, Poll}; -use std::time::Duration; use tokio::fs::File; use tokio::io::{AsyncBufReadExt, BufReader}; -use tokio_retry::strategy::{ExponentialBackoff, FixedInterval}; -use tokio_retry::{Condition, RetryIf}; use tokio_util::codec::{BytesCodec, FramedRead}; - -use tracing::{debug, error, event, info, instrument, trace}; +use tracing::{error, trace}; impl Client { - pub async fn stream_completion_text( - &self, - workspace_id: &str, - params: CompleteTextParams, - ) -> Result>, AppResponseError> { - let url = format!("{}/api/ai/{}/complete/stream", self.base_url, workspace_id); - let resp = self - .http_client_with_auth(Method::POST, &url) - .await? - .json(¶ms) - .send() - .await?; - log_request_id(&resp); - AppResponse::<()>::answer_response_stream(resp).await - } - pub async fn create_upload( &self, workspace_id: &str, @@ -107,9 +69,9 @@ impl Client { // Encode the parent directory to ensure it's URL-safe. let parent_dir = utf8_percent_encode(parent_dir, NON_ALPHANUMERIC).to_string(); let url = format!( - "{}/api/file_storage/{workspace_id}/upload_part/{parent_dir}/{file_id}/{upload_id}/{part_number}", - self.base_url - ); + "{}/api/file_storage/{workspace_id}/upload_part/{parent_dir}/{file_id}/{upload_id}/{part_number}", + self.base_url + ); let resp = self .http_client_with_auth(Method::PUT, &url) .await? @@ -141,169 +103,6 @@ impl Client { AppResponse::<()>::from_response(resp).await?.into_error() } - #[instrument(level = "debug", skip_all)] - pub async fn get_collab( - &self, - params: QueryCollabParams, - ) -> Result { - info!("get collab:{}", params); - // 2 seconds, 4 seconds, 8 seconds - let retry_strategy = ExponentialBackoff::from_millis(2).factor(1000).take(3); - let action = GetCollabAction::new(self.clone(), params); - RetryIf::spawn(retry_strategy, action, RetryGetCollabCondition).await - } - - #[instrument(level = "debug", skip_all, err)] - pub async fn post_realtime_msg( - &self, - device_id: &str, - msg: client_websocket::Message, - ) -> Result<(), AppResponseError> { - let device_id = device_id.to_string(); - let payload = - blocking_brotli_compress(msg.into_data(), 6, self.config.compression_buffer_size).await?; - - let msg = HttpRealtimeMessage { device_id, payload }.encode_to_vec(); - let body = Body::wrap_stream(stream::iter(vec![Ok::<_, reqwest::Error>(msg)])); - let url = format!("{}/api/realtime/post/stream", self.base_url); - let resp = self - .http_client_with_auth_compress(Method::POST, &url) - .await? - .body(body) - .send() - .await?; - crate::http::log_request_id(&resp); - AppResponse::<()>::from_response(resp).await?.into_error() - } - - #[instrument(level = "debug", skip_all, err)] - pub async fn create_collab_list( - &self, - workspace_id: &str, - params_list: Vec, - ) -> Result<(), AppResponseError> { - let url = self.batch_create_collab_url(workspace_id); - - let compression_tasks = params_list - .into_par_iter() - .filter_map(|params| { - let data = params.to_bytes().ok()?; - brotli_compress( - data, - self.config.compression_quality, - self.config.compression_buffer_size, - ) - .ok() - }) - .collect::>(); - - let mut framed_data = Vec::new(); - let mut size_count = 0; - for compressed in compression_tasks { - // The length of a u32 in bytes is 4. The server uses a u32 to read the size of each data frame, - // hence the frame size header is always 4 bytes. It's crucial not to alter this size value, - // as the server's logic for frame size reading is based on this fixed 4-byte length. - // note: - // the size of a u32 is a constant 4 bytes across all platforms that Rust supports. - let size = compressed.len() as u32; - framed_data.extend_from_slice(&size.to_be_bytes()); - framed_data.extend_from_slice(&compressed); - size_count += size; - } - event!( - tracing::Level::INFO, - "create batch collab with size: {}", - size_count - ); - let body = Body::wrap_stream(stream::once(async { Ok::<_, AppError>(framed_data) })); - let resp = self - .http_client_with_auth_compress(Method::POST, &url) - .await? - .timeout(Duration::from_secs(60)) - .body(body) - .send() - .await?; - - log_request_id(&resp); - AppResponse::<()>::from_response(resp).await?.into_error() - } - - /// Refreshes the access token using the stored refresh token. - /// - /// This function attempts to refresh the access token by sending a request to the authentication server - /// using the stored refresh token. If successful, it updates the stored access token with the new one - /// received from the server. - #[instrument(level = "debug", skip_all, err)] - pub async fn refresh_token(&self, reason: &str) -> Result<(), AppResponseError> { - let (tx, rx) = tokio::sync::oneshot::channel(); - self.refresh_ret_txs.write().push(tx); - - if !self.is_refreshing_token.load(Ordering::SeqCst) { - self.is_refreshing_token.store(true, Ordering::SeqCst); - - info!("refresh token reason:{}", reason); - let result = self.inner_refresh_token().await; - let txs = std::mem::take(&mut *self.refresh_ret_txs.write()); - for tx in txs { - let _ = tx.send(result.clone()); - } - self.is_refreshing_token.store(false, Ordering::SeqCst); - } else { - debug!("refresh token is already in progress"); - } - - // Wait for the result of the refresh token request. - match tokio::time::timeout(Duration::from_secs(60), rx).await { - Ok(Ok(result)) => result, - Ok(Err(err)) => Err(AppError::Internal(anyhow!("refresh token error: {}", err)).into()), - Err(_) => { - self.is_refreshing_token.store(false, Ordering::SeqCst); - Err(AppError::RequestTimeout("refresh token timeout".to_string()).into()) - }, - } - } - - async fn inner_refresh_token(&self) -> Result<(), AppResponseError> { - let retry_strategy = FixedInterval::new(Duration::from_secs(2)).take(4); - let action = RefreshTokenAction::new(self.token.clone(), self.gotrue_client.clone()); - match RetryIf::spawn(retry_strategy, action, RefreshTokenRetryCondition).await { - Ok(_) => { - event!(tracing::Level::INFO, "refresh token success"); - Ok(()) - }, - Err(err) => { - let err = AppError::from(err); - event!(tracing::Level::ERROR, "refresh token failed: {}", err); - - // If the error is an OAuth error, unset the token. - if err.is_unauthorized() { - self.token.write().unset(); - } - Err(err.into()) - }, - } - } - - pub async fn publish_collabs( - &self, - workspace_id: &str, - items: Vec>, - ) -> Result<(), AppResponseError> - where - Metadata: serde::Serialize + Send + 'static + Unpin, - Data: AsRef<[u8]> + Send + 'static + Unpin, - { - let publish_collab_stream = PublishCollabItemStream::new(items); - let url = format!("{}/api/workspace/{}/publish", self.base_url, workspace_id,); - let resp = self - .http_client_with_auth(Method::POST, &url) - .await? - .body(Body::wrap_stream(publish_collab_stream)) - .send() - .await?; - AppResponse::<()>::from_response(resp).await?.into_error() - } - /// Sends a POST request to import a file to the server. /// /// This function streams the contents of a file located at the provided `file_path` @@ -484,82 +283,6 @@ impl WSClientConnectURLProvider for Client { } } -// TODO(nathan): spawn for wasm -pub fn af_spawn(future: T) -> tokio::task::JoinHandle -where - T: Future + Send + 'static, - T::Output: Send + 'static, -{ - tokio::spawn(future) -} - -pub struct PublishCollabItemStream { - items: Vec>, - idx: usize, - done: bool, -} - -impl PublishCollabItemStream { - pub fn new(publish_collab_items: Vec>) -> Self { - PublishCollabItemStream { - items: publish_collab_items, - idx: 0, - done: false, - } - } -} - -impl Stream for PublishCollabItemStream -where - Metadata: Serialize + Send + 'static + Unpin, - Data: AsRef<[u8]> + Send + 'static + Unpin, -{ - type Item = Result; - - fn poll_next(mut self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll> { - let mut self_mut = self.as_mut(); - - if self_mut.idx >= self_mut.items.len() { - if !self_mut.done { - self_mut.done = true; - return Poll::Ready(Some(Ok((0_u32).to_le_bytes().to_vec().into()))); - } - return Poll::Ready(None); - } - - let item = &self_mut.items[self_mut.idx]; - match serialize_metadata_data(&item.meta, item.data.as_ref()) { - Err(e) => Poll::Ready(Some(Err(e))), - Ok(chunk) => { - self_mut.idx += 1; - Poll::Ready(Some(Ok::(chunk))) - }, - } - } -} - -fn serialize_metadata_data(m: Metadata, d: &[u8]) -> Result -where - Metadata: Serialize, -{ - let meta = serde_json::to_vec(&m)?; - - let mut chunk = Vec::with_capacity(8 + meta.len() + d.len()); - chunk.extend_from_slice(&(meta.len() as u32).to_le_bytes()); // Encode metadata length - chunk.extend_from_slice(&meta); - chunk.extend_from_slice(&(d.len() as u32).to_le_bytes()); // Encode data length - chunk.extend_from_slice(d); - - Ok(Bytes::from(chunk)) -} - -struct RetryGetCollabCondition; -impl Condition for RetryGetCollabCondition { - fn should_retry(&mut self, error: &AppResponseError) -> bool { - !error.is_record_not_found() - } -} - /// Calculates the MD5 hash of a file and returns the base64-encoded MD5 digest. /// /// # Arguments diff --git a/libs/client-api/src/http_view.rs b/libs/client-api/src/http_view.rs index 14101253d..a5b55cd6f 100644 --- a/libs/client-api/src/http_view.rs +++ b/libs/client-api/src/http_view.rs @@ -1,4 +1,6 @@ -use client_api_entity::workspace_dto::{CreatePageParams, Page, PageCollab, UpdatePageParams}; +use client_api_entity::workspace_dto::{ + CreatePageParams, CreateSpaceParams, Page, PageCollab, Space, UpdatePageParams, +}; use reqwest::Method; use serde_json::json; use shared_entity::response::{AppResponse, AppResponseError}; @@ -112,4 +114,19 @@ impl Client { .await? .into_data() } + + pub async fn create_space( + &self, + workspace_id: Uuid, + params: &CreateSpaceParams, + ) -> Result { + let url = format!("{}/api/workspace/{}/space", self.base_url, workspace_id,); + let resp = self + .http_client_with_auth(Method::POST, &url) + .await? + .json(params) + .send() + .await?; + AppResponse::::from_response(resp).await?.into_data() + } } diff --git a/libs/client-api/src/lib.rs b/libs/client-api/src/lib.rs index 2dea43cb2..0868fd2d3 100644 --- a/libs/client-api/src/lib.rs +++ b/libs/client-api/src/lib.rs @@ -8,6 +8,7 @@ mod http_collab; mod http_history; mod http_member; mod http_publish; +mod http_search; mod http_template; mod http_view; pub use http::*; @@ -15,23 +16,12 @@ pub use http::*; #[cfg(feature = "collab-sync")] pub mod collab_sync; -pub mod notify; - -#[cfg(not(target_arch = "wasm32"))] -mod native; -#[cfg(not(target_arch = "wasm32"))] -pub use native::*; - -#[cfg(target_arch = "wasm32")] -mod wasm; -#[cfg(target_arch = "wasm32")] -pub use wasm::*; - -#[cfg(not(target_arch = "wasm32"))] mod http_chat; - -mod http_search; +mod http_file; mod http_settings; +pub mod notify; +mod ping; +mod retry; pub mod ws; pub mod error { diff --git a/libs/client-api/src/native/mod.rs b/libs/client-api/src/native/mod.rs deleted file mode 100644 index a12859fa1..000000000 --- a/libs/client-api/src/native/mod.rs +++ /dev/null @@ -1,8 +0,0 @@ -mod http_native; -mod ping; -mod retry; - -#[allow(unused_imports)] -pub use http_native::*; -pub(crate) use ping::*; -pub(crate) use retry::*; diff --git a/libs/client-api/src/native/ping.rs b/libs/client-api/src/ping.rs similarity index 98% rename from libs/client-api/src/native/ping.rs rename to libs/client-api/src/ping.rs index 48ade444d..34911409a 100644 --- a/libs/client-api/src/native/ping.rs +++ b/libs/client-api/src/ping.rs @@ -1,4 +1,3 @@ -use crate::af_spawn; use crate::ws::{ConnectState, ConnectStateNotify}; use client_websocket::Message; use std::sync::Arc; @@ -52,7 +51,7 @@ impl ServerFixIntervalPing { let weak_ping_count = Arc::downgrade(&self.ping_count); let weak_state = Arc::downgrade(&self.state); let reconnect_per_ping = self.maximum_ping_count; - af_spawn(async move { + tokio::spawn(async move { loop { tokio::select! { _ = interval.tick() => { diff --git a/libs/client-api/src/native/retry.rs b/libs/client-api/src/retry.rs similarity index 77% rename from libs/client-api/src/native/retry.rs rename to libs/client-api/src/retry.rs index 02078e7c4..557b02531 100644 --- a/libs/client-api/src/native/retry.rs +++ b/libs/client-api/src/retry.rs @@ -1,18 +1,13 @@ -use crate::http::log_request_id; use crate::notify::ClientToken; use crate::ws::{ ConnectState, ConnectStateNotify, StateNotify, WSClientConnectURLProvider, WSError, }; -use crate::Client; + use app_error::gotrue::GoTrueError; -use client_api_entity::QueryCollabParams; use client_websocket::{connect_async, WebSocketStream}; use gotrue::grant::{Grant, RefreshTokenGrant}; use parking_lot::RwLock; use reqwest::header::HeaderMap; -use reqwest::Method; -use shared_entity::dto::workspace_dto::{CollabResponse, CollabTypeParam}; -use shared_entity::response::{AppResponse, AppResponseError}; use std::future::Future; use std::pin::Pin; use std::sync::{Arc, Weak}; @@ -143,42 +138,3 @@ impl Condition for RetryCondition { true } } - -pub(crate) struct GetCollabAction { - client: Client, - params: QueryCollabParams, -} - -impl GetCollabAction { - pub fn new(client: Client, params: QueryCollabParams) -> Self { - Self { client, params } - } -} - -impl Action for GetCollabAction { - type Future = Pin> + Send + Sync>>; - type Item = CollabResponse; - type Error = AppResponseError; - - fn run(&mut self) -> Self::Future { - let client = self.client.clone(); - let params = self.params.clone(); - let collab_type = self.params.collab_type.clone(); - - Box::pin(async move { - let url = format!( - "{}/api/workspace/v1/{}/collab/{}", - client.base_url, ¶ms.workspace_id, ¶ms.object_id - ); - let resp = client - .http_client_with_auth(Method::GET, &url) - .await? - .query(&CollabTypeParam { collab_type }) - .send() - .await?; - log_request_id(&resp); - let resp = AppResponse::::from_response(resp).await?; - resp.into_data() - }) - } -} diff --git a/libs/client-api/src/wasm/http_wasm.rs b/libs/client-api/src/wasm/http_wasm.rs deleted file mode 100644 index 1ba015ebf..000000000 --- a/libs/client-api/src/wasm/http_wasm.rs +++ /dev/null @@ -1,156 +0,0 @@ -use crate::http::log_request_id; -use crate::ws::{ConnectInfo, WSClientConnectURLProvider, WSClientHttpSender, WSError}; -use crate::Client; -use app_error::gotrue::GoTrueError; -use app_error::ErrorCode; -use async_trait::async_trait; -use client_api_entity::{CollabParams, QueryCollabParams}; -use gotrue::grant::{Grant, RefreshTokenGrant}; -use reqwest::Method; -use shared_entity::dto::workspace_dto::{CollabResponse, CollabTypeParam}; -use shared_entity::response::{AppResponse, AppResponseError}; -use std::future::Future; -use std::sync::atomic::Ordering; -use tracing::{info, instrument}; - -impl Client { - pub async fn create_collab_list( - &self, - workspace_id: &str, - _params_list: Vec, - ) -> Result<(), AppResponseError> { - let _url = self.batch_create_collab_url(workspace_id); - Err(AppResponseError::new( - ErrorCode::Unhandled, - "not implemented", - )) - } - - #[instrument(level = "debug", skip_all)] - pub async fn get_collab( - &self, - params: QueryCollabParams, - ) -> Result { - let url = format!( - "{}/api/workspace/v1/{}/collab/{}", - self.base_url, ¶ms.workspace_id, ¶ms.object_id - ); - let collab_type = params.collab_type.clone(); - let resp = self - .http_client_with_auth(Method::GET, &url) - .await? - .query(&CollabTypeParam { collab_type }) - .send() - .await?; - log_request_id(&resp); - let resp = AppResponse::::from_response(resp).await?; - resp.into_data() - } - - #[instrument(level = "debug", skip_all, err)] - pub async fn refresh_token(&self, reason: &str) -> Result<(), AppResponseError> { - let (tx, rx) = tokio::sync::oneshot::channel(); - self.refresh_ret_txs.write().push(tx); - - if !self.is_refreshing_token.load(Ordering::SeqCst) { - self.is_refreshing_token.store(true, Ordering::SeqCst); - - info!("refresh token reason:{}", reason); - let txs = std::mem::take(&mut *self.refresh_ret_txs.write()); - let result = self.inner_refresh_token().await; - for tx in txs { - let _ = tx.send(result.clone()); - } - self.is_refreshing_token.store(false, Ordering::SeqCst); - } - - rx.await - .map_err(|err| AppResponseError::new(ErrorCode::Internal, err.to_string()))??; - Ok(()) - } - - async fn inner_refresh_token(&self) -> Result<(), AppResponseError> { - // let policy = RetryPolicy::fixed(Duration::from_secs(2)).with_max_retries(4).with_jitter(false); - // let refresh_token = self - // .token - // .read() - // .as_ref() - // .ok_or(GoTrueError::NotLoggedIn( - // "fail to refresh user token".to_owned(), - // ))? - // .refresh_token - // .as_str() - // .to_owned(); - // match policy.retry_if(move || { - // let grant = Grant::RefreshToken(RefreshTokenGrant { refresh_token: refresh_token.clone() }); - // async move { - // self - // .gotrue_client - // .token(&grant).await - // } - // - // }, RefreshTokenRetryCondition).await { - // Ok(new_token) => { - // event!(tracing::Level::INFO, "refresh token success"); - // self.token.write().set(new_token); - // Ok(()) - // }, - // Err(err) => { - // let err = AppError::from(err); - // event!(tracing::Level::ERROR, "refresh token failed: {}", err); - // - // // If the error is an OAuth error, unset the token. - // if err.is_unauthorized() { - // self.token.write().unset(); - // } - // Err(err.into()) - // }, - // } - let refresh_token = self - .token - .read() - .as_ref() - .ok_or(GoTrueError::NotLoggedIn( - "fail to refresh user token".to_owned(), - ))? - .refresh_token - .as_str() - .to_owned(); - let new_token = self - .gotrue_client - .token(&Grant::RefreshToken(RefreshTokenGrant { refresh_token })) - .await?; - self.token.write().set(new_token); - Ok(()) - } -} - -pub fn af_spawn(future: T) -> tokio::task::JoinHandle -where - T: Future + 'static, - T::Output: Send + 'static, -{ - tokio::task::spawn_local(future) -} - -#[async_trait] -impl WSClientHttpSender for Client { - async fn send_ws_msg( - &self, - _device_id: &str, - _message: client_websocket::Message, - ) -> Result<(), WSError> { - Err(WSError::Internal(anyhow::Error::msg("not supported"))) - } -} - -#[async_trait] -impl WSClientConnectURLProvider for Client { - fn connect_ws_url(&self) -> String { - self.ws_addr.clone() - } - - async fn connect_info(&self) -> Result { - Err(WSError::Internal(anyhow::Error::msg("not supported"))) - } -} diff --git a/libs/client-api/src/wasm/mod.rs b/libs/client-api/src/wasm/mod.rs deleted file mode 100644 index b3550a7e3..000000000 --- a/libs/client-api/src/wasm/mod.rs +++ /dev/null @@ -1,7 +0,0 @@ -mod http_wasm; -mod ping; -mod retry; - -pub use http_wasm::*; -pub(crate) use ping::*; -pub(crate) use retry::*; diff --git a/libs/client-api/src/wasm/ping.rs b/libs/client-api/src/wasm/ping.rs deleted file mode 100644 index e9ce4f779..000000000 --- a/libs/client-api/src/wasm/ping.rs +++ /dev/null @@ -1,49 +0,0 @@ -use crate::ws::ConnectStateNotify; -use client_websocket::Message; -use std::sync::Arc; -use std::time::Duration; -use tokio::sync::broadcast::Sender; -use tokio::sync::mpsc::Receiver; -use tokio::sync::Mutex; -#[allow(dead_code)] -pub(crate) struct ServerFixIntervalPing { - duration: Duration, - ping_sender: Option>, - pong_recv: Option>, - #[allow(dead_code)] - stop_tx: tokio::sync::mpsc::Sender<()>, - stop_rx: Option>, - state: Arc>, - ping_count: Arc>, - maximum_ping_count: u32, -} - -impl ServerFixIntervalPing { - pub(crate) fn new( - duration: Duration, - state: Arc>, - ping_sender: Sender, - pong_recv: Receiver<()>, - maximum_ping_count: u32, - ) -> Self { - let (tx, rx) = tokio::sync::mpsc::channel(1000); - Self { - duration, - stop_tx: tx, - stop_rx: Some(rx), - state, - ping_sender: Some(ping_sender), - pong_recv: Some(pong_recv), - ping_count: Arc::new(Mutex::new(0)), - maximum_ping_count, - } - } - - pub(crate) async fn stop(&self) { - let _ = self.stop_tx.send(()).await; - } - - pub(crate) fn run(&mut self) { - // TODO(nathan): implement the ping for wasm - } -} diff --git a/libs/client-api/src/wasm/retry.rs b/libs/client-api/src/wasm/retry.rs deleted file mode 100644 index 5dc35f6be..000000000 --- a/libs/client-api/src/wasm/retry.rs +++ /dev/null @@ -1,24 +0,0 @@ -use crate::ws::{StateNotify, WSClientConnectURLProvider, WSError}; -use again::Condition; -use app_error::gotrue::GoTrueError; -use client_websocket::{connect_async, WebSocketStream}; -use reqwest::header::HeaderMap; -use std::sync::{Arc, Weak}; - -pub(crate) struct RefreshTokenRetryCondition; - -impl Condition for RefreshTokenRetryCondition { - fn is_retryable(&mut self, error: &GoTrueError) -> bool { - error.is_network_error() - } -} -pub async fn retry_connect( - connect_provider: Arc, - _state_notify: Weak, -) -> Result { - let url = connect_provider.connect_ws_url(); - let connect_info = connect_provider.connect_info().await?; - let headers: HeaderMap = connect_info.into(); - let stream = connect_async(url, headers).await?; - Ok(stream) -} diff --git a/libs/client-api/src/ws/client.rs b/libs/client-api/src/ws/client.rs index 2fa55dab0..259bf45db 100644 --- a/libs/client-api/src/ws/client.rs +++ b/libs/client-api/src/ws/client.rs @@ -14,17 +14,16 @@ use tokio::sync::oneshot; use tokio::sync::Mutex; use tracing::{error, info, trace, warn}; +use crate::ping::ServerFixIntervalPing; +use crate::retry::retry_connect; +use crate::ws::msg_queue::{AggregateMessageQueue, AggregateMessagesReceiver}; +use crate::ws::{ConnectState, ConnectStateNotify, WSError, WebSocketChannel}; use client_websocket::{CloseCode, CloseFrame, Message, WebSocketStream}; use collab_rt_entity::user::UserMessage; use collab_rt_entity::ClientCollabMessage; use collab_rt_entity::ServerCollabMessage; use collab_rt_entity::{RealtimeMessage, SystemMessage}; -use crate::ws::msg_queue::{AggregateMessageQueue, AggregateMessagesReceiver}; -use crate::ws::{ConnectState, ConnectStateNotify, WSError, WebSocketChannel}; -use crate::ServerFixIntervalPing; -use crate::{af_spawn, retry_connect}; - pub struct WSClientConfig { /// specifies the number of messages that the channel can hold at any given /// time. It is used to set the initial size of the channel's internal buffer @@ -181,7 +180,7 @@ impl WSClient { fn spawn_aggregate_message(&self) { let mut rx = self.rt_msg_sender.subscribe(); let weak_aggregate_queue = Arc::downgrade(&self.aggregate_queue); - af_spawn(async move { + tokio::spawn(async move { while let Ok(msg) = rx.recv().await { if let Some(aggregate_queue) = weak_aggregate_queue.upgrade() { aggregate_queue.push(msg).await; @@ -230,7 +229,7 @@ impl WSClient { } }; - af_spawn(async move { + tokio::spawn(async move { loop { tokio::select! { _ = &mut stop_ws_msg_loop_rx => break, @@ -265,7 +264,7 @@ impl WSClient { #[cfg(debug_assertions)] let cloned_skip_realtime_message = self.skip_realtime_message.clone(); let user_message_tx = self.user_channel.as_ref().clone(); - af_spawn(async move { + tokio::spawn(async move { while let Some(Ok(ws_msg)) = stream.next().await { match ws_msg { Message::Binary(data) => { @@ -462,7 +461,7 @@ async fn send_message( if message.is_binary() && message.len() > MAXIMUM_MESSAGE_SIZE { if let Some(http_sender) = http_sender.upgrade() { let cloned_device_id = device_id.to_string(); - af_spawn(async move { + tokio::spawn(async move { if let Err(err) = http_sender.send_ws_msg(&cloned_device_id, message).await { error!("Failed to send WebSocket message over HTTP: {}", err); } diff --git a/libs/client-api/src/ws/handler.rs b/libs/client-api/src/ws/handler.rs index 38f49e537..0a7ef0a8f 100644 --- a/libs/client-api/src/ws/handler.rs +++ b/libs/client-api/src/ws/handler.rs @@ -1,4 +1,3 @@ -use crate::af_spawn; use collab_rt_entity::ClientCollabMessage; use collab_rt_entity::RealtimeMessage; use futures_util::Sink; @@ -51,7 +50,7 @@ where let (tx, mut rx) = unbounded_channel::>(); let cloned_sender = self.rt_msg_sender.clone(); let object_id = self.object_id.clone(); - af_spawn(async move { + tokio::spawn(async move { while let Some(msg) = rx.recv().await { let _ = cloned_sender.send(msg); } @@ -66,7 +65,7 @@ where let (tx, rx) = unbounded_channel::>(); let mut recv = self.receiver.subscribe(); let object_id = self.object_id.clone(); - af_spawn(async move { + tokio::spawn(async move { while let Ok(msg) = recv.recv().await { if let Err(err) = tx.send(Ok(msg)) { trace!("Failed to send message to channel stream: {}", err); diff --git a/libs/client-websocket/src/error.rs b/libs/client-websocket/src/error.rs index f5b4cde62..60b2fa72c 100644 --- a/libs/client-websocket/src/error.rs +++ b/libs/client-websocket/src/error.rs @@ -59,7 +59,7 @@ pub enum Error { #[error("URL error: {0}")] Url(#[from] UrlError), #[error("HTTP error: {}", .0.status())] - Http(Response>>), + Http(Box>>>), #[error("HTTP format error: {0}")] HttpFormat(#[from] http::Error), #[error("Parsing blobs is unsupported")] diff --git a/libs/client-websocket/src/native.rs b/libs/client-websocket/src/native.rs index 2b0b0fc84..920f63884 100644 --- a/libs/client-websocket/src/native.rs +++ b/libs/client-websocket/src/native.rs @@ -152,7 +152,7 @@ impl From for crate::Error { Error::Utf8 => crate::Error::Utf8, Error::AttackAttempt => crate::Error::AttackAttempt, Error::Url(inner) => crate::Error::Url(inner.into()), - Error::Http(inner) => crate::Error::Http(inner), + Error::Http(inner) => crate::Error::Http(inner.into()), Error::HttpFormat(inner) => crate::Error::HttpFormat(inner), } } diff --git a/libs/database-entity/src/dto.rs b/libs/database-entity/src/dto.rs index ea021bfda..873406e03 100644 --- a/libs/database-entity/src/dto.rs +++ b/libs/database-entity/src/dto.rs @@ -624,6 +624,8 @@ pub struct AFWorkspace { pub created_at: DateTime, pub icon: String, pub member_count: Option, + #[serde(default)] + pub role: Option, // role of the user requesting the workspace } #[derive(Serialize, Deserialize)] diff --git a/libs/database/src/chat/chat_ops.rs b/libs/database/src/chat/chat_ops.rs index e2b563b8c..c0fffded7 100644 --- a/libs/database/src/chat/chat_ops.rs +++ b/libs/database/src/chat/chat_ops.rs @@ -33,7 +33,6 @@ pub async fn insert_chat( ))); } let rag_ids = json!(params.rag_ids); - sqlx::query!( r#" INSERT INTO af_chat (chat_id, name, workspace_id, rag_ids) @@ -145,6 +144,25 @@ pub async fn select_chat<'a, E: Executor<'a, Database = Postgres>>( } } +pub async fn select_chat_rag_ids<'a, E: Executor<'a, Database = Postgres>>( + executor: E, + chat_id: &str, +) -> Result, AppError> { + let chat_id = Uuid::from_str(chat_id)?; + let row = sqlx::query!( + r#" + SELECT rag_ids + FROM af_chat + WHERE chat_id = $1 AND deleted_at IS NULL + "#, + &chat_id, + ) + .fetch_one(executor) + .await?; + let rag_ids = serde_json::from_value::>(row.rag_ids).unwrap_or_default(); + Ok(rag_ids) +} + pub async fn insert_answer_message_with_transaction( transaction: &mut Transaction<'_, Postgres>, author: ChatAuthor, diff --git a/libs/database/src/file/file_storage.rs b/libs/database/src/file/file_storage.rs index f3d2acb55..8d3c2b0c8 100644 --- a/libs/database/src/file/file_storage.rs +++ b/libs/database/src/file/file_storage.rs @@ -25,7 +25,7 @@ pub trait BucketClient { async fn put_blob(&self, object_key: &str, content: &[u8]) -> Result<(), AppError>; - async fn put_blob_as_content_type( + async fn put_blob_with_content_type( &self, object_key: &str, stream: ByteStream, diff --git a/libs/database/src/file/s3_client_impl.rs b/libs/database/src/file/s3_client_impl.rs index b847a1618..8b20e9488 100644 --- a/libs/database/src/file/s3_client_impl.rs +++ b/libs/database/src/file/s3_client_impl.rs @@ -4,12 +4,13 @@ use app_error::AppError; use async_trait::async_trait; use aws_sdk_s3::operation::delete_object::DeleteObjectOutput; -use aws_sdk_s3::error::SdkError; use std::ops::Deref; use std::time::{Duration, SystemTime}; +use aws_sdk_s3::error::SdkError; use aws_sdk_s3::operation::delete_objects::DeleteObjectsOutput; use aws_sdk_s3::operation::get_object::GetObjectError; + use aws_sdk_s3::presigning::PresigningConfig; use aws_sdk_s3::primitives::ByteStream; use aws_sdk_s3::types::{CompletedMultipartUpload, CompletedPart, Delete, ObjectIdentifier}; @@ -133,12 +134,17 @@ impl BucketClient for AwsS3BucketClientImpl { .body(body) .send() .await - .map_err(|err| anyhow!("Failed to upload object to S3: {}", err))?; + .map_err(|err| match err { + SdkError::TimeoutError(_) | SdkError::DispatchFailure(_) | SdkError::ServiceError(_) => { + AppError::ServiceTemporaryUnavailable(format!("Failed to upload object to S3: {}", err)) + }, + _ => AppError::Internal(anyhow!("Failed to upload object to S3: {}", err)), + })?; Ok(()) } - async fn put_blob_as_content_type( + async fn put_blob_with_content_type( &self, object_key: &str, stream: ByteStream, @@ -158,7 +164,12 @@ impl BucketClient for AwsS3BucketClientImpl { .content_type(content_type) .send() .await - .map_err(|err| anyhow!("Failed to upload object to S3: {}", err))?; + .map_err(|err| match err { + SdkError::TimeoutError(_) | SdkError::DispatchFailure(_) | SdkError::ServiceError(_) => { + AppError::ServiceTemporaryUnavailable(format!("Failed to upload object to S3: {}", err)) + }, + _ => AppError::Internal(anyhow!("Failed to upload object to S3: {}", err)), + })?; Ok(()) } diff --git a/libs/database/src/index/collab_embeddings_ops.rs b/libs/database/src/index/collab_embeddings_ops.rs index ee60901b9..1722b54d7 100644 --- a/libs/database/src/index/collab_embeddings_ops.rs +++ b/libs/database/src/index/collab_embeddings_ops.rs @@ -77,34 +77,35 @@ pub async fn upsert_collab_embeddings( .await?; } - for r in records { - sqlx::query( - r#"INSERT INTO af_collab_embeddings (fragment_id, oid, partition_key, content_type, content, embedding, indexed_at) + if !records.is_empty() { + // replace existing collab embeddings + remove_collab_embeddings(tx, &records[0].object_id).await?; + for r in records { + sqlx::query( + r#"INSERT INTO af_collab_embeddings (fragment_id, oid, partition_key, content_type, content, embedding, indexed_at) VALUES ($1, $2, $3, $4, $5, $6, NOW()) ON CONFLICT (fragment_id) DO UPDATE SET content_type = $4, content = $5, embedding = $6, indexed_at = NOW()"#, - ) - .bind(&r.fragment_id) - .bind(&r.object_id) - .bind(crate::collab::partition_key_from_collab_type(&r.collab_type)) - .bind(r.content_type as i32) - .bind(&r.content) - .bind(r.embedding.clone().map(Vector::from)) - .execute(tx.deref_mut()) - .await?; + ) + .bind(&r.fragment_id) + .bind(&r.object_id) + .bind(crate::collab::partition_key_from_collab_type(&r.collab_type)) + .bind(r.content_type as i32) + .bind(&r.content) + .bind(r.embedding.clone().map(Vector::from)) + .execute(tx.deref_mut()) + .await?; + } } Ok(()) } pub async fn remove_collab_embeddings( tx: &mut Transaction<'_, sqlx::Postgres>, - ids: &[String], + object_id: &str, ) -> Result<(), sqlx::Error> { - sqlx::query!( - "DELETE FROM af_collab_embeddings WHERE fragment_id IN (SELECT unnest($1::text[]))", - ids - ) - .execute(tx.deref_mut()) - .await?; + sqlx::query!("DELETE FROM af_collab_embeddings WHERE oid = $1", object_id) + .execute(tx.deref_mut()) + .await?; Ok(()) } diff --git a/libs/database/src/pg_row.rs b/libs/database/src/pg_row.rs index 3d2c88124..7e456bf24 100644 --- a/libs/database/src/pg_row.rs +++ b/libs/database/src/pg_row.rs @@ -53,6 +53,7 @@ impl TryFrom for AFWorkspace { created_at, icon, member_count: None, + role: None, }) } } @@ -98,6 +99,7 @@ impl TryFrom for AFWorkspace { created_at, icon, member_count: Some(value.member_count), + role: None, }) } } diff --git a/libs/database/src/publish.rs b/libs/database/src/publish.rs index 0b8fb3be3..61d0a36cb 100644 --- a/libs/database/src/publish.rs +++ b/libs/database/src/publish.rs @@ -213,27 +213,11 @@ pub async fn select_workspace_publish_namespace( Ok(res) } -#[inline] -pub async fn insert_or_replace_publish_collabs( - pg_pool: &PgPool, +async fn delete_published_collabs( + txn: &mut sqlx::Transaction<'_, Postgres>, workspace_id: &Uuid, - publisher_uuid: &Uuid, - publish_items: Vec>>, + publish_names: &[String], ) -> Result<(), AppError> { - let item_count = publish_items.len(); - let mut view_ids: Vec = Vec::with_capacity(item_count); - let mut publish_names: Vec = Vec::with_capacity(item_count); - let mut metadatas: Vec = Vec::with_capacity(item_count); - let mut blobs: Vec> = Vec::with_capacity(item_count); - publish_items.into_iter().for_each(|item| { - view_ids.push(item.meta.view_id); - publish_names.push(item.meta.publish_name); - metadatas.push(item.meta.metadata); - blobs.push(item.data); - }); - - let mut txn = pg_pool.begin().await?; - let delete_publish_names = sqlx::query_scalar!( r#" DELETE FROM af_published_collab @@ -252,6 +236,30 @@ pub async fn insert_or_replace_publish_collabs( delete_publish_names ); } + Ok(()) +} + +#[inline] +pub async fn insert_or_replace_publish_collabs( + pg_pool: &PgPool, + workspace_id: &Uuid, + publisher_uuid: &Uuid, + publish_items: Vec>>, +) -> Result<(), AppError> { + let item_count = publish_items.len(); + let mut view_ids: Vec = Vec::with_capacity(item_count); + let mut publish_names: Vec = Vec::with_capacity(item_count); + let mut metadatas: Vec = Vec::with_capacity(item_count); + let mut blobs: Vec> = Vec::with_capacity(item_count); + publish_items.into_iter().for_each(|item| { + view_ids.push(item.meta.view_id); + publish_names.push(item.meta.publish_name); + metadatas.push(item.meta.metadata); + blobs.push(item.data); + }); + + let mut txn = pg_pool.begin().await?; + delete_published_collabs(&mut txn, workspace_id, &publish_names).await?; let res = sqlx::query!( r#" @@ -354,6 +362,15 @@ pub async fn update_published_collabs( workspace_id: &Uuid, patches: &[PatchPublishedCollab], ) -> Result<(), AppError> { + { + // Delete existing published collab records with the same publish names + let publish_names: Vec = patches + .iter() + .filter_map(|patch| patch.publish_name.clone()) + .collect(); + delete_published_collabs(txn, workspace_id, &publish_names).await?; + } + for patch in patches { let new_publish_name = match &patch.publish_name { Some(new_publish_name) => new_publish_name, @@ -365,7 +382,7 @@ pub async fn update_published_collabs( UPDATE af_published_collab SET publish_name = $1 WHERE workspace_id = $2 - AND view_id = $3 + AND view_id = $3 "#, patch.publish_name, workspace_id, diff --git a/libs/database/src/workspace.rs b/libs/database/src/workspace.rs index 93a7119a8..7d30c7be9 100644 --- a/libs/database/src/workspace.rs +++ b/libs/database/src/workspace.rs @@ -526,7 +526,10 @@ pub async fn delete_workspace_members( .unwrap_or(false); if is_owner { - return Err(AppError::NotEnoughPermissions); + return Err(AppError::NotEnoughPermissions { + user: member_email.to_string(), + workspace_id: workspace_id.to_string(), + }); } sqlx::query!( @@ -851,10 +854,32 @@ pub async fn select_member_count_for_workspaces<'a, E: Executor<'a, Database = P }; ret.insert(row.workspace_id, count); } - for workspace_id in workspace_ids.iter() { - if !ret.contains_key(workspace_id) { - ret.insert(*workspace_id, 0); - } + + Ok(ret) +} + +pub async fn select_roles_for_workspaces( + pg_pool: &PgPool, + user_uuid: &Uuid, + workspace_ids: &[Uuid], +) -> Result, AppError> { + let query_res = sqlx::query!( + r#" + SELECT workspace_id, role_id + FROM af_workspace_member + WHERE workspace_id = ANY($1) + AND uid = (SELECT uid FROM public.af_user WHERE uuid = $2) + "#, + workspace_ids, + user_uuid, + ) + .fetch_all(pg_pool) + .await?; + + let mut ret = HashMap::with_capacity(workspace_ids.len()); + for row in query_res { + let role = AFRole::from(row.role_id); + ret.insert(row.workspace_id, role); } Ok(ret) @@ -1466,7 +1491,8 @@ pub async fn select_publish_name_exists( SELECT 1 FROM af_published_collab WHERE workspace_id = $1 - AND publish_name = $2 + AND publish_name = $2 + AND unpublished_at IS NULL ) "#, workspace_uuid, diff --git a/libs/mailer/src/config.rs b/libs/mailer/src/config.rs index 71acd8bee..9f0631998 100644 --- a/libs/mailer/src/config.rs +++ b/libs/mailer/src/config.rs @@ -5,5 +5,6 @@ pub struct MailerSetting { pub smtp_host: String, pub smtp_port: u16, pub smtp_username: String, + pub smtp_email: String, pub smtp_password: Secret, } diff --git a/libs/mailer/src/sender.rs b/libs/mailer/src/sender.rs index 14d51c132..13564f8e3 100644 --- a/libs/mailer/src/sender.rs +++ b/libs/mailer/src/sender.rs @@ -5,21 +5,23 @@ use lettre::transport::smtp::authentication::Credentials; use lettre::Address; use lettre::AsyncSmtpTransport; use lettre::AsyncTransport; +use secrecy::ExposeSecret; #[derive(Clone)] pub struct Mailer { smtp_transport: AsyncSmtpTransport, - smtp_username: String, + smtp_email: String, handlers: Handlebars<'static>, } impl Mailer { pub async fn new( smtp_username: String, - smtp_password: String, + smtp_email: String, + smtp_password: secrecy::Secret, smtp_host: &str, smtp_port: u16, ) -> Result { - let creds = Credentials::new(smtp_username.clone(), smtp_password); + let creds = Credentials::new(smtp_username, smtp_password.expose_secret().to_string()); let smtp_transport = AsyncSmtpTransport::::relay(smtp_host)? .credentials(creds) .port(smtp_port) @@ -27,7 +29,7 @@ impl Mailer { let handlers = Handlebars::new(); Ok(Self { smtp_transport, - smtp_username, + smtp_email, handlers, }) } @@ -64,7 +66,7 @@ impl Mailer { let email = Message::builder() .from(lettre::message::Mailbox::new( Some("AppFlowy Notification".to_string()), - self.smtp_username.parse::
()?, + self.smtp_email.parse::
()?, )) .to(lettre::message::Mailbox::new( recipient_name, diff --git a/libs/shared-entity/src/dto/workspace_dto.rs b/libs/shared-entity/src/dto/workspace_dto.rs index 4741a1ef9..68076a831 100644 --- a/libs/shared-entity/src/dto/workspace_dto.rs +++ b/libs/shared-entity/src/dto/workspace_dto.rs @@ -123,15 +123,29 @@ pub struct CollabResponse { pub object_id: String, } +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct Space { + pub view_id: String, +} + #[derive(Debug, Clone, Serialize, Deserialize)] pub struct Page { pub view_id: String, } +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct CreateSpaceParams { + pub space_permission: SpacePermission, + pub name: String, + pub space_icon: String, + pub space_icon_color: String, +} + #[derive(Debug, Clone, Serialize, Deserialize)] pub struct CreatePageParams { pub parent_view_id: String, pub layout: ViewLayout, + pub name: Option, } #[derive(Debug, Clone, Serialize, Deserialize)] @@ -258,9 +272,17 @@ impl Default for ViewLayout { } } +#[derive(Eq, PartialEq, Debug, Hash, Clone, Serialize_repr, Deserialize_repr)] +#[repr(u8)] +pub enum SpacePermission { + PublicToAll = 0, + Private = 1, +} + #[derive(Default, Debug, Deserialize, Serialize)] pub struct QueryWorkspaceParam { pub include_member_count: Option, + pub include_role: Option, } #[derive(Default, Debug, Deserialize, Serialize)] diff --git a/script/run_ci_server.sh b/script/run_ci_server.sh index 0f9235c9c..84b87f27c 100755 --- a/script/run_ci_server.sh +++ b/script/run_ci_server.sh @@ -48,7 +48,6 @@ else export RUST_LOG=trace export APPFLOWY_CLOUD_VERSION=$IMAGE_VERSION export APPFLOWY_WORKER_VERSION=$IMAGE_VERSION - export APPFLOWY_HISTORY_VERSION=$IMAGE_VERSION export APPFLOWY_ADMIN_FRONTEND_VERSION=$IMAGE_VERSION docker compose -f docker-compose-ci.yml pull diff --git a/services/appflowy-collaborate/Cargo.toml b/services/appflowy-collaborate/Cargo.toml index 2e3151a8f..96232c813 100644 --- a/services/appflowy-collaborate/Cargo.toml +++ b/services/appflowy-collaborate/Cargo.toml @@ -87,7 +87,11 @@ lazy_static = "1.4.0" itertools = "0.12.0" validator = "0.16.1" rayon.workspace = true +tiktoken-rs = "0.6.0" +unicode-segmentation = "1.9.0" + [dev-dependencies] rand = "0.8.5" workspace-template.workspace = true +unicode-normalization = "0.1.24" diff --git a/services/appflowy-collaborate/src/application.rs b/services/appflowy-collaborate/src/application.rs index fd670fe8a..645162730 100644 --- a/services/appflowy-collaborate/src/application.rs +++ b/services/appflowy-collaborate/src/application.rs @@ -74,7 +74,6 @@ pub async fn run_actix_server( )), state.metrics.realtime_metrics.clone(), rt_cmd_recv, - state.redis_connection_manager.clone(), Duration::from_secs(config.collab.group_persistence_interval_secs), config.collab.edit_state_max_count, config.collab.edit_state_max_secs, @@ -113,14 +112,6 @@ pub async fn init_state(config: &Config, rt_cmd_tx: CLCommandSender) -> Result Result, - access_control: AccessControl, -) { - tokio::spawn(async move { - while let Ok(change) = listener.recv().await { - match change.action_type { - CollabMemberAction::INSERT | CollabMemberAction::UPDATE => { - if let Some(member_row) = change.new { - let permission_row = select_permission(&pg_pool, &member_row.permission_id).await; - if let Ok(Some(row)) = permission_row { - if let Err(err) = access_control - .update_policy( - &member_row.uid, - ObjectType::Collab(&member_row.oid), - ActionVariant::FromAccessLevel(&row.access_level), - ) - .await - { - error!( - "Failed to update the user:{} collab{} access control, error: {}", - member_row.uid, member_row.oid, err - ); - } - } - } else { - error!("The new collab member is None") - } - }, - CollabMemberAction::DELETE => { - if let (Some(oid), Some(uid)) = (change.old_oid(), change.old_uid()) { - if let Err(err) = access_control - .remove_policy(uid, &ObjectType::Collab(oid)) - .await - { - warn!( - "Failed to remove the user:{} collab{} access control, error: {}", - uid, oid, err - ); - } - } else { - warn!("The oid or uid is None") - } - }, - } - } - }); -} - -#[allow(clippy::upper_case_acronyms)] -#[derive(Deserialize, Clone, Debug)] -pub enum CollabMemberAction { - INSERT, - UPDATE, - DELETE, -} - -#[derive(Deserialize, Debug, Clone)] -pub struct CollabMemberNotification { - /// The old will be None if the row does not exist before - pub old: Option, - /// The new will be None if the row is deleted - pub new: Option, - /// Represent the action of the database. Such as INSERT, UPDATE, DELETE - pub action_type: CollabMemberAction, -} - -impl CollabMemberNotification { - pub fn old_uid(&self) -> Option<&i64> { - self.old.as_ref().map(|o| &o.uid) - } - - pub fn old_oid(&self) -> Option<&str> { - self.old.as_ref().map(|o| o.oid.as_str()) - } - pub fn new_uid(&self) -> Option<&i64> { - self.new.as_ref().map(|n| &n.uid) - } - pub fn new_oid(&self) -> Option<&str> { - self.new.as_ref().map(|n| n.oid.as_str()) - } -} diff --git a/services/appflowy-collaborate/src/collab/queue.rs b/services/appflowy-collaborate/src/collab/queue.rs deleted file mode 100644 index 914b42f2b..000000000 --- a/services/appflowy-collaborate/src/collab/queue.rs +++ /dev/null @@ -1,577 +0,0 @@ -use std::collections::HashMap; -use std::ops::DerefMut; -use std::sync::atomic::AtomicI64; -use std::sync::Arc; -use std::time::Duration; - -use anyhow::{anyhow, Context}; -use bytes::Bytes; -use collab::lock::Mutex; -use collab_entity::CollabType; -use serde::{Deserialize, Serialize}; -use sqlx::PgPool; - -use tokio::time::{interval, sleep, sleep_until, Instant}; -use tracing::{error, instrument, trace, warn}; - -use app_error::AppError; -use database::collab::cache::CollabCache; -use database_entity::dto::{AFCollabEmbeddings, CollabParams, QueryCollab, QueryCollabResult}; - -use crate::collab::queue_redis_ops::{ - get_pending_meta, remove_pending_meta, storage_cache_key, PendingWrite, WritePriority, - PENDING_WRITE_META_EXPIRE_SECS, -}; -use crate::collab::RedisSortedSet; -use crate::metrics::CollabMetrics; -use crate::state::RedisConnectionManager; - -type PendingWriteSet = Arc; -#[derive(Clone)] -pub struct StorageQueue { - collab_cache: CollabCache, - connection_manager: RedisConnectionManager, - pending_write_set: PendingWriteSet, - pending_id_counter: Arc, - total_queue_collab_count: Arc, - success_queue_collab_count: Arc, -} - -pub const REDIS_PENDING_WRITE_QUEUE: &str = "collab_pending_write_queue_v0"; - -impl StorageQueue { - pub fn new( - collab_cache: CollabCache, - connection_manager: RedisConnectionManager, - queue_name: &str, - ) -> Self { - Self::new_with_metrics(collab_cache, connection_manager, queue_name, None) - } - - pub fn new_with_metrics( - collab_cache: CollabCache, - connection_manager: RedisConnectionManager, - queue_name: &str, - metrics: Option>, - ) -> Self { - let next_duration = Arc::new(Mutex::from(Duration::from_secs(1))); - let pending_id_counter = Arc::new(AtomicI64::new(0)); - let pending_write_set = Arc::new(RedisSortedSet::new(connection_manager.clone(), queue_name)); - - let total_queue_collab_count = Arc::new(AtomicI64::new(0)); - let success_queue_collab_count = Arc::new(AtomicI64::new(0)); - - // Spawns a task that periodically writes pending collaboration objects to the database. - spawn_period_write( - next_duration.clone(), - collab_cache.clone(), - connection_manager.clone(), - pending_write_set.clone(), - metrics.clone(), - total_queue_collab_count.clone(), - success_queue_collab_count.clone(), - ); - - spawn_period_check_pg_conn_count(collab_cache.pg_pool().clone(), next_duration); - - Self { - collab_cache, - connection_manager, - pending_write_set, - pending_id_counter, - total_queue_collab_count, - success_queue_collab_count, - } - } - - /// Enqueues a object for deferred processing. High priority writes are processed before low priority writes. - /// - /// adds a write task to a pending queue, which is periodically flushed by another task that batches - /// and writes the queued collaboration objects to a PostgreSQL database. - /// - /// This data is stored temporarily in the `collab_cache` and is intended for later persistent storage - /// in the database. It can also be retrieved during subsequent calls in the [CollabStorageImpl::get_encode_collab] - /// to enhance performance and reduce database reads. - /// - #[instrument(level = "trace", skip_all)] - pub async fn push( - &self, - workspace_id: &str, - uid: &i64, - params: &CollabParams, - priority: WritePriority, - ) -> Result<(), AppError> { - trace!("queuing {} object to pending write queue", params.object_id,); - self - .collab_cache - .insert_encode_collab_to_mem(params) - .await?; - - let seq = self - .pending_id_counter - .fetch_add(1, std::sync::atomic::Ordering::SeqCst); - - let pending_write = PendingWrite { - object_id: params.object_id.clone(), - seq, - data_len: params.encoded_collab_v1.len(), - priority, - }; - - let pending_meta = PendingWriteMeta { - uid: *uid, - workspace_id: workspace_id.to_string(), - object_id: params.object_id.clone(), - collab_type: params.collab_type.clone(), - embeddings: params.embeddings.clone(), - }; - - self - .total_queue_collab_count - .fetch_add(1, std::sync::atomic::Ordering::Relaxed); - // If the queueing fails, write the data to the database immediately - if let Err(err) = self - .queue_pending(params, pending_write, pending_meta) - .await - { - error!( - "Failed to queue pending write for object {}: {:?}", - params.object_id, err - ); - - let mut transaction = self - .collab_cache - .pg_pool() - .begin() - .await - .context("acquire transaction to upsert collab") - .map_err(AppError::from)?; - self - .collab_cache - .insert_encode_collab_data(workspace_id, uid, params, &mut transaction) - .await?; - transaction - .commit() - .await - .context("fail to commit the transaction to upsert collab") - .map_err(AppError::from)?; - } else { - self - .success_queue_collab_count - .fetch_add(1, std::sync::atomic::Ordering::Relaxed); - trace!( - "did queue {}:{} object for deferred writing to disk", - params.object_id, - seq - ); - } - - Ok(()) - } - - #[cfg(debug_assertions)] - pub async fn clear(&self) -> Result<(), AppError> { - self.pending_write_set.clear().await?; - crate::collab::queue_redis_ops::remove_all_pending_meta(self.connection_manager.clone()) - .await?; - Ok(()) - } - - #[inline] - async fn queue_pending( - &self, - params: &CollabParams, - pending_write: PendingWrite, - pending_write_meta: PendingWriteMeta, - ) -> Result<(), anyhow::Error> { - trace!( - "queue pending write: {}:{}", - pending_write_meta.object_id, - pending_write_meta.collab_type - ); - const MAX_RETRIES: usize = 3; - const BASE_DELAY_MS: u64 = 200; - const BACKOFF_FACTOR: u64 = 2; - - // these serialization seems very fast, so we don't need to worry about the performance and no - // need to use spawn_blocking or block_in_place - let pending_write_data = serde_json::to_vec(&pending_write)?; - let pending_write_meta_data = serde_json::to_vec(&pending_write_meta)?; - - let key = storage_cache_key(¶ms.object_id, params.encoded_collab_v1.len()); - let mut conn = self.connection_manager.clone(); - for attempt in 0..MAX_RETRIES { - let mut pipe = redis::pipe(); - // Prepare the pipeline with both commands - // 1. ZADD to add the pending write to the queue - // 2. SETEX to add the pending metadata to the cache - pipe - // .atomic() - .cmd("ZADD") - .arg(self.pending_write_set.queue_name()) - .arg(pending_write.score()) - .arg(&pending_write_data) - .ignore() - .cmd("SETEX") - .arg(&key) - .arg(PENDING_WRITE_META_EXPIRE_SECS) - .arg(&pending_write_meta_data) - .ignore(); - - match pipe.query_async::<_, ()>(&mut conn).await { - Ok(_) => return Ok(()), - Err(e) => { - if attempt == MAX_RETRIES - 1 { - return Err(e.into()); - } - - // 200ms, 400ms, 800ms - let delay = BASE_DELAY_MS * BACKOFF_FACTOR.pow(attempt as u32); - sleep(Duration::from_millis(delay)).await; - }, - } - } - Err(anyhow!("Failed to execute redis pipeline after retries")) - } -} - -/// Spawn a task that periodically checks the number of active connections in the PostgreSQL pool -/// It aims to adjust the write interval based on the number of active connections. -fn spawn_period_check_pg_conn_count(pg_pool: PgPool, next_duration: Arc>) { - let mut interval = interval(tokio::time::Duration::from_secs(10)); - tokio::spawn(async move { - loop { - interval.tick().await; - // these range values are arbitrary and can be adjusted as needed - match pg_pool.size() { - 0..=40 => { - *next_duration.lock().await = Duration::from_secs(1); - }, - _ => { - *next_duration.lock().await = Duration::from_secs(5); - }, - } - } - }); -} - -fn spawn_period_write( - next_duration: Arc>, - collab_cache: CollabCache, - connection_manager: RedisConnectionManager, - pending_write_set: PendingWriteSet, - metrics: Option>, - total_queue_collab_count: Arc, - success_queue_collab_count: Arc, -) { - let total_write_count = Arc::new(AtomicI64::new(0)); - let success_write_count = Arc::new(AtomicI64::new(0)); - tokio::spawn(async move { - loop { - // The next_duration will be changed by spawn_period_check_pg_conn_count. When the number of - // active connections is high, the interval will be longer. - let instant = Instant::now() + *next_duration.lock().await; - sleep_until(instant).await; - - if let Some(metrics) = metrics.as_ref() { - metrics.record_write_collab( - success_write_count.load(std::sync::atomic::Ordering::Relaxed), - total_write_count.load(std::sync::atomic::Ordering::Relaxed), - ); - - metrics.record_queue_collab( - success_queue_collab_count.load(std::sync::atomic::Ordering::Relaxed), - total_queue_collab_count.load(std::sync::atomic::Ordering::Relaxed), - ); - } - - let chunk_keys = consume_pending_write(&pending_write_set, 20, 5).await; - if chunk_keys.is_empty() { - continue; - } - - for keys in chunk_keys { - trace!( - "start writing {} pending collaboration data to disk", - keys.len() - ); - let cloned_collab_cache = collab_cache.clone(); - let mut cloned_connection_manager = connection_manager.clone(); - let cloned_total_write_count = total_write_count.clone(); - let cloned_total_success_write_count = success_write_count.clone(); - - if let Ok(metas) = get_pending_meta(&keys, &mut cloned_connection_manager).await { - if metas.is_empty() { - error!("the pending write keys is not empty, but metas is empty"); - return; - } - - match retry_write_pending_to_disk(&cloned_collab_cache, metas).await { - Ok(success_result) => { - #[cfg(debug_assertions)] - tracing::info!("success write pending: {:?}", keys,); - - trace!("{:?}", success_result); - cloned_total_write_count.fetch_add( - success_result.expected as i64, - std::sync::atomic::Ordering::Relaxed, - ); - cloned_total_success_write_count.fetch_add( - success_result.success as i64, - std::sync::atomic::Ordering::Relaxed, - ); - }, - Err(err) => error!("{:?}", err), - } - // Remove pending metadata from Redis even if some records fail to write to disk after retries. - // Records that fail repeatedly are considered potentially corrupt or invalid. - let _ = remove_pending_meta(&keys, &mut cloned_connection_manager).await; - } - } - } - }); -} - -async fn retry_write_pending_to_disk( - collab_cache: &CollabCache, - mut metas: Vec, -) -> Result { - const RETRY_DELAYS: [Duration; 2] = [Duration::from_secs(1), Duration::from_secs(2)]; - - let expected = metas.len(); - let mut successes = Vec::with_capacity(metas.len()); - - for &delay in RETRY_DELAYS.iter() { - match write_pending_to_disk(&metas, collab_cache).await { - Ok(success_write_objects) => { - if !success_write_objects.is_empty() { - successes.extend_from_slice(&success_write_objects); - metas.retain(|meta| !success_write_objects.contains(&meta.object_id)); - } - - // If there are no more metas to process, return the successes - if metas.is_empty() { - return Ok(WritePendingResult { - expected, - success: successes.len(), - fail: 0, - }); - } - }, - Err(err) => { - warn!( - "Error writing to disk: {:?}, retrying after {:?}", - err, delay - ); - }, - } - - // Only sleep if there are more attempts left - if !metas.is_empty() { - sleep(delay).await; - } - } - - if expected >= successes.len() { - Ok(WritePendingResult { - expected, - success: successes.len(), - fail: expected - successes.len(), - }) - } else { - Err(AppError::Internal(anyhow!( - "the len of expected is less than success" - ))) - } -} - -#[derive(Debug)] -struct WritePendingResult { - expected: usize, - success: usize, - #[allow(dead_code)] - fail: usize, -} - -async fn write_pending_to_disk( - pending_metas: &[PendingWriteMeta], - collab_cache: &CollabCache, -) -> Result, AppError> { - let mut success_write_objects = Vec::with_capacity(pending_metas.len()); - // Convert pending metadata into query parameters for batch fetching - let queries = pending_metas - .iter() - .map(QueryCollab::from) - .collect::>(); - - // Retrieve encoded collaboration data in batch - let results = collab_cache.batch_get_encode_collab(queries).await; - - // Create a mapping from object IDs to their corresponding metadata - let meta_map = pending_metas - .iter() - .map(|meta| (meta.object_id.clone(), meta)) - .collect::>(); - - // Prepare collaboration data for writing to the database - let records = results - .into_iter() - .filter_map(|(object_id, result)| { - if let QueryCollabResult::Success { encode_collab_v1 } = result { - meta_map.get(&object_id).map(|meta| PendingWriteData { - uid: meta.uid, - workspace_id: meta.workspace_id.clone(), - object_id: meta.object_id.clone(), - collab_type: meta.collab_type.clone(), - encode_collab_v1: encode_collab_v1.into(), - embeddings: meta.embeddings.clone(), - }) - } else { - None - } - }) - .collect::>(); - - // Start a database transaction - let mut transaction = collab_cache - .pg_pool() - .begin() - .await - .context("Failed to acquire transaction for writing pending collaboration data") - .map_err(AppError::from)?; - - // Insert each record into the database within the transaction context - let mut action_description = String::new(); - for (index, record) in records.into_iter().enumerate() { - let params = CollabParams { - object_id: record.object_id.clone(), - collab_type: record.collab_type, - encoded_collab_v1: record.encode_collab_v1, - embeddings: record.embeddings, - }; - action_description = format!("{}", params); - let savepoint_name = format!("sp_{}", index); - - // using savepoint to rollback the transaction if the insert fails - sqlx::query(&format!("SAVEPOINT {}", savepoint_name)) - .execute(transaction.deref_mut()) - .await?; - if let Err(_err) = collab_cache - .insert_encode_collab_to_disk(&record.workspace_id, &record.uid, params, &mut transaction) - .await - { - sqlx::query(&format!("ROLLBACK TO SAVEPOINT {}", savepoint_name)) - .execute(transaction.deref_mut()) - .await?; - } else { - success_write_objects.push(record.object_id); - } - } - - // Commit the transaction to finalize all writes - match tokio::time::timeout(Duration::from_secs(10), transaction.commit()).await { - Ok(result) => { - result.map_err(AppError::from)?; - Ok(success_write_objects) - }, - Err(_) => { - error!( - "Timeout waiting for committing the transaction for pending write:{}", - action_description - ); - Err(AppError::Internal(anyhow!( - "Timeout when committing the transaction for pending collaboration data" - ))) - }, - } -} - -const MAXIMUM_CHUNK_SIZE: usize = 5 * 1024 * 1024; -#[inline] -pub async fn consume_pending_write( - pending_write_set: &PendingWriteSet, - maximum_consume_item: usize, - num_of_item_each_chunk: usize, -) -> Vec> { - let mut chunks = Vec::new(); - let mut current_chunk = Vec::with_capacity(maximum_consume_item); - let mut current_chunk_data_size = 0; - - if let Ok(items) = pending_write_set.pop(maximum_consume_item).await { - #[cfg(debug_assertions)] - if !items.is_empty() { - trace!("Consuming {} pending write items", items.len()); - } - - for item in items { - let item_size = item.data_len; - // Check if adding this item would exceed the maximum chunk size or item limit - if current_chunk_data_size + item_size > MAXIMUM_CHUNK_SIZE - || current_chunk.len() >= num_of_item_each_chunk - { - if !current_chunk.is_empty() { - chunks.push(std::mem::take(&mut current_chunk)); - } - current_chunk_data_size = 0; - } - - // Add the item to the current batch and update the batch size - current_chunk.push(item); - current_chunk_data_size += item_size; - } - } - - if !current_chunk.is_empty() { - chunks.push(current_chunk); - } - // Convert each batch of items into a batch of keys - chunks - .into_iter() - .map(|batch| { - batch - .into_iter() - .map(|pending| storage_cache_key(&pending.object_id, pending.data_len)) - .collect() - }) - .collect() -} - -#[derive(Debug, PartialEq, Serialize, Deserialize)] -pub struct PendingWriteMeta { - pub uid: i64, - pub workspace_id: String, - pub object_id: String, - pub collab_type: CollabType, - #[serde(default)] - pub embeddings: Option, -} - -impl From<&PendingWriteMeta> for QueryCollab { - fn from(meta: &PendingWriteMeta) -> Self { - QueryCollab { - object_id: meta.object_id.clone(), - collab_type: meta.collab_type.clone(), - } - } -} - -#[derive(PartialEq, Debug)] -pub struct PendingWriteData { - pub uid: i64, - pub workspace_id: String, - pub object_id: String, - pub collab_type: CollabType, - pub encode_collab_v1: Bytes, - pub embeddings: Option, -} - -impl From for CollabParams { - fn from(data: PendingWriteData) -> Self { - CollabParams { - object_id: data.object_id, - collab_type: data.collab_type, - encoded_collab_v1: data.encode_collab_v1, - embeddings: data.embeddings, - } - } -} diff --git a/services/appflowy-collaborate/src/collab/queue_redis_ops.rs b/services/appflowy-collaborate/src/collab/queue_redis_ops.rs deleted file mode 100644 index 59c4c544e..000000000 --- a/services/appflowy-collaborate/src/collab/queue_redis_ops.rs +++ /dev/null @@ -1,418 +0,0 @@ -use crate::collab::queue::PendingWriteMeta; -use crate::state::RedisConnectionManager; -use app_error::AppError; -use futures_util::StreamExt; -use redis::{AsyncCommands, AsyncIter, Script}; -use serde::{Deserialize, Serialize}; -use serde_repr::{Deserialize_repr, Serialize_repr}; - -pub(crate) const PENDING_WRITE_META_EXPIRE_SECS: u64 = 604800; // 7 days in seconds - -#[allow(dead_code)] -pub(crate) async fn remove_all_pending_meta( - mut connection_manager: RedisConnectionManager, -) -> Result<(), AppError> { - let pattern = format!("{}*", QUEUE_COLLAB_PREFIX); - let iter: AsyncIter = connection_manager - .scan_match(pattern) - .await - .map_err(|err| AppError::Internal(err.into()))?; - let keys: Vec<_> = iter.collect().await; - - if keys.is_empty() { - return Ok(()); - } - connection_manager - .del(keys) - .await - .map_err(|err| AppError::Internal(err.into()))?; - Ok(()) -} - -#[inline] -pub(crate) async fn get_pending_meta( - keys: &[String], - connection_manager: &mut RedisConnectionManager, -) -> Result, AppError> { - let results: Vec>> = connection_manager - .get(keys) - .await - .map_err(|err| AppError::Internal(err.into()))?; - - let metas = results - .into_iter() - .filter_map(|value| value.and_then(|data| serde_json::from_slice(&data).ok())) - .collect::>(); - - Ok(metas) -} - -#[inline] -pub(crate) async fn remove_pending_meta( - keys: &[String], - connection_manager: &mut RedisConnectionManager, -) -> Result<(), AppError> { - connection_manager - .del(keys) - .await - .map_err(|err| AppError::Internal(err.into()))?; - Ok(()) -} - -pub(crate) const QUEUE_COLLAB_PREFIX: &str = "storage_pending_meta_v0:"; - -#[inline] -pub(crate) fn storage_cache_key(object_id: &str, data_len: usize) -> String { - format!("{}{}:{}", QUEUE_COLLAB_PREFIX, object_id, data_len) -} - -#[derive(Clone)] -pub struct RedisSortedSet { - conn: RedisConnectionManager, - name: String, -} - -impl RedisSortedSet { - pub fn new(conn: RedisConnectionManager, name: &str) -> Self { - Self { - conn, - name: name.to_string(), - } - } - - pub async fn push(&self, item: PendingWrite) -> Result<(), anyhow::Error> { - let data = serde_json::to_vec(&item)?; - redis::cmd("ZADD") - .arg(&self.name) - .arg(item.score()) - .arg(data) - .query_async(&mut self.conn.clone()) - .await?; - Ok(()) - } - - pub async fn push_with_conn( - &self, - item: PendingWrite, - conn: &mut RedisConnectionManager, - ) -> Result<(), anyhow::Error> { - let data = serde_json::to_vec(&item)?; - redis::cmd("ZADD") - .arg(&self.name) - .arg(item.score()) - .arg(data) - .query_async(conn) - .await?; - Ok(()) - } - - pub fn queue_name(&self) -> &str { - &self.name - } - - /// Pops items from a Redis sorted set. - /// - /// This asynchronous function retrieves and removes the top `len` items from a Redis sorted set specified by `self.name`. - /// It uses a Lua script to atomically perform the operation to maintain data integrity during concurrent access. - /// - /// # Parameters - /// - `len`: The number of items to pop from the sorted set. If `len` is 0, the function returns an empty vector. - /// - pub async fn pop(&self, len: usize) -> Result, anyhow::Error> { - if len == 0 { - return Ok(vec![]); - } - - let script = Script::new( - r#" - local items = redis.call('ZRANGE', KEYS[1], 0, ARGV[1], 'WITHSCORES') - if #items > 0 then - redis.call('ZREMRANGEBYRANK', KEYS[1], 0, #items / 2 - 1) - end - return items - "#, - ); - let mut conn = self.conn.clone(); - let items: Vec<(String, f64)> = script - .key(&self.name) - .arg(len - 1) - .invoke_async(&mut conn) - .await?; - - let results = items - .iter() - .map(|(data, _score)| serde_json::from_str::(data).map_err(|e| e.into())) - .collect::, anyhow::Error>>()?; - - Ok(results) - } - - pub async fn peek(&self, n: usize) -> Result, anyhow::Error> { - let mut conn = self.conn.clone(); - let items: Vec<(String, f64)> = redis::cmd("ZREVRANGE") - .arg(&self.name) - .arg(0) - .arg(n - 1) - .arg("WITHSCORES") - .query_async(&mut conn) - .await?; - - let results = items - .iter() - .map(|(data, _score)| serde_json::from_str::(data).map_err(|e| e.into())) - .collect::, anyhow::Error>>()?; - - Ok(results) - } - pub async fn remove_items>( - &self, - items_to_remove: Vec, - ) -> Result<(), anyhow::Error> { - let mut conn = self.conn.clone(); - let mut pipe = redis::pipe(); - for item in items_to_remove { - pipe.cmd("ZREM").arg(&self.name).arg(item.as_ref()).ignore(); - } - pipe.query_async::<_, ()>(&mut conn).await?; - Ok(()) - } - - pub async fn clear(&self) -> Result<(), anyhow::Error> { - let mut conn = self.conn.clone(); - conn.del(&self.name).await?; - Ok(()) - } -} - -#[derive(Clone, Serialize, Deserialize, Debug)] -pub struct PendingWrite { - pub object_id: String, - pub seq: i64, - pub data_len: usize, - pub priority: WritePriority, -} - -impl PendingWrite { - pub fn score(&self) -> i64 { - match self.priority { - WritePriority::High => 0, - WritePriority::Low => self.seq + 1, - } - } -} - -#[derive(Clone, Serialize_repr, Deserialize_repr, Debug)] -#[repr(u8)] -pub enum WritePriority { - High = 0, - Low = 1, -} - -impl Eq for PendingWrite {} -impl PartialEq for PendingWrite { - fn eq(&self, other: &Self) -> bool { - self.object_id == other.object_id - } -} - -impl Ord for PendingWrite { - fn cmp(&self, other: &Self) -> std::cmp::Ordering { - match (&self.priority, &other.priority) { - (WritePriority::High, WritePriority::Low) => std::cmp::Ordering::Greater, - (WritePriority::Low, WritePriority::High) => std::cmp::Ordering::Less, - _ => { - // Assuming lower seq is higher priority - other.seq.cmp(&self.seq) - }, - } - } -} - -impl PartialOrd for PendingWrite { - fn partial_cmp(&self, other: &Self) -> Option { - Some(self.cmp(other)) - } -} - -#[cfg(test)] -mod tests { - use crate::collab::{PendingWrite, RedisSortedSet, WritePriority}; - use anyhow::Context; - use std::time::Duration; - - #[tokio::test] - async fn pending_write_sorted_set_test() { - let conn = redis_client().await.get_connection_manager().await.unwrap(); - let set_name = uuid::Uuid::new_v4().to_string(); - let sorted_set = RedisSortedSet::new(conn.clone(), &set_name); - - let pending_writes = vec![ - PendingWrite { - object_id: "o1".to_string(), - seq: 1, - data_len: 0, - priority: WritePriority::Low, - }, - PendingWrite { - object_id: "o2".to_string(), - seq: 2, - data_len: 0, - priority: WritePriority::Low, - }, - PendingWrite { - object_id: "o3".to_string(), - seq: 0, - data_len: 0, - priority: WritePriority::High, - }, - ]; - - for item in &pending_writes { - sorted_set.push(item.clone()).await.unwrap(); - } - - let pending_writes_from_sorted_set = sorted_set.pop(3).await.unwrap(); - assert_eq!(pending_writes_from_sorted_set[0].object_id, "o3"); - assert_eq!(pending_writes_from_sorted_set[1].object_id, "o1"); - assert_eq!(pending_writes_from_sorted_set[2].object_id, "o2"); - - let items = sorted_set.pop(2).await.unwrap(); - assert!(items.is_empty()); - } - - #[tokio::test] - async fn sorted_set_consume_partial_items_test() { - let conn = redis_client().await.get_connection_manager().await.unwrap(); - let set_name = uuid::Uuid::new_v4().to_string(); - let sorted_set_1 = RedisSortedSet::new(conn.clone(), &set_name); - - let pending_writes = vec![ - PendingWrite { - object_id: "o1".to_string(), - seq: 1, - data_len: 0, - priority: WritePriority::Low, - }, - PendingWrite { - object_id: "o1".to_string(), - seq: 1, - data_len: 0, - priority: WritePriority::Low, - }, - PendingWrite { - object_id: "o2".to_string(), - seq: 2, - data_len: 0, - priority: WritePriority::Low, - }, - PendingWrite { - object_id: "o3".to_string(), - seq: 0, - data_len: 0, - priority: WritePriority::High, - }, - ]; - - for item in &pending_writes { - sorted_set_1.push(item.clone()).await.unwrap(); - } - - let pending_writes_from_sorted_set = sorted_set_1.pop(1).await.unwrap(); - assert_eq!(pending_writes_from_sorted_set[0].object_id, "o3"); - - let sorted_set_2 = RedisSortedSet::new(conn.clone(), &set_name); - let pending_writes_from_sorted_set = sorted_set_2.pop(10).await.unwrap(); - assert_eq!(pending_writes_from_sorted_set.len(), 2); - assert_eq!(pending_writes_from_sorted_set[0].object_id, "o1"); - assert_eq!(pending_writes_from_sorted_set[1].object_id, "o2"); - - assert!(sorted_set_1.pop(10).await.unwrap().is_empty()); - assert!(sorted_set_2.pop(10).await.unwrap().is_empty()); - } - - #[tokio::test] - async fn large_num_set_test() { - let conn = redis_client().await.get_connection_manager().await.unwrap(); - let set_name = uuid::Uuid::new_v4().to_string(); - let sorted_set = RedisSortedSet::new(conn.clone(), &set_name); - assert!(sorted_set.pop(10).await.unwrap().is_empty()); - - for i in 0..100 { - let pending_write = PendingWrite { - object_id: format!("o{}", i), - seq: i, - data_len: 0, - priority: WritePriority::Low, - }; - sorted_set.push(pending_write).await.unwrap(); - } - - let set_1 = sorted_set.pop(20).await.unwrap(); - assert_eq!(set_1.len(), 20); - assert_eq!(set_1[19].object_id, "o19"); - - let set_2 = sorted_set.pop(30).await.unwrap(); - assert_eq!(set_2.len(), 30); - assert_eq!(set_2[0].object_id, "o20"); - assert_eq!(set_2[29].object_id, "o49"); - - let set_3 = sorted_set.pop(1).await.unwrap(); - assert_eq!(set_3.len(), 1); - assert_eq!(set_3[0].object_id, "o50"); - - let set_4 = sorted_set.pop(200).await.unwrap(); - assert_eq!(set_4.len(), 49); - } - - #[tokio::test] - async fn multi_threads_sorted_set_test() { - let conn = redis_client().await.get_connection_manager().await.unwrap(); - let set_name = uuid::Uuid::new_v4().to_string(); - let sorted_set = RedisSortedSet::new(conn.clone(), &set_name); - - let mut handles = vec![]; - for i in 0..100 { - let cloned_sorted_set = sorted_set.clone(); - let handle = tokio::spawn(async move { - let pending_write = PendingWrite { - object_id: format!("o{}", i), - seq: i, - data_len: 0, - priority: WritePriority::Low, - }; - tokio::time::sleep(Duration::from_millis(rand::random::() % 100)).await; - cloned_sorted_set - .push(pending_write) - .await - .expect("Failed to push data") - }); - handles.push(handle); - } - futures::future::join_all(handles).await; - - let mut handles = vec![]; - for _ in 0..10 { - let cloned_sorted_set = sorted_set.clone(); - let handle = tokio::spawn(async move { - tokio::time::sleep(Duration::from_millis(rand::random::() % 100)).await; - let items = cloned_sorted_set - .pop(10) - .await - .expect("Failed to pop items"); - assert_eq!(items.len(), 10, "Expected exactly 10 items to be popped"); - }); - handles.push(handle); - } - let results = futures::future::join_all(handles).await; - for result in results { - result.expect("A thread panicked or errored out"); - } - } - - async fn redis_client() -> redis::Client { - let redis_uri = "redis://localhost:6379"; - redis::Client::open(redis_uri) - .context("failed to connect to redis") - .unwrap() - } -} diff --git a/services/appflowy-collaborate/src/collab/storage.rs b/services/appflowy-collaborate/src/collab/storage.rs index 00f0701a6..3a0c6c55c 100644 --- a/services/appflowy-collaborate/src/collab/storage.rs +++ b/services/appflowy-collaborate/src/collab/storage.rs @@ -1,7 +1,6 @@ -use std::collections::HashMap; -use std::sync::Arc; -use std::time::Duration; +#![allow(unused_imports)] +use anyhow::{anyhow, Context}; use async_trait::async_trait; use collab::entity::EncodedCollab; use collab_entity::CollabType; @@ -10,10 +9,15 @@ use database::collab::cache::CollabCache; use itertools::{Either, Itertools}; use rayon::iter::{IntoParallelIterator, ParallelIterator}; use sqlx::Transaction; - +use std::collections::HashMap; +use std::ops::DerefMut; +use std::sync::Arc; +use std::time::Duration; +use tokio::sync::mpsc::{channel, Receiver, Sender}; use tokio::time::timeout; use tracing::warn; use tracing::{error, instrument, trace}; +use uuid::Uuid; use validator::Validate; use crate::command::{CLCommandSender, CollaborationCommand}; @@ -28,15 +32,28 @@ use database_entity::dto::{ }; use crate::collab::access_control::CollabStorageAccessControlImpl; -use crate::collab::queue::{StorageQueue, REDIS_PENDING_WRITE_QUEUE}; -use crate::collab::queue_redis_ops::WritePriority; use crate::collab::validator::CollabValidator; use crate::metrics::CollabMetrics; use crate::snapshot::SnapshotControl; -use crate::state::RedisConnectionManager; pub type CollabAccessControlStorage = CollabStorageImpl; +struct PendingCollabWrite { + workspace_id: String, + uid: i64, + params: CollabParams, +} + +impl PendingCollabWrite { + fn new(workspace_id: String, uid: i64, params: CollabParams) -> Self { + PendingCollabWrite { + workspace_id, + uid, + params, + } + } +} + /// A wrapper around the actual storage implementation that provides access control and caching. #[derive(Clone)] pub struct CollabStorageImpl { @@ -45,7 +62,8 @@ pub struct CollabStorageImpl { access_control: AC, snapshot_control: SnapshotControl, rt_cmd_sender: CLCommandSender, - queue: Arc, + queue: Sender, + metrics: Arc, } impl CollabStorageImpl @@ -57,14 +75,13 @@ where access_control: AC, snapshot_control: SnapshotControl, rt_cmd_sender: CLCommandSender, - redis_conn_manager: RedisConnectionManager, metrics: Arc, ) -> Self { - let queue = Arc::new(StorageQueue::new_with_metrics( + let (queue, reader) = channel(1000); + tokio::spawn(Self::periodic_write_task( cache.clone(), - redis_conn_manager, - REDIS_PENDING_WRITE_QUEUE, - Some(metrics), + metrics.clone(), + reader, )); Self { cache, @@ -72,9 +89,117 @@ where snapshot_control, rt_cmd_sender, queue, + metrics, } } + const PENDING_WRITE_BUF_CAPACITY: usize = 20; + async fn periodic_write_task( + cache: CollabCache, + metrics: Arc, + mut reader: Receiver, + ) { + let mut buf = Vec::with_capacity(Self::PENDING_WRITE_BUF_CAPACITY); + loop { + let n = reader + .recv_many(&mut buf, Self::PENDING_WRITE_BUF_CAPACITY) + .await; + if n == 0 { + break; + } + let pending = buf.drain(..n); + if let Err(e) = Self::persist(&cache, &metrics, pending).await { + tracing::error!("failed to persist {} collabs: {}", n, e); + } + } + } + + async fn persist( + cache: &CollabCache, + metrics: &CollabMetrics, + records: impl ExactSizeIterator, + ) -> Result<(), AppError> { + // Start a database transaction + let mut transaction = cache + .pg_pool() + .begin() + .await + .context("Failed to acquire transaction for writing pending collaboration data") + .map_err(AppError::from)?; + + let total_records = records.len(); + let mut successful_writes = 0; + // Insert each record into the database within the transaction context + let mut action_description = String::new(); + for (index, record) in records.into_iter().enumerate() { + let params = record.params; + action_description = format!("{}", params); + let savepoint_name = format!("sp_{}", index); + + // using savepoint to rollback the transaction if the insert fails + sqlx::query(&format!("SAVEPOINT {}", savepoint_name)) + .execute(transaction.deref_mut()) + .await?; + if let Err(_err) = cache + .insert_encode_collab_to_disk(&record.workspace_id, &record.uid, params, &mut transaction) + .await + { + sqlx::query(&format!("ROLLBACK TO SAVEPOINT {}", savepoint_name)) + .execute(transaction.deref_mut()) + .await?; + } else { + successful_writes += 1; + } + } + + metrics.record_write_collab(successful_writes, total_records as _); + + // Commit the transaction to finalize all writes + match tokio::time::timeout(Duration::from_secs(10), transaction.commit()).await { + Ok(result) => { + result.map_err(AppError::from)?; + Ok(()) + }, + Err(_) => { + error!( + "Timeout waiting for committing the transaction for pending write:{}", + action_description + ); + Err(AppError::Internal(anyhow!( + "Timeout when committing the transaction for pending collaboration data" + ))) + }, + } + } + + async fn insert_collab( + &self, + workspace_id: &str, + uid: &i64, + params: CollabParams, + ) -> AppResult<()> { + // Start a database transaction + let mut transaction = self + .cache + .pg_pool() + .begin() + .await + .context("Failed to acquire transaction for writing pending collaboration data") + .map_err(AppError::from)?; + self + .cache + .insert_encode_collab_to_disk(workspace_id, uid, params, &mut transaction) + .await?; + tokio::time::timeout(Duration::from_secs(10), transaction.commit()) + .await + .map_err(|_| { + AppError::Internal(anyhow!( + "Timeout when committing the transaction for pending collaboration data" + )) + })??; + Ok(()) + } + async fn check_write_workspace_permission( &self, workspace_id: &str, @@ -178,7 +303,6 @@ where workspace_id: &str, uid: &i64, params: CollabParams, - priority: WritePriority, ) -> Result<(), AppError> { trace!( "Queue insert collab:{}:{}", @@ -192,11 +316,13 @@ where ))); } - self - .queue - .push(workspace_id, uid, ¶ms, priority) - .await - .map_err(AppError::from) + let pending = PendingCollabWrite::new(workspace_id.into(), *uid, params); + if let Err(e) = self.queue.send(pending).await { + tracing::error!("Failed to queue insert collab doc state: {}", e); + } else { + self.metrics.record_queue_collab(1); + } + Ok(()) } async fn batch_insert_collabs( @@ -259,14 +385,11 @@ where .update_policy(uid, ¶ms.object_id, AFAccessLevel::FullAccess) .await?; } - let priority = if write_immediately { - WritePriority::High + if write_immediately { + self.insert_collab(workspace_id, uid, params).await?; } else { - WritePriority::Low - }; - self - .queue_insert_collab(workspace_id, uid, params, priority) - .await?; + self.queue_insert_collab(workspace_id, uid, params).await?; + } Ok(()) } diff --git a/services/appflowy-collaborate/src/group/broadcast.rs b/services/appflowy-collaborate/src/group/broadcast.rs index 4eda7943d..b5820c031 100644 --- a/services/appflowy-collaborate/src/group/broadcast.rs +++ b/services/appflowy-collaborate/src/group/broadcast.rs @@ -47,7 +47,6 @@ pub struct CollabBroadcast { edit_state: Arc, /// The last modified time of the document. pub modified_at: Arc>, - update_streaming: Arc, } unsafe impl Send for CollabBroadcast {} @@ -71,9 +70,7 @@ impl CollabBroadcast { buffer_capacity: usize, edit_state: Arc, collab: &Collab, - update_streaming: impl CollabUpdateStreaming, ) -> Self { - let update_streaming = Arc::new(update_streaming); let object_id = object_id.to_owned(); // broadcast channel let (sender, _) = channel(buffer_capacity); @@ -84,7 +81,6 @@ impl CollabBroadcast { doc_subscription: Default::default(), edit_state, modified_at: Arc::new(parking_lot::Mutex::new(Instant::now())), - update_streaming, }; this.observe_collab_changes(collab); this @@ -97,7 +93,6 @@ impl CollabBroadcast { let broadcast_sink = self.broadcast_sender.clone(); let modified_at = self.modified_at.clone(); let edit_state = self.edit_state.clone(); - let update_streaming = self.update_streaming.clone(); // Observer the document's update and broadcast it to all subscribers. When one of the clients // sends an update to the document that alters its state, the document observer will trigger @@ -115,10 +110,6 @@ impl CollabBroadcast { origin ); - let stream_update = event.update.clone(); - if let Err(err) = update_streaming.send_update(stream_update) { - warn!("fail to send updates to redis:{}", err) - } let payload = gen_update_message(&event.update); let msg = BroadcastSync::new(origin, cloned_oid.clone(), payload, seq_num); if let Err(err) = broadcast_sink.send(msg.into()) { @@ -215,11 +206,12 @@ impl CollabBroadcast { trace!("[realtime]: send {} => {}", message, cloned_user.user_device()); if let Err(err) = sink.send(message).await { - error!("fail to broadcast message:{}", err); + warn!("fail to broadcast message:{}", err); } } - Err(e) => { - error!("fail to receive message:{}", e); + Err(_) => { + // Err(RecvError::Closed) is returned when all Sender halves have dropped, + // indicating that no further values can be sent on the channel. break; }, } diff --git a/services/appflowy-collaborate/src/group/group_init.rs b/services/appflowy-collaborate/src/group/group_init.rs index 04a3aeea8..6f363ecef 100644 --- a/services/appflowy-collaborate/src/group/group_init.rs +++ b/services/appflowy-collaborate/src/group/group_init.rs @@ -1,4 +1,3 @@ -use std::collections::VecDeque; use std::fmt::Display; use std::sync::atomic::{AtomicBool, AtomicI64, AtomicU32, Ordering}; use std::sync::Arc; @@ -12,22 +11,18 @@ use collab_entity::CollabType; use dashmap::DashMap; use futures_util::{SinkExt, StreamExt}; use tokio::sync::mpsc; -use tracing::{error, event, info, trace}; -use yrs::updates::decoder::Decode; -use yrs::updates::encoder::Encode; -use yrs::Update; +use tracing::{event, info, trace}; use collab_rt_entity::user::RealtimeUser; use collab_rt_entity::CollabMessage; use collab_rt_entity::MessageByObjectId; -use collab_stream::client::CollabRedisStream; + use collab_stream::error::StreamError; -use collab_stream::model::{CollabUpdateEvent, StreamBinary}; -use collab_stream::stream_group::StreamGroup; + use database::collab::CollabStorage; use crate::error::RealtimeError; -use crate::group::broadcast::{CollabBroadcast, CollabUpdateStreaming, Subscription}; +use crate::group::broadcast::{CollabBroadcast, Subscription}; use crate::group::persistence::GroupPersistence; use crate::indexer::Indexer; use crate::metrics::CollabRealtimeMetrics; @@ -65,7 +60,6 @@ impl CollabGroup { metrics_calculate: Arc, storage: Arc, is_new_collab: bool, - collab_redis_stream: Arc, persistence_interval: Duration, edit_state_max_count: u32, edit_state_max_secs: i64, @@ -81,13 +75,7 @@ impl CollabGroup { )); let broadcast = { let lock = collab.read().await; - CollabBroadcast::new( - &object_id, - 10, - edit_state.clone(), - &lock, - CollabUpdateStreamingImpl::new(&workspace_id, &object_id, &collab_redis_stream).await?, - ) + CollabBroadcast::new(&object_id, 1000, edit_state.clone(), &lock) }; let (destroy_group_tx, rx) = mpsc::channel(1); @@ -382,82 +370,6 @@ impl EditState { } } -struct CollabUpdateStreamingImpl { - sender: mpsc::UnboundedSender>, - stopped: Arc, -} - -impl CollabUpdateStreamingImpl { - async fn new( - workspace_id: &str, - object_id: &str, - collab_redis_stream: &CollabRedisStream, - ) -> Result { - let stream = collab_redis_stream - .collab_update_stream(workspace_id, object_id, "collaborate_update_producer") - .await?; - let stopped = Arc::new(AtomicBool::new(false)); - let (sender, receiver) = mpsc::unbounded_channel(); - let cloned_stopped = stopped.clone(); - tokio::spawn(async move { - if let Err(err) = Self::consume_messages(receiver, stream).await { - error!("Failed to consume incoming updates: {}", err); - } - cloned_stopped.store(true, Ordering::SeqCst); - }); - Ok(Self { sender, stopped }) - } - - async fn consume_messages( - mut receiver: mpsc::UnboundedReceiver>, - mut stream: StreamGroup, - ) -> Result<(), RealtimeError> { - while let Some(update) = receiver.recv().await { - let mut update_count = 1; - let update = { - let mut updates = VecDeque::new(); - // there may be already more messages inside waiting, try to read them all right away - while let Ok(update) = receiver.try_recv() { - updates.push_back(Update::decode_v1(&update)?); - } - if updates.is_empty() { - update // no following messages - } else { - update_count += updates.len(); - // prepend first update and merge them all together - updates.push_front(Update::decode_v1(&update)?); - Update::merge_updates(updates).encode_v1() - } - }; - - let msg = StreamBinary::try_from(CollabUpdateEvent::UpdateV1 { - encode_update: update, - })?; - stream.insert_messages(vec![msg]).await?; - trace!("Sent cumulative ({}) collab update to redis", update_count); - } - Ok(()) - } - - pub fn is_stopped(&self) -> bool { - self.stopped.load(Ordering::SeqCst) - } -} - -impl CollabUpdateStreaming for CollabUpdateStreamingImpl { - fn send_update(&self, update: Vec) -> Result<(), RealtimeError> { - if self.is_stopped() { - Err(RealtimeError::Internal(anyhow::anyhow!( - "stream stopped processing incoming updates" - ))) - } else if let Err(err) = self.sender.send(update) { - Err(RealtimeError::Internal(err.into())) - } else { - Ok(()) - } - } -} - #[cfg(test)] mod tests { use crate::group::group_init::EditState; diff --git a/services/appflowy-collaborate/src/group/manager.rs b/services/appflowy-collaborate/src/group/manager.rs index 9f2543ca2..20b38ab0a 100644 --- a/services/appflowy-collaborate/src/group/manager.rs +++ b/services/appflowy-collaborate/src/group/manager.rs @@ -4,18 +4,16 @@ use std::time::Duration; use collab::core::collab::DataSource; use collab::core::origin::CollabOrigin; use collab::entity::EncodedCollab; -use collab::lock::{Mutex, RwLock}; +use collab::lock::RwLock; use collab::preclude::Collab; use collab_entity::CollabType; -use tracing::{error, instrument, trace}; +use tracing::{instrument, trace}; use access_control::collab::RealtimeAccessControl; use app_error::AppError; use collab_rt_entity::user::RealtimeUser; use collab_rt_entity::CollabMessage; -use collab_stream::client::{CollabRedisStream, CONTROL_STREAM_KEY}; -use collab_stream::model::CollabControlEvent; -use collab_stream::stream_group::StreamGroup; + use database::collab::{CollabStorage, GetCollabOrigin}; use database_entity::dto::QueryCollabParams; @@ -31,8 +29,6 @@ pub struct GroupManager { storage: Arc, access_control: Arc, metrics_calculate: Arc, - collab_redis_stream: Arc, - control_event_stream: Arc>, persistence_interval: Duration, edit_state_max_count: u32, edit_state_max_secs: i64, @@ -48,25 +44,16 @@ where storage: Arc, access_control: Arc, metrics_calculate: Arc, - collab_stream: CollabRedisStream, persistence_interval: Duration, edit_state_max_count: u32, edit_state_max_secs: i64, indexer_provider: Arc, ) -> Result { - let collab_stream = Arc::new(collab_stream); - let control_event_stream = collab_stream - .collab_control_stream(CONTROL_STREAM_KEY, "collaboration") - .await - .map_err(|err| RealtimeError::Internal(err.into()))?; - let control_event_stream = Arc::new(Mutex::from(control_event_stream)); Ok(Self { - state: GroupManagementState::new(metrics_calculate.clone(), control_event_stream.clone()), + state: GroupManagementState::new(metrics_calculate.clone()), storage, access_control, metrics_calculate, - collab_redis_stream: collab_stream, - control_event_stream, persistence_interval, edit_state_max_count, edit_state_max_secs, @@ -162,7 +149,7 @@ where } let result = load_collab(user.uid, object_id, params, self.storage.clone()).await; - let (collab, encode_collab) = { + let (collab, _encode_collab) = { let (mut collab, encode_collab) = match result { Ok(value) => value, Err(err) => { @@ -184,26 +171,6 @@ where (collab, encode_collab) }; - let cloned_control_event_stream = self.control_event_stream.clone(); - let open_event = CollabControlEvent::Open { - workspace_id: workspace_id.to_string(), - object_id: object_id.to_string(), - collab_type: collab_type.clone(), - doc_state: encode_collab.doc_state.to_vec(), - }; - trace!("Send control event: {}", open_event); - - tokio::spawn(async move { - if let Err(err) = cloned_control_event_stream - .lock() - .await - .insert_message(open_event) - .await - { - error!("Failed to insert open event to control stream: {}", err); - } - }); - trace!( "[realtime]: create group: uid:{},workspace_id:{},object_id:{}:{}", user.uid, @@ -233,7 +200,6 @@ where self.metrics_calculate.clone(), self.storage.clone(), is_new_collab, - self.collab_redis_stream.clone(), self.persistence_interval, self.edit_state_max_count, self.edit_state_max_secs, diff --git a/services/appflowy-collaborate/src/group/persistence.rs b/services/appflowy-collaborate/src/group/persistence.rs index 60e77d05c..0337fa3a6 100644 --- a/services/appflowy-collaborate/src/group/persistence.rs +++ b/services/appflowy-collaborate/src/group/persistence.rs @@ -134,7 +134,7 @@ where let lock = collab.read().await; if let Some(indexer) = &self.indexer { - match indexer.embedding_params(&lock) { + match indexer.embedding_params(&lock).await { Ok(embedding_params) => { drop(lock); // we no longer need the lock match indexer.embeddings(embedding_params).await { diff --git a/services/appflowy-collaborate/src/group/state.rs b/services/appflowy-collaborate/src/group/state.rs index d2edd8ca1..16222801c 100644 --- a/services/appflowy-collaborate/src/group/state.rs +++ b/services/appflowy-collaborate/src/group/state.rs @@ -1,4 +1,3 @@ -use collab::lock::Mutex; use dashmap::mapref::one::RefMut; use dashmap::try_result::TryResult; use dashmap::DashMap; @@ -13,8 +12,6 @@ use crate::error::RealtimeError; use crate::group::group_init::CollabGroup; use crate::metrics::CollabRealtimeMetrics; use collab_rt_entity::user::RealtimeUser; -use collab_stream::model::CollabControlEvent; -use collab_stream::stream_group::StreamGroup; #[derive(Clone)] pub(crate) struct GroupManagementState { @@ -24,14 +21,10 @@ pub(crate) struct GroupManagementState { metrics_calculate: Arc, /// By default, the number of groups to remove in a single batch is 50. remove_batch_size: usize, - control_event_stream: Arc>, } impl GroupManagementState { - pub(crate) fn new( - metrics_calculate: Arc, - control_event_stream: Arc>, - ) -> Self { + pub(crate) fn new(metrics_calculate: Arc) -> Self { let remove_batch_size = get_env_var("APPFLOWY_COLLABORATE_REMOVE_BATCH_SIZE", "50") .parse::() .unwrap_or(50); @@ -40,7 +33,6 @@ impl GroupManagementState { editing_by_user: Arc::new(DashMap::new()), metrics_calculate, remove_batch_size, - control_event_stream, } } @@ -127,18 +119,6 @@ impl GroupManagementState { pub(crate) async fn remove_group(&self, object_id: &str) { let entry = self.group_by_object_id.remove(object_id); - if let Err(err) = self - .control_event_stream - .lock() - .await - .insert_message(CollabControlEvent::Close { - object_id: object_id.to_string(), - }) - .await - { - error!("Failed to insert close event to control stream: {}", err); - } - if let Some(entry) = entry { let group = entry.1; group.stop().await; diff --git a/services/appflowy-collaborate/src/indexer/document_indexer.rs b/services/appflowy-collaborate/src/indexer/document_indexer.rs index eda937a2d..d4d5e392b 100644 --- a/services/appflowy-collaborate/src/indexer/document_indexer.rs +++ b/services/appflowy-collaborate/src/indexer/document_indexer.rs @@ -4,32 +4,52 @@ use anyhow::anyhow; use async_trait::async_trait; use collab::preclude::Collab; -use collab_document::document::DocumentBody; -use collab_document::error::DocumentError; -use collab_entity::CollabType; - +use crate::indexer::{DocumentDataExt, Indexer}; use app_error::AppError; use appflowy_ai_client::client::AppFlowyAIClient; use appflowy_ai_client::dto::{ - EmbeddingEncodingFormat, EmbeddingInput, EmbeddingOutput, EmbeddingRequest, EmbeddingsModel, + EmbeddingEncodingFormat, EmbeddingInput, EmbeddingModel, EmbeddingOutput, EmbeddingRequest, }; +use collab_document::document::DocumentBody; +use collab_document::error::DocumentError; +use collab_entity::CollabType; use database_entity::dto::{AFCollabEmbeddingParams, AFCollabEmbeddings, EmbeddingContentType}; -use crate::indexer::{DocumentDataExt, Indexer}; +use crate::config::get_env_var; +use crate::indexer::open_ai::{split_text_by_max_content_len, split_text_by_max_tokens}; +use tiktoken_rs::CoreBPE; +use tracing::trace; +use uuid::Uuid; pub struct DocumentIndexer { ai_client: AppFlowyAIClient, + tokenizer: Arc, + embedding_model: EmbeddingModel, + use_tiktoken: bool, } impl DocumentIndexer { pub fn new(ai_client: AppFlowyAIClient) -> Arc { - Arc::new(Self { ai_client }) + let tokenizer = tiktoken_rs::cl100k_base().unwrap(); + let use_tiktoken = get_env_var("APPFLOWY_AI_CONTENT_SPLITTER_TIKTOKEN", "false") + .parse::() + .unwrap_or(false); + + Arc::new(Self { + ai_client, + tokenizer: Arc::new(tokenizer), + embedding_model: EmbeddingModel::TextEmbedding3Small, + use_tiktoken, + }) } } #[async_trait] impl Indexer for DocumentIndexer { - fn embedding_params(&self, collab: &Collab) -> Result, AppError> { + async fn embedding_params( + &self, + collab: &Collab, + ) -> Result, AppError> { let object_id = collab.object_id().to_string(); let document = DocumentBody::from_collab(collab).ok_or_else(|| { anyhow!( @@ -42,16 +62,15 @@ impl Indexer for DocumentIndexer { match result { Ok(document_data) => { let content = document_data.to_plain_text(); - let plain_text_param = AFCollabEmbeddingParams { - fragment_id: object_id.clone(), - object_id: object_id.clone(), - collab_type: CollabType::Document, - content_type: EmbeddingContentType::PlainText, + create_embedding( + object_id, content, - embedding: None, - }; - - Ok(vec![plain_text_param]) + CollabType::Document, + &self.embedding_model, + self.tokenizer.clone(), + self.use_tiktoken, + ) + .await }, Err(err) => { if matches!(err, DocumentError::NoRequiredData) { @@ -80,12 +99,17 @@ impl Indexer for DocumentIndexer { .ai_client .embeddings(EmbeddingRequest { input: EmbeddingInput::StringArray(contents), - model: EmbeddingsModel::TextEmbedding3Small.to_string(), + model: EmbeddingModel::TextEmbedding3Small.to_string(), chunk_size: 2000, encoding_format: EmbeddingEncodingFormat::Float, - dimensions: 1536, + dimensions: EmbeddingModel::TextEmbedding3Small.default_dimensions(), }) .await?; + trace!( + "[Embedding] request {} embeddings, received {} embeddings", + params.len(), + resp.data.len() + ); for embedding in resp.data { let param = &mut params[embedding.index as usize]; @@ -112,3 +136,46 @@ impl Indexer for DocumentIndexer { })) } } + +async fn create_embedding( + object_id: String, + content: String, + collab_type: CollabType, + embedding_model: &EmbeddingModel, + tokenizer: Arc, + use_tiktoken: bool, +) -> Result, AppError> { + let split_contents = if use_tiktoken { + let max_tokens = embedding_model.default_dimensions() as usize; + if content.len() < 500 { + split_text_by_max_tokens(content, max_tokens, tokenizer.as_ref())? + } else { + tokio::task::spawn_blocking(move || { + split_text_by_max_tokens(content, max_tokens, tokenizer.as_ref()) + }) + .await?? + } + } else { + debug_assert!(matches!( + embedding_model, + EmbeddingModel::TextEmbedding3Small + )); + // We assume that every token is ~4 bytes. We're going to split document content into fragments + // of ~2000 tokens each. + split_text_by_max_content_len(content, 8000)? + }; + + Ok( + split_contents + .into_iter() + .map(|content| AFCollabEmbeddingParams { + fragment_id: Uuid::new_v4().to_string(), + object_id: object_id.clone(), + collab_type: collab_type.clone(), + content_type: EmbeddingContentType::PlainText, + content, + embedding: None, + }) + .collect(), + ) +} diff --git a/services/appflowy-collaborate/src/indexer/mod.rs b/services/appflowy-collaborate/src/indexer/mod.rs index 09581298b..e354f9e01 100644 --- a/services/appflowy-collaborate/src/indexer/mod.rs +++ b/services/appflowy-collaborate/src/indexer/mod.rs @@ -1,5 +1,6 @@ mod document_indexer; mod ext; +mod open_ai; mod provider; pub use document_indexer::DocumentIndexer; diff --git a/services/appflowy-collaborate/src/indexer/open_ai.rs b/services/appflowy-collaborate/src/indexer/open_ai.rs new file mode 100644 index 000000000..db5fb26a3 --- /dev/null +++ b/services/appflowy-collaborate/src/indexer/open_ai.rs @@ -0,0 +1,361 @@ +use app_error::AppError; +use tiktoken_rs::CoreBPE; +use unicode_segmentation::UnicodeSegmentation; + +/// ## Execution Time Comparison Results +/// +/// The following results were observed when running `execution_time_comparison_tests`: +/// +/// | Content Size (chars) | Direct Time (ms) | spawn_blocking Time (ms) | +/// |-----------------------|------------------|--------------------------| +/// | 500 | 1 | 1 | +/// | 1000 | 2 | 2 | +/// | 2000 | 5 | 5 | +/// | 5000 | 11 | 11 | +/// | 20000 | 49 | 48 | +/// +/// ## Guidelines for Using `spawn_blocking` +/// +/// - **Short Tasks (< 1 ms)**: +/// Use direct execution on the async runtime. The minimal execution time has negligible impact. +/// +/// - **Moderate Tasks (1–10 ms)**: +/// - For infrequent or low-concurrency tasks, direct execution is acceptable. +/// - For frequent or high-concurrency tasks, consider using `spawn_blocking` to avoid delays. +/// +/// - **Long Tasks (> 10 ms)**: +/// Always offload to a blocking thread with `spawn_blocking` to maintain runtime efficiency and responsiveness. +/// +/// Related blog: +/// https://tokio.rs/blog/2020-04-preemption +/// https://ryhl.io/blog/async-what-is-blocking/ +#[inline] +pub fn split_text_by_max_tokens( + content: String, + max_tokens: usize, + tokenizer: &CoreBPE, +) -> Result, AppError> { + if content.is_empty() { + return Ok(vec![]); + } + + let token_ids = tokenizer.encode_ordinary(&content); + let total_tokens = token_ids.len(); + if total_tokens <= max_tokens { + return Ok(vec![content]); + } + + let mut chunks = Vec::new(); + let mut start_idx = 0; + while start_idx < total_tokens { + let mut end_idx = (start_idx + max_tokens).min(total_tokens); + let mut decoded = false; + // Try to decode the chunk, adjust end_idx if decoding fails + while !decoded { + let token_chunk = &token_ids[start_idx..end_idx]; + // Attempt to decode the current chunk + match tokenizer.decode(token_chunk.to_vec()) { + Ok(chunk_text) => { + chunks.push(chunk_text); + start_idx = end_idx; + decoded = true; + }, + Err(_) => { + // If we can extend the chunk, do so + if end_idx < total_tokens { + end_idx += 1; + } else if start_idx + 1 < total_tokens { + // Skip the problematic token at start_idx + start_idx += 1; + end_idx = (start_idx + max_tokens).min(total_tokens); + } else { + // Cannot decode any further, break to avoid infinite loop + start_idx = total_tokens; + break; + } + }, + } + } + } + + Ok(chunks) +} + +#[inline] +pub fn split_text_by_max_content_len( + content: String, + max_content_len: usize, +) -> Result, AppError> { + if content.is_empty() { + return Ok(vec![]); + } + + if content.len() <= max_content_len { + return Ok(vec![content]); + } + + // Content is longer than max_content_len; need to split + let mut result = Vec::with_capacity(1 + content.len() / max_content_len); + let mut fragment = String::with_capacity(max_content_len); + let mut current_len = 0; + + for grapheme in content.graphemes(true) { + let grapheme_len = grapheme.len(); + if current_len + grapheme_len > max_content_len { + if !fragment.is_empty() { + result.push(std::mem::take(&mut fragment)); + } + current_len = 0; + + if grapheme_len > max_content_len { + // Push the grapheme as a fragment on its own + result.push(grapheme.to_string()); + continue; + } + } + fragment.push_str(grapheme); + current_len += grapheme_len; + } + + // Add the last fragment if it's not empty + if !fragment.is_empty() { + result.push(fragment); + } + Ok(result) +} + +#[cfg(test)] +mod tests { + + use crate::indexer::open_ai::{split_text_by_max_content_len, split_text_by_max_tokens}; + use tiktoken_rs::cl100k_base; + + #[test] + fn test_split_at_non_utf8() { + let max_tokens = 10; // Small number for testing + + // Content with multibyte characters (emojis) + let content = "Hello 😃 World 🌍! This is a test 🚀.".to_string(); + let tokenizer = cl100k_base().unwrap(); + let params = split_text_by_max_tokens(content.clone(), max_tokens, &tokenizer).unwrap(); + for content in params { + assert!(content.is_char_boundary(0)); + assert!(content.is_char_boundary(content.len())); + } + + let params = split_text_by_max_content_len(content.clone(), max_tokens).unwrap(); + for content in params { + assert!(content.is_char_boundary(0)); + assert!(content.is_char_boundary(content.len())); + } + } + #[test] + fn test_exact_boundary_split() { + let max_tokens = 5; // Set to 5 tokens for testing + let content = "The quick brown fox jumps over the lazy dog".to_string(); + let tokenizer = cl100k_base().unwrap(); + let params = split_text_by_max_tokens(content.clone(), max_tokens, &tokenizer).unwrap(); + + let total_tokens = tokenizer.encode_ordinary(&content).len(); + let expected_fragments = (total_tokens + max_tokens - 1) / max_tokens; + assert_eq!(params.len(), expected_fragments); + } + + #[test] + fn test_content_shorter_than_max_len() { + let max_tokens = 100; + let content = "Short content".to_string(); + let tokenizer = cl100k_base().unwrap(); + let params = split_text_by_max_tokens(content.clone(), max_tokens, &tokenizer).unwrap(); + + assert_eq!(params.len(), 1); + assert_eq!(params[0], content); + } + + #[test] + fn test_empty_content() { + let max_tokens = 10; + let content = "".to_string(); + let tokenizer = cl100k_base().unwrap(); + let params = split_text_by_max_tokens(content.clone(), max_tokens, &tokenizer).unwrap(); + assert_eq!(params.len(), 0); + + let params = split_text_by_max_content_len(content.clone(), max_tokens).unwrap(); + assert_eq!(params.len(), 0); + } + + #[test] + fn test_content_with_only_multibyte_characters() { + let max_tokens = 1; // Set to 1 token for testing + let content = "😀😃😄😁😆".to_string(); + let tokenizer = cl100k_base().unwrap(); + let params = split_text_by_max_tokens(content.clone(), max_tokens, &tokenizer).unwrap(); + + let emojis: Vec = content.chars().map(|c| c.to_string()).collect(); + for (param, emoji) in params.iter().zip(emojis.iter()) { + assert_eq!(param, emoji); + } + + let params = split_text_by_max_content_len(content.clone(), max_tokens).unwrap(); + for (param, emoji) in params.iter().zip(emojis.iter()) { + assert_eq!(param, emoji); + } + } + + #[test] + fn test_split_with_combining_characters() { + let max_tokens = 1; // Set to 1 token for testing + let content = "a\u{0301}e\u{0301}i\u{0301}o\u{0301}u\u{0301}".to_string(); // "áéíóú" + + let tokenizer = cl100k_base().unwrap(); + let params = split_text_by_max_tokens(content.clone(), max_tokens, &tokenizer).unwrap(); + let total_tokens = tokenizer.encode_ordinary(&content).len(); + assert_eq!(params.len(), total_tokens); + let reconstructed_content = params.join(""); + assert_eq!(reconstructed_content, content); + + let params = split_text_by_max_content_len(content.clone(), max_tokens).unwrap(); + let reconstructed_content: String = params.concat(); + assert_eq!(reconstructed_content, content); + } + + #[test] + fn test_large_content() { + let max_tokens = 1000; + let content = "a".repeat(5000); // 5000 characters + let tokenizer = cl100k_base().unwrap(); + let params = split_text_by_max_tokens(content.clone(), max_tokens, &tokenizer).unwrap(); + + let total_tokens = tokenizer.encode_ordinary(&content).len(); + let expected_fragments = (total_tokens + max_tokens - 1) / max_tokens; + assert_eq!(params.len(), expected_fragments); + } + + #[test] + fn test_non_ascii_characters() { + let max_tokens = 2; + let content = "áéíóú".to_string(); + let tokenizer = cl100k_base().unwrap(); + let params = split_text_by_max_tokens(content.clone(), max_tokens, &tokenizer).unwrap(); + + let total_tokens = tokenizer.encode_ordinary(&content).len(); + let expected_fragments = (total_tokens + max_tokens - 1) / max_tokens; + assert_eq!(params.len(), expected_fragments); + let reconstructed_content: String = params.concat(); + assert_eq!(reconstructed_content, content); + + let params = split_text_by_max_content_len(content.clone(), max_tokens).unwrap(); + let reconstructed_content: String = params.concat(); + assert_eq!(reconstructed_content, content); + } + + #[test] + fn test_content_with_leading_and_trailing_whitespace() { + let max_tokens = 3; + let content = " abcde ".to_string(); + let tokenizer = cl100k_base().unwrap(); + let params = split_text_by_max_tokens(content.clone(), max_tokens, &tokenizer).unwrap(); + + let total_tokens = tokenizer.encode_ordinary(&content).len(); + let expected_fragments = (total_tokens + max_tokens - 1) / max_tokens; + assert_eq!(params.len(), expected_fragments); + let reconstructed_content: String = params.concat(); + assert_eq!(reconstructed_content, content); + + let params = split_text_by_max_content_len(content.clone(), max_tokens).unwrap(); + let reconstructed_content: String = params.concat(); + assert_eq!(reconstructed_content, content); + } + + #[test] + fn test_content_with_multiple_zero_width_joiners() { + let max_tokens = 1; + let content = "👩‍👩‍👧‍👧👨‍👨‍👦‍👦".to_string(); + let tokenizer = cl100k_base().unwrap(); + let params = split_text_by_max_tokens(content.clone(), max_tokens, &tokenizer).unwrap(); + let reconstructed_content: String = params.concat(); + assert_eq!(reconstructed_content, content); + + let params = split_text_by_max_content_len(content.clone(), max_tokens).unwrap(); + let reconstructed_content: String = params.concat(); + assert_eq!(reconstructed_content, content); + } + + #[test] + fn test_content_with_long_combining_sequences() { + let max_tokens = 1; + let content = "a\u{0300}\u{0301}\u{0302}\u{0303}\u{0304}".to_string(); + let tokenizer = cl100k_base().unwrap(); + let params = split_text_by_max_tokens(content.clone(), max_tokens, &tokenizer).unwrap(); + let reconstructed_content: String = params.concat(); + assert_eq!(reconstructed_content, content); + + let params = split_text_by_max_content_len(content.clone(), max_tokens).unwrap(); + let reconstructed_content: String = params.concat(); + assert_eq!(reconstructed_content, content); + } +} + +// #[cfg(test)] +// mod execution_time_comparison_tests { +// use crate::indexer::document_indexer::split_text_by_max_tokens; +// use rand::distributions::Alphanumeric; +// use rand::{thread_rng, Rng}; +// use std::sync::Arc; +// use std::time::Instant; +// use tiktoken_rs::{cl100k_base, CoreBPE}; +// +// #[tokio::test] +// async fn test_execution_time_comparison() { +// let tokenizer = Arc::new(cl100k_base().unwrap()); +// let max_tokens = 100; +// +// let sizes = vec![500, 1000, 2000, 5000, 20000]; // Content sizes to test +// for size in sizes { +// let content = generate_random_string(size); +// +// // Measure direct execution time +// let direct_time = measure_direct_execution(content.clone(), max_tokens, &tokenizer); +// +// // Measure spawn_blocking execution time +// let spawn_blocking_time = +// measure_spawn_blocking_execution(content, max_tokens, Arc::clone(&tokenizer)).await; +// +// println!( +// "Content Size: {} | Direct Time: {}ms | spawn_blocking Time: {}ms", +// size, direct_time, spawn_blocking_time +// ); +// } +// } +// +// // Measure direct execution time +// fn measure_direct_execution(content: String, max_tokens: usize, tokenizer: &CoreBPE) -> u128 { +// let start = Instant::now(); +// split_text_by_max_tokens(content, max_tokens, tokenizer).unwrap(); +// start.elapsed().as_millis() +// } +// +// // Measure `spawn_blocking` execution time +// async fn measure_spawn_blocking_execution( +// content: String, +// max_tokens: usize, +// tokenizer: Arc, +// ) -> u128 { +// let start = Instant::now(); +// tokio::task::spawn_blocking(move || { +// split_text_by_max_tokens(content, max_tokens, tokenizer.as_ref()).unwrap() +// }) +// .await +// .unwrap(); +// start.elapsed().as_millis() +// } +// +// pub fn generate_random_string(len: usize) -> String { +// let rng = thread_rng(); +// rng +// .sample_iter(&Alphanumeric) +// .take(len) +// .map(char::from) +// .collect() +// } +// } diff --git a/services/appflowy-collaborate/src/indexer/provider.rs b/services/appflowy-collaborate/src/indexer/provider.rs index 036c0ade1..f56b6a078 100644 --- a/services/appflowy-collaborate/src/indexer/provider.rs +++ b/services/appflowy-collaborate/src/indexer/provider.rs @@ -26,7 +26,10 @@ use database_entity::dto::{AFCollabEmbeddingParams, AFCollabEmbeddings, CollabPa #[async_trait] pub trait Indexer: Send + Sync { - fn embedding_params(&self, collab: &Collab) -> Result, AppError>; + async fn embedding_params( + &self, + collab: &Collab, + ) -> Result, AppError>; async fn embeddings( &self, @@ -46,7 +49,7 @@ pub trait Indexer: Send + Sync { false, ) .map_err(|err| AppError::Internal(err.into()))?; - let embedding_params = self.embedding_params(&collab)?; + let embedding_params = self.embedding_params(&collab).await?; self.embeddings(embedding_params).await } } diff --git a/services/appflowy-collaborate/src/metrics.rs b/services/appflowy-collaborate/src/metrics.rs index 3a37c40c1..33b21cd33 100644 --- a/services/appflowy-collaborate/src/metrics.rs +++ b/services/appflowy-collaborate/src/metrics.rs @@ -1,9 +1,9 @@ -use std::sync::Arc; -use std::time::Duration; - +use prometheus_client::metrics::counter::Counter; use prometheus_client::metrics::gauge::Gauge; use prometheus_client::metrics::histogram::Histogram; use prometheus_client::registry::Registry; +use std::sync::Arc; +use std::time::Duration; use tokio::time::interval; use database::collab::CollabStorage; @@ -146,10 +146,9 @@ where pub struct CollabMetrics { success_write_snapshot_count: Gauge, total_write_snapshot_count: Gauge, - success_write_collab_count: Gauge, - total_write_collab_count: Gauge, - total_queue_collab_count: Gauge, - success_queue_collab_count: Gauge, + success_write_collab_count: Counter, + total_write_collab_count: Counter, + success_queue_collab_count: Counter, } impl CollabMetrics { @@ -159,7 +158,6 @@ impl CollabMetrics { total_write_snapshot_count: Default::default(), success_write_collab_count: Default::default(), total_write_collab_count: Default::default(), - total_queue_collab_count: Default::default(), success_queue_collab_count: Default::default(), } } @@ -192,11 +190,6 @@ impl CollabMetrics { "success queue collab", metrics.success_queue_collab_count.clone(), ); - realtime_registry.register( - "total_queue_collab_count", - "total queue pending collab", - metrics.total_queue_collab_count.clone(), - ); metrics } @@ -206,13 +199,12 @@ impl CollabMetrics { self.total_write_snapshot_count.set(total_attempt); } - pub fn record_write_collab(&self, success_attempt: i64, total_attempt: i64) { - self.success_write_collab_count.set(success_attempt); - self.total_write_collab_count.set(total_attempt); + pub fn record_write_collab(&self, success_attempt: u64, total_attempt: u64) { + self.success_write_collab_count.inc_by(success_attempt); + self.total_write_collab_count.inc_by(total_attempt); } - pub fn record_queue_collab(&self, success_attempt: i64, total_attempt: i64) { - self.success_queue_collab_count.set(success_attempt); - self.total_queue_collab_count.set(total_attempt); + pub fn record_queue_collab(&self, attempt: u64) { + self.success_queue_collab_count.inc_by(attempt); } } diff --git a/services/appflowy-collaborate/src/rt_server.rs b/services/appflowy-collaborate/src/rt_server.rs index c8b762a04..4944650c7 100644 --- a/services/appflowy-collaborate/src/rt_server.rs +++ b/services/appflowy-collaborate/src/rt_server.rs @@ -13,7 +13,7 @@ use tracing::{error, info, trace}; use access_control::collab::RealtimeAccessControl; use collab_rt_entity::user::{RealtimeUser, UserDevice}; use collab_rt_entity::MessageByObjectId; -use collab_stream::client::CollabRedisStream; + use database::collab::CollabStorage; use crate::client::client_msg_router::ClientMessageRouter; @@ -26,7 +26,7 @@ use crate::group::manager::GroupManager; use crate::indexer::IndexerProvider; use crate::metrics::spawn_metrics; use crate::rt_server::collaboration_runtime::COLLAB_RUNTIME; -use crate::state::RedisConnectionManager; + use crate::{CollabRealtimeMetrics, RealtimeClientWebsocketSink}; #[derive(Clone)] @@ -50,7 +50,6 @@ where access_control: Arc, metrics: Arc, command_recv: CLCommandReceiver, - redis_connection_manager: RedisConnectionManager, group_persistence_interval: Duration, edit_state_max_count: u32, edit_state_max_secs: i64, @@ -67,13 +66,11 @@ where } let connect_state = ConnectState::new(); - let collab_stream = CollabRedisStream::new_with_connection_manager(redis_connection_manager); let group_manager = Arc::new( GroupManager::new( storage.clone(), access_control.clone(), metrics.clone(), - collab_stream, group_persistence_interval, edit_state_max_count, edit_state_max_secs, diff --git a/services/appflowy-history/Cargo.toml b/services/appflowy-history_deprecated/Cargo.toml similarity index 100% rename from services/appflowy-history/Cargo.toml rename to services/appflowy-history_deprecated/Cargo.toml diff --git a/services/appflowy-history/Dockerfile b/services/appflowy-history_deprecated/Dockerfile similarity index 100% rename from services/appflowy-history/Dockerfile rename to services/appflowy-history_deprecated/Dockerfile diff --git a/services/appflowy-history/deploy.env b/services/appflowy-history_deprecated/deploy.env similarity index 100% rename from services/appflowy-history/deploy.env rename to services/appflowy-history_deprecated/deploy.env diff --git a/services/appflowy-history/src/api.rs b/services/appflowy-history_deprecated/src/api.rs similarity index 100% rename from services/appflowy-history/src/api.rs rename to services/appflowy-history_deprecated/src/api.rs diff --git a/services/appflowy-history/src/application.rs b/services/appflowy-history_deprecated/src/application.rs similarity index 100% rename from services/appflowy-history/src/application.rs rename to services/appflowy-history_deprecated/src/application.rs diff --git a/services/appflowy-history/src/biz/history.rs b/services/appflowy-history_deprecated/src/biz/history.rs similarity index 100% rename from services/appflowy-history/src/biz/history.rs rename to services/appflowy-history_deprecated/src/biz/history.rs diff --git a/services/appflowy-history/src/biz/mod.rs b/services/appflowy-history_deprecated/src/biz/mod.rs similarity index 100% rename from services/appflowy-history/src/biz/mod.rs rename to services/appflowy-history_deprecated/src/biz/mod.rs diff --git a/services/appflowy-history/src/biz/persistence.rs b/services/appflowy-history_deprecated/src/biz/persistence.rs similarity index 100% rename from services/appflowy-history/src/biz/persistence.rs rename to services/appflowy-history_deprecated/src/biz/persistence.rs diff --git a/services/appflowy-history/src/biz/snapshot.rs b/services/appflowy-history_deprecated/src/biz/snapshot.rs similarity index 100% rename from services/appflowy-history/src/biz/snapshot.rs rename to services/appflowy-history_deprecated/src/biz/snapshot.rs diff --git a/services/appflowy-history/src/config.rs b/services/appflowy-history_deprecated/src/config.rs similarity index 100% rename from services/appflowy-history/src/config.rs rename to services/appflowy-history_deprecated/src/config.rs diff --git a/services/appflowy-history/src/core/manager.rs b/services/appflowy-history_deprecated/src/core/manager.rs similarity index 100% rename from services/appflowy-history/src/core/manager.rs rename to services/appflowy-history_deprecated/src/core/manager.rs diff --git a/services/appflowy-history/src/core/mod.rs b/services/appflowy-history_deprecated/src/core/mod.rs similarity index 100% rename from services/appflowy-history/src/core/mod.rs rename to services/appflowy-history_deprecated/src/core/mod.rs diff --git a/services/appflowy-history/src/core/open_handle.rs b/services/appflowy-history_deprecated/src/core/open_handle.rs similarity index 100% rename from services/appflowy-history/src/core/open_handle.rs rename to services/appflowy-history_deprecated/src/core/open_handle.rs diff --git a/services/appflowy-history/src/error.rs b/services/appflowy-history_deprecated/src/error.rs similarity index 100% rename from services/appflowy-history/src/error.rs rename to services/appflowy-history_deprecated/src/error.rs diff --git a/services/appflowy-history/src/lib.rs b/services/appflowy-history_deprecated/src/lib.rs similarity index 100% rename from services/appflowy-history/src/lib.rs rename to services/appflowy-history_deprecated/src/lib.rs diff --git a/services/appflowy-history/src/main.rs b/services/appflowy-history_deprecated/src/main.rs similarity index 100% rename from services/appflowy-history/src/main.rs rename to services/appflowy-history_deprecated/src/main.rs diff --git a/services/appflowy-history/src/models.rs b/services/appflowy-history_deprecated/src/models.rs similarity index 100% rename from services/appflowy-history/src/models.rs rename to services/appflowy-history_deprecated/src/models.rs diff --git a/services/appflowy-history/src/response.rs b/services/appflowy-history_deprecated/src/response.rs similarity index 100% rename from services/appflowy-history/src/response.rs rename to services/appflowy-history_deprecated/src/response.rs diff --git a/services/appflowy-history/tests/edit_test/mock.rs b/services/appflowy-history_deprecated/tests/edit_test/mock.rs similarity index 100% rename from services/appflowy-history/tests/edit_test/mock.rs rename to services/appflowy-history_deprecated/tests/edit_test/mock.rs diff --git a/services/appflowy-history/tests/edit_test/mod.rs b/services/appflowy-history_deprecated/tests/edit_test/mod.rs similarity index 100% rename from services/appflowy-history/tests/edit_test/mod.rs rename to services/appflowy-history_deprecated/tests/edit_test/mod.rs diff --git a/services/appflowy-history/tests/edit_test/recv_update_test.rs b/services/appflowy-history_deprecated/tests/edit_test/recv_update_test.rs similarity index 100% rename from services/appflowy-history/tests/edit_test/recv_update_test.rs rename to services/appflowy-history_deprecated/tests/edit_test/recv_update_test.rs diff --git a/services/appflowy-history/tests/main.rs b/services/appflowy-history_deprecated/tests/main.rs similarity index 100% rename from services/appflowy-history/tests/main.rs rename to services/appflowy-history_deprecated/tests/main.rs diff --git a/services/appflowy-history/tests/stream_test/control_stream_test.rs b/services/appflowy-history_deprecated/tests/stream_test/control_stream_test.rs similarity index 100% rename from services/appflowy-history/tests/stream_test/control_stream_test.rs rename to services/appflowy-history_deprecated/tests/stream_test/control_stream_test.rs diff --git a/services/appflowy-history/tests/stream_test/encode_test.rs b/services/appflowy-history_deprecated/tests/stream_test/encode_test.rs similarity index 100% rename from services/appflowy-history/tests/stream_test/encode_test.rs rename to services/appflowy-history_deprecated/tests/stream_test/encode_test.rs diff --git a/services/appflowy-history/tests/stream_test/mod.rs b/services/appflowy-history_deprecated/tests/stream_test/mod.rs similarity index 100% rename from services/appflowy-history/tests/stream_test/mod.rs rename to services/appflowy-history_deprecated/tests/stream_test/mod.rs diff --git a/services/appflowy-history/tests/stream_test/update_stream_test.rs b/services/appflowy-history_deprecated/tests/stream_test/update_stream_test.rs similarity index 100% rename from services/appflowy-history/tests/stream_test/update_stream_test.rs rename to services/appflowy-history_deprecated/tests/stream_test/update_stream_test.rs diff --git a/services/appflowy-history/tests/util.rs b/services/appflowy-history_deprecated/tests/util.rs similarity index 100% rename from services/appflowy-history/tests/util.rs rename to services/appflowy-history_deprecated/tests/util.rs diff --git a/services/appflowy-worker/src/application.rs b/services/appflowy-worker/src/application.rs index 0617d911b..55cc2106c 100644 --- a/services/appflowy-worker/src/application.rs +++ b/services/appflowy-worker/src/application.rs @@ -145,7 +145,8 @@ pub struct AppState { async fn get_worker_mailer(config: &Config) -> Result { let mailer = Mailer::new( config.mailer.smtp_username.clone(), - config.mailer.smtp_password.expose_secret().clone(), + config.mailer.smtp_email.clone(), + config.mailer.smtp_password.clone(), &config.mailer.smtp_host, config.mailer.smtp_port, ) diff --git a/services/appflowy-worker/src/config.rs b/services/appflowy-worker/src/config.rs index a5bcea03e..21282e43b 100644 --- a/services/appflowy-worker/src/config.rs +++ b/services/appflowy-worker/src/config.rs @@ -48,6 +48,12 @@ impl Config { mailer: MailerSetting { smtp_host: get_env_var("APPFLOWY_MAILER_SMTP_HOST", "smtp.gmail.com"), smtp_port: get_env_var("APPFLOWY_MAILER_SMTP_PORT", "465").parse()?, + smtp_email: get_env_var("APPFLOWY_MAILER_SMTP_EMAIL", "sender@example.com"), + // `smtp_username` could be the same as `smtp_email`, but may not have to be. + // For example: + // - Azure Communication services uses a string of the format .. + // - SendGrid uses the string apikey + // Adapted from: https://github.com/AppFlowy-IO/AppFlowy-Cloud/issues/984 smtp_username: get_env_var("APPFLOWY_MAILER_SMTP_USERNAME", "sender@example.com"), smtp_password: get_env_var("APPFLOWY_MAILER_SMTP_PASSWORD", "password").into(), }, diff --git a/services/appflowy-worker/src/error.rs b/services/appflowy-worker/src/error.rs index 6b363690e..77985ae65 100644 --- a/services/appflowy-worker/src/error.rs +++ b/services/appflowy-worker/src/error.rs @@ -13,6 +13,12 @@ pub enum WorkerError { #[error(transparent)] ImportError(#[from] ImportError), + #[error("S3 service unavailable: {0}")] + S3ServiceUnavailable(String), + + #[error("Redis stream group not exist: {0}")] + StreamGroupNotExist(String), + #[error(transparent)] Internal(#[from] anyhow::Error), } diff --git a/services/appflowy-worker/src/import_worker/worker.rs b/services/appflowy-worker/src/import_worker/worker.rs index 8dae81755..6be6767d1 100644 --- a/services/appflowy-worker/src/import_worker/worker.rs +++ b/services/appflowy-worker/src/import_worker/worker.rs @@ -3,7 +3,7 @@ use crate::s3_client::{download_file, AutoRemoveDownloadedFile, S3StreamResponse use anyhow::anyhow; use aws_sdk_s3::primitives::ByteStream; -use crate::error::ImportError; +use crate::error::{ImportError, WorkerError}; use crate::mailer::ImportNotionMailerParam; use crate::s3_client::S3Client; @@ -46,7 +46,7 @@ use database::pg_row::AFImportTask; use serde::{Deserialize, Serialize}; use serde_json::from_str; use sqlx::types::chrono; -use sqlx::types::chrono::{DateTime, Utc}; +use sqlx::types::chrono::{DateTime, TimeZone, Utc}; use sqlx::PgPool; use std::collections::{HashMap, HashSet}; use std::env::temp_dir; @@ -80,10 +80,7 @@ pub async fn run_import_worker( tick_interval_secs: u64, ) -> Result<(), ImportError> { info!("Starting importer worker"); - if let Err(err) = ensure_consumer_group(stream_name, GROUP_NAME, &mut redis_client) - .await - .map_err(ImportError::Internal) - { + if let Err(err) = ensure_consumer_group(stream_name, GROUP_NAME, &mut redis_client).await { error!("Failed to ensure consumer group: {:?}", err); } @@ -179,6 +176,7 @@ async fn process_upcoming_tasks( loop { interval.tick().await; + let tasks: StreamReadReply = match redis_client .xread_options(&[stream_name], &[">"], &options) .await @@ -186,6 +184,17 @@ async fn process_upcoming_tasks( Ok(tasks) => tasks, Err(err) => { error!("Failed to read tasks from Redis stream: {:?}", err); + + // Use command: + // docker exec -it appflowy-cloud-redis-1 redis-cli FLUSHDB to generate the error + // NOGROUP: No such key 'import_task_stream' or consumer group 'import_task_group' in XREADGROUP with GROUP option + if let Some(code) = err.code() { + if code == "NOGROUP" { + if let Err(err) = ensure_consumer_group(stream_name, GROUP_NAME, redis_client).await { + error!("Failed to ensure consumer group: {:?}", err); + } + } + } continue; }, }; @@ -198,6 +207,7 @@ async fn process_upcoming_tasks( Ok(import_task) => { let stream_name = stream_name.to_string(); let group_name = group_name.to_string(); + let context = TaskContext { storage_dir: storage_dir.to_path_buf(), redis_client: redis_client.clone(), @@ -206,7 +216,8 @@ async fn process_upcoming_tasks( notifier: notifier.clone(), metrics: metrics.clone(), }; - task_handlers.push(spawn_local(async move { + + let handle = spawn_local(async move { consume_task( context, import_task, @@ -216,7 +227,8 @@ async fn process_upcoming_tasks( ) .await?; Ok::<(), ImportError>(()) - })); + }); + task_handlers.push(handle); }, Err(err) => { error!("Failed to deserialize task: {:?}", err); @@ -253,38 +265,53 @@ async fn consume_task( entry_id: String, ) -> Result<(), ImportError> { if let ImportTask::Notion(task) = &mut import_task { - if let Some(created_at_timestamp) = task.created_at { - if is_task_expired(created_at_timestamp, task.last_process_at) { - if let Ok(import_record) = select_import_task(&context.pg_pool, &task.task_id).await { - handle_expired_task( - &mut context, - &import_record, - task, - stream_name, - group_name, - &entry_id, - ) - .await?; - } + // If no created_at timestamp, proceed directly to processing + if task.created_at.is_none() { + return process_and_ack_task(context, import_task, stream_name, group_name, &entry_id).await; + } - return Ok(()); - } else if !check_blob_existence(&context.s3_client, &task.s3_key).await? { - task.last_process_at = Some(Utc::now().timestamp()); - trace!("[Import] {} file not found, re-add task", task.workspace_id); - re_add_task( - &mut context.redis_client, + // Check if the task is expired + if let Err(err) = is_task_expired(task.created_at.unwrap(), task.last_process_at) { + if let Ok(import_record) = select_import_task(&context.pg_pool, &task.task_id).await { + handle_expired_task( + &mut context, + &import_record, + task, stream_name, group_name, - import_task, &entry_id, + &err, ) .await?; - return Ok(()); } + return Ok(()); } - } - process_and_ack_task(context, import_task, stream_name, group_name, &entry_id).await + // Check if the blob exists + if check_blob_existence(&context.s3_client, &task.s3_key).await? { + if task.last_process_at.is_none() { + task.last_process_at = Some(Utc::now().timestamp()); + } + process_and_ack_task(context, import_task, stream_name, group_name, &entry_id).await + } else { + info!( + "[Import] {} zip file not found, queue task", + task.workspace_id + ); + push_task( + &mut context.redis_client, + stream_name, + group_name, + import_task, + &entry_id, + ) + .await?; + Ok(()) + } + } else { + // If the task is not a notion task, proceed directly to processing + process_and_ack_task(context, import_task, stream_name, group_name, &entry_id).await + } } async fn handle_expired_task( @@ -294,10 +321,11 @@ async fn handle_expired_task( stream_name: &str, group_name: &str, entry_id: &str, + reason: &str, ) -> Result<(), ImportError> { info!( - "[Import]: {} import is expired, delete workspace", - task.workspace_id + "[Import]: {} import is expired with reason:{}", + task.workspace_id, reason ); update_import_task_status( @@ -311,6 +339,7 @@ async fn handle_expired_task( ImportError::Internal(e.into()) })?; remove_workspace(&import_record.workspace_id, &context.pg_pool).await; + info!("[Import]: deleted workspace {}", task.workspace_id); if let Err(err) = context.s3_client.delete_blob(task.s3_key.as_str()).await { error!( @@ -318,7 +347,12 @@ async fn handle_expired_task( task.workspace_id, err ); } - let _ = xack_task(&mut context.redis_client, stream_name, group_name, entry_id).await; + if let Err(err) = xack_task(&mut context.redis_client, stream_name, group_name, entry_id).await { + error!( + "[Import] failed to acknowledge task:{} error:{:?}", + task.workspace_id, err + ); + } notify_user( task, Err(ImportError::UploadFileExpire), @@ -353,36 +387,60 @@ async fn process_and_ack_task( result } -fn is_task_expired(timestamp: i64, last_process_at: Option) -> bool { - if last_process_at.is_none() { - return false; - } - - match DateTime::::from_timestamp(timestamp, 0) { - None => { - info!("[Import] failed to parse timestamp: {}", timestamp); - true - }, +fn is_task_expired(created_timestamp: i64, last_process_at: Option) -> Result<(), String> { + match Utc.timestamp_opt(created_timestamp, 0).single() { + None => Err(format!( + "[Import] failed to parse timestamp: {}", + created_timestamp + )), Some(created_at) => { let now = Utc::now(); if created_at > now { - error!( + return Err(format!( "[Import] created_at is in the future: {} > {}", - created_at, now - ); - return false; + created_at.format("%m/%d/%y %H:%M"), + now.format("%m/%d/%y %H:%M") + )); } let elapsed = now - created_at; - let minutes = get_env_var("APPFLOWY_WORKER_IMPORT_TASK_EXPIRE_MINUTES", "20") + let hours = get_env_var("APPFLOWY_WORKER_IMPORT_TASK_PROCESS_EXPIRE_HOURS", "6") .parse::() - .unwrap_or(20); - elapsed.num_minutes() >= minutes + .unwrap_or(6); + + if elapsed.num_hours() >= hours { + return Err(format!( + "task is expired: created_at: {}, last_process_at: {:?}, elapsed: {} hours", + created_at.format("%m/%d/%y %H:%M"), + last_process_at, + elapsed.num_hours() + )); + } + + if last_process_at.is_none() { + return Ok(()); + } + + let elapsed = now - created_at; + let minutes = get_env_var("APPFLOWY_WORKER_IMPORT_TASK_EXPIRE_MINUTES", "30") + .parse::() + .unwrap_or(30); + + if elapsed.num_minutes() >= minutes { + Err(format!( + "[Import] task is expired: created_at: {}, last_process_at: {:?}, elapsed: {} minutes", + created_at.format("%m/%d/%y %H:%M"), + last_process_at, + elapsed.num_minutes() + )) + } else { + Ok(()) + } }, } } -async fn re_add_task( +async fn push_task( redis_client: &mut ConnectionManager, stream_name: &str, group_name: &str, @@ -449,10 +507,7 @@ async fn process_task( .parse() .unwrap_or(false); - info!( - "[Import]: Processing task: {}, retry interval: {}, streaming: {}", - import_task, retry_interval, streaming - ); + info!("[Import]: Processing task: {}", import_task); match import_task { ImportTask::Notion(task) => { @@ -496,6 +551,16 @@ async fn process_task( clean_up(&context.s3_client, &task).await; notify_user(&task, result, context.notifier, &context.metrics).await?; + + tokio::spawn(async move { + match fs::remove_dir_all(&unzip_dir_path).await { + Ok(_) => info!( + "[Import]: {} deleted unzip file: {:?}", + task.workspace_id, unzip_dir_path + ), + Err(err) => error!("Failed to delete unzip file: {:?}", err), + } + }); }, Err(err) => { // If there is any errors when download or unzip the file, we will remove the file from S3 and notify the user. @@ -679,10 +744,9 @@ async fn download_and_unzip_file( .await .map_err(|err| ImportError::Internal(err.into()))??; - trace!( - "[Import] {} finish unzip file: {:?}", - import_task.workspace_id, - unzip_file.unzip_dir + info!( + "[Import] {} finish unzip file to dir:{}, file:{:?}", + import_task.workspace_id, unzip_file.dir_name, unzip_file.unzip_dir ); Ok(unzip_file.unzip_dir) } @@ -1067,17 +1131,6 @@ async fn process_unzip_file( batch_upload_files_to_s3(&import_task.workspace_id, s3_client, upload_resources) .await .map_err(|err| ImportError::Internal(anyhow!("Failed to upload files to S3: {:?}", err)))?; - - // 10. delete zip file regardless of success or failure - match fs::remove_dir_all(unzip_dir_path).await { - Ok(_) => trace!( - "[Import]: {} deleted unzip file: {:?}", - import_task.workspace_id, - unzip_dir_path - ), - Err(err) => error!("Failed to delete unzip file: {:?}", err), - } - Ok(()) } @@ -1181,11 +1234,12 @@ async fn batch_upload_files_to_s3( .buffer_unordered(5); let results: Vec<_> = upload_stream.collect().await; let errors: Vec<_> = results.into_iter().filter_map(Result::err).collect(); - if errors.is_empty() { - Ok(()) - } else { - Err(anyhow!("Some uploads failed: {:?}", errors)) + + if !errors.is_empty() { + error!("Some uploads failed: {:?}", errors); } + + Ok(()) } async fn upload_file_to_s3( @@ -1201,12 +1255,29 @@ async fn upload_file_to_s3( return Err(anyhow!("File does not exist: {:?}", path)); } + let mut attempt = 0; + let max_retries = 2; + let object_key = format!("{}/{}/{}", workspace_id, object_id, file_id); - let byte_stream = ByteStream::from_path(path).await?; - client - .put_blob(&object_key, byte_stream, Some(file_type)) - .await?; - Ok(()) + while attempt <= max_retries { + let byte_stream = ByteStream::from_path(path).await?; + match client + .put_blob(&object_key, byte_stream, Some(file_type)) + .await + { + Ok(_) => return Ok(()), + Err(WorkerError::S3ServiceUnavailable(_)) if attempt < max_retries => { + attempt += 1; + tokio::time::sleep(Duration::from_secs(3)).await; + }, + Err(err) => return Err(err.into()), + } + } + + Err(anyhow!( + "Failed to upload file to S3 after {} attempts", + max_retries + 1 + )) } async fn get_encode_collab_from_bytes( @@ -1233,7 +1304,7 @@ async fn ensure_consumer_group( stream_key: &str, group_name: &str, redis_client: &mut ConnectionManager, -) -> Result<(), anyhow::Error> { +) -> Result<(), WorkerError> { let result: RedisResult<()> = redis_client .xgroup_create_mkstream(stream_key, group_name, "0") .await; @@ -1241,11 +1312,15 @@ async fn ensure_consumer_group( if let Err(redis_error) = result { if let Some(code) = redis_error.code() { if code == "BUSYGROUP" { - return Ok(()); // Group already exists, considered as success. + return Ok(()); + } + + if code == "NOGROUP" { + return Err(WorkerError::StreamGroupNotExist(group_name.to_string())); } } error!("Error when creating consumer group: {:?}", redis_error); - return Err(redis_error.into()); + return Err(WorkerError::Internal(redis_error.into())); } Ok(()) diff --git a/services/appflowy-worker/src/mailer.rs b/services/appflowy-worker/src/mailer.rs index 5c448778b..db362085b 100644 --- a/services/appflowy-worker/src/mailer.rs +++ b/services/appflowy-worker/src/mailer.rs @@ -58,8 +58,9 @@ mod tests { #[tokio::test] async fn render_import_report() { let mailer = Mailer::new( - "test mailer".to_string(), - "123".to_string(), + "smtp_username".to_string(), + "stmp_email".to_string(), + "smtp_password".to_string().into(), "localhost", 465, ) diff --git a/services/appflowy-worker/src/s3_client.rs b/services/appflowy-worker/src/s3_client.rs index d5d109c1b..f4514683e 100644 --- a/services/appflowy-worker/src/s3_client.rs +++ b/services/appflowy-worker/src/s3_client.rs @@ -128,10 +128,18 @@ impl S3Client for S3ClientImpl { .await { Ok(_) => Ok(()), - Err(err) => Err(WorkerError::from(anyhow!( - "Failed to put object to S3: {}", - err - ))), + Err(err) => match err { + SdkError::TimeoutError(_) | SdkError::DispatchFailure(_) | SdkError::ServiceError(_) => { + Err(WorkerError::S3ServiceUnavailable(format!( + "Failed to upload object to S3: {}", + err + ))) + }, + _ => Err(WorkerError::Internal(anyhow!( + "Failed to upload object to S3: {}", + err + ))), + }, } } diff --git a/src/api/ai.rs b/src/api/ai.rs index 109fc59b4..60ba4cdb9 100644 --- a/src/api/ai.rs +++ b/src/api/ai.rs @@ -5,7 +5,8 @@ use actix_web::web::{Data, Json}; use actix_web::{web, HttpRequest, HttpResponse, Scope}; use app_error::AppError; use appflowy_ai_client::dto::{ - CompleteTextResponse, LocalAIConfig, TranslateRowParams, TranslateRowResponse, + CalculateSimilarityParams, CompleteTextResponse, LocalAIConfig, SimilarityResponse, + TranslateRowParams, TranslateRowResponse, }; use futures_util::{stream, TryStreamExt}; @@ -25,6 +26,9 @@ pub fn ai_completion_scope() -> Scope { .service(web::resource("/summarize_row").route(web::post().to(summarize_row_handler))) .service(web::resource("/translate_row").route(web::post().to(translate_row_handler))) .service(web::resource("/local/config").route(web::get().to(local_ai_config_handler))) + .service( + web::resource("/calculate_similarity").route(web::post().to(calculate_similarity_handler)), + ) } async fn complete_text_handler( @@ -163,3 +167,18 @@ async fn local_ai_config_handler( .map_err(|err| AppError::AIServiceUnavailable(err.to_string()))?; Ok(AppResponse::Ok().with_data(config).into()) } + +#[instrument(level = "debug", skip_all, err)] +async fn calculate_similarity_handler( + state: web::Data, + payload: web::Json, +) -> actix_web::Result>> { + let params = payload.into_inner(); + + let response = state + .ai_client + .calculate_similarity(params) + .await + .map_err(|err| AppError::AIServiceUnavailable(err.to_string()))?; + Ok(AppResponse::Ok().with_data(response).into()) +} diff --git a/src/api/chat.rs b/src/api/chat.rs index 62da371e1..8dbc3e0db 100644 --- a/src/api/chat.rs +++ b/src/api/chat.rs @@ -95,7 +95,6 @@ async fn create_chat_handler( ) -> actix_web::Result> { let workspace_id = path.into_inner(); let params = payload.into_inner(); - trace!("create new chat: {:?}", params); create_chat(&state.pg_pool, params, &workspace_id).await?; Ok(AppResponse::Ok().into()) } @@ -242,10 +241,11 @@ async fn answer_stream_handler( let (_workspace_id, chat_id, question_id) = path.into_inner(); let (content, metadata) = chat::chat_ops::select_chat_message_content(&state.pg_pool, question_id).await?; + let rag_ids = chat::chat_ops::select_chat_rag_ids(&state.pg_pool, &chat_id).await?; let ai_model = ai_model_from_header(&req); match state .ai_client - .stream_question(&chat_id, &content, Some(metadata), &ai_model) + .stream_question(&chat_id, &content, Some(metadata), rag_ids, &ai_model) .await { Ok(answer_stream) => { @@ -275,10 +275,25 @@ async fn answer_stream_v2_handler( let (_workspace_id, chat_id, question_id) = path.into_inner(); let (content, metadata) = chat::chat_ops::select_chat_message_content(&state.pg_pool, question_id).await?; + let rag_ids = chat::chat_ops::select_chat_rag_ids(&state.pg_pool, &chat_id).await?; let ai_model = ai_model_from_header(&req); + + trace!( + "[Chat] stream answer for chat: {}, question: {}, rag_ids: {:?}", + chat_id, + content, + rag_ids + ); match state .ai_client - .stream_question_v2(&chat_id, &content, Some(metadata), &ai_model) + .stream_question_v2( + &chat_id, + question_id, + &content, + Some(metadata), + rag_ids, + &ai_model, + ) .await { Ok(answer_stream) => { diff --git a/src/api/data_import.rs b/src/api/data_import.rs index 8b3d28a1e..50215e624 100644 --- a/src/api/data_import.rs +++ b/src/api/data_import.rs @@ -220,13 +220,8 @@ async fn import_data_handler( "User:{} import data:{} to new workspace:{}, name:{}", uid, file.size, workspace_id, file.name, ); - let stream = ByteStream::from_path(&file.file_path).await.map_err(|e| { - AppError::Internal(anyhow!("Failed to create ByteStream from file path: {}", e)) - })?; - state - .bucket_client - .put_blob_as_content_type(&workspace_id, stream, "application/zip") - .await?; + + upload_file_with_retry(&state, &workspace_id, &file.file_path).await?; // This task will be deserialized into ImportTask let task_id = Uuid::new_v4(); @@ -260,6 +255,38 @@ async fn import_data_handler( Ok(AppResponse::Ok().into()) } +async fn upload_file_with_retry( + state: &AppState, + workspace_id: &str, + file_path: &PathBuf, +) -> Result<(), AppError> { + let mut attempt = 0; + let max_retries = 3; + + while attempt <= max_retries { + let stream = ByteStream::from_path(file_path).await.map_err(|e| { + AppError::Internal(anyhow!("Failed to create ByteStream from file path: {}", e)) + })?; + let result = state + .bucket_client + .put_blob_with_content_type(workspace_id, stream, "application/zip") + .await; + + match result { + Ok(_) => return Ok(()), + Err(AppError::ServiceTemporaryUnavailable(_)) if attempt < max_retries => { + attempt += 1; + tokio::time::sleep(std::time::Duration::from_secs(5)).await; + }, + Err(err) => return Err(err), + } + } + + Err(AppError::ServiceTemporaryUnavailable( + "Failed to upload file to S3".to_string(), + )) +} + async fn check_maximum_task(state: &Data, uid: i64) -> Result<(), AppError> { let count = num_pending_task(uid, &state.pg_pool).await?; let maximum_pending_task = get_env_var("MAXIMUM_IMPORT_PENDING_TASK", "3") diff --git a/src/api/workspace.rs b/src/api/workspace.rs index 58f4837b8..20ee96f2e 100644 --- a/src/api/workspace.rs +++ b/src/api/workspace.rs @@ -49,8 +49,8 @@ use crate::biz::workspace::ops::{ get_reactions_on_published_view, remove_comment_on_published_view, remove_reaction_on_comment, }; use crate::biz::workspace::page_view::{ - create_page, get_page_view_collab, move_page_to_trash, restore_all_pages_from_trash, - restore_page_from_trash, update_page, update_page_collab_data, + create_page, create_space, get_page_view_collab, move_page_to_trash, + restore_all_pages_from_trash, restore_page_from_trash, update_page, update_page_collab_data, }; use crate::biz::workspace::publish::get_workspace_default_publish_view_info_meta; use crate::domain::compression::{ @@ -127,6 +127,7 @@ pub fn workspace_scope() -> Scope { web::resource("/v1/{workspace_id}/collab/{object_id}/web-update") .route(web::post().to(post_web_update_handler)), ) + .service(web::resource("/{workspace_id}/space").route(web::post().to(post_space_handler))) .service( web::resource("/{workspace_id}/page-view").route(web::post().to(post_page_view_handler)), ) @@ -338,10 +339,16 @@ async fn list_workspace_handler( state: Data, query: web::Query, ) -> Result>> { + let QueryWorkspaceParam { + include_member_count, + include_role, + } = query.into_inner(); + let workspaces = workspace::ops::get_all_user_workspaces( &state.pg_pool, &uuid, - query.into_inner().include_member_count.unwrap_or(false), + include_member_count.unwrap_or(false), + include_role.unwrap_or(false), ) .await?; Ok(AppResponse::Ok().with_data(workspaces).into()) @@ -895,6 +902,28 @@ async fn post_web_update_handler( Ok(Json(AppResponse::Ok())) } +async fn post_space_handler( + user_uuid: UserUuid, + path: web::Path, + payload: Json, + state: Data, +) -> Result>> { + let uid = state.user_cache.get_user_uid(&user_uuid).await?; + let workspace_uuid = path.into_inner(); + let space = create_space( + &state.pg_pool, + &state.collab_access_control_storage, + uid, + workspace_uuid, + &payload.space_permission, + &payload.name, + &payload.space_icon, + &payload.space_icon_color, + ) + .await?; + Ok(Json(AppResponse::Ok().with_data(space))) +} + async fn post_page_view_handler( user_uuid: UserUuid, path: web::Path, @@ -910,6 +939,7 @@ async fn post_page_view_handler( workspace_uuid, &payload.parent_view_id, &payload.layout, + payload.name.as_deref(), ) .await?; Ok(Json(AppResponse::Ok().with_data(page))) diff --git a/src/application.rs b/src/application.rs index 652a42759..f91947c79 100644 --- a/src/application.rs +++ b/src/application.rs @@ -132,7 +132,6 @@ pub async fn run_actix_server( state.realtime_access_control.clone(), state.metrics.realtime_metrics.clone(), rt_cmd_recv, - state.redis_connection_manager.clone(), Duration::from_secs(config.collab.group_persistence_interval_secs), config.collab.edit_state_max_count, config.collab.edit_state_max_secs, @@ -300,7 +299,6 @@ pub async fn init_state(config: &Config, rt_cmd_tx: CLCommandSender) -> Result Result { let mailer = Mailer::new( config.mailer.smtp_username.clone(), - config.mailer.smtp_password.expose_secret().clone(), + config.mailer.smtp_email.clone(), + config.mailer.smtp_password.clone(), &config.mailer.smtp_host, config.mailer.smtp_port, ) diff --git a/src/biz/access_request/ops.rs b/src/biz/access_request/ops.rs index b54abf972..eaef98183 100644 --- a/src/biz/access_request/ops.rs +++ b/src/biz/access_request/ops.rs @@ -79,8 +79,15 @@ pub async fn get_access_request( ) -> Result { let access_request_with_view_id = select_access_request_by_request_id(pg_pool, access_request_id).await?; + if access_request_with_view_id.workspace.owner_uid != user_uid { - return Err(AppError::NotEnoughPermissions); + return Err(AppError::NotEnoughPermissions { + user: user_uid.to_string(), + workspace_id: access_request_with_view_id + .workspace + .workspace_id + .to_string(), + }); } let folder = get_latest_collab_folder( collab_storage, diff --git a/src/biz/chat/ops.rs b/src/biz/chat/ops.rs index 484d5df7a..ef23df17a 100644 --- a/src/biz/chat/ops.rs +++ b/src/biz/chat/ops.rs @@ -17,7 +17,7 @@ use shared_entity::dto::chat_dto::{ CreateChatParams, GetChatMessageParams, RepeatedChatMessage, UpdateChatMessageContentParams, }; use sqlx::PgPool; -use tracing::{error, info}; +use tracing::{error, info, trace}; use appflowy_ai_client::dto::AIModel; use validator::Validate; @@ -28,6 +28,7 @@ pub(crate) async fn create_chat( workspace_id: &str, ) -> Result<(), AppError> { params.validate()?; + trace!("[Chat] create chat {:?}", params); let mut txn = pg_pool.begin().await?; insert_chat(&mut txn, workspace_id, params).await?; @@ -60,7 +61,13 @@ pub async fn update_chat_message( // TODO(nathan): query the metadata from the database let new_answer = ai_client - .send_question(¶ms.chat_id, ¶ms.content, &ai_model, None) + .send_question( + ¶ms.chat_id, + params.message_id, + ¶ms.content, + &ai_model, + None, + ) .await?; let _answer = insert_answer_message( pg_pool, @@ -85,7 +92,13 @@ pub async fn generate_chat_message_answer( let (content, metadata) = chat::chat_ops::select_chat_message_content(pg_pool, question_message_id).await?; let new_answer = ai_client - .send_question(chat_id, &content, &ai_model, Some(metadata)) + .send_question( + chat_id, + question_message_id, + &content, + &ai_model, + Some(metadata), + ) .await?; info!("new_answer: {:?}", new_answer); @@ -174,7 +187,7 @@ pub async fn create_chat_message_stream( match params.message_type { ChatMessageType::System => {} ChatMessageType::User => { - let answer = match ai_client.send_question(&chat_id, ¶ms.content, &ai_model, Some(json!(params.metadata))).await { + let answer = match ai_client.send_question(&chat_id,question_id, ¶ms.content, &ai_model, Some(json!(params.metadata))).await { Ok(response) => response, Err(err) => { error!("Failed to send question to AI: {}", err); diff --git a/src/biz/collab/folder_view.rs b/src/biz/collab/folder_view.rs index 783b22a7e..46b77b8b2 100644 --- a/src/biz/collab/folder_view.rs +++ b/src/biz/collab/folder_view.rs @@ -2,7 +2,9 @@ use std::collections::HashSet; use app_error::AppError; use chrono::DateTime; -use collab_folder::{Folder, SectionItem, ViewLayout as CollabFolderViewLayout}; +use collab_folder::{ + hierarchy_builder::SpacePermission, Folder, SectionItem, ViewLayout as CollabFolderViewLayout, +}; use shared_entity::dto::workspace_dto::{ self, FavoriteFolderView, FolderView, FolderViewMinimal, RecentFolderView, TrashFolderView, ViewLayout, @@ -301,3 +303,10 @@ pub fn to_folder_view_layout(layout: workspace_dto::ViewLayout) -> collab_folder ViewLayout::Chat => collab_folder::ViewLayout::Chat, } } + +pub fn to_space_permission(space_permission: &workspace_dto::SpacePermission) -> SpacePermission { + match space_permission { + workspace_dto::SpacePermission::PublicToAll => SpacePermission::PublicToAll, + workspace_dto::SpacePermission::Private => SpacePermission::Private, + } +} diff --git a/src/biz/pg_listener.rs b/src/biz/pg_listener.rs index e04ba53d0..cb26a8235 100644 --- a/src/biz/pg_listener.rs +++ b/src/biz/pg_listener.rs @@ -1,6 +1,4 @@ -use access_control::casbin::notification::WorkspaceMemberNotification; use anyhow::Error; -use appflowy_collaborate::collab::notification::CollabMemberNotification; use database::listener::PostgresDBListener; use database::pg_row::AFUserNotification; use sqlx::PgPool; @@ -31,6 +29,4 @@ impl PgListeners { } } -pub type CollabMemberListener = PostgresDBListener; pub type UserListener = PostgresDBListener; -pub type WorkspaceMemberListener = PostgresDBListener; diff --git a/src/biz/search/ops.rs b/src/biz/search/ops.rs index 239a42161..0938ac057 100644 --- a/src/biz/search/ops.rs +++ b/src/biz/search/ops.rs @@ -2,7 +2,7 @@ use crate::api::metrics::RequestMetrics; use app_error::ErrorCode; use appflowy_ai_client::client::AppFlowyAIClient; use appflowy_ai_client::dto::{ - EmbeddingEncodingFormat, EmbeddingInput, EmbeddingOutput, EmbeddingRequest, EmbeddingsModel, + EmbeddingEncodingFormat, EmbeddingInput, EmbeddingModel, EmbeddingOutput, EmbeddingRequest, }; use database::index::{search_documents, SearchDocumentParams}; @@ -25,10 +25,10 @@ pub async fn search_document( let embeddings = ai_client .embeddings(EmbeddingRequest { input: EmbeddingInput::String(request.query.clone()), - model: EmbeddingsModel::TextEmbedding3Small.to_string(), + model: EmbeddingModel::TextEmbedding3Small.to_string(), chunk_size: 500, encoding_format: EmbeddingEncodingFormat::Float, - dimensions: 1536, + dimensions: EmbeddingModel::TextEmbedding3Small.default_dimensions(), }) .await .map_err(|e| AppResponseError::new(ErrorCode::Internal, e.to_string()))?; @@ -64,7 +64,7 @@ pub async fn search_document( user_id: uid, workspace_id, limit: request.limit.unwrap_or(10) as i32, - preview: request.preview_size.unwrap_or(180) as i32, + preview: request.preview_size.unwrap_or(500) as i32, embedding, }, total_tokens, diff --git a/src/biz/template/ops.rs b/src/biz/template/ops.rs index 1a830af34..76539ffe8 100644 --- a/src/biz/template/ops.rs +++ b/src/biz/template/ops.rs @@ -448,7 +448,7 @@ pub async fn upload_avatar( let object_key = avatar_object_key(&file_id); client - .put_blob_as_content_type( + .put_blob_with_content_type( &object_key, ByteStream::from(avatar.data.to_vec()), &content_type, diff --git a/src/biz/workspace/ops.rs b/src/biz/workspace/ops.rs index 67419b239..25a16ce2d 100644 --- a/src/biz/workspace/ops.rs +++ b/src/biz/workspace/ops.rs @@ -256,6 +256,7 @@ pub async fn get_all_user_workspaces( pg_pool: &PgPool, user_uuid: &Uuid, include_member_count: bool, + include_role: bool, ) -> Result, AppResponseError> { let workspaces = select_all_user_workspaces(pg_pool, user_uuid).await?; let mut workspaces = workspaces @@ -273,10 +274,23 @@ pub async fn get_all_user_workspaces( .iter() .map(|row| row.workspace_id) .collect::>(); - let member_count_by_workspace_id = select_member_count_for_workspaces(pg_pool, &ids).await?; + let mut member_count_by_workspace_id = + select_member_count_for_workspaces(pg_pool, &ids).await?; for workspace in workspaces.iter_mut() { - if let Some(member_count) = member_count_by_workspace_id.get(&workspace.workspace_id) { - workspace.member_count = Some(*member_count); + if let Some(member_count) = member_count_by_workspace_id.remove(&workspace.workspace_id) { + workspace.member_count = Some(member_count); + } + } + } + if include_role { + let ids = workspaces + .iter() + .map(|row| row.workspace_id) + .collect::>(); + let mut roles_by_workspace_id = select_roles_for_workspaces(pg_pool, user_uuid, &ids).await?; + for workspace in workspaces.iter_mut() { + if let Some(role) = roles_by_workspace_id.remove(&workspace.workspace_id) { + workspace.role = Some(role.clone()); } } } diff --git a/src/biz/workspace/page_view.rs b/src/biz/workspace/page_view.rs index b6e1fd9ae..ef61a2024 100644 --- a/src/biz/workspace/page_view.rs +++ b/src/biz/workspace/page_view.rs @@ -16,8 +16,9 @@ use database::user::select_web_user_from_uid; use database_entity::dto::{CollabParams, QueryCollab, QueryCollabParams, QueryCollabResult}; use itertools::Itertools; use rayon::iter::{IntoParallelIterator, ParallelIterator}; +use serde_json::json; use shared_entity::dto::workspace_dto::{ - FolderView, Page, PageCollab, PageCollabData, ViewIcon, ViewLayout, + FolderView, Page, PageCollab, PageCollabData, Space, SpacePermission, ViewIcon, ViewLayout, }; use sqlx::{PgPool, Transaction}; use std::collections::{HashMap, HashSet}; @@ -29,6 +30,7 @@ use yrs::Update; use crate::api::metrics::AppFlowyWebMetrics; use crate::biz::collab::folder_view::{ parse_extra_field_as_json, to_dto_view_icon, to_dto_view_layout, to_folder_view_icon, + to_space_permission, }; use crate::biz::collab::{ folder_view::view_is_space, @@ -42,6 +44,56 @@ struct FolderUpdate { pub encoded_updates: Vec, } +#[allow(clippy::too_many_arguments)] +pub async fn create_space( + pg_pool: &PgPool, + collab_storage: &CollabAccessControlStorage, + uid: i64, + workspace_id: Uuid, + space_permission: &SpacePermission, + name: &str, + space_icon: &str, + space_color: &str, +) -> Result { + let default_document_collab_params = prepare_default_document_collab_param()?; + let view_id = default_document_collab_params.object_id.clone(); + let collab_origin = GetCollabOrigin::User { uid }; + let mut folder = + get_latest_collab_folder(collab_storage, collab_origin, &workspace_id.to_string()).await?; + let folder_update = add_new_space_to_folder( + uid, + &workspace_id.to_string(), + &view_id, + &mut folder, + space_permission, + name, + space_icon, + space_color, + ) + .await?; + let mut transaction = pg_pool.begin().await?; + let action = format!("Create new space: {}", view_id); + collab_storage + .insert_new_collab_with_transaction( + &workspace_id.to_string(), + &uid, + default_document_collab_params, + &mut transaction, + &action, + ) + .await?; + insert_and_broadcast_workspace_folder_update( + uid, + workspace_id, + folder_update, + collab_storage, + &mut transaction, + ) + .await?; + transaction.commit().await?; + Ok(Space { view_id }) +} + pub async fn create_page( pg_pool: &PgPool, collab_storage: &CollabAccessControlStorage, @@ -49,13 +101,22 @@ pub async fn create_page( workspace_id: Uuid, parent_view_id: &str, view_layout: &ViewLayout, + name: Option<&str>, ) -> Result { if *view_layout != ViewLayout::Document { return Err(AppError::InvalidRequest( "Only document layout is supported for page creation".to_string(), )); } - create_document_page(pg_pool, collab_storage, uid, workspace_id, parent_view_id).await + create_document_page( + pg_pool, + collab_storage, + uid, + workspace_id, + parent_view_id, + name, + ) + .await } fn prepare_default_document_collab_param() -> Result { @@ -75,19 +136,63 @@ fn prepare_default_document_collab_param() -> Result { }) } +#[allow(clippy::too_many_arguments)] +async fn add_new_space_to_folder( + uid: i64, + workspace_id: &str, + view_id: &str, + folder: &mut Folder, + space_permission: &SpacePermission, + name: &str, + space_icon: &str, + space_color: &str, +) -> Result { + let encoded_update = { + let view = NestedChildViewBuilder::new(uid, workspace_id.to_string()) + .with_view_id(view_id) + .with_name(name) + .with_extra(|builder| { + let mut extra = builder + .is_space(true, to_space_permission(space_permission)) + .build(); + extra["space_icon_color"] = json!(space_color); + extra["space_icon"] = json!(space_icon); + extra + }) + .build() + .view; + let mut txn = folder.collab.transact_mut(); + folder.body.views.insert(&mut txn, view, None); + if *space_permission == SpacePermission::Private { + folder + .body + .views + .update_view(&mut txn, view_id, |update| update.set_private(true).done()); + } + txn.encode_update_v1() + }; + Ok(FolderUpdate { + updated_encoded_collab: folder_to_encoded_collab(folder)?, + encoded_updates: encoded_update, + }) +} + async fn add_new_view_to_folder( uid: i64, parent_view_id: &str, view_id: &str, folder: &mut Folder, + name: Option<&str>, ) -> Result { let encoded_update = { let view = NestedChildViewBuilder::new(uid, parent_view_id.to_string()) .with_view_id(view_id) + .with_name(name.unwrap_or_default()) .build() .view; let mut txn = folder.collab.transact_mut(); folder.body.views.insert(&mut txn, view, None); + txn.encode_update_v1() }; Ok(FolderUpdate { @@ -234,13 +339,15 @@ async fn create_document_page( uid: i64, workspace_id: Uuid, parent_view_id: &str, + name: Option<&str>, ) -> Result { let default_document_collab_params = prepare_default_document_collab_param()?; let view_id = default_document_collab_params.object_id.clone(); let collab_origin = GetCollabOrigin::User { uid }; let mut folder = get_latest_collab_folder(collab_storage, collab_origin, &workspace_id.to_string()).await?; - let folder_update = add_new_view_to_folder(uid, parent_view_id, &view_id, &mut folder).await?; + let folder_update = + add_new_view_to_folder(uid, parent_view_id, &view_id, &mut folder, name).await?; let mut transaction = pg_pool.begin().await?; let action = format!("Create new collab: {}", view_id); collab_storage diff --git a/src/biz/workspace/publish.rs b/src/biz/workspace/publish.rs index e2c2d26fe..61d78df78 100644 --- a/src/biz/workspace/publish.rs +++ b/src/biz/workspace/publish.rs @@ -74,7 +74,7 @@ fn check_collab_publish_name(publish_name: &str) -> Result<(), AppError> { // Only contain alphanumeric characters and hyphens for c in publish_name.chars() { - if !c.is_alphanumeric() && c != '-' { + if !c.is_alphanumeric() && c != '-' && c != '_' { return Err(AppError::PublishNameInvalidCharacter { character: c }); } } @@ -246,8 +246,9 @@ pub async fn list_collab_publish_info( async fn check_workspace_namespace(new_namespace: &str) -> Result<(), AppError> { // Must be url safe // Only contain alphanumeric characters and hyphens + // and underscores (discouraged) for c in new_namespace.chars() { - if !c.is_alphanumeric() && c != '-' { + if !c.is_alphanumeric() && c != '-' && c != '_' { return Err(AppError::CustomNamespaceInvalidCharacter { character: c }); } } diff --git a/src/config/config.rs b/src/config/config.rs index 79c5d2814..419e447fc 100644 --- a/src/config/config.rs +++ b/src/config/config.rs @@ -261,6 +261,7 @@ pub fn get_configuration() -> Result { smtp_host: get_env_var("APPFLOWY_MAILER_SMTP_HOST", "smtp.gmail.com"), smtp_port: get_env_var("APPFLOWY_MAILER_SMTP_PORT", "465").parse()?, smtp_username: get_env_var("APPFLOWY_MAILER_SMTP_USERNAME", "sender@example.com"), + smtp_email: get_env_var("APPFLOWY_MAILER_SMTP_EMAIL", "sender@example.com"), smtp_password: get_env_var("APPFLOWY_MAILER_SMTP_PASSWORD", "password").into(), }, apple_oauth: AppleOAuthSetting { diff --git a/tests/collab/collab_curd_test.rs b/tests/collab/collab_curd_test.rs index fbf9bdaec..1d49f422b 100644 --- a/tests/collab/collab_curd_test.rs +++ b/tests/collab/collab_curd_test.rs @@ -37,24 +37,6 @@ async fn get_collab_response_compatible_test() { assert_eq!(collab_resp.encode_collab, encode_collab); } -#[tokio::test] -#[should_panic] -async fn create_collab_workspace_id_equal_to_object_id_test() { - let mut test_client = TestClient::new_user().await; - let workspace_id = test_client.workspace_id().await; - // Only the object with [CollabType::Folder] can have the same object_id as workspace_id. But - // it should use create workspace API - test_client - .create_collab_with_data( - workspace_id.clone(), - &workspace_id, - CollabType::Unknown, - None, - ) - .await - .unwrap() -} - #[tokio::test] async fn batch_insert_collab_with_empty_payload_test() { let mut test_client = TestClient::new_user().await; diff --git a/tests/collab/storage_test.rs b/tests/collab/storage_test.rs index d97614cd5..e983ad89b 100644 --- a/tests/collab/storage_test.rs +++ b/tests/collab/storage_test.rs @@ -1,32 +1,20 @@ -use std::collections::HashMap; -use std::sync::Arc; -use std::time::Duration; - +use app_error::ErrorCode; +use client_api_test::*; use collab::core::transaction::DocTransactionExtension; use collab::entity::EncodedCollab; -use collab::lock::Mutex; use collab::preclude::{Doc, Transact}; use collab_entity::CollabType; -use sqlx::types::Uuid; -use sqlx::PgPool; -use tokio::time::sleep; - -use app_error::ErrorCode; -use appflowy_collaborate::collab::queue::StorageQueue; -use appflowy_collaborate::collab::WritePriority; -use client_api_test::*; -use database::collab::cache::CollabCache; use database::collab::mem_cache::CollabMemCache; use database::collab::CollabMetadata; use database_entity::dto::{ - CollabParams, CreateCollabParams, DeleteCollabParams, QueryCollab, QueryCollabParams, - QueryCollabResult, + CreateCollabParams, DeleteCollabParams, QueryCollab, QueryCollabParams, QueryCollabResult, }; +use sqlx::types::Uuid; +use std::collections::HashMap; use workspace_template::document::getting_started::GettingStartedTemplate; use workspace_template::WorkspaceTemplateBuilder; -use crate::collab::util::{generate_random_bytes, redis_connection_manager, test_encode_collab_v1}; -use crate::sql_test::util::{setup_db, test_create_user}; +use crate::collab::util::{redis_connection_manager, test_encode_collab_v1}; #[tokio::test] async fn success_insert_collab_test() { @@ -436,152 +424,3 @@ async fn insert_folder_data_success_test() { test_client.api_client.create_collab(params).await.unwrap(); } } - -#[sqlx::test(migrations = false)] -async fn simulate_small_data_set_write(pool: PgPool) { - // prepare test prerequisites - setup_db(&pool).await.unwrap(); - setup_log(); - let conn = redis_connection_manager().await; - let user_uuid = uuid::Uuid::new_v4(); - let name = user_uuid.to_string(); - let email = format!("{}@appflowy.io", name); - let user = test_create_user(&pool, user_uuid, &email, &name) - .await - .unwrap(); - - let collab_cache = CollabCache::new(conn.clone(), pool); - let queue_name = uuid::Uuid::new_v4().to_string(); - let storage_queue = StorageQueue::new(collab_cache.clone(), conn, &queue_name); - - let queries = Arc::new(Mutex::new(Vec::new())); - for i in 0..10 { - // sleep random seconds less than 2 seconds. because the runtime is single-threaded, - // we need sleep a little time to let the runtime switch to other tasks. - sleep(Duration::from_millis(i % 2)).await; - - let cloned_storage_queue = storage_queue.clone(); - let cloned_queries = queries.clone(); - let cloned_user = user.clone(); - let encode_collab = EncodedCollab::new_v1( - generate_random_bytes(1024), - generate_random_bytes(1024 * 1024), - ); - let params = CollabParams { - object_id: format!("object_id_{}", i), - collab_type: CollabType::Unknown, - encoded_collab_v1: encode_collab.encode_to_bytes().unwrap().into(), - embeddings: None, - }; - cloned_storage_queue - .push( - &cloned_user.workspace_id, - &cloned_user.uid, - ¶ms, - WritePriority::Low, - ) - .await - .unwrap(); - cloned_queries.lock().await.push((params, encode_collab)); - } - - // Allow some time for processing - sleep(Duration::from_secs(30)).await; - - // Check that all items are processed correctly - for (params, original_encode_collab) in queries.lock().await.iter() { - let query = QueryCollab { - object_id: params.object_id.clone(), - collab_type: params.collab_type.clone(), - }; - let encode_collab_from_disk = collab_cache - .get_encode_collab_from_disk(query) - .await - .unwrap(); - - assert_eq!( - encode_collab_from_disk.doc_state.len(), - original_encode_collab.doc_state.len(), - "doc_state length mismatch" - ); - assert_eq!( - encode_collab_from_disk.doc_state, - original_encode_collab.doc_state - ); - - assert_eq!( - encode_collab_from_disk.state_vector.len(), - original_encode_collab.state_vector.len(), - "state_vector length mismatch" - ); - assert_eq!( - encode_collab_from_disk.state_vector, - original_encode_collab.state_vector - ); - } -} - -#[sqlx::test(migrations = false)] -async fn simulate_large_data_set_write(pool: PgPool) { - // prepare test prerequisites - setup_db(&pool).await.unwrap(); - setup_log(); - - let conn = redis_connection_manager().await; - let user_uuid = uuid::Uuid::new_v4(); - let name = user_uuid.to_string(); - let email = format!("{}@appflowy.io", name); - let user = test_create_user(&pool, user_uuid, &email, &name) - .await - .unwrap(); - - let collab_cache = CollabCache::new(conn.clone(), pool); - let queue_name = uuid::Uuid::new_v4().to_string(); - let storage_queue = StorageQueue::new(collab_cache.clone(), conn, &queue_name); - - let origin_encode_collab = EncodedCollab::new_v1( - generate_random_bytes(10 * 1024), - generate_random_bytes(2 * 1024 * 1024), - ); - let params = CollabParams { - object_id: uuid::Uuid::new_v4().to_string(), - collab_type: CollabType::Unknown, - encoded_collab_v1: origin_encode_collab.encode_to_bytes().unwrap().into(), - embeddings: None, - }; - storage_queue - .push(&user.workspace_id, &user.uid, ¶ms, WritePriority::Low) - .await - .unwrap(); - - // Allow some time for processing - sleep(Duration::from_secs(30)).await; - - let query = QueryCollab { - object_id: params.object_id.clone(), - collab_type: params.collab_type.clone(), - }; - let encode_collab_from_disk = collab_cache - .get_encode_collab_from_disk(query) - .await - .unwrap(); - assert_eq!( - encode_collab_from_disk.doc_state.len(), - origin_encode_collab.doc_state.len(), - "doc_state length mismatch" - ); - assert_eq!( - encode_collab_from_disk.doc_state, - origin_encode_collab.doc_state - ); - - assert_eq!( - encode_collab_from_disk.state_vector.len(), - origin_encode_collab.state_vector.len(), - "state_vector length mismatch" - ); - assert_eq!( - encode_collab_from_disk.state_vector, - origin_encode_collab.state_vector - ); -} diff --git a/tests/file_test/delete_dir_test.rs b/tests/file_test/delete_dir_test.rs index 9167afdb4..13218258d 100644 --- a/tests/file_test/delete_dir_test.rs +++ b/tests/file_test/delete_dir_test.rs @@ -1,9 +1,9 @@ use crate::collab::util::generate_random_string; use app_error::ErrorCode; use bytes::Bytes; -use client_api::ChunkedBytes; use client_api_test::{generate_unique_registered_user_client, workspace_id_from_client}; use database_entity::file_dto::{CompleteUploadRequest, CompletedPartRequest, CreateUploadRequest}; +use infra::file_util::ChunkedBytes; use uuid::Uuid; #[tokio::test] diff --git a/tests/file_test/multiple_part_test.rs b/tests/file_test/multiple_part_test.rs index 22ac7d0db..4898b787d 100644 --- a/tests/file_test/multiple_part_test.rs +++ b/tests/file_test/multiple_part_test.rs @@ -4,12 +4,12 @@ use app_error::ErrorCode; use appflowy_cloud::api::file_storage::BlobPathV1; use aws_sdk_s3::types::CompletedPart; use bytes::Bytes; -use client_api::ChunkedBytes; use client_api_test::{generate_unique_registered_user_client, workspace_id_from_client}; use database::file::{BlobKey, BucketClient, ResponseBlob}; use database_entity::file_dto::{ CompleteUploadRequest, CompletedPartRequest, CreateUploadRequest, UploadPartData, }; +use infra::file_util::ChunkedBytes; use uuid::Uuid; #[tokio::test] diff --git a/tests/search/asset/appflowy_values.md b/tests/search/asset/appflowy_values.md new file mode 100644 index 000000000..bcefe5ece --- /dev/null +++ b/tests/search/asset/appflowy_values.md @@ -0,0 +1,54 @@ +# AppFlowy Values + +## Mission Driven + +- Our mission is to enable everyone to unleash the potential and achieve more with secure workplace tools. +- We are true believers in open source—a fundamentally superior approach to achieve the mission. +- We actively lead and support the AppFlowy open-source community, where a diverse group of people is empowered to + contribute to the common good. +- We think strategically, make wise decisions, and act accordingly, with an eye toward what’s sustainable in the long + run and not what’s convenient in the moment. + +## Aim High and Iterate + +1. We strive for excellence with a growth mindset. +2. We dream big, start small, and move fast. +3. We take smaller steps and ship smaller, simpler features. +4. We don’t wait, but instead iterate and work as part of the community. +5. We focus on results over process and prioritize progress over perfection. + +## Transparency + +1. We make information about AppFlowy public by default unless there is a compelling reason not to. +2. We are straightforward and kind with ourselves and each other. + +- We surface issues constructively and proactively. +- We say “why” and provide sufficient context for our actions rather than just disclosing the “what.” + +## Collaboration + +> We pride ourselves on being a great team. +> + +> We foster collaboration, value diversity and inclusion, and encourage sharing. +> + +> We thrive as individuals within the context of our team and succeed together. +> + +> We play very effectively with people of diverse backgrounds and cultures. +> + +> We make time to help each other in pursuit of our common goals. +> + +Honesty + +We are honest with ourselves. + +We admit mistakes freely and openly. + +We provide candid, helpful, timely feedback to colleagues with respect, regardless of their status or whether they +disagree with us. + +We are vulnerable in search of truth and don’t defend our point to just win over others. \ No newline at end of file diff --git a/tests/search/asset/kathryn_tennis_story.md b/tests/search/asset/kathryn_tennis_story.md new file mode 100644 index 000000000..d8fbb4dfc --- /dev/null +++ b/tests/search/asset/kathryn_tennis_story.md @@ -0,0 +1,54 @@ +Kathryn’s Journey to Becoming a Tennis Player + +Kathryn’s love for tennis began on a warm summer day when she was eight years old. She stumbled across a local park +where players were volleying back and forth. The sound of the ball hitting the racket and the sheer energy of the game +captivated her. That evening, she begged her parents for a tennis racket, and the very next weekend, she was on the +court for the first time. + +Learning the Basics + +Kathryn’s first lessons were clumsy but full of enthusiasm. She struggled with her serves, missed easy shots, and often +hit the ball over the fence. But every mistake made her more determined to improve. Her first coach, Mr. Evans, taught +her the fundamentals—how to grip the racket, the importance of footwork, and how to keep her eye on the ball. “Tennis is +about focus and persistence,” he would say, and Kathryn took that advice to heart. + +By the time she was 12, Kathryn was playing in local junior tournaments. At first, she lost more matches than she won, +but she never let the defeats discourage her. “Every loss teaches you something,” she told herself. Gradually, her +skills improved, and she started to win. + +The Turning Point + +As Kathryn entered high school, her passion for tennis only grew stronger. She spent hours after school practicing her +backhand and perfecting her serve. She joined her school’s tennis team, where she met her new coach, Ms. Carter. Unlike +her earlier coaches, Ms. Carter focused on strategy and mental toughness. + +“Kathryn, tennis isn’t just physical. It’s a mental game too,” she said one day after a tough match. “You need to stay +calm under pressure and think a few steps ahead of your opponent.” + +That advice changed everything for Kathryn. She began analyzing her matches, understanding her opponents’ patterns, and +using strategy to outplay them. By her senior year, she was the captain of her team and had won several regional +championships. + +Chasing the Dream + +After high school, Kathryn decided to pursue tennis seriously. She joined a competitive training academy, where the +practices were grueling, and the competition was fierce. There were times she doubted herself, especially after losing +matches to stronger players. But her love for the game kept her going. + +Her coaches helped her refine her technique, adding finesse to her volleys and power to her forehand. She also learned +to play smarter, conserving energy during long matches and capitalizing on her opponents’ weaknesses. + +Becoming a Player + +By the time Kathryn was in her early 20s, she was competing in national tournaments. She wasn’t the biggest name on the +court, but her hard work and persistence earned her respect. Each match was a chance to learn, grow, and prove herself. + +She eventually won her first title at a mid-level tournament, a moment she would never forget. Standing on the podium, +holding the trophy, she realized how far she had come—from the little girl who couldn’t hit a serve to a tennis player +with real potential. + +A Life of Tennis + +Today, Kathryn continues to play with the same passion she had when she first picked up a racket. She travels to +tournaments, trains every day, and inspires young players to follow their dreams. For her, tennis is more than a +sport—it’s a lifelong journey of growth, persistence, and joy. \ No newline at end of file diff --git a/tests/search/asset/the_five_dysfunctions_of_a_team.md b/tests/search/asset/the_five_dysfunctions_of_a_team.md new file mode 100644 index 000000000..10ee4ad97 --- /dev/null +++ b/tests/search/asset/the_five_dysfunctions_of_a_team.md @@ -0,0 +1,125 @@ +# *The Five Dysfunctions of a Team* by Patrick Lencioni + +*The Five Dysfunctions of a Team* by Patrick Lencioni is a compelling exploration of team dynamics and the common +pitfalls that undermine successful collaboration. Through the lens of a fictional story about a Silicon Valley startup, +DecisionTech, and its CEO Kathryn Petersen, Lencioni provides a practical framework to address and resolve issues that +commonly disrupt team cohesion and performance. Below is a chapter-by-chapter look at the book’s content, capturing its +essential lessons and actionable insights. + +--- + +## Part I: Underachievement + +In this introductory section, we meet Kathryn Petersen, the newly appointed CEO of DecisionTech, a struggling Silicon +Valley startup with a dysfunctional executive team. Kathryn steps into a role where the team is plagued by poor +communication, lack of trust, and weak commitment. + +Lencioni uses this setup to introduce readers to the core problems affecting team productivity and morale. Kathryn +realizes that the team’s challenges are deeply rooted in its dynamics rather than surface-level operational issues. +Through her initial observations, she identifies that turning around the team will require addressing foundational +issues like trust, respect, and open communication. + +--- + +## Part II: Lighting the Fire + +To start addressing these issues, Kathryn organizes an offsite meeting in Napa Valley. This setting becomes a +transformative space where Kathryn pushes the team to be present, vulnerable, and engaged. Her goal is to build trust, a +critical foundation for any team. + +Kathryn leads exercises that reveal personal histories, enabling the team members to see each other beyond their +professional roles. She also introduces the idea of constructive conflict, encouraging open discussion about +disagreements and differing opinions. Despite the discomfort this causes for some team members who are used to +individualistic work styles, Kathryn emphasizes that trust and openness are crucial for effective teamwork. + +--- + +## Part III: Heavy Lifting + +With initial trust in place, Kathryn shifts her focus to accountability and responsibility. This part highlights the +challenges team members face when taking ownership of collective goals. + +Kathryn holds the team to high standards, stressing the importance of addressing issues directly instead of avoiding +them. This section also examines the role of healthy conflict as a mechanism for growth, as team members begin to hold +each other accountable for their contributions. Through challenging conversations, they tackle topics like performance +expectations and role clarity. Kathryn’s persistence helps the team understand that embracing accountability is +essential for progress, even if it leads to uncomfortable discussions. + +--- + +## Part IV: Traction + +By this stage, Kathryn reinforces the team’s commitment to shared goals. The team starts experiencing the tangible +benefits of improved trust and open conflict. Accountability has now become an expected part of their routine, and +meetings are increasingly productive. + +As they move towards achieving measurable results, the focus shifts from individual successes to collective +achievements. Kathryn ensures that each member appreciates the value of prioritizing team success over personal gain. +Through this unified approach, the team’s motivation and performance visibly improve, demonstrating the power of +cohesive collaboration. + +--- + +## The Model: Overcoming the Five Dysfunctions + +Lencioni introduces a model that identifies the five key dysfunctions of a team and provides strategies to overcome +them: + +1. **Absence of Trust** + The lack of trust prevents team members from being vulnerable and open with each other. Lencioni suggests exercises + that encourage personal sharing to build this essential foundation. + +2. **Fear of Conflict** + Teams that avoid conflict miss out on critical discussions that lead to better decision-making. Lencioni recommends + fostering a safe environment where team members feel comfortable challenging each other’s ideas without fear of + reprisal. + +3. **Lack of Commitment** + Without clarity and buy-in, team decisions become fragmented. Leaders should ensure everyone understands and agrees + on goals to achieve genuine commitment. + +4. **Avoidance of Accountability** + When team members don’t hold each other accountable, performance suffers. Regular check-ins and peer accountability + encourage responsibility and consistency. + +5. **Inattention to Results** + Prioritizing individual goals over collective outcomes dilutes team success. Aligning rewards and recognition with + team achievements helps refocus efforts on shared objectives. + +--- + +## Understanding and Overcoming Each Dysfunction + +Each dysfunction is further broken down with practical strategies: + +- **Building Trust** + Kathryn’s personal history exercise is one example of building trust. By sharing backgrounds and opening up, team + members foster a culture of vulnerability and connection. + +- **Encouraging Conflict** + Constructive conflict allows ideas to be challenged and strengthened. Kathryn’s insistence on open debate helps the + team reach better, more robust decisions. + +- **Ensuring Commitment** + Lencioni highlights the importance of clarity and alignment, which Kathryn reinforces by facilitating discussions that + ensure all team members are on the same page about their goals. + +- **Embracing Accountability** + Accountability becomes ingrained as team members regularly check in with each other, creating a culture of mutual + responsibility and high standards. + +- **Focusing on Results** + Kathryn’s focus on collective achievements over individual successes aligns with Lencioni’s advice to reward team + efforts, ensuring the entire group works toward a shared purpose. + +--- + +## Final Thoughts + +*The Five Dysfunctions of a Team* illustrates the importance of cohesive team behavior and effective leadership in +overcoming common organizational challenges. Through Kathryn’s story, Lencioni provides a practical roadmap for leaders +and teams to diagnose and address dysfunctions, ultimately fostering an environment where trust, accountability, and +shared goals drive performance. + +This book remains a valuable resource for anyone seeking to understand and improve team dynamics, with lessons that +apply well beyond the workplace. \ No newline at end of file diff --git a/tests/search/document_search.rs b/tests/search/document_search.rs index 0b62ca9b9..e696c4369 100644 --- a/tests/search/document_search.rs +++ b/tests/search/document_search.rs @@ -1,13 +1,140 @@ +use std::path::PathBuf; use std::time::Duration; +use appflowy_ai_client::dto::CalculateSimilarityParams; +use client_api_test::{collect_answer, TestClient}; use collab::preclude::Collab; use collab_document::document::Document; +use collab_document::importer::md_importer::MDImporter; use collab_entity::CollabType; +use shared_entity::dto::chat_dto::{CreateChatMessageParams, CreateChatParams}; use tokio::time::sleep; - -use client_api_test::TestClient; use workspace_template::document::getting_started::getting_started_document_data; +#[tokio::test] +async fn test_embedding_when_create_document() { + let mut test_client = TestClient::new_user().await; + let workspace_id = test_client.workspace_id().await; + + let object_id_1 = uuid::Uuid::new_v4().to_string(); + let the_five_dysfunctions_of_a_team = + create_document_collab(&object_id_1, "the_five_dysfunctions_of_a_team.md").await; + let encoded_collab = the_five_dysfunctions_of_a_team.encode_collab().unwrap(); + test_client + .create_collab_with_data( + &workspace_id, + &object_id_1, + CollabType::Document, + encoded_collab, + ) + .await + .unwrap(); + + let object_id_2 = uuid::Uuid::new_v4().to_string(); + let tennis_player = create_document_collab(&object_id_2, "kathryn_tennis_story.md").await; + let encoded_collab = tennis_player.encode_collab().unwrap(); + test_client + .create_collab_with_data( + &workspace_id, + &object_id_2, + CollabType::Document, + encoded_collab, + ) + .await + .unwrap(); + + let search_resp = test_client + .api_client + .search_documents(&workspace_id, "Kathryn", 5, 100) + .await + .unwrap(); + // The number of returned documents affected by the max token size when splitting the document + // into chunks. + assert_eq!(search_resp.len(), 2); + + if ai_test_enabled() { + let previews = search_resp + .iter() + .map(|item| item.preview.clone().unwrap()) + .collect::>() + .join("\n"); + let params = CalculateSimilarityParams { + workspace_id: workspace_id.clone(), + input: previews, + expected: r#" + "Kathryn’s Journey to Becoming a Tennis Player Kathryn’s love for tennis began on a warm summer day w +yn decided to pursue tennis seriously. She joined a competitive training academy, where the +practice +mwork. Part III: Heavy Lifting With initial trust in place, Kathryn shifts her focus to accountabili +’s ideas without fear of +reprisal. Lack of Commitment Without clarity and buy-in, team decisions bec +The Five Dysfunctions of a Team by Patrick Lencioni The Five Dysfunctions of a Team by Patrick Lenci" + "# + .to_string(), + }; + let score = test_client + .api_client + .calculate_similarity(params) + .await + .unwrap() + .score; + + assert!( + score > 0.85, + "preview score should greater than 0.85, but got: {}", + score + ); + + // Create a chat to ask questions that related to the five dysfunctions of a team. + let chat_id = uuid::Uuid::new_v4().to_string(); + let params = CreateChatParams { + chat_id: chat_id.clone(), + name: "chat with the five dysfunctions of a team".to_string(), + rag_ids: vec![object_id_1], + }; + + test_client + .api_client + .create_chat(&workspace_id, params) + .await + .unwrap(); + + let params = CreateChatMessageParams::new_user("Tell me what Kathryn concisely?"); + let question = test_client + .api_client + .create_question(&workspace_id, &chat_id, params) + .await + .unwrap(); + let answer_stream = test_client + .api_client + .stream_answer_v2(&workspace_id, &chat_id, question.message_id) + .await + .unwrap(); + let answer = collect_answer(answer_stream).await; + + let params = CalculateSimilarityParams { + workspace_id, + input: answer.clone(), + expected: r#" + Kathryn Petersen is the newly appointed CEO of DecisionTech, a struggling Silicon Valley startup. + She steps into a role facing a dysfunctional executive team characterized by poor communication, + lack of trust, and weak commitment. Throughout the narrative, Kathryn focuses on addressing + foundational team issues by fostering trust, encouraging open conflict, and promoting accountability, + ultimately leading her team toward improved collaboration and performance. + "# + .to_string(), + }; + let score = test_client + .api_client + .calculate_similarity(params) + .await + .unwrap() + .score; + + assert!(score > 0.9, "score: {}, input:{}", score, answer); + } +} + #[ignore] #[tokio::test] async fn test_document_indexing_and_search() { @@ -44,7 +171,6 @@ async fn test_document_indexing_and_search() { sleep(Duration::from_millis(2000)).await; // document should get automatically indexed after opening if it wasn't indexed before - let search_resp = test_client .api_client .search_documents(&workspace_id, "Appflowy", 1, 20) @@ -53,5 +179,22 @@ async fn test_document_indexing_and_search() { assert_eq!(search_resp.len(), 1); let item = &search_resp[0]; assert_eq!(item.object_id, object_id); - assert_eq!(item.preview.as_deref(), Some("\nWelcome to AppFlowy")); + + let preview = item.preview.clone().unwrap(); + assert!(preview.contains("Welcome to AppFlowy")); +} + +async fn create_document_collab(document_id: &str, file_name: &str) -> Document { + let file_path = PathBuf::from(format!("tests/search/asset/{}", file_name)); + let md = std::fs::read_to_string(file_path).unwrap(); + let importer = MDImporter::new(None); + let document_data = importer.import(document_id, md).unwrap(); + Document::create(document_id, document_data).unwrap() +} + +pub fn ai_test_enabled() -> bool { + if cfg!(feature = "ai-test-enabled") { + return true; + } + false } diff --git a/tests/workspace/invitation_crud.rs b/tests/workspace/invitation_crud.rs index b258c47c2..9964e689a 100644 --- a/tests/workspace/invitation_crud.rs +++ b/tests/workspace/invitation_crud.rs @@ -15,6 +15,15 @@ async fn invite_workspace_crud() { .workspace_id; let (bob_client, bob) = generate_unique_registered_user_client().await; + let bob_workspace_id = bob_client + .get_workspaces() + .await + .unwrap() + .first() + .unwrap() + .workspace_id; + + // alice invite bob to alice's workspace alice_client .invite_workspace_members( alice_workspace_id.to_string().as_str(), @@ -94,16 +103,49 @@ async fn invite_workspace_crud() { .unwrap(); assert_eq!(accepted_invs.len(), 1); - // workspace now have 2 members - let member_count = alice_client - .get_workspaces_opt(QueryWorkspaceParam { - include_member_count: Some(true), - }) - .await - .unwrap() - .first() - .unwrap() - .member_count - .unwrap(); - assert_eq!(member_count, 2); + { + // alice's view of the workspaces + let workspaces = alice_client + .get_workspaces_opt(QueryWorkspaceParam { + include_member_count: Some(true), + include_role: Some(true), + }) + .await + .unwrap(); + + assert_eq!(workspaces.len(), 1); + assert_eq!(workspaces[0].workspace_id, alice_workspace_id); + assert_eq!(workspaces[0].member_count, Some(2)); + assert_eq!(workspaces[0].role, Some(AFRole::Owner)); + } + + { + // bob's view of the workspaces + // bob should see 2 workspaces, one is his own and the other is alice's + let workspaces = bob_client + .get_workspaces_opt(QueryWorkspaceParam { + include_member_count: Some(true), + include_role: Some(true), + }) + .await + .unwrap(); + assert_eq!(workspaces.len(), 2); + { + let alice_workspace = workspaces + .iter() + .find(|w| w.workspace_id == alice_workspace_id) + .unwrap(); + assert_eq!(alice_workspace.member_count, Some(2)); + assert_eq!(alice_workspace.role, Some(AFRole::Member)); + } + { + let bob_workspace = workspaces + .iter() + .find(|w| w.workspace_id == bob_workspace_id) + .unwrap(); + println!("{:?}", bob_workspace); + assert_eq!(bob_workspace.member_count, Some(1)); + assert_eq!(bob_workspace.role, Some(AFRole::Owner)); + } + } } diff --git a/tests/workspace/page_view.rs b/tests/workspace/page_view.rs index d4e922395..88734fed7 100644 --- a/tests/workspace/page_view.rs +++ b/tests/workspace/page_view.rs @@ -7,9 +7,10 @@ use client_api_test::{ use collab::{core::origin::CollabClient, preclude::Collab}; use collab_entity::CollabType; use collab_folder::{CollabOrigin, Folder}; -use serde_json::json; +use serde_json::{json, Value}; use shared_entity::dto::workspace_dto::{ - CreatePageParams, IconType, UpdatePageParams, ViewIcon, ViewLayout, + CreatePageParams, CreateSpaceParams, IconType, SpacePermission, UpdatePageParams, ViewIcon, + ViewLayout, }; use tokio::time::sleep; use uuid::Uuid; @@ -100,6 +101,7 @@ async fn create_new_document_page() { &CreatePageParams { parent_view_id: general_space.view_id.clone(), layout: ViewLayout::Document, + name: Some("New document".to_string()), }, ) .await @@ -114,11 +116,12 @@ async fn create_new_document_page() { .into_iter() .find(|v| v.name == "General") .unwrap(); - general_space + let view = general_space .children .iter() .find(|v| v.view_id == page.view_id) .unwrap(); + assert_eq!(view.name, "New document"); c.get_collab(QueryCollabParams { workspace_id: workspace_id.to_string(), inner: QueryCollab { @@ -281,3 +284,72 @@ async fn update_page() { Some(json!({"is_pinned": true}).to_string()) ); } + +#[tokio::test] +async fn create_space() { + let registered_user = generate_unique_registered_user().await; + let mut app_client = TestClient::user_with_new_device(registered_user.clone()).await; + let web_client = TestClient::user_with_new_device(registered_user.clone()).await; + let workspace_id = app_client.workspace_id().await; + app_client.open_workspace_collab(&workspace_id).await; + app_client + .wait_object_sync_complete(&workspace_id) + .await + .unwrap(); + let workspace_uuid = Uuid::parse_str(&workspace_id).unwrap(); + let public_space = web_client + .api_client + .create_space( + workspace_uuid, + &CreateSpaceParams { + space_permission: SpacePermission::PublicToAll, + name: "Public Space".to_string(), + space_icon: "space_icon_1".to_string(), + space_icon_color: "0xFFA34AFD".to_string(), + }, + ) + .await + .unwrap(); + web_client + .api_client + .create_space( + workspace_uuid, + &CreateSpaceParams { + space_permission: SpacePermission::Private, + name: "Private Space".to_string(), + space_icon: "space_icon_2".to_string(), + space_icon_color: "0xFFA34AFD".to_string(), + }, + ) + .await + .unwrap(); + let folder = get_latest_folder(&app_client, &workspace_id).await; + let view = folder.get_view(&public_space.view_id).unwrap(); + let space_info: Value = serde_json::from_str(view.extra.as_ref().unwrap()).unwrap(); + assert!(space_info["is_space"].as_bool().unwrap()); + assert_eq!( + space_info["space_permission"].as_u64().unwrap() as u8, + SpacePermission::PublicToAll as u8 + ); + assert_eq!(space_info["space_icon"].as_str().unwrap(), "space_icon_1"); + assert_eq!( + space_info["space_icon_color"].as_str().unwrap(), + "0xFFA34AFD" + ); + let folder_view = web_client + .api_client + .get_workspace_folder(&workspace_id, Some(2), Some(workspace_id.to_string())) + .await + .unwrap(); + folder_view + .children + .iter() + .find(|v| v.name == "Public Space") + .unwrap(); + let private_space = folder_view + .children + .iter() + .find(|v| v.name == "Private Space") + .unwrap(); + assert!(private_space.is_private); +} diff --git a/tests/workspace/publish.rs b/tests/workspace/publish.rs index ce7605605..72f774d00 100644 --- a/tests/workspace/publish.rs +++ b/tests/workspace/publish.rs @@ -39,7 +39,7 @@ async fn test_set_publish_namespace_set() { .unwrap(); } - let new_namespace = uuid::Uuid::new_v4().to_string(); + let new_namespace = format!("namespace_{}", uuid::Uuid::new_v4()); c.set_workspace_publish_namespace(&workspace_id.to_string(), new_namespace.clone()) .await .unwrap(); @@ -156,9 +156,9 @@ async fn test_publish_doc() { assert_eq!(err.code, ErrorCode::PublishNameTooLong, "{:?}", err); } - let publish_name_1 = "publish-name-1"; + let publish_name_1 = "publish_name-1"; let view_id_1 = uuid::Uuid::new_v4(); - let publish_name_2 = "publish-name-2"; + let publish_name_2 = "publish_name-2"; let view_id_2 = uuid::Uuid::new_v4(); // User publishes two collabs @@ -1709,3 +1709,69 @@ async fn test_republish_doc() { assert_eq!(err.code, ErrorCode::RecordNotFound, "{:?}", err); } } + +#[tokio::test] +async fn test_republish_patch() { + let (c, _user) = generate_unique_registered_user_client().await; + let workspace_id = get_first_workspace_string(&c).await; + let my_namespace = uuid::Uuid::new_v4().to_string(); + c.set_workspace_publish_namespace(&workspace_id.to_string(), my_namespace.clone()) + .await + .unwrap(); + + let publish_name = "my-publish-name"; + let view_id = uuid::Uuid::new_v4(); + + // User publishes 1 doc + c.publish_collabs::( + &workspace_id, + vec![PublishCollabItem { + meta: PublishCollabMetadata { + view_id, + publish_name: publish_name.to_string(), + metadata: MyCustomMetadata { + title: "my_title_1".to_string(), + }, + }, + data: "yrs_encoded_data_1".as_bytes(), + }], + ) + .await + .unwrap(); + + // user unpublishes the doc + c.unpublish_collabs(&workspace_id, &[view_id]) + .await + .unwrap(); + + // User publish another doc + let publish_name_2 = "my-publish-name-2"; + let view_id_2 = uuid::Uuid::new_v4(); + c.publish_collabs::( + &workspace_id, + vec![PublishCollabItem { + meta: PublishCollabMetadata { + view_id: view_id_2, + publish_name: publish_name_2.to_string(), + metadata: MyCustomMetadata { + title: "my_title_1".to_string(), + }, + }, + data: "yrs_encoded_data_1".as_bytes(), + }], + ) + .await + .unwrap(); + + // User change the publish name of the document to publish_name + // which should be allowed since the original document is already unpublished + c.patch_published_collabs( + &workspace_id, + &[PatchPublishedCollab { + view_id: view_id_2, + publish_name: Some(publish_name.to_string()), + }], + ) + .await + .unwrap(); +} diff --git a/tests/workspace/workspace_folder.rs b/tests/workspace/workspace_folder.rs index e5478b680..8b3738c13 100644 --- a/tests/workspace/workspace_folder.rs +++ b/tests/workspace/workspace_folder.rs @@ -1,7 +1,4 @@ -use client_api::entity::{CreateCollabParams, QueryCollabParams}; use client_api_test::generate_unique_registered_user_client; -use collab::core::origin::CollabClient; -use collab_folder::{CollabOrigin, Folder}; #[tokio::test] async fn get_workpace_folder() { @@ -34,68 +31,3 @@ async fn get_workpace_folder() { .unwrap(); assert_eq!(folder_view.children.len(), 2); } - -#[tokio::test] -async fn get_section_items() { - let (c, _user) = generate_unique_registered_user_client().await; - let user_workspace_info = c.get_user_workspace_info().await.unwrap(); - let workspaces = c.get_workspaces().await.unwrap(); - assert_eq!(workspaces.len(), 1); - let workspace_id = workspaces[0].workspace_id.to_string(); - let folder_collab = c - .get_collab(QueryCollabParams::new( - workspace_id.clone(), - collab_entity::CollabType::Folder, - workspace_id.clone(), - )) - .await - .unwrap() - .encode_collab; - let uid = user_workspace_info.user_profile.uid; - let mut folder = Folder::from_collab_doc_state( - uid, - CollabOrigin::Client(CollabClient::new(uid, c.device_id.clone())), - folder_collab.into(), - &workspace_id, - vec![], - ) - .unwrap(); - let views = folder.get_views_belong_to(&workspace_id); - let new_favorite_id = views[0].children[0].id.clone(); - let to_be_deleted_favorite_id = views[0].children[1].id.clone(); - folder.add_favorite_view_ids(vec![ - new_favorite_id.clone(), - to_be_deleted_favorite_id.clone(), - ]); - folder.add_trash_view_ids(vec![to_be_deleted_favorite_id.clone()]); - let recent_id = folder.get_views_belong_to(&new_favorite_id)[0].id.clone(); - folder.add_recent_view_ids(vec![recent_id.clone()]); - let collab_type = collab_entity::CollabType::Folder; - c.update_collab(CreateCollabParams { - workspace_id: workspace_id.clone(), - collab_type: collab_type.clone(), - object_id: workspace_id.clone(), - encoded_collab_v1: folder - .encode_collab_v1(|collab| collab_type.validate_require_data(collab)) - .unwrap() - .encode_to_bytes() - .unwrap(), - }) - .await - .unwrap(); - let favorite_section_items = c.get_workspace_favorite(&workspace_id).await.unwrap(); - assert_eq!(favorite_section_items.views.len(), 1); - assert_eq!( - favorite_section_items.views[0].view.view_id, - new_favorite_id - ); - let trash_section_items = c.get_workspace_trash(&workspace_id).await.unwrap(); - assert_eq!(trash_section_items.views.len(), 1); - assert_eq!( - trash_section_items.views[0].view.view_id, - to_be_deleted_favorite_id - ); - let recent_section_items = c.get_workspace_recent(&workspace_id).await.unwrap(); - assert_eq!(recent_section_items.views.len(), 1); - assert_eq!(recent_section_items.views[0].view.view_id, recent_id); -} diff --git a/xtask/src/main.rs b/xtask/src/main.rs index 9b6a72d6c..f04351688 100644 --- a/xtask/src/main.rs +++ b/xtask/src/main.rs @@ -24,16 +24,16 @@ async fn main() -> Result<()> { .spawn() .context("Failed to start AppFlowy-Cloud process")?; - let mut appflowy_history_cmd = Command::new("cargo") - .args([ - "run", - // "--features", - // "verbose_log", - "--manifest-path", - "./services/appflowy-history/Cargo.toml", - ]) - .spawn() - .context("Failed to start AppFlowy-History process")?; + // let mut appflowy_history_cmd = Command::new("cargo") + // .args([ + // "run", + // // "--features", + // // "verbose_log", + // "--manifest-path", + // "./services/appflowy-history/Cargo.toml", + // ]) + // .spawn() + // .context("Failed to start AppFlowy-History process")?; let mut appflowy_worker_cmd = Command::new("cargo") .args([ @@ -48,9 +48,9 @@ async fn main() -> Result<()> { status = appflowy_cloud_cmd.wait() => { handle_process_exit(status?, appflowy_cloud_bin_name)?; }, - status = appflowy_history_cmd.wait() => { - handle_process_exit(status?, history)?; - } + // status = appflowy_history_cmd.wait() => { + // handle_process_exit(status?, history)?; + // } status = appflowy_worker_cmd.wait() => { handle_process_exit(status?, worker)?; }