diff --git a/packages/backend/server/migrations/20241023065619_blobs/migration.sql b/packages/backend/server/migrations/20241023065619_blobs/migration.sql new file mode 100644 index 0000000000000..22fac2f05801b --- /dev/null +++ b/packages/backend/server/migrations/20241023065619_blobs/migration.sql @@ -0,0 +1,14 @@ +-- CreateTable +CREATE TABLE "blobs" ( + "workspace_id" VARCHAR NOT NULL, + "key" VARCHAR NOT NULL, + "size" INTEGER NOT NULL, + "mime" VARCHAR NOT NULL, + "created_at" TIMESTAMPTZ(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "deleted_at" TIMESTAMPTZ(3), + + CONSTRAINT "blobs_pkey" PRIMARY KEY ("workspace_id","key") +); + +-- AddForeignKey +ALTER TABLE "blobs" ADD CONSTRAINT "blobs_workspace_id_fkey" FOREIGN KEY ("workspace_id") REFERENCES "workspaces"("id") ON DELETE CASCADE ON UPDATE CASCADE; diff --git a/packages/backend/server/schema.prisma b/packages/backend/server/schema.prisma index 9f810ceef4df4..ce5315b46ecd3 100644 --- a/packages/backend/server/schema.prisma +++ b/packages/backend/server/schema.prisma @@ -106,6 +106,7 @@ model Workspace { permissions WorkspaceUserPermission[] pagePermissions WorkspacePageUserPermission[] features WorkspaceFeature[] + blobs Blob[] @@map("workspaces") } @@ -568,3 +569,19 @@ model Invoice { @@index([targetId]) @@map("invoices") } +// Blob table only exists for fast non-data queries. +// like, total size of blobs in a workspace, or blob list for sync service. +// it should only be a map of metadata of blobs stored anywhere else +model Blob { + workspaceId String @map("workspace_id") @db.VarChar + key String @db.VarChar + size Int @db.Integer + mime String @db.VarChar + createdAt DateTime @default(now()) @map("created_at") @db.Timestamptz(3) + deletedAt DateTime? @map("deleted_at") @db.Timestamptz(3) + + workspace Workspace @relation(fields: [workspaceId], references: [id], onDelete: Cascade) + + @@id([workspaceId, key]) + @@map("blobs") +} diff --git a/packages/backend/server/src/core/doc/storage/doc.ts b/packages/backend/server/src/core/doc/storage/doc.ts index 6010855484812..4bcbf4067be68 100644 --- a/packages/backend/server/src/core/doc/storage/doc.ts +++ b/packages/backend/server/src/core/doc/storage/doc.ts @@ -1,8 +1,10 @@ import { applyUpdate, + diffUpdate, Doc, encodeStateAsUpdate, encodeStateVector, + encodeStateVectorFromUpdate, mergeUpdates, UndoManager, } from 'yjs'; @@ -19,6 +21,12 @@ export interface DocRecord { editor?: string; } +export interface DocDiff { + missing: Uint8Array; + state: Uint8Array; + timestamp: number; +} + export interface DocUpdate { bin: Uint8Array; timestamp: number; @@ -96,6 +104,27 @@ export abstract class DocStorageAdapter extends Connection { return snapshot; } + async getDocDiff( + spaceId: string, + docId: string, + stateVector?: Uint8Array + ): Promise { + const doc = await this.getDoc(spaceId, docId); + + if (!doc) { + return null; + } + + const missing = stateVector ? diffUpdate(doc.bin, stateVector) : doc.bin; + const state = encodeStateVectorFromUpdate(doc.bin); + + return { + missing, + state, + timestamp: doc.timestamp, + }; + } + abstract pushDocUpdates( spaceId: string, docId: string, diff --git a/packages/backend/server/src/core/doc/storage/index.ts b/packages/backend/server/src/core/doc/storage/index.ts index 6ba0e23dd1119..7521fb6cca367 100644 --- a/packages/backend/server/src/core/doc/storage/index.ts +++ b/packages/backend/server/src/core/doc/storage/index.ts @@ -1,4 +1,6 @@ -// TODO(@forehalo): share with frontend +// This is a totally copy of definitions in [@affine/space-store] +// because currently importing cross workspace package from [@affine/server] is not yet supported +// should be kept updated with the original definitions in [@affine/space-store] import type { BlobStorageAdapter } from './blob'; import { Connection } from './connection'; import type { DocStorageAdapter } from './doc'; diff --git a/packages/backend/server/src/core/quota/resolver.ts b/packages/backend/server/src/core/quota/resolver.ts index c8afaa2237a01..309f43833ab6d 100644 --- a/packages/backend/server/src/core/quota/resolver.ts +++ b/packages/backend/server/src/core/quota/resolver.ts @@ -11,6 +11,7 @@ import { CurrentUser } from '../auth/session'; import { EarlyAccessType } from '../features'; import { UserType } from '../user'; import { QuotaService } from './service'; +import { QuotaManagementService } from './storage'; registerEnumType(EarlyAccessType, { name: 'EarlyAccessType', @@ -55,9 +56,18 @@ class UserQuotaType { humanReadable!: UserQuotaHumanReadableType; } +@ObjectType('UserQuotaUsage') +class UserQuotaUsageType { + @Field(() => SafeIntResolver, { name: 'storageQuota' }) + storageQuota!: number; +} + @Resolver(() => UserType) export class QuotaManagementResolver { - constructor(private readonly quota: QuotaService) {} + constructor( + private readonly quota: QuotaService, + private readonly management: QuotaManagementService + ) {} @ResolveField(() => UserQuotaType, { name: 'quota', nullable: true }) async getQuota(@CurrentUser() me: UserType) { @@ -65,4 +75,15 @@ export class QuotaManagementResolver { return quota.feature; } + + @ResolveField(() => UserQuotaUsageType, { name: 'quotaUsage' }) + async getQuotaUsage( + @CurrentUser() me: UserType + ): Promise { + const usage = await this.management.getUserStorageUsage(me.id); + + return { + storageQuota: usage, + }; + } } diff --git a/packages/backend/server/src/core/quota/storage.ts b/packages/backend/server/src/core/quota/storage.ts index 8476a48973f99..a74942eeec579 100644 --- a/packages/backend/server/src/core/quota/storage.ts +++ b/packages/backend/server/src/core/quota/storage.ts @@ -77,7 +77,7 @@ export class QuotaManagementService { return this.quota.hasWorkspaceQuota(workspaceId, QuotaType.TeamPlanV1); } - async getUserUsage(userId: string) { + async getUserStorageUsage(userId: string) { const workspaces = await this.permissions.getOwnedWorkspaces(userId); const sizes = await Promise.allSettled( @@ -125,7 +125,7 @@ export class QuotaManagementService { async getQuotaCalculator(userId: string) { const quota = await this.getUserQuota(userId); const { storageQuota, businessBlobLimit } = quota; - const usedSize = await this.getUserUsage(userId); + const usedSize = await this.getUserStorageUsage(userId); return this.generateQuotaCalculator( storageQuota, @@ -183,7 +183,7 @@ export class QuotaManagementService { humanReadable, } = await this.getWorkspaceQuota(owner.id, workspaceId); // get all workspaces size of owner used - const usedSize = await this.getUserUsage(owner.id); + const usedSize = await this.getUserStorageUsage(owner.id); // relax restrictions if workspace has unlimited feature // todo(@darkskygit): need a mechanism to allow feature as a middleware to edit quota const unlimited = await this.feature.hasWorkspaceFeature( diff --git a/packages/backend/server/src/core/storage/wrappers/blob.ts b/packages/backend/server/src/core/storage/wrappers/blob.ts index 9aa991ab4b924..38b63df950ce8 100644 --- a/packages/backend/server/src/core/storage/wrappers/blob.ts +++ b/packages/backend/server/src/core/storage/wrappers/blob.ts @@ -1,33 +1,42 @@ -import { Injectable } from '@nestjs/common'; +import { Injectable, Logger } from '@nestjs/common'; +import { PrismaClient } from '@prisma/client'; import { - type BlobInputType, - Cache, + autoMetadata, Config, EventEmitter, type EventPayload, - type ListObjectsMetadata, + type GetObjectMetadata, + ListObjectsMetadata, OnEvent, + PutObjectMetadata, type StorageProvider, StorageProviderFactory, } from '../../../fundamentals'; @Injectable() export class WorkspaceBlobStorage { + private readonly logger = new Logger(WorkspaceBlobStorage.name); public readonly provider: StorageProvider; constructor( private readonly config: Config, private readonly event: EventEmitter, private readonly storageFactory: StorageProviderFactory, - private readonly cache: Cache + private readonly db: PrismaClient ) { this.provider = this.storageFactory.create(this.config.storages.blob); } - async put(workspaceId: string, key: string, blob: BlobInputType) { - await this.provider.put(`${workspaceId}/${key}`, blob); - await this.cache.delete(`blob-list:${workspaceId}`); + async put(workspaceId: string, key: string, blob: Buffer) { + const meta: PutObjectMetadata = autoMetadata(blob); + + await this.provider.put(`${workspaceId}/${key}`, blob, meta); + this.trySyncBlobMeta(workspaceId, key, { + contentType: meta.contentType ?? 'application/octet-stream', + contentLength: blob.length, + lastModified: new Date(), + }); } async get(workspaceId: string, key: string) { @@ -35,41 +44,141 @@ export class WorkspaceBlobStorage { } async list(workspaceId: string) { - const cachedList = await this.cache.list( - `blob-list:${workspaceId}`, - 0, - -1 - ); - - if (cachedList.length > 0) { - return cachedList; + const blobsInDb = await this.db.blob.findMany({ + where: { + workspaceId, + deletedAt: null, + }, + }); + + if (blobsInDb.length > 0) { + return blobsInDb; } const blobs = await this.provider.list(workspaceId + '/'); - - blobs.forEach(item => { - // trim workspace prefix - item.key = item.key.slice(workspaceId.length + 1); + blobs.forEach(blob => { + blob.key = blob.key.slice(workspaceId.length + 1); }); - await this.cache.pushBack(`blob-list:${workspaceId}`, ...blobs); + this.trySyncBlobsMeta(workspaceId, blobs); - return blobs; + return blobs.map(blob => ({ + key: blob.key, + size: blob.contentLength, + createdAt: blob.lastModified, + mime: 'application/octet-stream', + })); } - /** - * we won't really delete the blobs until the doc blobs manager is implemented sounded - */ - async delete(_workspaceId: string, _key: string) { - // return this.provider.delete(`${workspaceId}/${key}`); + async delete(workspaceId: string, key: string, permanently = false) { + if (permanently) { + await this.provider.delete(`${workspaceId}/${key}`); + await this.db.blob.deleteMany({ + where: { + workspaceId, + key, + }, + }); + } else { + await this.db.blob.update({ + where: { + workspaceId_key: { + workspaceId, + key, + }, + }, + data: { + deletedAt: new Date(), + }, + }); + } + } + + async release(workspaceId: string) { + const deletedBlobs = await this.db.blob.findMany({ + where: { + workspaceId, + deletedAt: { + not: null, + }, + }, + }); + + deletedBlobs.forEach(blob => { + this.event.emit('workspace.blob.deleted', { + workspaceId: workspaceId, + key: blob.key, + }); + }); } async totalSize(workspaceId: string) { const blobs = await this.list(workspaceId); - // how could we ignore the ones get soft-deleted? return blobs.reduce((acc, item) => acc + item.size, 0); } + private trySyncBlobsMeta(workspaceId: string, blobs: ListObjectsMetadata[]) { + for (const blob of blobs) { + this.trySyncBlobMeta(workspaceId, blob.key); + } + } + + private trySyncBlobMeta( + workspaceId: string, + key: string, + meta?: GetObjectMetadata + ) { + setImmediate(() => { + this.syncBlobMeta(workspaceId, key, meta).catch(() => { + /* never throw */ + }); + }); + } + + private async syncBlobMeta( + workspaceId: string, + key: string, + meta?: GetObjectMetadata + ) { + try { + if (!meta) { + const blob = await this.get(workspaceId, key); + meta = blob.metadata; + } + + if (meta) { + await this.db.blob.upsert({ + where: { + workspaceId_key: { + workspaceId, + key, + }, + }, + update: { + mime: meta.contentType, + size: meta.contentLength, + }, + create: { + workspaceId, + key, + mime: meta.contentType, + size: meta.contentLength, + }, + }); + } else { + await this.db.blob.deleteMany({ + where: { + workspaceId, + key, + }, + }); + } + } catch (e) { + // never throw + this.logger.error('failed to sync blob meta to DB', e); + } + } + @OnEvent('workspace.deleted') async onWorkspaceDeleted(workspaceId: EventPayload<'workspace.deleted'>) { const blobs = await this.list(workspaceId); @@ -78,7 +187,7 @@ export class WorkspaceBlobStorage { blobs.forEach(blob => { this.event.emit('workspace.blob.deleted', { workspaceId: workspaceId, - name: blob.key, + key: blob.key, }); }); } @@ -86,8 +195,8 @@ export class WorkspaceBlobStorage { @OnEvent('workspace.blob.deleted') async onDeleteWorkspaceBlob({ workspaceId, - name, + key, }: EventPayload<'workspace.blob.deleted'>) { - await this.delete(workspaceId, name); + await this.delete(workspaceId, key, true); } } diff --git a/packages/backend/server/src/core/sync/gateway.ts b/packages/backend/server/src/core/sync/gateway.ts index d5686c658fd42..d1c1c5ff7759a 100644 --- a/packages/backend/server/src/core/sync/gateway.ts +++ b/packages/backend/server/src/core/sync/gateway.ts @@ -8,7 +8,6 @@ import { WebSocketGateway, } from '@nestjs/websockets'; import { Socket } from 'socket.io'; -import { diffUpdate, encodeStateVectorFromUpdate } from 'yjs'; import { AlreadyInSpace, @@ -83,6 +82,9 @@ interface LeaveSpaceAwarenessMessage { docId: string; } +/** + * @deprecated + */ interface PushDocUpdatesMessage { spaceType: SpaceType; spaceId: string; @@ -90,6 +92,13 @@ interface PushDocUpdatesMessage { updates: string[]; } +interface PushDocUpdateMessage { + spaceType: SpaceType; + spaceId: string; + docId: string; + update: string; +} + interface LoadDocMessage { spaceType: SpaceType; spaceId: string; @@ -97,6 +106,12 @@ interface LoadDocMessage { stateVector?: string; } +interface DeleteDocMessage { + spaceType: SpaceType; + spaceId: string; + docId: string; +} + interface LoadDocTimestampsMessage { spaceType: SpaceType; spaceId: string; @@ -114,6 +129,7 @@ interface UpdateAwarenessMessage { docId: string; awarenessUpdate: string; } + @WebSocketGateway() export class SpaceSyncGateway implements OnGatewayConnection, OnGatewayDisconnect @@ -182,26 +198,6 @@ export class SpaceSyncGateway } } - async joinWorkspace( - client: Socket, - room: `${string}:${'sync' | 'awareness'}` - ) { - await client.join(room); - } - - async leaveWorkspace( - client: Socket, - room: `${string}:${'sync' | 'awareness'}` - ) { - await client.leave(room); - } - - assertInWorkspace(client: Socket, room: `${string}:${'sync' | 'awareness'}`) { - if (!client.rooms.has(room)) { - throw new NotInSpace({ spaceId: room.split(':')[0] }); - } - } - // v3 @SubscribeMessage('space:join') async onJoinSpace( @@ -233,36 +229,42 @@ export class SpaceSyncGateway @MessageBody() { spaceType, spaceId, docId, stateVector }: LoadDocMessage ): Promise< - EventResponse<{ missing: string; state?: string; timestamp: number }> + EventResponse<{ missing: string; state: string; timestamp: number }> > { const adapter = this.selectAdapter(client, spaceType); adapter.assertIn(spaceId); - const doc = await adapter.get(spaceId, docId); + const doc = await adapter.diff( + spaceId, + docId, + stateVector ? Buffer.from(stateVector, 'base64') : undefined + ); if (!doc) { throw new DocNotFound({ spaceId, docId }); } - const missing = Buffer.from( - stateVector - ? diffUpdate(doc.bin, Buffer.from(stateVector, 'base64')) - : doc.bin - ).toString('base64'); - - const state = Buffer.from(encodeStateVectorFromUpdate(doc.bin)).toString( - 'base64' - ); - return { data: { - missing, - state, + missing: Buffer.from(doc.missing).toString('base64'), + state: Buffer.from(doc.state).toString('base64'), timestamp: doc.timestamp, }, }; } + @SubscribeMessage('space:delete-doc') + async onDeleteSpaceDoc( + @ConnectedSocket() client: Socket, + @MessageBody() { spaceType, spaceId, docId }: DeleteDocMessage + ) { + const adapter = this.selectAdapter(client, spaceType); + await adapter.delete(spaceId, docId); + } + + /** + * @deprecated use [space:push-doc-update] instead, client should always merge updates on their own + */ @SubscribeMessage('space:push-doc-updates') async onReceiveDocUpdates( @ConnectedSocket() client: Socket, @@ -307,6 +309,51 @@ export class SpaceSyncGateway }; } + @SubscribeMessage('space:push-doc-update') + async onReceiveDocUpdate( + @ConnectedSocket() client: Socket, + @CurrentUser() user: CurrentUser, + @MessageBody() + message: PushDocUpdateMessage + ): Promise> { + const { spaceType, spaceId, docId, update } = message; + const adapter = this.selectAdapter(client, spaceType); + + // TODO(@forehalo): we might need to check write permission before push updates + const timestamp = await adapter.push( + spaceId, + docId, + [Buffer.from(update, 'base64')], + user.id + ); + + // TODO(@forehalo): separate different version of clients into different rooms, + // so the clients won't receive useless updates events + client.to(adapter.room(spaceId)).emit('space:broadcast-doc-updates', { + spaceType, + spaceId, + docId, + updates: [update], + timestamp, + }); + + client.to(adapter.room(spaceId)).emit('space:broadcast-doc-update', { + spaceType, + spaceId, + docId, + update, + timestamp, + editor: user.id, + }); + + return { + data: { + accepted: true, + timestamp, + }, + }; + } + @SubscribeMessage('space:load-doc-timestamps') async onLoadDocTimestamps( @ConnectedSocket() client: Socket, @@ -600,9 +647,14 @@ abstract class SyncSocketAdapter { return this.storage.pushDocUpdates(spaceId, docId, updates, editorId); } - get(spaceId: string, docId: string) { + diff(spaceId: string, docId: string, stateVector?: Uint8Array) { + this.assertIn(spaceId); + return this.storage.getDocDiff(spaceId, docId, stateVector); + } + + delete(spaceId: string, docId: string) { this.assertIn(spaceId); - return this.storage.getDoc(spaceId, docId); + return this.storage.deleteDoc(spaceId, docId); } getTimestamps(spaceId: string, timestamp?: number) { @@ -630,9 +682,9 @@ class WorkspaceSyncAdapter extends SyncSocketAdapter { return super.push(spaceId, id.guid, updates, editorId); } - override get(spaceId: string, docId: string) { + override diff(spaceId: string, docId: string, stateVector?: Uint8Array) { const id = new DocID(docId, spaceId); - return this.storage.getDoc(spaceId, id.guid); + return this.storage.getDocDiff(spaceId, id.guid, stateVector); } async assertAccessible( diff --git a/packages/backend/server/src/core/workspaces/resolvers/blob.ts b/packages/backend/server/src/core/workspaces/resolvers/blob.ts index da61143a99ce4..448d23741c2cf 100644 --- a/packages/backend/server/src/core/workspaces/resolvers/blob.ts +++ b/packages/backend/server/src/core/workspaces/resolvers/blob.ts @@ -1,29 +1,40 @@ import { Logger, UseGuards } from '@nestjs/common'; import { Args, + Field, Int, Mutation, + ObjectType, Parent, Query, ResolveField, Resolver, } from '@nestjs/graphql'; -import { SafeIntResolver } from 'graphql-scalars'; import GraphQLUpload from 'graphql-upload/GraphQLUpload.mjs'; import type { FileUpload } from '../../../fundamentals'; -import { - BlobQuotaExceeded, - CloudThrottlerGuard, - MakeCache, - PreventCache, -} from '../../../fundamentals'; +import { BlobQuotaExceeded, CloudThrottlerGuard } from '../../../fundamentals'; import { CurrentUser } from '../../auth'; import { Permission, PermissionService } from '../../permission'; import { QuotaManagementService } from '../../quota'; import { WorkspaceBlobStorage } from '../../storage'; import { WorkspaceBlobSizes, WorkspaceType } from '../types'; +@ObjectType() +class ListedBlob { + @Field() + key!: string; + + @Field() + mime!: string; + + @Field() + size!: number; + + @Field() + createdAt!: string; +} + @UseGuards(CloudThrottlerGuard) @Resolver(() => WorkspaceType) export class WorkspaceBlobResolver { @@ -34,7 +45,7 @@ export class WorkspaceBlobResolver { private readonly storage: WorkspaceBlobStorage ) {} - @ResolveField(() => [String], { + @ResolveField(() => [ListedBlob], { description: 'List blobs of workspace', complexity: 2, }) @@ -44,9 +55,7 @@ export class WorkspaceBlobResolver { ) { await this.permissions.checkWorkspace(workspace.id, user.id); - return this.storage - .list(workspace.id) - .then(list => list.map(item => item.key)); + return this.storage.list(workspace.id); } @ResolveField(() => Int, { @@ -64,7 +73,6 @@ export class WorkspaceBlobResolver { description: 'List blobs of workspace', deprecationReason: 'use `workspace.blobs` instead', }) - @MakeCache(['blobs'], ['workspaceId']) async listBlobs( @CurrentUser() user: CurrentUser, @Args('workspaceId') workspaceId: string @@ -76,42 +84,15 @@ export class WorkspaceBlobResolver { .then(list => list.map(item => item.key)); } - /** - * @deprecated use `user.storageUsage` instead - */ @Query(() => WorkspaceBlobSizes, { - deprecationReason: 'use `user.storageUsage` instead', + deprecationReason: 'use `user.quotaUsage` instead', }) async collectAllBlobSizes(@CurrentUser() user: CurrentUser) { - const size = await this.quota.getUserUsage(user.id); + const size = await this.quota.getUserStorageUsage(user.id); return { size }; } - /** - * @deprecated mutation `setBlob` will check blob limit & quota usage - */ - @Query(() => WorkspaceBlobSizes, { - deprecationReason: 'no more needed', - }) - async checkBlobSize( - @CurrentUser() user: CurrentUser, - @Args('workspaceId') workspaceId: string, - @Args('size', { type: () => SafeIntResolver }) blobSize: number - ) { - const canWrite = await this.permissions.tryCheckWorkspace( - workspaceId, - user.id, - Permission.Write - ); - if (canWrite) { - const size = await this.quota.checkBlobQuota(workspaceId, blobSize); - return { size }; - } - return false; - } - @Mutation(() => String) - @PreventCache(['blobs'], ['workspaceId']) async setBlob( @CurrentUser() user: CurrentUser, @Args('workspaceId') workspaceId: string, @@ -160,11 +141,35 @@ export class WorkspaceBlobResolver { } @Mutation(() => Boolean) - @PreventCache(['blobs'], ['workspaceId']) async deleteBlob( @CurrentUser() user: CurrentUser, @Args('workspaceId') workspaceId: string, - @Args('hash') name: string + @Args('hash', { + type: () => String, + deprecationReason: 'use parameter [key]', + nullable: true, + }) + hash?: string, + @Args('key', { type: () => String, nullable: true }) key?: string, + @Args('permanently', { type: () => Boolean, defaultValue: false }) + permanently = false + ) { + key = key ?? hash; + if (!key) { + return false; + } + + await this.permissions.checkWorkspace(workspaceId, user.id); + + await this.storage.delete(workspaceId, key, permanently); + + return true; + } + + @Mutation(() => Boolean) + async releaseDeletedBlobs( + @CurrentUser() user: CurrentUser, + @Args('workspaceId') workspaceId: string ) { await this.permissions.checkWorkspace( workspaceId, @@ -172,7 +177,7 @@ export class WorkspaceBlobResolver { Permission.Write ); - await this.storage.delete(workspaceId, name); + await this.storage.release(workspaceId); return true; } diff --git a/packages/backend/server/src/fundamentals/event/def.ts b/packages/backend/server/src/fundamentals/event/def.ts index 33db14b7a2832..c9cb42381350a 100644 --- a/packages/backend/server/src/fundamentals/event/def.ts +++ b/packages/backend/server/src/fundamentals/event/def.ts @@ -7,7 +7,7 @@ export interface WorkspaceEvents { blob: { deleted: Payload<{ workspaceId: Workspace['id']; - name: string; + key: string; }>; }; } diff --git a/packages/backend/server/src/fundamentals/storage/providers/fs.ts b/packages/backend/server/src/fundamentals/storage/providers/fs.ts index 282396c98d0f0..41e17f3e6679d 100644 --- a/packages/backend/server/src/fundamentals/storage/providers/fs.ts +++ b/packages/backend/server/src/fundamentals/storage/providers/fs.ts @@ -119,7 +119,7 @@ export class FsStorageProvider implements StorageProvider { results.push({ key: res, lastModified: stat.mtime, - size: stat.size, + contentLength: stat.size, }); } } @@ -216,7 +216,7 @@ export class FsStorageProvider implements StorageProvider { raw: PutObjectMetadata ) { try { - const metadata = await autoMetadata(blob, raw); + const metadata = autoMetadata(blob, raw); if (raw.checksumCRC32 && metadata.checksumCRC32 !== raw.checksumCRC32) { throw new Error( @@ -224,6 +224,12 @@ export class FsStorageProvider implements StorageProvider { ); } + if (raw.contentLength && metadata.contentLength !== raw.contentLength) { + throw new Error( + 'The content length of the uploaded file is not matched with the one you provide, the file may be corrupted and the uploading will not be processed.' + ); + } + writeFileSync( this.join(`${key}.metadata.json`), JSON.stringify({ diff --git a/packages/backend/server/src/fundamentals/storage/providers/provider.ts b/packages/backend/server/src/fundamentals/storage/providers/provider.ts index 46f8ef688db35..5b2bad24b2473 100644 --- a/packages/backend/server/src/fundamentals/storage/providers/provider.ts +++ b/packages/backend/server/src/fundamentals/storage/providers/provider.ts @@ -21,7 +21,7 @@ export interface PutObjectMetadata { export interface ListObjectsMetadata { key: string; lastModified: Date; - size: number; + contentLength: number; } export type BlobInputType = Buffer | Readable | string; diff --git a/packages/backend/server/src/fundamentals/storage/providers/utils.ts b/packages/backend/server/src/fundamentals/storage/providers/utils.ts index a0eab7d8c5451..4c7e9323f2888 100644 --- a/packages/backend/server/src/fundamentals/storage/providers/utils.ts +++ b/packages/backend/server/src/fundamentals/storage/providers/utils.ts @@ -14,19 +14,19 @@ export async function toBuffer(input: BlobInputType): Promise { : Buffer.from(input); } -export async function autoMetadata( +export function autoMetadata( blob: Buffer, - raw: PutObjectMetadata -): Promise { + raw: PutObjectMetadata = {} +): PutObjectMetadata { const metadata = { ...raw, }; - try { - // length - if (!metadata.contentLength) { - metadata.contentLength = blob.length; - } + if (!metadata.contentLength) { + metadata.contentLength = blob.byteLength; + } + + try { // checksum if (!metadata.checksumCRC32) { metadata.checksumCRC32 = crc32(blob).toString(16); @@ -34,15 +34,11 @@ export async function autoMetadata( // mime type if (!metadata.contentType) { - try { - metadata.contentType = getMime(blob); - } catch { - // ignore - } + metadata.contentType = getMime(blob); } - - return metadata; } catch { - return metadata; + // noop } + + return metadata; } diff --git a/packages/backend/server/src/plugins/storage/providers/s3.ts b/packages/backend/server/src/plugins/storage/providers/s3.ts index 3a242704dbf28..451bd5f6afc77 100644 --- a/packages/backend/server/src/plugins/storage/providers/s3.ts +++ b/packages/backend/server/src/plugins/storage/providers/s3.ts @@ -50,7 +50,7 @@ export class S3StorageProvider implements StorageProvider { ): Promise { const blob = await toBuffer(body); - metadata = await autoMetadata(blob, metadata); + metadata = autoMetadata(blob, metadata); try { await this.client.send( @@ -140,7 +140,7 @@ export class S3StorageProvider implements StorageProvider { listResult.Contents.map(r => ({ key: r.Key!, lastModified: r.LastModified!, - size: r.Size!, + contentLength: r.Size!, })) ); } diff --git a/packages/backend/server/src/schema.gql b/packages/backend/server/src/schema.gql index c138ef63f36f6..c463efc2b39de 100644 --- a/packages/backend/server/src/schema.gql +++ b/packages/backend/server/src/schema.gql @@ -449,6 +449,13 @@ input ListUserInput { skip: Int = 0 } +type ListedBlob { + createdAt: String! + key: String! + mime: String! + size: Int! +} + input ManageUserInput { """User email""" email: String @@ -496,7 +503,7 @@ type Mutation { """Create a new workspace""" createWorkspace(init: Upload): WorkspaceType! deleteAccount: DeleteAccount! - deleteBlob(hash: String!, workspaceId: String!): Boolean! + deleteBlob(hash: String @deprecated(reason: "use parameter [key]"), key: String, permanently: Boolean! = false, workspaceId: String!): Boolean! """Delete a user account""" deleteUser(id: String!): DeleteAccount! @@ -511,6 +518,7 @@ type Mutation { leaveWorkspace(sendLeaveMail: Boolean, workspaceId: String!, workspaceName: String!): Boolean! publishPage(mode: PublicPageMode = Page, pageId: String!, workspaceId: String!): WorkspacePage! recoverDoc(guid: String!, timestamp: DateTime!, workspaceId: String!): DateTime! + releaseDeletedBlobs(workspaceId: String!): Boolean! """Remove user avatar""" removeAvatar: RemoveAvatar! @@ -584,8 +592,7 @@ enum PublicPageMode { } type Query { - checkBlobSize(size: SafeInt!, workspaceId: String!): WorkspaceBlobSizes! @deprecated(reason: "no more needed") - collectAllBlobSizes: WorkspaceBlobSizes! @deprecated(reason: "use `user.storageUsage` instead") + collectAllBlobSizes: WorkspaceBlobSizes! @deprecated(reason: "use `user.quotaUsage` instead") """Get current user""" currentUser: UserType @@ -885,6 +892,10 @@ type UserQuotaHumanReadable { storageQuota: String! } +type UserQuotaUsage { + storageQuota: SafeInt! +} + type UserType { """User avatar url""" avatarUrl: String @@ -913,6 +924,7 @@ type UserType { """User name""" name: String! quota: UserQuota + quotaUsage: UserQuotaUsage! subscriptions: [SubscriptionType!]! token: tokenType! @deprecated(reason: "use [/api/auth/sign-in?native=true] instead") } @@ -962,7 +974,7 @@ type WorkspaceType { availableFeatures: [FeatureType!]! """List blobs of workspace""" - blobs: [String!]! + blobs: [ListedBlob!]! """Blobs size of workspace""" blobsSize: Int! diff --git a/packages/backend/server/tests/utils/blobs.ts b/packages/backend/server/tests/utils/blobs.ts index 514fbeee468d1..f6832c41c401d 100644 --- a/packages/backend/server/tests/utils/blobs.ts +++ b/packages/backend/server/tests/utils/blobs.ts @@ -54,35 +54,16 @@ export async function collectAllBlobSizes( .send({ query: ` query { - collectAllBlobSizes { - size + currentUser { + quotaUsage { + storageQuota + } } } `, }) .expect(200); - return res.body.data.collectAllBlobSizes.size; -} - -export async function checkBlobSize( - app: INestApplication, - token: string, - workspaceId: string, - size: number -): Promise { - const res = await request(app.getHttpServer()) - .post(gql) - .auth(token, { type: 'bearer' }) - .send({ - query: `query checkBlobSize($workspaceId: String!, $size: SafeInt!) { - checkBlobSize(workspaceId: $workspaceId, size: $size) { - size - } - }`, - variables: { workspaceId, size }, - }) - .expect(200); - return res.body.data.checkBlobSize.size; + return res.body.data.currentUser.quotaUsage.storageQuota; } export async function setBlob( diff --git a/packages/backend/server/tests/workspace-blobs.e2e.ts b/packages/backend/server/tests/workspace/blobs.e2e.ts similarity index 76% rename from packages/backend/server/tests/workspace-blobs.e2e.ts rename to packages/backend/server/tests/workspace/blobs.e2e.ts index 3e81d5fbf7aff..08dc6947fce4d 100644 --- a/packages/backend/server/tests/workspace-blobs.e2e.ts +++ b/packages/backend/server/tests/workspace/blobs.e2e.ts @@ -2,11 +2,10 @@ import type { INestApplication } from '@nestjs/common'; import test from 'ava'; import request from 'supertest'; -import { AppModule } from '../src/app.module'; -import { FeatureManagementService, FeatureType } from '../src/core/features'; -import { QuotaService, QuotaType } from '../src/core/quota'; +import { AppModule } from '../../src/app.module'; +import { FeatureManagementService, FeatureType } from '../../src/core/features'; +import { QuotaService, QuotaType } from '../../src/core/quota'; import { - checkBlobSize, collectAllBlobSizes, createTestingApp, createWorkspace, @@ -14,7 +13,7 @@ import { listBlobs, setBlob, signUp, -} from './utils'; +} from '../utils'; const OneMB = 1024 * 1024; @@ -114,58 +113,6 @@ test('should calc all blobs size', async t => { const size = await collectAllBlobSizes(app, u1.token.token); t.is(size, 8, 'failed to collect all blob sizes'); - - const size1 = await checkBlobSize( - app, - u1.token.token, - workspace1.id, - 10 * 1024 * 1024 * 1024 - 8 - ); - t.is(size1, 0, 'failed to check blob size'); - - const size2 = await checkBlobSize( - app, - u1.token.token, - workspace1.id, - 10 * 1024 * 1024 * 1024 - 7 - ); - t.is(size2, -1, 'failed to check blob size'); -}); - -test('should be able calc quota after switch plan', async t => { - const u1 = await signUp(app, 'darksky', 'darksky@affine.pro', '1'); - - const workspace1 = await createWorkspace(app, u1.token.token); - - const buffer1 = Buffer.from([0, 0]); - await setBlob(app, u1.token.token, workspace1.id, buffer1); - const buffer2 = Buffer.from([0, 1]); - await setBlob(app, u1.token.token, workspace1.id, buffer2); - - const workspace2 = await createWorkspace(app, u1.token.token); - - const buffer3 = Buffer.from([0, 0]); - await setBlob(app, u1.token.token, workspace2.id, buffer3); - const buffer4 = Buffer.from([0, 1]); - await setBlob(app, u1.token.token, workspace2.id, buffer4); - - const size1 = await checkBlobSize( - app, - u1.token.token, - workspace1.id, - 10 * 1024 * 1024 * 1024 - 8 - ); - t.is(size1, 0, 'failed to check free plan blob size'); - - await quota.switchUserQuota(u1.id, QuotaType.ProPlanV1); - - const size2 = await checkBlobSize( - app, - u1.token.token, - workspace1.id, - 100 * 1024 * 1024 * 1024 - 8 - ); - t.is(size2, 0, 'failed to check pro plan blob size'); }); test('should reject blob exceeded limit', async t => { diff --git a/packages/common/nbstore/package.json b/packages/common/nbstore/package.json index 8d39c3491e3c8..8d0e26c0ebcda 100644 --- a/packages/common/nbstore/package.json +++ b/packages/common/nbstore/package.json @@ -8,7 +8,8 @@ ".": "./src/index.ts", "./op": "./src/op/index.ts", "./idb": "./src/impls/idb/index.ts", - "./idb/v1": "./src/impls/idb/v1/index.ts" + "./idb/v1": "./src/impls/idb/v1/index.ts", + "./cloud": "./src/impls/cloud/index.ts" }, "dependencies": { "@datastructures-js/binary-search-tree": "^5.3.2", @@ -20,11 +21,15 @@ "yjs": "patch:yjs@npm%3A13.6.18#~/.yarn/patches/yjs-npm-13.6.18-ad0d5f7c43.patch" }, "devDependencies": { + "@affine/graphql": "workspace:*", "fake-indexeddb": "^6.0.0", "idb": "^8.0.0", + "socket.io-client": "^4.7.5", "vitest": "2.1.4" }, "peerDependencies": { - "idb": "^8.0.0" + "@affine/graphql": "workspace:*", + "idb": "^8.0.0", + "socket.io-client": "^4.7.5" } } diff --git a/packages/common/nbstore/src/impls/cloud/blob.ts b/packages/common/nbstore/src/impls/cloud/blob.ts new file mode 100644 index 0000000000000..671687d75ced1 --- /dev/null +++ b/packages/common/nbstore/src/impls/cloud/blob.ts @@ -0,0 +1,72 @@ +import { + deleteBlobMutation, + gqlFetcherFactory, + listBlobsQuery, + releaseDeletedBlobsMutation, + setBlobMutation, +} from '@affine/graphql'; + +import { DummyConnection } from '../../connection'; +import { type BlobRecord, BlobStorage } from '../../storage'; + +export class CloudBlobStorage extends BlobStorage { + private readonly gql = gqlFetcherFactory(this.options.peer + '/graphql'); + override connection = new DummyConnection(); + + override async get(key: string) { + const res = await fetch( + this.options.peer + '/api/workspaces/' + this.spaceId + '/blobs/' + key, + { cache: 'default' } + ); + + if (!res.ok) { + return null; + } + + const data = await res.arrayBuffer(); + + return { + key, + data: new Uint8Array(data), + mime: res.headers.get('content-type') || '', + size: data.byteLength, + createdAt: new Date(res.headers.get('last-modified') || Date.now()), + }; + } + + override async set(blob: BlobRecord) { + await this.gql({ + query: setBlobMutation, + variables: { + workspaceId: this.spaceId, + blob: new File([blob.data], blob.key, { type: blob.mime }), + }, + }); + } + + override async delete(key: string, permanently: boolean) { + await this.gql({ + query: deleteBlobMutation, + variables: { workspaceId: this.spaceId, key, permanently }, + }); + } + + override async release() { + await this.gql({ + query: releaseDeletedBlobsMutation, + variables: { workspaceId: this.spaceId }, + }); + } + + override async list() { + const res = await this.gql({ + query: listBlobsQuery, + variables: { workspaceId: this.spaceId }, + }); + + return res.workspace.blobs.map(blob => ({ + ...blob, + createdAt: new Date(blob.createdAt), + })); + } +} diff --git a/packages/common/nbstore/src/impls/cloud/doc.ts b/packages/common/nbstore/src/impls/cloud/doc.ts new file mode 100644 index 0000000000000..327ac777fa90c --- /dev/null +++ b/packages/common/nbstore/src/impls/cloud/doc.ts @@ -0,0 +1,199 @@ +import { noop } from 'lodash-es'; +import type { SocketOptions } from 'socket.io-client'; + +import { share } from '../../connection'; +import { + type DocClock, + type DocClocks, + DocStorage, + type DocStorageOptions, + type DocUpdate, +} from '../../storage'; +import { + base64ToUint8Array, + type ServerEventsMap, + SocketConnection, + uint8ArrayToBase64, +} from './socket'; + +interface CloudDocStorageOptions extends DocStorageOptions { + socketOptions: SocketOptions; +} + +export class CloudDocStorage extends DocStorage { + connection = share( + new SocketConnection(this.peer, this.options.socketOptions) + ); + + private get socket() { + return this.connection.inner; + } + + override async connect(): Promise { + await super.connect(); + this.connection.onStatusChanged(status => { + if (status === 'connected') { + this.join().catch(noop); + this.socket.on('space:broadcast-doc-update', this.onServerUpdate); + } + }); + } + + override async disconnect(): Promise { + this.socket.emit('space:leave', { + spaceType: this.spaceType, + spaceId: this.spaceId, + }); + this.socket.off('space:broadcast-doc-update', this.onServerUpdate); + await super.connect(); + } + + async join() { + try { + const res = await this.socket.emitWithAck('space:join', { + spaceType: this.spaceType, + spaceId: this.spaceId, + clientVersion: BUILD_CONFIG.appVersion, + }); + + if ('error' in res) { + this.connection.setStatus('closed', new Error(res.error.message)); + } + } catch (e) { + this.connection.setStatus('error', e as Error); + } + } + + onServerUpdate: ServerEventsMap['space:broadcast-doc-update'] = message => { + if ( + this.spaceType === message.spaceType && + this.spaceId === message.spaceId + ) { + this.emit('update', { + docId: message.docId, + bin: base64ToUint8Array(message.update), + timestamp: new Date(message.timestamp), + editor: message.editor, + }); + } + }; + + override async getDocSnapshot(docId: string) { + const response = await this.socket.emitWithAck('space:load-doc', { + spaceType: this.spaceType, + spaceId: this.spaceId, + docId, + }); + + if ('error' in response) { + // TODO: use [UserFriendlyError] + throw new Error(response.error.message); + } + + return { + docId, + bin: base64ToUint8Array(response.data.missing), + timestamp: new Date(response.data.timestamp), + }; + } + + override async getDocDiff(docId: string, state?: Uint8Array) { + const response = await this.socket.emitWithAck('space:load-doc', { + spaceType: this.spaceType, + spaceId: this.spaceId, + docId, + stateVector: state ? await uint8ArrayToBase64(state) : void 0, + }); + + if ('error' in response) { + // TODO: use [UserFriendlyError] + throw new Error(response.error.message); + } + + return { + docId, + missing: base64ToUint8Array(response.data.missing), + state: base64ToUint8Array(response.data.state), + timestamp: new Date(response.data.timestamp), + }; + } + + override async pushDocUpdate(update: DocUpdate) { + const response = await this.socket.emitWithAck('space:push-doc-update', { + spaceType: this.spaceType, + spaceId: this.spaceId, + docId: update.docId, + updates: await uint8ArrayToBase64(update.bin), + }); + + if ('error' in response) { + // TODO(@forehalo): use [UserFriendlyError] + throw new Error(response.error.message); + } + + return { + docId: update.docId, + timestamp: new Date(response.data.timestamp), + }; + } + + /** + * Just a rough implementation, cloud doc storage should not need this method. + */ + override async getDocTimestamp(docId: string): Promise { + const response = await this.socket.emitWithAck('space:load-doc', { + spaceType: this.spaceType, + spaceId: this.spaceId, + docId, + }); + + if ('error' in response) { + // TODO: use [UserFriendlyError] + throw new Error(response.error.message); + } + + return { + docId, + timestamp: new Date(response.data.timestamp), + }; + } + + override async getDocTimestamps(after?: Date) { + const response = await this.socket.emitWithAck( + 'space:load-doc-timestamps', + { + spaceType: this.spaceType, + spaceId: this.spaceId, + timestamp: after ? after.getTime() : undefined, + } + ); + + if ('error' in response) { + // TODO(@forehalo): use [UserFriendlyError] + throw new Error(response.error.message); + } + + return Object.entries(response.data).reduce((ret, [docId, timestamp]) => { + ret[docId] = new Date(timestamp); + return ret; + }, {} as DocClocks); + } + + override async deleteDoc(docId: string) { + this.socket.emit('space:delete-doc', { + spaceType: this.spaceType, + spaceId: this.spaceId, + docId, + }); + } + + protected async setDocSnapshot() { + return false; + } + protected async getDocUpdates() { + return []; + } + protected async markUpdatesMerged() { + return 0; + } +} diff --git a/packages/common/nbstore/src/impls/cloud/index.ts b/packages/common/nbstore/src/impls/cloud/index.ts new file mode 100644 index 0000000000000..d476ae6eb9b92 --- /dev/null +++ b/packages/common/nbstore/src/impls/cloud/index.ts @@ -0,0 +1,2 @@ +export * from './blob'; +export * from './doc'; diff --git a/packages/common/nbstore/src/impls/cloud/socket.ts b/packages/common/nbstore/src/impls/cloud/socket.ts new file mode 100644 index 0000000000000..6e84a433e372a --- /dev/null +++ b/packages/common/nbstore/src/impls/cloud/socket.ts @@ -0,0 +1,173 @@ +import { + Manager as SocketIOManager, + type Socket as SocketIO, + type SocketOptions, +} from 'socket.io-client'; + +import { Connection, type ConnectionStatus } from '../../connection'; + +// TODO(@forehalo): use [UserFriendlyError] +interface EventError { + name: string; + message: string; +} + +type WebsocketResponse = + | { + error: EventError; + } + | { + data: T; + }; + +interface ServerEvents { + 'space:broadcast-doc-update': { + spaceType: string; + spaceId: string; + docId: string; + update: string; + timestamp: number; + editor: string; + }; +} + +interface ClientEvents { + 'space:join': [ + { spaceType: string; spaceId: string; clientVersion: string }, + { clientId: string }, + ]; + 'space:leave': { spaceType: string; spaceId: string }; + 'space:join-awareness': [ + { + spaceType: string; + spaceId: string; + docId: string; + clientVersion: string; + }, + { clientId: string }, + ]; + 'space:leave-awareness': { + spaceType: string; + spaceId: string; + docId: string; + }; + + 'space:push-doc-update': [ + { spaceType: string; spaceId: string; docId: string; updates: string }, + { timestamp: number }, + ]; + 'space:load-doc-timestamps': [ + { + spaceType: string; + spaceId: string; + timestamp?: number; + }, + Record, + ]; + 'space:load-doc': [ + { + spaceType: string; + spaceId: string; + docId: string; + stateVector?: string; + }, + { + missing: string; + state: string; + timestamp: number; + }, + ]; + 'space:delete-doc': { spaceType: string; spaceId: string; docId: string }; +} + +export type ServerEventsMap = { + [Key in keyof ServerEvents]: (data: ServerEvents[Key]) => void; +}; + +export type ClientEventsMap = { + [Key in keyof ClientEvents]: ClientEvents[Key] extends Array + ? ( + data: ClientEvents[Key][0], + ack: (res: WebsocketResponse) => void + ) => void + : (data: ClientEvents[Key]) => void; +}; + +export type Socket = SocketIO; + +export function uint8ArrayToBase64(array: Uint8Array): Promise { + return new Promise(resolve => { + // Create a blob from the Uint8Array + const blob = new Blob([array]); + + const reader = new FileReader(); + reader.onload = function () { + const dataUrl = reader.result as string | null; + if (!dataUrl) { + resolve(''); + return; + } + // The result includes the `data:` URL prefix and the MIME type. We only want the Base64 data + const base64 = dataUrl.split(',')[1]; + resolve(base64); + }; + + reader.readAsDataURL(blob); + }); +} + +export function base64ToUint8Array(base64: string) { + const binaryString = atob(base64); + const binaryArray = binaryString.split('').map(function (char) { + return char.charCodeAt(0); + }); + return new Uint8Array(binaryArray); +} + +export class SocketConnection extends Connection { + manager = new SocketIOManager(this.endpoint, { + autoConnect: false, + transports: ['websocket'], + secure: new URL(this.endpoint).protocol === 'https:', + }); + + constructor( + private readonly endpoint: string, + private readonly socketOptions: SocketOptions + ) { + super(); + } + + override get shareId() { + return `socket:${this.endpoint}`; + } + + override async doConnect() { + const conn = this.manager.socket('/', this.socketOptions); + + await new Promise((resolve, reject) => { + conn.once('connect', () => { + resolve(); + }); + conn.once('connect_error', err => { + reject(err); + }); + conn.open(); + }); + + return conn; + } + + override async doDisconnect(conn: Socket) { + conn.close(); + } + + /** + * Socket connection allow explicitly set status by user + * + * used when join space failed + */ + override setStatus(status: ConnectionStatus, error?: Error) { + super.setStatus(status, error); + } +} diff --git a/packages/common/nbstore/src/impls/index.ts b/packages/common/nbstore/src/impls/index.ts index 8bce204798b51..bff2874ad4467 100644 --- a/packages/common/nbstore/src/impls/index.ts +++ b/packages/common/nbstore/src/impls/index.ts @@ -1,4 +1,5 @@ import type { Storage } from '../storage'; +import { CloudBlobStorage, CloudDocStorage } from './cloud'; import { IndexedDBBlobStorage, IndexedDBDocStorage, @@ -19,7 +20,9 @@ const idbv1: StorageConstructor[] = [ IndexedDBV1BlobStorage, ]; -export const storages: StorageConstructor[] = [...idbv1, ...idb]; +const cloud: StorageConstructor[] = [CloudDocStorage, CloudBlobStorage]; + +export const storages: StorageConstructor[] = cloud.concat(idbv1, idb); const AvailableStorageImplementations = storages.reduce( (acc, curr) => { diff --git a/packages/frontend/core/src/modules/cloud/stores/user-quota.ts b/packages/frontend/core/src/modules/cloud/stores/user-quota.ts index 0413d46dcb8b5..70698060aec26 100644 --- a/packages/frontend/core/src/modules/cloud/stores/user-quota.ts +++ b/packages/frontend/core/src/modules/cloud/stores/user-quota.ts @@ -23,7 +23,7 @@ export class UserQuotaStore extends Store { return { userId: data.currentUser.id, quota: data.currentUser.quota, - used: data.collectAllBlobSizes.size, + used: data.currentUser.quotaUsage.storageQuota, }; } } diff --git a/packages/frontend/core/src/modules/workspace-engine/impls/engine/blob-cloud.ts b/packages/frontend/core/src/modules/workspace-engine/impls/engine/blob-cloud.ts index 3eacf5dbfb691..e6549b0e05e8f 100644 --- a/packages/frontend/core/src/modules/workspace-engine/impls/engine/blob-cloud.ts +++ b/packages/frontend/core/src/modules/workspace-engine/impls/engine/blob-cloud.ts @@ -70,7 +70,7 @@ export class CloudBlobStorage implements BlobStorage { query: deleteBlobMutation, variables: { workspaceId: key, - hash: key, + key, }, }); } @@ -82,6 +82,6 @@ export class CloudBlobStorage implements BlobStorage { workspaceId: this.workspaceId, }, }); - return result.listBlobs; + return result.workspace.blobs.map(blob => blob.key); } } diff --git a/packages/frontend/graphql/src/graphql/blob-delete.gql b/packages/frontend/graphql/src/graphql/blob-delete.gql index 790c326465909..ab78a54d88c6a 100644 --- a/packages/frontend/graphql/src/graphql/blob-delete.gql +++ b/packages/frontend/graphql/src/graphql/blob-delete.gql @@ -1,3 +1,7 @@ -mutation deleteBlob($workspaceId: String!, $hash: String!) { - deleteBlob(workspaceId: $workspaceId, hash: $hash) +mutation deleteBlob( + $workspaceId: String! + $key: String! + $permanently: Boolean +) { + deleteBlob(workspaceId: $workspaceId, key: $key, permanently: $permanently) } diff --git a/packages/frontend/graphql/src/graphql/blob-list.gql b/packages/frontend/graphql/src/graphql/blob-list.gql index 67a19c718c1bb..5b314f0cc5cfa 100644 --- a/packages/frontend/graphql/src/graphql/blob-list.gql +++ b/packages/frontend/graphql/src/graphql/blob-list.gql @@ -1,3 +1,10 @@ query listBlobs($workspaceId: String!) { - listBlobs(workspaceId: $workspaceId) + workspace(id: $workspaceId) { + blobs { + key + size + mime + createdAt + } + } } diff --git a/packages/frontend/graphql/src/graphql/blob-release-deleted.gql b/packages/frontend/graphql/src/graphql/blob-release-deleted.gql new file mode 100644 index 0000000000000..d51d0ef7ca68d --- /dev/null +++ b/packages/frontend/graphql/src/graphql/blob-release-deleted.gql @@ -0,0 +1,3 @@ +mutation releaseDeletedBlobs($workspaceId: String!) { + releaseDeletedBlobs(workspaceId: $workspaceId) +} diff --git a/packages/frontend/graphql/src/graphql/index.ts b/packages/frontend/graphql/src/graphql/index.ts index 638cfd3846dea..d22e5dc345920 100644 --- a/packages/frontend/graphql/src/graphql/index.ts +++ b/packages/frontend/graphql/src/graphql/index.ts @@ -47,19 +47,37 @@ export const deleteBlobMutation = { definitionName: 'deleteBlob', containsFile: false, query: ` -mutation deleteBlob($workspaceId: String!, $hash: String!) { - deleteBlob(workspaceId: $workspaceId, hash: $hash) +mutation deleteBlob($workspaceId: String!, $key: String!, $permanently: Boolean) { + deleteBlob(workspaceId: $workspaceId, key: $key, permanently: $permanently) }`, }; export const listBlobsQuery = { id: 'listBlobsQuery' as const, operationName: 'listBlobs', - definitionName: 'listBlobs', + definitionName: 'workspace', containsFile: false, query: ` query listBlobs($workspaceId: String!) { - listBlobs(workspaceId: $workspaceId) + workspace(id: $workspaceId) { + blobs { + key + size + mime + createdAt + } + } +}`, +}; + +export const releaseDeletedBlobsMutation = { + id: 'releaseDeletedBlobsMutation' as const, + operationName: 'releaseDeletedBlobs', + definitionName: 'releaseDeletedBlobs', + containsFile: false, + query: ` +mutation releaseDeletedBlobs($workspaceId: String!) { + releaseDeletedBlobs(workspaceId: $workspaceId) }`, }; @@ -885,7 +903,7 @@ mutation publishPage($workspaceId: String!, $pageId: String!, $mode: PublicPageM export const quotaQuery = { id: 'quotaQuery' as const, operationName: 'quota', - definitionName: 'currentUser,collectAllBlobSizes', + definitionName: 'currentUser', containsFile: false, query: ` query quota { @@ -905,9 +923,9 @@ query quota { memberLimit } } - } - collectAllBlobSizes { - size + quotaUsage { + storageQuota + } } }`, }; diff --git a/packages/frontend/graphql/src/graphql/quota.gql b/packages/frontend/graphql/src/graphql/quota.gql index c0268b6443584..682c2e5bc3550 100644 --- a/packages/frontend/graphql/src/graphql/quota.gql +++ b/packages/frontend/graphql/src/graphql/quota.gql @@ -15,8 +15,8 @@ query quota { memberLimit } } - } - collectAllBlobSizes { - size + quotaUsage { + storageQuota + } } } diff --git a/packages/frontend/graphql/src/schema.ts b/packages/frontend/graphql/src/schema.ts index 4a7fd44c5564b..4c554e068e5b5 100644 --- a/packages/frontend/graphql/src/schema.ts +++ b/packages/frontend/graphql/src/schema.ts @@ -517,6 +517,14 @@ export interface ListUserInput { skip?: InputMaybe; } +export interface ListedBlob { + __typename?: 'ListedBlob'; + createdAt: Scalars['String']['output']; + key: Scalars['String']['output']; + mime: Scalars['String']['output']; + size: Scalars['Int']['output']; +} + export interface ManageUserInput { /** User email */ email?: InputMaybe; @@ -569,6 +577,7 @@ export interface Mutation { leaveWorkspace: Scalars['Boolean']['output']; publishPage: WorkspacePage; recoverDoc: Scalars['DateTime']['output']; + releaseDeletedBlobs: Scalars['Boolean']['output']; /** Remove user avatar */ removeAvatar: RemoveAvatar; removeWorkspaceFeature: Scalars['Int']['output']; @@ -673,7 +682,9 @@ export interface MutationCreateWorkspaceArgs { } export interface MutationDeleteBlobArgs { - hash: Scalars['String']['input']; + hash?: InputMaybe; + key?: InputMaybe; + permanently?: Scalars['Boolean']['input']; workspaceId: Scalars['String']['input']; } @@ -731,6 +742,10 @@ export interface MutationRecoverDocArgs { workspaceId: Scalars['String']['input']; } +export interface MutationReleaseDeletedBlobsArgs { + workspaceId: Scalars['String']['input']; +} + export interface MutationRemoveWorkspaceFeatureArgs { feature: FeatureType; workspaceId: Scalars['String']['input']; @@ -882,9 +897,7 @@ export enum PublicPageMode { export interface Query { __typename?: 'Query'; - /** @deprecated no more needed */ - checkBlobSize: WorkspaceBlobSizes; - /** @deprecated use `user.storageUsage` instead */ + /** @deprecated use `user.quotaUsage` instead */ collectAllBlobSizes: WorkspaceBlobSizes; /** Get current user */ currentUser: Maybe; @@ -925,11 +938,6 @@ export interface Query { workspaces: Array; } -export interface QueryCheckBlobSizeArgs { - size: Scalars['SafeInt']['input']; - workspaceId: Scalars['String']['input']; -} - export interface QueryErrorArgs { name: ErrorNames; } @@ -1225,6 +1233,11 @@ export interface UserQuotaHumanReadable { storageQuota: Scalars['String']['output']; } +export interface UserQuotaUsage { + __typename?: 'UserQuotaUsage'; + storageQuota: Scalars['SafeInt']['output']; +} + export interface UserType { __typename?: 'UserType'; /** User avatar url */ @@ -1250,6 +1263,7 @@ export interface UserType { /** User name */ name: Scalars['String']['output']; quota: Maybe; + quotaUsage: UserQuotaUsage; subscriptions: Array; /** @deprecated use [/api/auth/sign-in?native=true] instead */ token: TokenType; @@ -1313,7 +1327,7 @@ export interface WorkspaceType { /** Available features of workspace */ availableFeatures: Array; /** List blobs of workspace */ - blobs: Array; + blobs: Array; /** Blobs size of workspace */ blobsSize: Scalars['Int']['output']; /** Workspace created date */ @@ -1417,7 +1431,8 @@ export type AdminServerConfigQuery = { export type DeleteBlobMutationVariables = Exact<{ workspaceId: Scalars['String']['input']; - hash: Scalars['String']['input']; + key: Scalars['String']['input']; + permanently?: InputMaybe; }>; export type DeleteBlobMutation = { @@ -1429,7 +1444,28 @@ export type ListBlobsQueryVariables = Exact<{ workspaceId: Scalars['String']['input']; }>; -export type ListBlobsQuery = { __typename?: 'Query'; listBlobs: Array }; +export type ListBlobsQuery = { + __typename?: 'Query'; + workspace: { + __typename?: 'WorkspaceType'; + blobs: Array<{ + __typename?: 'ListedBlob'; + key: string; + size: number; + mime: string; + createdAt: string; + }>; + }; +}; + +export type ReleaseDeletedBlobsMutationVariables = Exact<{ + workspaceId: Scalars['String']['input']; +}>; + +export type ReleaseDeletedBlobsMutation = { + __typename?: 'Mutation'; + releaseDeletedBlobs: boolean; +}; export type SetBlobMutationVariables = Exact<{ workspaceId: Scalars['String']['input']; @@ -2196,8 +2232,8 @@ export type QuotaQuery = { memberLimit: string; }; } | null; + quotaUsage: { __typename?: 'UserQuotaUsage'; storageQuota: number }; } | null; - collectAllBlobSizes: { __typename?: 'WorkspaceBlobSizes'; size: number }; }; export type RecoverDocMutationVariables = Exact<{ @@ -2953,6 +2989,11 @@ export type Mutations = variables: DeleteBlobMutationVariables; response: DeleteBlobMutation; } + | { + name: 'releaseDeletedBlobsMutation'; + variables: ReleaseDeletedBlobsMutationVariables; + response: ReleaseDeletedBlobsMutation; + } | { name: 'setBlobMutation'; variables: SetBlobMutationVariables; diff --git a/yarn.lock b/yarn.lock index 784182955d3fe..0a341daab005e 100644 --- a/yarn.lock +++ b/yarn.lock @@ -722,6 +722,7 @@ __metadata: version: 0.0.0-use.local resolution: "@affine/nbstore@workspace:packages/common/nbstore" dependencies: + "@affine/graphql": "workspace:*" "@datastructures-js/binary-search-tree": "npm:^5.3.2" "@toeverything/infra": "workspace:*" eventemitter2: "npm:^6.4.9" @@ -730,10 +731,13 @@ __metadata: lodash-es: "npm:^4.17.21" nanoid: "npm:^5.0.7" rxjs: "npm:^7.8.1" + socket.io-client: "npm:^4.7.5" vitest: "npm:2.1.4" yjs: "patch:yjs@npm%3A13.6.18#~/.yarn/patches/yjs-npm-13.6.18-ad0d5f7c43.patch" peerDependencies: + "@affine/graphql": "workspace:*" idb: ^8.0.0 + socket.io-client: ^4.7.5 languageName: unknown linkType: soft @@ -11491,32 +11495,42 @@ __metadata: languageName: node linkType: hard -"@shikijs/core@npm:1.24.0": - version: 1.24.0 - resolution: "@shikijs/core@npm:1.24.0" +"@shikijs/core@npm:1.22.2": + version: 1.22.2 + resolution: "@shikijs/core@npm:1.22.2" dependencies: - "@shikijs/engine-javascript": "npm:1.24.0" - "@shikijs/engine-oniguruma": "npm:1.24.0" - "@shikijs/types": "npm:1.24.0" + "@shikijs/engine-javascript": "npm:1.22.2" + "@shikijs/engine-oniguruma": "npm:1.22.2" + "@shikijs/types": "npm:1.22.2" "@shikijs/vscode-textmate": "npm:^9.3.0" "@types/hast": "npm:^3.0.4" hast-util-to-html: "npm:^9.0.3" - checksum: 10/7a8944676d37c902b83b0585164675d55185881079bb75ce5411cb01fe6f4c5fd8411291e99a399c2d9f6b21955fbb6a7e3040e16f8bf43b2bafaa033320e12f + checksum: 10/c5203e1cbef8e159fc4ef4556b350dc055d9d15af57cb12ea699c94ecd603e58f7000e106fd5d103e1a1c8d1cc975cd7c573e9bacaa01f7e5eaa05b921d1ee38 languageName: node linkType: hard -"@shikijs/engine-javascript@npm:1.24.0": - version: 1.24.0 - resolution: "@shikijs/engine-javascript@npm:1.24.0" +"@shikijs/engine-javascript@npm:1.22.2": + version: 1.22.2 + resolution: "@shikijs/engine-javascript@npm:1.22.2" dependencies: - "@shikijs/types": "npm:1.24.0" + "@shikijs/types": "npm:1.22.2" "@shikijs/vscode-textmate": "npm:^9.3.0" - oniguruma-to-es: "npm:0.7.0" - checksum: 10/f5a1832bcbad0761292172dbfdd4dbedbd87f9f31f786606798228d5355ccb8d10bf672741a6e1d2be48570b8cf810402704a4131e82ac10797806b5301e34d8 + oniguruma-to-js: "npm:0.4.3" + checksum: 10/162f089f7ec7bc8e6877e1047bdf339a7446b7407ad0bffcb4b7372263ae5aae0be429f1c87054326be79d4e1bbe55849c010ea4aa499e83816ce009e490938b languageName: node linkType: hard -"@shikijs/engine-oniguruma@npm:1.24.0, @shikijs/engine-oniguruma@npm:^1.24.0": +"@shikijs/engine-oniguruma@npm:1.22.2": + version: 1.22.2 + resolution: "@shikijs/engine-oniguruma@npm:1.22.2" + dependencies: + "@shikijs/types": "npm:1.22.2" + "@shikijs/vscode-textmate": "npm:^9.3.0" + checksum: 10/924fff6c3d0e464ab2bde326076535fc1f98c0c90ceba1811b89f25c7b9df36a0fbec509b2859f5e2097e29f8087e90fa6b951fa8183d44c1abaa21a49c71e9e + languageName: node + linkType: hard + +"@shikijs/engine-oniguruma@npm:^1.24.0": version: 1.24.0 resolution: "@shikijs/engine-oniguruma@npm:1.24.0" dependencies: @@ -11526,6 +11540,16 @@ __metadata: languageName: node linkType: hard +"@shikijs/types@npm:1.22.2": + version: 1.22.2 + resolution: "@shikijs/types@npm:1.22.2" + dependencies: + "@shikijs/vscode-textmate": "npm:^9.3.0" + "@types/hast": "npm:^3.0.4" + checksum: 10/bba6e4d8ef76fae30e9c298539e6b6b4f82360d894352fc54882531e71b5f5c490a1a49ae83d4133a0de85c7d58ec678c6ceb5f5f5d012cd09a289d8845b1737 + languageName: node + linkType: hard + "@shikijs/types@npm:1.24.0, @shikijs/types@npm:^1.24.0": version: 1.24.0 resolution: "@shikijs/types@npm:1.24.0" @@ -18915,13 +18939,6 @@ __metadata: languageName: node linkType: hard -"emoji-regex-xs@npm:^1.0.0": - version: 1.0.0 - resolution: "emoji-regex-xs@npm:1.0.0" - checksum: 10/e216ec4270f765e1097cefc1b9518a7166b872b4424c60a85d79765f318d989cd458e036c76c13e9ce2ed1fe1bb5935a7fd5c1fab7600668bc8e92a789045b3c - languageName: node - linkType: hard - "emoji-regex@npm:^10.3.0": version: 10.4.0 resolution: "emoji-regex@npm:10.4.0" @@ -26468,14 +26485,12 @@ __metadata: languageName: node linkType: hard -"oniguruma-to-es@npm:0.7.0": - version: 0.7.0 - resolution: "oniguruma-to-es@npm:0.7.0" +"oniguruma-to-js@npm:0.4.3": + version: 0.4.3 + resolution: "oniguruma-to-js@npm:0.4.3" dependencies: - emoji-regex-xs: "npm:^1.0.0" - regex: "npm:^5.0.2" - regex-recursion: "npm:^4.3.0" - checksum: 10/766f2c4a9a9eb97070914ebbd78517d073c58f2558994cffb58b064facf860b8f568c7146281e527c796631e93ac23cc1c4b897436189033785429a4486ad41d + regex: "npm:^4.3.2" + checksum: 10/af64a77f4e428c2652387014596138c51bd61d67b0bbe957cd10ff73b4ec14567701ff9286342ab804cfa00486a9a0ff189da8391721c21c898ea8e26b62e74f languageName: node linkType: hard @@ -28783,28 +28798,10 @@ __metadata: languageName: node linkType: hard -"regex-recursion@npm:^4.3.0": - version: 4.3.0 - resolution: "regex-recursion@npm:4.3.0" - dependencies: - regex-utilities: "npm:^2.3.0" - checksum: 10/bbb7fcd6542c980cb3a4571186928826b263759e89bbc1c7b313d9f1064b6b1878c414a696b9cee01156a42225e508a62003f3edaab52a0a3344debf3211ebd8 - languageName: node - linkType: hard - -"regex-utilities@npm:^2.3.0": - version: 2.3.0 - resolution: "regex-utilities@npm:2.3.0" - checksum: 10/d11519c31f379488cbc6278b8645d72f16339ee325c79a4b8b3a6477738016a52983158dc69ae1b5867f8b06978ff5d83933520257a57f7e5c3e4ac6a1ea3cc7 - languageName: node - linkType: hard - -"regex@npm:^5.0.2": - version: 5.0.2 - resolution: "regex@npm:5.0.2" - dependencies: - regex-utilities: "npm:^2.3.0" - checksum: 10/c9dab5adc2df30a37bed0665b4830be170e413e48bb0fc149388161995dc250049ce0aa5e579757b3c6c0ecb8cb2b9afe50d3a5de229cbed36132ff9cc93efa6 +"regex@npm:^4.3.2": + version: 4.4.0 + resolution: "regex@npm:4.4.0" + checksum: 10/0a32dcf2913287f5225a48aff11d26423711734307135c4dde489e71be35939b0d1fc253ddb9d81549e27451a4346a6401e87c10b1f4567fb928ad67279fbe31 languageName: node linkType: hard @@ -29976,16 +29973,16 @@ __metadata: linkType: hard "shiki@npm:^1.12.0, shiki@npm:^1.14.1": - version: 1.24.0 - resolution: "shiki@npm:1.24.0" + version: 1.22.2 + resolution: "shiki@npm:1.22.2" dependencies: - "@shikijs/core": "npm:1.24.0" - "@shikijs/engine-javascript": "npm:1.24.0" - "@shikijs/engine-oniguruma": "npm:1.24.0" - "@shikijs/types": "npm:1.24.0" + "@shikijs/core": "npm:1.22.2" + "@shikijs/engine-javascript": "npm:1.22.2" + "@shikijs/engine-oniguruma": "npm:1.22.2" + "@shikijs/types": "npm:1.22.2" "@shikijs/vscode-textmate": "npm:^9.3.0" "@types/hast": "npm:^3.0.4" - checksum: 10/1e4f119fcd365a3ebfe50dec3d9c5f40672f4530b1b42587e03edbffb77395dcf8e4e70dccf1fd9b68eeff5769aa7d1cfdb7ea139e8312c1f58507c5d222a2c5 + checksum: 10/5da1925609662cc773a807c3e9223805dc3323eaf2081aaf6633f3c20846485cab1429601f0a8a4a4aab83c80382b5b03ea9c94edffd3db968ab68de2484315c languageName: node linkType: hard