Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat(nbstore): add cloud implementation #8810

Merged
merged 1 commit into from
Dec 10, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
-- CreateTable
CREATE TABLE "blobs" (
"workspace_id" VARCHAR NOT NULL,
"key" VARCHAR NOT NULL,
"size" INTEGER NOT NULL,
"mime" VARCHAR NOT NULL,
"created_at" TIMESTAMPTZ(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
"deleted_at" TIMESTAMPTZ(3),

CONSTRAINT "blobs_pkey" PRIMARY KEY ("workspace_id","key")
);

-- AddForeignKey
ALTER TABLE "blobs" ADD CONSTRAINT "blobs_workspace_id_fkey" FOREIGN KEY ("workspace_id") REFERENCES "workspaces"("id") ON DELETE CASCADE ON UPDATE CASCADE;
17 changes: 17 additions & 0 deletions packages/backend/server/schema.prisma
Original file line number Diff line number Diff line change
Expand Up @@ -106,6 +106,7 @@ model Workspace {
permissions WorkspaceUserPermission[]
pagePermissions WorkspacePageUserPermission[]
features WorkspaceFeature[]
blobs Blob[]

@@map("workspaces")
}
Expand Down Expand Up @@ -568,3 +569,19 @@ model Invoice {
@@index([targetId])
@@map("invoices")
}
// Blob table only exists for fast non-data queries.
// like, total size of blobs in a workspace, or blob list for sync service.
// it should only be a map of metadata of blobs stored anywhere else
model Blob {
workspaceId String @map("workspace_id") @db.VarChar
key String @db.VarChar
size Int @db.Integer
mime String @db.VarChar
createdAt DateTime @default(now()) @map("created_at") @db.Timestamptz(3)
deletedAt DateTime? @map("deleted_at") @db.Timestamptz(3)

workspace Workspace @relation(fields: [workspaceId], references: [id], onDelete: Cascade)

@@id([workspaceId, key])
@@map("blobs")
}
29 changes: 29 additions & 0 deletions packages/backend/server/src/core/doc/storage/doc.ts
Original file line number Diff line number Diff line change
@@ -1,8 +1,10 @@
import {
applyUpdate,
diffUpdate,
Doc,
encodeStateAsUpdate,
encodeStateVector,
encodeStateVectorFromUpdate,
mergeUpdates,
UndoManager,
} from 'yjs';
Expand All @@ -19,6 +21,12 @@ export interface DocRecord {
editor?: string;
}

export interface DocDiff {
missing: Uint8Array;
state: Uint8Array;
timestamp: number;
}

export interface DocUpdate {
bin: Uint8Array;
timestamp: number;
Expand Down Expand Up @@ -96,6 +104,27 @@ export abstract class DocStorageAdapter extends Connection {
return snapshot;
}

async getDocDiff(
spaceId: string,
docId: string,
stateVector?: Uint8Array
): Promise<DocDiff | null> {
const doc = await this.getDoc(spaceId, docId);

if (!doc) {
return null;
}

const missing = stateVector ? diffUpdate(doc.bin, stateVector) : doc.bin;
const state = encodeStateVectorFromUpdate(doc.bin);

return {
missing,
state,
timestamp: doc.timestamp,
};
}

abstract pushDocUpdates(
spaceId: string,
docId: string,
Expand Down
4 changes: 3 additions & 1 deletion packages/backend/server/src/core/doc/storage/index.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,6 @@
// TODO(@forehalo): share with frontend
// This is a totally copy of definitions in [@affine/space-store]
// because currently importing cross workspace package from [@affine/server] is not yet supported
// should be kept updated with the original definitions in [@affine/space-store]
import type { BlobStorageAdapter } from './blob';
forehalo marked this conversation as resolved.
Show resolved Hide resolved
import { Connection } from './connection';
import type { DocStorageAdapter } from './doc';
Expand Down
23 changes: 22 additions & 1 deletion packages/backend/server/src/core/quota/resolver.ts
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ import { CurrentUser } from '../auth/session';
import { EarlyAccessType } from '../features';
import { UserType } from '../user';
import { QuotaService } from './service';
import { QuotaManagementService } from './storage';

registerEnumType(EarlyAccessType, {
name: 'EarlyAccessType',
Expand Down Expand Up @@ -55,14 +56,34 @@ class UserQuotaType {
humanReadable!: UserQuotaHumanReadableType;
}

@ObjectType('UserQuotaUsage')
class UserQuotaUsageType {
@Field(() => SafeIntResolver, { name: 'storageQuota' })
storageQuota!: number;
}

@Resolver(() => UserType)
export class QuotaManagementResolver {
constructor(private readonly quota: QuotaService) {}
constructor(
private readonly quota: QuotaService,
private readonly management: QuotaManagementService
) {}

@ResolveField(() => UserQuotaType, { name: 'quota', nullable: true })
async getQuota(@CurrentUser() me: UserType) {
const quota = await this.quota.getUserQuota(me.id);

return quota.feature;
}

@ResolveField(() => UserQuotaUsageType, { name: 'quotaUsage' })
async getQuotaUsage(
@CurrentUser() me: UserType
): Promise<UserQuotaUsageType> {
const usage = await this.management.getUserStorageUsage(me.id);

return {
storageQuota: usage,
};
}
}
6 changes: 3 additions & 3 deletions packages/backend/server/src/core/quota/storage.ts
Original file line number Diff line number Diff line change
Expand Up @@ -77,7 +77,7 @@ export class QuotaManagementService {
return this.quota.hasWorkspaceQuota(workspaceId, QuotaType.TeamPlanV1);
}

async getUserUsage(userId: string) {
async getUserStorageUsage(userId: string) {
const workspaces = await this.permissions.getOwnedWorkspaces(userId);

const sizes = await Promise.allSettled(
Expand Down Expand Up @@ -125,7 +125,7 @@ export class QuotaManagementService {
async getQuotaCalculator(userId: string) {
const quota = await this.getUserQuota(userId);
const { storageQuota, businessBlobLimit } = quota;
const usedSize = await this.getUserUsage(userId);
const usedSize = await this.getUserStorageUsage(userId);

return this.generateQuotaCalculator(
storageQuota,
Expand Down Expand Up @@ -183,7 +183,7 @@ export class QuotaManagementService {
humanReadable,
} = await this.getWorkspaceQuota(owner.id, workspaceId);
// get all workspaces size of owner used
const usedSize = await this.getUserUsage(owner.id);
const usedSize = await this.getUserStorageUsage(owner.id);
// relax restrictions if workspace has unlimited feature
// todo(@darkskygit): need a mechanism to allow feature as a middleware to edit quota
const unlimited = await this.feature.hasWorkspaceFeature(
Expand Down
Loading
Loading