Skip to content

Commit

Permalink
feat: configure separate remote object storage
Browse files Browse the repository at this point in the history
  • Loading branch information
@caipira113
caipira113 committed Oct 21, 2024
1 parent eb7e96e commit 4e26335
Show file tree
Hide file tree
Showing 12 changed files with 594 additions and 118 deletions.
38 changes: 38 additions & 0 deletions packages/backend/migration/1729518620697-remoteObjectStorage.js
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
/*
* SPDX-FileCopyrightText: syuilo and misskey-project
* SPDX-License-Identifier: AGPL-3.0-only
*/

export class RemoteObjectStorage1729518620697 {
async up(queryRunner) {
await queryRunner.query(`ALTER TABLE "meta" ADD COLUMN useRemoteObjectStorage BOOLEAN DEFAULT false;`);
await queryRunner.query(`ALTER TABLE "meta" ADD COLUMN remoteObjectStorageBucket VARCHAR(1024) NOT NULL;`);
await queryRunner.query(`ALTER TABLE "meta" ADD COLUMN remoteObjectStoragePrefix VARCHAR(1024) NOT NULL;`);
await queryRunner.query(`ALTER TABLE "meta" ADD COLUMN remoteObjectStorageBaseUrl VARCHAR(1024);`);
await queryRunner.query(`ALTER TABLE "meta" ADD COLUMN remoteObjectStorageEndpoint VARCHAR(1024);`);
await queryRunner.query(`ALTER TABLE "meta" ADD COLUMN remoteObjectStorageRegion VARCHAR(1024);`);
await queryRunner.query(`ALTER TABLE "meta" ADD COLUMN remoteObjectStorageAccessKey VARCHAR(1024);`);
await queryRunner.query(`ALTER TABLE "meta" ADD COLUMN remoteObjectStorageSecretKey VARCHAR(1024);`);
await queryRunner.query(`ALTER TABLE "meta" ADD COLUMN remoteObjectStoragePort INTEGER;`);
await queryRunner.query(`ALTER TABLE "meta" ADD COLUMN remoteObjectStorageUseSSL BOOLEAN DEFAULT true;`);
await queryRunner.query(`ALTER TABLE "meta" ADD COLUMN remoteObjectStorageUseProxy BOOLEAN DEFAULT true;`);
await queryRunner.query(`ALTER TABLE "meta" ADD COLUMN remoteObjectStorageSetPublicRead BOOLEAN DEFAULT false;`);
await queryRunner.query(`ALTER TABLE "meta" ADD COLUMN remoteObjectStorageS3ForcePathStyle BOOLEAN DEFAULT true;`);
}

async down(queryRunner) {
await queryRunner.query(`ALTER TABLE "meta" DROP COLUMN remoteObjectStorageS3ForcePathStyle;`);
await queryRunner.query(`ALTER TABLE "meta" DROP COLUMN remoteObjectStorageSetPublicRead;`);
await queryRunner.query(`ALTER TABLE "meta" DROP COLUMN remoteObjectStorageUseProxy;`);
await queryRunner.query(`ALTER TABLE "meta" DROP COLUMN remoteObjectStorageUseSSL;`);
await queryRunner.query(`ALTER TABLE "meta" DROP COLUMN remoteObjectStoragePort;`);
await queryRunner.query(`ALTER TABLE "meta" DROP COLUMN remoteObjectStorageSecretKey;`);
await queryRunner.query(`ALTER TABLE "meta" DROP COLUMN remoteObjectStorageAccessKey;`);
await queryRunner.query(`ALTER TABLE "meta" DROP COLUMN remoteObjectStorageRegion;`);
await queryRunner.query(`ALTER TABLE "meta" DROP COLUMN remoteObjectStorageEndpoint;`);
await queryRunner.query(`ALTER TABLE "meta" DROP COLUMN remoteObjectStorageBaseUrl;`);
await queryRunner.query(`ALTER TABLE "meta" DROP COLUMN remoteObjectStoragePrefix;`);
await queryRunner.query(`ALTER TABLE "meta" DROP COLUMN remoteObjectStorageBucket;`);
await queryRunner.query(`ALTER TABLE "meta" DROP COLUMN useRemoteObjectStorage;`);
}
}
80 changes: 61 additions & 19 deletions packages/backend/src/core/DriveService.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -146,7 +146,7 @@ export class DriveService {
* @param size Size for original
*/
@bindThis
private async save(file: MiDriveFile, path: string, name: string, type: string, hash: string, size: number): Promise<MiDriveFile> {
private async save(file: MiDriveFile, path: string, name: string, type: string, hash: string, size: number, isRemote = false): Promise<MiDriveFile> {
// thunbnail, webpublic を必要なら生成
const alts = await this.generateAlts(path, type, !file.uri);

Expand All @@ -169,11 +169,37 @@ export class DriveService {
ext = '';
}

const baseUrl = this.meta.objectStorageBaseUrl
?? `${ this.meta.objectStorageUseSSL ? 'https' : 'http' }://${ this.meta.objectStorageEndpoint }${ this.meta.objectStoragePort ? `:${this.meta.objectStoragePort}` : '' }/${ this.meta.objectStorageBucket }`;
const useRemoteObjectStorage = isRemote && this.meta.useRemoteObjectStorage;

const objectStorageBaseUrl = useRemoteObjectStorage
? this.meta.remoteObjectStorageBaseUrl
: this.meta.objectStorageBaseUrl;

const objectStorageUseSSL = useRemoteObjectStorage
? this.meta.remoteObjectStorageUseSSL
: this.meta.objectStorageUseSSL;

const objectStorageEndpoint = useRemoteObjectStorage
? this.meta.remoteObjectStorageEndpoint
: this.meta.objectStorageEndpoint;

const objectStoragePort = useRemoteObjectStorage
? this.meta.remoteObjectStoragePort
: this.meta.objectStoragePort;

const objectStorageBucket = useRemoteObjectStorage
? this.meta.remoteObjectStorageBucket
: this.meta.objectStorageBucket;

const objectStoragePrefix = useRemoteObjectStorage
? this.meta.remoteObjectStoragePrefix
: this.meta.objectStoragePrefix;

const baseUrl = objectStorageBaseUrl
?? `${objectStorageUseSSL ? 'https' : 'http'}://${objectStorageEndpoint}${objectStoragePort ? `:${objectStoragePort}` : ''}/${objectStorageBucket}`;

// for original
const key = `${this.meta.objectStoragePrefix}/${randomUUID()}${ext}`;
const key = `${objectStoragePrefix}/${randomUUID()}${ext}`;
const url = `${ baseUrl }/${ key }`;

// for alts
Expand All @@ -186,23 +212,23 @@ export class DriveService {
//#region Uploads
this.registerLogger.info(`uploading original: ${key}`);
const uploads = [
this.upload(key, fs.createReadStream(path), type, null, name),
this.upload(key, fs.createReadStream(path), type, isRemote, null, name),
];

if (alts.webpublic) {
webpublicKey = `${this.meta.objectStoragePrefix}/webpublic-${randomUUID()}.${alts.webpublic.ext}`;
webpublicUrl = `${ baseUrl }/${ webpublicKey }`;

this.registerLogger.info(`uploading webpublic: ${webpublicKey}`);
uploads.push(this.upload(webpublicKey, alts.webpublic.data, alts.webpublic.type, alts.webpublic.ext, name));
uploads.push(this.upload(webpublicKey, alts.webpublic.data, alts.webpublic.type, isRemote, alts.webpublic.ext, name));
}

if (alts.thumbnail) {
thumbnailKey = `${this.meta.objectStoragePrefix}/thumbnail-${randomUUID()}.${alts.thumbnail.ext}`;
thumbnailUrl = `${ baseUrl }/${ thumbnailKey }`;

this.registerLogger.info(`uploading thumbnail: ${thumbnailKey}`);
uploads.push(this.upload(thumbnailKey, alts.thumbnail.data, alts.thumbnail.type, alts.thumbnail.ext, `${name}.thumbnail`));
uploads.push(this.upload(thumbnailKey, alts.thumbnail.data, alts.thumbnail.type, isRemote, alts.thumbnail.ext, `${name}.thumbnail`));
}

await Promise.all(uploads);
Expand Down Expand Up @@ -371,12 +397,22 @@ export class DriveService {
* Upload to ObjectStorage
*/
@bindThis
private async upload(key: string, stream: fs.ReadStream | Buffer, type: string, ext?: string | null, filename?: string) {
private async upload(key: string, stream: fs.ReadStream | Buffer, type: string, isRemote = false, ext?: string | null, filename?: string) {
if (type === 'image/apng') type = 'image/png';
if (!FILE_TYPE_BROWSERSAFE.includes(type)) type = 'application/octet-stream';

const useRemoteObjectStorage = isRemote && this.meta.useRemoteObjectStorage;

const objectStorageBucket = useRemoteObjectStorage
? this.meta.remoteObjectStorageBucket
: this.meta.objectStorageBucket;

const objectStorageSetPublicRead = useRemoteObjectStorage
? this.meta.remoteObjectStorageSetPublicRead
: this.meta.objectStorageSetPublicRead;

const params = {
Bucket: this.meta.objectStorageBucket,
Bucket: objectStorageBucket,
Key: key,
Body: stream,
ContentType: type,
Expand All @@ -389,9 +425,9 @@ export class DriveService {
// 許可されているファイル形式でしか拡張子をつけない
ext ? correctFilename(filename, ext) : filename,
);
if (this.meta.objectStorageSetPublicRead) params.ACL = 'public-read';
if (objectStorageSetPublicRead) params.ACL = 'public-read';

await this.s3Service.upload(this.meta, params)
await this.s3Service.upload(this.meta, params, isRemote)
.then(
result => {
if ('Bucket' in result) { // CompleteMultipartUploadCommandOutput
Expand Down Expand Up @@ -630,7 +666,8 @@ export class DriveService {
}
}
} else {
file = await (this.save(file, path, detectedName, info.type.mime, info.md5, info.size));
const isRemote = user ? this.userEntityService.isRemoteUser(user) : false;
file = await (this.save(file, path, detectedName, info.type.mime, info.md5, info.size, isRemote));
}

this.registerLogger.succ(`drive file has been created ${file.id}`);
Expand Down Expand Up @@ -740,7 +777,7 @@ export class DriveService {
}

@bindThis
public async deleteFileSync(file: MiDriveFile, isExpired = false, deleter?: MiUser) {
public async deleteFileSync(file: MiDriveFile, isExpired = false, isRemote = false, deleter?: MiUser) {
if (file.storedInternal) {
this.internalStorageService.del(file.accessKey!);

Expand All @@ -754,14 +791,14 @@ export class DriveService {
} else if (!file.isLink) {
const promises = [];

promises.push(this.deleteObjectStorageFile(file.accessKey!));
promises.push(this.deleteObjectStorageFile(file.accessKey!, isRemote));

if (file.thumbnailUrl) {
promises.push(this.deleteObjectStorageFile(file.thumbnailAccessKey!));
promises.push(this.deleteObjectStorageFile(file.thumbnailAccessKey!, isRemote));
}

if (file.webpublicUrl) {
promises.push(this.deleteObjectStorageFile(file.webpublicAccessKey!));
promises.push(this.deleteObjectStorageFile(file.webpublicAccessKey!, isRemote));
}

await Promise.all(promises);
Expand Down Expand Up @@ -815,14 +852,19 @@ export class DriveService {
}

@bindThis
public async deleteObjectStorageFile(key: string) {
public async deleteObjectStorageFile(key: string, isRemote = false) {
const useRemoteObjectStorage = isRemote && this.meta.useRemoteObjectStorage;
const objectStorageBucket = useRemoteObjectStorage
? this.meta.remoteObjectStorageBucket
: this.meta.objectStorageBucket;

try {
const param = {
Bucket: this.meta.objectStorageBucket,
Bucket: objectStorageBucket,
Key: key,
} as DeleteObjectCommandInput;

await this.s3Service.delete(this.meta, param);
await this.s3Service.delete(this.meta, param, isRemote);
} catch (err: any) {
if (err.name === 'NoSuchKey') {
this.deleteLogger.warn(`The object storage had no such key to delete: ${key}. Skipping this.`, err as Error);
Expand Down
63 changes: 44 additions & 19 deletions packages/backend/src/core/S3Service.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,39 +19,64 @@ import type { DeleteObjectCommandInput, PutObjectCommandInput } from '@aws-sdk/c
export class S3Service {
constructor(
private httpRequestService: HttpRequestService,
) {
}
) {}

@bindThis
public getS3Client(meta: MiMeta): S3Client {
const u = meta.objectStorageEndpoint
? `${meta.objectStorageUseSSL ? 'https' : 'http'}://${meta.objectStorageEndpoint}`
: `${meta.objectStorageUseSSL ? 'https' : 'http'}://example.net`; // dummy url to select http(s) agent
public getS3Client(meta: MiMeta, isRemote = false): S3Client {
const useRemoteObjectStorage = isRemote && meta.useRemoteObjectStorage;

const objectStorageEndpoint = useRemoteObjectStorage
? meta.remoteObjectStorageEndpoint
: meta.objectStorageEndpoint;

const objectStorageUseSSL = useRemoteObjectStorage
? meta.remoteObjectStorageUseSSL
: meta.objectStorageUseSSL;

const objectStorageAccessKey = useRemoteObjectStorage
? meta.remoteObjectStorageAccessKey
: meta.objectStorageAccessKey;

const objectStorageSecretKey = useRemoteObjectStorage
? meta.remoteObjectStorageSecretKey
: meta.objectStorageSecretKey;

const objectStorageRegion = useRemoteObjectStorage
? meta.remoteObjectStorageRegion
: meta.objectStorageRegion;

const objectStorageS3ForcePathStyle = useRemoteObjectStorage
? meta.remoteObjectStorageS3ForcePathStyle
: meta.objectStorageS3ForcePathStyle;

const u = objectStorageEndpoint
? `${objectStorageUseSSL ? 'https' : 'http'}://${objectStorageEndpoint}`
: `${objectStorageUseSSL ? 'https' : 'http'}://example.net`; // dummy url to select http(s) agent

const agent = this.httpRequestService.getAgentByUrl(new URL(u), !meta.objectStorageUseProxy);
const agent = this.httpRequestService.getAgentByUrl(new URL(u), !objectStorageUseSSL);
const handlerOption: NodeHttpHandlerOptions = {};
if (meta.objectStorageUseSSL) {
if (objectStorageUseSSL) {
handlerOption.httpsAgent = agent as https.Agent;
} else {
handlerOption.httpAgent = agent as http.Agent;
}

return new S3Client({
endpoint: meta.objectStorageEndpoint ? u : undefined,
credentials: (meta.objectStorageAccessKey !== null && meta.objectStorageSecretKey !== null) ? {
accessKeyId: meta.objectStorageAccessKey,
secretAccessKey: meta.objectStorageSecretKey,
endpoint: objectStorageEndpoint ? u : undefined,
credentials: (objectStorageAccessKey !== null && objectStorageSecretKey !== null) ? {
accessKeyId: objectStorageAccessKey,
secretAccessKey: objectStorageSecretKey,
} : undefined,
region: meta.objectStorageRegion ? meta.objectStorageRegion : undefined, // 空文字列もundefinedにするため ?? は使わない
tls: meta.objectStorageUseSSL,
forcePathStyle: meta.objectStorageEndpoint ? meta.objectStorageS3ForcePathStyle : false, // AWS with endPoint omitted
region: objectStorageRegion || undefined, // 空文字列もundefinedにするため ?? は使わない
tls: objectStorageUseSSL,
forcePathStyle: objectStorageEndpoint ? objectStorageS3ForcePathStyle : false, // AWS with endPoint omitted
requestHandler: new NodeHttpHandler(handlerOption),
});
}

@bindThis
public async upload(meta: MiMeta, input: PutObjectCommandInput) {
const client = this.getS3Client(meta);
public async upload(meta: MiMeta, input: PutObjectCommandInput, isRemote = false) {
const client = this.getS3Client(meta, isRemote);
return new Upload({
client,
params: input,
Expand All @@ -62,8 +87,8 @@ export class S3Service {
}

@bindThis
public delete(meta: MiMeta, input: DeleteObjectCommandInput) {
const client = this.getS3Client(meta);
public delete(meta: MiMeta, input: DeleteObjectCommandInput, isRemote = false) {
const client = this.getS3Client(meta, isRemote);
return client.send(new DeleteObjectCommand(input));
}
}
72 changes: 72 additions & 0 deletions packages/backend/src/models/Meta.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -481,6 +481,78 @@ export class MiMeta {
})
public objectStorageS3ForcePathStyle: boolean;

@Column('boolean', {
default: false,
})
public useRemoteObjectStorage: boolean;

@Column('varchar', {
length: 1024,
nullable: true,
})
public remoteObjectStorageBucket: string | null;

@Column('varchar', {
length: 1024,
nullable: true,
})
public remoteObjectStoragePrefix: string | null;

@Column('varchar', {
length: 1024,
nullable: true,
})
public remoteObjectStorageBaseUrl: string | null;

@Column('varchar', {
length: 1024,
nullable: true,
})
public remoteObjectStorageEndpoint: string | null;

@Column('varchar', {
length: 1024,
nullable: true,
})
public remoteObjectStorageRegion: string | null;

@Column('varchar', {
length: 1024,
nullable: true,
})
public remoteObjectStorageAccessKey: string | null;

@Column('varchar', {
length: 1024,
nullable: true,
})
public remoteObjectStorageSecretKey: string | null;

@Column('integer', {
nullable: true,
})
public remoteObjectStoragePort: number | null;

@Column('boolean', {
default: true,
})
public remoteObjectStorageUseSSL: boolean;

@Column('boolean', {
default: true,
})
public remoteObjectStorageUseProxy: boolean;

@Column('boolean', {
default: false,
})
public remoteObjectStorageSetPublicRead: boolean;

@Column('boolean', {
default: true,
})
public remoteObjectStorageS3ForcePathStyle: boolean;

@Column('boolean', {
default: false,
})
Expand Down
2 changes: 1 addition & 1 deletion packages/backend/src/queue/processors/CleanRemoteFilesProcessorService.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,7 @@ export class CleanRemoteFilesProcessorService {

cursor = files.at(-1)?.id ?? null;

await Promise.all(files.map(file => this.driveService.deleteFileSync(file, true)));
await Promise.all(files.map(file => this.driveService.deleteFileSync(file, true, true)));

deletedCount += 8;

Expand Down
Loading

1 comment on commit 4e26335

@github-actions
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Chromatic detects changes. Please review the changes on Chromatic.

Please sign in to comment.