Skip to content

Commit

Permalink
feat: configure separate remote object storage
Browse files Browse the repository at this point in the history
  • Loading branch information
@caipira113
caipira113 committed Oct 21, 2024
1 parent eb7e96e commit 1c6801c
Show file tree
Hide file tree
Showing 12 changed files with 595 additions and 118 deletions.
38 changes: 38 additions & 0 deletions packages/backend/migration/1729518620697-remoteObjectStorage.js
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
/*
* SPDX-FileCopyrightText: syuilo and misskey-project
* SPDX-License-Identifier: AGPL-3.0-only
*/

export class RemoteObjectStorage1729518620697 {
async up(queryRunner) {
await queryRunner.query(`ALTER TABLE "meta" ADD "useRemoteObjectStorage" boolean NOT NULL DEFAULT false`);
await queryRunner.query(`ALTER TABLE "meta" ADD "remoteObjectStorageBucket" character varying(1024)`);
await queryRunner.query(`ALTER TABLE "meta" ADD "remoteObjectStoragePrefix" character varying(1024)`);
await queryRunner.query(`ALTER TABLE "meta" ADD "remoteObjectStorageBaseUrl" character varying(1024)`);
await queryRunner.query(`ALTER TABLE "meta" ADD "remoteObjectStorageEndpoint" character varying(1024)`);
await queryRunner.query(`ALTER TABLE "meta" ADD "remoteObjectStorageRegion" character varying(1024)`);
await queryRunner.query(`ALTER TABLE "meta" ADD "remoteObjectStorageAccessKey" character varying(1024)`);
await queryRunner.query(`ALTER TABLE "meta" ADD "remoteObjectStorageSecretKey" character varying(1024)`);
await queryRunner.query(`ALTER TABLE "meta" ADD "remoteObjectStoragePort" integer`);
await queryRunner.query(`ALTER TABLE "meta" ADD "remoteObjectStorageUseSSL" boolean DEFAULT true`);
await queryRunner.query(`ALTER TABLE "meta" ADD "remoteObjectStorageUseProxy" boolean DEFAULT true`);
await queryRunner.query(`ALTER TABLE "meta" ADD "remoteObjectStorageSetPublicRead" boolean NOT NULL DEFAULT false`);
await queryRunner.query(`ALTER TABLE "meta" ADD "remoteObjectStorageS3ForcePathStyle" boolean NOT NULL DEFAULT true`);
}

async down(queryRunner) {
await queryRunner.query(`ALTER TABLE "meta" DROP COLUMN "remoteObjectStorageS3ForcePathStyle";`);
await queryRunner.query(`ALTER TABLE "meta" DROP COLUMN "remoteObjectStorageSetPublicRead";`);
await queryRunner.query(`ALTER TABLE "meta" DROP COLUMN "remoteObjectStorageUseProxy";`);
await queryRunner.query(`ALTER TABLE "meta" DROP COLUMN "remoteObjectStorageUseSSL";`);
await queryRunner.query(`ALTER TABLE "meta" DROP COLUMN "remoteObjectStoragePort";`);
await queryRunner.query(`ALTER TABLE "meta" DROP COLUMN "remoteObjectStorageSecretKey";`);
await queryRunner.query(`ALTER TABLE "meta" DROP COLUMN "remoteObjectStorageAccessKey";`);
await queryRunner.query(`ALTER TABLE "meta" DROP COLUMN "remoteObjectStorageRegion";`);
await queryRunner.query(`ALTER TABLE "meta" DROP COLUMN "remoteObjectStorageEndpoint";`);
await queryRunner.query(`ALTER TABLE "meta" DROP COLUMN "remoteObjectStorageBaseUrl";`);
await queryRunner.query(`ALTER TABLE "meta" DROP COLUMN "remoteObjectStoragePrefix";`);
await queryRunner.query(`ALTER TABLE "meta" DROP COLUMN "remoteObjectStorageBucket";`);
await queryRunner.query(`ALTER TABLE "meta" DROP COLUMN "useRemoteObjectStorage";`);
}
}
81 changes: 62 additions & 19 deletions packages/backend/src/core/DriveService.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -144,9 +144,10 @@ export class DriveService {
* @param type Content-Type for original
* @param hash Hash for original
* @param size Size for original
* @param isRemote If true, file is remote file
*/
@bindThis
private async save(file: MiDriveFile, path: string, name: string, type: string, hash: string, size: number): Promise<MiDriveFile> {
private async save(file: MiDriveFile, path: string, name: string, type: string, hash: string, size: number, isRemote = false): Promise<MiDriveFile> {
// thunbnail, webpublic を必要なら生成
const alts = await this.generateAlts(path, type, !file.uri);

Expand All @@ -169,11 +170,37 @@ export class DriveService {
ext = '';
}

const baseUrl = this.meta.objectStorageBaseUrl
?? `${ this.meta.objectStorageUseSSL ? 'https' : 'http' }://${ this.meta.objectStorageEndpoint }${ this.meta.objectStoragePort ? `:${this.meta.objectStoragePort}` : '' }/${ this.meta.objectStorageBucket }`;
const useRemoteObjectStorage = isRemote && this.meta.useRemoteObjectStorage;

const objectStorageBaseUrl = useRemoteObjectStorage
? this.meta.remoteObjectStorageBaseUrl
: this.meta.objectStorageBaseUrl;

const objectStorageUseSSL = useRemoteObjectStorage
? this.meta.remoteObjectStorageUseSSL
: this.meta.objectStorageUseSSL;

const objectStorageEndpoint = useRemoteObjectStorage
? this.meta.remoteObjectStorageEndpoint
: this.meta.objectStorageEndpoint;

const objectStoragePort = useRemoteObjectStorage
? this.meta.remoteObjectStoragePort
: this.meta.objectStoragePort;

const objectStorageBucket = useRemoteObjectStorage
? this.meta.remoteObjectStorageBucket
: this.meta.objectStorageBucket;

const objectStoragePrefix = useRemoteObjectStorage
? this.meta.remoteObjectStoragePrefix
: this.meta.objectStoragePrefix;

const baseUrl = objectStorageBaseUrl
?? `${objectStorageUseSSL ? 'https' : 'http'}://${objectStorageEndpoint}${objectStoragePort ? `:${objectStoragePort}` : ''}/${objectStorageBucket}`;

// for original
const key = `${this.meta.objectStoragePrefix}/${randomUUID()}${ext}`;
const key = `${objectStoragePrefix}/${randomUUID()}${ext}`;
const url = `${ baseUrl }/${ key }`;

// for alts
Expand All @@ -186,23 +213,23 @@ export class DriveService {
//#region Uploads
this.registerLogger.info(`uploading original: ${key}`);
const uploads = [
this.upload(key, fs.createReadStream(path), type, null, name),
this.upload(key, fs.createReadStream(path), type, isRemote, null, name),
];

if (alts.webpublic) {
webpublicKey = `${this.meta.objectStoragePrefix}/webpublic-${randomUUID()}.${alts.webpublic.ext}`;
webpublicUrl = `${ baseUrl }/${ webpublicKey }`;

this.registerLogger.info(`uploading webpublic: ${webpublicKey}`);
uploads.push(this.upload(webpublicKey, alts.webpublic.data, alts.webpublic.type, alts.webpublic.ext, name));
uploads.push(this.upload(webpublicKey, alts.webpublic.data, alts.webpublic.type, isRemote, alts.webpublic.ext, name));
}

if (alts.thumbnail) {
thumbnailKey = `${this.meta.objectStoragePrefix}/thumbnail-${randomUUID()}.${alts.thumbnail.ext}`;
thumbnailUrl = `${ baseUrl }/${ thumbnailKey }`;

this.registerLogger.info(`uploading thumbnail: ${thumbnailKey}`);
uploads.push(this.upload(thumbnailKey, alts.thumbnail.data, alts.thumbnail.type, alts.thumbnail.ext, `${name}.thumbnail`));
uploads.push(this.upload(thumbnailKey, alts.thumbnail.data, alts.thumbnail.type, isRemote, alts.thumbnail.ext, `${name}.thumbnail`));
}

await Promise.all(uploads);
Expand Down Expand Up @@ -371,12 +398,22 @@ export class DriveService {
* Upload to ObjectStorage
*/
@bindThis
private async upload(key: string, stream: fs.ReadStream | Buffer, type: string, ext?: string | null, filename?: string) {
private async upload(key: string, stream: fs.ReadStream | Buffer, type: string, isRemote = false, ext?: string | null, filename?: string) {
if (type === 'image/apng') type = 'image/png';
if (!FILE_TYPE_BROWSERSAFE.includes(type)) type = 'application/octet-stream';

const useRemoteObjectStorage = isRemote && this.meta.useRemoteObjectStorage;

const objectStorageBucket = useRemoteObjectStorage
? this.meta.remoteObjectStorageBucket
: this.meta.objectStorageBucket;

const objectStorageSetPublicRead = useRemoteObjectStorage
? this.meta.remoteObjectStorageSetPublicRead
: this.meta.objectStorageSetPublicRead;

const params = {
Bucket: this.meta.objectStorageBucket,
Bucket: objectStorageBucket,
Key: key,
Body: stream,
ContentType: type,
Expand All @@ -389,9 +426,9 @@ export class DriveService {
// 許可されているファイル形式でしか拡張子をつけない
ext ? correctFilename(filename, ext) : filename,
);
if (this.meta.objectStorageSetPublicRead) params.ACL = 'public-read';
if (objectStorageSetPublicRead) params.ACL = 'public-read';

await this.s3Service.upload(this.meta, params)
await this.s3Service.upload(this.meta, params, isRemote)
.then(
result => {
if ('Bucket' in result) { // CompleteMultipartUploadCommandOutput
Expand Down Expand Up @@ -630,7 +667,8 @@ export class DriveService {
}
}
} else {
file = await (this.save(file, path, detectedName, info.type.mime, info.md5, info.size));
const isRemote = user ? this.userEntityService.isRemoteUser(user) : false;
file = await (this.save(file, path, detectedName, info.type.mime, info.md5, info.size, isRemote));
}

this.registerLogger.succ(`drive file has been created ${file.id}`);
Expand Down Expand Up @@ -740,7 +778,7 @@ export class DriveService {
}

@bindThis
public async deleteFileSync(file: MiDriveFile, isExpired = false, deleter?: MiUser) {
public async deleteFileSync(file: MiDriveFile, isExpired = false, isRemote = false, deleter?: MiUser) {
if (file.storedInternal) {
this.internalStorageService.del(file.accessKey!);

Expand All @@ -754,14 +792,14 @@ export class DriveService {
} else if (!file.isLink) {
const promises = [];

promises.push(this.deleteObjectStorageFile(file.accessKey!));
promises.push(this.deleteObjectStorageFile(file.accessKey!, isRemote));

if (file.thumbnailUrl) {
promises.push(this.deleteObjectStorageFile(file.thumbnailAccessKey!));
promises.push(this.deleteObjectStorageFile(file.thumbnailAccessKey!, isRemote));
}

if (file.webpublicUrl) {
promises.push(this.deleteObjectStorageFile(file.webpublicAccessKey!));
promises.push(this.deleteObjectStorageFile(file.webpublicAccessKey!, isRemote));
}

await Promise.all(promises);
Expand Down Expand Up @@ -815,14 +853,19 @@ export class DriveService {
}

@bindThis
public async deleteObjectStorageFile(key: string) {
public async deleteObjectStorageFile(key: string, isRemote = false) {
const useRemoteObjectStorage = isRemote && this.meta.useRemoteObjectStorage;
const objectStorageBucket = useRemoteObjectStorage
? this.meta.remoteObjectStorageBucket
: this.meta.objectStorageBucket;

try {
const param = {
Bucket: this.meta.objectStorageBucket,
Bucket: objectStorageBucket,
Key: key,
} as DeleteObjectCommandInput;

await this.s3Service.delete(this.meta, param);
await this.s3Service.delete(this.meta, param, isRemote);
} catch (err: any) {
if (err.name === 'NoSuchKey') {
this.deleteLogger.warn(`The object storage had no such key to delete: ${key}. Skipping this.`, err as Error);
Expand Down
63 changes: 44 additions & 19 deletions packages/backend/src/core/S3Service.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,39 +19,64 @@ import type { DeleteObjectCommandInput, PutObjectCommandInput } from '@aws-sdk/c
export class S3Service {
constructor(
private httpRequestService: HttpRequestService,
) {
}
) {}

@bindThis
public getS3Client(meta: MiMeta): S3Client {
const u = meta.objectStorageEndpoint
? `${meta.objectStorageUseSSL ? 'https' : 'http'}://${meta.objectStorageEndpoint}`
: `${meta.objectStorageUseSSL ? 'https' : 'http'}://example.net`; // dummy url to select http(s) agent
public getS3Client(meta: MiMeta, isRemote = false): S3Client {
const useRemoteObjectStorage = isRemote && meta.useRemoteObjectStorage;

const objectStorageEndpoint = useRemoteObjectStorage
? meta.remoteObjectStorageEndpoint
: meta.objectStorageEndpoint;

const objectStorageUseSSL = useRemoteObjectStorage
? meta.remoteObjectStorageUseSSL
: meta.objectStorageUseSSL;

const objectStorageAccessKey = useRemoteObjectStorage
? meta.remoteObjectStorageAccessKey
: meta.objectStorageAccessKey;

const objectStorageSecretKey = useRemoteObjectStorage
? meta.remoteObjectStorageSecretKey
: meta.objectStorageSecretKey;

const objectStorageRegion = useRemoteObjectStorage
? meta.remoteObjectStorageRegion
: meta.objectStorageRegion;

const objectStorageS3ForcePathStyle = useRemoteObjectStorage
? meta.remoteObjectStorageS3ForcePathStyle
: meta.objectStorageS3ForcePathStyle;

const u = objectStorageEndpoint
? `${objectStorageUseSSL ? 'https' : 'http'}://${objectStorageEndpoint}`
: `${objectStorageUseSSL ? 'https' : 'http'}://example.net`; // dummy url to select http(s) agent

const agent = this.httpRequestService.getAgentByUrl(new URL(u), !meta.objectStorageUseProxy);
const agent = this.httpRequestService.getAgentByUrl(new URL(u), !objectStorageUseSSL);
const handlerOption: NodeHttpHandlerOptions = {};
if (meta.objectStorageUseSSL) {
if (objectStorageUseSSL) {
handlerOption.httpsAgent = agent as https.Agent;
} else {
handlerOption.httpAgent = agent as http.Agent;
}

return new S3Client({
endpoint: meta.objectStorageEndpoint ? u : undefined,
credentials: (meta.objectStorageAccessKey !== null && meta.objectStorageSecretKey !== null) ? {
accessKeyId: meta.objectStorageAccessKey,
secretAccessKey: meta.objectStorageSecretKey,
endpoint: objectStorageEndpoint ? u : undefined,
credentials: (objectStorageAccessKey !== null && objectStorageSecretKey !== null) ? {
accessKeyId: objectStorageAccessKey,
secretAccessKey: objectStorageSecretKey,
} : undefined,
region: meta.objectStorageRegion ? meta.objectStorageRegion : undefined, // 空文字列もundefinedにするため ?? は使わない
tls: meta.objectStorageUseSSL,
forcePathStyle: meta.objectStorageEndpoint ? meta.objectStorageS3ForcePathStyle : false, // AWS with endPoint omitted
region: objectStorageRegion || undefined, // 空文字列もundefinedにするため ?? は使わない
tls: objectStorageUseSSL,
forcePathStyle: objectStorageEndpoint ? objectStorageS3ForcePathStyle : false, // AWS with endPoint omitted
requestHandler: new NodeHttpHandler(handlerOption),
});
}

@bindThis
public async upload(meta: MiMeta, input: PutObjectCommandInput) {
const client = this.getS3Client(meta);
public async upload(meta: MiMeta, input: PutObjectCommandInput, isRemote = false) {
const client = this.getS3Client(meta, isRemote);
return new Upload({
client,
params: input,
Expand All @@ -62,8 +87,8 @@ export class S3Service {
}

@bindThis
public delete(meta: MiMeta, input: DeleteObjectCommandInput) {
const client = this.getS3Client(meta);
public delete(meta: MiMeta, input: DeleteObjectCommandInput, isRemote = false) {
const client = this.getS3Client(meta, isRemote);
return client.send(new DeleteObjectCommand(input));
}
}
72 changes: 72 additions & 0 deletions packages/backend/src/models/Meta.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -481,6 +481,78 @@ export class MiMeta {
})
public objectStorageS3ForcePathStyle: boolean;

@Column('boolean', {
default: false,
})
public useRemoteObjectStorage: boolean;

@Column('varchar', {
length: 1024,
nullable: true,
})
public remoteObjectStorageBucket: string | null;

@Column('varchar', {
length: 1024,
nullable: true,
})
public remoteObjectStoragePrefix: string | null;

@Column('varchar', {
length: 1024,
nullable: true,
})
public remoteObjectStorageBaseUrl: string | null;

@Column('varchar', {
length: 1024,
nullable: true,
})
public remoteObjectStorageEndpoint: string | null;

@Column('varchar', {
length: 1024,
nullable: true,
})
public remoteObjectStorageRegion: string | null;

@Column('varchar', {
length: 1024,
nullable: true,
})
public remoteObjectStorageAccessKey: string | null;

@Column('varchar', {
length: 1024,
nullable: true,
})
public remoteObjectStorageSecretKey: string | null;

@Column('integer', {
nullable: true,
})
public remoteObjectStoragePort: number | null;

@Column('boolean', {
default: true,
})
public remoteObjectStorageUseSSL: boolean;

@Column('boolean', {
default: true,
})
public remoteObjectStorageUseProxy: boolean;

@Column('boolean', {
default: false,
})
public remoteObjectStorageSetPublicRead: boolean;

@Column('boolean', {
default: true,
})
public remoteObjectStorageS3ForcePathStyle: boolean;

@Column('boolean', {
default: false,
})
Expand Down
2 changes: 1 addition & 1 deletion packages/backend/src/queue/processors/CleanRemoteFilesProcessorService.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,7 @@ export class CleanRemoteFilesProcessorService {

cursor = files.at(-1)?.id ?? null;

await Promise.all(files.map(file => this.driveService.deleteFileSync(file, true)));
await Promise.all(files.map(file => this.driveService.deleteFileSync(file, true, true)));

deletedCount += 8;

Expand Down
Loading

1 comment on commit 1c6801c

@github-actions
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Chromatic detects changes. Please review the changes on Chromatic.

Please sign in to comment.