diff --git a/src/helpers/copy.js b/src/helpers/copy.js index 6b86ef5..72b30ce 100644 --- a/src/helpers/copy.js +++ b/src/helpers/copy.js @@ -13,9 +13,10 @@ export default async function copyHelper(req, daCtx) { const formData = await req.formData(); if (!formData) return {}; const fullDest = formData.get('destination'); + const continuationToken = formData.get('continuation-token'); const lower = fullDest.slice(1).toLowerCase(); const sanitized = lower.endsWith('/') ? lower.slice(0, -1) : lower; const destination = sanitized.split('/').slice(1).join('/'); const source = daCtx.key; - return { source, destination }; + return { source, destination, continuationToken }; } diff --git a/src/helpers/delete.js b/src/helpers/delete.js new file mode 100644 index 0000000..94e42a7 --- /dev/null +++ b/src/helpers/delete.js @@ -0,0 +1,22 @@ +/* + * Copyright 2024 Adobe. All rights reserved. + * This file is licensed to you under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under + * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS + * OF ANY KIND, either express or implied. See the License for the specific language + * governing permissions and limitations under the License. + */ + +export default async function deleteHelper(req) { + try { + const formData = await req.formData(); + if (!formData) return {}; + const continuationToken = formData.get('continuation-token'); + return { continuationToken }; + } catch { + return {}; + } +} diff --git a/src/routes/source.js b/src/routes/source.js index 0aefccc..0d04125 100644 --- a/src/routes/source.js +++ b/src/routes/source.js @@ -14,6 +14,7 @@ import putObject from '../storage/object/put.js'; import deleteObjects from '../storage/object/delete.js'; import putHelper from '../helpers/source.js'; +import deleteHelper from '../helpers/delete.js'; import { postObjectVersion } from '../storage/version/put.js'; async function invalidateCollab(api, url, env) { @@ -25,13 +26,18 @@ async function invalidateCollab(api, url, env) { } export async function deleteSource({ req, env, daCtx }) { + const details = await deleteHelper(req); await postObjectVersion(req, env, daCtx); - const resp = await deleteObjects(env, daCtx); - - if (resp.status === 204) { - const initiator = req.headers.get('x-da-initiator'); - if (initiator !== 'collab') { - await invalidateCollab('deleteadmin', req.url, env); + const resp = await deleteObjects(env, daCtx, details); + + if (resp.status === 204 || resp.status === 200) { + try { + const initiator = req.headers.get('x-da-initiator'); + if (initiator !== 'collab') { + await invalidateCollab('deleteadmin', req.url, env); + } + } catch { + // collab not available } } return resp; diff --git a/src/storage/object/copy.js b/src/storage/object/copy.js index 9196f3c..6858224 100644 --- a/src/storage/object/copy.js +++ b/src/storage/object/copy.js @@ -11,18 +11,11 @@ */ import { S3Client, - ListObjectsV2Command, CopyObjectCommand, } from '@aws-sdk/client-s3'; import getS3Config from '../utils/config.js'; - -function buildInput(org, key) { - return { - Bucket: `${org}-content`, - Prefix: `${key}/`, - }; -} +import { listCommand } from '../utils/list.js'; export const copyFile = async (client, daCtx, sourceKey, details, isRename) => { const Key = `${sourceKey.replace(details.source, details.destination)}`; @@ -51,46 +44,25 @@ export const copyFile = async (client, daCtx, sourceKey, details, isRename) => { try { await client.send(new CopyObjectCommand(input)); } catch (e) { - // eslint-disable-next-line no-console - console.log(e.$metadata); + console.log({ + code: e.$metadata.httpStatusCode, + dest: Key, + src: `${daCtx.org}-content/${sourceKey}`, + }); } }; export default async function copyObject(env, daCtx, details, isRename) { - if (details.source === details.destination) { - return { body: '', status: 409 }; - } + if (details.source === details.destination) return { body: '', status: 409 }; const config = getS3Config(env); const client = new S3Client(config); - const input = buildInput(daCtx.org, details.source); - - let ContinuationToken; - - // The input prefix has a forward slash to prevent (drafts + drafts-new, etc.). - // Which means the list will only pickup children. This adds to the initial list. - const sourceKeys = [details.source, `${details.source}.props`]; - - do { - try { - const command = new ListObjectsV2Command({ ...input, ContinuationToken }); - const resp = await client.send(command); - const { Contents = [], NextContinuationToken } = resp; - sourceKeys.push(...Contents.map(({ Key }) => Key)); - ContinuationToken = NextContinuationToken; - } catch (e) { - return { body: '', status: 404 }; - } - } while (ContinuationToken); + const { sourceKeys, continuationToken } = await listCommand(daCtx, details, client); - await Promise.all( - new Array(1).fill(null).map(async () => { - while (sourceKeys.length) { - await copyFile(client, daCtx, sourceKeys.pop(), details, isRename); - } - }), - ); + await Promise.all(sourceKeys.map(async (key) => { + await copyFile(client, daCtx, key, details, isRename); + })); - return { status: 204 }; + return { body: JSON.stringify({ continuationToken }), status: 200 }; } diff --git a/src/storage/object/delete.js b/src/storage/object/delete.js index 706ede9..6a6c7a1 100644 --- a/src/storage/object/delete.js +++ b/src/storage/object/delete.js @@ -12,18 +12,10 @@ import { S3Client, DeleteObjectCommand, - ListObjectsV2Command, } from '@aws-sdk/client-s3'; import { getSignedUrl } from '@aws-sdk/s3-request-presigner'; - import getS3Config from '../utils/config.js'; - -function buildInput(org, key) { - return { - Bucket: `${org}-content`, - Prefix: `${key}/`, - }; -} +import { listCommand } from '../utils/list.js'; export async function deleteObject(client, org, Key) { try { @@ -37,40 +29,15 @@ export async function deleteObject(client, org, Key) { } } -export default async function deleteObjects(env, daCtx) { +export default async function deleteObjects(env, daCtx, details) { const config = getS3Config(env); const client = new S3Client(config); - const input = buildInput(daCtx.org, daCtx.key); - - let ContinuationToken; - - // The input prefix has a forward slash to prevent (drafts + drafts-new, etc.). - // Which means the list will only pickup children. This adds to the initial list. - const sourceKeys = [daCtx.key, `${daCtx.key}.props`]; - - do { - try { - const command = new ListObjectsV2Command({ ...input, ContinuationToken }); - const resp = await client.send(command); - - const { Contents = [], NextContinuationToken } = resp; - sourceKeys.push(...Contents.map(({ Key }) => Key)); - await Promise.all( - new Array(1).fill(null).map(async () => { - while (sourceKeys.length) { - await deleteObject(client, daCtx.org, sourceKeys.pop()); - } - }), - ); + const { sourceKeys, continuationToken } = await listCommand(daCtx, details, client); - ContinuationToken = NextContinuationToken; - } catch (e) { - // eslint-disable-next-line no-console - console.log(e); - return { body: '', status: 404 }; - } - } while (ContinuationToken); + await Promise.all(sourceKeys.map(async (key) => { + await deleteObject(client, daCtx.org, key); + })); - return { body: null, status: 204 }; + return { body: JSON.stringify({ continuationToken }), status: 200 }; } diff --git a/src/storage/utils/list.js b/src/storage/utils/list.js index 6241601..a3cd132 100644 --- a/src/storage/utils/list.js +++ b/src/storage/utils/list.js @@ -9,6 +9,10 @@ * OF ANY KIND, either express or implied. See the License for the specific language * governing permissions and limitations under the License. */ +import { + ListObjectsV2Command, +} from '@aws-sdk/client-s3'; + export default function formatList(resp, daCtx) { function compare(a, b) { if (a.name < b.name) return -1; @@ -69,3 +73,37 @@ export default function formatList(resp, daCtx) { return combined.sort(compare); } + +function buildInput(org, key) { + return { + Bucket: `${org}-content`, + Prefix: `${key}/`, + MaxKeys: 900, + }; +} + +export async function listCommand(daCtx, details, s3client) { + // There's no need to use the list command if the item has an extension + if (daCtx.ext) return { sourceKeys: [daCtx.key] }; + + const input = buildInput(daCtx.org, daCtx.key); + const { continuationToken } = details; + + // The input prefix has a forward slash to prevent (drafts + drafts-new, etc.). + // Which means the list will only pickup children. This adds to the initial list. + const sourceKeys = []; + if (!continuationToken) sourceKeys.push(daCtx.key, `${daCtx.key}.props`); + + try { + const commandInput = { ...input, ContinuationToken: continuationToken }; + const command = new ListObjectsV2Command(commandInput); + const resp = await s3client.send(command); + + const { Contents = [], NextContinuationToken } = resp; + sourceKeys.push(...Contents.map(({ Key }) => Key)); + + return { sourceKeys, continuationToken: NextContinuationToken }; + } catch (e) { + return { body: '', status: 404 }; + } +} diff --git a/test/storage/object/copy.test.js b/test/storage/object/copy.test.js index fca7079..3ba0a6b 100644 --- a/test/storage/object/copy.test.js +++ b/test/storage/object/copy.test.js @@ -23,11 +23,17 @@ describe('Object copy', () => { }); it('does not allow copying to the same location', async () => { + const ctx = { + org: 'foo', + key: 'mydir', + users: [{email: 'haha@foo.com'}], + }; + const details = { source: 'mydir', destination: 'mydir', }; - const resp = await copyObject({}, {}, details, false); + const resp = await copyObject({}, ctx, details, false); assert.strictEqual(resp.status, 409); }); @@ -41,6 +47,7 @@ describe('Object copy', () => { const ctx = { org: 'foo', + key: 'mydir', users: [{email: 'haha@foo.com'}], }; const details = { @@ -50,7 +57,7 @@ describe('Object copy', () => { await copyObject({}, ctx, details, false); assert.strictEqual(s3Sent.length, 3); - const input = s3Sent[0]; + const input = s3Sent[2]; assert.strictEqual(input.Bucket, 'foo-content'); assert.strictEqual(input.CopySource, 'foo-content/mydir/xyz.html'); assert.strictEqual(input.Key, 'mydir/newdir/xyz.html'); @@ -71,7 +78,7 @@ describe('Object copy', () => { s3Sent.push(input); })); - const ctx = { org: 'testorg' }; + const ctx = { key: 'mydir/dir1', org: 'testorg' }; const details = { source: 'mydir/dir1', destination: 'mydir/dir2', @@ -80,7 +87,7 @@ describe('Object copy', () => { assert.strictEqual(s3Sent.length, 3); - const input = s3Sent[0]; + const input = s3Sent[2]; assert.strictEqual(input.Bucket, 'testorg-content'); assert.strictEqual(input.CopySource, 'testorg-content/mydir/dir1/myfile.html'); assert.strictEqual(input.Key, 'mydir/dir2/myfile.html');