GH-79 Copy and delete performance

src/helpers/copy.js

@@ -13,9 +13,10 @@ export default async function copyHelper(req, daCtx) {
   const formData = await req.formData();
   if (!formData) return {};
   const fullDest = formData.get('destination');
+  const continuationToken = formData.get('continuation-token');
   const lower = fullDest.slice(1).toLowerCase();
   const sanitized = lower.endsWith('/') ? lower.slice(0, -1) : lower;
   const destination = sanitized.split('/').slice(1).join('/');
   const source = daCtx.key;
-  return { source, destination };
+  return { source, destination, continuationToken };
 }

src/helpers/delete.js

@@ -0,0 +1,22 @@
+/*
+ * Copyright 2024 Adobe. All rights reserved.
+ * This file is licensed to you under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License. You may obtain a copy
+ * of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under
+ * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS
+ * OF ANY KIND, either express or implied. See the License for the specific language
+ * governing permissions and limitations under the License.
+ */
+
+export default async function deleteHelper(req) {
+  try {
+    const formData = await req.formData();
+    if (!formData) return {};
+    const continuationToken = formData.get('continuation-token');
+    return { continuationToken };
+  } catch {
+    return {};
+  }
+}

src/routes/source.js

@@ -14,6 +14,7 @@ import putObject from '../storage/object/put.js';
 import deleteObjects from '../storage/object/delete.js';
 
 import putHelper from '../helpers/source.js';
+import deleteHelper from '../helpers/delete.js';
 import { postObjectVersion } from '../storage/version/put.js';
 
 async function invalidateCollab(api, url, env) {
@@ -25,13 +26,18 @@ async function invalidateCollab(api, url, env) {
 }
 
 export async function deleteSource({ req, env, daCtx }) {
+  const details = await deleteHelper(req);
   await postObjectVersion(req, env, daCtx);
-  const resp = await deleteObjects(env, daCtx);
-
-  if (resp.status === 204) {
-    const initiator = req.headers.get('x-da-initiator');
-    if (initiator !== 'collab') {
-      await invalidateCollab('deleteadmin', req.url, env);
+  const resp = await deleteObjects(env, daCtx, details);
+
+  if (resp.status === 204 || resp.status === 200) {
+    try {
+      const initiator = req.headers.get('x-da-initiator');
+      if (initiator !== 'collab') {
+        await invalidateCollab('deleteadmin', req.url, env);
+      }
+    } catch {
+      // collab not available
     }
   }
   return resp;

src/storage/object/copy.js

@@ -11,18 +11,11 @@
  */
 import {
   S3Client,
-  ListObjectsV2Command,
   CopyObjectCommand,
 } from '@aws-sdk/client-s3';
 
 import getS3Config from '../utils/config.js';
-
-function buildInput(org, key) {
-  return {
-    Bucket: `${org}-content`,
-    Prefix: `${key}/`,
-  };
-}
+import { listCommand } from '../utils/list.js';
 
 export const copyFile = async (client, daCtx, sourceKey, details, isRename) => {
   const Key = `${sourceKey.replace(details.source, details.destination)}`;
@@ -51,46 +44,25 @@
   try {
     await client.send(new CopyObjectCommand(input));
   } catch (e) {
-    // eslint-disable-next-line no-console
-    console.log(e.$metadata);
+    console.log({
+      code: e.$metadata.httpStatusCode,
+      dest: Key,
+      src: `${daCtx.org}-content/${sourceKey}`,
+    });
   }
 };
 
 export default async function copyObject(env, daCtx, details, isRename) {
-  if (details.source === details.destination) {
-    return { body: '', status: 409 };
-  }
+  if (details.source === details.destination) return { body: '', status: 409 };
 
   const config = getS3Config(env);
   const client = new S3Client(config);
-  const input = buildInput(daCtx.org, details.source);
-
-  let ContinuationToken;
-
-  // The input prefix has a forward slash to prevent (drafts + drafts-new, etc.).
-  // Which means the list will only pickup children. This adds to the initial list.
-  const sourceKeys = [details.source, `${details.source}.props`];
-
-  do {
-    try {
-      const command = new ListObjectsV2Command({ ...input, ContinuationToken });
-      const resp = await client.send(command);
 
-      const { Contents = [], NextContinuationToken } = resp;
-      sourceKeys.push(...Contents.map(({ Key }) => Key));
-      ContinuationToken = NextContinuationToken;
-    } catch (e) {
-      return { body: '', status: 404 };
-    }
-  } while (ContinuationToken);
+  const { sourceKeys, continuationToken } = await listCommand(daCtx, details, client);
 
-  await Promise.all(
-    new Array(1).fill(null).map(async () => {
-      while (sourceKeys.length) {
-        await copyFile(client, daCtx, sourceKeys.pop(), details, isRename);
-      }
-    }),
-  );
+  await Promise.all(sourceKeys.map(async (key) => {
+    await copyFile(client, daCtx, key, details, isRename);
+  }));
 
-  return { status: 204 };
+  return { body: JSON.stringify({ continuationToken }), status: 200 };
 }

src/storage/object/delete.js

@@ -12,18 +12,10 @@
 import {
   S3Client,
   DeleteObjectCommand,
-  ListObjectsV2Command,
 } from '@aws-sdk/client-s3';
 import { getSignedUrl } from '@aws-sdk/s3-request-presigner';
-
 import getS3Config from '../utils/config.js';
-
-function buildInput(org, key) {
-  return {
-    Bucket: `${org}-content`,
-    Prefix: `${key}/`,
-  };
-}
+import { listCommand } from '../utils/list.js';
 
 export async function deleteObject(client, org, Key) {
   try {
@@ -37,40 +29,15 @@ export async function deleteObject(client, org, Key) {
   }
 }
 
-export default async function deleteObjects(env, daCtx) {
+export default async function deleteObjects(env, daCtx, details) {
   const config = getS3Config(env);
   const client = new S3Client(config);
-  const input = buildInput(daCtx.org, daCtx.key);
-
-  let ContinuationToken;
-
-  // The input prefix has a forward slash to prevent (drafts + drafts-new, etc.).
-  // Which means the list will only pickup children. This adds to the initial list.
-  const sourceKeys = [daCtx.key, `${daCtx.key}.props`];
-
-  do {
-    try {
-      const command = new ListObjectsV2Command({ ...input, ContinuationToken });
-      const resp = await client.send(command);
-
-      const { Contents = [], NextContinuationToken } = resp;
-      sourceKeys.push(...Contents.map(({ Key }) => Key));
 
-      await Promise.all(
-        new Array(1).fill(null).map(async () => {
-          while (sourceKeys.length) {
-            await deleteObject(client, daCtx.org, sourceKeys.pop());
-          }
-        }),
-      );
+  const { sourceKeys, continuationToken } = await listCommand(daCtx, details, client);
 
-      ContinuationToken = NextContinuationToken;
-    } catch (e) {
-      // eslint-disable-next-line no-console
-      console.log(e);
-      return { body: '', status: 404 };
-    }
-  } while (ContinuationToken);
+  await Promise.all(sourceKeys.map(async (key) => {
+    await deleteObject(client, daCtx.org, key);
+  }));
 
-  return { body: null, status: 204 };
+  return { body: JSON.stringify({ continuationToken }), status: 200 };
 }

src/storage/utils/list.js

@@ -9,6 +9,10 @@
  * OF ANY KIND, either express or implied. See the License for the specific language
  * governing permissions and limitations under the License.
  */
+import {
+  ListObjectsV2Command,
+} from '@aws-sdk/client-s3';
+
 export default function formatList(resp, daCtx) {
   function compare(a, b) {
     if (a.name < b.name) return -1;
@@ -69,3 +73,37 @@ export default function formatList(resp, daCtx) {
 
   return combined.sort(compare);
 }
+
+function buildInput(org, key) {
+  return {
+    Bucket: `${org}-content`,
+    Prefix: `${key}/`,
+    MaxKeys: 900,
+  };
+}
+
+export async function listCommand(daCtx, details, s3client) {
+  // There's no need to use the list command if the item has an extension
+  if (daCtx.ext) return { sourceKeys: [daCtx.key] };
+
+  const input = buildInput(daCtx.org, daCtx.key);
+  const { continuationToken } = details;
+
+  // The input prefix has a forward slash to prevent (drafts + drafts-new, etc.).
+  // Which means the list will only pickup children. This adds to the initial list.
+  const sourceKeys = [];
+  if (!continuationToken) sourceKeys.push(daCtx.key, `${daCtx.key}.props`);
+
+  try {
+    const commandInput = { ...input, ContinuationToken: continuationToken };
+    const command = new ListObjectsV2Command(commandInput);
+    const resp = await s3client.send(command);
+
+    const { Contents = [], NextContinuationToken } = resp;
+    sourceKeys.push(...Contents.map(({ Key }) => Key));
+
+    return { sourceKeys, continuationToken: NextContinuationToken };
+  } catch (e) {
+    return { body: '', status: 404 };
+  }
+}

test/storage/object/copy.test.js

@@ -23,11 +23,17 @@ describe('Object copy', () => {
   });
 
   it('does not allow copying to the same location', async () => {
+    const ctx = {
+      org: 'foo',
+      key: 'mydir',
+      users: [{email: '[email protected]'}],
+    };
+
     const details = {
       source: 'mydir',
       destination: 'mydir',
     };
-    const resp = await copyObject({}, {}, details, false);
+    const resp = await copyObject({}, ctx, details, false);
     assert.strictEqual(resp.status, 409);
   });
 
@@ -41,6 +47,7 @@ describe('Object copy', () => {
 
     const ctx = {
       org: 'foo',
+      key: 'mydir',
       users: [{email: '[email protected]'}],
     };
     const details = {
@@ -50,7 +57,7 @@ describe('Object copy', () => {
     await copyObject({}, ctx, details, false);
 
     assert.strictEqual(s3Sent.length, 3);
-    const input = s3Sent[0];
+    const input = s3Sent[2];
     assert.strictEqual(input.Bucket, 'foo-content');
     assert.strictEqual(input.CopySource, 'foo-content/mydir/xyz.html');
     assert.strictEqual(input.Key, 'mydir/newdir/xyz.html');
@@ -71,7 +78,7 @@ describe('Object copy', () => {
       s3Sent.push(input);
     }));
 
-    const ctx = { org: 'testorg' };
+    const ctx = { key: 'mydir/dir1', org: 'testorg' };
     const details = {
       source: 'mydir/dir1',
       destination: 'mydir/dir2',
@@ -80,7 +87,7 @@ describe('Object copy', () => {
 
 
     assert.strictEqual(s3Sent.length, 3);
-    const input = s3Sent[0];
+    const input = s3Sent[2];
     assert.strictEqual(input.Bucket, 'testorg-content');
     assert.strictEqual(input.CopySource, 'testorg-content/mydir/dir1/myfile.html');
     assert.strictEqual(input.Key, 'mydir/dir2/myfile.html');