GH-79 Copy and delete performance (#92)

* GH-79 Copy and delete performance * Copy and delete now share a listCommand - DRY * Copy and delete now pass down a continuationToken * Delete will now look for formData Resolves: GH-79 * Update delete response codes. * JOB Support for copy * Update src/storage/object/copy.js Co-authored-by: Chris Peyer <[email protected]> * Fix var reference. * Fix bug on delete. Reduce max keys for sub-request limit. * Merge in branch verdelfix branch Fix key passed to postObjectVersionWithLabel when deleting * Ensure DaCtx is complete for versioning calls. --------- Co-authored-by: Chris Millar <[email protected]> Co-authored-by: Chris Peyer <[email protected]> Co-authored-by: David Bosschaert <[email protected]>
adobe · Dec 4, 2024 · b4958a5 · b4958a5
1 parent 92d14ce
commit b4958a5
Show file tree

Hide file tree

Showing 9 changed files with 588 additions and 302 deletions.
diff --git a/src/helpers/copy.js b/src/helpers/copy.js
@@ -13,9 +13,10 @@ export default async function copyHelper(req, daCtx) {
   const formData = await req.formData();
   if (!formData) return {};
   const fullDest = formData.get('destination');
+  const continuationToken = formData.get('continuation-token');
   const lower = fullDest.slice(1).toLowerCase();
   const sanitized = lower.endsWith('/') ? lower.slice(0, -1) : lower;
   const destination = sanitized.split('/').slice(1).join('/');
   const source = daCtx.key;
-  return { source, destination };
+  return { source, destination, continuationToken };
 }
diff --git a/src/helpers/delete.js b/src/helpers/delete.js
@@ -0,0 +1,22 @@
+/*
+ * Copyright 2024 Adobe. All rights reserved.
+ * This file is licensed to you under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License. You may obtain a copy
+ * of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under
+ * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS
+ * OF ANY KIND, either express or implied. See the License for the specific language
+ * governing permissions and limitations under the License.
+ */
+
+export default async function deleteHelper(req) {
+  try {
+    const formData = await req.formData();
+    if (!formData) return {};
+    const continuationToken = formData.get('continuation-token');
+    return { continuationToken };
+  } catch {
+    return {};
+  }
+}
diff --git a/src/routes/source.js b/src/routes/source.js
@@ -14,6 +14,7 @@ import putObject from '../storage/object/put.js';
 import deleteObjects from '../storage/object/delete.js';
 
 import putHelper from '../helpers/source.js';
+import deleteHelper from '../helpers/delete.js';
 
 async function invalidateCollab(api, url, env) {
   const invPath = `/api/v1/${api}?doc=${url}`;
@@ -23,8 +24,9 @@ async function invalidateCollab(api, url, env) {
   await env.dacollab.fetch(invURL);
 }
 
-export async function deleteSource({ env, daCtx }) {
-  return /* await */ deleteObjects(env, daCtx);
+export async function deleteSource({ req, env, daCtx }) {
+  const details = await deleteHelper(req);
+  return /* await */ deleteObjects(env, daCtx, details);
 }
 
 export async function postSource({ req, env, daCtx }) {

diff --git a/src/storage/object/copy.js b/src/storage/object/copy.js
@@ -11,18 +11,13 @@
  */
 import {
   S3Client,
-  ListObjectsV2Command,
   CopyObjectCommand,
 } from '@aws-sdk/client-s3';
 
 import getS3Config from '../utils/config.js';
+import { listCommand } from '../utils/list.js';
 
-function buildInput(org, key) {
-  return {
-    Bucket: `${org}-content`,
-    Prefix: `${key}/`,
-  };
-}
+const MAX_KEYS = 900;
 
 export const copyFile = async (client, daCtx, sourceKey, details, isRename) => {
   const Key = `${sourceKey.replace(details.source, details.destination)}`;
@@ -51,46 +46,52 @@ export const copyFile = async (client, daCtx, sourceKey, details, isRename) => {
   try {
     await client.send(new CopyObjectCommand(input));
   } catch (e) {
-    // eslint-disable-next-line no-console
-    console.log(e.$metadata);
+    console.log({
+      code: e.$metadata.httpStatusCode,
+      dest: Key,
+      src: `${daCtx.org}-content/${sourceKey}`,
+    });
   }
 };
 
 export default async function copyObject(env, daCtx, details, isRename) {
-  if (details.source === details.destination) {
-    return { body: '', status: 409 };
-  }
+  if (details.source === details.destination) return { body: '', status: 409 };
 
   const config = getS3Config(env);
   const client = new S3Client(config);
-  const input = buildInput(daCtx.org, details.source);
-
-  let ContinuationToken;
-
-  // The input prefix has a forward slash to prevent (drafts + drafts-new, etc.).
-  // Which means the list will only pickup children. This adds to the initial list.
-  const sourceKeys = [details.source, `${details.source}.props`];
-
-  do {
-    try {
-      const command = new ListObjectsV2Command({ ...input, ContinuationToken });
-      const resp = await client.send(command);
 
-      const { Contents = [], NextContinuationToken } = resp;
-      sourceKeys.push(...Contents.map(({ Key }) => Key));
-      ContinuationToken = NextContinuationToken;
-    } catch (e) {
-      return { body: '', status: 404 };
-    }
-  } while (ContinuationToken);
+  let sourceKeys;
+  let remainingKeys = [];
+  let continuationToken;
 
-  await Promise.all(
-    new Array(1).fill(null).map(async () => {
-      while (sourceKeys.length) {
-        await copyFile(client, daCtx, sourceKeys.pop(), details, isRename);
+  try {
+    if (details.continuationToken) {
+      continuationToken = details.continuationToken;
+      remainingKeys = await env.DA_JOBS.get(continuationToken, { type: 'json' });
+      sourceKeys = remainingKeys.splice(0, MAX_KEYS);
+    } else {
+      let resp = await listCommand(daCtx, details, client);
+      sourceKeys = resp.sourceKeys;
+      if (resp.continuationToken) {
+        continuationToken = `copy-${details.source}-${details.destination}-${crypto.randomUUID()}`;
+        while (resp.continuationToken) {
+          resp = await listCommand(daCtx, { continuationToken: resp.continuationToken }, client);
+          remainingKeys.push(...resp.sourceKeys);
+        }
       }
-    }),
-  );
+    }
+    await Promise.all(sourceKeys.map(async (key) => {
+      await copyFile(client, daCtx, key, details, isRename);
+    }));
 
-  return { status: 204 };
+    if (remainingKeys.length) {
+      await env.DA_JOBS.put(continuationToken, JSON.stringify(remainingKeys));
+      return { body: JSON.stringify({ continuationToken }), status: 206 };
+    } else if (continuationToken) {
+      await env.DA_JOBS.delete(continuationToken);
+    }
+    return { status: 204 };
+  } catch (e) {
+    return { body: '', status: 404 };
+  }
 }
diff --git a/src/storage/object/delete.js b/src/storage/object/delete.js
@@ -12,19 +12,11 @@
 import {
   S3Client,
   DeleteObjectCommand,
-  ListObjectsV2Command,
 } from '@aws-sdk/client-s3';
 import { getSignedUrl } from '@aws-sdk/s3-request-presigner';
-
 import getS3Config from '../utils/config.js';
 import { postObjectVersionWithLabel } from '../version/put.js';
-
-function buildInput(org, key) {
-  return {
-    Bucket: `${org}-content`,
-    Prefix: `${key}/`,
-  };
-}
+import { listCommand } from '../utils/list.js';
 
 async function invalidateCollab(api, url, env) {
   const invPath = `/api/v1/${api}?doc=${url}`;
@@ -37,8 +29,9 @@ async function invalidateCollab(api, url, env) {
 export async function deleteObject(client, daCtx, Key, env, isMove = false) {
   const fname = Key.split('/').pop();
 
-  if (fname.includes('.') && !Key.endsWith('.props')) {
-    await postObjectVersionWithLabel(isMove ? 'Moved' : 'Deleted', env, daCtx);
+  if (fname.includes('.') && !fname.startsWith('.') && !fname.endsWith('.props')) {
+    const tmpCtx = { ...daCtx, key: Key }; // For next calls, ctx needs the passed
+    await postObjectVersionWithLabel(isMove ? 'Moved' : 'Deleted', env, tmpCtx);
   }
 
   let resp;
@@ -59,40 +52,21 @@ export async function deleteObject(client, daCtx, Key, env, isMove = false) {
   return resp;
 }
 
-export default async function deleteObjects(env, daCtx) {
+export default async function deleteObjects(env, daCtx, details) {
   const config = getS3Config(env);
   const client = new S3Client(config);
-  const input = buildInput(daCtx.org, daCtx.key);
-
-  let ContinuationToken;
-
-  // The input prefix has a forward slash to prevent (drafts + drafts-new, etc.).
-  // Which means the list will only pickup children. This adds to the initial list.
-  const sourceKeys = [daCtx.key, `${daCtx.key}.props`];
-
-  do {
-    try {
-      const command = new ListObjectsV2Command({ ...input, ContinuationToken });
-      const resp = await client.send(command);
 
-      const { Contents = [], NextContinuationToken } = resp;
-      sourceKeys.push(...Contents.map(({ Key }) => Key));
-
-      await Promise.all(
-        new Array(1).fill(null).map(async () => {
-          while (sourceKeys.length) {
-            await deleteObject(client, daCtx, sourceKeys.pop(), env);
-          }
-        }),
-      );
+  try {
+    const { sourceKeys, continuationToken } = await listCommand(daCtx, details, client);
+    await Promise.all(sourceKeys.map(async (key) => {
+      await deleteObject(client, daCtx, key, env);
+    }));
 
-      ContinuationToken = NextContinuationToken;
-    } catch (e) {
-      // eslint-disable-next-line no-console
-      console.log(e);
-      return { body: '', status: 404 };
+    if (continuationToken) {
+      return { body: JSON.stringify({ continuationToken }), status: 206 };
     }
-  } while (ContinuationToken);
-
-  return { body: null, status: 204 };
+    return { status: 204 };
+  } catch (e) {
+    return { body: '', status: 404 };
+  }
 }
diff --git a/src/storage/utils/list.js b/src/storage/utils/list.js
@@ -9,6 +9,10 @@
  * OF ANY KIND, either express or implied. See the License for the specific language
  * governing permissions and limitations under the License.
  */
+import {
+  ListObjectsV2Command,
+} from '@aws-sdk/client-s3';
+
 export default function formatList(resp, daCtx) {
   function compare(a, b) {
     if (a.name < b.name) return -1;
@@ -69,3 +73,40 @@ export default function formatList(resp, daCtx) {
 
   return combined.sort(compare);
 }
+
+function buildInput(org, key) {
+  return {
+    Bucket: `${org}-content`,
+    Prefix: `${key}/`,
+    MaxKeys: 300,
+  };
+}
+
+/**
+ * Lists a files in a bucket under the specified key.
+ * @param {DaCtx} daCtx the DA Context
+ * @param {Object} details contains any prevous Continuation token
+ * @param s3client
+ * @return {Promise<{sourceKeys: String[], continuationToken: String}>}
+ */
+export async function listCommand(daCtx, details, s3client) {
+  // There's no need to use the list command if the item has an extension
+  if (daCtx.ext) return { sourceKeys: [daCtx.key] };
+
+  const input = buildInput(daCtx.org, daCtx.key);
+  const { continuationToken } = details;
+
+  // The input prefix has a forward slash to prevent (drafts + drafts-new, etc.).
+  // Which means the list will only pickup children. This adds to the initial list.
+  const sourceKeys = [];
+  if (!continuationToken) sourceKeys.push(daCtx.key, `${daCtx.key}.props`);
+
+  const commandInput = { ...input, ContinuationToken: continuationToken };
+  const command = new ListObjectsV2Command(commandInput);
+  const resp = await s3client.send(command);
+
+  const { Contents = [], NextContinuationToken } = resp;
+  sourceKeys.push(...Contents.map(({ Key }) => Key));
+
+  return { sourceKeys, continuationToken: NextContinuationToken };
+}