Single Bucket - Copy/Rename

.nycrc.json

@@ -7,9 +7,8 @@
   "check-coverage": true,
   "all": true,
   "include": ["src/**/*.js"],
-  "lines": 45,
+  "lines": 60,
   "branches": 80,
-  "statements": 45,
-  "functions": 45,
-  "skip-full": true
+  "statements": 60,
+  "functions": 60
 }

package.json

@@ -13,6 +13,7 @@
     "lint": "eslint .",
     "test": "c8 mocha --spec=test/**/*.test.js && mocha --spec=test/it/**/*.spec.js",
     "test:unit": "c8 mocha --spec=test/**/*.test.js",
+    "test:perf": "mocha --spec=test/**/*.perf.js",
     "test:it": "mocha --spec=test/it/**/*.spec.js",
     "dev": "wrangler dev --env dev",
     "deploy:prod": "wrangler deploy",

src/helpers/copy.js

@@ -9,6 +9,13 @@
  * OF ANY KIND, either express or implied. See the License for the specific language
  * governing permissions and limitations under the License.
  */
+
+/**
+ * Creates the copy source/dest object from the current contexts
+ * @param {Request} req the Request object
+ * @param {Object} daCtx the DA Context
+ * @return {Promise<{destination: string, source: string}|{}>}
+ */
 export default async function copyHelper(req, daCtx) {
   const formData = await req.formData();
   if (!formData) return {};

src/storage/object/copy.js

@@ -9,88 +9,114 @@
  * OF ANY KIND, either express or implied. See the License for the specific language
  * governing permissions and limitations under the License.
  */
-import {
-  S3Client,
-  ListObjectsV2Command,
-  CopyObjectCommand,
-} from '@aws-sdk/client-s3';
 
-import getS3Config from '../utils/config.js';
+const limit = 100;
 
-function buildInput(org, key) {
-  return {
-    Bucket: `${org}-content`,
-    Prefix: `${key}/`,
-  };
-}
-
-export const copyFile = async (client, daCtx, sourceKey, details, isRename) => {
-  const Key = `${sourceKey.replace(details.source, details.destination)}`;
-
-  const input = {
-    Bucket: `${daCtx.org}-content`,
-    Key,
-    CopySource: `${daCtx.org}-content/${sourceKey}`,
-  };
+/**
+ * Copies the specified file from the source to the destination.
+ * @param {Object} env the CloudFlare environment
+ * @param {Object} daCtx the DA Context
+ * @param {String} sourceKey the key for the source file
+ * @param {String} destinationKey the key for the destination file
+ * @param {Boolean} isRename whether this is a rename operation
+ * @return {Promise<Object>} the status of the copy operation
+ */
+const copyFile = async (env, daCtx, sourceKey, destinationKey, isRename) => {
+  try {
+    const obj = await env.DA_CONTENT.get(sourceKey);
+    if (!obj) {
+      return { success: false, source: sourceKey, destination: destinationKey };
+    }
 
-  // We only want to keep the history if this was a rename. In case of an actual
-  // copy we should start with clean history. The history is associated with the
-  // ID of the object, so we need to generate a new ID for the object and also a
-  // new ID for the version. We set the user to the user making the copy.
-  if (!isRename) {
-    input.Metadata = {
-      ID: crypto.randomUUID(),
-      Version: crypto.randomUUID(),
-      Timestamp: `${Date.now()}`,
-      Users: JSON.stringify(daCtx.users),
-      Path: Key,
+    const body = await obj.text();
+    const { httpMetadata } = obj;
+    // We want to keep the history if this was a rename. In case of an actual
+    // copy we should start with clean history. The history is associated with the
+    // ID of the object, so we need to generate a new ID for the object and also a
+    // new ID for the version. We set the user to the user making the copy.
+    const customMetadata = {
+      id: crypto.randomUUID(),
+      version: crypto.randomUUID(),
+      timestamp: `${Date.now()}`,
+      users: JSON.stringify(daCtx.users),
+      path: destinationKey,
     };
-    input.MetadataDirective = 'REPLACE';
-  }
+    if (isRename) Object.assign(customMetadata, obj.customMetadata, { path: destinationKey });
 
-  try {
-    await client.send(new CopyObjectCommand(input));
+    await env.DA_CONTENT.put(destinationKey, body, { httpMetadata, customMetadata });
+    if (isRename) await env.DA_CONTENT.delete(sourceKey);
+    return { success: true, source: sourceKey, destination: destinationKey };
+    /* c8 ignore next 4 */
   } catch (e) {
     // eslint-disable-next-line no-console
-    console.log(e.$metadata);
+    console.error(`Failed to copy: ${sourceKey} to ${destinationKey}`, e);
+    return { success: false, source: sourceKey, destination: destinationKey };
+  }
+};
+
+const copyFiles = async (env, daCtx, detailsList, isRename) => {
+  const results = [];
+  while (detailsList.length > 0) {
+    const promises = [];
+    do {
+      const { src, dest } = detailsList.shift();
+      promises.push(copyFile(env, daCtx, src, dest, isRename));
+    } while (detailsList.length > 0 && promises.length <= limit);
+    await Promise.all(promises).then((values) => results.push(...values));
   }
+  return results;
 };
 
-export default async function copyObject(env, daCtx, details, isRename) {
+/**
+ * Copies a directory (and contents) or a single file to location.
+ * @param {Object} env the CloudFlare environment
+ * @param {Object} daCtx the DA Context
+ * @param {Object} details the source & details of the copy operation
+ * @param {string} details.source the source directory or file
+ * @param {string} details.destination the destination directory or file
+ * @param {Boolean=false} isRename whether this is a rename operation
+ * @return {Promise<{ status }>}
+ */
+export default async function copyObject(env, daCtx, details, isRename = false) {
   if (details.source === details.destination) {
     return { body: '', status: 409 };
   }
+  const results = [];
+  const src = details.source.length ? `${daCtx.org}/${details.source}` : daCtx.org;
+  const dest = `${daCtx.org}/${details.destination}`;
+  const obj = await env.DA_CONTENT.head(src);
+  // Head won't return for a folder so this must be a file copy.
+  if (obj) {
+    await copyFile(env, daCtx, src, dest, isRename).then((value) => results.push(value));
+  } else {
+    let cursor;
+    // The input prefix has a forward slash to prevent (drafts + drafts-new, etc.).
+    // Which means the list will only pickup children. This adds to the initial list.
+    const detailsList = [{ src: `${src}.props`, dest: `${dest}.props` }];
+    do {
+      const input = {
+        prefix: `${src}/`,
+        limit,
+        cursor,
+      };
+      const r2list = await env.DA_CONTENT.list(input);
+      const { objects } = r2list;
+      cursor = r2list.cursor;
+      // List of objects to copy
+      detailsList.push(...objects
+        // Do not save root props file to new folder under *original*
+        .filter(({ key }) => key !== `${src}.props`)
+        .map(({ key }) => ({ src: key, dest: `${key.replace(src, dest)}` })));
+    } while (cursor);
+    await copyFiles(env, daCtx, detailsList, isRename).then((values) => results.push(...values));
+  }
 
-  const config = getS3Config(env);
-  const client = new S3Client(config);
-  const input = buildInput(daCtx.org, details.source);
-
-  let ContinuationToken;
-
-  // The input prefix has a forward slash to prevent (drafts + drafts-new, etc.).
-  // Which means the list will only pickup children. This adds to the initial list.
-  const sourceKeys = [details.source, `${details.source}.props`];
-
-  do {
-    try {
-      const command = new ListObjectsV2Command({ ...input, ContinuationToken });
-      const resp = await client.send(command);
-
-      const { Contents = [], NextContinuationToken } = resp;
-      sourceKeys.push(...Contents.map(({ Key }) => Key));
-      ContinuationToken = NextContinuationToken;
-    } catch (e) {
-      return { body: '', status: 404 };
-    }
-  } while (ContinuationToken);
-
-  await Promise.all(
-    new Array(1).fill(null).map(async () => {
-      while (sourceKeys.length) {
-        await copyFile(client, daCtx, sourceKeys.pop(), details, isRename);
-      }
-    }),
-  );
+  // Retry failures
+  const retries = results.filter(({ success }) => !success).map(({ source, destination }) => ({ src: source, dest: destination }));
+  if (retries.length > 0) {
+    const retryResults = await copyFiles(env, daCtx, retries, isRename);
+    results.push(...retryResults);
+  }
 
   return { status: 204 };
 }

test/helpers/copy.test.js

@@ -0,0 +1,45 @@
+/*
+ * Copyright 2024 Adobe. All rights reserved.
+ * This file is licensed to you under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License. You may obtain a copy
+ * of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under
+ * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS
+ * OF ANY KIND, either express or implied. See the License for the specific language
+ * governing permissions and limitations under the License.
+ */
+
+import assert from 'node:assert';
+
+import copyHelper from '../../src/helpers/copy.js';
+
+describe('Copy helper', () => {
+  it('handles no form data', async () => {
+    const req = {
+      formData: async () => undefined,
+    };
+    const details = await copyHelper(req, {})
+    assert.deepStrictEqual(details, {});
+  });
+
+  it('sanitizes a folder path', async () => {
+    const req = {
+      formData: async () => ({
+        get: () => '/foo/bar/',
+      }),
+    };
+    const details = await copyHelper(req, { key: 'baz' });
+    assert.deepEqual(details, { source: 'baz', destination: 'bar' });
+  });
+
+  it('sanitizes a file path', async () => {
+    const req = {
+      formData: async () => ({
+        get: () => '/FOO/BAR',
+      }),
+    };
+    const details = await copyHelper(req, { key: 'baz' });
+    assert.deepEqual(details, { source: 'baz', destination: 'bar' });
+  });
+});

test/it/post.spec.js

@@ -59,6 +59,80 @@ describe('POST/PUT HTTP Requests', () => {
           assert.strictEqual(r2o.objects.length, 1);
         });
       });
+
+      describe('/copy', () => {
+        it('copies a file', async () => {
+          const body = new FormData();
+          body.append('destination', '/wknd/new-folder/index.html' );
+          const opts =  {
+            body,
+            method
+          };
+          const req = new Request('https://admin.da.live/copy/wknd/index.html', opts);
+          const resp = await worker.fetch(req, env);
+          assert.strictEqual(resp.status, 204);
+          const head = await env.DA_CONTENT.head('wknd/new-folder/index.html');
+          assert(head);
+        });
+
+        it('copies a folder', async () => {
+          for (let i = 0; i < 5; i++) {
+            await env.DA_CONTENT.put(`wknd/pages/index${i}.html`, 'Hello, World!');
+          }
+          const body = new FormData();
+          body.append('destination', '/wknd/new-folder' );
+          const opts =  {
+            body,
+            method
+          };
+          const req = new Request('https://admin.da.live/copy/wknd/pages', opts);
+          const resp = await worker.fetch(req, env);
+          assert.strictEqual(resp.status, 204);
+          for (let i = 0; i < 5; i++) {
+            const head = await env.DA_CONTENT.head(`wknd/new-folder/index${i}.html`);
+            assert(head);
+          }
+        });
+      });
+
+      describe('/rename', () => {
+        it('renames a file', async () => {
+          const body = new FormData();
+          body.append('newname', 'renamed.html' );
+          const opts =  {
+            body,
+            method
+          };
+          const req = new Request('https://admin.da.live/rename/wknd/index.html', opts);
+          const resp = await worker.fetch(req, env);
+          assert.strictEqual(resp.status, 204);
+          let head = await env.DA_CONTENT.head('wknd/renamed.html');
+          assert(head);
+          head = await env.DA_CONTENT.head('wknd/index.html');
+          assert.ifError(head);
+        });
+
+        it('renames a folder', async () => {
+          for (let i = 0; i < 5; i++) {
+            await env.DA_CONTENT.put(`wknd/pages/index${i}.html`, 'Hello, World!');
+          }
+          const body = new FormData();
+          body.append('newname', 'new-folder' );
+          const opts =  {
+            body,
+            method
+          };
+          const req = new Request('https://admin.da.live/rename/wknd/pages', opts);
+          const resp = await worker.fetch(req, env);
+          assert.strictEqual(resp.status, 204);
+          for (let i = 0; i < 5; i++) {
+            let head = await env.DA_CONTENT.head(`wknd/new-folder/index${i}.html`);
+            assert(head);
+            head = await env.DA_CONTENT.head(`wknd/pages/index${i}.html`);
+            assert.ifError(head);
+          }
+        });
+      });
     });
   }
 });

test/routes/copy.test.js

@@ -0,0 +1,30 @@
+/*
+ * Copyright 2024 Adobe. All rights reserved.
+ * This file is licensed to you under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License. You may obtain a copy
+ * of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under
+ * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS
+ * OF ANY KIND, either express or implied. See the License for the specific language
+ * governing permissions and limitations under the License.
+ */
+
+import assert from 'node:assert';
+import esmock from 'esmock';
+
+describe('Copy Handler', () => {
+  const params = { req: {}, env: {}, daCtx: {} }
+  it('handles valid request', async () => {
+    const copyHandler = await esmock('../../src/routes/copy.js', {
+      '../../src/helpers/copy.js': {
+        default: async () => ({ source: 'mydir', destination: 'mydir' })
+      },
+      '../../src/storage/object/copy.js': {
+        default: async () => ({ status: 201 })
+      }
+    });
+    const resp = await copyHandler(params);
+    assert.deepStrictEqual(resp, { status: 201 });
+  });
+});