Impl Version On Delete for Single Bucket Branch (#96)

* Update deps.

* Version a file when a delete is requested.

* Update move to version before delete

* Incorporate bug where DaCtx contains wrong key

* Fix key context and update tests.

* Always pass a full DaCtx object.

* Check for file on versioning

* Bad merge.

src/routes/source.js

@@ -12,38 +12,18 @@
 import getObject from '../storage/object/get.js';
 import putObject from '../storage/object/put.js';
 import deleteObjects from '../storage/object/delete.js';
-
 import putHelper from '../helpers/source.js';
+import { syncCollab } from '../storage/utils/collab.js';
 
-async function invalidateCollab(api, url, env) {
-  const invPath = `/api/v1/${api}?doc=${url}`;
-
-  // Use dacollab service binding, hostname is not relevant
-  const invURL = `https://localhost${invPath}`;
-  await env.dacollab.fetch(invURL);
-}
-
-export async function deleteSource({ req, env, daCtx }) {
-  const resp = await deleteObjects(env, daCtx);
-
-  if (resp.status === 204) {
-    const initiator = req.headers.get('x-da-initiator');
-    if (initiator !== 'collab') {
-      await invalidateCollab('deleteadmin', req.url, env);
-    }
-  }
-  return resp;
+export async function deleteSource({ env, daCtx }) {
+  return deleteObjects(env, daCtx);
 }
 
 export async function postSource({ req, env, daCtx }) {
   const obj = await putHelper(req, env, daCtx);
   const resp = await putObject(env, daCtx, obj);
-
-  if (resp.status === 201 || resp.status === 200) {
-    const initiator = req.headers.get('x-da-initiator');
-    if (initiator !== 'collab') {
-      await invalidateCollab('syncadmin', req.url, env);
-    }
+  if (resp.status === 200 || resp.status === 201) {
+    await syncCollab(env, daCtx);
   }
   return resp;
 }

src/storage/object/delete.js

@@ -9,35 +9,54 @@
  * OF ANY KIND, either express or implied. See the License for the specific language
  * governing permissions and limitations under the License.
  */
+import { deleteFromCollab } from '../utils/collab.js';
+import { postObjectVersionWithLabel } from '../version/put.js';
 
+/**
+ * Deletes an object in the storage, creating a version of it if necessary.
+ *
+ * @param {Object} env the CloudFlare environment
+ * @param {DaCtx} daCtx the DA Context
+ * @param {String} key the key of the object to delete (excluding Org)
+ * @param {Boolean} isMove if this was initiated by a move operation
+ * @return {Promise<void>}
+ */
+export async function deleteObject(env, daCtx, key, isMove = false) {
+  const fname = key.split('/').pop();
+
+  const tmpCtx = { ...daCtx, key }; // For next calls, ctx needs the passed key, as it could contain a folder
+  if (fname.indexOf('.') > 0 && !key.endsWith('.props')) {
+    await postObjectVersionWithLabel(env, tmpCtx, isMove ? 'Moved' : 'Deleted');
+  }
+  await env.DA_CONTENT.delete(`${daCtx.org}/${key}`);
+  await deleteFromCollab(env, tmpCtx);
+}
 /**
  * Deletes one or more objects in the storage. Object is specified by the key in the daCtx or a list passed in.
  * Note: folders can not be specified in the `keys` list.
  *
  * @param {Object} env the CloudFlare environment
  * @param {DaCtx} daCtx the DA Context
- * @param {String[]} [keys=[]] the list of keys to delete (excluding the Org)
  * @return {Promise<{body: null, status: number}>}
  */
-export default async function deleteObjects(env, daCtx, keys = []) {
-  if (keys.length) {
-    const fullKeys = keys.map((key) => `${daCtx.org}/${key}`);
-    await env.DA_CONTENT.delete(fullKeys);
-    return { body: null, status: 204 };
-  }
-
-  const fullKey = `${daCtx.org}/${daCtx.key}`;
-  const prefix = `${fullKey}/`;
+export default async function deleteObjects(env, daCtx) {
+  const keys = [];
+  const prefix = `${daCtx.org}/${daCtx.key}/`;
   // The input prefix has a forward slash to prevent (drafts + drafts-new, etc.).
   // Which means the list will only pickup children. This adds to the initial list.
-  keys.push(fullKey, `${fullKey}.props`);
+  keys.push(daCtx.key, `${daCtx.key}.props`);
   let truncated = false;
   do {
-    const r2objects = await env.DA_CONTENT.list({ prefix, limit: 500 });
+    const r2objects = await env.DA_CONTENT.list({ prefix, limit: 100 });
     const { objects } = r2objects;
     truncated = r2objects.truncated;
-    keys.push(...objects.map(({ key }) => key));
-    await env.DA_CONTENT.delete(keys);
+    keys.push(...objects.map(({ key }) => key.split('/').slice(1).join('/')));
+    const promises = [];
+    keys.forEach((k) => {
+      promises.push(deleteObject(env, daCtx, k));
+    });
+    await Promise.all(promises);
+    keys.length = 0;
   } while (truncated);
   return { body: null, status: 204 };
 }

src/storage/object/get.js

@@ -13,12 +13,12 @@
 /**
  * Retrieve a specified object.
  * @param {Object} env the CloudFlare environment
- * @param {String} org the org of the object
- * @param {String} key the key of the object
+ * @param {DaCtx} daCtx the DA Context
  * @param {boolean} head flag to only retrieve head info or body as well.
  * @return {Promise<{Object}>} response object
  */
-export default async function getObject(env, { org, key }, head = false) {
+export default async function getObject(env, daCtx, head = false) {
+  const { org, key } = daCtx;
   const daKey = `${org}/${key}`;
 
   let obj;

src/storage/object/move.js

@@ -9,7 +9,8 @@
  * OF ANY KIND, either express or implied. See the License for the specific language
  * governing permissions and limitations under the License.
  */
-import { copyFile, copyFiles } from '../utils/copy.js';
+import { copyFile } from '../utils/copy.js';
+import { deleteObject } from './delete.js';
 
 const limit = 100;
 
@@ -24,7 +25,10 @@ const limit = 100;
  */
 export default async function moveObject(env, daCtx, details) {
   if (daCtx.isFile) {
-    await copyFile(env, daCtx, details.source, details.destination, true);
+    const res = await copyFile(env, daCtx, details.source, details.destination, true);
+    if (res.success) {
+      await deleteObject(env, daCtx, details.source, true);
+    }
     return { status: 204 };
   }
 
@@ -53,14 +57,21 @@ export default async function moveObject(env, daCtx, details) {
             dest: src.replace(details.source, details.destination),
           };
         }));
-      await copyFiles(env, daCtx, sourceList, true)
-        .then(async (values) => {
-          const successes = values
-            .filter((item) => item.success)
-            .map((item) => item.source);
-          await env.DA_CONTENT.delete(successes);
-          results.push(...values);
-        });
+
+      const promises = [];
+      sourceList.forEach(({ src, dest }) => {
+        promises.push(
+          copyFile(env, daCtx, src, dest, true)
+            .then(async (res) => {
+              if (res.success) {
+                await deleteObject(env, daCtx, src, true);
+              }
+              return res;
+            })
+            .then((res) => results.push(res)),
+        );
+      });
+      await Promise.allSettled(promises);
       sourceList.length = 0;
       /* c8 ignore next 3 */
     } catch (e) {

src/storage/utils/collab.js

@@ -0,0 +1,41 @@
+/*
+ * Copyright 2024 Adobe. All rights reserved.
+ * This file is licensed to you under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License. You may obtain a copy
+ * of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under
+ * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS
+ * OF ANY KIND, either express or implied. See the License for the specific language
+ * governing permissions and limitations under the License.
+ */
+
+async function invalidateCollab(env, daCtx, api) {
+  if (daCtx.initiator === 'collab' || !daCtx.key.endsWith('.html')) {
+    return;
+  }
+  const invPath = `/api/v1/${api}?doc=${daCtx.origin}/${daCtx.api}/${daCtx.org}/${daCtx.key}`;
+  // Use dacollab service binding, hostname is not relevant
+  const invURL = `https://localhost${invPath}`;
+  await env.dacollab.fetch(invURL);
+}
+
+/**
+ * Removes the specified URL from the Collab cache.
+ * @param {Object} env the CloudFlare environment
+ * @param {DaCtx} daCtx the DA Context
+ * @return {Promise<void>}
+ */
+export async function deleteFromCollab(env, daCtx) {
+  await invalidateCollab(env, daCtx, 'deleteadmin');
+}
+
+/**
+ * Forces a sync in DaCollab for the specified URL.
+ * @param {Object} env the CloudFlare environment
+ * @param {DaCtx} daCtx the DA Context
+ * @return {Promise<void>}
+ */
+export async function syncCollab(env, daCtx) {
+  await invalidateCollab(env, daCtx, 'syncadmin');
+}

src/storage/version/get.js

@@ -11,6 +11,14 @@
  */
 import getObject from '../object/get.js';
 
-export async function getObjectVersion(env, { org, key }, head) {
-  return getObject(env, { org, key: `.da-versions/${key}` }, head);
+/**
+ * Gets a specified object version.
+ * @param {Object} env the CloudFlare environment
+ * @param {DaCtx} daCtx the DA Context
+ * @param {Boolean} head only retrieve the head info
+ * @return {Promise<Promise<{Object}>|*>}
+ */
+export async function getObjectVersion(env, daCtx, head) {
+  const tmpCtx = { ...daCtx, key: `.da-versions/${daCtx.key}` };
+  return getObject(env, tmpCtx, head);
 }

src/storage/version/list.js

@@ -12,18 +12,24 @@
 import getObject from '../object/get.js';
 import listObjects from '../object/list.js';
 
-export async function listObjectVersions(env, { org, key }) {
-  const current = await getObject(env, { org, key }, true);
+/**
+ * Lists all versions of an object.
+ *
+ * @param {Object} env the CloudFlare environment
+ * @param {DaCtx} daCtx the DA Context
+ * @return {Promise<{body: string, status: number}|{body: string, contentType: string, status: number}|number>}
+ */
+export async function listObjectVersions(env, daCtx) {
+  const { org } = daCtx;
+  const current = await getObject(env, daCtx, true);
   if (current.status === 404 || !current.metadata.id) {
     return 404;
   }
   const objects = await listObjects(env, { org, key: `.da-versions/${current.metadata.id}` });
   if (objects) {
     const promises = await Promise.all(objects.map(async (entry) => {
-      const entryResp = await getObject(env, {
-        org,
-        key: `.da-versions/${current.metadata.id}/${entry.name}.${entry.ext}`,
-      }, true);
+      const tmpCtx = { ...daCtx, key: `.da-versions/${current.metadata.id}/${entry.name}.${entry.ext}` };
+      const entryResp = await getObject(env, tmpCtx, true);
       const timestamp = parseInt(entryResp.metadata.timestamp || '0', 10);
       const users = JSON.parse(entryResp.metadata.users || '[{"email":"anonymous"}]');
       const { label, path } = entryResp.metadata;

src/storage/version/put.js

@@ -61,7 +61,8 @@ export async function putObjectWithVersion(env, daCtx, update, body) {
   // While we are automatically storing the body once for the 'Collab Parse' changes, we never
   // do a HEAD, because we may need the content. Once we don't need to do this automatic store
   // any more, we can change the 'false' argument in the next line back to !body.
-  const current = await getObject(env, { org: daCtx.org, key: update.key }, false);
+  const tmpCtx = { ...daCtx, key: update.key };
+  const current = await getObject(env, tmpCtx, false);
   const id = current.metadata?.id || crypto.randomUUID();
   const version = current.metadata?.version || crypto.randomUUID();
   const users = JSON.stringify(daCtx.users);
@@ -127,6 +128,24 @@ export async function putObjectWithVersion(env, daCtx, update, body) {
   return 201;
 }
 
+/**
+ * Create a version of an object in its current state, with an optional label.
+ * @param {Object} env the CloudFlare environment
+ * @param {Object} daCtx the DA context
+ * @param {String} label the label for the version
+ * @return {Promise<{status: number}>} the response object
+ */
+export async function postObjectVersionWithLabel(env, daCtx, label) {
+  const { body, contentType } = await getObject(env, daCtx);
+  const { key } = daCtx;
+
+  const resp = await putObjectWithVersion(env, daCtx, {
+    key, body, type: contentType, label,
+  }, true);
+
+  return { status: resp };
+}
+
 /**
  * Create a version of an object in its current state, with an optional label.
  * @param {Request} req request object
@@ -143,12 +162,5 @@ export async function postObjectVersion(req, env, daCtx) {
   }
   const label = reqJSON?.label;
 
-  const { body, contentType } = await getObject(env, daCtx);
-  const { key } = daCtx;
-
-  const resp = await putObjectWithVersion(env, daCtx, {
-    key, body, type: contentType, label,
-  }, true);
-
-  return { status: resp };
+  return postObjectVersionWithLabel(env, daCtx, label);
 }

src/utils/daCtx.js

@@ -15,11 +15,12 @@ import { getUsers, isAuthorized } from './auth.js';
 /**
  * Gets Dark Alley Context
  * @param {Request} req the request object
- * @param env the Cloudflare environment context
+ * @param {Object} env the Cloudflare environment context
  * @returns {DaCtx} The Dark Alley Context.
  */
 export default async function getDaCtx(req, env) {
-  let { pathname } = new URL(req.url);
+  const url = new URL(req.url);
+  let { pathname } = url;
   // Remove proxied api route
   if (pathname.startsWith('/api')) pathname = pathname.replace('/api', '');
 
@@ -40,10 +41,12 @@ export default async function getDaCtx(req, env) {
   // Set base details
   const daCtx = {
     path: pathname,
+    origin: url.origin,
     api,
     org,
     users,
     fullKey,
+    initiator: req.headers.get('x-da-initiator'),
   };
 
   // Get org properties

test/it/delete.spec.js

@@ -32,11 +32,22 @@ describe('DELETE HTTP Requests',  () => {
     });
 
     it('handles existing file', async () => {
+      const before = await env.DA_CONTENT.get('wknd/index.html');
+      const id = before.customMetadata.id;
+      const input = {
+        prefix: `wknd/.da-versions/${id}/`,
+        delimiter: '/',
+      }
+      let list = await env.DA_CONTENT.list(input);
+      assert.strictEqual(list.objects.length, 0);
+
       const req = new Request('https://admin.da.live/source/wknd/index.html', { method: 'DELETE' });
       const resp = await worker.fetch(req, env);
       assert.strictEqual(resp.status, 204);
-      const obj = await env.DA_CONTENT.get('wknd/index.html');
-      assert.ifError(obj);
+      const after = await env.DA_CONTENT.get('wknd/index.html');
+      assert.ifError(after);
+      list = await env.DA_CONTENT.list(input);
+      assert.strictEqual(list.objects.length, 1);
     });
   });
 })

test/mocks/miniflare.js

@@ -35,6 +35,7 @@ const config = {
  * @return {Miniflare}
  */
 export async function getMiniflare() {
+  const collabCalls = [];
   const mf = new Miniflare({
     modules: true,
     // Need a script to initialize Miniflare
@@ -46,11 +47,7 @@ export async function getMiniflare() {
         }
       `,
     serviceBindings: {
-      dacollab() {
-        return {
-          fetch: () => { /* no-op fetch */ },
-        };
-      },
+      dacollab(request) { /* no op */ },
     },
     kvNamespaces: { DA_AUTH: 'DA_AUTH', DA_CONFIG: 'DA_CONFIG' },
     r2Buckets: { DA_CONTENT: 'DA_CONTENT' },