From f176ab44ae282bde1626a8e674eb1129fd1f1ea3 Mon Sep 17 00:00:00 2001 From: David Bosschaert Date: Wed, 1 May 2024 13:03:54 +0100 Subject: [PATCH 1/8] Simple approach to automatically store the document body 'every now and then' when a version is created. --- src/storage/object/put.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/storage/object/put.js b/src/storage/object/put.js index ff84728..8f247ed 100644 --- a/src/storage/object/put.js +++ b/src/storage/object/put.js @@ -85,11 +85,13 @@ export default async function putObject(env, daCtx, obj) { let status = 201; if (obj) { if (obj.data) { + const storeBody = (Date.now() % 20 === 0); + const isFile = obj.data instanceof File; const { body, type } = isFile ? await getFileBody(obj.data) : getObjectBody(obj.data); status = await putObjectWithVersion(env, daCtx, { org, key, body, type, - }); + }, storeBody); } } else { const { body, type } = getObjectBody({}); From 72dd7cb9c6215d043ec6c5954328fa442adc9852 Mon Sep 17 00:00:00 2001 From: David Bosschaert Date: Fri, 10 May 2024 16:49:24 +0100 Subject: [PATCH 2/8] Store content every hour --- package-lock.json | 141 ++++++++++++++++++++++++++++--------- package.json | 2 +- src/storage/object/put.js | 4 +- src/storage/version/put.js | 13 +++- 4 files changed, 120 insertions(+), 40 deletions(-) diff --git a/package-lock.json b/package-lock.json index f80582a..f1df2ed 100644 --- a/package-lock.json +++ b/package-lock.json @@ -22,7 +22,7 @@ "eslint": "8.56.0", "esmock": "^2.6.4", "mocha": "^10.2.0", - "wrangler": "^3.17.1" + "wrangler": "^3.55.0" } }, "node_modules/@aashutoshrathi/word-wrap": { @@ -928,18 +928,37 @@ "dev": true }, "node_modules/@cloudflare/kv-asset-handler": { - "version": "0.2.0", - "resolved": "https://registry.npmjs.org/@cloudflare/kv-asset-handler/-/kv-asset-handler-0.2.0.tgz", - "integrity": "sha512-MVbXLbTcAotOPUj0pAMhVtJ+3/kFkwJqc5qNOleOZTv6QkZZABDMS21dSrSlVswEHwrpWC03e4fWytjqKvuE2A==", + "version": "0.3.2", + "resolved": "https://registry.npmjs.org/@cloudflare/kv-asset-handler/-/kv-asset-handler-0.3.2.tgz", + "integrity": "sha512-EeEjMobfuJrwoctj7FA1y1KEbM0+Q1xSjobIEyie9k4haVEBB7vkDvsasw1pM3rO39mL2akxIAzLMUAtrMHZhA==", "dev": true, "dependencies": { "mime": "^3.0.0" + }, + "engines": { + "node": ">=16.13" + } + }, + "node_modules/@cloudflare/workerd-darwin-64": { + "version": "1.20240419.0", + "resolved": "https://registry.npmjs.org/@cloudflare/workerd-darwin-64/-/workerd-darwin-64-1.20240419.0.tgz", + "integrity": "sha512-PGVe9sYWULHfvGhN0IZh8MsskNG/ufnBSqPbgFCxJHCTrVXLPuC35EoVaforyqjKRwj3U35XMyGo9KHcGnTeHQ==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=16" } }, "node_modules/@cloudflare/workerd-darwin-arm64": { - "version": "1.20240129.0", - "resolved": "https://registry.npmjs.org/@cloudflare/workerd-darwin-arm64/-/workerd-darwin-arm64-1.20240129.0.tgz", - "integrity": "sha512-t0q8ABkmumG1zRM/MZ/vIv/Ysx0vTAXnQAPy/JW5aeQi/tqrypXkO9/NhPc0jbF/g/hIPrWEqpDgEp3CB7Da7Q==", + "version": "1.20240419.0", + "resolved": "https://registry.npmjs.org/@cloudflare/workerd-darwin-arm64/-/workerd-darwin-arm64-1.20240419.0.tgz", + "integrity": "sha512-z4etQSPiD5Gcjs962LiC7ZdmXnN6SGof5KrYoFiSI9X9kUvpuGH/lnjVVPd+NnVNeDU2kzmcAIgyZjkjTaqVXQ==", "cpu": [ "arm64" ], @@ -952,6 +971,54 @@ "node": ">=16" } }, + "node_modules/@cloudflare/workerd-linux-64": { + "version": "1.20240419.0", + "resolved": "https://registry.npmjs.org/@cloudflare/workerd-linux-64/-/workerd-linux-64-1.20240419.0.tgz", + "integrity": "sha512-lBwhg0j3sYTFMsEb4bOClbVje8nqrYOu0H3feQlX+Eks94JIhWPkf8ywK4at/BUc1comPMhCgzDHwc2OMPUGgg==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=16" + } + }, + "node_modules/@cloudflare/workerd-linux-arm64": { + "version": "1.20240419.0", + "resolved": "https://registry.npmjs.org/@cloudflare/workerd-linux-arm64/-/workerd-linux-arm64-1.20240419.0.tgz", + "integrity": "sha512-ZMY6wwWkxL+WPq8ydOp/irSYjAnMhBz1OC1+4z+OANtDs2beaZODmq7LEB3hb5WUAaTPY7DIjZh3DfDfty0nYg==", + "cpu": [ + "arm64" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=16" + } + }, + "node_modules/@cloudflare/workerd-windows-64": { + "version": "1.20240419.0", + "resolved": "https://registry.npmjs.org/@cloudflare/workerd-windows-64/-/workerd-windows-64-1.20240419.0.tgz", + "integrity": "sha512-YJjgaJN2yGTkV7Cr4K3i8N4dUwVQTclT3Pr3NpRZCcLjTszwlE53++XXDnHMKGXBbSguIizaVbmcU2EtmIXyeQ==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=16" + } + }, "node_modules/@cspotcode/source-map-support": { "version": "0.8.1", "resolved": "https://registry.npmjs.org/@cspotcode/source-map-support/-/source-map-support-0.8.1.tgz", @@ -1118,9 +1185,9 @@ "dev": true }, "node_modules/@fastify/busboy": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/@fastify/busboy/-/busboy-2.1.0.tgz", - "integrity": "sha512-+KpH+QxZU7O4675t3mnkQKcZZg56u+K/Ct2K+N2AZYNVK8kyeo/bI18tI8aPm3tvNNRyTWfj6s5tnGNlcbQRsA==", + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/@fastify/busboy/-/busboy-2.1.1.tgz", + "integrity": "sha512-vBZP4NlzfOlerQTnba4aqZoMhE/a9HY7HRqoOPaETQcSQuWEIyZMHGfVu6w9wGtGK5fED5qRs2DteVCjOH60sA==", "dev": true, "engines": { "node": ">=14" @@ -4526,9 +4593,9 @@ } }, "node_modules/miniflare": { - "version": "3.20240129.0", - "resolved": "https://registry.npmjs.org/miniflare/-/miniflare-3.20240129.0.tgz", - "integrity": "sha512-27pDhlP2G/4gXmvnSt6LjMQ8KrkmbJElIQmn+BLjdiyIx+zXY4E8MSPJmi9flgf0dn3wtjuHO2ASenuopqqxrw==", + "version": "3.20240419.1", + "resolved": "https://registry.npmjs.org/miniflare/-/miniflare-3.20240419.1.tgz", + "integrity": "sha512-Q9n0W07uUD/u0c/b03E4iogeXOAMjZnE3P7B5Yi8sPaZAx6TYWwjurGBja+Pg2yILN2iMaliEobfVyAKss33cA==", "dev": true, "dependencies": { "@cspotcode/source-map-support": "0.8.1", @@ -4539,7 +4606,7 @@ "glob-to-regexp": "^0.4.1", "stoppable": "^1.1.0", "undici": "^5.28.2", - "workerd": "1.20240129.0", + "workerd": "1.20240419.0", "ws": "^8.11.0", "youch": "^3.2.2", "zod": "^3.20.6" @@ -4552,9 +4619,9 @@ } }, "node_modules/miniflare/node_modules/ws": { - "version": "8.16.0", - "resolved": "https://registry.npmjs.org/ws/-/ws-8.16.0.tgz", - "integrity": "sha512-HS0c//TP7Ina87TfiPUz1rQzMhHrl/SG2guqRcTOIUYD2q8uhUdNHZYJUaQ8aTGPzCh+c6oawMKW35nFl1dxyQ==", + "version": "8.17.0", + "resolved": "https://registry.npmjs.org/ws/-/ws-8.17.0.tgz", + "integrity": "sha512-uJq6108EgZMAl20KagGkzCKfMEjxmKvZHG7Tlq0Z6nOky7YF7aq4mOx6xK8TJ/i1LeK4Qus7INktacctDgY8Ow==", "dev": true, "engines": { "node": ">=10.0.0" @@ -6527,9 +6594,9 @@ "dev": true }, "node_modules/workerd": { - "version": "1.20240129.0", - "resolved": "https://registry.npmjs.org/workerd/-/workerd-1.20240129.0.tgz", - "integrity": "sha512-t4pnsmjjk/u+GdVDgH2M1AFmJaBUABshYK/vT/HNrAXsHSwN6VR8Yqw0JQ845OokO34VLkuUtYQYyxHHKpdtsw==", + "version": "1.20240419.0", + "resolved": "https://registry.npmjs.org/workerd/-/workerd-1.20240419.0.tgz", + "integrity": "sha512-9yV98KpkQgG+bdEsKEW8i1AYZgxns6NVSfdOVEB2Ue1pTMtIEYfUyqUE+O2amisRrfaC3Pw4EvjtTmVaoetfeg==", "dev": true, "hasInstallScript": true, "bin": { @@ -6539,11 +6606,11 @@ "node": ">=16" }, "optionalDependencies": { - "@cloudflare/workerd-darwin-64": "1.20240129.0", - "@cloudflare/workerd-darwin-arm64": "1.20240129.0", - "@cloudflare/workerd-linux-64": "1.20240129.0", - "@cloudflare/workerd-linux-arm64": "1.20240129.0", - "@cloudflare/workerd-windows-64": "1.20240129.0" + "@cloudflare/workerd-darwin-64": "1.20240419.0", + "@cloudflare/workerd-darwin-arm64": "1.20240419.0", + "@cloudflare/workerd-linux-64": "1.20240419.0", + "@cloudflare/workerd-linux-arm64": "1.20240419.0", + "@cloudflare/workerd-windows-64": "1.20240419.0" } }, "node_modules/workerpool": { @@ -6553,18 +6620,18 @@ "dev": true }, "node_modules/wrangler": { - "version": "3.26.0", - "resolved": "https://registry.npmjs.org/wrangler/-/wrangler-3.26.0.tgz", - "integrity": "sha512-2FKDyL0wV6ws+9AHkQl5/Yzn17kG9jlpgyT7wqCDkhb5q+TCL/I8N5IKVwXe8tRrTluBI1QQZRRymoA5nu0pHw==", + "version": "3.55.0", + "resolved": "https://registry.npmjs.org/wrangler/-/wrangler-3.55.0.tgz", + "integrity": "sha512-VhtCioKxOdVqkHa8jQ6C6bX3by2Ko0uM0DKzrA+6lBZvfDUlGDWSOPiG+1fOHBHj2JTVBntxWCztXP6L+Udr8w==", "dev": true, "dependencies": { - "@cloudflare/kv-asset-handler": "^0.2.0", + "@cloudflare/kv-asset-handler": "0.3.2", "@esbuild-plugins/node-globals-polyfill": "^0.2.3", "@esbuild-plugins/node-modules-polyfill": "^0.2.2", "blake3-wasm": "^2.1.5", "chokidar": "^3.5.3", "esbuild": "0.17.19", - "miniflare": "3.20240129.0", + "miniflare": "3.20240419.1", "nanoid": "^3.3.3", "path-to-regexp": "^6.2.0", "resolve": "^1.22.8", @@ -6582,6 +6649,14 @@ }, "optionalDependencies": { "fsevents": "~2.3.2" + }, + "peerDependencies": { + "@cloudflare/workers-types": "^4.20240419.0" + }, + "peerDependenciesMeta": { + "@cloudflare/workers-types": { + "optional": true + } } }, "node_modules/wrap-ansi": { @@ -6730,9 +6805,9 @@ } }, "node_modules/zod": { - "version": "3.22.4", - "resolved": "https://registry.npmjs.org/zod/-/zod-3.22.4.tgz", - "integrity": "sha512-iC+8Io04lddc+mVqQ9AZ7OQ2MrUKGN+oIQyq1vemgt46jwCwLfhq7/pwnBnNXXXZb8VTVLKwp9EDkx+ryxIWmg==", + "version": "3.23.8", + "resolved": "https://registry.npmjs.org/zod/-/zod-3.23.8.tgz", + "integrity": "sha512-XBx9AXhXktjUqnepgTiE5flcKIYWi/rme0Eaj+5Y0lftuGBq+jyRu/md4WnuxqgP1ubdpNCsYEYPxrzVHD8d6g==", "dev": true, "funding": { "url": "https://github.com/sponsors/colinhacks" diff --git a/package.json b/package.json index a13c275..0cbc2cf 100644 --- a/package.json +++ b/package.json @@ -27,7 +27,7 @@ "eslint": "8.56.0", "esmock": "^2.6.4", "mocha": "^10.2.0", - "wrangler": "^3.17.1" + "wrangler": "^3.55.0" }, "lint-staged": { "*.js": "eslint", diff --git a/src/storage/object/put.js b/src/storage/object/put.js index 8f247ed..2fe5010 100644 --- a/src/storage/object/put.js +++ b/src/storage/object/put.js @@ -85,13 +85,11 @@ export default async function putObject(env, daCtx, obj) { let status = 201; if (obj) { if (obj.data) { - const storeBody = (Date.now() % 20 === 0); - const isFile = obj.data instanceof File; const { body, type } = isFile ? await getFileBody(obj.data) : getObjectBody(obj.data); status = await putObjectWithVersion(env, daCtx, { org, key, body, type, - }, storeBody); + }, true); } } else { const { body, type } = getObjectBody({}); diff --git a/src/storage/version/put.js b/src/storage/version/put.js index 9c8f285..7ae4224 100644 --- a/src/storage/version/put.js +++ b/src/storage/version/put.js @@ -89,18 +89,25 @@ export async function putObjectWithVersion(env, daCtx, update, body) { const config = getS3Config(env); const current = await getObject(env, update, !body); + const lfs = current.metadata?.lastfullstore || 0; + + // Store only if we requested the body and the last full store was more than an hour ago + const storeBody = body && (Date.now() - lfs > 1000 * 60 * 60); + const ID = current.metadata?.id || crypto.randomUUID(); const Version = current.metadata?.version || crypto.randomUUID(); const Users = JSON.stringify(daCtx.users); const input = buildInput(update); const Timestamp = `${Date.now()}`; const Path = update.key; + const Lastfullstore = storeBody ? Timestamp : lfs; + if (current.status === 404) { const client = ifNoneMatch(config); const command = new PutObjectCommand({ ...input, Metadata: { - ID, Version, Users, Timestamp, Path, + ID, Version, Users, Timestamp, Path, Lastfullstore, }, }); try { @@ -116,7 +123,7 @@ export async function putObjectWithVersion(env, daCtx, update, body) { const versionResp = await putVersion(config, { Bucket: input.Bucket, - Body: current.body, + Body: storeBody ? current.body : '', ID, Version, Ext: daCtx.ext, @@ -135,7 +142,7 @@ export async function putObjectWithVersion(env, daCtx, update, body) { const command = new PutObjectCommand({ ...input, Metadata: { - ID, Version: crypto.randomUUID(), Users, Timestamp, Path, + ID, Version: crypto.randomUUID(), Users, Timestamp, Path, Lastfullstore, }, }); try { From 3bb5651a26227030a4769bff4aa7c6556a3ae197 Mon Sep 17 00:00:00 2001 From: David Bosschaert Date: Mon, 13 May 2024 11:12:00 +0100 Subject: [PATCH 3/8] Only update LastFullStore on version creation --- src/storage/version/put.js | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/storage/version/put.js b/src/storage/version/put.js index 7ae4224..3987017 100644 --- a/src/storage/version/put.js +++ b/src/storage/version/put.js @@ -89,25 +89,19 @@ export async function putObjectWithVersion(env, daCtx, update, body) { const config = getS3Config(env); const current = await getObject(env, update, !body); - const lfs = current.metadata?.lastfullstore || 0; - - // Store only if we requested the body and the last full store was more than an hour ago - const storeBody = body && (Date.now() - lfs > 1000 * 60 * 60); - const ID = current.metadata?.id || crypto.randomUUID(); const Version = current.metadata?.version || crypto.randomUUID(); const Users = JSON.stringify(daCtx.users); const input = buildInput(update); const Timestamp = `${Date.now()}`; const Path = update.key; - const Lastfullstore = storeBody ? Timestamp : lfs; if (current.status === 404) { const client = ifNoneMatch(config); const command = new PutObjectCommand({ ...input, Metadata: { - ID, Version, Users, Timestamp, Path, Lastfullstore, + ID, Version, Users, Timestamp, Path, }, }); try { @@ -121,9 +115,15 @@ export async function putObjectWithVersion(env, daCtx, update, body) { } } + const lfs = current.metadata?.lastfullstore || '0'; + + // Store only if we requested the body and the last full store was more than an hour ago + const storeBody = body && (Date.now() - lfs > 1000 * 60 * 60); + const Lastfullstore = storeBody ? Timestamp : lfs; + const versionResp = await putVersion(config, { Bucket: input.Bucket, - Body: storeBody ? current.body : '', + Body: (storeBody ? current.body : ''), ID, Version, Ext: daCtx.ext, From ed959e39b6ac85fd92f2693cc1709315510a2e72 Mon Sep 17 00:00:00 2001 From: David Bosschaert Date: Mon, 13 May 2024 12:32:26 +0100 Subject: [PATCH 4/8] Unit tests --- test/storage/version/put.test.js | 179 +++++++++++++++++++++++++++++++ 1 file changed, 179 insertions(+) diff --git a/test/storage/version/put.test.js b/test/storage/version/put.test.js index a6ca820..769da77 100644 --- a/test/storage/version/put.test.js +++ b/test/storage/version/put.test.js @@ -185,4 +185,183 @@ describe('Version Put', () => { assert.equal('existing label', input.Metadata.Label); assert.equal('/y/z', input.Metadata.Path); }); + + it('Put Object With Version store content', async () => { + const mockGetObject = async (e, u, h) => { + if (!h) { + return { + body: 'prevbody', + metadata: { + id: 'x123', + version: 'aaa-bbb', + }, + status: 200 + }; + } + } + + const s3VersionSent = []; + const mockS3VersionClient = { + send: (c) => { + s3VersionSent.push(c); + return { $metadata: { httpStatusCode: 200 } }; + } + }; + const mockIfNoneMatch = () => mockS3VersionClient; + + const s3Sent = []; + const mockS3Client = { + send: (c) => { + s3Sent.push(c); + return { $metadata: { httpStatusCode: 200 } }; + } + }; + const mockIfMatch = () => mockS3Client + + const { putObjectWithVersion } = await esmock('../../../src/storage/version/put.js', { + '../../../src/storage/object/get.js': { + default: mockGetObject + }, + '../../../src/storage/utils/version.js': { + ifNoneMatch: mockIfNoneMatch, + ifMatch: mockIfMatch, + }, + }); + + const env = {}; + const daCtx= { ext: 'html' }; + const update = { body: 'new-body', org: 'myorg', key: '/a/x.html' }; + const resp = await putObjectWithVersion(env, daCtx, update, true); + assert.equal(200, resp); + assert.equal(1, s3VersionSent.length); + assert.equal('prevbody', s3VersionSent[0].input.Body); + assert.equal('myorg-content', s3VersionSent[0].input.Bucket); + assert.equal('.da-versions/x123/aaa-bbb.html', s3VersionSent[0].input.Key); + assert.equal('[{"email":"anonymous"}]', s3VersionSent[0].input.Metadata.Users); + assert.equal('/a/x.html', s3VersionSent[0].input.Metadata.Path); + assert(s3VersionSent[0].input.Metadata.Timestamp > 0); + + assert.equal(1, s3Sent.length); + assert.equal('new-body', s3Sent[0].input.Body); + assert.equal('myorg-content', s3Sent[0].input.Bucket); + assert.equal('/a/x.html', s3Sent[0].input.Key); + assert.equal('x123', s3Sent[0].input.Metadata.ID); + assert.equal('/a/x.html', s3Sent[0].input.Metadata.Path); + assert.notEqual('aaa-bbb', s3Sent[0].input.Metadata.Version); + assert(s3Sent[0].input.Metadata.Timestamp > 0); + assert.equal(s3Sent[0].input.Metadata.Lastfullstore, s3Sent[0].input.Metadata.Timestamp); + }); + + it('Put Object With Version don\'t store content', async () => { + const mockGetObject = async (e, u, h) => { + if (!h) { + return { + body: 'prevbody', + metadata: { + id: 'q123-456', + lastfullstore: Date.now(), + version: 'ver123', + }, + status: 201 + }; + } + } + + const s3VersionSent = []; + const mockS3VersionClient = { + send: (c) => { + s3VersionSent.push(c); + return { $metadata: { httpStatusCode: 200 } }; + } + }; + const mockIfNoneMatch = () => mockS3VersionClient; + + const s3Sent = []; + const mockS3Client = { + send: (c) => { + s3Sent.push(c); + return { $metadata: { httpStatusCode: 202 } }; + } + }; + const mockIfMatch = () => mockS3Client + + const { putObjectWithVersion } = await esmock('../../../src/storage/version/put.js', { + '../../../src/storage/object/get.js': { + default: mockGetObject + }, + '../../../src/storage/utils/version.js': { + ifNoneMatch: mockIfNoneMatch, + ifMatch: mockIfMatch, + }, + }); + + const env = {}; + const daCtx= { ext: 'html', users: [{"email": "foo@acme.org"}, {"email": "bar@acme.org"}] }; + const update = { body: 'new-body', org: 'myorg', key: '/a/x.html' }; + const resp = await putObjectWithVersion(env, daCtx, update, true); + assert.equal(202, resp); + assert.equal(1, s3VersionSent.length); + assert.equal('', s3VersionSent[0].input.Body); + assert.equal('myorg-content', s3VersionSent[0].input.Bucket); + assert.equal('.da-versions/q123-456/ver123.html', s3VersionSent[0].input.Key); + assert.equal('[{"email":"anonymous"}]', s3VersionSent[0].input.Metadata.Users); + assert.equal('/a/x.html', s3VersionSent[0].input.Metadata.Path); + assert(s3VersionSent[0].input.Metadata.Timestamp > 0); + + assert.equal(1, s3Sent.length); + assert.equal('new-body', s3Sent[0].input.Body); + assert.equal('myorg-content', s3Sent[0].input.Bucket); + assert.equal('/a/x.html', s3Sent[0].input.Key); + assert.equal('q123-456', s3Sent[0].input.Metadata.ID); + assert.equal('/a/x.html', s3Sent[0].input.Metadata.Path); + assert.equal('[{\"email\":\"foo@acme.org\"},{\"email\":\"bar@acme.org\"}]', s3Sent[0].input.Metadata.Users); + assert.notEqual('aaa-bbb', s3Sent[0].input.Metadata.Version); + assert(s3Sent[0].input.Metadata.Timestamp > 0); + assert.equal(s3Sent[0].input.Metadata.Lastfullstore, s3Sent[0].input.Metadata.Timestamp); + }); + + it('Put First Object With Version', async () => { + const mockGetObject = async (e, u, h) => { + if (!h) { + return { + status: 404 + }; + } + } + + const s3Sent = []; + const mockS3Client = { + send: (c) => { + s3Sent.push(c); + return { + $metadata: { + httpStatusCode: 200 + } + }; + } + }; + const mockIfNoneMatch = () => mockS3Client; + + const { putObjectWithVersion } = await esmock('../../../src/storage/version/put.js', { + '../../../src/storage/object/get.js': { + default: mockGetObject + }, + '../../../src/storage/utils/version.js': { + ifNoneMatch: mockIfNoneMatch + }, + }); + + const env = {}; + const daCtx= {}; + const update = { org: 'myorg', key: '/a/b/c' }; + const resp = await putObjectWithVersion(env, daCtx, update, true); + assert.equal(201, resp); + + assert.equal(1, s3Sent.length); + assert.equal('myorg-content', s3Sent[0].input.Bucket); + assert(s3Sent[0].input.Metadata.ID); + assert.equal('/a/b/c', s3Sent[0].input.Metadata.Path); + assert(s3Sent[0].input.Metadata.Timestamp > 0); + assert(s3Sent[0].input.Metadata.Version); + }); }); From 20155630fbaef4151e5e0c80182e314a40f11645 Mon Sep 17 00:00:00 2001 From: David Bosschaert Date: Tue, 14 May 2024 16:26:01 +0100 Subject: [PATCH 5/8] Add stage env --- wrangler.toml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/wrangler.toml b/wrangler.toml index d96d934..3b87621 100644 --- a/wrangler.toml +++ b/wrangler.toml @@ -15,6 +15,11 @@ kv_namespaces = [ [env.stage] vars = { ENVIRONMENT = "stage", DA_COLLAB = "https://collab.da.live" } + +services = [ + { binding = "dacollab", service = "da-collab-stage" } +] + kv_namespaces = [ { binding = "DA_AUTH", id = "21693f3b20f54fcbb850ddc8947335ba" }, { binding = "DA_CONFIG", id = "c44cb8dc69f041dc87c5aaef41b97df9" } From 7befd96d16170caf893ef8fd3ceb772208c0ca31 Mon Sep 17 00:00:00 2001 From: David Bosschaert Date: Mon, 20 May 2024 11:40:47 +0100 Subject: [PATCH 6/8] Do a one-off store to prepare for the parsing changes --- package-lock.json | 66 +++++++++++++++++++------------------- package.json | 2 +- src/storage/version/put.js | 10 +++--- 3 files changed, 39 insertions(+), 39 deletions(-) diff --git a/package-lock.json b/package-lock.json index f1df2ed..5c77de8 100644 --- a/package-lock.json +++ b/package-lock.json @@ -22,7 +22,7 @@ "eslint": "8.56.0", "esmock": "^2.6.4", "mocha": "^10.2.0", - "wrangler": "^3.55.0" + "wrangler": "^3.57.0" } }, "node_modules/@aashutoshrathi/word-wrap": { @@ -940,9 +940,9 @@ } }, "node_modules/@cloudflare/workerd-darwin-64": { - "version": "1.20240419.0", - "resolved": "https://registry.npmjs.org/@cloudflare/workerd-darwin-64/-/workerd-darwin-64-1.20240419.0.tgz", - "integrity": "sha512-PGVe9sYWULHfvGhN0IZh8MsskNG/ufnBSqPbgFCxJHCTrVXLPuC35EoVaforyqjKRwj3U35XMyGo9KHcGnTeHQ==", + "version": "1.20240512.0", + "resolved": "https://registry.npmjs.org/@cloudflare/workerd-darwin-64/-/workerd-darwin-64-1.20240512.0.tgz", + "integrity": "sha512-VMp+CsSHFALQiBzPdQ5dDI4T1qwLu0mQ0aeKVNDosXjueN0f3zj/lf+mFil5/9jBbG3t4mG0y+6MMnalP9Lobw==", "cpu": [ "x64" ], @@ -956,9 +956,9 @@ } }, "node_modules/@cloudflare/workerd-darwin-arm64": { - "version": "1.20240419.0", - "resolved": "https://registry.npmjs.org/@cloudflare/workerd-darwin-arm64/-/workerd-darwin-arm64-1.20240419.0.tgz", - "integrity": "sha512-z4etQSPiD5Gcjs962LiC7ZdmXnN6SGof5KrYoFiSI9X9kUvpuGH/lnjVVPd+NnVNeDU2kzmcAIgyZjkjTaqVXQ==", + "version": "1.20240512.0", + "resolved": "https://registry.npmjs.org/@cloudflare/workerd-darwin-arm64/-/workerd-darwin-arm64-1.20240512.0.tgz", + "integrity": "sha512-lZktXGmzMrB5rJqY9+PmnNfv1HKlj/YLZwMjPfF0WVKHUFdvQbAHsi7NlKv6mW9uIvlZnS+K4sIkWc0MDXcRnA==", "cpu": [ "arm64" ], @@ -972,9 +972,9 @@ } }, "node_modules/@cloudflare/workerd-linux-64": { - "version": "1.20240419.0", - "resolved": "https://registry.npmjs.org/@cloudflare/workerd-linux-64/-/workerd-linux-64-1.20240419.0.tgz", - "integrity": "sha512-lBwhg0j3sYTFMsEb4bOClbVje8nqrYOu0H3feQlX+Eks94JIhWPkf8ywK4at/BUc1comPMhCgzDHwc2OMPUGgg==", + "version": "1.20240512.0", + "resolved": "https://registry.npmjs.org/@cloudflare/workerd-linux-64/-/workerd-linux-64-1.20240512.0.tgz", + "integrity": "sha512-wrHvqCZZqXz6Y3MUTn/9pQNsvaoNjbJpuA6vcXsXu8iCzJi911iVW2WUEBX+MpUWD+mBIP0oXni5tTlhkokOPw==", "cpu": [ "x64" ], @@ -988,9 +988,9 @@ } }, "node_modules/@cloudflare/workerd-linux-arm64": { - "version": "1.20240419.0", - "resolved": "https://registry.npmjs.org/@cloudflare/workerd-linux-arm64/-/workerd-linux-arm64-1.20240419.0.tgz", - "integrity": "sha512-ZMY6wwWkxL+WPq8ydOp/irSYjAnMhBz1OC1+4z+OANtDs2beaZODmq7LEB3hb5WUAaTPY7DIjZh3DfDfty0nYg==", + "version": "1.20240512.0", + "resolved": "https://registry.npmjs.org/@cloudflare/workerd-linux-arm64/-/workerd-linux-arm64-1.20240512.0.tgz", + "integrity": "sha512-YPezHMySL9J9tFdzxz390eBswQ//QJNYcZolz9Dgvb3FEfdpK345cE/bsWbMOqw5ws2f82l388epoenghtYvAg==", "cpu": [ "arm64" ], @@ -1004,9 +1004,9 @@ } }, "node_modules/@cloudflare/workerd-windows-64": { - "version": "1.20240419.0", - "resolved": "https://registry.npmjs.org/@cloudflare/workerd-windows-64/-/workerd-windows-64-1.20240419.0.tgz", - "integrity": "sha512-YJjgaJN2yGTkV7Cr4K3i8N4dUwVQTclT3Pr3NpRZCcLjTszwlE53++XXDnHMKGXBbSguIizaVbmcU2EtmIXyeQ==", + "version": "1.20240512.0", + "resolved": "https://registry.npmjs.org/@cloudflare/workerd-windows-64/-/workerd-windows-64-1.20240512.0.tgz", + "integrity": "sha512-SxKapDrIYSscMR7lGIp/av0l6vokjH4xQ9ACxHgXh+OdOus9azppSmjaPyw4/ePvg7yqpkaNjf9o258IxWtvKQ==", "cpu": [ "x64" ], @@ -4593,9 +4593,9 @@ } }, "node_modules/miniflare": { - "version": "3.20240419.1", - "resolved": "https://registry.npmjs.org/miniflare/-/miniflare-3.20240419.1.tgz", - "integrity": "sha512-Q9n0W07uUD/u0c/b03E4iogeXOAMjZnE3P7B5Yi8sPaZAx6TYWwjurGBja+Pg2yILN2iMaliEobfVyAKss33cA==", + "version": "3.20240512.0", + "resolved": "https://registry.npmjs.org/miniflare/-/miniflare-3.20240512.0.tgz", + "integrity": "sha512-X0PlKR0AROKpxFoJNmRtCMIuJxj+ngEcyTOlEokj2rAQ0TBwUhB4/1uiPvdI6ofW5NugPOD1uomAv+gLjwsLDQ==", "dev": true, "dependencies": { "@cspotcode/source-map-support": "0.8.1", @@ -4606,7 +4606,7 @@ "glob-to-regexp": "^0.4.1", "stoppable": "^1.1.0", "undici": "^5.28.2", - "workerd": "1.20240419.0", + "workerd": "1.20240512.0", "ws": "^8.11.0", "youch": "^3.2.2", "zod": "^3.20.6" @@ -6594,9 +6594,9 @@ "dev": true }, "node_modules/workerd": { - "version": "1.20240419.0", - "resolved": "https://registry.npmjs.org/workerd/-/workerd-1.20240419.0.tgz", - "integrity": "sha512-9yV98KpkQgG+bdEsKEW8i1AYZgxns6NVSfdOVEB2Ue1pTMtIEYfUyqUE+O2amisRrfaC3Pw4EvjtTmVaoetfeg==", + "version": "1.20240512.0", + "resolved": "https://registry.npmjs.org/workerd/-/workerd-1.20240512.0.tgz", + "integrity": "sha512-VUBmR1PscAPHEE0OF/G2K7/H1gnr9aDWWZzdkIgWfNKkv8dKFCT75H+GJtUHjfwqz3rYCzaNZmatSXOpLGpF8A==", "dev": true, "hasInstallScript": true, "bin": { @@ -6606,11 +6606,11 @@ "node": ">=16" }, "optionalDependencies": { - "@cloudflare/workerd-darwin-64": "1.20240419.0", - "@cloudflare/workerd-darwin-arm64": "1.20240419.0", - "@cloudflare/workerd-linux-64": "1.20240419.0", - "@cloudflare/workerd-linux-arm64": "1.20240419.0", - "@cloudflare/workerd-windows-64": "1.20240419.0" + "@cloudflare/workerd-darwin-64": "1.20240512.0", + "@cloudflare/workerd-darwin-arm64": "1.20240512.0", + "@cloudflare/workerd-linux-64": "1.20240512.0", + "@cloudflare/workerd-linux-arm64": "1.20240512.0", + "@cloudflare/workerd-windows-64": "1.20240512.0" } }, "node_modules/workerpool": { @@ -6620,9 +6620,9 @@ "dev": true }, "node_modules/wrangler": { - "version": "3.55.0", - "resolved": "https://registry.npmjs.org/wrangler/-/wrangler-3.55.0.tgz", - "integrity": "sha512-VhtCioKxOdVqkHa8jQ6C6bX3by2Ko0uM0DKzrA+6lBZvfDUlGDWSOPiG+1fOHBHj2JTVBntxWCztXP6L+Udr8w==", + "version": "3.57.0", + "resolved": "https://registry.npmjs.org/wrangler/-/wrangler-3.57.0.tgz", + "integrity": "sha512-izK3AZtlFoTq8N0EZjLOQ7hqwsjaXCc1cbNKuhsLJjDX1jB1YZBDPhIhtXL4VVzkJAcH+0Zw2gguOePFCHNaxw==", "dev": true, "dependencies": { "@cloudflare/kv-asset-handler": "0.3.2", @@ -6631,7 +6631,7 @@ "blake3-wasm": "^2.1.5", "chokidar": "^3.5.3", "esbuild": "0.17.19", - "miniflare": "3.20240419.1", + "miniflare": "3.20240512.0", "nanoid": "^3.3.3", "path-to-regexp": "^6.2.0", "resolve": "^1.22.8", @@ -6651,7 +6651,7 @@ "fsevents": "~2.3.2" }, "peerDependencies": { - "@cloudflare/workers-types": "^4.20240419.0" + "@cloudflare/workers-types": "^4.20240512.0" }, "peerDependenciesMeta": { "@cloudflare/workers-types": { diff --git a/package.json b/package.json index 0cbc2cf..217538a 100644 --- a/package.json +++ b/package.json @@ -27,7 +27,7 @@ "eslint": "8.56.0", "esmock": "^2.6.4", "mocha": "^10.2.0", - "wrangler": "^3.55.0" + "wrangler": "^3.57.0" }, "lint-staged": { "*.js": "eslint", diff --git a/src/storage/version/put.js b/src/storage/version/put.js index 3987017..204a9cf 100644 --- a/src/storage/version/put.js +++ b/src/storage/version/put.js @@ -115,11 +115,11 @@ export async function putObjectWithVersion(env, daCtx, update, body) { } } - const lfs = current.metadata?.lastfullstore || '0'; + const pps = current.metadata?.preparsingstore || '0'; - // Store only if we requested the body and the last full store was more than an hour ago - const storeBody = body && (Date.now() - lfs > 1000 * 60 * 60); - const Lastfullstore = storeBody ? Timestamp : lfs; + // Store the body if preparsingstore is not defined, so a once-off store + const storeBody = body && pps === '0'; + const Preparsingstore = storeBody ? Timestamp : pps; const versionResp = await putVersion(config, { Bucket: input.Bucket, @@ -142,7 +142,7 @@ export async function putObjectWithVersion(env, daCtx, update, body) { const command = new PutObjectCommand({ ...input, Metadata: { - ID, Version: crypto.randomUUID(), Users, Timestamp, Path, Lastfullstore, + ID, Version: crypto.randomUUID(), Users, Timestamp, Path, Preparsingstore, }, }); try { From 4ebccc49e1841ae0ec7fb074147f99a4c8ae2d50 Mon Sep 17 00:00:00 2001 From: David Bosschaert Date: Mon, 20 May 2024 11:59:50 +0100 Subject: [PATCH 7/8] Update test --- test/storage/version/put.test.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/storage/version/put.test.js b/test/storage/version/put.test.js index 769da77..0fe0887 100644 --- a/test/storage/version/put.test.js +++ b/test/storage/version/put.test.js @@ -249,7 +249,7 @@ describe('Version Put', () => { assert.equal('/a/x.html', s3Sent[0].input.Metadata.Path); assert.notEqual('aaa-bbb', s3Sent[0].input.Metadata.Version); assert(s3Sent[0].input.Metadata.Timestamp > 0); - assert.equal(s3Sent[0].input.Metadata.Lastfullstore, s3Sent[0].input.Metadata.Timestamp); + assert.equal(s3Sent[0].input.Metadata.Preparsingstore, s3Sent[0].input.Metadata.Timestamp); }); it('Put Object With Version don\'t store content', async () => { @@ -259,7 +259,7 @@ describe('Version Put', () => { body: 'prevbody', metadata: { id: 'q123-456', - lastfullstore: Date.now(), + preparsingstore: Date.now(), version: 'ver123', }, status: 201 @@ -317,7 +317,7 @@ describe('Version Put', () => { assert.equal('[{\"email\":\"foo@acme.org\"},{\"email\":\"bar@acme.org\"}]', s3Sent[0].input.Metadata.Users); assert.notEqual('aaa-bbb', s3Sent[0].input.Metadata.Version); assert(s3Sent[0].input.Metadata.Timestamp > 0); - assert.equal(s3Sent[0].input.Metadata.Lastfullstore, s3Sent[0].input.Metadata.Timestamp); + assert.equal(s3Sent[0].input.Metadata.Preparsingstore, s3Sent[0].input.Metadata.Timestamp); }); it('Put First Object With Version', async () => { From 7b65790a526e227ecd6cff424014aeeb081bd22d Mon Sep 17 00:00:00 2001 From: David Bosschaert Date: Mon, 27 May 2024 16:37:50 +0100 Subject: [PATCH 8/8] Label the pre-parsing store version --- src/storage/version/put.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/storage/version/put.js b/src/storage/version/put.js index e6dd406..8d199b0 100644 --- a/src/storage/version/put.js +++ b/src/storage/version/put.js @@ -120,6 +120,7 @@ export async function putObjectWithVersion(env, daCtx, update, body) { // Store the body if preparsingstore is not defined, so a once-off store const storeBody = body && pps === '0'; const Preparsingstore = storeBody ? Timestamp : pps; + const Label = storeBody ? 'Collab Parse' : undefined; const versionResp = await putVersion(config, { Bucket: input.Bucket, @@ -131,6 +132,7 @@ export async function putObjectWithVersion(env, daCtx, update, body) { Users: current.metadata?.users || JSON.stringify([{ email: 'anonymous' }]), Timestamp: current.metadata?.timestamp || Timestamp, Path: current.metadata?.path || Path, + Label, }, });