-
Notifications
You must be signed in to change notification settings - Fork 9
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
GH-79 Copy and delete performance #92
Changes from all commits
c0e5dac
5a3dfef
82da50b
1c2fec2
7fc759f
7ff484b
1cf384f
56798d0
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,22 @@ | ||
/* | ||
* Copyright 2024 Adobe. All rights reserved. | ||
* This file is licensed to you under the Apache License, Version 2.0 (the "License"); | ||
* you may not use this file except in compliance with the License. You may obtain a copy | ||
* of the License at http://www.apache.org/licenses/LICENSE-2.0 | ||
* | ||
* Unless required by applicable law or agreed to in writing, software distributed under | ||
* the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS | ||
* OF ANY KIND, either express or implied. See the License for the specific language | ||
* governing permissions and limitations under the License. | ||
*/ | ||
|
||
export default async function deleteHelper(req) { | ||
try { | ||
const formData = await req.formData(); | ||
if (!formData) return {}; | ||
const continuationToken = formData.get('continuation-token'); | ||
return { continuationToken }; | ||
} catch { | ||
return {}; | ||
} | ||
} |
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -11,18 +11,13 @@ | |
*/ | ||
import { | ||
S3Client, | ||
ListObjectsV2Command, | ||
CopyObjectCommand, | ||
} from '@aws-sdk/client-s3'; | ||
|
||
import getS3Config from '../utils/config.js'; | ||
import { listCommand } from '../utils/list.js'; | ||
|
||
function buildInput(org, key) { | ||
return { | ||
Bucket: `${org}-content`, | ||
Prefix: `${key}/`, | ||
}; | ||
} | ||
const MAX_KEYS = 900; | ||
|
||
export const copyFile = async (client, daCtx, sourceKey, details, isRename) => { | ||
const Key = `${sourceKey.replace(details.source, details.destination)}`; | ||
|
@@ -51,46 +46,52 @@ | |
try { | ||
await client.send(new CopyObjectCommand(input)); | ||
} catch (e) { | ||
// eslint-disable-next-line no-console | ||
console.log(e.$metadata); | ||
console.log({ | ||
code: e.$metadata.httpStatusCode, | ||
dest: Key, | ||
src: `${daCtx.org}-content/${sourceKey}`, | ||
}); | ||
} | ||
}; | ||
|
||
export default async function copyObject(env, daCtx, details, isRename) { | ||
if (details.source === details.destination) { | ||
return { body: '', status: 409 }; | ||
} | ||
if (details.source === details.destination) return { body: '', status: 409 }; | ||
|
||
const config = getS3Config(env); | ||
const client = new S3Client(config); | ||
const input = buildInput(daCtx.org, details.source); | ||
|
||
let ContinuationToken; | ||
|
||
// The input prefix has a forward slash to prevent (drafts + drafts-new, etc.). | ||
// Which means the list will only pickup children. This adds to the initial list. | ||
const sourceKeys = [details.source, `${details.source}.props`]; | ||
|
||
do { | ||
try { | ||
const command = new ListObjectsV2Command({ ...input, ContinuationToken }); | ||
const resp = await client.send(command); | ||
|
||
const { Contents = [], NextContinuationToken } = resp; | ||
sourceKeys.push(...Contents.map(({ Key }) => Key)); | ||
ContinuationToken = NextContinuationToken; | ||
} catch (e) { | ||
return { body: '', status: 404 }; | ||
} | ||
} while (ContinuationToken); | ||
let sourceKeys; | ||
let remainingKeys = []; | ||
let continuationToken; | ||
|
||
await Promise.all( | ||
new Array(1).fill(null).map(async () => { | ||
while (sourceKeys.length) { | ||
await copyFile(client, daCtx, sourceKeys.pop(), details, isRename); | ||
try { | ||
if (details.continuationToken) { | ||
continuationToken = details.continuationToken; | ||
remainingKeys = await env.DA_JOBS.get(continuationToken, { type: 'json' }); | ||
sourceKeys = remainingKeys.splice(0, MAX_KEYS); | ||
} else { | ||
let resp = await listCommand(daCtx, details, client); | ||
sourceKeys = resp.sourceKeys; | ||
if (resp.continuationToken) { | ||
continuationToken = `copy-${details.source}-${details.destination}-${crypto.randomUUID()}`; | ||
while (resp.continuationToken) { | ||
resp = await listCommand(daCtx, { continuationToken: resp.continuationToken }, client); | ||
remainingKeys.push(...resp.sourceKeys); | ||
} | ||
} | ||
}), | ||
); | ||
} | ||
await Promise.all(sourceKeys.map(async (key) => { | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I know for the client-side operations @auniverseaway is now using a queue to limit the number of concurrent operations. Is that something that could be needed if a very large copy operation is happening? There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. As long as we stay under 1000 sub-requests, we should be fine. The Queue class on client-side is for when we have 2000+, want to send all immediately, and don't want the browser to choke with too many requests. |
||
await copyFile(client, daCtx, key, details, isRename); | ||
})); | ||
|
||
return { status: 204 }; | ||
if (remainingKeys.length) { | ||
await env.DA_JOBS.put(continuationToken, JSON.stringify(remainingKeys)); | ||
return { body: JSON.stringify({ continuationToken }), status: 206 }; | ||
} else if (continuationToken) { | ||
await env.DA_JOBS.delete(continuationToken); | ||
} | ||
return { status: 204 }; | ||
} catch (e) { | ||
return { body: '', status: 404 }; | ||
} | ||
} |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Why is the await commented out instead of deleted?
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
I do not know. I left it there as it was from the original variation of this work.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
I did that 😄
The reason is that I find it problematic that all of a sudden the linter wants you to lose the await on an async function if its called on
return
. If you change the function later and want to add some statements before returning you might forget to add theawait
at that point.So I just added the await in a comment as a reminder.
But if you don't like it feel free to remove 😄