Skip to content

Commit

Permalink
Avoid error when putting objects in R2/S3
Browse files Browse the repository at this point in the history
This fixes the following error that shows up in the da-admin console:
  [ERROR] Are you using a Stream of unknown length as the Body of a PutObject
  request? Consider using Upload instead from @aws-sdk/lib-storage.
  • Loading branch information
bosschaert committed May 10, 2024
1 parent 725b972 commit 508b94b
Show file tree
Hide file tree
Showing 4 changed files with 167 additions and 39 deletions.
141 changes: 108 additions & 33 deletions package-lock.json

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion package.json
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@
"eslint": "8.56.0",
"esmock": "^2.6.4",
"mocha": "^10.2.0",
"wrangler": "^3.17.1"
"wrangler": "^3.55.0"
},
"lint-staged": {
"*.js": "eslint",
Expand Down
34 changes: 29 additions & 5 deletions src/storage/version/put.js
Original file line number Diff line number Diff line change
Expand Up @@ -20,12 +20,34 @@ import {
} from '../utils/version.js';
import getObject from '../object/get.js';

export async function getBodyLength(body) {
if (body === undefined) {
return {};
}

let content;
if (Object.hasOwn(body, 'transformToString') && typeof body.transformToString === 'function') {
content = await body.transformToString();
} else {
content = body;
}

let length;
if (typeof content === 'string' || content instanceof String) {
// get string length in bytes
length = new Blob([content]).size;
}
return { content, length };
}

export async function putVersion(config, {
Bucket, Body, ID, Version, Ext, Metadata,
}, noneMatch = true) {
const { content, length } = await getBodyLength(Body);

const client = noneMatch ? ifNoneMatch(config) : createBucketIfMissing(new S3Client(config));
const input = {
Bucket, Key: `.da-versions/${ID}/${Version}.${Ext}`, Body, Metadata,
Bucket, Key: `.da-versions/${ID}/${Version}.${Ext}`, Body: content, Metadata, ContentLength: length,
};
const command = new PutObjectCommand(input);
try {
Expand All @@ -36,12 +58,14 @@ export async function putVersion(config, {
}
}

function buildInput({
async function buildInput({
org, key, body, type,
}) {
const { content, length } = await getBodyLength(body);

const Bucket = `${org}-content`;
return {
Bucket, Key: key, Body: body, ContentType: type,
Bucket, Key: key, Body: content, ContentType: type, ContentLength: length,
};
}

Expand All @@ -54,7 +78,7 @@ export async function postObjectVersion(req, env, daCtx) {
}

const config = getS3Config(env);
const update = buildInput(daCtx);
const update = await buildInput(daCtx);
const current = await getObject(env, daCtx);
if (current.status === 404 || !current.metadata?.id || !current.metadata?.version) {
return 404;
Expand Down Expand Up @@ -92,7 +116,7 @@ export async function putObjectWithVersion(env, daCtx, update, body) {
const ID = current.metadata?.id || crypto.randomUUID();
const Version = current.metadata?.version || crypto.randomUUID();
const Users = JSON.stringify(daCtx.users);
const input = buildInput(update);
const input = await buildInput(update);

Check warning on line 119 in src/storage/version/put.js

View check run for this annotation

Codecov / codecov/patch

src/storage/version/put.js#L119

Added line #L119 was not covered by tests
const Timestamp = `${Date.now()}`;
const Path = update.key;
if (current.status === 404) {
Expand Down
29 changes: 29 additions & 0 deletions test/storage/version/put.test.js
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,8 @@
import assert from 'assert';
import esmock from 'esmock';

import { getBodyLength } from '../../../src/storage/version/put.js';

describe('Version Put', () => {
it('Post Object Version', async () => {
const mockGetObject = async () => {
Expand Down Expand Up @@ -185,4 +187,31 @@ describe('Version Put', () => {
assert.equal('existing label', input.Metadata.Label);
assert.equal('/y/z', input.Metadata.Path);
});

it('Test getBodyLength String', async () => {
const {content, length} = await getBodyLength('hello');
assert.equal('hello', content);
assert.equal(5, length);
});

it('Test getBodyLength with Transform to String', async () => {
const stream = { transformToString: () => '🥳' };

const { content, length } = await getBodyLength(stream);
assert.equal('🥳', content);
assert.equal(4, length);
});

it('Test getBodyLength with unknown', async () => {
const body = {};
const { content, length } = await getBodyLength(body);
assert.equal(body, content);
assert.equal(undefined, length);
});

it('Test getBodyLength with undefined', async () => {
const { content, length } = await getBodyLength(undefined);
assert.equal(undefined, content);
assert.equal(undefined, length);
});
});

0 comments on commit 508b94b

Please sign in to comment.