Skip to content

Commit

Permalink
feat: use url encoding type for list calls (#14010)
Browse files Browse the repository at this point in the history
  • Loading branch information
@pranavosu
pranavosu authored Nov 14, 2024
2 parents 405b2cc + 2736726 commit d8c0a68
Show file tree
Hide file tree
Showing 6 changed files with 362 additions and 48 deletions.
164 changes: 122 additions & 42 deletions packages/storage/__tests__/providers/s3/apis/internal/list.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -184,11 +184,11 @@ describe('list API', () => {
expect(listObjectsV2).toHaveBeenCalledTimes(1);
await expect(listObjectsV2).toBeLastCalledWithConfigAndInput(
listObjectClientConfig,
{
expect.objectContaining({
Bucket: bucket,
MaxKeys: 1000,
Prefix: expectedKey,
},
}),
);
});
});
Expand Down Expand Up @@ -226,12 +226,12 @@ describe('list API', () => {
expect(listObjectsV2).toHaveBeenCalledTimes(1);
await expect(listObjectsV2).toBeLastCalledWithConfigAndInput(
listObjectClientConfig,
{
expect.objectContaining({
Bucket: bucket,
Prefix: expectedKey,
ContinuationToken: nextToken,
MaxKeys: customPageSize,
},
}),
);
});
});
Expand Down Expand Up @@ -260,11 +260,11 @@ describe('list API', () => {
expect(listObjectsV2).toHaveBeenCalledTimes(1);
await expect(listObjectsV2).toBeLastCalledWithConfigAndInput(
listObjectClientConfig,
{
expect.objectContaining({
Bucket: bucket,
MaxKeys: 1000,
Prefix: expectedKey,
},
}),
);
});
});
Expand Down Expand Up @@ -297,23 +297,23 @@ describe('list API', () => {
await expect(listObjectsV2).toHaveBeenNthCalledWithConfigAndInput(
1,
listObjectClientConfig,
{
expect.objectContaining({
Bucket: bucket,
Prefix: expectedKey,
MaxKeys: 1000,
ContinuationToken: undefined,
},
}),
);
// last input receives TEST_TOKEN as the Continuation Token
await expect(listObjectsV2).toHaveBeenNthCalledWithConfigAndInput(
3,
listObjectClientConfig,
{
expect.objectContaining({
Bucket: bucket,
Prefix: expectedKey,
MaxKeys: 1000,
ContinuationToken: nextToken,
},
}),
);
});
},
Expand Down Expand Up @@ -348,11 +348,11 @@ describe('list API', () => {
region: mockRegion,
userAgentValue: expect.any(String),
},
{
expect.objectContaining({
Bucket: mockBucketName,
MaxKeys: 1000,
Prefix: `public/${inputKey}`,
},
}),
);
});

Expand Down Expand Up @@ -382,11 +382,11 @@ describe('list API', () => {
region,
userAgentValue: expect.any(String),
},
{
expect.objectContaining({
Bucket: bucket,
MaxKeys: 1000,
Prefix: `public/${inputKey}`,
},
}),
);
});
});
Expand Down Expand Up @@ -444,11 +444,11 @@ describe('list API', () => {
expect(listObjectsV2).toHaveBeenCalledTimes(1);
await expect(listObjectsV2).toBeLastCalledWithConfigAndInput(
listObjectClientConfig,
{
expect.objectContaining({
Bucket: bucket,
MaxKeys: 1000,
Prefix: resolvePath(inputPath),
},
}),
);
},
);
Expand Down Expand Up @@ -487,12 +487,12 @@ describe('list API', () => {
expect(listObjectsV2).toHaveBeenCalledTimes(1);
await expect(listObjectsV2).toBeLastCalledWithConfigAndInput(
listObjectClientConfig,
{
expect.objectContaining({
Bucket: bucket,
Prefix: resolvePath(inputPath),
ContinuationToken: nextToken,
MaxKeys: customPageSize,
},
}),
);
},
);
Expand All @@ -516,11 +516,11 @@ describe('list API', () => {
expect(listObjectsV2).toHaveBeenCalledTimes(1);
await expect(listObjectsV2).toBeLastCalledWithConfigAndInput(
listObjectClientConfig,
{
expect.objectContaining({
Bucket: bucket,
MaxKeys: 1000,
Prefix: resolvePath(path),
},
}),
);
},
);
Expand Down Expand Up @@ -552,23 +552,23 @@ describe('list API', () => {
await expect(listObjectsV2).toHaveBeenNthCalledWithConfigAndInput(
1,
listObjectClientConfig,
{
expect.objectContaining({
Bucket: bucket,
Prefix: resolvedPath,
MaxKeys: 1000,
ContinuationToken: undefined,
},
}),
);
// last input receives TEST_TOKEN as the Continuation Token
await expect(listObjectsV2).toHaveBeenNthCalledWithConfigAndInput(
3,
listObjectClientConfig,
{
expect.objectContaining({
Bucket: bucket,
Prefix: resolvedPath,
MaxKeys: 1000,
ContinuationToken: nextToken,
},
}),
);
},
);
Expand Down Expand Up @@ -602,11 +602,11 @@ describe('list API', () => {
region: mockRegion,
userAgentValue: expect.any(String),
},
{
expect.objectContaining({
Bucket: mockBucketName,
MaxKeys: 1000,
Prefix: 'path/',
},
}),
);
});

Expand Down Expand Up @@ -636,11 +636,11 @@ describe('list API', () => {
region,
userAgentValue: expect.any(String),
},
{
expect.objectContaining({
Bucket: bucket,
MaxKeys: 1000,
Prefix: 'path/',
},
}),
);
});
});
Expand All @@ -664,11 +664,11 @@ describe('list API', () => {
expect(listObjectsV2).toHaveBeenCalledTimes(1);
await expect(listObjectsV2).toBeLastCalledWithConfigAndInput(
listObjectClientConfig,
{
expect.objectContaining({
Bucket: bucket,
MaxKeys: 1000,
Prefix: 'public/',
},
}),
);
expect(error.$metadata.httpStatusCode).toBe(404);
}
Expand Down Expand Up @@ -772,12 +772,12 @@ describe('list API', () => {
expect(listObjectsV2).toHaveBeenCalledTimes(1);
await expect(listObjectsV2).toBeLastCalledWithConfigAndInput(
listObjectClientConfig,
{
expect.objectContaining({
Bucket: bucket,
MaxKeys: 1000,
Prefix: mockedPath,
Delimiter: '/',
},
}),
);
});

Expand Down Expand Up @@ -806,12 +806,12 @@ describe('list API', () => {
expect(listObjectsV2).toHaveBeenCalledTimes(1);
await expect(listObjectsV2).toBeLastCalledWithConfigAndInput(
listObjectClientConfig,
{
expect.objectContaining({
Bucket: bucket,
MaxKeys: 1000,
Prefix: mockedPath,
Delimiter: '/',
},
}),
);
});

Expand All @@ -828,12 +828,12 @@ describe('list API', () => {
expect(listObjectsV2).toHaveBeenCalledTimes(1);
await expect(listObjectsV2).toBeLastCalledWithConfigAndInput(
listObjectClientConfig,
{
expect.objectContaining({
Bucket: bucket,
MaxKeys: 3,
Prefix: mockedPath,
Delimiter: '/',
},
}),
);
});

Expand All @@ -850,12 +850,12 @@ describe('list API', () => {
expect(listObjectsV2).toHaveBeenCalledTimes(1);
await expect(listObjectsV2).toBeLastCalledWithConfigAndInput(
listObjectClientConfig,
{
expect.objectContaining({
Bucket: bucket,
MaxKeys: 1000,
Prefix: mockedPath,
Delimiter: '-',
},
}),
);
});

Expand All @@ -871,12 +871,12 @@ describe('list API', () => {
expect(listObjectsV2).toHaveBeenCalledTimes(1);
await expect(listObjectsV2).toBeLastCalledWithConfigAndInput(
listObjectClientConfig,
{
expect.objectContaining({
Bucket: bucket,
MaxKeys: 1000,
Prefix: mockedPath,
Delimiter: undefined,
},
}),
);
});

Expand All @@ -887,12 +887,12 @@ describe('list API', () => {
expect(listObjectsV2).toHaveBeenCalledTimes(1);
await expect(listObjectsV2).toBeLastCalledWithConfigAndInput(
listObjectClientConfig,
{
expect.objectContaining({
Bucket: bucket,
MaxKeys: 1000,
Prefix: mockedPath,
Delimiter: undefined,
},
}),
);
});
});
Expand Down Expand Up @@ -1024,4 +1024,84 @@ describe('list API', () => {
});
});
});

describe.each([
{
type: 'Prefix',
listFunction: (options?: any) =>
list(Amplify, {
prefix: 'some folder with unprintable unicode/',
options,
}),
key: 'key',
},
{
type: 'Path',
listFunction: (options?: any) =>
list(Amplify, {
path: 'public/some folder with unprintable unicode/',
options,
}),
key: 'path',
},
])('Encoding for List with $type', ({ listFunction, key }) => {
afterEach(() => {
mockListObject.mockClear();
});
it('should decode encoded list output', async () => {
const encodedBadKeys = [
'some+folder+with+spaces/',
'real%0A%0A%0A%0A%0A%0A%0A%0A%0Afunny%0A%0A%0A%0A%0A%0A%0A%0A%0Abiz',
'some+folder+with+%E3%81%8A%E3%81%AF%E3%82%88%E3%81%86+multibyte+unicode/',
'bad%3Cdiv%3Ekey',
'bad%00key',
'bad%01key',
];

mockListObject.mockReturnValueOnce({
Name: bucket,
Prefix: 'public/some+folder+with++unprintable+unicode/',
Delimiter: 'bad%08key',
MaxKeys: 1000,
StartAfter: 'bad%7Fbiz/',
EncodingType: 'url',
Contents: encodedBadKeys.map(badKey => ({
...listObjectClientBaseResultItem,
Key: key === 'key' ? `public/${badKey}` : badKey,
})),
});

const result = await listFunction({
subpathStrategy: { strategy: 'exclude', delimiter: 'bad\x08key' },
});

expect(listObjectsV2).toBeLastCalledWithConfigAndInput(
expect.any(Object),
expect.objectContaining({
Bucket: bucket,
EncodingType: 'url',
}),
);

const decodedKeys = [
'some folder with spaces/',
'real\x0a\x0a\x0a\x0a\x0a\x0a\x0a\x0a\x0afunny\x0a\x0a\x0a\x0a\x0a\x0a\x0a\x0a\x0abiz',
'some folder with おはよう multibyte unicode/',
'bad<div>key',
'bad\x00key',
'bad\x01key',
];

const expectedResult = {
items: decodedKeys.map(decodedKey => ({
[key]: decodedKey,
eTag: 'eTag',
lastModified: 'lastModified',
size: 'size',
})),
nextToken: undefined,
};
expect(result).toEqual(expectedResult);
});
});
});
Loading

0 comments on commit d8c0a68

Please sign in to comment.