Skip to content

Commit

Permalink
feat(storage): add customEndpoint to internal apis in advanced options (
Browse files Browse the repository at this point in the history
#13961)

* feat: add baseEndpoint to advanced options

* feat: add baseEndpoint to customEndpoint

* feat: thread baseEndpoint through resolved config to endpoint resolver

* add customEndpoint advanced option to internals storage data-plane apis

* add customEndpoint advanced option to internals storage control-plane apis

* fix unit test

* code cleanup

* increase bundle size

* wire up customEndpoint on copy API

* increase the bundle size

* add customEndpoint unit tests for all data and control apis

* increase bundle size

* update ts docs

* add additional error unit tests for endpointResolver

* add unit tests for internals/ apis

* code cleanup

* address feedback

* add comment for ForcePathStyleEndpointNotSupported ErrorCode

* increase bundle size

* remove docs links from error recovery message

---------

Co-authored-by: Erin Beal <[email protected]>
Co-authored-by: Ashwin Kumar <[email protected]>
  • Loading branch information
3 people authored Nov 5, 2024
1 parent 4ff7b52 commit c12f2e0
Show file tree
Hide file tree
Showing 38 changed files with 690 additions and 74 deletions.
14 changes: 7 additions & 7 deletions packages/aws-amplify/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -461,43 +461,43 @@
"name": "[Storage] copy (S3)",
"path": "./dist/esm/storage/index.mjs",
"import": "{ copy }",
"limit": "16.15 kB"
"limit": "16.39 kB"
},
{
"name": "[Storage] downloadData (S3)",
"path": "./dist/esm/storage/index.mjs",
"import": "{ downloadData }",
"limit": "16.48 kB"
"limit": "16.73 kB"
},
{
"name": "[Storage] getProperties (S3)",
"path": "./dist/esm/storage/index.mjs",
"import": "{ getProperties }",
"limit": "15.72 kB"
"limit": "15.99 kB"
},
{
"name": "[Storage] getUrl (S3)",
"path": "./dist/esm/storage/index.mjs",
"import": "{ getUrl }",
"limit": "16.98 kB"
"limit": "17.22 kB"
},
{
"name": "[Storage] list (S3)",
"path": "./dist/esm/storage/index.mjs",
"import": "{ list }",
"limit": "16.45 kB"
"limit": "16.69 kB"
},
{
"name": "[Storage] remove (S3)",
"path": "./dist/esm/storage/index.mjs",
"import": "{ remove }",
"limit": "15.59 kB"
"limit": "15.83 kB"
},
{
"name": "[Storage] uploadData (S3)",
"path": "./dist/esm/storage/index.mjs",
"import": "{ uploadData }",
"limit": "22.56 kB"
"limit": "22.81 kB"
}
]
}
2 changes: 2 additions & 0 deletions packages/storage/__tests__/internals/apis/copy.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ describe('copy (internals)', () => {
});

it('should pass advanced option locationCredentialsProvider to internal list', async () => {
const customEndpoint = 's3.dualstack.us-east-2.amazonaws.com';
const locationCredentialsProvider = async () => ({
credentials: {
accessKeyId: 'akid',
Expand All @@ -40,6 +41,7 @@ describe('copy (internals)', () => {
},
options: {
locationCredentialsProvider,
customEndpoint,
},
};
const result = await advancedCopy(copyInputWithAdvancedOptions);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@ describe('downloadData (internal)', () => {
const useAccelerateEndpoint = true;
const expectedBucketOwner = '012345678901';
const bucket = { bucketName: 'bucket', region: 'us-east-1' };
const customEndpoint = 's3.dualstack.us-east-2.amazonaws.com';
const locationCredentialsProvider = async () => ({
credentials: {
accessKeyId: 'akid',
Expand All @@ -45,6 +46,7 @@ describe('downloadData (internal)', () => {
const output = await advancedDownloadData({
path: 'input/path/to/mock/object',
options: {
customEndpoint,
useAccelerateEndpoint,
bucket,
locationCredentialsProvider,
Expand All @@ -58,6 +60,7 @@ describe('downloadData (internal)', () => {
expect(mockedDownloadDataInternal).toHaveBeenCalledWith({
path: 'input/path/to/mock/object',
options: {
customEndpoint,
useAccelerateEndpoint,
bucket,
locationCredentialsProvider,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -31,10 +31,11 @@ const MOCK_ACCESS_CREDENTIALS = {
SessionToken: MOCK_SESSION_TOKEN,
Expiration: MOCK_EXPIRATION_DATE,
};
const MOCK_CUSTOM_ENDPOINT = 's3-accesspoint.dualstack.us-east-2.amazonaws.com';
const MOCK_CREDENTIAL_PROVIDER = jest.fn().mockResolvedValue(MOCK_CREDENTIALS);

const sharedGetDataAccessParams: GetDataAccessInput = {
accountId: MOCK_ACCOUNT_ID,
customEndpoint: MOCK_CUSTOM_ENDPOINT,
credentialsProvider: MOCK_CREDENTIAL_PROVIDER,
durationSeconds: 900,
permission: 'READWRITE',
Expand Down Expand Up @@ -62,6 +63,7 @@ describe('getDataAccess', () => {
expect(getDataAccessClientMock).toHaveBeenCalledWith(
expect.objectContaining({
credentials: expect.any(Function),
customEndpoint: MOCK_CUSTOM_ENDPOINT,
region: MOCK_REGION,
userAgentValue: expect.stringContaining('storage/8'),
}),
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ describe('getProperties (internal)', () => {
const useAccelerateEndpoint = true;
const expectedBucketOwner = '012345678901';
const bucket = { bucketName: 'bucket', region: 'us-east-1' };
const customEndpoint = 's3.dualstack.us-east-2.amazonaws.com';
const locationCredentialsProvider = async () => ({
credentials: {
accessKeyId: 'akid',
Expand All @@ -34,6 +35,7 @@ describe('getProperties (internal)', () => {
const result = await advancedGetProperties({
path: 'input/path/to/mock/object',
options: {
customEndpoint,
useAccelerateEndpoint,
bucket,
expectedBucketOwner,
Expand All @@ -46,6 +48,7 @@ describe('getProperties (internal)', () => {
{
path: 'input/path/to/mock/object',
options: {
customEndpoint,
useAccelerateEndpoint,
bucket,
expectedBucketOwner,
Expand Down
3 changes: 3 additions & 0 deletions packages/storage/__tests__/internals/apis/getUrl.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@ describe('getUrl (internal)', () => {
const contentDisposition = 'inline; filename="example.jpg"';
const contentType = 'image/jpeg';
const bucket = { bucketName: 'bucket', region: 'us-east-1' };
const customEndpoint = 's3.dualstack.us-east-2.amazonaws.com';
const locationCredentialsProvider = async () => ({
credentials: {
accessKeyId: 'akid',
Expand All @@ -43,6 +44,7 @@ describe('getUrl (internal)', () => {
const result = await advancedGetUrl({
path: 'input/path/to/mock/object',
options: {
customEndpoint,
useAccelerateEndpoint,
bucket,
validateObjectExistence,
Expand All @@ -59,6 +61,7 @@ describe('getUrl (internal)', () => {
{
path: 'input/path/to/mock/object',
options: {
customEndpoint,
useAccelerateEndpoint,
bucket,
validateObjectExistence,
Expand Down
3 changes: 3 additions & 0 deletions packages/storage/__tests__/internals/apis/list.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ describe('list (internals)', () => {
const useAccelerateEndpoint = true;
const expectedBucketOwner = '012345678901';
const bucket = { bucketName: 'bucket', region: 'us-east-1' };
const customEndpoint = 's3.dualstack.us-east-2.amazonaws.com';
const locationCredentialsProvider = async () => ({
credentials: {
accessKeyId: 'akid',
Expand All @@ -31,6 +32,7 @@ describe('list (internals)', () => {
const result = await advancedList({
path: 'input/path/to/mock/object',
options: {
customEndpoint,
useAccelerateEndpoint,
bucket,
expectedBucketOwner,
Expand All @@ -43,6 +45,7 @@ describe('list (internals)', () => {
{
path: 'input/path/to/mock/object',
options: {
customEndpoint,
useAccelerateEndpoint,
bucket,
expectedBucketOwner,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ const mockCredentialsProvider = jest
.mockResolvedValue({ credentials: mockCredentials });
const mockNextToken = '123';
const mockPageSize = 123;
const mockCustomEndpoint = 's3-accesspoint.dualstack.us-east-2.amazonaws.com';

describe('listCallerAccessGrants', () => {
afterEach(() => {
Expand All @@ -36,6 +37,7 @@ describe('listCallerAccessGrants', () => {
});
await listCallerAccessGrants({
accountId: mockAccountId,
customEndpoint: mockCustomEndpoint,
region: mockRegion,
credentialsProvider: mockCredentialsProvider,
nextToken: mockNextToken,
Expand All @@ -45,6 +47,7 @@ describe('listCallerAccessGrants', () => {
expect.objectContaining({
region: mockRegion,
credentials: expect.any(Function),
customEndpoint: mockCustomEndpoint,
}),
expect.objectContaining({
AccountId: mockAccountId,
Expand Down
3 changes: 3 additions & 0 deletions packages/storage/__tests__/internals/apis/remove.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ describe('remove (internal)', () => {
const useAccelerateEndpoint = true;
const expectedBucketOwner = '012345678901';
const bucket = { bucketName: 'bucket', region: 'us-east-1' };
const customEndpoint = 's3.dualstack.us-east-2.amazonaws.com';
const locationCredentialsProvider = async () => ({
credentials: {
accessKeyId: 'akid',
Expand All @@ -35,6 +36,7 @@ describe('remove (internal)', () => {
const result = await advancedRemove({
path: 'input/path/to/mock/object',
options: {
customEndpoint,
useAccelerateEndpoint,
bucket,
expectedBucketOwner,
Expand All @@ -48,6 +50,7 @@ describe('remove (internal)', () => {
{
path: 'input/path/to/mock/object',
options: {
customEndpoint,
useAccelerateEndpoint,
bucket,
expectedBucketOwner,
Expand Down
4 changes: 4 additions & 0 deletions packages/storage/__tests__/internals/apis/uploadData.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,8 @@ describe('uploadData (internal)', () => {
const useAccelerateEndpoint = true;
const expectedBucketOwner = '012345678901';
const bucket = { bucketName: 'bucket', region: 'us-east-1' };
const customEndpoint = 's3.dualstack.us-east-2.amazonaws.com';

const locationCredentialsProvider = async () => ({
credentials: {
accessKeyId: 'akid',
Expand All @@ -37,6 +39,7 @@ describe('uploadData (internal)', () => {
path: 'input/path/to/mock/object',
data: 'data',
options: {
customEndpoint,
useAccelerateEndpoint,
bucket,
locationCredentialsProvider,
Expand All @@ -54,6 +57,7 @@ describe('uploadData (internal)', () => {
path: 'input/path/to/mock/object',
data: 'data',
options: {
customEndpoint,
useAccelerateEndpoint,
bucket,
locationCredentialsProvider,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -42,4 +42,38 @@ const abortMultipartUploadHappyCase: ApiFunctionalTestCase<
},
];

export default [abortMultipartUploadHappyCase];
const abortMultipartUploadHappyCaseCustomEndpoint: ApiFunctionalTestCase<
typeof abortMultipartUpload
> = [
'happy case',
'abortMultipartUpload with custom endpoint',
abortMultipartUpload,
{
...defaultConfig,
customEndpoint: 'custom.endpoint.com',
forcePathStyle: true,
},
{
Bucket: 'bucket',
Key: 'key',
UploadId: 'uploadId',
},
expect.objectContaining({
url: expect.objectContaining({
href: 'https://custom.endpoint.com/bucket/key?uploadId=uploadId',
}),
}),
{
status: 204,
headers: DEFAULT_RESPONSE_HEADERS,
body: '',
},
expect.objectContaining({
/** skip validating response */
}) as any,
];

export default [
abortMultipartUploadHappyCase,
abortMultipartUploadHappyCaseCustomEndpoint,
];
Original file line number Diff line number Diff line change
Expand Up @@ -81,7 +81,6 @@ const completeMultipartUploadHappyCase: ApiFunctionalTestCase<
},
];

// API reference: https://docs.aws.amazon.com/AmazonS3/latest/API/API_CompleteMultipartUpload.html
const completeMultipartUploadHappyCaseIfNoneMatch: ApiFunctionalTestCase<
typeof completeMultipartUpload
> = [
Expand All @@ -104,7 +103,46 @@ const completeMultipartUploadHappyCaseIfNoneMatch: ApiFunctionalTestCase<
completeMultipartUploadHappyCase[7],
];

// API reference: https://docs.aws.amazon.com/AmazonS3/latest/API/API_CompleteMultipartUpload.html
const completeMultipartUploadHappyCaseCustomEndpoint: ApiFunctionalTestCase<
typeof completeMultipartUpload
> = [
'happy case',
'completeMultipartUpload with custom endpoint',
completeMultipartUpload,
{
...defaultConfig,
customEndpoint: 'custom.endpoint.com',
forcePathStyle: true,
},
{
Bucket: 'bucket',
Key: 'key',
MultipartUpload: {
Parts: [
{
ETag: 'etag1',
PartNumber: 1,
ChecksumCRC32: 'test-checksum-1',
},
],
},
UploadId: 'uploadId',
},
expect.objectContaining({
url: expect.objectContaining({
href: 'https://custom.endpoint.com/bucket/key?uploadId=uploadId',
}),
}),
{
status: 200,
headers: { ...DEFAULT_RESPONSE_HEADERS },
body: '',
},
expect.objectContaining({
/** skip validating response */
}) as any,
];

const completeMultipartUploadErrorCase: ApiFunctionalTestCase<
typeof completeMultipartUpload
> = [
Expand Down Expand Up @@ -167,6 +205,7 @@ const completeMultipartUploadErrorWith200CodeCase: ApiFunctionalTestCase<
export default [
completeMultipartUploadHappyCase,
completeMultipartUploadHappyCaseIfNoneMatch,
completeMultipartUploadHappyCaseCustomEndpoint,
completeMultipartUploadErrorCase,
completeMultipartUploadErrorWith200CodeCase,
];
Original file line number Diff line number Diff line change
Expand Up @@ -58,4 +58,34 @@ const copyObjectHappyCase: ApiFunctionalTestCase<typeof copyObject> = [
},
];

export default [copyObjectHappyCase];
const copyObjectHappyCaseCustomEndpoint: ApiFunctionalTestCase<
typeof copyObject
> = [
'happy case',
'getObject with custom endpoint',
copyObject,
{
...defaultConfig,
customEndpoint: 'custom.endpoint.com',
forcePathStyle: true,
},
{
Bucket: 'bucket',
Key: 'key',
CopySource: 'sourceBucket/sourceKey',
},
expect.objectContaining({
url: expect.objectContaining({
href: 'https://custom.endpoint.com/bucket/key',
}),
}),
{
status: 200,
headers: DEFAULT_RESPONSE_HEADERS,
body: '',
},
expect.objectContaining({
/** skip validating response */
}) as any,
];
export default [copyObjectHappyCase, copyObjectHappyCaseCustomEndpoint];
Loading

0 comments on commit c12f2e0

Please sign in to comment.