From c12f2e0971031ecf3a6d49c58c7cf5c271fb4d19 Mon Sep 17 00:00:00 2001 From: Ashwin Kumar Date: Tue, 5 Nov 2024 10:32:02 -0800 Subject: [PATCH] feat(storage): add customEndpoint to internal apis in advanced options (#13961) * feat: add baseEndpoint to advanced options * feat: add baseEndpoint to customEndpoint * feat: thread baseEndpoint through resolved config to endpoint resolver * add customEndpoint advanced option to internals storage data-plane apis * add customEndpoint advanced option to internals storage control-plane apis * fix unit test * code cleanup * increase bundle size * wire up customEndpoint on copy API * increase the bundle size * add customEndpoint unit tests for all data and control apis * increase bundle size * update ts docs * add additional error unit tests for endpointResolver * add unit tests for internals/ apis * code cleanup * address feedback * add comment for ForcePathStyleEndpointNotSupported ErrorCode * increase bundle size * remove docs links from error recovery message --------- Co-authored-by: Erin Beal Co-authored-by: Ashwin Kumar --- packages/aws-amplify/package.json | 14 +-- .../__tests__/internals/apis/copy.test.ts | 2 + .../internals/apis/downloadData.test.ts | 3 + .../internals/apis/getDataAccess.test.ts | 4 +- .../internals/apis/getProperties.test.ts | 3 + .../__tests__/internals/apis/getUrl.test.ts | 3 + .../__tests__/internals/apis/list.test.ts | 3 + .../apis/listCallerAccessGrants.test.ts | 3 + .../__tests__/internals/apis/remove.test.ts | 3 + .../internals/apis/uploadData.test.ts | 4 + .../client/S3/cases/abortMultipartUpload.ts | 36 +++++- .../S3/cases/completeMultipartUpload.ts | 43 ++++++- .../s3/utils/client/S3/cases/copyObject.ts | 32 ++++- .../client/S3/cases/createMultipartUpload.ts | 32 ++++- .../s3/utils/client/S3/cases/deleteObject.ts | 32 ++++- .../s3/utils/client/S3/cases/getDataAccess.ts | 72 +++++++++++- .../s3/utils/client/S3/cases/getObject.ts | 110 ++++++++++++++++-- .../s3/utils/client/S3/cases/headObject.ts | 32 ++++- .../client/S3/cases/listCallerAccessGrants.ts | 32 +++++ .../s3/utils/client/S3/cases/listObjectsV2.ts | 45 +++++++ .../s3/utils/client/S3/cases/listParts.ts | 33 +++++- .../s3/utils/client/S3/cases/putObject.ts | 33 +++++- .../s3/utils/client/S3/cases/uploadPart.ts | 34 +++++- .../storage/src/errors/types/validation.ts | 12 ++ packages/storage/src/internals/apis/copy.ts | 1 + .../src/internals/apis/downloadData.ts | 1 + .../src/internals/apis/getDataAccess.ts | 1 + .../src/internals/apis/getProperties.ts | 1 + packages/storage/src/internals/apis/getUrl.ts | 1 + packages/storage/src/internals/apis/list.ts | 1 + .../internals/apis/listCallerAccessGrants.ts | 10 +- packages/storage/src/internals/apis/remove.ts | 1 + .../storage/src/internals/apis/uploadData.ts | 1 + .../storage/src/internals/types/inputs.ts | 39 +++---- .../src/providers/s3/apis/internal/copy.ts | 1 + .../s3/utils/client/s3control/base.ts | 34 ++++-- .../providers/s3/utils/client/s3data/base.ts | 48 +++++--- .../s3/utils/resolveS3ConfigAndInput.ts | 4 + 38 files changed, 690 insertions(+), 74 deletions(-) diff --git a/packages/aws-amplify/package.json b/packages/aws-amplify/package.json index 2a808815c19..9dab516b859 100644 --- a/packages/aws-amplify/package.json +++ b/packages/aws-amplify/package.json @@ -461,43 +461,43 @@ "name": "[Storage] copy (S3)", "path": "./dist/esm/storage/index.mjs", "import": "{ copy }", - "limit": "16.15 kB" + "limit": "16.39 kB" }, { "name": "[Storage] downloadData (S3)", "path": "./dist/esm/storage/index.mjs", "import": "{ downloadData }", - "limit": "16.48 kB" + "limit": "16.73 kB" }, { "name": "[Storage] getProperties (S3)", "path": "./dist/esm/storage/index.mjs", "import": "{ getProperties }", - "limit": "15.72 kB" + "limit": "15.99 kB" }, { "name": "[Storage] getUrl (S3)", "path": "./dist/esm/storage/index.mjs", "import": "{ getUrl }", - "limit": "16.98 kB" + "limit": "17.22 kB" }, { "name": "[Storage] list (S3)", "path": "./dist/esm/storage/index.mjs", "import": "{ list }", - "limit": "16.45 kB" + "limit": "16.69 kB" }, { "name": "[Storage] remove (S3)", "path": "./dist/esm/storage/index.mjs", "import": "{ remove }", - "limit": "15.59 kB" + "limit": "15.83 kB" }, { "name": "[Storage] uploadData (S3)", "path": "./dist/esm/storage/index.mjs", "import": "{ uploadData }", - "limit": "22.56 kB" + "limit": "22.81 kB" } ] } diff --git a/packages/storage/__tests__/internals/apis/copy.test.ts b/packages/storage/__tests__/internals/apis/copy.test.ts index f7fc957cae7..2692f4f6a68 100644 --- a/packages/storage/__tests__/internals/apis/copy.test.ts +++ b/packages/storage/__tests__/internals/apis/copy.test.ts @@ -17,6 +17,7 @@ describe('copy (internals)', () => { }); it('should pass advanced option locationCredentialsProvider to internal list', async () => { + const customEndpoint = 's3.dualstack.us-east-2.amazonaws.com'; const locationCredentialsProvider = async () => ({ credentials: { accessKeyId: 'akid', @@ -40,6 +41,7 @@ describe('copy (internals)', () => { }, options: { locationCredentialsProvider, + customEndpoint, }, }; const result = await advancedCopy(copyInputWithAdvancedOptions); diff --git a/packages/storage/__tests__/internals/apis/downloadData.test.ts b/packages/storage/__tests__/internals/apis/downloadData.test.ts index 6175e91e7d0..f18ea441e69 100644 --- a/packages/storage/__tests__/internals/apis/downloadData.test.ts +++ b/packages/storage/__tests__/internals/apis/downloadData.test.ts @@ -31,6 +31,7 @@ describe('downloadData (internal)', () => { const useAccelerateEndpoint = true; const expectedBucketOwner = '012345678901'; const bucket = { bucketName: 'bucket', region: 'us-east-1' }; + const customEndpoint = 's3.dualstack.us-east-2.amazonaws.com'; const locationCredentialsProvider = async () => ({ credentials: { accessKeyId: 'akid', @@ -45,6 +46,7 @@ describe('downloadData (internal)', () => { const output = await advancedDownloadData({ path: 'input/path/to/mock/object', options: { + customEndpoint, useAccelerateEndpoint, bucket, locationCredentialsProvider, @@ -58,6 +60,7 @@ describe('downloadData (internal)', () => { expect(mockedDownloadDataInternal).toHaveBeenCalledWith({ path: 'input/path/to/mock/object', options: { + customEndpoint, useAccelerateEndpoint, bucket, locationCredentialsProvider, diff --git a/packages/storage/__tests__/internals/apis/getDataAccess.test.ts b/packages/storage/__tests__/internals/apis/getDataAccess.test.ts index 630e7835f9e..34c41fe2bc7 100644 --- a/packages/storage/__tests__/internals/apis/getDataAccess.test.ts +++ b/packages/storage/__tests__/internals/apis/getDataAccess.test.ts @@ -31,10 +31,11 @@ const MOCK_ACCESS_CREDENTIALS = { SessionToken: MOCK_SESSION_TOKEN, Expiration: MOCK_EXPIRATION_DATE, }; +const MOCK_CUSTOM_ENDPOINT = 's3-accesspoint.dualstack.us-east-2.amazonaws.com'; const MOCK_CREDENTIAL_PROVIDER = jest.fn().mockResolvedValue(MOCK_CREDENTIALS); - const sharedGetDataAccessParams: GetDataAccessInput = { accountId: MOCK_ACCOUNT_ID, + customEndpoint: MOCK_CUSTOM_ENDPOINT, credentialsProvider: MOCK_CREDENTIAL_PROVIDER, durationSeconds: 900, permission: 'READWRITE', @@ -62,6 +63,7 @@ describe('getDataAccess', () => { expect(getDataAccessClientMock).toHaveBeenCalledWith( expect.objectContaining({ credentials: expect.any(Function), + customEndpoint: MOCK_CUSTOM_ENDPOINT, region: MOCK_REGION, userAgentValue: expect.stringContaining('storage/8'), }), diff --git a/packages/storage/__tests__/internals/apis/getProperties.test.ts b/packages/storage/__tests__/internals/apis/getProperties.test.ts index 97e85210e36..aa0c2c9815e 100644 --- a/packages/storage/__tests__/internals/apis/getProperties.test.ts +++ b/packages/storage/__tests__/internals/apis/getProperties.test.ts @@ -23,6 +23,7 @@ describe('getProperties (internal)', () => { const useAccelerateEndpoint = true; const expectedBucketOwner = '012345678901'; const bucket = { bucketName: 'bucket', region: 'us-east-1' }; + const customEndpoint = 's3.dualstack.us-east-2.amazonaws.com'; const locationCredentialsProvider = async () => ({ credentials: { accessKeyId: 'akid', @@ -34,6 +35,7 @@ describe('getProperties (internal)', () => { const result = await advancedGetProperties({ path: 'input/path/to/mock/object', options: { + customEndpoint, useAccelerateEndpoint, bucket, expectedBucketOwner, @@ -46,6 +48,7 @@ describe('getProperties (internal)', () => { { path: 'input/path/to/mock/object', options: { + customEndpoint, useAccelerateEndpoint, bucket, expectedBucketOwner, diff --git a/packages/storage/__tests__/internals/apis/getUrl.test.ts b/packages/storage/__tests__/internals/apis/getUrl.test.ts index d1b83149b7a..fcffafd3f2e 100644 --- a/packages/storage/__tests__/internals/apis/getUrl.test.ts +++ b/packages/storage/__tests__/internals/apis/getUrl.test.ts @@ -32,6 +32,7 @@ describe('getUrl (internal)', () => { const contentDisposition = 'inline; filename="example.jpg"'; const contentType = 'image/jpeg'; const bucket = { bucketName: 'bucket', region: 'us-east-1' }; + const customEndpoint = 's3.dualstack.us-east-2.amazonaws.com'; const locationCredentialsProvider = async () => ({ credentials: { accessKeyId: 'akid', @@ -43,6 +44,7 @@ describe('getUrl (internal)', () => { const result = await advancedGetUrl({ path: 'input/path/to/mock/object', options: { + customEndpoint, useAccelerateEndpoint, bucket, validateObjectExistence, @@ -59,6 +61,7 @@ describe('getUrl (internal)', () => { { path: 'input/path/to/mock/object', options: { + customEndpoint, useAccelerateEndpoint, bucket, validateObjectExistence, diff --git a/packages/storage/__tests__/internals/apis/list.test.ts b/packages/storage/__tests__/internals/apis/list.test.ts index d72b95ad0bd..16ea0e5037b 100644 --- a/packages/storage/__tests__/internals/apis/list.test.ts +++ b/packages/storage/__tests__/internals/apis/list.test.ts @@ -20,6 +20,7 @@ describe('list (internals)', () => { const useAccelerateEndpoint = true; const expectedBucketOwner = '012345678901'; const bucket = { bucketName: 'bucket', region: 'us-east-1' }; + const customEndpoint = 's3.dualstack.us-east-2.amazonaws.com'; const locationCredentialsProvider = async () => ({ credentials: { accessKeyId: 'akid', @@ -31,6 +32,7 @@ describe('list (internals)', () => { const result = await advancedList({ path: 'input/path/to/mock/object', options: { + customEndpoint, useAccelerateEndpoint, bucket, expectedBucketOwner, @@ -43,6 +45,7 @@ describe('list (internals)', () => { { path: 'input/path/to/mock/object', options: { + customEndpoint, useAccelerateEndpoint, bucket, expectedBucketOwner, diff --git a/packages/storage/__tests__/internals/apis/listCallerAccessGrants.test.ts b/packages/storage/__tests__/internals/apis/listCallerAccessGrants.test.ts index ebe724e688c..43d96f24488 100644 --- a/packages/storage/__tests__/internals/apis/listCallerAccessGrants.test.ts +++ b/packages/storage/__tests__/internals/apis/listCallerAccessGrants.test.ts @@ -21,6 +21,7 @@ const mockCredentialsProvider = jest .mockResolvedValue({ credentials: mockCredentials }); const mockNextToken = '123'; const mockPageSize = 123; +const mockCustomEndpoint = 's3-accesspoint.dualstack.us-east-2.amazonaws.com'; describe('listCallerAccessGrants', () => { afterEach(() => { @@ -36,6 +37,7 @@ describe('listCallerAccessGrants', () => { }); await listCallerAccessGrants({ accountId: mockAccountId, + customEndpoint: mockCustomEndpoint, region: mockRegion, credentialsProvider: mockCredentialsProvider, nextToken: mockNextToken, @@ -45,6 +47,7 @@ describe('listCallerAccessGrants', () => { expect.objectContaining({ region: mockRegion, credentials: expect.any(Function), + customEndpoint: mockCustomEndpoint, }), expect.objectContaining({ AccountId: mockAccountId, diff --git a/packages/storage/__tests__/internals/apis/remove.test.ts b/packages/storage/__tests__/internals/apis/remove.test.ts index 2f67997e10c..2adab6dd0ef 100644 --- a/packages/storage/__tests__/internals/apis/remove.test.ts +++ b/packages/storage/__tests__/internals/apis/remove.test.ts @@ -23,6 +23,7 @@ describe('remove (internal)', () => { const useAccelerateEndpoint = true; const expectedBucketOwner = '012345678901'; const bucket = { bucketName: 'bucket', region: 'us-east-1' }; + const customEndpoint = 's3.dualstack.us-east-2.amazonaws.com'; const locationCredentialsProvider = async () => ({ credentials: { accessKeyId: 'akid', @@ -35,6 +36,7 @@ describe('remove (internal)', () => { const result = await advancedRemove({ path: 'input/path/to/mock/object', options: { + customEndpoint, useAccelerateEndpoint, bucket, expectedBucketOwner, @@ -48,6 +50,7 @@ describe('remove (internal)', () => { { path: 'input/path/to/mock/object', options: { + customEndpoint, useAccelerateEndpoint, bucket, expectedBucketOwner, diff --git a/packages/storage/__tests__/internals/apis/uploadData.test.ts b/packages/storage/__tests__/internals/apis/uploadData.test.ts index 642e4776e74..e9345d12cc6 100644 --- a/packages/storage/__tests__/internals/apis/uploadData.test.ts +++ b/packages/storage/__tests__/internals/apis/uploadData.test.ts @@ -21,6 +21,8 @@ describe('uploadData (internal)', () => { const useAccelerateEndpoint = true; const expectedBucketOwner = '012345678901'; const bucket = { bucketName: 'bucket', region: 'us-east-1' }; + const customEndpoint = 's3.dualstack.us-east-2.amazonaws.com'; + const locationCredentialsProvider = async () => ({ credentials: { accessKeyId: 'akid', @@ -37,6 +39,7 @@ describe('uploadData (internal)', () => { path: 'input/path/to/mock/object', data: 'data', options: { + customEndpoint, useAccelerateEndpoint, bucket, locationCredentialsProvider, @@ -54,6 +57,7 @@ describe('uploadData (internal)', () => { path: 'input/path/to/mock/object', data: 'data', options: { + customEndpoint, useAccelerateEndpoint, bucket, locationCredentialsProvider, diff --git a/packages/storage/__tests__/providers/s3/utils/client/S3/cases/abortMultipartUpload.ts b/packages/storage/__tests__/providers/s3/utils/client/S3/cases/abortMultipartUpload.ts index cc81a2be88f..5eba75535a6 100644 --- a/packages/storage/__tests__/providers/s3/utils/client/S3/cases/abortMultipartUpload.ts +++ b/packages/storage/__tests__/providers/s3/utils/client/S3/cases/abortMultipartUpload.ts @@ -42,4 +42,38 @@ const abortMultipartUploadHappyCase: ApiFunctionalTestCase< }, ]; -export default [abortMultipartUploadHappyCase]; +const abortMultipartUploadHappyCaseCustomEndpoint: ApiFunctionalTestCase< + typeof abortMultipartUpload +> = [ + 'happy case', + 'abortMultipartUpload with custom endpoint', + abortMultipartUpload, + { + ...defaultConfig, + customEndpoint: 'custom.endpoint.com', + forcePathStyle: true, + }, + { + Bucket: 'bucket', + Key: 'key', + UploadId: 'uploadId', + }, + expect.objectContaining({ + url: expect.objectContaining({ + href: 'https://custom.endpoint.com/bucket/key?uploadId=uploadId', + }), + }), + { + status: 204, + headers: DEFAULT_RESPONSE_HEADERS, + body: '', + }, + expect.objectContaining({ + /** skip validating response */ + }) as any, +]; + +export default [ + abortMultipartUploadHappyCase, + abortMultipartUploadHappyCaseCustomEndpoint, +]; diff --git a/packages/storage/__tests__/providers/s3/utils/client/S3/cases/completeMultipartUpload.ts b/packages/storage/__tests__/providers/s3/utils/client/S3/cases/completeMultipartUpload.ts index 74a914767c4..140267b751b 100644 --- a/packages/storage/__tests__/providers/s3/utils/client/S3/cases/completeMultipartUpload.ts +++ b/packages/storage/__tests__/providers/s3/utils/client/S3/cases/completeMultipartUpload.ts @@ -81,7 +81,6 @@ const completeMultipartUploadHappyCase: ApiFunctionalTestCase< }, ]; -// API reference: https://docs.aws.amazon.com/AmazonS3/latest/API/API_CompleteMultipartUpload.html const completeMultipartUploadHappyCaseIfNoneMatch: ApiFunctionalTestCase< typeof completeMultipartUpload > = [ @@ -104,7 +103,46 @@ const completeMultipartUploadHappyCaseIfNoneMatch: ApiFunctionalTestCase< completeMultipartUploadHappyCase[7], ]; -// API reference: https://docs.aws.amazon.com/AmazonS3/latest/API/API_CompleteMultipartUpload.html +const completeMultipartUploadHappyCaseCustomEndpoint: ApiFunctionalTestCase< + typeof completeMultipartUpload +> = [ + 'happy case', + 'completeMultipartUpload with custom endpoint', + completeMultipartUpload, + { + ...defaultConfig, + customEndpoint: 'custom.endpoint.com', + forcePathStyle: true, + }, + { + Bucket: 'bucket', + Key: 'key', + MultipartUpload: { + Parts: [ + { + ETag: 'etag1', + PartNumber: 1, + ChecksumCRC32: 'test-checksum-1', + }, + ], + }, + UploadId: 'uploadId', + }, + expect.objectContaining({ + url: expect.objectContaining({ + href: 'https://custom.endpoint.com/bucket/key?uploadId=uploadId', + }), + }), + { + status: 200, + headers: { ...DEFAULT_RESPONSE_HEADERS }, + body: '', + }, + expect.objectContaining({ + /** skip validating response */ + }) as any, +]; + const completeMultipartUploadErrorCase: ApiFunctionalTestCase< typeof completeMultipartUpload > = [ @@ -167,6 +205,7 @@ const completeMultipartUploadErrorWith200CodeCase: ApiFunctionalTestCase< export default [ completeMultipartUploadHappyCase, completeMultipartUploadHappyCaseIfNoneMatch, + completeMultipartUploadHappyCaseCustomEndpoint, completeMultipartUploadErrorCase, completeMultipartUploadErrorWith200CodeCase, ]; diff --git a/packages/storage/__tests__/providers/s3/utils/client/S3/cases/copyObject.ts b/packages/storage/__tests__/providers/s3/utils/client/S3/cases/copyObject.ts index a1a52247846..c20c0394a0e 100644 --- a/packages/storage/__tests__/providers/s3/utils/client/S3/cases/copyObject.ts +++ b/packages/storage/__tests__/providers/s3/utils/client/S3/cases/copyObject.ts @@ -58,4 +58,34 @@ const copyObjectHappyCase: ApiFunctionalTestCase = [ }, ]; -export default [copyObjectHappyCase]; +const copyObjectHappyCaseCustomEndpoint: ApiFunctionalTestCase< + typeof copyObject +> = [ + 'happy case', + 'getObject with custom endpoint', + copyObject, + { + ...defaultConfig, + customEndpoint: 'custom.endpoint.com', + forcePathStyle: true, + }, + { + Bucket: 'bucket', + Key: 'key', + CopySource: 'sourceBucket/sourceKey', + }, + expect.objectContaining({ + url: expect.objectContaining({ + href: 'https://custom.endpoint.com/bucket/key', + }), + }), + { + status: 200, + headers: DEFAULT_RESPONSE_HEADERS, + body: '', + }, + expect.objectContaining({ + /** skip validating response */ + }) as any, +]; +export default [copyObjectHappyCase, copyObjectHappyCaseCustomEndpoint]; diff --git a/packages/storage/__tests__/providers/s3/utils/client/S3/cases/createMultipartUpload.ts b/packages/storage/__tests__/providers/s3/utils/client/S3/cases/createMultipartUpload.ts index e027397e569..b53ae0b48e8 100644 --- a/packages/storage/__tests__/providers/s3/utils/client/S3/cases/createMultipartUpload.ts +++ b/packages/storage/__tests__/providers/s3/utils/client/S3/cases/createMultipartUpload.ts @@ -42,4 +42,34 @@ const createMultiPartUploadHappyCase: ApiFunctionalTestCase< }, ]; -export default [createMultiPartUploadHappyCase]; +const createMultiPartUploadHappyCaseCustomEndpoint: ApiFunctionalTestCase< + typeof createMultipartUpload +> = [ + 'happy case', + 'createMultipartUpload with custom endpoint', + createMultipartUpload, + { + ...defaultConfig, + customEndpoint: 'custom.endpoint.com', + forcePathStyle: true, + }, + putObjectRequest, + expect.objectContaining({ + url: expect.objectContaining({ + href: 'https://custom.endpoint.com/bucket/key?uploads', + }), + }), + { + status: 200, + headers: { ...DEFAULT_RESPONSE_HEADERS }, + body: '', + }, + expect.objectContaining({ + /** skip validating response */ + }) as any, +]; + +export default [ + createMultiPartUploadHappyCase, + createMultiPartUploadHappyCaseCustomEndpoint, +]; diff --git a/packages/storage/__tests__/providers/s3/utils/client/S3/cases/deleteObject.ts b/packages/storage/__tests__/providers/s3/utils/client/S3/cases/deleteObject.ts index 614a3c1fff6..0d591b6bfcc 100644 --- a/packages/storage/__tests__/providers/s3/utils/client/S3/cases/deleteObject.ts +++ b/packages/storage/__tests__/providers/s3/utils/client/S3/cases/deleteObject.ts @@ -36,4 +36,34 @@ const deleteObjectHappyCase: ApiFunctionalTestCase = [ }, ]; -export default [deleteObjectHappyCase]; +const deleteObjectHappyCaseCustomEndpoint: ApiFunctionalTestCase< + typeof deleteObject +> = [ + 'happy case', + 'deleteObject with custom endpoint', + deleteObject, + { + ...defaultConfig, + customEndpoint: 'custom.endpoint.com', + forcePathStyle: true, + }, + { + Bucket: 'bucket', + Key: 'key', + }, + expect.objectContaining({ + url: expect.objectContaining({ + href: 'https://custom.endpoint.com/bucket/key', + }), + }), + { + status: 200, + headers: DEFAULT_RESPONSE_HEADERS, + body: '', + }, + expect.objectContaining({ + /** skip validating response */ + }) as any, +]; + +export default [deleteObjectHappyCase, deleteObjectHappyCaseCustomEndpoint]; diff --git a/packages/storage/__tests__/providers/s3/utils/client/S3/cases/getDataAccess.ts b/packages/storage/__tests__/providers/s3/utils/client/S3/cases/getDataAccess.ts index 851bc993a7c..6e944a058a6 100644 --- a/packages/storage/__tests__/providers/s3/utils/client/S3/cases/getDataAccess.ts +++ b/packages/storage/__tests__/providers/s3/utils/client/S3/cases/getDataAccess.ts @@ -71,6 +71,38 @@ const getDataAccessHappyCase: ApiFunctionalTestCase = [ }, ]; +const getDataAccessHappyCaseCustomEndpoint: ApiFunctionalTestCase< + typeof getDataAccess +> = [ + 'happy case', + 'getDataAccess with custom endpoint', + getDataAccess, + { + ...defaultConfig, + customEndpoint: 'custom.endpoint.com', + }, + { + AccountId: MOCK_ACCOUNT_ID, + Target: 's3://my-bucket/path/to/object.md', + Permission: 'READWRITE', + }, + expect.objectContaining({ + url: expect.objectContaining({ + href: 'https://accountid.custom.endpoint.com/v20180820/accessgrantsinstance/dataaccess?permission=READWRITE&target=s3%3A%2F%2Fmy-bucket%2Fpath%2Fto%2Fobject.md', + }), + }), + { + status: 200, + headers: { + ...DEFAULT_RESPONSE_HEADERS, + }, + body: '', + }, + expect.objectContaining({ + /** skip validating response */ + }) as any, +]; + const getDataAccessErrorCase: ApiFunctionalTestCase = [ 'error case', 'getDataAccess', @@ -99,4 +131,42 @@ const getDataAccessErrorCase: ApiFunctionalTestCase = [ }, ]; -export default [getDataAccessHappyCase, getDataAccessErrorCase]; +const getDataAccessErrorCaseInvalidCustomEndpoint: ApiFunctionalTestCase< + typeof getDataAccess +> = [ + 'error case', + 'getDataAccess with invalid custom endpoint', + getDataAccess, + { + ...defaultConfig, + customEndpoint: 'http://custom.endpoint.com', + }, + { + AccountId: MOCK_ACCOUNT_ID, + Target: 's3://my-bucket/path/to/object.md', + Permission: 'READWRITE', + }, + expect.objectContaining({ + url: expect.objectContaining({ + href: 'https://accountid.custom.endpoint.com/v20180820/accessgrantsinstance/dataaccess?permission=READWRITE&target=s3%3A%2F%2Fmy-bucket%2Fpath%2Fto%2Fobject.md', + }), + }), + { + status: 200, + headers: { + ...DEFAULT_RESPONSE_HEADERS, + }, + body: '', + }, + { + message: 'Invalid S3 custom endpoint.', + name: 'InvalidCustomEndpoint', + }, +]; + +export default [ + getDataAccessHappyCase, + getDataAccessHappyCaseCustomEndpoint, + getDataAccessErrorCase, + getDataAccessErrorCaseInvalidCustomEndpoint, +]; diff --git a/packages/storage/__tests__/providers/s3/utils/client/S3/cases/getObject.ts b/packages/storage/__tests__/providers/s3/utils/client/S3/cases/getObject.ts index a35c813f3d8..2a2ddf98f68 100644 --- a/packages/storage/__tests__/providers/s3/utils/client/S3/cases/getObject.ts +++ b/packages/storage/__tests__/providers/s3/utils/client/S3/cases/getObject.ts @@ -143,14 +143,16 @@ const getObjectHappyCase: ApiFunctionalTestCase = [ }, ]; -const getObjectAccelerateEndpoint: ApiFunctionalTestCase = [ +const getObjectHappyCaseAccelerateEndpoint: ApiFunctionalTestCase< + typeof getObject +> = [ 'happy case', 'getObject with accelerate endpoint', getObject, { ...defaultConfig, useAccelerateEndpoint: true, - } as Parameters[0], + }, { Bucket: 'bucket', Key: 'key', @@ -170,15 +172,17 @@ const getObjectAccelerateEndpoint: ApiFunctionalTestCase = [ }) as any, ]; -const getObjectCustomEndpoint: ApiFunctionalTestCase = [ +const getObjectHappyCaseCustomEndpoint: ApiFunctionalTestCase< + typeof getObject +> = [ 'happy case', 'getObject with custom endpoint', getObject, { ...defaultConfig, - customEndpoint: 'https://custom.endpoint.com', + customEndpoint: 'custom.endpoint.com', forcePathStyle: true, - } as Parameters[0], + }, { Bucket: 'bucket', Key: 'key', @@ -198,8 +202,100 @@ const getObjectCustomEndpoint: ApiFunctionalTestCase = [ }) as any, ]; +const getObjectErrorCaseAccelerateEndpoint: ApiFunctionalTestCase< + typeof getObject +> = [ + 'error case', + 'getObject with accelerate endpoint and forcePathStyle', + getObject, + { + ...defaultConfig, + useAccelerateEndpoint: true, + forcePathStyle: true, + }, + { + Bucket: 'bucket', + Key: 'key', + }, + expect.objectContaining({ + url: expect.objectContaining({ + href: 'https://bucket.s3-accelerate.amazonaws.com/key', + }), + }), + { + status: 400, + headers: DEFAULT_RESPONSE_HEADERS, + body: 'mockBody', + }, + { + message: 'Path style URLs are not supported with S3 Transfer Acceleration.', + name: 'ForcePathStyleEndpointNotSupported', + }, +]; + +const getObjectErrorCaseInvalidCustomEndpoint: ApiFunctionalTestCase< + typeof getObject +> = [ + 'error case', + 'getObject with invalid custom endpoint', + getObject, + { + ...defaultConfig, + customEndpoint: 'http://custom.endpoint.com', + forcePathStyle: true, + }, + { + Bucket: 'bucket', + Key: 'key', + }, + expect.objectContaining({ + url: expect.objectContaining({ + href: 'https://custom.endpoint.com/bucket/key', + }), + }), + { + status: 400, + headers: DEFAULT_RESPONSE_HEADERS, + body: 'mockBody', + }, + { + message: 'Invalid S3 custom endpoint.', + name: 'InvalidCustomEndpoint', + }, +]; + +const getObjectErrorCaseInvalidBucketName: ApiFunctionalTestCase< + typeof getObject +> = [ + 'error case', + 'getObject with incompatible Dns bucket name', + getObject, + defaultConfig, + { + Bucket: 'incompatibleDnsCompatibleBucketName', + Key: 'key', + }, + expect.objectContaining({ + url: expect.objectContaining({ + href: 'https://incompatibleDnsCompatibleBucketName.s3.us-east-1.amazonaws.com/key', + }), + }), + { + status: 400, + headers: DEFAULT_RESPONSE_HEADERS, + body: 'mockBody', + }, + { + message: `The bucket name isn't DNS compatible.`, + name: 'DnsIncompatibleBucketName', + }, +]; + export default [ getObjectHappyCase, - getObjectAccelerateEndpoint, - getObjectCustomEndpoint, + getObjectHappyCaseAccelerateEndpoint, + getObjectHappyCaseCustomEndpoint, + getObjectErrorCaseAccelerateEndpoint, + getObjectErrorCaseInvalidCustomEndpoint, + getObjectErrorCaseInvalidBucketName, ]; diff --git a/packages/storage/__tests__/providers/s3/utils/client/S3/cases/headObject.ts b/packages/storage/__tests__/providers/s3/utils/client/S3/cases/headObject.ts index 0cc016a7813..a392e121c8c 100644 --- a/packages/storage/__tests__/providers/s3/utils/client/S3/cases/headObject.ts +++ b/packages/storage/__tests__/providers/s3/utils/client/S3/cases/headObject.ts @@ -48,4 +48,34 @@ const headObjectHappyCase: ApiFunctionalTestCase = [ }, ]; -export default [headObjectHappyCase]; +const headObjectHappyCaseCustomEndpoint: ApiFunctionalTestCase< + typeof headObject +> = [ + 'happy case', + 'headObject with custom endpoint', + headObject, + { + ...defaultConfig, + customEndpoint: 'custom.endpoint.com', + forcePathStyle: true, + }, + { + Bucket: 'bucket', + Key: 'key', + }, + expect.objectContaining({ + url: expect.objectContaining({ + href: 'https://custom.endpoint.com/bucket/key', + }), + }), + { + status: 200, + headers: DEFAULT_RESPONSE_HEADERS, + body: '', + }, + expect.objectContaining({ + /** skip validating response */ + }) as any, +]; + +export default [headObjectHappyCase, headObjectHappyCaseCustomEndpoint]; diff --git a/packages/storage/__tests__/providers/s3/utils/client/S3/cases/listCallerAccessGrants.ts b/packages/storage/__tests__/providers/s3/utils/client/S3/cases/listCallerAccessGrants.ts index 63499b7234c..175e6d8b0da 100644 --- a/packages/storage/__tests__/providers/s3/utils/client/S3/cases/listCallerAccessGrants.ts +++ b/packages/storage/__tests__/providers/s3/utils/client/S3/cases/listCallerAccessGrants.ts @@ -137,6 +137,37 @@ const listCallerAccessGrantsHappyCaseMultipleGrants: ApiFunctionalTestCase< }, ]; +const listCallerAccessGrantsHappyCaseCustomEndpoint: ApiFunctionalTestCase< + typeof listCallerAccessGrants +> = [ + 'happy case', + 'listCallerAccessGrants with custom endpoint', + listCallerAccessGrants, + { + ...defaultConfig, + customEndpoint: 'custom.endpoint.com', + }, + { + AccountId: MOCK_ACCOUNT_ID, + GrantScope: 's3://my-bucket/path/to/', + }, + expect.objectContaining({ + url: expect.objectContaining({ + href: 'https://accountid.custom.endpoint.com/v20180820/accessgrantsinstance/caller/grants?grantscope=s3%3A%2F%2Fmy-bucket%2Fpath%2Fto%2F', + }), + }), + { + status: 200, + headers: { + ...DEFAULT_RESPONSE_HEADERS, + }, + body: '', + }, + expect.objectContaining({ + /** skip validating response */ + }) as any, +]; + const listCallerAccessGrantsErrorCase: ApiFunctionalTestCase< typeof listCallerAccessGrants > = [ @@ -170,5 +201,6 @@ const listCallerAccessGrantsErrorCase: ApiFunctionalTestCase< export default [ listCallerAccessGrantsHappyCaseSingleGrant, listCallerAccessGrantsHappyCaseMultipleGrants, + listCallerAccessGrantsHappyCaseCustomEndpoint, listCallerAccessGrantsErrorCase, ]; diff --git a/packages/storage/__tests__/providers/s3/utils/client/S3/cases/listObjectsV2.ts b/packages/storage/__tests__/providers/s3/utils/client/S3/cases/listObjectsV2.ts index a14ec159fa3..aa60b74f906 100644 --- a/packages/storage/__tests__/providers/s3/utils/client/S3/cases/listObjectsV2.ts +++ b/packages/storage/__tests__/providers/s3/utils/client/S3/cases/listObjectsV2.ts @@ -376,9 +376,54 @@ const listObjectsV2ErrorCaseMissingTruncated: ApiFunctionalTestCase< }, ]; +const listObjectsV2HappyCaseCustomEndpoint: ApiFunctionalTestCase< + typeof listObjectsV2 +> = [ + 'happy case', + 'listObjectsV2 with custom endpoint', + listObjectsV2, + { + ...defaultConfig, + customEndpoint: 'custom.endpoint.com', + forcePathStyle: true, + }, + { + Bucket: 'bucket', + Prefix: 'Prefix', + }, + expect.objectContaining({ + url: expect.objectContaining({ + href: 'https://custom.endpoint.com/bucket?list-type=2&prefix=Prefix', + }), + }), + { + status: 200, + headers: DEFAULT_RESPONSE_HEADERS, + body: ` + + bucket + + 1 + 1000 + false + + ExampleObject.txt + 2013-09-17T18:07:53.000Z + "599bab3ed2c697f1d26842727561fd94" + 857 + REDUCED_REDUNDANCY + + `, + }, + expect.objectContaining({ + /** skip validating response */ + }) as any, +]; + export default [ listObjectsV2HappyCaseTruncated, listObjectsV2HappyCaseComplete, + listObjectsV2HappyCaseCustomEndpoint, listObjectsV2ErrorCaseKeyCount, listObjectsV2ErrorCaseMissingTruncated, listObjectsV2ErrorCaseMissingToken, diff --git a/packages/storage/__tests__/providers/s3/utils/client/S3/cases/listParts.ts b/packages/storage/__tests__/providers/s3/utils/client/S3/cases/listParts.ts index 059dfcaec0a..63f2a37e06c 100644 --- a/packages/storage/__tests__/providers/s3/utils/client/S3/cases/listParts.ts +++ b/packages/storage/__tests__/providers/s3/utils/client/S3/cases/listParts.ts @@ -66,4 +66,35 @@ const listPartsHappyCase: ApiFunctionalTestCase = [ }, ]; -export default [listPartsHappyCase]; +const listPartsHappyCaseCustomEndpoint: ApiFunctionalTestCase< + typeof listParts +> = [ + 'happy case', + 'listParts with custom endpoint', + listParts, + { + ...defaultConfig, + customEndpoint: 'custom.endpoint.com', + forcePathStyle: true, + }, + { + Bucket: 'bucket', + Key: 'key', + UploadId: 'uploadId', + }, + expect.objectContaining({ + url: expect.objectContaining({ + href: 'https://custom.endpoint.com/bucket/key?uploadId=uploadId', + }), + }), + { + status: 200, + headers: DEFAULT_RESPONSE_HEADERS, + body: '', + }, + expect.objectContaining({ + /** skip validating response */ + }) as any, +]; + +export default [listPartsHappyCase, listPartsHappyCaseCustomEndpoint]; diff --git a/packages/storage/__tests__/providers/s3/utils/client/S3/cases/putObject.ts b/packages/storage/__tests__/providers/s3/utils/client/S3/cases/putObject.ts index 867ee3f0af2..6ee6f1e62fa 100644 --- a/packages/storage/__tests__/providers/s3/utils/client/S3/cases/putObject.ts +++ b/packages/storage/__tests__/providers/s3/utils/client/S3/cases/putObject.ts @@ -68,7 +68,32 @@ const putObjectHappyCase: ApiFunctionalTestCase = [ }, ]; -const pubObjectDefaultContentType: ApiFunctionalTestCase = [ +const putObjectHappyCaseCustomEndpoint: ApiFunctionalTestCase< + typeof putObject +> = [ + 'happy case', + 'putObject with custom endpoint', + putObject, + { + ...defaultConfig, + customEndpoint: 'custom.endpoint.com', + forcePathStyle: true, + }, + putObjectRequest, + expect.objectContaining({ + url: expect.objectContaining({ + href: 'https://custom.endpoint.com/bucket/key', + }), + }), + putObjectSuccessResponse, + expect.objectContaining({ + /** skip validating response */ + }) as any, +]; + +const pubObjectHappyCaseDefaultContentType: ApiFunctionalTestCase< + typeof putObject +> = [ 'happy case', 'putObject default content type', putObject, @@ -86,4 +111,8 @@ const pubObjectDefaultContentType: ApiFunctionalTestCase = [ expect.anything(), ]; -export default [putObjectHappyCase, pubObjectDefaultContentType]; +export default [ + putObjectHappyCase, + putObjectHappyCaseCustomEndpoint, + pubObjectHappyCaseDefaultContentType, +]; diff --git a/packages/storage/__tests__/providers/s3/utils/client/S3/cases/uploadPart.ts b/packages/storage/__tests__/providers/s3/utils/client/S3/cases/uploadPart.ts index 4a46891c849..34d0d6f7f38 100644 --- a/packages/storage/__tests__/providers/s3/utils/client/S3/cases/uploadPart.ts +++ b/packages/storage/__tests__/providers/s3/utils/client/S3/cases/uploadPart.ts @@ -44,4 +44,36 @@ const uploadPartHappyCase: ApiFunctionalTestCase = [ }, ]; -export default [uploadPartHappyCase]; +const uploadPartHappyCaseCustomEndpoint: ApiFunctionalTestCase< + typeof uploadPart +> = [ + 'happy case', + 'uploadPart with custom endpoint', + uploadPart, + { + ...defaultConfig, + customEndpoint: 'custom.endpoint.com', + forcePathStyle: true, + }, + { + Bucket: 'bucket', + Key: 'key', + PartNumber: 1, + UploadId: 'uploadId', + }, + expect.objectContaining({ + url: expect.objectContaining({ + href: 'https://custom.endpoint.com/bucket/key?partNumber=1&uploadId=uploadId', + }), + }), + { + status: 200, + headers: { ...DEFAULT_RESPONSE_HEADERS, etag: 'etag' }, + body: '', + }, + expect.objectContaining({ + /** skip validating response */ + }) as any, +]; + +export default [uploadPartHappyCase, uploadPartHappyCaseCustomEndpoint]; diff --git a/packages/storage/src/errors/types/validation.ts b/packages/storage/src/errors/types/validation.ts index de15d0a89ec..a56662adec4 100644 --- a/packages/storage/src/errors/types/validation.ts +++ b/packages/storage/src/errors/types/validation.ts @@ -25,6 +25,9 @@ export enum StorageValidationErrorCode { InvalidLocationCredentialsCacheSize = 'InvalidLocationCredentialsCacheSize', LocationCredentialsStoreDestroyed = 'LocationCredentialsStoreDestroyed', InvalidS3Uri = 'InvalidS3Uri', + InvalidCustomEndpoint = 'InvalidCustomEndpoint', + ForcePathStyleEndpointNotSupported = 'ForcePathStyleEndpointNotSupported', + DnsIncompatibleBucketName = 'DnsIncompatibleBucketName', } export const validationErrorMap: AmplifyErrorMap = { @@ -95,4 +98,13 @@ export const validationErrorMap: AmplifyErrorMap = { [StorageValidationErrorCode.InvalidCopyOperationStorageBucket]: { message: 'Missing bucket option in either source or destination.', }, + [StorageValidationErrorCode.InvalidCustomEndpoint]: { + message: 'Invalid S3 custom endpoint.', + }, + [StorageValidationErrorCode.ForcePathStyleEndpointNotSupported]: { + message: 'Path style URLs are not supported with S3 Transfer Acceleration.', + }, + [StorageValidationErrorCode.DnsIncompatibleBucketName]: { + message: `The bucket name isn't DNS compatible.`, + }, }; diff --git a/packages/storage/src/internals/apis/copy.ts b/packages/storage/src/internals/apis/copy.ts index eb53241e89e..3286ab99462 100644 --- a/packages/storage/src/internals/apis/copy.ts +++ b/packages/storage/src/internals/apis/copy.ts @@ -27,6 +27,7 @@ export const copy = (input: CopyInput) => options: { // Advanced options locationCredentialsProvider: input.options?.locationCredentialsProvider, + customEndpoint: input?.options?.customEndpoint, }, // Type casting is necessary because `copyInternal` supports both Gen1 and Gen2 signatures, but here // given in input can only be Gen2 signature, the return can only ben Gen2 signature. diff --git a/packages/storage/src/internals/apis/downloadData.ts b/packages/storage/src/internals/apis/downloadData.ts index 8f097f82bb6..bd862d9d9b4 100644 --- a/packages/storage/src/internals/apis/downloadData.ts +++ b/packages/storage/src/internals/apis/downloadData.ts @@ -18,6 +18,7 @@ export const downloadData = (input: DownloadDataInput): DownloadDataOutput => bytesRange: input?.options?.bytesRange, onProgress: input?.options?.onProgress, expectedBucketOwner: input?.options?.expectedBucketOwner, + customEndpoint: input?.options?.customEndpoint, }, // Type casting is necessary because `downloadDataInternal` supports both Gen1 and Gen2 signatures, but here // given in input can only be Gen2 signature, the return can only ben Gen2 signature. diff --git a/packages/storage/src/internals/apis/getDataAccess.ts b/packages/storage/src/internals/apis/getDataAccess.ts index 3a6af14441a..070bf617078 100644 --- a/packages/storage/src/internals/apis/getDataAccess.ts +++ b/packages/storage/src/internals/apis/getDataAccess.ts @@ -33,6 +33,7 @@ export const getDataAccess = async ( const result = await getDataAccessClient( { credentials: clientCredentialsProvider, + customEndpoint: input.customEndpoint, region: input.region, userAgentValue: getStorageUserAgentValue(StorageAction.GetDataAccess), }, diff --git a/packages/storage/src/internals/apis/getProperties.ts b/packages/storage/src/internals/apis/getProperties.ts index 54e03c6d60b..213e184edae 100644 --- a/packages/storage/src/internals/apis/getProperties.ts +++ b/packages/storage/src/internals/apis/getProperties.ts @@ -20,6 +20,7 @@ export const getProperties = ( bucket: input?.options?.bucket, locationCredentialsProvider: input?.options?.locationCredentialsProvider, expectedBucketOwner: input?.options?.expectedBucketOwner, + customEndpoint: input?.options?.customEndpoint, }, // Type casting is necessary because `getPropertiesInternal` supports both Gen1 and Gen2 signatures, but here // given in input can only be Gen2 signature, the return can only ben Gen2 signature. diff --git a/packages/storage/src/internals/apis/getUrl.ts b/packages/storage/src/internals/apis/getUrl.ts index 3c19b922e17..ef82f107c67 100644 --- a/packages/storage/src/internals/apis/getUrl.ts +++ b/packages/storage/src/internals/apis/getUrl.ts @@ -24,6 +24,7 @@ export const getUrl = (input: GetUrlInput) => // Advanced options locationCredentialsProvider: input?.options?.locationCredentialsProvider, + customEndpoint: input?.options?.customEndpoint, }, // Type casting is necessary because `getPropertiesInternal` supports both Gen1 and Gen2 signatures, but here // given in input can only be Gen2 signature, the return can only ben Gen2 signature. diff --git a/packages/storage/src/internals/apis/list.ts b/packages/storage/src/internals/apis/list.ts index 4ce0a72eb3f..60c9184bd7f 100644 --- a/packages/storage/src/internals/apis/list.ts +++ b/packages/storage/src/internals/apis/list.ts @@ -39,6 +39,7 @@ export function list(input: ListInput): Promise { pageSize: (input as ListPaginateInput).options?.pageSize, // Advanced options locationCredentialsProvider: input.options?.locationCredentialsProvider, + customEndpoint: input?.options?.customEndpoint, }, // Type casting is necessary because `listInternal` supports both Gen1 and Gen2 signatures, but here // given in input can only be Gen2 signature, the return can only ben Gen2 signature. diff --git a/packages/storage/src/internals/apis/listCallerAccessGrants.ts b/packages/storage/src/internals/apis/listCallerAccessGrants.ts index c0da06b4f93..47fee1c051a 100644 --- a/packages/storage/src/internals/apis/listCallerAccessGrants.ts +++ b/packages/storage/src/internals/apis/listCallerAccessGrants.ts @@ -20,7 +20,14 @@ import { MAX_PAGE_SIZE } from '../utils/constants'; export const listCallerAccessGrants = async ( input: ListCallerAccessGrantsInput, ): Promise => { - const { credentialsProvider, accountId, region, nextToken, pageSize } = input; + const { + credentialsProvider, + accountId, + region, + nextToken, + pageSize, + customEndpoint, + } = input; logger.debug(`listing available locations from account ${input.accountId}`); @@ -40,6 +47,7 @@ export const listCallerAccessGrants = async ( await listCallerAccessGrantsClient( { credentials: clientCredentialsProvider, + customEndpoint, region, userAgentValue: getStorageUserAgentValue( StorageAction.ListCallerAccessGrants, diff --git a/packages/storage/src/internals/apis/remove.ts b/packages/storage/src/internals/apis/remove.ts index 22864c3156b..96530325e2c 100644 --- a/packages/storage/src/internals/apis/remove.ts +++ b/packages/storage/src/internals/apis/remove.ts @@ -18,6 +18,7 @@ export const remove = (input: RemoveInput): Promise => bucket: input?.options?.bucket, expectedBucketOwner: input?.options?.expectedBucketOwner, locationCredentialsProvider: input?.options?.locationCredentialsProvider, + customEndpoint: input?.options?.customEndpoint, }, // Type casting is necessary because `removeInternal` supports both Gen1 and Gen2 signatures, but here // given in input can only be Gen2 signature, the return can only ben Gen2 signature. diff --git a/packages/storage/src/internals/apis/uploadData.ts b/packages/storage/src/internals/apis/uploadData.ts index 58b04ac2211..1ebabdb6165 100644 --- a/packages/storage/src/internals/apis/uploadData.ts +++ b/packages/storage/src/internals/apis/uploadData.ts @@ -27,6 +27,7 @@ export const uploadData = (input: UploadDataInput) => { // Advanced options locationCredentialsProvider: options?.locationCredentialsProvider, + customEndpoint: options?.customEndpoint, }, // Type casting is necessary because `uploadDataInternal` supports both Gen1 and Gen2 signatures, but here // given in input can only be Gen2 signature, the return can only ben Gen2 signature. diff --git a/packages/storage/src/internals/types/inputs.ts b/packages/storage/src/internals/types/inputs.ts index 94c92ee849b..a79d171ff08 100644 --- a/packages/storage/src/internals/types/inputs.ts +++ b/packages/storage/src/internals/types/inputs.ts @@ -26,6 +26,7 @@ import { Permission, PrefixType, Privilege } from './common'; export interface ListCallerAccessGrantsInput extends ListLocationsInput { accountId: string; credentialsProvider: CredentialsProvider; + customEndpoint?: string; region: string; } @@ -35,6 +36,7 @@ export interface ListCallerAccessGrantsInput extends ListLocationsInput { export interface GetDataAccessInput { accountId: string; credentialsProvider: CredentialsProvider; + customEndpoint?: string; durationSeconds?: number; permission: Permission; prefixType?: PrefixType; @@ -43,14 +45,17 @@ export interface GetDataAccessInput { scope: string; } +export interface AdvancedOptions { + locationCredentialsProvider?: CredentialsProvider; + customEndpoint?: string; +} + /** * @internal */ export type ListAllInput = ExtendInputWithAdvancedOptions< ListAllWithPathInput, - { - locationCredentialsProvider?: CredentialsProvider; - } + AdvancedOptions >; /** @@ -58,9 +63,7 @@ export type ListAllInput = ExtendInputWithAdvancedOptions< */ export type ListPaginateInput = ExtendInputWithAdvancedOptions< ListPaginateWithPathInput, - { - locationCredentialsProvider?: CredentialsProvider; - } + AdvancedOptions >; /** @@ -73,9 +76,7 @@ export type ListInput = ListAllInput | ListPaginateInput; */ export type RemoveInput = ExtendInputWithAdvancedOptions< RemoveWithPathInput, - { - locationCredentialsProvider?: CredentialsProvider; - } + AdvancedOptions >; /** @@ -83,9 +84,7 @@ export type RemoveInput = ExtendInputWithAdvancedOptions< */ export type GetPropertiesInput = ExtendInputWithAdvancedOptions< GetPropertiesWithPathInput, - { - locationCredentialsProvider?: CredentialsProvider; - } + AdvancedOptions >; /** @@ -93,9 +92,7 @@ export type GetPropertiesInput = ExtendInputWithAdvancedOptions< */ export type GetUrlInput = ExtendInputWithAdvancedOptions< GetUrlWithPathInput, - { - locationCredentialsProvider?: CredentialsProvider; - } + AdvancedOptions >; /** @@ -103,16 +100,12 @@ export type GetUrlInput = ExtendInputWithAdvancedOptions< */ export type CopyInput = ExtendCopyInputWithAdvancedOptions< CopyWithPathInput, - { - locationCredentialsProvider?: CredentialsProvider; - } + AdvancedOptions >; export type UploadDataInput = ExtendInputWithAdvancedOptions< UploadDataWithPathInput, - { - locationCredentialsProvider?: CredentialsProvider; - } + AdvancedOptions >; /** @@ -120,9 +113,7 @@ export type UploadDataInput = ExtendInputWithAdvancedOptions< */ export type DownloadDataInput = ExtendInputWithAdvancedOptions< DownloadDataWithPathInput, - { - locationCredentialsProvider?: CredentialsProvider; - } + AdvancedOptions >; /** diff --git a/packages/storage/src/providers/s3/apis/internal/copy.ts b/packages/storage/src/providers/s3/apis/internal/copy.ts index 5098096a81f..85898ea228f 100644 --- a/packages/storage/src/providers/s3/apis/internal/copy.ts +++ b/packages/storage/src/providers/s3/apis/internal/copy.ts @@ -80,6 +80,7 @@ const copyWithPath = async ( path: input.destination.path, options: { locationCredentialsProvider: input.options?.locationCredentialsProvider, + customEndpoint: input.options?.customEndpoint, ...input.destination, }, }); // resolveS3ConfigAndInput does not make extra API calls or storage access if called repeatedly. diff --git a/packages/storage/src/providers/s3/utils/client/s3control/base.ts b/packages/storage/src/providers/s3/utils/client/s3control/base.ts index 590f2b26120..455c9ac4cbc 100644 --- a/packages/storage/src/providers/s3/utils/client/s3control/base.ts +++ b/packages/storage/src/providers/s3/utils/client/s3control/base.ts @@ -12,6 +12,8 @@ import { } from '@aws-amplify/core/internals/aws-client-utils'; import { createRetryDecider, createXmlErrorParser } from '../utils'; +import { assertValidationError } from '../../../../../errors/utils/assertValidationError'; +import { StorageValidationErrorCode } from '../../../../../errors/types/validation'; /** * The service name used to sign requests if the API requires authentication. @@ -26,6 +28,20 @@ export const SERVICE_NAME = 's3'; export type S3EndpointResolverOptions = EndpointResolverOptions & { /** * Fully qualified custom endpoint for S3. If this is set, this endpoint will be used regardless of region. + * + * A fully qualified custom endpoint for S3. If set, this endpoint will override + * the default S3 control endpoint and be used regardless of the specified region configuration. + * + * Refer to AWS documentation for more details on available endpoints: + * https://docs.aws.amazon.com/general/latest/gr/s3.html#s3_region + * + * @example + * ```ts + * // Examples of S3 custom endpoints + * const endpoint1 = "s3-control.us-east-2.amazonaws.com"; + * const endpoint2 = "s3-control.dualstack.us-east-2.amazonaws.com"; + * const endpoint3 = "s3-control-fips.dualstack.us-east-2.amazonaws.com"; + * ``` */ customEndpoint?: string; }; @@ -35,22 +51,22 @@ export type S3EndpointResolverOptions = EndpointResolverOptions & { */ const endpointResolver = ( options: S3EndpointResolverOptions, - apiInput?: { AccountId?: string }, + apiInput?: { AccountId: string }, ) => { const { region, customEndpoint } = options; - const { AccountId: accountId } = apiInput || {}; + // TODO(ashwinkumar6): make accountId a required param + const { AccountId: accountId } = apiInput ?? {}; let endpoint: URL; - // 1. get base endpoint + if (customEndpoint) { - endpoint = new AmplifyUrl(customEndpoint); - } else if (accountId) { - // Control plane operations - endpoint = new AmplifyUrl( - `https://${accountId}.s3-control.${region}.${getDnsSuffix(region)}`, + assertValidationError( + !customEndpoint.includes('://'), + StorageValidationErrorCode.InvalidCustomEndpoint, ); + endpoint = new AmplifyUrl(`https://${accountId}.${customEndpoint}`); } else { endpoint = new AmplifyUrl( - `https://s3-control.${region}.${getDnsSuffix(region)}`, + `https://${accountId}.s3-control.${region}.${getDnsSuffix(region)}`, ); } diff --git a/packages/storage/src/providers/s3/utils/client/s3data/base.ts b/packages/storage/src/providers/s3/utils/client/s3data/base.ts index c7aef5c033c..fdf6160d077 100644 --- a/packages/storage/src/providers/s3/utils/client/s3data/base.ts +++ b/packages/storage/src/providers/s3/utils/client/s3data/base.ts @@ -12,6 +12,9 @@ import { } from '@aws-amplify/core/internals/aws-client-utils'; import { createRetryDecider, createXmlErrorParser } from '../utils'; +import { LOCAL_TESTING_S3_ENDPOINT } from '../../constants'; +import { assertValidationError } from '../../../../../errors/utils/assertValidationError'; +import { StorageValidationErrorCode } from '../../../../../errors/types/validation'; const DOMAIN_PATTERN = /^[a-z0-9][a-z0-9.-]{1,61}[a-z0-9]$/; const IP_ADDRESS_PATTERN = /(\d+\.){3}\d+/; @@ -33,12 +36,22 @@ export type S3EndpointResolverOptions = EndpointResolverOptions & { */ useAccelerateEndpoint?: boolean; /** - * Fully qualified custom endpoint for S3. If this is set, this endpoint will be used regardless of region or - * useAccelerateEndpoint config. - * The path of this endpoint + * A fully qualified custom endpoint for S3. If set, this endpoint will override + * the default S3 endpoint and be used regardless of the specified region or + * `useAccelerateEndpoint` configuration. + * + * Refer to AWS documentation for more details on available endpoints: + * https://docs.aws.amazon.com/general/latest/gr/s3.html#s3_region + * + * @example + * ```ts + * // Examples of S3 custom endpoints + * const endpoint1 = "s3.us-east-2.amazonaws.com"; + * const endpoint2 = "s3.dualstack.us-east-2.amazonaws.com"; + * const endpoint3 = "s3-fips.dualstack.us-east-2.amazonaws.com"; + * ``` */ customEndpoint?: string; - /** * Whether to force path style URLs for S3 objects (e.g., https://s3.amazonaws.com// instead of * https://.s3.amazonaws.com/ @@ -59,22 +72,31 @@ const endpointResolver = ( let endpoint: URL; // 1. get base endpoint if (customEndpoint) { - endpoint = new AmplifyUrl(customEndpoint); - } else if (useAccelerateEndpoint) { - if (forcePathStyle) { - throw new Error( - 'Path style URLs are not supported with S3 Transfer Acceleration.', - ); + if (customEndpoint === LOCAL_TESTING_S3_ENDPOINT) { + endpoint = new AmplifyUrl(customEndpoint); } + assertValidationError( + !customEndpoint.includes('://'), + StorageValidationErrorCode.InvalidCustomEndpoint, + ); + endpoint = new AmplifyUrl(`https://${customEndpoint}`); + } else if (useAccelerateEndpoint) { + // this ErrorCode isn't expose yet since forcePathStyle param isn't publicly exposed + assertValidationError( + !forcePathStyle, + StorageValidationErrorCode.ForcePathStyleEndpointNotSupported, + ); endpoint = new AmplifyUrl(`https://s3-accelerate.${getDnsSuffix(region)}`); } else { endpoint = new AmplifyUrl(`https://s3.${region}.${getDnsSuffix(region)}`); } // 2. inject bucket name if (apiInput?.Bucket) { - if (!isDnsCompatibleBucketName(apiInput.Bucket)) { - throw new Error(`Invalid bucket name: "${apiInput.Bucket}".`); - } + assertValidationError( + isDnsCompatibleBucketName(apiInput.Bucket), + StorageValidationErrorCode.DnsIncompatibleBucketName, + ); + if (forcePathStyle || apiInput.Bucket.includes('.')) { endpoint.pathname = `/${apiInput.Bucket}`; } else { diff --git a/packages/storage/src/providers/s3/utils/resolveS3ConfigAndInput.ts b/packages/storage/src/providers/s3/utils/resolveS3ConfigAndInput.ts index f19be5ddca7..7cb4c55316e 100644 --- a/packages/storage/src/providers/s3/utils/resolveS3ConfigAndInput.ts +++ b/packages/storage/src/providers/s3/utils/resolveS3ConfigAndInput.ts @@ -29,6 +29,7 @@ interface S3ApiOptions { targetIdentityId?: string; useAccelerateEndpoint?: boolean; locationCredentialsProvider?: LocationCredentialsProvider; + customEndpoint?: string; bucket?: StorageBucket; } @@ -133,6 +134,9 @@ export const resolveS3ConfigAndInput = async ( credentials: credentialsProvider, region, useAccelerateEndpoint: apiOptions?.useAccelerateEndpoint, + ...(apiOptions?.customEndpoint + ? { customEndpoint: apiOptions.customEndpoint } + : {}), ...(dangerouslyConnectToHttpEndpointForTesting ? { customEndpoint: LOCAL_TESTING_S3_ENDPOINT,