From 084b4ff078211397eb99055d59c03982acf44467 Mon Sep 17 00:00:00 2001 From: AllanZhengYP Date: Thu, 11 Jul 2024 11:15:39 -0700 Subject: [PATCH] feat(storage): add cred store creation implementation (#13575) Co-authored-by: israx <70438514+israx@users.noreply.github.com> --- packages/aws-amplify/package.json | 14 +- .../locationCredentialsStore/create.test.ts | 152 ++++++++++++- .../validators.test.ts | 208 ++++++++++++++++++ .../storage/src/errors/types/validation.ts | 26 ++- .../locationCredentialsStore/constants.ts | 5 + .../locationCredentialsStore/create.ts | 68 +++++- .../locationCredentialsStore/store.ts | 11 +- .../locationCredentialsStore/validators.ts | 86 ++++++++ 8 files changed, 548 insertions(+), 22 deletions(-) create mode 100644 packages/storage/__tests__/storageBrowser/locationCredentialsStore/validators.test.ts create mode 100644 packages/storage/src/storageBrowser/locationCredentialsStore/constants.ts create mode 100644 packages/storage/src/storageBrowser/locationCredentialsStore/validators.ts diff --git a/packages/aws-amplify/package.json b/packages/aws-amplify/package.json index c1ba562ba9c..d4cfecc01d7 100644 --- a/packages/aws-amplify/package.json +++ b/packages/aws-amplify/package.json @@ -461,43 +461,43 @@ "name": "[Storage] copy (S3)", "path": "./dist/esm/storage/index.mjs", "import": "{ copy }", - "limit": "14.71 kB" + "limit": "14.9 kB" }, { "name": "[Storage] downloadData (S3)", "path": "./dist/esm/storage/index.mjs", "import": "{ downloadData }", - "limit": "15.31 kB" + "limit": "15.49 kB" }, { "name": "[Storage] getProperties (S3)", "path": "./dist/esm/storage/index.mjs", "import": "{ getProperties }", - "limit": "14.58 kB" + "limit": "14.77 kB" }, { "name": "[Storage] getUrl (S3)", "path": "./dist/esm/storage/index.mjs", "import": "{ getUrl }", - "limit": "15.68 kB" + "limit": "15.87 kB" }, { "name": "[Storage] list (S3)", "path": "./dist/esm/storage/index.mjs", "import": "{ list }", - "limit": "15.18 kB" + "limit": "15.36 kB" }, { "name": "[Storage] remove (S3)", "path": "./dist/esm/storage/index.mjs", "import": "{ remove }", - "limit": "14.45 kB" + "limit": "14.63 kB" }, { "name": "[Storage] uploadData (S3)", "path": "./dist/esm/storage/index.mjs", "import": "{ uploadData }", - "limit": "19.77 kB" + "limit": "19.94 kB" } ] } diff --git a/packages/storage/__tests__/storageBrowser/locationCredentialsStore/create.test.ts b/packages/storage/__tests__/storageBrowser/locationCredentialsStore/create.test.ts index 1bd8f08c6a2..fa1ba483ada 100644 --- a/packages/storage/__tests__/storageBrowser/locationCredentialsStore/create.test.ts +++ b/packages/storage/__tests__/storageBrowser/locationCredentialsStore/create.test.ts @@ -1,17 +1,157 @@ // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 +import { AWSCredentials } from '@aws-amplify/core/internals/utils'; + +import { createLocationCredentialsStore } from '../../../src/storageBrowser/locationCredentialsStore/create'; +import { + createStore, + getValue, + removeStore, +} from '../../../src/storageBrowser/locationCredentialsStore/registry'; +import { validateCredentialsProviderLocation } from '../../../src/storageBrowser/locationCredentialsStore/validators'; +import { LocationCredentialsStore } from '../../../src/storageBrowser/types'; +import { + StorageValidationErrorCode, + validationErrorMap, +} from '../../../src/errors/types/validation'; + +jest.mock('../../../src/storageBrowser/locationCredentialsStore/registry'); +jest.mock('../../../src/storageBrowser/locationCredentialsStore/validators'); + +const mockedCredentials = 'MOCK_CREDS' as any as AWSCredentials; describe('createLocationCredentialsStore', () => { - it.todo('should create a store'); + it('should create a store', () => { + const refreshHandler = jest.fn(); + const store = createLocationCredentialsStore({ handler: refreshHandler }); + + expect(createStore).toHaveBeenCalledWith(refreshHandler); + expect(store.getProvider).toBeDefined(); + expect(store.destroy).toBeDefined(); + }); + describe('created store', () => { - describe('getValue()', () => { - it.todo('should call getValue() from store'); - it.todo( - 'should validate credentials location with resolved common scope', + describe('getProvider()', () => { + let store: LocationCredentialsStore; + + beforeEach(() => { + store = createLocationCredentialsStore({ handler: jest.fn() }); + }); + + afterEach(() => { + jest.clearAllMocks(); + store.destroy(); + }); + + it('should call getValue() from store', async () => { + expect.assertions(2); + jest + .mocked(getValue) + .mockResolvedValue({ credentials: mockedCredentials }); + + const locationCredentialsProvider = store.getProvider({ + scope: 's3://bucket/path/*', + permission: 'READ', + }); + const { credentials } = await locationCredentialsProvider({ + locations: [{ bucket: 'bucket', path: 'path/to/object' }], + permission: 'READ', + }); + expect(credentials).toEqual(mockedCredentials); + expect(getValue).toHaveBeenCalledWith( + expect.objectContaining({ + location: { + scope: 's3://bucket/path/*', + permission: 'READ', + }, + forceRefresh: false, + }), + ); + }); + + it('should validate credentials location with resolved common scope', async () => { + expect.assertions(1); + jest + .mocked(getValue) + .mockResolvedValue({ credentials: mockedCredentials }); + const locationCredentialsProvider = store.getProvider({ + scope: 's3://bucket/path/*', + permission: 'READWRITE', + }); + await locationCredentialsProvider({ + locations: [ + { bucket: 'bucket', path: 'path/to/object' }, + { bucket: 'bucket', path: 'path/to/object2' }, + { bucket: 'bucket', path: 'path/folder' }, + ], + permission: 'READ', + }); + expect(validateCredentialsProviderLocation).toHaveBeenCalledWith( + { bucket: 'bucket', path: 'path/', permission: 'READ' }, + { bucket: 'bucket', path: 'path/*', permission: 'READWRITE' }, + ); + }); + + it('should throw if action requires cross-bucket permission', async () => { + expect.assertions(1); + jest + .mocked(getValue) + .mockResolvedValue({ credentials: mockedCredentials }); + const locationCredentialsProvider = store.getProvider({ + scope: 's3://bucket/path/*', + permission: 'READWRITE', + }); + try { + await locationCredentialsProvider({ + locations: [ + { bucket: 'bucket-1', path: 'path/to/object' }, + { bucket: 'bucket-2', path: 'path/to/object2' }, + ], + permission: 'READ', + }); + } catch (e: any) { + expect(e.message).toEqual( + validationErrorMap[ + StorageValidationErrorCode.LocationCredentialsCrossBucket + ].message, + ); + } + }); + + it.each(['invalid-s3-uri', 's3://', 's3:///'])( + 'should throw if location credentials provider scope is not a valid S3 URI "%s"', + async invalidScope => { + expect.assertions(1); + jest + .mocked(getValue) + .mockResolvedValue({ credentials: mockedCredentials }); + const locationCredentialsProvider = store.getProvider({ + scope: invalidScope, + permission: 'READWRITE', + }); + try { + await locationCredentialsProvider({ + locations: [{ bucket: 'XXXXXXXX', path: 'path/to/object' }], + permission: 'READ', + }); + } catch (e: any) { + expect(e.message).toEqual( + validationErrorMap[StorageValidationErrorCode.InvalidS3Uri] + .message, + ); + } + }, ); }); + describe('destroy()', () => { - it.todo('should call removeStore() from store'); + it('should call removeStore() from store', () => { + const store = createLocationCredentialsStore({ + handler: jest.fn(), + }); + store.destroy(); + expect(removeStore).toHaveBeenCalled(); + }); }); }); }); diff --git a/packages/storage/__tests__/storageBrowser/locationCredentialsStore/validators.test.ts b/packages/storage/__tests__/storageBrowser/locationCredentialsStore/validators.test.ts new file mode 100644 index 00000000000..774b5663d34 --- /dev/null +++ b/packages/storage/__tests__/storageBrowser/locationCredentialsStore/validators.test.ts @@ -0,0 +1,208 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { validateCredentialsProviderLocation } from '../../../src/storageBrowser/locationCredentialsStore/validators'; +import { + StorageValidationErrorCode, + validationErrorMap, +} from '../../../src/errors/types/validation'; + +jest.mock('../../../src/storageBrowser/locationCredentialsStore/registry'); + +const mockBucket = 'MOCK_BUCKET'; + +describe('validateCredentialsProviderLocation', () => { + it('should NOT throw if action path matches credentials path prefix', () => { + expect(() => { + validateCredentialsProviderLocation( + { + bucket: mockBucket, + path: 'path/to/object', + permission: 'READ', + }, + { + bucket: mockBucket, + path: 'path/to/*', + permission: 'READ', + }, + ); + }).not.toThrow(); + }); + + it('should throw if action path does not path credentials path prefix', () => { + expect(() => { + validateCredentialsProviderLocation( + { + bucket: mockBucket, + path: 'path/to/object', + permission: 'READ', + }, + { + bucket: mockBucket, + path: 'path/to/other/*', + permission: 'READ', + }, + ); + }).toThrow( + validationErrorMap[ + StorageValidationErrorCode.LocationCredentialsPathMismatch + ].message, + ); + }); + + it('should NOT throw if action path matches credentials object path', () => { + expect(() => { + validateCredentialsProviderLocation( + { + bucket: mockBucket, + path: 'path/to/object', + permission: 'READ', + }, + { + bucket: mockBucket, + path: 'path/to/object', + permission: 'READ', + }, + ); + }).not.toThrow(); + }); + + it('should throw if action path does not match credentials object path', () => { + expect(() => { + validateCredentialsProviderLocation( + { + bucket: mockBucket, + path: 'path/to/object', + permission: 'READ', + }, + { + bucket: mockBucket, + path: 'path/to/object2', + permission: 'READ', + }, + ); + }).toThrow( + validationErrorMap[ + StorageValidationErrorCode.LocationCredentialsPathMismatch + ].message, + ); + }); + + it('should throw if action bucket and credentials bucket does not match', () => { + expect(() => { + validateCredentialsProviderLocation( + { + bucket: 'bucket-1', + path: 'path/to/object', + permission: 'READ', + }, + { + bucket: 'bucket-2', + path: 'path/to/object', + permission: 'READ', + }, + ); + }).toThrow( + validationErrorMap[ + StorageValidationErrorCode.LocationCredentialsBucketMismatch + ].message, + ); + }); + + it('should not throw if READ action permission matches READWRITE credentials permission', () => { + expect(() => { + validateCredentialsProviderLocation( + { + bucket: mockBucket, + path: 'path/to/object', + permission: 'READ', + }, + { + bucket: mockBucket, + path: 'path/to/*', + permission: 'READWRITE', + }, + ); + }).not.toThrow(); + }); + + it('should not throw if WRITE action permission matches READWRITE credentials permission', () => { + expect(() => { + validateCredentialsProviderLocation( + { + bucket: mockBucket, + path: 'path/to/object', + permission: 'WRITE', + }, + { + bucket: mockBucket, + path: 'path/to/*', + permission: 'READWRITE', + }, + ); + }).not.toThrow(); + }); + + it('should throw if READ action permission does not match WRITE credentials permission', () => { + expect(() => { + validateCredentialsProviderLocation( + { + bucket: mockBucket, + path: 'path/to/object', + permission: 'READ', + }, + { + bucket: mockBucket, + path: 'path/to/*', + permission: 'WRITE', + }, + ); + }).toThrow( + validationErrorMap[ + StorageValidationErrorCode.LocationCredentialsPermissionMismatch + ].message, + ); + }); + + it('should throw if WRITE action permission does not match READ credentials permission', () => { + expect(() => { + validateCredentialsProviderLocation( + { + bucket: mockBucket, + path: 'path/to/object', + permission: 'WRITE', + }, + { + bucket: mockBucket, + path: 'path/to/*', + permission: 'READ', + }, + ); + }).toThrow( + validationErrorMap[ + StorageValidationErrorCode.LocationCredentialsPermissionMismatch + ].message, + ); + }); + + it('should throw if READWRITE action permission does not match READ credentials permission', () => { + expect(() => { + validateCredentialsProviderLocation( + { + bucket: mockBucket, + path: 'path/to/object', + permission: 'READWRITE', + }, + { + bucket: mockBucket, + path: 'path/to/*', + permission: 'READ', + }, + ); + }).toThrow( + validationErrorMap[ + StorageValidationErrorCode.LocationCredentialsPermissionMismatch + ].message, + ); + }); +}); diff --git a/packages/storage/src/errors/types/validation.ts b/packages/storage/src/errors/types/validation.ts index 5741f807874..281272863de 100644 --- a/packages/storage/src/errors/types/validation.ts +++ b/packages/storage/src/errors/types/validation.ts @@ -21,8 +21,17 @@ export enum StorageValidationErrorCode { UrlExpirationMaxLimitExceed = 'UrlExpirationMaxLimitExceed', InvalidLocationCredentialsCacheSize = 'InvalidLocationCredentialsCacheSize', LocationCredentialsStoreDestroyed = 'LocationCredentialsStoreDestroyed', + LocationCredentialsBucketMismatch = 'LocationCredentialsBucketMismatch', + LocationCredentialsCrossBucket = 'LocationCredentialsCrossBucket', + LocationCredentialsPathMismatch = 'LocationCredentialsPathMismatch', + LocationCredentialsPermissionMismatch = 'LocationCredentialsPermissionMismatch', + InvalidS3Uri = 'InvalidS3Uri', } +// Common error message strings to save some bytes +const LOCATION_SPECIFIC_CREDENTIALS = 'Location-specific credentials'; +const DOES_NOT_MATCH = 'does not match that required for the API call'; + export const validationErrorMap: AmplifyErrorMap = { [StorageValidationErrorCode.NoCredentials]: { message: 'Credentials should not be empty.', @@ -76,6 +85,21 @@ export const validationErrorMap: AmplifyErrorMap = { message: 'locationCredentialsCacheSize must be a positive integer.', }, [StorageValidationErrorCode.LocationCredentialsStoreDestroyed]: { - message: 'The location-specific credentials store has been destroyed', + message: `${LOCATION_SPECIFIC_CREDENTIALS} store has been destroyed.`, + }, + [StorageValidationErrorCode.InvalidS3Uri]: { + message: 'Invalid S3 URI.', + }, + [StorageValidationErrorCode.LocationCredentialsCrossBucket]: { + message: `${LOCATION_SPECIFIC_CREDENTIALS} cannot be used across buckets.`, + }, + [StorageValidationErrorCode.LocationCredentialsBucketMismatch]: { + message: `${LOCATION_SPECIFIC_CREDENTIALS} bucket ${DOES_NOT_MATCH}.`, + }, + [StorageValidationErrorCode.LocationCredentialsPathMismatch]: { + message: `${LOCATION_SPECIFIC_CREDENTIALS} path ${DOES_NOT_MATCH}.`, + }, + [StorageValidationErrorCode.LocationCredentialsPermissionMismatch]: { + message: `${LOCATION_SPECIFIC_CREDENTIALS} permission ${DOES_NOT_MATCH}.`, }, }; diff --git a/packages/storage/src/storageBrowser/locationCredentialsStore/constants.ts b/packages/storage/src/storageBrowser/locationCredentialsStore/constants.ts new file mode 100644 index 00000000000..67b22505b95 --- /dev/null +++ b/packages/storage/src/storageBrowser/locationCredentialsStore/constants.ts @@ -0,0 +1,5 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +export const CREDENTIALS_STORE_DEFAULT_SIZE = 10; +export const CREDENTIALS_REFRESH_WINDOW_MS = 30_000; diff --git a/packages/storage/src/storageBrowser/locationCredentialsStore/create.ts b/packages/storage/src/storageBrowser/locationCredentialsStore/create.ts index d4f91341b06..dc8da21bff5 100644 --- a/packages/storage/src/storageBrowser/locationCredentialsStore/create.ts +++ b/packages/storage/src/storageBrowser/locationCredentialsStore/create.ts @@ -1,5 +1,3 @@ -/* eslint-disable unused-imports/no-unused-vars */ - // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 @@ -8,9 +6,15 @@ import { LocationCredentialsHandler, LocationCredentialsStore, } from '../types'; -import { LocationCredentialsProvider } from '../../providers/s3/types/options'; +import { StorageValidationErrorCode } from '../../errors/types/validation'; +import { assertValidationError } from '../../errors/utils/assertValidationError'; +import { + BucketLocation, + LocationCredentialsProvider, +} from '../../providers/s3/types/options'; import { createStore, getValue, removeStore } from './registry'; +import { validateCredentialsProviderLocation } from './validators'; export const createLocationCredentialsStore = (input: { handler: LocationCredentialsHandler; @@ -24,7 +28,18 @@ export const createLocationCredentialsStore = (input: { locations, forceRefresh = false, }: Parameters[0]) => { - // TODO(@AllanZhengYP) validate input + const actionBucketLocation = resolveCommonBucketLocation(locations); + const providerBucketLocation = parseS3Uri(providerLocation.scope); + validateCredentialsProviderLocation( + { + ...actionBucketLocation, + permission, + }, + { + ...providerBucketLocation, + permission: providerLocation.permission, + }, + ); return getValue({ storeSymbol, @@ -43,3 +58,48 @@ export const createLocationCredentialsStore = (input: { return store; }; + +type S3Uri = string; + +const parseS3Uri = (uri: S3Uri): BucketLocation => { + const s3UrlSchemaRegex = /^s3:\/\//; + // TODO(@AllanZhengYP): Provide more info to error message: url + assertValidationError( + s3UrlSchemaRegex.test(uri), + StorageValidationErrorCode.InvalidS3Uri, + ); + const [bucket, ...pathParts] = uri.replace(s3UrlSchemaRegex, '').split('/'); + assertValidationError(!!bucket, StorageValidationErrorCode.InvalidS3Uri); + const path = pathParts.join('/'); + + return { + bucket, + path, + }; +}; + +/** + * Given a list of bucket and path combinations, verify they have the same + * bucket and resolves the longest common prefix for multiple given paths. + */ +const resolveCommonBucketLocation = ( + locations: BucketLocation[], +): BucketLocation => { + let { bucket: commonBucket, path: commonPath } = locations[0]; + + for (const location of locations) { + const { bucket, path } = location; + assertValidationError( + bucket === commonBucket, + StorageValidationErrorCode.LocationCredentialsCrossBucket, + ); + while (commonPath !== '' && !path.startsWith(commonPath)) { + commonPath = commonPath.slice(0, -1); + } + } + + return { + bucket: commonBucket, + path: commonPath, + }; +}; diff --git a/packages/storage/src/storageBrowser/locationCredentialsStore/store.ts b/packages/storage/src/storageBrowser/locationCredentialsStore/store.ts index 1253d010b22..ac1901d669b 100644 --- a/packages/storage/src/storageBrowser/locationCredentialsStore/store.ts +++ b/packages/storage/src/storageBrowser/locationCredentialsStore/store.ts @@ -10,17 +10,19 @@ import { CredentialsLocation, LocationCredentialsHandler } from '../types'; import { assertValidationError } from '../../errors/utils/assertValidationError'; import { StorageValidationErrorCode } from '../../errors/types/validation'; -const CREDENTIALS_STORE_DEFAULT_SIZE = 10; -const CREDENTIALS_REFRESH_WINDOW_MS = 30_000; +import { + CREDENTIALS_REFRESH_WINDOW_MS, + CREDENTIALS_STORE_DEFAULT_SIZE, +} from './constants'; interface StoreValue extends CredentialsLocation { credentials?: AWSCredentials; inflightCredentials?: Promise<{ credentials: AWSCredentials }>; } -type S3Url = string; +type S3Uri = string; -type CacheKey = `${S3Url}_${Permission}`; +type CacheKey = `${S3Uri}_${Permission}`; const createCacheKey = (location: CredentialsLocation): CacheKey => `${location.scope}_${location.permission}`; @@ -153,6 +155,7 @@ const setCacheRecord = ( // So first key is the last recently inserted. const [oldestKey] = store.values.keys(); store.values.delete(oldestKey); + // TODO(@AllanZhengYP): Add log info when record is evicted. } // Add latest used value to the cache. store.values.set(key, value); diff --git a/packages/storage/src/storageBrowser/locationCredentialsStore/validators.ts b/packages/storage/src/storageBrowser/locationCredentialsStore/validators.ts new file mode 100644 index 00000000000..3409a899fdf --- /dev/null +++ b/packages/storage/src/storageBrowser/locationCredentialsStore/validators.ts @@ -0,0 +1,86 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { StorageValidationErrorCode } from '../../errors/types/validation'; +import { assertValidationError } from '../../errors/utils/assertValidationError'; +import { BucketLocation, Permission } from '../../providers/s3/types/options'; + +interface CredentialsBucketLocation extends BucketLocation { + permission: Permission; +} + +/** + * @internal + */ +export const validateCredentialsProviderLocation = ( + actionLocation: CredentialsBucketLocation, + providerLocation: CredentialsBucketLocation, +): void => { + validateLocationBucket({ + actionBucket: actionLocation.bucket, + providerBucket: providerLocation.bucket, + }); + validateLocationPath({ + actionPath: actionLocation.path, + providerPath: providerLocation.path, + }); + validateLocationPermission({ + actionPermission: actionLocation.permission, + providerPermission: providerLocation.permission, + }); +}; + +const validateLocationBucket = (input: { + actionBucket: string; + providerBucket?: string; +}): void => { + const { actionBucket, providerBucket } = input; + if (!providerBucket) { + return; + } + assertValidationError( + actionBucket === providerBucket, + StorageValidationErrorCode.LocationCredentialsBucketMismatch, + ); +}; + +const validateLocationPath = (input: { + actionPath: string; + providerPath?: string; +}): void => { + const { actionPath, providerPath } = input; + if (!providerPath) { + return; + } + if (providerPath.endsWith('*')) { + // Verify if the action path has prefix required by the provider; + const providerPathPrefix = providerPath.replace(/\*$/, ''); + assertValidationError( + actionPath.startsWith(providerPathPrefix), + StorageValidationErrorCode.LocationCredentialsPathMismatch, + ); + } else { + // If provider path is scoped to an object, verify if the action path points to the same object. + // TODO(@AllanZhengYP) Provider more info in error message: actionPath, providerPath. + assertValidationError( + actionPath === providerPath, + StorageValidationErrorCode.LocationCredentialsPathMismatch, + ); + } +}; + +const validateLocationPermission = (input: { + actionPermission: Permission; + providerPermission?: Permission; +}) => { + const { actionPermission, providerPermission } = input; + if (!providerPermission) { + return; + } + // TODO(@AllanZhengYP) Provide more info in error message: `API needs permission ${actionPermission}, but provided + // location credentials provider with permission ${providerPermission}.` + assertValidationError( + providerPermission.includes(actionPermission), + StorageValidationErrorCode.LocationCredentialsPermissionMismatch, + ); +};