From feba83a462420d47570d2a144de015b039102e1f Mon Sep 17 00:00:00 2001 From: Kirill Date: Sun, 11 Feb 2024 23:05:34 +0100 Subject: [PATCH] Upgrade AWS SDK to v2 Initial attempt --- pom.xml | 19 +++- .../repository/S3ConfigurationRepository.java | 6 +- .../S3ConfigurationRepositoryBuilder.java | 26 +++-- .../baigan/repository/aws/S3FileLoader.java | 32 +++--- .../S3ConfigurationRepositoryEnd2EndIT.java | 99 ++++++++++--------- 5 files changed, 101 insertions(+), 81 deletions(-) diff --git a/pom.xml b/pom.xml index ce0318b..644094b 100644 --- a/pom.xml +++ b/pom.xml @@ -48,7 +48,7 @@ 2.0.7 5.3.1 2.2 - 1.12.475 + 2.26.15 1.19.1 @@ -61,6 +61,13 @@ pom import + + software.amazon.awssdk + bom + ${awssdk.version} + pom + import + @@ -111,10 +118,14 @@ - com.amazonaws - aws-java-sdk-s3 - ${awssdk.version} + software.amazon.awssdk + s3 + + software.amazon.awssdk + kms + + diff --git a/src/main/java/org/zalando/baigan/repository/S3ConfigurationRepository.java b/src/main/java/org/zalando/baigan/repository/S3ConfigurationRepository.java index 951fa03..d4634eb 100644 --- a/src/main/java/org/zalando/baigan/repository/S3ConfigurationRepository.java +++ b/src/main/java/org/zalando/baigan/repository/S3ConfigurationRepository.java @@ -1,12 +1,12 @@ package org.zalando.baigan.repository; -import com.amazonaws.services.kms.AWSKMS; -import com.amazonaws.services.s3.AmazonS3; import com.google.common.collect.ImmutableMap; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.zalando.baigan.model.Configuration; import org.zalando.baigan.repository.aws.S3FileLoader; +import software.amazon.awssdk.services.kms.KmsClient; +import software.amazon.awssdk.services.s3.S3Client; import javax.annotation.Nonnull; import java.time.Duration; @@ -35,7 +35,7 @@ public class S3ConfigurationRepository implements ConfigurationRepository { S3ConfigurationRepository(@Nonnull final String bucketName, @Nonnull final String key, final Duration refreshInterval, final ScheduledExecutorService executor, - final AmazonS3 s3Client, final AWSKMS kmsClient, ConfigurationParser configurationParser) { + final S3Client s3Client, final KmsClient kmsClient, ConfigurationParser configurationParser) { checkNotNull(bucketName, "bucketName is required"); checkNotNull(key, "key is required"); checkArgument(!refreshInterval.isNegative(), "refreshInterval has to be >= 0"); diff --git a/src/main/java/org/zalando/baigan/repository/S3ConfigurationRepositoryBuilder.java b/src/main/java/org/zalando/baigan/repository/S3ConfigurationRepositoryBuilder.java index c3664f9..502a3f4 100644 --- a/src/main/java/org/zalando/baigan/repository/S3ConfigurationRepositoryBuilder.java +++ b/src/main/java/org/zalando/baigan/repository/S3ConfigurationRepositoryBuilder.java @@ -1,10 +1,8 @@ package org.zalando.baigan.repository; -import com.amazonaws.services.kms.AWSKMS; -import com.amazonaws.services.kms.AWSKMSClientBuilder; -import com.amazonaws.services.s3.AmazonS3; -import com.amazonaws.services.s3.AmazonS3ClientBuilder; import com.fasterxml.jackson.databind.ObjectMapper; +import software.amazon.awssdk.services.kms.KmsClient; +import software.amazon.awssdk.services.s3.S3Client; import javax.annotation.Nonnull; import java.time.Duration; @@ -25,8 +23,8 @@ public class S3ConfigurationRepositoryBuilder { private ScheduledExecutorService executor; - private AmazonS3 s3Client; - private AWSKMS kmsClient; + private S3Client s3Client; + private KmsClient kmsClient; private Duration refreshInterval = Duration.ofMinutes(1); private String bucketName; private String key; @@ -39,20 +37,20 @@ public S3ConfigurationRepositoryBuilder(final ConfigurationParser configurationP /** * @param s3Client The S3 client to be used to fetch the configuration file. - * If the S3 client is not specified explicitly, the builder - * uses {@link AmazonS3ClientBuilder#defaultClient()} + * If the S3 client is not specified explicitly, Baigan builds a default + * client using {@link S3Client#builder()}. */ - public S3ConfigurationRepositoryBuilder s3Client(final AmazonS3 s3Client) { + public S3ConfigurationRepositoryBuilder s3Client(final S3Client s3Client) { this.s3Client = s3Client; return this; } /** * @param kmsClient The KMS client to be used to decrypt the configuration file. - * If the KMS client is not specified explicitly, the builder - * uses {@link AWSKMSClientBuilder#defaultClient()} + * If the KMS client is not specified explicitly, Baigan builds a default + * client using {@link KmsClient#builder()}. */ - public S3ConfigurationRepositoryBuilder kmsClient(final AWSKMS kmsClient) { + public S3ConfigurationRepositoryBuilder kmsClient(final KmsClient kmsClient) { this.kmsClient = kmsClient; return this; } @@ -115,10 +113,10 @@ public S3ConfigurationRepository build() { executor = new ScheduledThreadPoolExecutor(1); } if (s3Client == null) { - s3Client = AmazonS3ClientBuilder.defaultClient(); + s3Client = S3Client.builder().build(); } if (kmsClient == null) { - kmsClient = AWSKMSClientBuilder.defaultClient(); + kmsClient = KmsClient.builder().build(); } if (objectMapper != null) { configurationParser.setObjectMapper(objectMapper); diff --git a/src/main/java/org/zalando/baigan/repository/aws/S3FileLoader.java b/src/main/java/org/zalando/baigan/repository/aws/S3FileLoader.java index fd29d3f..b1ca966 100644 --- a/src/main/java/org/zalando/baigan/repository/aws/S3FileLoader.java +++ b/src/main/java/org/zalando/baigan/repository/aws/S3FileLoader.java @@ -1,13 +1,15 @@ package org.zalando.baigan.repository.aws; -import com.amazonaws.services.kms.AWSKMS; -import com.amazonaws.services.kms.model.DecryptRequest; -import com.amazonaws.services.kms.model.DependencyTimeoutException; -import com.amazonaws.services.kms.model.KMSInternalException; -import com.amazonaws.services.s3.AmazonS3; import com.google.common.io.BaseEncoding; import net.jodah.failsafe.Failsafe; import net.jodah.failsafe.RetryPolicy; +import software.amazon.awssdk.core.SdkBytes; +import software.amazon.awssdk.services.kms.KmsClient; +import software.amazon.awssdk.services.kms.model.DecryptRequest; +import software.amazon.awssdk.services.kms.model.DependencyTimeoutException; +import software.amazon.awssdk.services.kms.model.KmsInternalException; +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.model.GetObjectRequest; import javax.annotation.Nonnull; import java.nio.ByteBuffer; @@ -28,17 +30,17 @@ public class S3FileLoader { private static final int RETRY_SECONDS_WAIT = 10; private final RetryPolicy retryPolicy = new RetryPolicy() - .handle(KMSInternalException.class) + .handle(KmsInternalException.class) .handle(DependencyTimeoutException.class) .withBackoff(1, RETRY_SECONDS_WAIT, SECONDS) .withMaxRetries(MAX_RETRIES); - private final AmazonS3 s3Client; - private final AWSKMS kmsClient; + private final S3Client s3Client; + private final KmsClient kmsClient; private final String bucketName; private final String key; - public S3FileLoader(@Nonnull String bucketName, @Nonnull String key, @Nonnull AmazonS3 s3Client, @Nonnull AWSKMS kmsClient) { + public S3FileLoader(@Nonnull String bucketName, @Nonnull String key, @Nonnull S3Client s3Client, @Nonnull KmsClient kmsClient) { this.s3Client = s3Client; this.kmsClient = kmsClient; this.bucketName = bucketName; @@ -46,7 +48,11 @@ public S3FileLoader(@Nonnull String bucketName, @Nonnull String key, @Nonnull Am } public String loadContent() { - final String configurationText = s3Client.getObjectAsString(bucketName, key); + final GetObjectRequest request = GetObjectRequest.builder() + .bucket(bucketName) + .key(key) + .build(); + final String configurationText = s3Client.getObjectAsBytes(request).asUtf8String(); return decryptIfNecessary(configurationText); } @@ -68,8 +74,10 @@ private String decryptIfNecessary(final String candidate) { } private ByteBuffer decryptValue(final byte[] encryptedBytes) { - final DecryptRequest request = new DecryptRequest().withCiphertextBlob(ByteBuffer.wrap(encryptedBytes)); - return Failsafe.with(retryPolicy).get(() -> kmsClient.decrypt(request).getPlaintext()); + final DecryptRequest request = DecryptRequest.builder() + .ciphertextBlob(SdkBytes.fromByteArray(encryptedBytes)) + .build(); + return Failsafe.with(retryPolicy).get(() -> kmsClient.decrypt(request).plaintext().asByteBuffer()); } private static Optional getEncryptedValue(final String value) { diff --git a/src/test/java/org/zalando/baigan/e2e/s3repo/S3ConfigurationRepositoryEnd2EndIT.java b/src/test/java/org/zalando/baigan/e2e/s3repo/S3ConfigurationRepositoryEnd2EndIT.java index 855a905..d6037e1 100644 --- a/src/test/java/org/zalando/baigan/e2e/s3repo/S3ConfigurationRepositoryEnd2EndIT.java +++ b/src/test/java/org/zalando/baigan/e2e/s3repo/S3ConfigurationRepositoryEnd2EndIT.java @@ -1,15 +1,5 @@ package org.zalando.baigan.e2e.s3repo; -import com.amazonaws.auth.AWSStaticCredentialsProvider; -import com.amazonaws.auth.BasicAWSCredentials; -import com.amazonaws.client.builder.AwsClientBuilder; -import com.amazonaws.regions.Regions; -import com.amazonaws.services.kms.AWSKMS; -import com.amazonaws.services.kms.AWSKMSClientBuilder; -import com.amazonaws.services.s3.AmazonS3; -import com.amazonaws.services.s3.AmazonS3ClientBuilder; -import com.amazonaws.services.s3.model.AmazonS3Exception; -import com.amazonaws.services.s3.model.CreateBucketRequest; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.TestInstance; @@ -29,6 +19,15 @@ import org.zalando.baigan.e2e.configs.SomeConfiguration; import org.zalando.baigan.repository.RepositoryFactory; import org.zalando.baigan.repository.S3ConfigurationRepository; +import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; +import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; +import software.amazon.awssdk.core.sync.RequestBody; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.kms.KmsClient; +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.model.CreateBucketRequest; +import software.amazon.awssdk.services.s3.model.PutObjectRequest; +import software.amazon.awssdk.services.s3.model.S3Exception; import java.time.Duration; import java.util.List; @@ -48,11 +47,16 @@ public class S3ConfigurationRepositoryEnd2EndIT { public static final String S3_CONFIG_BUCKET = "some-bucket"; public static final String S3_CONFIG_KEY = "some-key"; + public static final PutObjectRequest PUT_OBJECT_REQUEST = PutObjectRequest.builder() + .bucket(S3_CONFIG_BUCKET) + .key(S3_CONFIG_KEY) + .build(); + private static final Duration CONFIG_REFRESH_INTERVAL = Duration.ofMillis(100); private static final long TIME_TO_WAIT_FOR_CONFIG_REFRESH = CONFIG_REFRESH_INTERVAL.plusMillis(100).toMillis(); @Autowired - private AmazonS3 s3; + private S3Client s3; @Autowired private SomeConfiguration someConfiguration; @@ -69,9 +73,10 @@ public void givenS3Configuration_whenConfigurationIsChangedOnS3_thenConfiguratio assertThat(someConfiguration.topLevelGenerics(), nullValue()); s3.putObject( - S3_CONFIG_BUCKET, - S3_CONFIG_KEY, - "[{\"alias\": \"some.configuration.some.value\", \"defaultValue\": \"some value\"}]" + PUT_OBJECT_REQUEST, + RequestBody.fromString( + "[{\"alias\": \"some.configuration.some.value\", \"defaultValue\": \"some value\"}]" + ) ); Thread.sleep(TIME_TO_WAIT_FOR_CONFIG_REFRESH); assertThat(someConfiguration.isThisTrue(), nullValue()); @@ -80,13 +85,14 @@ public void givenS3Configuration_whenConfigurationIsChangedOnS3_thenConfiguratio assertThat(someConfiguration.configList(), nullValue()); s3.putObject( - S3_CONFIG_BUCKET, - S3_CONFIG_KEY, + PUT_OBJECT_REQUEST, + RequestBody.fromString( "[{ \"alias\": \"some.configuration.some.config\", \"defaultValue\": {\"config_key\":\"a value\"}}," + "{ \"alias\": \"some.non.existing.config\", \"defaultValue\": {\"other_config_key\":\"other value\"}}," + "{ \"alias\": \"some.configuration.is.this.true\", \"defaultValue\": true}, " + "{ \"alias\": \"some.configuration.some.value\", \"defaultValue\": \"some value\"}, " + "{ \"alias\": \"some.configuration.config.list\", \"defaultValue\": [\"A\",\"B\"]}]" + ) ); Thread.sleep(TIME_TO_WAIT_FOR_CONFIG_REFRESH); assertThat(someConfiguration.someConfig(), equalTo(new SomeConfigObject("a value"))); @@ -98,19 +104,19 @@ public void givenS3Configuration_whenConfigurationIsChangedOnS3_thenConfiguratio @Test public void givenS3Configuration_whenTheS3FileIsUpdatedWithInvalidConfig_thenTheConfigurationIsNotUpdated() throws InterruptedException { s3.putObject( - S3_CONFIG_BUCKET, - S3_CONFIG_KEY, + PUT_OBJECT_REQUEST, + RequestBody.fromString( "[{\"alias\": \"some.configuration.is.this.true\", \"defaultValue\": true}, " + "{\"alias\": \"some.configuration.some.value\", \"defaultValue\": \"some value\"}]" + ) ); Thread.sleep(TIME_TO_WAIT_FOR_CONFIG_REFRESH); assertThat(someConfiguration.isThisTrue(), equalTo(true)); assertThat(someConfiguration.someValue(), equalTo("some value")); s3.putObject( - S3_CONFIG_BUCKET, - S3_CONFIG_KEY, - "an: invalid\"} config" + PUT_OBJECT_REQUEST, + RequestBody.fromString("an: invalid\"} config") ); Thread.sleep(TIME_TO_WAIT_FOR_CONFIG_REFRESH); assertThat(someConfiguration.isThisTrue(), equalTo(true)); @@ -135,11 +141,11 @@ ScheduledThreadPoolExecutor baiganRefresherPoolExecutor() { @Bean S3ConfigurationRepository configurationRepository( RepositoryFactory repositoryFactory, - AmazonS3 amazonS3, - AWSKMS kms, + S3Client amazonS3, + KmsClient kms, ScheduledThreadPoolExecutor executorService ) { - amazonS3.putObject(S3_CONFIG_BUCKET, S3_CONFIG_KEY, "[]"); + amazonS3.putObject(PUT_OBJECT_REQUEST, RequestBody.fromString("[]")); return repositoryFactory.s3ConfigurationRepository() .bucketName(S3_CONFIG_BUCKET) .key(S3_CONFIG_KEY) @@ -153,38 +159,35 @@ S3ConfigurationRepository configurationRepository( @Container private static final LocalStackContainer localstack = new LocalStackContainer( DockerImageName.parse("localstack/localstack:2.1.0") - ).withServices(S3, KMS).withEnv("DEFAULT_REGION", Regions.EU_CENTRAL_1.getName()); + ).withServices(S3, KMS).withEnv("DEFAULT_REGION", Region.EU_CENTRAL_1.id()); @Bean - AWSKMS kms() { + KmsClient kms() { localstack.start(); - return AWSKMSClientBuilder.standard().withEndpointConfiguration( - new AwsClientBuilder.EndpointConfiguration( - localstack.getEndpointOverride(KMS).toString(), localstack.getRegion() - ) - ).build(); + return KmsClient.builder() + .endpointOverride(localstack.getEndpoint()) + .region(Region.of(localstack.getRegion())) + .build(); } @Bean - AmazonS3 amazonS3() { + S3Client amazonS3() { localstack.start(); - AmazonS3 s3 = AmazonS3ClientBuilder.standard().withEndpointConfiguration( - new AwsClientBuilder.EndpointConfiguration( - localstack.getEndpointOverride(S3).toString(), localstack.getRegion() - ) - ).withCredentials( - new AWSStaticCredentialsProvider( - new BasicAWSCredentials(localstack.getAccessKey(), localstack.getSecretKey()) + S3Client s3 = S3Client + .builder() + .endpointOverride(localstack.getEndpoint()) + .credentialsProvider( + StaticCredentialsProvider.create( + AwsBasicCredentials.create(localstack.getAccessKey(), localstack.getSecretKey()) + ) ) - ).build(); - - try { - s3.createBucket(new CreateBucketRequest(S3_CONFIG_BUCKET, localstack.getRegion())); - } catch (AmazonS3Exception e) { - if (!e.getErrorCode().equals("BucketAlreadyOwnedByYou")) { - throw e; - } - } + .region(Region.of(localstack.getRegion())) + .build(); + + CreateBucketRequest request = CreateBucketRequest.builder() + .bucket(S3_CONFIG_BUCKET) + .build(); + s3.createBucket(request); return s3; }