From 3c75aa8310b30e549d60a3bd2e6de784b6c4fc1a Mon Sep 17 00:00:00 2001 From: BuildKite on behalf of Wellcome Collection Date: Thu, 11 Apr 2024 13:22:51 +0000 Subject: [PATCH 01/28] Bump scala-libs to v32.41.0 --- project/Dependencies.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project/Dependencies.scala b/project/Dependencies.scala index d6c485394..722fc5e95 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -1,7 +1,7 @@ import sbt._ object WellcomeDependencies { - val defaultVersion = "32.40.4" // This is automatically bumped by the scala-libs release process, do not edit this line manually + val defaultVersion = "32.41.0" // This is automatically bumped by the scala-libs release process, do not edit this line manually lazy val versions = new { val fixtures = defaultVersion From bf9ac5965e4fd4b2793ed7842a4f2231a7ee7d5b Mon Sep 17 00:00:00 2001 From: Agnes Garoux Date: Fri, 12 Apr 2024 09:25:36 +0100 Subject: [PATCH 02/28] Try localstack3.0.0 to fix sqs failure --- bag_replicator/docker-compose.yml | 36 ++++++++++++++++--------------- 1 file changed, 19 insertions(+), 17 deletions(-) diff --git a/bag_replicator/docker-compose.yml b/bag_replicator/docker-compose.yml index 0a34e490d..bd3344c21 100644 --- a/bag_replicator/docker-compose.yml +++ b/bag_replicator/docker-compose.yml @@ -1,17 +1,19 @@ -localstack: - image: "760097843905.dkr.ecr.eu-west-1.amazonaws.com/localstack/localstack" - environment: - - SERVICES=sqs - ports: - - "4566:4566" -s3: - image: "760097843905.dkr.ecr.eu-west-1.amazonaws.com/zenko/cloudserver:8.1.8" - environment: - - "S3BACKEND=mem" - ports: - - "33333:8000" -azurite: - image: "mcr.microsoft.com/azure-storage/azurite" - ports: - - "10000:10000" - command: ["azurite", "--blobHost", "0.0.0.0"] +version: "3.0.0" +services: + localstack: + image: "public.ecr.aws/localstack/localstack:3.0.0" + environment: + - SERVICES=sqs + ports: + - "4566:4566" + s3: + image: "760097843905.dkr.ecr.eu-west-1.amazonaws.com/zenko/cloudserver:8.1.8" + environment: + - "S3BACKEND=mem" + ports: + - "33333:8000" + azurite: + image: "mcr.microsoft.com/azure-storage/azurite" + ports: + - "10000:10000" + command: ["azurite", "--blobHost", "0.0.0.0"] From f35a18b0b40e8372000222715122bc2f9d713663 Mon Sep 17 00:00:00 2001 From: Agnes Garoux Date: Fri, 12 Apr 2024 09:49:55 +0100 Subject: [PATCH 03/28] Don't set the version --- bag_replicator/docker-compose.yml | 36 +++++++++++++++---------------- 1 file changed, 17 insertions(+), 19 deletions(-) diff --git a/bag_replicator/docker-compose.yml b/bag_replicator/docker-compose.yml index bd3344c21..612e7990f 100644 --- a/bag_replicator/docker-compose.yml +++ b/bag_replicator/docker-compose.yml @@ -1,19 +1,17 @@ -version: "3.0.0" -services: - localstack: - image: "public.ecr.aws/localstack/localstack:3.0.0" - environment: - - SERVICES=sqs - ports: - - "4566:4566" - s3: - image: "760097843905.dkr.ecr.eu-west-1.amazonaws.com/zenko/cloudserver:8.1.8" - environment: - - "S3BACKEND=mem" - ports: - - "33333:8000" - azurite: - image: "mcr.microsoft.com/azure-storage/azurite" - ports: - - "10000:10000" - command: ["azurite", "--blobHost", "0.0.0.0"] +localstack: + image: "public.ecr.aws/localstack/localstack:3.0.0" + environment: + - SERVICES=sqs + ports: + - "4566:4566" +s3: + image: "760097843905.dkr.ecr.eu-west-1.amazonaws.com/zenko/cloudserver:8.1.8" + environment: + - "S3BACKEND=mem" + ports: + - "33333:8000" +azurite: + image: "mcr.microsoft.com/azure-storage/azurite" + ports: + - "10000:10000" + command: ["azurite", "--blobHost", "0.0.0.0"] From 1cfe38e54ad33c746d9c6ecb718ced3f87dedb86 Mon Sep 17 00:00:00 2001 From: Agnes Garoux Date: Fri, 12 Apr 2024 09:59:02 +0100 Subject: [PATCH 04/28] Dependencies --- project/Dependencies.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/project/Dependencies.scala b/project/Dependencies.scala index 722fc5e95..89ec7ecbf 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -102,12 +102,12 @@ object ExternalDependencies { // This should match the version of aws used in scala-libs; see // https://github.com/wellcomecollection/scala-libs/blob/main/project/Dependencies.scala - val aws = "2.19.0" + val aws = "2.25.28" // These are the "Common Runtime Libraries", which you're encouraged to use for // better performance. // See https://docs.aws.amazon.com/sdkref/latest/guide/common-runtime.html - val awsCrt = "0.20.3" + val awsCrt = "0.29.16" } val azureDependencies: Seq[ModuleID] = Seq( From cc6068d3f54c3180c7de507583e0cdb536bdaa02 Mon Sep 17 00:00:00 2001 From: Agnes Garoux Date: Fri, 12 Apr 2024 10:20:17 +0100 Subject: [PATCH 05/28] CRT not used as default anymore? --- .../storage/transfer/fixtures/S3TransferManagerFixtures.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bag_replicator/src/test/scala/weco/storage/transfer/fixtures/S3TransferManagerFixtures.scala b/bag_replicator/src/test/scala/weco/storage/transfer/fixtures/S3TransferManagerFixtures.scala index 12fa2e42b..12ffe60da 100644 --- a/bag_replicator/src/test/scala/weco/storage/transfer/fixtures/S3TransferManagerFixtures.scala +++ b/bag_replicator/src/test/scala/weco/storage/transfer/fixtures/S3TransferManagerFixtures.scala @@ -9,7 +9,7 @@ import java.net.URI trait S3TransferManagerFixtures extends S3Fixtures { def createS3TransferManagerWithEndpoint(endpoint: String): S3TransferManager = { val s3AsyncClient = - S3AsyncClient.builder() + S3AsyncClient.crtBuilder() .credentialsProvider(s3Credentials) .forcePathStyle(true) .endpointOverride(new URI(endpoint)) From 6c1f4556aafc73505de9c54065f8749e12e835e1 Mon Sep 17 00:00:00 2001 From: Agnes Garoux Date: Fri, 12 Apr 2024 12:12:04 +0100 Subject: [PATCH 06/28] ugh --- .../storage/transfer/fixtures/S3TransferManagerFixtures.scala | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bag_replicator/src/test/scala/weco/storage/transfer/fixtures/S3TransferManagerFixtures.scala b/bag_replicator/src/test/scala/weco/storage/transfer/fixtures/S3TransferManagerFixtures.scala index 12ffe60da..28eec6ee1 100644 --- a/bag_replicator/src/test/scala/weco/storage/transfer/fixtures/S3TransferManagerFixtures.scala +++ b/bag_replicator/src/test/scala/weco/storage/transfer/fixtures/S3TransferManagerFixtures.scala @@ -1,6 +1,7 @@ package weco.storage.transfer.fixtures import software.amazon.awssdk.services.s3.S3AsyncClient +import software.amazon.awssdk.services.s3.internal.crt.S3CrtAsyncClient import software.amazon.awssdk.transfer.s3.S3TransferManager import weco.storage.fixtures.S3Fixtures @@ -9,7 +10,7 @@ import java.net.URI trait S3TransferManagerFixtures extends S3Fixtures { def createS3TransferManagerWithEndpoint(endpoint: String): S3TransferManager = { val s3AsyncClient = - S3AsyncClient.crtBuilder() + S3CrtAsyncClient.builder() .credentialsProvider(s3Credentials) .forcePathStyle(true) .endpointOverride(new URI(endpoint)) From f34fe6d6557acf9c82699da6d40a8a3f7dcae8b0 Mon Sep 17 00:00:00 2001 From: Agnes Garoux Date: Fri, 12 Apr 2024 12:17:29 +0100 Subject: [PATCH 07/28] unused import --- .../storage/transfer/fixtures/S3TransferManagerFixtures.scala | 1 - 1 file changed, 1 deletion(-) diff --git a/bag_replicator/src/test/scala/weco/storage/transfer/fixtures/S3TransferManagerFixtures.scala b/bag_replicator/src/test/scala/weco/storage/transfer/fixtures/S3TransferManagerFixtures.scala index 28eec6ee1..74a105edd 100644 --- a/bag_replicator/src/test/scala/weco/storage/transfer/fixtures/S3TransferManagerFixtures.scala +++ b/bag_replicator/src/test/scala/weco/storage/transfer/fixtures/S3TransferManagerFixtures.scala @@ -1,6 +1,5 @@ package weco.storage.transfer.fixtures -import software.amazon.awssdk.services.s3.S3AsyncClient import software.amazon.awssdk.services.s3.internal.crt.S3CrtAsyncClient import software.amazon.awssdk.transfer.s3.S3TransferManager import weco.storage.fixtures.S3Fixtures From 3858e1a8caf7a704d2d3ec945a8b907bbc4c23ca Mon Sep 17 00:00:00 2001 From: Agnes Garoux Date: Fri, 12 Apr 2024 14:06:50 +0100 Subject: [PATCH 08/28] take??? --- .../scala/weco/storage_service/bag_replicator/Main.scala | 8 ++++++-- .../transfer/fixtures/S3TransferManagerFixtures.scala | 4 ++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/bag_replicator/src/main/scala/weco/storage_service/bag_replicator/Main.scala b/bag_replicator/src/main/scala/weco/storage_service/bag_replicator/Main.scala index 3b3dd8912..6015c5274 100644 --- a/bag_replicator/src/main/scala/weco/storage_service/bag_replicator/Main.scala +++ b/bag_replicator/src/main/scala/weco/storage_service/bag_replicator/Main.scala @@ -3,7 +3,7 @@ package weco.storage_service.bag_replicator import akka.actor.ActorSystem import com.azure.storage.blob.{BlobServiceClient, BlobServiceClientBuilder} import com.typesafe.config.Config -import software.amazon.awssdk.services.s3.S3Client +import software.amazon.awssdk.services.s3.{S3Client, S3AsyncClient} import software.amazon.awssdk.services.s3.presigner.S3Presigner import software.amazon.awssdk.services.sqs.SqsAsyncClient import software.amazon.awssdk.transfer.s3.S3TransferManager @@ -52,8 +52,12 @@ object Main extends WellcomeTypesafeApp { implicit val s3Client: S3Client = S3Client.builder().build() + implicit val s3AsyncClient: S3AsyncClient = + S3AsyncClient.crtBuilder().build() + implicit val s3TransferManager: S3TransferManager = - S3TransferManager.builder().build() + S3TransferManager.builder().s3Client(s3AsyncClient).build() + implicit val s3Presigner: S3Presigner = S3Presigner.builder().build() diff --git a/bag_replicator/src/test/scala/weco/storage/transfer/fixtures/S3TransferManagerFixtures.scala b/bag_replicator/src/test/scala/weco/storage/transfer/fixtures/S3TransferManagerFixtures.scala index 74a105edd..12ffe60da 100644 --- a/bag_replicator/src/test/scala/weco/storage/transfer/fixtures/S3TransferManagerFixtures.scala +++ b/bag_replicator/src/test/scala/weco/storage/transfer/fixtures/S3TransferManagerFixtures.scala @@ -1,6 +1,6 @@ package weco.storage.transfer.fixtures -import software.amazon.awssdk.services.s3.internal.crt.S3CrtAsyncClient +import software.amazon.awssdk.services.s3.S3AsyncClient import software.amazon.awssdk.transfer.s3.S3TransferManager import weco.storage.fixtures.S3Fixtures @@ -9,7 +9,7 @@ import java.net.URI trait S3TransferManagerFixtures extends S3Fixtures { def createS3TransferManagerWithEndpoint(endpoint: String): S3TransferManager = { val s3AsyncClient = - S3CrtAsyncClient.builder() + S3AsyncClient.crtBuilder() .credentialsProvider(s3Credentials) .forcePathStyle(true) .endpointOverride(new URI(endpoint)) From 405435f014db71379c2f4175db6fa7c6f0ddc683 Mon Sep 17 00:00:00 2001 From: Agnes Garoux Date: Fri, 12 Apr 2024 14:17:55 +0100 Subject: [PATCH 09/28] what's going on --- .../main/scala/weco/storage_service/bag_replicator/Main.scala | 1 - 1 file changed, 1 deletion(-) diff --git a/bag_replicator/src/main/scala/weco/storage_service/bag_replicator/Main.scala b/bag_replicator/src/main/scala/weco/storage_service/bag_replicator/Main.scala index 6015c5274..aead8aab3 100644 --- a/bag_replicator/src/main/scala/weco/storage_service/bag_replicator/Main.scala +++ b/bag_replicator/src/main/scala/weco/storage_service/bag_replicator/Main.scala @@ -54,7 +54,6 @@ object Main extends WellcomeTypesafeApp { S3Client.builder().build() implicit val s3AsyncClient: S3AsyncClient = S3AsyncClient.crtBuilder().build() - implicit val s3TransferManager: S3TransferManager = S3TransferManager.builder().s3Client(s3AsyncClient).build() From 3e4e788e494e3e205debb28a166daafca0a963dd Mon Sep 17 00:00:00 2001 From: Agnes Garoux Date: Fri, 12 Apr 2024 14:25:25 +0100 Subject: [PATCH 10/28] dang --- .../main/scala/weco/storage_service/bag_replicator/Main.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bag_replicator/src/main/scala/weco/storage_service/bag_replicator/Main.scala b/bag_replicator/src/main/scala/weco/storage_service/bag_replicator/Main.scala index aead8aab3..0cf9ea23c 100644 --- a/bag_replicator/src/main/scala/weco/storage_service/bag_replicator/Main.scala +++ b/bag_replicator/src/main/scala/weco/storage_service/bag_replicator/Main.scala @@ -3,7 +3,7 @@ package weco.storage_service.bag_replicator import akka.actor.ActorSystem import com.azure.storage.blob.{BlobServiceClient, BlobServiceClientBuilder} import com.typesafe.config.Config -import software.amazon.awssdk.services.s3.{S3Client, S3AsyncClient} +import software.amazon.awssdk.services.s3.{S3AsyncClient, S3Client} import software.amazon.awssdk.services.s3.presigner.S3Presigner import software.amazon.awssdk.services.sqs.SqsAsyncClient import software.amazon.awssdk.transfer.s3.S3TransferManager From 0f9f5e92155c3fc61e8b9170c64f607e6fc8bd2d Mon Sep 17 00:00:00 2001 From: Agnes Garoux Date: Fri, 12 Apr 2024 14:32:19 +0100 Subject: [PATCH 11/28] revert changes --- .../scala/weco/storage_service/bag_replicator/Main.scala | 7 +++---- .../transfer/fixtures/S3TransferManagerFixtures.scala | 2 +- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/bag_replicator/src/main/scala/weco/storage_service/bag_replicator/Main.scala b/bag_replicator/src/main/scala/weco/storage_service/bag_replicator/Main.scala index 0cf9ea23c..953665a51 100644 --- a/bag_replicator/src/main/scala/weco/storage_service/bag_replicator/Main.scala +++ b/bag_replicator/src/main/scala/weco/storage_service/bag_replicator/Main.scala @@ -3,7 +3,7 @@ package weco.storage_service.bag_replicator import akka.actor.ActorSystem import com.azure.storage.blob.{BlobServiceClient, BlobServiceClientBuilder} import com.typesafe.config.Config -import software.amazon.awssdk.services.s3.{S3AsyncClient, S3Client} +import software.amazon.awssdk.services.s3.S3Client import software.amazon.awssdk.services.s3.presigner.S3Presigner import software.amazon.awssdk.services.sqs.SqsAsyncClient import software.amazon.awssdk.transfer.s3.S3TransferManager @@ -52,10 +52,9 @@ object Main extends WellcomeTypesafeApp { implicit val s3Client: S3Client = S3Client.builder().build() - implicit val s3AsyncClient: S3AsyncClient = - S3AsyncClient.crtBuilder().build() + implicit val s3TransferManager: S3TransferManager = - S3TransferManager.builder().s3Client(s3AsyncClient).build() + S3TransferManager.builder().build() implicit val s3Presigner: S3Presigner = S3Presigner.builder().build() diff --git a/bag_replicator/src/test/scala/weco/storage/transfer/fixtures/S3TransferManagerFixtures.scala b/bag_replicator/src/test/scala/weco/storage/transfer/fixtures/S3TransferManagerFixtures.scala index 12ffe60da..12fa2e42b 100644 --- a/bag_replicator/src/test/scala/weco/storage/transfer/fixtures/S3TransferManagerFixtures.scala +++ b/bag_replicator/src/test/scala/weco/storage/transfer/fixtures/S3TransferManagerFixtures.scala @@ -9,7 +9,7 @@ import java.net.URI trait S3TransferManagerFixtures extends S3Fixtures { def createS3TransferManagerWithEndpoint(endpoint: String): S3TransferManager = { val s3AsyncClient = - S3AsyncClient.crtBuilder() + S3AsyncClient.builder() .credentialsProvider(s3Credentials) .forcePathStyle(true) .endpointOverride(new URI(endpoint)) From acbc622c05355499579028cabf8bd8f019d55cac Mon Sep 17 00:00:00 2001 From: Agnes Garoux Date: Fri, 12 Apr 2024 14:48:43 +0100 Subject: [PATCH 12/28] use localstack 3.0.0 in common --- common/docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/docker-compose.yml b/common/docker-compose.yml index 3b4c3fc1c..662748ba4 100644 --- a/common/docker-compose.yml +++ b/common/docker-compose.yml @@ -1,7 +1,7 @@ version: "2.1" services: localstack: - image: "760097843905.dkr.ecr.eu-west-1.amazonaws.com/localstack/localstack" + image: "public.ecr.aws/localstack/localstack:3.0.0" environment: - SERVICES=sqs ports: From 5a30ebb38ab4b78370f1bcaefd64ddf5efc77322 Mon Sep 17 00:00:00 2001 From: Agnes Garoux Date: Fri, 12 Apr 2024 14:57:11 +0100 Subject: [PATCH 13/28] try to set region as env var --- bag_replicator/docker-compose.yml | 38 +++++++++++++++++-------------- 1 file changed, 21 insertions(+), 17 deletions(-) diff --git a/bag_replicator/docker-compose.yml b/bag_replicator/docker-compose.yml index 612e7990f..633873b25 100644 --- a/bag_replicator/docker-compose.yml +++ b/bag_replicator/docker-compose.yml @@ -1,17 +1,21 @@ -localstack: - image: "public.ecr.aws/localstack/localstack:3.0.0" - environment: - - SERVICES=sqs - ports: - - "4566:4566" -s3: - image: "760097843905.dkr.ecr.eu-west-1.amazonaws.com/zenko/cloudserver:8.1.8" - environment: - - "S3BACKEND=mem" - ports: - - "33333:8000" -azurite: - image: "mcr.microsoft.com/azure-storage/azurite" - ports: - - "10000:10000" - command: ["azurite", "--blobHost", "0.0.0.0"] +version: "3.0.0" +services: + localstack: + image: "public.ecr.aws/localstack/localstack:3.0.0" + environment: + - SERVICES=sqs + - USE_SINGLE_REGION=1 + - DEFAULT_REGION=eu-west-1 + ports: + - "4566:4566" + s3: + image: "760097843905.dkr.ecr.eu-west-1.amazonaws.com/zenko/cloudserver:8.1.8" + environment: + - "S3BACKEND=mem" + ports: + - "33333:8000" + azurite: + image: "mcr.microsoft.com/azure-storage/azurite" + ports: + - "10000:10000" + command: ["azurite", "--blobHost", "0.0.0.0"] From 162dbd5494b27374c2befe63ff4b8c5222c4bbdc Mon Sep 17 00:00:00 2001 From: Agnes Garoux Date: Fri, 12 Apr 2024 15:12:23 +0100 Subject: [PATCH 14/28] The provided DefaultS3AsyncClient is not an instance of S3CrtAsyncClient --- .../storage/transfer/fixtures/S3TransferManagerFixtures.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bag_replicator/src/test/scala/weco/storage/transfer/fixtures/S3TransferManagerFixtures.scala b/bag_replicator/src/test/scala/weco/storage/transfer/fixtures/S3TransferManagerFixtures.scala index 12fa2e42b..12ffe60da 100644 --- a/bag_replicator/src/test/scala/weco/storage/transfer/fixtures/S3TransferManagerFixtures.scala +++ b/bag_replicator/src/test/scala/weco/storage/transfer/fixtures/S3TransferManagerFixtures.scala @@ -9,7 +9,7 @@ import java.net.URI trait S3TransferManagerFixtures extends S3Fixtures { def createS3TransferManagerWithEndpoint(endpoint: String): S3TransferManager = { val s3AsyncClient = - S3AsyncClient.builder() + S3AsyncClient.crtBuilder() .credentialsProvider(s3Credentials) .forcePathStyle(true) .endpointOverride(new URI(endpoint)) From d28d3d5586cab7d913dbc9d0f736ec39a1c19030 Mon Sep 17 00:00:00 2001 From: Agnes Garoux Date: Fri, 12 Apr 2024 15:32:30 +0100 Subject: [PATCH 15/28] use S3AsyncClient.crtCreate --- .../storage_service/bag_replicator/Main.scala | 23 ++++++++----------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/bag_replicator/src/main/scala/weco/storage_service/bag_replicator/Main.scala b/bag_replicator/src/main/scala/weco/storage_service/bag_replicator/Main.scala index 953665a51..ed5fd21e5 100644 --- a/bag_replicator/src/main/scala/weco/storage_service/bag_replicator/Main.scala +++ b/bag_replicator/src/main/scala/weco/storage_service/bag_replicator/Main.scala @@ -3,7 +3,8 @@ package weco.storage_service.bag_replicator import akka.actor.ActorSystem import com.azure.storage.blob.{BlobServiceClient, BlobServiceClientBuilder} import com.typesafe.config.Config -import software.amazon.awssdk.services.s3.S3Client + +import software.amazon.awssdk.services.s3.{S3AsyncClient, S3Client} import software.amazon.awssdk.services.s3.presigner.S3Presigner import software.amazon.awssdk.services.sqs.SqsAsyncClient import software.amazon.awssdk.transfer.s3.S3TransferManager @@ -24,16 +25,8 @@ import weco.storage_service.bag_replicator.replicator.azure.AzureReplicator import weco.storage_service.bag_replicator.replicator.models.ReplicationSummary import weco.storage_service.bag_replicator.replicator.s3.S3Replicator import weco.storage_service.bag_replicator.services.BagReplicatorWorker -import weco.storage_service.config.builders.{ - IngestUpdaterBuilder, - OperationNameBuilder, - OutgoingPublisherBuilder -} -import weco.storage_service.ingests.models.{ - AmazonS3StorageProvider, - AzureBlobStorageProvider, - StorageProvider -} +import weco.storage_service.config.builders.{IngestUpdaterBuilder, OperationNameBuilder, OutgoingPublisherBuilder} +import weco.storage_service.ingests.models.{AmazonS3StorageProvider, AzureBlobStorageProvider, StorageProvider} import weco.storage_service.storage.models.IngestStepResult import weco.typesafe.WellcomeTypesafeApp import weco.typesafe.config.builders.EnrichConfig._ @@ -53,8 +46,12 @@ object Main extends WellcomeTypesafeApp { implicit val s3Client: S3Client = S3Client.builder().build() + + implicit val s3AsyncClient: S3AsyncClient = + S3AsyncClient.crtCreate + implicit val s3TransferManager: S3TransferManager = - S3TransferManager.builder().build() + S3TransferManager.builder().s3Client(s3AsyncClient).build() implicit val s3Presigner: S3Presigner = S3Presigner.builder().build() @@ -63,7 +60,7 @@ object Main extends WellcomeTypesafeApp { CloudWatchBuilder.buildCloudWatchMetrics(config) implicit val sqsClient: SqsAsyncClient = - SqsAsyncClient.builder().build() + SqsAsyncClient.builder().build()use implicit val lockDao: DynamoLockDao = DynamoLockDaoBuilder.buildDynamoLockDao(config) From 425a1151042c0ccefeec161c394bae34936166ab Mon Sep 17 00:00:00 2001 From: Agnes Garoux Date: Fri, 12 Apr 2024 15:34:08 +0100 Subject: [PATCH 16/28] typo --- .../main/scala/weco/storage_service/bag_replicator/Main.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bag_replicator/src/main/scala/weco/storage_service/bag_replicator/Main.scala b/bag_replicator/src/main/scala/weco/storage_service/bag_replicator/Main.scala index ed5fd21e5..97833dd5a 100644 --- a/bag_replicator/src/main/scala/weco/storage_service/bag_replicator/Main.scala +++ b/bag_replicator/src/main/scala/weco/storage_service/bag_replicator/Main.scala @@ -60,7 +60,7 @@ object Main extends WellcomeTypesafeApp { CloudWatchBuilder.buildCloudWatchMetrics(config) implicit val sqsClient: SqsAsyncClient = - SqsAsyncClient.builder().build()use + SqsAsyncClient.builder().build() implicit val lockDao: DynamoLockDao = DynamoLockDaoBuilder.buildDynamoLockDao(config) From da363b85dff1f25c5aa32ff02e723d7f24669f11 Mon Sep 17 00:00:00 2001 From: Agnes Garoux Date: Mon, 15 Apr 2024 10:02:06 +0100 Subject: [PATCH 17/28] rollback --- bag_replicator/docker-compose.yml | 2 -- .../scala/weco/storage_service/bag_replicator/Main.scala | 8 ++------ .../transfer/fixtures/S3TransferManagerFixtures.scala | 2 +- 3 files changed, 3 insertions(+), 9 deletions(-) diff --git a/bag_replicator/docker-compose.yml b/bag_replicator/docker-compose.yml index 633873b25..bd3344c21 100644 --- a/bag_replicator/docker-compose.yml +++ b/bag_replicator/docker-compose.yml @@ -4,8 +4,6 @@ services: image: "public.ecr.aws/localstack/localstack:3.0.0" environment: - SERVICES=sqs - - USE_SINGLE_REGION=1 - - DEFAULT_REGION=eu-west-1 ports: - "4566:4566" s3: diff --git a/bag_replicator/src/main/scala/weco/storage_service/bag_replicator/Main.scala b/bag_replicator/src/main/scala/weco/storage_service/bag_replicator/Main.scala index 97833dd5a..7bc5f2cc4 100644 --- a/bag_replicator/src/main/scala/weco/storage_service/bag_replicator/Main.scala +++ b/bag_replicator/src/main/scala/weco/storage_service/bag_replicator/Main.scala @@ -4,7 +4,7 @@ import akka.actor.ActorSystem import com.azure.storage.blob.{BlobServiceClient, BlobServiceClientBuilder} import com.typesafe.config.Config -import software.amazon.awssdk.services.s3.{S3AsyncClient, S3Client} +import software.amazon.awssdk.services.s3.S3Client import software.amazon.awssdk.services.s3.presigner.S3Presigner import software.amazon.awssdk.services.sqs.SqsAsyncClient import software.amazon.awssdk.transfer.s3.S3TransferManager @@ -46,12 +46,8 @@ object Main extends WellcomeTypesafeApp { implicit val s3Client: S3Client = S3Client.builder().build() - - implicit val s3AsyncClient: S3AsyncClient = - S3AsyncClient.crtCreate - implicit val s3TransferManager: S3TransferManager = - S3TransferManager.builder().s3Client(s3AsyncClient).build() + S3TransferManager.builder().build() implicit val s3Presigner: S3Presigner = S3Presigner.builder().build() diff --git a/bag_replicator/src/test/scala/weco/storage/transfer/fixtures/S3TransferManagerFixtures.scala b/bag_replicator/src/test/scala/weco/storage/transfer/fixtures/S3TransferManagerFixtures.scala index 12ffe60da..12fa2e42b 100644 --- a/bag_replicator/src/test/scala/weco/storage/transfer/fixtures/S3TransferManagerFixtures.scala +++ b/bag_replicator/src/test/scala/weco/storage/transfer/fixtures/S3TransferManagerFixtures.scala @@ -9,7 +9,7 @@ import java.net.URI trait S3TransferManagerFixtures extends S3Fixtures { def createS3TransferManagerWithEndpoint(endpoint: String): S3TransferManager = { val s3AsyncClient = - S3AsyncClient.crtBuilder() + S3AsyncClient.builder() .credentialsProvider(s3Credentials) .forcePathStyle(true) .endpointOverride(new URI(endpoint)) From f2f22d2124573ed0e63fd60547f072259bdb8f67 Mon Sep 17 00:00:00 2001 From: Agnes Garoux Date: Mon, 15 Apr 2024 10:45:03 +0100 Subject: [PATCH 18/28] use crt builder in Main --- .../scala/weco/storage_service/bag_replicator/Main.scala | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/bag_replicator/src/main/scala/weco/storage_service/bag_replicator/Main.scala b/bag_replicator/src/main/scala/weco/storage_service/bag_replicator/Main.scala index 7bc5f2cc4..00ed5cd81 100644 --- a/bag_replicator/src/main/scala/weco/storage_service/bag_replicator/Main.scala +++ b/bag_replicator/src/main/scala/weco/storage_service/bag_replicator/Main.scala @@ -4,7 +4,7 @@ import akka.actor.ActorSystem import com.azure.storage.blob.{BlobServiceClient, BlobServiceClientBuilder} import com.typesafe.config.Config -import software.amazon.awssdk.services.s3.S3Client +import software.amazon.awssdk.services.s3.{S3Client, S3AsyncClient} import software.amazon.awssdk.services.s3.presigner.S3Presigner import software.amazon.awssdk.services.sqs.SqsAsyncClient import software.amazon.awssdk.transfer.s3.S3TransferManager @@ -46,8 +46,11 @@ object Main extends WellcomeTypesafeApp { implicit val s3Client: S3Client = S3Client.builder().build() + implicit val s3AsyncClient: S3AsyncClient = + S3AsyncClient.crtBuilder().build() + implicit val s3TransferManager: S3TransferManager = - S3TransferManager.builder().build() + S3TransferManager.builder().s3Client(s3AsyncClient).build() implicit val s3Presigner: S3Presigner = S3Presigner.builder().build() From a5a1f114957f6ab4404026a1e84710ce28ffea51 Mon Sep 17 00:00:00 2001 From: Agnes Garoux Date: Mon, 15 Apr 2024 10:57:11 +0100 Subject: [PATCH 19/28] use crt builder in fixtures --- .../main/scala/weco/storage_service/bag_replicator/Main.scala | 1 - .../storage/transfer/fixtures/S3TransferManagerFixtures.scala | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/bag_replicator/src/main/scala/weco/storage_service/bag_replicator/Main.scala b/bag_replicator/src/main/scala/weco/storage_service/bag_replicator/Main.scala index 00ed5cd81..fe6e6843c 100644 --- a/bag_replicator/src/main/scala/weco/storage_service/bag_replicator/Main.scala +++ b/bag_replicator/src/main/scala/weco/storage_service/bag_replicator/Main.scala @@ -3,7 +3,6 @@ package weco.storage_service.bag_replicator import akka.actor.ActorSystem import com.azure.storage.blob.{BlobServiceClient, BlobServiceClientBuilder} import com.typesafe.config.Config - import software.amazon.awssdk.services.s3.{S3Client, S3AsyncClient} import software.amazon.awssdk.services.s3.presigner.S3Presigner import software.amazon.awssdk.services.sqs.SqsAsyncClient diff --git a/bag_replicator/src/test/scala/weco/storage/transfer/fixtures/S3TransferManagerFixtures.scala b/bag_replicator/src/test/scala/weco/storage/transfer/fixtures/S3TransferManagerFixtures.scala index 12fa2e42b..12ffe60da 100644 --- a/bag_replicator/src/test/scala/weco/storage/transfer/fixtures/S3TransferManagerFixtures.scala +++ b/bag_replicator/src/test/scala/weco/storage/transfer/fixtures/S3TransferManagerFixtures.scala @@ -9,7 +9,7 @@ import java.net.URI trait S3TransferManagerFixtures extends S3Fixtures { def createS3TransferManagerWithEndpoint(endpoint: String): S3TransferManager = { val s3AsyncClient = - S3AsyncClient.builder() + S3AsyncClient.crtBuilder() .credentialsProvider(s3Credentials) .forcePathStyle(true) .endpointOverride(new URI(endpoint)) From 65426488786f8a68277f30d41b9a8df01a8debdd Mon Sep 17 00:00:00 2001 From: Agnes Garoux Date: Mon, 15 Apr 2024 11:12:04 +0100 Subject: [PATCH 20/28] azurite doesn't like v3.0.0 --- bag_replicator/docker-compose.yml | 36 +++++++++++++++---------------- 1 file changed, 17 insertions(+), 19 deletions(-) diff --git a/bag_replicator/docker-compose.yml b/bag_replicator/docker-compose.yml index bd3344c21..612e7990f 100644 --- a/bag_replicator/docker-compose.yml +++ b/bag_replicator/docker-compose.yml @@ -1,19 +1,17 @@ -version: "3.0.0" -services: - localstack: - image: "public.ecr.aws/localstack/localstack:3.0.0" - environment: - - SERVICES=sqs - ports: - - "4566:4566" - s3: - image: "760097843905.dkr.ecr.eu-west-1.amazonaws.com/zenko/cloudserver:8.1.8" - environment: - - "S3BACKEND=mem" - ports: - - "33333:8000" - azurite: - image: "mcr.microsoft.com/azure-storage/azurite" - ports: - - "10000:10000" - command: ["azurite", "--blobHost", "0.0.0.0"] +localstack: + image: "public.ecr.aws/localstack/localstack:3.0.0" + environment: + - SERVICES=sqs + ports: + - "4566:4566" +s3: + image: "760097843905.dkr.ecr.eu-west-1.amazonaws.com/zenko/cloudserver:8.1.8" + environment: + - "S3BACKEND=mem" + ports: + - "33333:8000" +azurite: + image: "mcr.microsoft.com/azure-storage/azurite" + ports: + - "10000:10000" + command: ["azurite", "--blobHost", "0.0.0.0"] From a2edc8d68f9381801a2292fb43a76ce7ebc8e56b Mon Sep 17 00:00:00 2001 From: Agnes Garoux Date: Mon, 15 Apr 2024 11:13:05 +0100 Subject: [PATCH 21/28] formatting --- .../main/scala/weco/storage_service/bag_replicator/Main.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bag_replicator/src/main/scala/weco/storage_service/bag_replicator/Main.scala b/bag_replicator/src/main/scala/weco/storage_service/bag_replicator/Main.scala index fe6e6843c..4a15bb67e 100644 --- a/bag_replicator/src/main/scala/weco/storage_service/bag_replicator/Main.scala +++ b/bag_replicator/src/main/scala/weco/storage_service/bag_replicator/Main.scala @@ -3,7 +3,7 @@ package weco.storage_service.bag_replicator import akka.actor.ActorSystem import com.azure.storage.blob.{BlobServiceClient, BlobServiceClientBuilder} import com.typesafe.config.Config -import software.amazon.awssdk.services.s3.{S3Client, S3AsyncClient} +import software.amazon.awssdk.services.s3.{S3AsyncClient, S3Client} import software.amazon.awssdk.services.s3.presigner.S3Presigner import software.amazon.awssdk.services.sqs.SqsAsyncClient import software.amazon.awssdk.transfer.s3.S3TransferManager From 79b9787344290da6a8ab254fc013c34ea6580395 Mon Sep 17 00:00:00 2001 From: Agnes Garoux Date: Mon, 15 Apr 2024 12:05:24 +0100 Subject: [PATCH 22/28] set AWS_REGION for localstack SQS --- bag_replicator/docker-compose.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/bag_replicator/docker-compose.yml b/bag_replicator/docker-compose.yml index 612e7990f..c2527b420 100644 --- a/bag_replicator/docker-compose.yml +++ b/bag_replicator/docker-compose.yml @@ -2,6 +2,7 @@ localstack: image: "public.ecr.aws/localstack/localstack:3.0.0" environment: - SERVICES=sqs + - AWS_REGION=us-east-1 ports: - "4566:4566" s3: From 8b243566ed85f7fa3fc9d5f9ff41f89bdb2d6e77 Mon Sep 17 00:00:00 2001 From: Agnes Garoux Date: Mon, 15 Apr 2024 14:20:15 +0100 Subject: [PATCH 23/28] ALLOW_NONSTANDARD_REGIONS in services using localstack/sqs --- bag_register/docker-compose.yml | 3 ++- bag_replicator/docker-compose.yml | 2 +- bag_root_finder/docker-compose.yml | 3 ++- bag_tagger/docker-compose.yml | 3 ++- bag_unpacker/docker-compose.yml | 3 ++- bag_verifier/docker-compose.yml | 3 ++- bag_versioner/docker-compose.yml | 3 ++- common/docker-compose.yml | 1 + indexer/bag_indexer/docker-compose.yml | 3 ++- indexer/file_finder/docker-compose.yml | 3 ++- indexer/file_indexer/docker-compose.yml | 3 ++- indexer/ingests_indexer/docker-compose.yml | 3 ++- ingests/ingests_worker/docker-compose.yml | 3 ++- notifier/docker-compose.yml | 3 ++- replica_aggregator/docker-compose.yml | 3 ++- 15 files changed, 28 insertions(+), 14 deletions(-) diff --git a/bag_register/docker-compose.yml b/bag_register/docker-compose.yml index 96b959542..fd855db0b 100644 --- a/bag_register/docker-compose.yml +++ b/bag_register/docker-compose.yml @@ -1,7 +1,8 @@ localstack: - image: "760097843905.dkr.ecr.eu-west-1.amazonaws.com/localstack/localstack" + image: "public.ecr.aws/localstack/localstack:3.0.0" environment: - SERVICES=sqs + - ALLOW_NONSTANDARD_REGIONS=1 ports: - "4566:4566" s3: diff --git a/bag_replicator/docker-compose.yml b/bag_replicator/docker-compose.yml index c2527b420..c3ae4b88c 100644 --- a/bag_replicator/docker-compose.yml +++ b/bag_replicator/docker-compose.yml @@ -2,7 +2,7 @@ localstack: image: "public.ecr.aws/localstack/localstack:3.0.0" environment: - SERVICES=sqs - - AWS_REGION=us-east-1 + - ALLOW_NONSTANDARD_REGIONS=1 ports: - "4566:4566" s3: diff --git a/bag_root_finder/docker-compose.yml b/bag_root_finder/docker-compose.yml index 96b959542..fd855db0b 100644 --- a/bag_root_finder/docker-compose.yml +++ b/bag_root_finder/docker-compose.yml @@ -1,7 +1,8 @@ localstack: - image: "760097843905.dkr.ecr.eu-west-1.amazonaws.com/localstack/localstack" + image: "public.ecr.aws/localstack/localstack:3.0.0" environment: - SERVICES=sqs + - ALLOW_NONSTANDARD_REGIONS=1 ports: - "4566:4566" s3: diff --git a/bag_tagger/docker-compose.yml b/bag_tagger/docker-compose.yml index 0a34e490d..c3ae4b88c 100644 --- a/bag_tagger/docker-compose.yml +++ b/bag_tagger/docker-compose.yml @@ -1,7 +1,8 @@ localstack: - image: "760097843905.dkr.ecr.eu-west-1.amazonaws.com/localstack/localstack" + image: "public.ecr.aws/localstack/localstack:3.0.0" environment: - SERVICES=sqs + - ALLOW_NONSTANDARD_REGIONS=1 ports: - "4566:4566" s3: diff --git a/bag_unpacker/docker-compose.yml b/bag_unpacker/docker-compose.yml index 96b959542..fd855db0b 100644 --- a/bag_unpacker/docker-compose.yml +++ b/bag_unpacker/docker-compose.yml @@ -1,7 +1,8 @@ localstack: - image: "760097843905.dkr.ecr.eu-west-1.amazonaws.com/localstack/localstack" + image: "public.ecr.aws/localstack/localstack:3.0.0" environment: - SERVICES=sqs + - ALLOW_NONSTANDARD_REGIONS=1 ports: - "4566:4566" s3: diff --git a/bag_verifier/docker-compose.yml b/bag_verifier/docker-compose.yml index 55dbfd0b1..95d6ae6ac 100644 --- a/bag_verifier/docker-compose.yml +++ b/bag_verifier/docker-compose.yml @@ -1,9 +1,10 @@ version: "2.1" services: localstack: - image: "760097843905.dkr.ecr.eu-west-1.amazonaws.com/localstack/localstack" + image: "public.ecr.aws/localstack/localstack:3.0.0" environment: - SERVICES=sqs + - ALLOW_NONSTANDARD_REGIONS=1 ports: - "4566:4566" s3: diff --git a/bag_versioner/docker-compose.yml b/bag_versioner/docker-compose.yml index 536eff3ea..a1e5f61c7 100644 --- a/bag_versioner/docker-compose.yml +++ b/bag_versioner/docker-compose.yml @@ -1,7 +1,8 @@ localstack: - image: "760097843905.dkr.ecr.eu-west-1.amazonaws.com/localstack/localstack" + image: "public.ecr.aws/localstack/localstack:3.0.0" environment: - SERVICES=sqs + - ALLOW_NONSTANDARD_REGIONS=1 ports: - "4566:4566" dynamodb: diff --git a/common/docker-compose.yml b/common/docker-compose.yml index 662748ba4..bdbea4513 100644 --- a/common/docker-compose.yml +++ b/common/docker-compose.yml @@ -4,6 +4,7 @@ services: image: "public.ecr.aws/localstack/localstack:3.0.0" environment: - SERVICES=sqs + - ALLOW_NONSTANDARD_REGIONS=1 ports: - "4566:4566" s3: diff --git a/indexer/bag_indexer/docker-compose.yml b/indexer/bag_indexer/docker-compose.yml index 29afed8e8..132d672a6 100644 --- a/indexer/bag_indexer/docker-compose.yml +++ b/indexer/bag_indexer/docker-compose.yml @@ -11,8 +11,9 @@ elasticsearch: - "discovery.type=single-node" - "xpack.security.enabled=false" localstack: - image: "760097843905.dkr.ecr.eu-west-1.amazonaws.com/localstack/localstack" + image: "public.ecr.aws/localstack/localstack:3.0.0" environment: - SERVICES=sqs + - ALLOW_NONSTANDARD_REGIONS=1 ports: - "4566:4566" diff --git a/indexer/file_finder/docker-compose.yml b/indexer/file_finder/docker-compose.yml index 45ca56c7a..ce4d0ec51 100644 --- a/indexer/file_finder/docker-compose.yml +++ b/indexer/file_finder/docker-compose.yml @@ -1,6 +1,7 @@ localstack: - image: "760097843905.dkr.ecr.eu-west-1.amazonaws.com/localstack/localstack" + image: "public.ecr.aws/localstack/localstack:3.0.0" environment: - SERVICES=sqs + - ALLOW_NONSTANDARD_REGIONS=1 ports: - "4566:4566" diff --git a/indexer/file_indexer/docker-compose.yml b/indexer/file_indexer/docker-compose.yml index 29afed8e8..132d672a6 100644 --- a/indexer/file_indexer/docker-compose.yml +++ b/indexer/file_indexer/docker-compose.yml @@ -11,8 +11,9 @@ elasticsearch: - "discovery.type=single-node" - "xpack.security.enabled=false" localstack: - image: "760097843905.dkr.ecr.eu-west-1.amazonaws.com/localstack/localstack" + image: "public.ecr.aws/localstack/localstack:3.0.0" environment: - SERVICES=sqs + - ALLOW_NONSTANDARD_REGIONS=1 ports: - "4566:4566" diff --git a/indexer/ingests_indexer/docker-compose.yml b/indexer/ingests_indexer/docker-compose.yml index 29afed8e8..132d672a6 100644 --- a/indexer/ingests_indexer/docker-compose.yml +++ b/indexer/ingests_indexer/docker-compose.yml @@ -11,8 +11,9 @@ elasticsearch: - "discovery.type=single-node" - "xpack.security.enabled=false" localstack: - image: "760097843905.dkr.ecr.eu-west-1.amazonaws.com/localstack/localstack" + image: "public.ecr.aws/localstack/localstack:3.0.0" environment: - SERVICES=sqs + - ALLOW_NONSTANDARD_REGIONS=1 ports: - "4566:4566" diff --git a/ingests/ingests_worker/docker-compose.yml b/ingests/ingests_worker/docker-compose.yml index 45ca56c7a..ce4d0ec51 100644 --- a/ingests/ingests_worker/docker-compose.yml +++ b/ingests/ingests_worker/docker-compose.yml @@ -1,6 +1,7 @@ localstack: - image: "760097843905.dkr.ecr.eu-west-1.amazonaws.com/localstack/localstack" + image: "public.ecr.aws/localstack/localstack:3.0.0" environment: - SERVICES=sqs + - ALLOW_NONSTANDARD_REGIONS=1 ports: - "4566:4566" diff --git a/notifier/docker-compose.yml b/notifier/docker-compose.yml index 45ca56c7a..ce4d0ec51 100644 --- a/notifier/docker-compose.yml +++ b/notifier/docker-compose.yml @@ -1,6 +1,7 @@ localstack: - image: "760097843905.dkr.ecr.eu-west-1.amazonaws.com/localstack/localstack" + image: "public.ecr.aws/localstack/localstack:3.0.0" environment: - SERVICES=sqs + - ALLOW_NONSTANDARD_REGIONS=1 ports: - "4566:4566" diff --git a/replica_aggregator/docker-compose.yml b/replica_aggregator/docker-compose.yml index 3e59669e7..c753c8f55 100644 --- a/replica_aggregator/docker-compose.yml +++ b/replica_aggregator/docker-compose.yml @@ -3,8 +3,9 @@ dynamodb: ports: - "45678:8000" localstack: - image: "760097843905.dkr.ecr.eu-west-1.amazonaws.com/localstack/localstack" + image: "public.ecr.aws/localstack/localstack:3.0.0" environment: - SERVICES=sqs + - ALLOW_NONSTANDARD_REGIONS=1 ports: - "4566:4566" From 9c6c8bd047be9bad7b74b79e8e4aa0a764d2d857 Mon Sep 17 00:00:00 2001 From: Agnes Garoux Date: Mon, 15 Apr 2024 14:21:53 +0100 Subject: [PATCH 24/28] formatting --- .../weco/storage_service/bag_replicator/Main.scala | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/bag_replicator/src/main/scala/weco/storage_service/bag_replicator/Main.scala b/bag_replicator/src/main/scala/weco/storage_service/bag_replicator/Main.scala index 4a15bb67e..7933aa52a 100644 --- a/bag_replicator/src/main/scala/weco/storage_service/bag_replicator/Main.scala +++ b/bag_replicator/src/main/scala/weco/storage_service/bag_replicator/Main.scala @@ -24,8 +24,16 @@ import weco.storage_service.bag_replicator.replicator.azure.AzureReplicator import weco.storage_service.bag_replicator.replicator.models.ReplicationSummary import weco.storage_service.bag_replicator.replicator.s3.S3Replicator import weco.storage_service.bag_replicator.services.BagReplicatorWorker -import weco.storage_service.config.builders.{IngestUpdaterBuilder, OperationNameBuilder, OutgoingPublisherBuilder} -import weco.storage_service.ingests.models.{AmazonS3StorageProvider, AzureBlobStorageProvider, StorageProvider} +import weco.storage_service.config.builders.{ + IngestUpdaterBuilder, + OperationNameBuilder, + OutgoingPublisherBuilder +} +import weco.storage_service.ingests.models.{ + AmazonS3StorageProvider, + AzureBlobStorageProvider, + StorageProvider +} import weco.storage_service.storage.models.IngestStepResult import weco.typesafe.WellcomeTypesafeApp import weco.typesafe.config.builders.EnrichConfig._ From b453bbd487d58235c782d7040f1d64e8deab7c10 Mon Sep 17 00:00:00 2001 From: Agnes Garoux Date: Mon, 15 Apr 2024 14:52:10 +0100 Subject: [PATCH 25/28] with S3CrtAsyncClient --- .../transfer/fixtures/S3TransferManagerFixtures.scala | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/bag_replicator/src/test/scala/weco/storage/transfer/fixtures/S3TransferManagerFixtures.scala b/bag_replicator/src/test/scala/weco/storage/transfer/fixtures/S3TransferManagerFixtures.scala index 12ffe60da..7d9eb6058 100644 --- a/bag_replicator/src/test/scala/weco/storage/transfer/fixtures/S3TransferManagerFixtures.scala +++ b/bag_replicator/src/test/scala/weco/storage/transfer/fixtures/S3TransferManagerFixtures.scala @@ -1,6 +1,7 @@ package weco.storage.transfer.fixtures -import software.amazon.awssdk.services.s3.S3AsyncClient +//import software.amazon.awssdk.services.s3.S3AsyncClient +import software.amazon.awssdk.services.s3.internal.crt.S3CrtAsyncClient import software.amazon.awssdk.transfer.s3.S3TransferManager import weco.storage.fixtures.S3Fixtures @@ -9,7 +10,7 @@ import java.net.URI trait S3TransferManagerFixtures extends S3Fixtures { def createS3TransferManagerWithEndpoint(endpoint: String): S3TransferManager = { val s3AsyncClient = - S3AsyncClient.crtBuilder() + S3CrtAsyncClient.builder() .credentialsProvider(s3Credentials) .forcePathStyle(true) .endpointOverride(new URI(endpoint)) From 238369ac64fc54328d8c9826d4aa1a6c149c6b7a Mon Sep 17 00:00:00 2001 From: Agnes Garoux Date: Mon, 15 Apr 2024 15:46:13 +0100 Subject: [PATCH 26/28] use S3AsyncClient.crtBuilder() --- .../transfer/fixtures/S3TransferManagerFixtures.scala | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/bag_replicator/src/test/scala/weco/storage/transfer/fixtures/S3TransferManagerFixtures.scala b/bag_replicator/src/test/scala/weco/storage/transfer/fixtures/S3TransferManagerFixtures.scala index 7d9eb6058..12ffe60da 100644 --- a/bag_replicator/src/test/scala/weco/storage/transfer/fixtures/S3TransferManagerFixtures.scala +++ b/bag_replicator/src/test/scala/weco/storage/transfer/fixtures/S3TransferManagerFixtures.scala @@ -1,7 +1,6 @@ package weco.storage.transfer.fixtures -//import software.amazon.awssdk.services.s3.S3AsyncClient -import software.amazon.awssdk.services.s3.internal.crt.S3CrtAsyncClient +import software.amazon.awssdk.services.s3.S3AsyncClient import software.amazon.awssdk.transfer.s3.S3TransferManager import weco.storage.fixtures.S3Fixtures @@ -10,7 +9,7 @@ import java.net.URI trait S3TransferManagerFixtures extends S3Fixtures { def createS3TransferManagerWithEndpoint(endpoint: String): S3TransferManager = { val s3AsyncClient = - S3CrtAsyncClient.builder() + S3AsyncClient.crtBuilder() .credentialsProvider(s3Credentials) .forcePathStyle(true) .endpointOverride(new URI(endpoint)) From 9b5d2d7e56fbf3ef3c19ea4614f6c58e08eb8dff Mon Sep 17 00:00:00 2001 From: Agnes Garoux Date: Tue, 16 Apr 2024 14:54:13 +0100 Subject: [PATCH 27/28] use no_alpine sbt-wrapper image --- builds/run_sbt_task_in_docker.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/builds/run_sbt_task_in_docker.sh b/builds/run_sbt_task_in_docker.sh index 896c811dd..301a17432 100755 --- a/builds/run_sbt_task_in_docker.sh +++ b/builds/run_sbt_task_in_docker.sh @@ -52,8 +52,9 @@ docker run --tty --rm \ --volume ~/.ivy2:/root/.ivy2 \ --volume "$HOST_COURSIER_CACHE:/root/$LINUX_COURSIER_CACHE" \ --volume /var/run/docker.sock:/var/run/docker.sock \ - --volume "${DOCKER_CONFIG:-$HOME/.docker}:/root/.docker" \ + --volume ~/.aws:/root/.aws \ --net host \ --volume "$ROOT:$ROOT" \ --workdir "$ROOT" \ - "$ECR_REGISTRY/wellcome/sbt_wrapper" "$@" + --env AWS_REGION="eu-west-1" \ + "$ECR_REGISTRY/wellcome/sbt_wrapper:no_alpine" "$@" From ce1ed2f490587e6f4560826aa51d3f4d96a66e2b Mon Sep 17 00:00:00 2001 From: Agnes Garoux Date: Tue, 16 Apr 2024 15:04:51 +0100 Subject: [PATCH 28/28] revert setting region --- builds/run_sbt_task_in_docker.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/builds/run_sbt_task_in_docker.sh b/builds/run_sbt_task_in_docker.sh index 301a17432..5e4b17362 100755 --- a/builds/run_sbt_task_in_docker.sh +++ b/builds/run_sbt_task_in_docker.sh @@ -52,9 +52,8 @@ docker run --tty --rm \ --volume ~/.ivy2:/root/.ivy2 \ --volume "$HOST_COURSIER_CACHE:/root/$LINUX_COURSIER_CACHE" \ --volume /var/run/docker.sock:/var/run/docker.sock \ - --volume ~/.aws:/root/.aws \ + --volume "${DOCKER_CONFIG:-$HOME/.docker}:/root/.docker" \ --net host \ --volume "$ROOT:$ROOT" \ --workdir "$ROOT" \ - --env AWS_REGION="eu-west-1" \ "$ECR_REGISTRY/wellcome/sbt_wrapper:no_alpine" "$@"