From e24ec62c6379928590ddefdefe5bb0a883d652d2 Mon Sep 17 00:00:00 2001 From: Munish Chouhan Date: Wed, 20 Nov 2024 16:51:18 +0100 Subject: [PATCH 1/3] Add /v1alpha2/validate-creds (#752) Signed-off-by: munishchouhan --- .../wave/controller/ValidateController.groovy | 11 +++- .../ValidateCredsControllerTest.groovy | 63 +++++++++++++++++++ 2 files changed, 72 insertions(+), 2 deletions(-) diff --git a/src/main/groovy/io/seqera/wave/controller/ValidateController.groovy b/src/main/groovy/io/seqera/wave/controller/ValidateController.groovy index fd23fbdc6..cebc541b5 100644 --- a/src/main/groovy/io/seqera/wave/controller/ValidateController.groovy +++ b/src/main/groovy/io/seqera/wave/controller/ValidateController.groovy @@ -18,6 +18,7 @@ package io.seqera.wave.controller +import io.micronaut.http.annotation.Body import io.micronaut.http.annotation.Controller import io.micronaut.http.annotation.Post import io.micronaut.scheduling.TaskExecutors @@ -26,15 +27,21 @@ import io.seqera.wave.auth.RegistryAuthService import jakarta.inject.Inject import jakarta.validation.Valid -@Controller("/validate-creds") +@Controller("/") @ExecuteOn(TaskExecutors.BLOCKING) class ValidateController { @Inject RegistryAuthService loginService - @Post + @Deprecated + @Post("/validate-creds") Boolean validateCreds(@Valid ValidateRegistryCredsRequest request){ loginService.validateUser(request.registry, request.userName, request.password) } + @Post("/v1alpha2/validate-creds") + Boolean validateCredsV2(@Valid @Body ValidateRegistryCredsRequest request){ + loginService.validateUser(request.registry, request.userName, request.password) + } + } diff --git a/src/test/groovy/io/seqera/wave/controller/ValidateCredsControllerTest.groovy b/src/test/groovy/io/seqera/wave/controller/ValidateCredsControllerTest.groovy index 586d48616..884728787 100644 --- a/src/test/groovy/io/seqera/wave/controller/ValidateCredsControllerTest.groovy +++ b/src/test/groovy/io/seqera/wave/controller/ValidateCredsControllerTest.groovy @@ -127,4 +127,67 @@ class ValidateCredsControllerTest extends Specification implements SecureDockerR 'nope' | 'yepes' | "https://quay.io" | false 'test' | 'test' | 'test' | true } + + void 'should validate username required'() { + when: + HttpRequest request = HttpRequest.POST("/v1alpha2/validate-creds", [ + password: 'test', + ]) + client.toBlocking().exchange(request, Boolean) + then: + def e = thrown(HttpClientResponseException) + } + + void 'should validate pwd required'() { + when: + HttpRequest request = HttpRequest.POST("/v1alpha2/validate-creds", [ + userName: 'test', + ]) + client.toBlocking().exchange(request, Boolean) + then: + def e = thrown(HttpClientResponseException) + } + + void 'should validate the test user'() { + given: + def req = [ + userName:'test', + password:'test', + registry: getTestRegistryUrl('test') ] + and: + HttpRequest request = HttpRequest.POST("/v1alpha2/validate-creds", req) + when: + HttpResponse response = client.toBlocking().exchange(request, Boolean) + then: + response.status() == HttpStatus.OK + and: + response.body() + } + + void 'test validateController valid login'() { + given: + def req = [ + userName: USER, + password: PWD, + registry: getTestRegistryUrl(REGISTRY_URL) + ] + HttpRequest request = HttpRequest.POST("/v1alpha2/validate-creds", req) + when: + HttpResponse response = client.toBlocking().exchange(request, Boolean) + + then: + response.status() == HttpStatus.OK + and: + response.body() == VALID + + where: + USER | PWD | REGISTRY_URL | VALID + 'test' | 'test' | 'test' | true + 'nope' | 'yepes' | 'test' | false + dockerUsername | dockerPassword | "https://registry-1.docker.io" | true + 'nope' | 'yepes' | "https://registry-1.docker.io" | false + quayUsername | quayPassword | "https://quay.io" | true + 'nope' | 'yepes' | "https://quay.io" | false + 'test' | 'test' | 'test' | true + } } From 86ef526c6b1a86693d5d7293e64ec09716ffe1b7 Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Wed, 20 Nov 2024 16:53:20 +0100 Subject: [PATCH 2/3] Check block existence with object operation (#750) Signed-off-by: Paolo Di Tommaso Co-authored-by: Munish Chouhan --- .../blob/impl/BlobCacheServiceImpl.groovy | 50 ++++++++++--------- .../blob/BlobStateStoreImplTest.groovy | 8 +++ 2 files changed, 34 insertions(+), 24 deletions(-) diff --git a/src/main/groovy/io/seqera/wave/service/blob/impl/BlobCacheServiceImpl.groovy b/src/main/groovy/io/seqera/wave/service/blob/impl/BlobCacheServiceImpl.groovy index 606a4ecd3..c6c3b61d7 100644 --- a/src/main/groovy/io/seqera/wave/service/blob/impl/BlobCacheServiceImpl.groovy +++ b/src/main/groovy/io/seqera/wave/service/blob/impl/BlobCacheServiceImpl.groovy @@ -17,9 +17,6 @@ */ package io.seqera.wave.service.blob.impl -import java.net.http.HttpClient -import java.net.http.HttpRequest -import java.net.http.HttpResponse import groovy.transform.CompileStatic import groovy.util.logging.Slf4j @@ -29,7 +26,6 @@ import io.seqera.wave.configuration.BlobCacheConfig import io.seqera.wave.configuration.HttpClientConfig import io.seqera.wave.core.RegistryProxyService import io.seqera.wave.core.RoutePath -import io.seqera.wave.http.HttpClientFactory import io.seqera.wave.service.blob.BlobCacheService import io.seqera.wave.service.blob.BlobEntry import io.seqera.wave.service.blob.BlobSigningService @@ -38,14 +34,16 @@ import io.seqera.wave.service.job.JobHandler import io.seqera.wave.service.job.JobService import io.seqera.wave.service.job.JobSpec import io.seqera.wave.service.job.JobState +import io.seqera.wave.util.BucketTokenizer import io.seqera.wave.util.Escape -import io.seqera.wave.util.Retryable import io.seqera.wave.util.StringUtils import jakarta.annotation.PostConstruct import jakarta.inject.Inject import jakarta.inject.Named import jakarta.inject.Singleton -import static io.seqera.wave.WaveDefault.HTTP_SERVER_ERRORS +import software.amazon.awssdk.services.s3.S3Client +import software.amazon.awssdk.services.s3.model.HeadObjectRequest +import software.amazon.awssdk.services.s3.model.S3Exception /** * Implements cache for container image layer blobs * @@ -79,11 +77,12 @@ class BlobCacheServiceImpl implements BlobCacheService, JobHandler { @Inject private HttpClientConfig httpConfig - private HttpClient httpClient + @Inject + @Named('BlobS3Client') + private S3Client s3Client @PostConstruct private void init() { - httpClient = HttpClientFactory.followRedirectsHttpClient() log.info "Creating Blob cache service - $blobConfig" } @@ -98,7 +97,7 @@ class BlobCacheServiceImpl implements BlobCacheService, JobHandler { // therefore it's safe to check and return directly // if it exists (no risk of returning a partial upload) // https://developers.cloudflare.com/r2/reference/consistency/ - if( blobExists(info.locationUri) && !debug ) { + if( blobExists(info.objectUri) && !debug ) { log.debug "== Blob cache exists for object '${info.locationUri}'" return info.cached() } @@ -113,21 +112,24 @@ class BlobCacheServiceImpl implements BlobCacheService, JobHandler { return result?.withLocation(locationUri) } - protected boolean blobExists(String uri) { - final request = HttpRequest - .newBuilder(new URI(uri)) - .method("HEAD", HttpRequest.BodyPublishers.noBody()) - .build() - - // retry strategy - final retryable = Retryable - .>of(httpConfig) - .retryIf((response) -> response.statusCode() in HTTP_SERVER_ERRORS) - .onRetry((event) -> log.warn("Unable to connect '$uri' - event: $event")) - - // submit the request - final resp = retryable.apply(()-> httpClient.send(request, HttpResponse.BodyHandlers.ofString())) - return resp.statusCode() == 200 + protected boolean blobExists(String blobLocation) { + try { + final object = BucketTokenizer.from(blobLocation) + final request = HeadObjectRequest + .builder() + .bucket(object.bucket) + .key(object.key) + .build() as HeadObjectRequest + // Execute the request + s3Client.headObject(request) + return true + } + catch (S3Exception e) { + if (e.statusCode() != 404) { + log.error "Unexpected response=${e.statusCode()} checking existence for object=${blobLocation} - cause: ${e.message}" + } + return false + } } /** diff --git a/src/test/groovy/io/seqera/wave/service/blob/BlobStateStoreImplTest.groovy b/src/test/groovy/io/seqera/wave/service/blob/BlobStateStoreImplTest.groovy index 61ad3465f..2898dfd29 100644 --- a/src/test/groovy/io/seqera/wave/service/blob/BlobStateStoreImplTest.groovy +++ b/src/test/groovy/io/seqera/wave/service/blob/BlobStateStoreImplTest.groovy @@ -23,10 +23,14 @@ import spock.lang.Specification import java.time.Duration import io.micronaut.context.annotation.Property +import io.micronaut.test.annotation.MockBean import io.micronaut.test.extensions.spock.annotation.MicronautTest import io.seqera.wave.configuration.BlobCacheConfig import io.seqera.wave.store.state.impl.StateProvider import jakarta.inject.Inject +import jakarta.inject.Named +import software.amazon.awssdk.services.s3.S3Client + /** * * @author Paolo Di Tommaso @@ -37,6 +41,10 @@ import jakarta.inject.Inject @MicronautTest class BlobStateStoreImplTest extends Specification { + @MockBean(S3Client) + @Named('BlobS3Client') + S3Client mockS3Client() { Mock(S3Client) } + @Inject BlobStoreImpl store From 84b13df279e79ab7f6e54d86e3cdc6607c108439 Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Wed, 20 Nov 2024 18:44:05 +0100 Subject: [PATCH 3/3] [release] Bump version 1.15.1 Signed-off-by: Paolo Di Tommaso --- VERSION | 2 +- changelog.txt | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 141f2e805..ace44233b 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.15.0 +1.15.1 diff --git a/changelog.txt b/changelog.txt index 5e5b7fdad..3aa45f8dd 100644 --- a/changelog.txt +++ b/changelog.txt @@ -1,4 +1,9 @@ # Wave changelog +1.15.1 - 20 Nov 2024 +- Check block existence with object operation (#750) [86ef526c] +- Add /v1alpha2/validate-creds (#752) [e24ec62c] +- Switched env vars around (#748) [080d5cce] + 1.15.0 - 18 Nov 2024 - Migration to virtual threads - phase 1 (#746) [aaf0420c] - Use runAsync instead supplyAsync [ffd0dacd]