Skip to content

Commit

Permalink
wip
Browse files Browse the repository at this point in the history
Signed-off-by: Paolo Di Tommaso <[email protected]>
  • Loading branch information
pditommaso committed Oct 5, 2024
1 parent 88a85e1 commit 599684e
Show file tree
Hide file tree
Showing 7 changed files with 134 additions and 21 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -194,8 +194,8 @@ class BuildRequest {
this.scanId = opts.scanId
this.buildContext = opts.buildContext as BuildContext
this.format = opts.format as BuildFormat
this.buildId = opts.buildId
this.maxDuration = opts.maxDuration as Duration
this.buildId = opts.buildId
}

@Override
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -78,7 +78,7 @@ class BuildResult {

@Override
String toString() {
return "BuildResult[id=$buildId; exitStatus=$exitStatus; duration=$duration]"
return "BuildResult[buildId=$buildId; exitStatus=$exitStatus; duration=$duration]"
}

static BuildResult completed(String buildId, Integer exitStatus, String logs, Instant startTime, String digest) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -85,7 +85,7 @@ class ContainerBuildServiceImpl implements ContainerBuildService, JobHandler<Bui
private ApplicationEventPublisher<BuildEvent> eventPublisher

@Inject
private BuildStateStore buildStateStore
private BuildStateStore buildStore

@Inject
@Named(TaskExecutors.IO)
Expand Down Expand Up @@ -148,7 +148,7 @@ class ContainerBuildServiceImpl implements ContainerBuildService, JobHandler<Bui
*/
@Override
CompletableFuture<BuildResult> buildResult(String targetImage) {
return buildStateStore
return buildStore
.awaitBuild(targetImage)
}

Expand Down Expand Up @@ -191,7 +191,7 @@ class ContainerBuildServiceImpl implements ContainerBuildService, JobHandler<Bui
catch (Throwable e) {
log.error "== Container build unexpected exception: ${e.message} - request=$req", e
final result = BuildResult.failed(req.buildId, e.message, req.startTime)
buildStateStore.storeBuild(req.targetImage, new BuildEntry(req, result), buildConfig.failureDuration)
buildStore.storeBuild(req.targetImage, new BuildEntry(req, result), buildConfig.failureDuration)
eventPublisher.publishEvent(new BuildEvent(req, result))
}
}
Expand All @@ -203,7 +203,7 @@ class ContainerBuildServiceImpl implements ContainerBuildService, JobHandler<Bui
rateLimiterService.acquireBuild(new AcquireRequest(request.identity.userId as String, request.ip))
}
catch (Exception e) {
buildStateStore.removeBuild(request.targetImage)
buildStore.removeBuild(request.targetImage)
throw e
}

Expand All @@ -218,7 +218,7 @@ class ContainerBuildServiceImpl implements ContainerBuildService, JobHandler<Bui
protected BuildTrack checkOrSubmit(BuildRequest request) {
// try to store a new build status for the given target image
// this returns true if and only if such container image was not set yet
final result = buildStateStore.putIfAbsentAndCount(request.targetImage, BuildEntry.create(request))
final result = buildStore.putIfAbsentAndCount(request.targetImage, BuildEntry.create(request))
if( result.succeed ) {
// NOTE: when the entry is stored, the buildId is automatically incremented
// therefore the request reference should be overridden
Expand All @@ -230,7 +230,7 @@ class ContainerBuildServiceImpl implements ContainerBuildService, JobHandler<Bui
}
// since it was unable to initialise the build result status
// this means the build status already exists, retrieve it
final ret2 = buildStateStore.getBuildResult(request.targetImage)
final ret2 = buildStore.getBuildResult(request.targetImage)
if( ret2 ) {
log.info "== Container build hit cache - request=$request"
// note: mark as cached only if the build result is 'done'
Expand Down Expand Up @@ -313,7 +313,7 @@ class ContainerBuildServiceImpl implements ContainerBuildService, JobHandler<Bui

@Override
BuildEntry getJobEntry(JobSpec job) {
buildStateStore.getBuild(job.entryKey)
buildStore.getBuild(job.entryKey)
}

@Override
Expand All @@ -332,15 +332,15 @@ class ContainerBuildServiceImpl implements ContainerBuildService, JobHandler<Bui
final result = state.completed()
? BuildResult.completed(buildId, state.exitCode, state.stdout, job.creationTime, digest)
: BuildResult.failed(buildId, state.stdout, job.creationTime)
buildStateStore.storeBuild(job.entryKey, entry.withResult(result), ttl)
buildStore.storeBuild(job.entryKey, entry.withResult(result), ttl)
eventPublisher.publishEvent(new BuildEvent(entry.request, result))
log.info "== Container build completed '${entry.request.targetImage}' - operation=${job.operationName}; exit=${state.exitCode}; status=${state.status}; duration=${result.duration}"
}

@Override
void onJobException(JobSpec job, BuildEntry entry, Throwable error) {
final result= BuildResult.failed(entry.request.buildId, error.message, job.creationTime)
buildStateStore.storeBuild(job.entryKey, entry.withResult(result), buildConfig.failureDuration)
buildStore.storeBuild(job.entryKey, entry.withResult(result), buildConfig.failureDuration)
eventPublisher.publishEvent(new BuildEvent(entry.request, result))
log.error("== Container build exception '${entry.request.targetImage}' - operation=${job.operationName}; cause=${error.message}", error)
}
Expand All @@ -349,7 +349,7 @@ class ContainerBuildServiceImpl implements ContainerBuildService, JobHandler<Bui
void onJobTimeout(JobSpec job, BuildEntry entry) {
final buildId = entry.request.buildId
final result= BuildResult.failed(buildId, "Container image build timed out '${entry.request.targetImage}'", job.creationTime)
buildStateStore.storeBuild(job.entryKey, entry.withResult(result), buildConfig.failureDuration)
buildStore.storeBuild(job.entryKey, entry.withResult(result), buildConfig.failureDuration)
eventPublisher.publishEvent(new BuildEvent(entry.request, result))
log.warn "== Container build time out '${entry.request.targetImage}'; operation=${job.operationName}; duration=${result.duration}"
}
Expand All @@ -373,7 +373,7 @@ class ContainerBuildServiceImpl implements ContainerBuildService, JobHandler<Bui
*/
@Override
WaveBuildRecord getBuildRecord(String buildId) {
final entry = buildStateStore.findByRequestId(buildId)
final entry = buildStore.findByRequestId(buildId)
return entry
? WaveBuildRecord.fromEntry(entry)
: persistenceService.loadBuild(buildId)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -52,11 +52,34 @@ abstract class AbstractStateStore<V> implements StateStore<String,V> {
return getPrefix() + 'request-id/' + requestId
}

/**
* Defines the counter for auto-increment operations. By default
* uses the entry "key". Subclasses can provide a custom logic to use a
* different counter key.
*
* @param key
* The entry for which the increment should be performed
* @param value
* The entry value for which the increment should be performed
* @return
* The counter key that by default is the entry key.
*/
protected String counterKey(String key, V value) {
return key
}

/**
* Defines the Lua script that's applied to increment the entry counter.
*
* It assumes the entry is serialised as JSON object and it contains a {@code count} attribute
* that will be update with the store counter value.
*
* @return The Lua script used to increment the entry count.
*/
protected String counterScript() {
// NOTE:
// "value" is expected to be a Lua variable holding the JSON object
// "counter_value" is expected to be a Lua variable holding the new count value
/string.gsub(value, '"count"%s*:%s*(%d+)', '"count":' .. counter_value)/
}

Expand Down
9 changes: 5 additions & 4 deletions src/main/groovy/io/seqera/wave/store/state/CountResult.groovy
Original file line number Diff line number Diff line change
Expand Up @@ -21,12 +21,13 @@ package io.seqera.wave.store.state
import groovy.transform.Canonical

/**
*
* Model the result object of state auto-increment operation
*
* @author Paolo Di Tommaso <[email protected]>
*/
@Canonical
class CountResult<V> {
Boolean succeed
V value
Integer count
final Boolean succeed
final V value
final Integer count
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,89 @@
/*
* Wave, containers provisioning service
* Copyright (c) 2023-2024, Seqera Labs
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/

package io.seqera.wave.service.builder

import spock.lang.Specification

import java.nio.file.Path

import io.seqera.wave.core.ContainerPlatform
import io.seqera.wave.tower.PlatformId

/**
*
* @author Paolo Di Tommaso <[email protected]>
*/
class BuildEntryTest extends Specification {

def 'should create entry with constructor'() {
given:
def request = Mock(BuildRequest)
def result = Mock(BuildResult)
when:
def entry = new BuildEntry(request, result)
then:
entry.request == request
entry.result == result
}

def 'should create entry with result'() {
given:
def request = Mock(BuildRequest)
def result1 = Mock(BuildResult)
def result2 = Mock(BuildResult)
when:
def entry1 = new BuildEntry(request, result1)
then:
entry1.request == request
entry1.result == result1

when:
def entry2 = entry1.withResult(result2)
then:
entry1.request == request
entry1.result == result1
and:
entry2.request == request
entry2.result == result2
and:
result1 != result2
}

def 'should create entry with factory' () {
given:
def request = new BuildRequest(
containerId: '12345',
buildId: "bd-12345_1",
containerFile: 'FROM foo',
workspace: Path.of("/some/path"),
targetImage: 'some/targer:12345',

Check failure on line 75 in src/test/groovy/io/seqera/wave/service/builder/BuildEntryTest.groovy

View workflow job for this annotation

GitHub Actions / Check for spelling errors

targer ==> target, larger, tagger

Check failure on line 75 in src/test/groovy/io/seqera/wave/service/builder/BuildEntryTest.groovy

View workflow job for this annotation

GitHub Actions / Check for spelling errors

targer ==> target, larger, tagger
identity: PlatformId.NULL,
platform: ContainerPlatform.DEFAULT,
cacheRepository: 'cacherepo',
ip: "1.2.3.4",
configJson: '{"config":"json"}',
scanId: 'scan12345',
)
when:
def entry = BuildEntry.create(request)
then:
entry.request == request
entry.result == BuildResult.create(request)
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -137,7 +137,7 @@ class ContainerBuildServiceTest extends Specification {
and:
def store = Mock(BuildStateStore)
def jobService = Mock(JobService)
def builder = new ContainerBuildServiceImpl(buildStateStore: store, buildConfig: buildConfig, jobService: jobService)
def builder = new ContainerBuildServiceImpl(buildStore: store, buildConfig: buildConfig, jobService: jobService)
def RESPONSE = Mock(JobSpec)

when:
Expand Down Expand Up @@ -350,7 +350,7 @@ class ContainerBuildServiceTest extends Specification {
def mockBuildStore = Mock(BuildStateStore)
def mockProxyService = Mock(RegistryProxyService)
def mockEventPublisher = Mock(ApplicationEventPublisher<BuildEvent>)
def service = new ContainerBuildServiceImpl(buildStateStore: mockBuildStore, proxyService: mockProxyService, eventPublisher: mockEventPublisher, buildConfig: buildConfig)
def service = new ContainerBuildServiceImpl(buildStore: mockBuildStore, proxyService: mockProxyService, eventPublisher: mockEventPublisher, buildConfig: buildConfig)
def job = JobSpec.build('1', 'operationName', Instant.now(), Duration.ofMinutes(1), Path.of('/work/dir'))
def state = JobState.succeeded('logs')
def res = BuildResult.create('1')
Expand Down Expand Up @@ -378,7 +378,7 @@ class ContainerBuildServiceTest extends Specification {
def mockBuildStore = Mock(BuildStateStore)
def mockProxyService = Mock(RegistryProxyService)
def mockEventPublisher = Mock(ApplicationEventPublisher<BuildEvent>)
def service = new ContainerBuildServiceImpl(buildStateStore: mockBuildStore, proxyService: mockProxyService, eventPublisher: mockEventPublisher, buildConfig: buildConfig)
def service = new ContainerBuildServiceImpl(buildStore: mockBuildStore, proxyService: mockProxyService, eventPublisher: mockEventPublisher, buildConfig: buildConfig)
def job = JobSpec.build('1', 'operationName', Instant.now(), Duration.ofMinutes(1), Path.of('/work/dir'))
def error = new Exception('error')
def res = BuildResult.create('1')
Expand All @@ -404,7 +404,7 @@ class ContainerBuildServiceTest extends Specification {
def mockBuildStore = Mock(BuildStateStore)
def mockProxyService = Mock(RegistryProxyService)
def mockEventPublisher = Mock(ApplicationEventPublisher<BuildEvent>)
def service = new ContainerBuildServiceImpl(buildStateStore: mockBuildStore, proxyService: mockProxyService, eventPublisher: mockEventPublisher, buildConfig: buildConfig)
def service = new ContainerBuildServiceImpl(buildStore: mockBuildStore, proxyService: mockProxyService, eventPublisher: mockEventPublisher, buildConfig: buildConfig)
def job = JobSpec.build('1', 'operationName', Instant.now(), Duration.ofMinutes(1), Path.of('/work/dir'))
def res = BuildResult.create('1')
def req = new BuildRequest(
Expand Down

0 comments on commit 599684e

Please sign in to comment.