diff --git a/src/main/java/org/jboss/pnc/cleaner/logverifier/BifrostClient.java b/src/main/java/org/jboss/pnc/cleaner/logverifier/BifrostClient.java deleted file mode 100644 index aa1322a..0000000 --- a/src/main/java/org/jboss/pnc/cleaner/logverifier/BifrostClient.java +++ /dev/null @@ -1,30 +0,0 @@ -/* - * JBoss, Home of Professional Open Source. - * Copyright 2019-2022 Red Hat, Inc., and individual contributors - * as indicated by the @author tags. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.jboss.pnc.cleaner.logverifier; - -import org.eclipse.microprofile.rest.client.inject.RegisterRestClient; -import org.jboss.pnc.api.bifrost.rest.Bifrost; - -/** - * @author Matej Lazar We don't need build-log-verifier anymore. Candidate - * for removal - */ -@RegisterRestClient -@Deprecated -public interface BifrostClient extends Bifrost { -} diff --git a/src/main/java/org/jboss/pnc/cleaner/logverifier/BuildLogVerifier.java b/src/main/java/org/jboss/pnc/cleaner/logverifier/BuildLogVerifier.java deleted file mode 100644 index 52f9e9c..0000000 --- a/src/main/java/org/jboss/pnc/cleaner/logverifier/BuildLogVerifier.java +++ /dev/null @@ -1,193 +0,0 @@ -/* - * JBoss, Home of Professional Open Source. - * Copyright 2019-2022 Red Hat, Inc., and individual contributors - * as indicated by the @author tags. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.jboss.pnc.cleaner.logverifier; - -import io.micrometer.core.annotation.Timed; -import io.micrometer.core.instrument.Counter; -import io.micrometer.core.instrument.MeterRegistry; -import io.opentelemetry.api.GlobalOpenTelemetry; -import io.opentelemetry.api.trace.Span; -import io.opentelemetry.api.trace.SpanBuilder; -import io.opentelemetry.api.trace.SpanKind; -import io.opentelemetry.context.Scope; -import jakarta.annotation.PostConstruct; -import org.eclipse.microprofile.config.inject.ConfigProperty; -import org.eclipse.microprofile.rest.client.inject.RestClient; -import org.jboss.pnc.api.bifrost.dto.MetaData; -import org.jboss.pnc.api.bifrost.enums.Direction; -import org.jboss.pnc.cleaner.orchApi.OrchClientProducer; -import org.jboss.pnc.client.BuildClient; -import org.jboss.pnc.client.RemoteCollection; -import org.jboss.pnc.client.RemoteResourceException; -import org.jboss.pnc.common.otel.OtelUtils; -import org.jboss.pnc.dto.Build; -import org.jboss.pnc.rest.api.parameters.BuildsFilterParameters; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import jakarta.enterprise.context.ApplicationScoped; -import jakarta.inject.Inject; -import java.io.IOException; -import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.concurrent.atomic.AtomicInteger; - -/** - * @author Matej Lazar We don't need build-log-verifier anymore. Candidate for - * removal - */ -@ApplicationScoped -@Deprecated -public class BuildLogVerifier { - - private static final String className = BuildLogVerifier.class.getName(); - - private final Logger logger = LoggerFactory.getLogger(BuildLogVerifier.class); - private final Integer DEFAULT_BIFROST_BATCH_SIZE = 10000; - - @Inject - @RestClient - BifrostClient bifrost; - - @Inject - BuildClient buildClient; - - @Inject - OrchClientProducer orchClientProducer; - - @ConfigProperty(name = "buildLogVerifierScheduler.maxRetries") - private Integer maxRetries; - - private final Map buildESLogErrorCounter = new HashMap<>(); - - public static final String BUILD_OUTPUT_OK_KEY = "BUILD_OUTPUT_OK"; - - @Inject - MeterRegistry registry; - - private Counter errCounter; - private Counter warnCounter; - - @PostConstruct - void initMetrics() { - errCounter = registry.counter(className + ".error.count"); - warnCounter = registry.counter(className + ".warning.count"); - } - - public BuildLogVerifier() { - } - - @Timed - public int verifyUnflaggedBuilds() { - - logger.info("Verifying log checksums ..."); - Collection unverifiedBuilds = getUnverifiedBuilds().getAll(); - logger.info("Found {} unverified builds.", unverifiedBuilds.size()); - unverifiedBuilds.forEach(build -> verify(build.getId(), build.getBuildOutputChecksum())); - return unverifiedBuilds.size(); - } - - @Timed - void verify(String buildId, String checksum) { - - try { - logger.info("Verifying log for build id: {}", buildId); - String esChecksum = getESChecksum(buildId); - if (checksum.equals(esChecksum)) { - logger.info("Build output checksum OK. BuildId: {}, Checksum: {}.", buildId, checksum); - flagPncBuild(buildId, true); - - removeRetryCounter(buildId); - } else { - warnCounter.increment(); - logger.warn( - "Build output checksum MISMATCH. BuildId: {}, Db checksum: {}, ElasticSearch checksum {}.", - buildId, - checksum, - esChecksum); - - handleMismatchWithRetries(buildId); - } - } catch (IOException e) { - errCounter.increment(); - logger.error("Cannot verify checksum for buildId: " + buildId + ".", e); - } - } - - private void removeRetryCounter(String buildId) { - buildESLogErrorCounter.remove(buildId); - } - - @Timed - void handleMismatchWithRetries(String buildId) { - if (!buildESLogErrorCounter.containsKey(buildId)) { - buildESLogErrorCounter.put(buildId, new AtomicInteger(0)); - } - - AtomicInteger numOfRetries = buildESLogErrorCounter.get(buildId); - if (numOfRetries.get() >= maxRetries) { - warnCounter.increment(); - logger.warn("Marking build with id: {} as mismatch", buildId); - flagPncBuild(buildId, false); - removeRetryCounter(buildId); - return; - } - - warnCounter.increment(); - logger.warn("Increasing retry counter (counter: {}) for build with id: {}", numOfRetries, buildId); - buildESLogErrorCounter.get(buildId).incrementAndGet(); - } - - @Timed - String getESChecksum(String buildId) throws IOException { - String matchFilters = "mdc.processContext.keyword:build-" + buildId + "," - + "loggerName.keyword:org.jboss.pnc._userlog_.build-log"; - - MetaData metaData = bifrost - .getMetaData(matchFilters, null, null, Direction.ASC, null, DEFAULT_BIFROST_BATCH_SIZE); - return metaData.getMd5Digest(); - } - - private void flagPncBuild(String buildId, boolean checksumMatch) { - try (BuildClient buildClientAuthenticated = orchClientProducer.getAuthenticatedBuildClient()) { - buildClientAuthenticated.addAttribute(buildId, BUILD_OUTPUT_OK_KEY, Boolean.toString(checksumMatch)); - } catch (RemoteResourceException e) { - errCounter.increment(); - logger.error("Cannot set {} attribute to build id: {}.", checksumMatch, buildId); - } - } - - @Timed - RemoteCollection getUnverifiedBuilds() { - BuildsFilterParameters buildsFilterParameters = new BuildsFilterParameters(); - buildsFilterParameters.setRunning(false); - List attributes = Collections.singletonList("!" + BUILD_OUTPUT_OK_KEY); - try { - String query = "buildOutputChecksum!=null"; - return buildClient.getAll(buildsFilterParameters, attributes, Optional.empty(), Optional.of(query)); - } catch (RemoteResourceException e) { - errCounter.increment(); - logger.error("Cannot read remote builds.", e); - return RemoteCollection.empty(); - } - } -} diff --git a/src/main/java/org/jboss/pnc/cleaner/logverifier/BuildLogVerifierScheduler.java b/src/main/java/org/jboss/pnc/cleaner/logverifier/BuildLogVerifierScheduler.java deleted file mode 100644 index e583325..0000000 --- a/src/main/java/org/jboss/pnc/cleaner/logverifier/BuildLogVerifierScheduler.java +++ /dev/null @@ -1,64 +0,0 @@ -/* - * JBoss, Home of Professional Open Source. - * Copyright 2019-2022 Red Hat, Inc., and individual contributors - * as indicated by the @author tags. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.jboss.pnc.cleaner.logverifier; - -/** - * JBoss, Home of Professional Open Source. - * Copyright 2014 Red Hat, Inc., and individual contributors - * as indicated by the @author tags. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -import io.micrometer.core.annotation.Timed; -import io.opentelemetry.instrumentation.annotations.WithSpan; -import io.quarkus.scheduler.Scheduled; - -import jakarta.inject.Inject; - -/** - * @author Matej Lazar We don't need build-log-verifier anymore. Candidate for - * removal - */ -@Deprecated -public class BuildLogVerifierScheduler { - - @Inject - BuildLogVerifier buildLogVerifier; - - /** - * We don't need build-log-verifier anymore. Candidate for removal - */ - @Timed - @Scheduled(cron = "{buildLogVerifierScheduler.cron}", concurrentExecution = Scheduled.ConcurrentExecution.SKIP) - @WithSpan - @Deprecated - public void verifyBuildLogs() { - buildLogVerifier.verifyUnflaggedBuilds(); - } - -} diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties index 2b8b7c6..be047b4 100644 --- a/src/main/resources/application.properties +++ b/src/main/resources/application.properties @@ -49,13 +49,6 @@ temporaryBuildsCleaner.lifespan=14 # Starts every day at 0:15AM temporaryBuildsCleaner.cron=0 15 0 ? * Sun -#Build Log Verifier -# Deprecated -#run every 15 min from year 1970, effectively disabling it -buildLogVerifierScheduler.cron=0 */15 * ? * * 1970 -buildLogVerifierScheduler.maxRetries=5 -%test.buildLogVerifierScheduler.maxRetries=1 - #Build Archiver #run every 30 min buildArchiverScheduler.cron=0 */30 * ? * * @@ -65,8 +58,7 @@ applicationUri=0.0.0.0:8080 quarkus.index-dependency.pncapi.group-id=org.jboss.pnc quarkus.index-dependency.pncapi.artifact-id=pnc-api -org.jboss.pnc.cleaner.logverifier.BifrostClient/mp-rest/url=http://localhost:8081/ -org.jboss.pnc.cleaner.archiver.FinalLogClient/mp-rest/url=${org.jboss.pnc.cleaner.logverifier.BifrostClient/mp-rest/url} +org.jboss.pnc.cleaner.archiver.FinalLogClient/mp-rest/url=http://localhost:8081/ quarkus.datasource.db-kind = postgresql prod.quarkus.datasource.jdbc.url = ${QUARKUS_DATASOURCE_URL:} diff --git a/src/test/java/org/jboss/pnc/cleaner/logverifier/LogVerifierTest.java b/src/test/java/org/jboss/pnc/cleaner/logverifier/LogVerifierTest.java deleted file mode 100644 index a37c425..0000000 --- a/src/test/java/org/jboss/pnc/cleaner/logverifier/LogVerifierTest.java +++ /dev/null @@ -1,98 +0,0 @@ -/* - * JBoss, Home of Professional Open Source. - * Copyright 2019-2022 Red Hat, Inc., and individual contributors - * as indicated by the @author tags. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.jboss.pnc.cleaner.logverifier; - -import io.quarkus.test.junit.QuarkusTest; -import lombok.extern.slf4j.Slf4j; -import org.jboss.pnc.api.bifrost.dto.MetaData; -import org.jboss.pnc.cleaner.mock.BifrostProvider; -import org.jboss.pnc.cleaner.mock.OrchBuildProvider; -import org.jboss.pnc.dto.Build; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.MethodOrderer; -import org.junit.jupiter.api.Order; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.TestMethodOrder; - -import jakarta.inject.Inject; -import java.util.HashMap; - -import static org.jboss.pnc.cleaner.logverifier.BuildLogVerifier.BUILD_OUTPUT_OK_KEY; - -/** - * @author Matej Lazar - */ -@Slf4j -@TestMethodOrder(MethodOrderer.OrderAnnotation.class) -@QuarkusTest -public class LogVerifierTest { - - @Inject - BuildLogVerifier buildLogVerifier; - - @Inject - OrchBuildProvider orchBuildProvider; - - @Inject - BifrostProvider bifrostProvider; - - @Inject - OrchClientConfigurationMock orchClientConfiguration; - - @Test - @Order(-1) // Before all methods - public void prepare() { - log.info("Overriding configuration ..."); - orchClientConfiguration.setPort(8081); // Quarkus test server port - } - - @Test - @Order(Integer.MAX_VALUE) // After all methods - public void cleanup() { - log.info("Restoring configuration ..."); - orchClientConfiguration.setPort(8082); // wiremock server used by other tests - } - - @Test - public void shouldMarkMatchedChecksum() { - // given - Build build1 = Build.builder().id("1").buildOutputChecksum("match").attributes(new HashMap<>()).build(); - orchBuildProvider.addBuild(build1); - Build build2 = Build.builder().id("2").buildOutputChecksum("dont-match").attributes(new HashMap<>()).build(); - orchBuildProvider.addBuild(build2); - - bifrostProvider.addMetaData("build-1", new MetaData("match")); - bifrostProvider.addMetaData("build-2", new MetaData("wrong")); - - // when #1 - buildLogVerifier.verifyUnflaggedBuilds(); - - // then #1 - Build build1Updated = orchBuildProvider.getById("1"); - Assertions.assertEquals(Boolean.TRUE.toString(), build1Updated.getAttributes().get(BUILD_OUTPUT_OK_KEY)); - Build build2Updated = orchBuildProvider.getById("2"); - Assertions.assertEquals(null, build2Updated.getAttributes().get(BUILD_OUTPUT_OK_KEY)); - - // when #2 (simulate retry for build 2) - buildLogVerifier.verifyUnflaggedBuilds(); - - // then #2 - build2Updated = orchBuildProvider.getById("2"); - Assertions.assertEquals(Boolean.FALSE.toString(), build2Updated.getAttributes().get(BUILD_OUTPUT_OK_KEY)); - } -} diff --git a/src/test/java/org/jboss/pnc/cleaner/logverifier/OrchClientConfigurationMock.java b/src/test/java/org/jboss/pnc/cleaner/logverifier/OrchClientConfigurationMock.java deleted file mode 100644 index 358014b..0000000 --- a/src/test/java/org/jboss/pnc/cleaner/logverifier/OrchClientConfigurationMock.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * JBoss, Home of Professional Open Source. - * Copyright 2019-2022 Red Hat, Inc., and individual contributors - * as indicated by the @author tags. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.jboss.pnc.cleaner.logverifier; - -import io.quarkus.test.Mock; -import org.jboss.pnc.cleaner.orchApi.OrchClientConfiguration; - -import jakarta.enterprise.context.ApplicationScoped; - -/** - * @author Matej Lazar - */ -@Mock -@ApplicationScoped -public class OrchClientConfigurationMock extends OrchClientConfiguration { - - public void setPort(Integer port) { - this.port = port; - } -}