From 40ec3e392e5333e8b63dedea0c173231652ab541 Mon Sep 17 00:00:00 2001 From: Nick Cross Date: Tue, 21 Nov 2023 15:07:18 +0000 Subject: [PATCH 01/15] Add logging. Fix project naming in gitlab. Handle PKIX issues. --- .../container/analyser/deploy/DeployCommand.java | 1 + .../hacbs/container/analyser/deploy/git/Git.java | 7 +++++-- .../container/analyser/deploy/git/GitHub.java | 10 ++++++++++ .../container/analyser/deploy/git/GitLab.java | 13 +++++++++++++ .../mavenrepository/MavenRepositoryDeployer.java | 2 ++ .../container/analyser/deploy/git/GitTest.java | 14 +++++++++++--- 6 files changed, 42 insertions(+), 5 deletions(-) diff --git a/java-components/build-request-processor/src/main/java/com/redhat/hacbs/container/analyser/deploy/DeployCommand.java b/java-components/build-request-processor/src/main/java/com/redhat/hacbs/container/analyser/deploy/DeployCommand.java index c29ab9858..101802da5 100644 --- a/java-components/build-request-processor/src/main/java/com/redhat/hacbs/container/analyser/deploy/DeployCommand.java +++ b/java-components/build-request-processor/src/main/java/com/redhat/hacbs/container/analyser/deploy/DeployCommand.java @@ -155,6 +155,7 @@ public void run() { try { // Save the source first regardless of deployment checks if (isNotEmpty(gitIdentity) && gitToken.isPresent()) { + Log.infof("Git credentials are identity '%s' and URL '%s'", gitIdentity, gitURL); var git = Git.builder(gitURL, gitIdentity, gitToken.get()); git.create(scmUri); git.add(sourcePath, commit, imageId); diff --git a/java-components/build-request-processor/src/main/java/com/redhat/hacbs/container/analyser/deploy/git/Git.java b/java-components/build-request-processor/src/main/java/com/redhat/hacbs/container/analyser/deploy/git/Git.java index b70f6684b..b926a127a 100644 --- a/java-components/build-request-processor/src/main/java/com/redhat/hacbs/container/analyser/deploy/git/Git.java +++ b/java-components/build-request-processor/src/main/java/com/redhat/hacbs/container/analyser/deploy/git/Git.java @@ -61,6 +61,7 @@ protected void pushRepository(Path path, String httpTransportUrl, String commit, Log.infof("Updating current origin of %s to %s", jConfig.getString("remote", "origin", "url"), httpTransportUrl); jConfig.setString("remote", "origin", "url", httpTransportUrl); + jConfig.setBoolean("http", null, "sslVerify", false); jConfig.save(); Log.infof("Pushing to %s with content from %s (branch %s, commit %s, tag %s)", httpTransportUrl, path, jRepo.getBranch(), commit, tagName); @@ -97,11 +98,13 @@ protected void pushRepository(Path path, String httpTransportUrl, String commit, * @return a reformatted name to use as the new repository name. * @throws URISyntaxException if an error occurs. */ - protected static String parseScmURI(String scmUri) + protected String parseScmURI(String scmUri) throws URISyntaxException { String path = new URI(scmUri).getPath().substring(1); String group = path.substring(0, path.lastIndexOf("/")); String name = (path.endsWith(".git") ? path.substring(0, path.length() - 4) : path).substring(group.length() + 1); - return group + "--" + name; + return group + groupSplit() + name; } + + abstract String groupSplit(); } diff --git a/java-components/build-request-processor/src/main/java/com/redhat/hacbs/container/analyser/deploy/git/GitHub.java b/java-components/build-request-processor/src/main/java/com/redhat/hacbs/container/analyser/deploy/git/GitHub.java index ded42ed24..ff8198085 100644 --- a/java-components/build-request-processor/src/main/java/com/redhat/hacbs/container/analyser/deploy/git/GitHub.java +++ b/java-components/build-request-processor/src/main/java/com/redhat/hacbs/container/analyser/deploy/git/GitHub.java @@ -49,6 +49,11 @@ public GitHub(String endpoint, String identity, String token) Log.infof("Type %s", type); } + GitHub() { + owner = null; + github = null; + } + @Override public void create(String scmUri) throws IOException, URISyntaxException { @@ -85,4 +90,9 @@ public void add(Path path, String commit, String imageId) { } pushRepository(path, repository.getHttpTransportUrl(), commit, imageId); } + + @Override + String groupSplit() { + return "--"; + } } diff --git a/java-components/build-request-processor/src/main/java/com/redhat/hacbs/container/analyser/deploy/git/GitLab.java b/java-components/build-request-processor/src/main/java/com/redhat/hacbs/container/analyser/deploy/git/GitLab.java index 6624e55e4..58cf2216b 100644 --- a/java-components/build-request-processor/src/main/java/com/redhat/hacbs/container/analyser/deploy/git/GitLab.java +++ b/java-components/build-request-processor/src/main/java/com/redhat/hacbs/container/analyser/deploy/git/GitLab.java @@ -24,12 +24,20 @@ public GitLab(String endpoint, String identity, String token) { gitLabApi = new GitLabApi(endpoint, token); owner = identity; credentialsProvider = new UsernamePasswordCredentialsProvider("", token); + + gitLabApi.setIgnoreCertificateErrors(true); + } + + GitLab() { + owner = null; + gitLabApi = null; } @Override public void create(String scmUri) throws URISyntaxException { String name = parseScmURI(scmUri); + Log.infof("Creating repository with name %s", name); try { project = gitLabApi.getProjectApi().getUserProjectsStream(owner, new ProjectFilter().withSearch(name)) .filter(p -> p.getName().equals(name)) @@ -62,4 +70,9 @@ public void add(Path path, String commit, String imageId) } pushRepository(path, project.getHttpUrlToRepo(), commit, imageId); } + + @Override + String groupSplit() { + return "-"; + } } diff --git a/java-components/build-request-processor/src/main/java/com/redhat/hacbs/container/analyser/deploy/mavenrepository/MavenRepositoryDeployer.java b/java-components/build-request-processor/src/main/java/com/redhat/hacbs/container/analyser/deploy/mavenrepository/MavenRepositoryDeployer.java index 08d87c127..aa17eb66e 100644 --- a/java-components/build-request-processor/src/main/java/com/redhat/hacbs/container/analyser/deploy/mavenrepository/MavenRepositoryDeployer.java +++ b/java-components/build-request-processor/src/main/java/com/redhat/hacbs/container/analyser/deploy/mavenrepository/MavenRepositoryDeployer.java @@ -49,6 +49,8 @@ public MavenRepositoryDeployer(BootstrapMavenContext mvnCtx, String username, St this.system = mvnCtx.getRepositorySystem(); this.session = MavenRepositorySystemUtils.newSession(); + Log.infof("Maven credentials are username '%s' and repository '%s'", username, repository); + // https://maven.apache.org/resolver/third-party-integrations.html states a local repository manager should be added. session.setLocalRepositoryManager(system.newLocalRepositoryManager(session, new LocalRepository(artifacts.toFile()))); } diff --git a/java-components/build-request-processor/src/test/java/com/redhat/hacbs/container/analyser/deploy/git/GitTest.java b/java-components/build-request-processor/src/test/java/com/redhat/hacbs/container/analyser/deploy/git/GitTest.java index 590210392..56afb4cc7 100644 --- a/java-components/build-request-processor/src/test/java/com/redhat/hacbs/container/analyser/deploy/git/GitTest.java +++ b/java-components/build-request-processor/src/test/java/com/redhat/hacbs/container/analyser/deploy/git/GitTest.java @@ -37,12 +37,15 @@ public void clearLogs() { @Test public void parseScmURI() throws URISyntaxException { - String result = Git.parseScmURI("https://github.com/apache/commons-codec.git"); + + String result = new GitHub().parseScmURI("https://github.com/apache/commons-codec.git"); assertEquals("apache--commons-codec", result); - result = Git.parseScmURI("https://gitlab.com/rnc/testRepo"); + result = new GitHub().parseScmURI("https://gitlab.com/rnc/testRepo"); assertEquals("rnc--testRepo", result); - result = Git.parseScmURI("file:///rnc/testRepo"); + result = new GitHub().parseScmURI("file:///rnc/testRepo"); assertEquals("rnc--testRepo", result); + result = new GitLab().parseScmURI("https://gitlab.com/rnc/testRepo"); + assertEquals("rnc-testRepo", result); } @Test @@ -73,6 +76,11 @@ public void create(String name) { @Override public void add(Path path, String commit, String imageId) { } + + @Override + public String groupSplit() { + return null; + } }; test.pushRepository( initialRepo, From 4f9ce332288151ea80c0d445357d60916d93f4bc Mon Sep 17 00:00:00 2001 From: Nick Cross Date: Wed, 22 Nov 2023 14:13:56 +0000 Subject: [PATCH 02/15] Make SSL verification configurable --- .../base/jvmbuildservice.io_jbsconfigs.yaml | 2 ++ deploy/overlays/dev-template/config.yaml | 1 + deploy/patch-yaml.sh | 4 ++++ .../analyser/deploy/DeployCommand.java | 5 +++- .../container/analyser/deploy/git/Git.java | 24 ++++++++++++++++--- .../container/analyser/deploy/git/GitHub.java | 3 ++- .../container/analyser/deploy/git/GitLab.java | 7 ++++-- .../analyser/deploy/git/GitTest.java | 4 ++-- .../crds/jvmbuildservice.io_jbsconfigs.yaml | 2 ++ .../v1alpha1/jbsconfig_types.go | 5 ++-- .../dependencybuild/buildrecipeyaml.go | 3 +++ 11 files changed, 49 insertions(+), 11 deletions(-) diff --git a/deploy/crds/base/jvmbuildservice.io_jbsconfigs.yaml b/deploy/crds/base/jvmbuildservice.io_jbsconfigs.yaml index dfadb6a02..428148434 100644 --- a/deploy/crds/base/jvmbuildservice.io_jbsconfigs.yaml +++ b/deploy/crds/base/jvmbuildservice.io_jbsconfigs.yaml @@ -89,6 +89,8 @@ spec: type: boolean gitSourceArchive: properties: + disableSSLVerification: + type: string identity: type: string url: diff --git a/deploy/overlays/dev-template/config.yaml b/deploy/overlays/dev-template/config.yaml index d1fd2237a..2804f7775 100644 --- a/deploy/overlays/dev-template/config.yaml +++ b/deploy/overlays/dev-template/config.yaml @@ -16,6 +16,7 @@ spec: gitSourceArchive: identity: GIT_DEPLOY_IDENTITY url: GIT_DEPLOY_URL + disableSSLVerification: GIT_DISABLE_SSL_VERIFICATION relocationPatterns: - relocationPattern: buildPolicy: "default" diff --git a/deploy/patch-yaml.sh b/deploy/patch-yaml.sh index ccb45e3eb..fc0c44887 100755 --- a/deploy/patch-yaml.sh +++ b/deploy/patch-yaml.sh @@ -53,5 +53,9 @@ fi if [ -z "${GIT_DEPLOY_IDENTITY}" ]; then GIT_DEPLOY_IDENTITY="" fi +if [ -z "${GIT_DISABLE_SSL_VERIFICATION}" ]; then + GIT_DISABLE_SSL_VERIFICATION="false" +fi find $DIR -path \*development\*.yaml -exec $SED -i s/GIT_DEPLOY_URL/${GIT_DEPLOY_URL}/ {} \; find $DIR -path \*development\*.yaml -exec $SED -i s%GIT_DEPLOY_IDENTITY%${GIT_DEPLOY_IDENTITY}% {} \; +find $DIR -path \*development\*.yaml -exec $SED -i s%GIT_DISABLE_SSL_VERIFICATION%${GIT_DISABLE_SSL_VERIFICATION}% {} \; diff --git a/java-components/build-request-processor/src/main/java/com/redhat/hacbs/container/analyser/deploy/DeployCommand.java b/java-components/build-request-processor/src/main/java/com/redhat/hacbs/container/analyser/deploy/DeployCommand.java index 101802da5..7fd83b2a1 100644 --- a/java-components/build-request-processor/src/main/java/com/redhat/hacbs/container/analyser/deploy/DeployCommand.java +++ b/java-components/build-request-processor/src/main/java/com/redhat/hacbs/container/analyser/deploy/DeployCommand.java @@ -134,6 +134,9 @@ public class DeployCommand implements Runnable { @CommandLine.Option(names = "--git-identity") String gitIdentity; + @CommandLine.Option(names = "--git-disable-ssl-verification") + boolean gitDisableSSLVerification; + @CommandLine.Option(names = "--build-id") String buildId; // Testing only ; used to disable image deployment @@ -156,7 +159,7 @@ public void run() { // Save the source first regardless of deployment checks if (isNotEmpty(gitIdentity) && gitToken.isPresent()) { Log.infof("Git credentials are identity '%s' and URL '%s'", gitIdentity, gitURL); - var git = Git.builder(gitURL, gitIdentity, gitToken.get()); + var git = Git.builder(gitURL, gitIdentity, gitToken.get(), gitDisableSSLVerification); git.create(scmUri); git.add(sourcePath, commit, imageId); } diff --git a/java-components/build-request-processor/src/main/java/com/redhat/hacbs/container/analyser/deploy/git/Git.java b/java-components/build-request-processor/src/main/java/com/redhat/hacbs/container/analyser/deploy/git/Git.java index b926a127a..71bc59c4e 100644 --- a/java-components/build-request-processor/src/main/java/com/redhat/hacbs/container/analyser/deploy/git/Git.java +++ b/java-components/build-request-processor/src/main/java/com/redhat/hacbs/container/analyser/deploy/git/Git.java @@ -19,6 +19,8 @@ public abstract class Git { protected CredentialsProvider credentialsProvider; + protected boolean disableSSLVerification; + public abstract void create(String name) throws IOException, URISyntaxException; @@ -35,14 +37,28 @@ public abstract void add(Path path, String commit, String imageId) */ public static Git builder(String endpoint, String identity, String token) throws IOException { + return builder(endpoint, identity, token, true); + } + + /** + * + * @param endpoint URL of the GitHub or GitLab instance. + * @param identity Might be user or organisation name. + * @param token Authorisation token. + * @param disableSSLVerification Whether to enable SSLVerification (Default: true). + * @return Valid Git instance + * @throws IOException if an error occurs + */ + public static Git builder(String endpoint, String identity, String token, boolean disableSSLVerification) + throws IOException { // TODO: This could be a bit presumptuous to assume // an on-premise installation will always contain some determinable // information. Alternative would be forcing the user to configure // endpoint, token, AND type [gitlab|github] if (endpoint != null && endpoint.contains("gitlab")) { - return new GitLab(endpoint, identity, token); + return new GitLab(endpoint, identity, token, disableSSLVerification); } else { - return new GitHub(endpoint, identity, token); + return new GitHub(endpoint, identity, token, disableSSLVerification); } } @@ -61,7 +77,9 @@ protected void pushRepository(Path path, String httpTransportUrl, String commit, Log.infof("Updating current origin of %s to %s", jConfig.getString("remote", "origin", "url"), httpTransportUrl); jConfig.setString("remote", "origin", "url", httpTransportUrl); - jConfig.setBoolean("http", null, "sslVerify", false); + if (disableSSLVerification) { + jConfig.setBoolean("http", null, "sslVerify", false); + } jConfig.save(); Log.infof("Pushing to %s with content from %s (branch %s, commit %s, tag %s)", httpTransportUrl, path, jRepo.getBranch(), commit, tagName); diff --git a/java-components/build-request-processor/src/main/java/com/redhat/hacbs/container/analyser/deploy/git/GitHub.java b/java-components/build-request-processor/src/main/java/com/redhat/hacbs/container/analyser/deploy/git/GitHub.java index ff8198085..240bf72fc 100644 --- a/java-components/build-request-processor/src/main/java/com/redhat/hacbs/container/analyser/deploy/git/GitHub.java +++ b/java-components/build-request-processor/src/main/java/com/redhat/hacbs/container/analyser/deploy/git/GitHub.java @@ -29,7 +29,7 @@ enum Type { private GHRepository repository; - public GitHub(String endpoint, String identity, String token) + public GitHub(String endpoint, String identity, String token, boolean ssl) throws IOException { if (isNotEmpty(token)) { github = new GitHubBuilder().withEndpoint(endpoint == null ? GITHUB_URL : endpoint) @@ -41,6 +41,7 @@ public GitHub(String endpoint, String identity, String token) } owner = identity; credentialsProvider = new UsernamePasswordCredentialsProvider(token, ""); + disableSSLVerification = ssl; switch (github.getUser(identity).getType()) { case "User" -> type = Type.USER; diff --git a/java-components/build-request-processor/src/main/java/com/redhat/hacbs/container/analyser/deploy/git/GitLab.java b/java-components/build-request-processor/src/main/java/com/redhat/hacbs/container/analyser/deploy/git/GitLab.java index 58cf2216b..ba726eeb5 100644 --- a/java-components/build-request-processor/src/main/java/com/redhat/hacbs/container/analyser/deploy/git/GitLab.java +++ b/java-components/build-request-processor/src/main/java/com/redhat/hacbs/container/analyser/deploy/git/GitLab.java @@ -20,12 +20,15 @@ public class GitLab extends Git { private Project project; - public GitLab(String endpoint, String identity, String token) { + public GitLab(String endpoint, String identity, String token, boolean ssl) { gitLabApi = new GitLabApi(endpoint, token); owner = identity; credentialsProvider = new UsernamePasswordCredentialsProvider("", token); + disableSSLVerification = ssl; - gitLabApi.setIgnoreCertificateErrors(true); + if (disableSSLVerification) { + gitLabApi.setIgnoreCertificateErrors(true); + } } GitLab() { diff --git a/java-components/build-request-processor/src/test/java/com/redhat/hacbs/container/analyser/deploy/git/GitTest.java b/java-components/build-request-processor/src/test/java/com/redhat/hacbs/container/analyser/deploy/git/GitTest.java index 56afb4cc7..31095c655 100644 --- a/java-components/build-request-processor/src/test/java/com/redhat/hacbs/container/analyser/deploy/git/GitTest.java +++ b/java-components/build-request-processor/src/test/java/com/redhat/hacbs/container/analyser/deploy/git/GitTest.java @@ -106,11 +106,11 @@ public String groupSplit() { @Test public void testIdentity() throws IOException { - new GitHub(null, "cekit", null); + new GitHub(null, "cekit", null, true); List logRecords = LogCollectingTestResource.current().getRecords(); assertTrue(logRecords.stream().anyMatch(r -> LogCollectingTestResource.format(r).matches("Type ORGANISATION"))); LogCollectingTestResource.current().clear(); - new GitHub(null, "rnc", null); + new GitHub(null, "rnc", null, true); logRecords = LogCollectingTestResource.current().getRecords(); assertTrue(logRecords.stream().anyMatch(r -> LogCollectingTestResource.format(r).matches("Type USER"))); } diff --git a/java-components/resource-model/src/main/resources/crds/jvmbuildservice.io_jbsconfigs.yaml b/java-components/resource-model/src/main/resources/crds/jvmbuildservice.io_jbsconfigs.yaml index dfadb6a02..428148434 100644 --- a/java-components/resource-model/src/main/resources/crds/jvmbuildservice.io_jbsconfigs.yaml +++ b/java-components/resource-model/src/main/resources/crds/jvmbuildservice.io_jbsconfigs.yaml @@ -89,6 +89,8 @@ spec: type: boolean gitSourceArchive: properties: + disableSSLVerification: + type: string identity: type: string url: diff --git a/pkg/apis/jvmbuildservice/v1alpha1/jbsconfig_types.go b/pkg/apis/jvmbuildservice/v1alpha1/jbsconfig_types.go index 2e44361ce..8889d495d 100644 --- a/pkg/apis/jvmbuildservice/v1alpha1/jbsconfig_types.go +++ b/pkg/apis/jvmbuildservice/v1alpha1/jbsconfig_types.go @@ -111,8 +111,9 @@ type MavenDeployment struct { } type GitSourceArchive struct { - Identity string `json:"identity,omitempty"` - URL string `json:"url,omitempty"` + Identity string `json:"identity,omitempty"` + URL string `json:"url,omitempty"` + DisableSSLVerification string `json:"disableSSLVerification,omitempty"` } type RelocationPatternElement struct { diff --git a/pkg/reconciler/dependencybuild/buildrecipeyaml.go b/pkg/reconciler/dependencybuild/buildrecipeyaml.go index 47f3eee82..de74f0367 100644 --- a/pkg/reconciler/dependencybuild/buildrecipeyaml.go +++ b/pkg/reconciler/dependencybuild/buildrecipeyaml.go @@ -666,6 +666,9 @@ func imageRegistryCommands(imageId string, recipe *v1alpha12.BuildRecipe, db *v1 if jbsConfig.Spec.GitSourceArchive.URL != "" { mavenArgs = append(mavenArgs, "--git-url="+jbsConfig.Spec.GitSourceArchive.URL) } + if jbsConfig.Spec.GitSourceArchive.DisableSSLVerification != "" { + mavenArgs = append(mavenArgs, "--git-disable-ssl-verification="+jbsConfig.Spec.GitSourceArchive.DisableSSLVerification) + } deployArgs = append(deployArgs, mavenArgs...) hermeticPreBuildImageArgs := []string{ From fa3250c79039dd8fbb363165999511a09b988357 Mon Sep 17 00:00:00 2001 From: Nick Cross Date: Thu, 23 Nov 2023 14:44:15 +0000 Subject: [PATCH 03/15] Correct typing on SSL Verification --- deploy/crds/base/jvmbuildservice.io_jbsconfigs.yaml | 2 +- deploy/patch-yaml.sh | 2 +- .../main/resources/crds/jvmbuildservice.io_jbsconfigs.yaml | 2 +- pkg/apis/jvmbuildservice/v1alpha1/jbsconfig_types.go | 2 +- pkg/reconciler/dependencybuild/buildrecipeyaml.go | 4 ++-- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/deploy/crds/base/jvmbuildservice.io_jbsconfigs.yaml b/deploy/crds/base/jvmbuildservice.io_jbsconfigs.yaml index 428148434..edb422d83 100644 --- a/deploy/crds/base/jvmbuildservice.io_jbsconfigs.yaml +++ b/deploy/crds/base/jvmbuildservice.io_jbsconfigs.yaml @@ -90,7 +90,7 @@ spec: gitSourceArchive: properties: disableSSLVerification: - type: string + type: boolean identity: type: string url: diff --git a/deploy/patch-yaml.sh b/deploy/patch-yaml.sh index fc0c44887..f1871631c 100755 --- a/deploy/patch-yaml.sh +++ b/deploy/patch-yaml.sh @@ -56,6 +56,6 @@ fi if [ -z "${GIT_DISABLE_SSL_VERIFICATION}" ]; then GIT_DISABLE_SSL_VERIFICATION="false" fi -find $DIR -path \*development\*.yaml -exec $SED -i s/GIT_DEPLOY_URL/${GIT_DEPLOY_URL}/ {} \; +find $DIR -path \*development\*.yaml -exec $SED -i s%GIT_DEPLOY_URL%${GIT_DEPLOY_URL}% {} \; find $DIR -path \*development\*.yaml -exec $SED -i s%GIT_DEPLOY_IDENTITY%${GIT_DEPLOY_IDENTITY}% {} \; find $DIR -path \*development\*.yaml -exec $SED -i s%GIT_DISABLE_SSL_VERIFICATION%${GIT_DISABLE_SSL_VERIFICATION}% {} \; diff --git a/java-components/resource-model/src/main/resources/crds/jvmbuildservice.io_jbsconfigs.yaml b/java-components/resource-model/src/main/resources/crds/jvmbuildservice.io_jbsconfigs.yaml index 428148434..edb422d83 100644 --- a/java-components/resource-model/src/main/resources/crds/jvmbuildservice.io_jbsconfigs.yaml +++ b/java-components/resource-model/src/main/resources/crds/jvmbuildservice.io_jbsconfigs.yaml @@ -90,7 +90,7 @@ spec: gitSourceArchive: properties: disableSSLVerification: - type: string + type: boolean identity: type: string url: diff --git a/pkg/apis/jvmbuildservice/v1alpha1/jbsconfig_types.go b/pkg/apis/jvmbuildservice/v1alpha1/jbsconfig_types.go index 8889d495d..7fdb28631 100644 --- a/pkg/apis/jvmbuildservice/v1alpha1/jbsconfig_types.go +++ b/pkg/apis/jvmbuildservice/v1alpha1/jbsconfig_types.go @@ -113,7 +113,7 @@ type MavenDeployment struct { type GitSourceArchive struct { Identity string `json:"identity,omitempty"` URL string `json:"url,omitempty"` - DisableSSLVerification string `json:"disableSSLVerification,omitempty"` + DisableSSLVerification bool `json:"disableSSLVerification,omitempty"` } type RelocationPatternElement struct { diff --git a/pkg/reconciler/dependencybuild/buildrecipeyaml.go b/pkg/reconciler/dependencybuild/buildrecipeyaml.go index de74f0367..5ec8ebe4d 100644 --- a/pkg/reconciler/dependencybuild/buildrecipeyaml.go +++ b/pkg/reconciler/dependencybuild/buildrecipeyaml.go @@ -666,8 +666,8 @@ func imageRegistryCommands(imageId string, recipe *v1alpha12.BuildRecipe, db *v1 if jbsConfig.Spec.GitSourceArchive.URL != "" { mavenArgs = append(mavenArgs, "--git-url="+jbsConfig.Spec.GitSourceArchive.URL) } - if jbsConfig.Spec.GitSourceArchive.DisableSSLVerification != "" { - mavenArgs = append(mavenArgs, "--git-disable-ssl-verification="+jbsConfig.Spec.GitSourceArchive.DisableSSLVerification) + if jbsConfig.Spec.GitSourceArchive.DisableSSLVerification { + mavenArgs = append(mavenArgs, "--git-disable-ssl-verification") } deployArgs = append(deployArgs, mavenArgs...) From e1b9ff122db896fbda5933864410361f91bff255 Mon Sep 17 00:00:00 2001 From: Stuart Douglas Date: Sat, 25 Nov 2023 11:57:38 +1100 Subject: [PATCH 04/15] STONEBLD-1955 store all contaminate information --- .../jvmbuildservice.io_dependencybuilds.yaml | 8 +++ .../analyser/deploy/DeployCommand.java | 52 +++++++++++-------- .../src/main/resources/application.properties | 2 +- .../jvmbuildservice.io_dependencybuilds.yaml | 8 +++ .../v1alpha1/dependencybuild_types.go | 16 +++++- .../v1alpha1/zz_generated.deepcopy.go | 8 ++- pkg/reconciler/artifactbuild/artifactbuild.go | 16 +++--- .../artifactbuild/artifactbuild_test.go | 12 +++-- .../dependencybuild/dependencybuild.go | 22 +++++--- .../dependencybuild/dependencybuild_test.go | 2 +- 10 files changed, 102 insertions(+), 44 deletions(-) diff --git a/deploy/crds/base/jvmbuildservice.io_dependencybuilds.yaml b/deploy/crds/base/jvmbuildservice.io_dependencybuilds.yaml index 6fa3f883c..6ea6a18e4 100644 --- a/deploy/crds/base/jvmbuildservice.io_dependencybuilds.yaml +++ b/deploy/crds/base/jvmbuildservice.io_dependencybuilds.yaml @@ -262,12 +262,20 @@ spec: contaminates: items: properties: + allowed: + type: boolean + buildId: + type: string contaminatedArtifacts: items: type: string type: array gav: type: string + rebuildAvailable: + type: boolean + source: + type: string type: object type: array deployedArtifacts: diff --git a/java-components/build-request-processor/src/main/java/com/redhat/hacbs/container/analyser/deploy/DeployCommand.java b/java-components/build-request-processor/src/main/java/com/redhat/hacbs/container/analyser/deploy/DeployCommand.java index 7fd83b2a1..50cd26265 100644 --- a/java-components/build-request-processor/src/main/java/com/redhat/hacbs/container/analyser/deploy/DeployCommand.java +++ b/java-components/build-request-processor/src/main/java/com/redhat/hacbs/container/analyser/deploy/DeployCommand.java @@ -60,6 +60,7 @@ public class DeployCommand implements Runnable { private static final String DOT = "."; private static final Set ALLOWED_CONTAMINANTS = Set.of("-tests.jar"); public static final String IMAGE_DIGEST_OUTPUT = "Image Digest: "; + public static final String BUILD_ID = "build-id"; final BeanManager beanManager; final ResultsUpdater resultsUpdater; @@ -166,9 +167,7 @@ public void run() { Set gavs = new HashSet<>(); Map> contaminatedPaths = new HashMap<>(); - Map> contaminatedGavs = new HashMap<>(); - Map> allowedContaminatedPaths = new HashMap<>(); - Map> allowedContaminatedGavs = new HashMap<>(); + Map contaminatedGavs = new HashMap<>(); // Represents directories that should not be deployed i.e. if a single artifact (barring test jars) is // contaminated then none of the artifacts will be deployed. Set toRemove = new HashSet<>(); @@ -184,12 +183,11 @@ public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IO //we check every file as we also want to catch .tar.gz etc var info = ClassFileTracker.readTrackingDataFromFile(Files.newInputStream(file), name); for (var i : info) { - if (!allowedSources.contains(i.source)) { - Log.errorf("%s was contaminated by %s from %s", name, i.gav, i.source); - if (ALLOWED_CONTAMINANTS.stream().noneMatch(a -> file.getFileName().toString().endsWith(a))) { - gav.ifPresent(g -> contaminatedGavs.computeIfAbsent(i.gav, s -> new HashSet<>()) - .add(g.getGroupId() + ":" + g.getArtifactId() + ":" + g.getVersion())); - int index = name.lastIndexOf("/"); + Log.errorf("%s was contaminated by %s from %s", name, i.gav, i.source); + if (ALLOWED_CONTAMINANTS.stream().noneMatch(a -> file.getFileName().toString().endsWith(a))) { + int index = name.lastIndexOf("/"); + boolean allowed = allowedSources.contains(i.source); + if (!allowed) { if (index != -1) { contaminatedPaths.computeIfAbsent(name.substring(0, index), s -> new HashSet<>()).add(i.gav); @@ -197,12 +195,23 @@ public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IO contaminatedPaths.computeIfAbsent("", s -> new HashSet<>()).add(i.gav); } toRemove.add(file.getParent()); - } else { - Log.debugf("Ignoring contaminant for %s", file.getFileName()); } - } else { + gav.ifPresent(g -> contaminatedGavs.computeIfAbsent(i.gav, s -> { + Contaminates contaminates = new Contaminates(); + contaminates.setGav(i.gav); + contaminates.setAllowed(allowed); + contaminates.setSource(i.source); + contaminates.setBuildId(i.getAttributes().get(BUILD_ID)); + contaminates.setContaminatedArtifacts(new ArrayList<>()); + return contaminates; + }) + .getContaminatedArtifacts() + .add(g.getGroupId() + ":" + g.getArtifactId() + ":" + g.getVersion())); + } else { + Log.debugf("Ignoring contaminant for %s", file.getFileName()); } + } if (gav.isPresent()) { //now add our own tracking data @@ -225,7 +234,7 @@ public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IO + gav.getVersion(), "rebuilt", Map.of("scm-uri", scmUri, "scm-commit", commit, "hermetic", - Boolean.toString(hermetic), "build-id", buildId)), + Boolean.toString(hermetic), BUILD_ID, buildId)), Files.newOutputStream(temp), false); Files.delete(file); Files.move(temp, file); @@ -269,14 +278,12 @@ public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) { }); throw new RuntimeException("deploy failed"); } - //we still deploy, but without the contaminates - // This means the build failed to produce any deployable output. - // If everything is contaminated we still need the task to succeed so we can resolve the contamination. for (var i : contaminatedGavs.entrySet()) { - gavs.removeAll(i.getValue()); + if (!i.getValue().getAllowed()) { + gavs.removeAll(i.getValue().getContaminatedArtifacts()); + } } generateBuildSbom(); - if (isNotEmpty(mvnRepo) && mvnPassword.isEmpty()) { Log.infof("Maven repository specified as %s and no password specified", mvnRepo); URL url = new URL(mvnRepo); @@ -304,6 +311,10 @@ public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) { .getAuthorizationToken()); } } + + //we still deploy, but without the contaminates + // This means the build failed to produce any deployable output. + // If everything is contaminated we still need the task to succeed so we can resolve the contamination. if (!gavs.isEmpty()) { try { cleanBrokenSymlinks(sourcePath); @@ -323,10 +334,7 @@ public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) { List newContaminates = new ArrayList<>(); for (var i : contaminatedGavs.entrySet()) { - Contaminates contaminates = new Contaminates(); - contaminates.setContaminatedArtifacts(new ArrayList<>(i.getValue())); - contaminates.setGav(i.getKey()); - newContaminates.add(contaminates); + newContaminates.add(i.getValue()); } String serialisedContaminants = ResultsUpdater.MAPPER.writeValueAsString(newContaminates); Log.infof("Updating results %s with contaminants %s and deployed resources %s", diff --git a/java-components/management-console/src/main/resources/application.properties b/java-components/management-console/src/main/resources/application.properties index 6f2acf1be..c7c1300b3 100644 --- a/java-components/management-console/src/main/resources/application.properties +++ b/java-components/management-console/src/main/resources/application.properties @@ -33,4 +33,4 @@ quarkus.quinoa.build-dir=dist quarkus.quinoa.enable-spa-routing=true quarkus.resteasy-reactive.path=/api/ -%dev.sbom-discovery.enabled=false +%dev.sbom-discovery.enabled=true diff --git a/java-components/resource-model/src/main/resources/crds/jvmbuildservice.io_dependencybuilds.yaml b/java-components/resource-model/src/main/resources/crds/jvmbuildservice.io_dependencybuilds.yaml index 6fa3f883c..6ea6a18e4 100644 --- a/java-components/resource-model/src/main/resources/crds/jvmbuildservice.io_dependencybuilds.yaml +++ b/java-components/resource-model/src/main/resources/crds/jvmbuildservice.io_dependencybuilds.yaml @@ -262,12 +262,20 @@ spec: contaminates: items: properties: + allowed: + type: boolean + buildId: + type: string contaminatedArtifacts: items: type: string type: array gav: type: string + rebuildAvailable: + type: boolean + source: + type: string type: object type: array deployedArtifacts: diff --git a/pkg/apis/jvmbuildservice/v1alpha1/dependencybuild_types.go b/pkg/apis/jvmbuildservice/v1alpha1/dependencybuild_types.go index 46e3a338a..f00c6aabf 100644 --- a/pkg/apis/jvmbuildservice/v1alpha1/dependencybuild_types.go +++ b/pkg/apis/jvmbuildservice/v1alpha1/dependencybuild_types.go @@ -28,7 +28,7 @@ type DependencyBuildStatus struct { Conditions []metav1.Condition `json:"conditions,omitempty"` State string `json:"state,omitempty"` Message string `json:"message,omitempty"` - Contaminants []Contaminant `json:"contaminates,omitempty"` + Contaminants []*Contaminant `json:"contaminates,omitempty"` // PotentialBuildRecipes additional recipes to try if the current recipe fails PotentialBuildRecipes []*BuildRecipe `json:"potentialBuildRecipes,omitempty"` CommitTime int64 `json:"commitTime,omitempty"` @@ -115,6 +115,16 @@ func (r *DependencyBuildStatus) CurrentBuildAttempt() *BuildAttempt { return r.BuildAttempts[len(r.BuildAttempts)-1] } +func (r *DependencyBuildStatus) ProblemContaminates() []*Contaminant { + problemContaminates := []*Contaminant{} + for _, i := range r.Contaminants { + if !i.Allowed { + problemContaminates = append(problemContaminates, i) + } + } + return problemContaminates +} + type BuildRecipe struct { //Deprecated Pipeline string `json:"pipeline,omitempty"` @@ -136,6 +146,10 @@ type BuildRecipe struct { type Contaminant struct { GAV string `json:"gav,omitempty"` ContaminatedArtifacts []string `json:"contaminatedArtifacts,omitempty"` + BuildId string `json:"buildId,omitempty"` + Source string `json:"source,omitempty"` + Allowed bool `json:"allowed,omitempty"` + RebuildAvailable bool `json:"rebuildAvailable,omitempty"` } type AdditionalDownload struct { Uri string `json:"uri,omitempty"` diff --git a/pkg/apis/jvmbuildservice/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/jvmbuildservice/v1alpha1/zz_generated.deepcopy.go index ec22b012b..f2651e6ad 100644 --- a/pkg/apis/jvmbuildservice/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/apis/jvmbuildservice/v1alpha1/zz_generated.deepcopy.go @@ -410,9 +410,13 @@ func (in *DependencyBuildStatus) DeepCopyInto(out *DependencyBuildStatus) { } if in.Contaminants != nil { in, out := &in.Contaminants, &out.Contaminants - *out = make([]Contaminant, len(*in)) + *out = make([]*Contaminant, len(*in)) for i := range *in { - (*in)[i].DeepCopyInto(&(*out)[i]) + if (*in)[i] != nil { + in, out := &(*in)[i], &(*out)[i] + *out = new(Contaminant) + (*in).DeepCopyInto(*out) + } } } if in.PotentialBuildRecipes != nil { diff --git a/pkg/reconciler/artifactbuild/artifactbuild.go b/pkg/reconciler/artifactbuild/artifactbuild.go index 0644a4fd4..3d55a559a 100644 --- a/pkg/reconciler/artifactbuild/artifactbuild.go +++ b/pkg/reconciler/artifactbuild/artifactbuild.go @@ -383,15 +383,17 @@ func (r *ReconcileArtifactBuild) handleStateComplete(ctx context.Context, log lo if db.Status.State != v1alpha1.DependencyBuildStateContaminated { continue } - var newContaminates []v1alpha1.Contaminant - for _, contaminant := range db.Status.Contaminants { - if contaminant.GAV != abr.Spec.GAV { - newContaminates = append(newContaminates, contaminant) + allOk := true + for i := range db.Status.Contaminants { + contaminant := db.Status.Contaminants[i] + if contaminant.GAV == abr.Spec.GAV { + contaminant.RebuildAvailable = true + } else if !contaminant.Allowed && !contaminant.RebuildAvailable { + allOk = false } } - log.Info("Attempting to resolve contamination for dependencybuild", "dependencybuild", db.Name+"-"+db.Spec.ScmInfo.SCMURL+"-"+db.Spec.ScmInfo.Tag, "old", db.Status.Contaminants, "new", newContaminates) - db.Status.Contaminants = newContaminates - if len(db.Status.Contaminants) == 0 { + if allOk { + log.Info("Attempting to resolve contamination for dependencybuild as all contaminates are ready", "dependencybuild", db.Name+"-"+db.Spec.ScmInfo.SCMURL+"-"+db.Spec.ScmInfo.Tag) //TODO: we could have a situation where there are still some contamination, but not for artifacts that we care about //kick off the build again log.Info("Contamination resolved, moving to state new", "dependencybuild", db.Name+"-"+db.Spec.ScmInfo.SCMURL+"-"+db.Spec.ScmInfo.Tag) diff --git a/pkg/reconciler/artifactbuild/artifactbuild_test.go b/pkg/reconciler/artifactbuild/artifactbuild_test.go index 875ade879..275403e69 100644 --- a/pkg/reconciler/artifactbuild/artifactbuild_test.go +++ b/pkg/reconciler/artifactbuild/artifactbuild_test.go @@ -311,7 +311,7 @@ func TestStateBuilding(t *testing.T) { Labels: map[string]string{DependencyBuildIdLabel: util.HashString("")}, }, Spec: v1alpha1.DependencyBuildSpec{}, - Status: v1alpha1.DependencyBuildStatus{State: v1alpha1.DependencyBuildStateContaminated, Contaminants: []v1alpha1.Contaminant{{GAV: "com.test:test:1.0", ContaminatedArtifacts: []string{"a:b:1"}}}}, + Status: v1alpha1.DependencyBuildStatus{State: v1alpha1.DependencyBuildStateContaminated, Contaminants: []*v1alpha1.Contaminant{{GAV: "com.test:test:1.0", ContaminatedArtifacts: []string{"a:b:1"}}}}, } g.Expect(controllerutil.SetOwnerReference(abr, db, reconciler.scheme)) g.Expect(client.Create(ctx, db)) @@ -357,7 +357,7 @@ func TestStateCompleteFixingContamination(t *testing.T) { Labels: map[string]string{DependencyBuildIdLabel: util.HashString("")}, }, Spec: v1alpha1.DependencyBuildSpec{}, - Status: v1alpha1.DependencyBuildStatus{State: v1alpha1.DependencyBuildStateContaminated, Contaminants: []v1alpha1.Contaminant{{GAV: "com.test:test:1.0", ContaminatedArtifacts: []string{"a:b:1"}}}}, + Status: v1alpha1.DependencyBuildStatus{State: v1alpha1.DependencyBuildStateContaminated, Contaminants: []*v1alpha1.Contaminant{{GAV: "com.test:test:1.0", ContaminatedArtifacts: []string{"a:b:1"}}}}, } client, reconciler = setupClientAndReconciler(abr, contaiminated) } @@ -367,6 +367,12 @@ func TestStateCompleteFixingContamination(t *testing.T) { g.Expect(reconciler.Reconcile(ctx, reconcile.Request{NamespacedName: types.NamespacedName{Namespace: metav1.NamespaceDefault, Name: "test"}})) db := v1alpha1.DependencyBuild{} g.Expect(client.Get(ctx, types.NamespacedName{Namespace: metav1.NamespaceDefault, Name: contaminatedName}, &db)) - g.Expect(db.Status.Contaminants).Should(BeEmpty()) + allOk := true + for _, i := range db.Status.Contaminants { + if !i.Allowed && !i.RebuildAvailable { + allOk = false + } + } + g.Expect(allOk).Should(BeTrue()) }) } diff --git a/pkg/reconciler/dependencybuild/dependencybuild.go b/pkg/reconciler/dependencybuild/dependencybuild.go index 04a44302a..d0559c1b7 100644 --- a/pkg/reconciler/dependencybuild/dependencybuild.go +++ b/pkg/reconciler/dependencybuild/dependencybuild.go @@ -661,7 +661,7 @@ func (r *ReconcileDependencyBuild) handleBuildPipelineRunReceived(ctx context.Co digest = i.Value.StringVal } else if i.Name == artifactbuild.PipelineResultContaminants { - db.Status.Contaminants = []v1alpha1.Contaminant{} + db.Status.Contaminants = []*v1alpha1.Contaminant{} //unmarshal directly into the contaminants field err := json.Unmarshal([]byte(i.Value.StringVal), &db.Status.Contaminants) if err != nil { @@ -689,7 +689,7 @@ func (r *ReconcileDependencyBuild) handleBuildPipelineRunReceived(ctx context.Co for _, i := range pr.Status.Results { if i.Name == artifactbuild.PipelineResultContaminants { - db.Status.Contaminants = []v1alpha1.Contaminant{} + db.Status.Contaminants = []*v1alpha1.Contaminant{} //unmarshal directly into the contaminants field err := json.Unmarshal([]byte(i.Value.StringVal), &db.Status.Contaminants) if err != nil { @@ -712,7 +712,8 @@ func (r *ReconcileDependencyBuild) handleBuildPipelineRunReceived(ctx context.Co } } - if len(db.Status.Contaminants) == 0 { + problemContaminates := db.Status.ProblemContaminates() + if len(problemContaminates) == 0 { db.Status.State = v1alpha1.DependencyBuildStateComplete } else { r.eventRecorder.Eventf(db, v1.EventTypeWarning, "BuildContaminated", "The DependencyBuild %s/%s was contaminated with community dependencies", db.Namespace, db.Name) @@ -720,7 +721,7 @@ func (r *ReconcileDependencyBuild) handleBuildPipelineRunReceived(ctx context.Co //most likely shaded in //we don't need to update the status here, it will be handled by the handleStateComplete method //even though there are contaminates they may not be in artifacts we care about - err := r.handleBuildCompletedWithContaminants(ctx, db, log) + err := r.handleBuildCompletedWithContaminants(ctx, db, log, problemContaminates) if err != nil { return reconcile.Result{}, err } @@ -815,7 +816,7 @@ func (r *ReconcileDependencyBuild) dependencyBuildForPipelineRun(ctx context.Con // no actual request for these artifacts. This can change if new artifacts are requested, so even when complete // we still need to verify that hte build is ok // this method will always update the status if it does not return an error -func (r *ReconcileDependencyBuild) handleBuildCompletedWithContaminants(ctx context.Context, db *v1alpha1.DependencyBuild, l logr.Logger) error { +func (r *ReconcileDependencyBuild) handleBuildCompletedWithContaminants(ctx context.Context, db *v1alpha1.DependencyBuild, l logr.Logger, problemContaminates []*v1alpha1.Contaminant) error { ownerGavs := map[string]bool{} db.Status.State = v1alpha1.DependencyBuildStateComplete @@ -838,7 +839,7 @@ func (r *ReconcileDependencyBuild) handleBuildCompletedWithContaminants(ctx cont ownerGavs[ab.Spec.GAV] = true } } - for _, contaminant := range db.Status.Contaminants { + for _, contaminant := range problemContaminates { for _, artifact := range contaminant.ContaminatedArtifacts { if ownerGavs[artifact] { db.Status.State = v1alpha1.DependencyBuildStateContaminated @@ -883,7 +884,14 @@ func (r *ReconcileDependencyBuild) handleBuildCompletedWithContaminants(ctx cont } func (r *ReconcileDependencyBuild) handleStateContaminated(ctx context.Context, db *v1alpha1.DependencyBuild) (reconcile.Result, error) { contaminants := db.Status.Contaminants - if len(contaminants) == 0 { + allOk := true + for _, i := range contaminants { + if !i.Allowed && !i.RebuildAvailable { + allOk = false + break + } + } + if allOk { //all fixed, just set the state back to building and try again //this is triggered when contaminants are removed by the ABR controller //setting it back to building should re-try the recipe that actually worked diff --git a/pkg/reconciler/dependencybuild/dependencybuild_test.go b/pkg/reconciler/dependencybuild/dependencybuild_test.go index 055ab6035..d27d3f8af 100644 --- a/pkg/reconciler/dependencybuild/dependencybuild_test.go +++ b/pkg/reconciler/dependencybuild/dependencybuild_test.go @@ -491,7 +491,7 @@ func TestStateBuilding(t *testing.T) { g.Expect(client.Update(ctx, pr)).Should(BeNil()) db := getBuild(client, g) g.Expect(controllerutil.SetOwnerReference(&ab, db, reconciler.scheme)).Should(BeNil()) - db.Status.Contaminants = []v1alpha1.Contaminant{{GAV: "com.acme:foo:1.0", ContaminatedArtifacts: []string{TestArtifact}}} + db.Status.Contaminants = []*v1alpha1.Contaminant{{GAV: "com.acme:foo:1.0", ContaminatedArtifacts: []string{TestArtifact}}} g.Expect(client.Update(ctx, db)) g.Expect(reconciler.Reconcile(ctx, reconcile.Request{NamespacedName: taskRunName})) g.Expect(reconciler.Reconcile(ctx, reconcile.Request{NamespacedName: buildName})) From 123378ef225602cb5ad18e43231d38b0a6b6f022 Mon Sep 17 00:00:00 2001 From: Stuart Douglas Date: Mon, 27 Nov 2023 09:30:39 +1100 Subject: [PATCH 05/15] Clean up some code Move to a single loop when reading results --- .../dependencybuild/dependencybuild.go | 64 ++++++------------- 1 file changed, 19 insertions(+), 45 deletions(-) diff --git a/pkg/reconciler/dependencybuild/dependencybuild.go b/pkg/reconciler/dependencybuild/dependencybuild.go index d0559c1b7..1c0c8b4a9 100644 --- a/pkg/reconciler/dependencybuild/dependencybuild.go +++ b/pkg/reconciler/dependencybuild/dependencybuild.go @@ -355,11 +355,9 @@ func (r *ReconcileDependencyBuild) handleAnalyzeBuildPipelineRunReceived(ctx con if len(unmarshalled.Image) > 0 { log.Info(fmt.Sprintf("Found preexisting shared build with deployed GAVs %#v from image %#v", unmarshalled.Gavs, unmarshalled.Image)) db.Status.State = v1alpha1.DependencyBuildStateComplete - con, err := r.createRebuiltArtifacts(ctx, log, pr, &db, unmarshalled.Image, unmarshalled.Digest, unmarshalled.Gavs) + err := r.createRebuiltArtifacts(ctx, log, pr, &db, unmarshalled.Image, unmarshalled.Digest, unmarshalled.Gavs) if err != nil { return reconcile.Result{}, err - } else if !con { - return reconcile.Result{}, nil } } else { db.Status.State = v1alpha1.DependencyBuildStateSubmitBuild @@ -654,13 +652,13 @@ func (r *ReconcileDependencyBuild) handleBuildPipelineRunReceived(ctx context.Co var verificationResults string var gavs []string var hermeticBuildImage string + var deployed []string for _, i := range pr.Status.Results { if i.Name == PipelineResultImage { image = i.Value.StringVal } else if i.Name == PipelineResultImageDigest { digest = i.Value.StringVal } else if i.Name == artifactbuild.PipelineResultContaminants { - db.Status.Contaminants = []*v1alpha1.Contaminant{} //unmarshal directly into the contaminants field err := json.Unmarshal([]byte(i.Value.StringVal), &db.Status.Contaminants) @@ -670,13 +668,21 @@ func (r *ReconcileDependencyBuild) handleBuildPipelineRunReceived(ctx context.Co } else if i.Name == artifactbuild.PipelineResultPassedVerification { parseBool, _ := strconv.ParseBool(i.Value.StringVal) passedVerification = !parseBool + } else if i.Name == artifactbuild.PipelineResultVerificationResult { + verificationResults = i.Value.StringVal } else if i.Name == artifactbuild.PipelineResultGavs { deployed := strings.Split(i.Value.StringVal, ",") db.Status.DeployedArtifacts = deployed - } else if i.Name == artifactbuild.PipelineResultVerificationResult { - verificationResults = i.Value.StringVal + } else if i.Name == artifactbuild.PipelineResultDeployedResources && len(i.Value.StringVal) > 0 { + //we need to create 'DeployedArtifact' resources for the objects that were deployed + deployed = strings.Split(i.Value.StringVal, ",") } } + err = r.createRebuiltArtifacts(ctx, log, pr, db, image, digest, deployed) + if err != nil { + return reconcile.Result{}, err + } + run.Results = &v1alpha1.BuildPipelineRunResults{ Image: image, ImageDigest: digest, @@ -685,33 +691,6 @@ func (r *ReconcileDependencyBuild) handleBuildPipelineRunReceived(ctx context.Co Gavs: gavs, HermeticBuildImage: hermeticBuildImage, } - - for _, i := range pr.Status.Results { - if i.Name == artifactbuild.PipelineResultContaminants { - - db.Status.Contaminants = []*v1alpha1.Contaminant{} - //unmarshal directly into the contaminants field - err := json.Unmarshal([]byte(i.Value.StringVal), &db.Status.Contaminants) - if err != nil { - return reconcile.Result{}, err - } - } else if i.Name == artifactbuild.PipelineResultDeployedResources && len(i.Value.StringVal) > 0 { - //we need to create 'DeployedArtifact' resources for the objects that were deployed - deployed := strings.Split(i.Value.StringVal, ",") - - con, err := r.createRebuiltArtifacts(ctx, log, pr, db, image, digest, deployed) - - if err != nil { - return reconcile.Result{}, err - } else if !con { - return reconcile.Result{}, nil - } - } else if i.Name == artifactbuild.PipelineResultPassedVerification { - parseBool, _ := strconv.ParseBool(i.Value.StringVal) - db.Status.FailedVerification = !parseBool - } - } - problemContaminates := db.Status.ProblemContaminates() if len(problemContaminates) == 0 { db.Status.State = v1alpha1.DependencyBuildStateComplete @@ -902,7 +881,7 @@ func (r *ReconcileDependencyBuild) handleStateContaminated(ctx context.Context, } func (r *ReconcileDependencyBuild) createRebuiltArtifacts(ctx context.Context, log logr.Logger, pr *pipelinev1beta1.PipelineRun, db *v1alpha1.DependencyBuild, - image string, digest string, deployed []string) (bool, error) { + image string, digest string, deployed []string) error { db.Status.DeployedArtifacts = deployed for _, i := range deployed { @@ -911,7 +890,7 @@ func (r *ReconcileDependencyBuild) createRebuiltArtifacts(ctx context.Context, l ra.Namespace = pr.Namespace ra.Name = artifactbuild.CreateABRName(i) if err := controllerutil.SetOwnerReference(db, &ra, r.scheme); err != nil { - return false, err + return err } ra.Spec.GAV = i ra.Spec.Image = image @@ -919,29 +898,24 @@ func (r *ReconcileDependencyBuild) createRebuiltArtifacts(ctx context.Context, l err := r.client.Create(ctx, &ra) if err != nil { if !errors.IsAlreadyExists(err) { - return false, err + return err } else { //if it already exists we update the image field - err := r.client.Get(ctx, types.NamespacedName{Namespace: ra.Namespace, Name: ra.Name}, &ra) + err = r.client.Get(ctx, types.NamespacedName{Namespace: ra.Namespace, Name: ra.Name}, &ra) if err != nil { - if !errors.IsNotFound(err) { - return false, err - } - //on not found we don't return the error - //no need to retry it would just result in an infinite loop - return false, nil + return err } ra.Spec.Image = image ra.Spec.Digest = digest log.Info(fmt.Sprintf("Updating existing RebuiltArtifact %s to reference image %s", ra.Name, ra.Spec.Image), "action", "UPDATE") err = r.client.Update(ctx, &ra) if err != nil { - return false, err + return err } } } } - return true, nil + return nil } func (r *ReconcileDependencyBuild) createLookupBuildInfoPipeline(ctx context.Context, log logr.Logger, db *v1alpha1.DependencyBuild, jbsConfig *v1alpha1.JBSConfig, additionalMemory int, systemConfig *v1alpha1.SystemConfig) (*pipelinev1beta1.PipelineSpec, error) { From a24032383827ffec0588074260e8a82cb01a5a52 Mon Sep 17 00:00:00 2001 From: Nick Cross Date: Fri, 24 Nov 2023 09:12:58 +0000 Subject: [PATCH 06/15] Fix coverage --- .codecov.yml | 3 +- .github/workflows/codecov-main.yaml | 19 ------------- .github/workflows/go-ci.yaml | 14 +++++----- .github/workflows/{build.yml => java-ci.yml} | 16 ++++++++--- .github/workflows/kube-linter.yaml | 3 +- .github/workflows/minikube.yaml | 2 +- java-components/pom.xml | 29 ++++++++++++++++++++ 7 files changed, 51 insertions(+), 35 deletions(-) delete mode 100644 .github/workflows/codecov-main.yaml rename .github/workflows/{build.yml => java-ci.yml} (50%) diff --git a/.codecov.yml b/.codecov.yml index 6eae8911b..6d314a9e9 100644 --- a/.codecov.yml +++ b/.codecov.yml @@ -1,4 +1,3 @@ -comment: off coverage: status: project: @@ -6,7 +5,7 @@ coverage: target: auto threshold: 1% removed_code_behavior: adjust_base - patch: + patch: default: informational: true target: auto diff --git a/.github/workflows/codecov-main.yaml b/.github/workflows/codecov-main.yaml deleted file mode 100644 index 134c7a9a0..000000000 --- a/.github/workflows/codecov-main.yaml +++ /dev/null @@ -1,19 +0,0 @@ -name: Codecov report on push to the "main" branch -on: - push: - branches: [ main ] -jobs: - coverage: - name: Unit tests and coverage report - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-go@v4 - with: - go-version-file: './go.mod' - - name: Build - run: make build - - name: Run tests - run: make test - - name: Codecov - uses: codecov/codecov-action@v3 diff --git a/.github/workflows/go-ci.yaml b/.github/workflows/go-ci.yaml index 791f547b0..9c0091f49 100644 --- a/.github/workflows/go-ci.yaml +++ b/.github/workflows/go-ci.yaml @@ -1,4 +1,4 @@ -name: Validate PR - golang CI +name: GoLang CI on: pull_request: branches: [ main ] @@ -19,12 +19,12 @@ jobs: name: Check sources runs-on: ubuntu-latest steps: + - name: Check out code + uses: actions/checkout@v4 - name: Install Go uses: actions/setup-go@v4 with: - go-version: 1.19.x - - name: Check out code - uses: actions/checkout@v4 + go-version-file: './go.mod' - name: Check go mod status run: | go mod tidy @@ -56,12 +56,12 @@ jobs: name: Golang Unit tests runs-on: ubuntu-latest steps: + - name: Check out code + uses: actions/checkout@v4 - name: Install Go uses: actions/setup-go@v4 with: - go-version: 1.19.x - - name: Check out code - uses: actions/checkout@v4 + go-version-file: './go.mod' - name: Build run: make build - name: Test diff --git a/.github/workflows/build.yml b/.github/workflows/java-ci.yml similarity index 50% rename from .github/workflows/build.yml rename to .github/workflows/java-ci.yml index a8e03b767..9b262840e 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/java-ci.yml @@ -1,14 +1,20 @@ name: Java CI -on: [push, pull_request] - +on: + pull_request: + branches: [ main ] jobs: build: - runs-on: ubuntu-latest - steps: - uses: actions/checkout@v4 + - name: Cache local Maven repository + uses: actions/cache@v3 + with: + path: ~/.m2/repository + key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} + restore-keys: | + ${{ runner.os }}-maven- - name: Set up JDK 17 uses: actions/setup-java@v3 with: @@ -17,3 +23,5 @@ jobs: - name: Build with Maven working-directory: java-components run: mvn -V -B package -Dvalidate-format + - name: Codecov + uses: codecov/codecov-action@v3 diff --git a/.github/workflows/kube-linter.yaml b/.github/workflows/kube-linter.yaml index 4b69b06d1..d243dd34b 100644 --- a/.github/workflows/kube-linter.yaml +++ b/.github/workflows/kube-linter.yaml @@ -1,4 +1,4 @@ -name: Check Kubernetes YAMLs with kube-linter +name: Validate Kubernetes YAMLs on: pull_request: @@ -7,7 +7,6 @@ on: - 'deploy/crds/base/**.ya?ml' - 'deploy/operator/base/**.ya?ml' - 'deploy/operator/config/**.ya?ml' - jobs: kube-linter: name: Kube linter diff --git a/.github/workflows/minikube.yaml b/.github/workflows/minikube.yaml index 80a326d77..847c1bfbd 100644 --- a/.github/workflows/minikube.yaml +++ b/.github/workflows/minikube.yaml @@ -1,4 +1,4 @@ -name: Validate PR - Minikube Tests +name: Minikube Tests on: pull_request: branches: [ main ] diff --git a/java-components/pom.xml b/java-components/pom.xml index 781f60c3e..96a5cbc1d 100644 --- a/java-components/pom.xml +++ b/java-components/pom.xml @@ -267,6 +267,14 @@ + + + io.quarkus + quarkus-jacoco + test + + + @@ -310,6 +318,27 @@ + + + + org.jacoco + jacoco-maven-plugin + 0.8.10 + + + default-prepare-agent + + prepare-agent + + + *QuarkusClassLoader + ${project.build.directory}/jacoco-quarkus.exec + true + + + + + From 752da772bbc7d4a6df1679a6408506de23189ae2 Mon Sep 17 00:00:00 2001 From: Stuart Douglas Date: Fri, 10 Nov 2023 09:21:44 +1100 Subject: [PATCH 07/15] Remove quaytoken This is now handled by the image controller --- deploy/base-development.sh | 5 ----- deploy/operator/base/deployment.yaml | 9 --------- deploy/operator/overlays/ci-template/kustomization.yaml | 8 -------- deploy/operator/overlays/dev-template/kustomization.yaml | 7 ------- 4 files changed, 29 deletions(-) diff --git a/deploy/base-development.sh b/deploy/base-development.sh index 039bd112d..6c47cef74 100755 --- a/deploy/base-development.sh +++ b/deploy/base-development.sh @@ -18,11 +18,6 @@ DIR=`dirname $0` kubectl apply -f $DIR/namespace.yaml kubectl config set-context --current --namespace=test-jvm-namespace -if [ -n "$QUAY_ORG" ] && [ -n "$QUAY_TOKEN" ]; then - kubectl delete --ignore-not-found secret -n image-controller quaytoken - kubectl create secret generic -n image-controller quaytoken --from-literal "quaytoken=$QUAY_TOKEN" --from-literal "organization=$QUAY_ORG" -fi - echo -e "\033[0;32mSecrets...\033[0m" kubectl create --dry-run=client -o=yaml secret generic jvm-build-image-secrets --from-file=.dockerconfigjson=$HOME/.docker/config.json --type=kubernetes.io/dockerconfigjson | kubectl apply -f - kubectl create --dry-run=client -o=yaml secret generic jvm-build-git-secrets --from-literal .git-credentials=" diff --git a/deploy/operator/base/deployment.yaml b/deploy/operator/base/deployment.yaml index 2f59813bd..e86c18e75 100644 --- a/deploy/operator/base/deployment.yaml +++ b/deploy/operator/base/deployment.yaml @@ -13,11 +13,6 @@ spec: labels: app: hacbs-jvm-operator spec: - volumes: - - name: quaytoken - secret: - optional: false - secretName: quaytoken securityContext: runAsNonRoot: true containers: @@ -37,10 +32,6 @@ spec: limits: memory: "1024Mi" cpu: "500m" - volumeMounts: - - mountPath: "/workspace" - name: quaytoken - readOnly: true securityContext: readOnlyRootFilesystem: true serviceAccountName: hacbs-jvm-operator diff --git a/deploy/operator/overlays/ci-template/kustomization.yaml b/deploy/operator/overlays/ci-template/kustomization.yaml index 8a911b7cb..c4965c4e3 100644 --- a/deploy/operator/overlays/ci-template/kustomization.yaml +++ b/deploy/operator/overlays/ci-template/kustomization.yaml @@ -6,11 +6,3 @@ resources: # and I'm bypassing that gross env var munging we had to do in infra-deps; also, the deployment yaml should be pretty static at this point - base-deployment.yaml -patches: - - patch: |- - - op: replace - path: /spec/template/spec/volumes/0/secret/optional - value: true - target: - kind: Deployment - name: hacbs-jvm-operator diff --git a/deploy/operator/overlays/dev-template/kustomization.yaml b/deploy/operator/overlays/dev-template/kustomization.yaml index 7999e7947..7516a2735 100644 --- a/deploy/operator/overlays/dev-template/kustomization.yaml +++ b/deploy/operator/overlays/dev-template/kustomization.yaml @@ -21,11 +21,4 @@ patches: target: kind: Deployment name: hacbs-jvm-operator -- patch: |- - - op: replace - path: /spec/template/spec/volumes/0/secret/optional - value: true - target: - kind: Deployment - name: hacbs-jvm-operator - path: namespace.yaml From 1e4185be8f482e13da397b899dd90051c52e8db8 Mon Sep 17 00:00:00 2001 From: Stuart Douglas Date: Tue, 28 Nov 2023 19:02:38 +1100 Subject: [PATCH 08/15] Minor fix --- pkg/reconciler/dependencybuild/dependencybuild.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pkg/reconciler/dependencybuild/dependencybuild.go b/pkg/reconciler/dependencybuild/dependencybuild.go index 1c0c8b4a9..2ced02e03 100644 --- a/pkg/reconciler/dependencybuild/dependencybuild.go +++ b/pkg/reconciler/dependencybuild/dependencybuild.go @@ -650,7 +650,6 @@ func (r *ReconcileDependencyBuild) handleBuildPipelineRunReceived(ctx context.Co var digest string var passedVerification bool var verificationResults string - var gavs []string var hermeticBuildImage string var deployed []string for _, i := range pr.Status.Results { @@ -676,6 +675,8 @@ func (r *ReconcileDependencyBuild) handleBuildPipelineRunReceived(ctx context.Co } else if i.Name == artifactbuild.PipelineResultDeployedResources && len(i.Value.StringVal) > 0 { //we need to create 'DeployedArtifact' resources for the objects that were deployed deployed = strings.Split(i.Value.StringVal, ",") + } else if i.Name == artifactbuild.PipelineResultHermeticBuildImage { + hermeticBuildImage = i.Value.StringVal } } err = r.createRebuiltArtifacts(ctx, log, pr, db, image, digest, deployed) @@ -688,7 +689,7 @@ func (r *ReconcileDependencyBuild) handleBuildPipelineRunReceived(ctx context.Co ImageDigest: digest, Verified: passedVerification, VerificationResults: verificationResults, - Gavs: gavs, + Gavs: deployed, HermeticBuildImage: hermeticBuildImage, } problemContaminates := db.Status.ProblemContaminates() From fafb6a0b1df80c8bb2924394c1f6453fc97d417a Mon Sep 17 00:00:00 2001 From: Nick Cross Date: Tue, 28 Nov 2023 08:49:34 +0000 Subject: [PATCH 09/15] Run coverage for push to main --- .github/workflows/go-ci.yaml | 5 +++++ .github/workflows/java-ci.yml | 2 ++ 2 files changed, 7 insertions(+) diff --git a/.github/workflows/go-ci.yaml b/.github/workflows/go-ci.yaml index 9c0091f49..0ce7f6b38 100644 --- a/.github/workflows/go-ci.yaml +++ b/.github/workflows/go-ci.yaml @@ -1,11 +1,14 @@ name: GoLang CI on: + push: + branches: [ main ] pull_request: branches: [ main ] jobs: lint: name: Lint runs-on: ubuntu-latest + if: github.event_name == 'pull_request' steps: - uses: actions/checkout@v4 - uses: actions/setup-go@v4 @@ -18,6 +21,7 @@ jobs: go: name: Check sources runs-on: ubuntu-latest + if: github.event_name == 'pull_request' steps: - name: Check out code uses: actions/checkout@v4 @@ -71,6 +75,7 @@ jobs: security_scan: name: Security scan runs-on: ubuntu-latest + if: github.event_name == 'pull_request' steps: - uses: actions/checkout@v4 - uses: actions/setup-go@v4 diff --git a/.github/workflows/java-ci.yml b/.github/workflows/java-ci.yml index 9b262840e..23c56ddd5 100644 --- a/.github/workflows/java-ci.yml +++ b/.github/workflows/java-ci.yml @@ -1,6 +1,8 @@ name: Java CI on: + push: + branches: [ main ] pull_request: branches: [ main ] jobs: From eeb8b2de0b7956afc6a0048f9625b06c78b188d8 Mon Sep 17 00:00:00 2001 From: Nick Cross Date: Thu, 23 Nov 2023 15:53:43 +0000 Subject: [PATCH 10/15] Add GIT_ARCHIVE results to TaskResult --- .../analyser/deploy/DeployCommand.java | 14 +++++--- .../container/analyser/deploy/git/Git.java | 15 +++++++-- .../container/analyser/deploy/git/GitHub.java | 7 ++-- .../container/analyser/deploy/git/GitLab.java | 6 ++-- .../analyser/deploy/git/GitTest.java | 33 +++++++++++++++---- 5 files changed, 57 insertions(+), 18 deletions(-) diff --git a/java-components/build-request-processor/src/main/java/com/redhat/hacbs/container/analyser/deploy/DeployCommand.java b/java-components/build-request-processor/src/main/java/com/redhat/hacbs/container/analyser/deploy/DeployCommand.java index 50cd26265..7c05c4610 100644 --- a/java-components/build-request-processor/src/main/java/com/redhat/hacbs/container/analyser/deploy/DeployCommand.java +++ b/java-components/build-request-processor/src/main/java/com/redhat/hacbs/container/analyser/deploy/DeployCommand.java @@ -157,17 +157,19 @@ public DeployCommand(BeanManager beanManager, public void run() { try { + Set gavs = new HashSet<>(); + Map> contaminatedPaths = new HashMap<>(); + Map> contaminatedGavs = new HashMap<>(); + Map archivedSourceTags = new HashMap<>(); + // Save the source first regardless of deployment checks if (isNotEmpty(gitIdentity) && gitToken.isPresent()) { Log.infof("Git credentials are identity '%s' and URL '%s'", gitIdentity, gitURL); var git = Git.builder(gitURL, gitIdentity, gitToken.get(), gitDisableSSLVerification); git.create(scmUri); - git.add(sourcePath, commit, imageId); + archivedSourceTags = git.add(sourcePath, commit, imageId); } - Set gavs = new HashSet<>(); - Map> contaminatedPaths = new HashMap<>(); - Map contaminatedGavs = new HashMap<>(); // Represents directories that should not be deployed i.e. if a single artifact (barring test jars) is // contaminated then none of the artifacts will be deployed. Set toRemove = new HashSet<>(); @@ -337,13 +339,15 @@ public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) { newContaminates.add(i.getValue()); } String serialisedContaminants = ResultsUpdater.MAPPER.writeValueAsString(newContaminates); + String serialisedGitArchive = ResultsUpdater.MAPPER.writeValueAsString(archivedSourceTags); Log.infof("Updating results %s with contaminants %s and deployed resources %s", taskRun, serialisedContaminants, gavs); resultsUpdater.updateResults(taskRun, Map.of( "CONTAMINANTS", serialisedContaminants, "DEPLOYED_RESOURCES", String.join(",", gavs), "IMAGE_URL", imageName == null ? "" : imageName, - "IMAGE_DIGEST", imageDigest == null ? "" : "sha256:" + imageDigest)); + "IMAGE_DIGEST", imageDigest == null ? "" : "sha256:" + imageDigest, + "GIT_ARCHIVE", serialisedGitArchive)); } } catch (Exception e) { Log.error("Deployment failed", e); diff --git a/java-components/build-request-processor/src/main/java/com/redhat/hacbs/container/analyser/deploy/git/Git.java b/java-components/build-request-processor/src/main/java/com/redhat/hacbs/container/analyser/deploy/git/Git.java index 71bc59c4e..e51e6cdba 100644 --- a/java-components/build-request-processor/src/main/java/com/redhat/hacbs/container/analyser/deploy/git/Git.java +++ b/java-components/build-request-processor/src/main/java/com/redhat/hacbs/container/analyser/deploy/git/Git.java @@ -4,10 +4,13 @@ import java.net.URI; import java.net.URISyntaxException; import java.nio.file.Path; +import java.util.Collections; +import java.util.Map; import java.util.UUID; import org.eclipse.jgit.api.errors.GitAPIException; import org.eclipse.jgit.lib.Ref; +import org.eclipse.jgit.lib.Repository; import org.eclipse.jgit.lib.StoredConfig; import org.eclipse.jgit.transport.CredentialsProvider; import org.eclipse.jgit.transport.PushResult; @@ -24,7 +27,7 @@ public abstract class Git { public abstract void create(String name) throws IOException, URISyntaxException; - public abstract void add(Path path, String commit, String imageId) + public abstract Map add(Path path, String commit, String imageId) throws IOException; /** @@ -62,7 +65,7 @@ public static Git builder(String endpoint, String identity, String token, boolea } } - protected void pushRepository(Path path, String httpTransportUrl, String commit, String imageId) { + protected Map pushRepository(Path path, String httpTransportUrl, String commit, String imageId) { try (var jGit = org.eclipse.jgit.api.Git.init().setDirectory(path.toFile()).call()) { // Find the tag name associated with the commit. Then append the unique imageId. This is from the Go code // and is a hash of abr.Status.SCMInfo.SCMURL + abr.Status.SCMInfo.Tag + abr.Status.SCMInfo.Path @@ -87,6 +90,7 @@ protected void pushRepository(Path path, String httpTransportUrl, String commit, Ref tagRefStable = jGit.tag().setAnnotated(true).setName(tagName + "-" + imageId).setForceUpdate(true).call(); Ref tagRefUnique = jGit.tag().setAnnotated(true).setName(tagName + "-" + UUID.randomUUID()).setForceUpdate(true) .call(); + Iterable results = jGit.push().setForce(true).setRemote("origin") .add(jRepo.getBranch()) // Push the default branch else GitHub doesn't show the code. .add(tagRefStable) @@ -95,7 +99,8 @@ protected void pushRepository(Path path, String httpTransportUrl, String commit, for (PushResult result : results) { result.getRemoteUpdates().forEach(r -> { - if (!r.getStatus().equals(RemoteRefUpdate.Status.OK)) { + if (!r.getStatus().equals(RemoteRefUpdate.Status.OK) + && !r.getStatus().equals(RemoteRefUpdate.Status.UP_TO_DATE)) { Log.errorf("Push failure " + r); throw new RuntimeException("Failed to push updates due to " + r.getMessage()); } @@ -103,6 +108,10 @@ protected void pushRepository(Path path, String httpTransportUrl, String commit, Log.debugf("Pushed " + result.getMessages() + " " + result.getURI() + " updates: " + result.getRemoteUpdates()); } + + return Collections.singletonMap(Repository.shortenRefName(tagRefUnique.getName()), + jRepo.getRefDatabase().peel(tagRefUnique).getPeeledObjectId().getName()); + } catch (GitAPIException | IOException e) { throw new RuntimeException(e); } diff --git a/java-components/build-request-processor/src/main/java/com/redhat/hacbs/container/analyser/deploy/git/GitHub.java b/java-components/build-request-processor/src/main/java/com/redhat/hacbs/container/analyser/deploy/git/GitHub.java index 240bf72fc..23d0cb197 100644 --- a/java-components/build-request-processor/src/main/java/com/redhat/hacbs/container/analyser/deploy/git/GitHub.java +++ b/java-components/build-request-processor/src/main/java/com/redhat/hacbs/container/analyser/deploy/git/GitHub.java @@ -5,6 +5,7 @@ import java.io.IOException; import java.net.URISyntaxException; import java.nio.file.Path; +import java.util.Map; import org.eclipse.jgit.transport.UsernamePasswordCredentialsProvider; import org.kohsuke.github.GHRepository; @@ -62,6 +63,7 @@ public void create(String scmUri) if (type == Type.USER) { repository = github.getUser(owner).getRepository(name); if (repository == null) { + Log.infof("Creating repository with name %s", name); repository = github.createRepository(name) .wiki(false) .defaultBranch("main") @@ -73,6 +75,7 @@ public void create(String scmUri) } else { repository = github.getOrganization(owner).getRepository(name); if (repository == null) { + Log.infof("Creating repository with name %s", name); repository = github.getOrganization(owner).createRepository(name) .wiki(false) .defaultBranch("main") @@ -85,11 +88,11 @@ public void create(String scmUri) } @Override - public void add(Path path, String commit, String imageId) { + public Map add(Path path, String commit, String imageId) { if (repository == null) { throw new RuntimeException("Call create first"); } - pushRepository(path, repository.getHttpTransportUrl(), commit, imageId); + return pushRepository(path, repository.getHttpTransportUrl(), commit, imageId); } @Override diff --git a/java-components/build-request-processor/src/main/java/com/redhat/hacbs/container/analyser/deploy/git/GitLab.java b/java-components/build-request-processor/src/main/java/com/redhat/hacbs/container/analyser/deploy/git/GitLab.java index ba726eeb5..04bb6d3e6 100644 --- a/java-components/build-request-processor/src/main/java/com/redhat/hacbs/container/analyser/deploy/git/GitLab.java +++ b/java-components/build-request-processor/src/main/java/com/redhat/hacbs/container/analyser/deploy/git/GitLab.java @@ -3,6 +3,7 @@ import java.io.IOException; import java.net.URISyntaxException; import java.nio.file.Path; +import java.util.Map; import org.eclipse.jgit.transport.UsernamePasswordCredentialsProvider; import org.gitlab4j.api.GitLabApi; @@ -49,6 +50,7 @@ public void create(String scmUri) Log.warnf("Repository %s already exists", name); } else { // Can't set public visibility after creation for some reason with this API. + Log.infof("Creating repository with name %s", name); project = gitLabApi.getProjectApi().createProject(name, null, null, @@ -66,12 +68,12 @@ public void create(String scmUri) } @Override - public void add(Path path, String commit, String imageId) + public Map add(Path path, String commit, String imageId) throws IOException { if (project == null) { throw new RuntimeException("Call create first"); } - pushRepository(path, project.getHttpUrlToRepo(), commit, imageId); + return pushRepository(path, project.getHttpUrlToRepo(), commit, imageId); } @Override diff --git a/java-components/build-request-processor/src/test/java/com/redhat/hacbs/container/analyser/deploy/git/GitTest.java b/java-components/build-request-processor/src/test/java/com/redhat/hacbs/container/analyser/deploy/git/GitTest.java index 31095c655..3809b00d4 100644 --- a/java-components/build-request-processor/src/test/java/com/redhat/hacbs/container/analyser/deploy/git/GitTest.java +++ b/java-components/build-request-processor/src/test/java/com/redhat/hacbs/container/analyser/deploy/git/GitTest.java @@ -10,11 +10,14 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.util.List; +import java.util.Map; import java.util.Objects; import java.util.logging.LogRecord; import org.apache.commons.io.FileUtils; import org.eclipse.jgit.api.errors.GitAPIException; +import org.eclipse.jgit.lib.Ref; +import org.eclipse.jgit.lib.Repository; import org.eclipse.jgit.transport.TagOpt; import org.eclipse.jgit.transport.URIish; import org.junit.jupiter.api.BeforeEach; @@ -54,6 +57,7 @@ public void testPush() Path initialRepo = Files.createTempDirectory("initial-repo"); Path testRepo = Files.createTempDirectory("test-repo"); String testRepoURI = "file://" + testRepo; + String imageID = "75ecd81c7a2b384151c990975eb1dd10"; try (var testRepository = org.eclipse.jgit.api.Git.init().setDirectory(testRepo.toFile()).call(); var initialRepository = org.eclipse.jgit.api.Git.init().setDirectory(initialRepo.toFile()).call()) { Path repoRoot = Paths.get(Objects.requireNonNull(getClass().getResource("/")).toURI()).getParent().getParent() @@ -74,7 +78,8 @@ public void create(String name) { } @Override - public void add(Path path, String commit, String imageId) { + public Map add(Path path, String commit, String imageId) { + return null; } @Override @@ -82,11 +87,11 @@ public String groupSplit() { return null; } }; - test.pushRepository( + Map tagResults = test.pushRepository( initialRepo, testRepoURI, "c396268fb90335bde5c9272b9a194c3d4302bf24", - "75ecd81c7a2b384151c990975eb1dd10"); + imageID); List logRecords = LogCollectingTestResource.current().getRecords(); @@ -98,9 +103,25 @@ public String groupSplit() { .anyMatch( r -> LogCollectingTestResource.format(r).matches("Updating current origin of.*to " + testRepoURI))); - assertEquals(2, testRepository.tagList().call().size()); - assertTrue(testRepository.tagList().call().stream() - .anyMatch(r -> r.getName().equals("refs/tags/0.1-75ecd81c7a2b384151c990975eb1dd10"))); + List tags = testRepository.tagList().call(); + assertEquals(2, tags.size()); + assertEquals(1, tagResults.size()); + + assertTrue(tags.stream().anyMatch(r -> r.getName().equals("refs/tags/0.1-75ecd81c7a2b384151c990975eb1dd10"))); + + tagResults.forEach((k, v) -> { + var found = tags.stream().filter(t -> Repository.shortenRefName(t.getName()).matches(k)).findFirst(); + assertTrue(found.isPresent()); + try { + assertTrue(v.matches(testRepository.getRepository() + .getRefDatabase() + .peel(found.get()) + .getPeeledObjectId() + .getName())); + } catch (IOException e) { + throw new RuntimeException(e); + } + }); } } From cddc802e5650c77d1cf191e32bd2a34f2d3df09c Mon Sep 17 00:00:00 2001 From: Nick Cross Date: Sat, 25 Nov 2023 17:17:22 +0000 Subject: [PATCH 11/15] Add GitArchive information to BuildAttempt --- .../jvmbuildservice.io_dependencybuilds.yaml | 10 +++++++ .../analyser/deploy/DeployCommand.java | 4 +-- .../container/analyser/deploy/git/Git.java | 29 +++++++++++++++---- .../container/analyser/deploy/git/GitHub.java | 3 +- .../container/analyser/deploy/git/GitLab.java | 4 +-- .../hacbs/container/verifier/DiffUtils.java | 2 -- .../hacbs/container/verifier/MavenUtils.java | 4 +-- .../verifier/VerifyBuiltArtifactsCommand.java | 6 ++-- .../hacbs/container/verifier/asm/JarInfo.java | 5 ++-- .../analyser/deploy/git/GitTest.java | 28 +++++++----------- .../services/CachePomScmLocator.java | 8 ++--- .../jvmbuildservice.io_dependencybuilds.yaml | 10 +++++++ .../v1alpha1/dependencybuild_types.go | 8 +++++ pkg/reconciler/artifactbuild/artifactbuild.go | 13 ++------- .../dependencybuild/buildrecipeyaml.go | 2 ++ .../dependencybuild/dependencybuild.go | 14 +++++++++ 16 files changed, 94 insertions(+), 56 deletions(-) diff --git a/deploy/crds/base/jvmbuildservice.io_dependencybuilds.yaml b/deploy/crds/base/jvmbuildservice.io_dependencybuilds.yaml index 6ea6a18e4..19aa3b279 100644 --- a/deploy/crds/base/jvmbuildservice.io_dependencybuilds.yaml +++ b/deploy/crds/base/jvmbuildservice.io_dependencybuilds.yaml @@ -86,6 +86,16 @@ spec: items: type: string type: array + gitArchive: + description: The git archive source information + properties: + sha: + type: string + tag: + type: string + url: + type: string + type: object hermeticBuildImage: description: The hermetic build image produced by the build diff --git a/java-components/build-request-processor/src/main/java/com/redhat/hacbs/container/analyser/deploy/DeployCommand.java b/java-components/build-request-processor/src/main/java/com/redhat/hacbs/container/analyser/deploy/DeployCommand.java index 7c05c4610..cb3c8e3e2 100644 --- a/java-components/build-request-processor/src/main/java/com/redhat/hacbs/container/analyser/deploy/DeployCommand.java +++ b/java-components/build-request-processor/src/main/java/com/redhat/hacbs/container/analyser/deploy/DeployCommand.java @@ -159,8 +159,8 @@ public void run() { try { Set gavs = new HashSet<>(); Map> contaminatedPaths = new HashMap<>(); - Map> contaminatedGavs = new HashMap<>(); - Map archivedSourceTags = new HashMap<>(); + Map contaminatedGavs = new HashMap<>(); + Git.GitStatus archivedSourceTags = new Git.GitStatus(); // Save the source first regardless of deployment checks if (isNotEmpty(gitIdentity) && gitToken.isPresent()) { diff --git a/java-components/build-request-processor/src/main/java/com/redhat/hacbs/container/analyser/deploy/git/Git.java b/java-components/build-request-processor/src/main/java/com/redhat/hacbs/container/analyser/deploy/git/Git.java index e51e6cdba..1628480c3 100644 --- a/java-components/build-request-processor/src/main/java/com/redhat/hacbs/container/analyser/deploy/git/Git.java +++ b/java-components/build-request-processor/src/main/java/com/redhat/hacbs/container/analyser/deploy/git/Git.java @@ -4,8 +4,6 @@ import java.net.URI; import java.net.URISyntaxException; import java.nio.file.Path; -import java.util.Collections; -import java.util.Map; import java.util.UUID; import org.eclipse.jgit.api.errors.GitAPIException; @@ -27,7 +25,7 @@ public abstract class Git { public abstract void create(String name) throws IOException, URISyntaxException; - public abstract Map add(Path path, String commit, String imageId) + public abstract GitStatus add(Path path, String commit, String imageId) throws IOException; /** @@ -65,7 +63,7 @@ public static Git builder(String endpoint, String identity, String token, boolea } } - protected Map pushRepository(Path path, String httpTransportUrl, String commit, String imageId) { + protected GitStatus pushRepository(Path path, String httpTransportUrl, String commit, String imageId) { try (var jGit = org.eclipse.jgit.api.Git.init().setDirectory(path.toFile()).call()) { // Find the tag name associated with the commit. Then append the unique imageId. This is from the Go code // and is a hash of abr.Status.SCMInfo.SCMURL + abr.Status.SCMInfo.Tag + abr.Status.SCMInfo.Path @@ -109,9 +107,8 @@ protected Map pushRepository(Path path, String httpTransportUrl, + result.getRemoteUpdates()); } - return Collections.singletonMap(Repository.shortenRefName(tagRefUnique.getName()), + return new GitStatus(httpTransportUrl, Repository.shortenRefName(tagRefUnique.getName()), jRepo.getRefDatabase().peel(tagRefUnique).getPeeledObjectId().getName()); - } catch (GitAPIException | IOException e) { throw new RuntimeException(e); } @@ -134,4 +131,24 @@ protected String parseScmURI(String scmUri) } abstract String groupSplit(); + + public static class GitStatus { + public String url; + public String tag; + public String sha; + + public GitStatus() { + } + + public GitStatus(String url, String tag, String sha) { + this.url = url; + this.tag = tag; + this.sha = sha; + } + + @Override + public String toString() { + return "GitStatus{url='" + url + "', tag='" + tag + "', sha='" + sha + "'}"; + } + } } diff --git a/java-components/build-request-processor/src/main/java/com/redhat/hacbs/container/analyser/deploy/git/GitHub.java b/java-components/build-request-processor/src/main/java/com/redhat/hacbs/container/analyser/deploy/git/GitHub.java index 23d0cb197..92fe9dc0f 100644 --- a/java-components/build-request-processor/src/main/java/com/redhat/hacbs/container/analyser/deploy/git/GitHub.java +++ b/java-components/build-request-processor/src/main/java/com/redhat/hacbs/container/analyser/deploy/git/GitHub.java @@ -5,7 +5,6 @@ import java.io.IOException; import java.net.URISyntaxException; import java.nio.file.Path; -import java.util.Map; import org.eclipse.jgit.transport.UsernamePasswordCredentialsProvider; import org.kohsuke.github.GHRepository; @@ -88,7 +87,7 @@ public void create(String scmUri) } @Override - public Map add(Path path, String commit, String imageId) { + public GitStatus add(Path path, String commit, String imageId) { if (repository == null) { throw new RuntimeException("Call create first"); } diff --git a/java-components/build-request-processor/src/main/java/com/redhat/hacbs/container/analyser/deploy/git/GitLab.java b/java-components/build-request-processor/src/main/java/com/redhat/hacbs/container/analyser/deploy/git/GitLab.java index 04bb6d3e6..206f0e475 100644 --- a/java-components/build-request-processor/src/main/java/com/redhat/hacbs/container/analyser/deploy/git/GitLab.java +++ b/java-components/build-request-processor/src/main/java/com/redhat/hacbs/container/analyser/deploy/git/GitLab.java @@ -3,7 +3,6 @@ import java.io.IOException; import java.net.URISyntaxException; import java.nio.file.Path; -import java.util.Map; import org.eclipse.jgit.transport.UsernamePasswordCredentialsProvider; import org.gitlab4j.api.GitLabApi; @@ -41,7 +40,6 @@ public GitLab(String endpoint, String identity, String token, boolean ssl) { public void create(String scmUri) throws URISyntaxException { String name = parseScmURI(scmUri); - Log.infof("Creating repository with name %s", name); try { project = gitLabApi.getProjectApi().getUserProjectsStream(owner, new ProjectFilter().withSearch(name)) .filter(p -> p.getName().equals(name)) @@ -68,7 +66,7 @@ public void create(String scmUri) } @Override - public Map add(Path path, String commit, String imageId) + public GitStatus add(Path path, String commit, String imageId) throws IOException { if (project == null) { throw new RuntimeException("Call create first"); diff --git a/java-components/build-request-processor/src/main/java/com/redhat/hacbs/container/verifier/DiffUtils.java b/java-components/build-request-processor/src/main/java/com/redhat/hacbs/container/verifier/DiffUtils.java index 8a3587edc..3ba20fb04 100644 --- a/java-components/build-request-processor/src/main/java/com/redhat/hacbs/container/verifier/DiffUtils.java +++ b/java-components/build-request-processor/src/main/java/com/redhat/hacbs/container/verifier/DiffUtils.java @@ -10,12 +10,10 @@ import java.util.stream.Collectors; import org.apache.commons.lang3.builder.DiffResult; -import org.jboss.logging.Logger; import com.redhat.hacbs.container.verifier.asm.AsmDiffable; public class DiffUtils { - private static final Logger Log = Logger.getLogger(DiffUtils.class); public record DiffResults(Set shared, Set added, Set deleted, Map> diffResults, diff --git a/java-components/build-request-processor/src/main/java/com/redhat/hacbs/container/verifier/MavenUtils.java b/java-components/build-request-processor/src/main/java/com/redhat/hacbs/container/verifier/MavenUtils.java index d381957e1..4b59231e1 100644 --- a/java-components/build-request-processor/src/main/java/com/redhat/hacbs/container/verifier/MavenUtils.java +++ b/java-components/build-request-processor/src/main/java/com/redhat/hacbs/container/verifier/MavenUtils.java @@ -28,10 +28,10 @@ import org.eclipse.aether.util.repository.AuthenticationBuilder; import org.eclipse.aether.util.repository.DefaultAuthenticationSelector; import org.eclipse.aether.util.repository.DefaultMirrorSelector; -import org.jboss.logging.Logger; + +import io.quarkus.logging.Log; public class MavenUtils { - private static final Logger Log = Logger.getLogger(MavenUtils.class); public static Settings newSettings(Path globalSettingsFile, Path settingsFile) { try { diff --git a/java-components/build-request-processor/src/main/java/com/redhat/hacbs/container/verifier/VerifyBuiltArtifactsCommand.java b/java-components/build-request-processor/src/main/java/com/redhat/hacbs/container/verifier/VerifyBuiltArtifactsCommand.java index e8dec461b..b5434f920 100644 --- a/java-components/build-request-processor/src/main/java/com/redhat/hacbs/container/verifier/VerifyBuiltArtifactsCommand.java +++ b/java-components/build-request-processor/src/main/java/com/redhat/hacbs/container/verifier/VerifyBuiltArtifactsCommand.java @@ -36,20 +36,19 @@ import org.eclipse.aether.repository.LocalRepository; import org.eclipse.aether.repository.RemoteRepository; import org.eclipse.aether.repository.RemoteRepository.Builder; -import org.jboss.logging.Logger; import com.redhat.hacbs.container.results.ResultsUpdater; import com.redhat.hacbs.container.verifier.asm.JarInfo; import io.quarkus.bootstrap.resolver.maven.BootstrapMavenContext; import io.quarkus.bootstrap.resolver.maven.BootstrapMavenException; +import io.quarkus.logging.Log; import picocli.CommandLine; import picocli.CommandLine.Command; import picocli.CommandLine.Option; @Command(name = "verify-built-artifacts") public class VerifyBuiltArtifactsCommand implements Callable { - private static final Logger Log = Logger.getLogger(VerifyBuiltArtifactsCommand.class); static class LocalOptions { @Option(required = true, names = { "-of", "--original-file" }) @@ -139,7 +138,6 @@ public Integer call() { session.setReadOnly(); } - Log.debugf("Deploy path: %s", options.mavenOptions.deployPath); var futureResults = new HashMap>>(); Files.walkFileTree(options.mavenOptions.deployPath, new SimpleFileVisitor<>() { @@ -191,7 +189,7 @@ public List call() throws Exception { } if (taskRunName != null) { var json = ResultsUpdater.MAPPER.writeValueAsString(verificationResults); - io.quarkus.logging.Log.infof("Writing verification results %s", json); + Log.infof("Writing verification results %s", json); resultsUpdater.get().updateResults(taskRunName, Map.of("VERIFICATION_RESULTS", json)); } return (failed && !reportOnly ? 1 : 0); diff --git a/java-components/build-request-processor/src/main/java/com/redhat/hacbs/container/verifier/asm/JarInfo.java b/java-components/build-request-processor/src/main/java/com/redhat/hacbs/container/verifier/asm/JarInfo.java index a90bd4ee4..94cf4fe3e 100644 --- a/java-components/build-request-processor/src/main/java/com/redhat/hacbs/container/verifier/asm/JarInfo.java +++ b/java-components/build-request-processor/src/main/java/com/redhat/hacbs/container/verifier/asm/JarInfo.java @@ -16,15 +16,14 @@ import java.util.regex.Pattern; import org.apache.commons.lang3.StringUtils; -import org.jboss.logging.Logger; import org.objectweb.asm.ClassReader; import org.objectweb.asm.tree.ClassNode; import com.redhat.hacbs.container.verifier.DiffUtils; -public record JarInfo(String name, Map classes) implements AsmDiffable { +import io.quarkus.logging.Log; - private static final Logger Log = Logger.getLogger(JarInfo.class); +public record JarInfo(String name, Map classes) implements AsmDiffable { // diffClass excluding name diff --git a/java-components/build-request-processor/src/test/java/com/redhat/hacbs/container/analyser/deploy/git/GitTest.java b/java-components/build-request-processor/src/test/java/com/redhat/hacbs/container/analyser/deploy/git/GitTest.java index 3809b00d4..c36340777 100644 --- a/java-components/build-request-processor/src/test/java/com/redhat/hacbs/container/analyser/deploy/git/GitTest.java +++ b/java-components/build-request-processor/src/test/java/com/redhat/hacbs/container/analyser/deploy/git/GitTest.java @@ -10,7 +10,6 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.util.List; -import java.util.Map; import java.util.Objects; import java.util.logging.LogRecord; @@ -78,7 +77,7 @@ public void create(String name) { } @Override - public Map add(Path path, String commit, String imageId) { + public GitStatus add(Path path, String commit, String imageId) { return null; } @@ -87,7 +86,7 @@ public String groupSplit() { return null; } }; - Map tagResults = test.pushRepository( + Git.GitStatus tagResults = test.pushRepository( initialRepo, testRepoURI, "c396268fb90335bde5c9272b9a194c3d4302bf24", @@ -105,23 +104,16 @@ public String groupSplit() { List tags = testRepository.tagList().call(); assertEquals(2, tags.size()); - assertEquals(1, tagResults.size()); - assertTrue(tags.stream().anyMatch(r -> r.getName().equals("refs/tags/0.1-75ecd81c7a2b384151c990975eb1dd10"))); - tagResults.forEach((k, v) -> { - var found = tags.stream().filter(t -> Repository.shortenRefName(t.getName()).matches(k)).findFirst(); - assertTrue(found.isPresent()); - try { - assertTrue(v.matches(testRepository.getRepository() - .getRefDatabase() - .peel(found.get()) - .getPeeledObjectId() - .getName())); - } catch (IOException e) { - throw new RuntimeException(e); - } - }); + var found = tags.stream().filter(t -> Repository.shortenRefName(t.getName()).matches(tagResults.tag)).findFirst(); + assertTrue(found.isPresent()); + assertTrue(tagResults.url.contains(testRepoURI)); + assertTrue(tagResults.sha.matches(testRepository.getRepository() + .getRefDatabase() + .peel(found.get()) + .getPeeledObjectId() + .getName())); } } diff --git a/java-components/cache/src/main/java/com/redhat/hacbs/artifactcache/services/CachePomScmLocator.java b/java-components/cache/src/main/java/com/redhat/hacbs/artifactcache/services/CachePomScmLocator.java index 62ce2cadb..d207ba078 100644 --- a/java-components/cache/src/main/java/com/redhat/hacbs/artifactcache/services/CachePomScmLocator.java +++ b/java-components/cache/src/main/java/com/redhat/hacbs/artifactcache/services/CachePomScmLocator.java @@ -9,14 +9,14 @@ import org.apache.maven.model.Model; import org.apache.maven.model.io.xpp3.MavenXpp3Reader; -import org.jboss.logging.Logger; import com.redhat.hacbs.recipies.scm.AbstractPomScmLocator; +import io.quarkus.logging.Log; + @ApplicationScoped public class CachePomScmLocator extends AbstractPomScmLocator { - private static final Logger log = Logger.getLogger(CachePomScmLocator.class); @Inject CacheFacade cache; @@ -27,7 +27,7 @@ protected AbstractPomScmLocator.PomClient createPomClient() { public Optional getPom(String group, String artifact, String version) { var response = cache.getArtifactFile("default", group.replace(".", "/"), artifact, version, artifact + "-" + version + ".pom", false); - if (!response.isPresent()) { + if (response.isEmpty()) { return Optional.empty(); } try { @@ -38,7 +38,7 @@ public Optional getPom(String group, String artifact, String version) { } } catch (Exception e) { - log.errorf(e, "Failed to get pom for %s:%s:%s", group, artifact, version); + Log.errorf(e, "Failed to get pom for %s:%s:%s", group, artifact, version); return Optional.empty(); } finally { try { diff --git a/java-components/resource-model/src/main/resources/crds/jvmbuildservice.io_dependencybuilds.yaml b/java-components/resource-model/src/main/resources/crds/jvmbuildservice.io_dependencybuilds.yaml index 6ea6a18e4..19aa3b279 100644 --- a/java-components/resource-model/src/main/resources/crds/jvmbuildservice.io_dependencybuilds.yaml +++ b/java-components/resource-model/src/main/resources/crds/jvmbuildservice.io_dependencybuilds.yaml @@ -86,6 +86,16 @@ spec: items: type: string type: array + gitArchive: + description: The git archive source information + properties: + sha: + type: string + tag: + type: string + url: + type: string + type: object hermeticBuildImage: description: The hermetic build image produced by the build diff --git a/pkg/apis/jvmbuildservice/v1alpha1/dependencybuild_types.go b/pkg/apis/jvmbuildservice/v1alpha1/dependencybuild_types.go index f00c6aabf..16514bfcc 100644 --- a/pkg/apis/jvmbuildservice/v1alpha1/dependencybuild_types.go +++ b/pkg/apis/jvmbuildservice/v1alpha1/dependencybuild_types.go @@ -92,11 +92,19 @@ type BuildPipelineRunResults struct { Gavs []string `json:"gavs,omitempty"` // The hermetic build image produced by the build HermeticBuildImage string `json:"hermeticBuildImage,omitempty"` + // The git archive source information + GitArchive GitArchive `json:"gitArchive,omitempty"` // The Tekton results PipelineResults *PipelineResults `json:"pipelineResults,omitempty"` } +type GitArchive struct { + URL string `json:"url,omitempty"` + Tag string `json:"tag,omitempty"` + SHA string `json:"sha,omitempty"` +} + func (r *DependencyBuildStatus) GetBuildPipelineRun(pipeline string) *BuildAttempt { for i := range r.BuildAttempts { ba := r.BuildAttempts[i] diff --git a/pkg/reconciler/artifactbuild/artifactbuild.go b/pkg/reconciler/artifactbuild/artifactbuild.go index 3d55a559a..91816bea9 100644 --- a/pkg/reconciler/artifactbuild/artifactbuild.go +++ b/pkg/reconciler/artifactbuild/artifactbuild.go @@ -36,14 +36,6 @@ const ( DependencyBuildIdLabel = "jvmbuildservice.io/dependencybuild-id" PipelineRunLabel = "jvmbuildservice.io/pipelinerun" - PipelineResultScmUrl = "scm-url" - PipelineResultScmTag = "scm-tag" - PipelineResultScmHash = "scm-hash" - PipelineResultScmType = "scm-type" - PipelineResultContextPath = "context" - PipelineResultMessage = "message" - PipelineResultPrivate = "private" - PreBuildTaskName = "pre-build" BuildTaskName = "build" HermeticBuildTaskName = "hermetic-build" @@ -54,12 +46,13 @@ const ( PipelineResultVerificationResult = "VERIFICATION_RESULTS" PipelineResultPassedVerification = "PASSED_VERIFICATION" //#nosec PipelineResultHermeticBuildImage = "HERMETIC_BUILD_IMAGE" + PipelineResultGitArchive = "GIT_ARCHIVE" PipelineResultGavs = "GAVS" RebuildAnnotation = "jvmbuildservice.io/rebuild" - //annotation that is applied after a rebuild, it will affect the dependencybuild behaviour + // RebuiltAnnotation annotation that is applied after a rebuild, it will affect the dependencybuild behaviour RebuiltAnnotation = "jvmbuildservice.io/rebuilt" - //if this annotation is present it will be deleted after a set time to live + // HoursToLive if this annotation is present it will be deleted after a set time to live //useful when doing builds that are being deployed to maven, and you don't want to accumulate them in the cluster HoursToLive = "jvmbuildservice.io/hours-to-live" ) diff --git a/pkg/reconciler/dependencybuild/buildrecipeyaml.go b/pkg/reconciler/dependencybuild/buildrecipeyaml.go index 5ec8ebe4d..bf05dcc3e 100644 --- a/pkg/reconciler/dependencybuild/buildrecipeyaml.go +++ b/pkg/reconciler/dependencybuild/buildrecipeyaml.go @@ -248,6 +248,7 @@ func createPipelineSpec(tool string, commitTime int64, jbsConfig *v1alpha12.JBSC {Name: PipelineResultImageDigest}, {Name: artifactbuild.PipelineResultPassedVerification}, {Name: artifactbuild.PipelineResultVerificationResult}, + {Name: artifactbuild.PipelineResultGitArchive}, }...), Steps: []pipelinev1beta1.Step{ { @@ -296,6 +297,7 @@ func createPipelineSpec(tool string, commitTime int64, jbsConfig *v1alpha12.JBSC {Name: PipelineResultImageDigest}, {Name: artifactbuild.PipelineResultPassedVerification}, {Name: artifactbuild.PipelineResultVerificationResult}, + {Name: artifactbuild.PipelineResultGitArchive}, }, Steps: []pipelinev1beta1.Step{ { diff --git a/pkg/reconciler/dependencybuild/dependencybuild.go b/pkg/reconciler/dependencybuild/dependencybuild.go index 2ced02e03..10bb1adf6 100644 --- a/pkg/reconciler/dependencybuild/dependencybuild.go +++ b/pkg/reconciler/dependencybuild/dependencybuild.go @@ -652,6 +652,8 @@ func (r *ReconcileDependencyBuild) handleBuildPipelineRunReceived(ctx context.Co var verificationResults string var hermeticBuildImage string var deployed []string + var gitArchive v1alpha1.GitArchive + for _, i := range pr.Status.Results { if i.Name == PipelineResultImage { image = i.Value.StringVal @@ -668,8 +670,13 @@ func (r *ReconcileDependencyBuild) handleBuildPipelineRunReceived(ctx context.Co parseBool, _ := strconv.ParseBool(i.Value.StringVal) passedVerification = !parseBool } else if i.Name == artifactbuild.PipelineResultVerificationResult { + // Note: The TaskRun stores this as + // VERIFICATION_RESULTS {"commons-lang:commons-lang:jar:2.5":[]} + // But this is now stored as + // "verificationFailures": "{\"commons-lang:commons-lang:jar:2.5\":[]}" verificationResults = i.Value.StringVal } else if i.Name == artifactbuild.PipelineResultGavs { + // TODO: What is the difference between this and PipelineResultDeployedResources? deployed := strings.Split(i.Value.StringVal, ",") db.Status.DeployedArtifacts = deployed } else if i.Name == artifactbuild.PipelineResultDeployedResources && len(i.Value.StringVal) > 0 { @@ -677,6 +684,11 @@ func (r *ReconcileDependencyBuild) handleBuildPipelineRunReceived(ctx context.Co deployed = strings.Split(i.Value.StringVal, ",") } else if i.Name == artifactbuild.PipelineResultHermeticBuildImage { hermeticBuildImage = i.Value.StringVal + } else if i.Name == artifactbuild.PipelineResultGitArchive { + err := json.Unmarshal([]byte(i.Value.StringVal), &gitArchive) + if err != nil { + return reconcile.Result{}, err + } } } err = r.createRebuiltArtifacts(ctx, log, pr, db, image, digest, deployed) @@ -690,8 +702,10 @@ func (r *ReconcileDependencyBuild) handleBuildPipelineRunReceived(ctx context.Co Verified: passedVerification, VerificationResults: verificationResults, Gavs: deployed, + GitArchive: gitArchive, HermeticBuildImage: hermeticBuildImage, } + problemContaminates := db.Status.ProblemContaminates() if len(problemContaminates) == 0 { db.Status.State = v1alpha1.DependencyBuildStateComplete From 5bacac1f4eccb42bd49bc8e9a9c663d828d462df Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 28 Nov 2023 11:49:25 +0000 Subject: [PATCH 12/15] Bump maven-resolver.version from 1.9.16 to 1.9.18 in /java-components Bumps `maven-resolver.version` from 1.9.16 to 1.9.18. Updates `org.apache.maven.resolver:maven-resolver-api` from 1.9.16 to 1.9.18 - [Release notes](https://github.com/apache/maven-resolver/releases) - [Commits](https://github.com/apache/maven-resolver/compare/maven-resolver-1.9.16...maven-resolver-1.9.18) Updates `org.apache.maven.resolver:maven-resolver-connector-basic` from 1.9.16 to 1.9.18 - [Release notes](https://github.com/apache/maven-resolver/releases) - [Commits](https://github.com/apache/maven-resolver/compare/maven-resolver-1.9.16...maven-resolver-1.9.18) Updates `org.apache.maven.resolver:maven-resolver-impl` from 1.9.16 to 1.9.18 - [Release notes](https://github.com/apache/maven-resolver/releases) - [Commits](https://github.com/apache/maven-resolver/compare/maven-resolver-1.9.16...maven-resolver-1.9.18) Updates `org.apache.maven.resolver:maven-resolver-spi` from 1.9.16 to 1.9.18 - [Release notes](https://github.com/apache/maven-resolver/releases) - [Commits](https://github.com/apache/maven-resolver/compare/maven-resolver-1.9.16...maven-resolver-1.9.18) Updates `org.apache.maven.resolver:maven-resolver-transport-file` from 1.9.16 to 1.9.18 - [Release notes](https://github.com/apache/maven-resolver/releases) - [Commits](https://github.com/apache/maven-resolver/compare/maven-resolver-1.9.16...maven-resolver-1.9.18) Updates `org.apache.maven.resolver:maven-resolver-transport-http` from 1.9.16 to 1.9.18 Updates `org.apache.maven.resolver:maven-resolver-transport-wagon` from 1.9.16 to 1.9.18 - [Release notes](https://github.com/apache/maven-resolver/releases) - [Commits](https://github.com/apache/maven-resolver/compare/maven-resolver-1.9.16...maven-resolver-1.9.18) Updates `org.apache.maven.resolver:maven-resolver-util` from 1.9.16 to 1.9.18 - [Release notes](https://github.com/apache/maven-resolver/releases) - [Commits](https://github.com/apache/maven-resolver/compare/maven-resolver-1.9.16...maven-resolver-1.9.18) --- updated-dependencies: - dependency-name: org.apache.maven.resolver:maven-resolver-api dependency-type: direct:production update-type: version-update:semver-patch - dependency-name: org.apache.maven.resolver:maven-resolver-connector-basic dependency-type: direct:production update-type: version-update:semver-patch - dependency-name: org.apache.maven.resolver:maven-resolver-impl dependency-type: direct:production update-type: version-update:semver-patch - dependency-name: org.apache.maven.resolver:maven-resolver-spi dependency-type: direct:production update-type: version-update:semver-patch - dependency-name: org.apache.maven.resolver:maven-resolver-transport-file dependency-type: direct:production update-type: version-update:semver-patch - dependency-name: org.apache.maven.resolver:maven-resolver-transport-http dependency-type: direct:production update-type: version-update:semver-patch - dependency-name: org.apache.maven.resolver:maven-resolver-transport-wagon dependency-type: direct:production update-type: version-update:semver-patch - dependency-name: org.apache.maven.resolver:maven-resolver-util dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- java-components/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/java-components/pom.xml b/java-components/pom.xml index 96a5cbc1d..948c4df40 100644 --- a/java-components/pom.xml +++ b/java-components/pom.xml @@ -60,7 +60,7 @@ 8.0.3 9.6 7.1.0 - 1.9.16 + 1.9.18 From 4fd1a2223a62e0a0e43d9550ebe76a61a9cd64ba Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 28 Nov 2023 11:49:33 +0000 Subject: [PATCH 13/15] Bump com.amazonaws:aws-java-sdk-codeartifact in /java-components Bumps [com.amazonaws:aws-java-sdk-codeartifact](https://github.com/aws/aws-sdk-java) from 1.12.592 to 1.12.598. - [Changelog](https://github.com/aws/aws-sdk-java/blob/master/CHANGELOG.md) - [Commits](https://github.com/aws/aws-sdk-java/compare/1.12.592...1.12.598) --- updated-dependencies: - dependency-name: com.amazonaws:aws-java-sdk-codeartifact dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- java-components/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/java-components/pom.xml b/java-components/pom.xml index 96a5cbc1d..c617280b5 100644 --- a/java-components/pom.xml +++ b/java-components/pom.xml @@ -51,7 +51,7 @@ 6.9.2 3.24.2 - 1.12.592 + 1.12.598 3.0.5 8.4 From eb1c5242424fce69c93a46c53d3a0ea7484fcd2a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 28 Nov 2023 12:20:55 +0000 Subject: [PATCH 14/15] Bump org.kohsuke:github-api from 1.316 to 1.318 in /java-components Bumps [org.kohsuke:github-api](https://github.com/hub4j/github-api) from 1.316 to 1.318. - [Release notes](https://github.com/hub4j/github-api/releases) - [Changelog](https://github.com/hub4j/github-api/blob/main/CHANGELOG.md) - [Commits](https://github.com/hub4j/github-api/compare/github-api-1.316...github-api-1.318) --- updated-dependencies: - dependency-name: org.kohsuke:github-api dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- java-components/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/java-components/pom.xml b/java-components/pom.xml index 96a5cbc1d..63b56b9dc 100644 --- a/java-components/pom.xml +++ b/java-components/pom.xml @@ -55,7 +55,7 @@ 3.0.5 8.4 - 1.316 + 1.318 ${github-api.version}.0 8.0.3 9.6 From 0da3af7cb7271a233cb1c163bee38e80409d9c46 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 28 Nov 2023 12:21:13 +0000 Subject: [PATCH 15/15] Bump quarkus.platform.version from 3.5.2 to 3.5.3 in /java-components Bumps `quarkus.platform.version` from 3.5.2 to 3.5.3. Updates `io.quarkus.platform:quarkus-bom` from 3.5.2 to 3.5.3 - [Commits](https://github.com/quarkusio/quarkus-platform/compare/3.5.2...3.5.3) Updates `io.quarkus.platform:quarkus-amazon-services-bom` from 3.5.2 to 3.5.3 - [Commits](https://github.com/quarkusio/quarkus-platform/compare/3.5.2...3.5.3) Updates `io.quarkus:quarkus-ide-config` from 3.5.2 to 3.5.3 - [Release notes](https://github.com/quarkusio/quarkus/releases) - [Commits](https://github.com/quarkusio/quarkus/compare/3.5.2...3.5.3) Updates `io.quarkus.platform:quarkus-maven-plugin` from 3.5.2 to 3.5.3 - [Commits](https://github.com/quarkusio/quarkus-platform/compare/3.5.2...3.5.3) --- updated-dependencies: - dependency-name: io.quarkus.platform:quarkus-bom dependency-type: direct:production update-type: version-update:semver-patch - dependency-name: io.quarkus.platform:quarkus-amazon-services-bom dependency-type: direct:production update-type: version-update:semver-patch - dependency-name: io.quarkus:quarkus-ide-config dependency-type: direct:production update-type: version-update:semver-patch - dependency-name: io.quarkus.platform:quarkus-maven-plugin dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- java-components/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/java-components/pom.xml b/java-components/pom.xml index 96a5cbc1d..93cb01248 100644 --- a/java-components/pom.xml +++ b/java-components/pom.xml @@ -42,7 +42,7 @@ UTF-8 quarkus-bom io.quarkus.platform - 3.5.2 + 3.5.3 3.2.2