diff --git a/.codecov.yml b/.codecov.yml index 6eae8911b..6d314a9e9 100644 --- a/.codecov.yml +++ b/.codecov.yml @@ -1,4 +1,3 @@ -comment: off coverage: status: project: @@ -6,7 +5,7 @@ coverage: target: auto threshold: 1% removed_code_behavior: adjust_base - patch: + patch: default: informational: true target: auto diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml deleted file mode 100644 index a8e03b767..000000000 --- a/.github/workflows/build.yml +++ /dev/null @@ -1,19 +0,0 @@ -name: Java CI - -on: [push, pull_request] - -jobs: - build: - - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v4 - - name: Set up JDK 17 - uses: actions/setup-java@v3 - with: - distribution: 'temurin' - java-version: 17 - - name: Build with Maven - working-directory: java-components - run: mvn -V -B package -Dvalidate-format diff --git a/.github/workflows/codecov-main.yaml b/.github/workflows/codecov-main.yaml deleted file mode 100644 index 134c7a9a0..000000000 --- a/.github/workflows/codecov-main.yaml +++ /dev/null @@ -1,19 +0,0 @@ -name: Codecov report on push to the "main" branch -on: - push: - branches: [ main ] -jobs: - coverage: - name: Unit tests and coverage report - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-go@v4 - with: - go-version-file: './go.mod' - - name: Build - run: make build - - name: Run tests - run: make test - - name: Codecov - uses: codecov/codecov-action@v3 diff --git a/.github/workflows/go-ci.yaml b/.github/workflows/go-ci.yaml index 791f547b0..0ce7f6b38 100644 --- a/.github/workflows/go-ci.yaml +++ b/.github/workflows/go-ci.yaml @@ -1,11 +1,14 @@ -name: Validate PR - golang CI +name: GoLang CI on: + push: + branches: [ main ] pull_request: branches: [ main ] jobs: lint: name: Lint runs-on: ubuntu-latest + if: github.event_name == 'pull_request' steps: - uses: actions/checkout@v4 - uses: actions/setup-go@v4 @@ -18,13 +21,14 @@ jobs: go: name: Check sources runs-on: ubuntu-latest + if: github.event_name == 'pull_request' steps: + - name: Check out code + uses: actions/checkout@v4 - name: Install Go uses: actions/setup-go@v4 with: - go-version: 1.19.x - - name: Check out code - uses: actions/checkout@v4 + go-version-file: './go.mod' - name: Check go mod status run: | go mod tidy @@ -56,12 +60,12 @@ jobs: name: Golang Unit tests runs-on: ubuntu-latest steps: + - name: Check out code + uses: actions/checkout@v4 - name: Install Go uses: actions/setup-go@v4 with: - go-version: 1.19.x - - name: Check out code - uses: actions/checkout@v4 + go-version-file: './go.mod' - name: Build run: make build - name: Test @@ -71,6 +75,7 @@ jobs: security_scan: name: Security scan runs-on: ubuntu-latest + if: github.event_name == 'pull_request' steps: - uses: actions/checkout@v4 - uses: actions/setup-go@v4 diff --git a/.github/workflows/java-ci.yml b/.github/workflows/java-ci.yml new file mode 100644 index 000000000..23c56ddd5 --- /dev/null +++ b/.github/workflows/java-ci.yml @@ -0,0 +1,29 @@ +name: Java CI + +on: + push: + branches: [ main ] + pull_request: + branches: [ main ] +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Cache local Maven repository + uses: actions/cache@v3 + with: + path: ~/.m2/repository + key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} + restore-keys: | + ${{ runner.os }}-maven- + - name: Set up JDK 17 + uses: actions/setup-java@v3 + with: + distribution: 'temurin' + java-version: 17 + - name: Build with Maven + working-directory: java-components + run: mvn -V -B package -Dvalidate-format + - name: Codecov + uses: codecov/codecov-action@v3 diff --git a/.github/workflows/kube-linter.yaml b/.github/workflows/kube-linter.yaml index 4b69b06d1..d243dd34b 100644 --- a/.github/workflows/kube-linter.yaml +++ b/.github/workflows/kube-linter.yaml @@ -1,4 +1,4 @@ -name: Check Kubernetes YAMLs with kube-linter +name: Validate Kubernetes YAMLs on: pull_request: @@ -7,7 +7,6 @@ on: - 'deploy/crds/base/**.ya?ml' - 'deploy/operator/base/**.ya?ml' - 'deploy/operator/config/**.ya?ml' - jobs: kube-linter: name: Kube linter diff --git a/.github/workflows/minikube.yaml b/.github/workflows/minikube.yaml index 80a326d77..847c1bfbd 100644 --- a/.github/workflows/minikube.yaml +++ b/.github/workflows/minikube.yaml @@ -1,4 +1,4 @@ -name: Validate PR - Minikube Tests +name: Minikube Tests on: pull_request: branches: [ main ] diff --git a/deploy/base-development.sh b/deploy/base-development.sh index 039bd112d..6c47cef74 100755 --- a/deploy/base-development.sh +++ b/deploy/base-development.sh @@ -18,11 +18,6 @@ DIR=`dirname $0` kubectl apply -f $DIR/namespace.yaml kubectl config set-context --current --namespace=test-jvm-namespace -if [ -n "$QUAY_ORG" ] && [ -n "$QUAY_TOKEN" ]; then - kubectl delete --ignore-not-found secret -n image-controller quaytoken - kubectl create secret generic -n image-controller quaytoken --from-literal "quaytoken=$QUAY_TOKEN" --from-literal "organization=$QUAY_ORG" -fi - echo -e "\033[0;32mSecrets...\033[0m" kubectl create --dry-run=client -o=yaml secret generic jvm-build-image-secrets --from-file=.dockerconfigjson=$HOME/.docker/config.json --type=kubernetes.io/dockerconfigjson | kubectl apply -f - kubectl create --dry-run=client -o=yaml secret generic jvm-build-git-secrets --from-literal .git-credentials=" diff --git a/deploy/crds/base/jvmbuildservice.io_dependencybuilds.yaml b/deploy/crds/base/jvmbuildservice.io_dependencybuilds.yaml index 6fa3f883c..19aa3b279 100644 --- a/deploy/crds/base/jvmbuildservice.io_dependencybuilds.yaml +++ b/deploy/crds/base/jvmbuildservice.io_dependencybuilds.yaml @@ -86,6 +86,16 @@ spec: items: type: string type: array + gitArchive: + description: The git archive source information + properties: + sha: + type: string + tag: + type: string + url: + type: string + type: object hermeticBuildImage: description: The hermetic build image produced by the build @@ -262,12 +272,20 @@ spec: contaminates: items: properties: + allowed: + type: boolean + buildId: + type: string contaminatedArtifacts: items: type: string type: array gav: type: string + rebuildAvailable: + type: boolean + source: + type: string type: object type: array deployedArtifacts: diff --git a/deploy/crds/base/jvmbuildservice.io_jbsconfigs.yaml b/deploy/crds/base/jvmbuildservice.io_jbsconfigs.yaml index dfadb6a02..edb422d83 100644 --- a/deploy/crds/base/jvmbuildservice.io_jbsconfigs.yaml +++ b/deploy/crds/base/jvmbuildservice.io_jbsconfigs.yaml @@ -89,6 +89,8 @@ spec: type: boolean gitSourceArchive: properties: + disableSSLVerification: + type: boolean identity: type: string url: diff --git a/deploy/operator/base/deployment.yaml b/deploy/operator/base/deployment.yaml index 2f59813bd..e86c18e75 100644 --- a/deploy/operator/base/deployment.yaml +++ b/deploy/operator/base/deployment.yaml @@ -13,11 +13,6 @@ spec: labels: app: hacbs-jvm-operator spec: - volumes: - - name: quaytoken - secret: - optional: false - secretName: quaytoken securityContext: runAsNonRoot: true containers: @@ -37,10 +32,6 @@ spec: limits: memory: "1024Mi" cpu: "500m" - volumeMounts: - - mountPath: "/workspace" - name: quaytoken - readOnly: true securityContext: readOnlyRootFilesystem: true serviceAccountName: hacbs-jvm-operator diff --git a/deploy/operator/overlays/ci-template/kustomization.yaml b/deploy/operator/overlays/ci-template/kustomization.yaml index 8a911b7cb..c4965c4e3 100644 --- a/deploy/operator/overlays/ci-template/kustomization.yaml +++ b/deploy/operator/overlays/ci-template/kustomization.yaml @@ -6,11 +6,3 @@ resources: # and I'm bypassing that gross env var munging we had to do in infra-deps; also, the deployment yaml should be pretty static at this point - base-deployment.yaml -patches: - - patch: |- - - op: replace - path: /spec/template/spec/volumes/0/secret/optional - value: true - target: - kind: Deployment - name: hacbs-jvm-operator diff --git a/deploy/operator/overlays/dev-template/kustomization.yaml b/deploy/operator/overlays/dev-template/kustomization.yaml index 7999e7947..7516a2735 100644 --- a/deploy/operator/overlays/dev-template/kustomization.yaml +++ b/deploy/operator/overlays/dev-template/kustomization.yaml @@ -21,11 +21,4 @@ patches: target: kind: Deployment name: hacbs-jvm-operator -- patch: |- - - op: replace - path: /spec/template/spec/volumes/0/secret/optional - value: true - target: - kind: Deployment - name: hacbs-jvm-operator - path: namespace.yaml diff --git a/deploy/overlays/dev-template/config.yaml b/deploy/overlays/dev-template/config.yaml index d1fd2237a..2804f7775 100644 --- a/deploy/overlays/dev-template/config.yaml +++ b/deploy/overlays/dev-template/config.yaml @@ -16,6 +16,7 @@ spec: gitSourceArchive: identity: GIT_DEPLOY_IDENTITY url: GIT_DEPLOY_URL + disableSSLVerification: GIT_DISABLE_SSL_VERIFICATION relocationPatterns: - relocationPattern: buildPolicy: "default" diff --git a/deploy/patch-yaml.sh b/deploy/patch-yaml.sh index ccb45e3eb..f1871631c 100755 --- a/deploy/patch-yaml.sh +++ b/deploy/patch-yaml.sh @@ -53,5 +53,9 @@ fi if [ -z "${GIT_DEPLOY_IDENTITY}" ]; then GIT_DEPLOY_IDENTITY="" fi -find $DIR -path \*development\*.yaml -exec $SED -i s/GIT_DEPLOY_URL/${GIT_DEPLOY_URL}/ {} \; +if [ -z "${GIT_DISABLE_SSL_VERIFICATION}" ]; then + GIT_DISABLE_SSL_VERIFICATION="false" +fi +find $DIR -path \*development\*.yaml -exec $SED -i s%GIT_DEPLOY_URL%${GIT_DEPLOY_URL}% {} \; find $DIR -path \*development\*.yaml -exec $SED -i s%GIT_DEPLOY_IDENTITY%${GIT_DEPLOY_IDENTITY}% {} \; +find $DIR -path \*development\*.yaml -exec $SED -i s%GIT_DISABLE_SSL_VERIFICATION%${GIT_DISABLE_SSL_VERIFICATION}% {} \; diff --git a/java-components/build-request-processor/src/main/java/com/redhat/hacbs/container/analyser/deploy/DeployCommand.java b/java-components/build-request-processor/src/main/java/com/redhat/hacbs/container/analyser/deploy/DeployCommand.java index c29ab9858..cb3c8e3e2 100644 --- a/java-components/build-request-processor/src/main/java/com/redhat/hacbs/container/analyser/deploy/DeployCommand.java +++ b/java-components/build-request-processor/src/main/java/com/redhat/hacbs/container/analyser/deploy/DeployCommand.java @@ -60,6 +60,7 @@ public class DeployCommand implements Runnable { private static final String DOT = "."; private static final Set ALLOWED_CONTAMINANTS = Set.of("-tests.jar"); public static final String IMAGE_DIGEST_OUTPUT = "Image Digest: "; + public static final String BUILD_ID = "build-id"; final BeanManager beanManager; final ResultsUpdater resultsUpdater; @@ -134,6 +135,9 @@ public class DeployCommand implements Runnable { @CommandLine.Option(names = "--git-identity") String gitIdentity; + @CommandLine.Option(names = "--git-disable-ssl-verification") + boolean gitDisableSSLVerification; + @CommandLine.Option(names = "--build-id") String buildId; // Testing only ; used to disable image deployment @@ -153,18 +157,19 @@ public DeployCommand(BeanManager beanManager, public void run() { try { + Set gavs = new HashSet<>(); + Map> contaminatedPaths = new HashMap<>(); + Map contaminatedGavs = new HashMap<>(); + Git.GitStatus archivedSourceTags = new Git.GitStatus(); + // Save the source first regardless of deployment checks if (isNotEmpty(gitIdentity) && gitToken.isPresent()) { - var git = Git.builder(gitURL, gitIdentity, gitToken.get()); + Log.infof("Git credentials are identity '%s' and URL '%s'", gitIdentity, gitURL); + var git = Git.builder(gitURL, gitIdentity, gitToken.get(), gitDisableSSLVerification); git.create(scmUri); - git.add(sourcePath, commit, imageId); + archivedSourceTags = git.add(sourcePath, commit, imageId); } - Set gavs = new HashSet<>(); - Map> contaminatedPaths = new HashMap<>(); - Map> contaminatedGavs = new HashMap<>(); - Map> allowedContaminatedPaths = new HashMap<>(); - Map> allowedContaminatedGavs = new HashMap<>(); // Represents directories that should not be deployed i.e. if a single artifact (barring test jars) is // contaminated then none of the artifacts will be deployed. Set toRemove = new HashSet<>(); @@ -180,12 +185,11 @@ public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IO //we check every file as we also want to catch .tar.gz etc var info = ClassFileTracker.readTrackingDataFromFile(Files.newInputStream(file), name); for (var i : info) { - if (!allowedSources.contains(i.source)) { - Log.errorf("%s was contaminated by %s from %s", name, i.gav, i.source); - if (ALLOWED_CONTAMINANTS.stream().noneMatch(a -> file.getFileName().toString().endsWith(a))) { - gav.ifPresent(g -> contaminatedGavs.computeIfAbsent(i.gav, s -> new HashSet<>()) - .add(g.getGroupId() + ":" + g.getArtifactId() + ":" + g.getVersion())); - int index = name.lastIndexOf("/"); + Log.errorf("%s was contaminated by %s from %s", name, i.gav, i.source); + if (ALLOWED_CONTAMINANTS.stream().noneMatch(a -> file.getFileName().toString().endsWith(a))) { + int index = name.lastIndexOf("/"); + boolean allowed = allowedSources.contains(i.source); + if (!allowed) { if (index != -1) { contaminatedPaths.computeIfAbsent(name.substring(0, index), s -> new HashSet<>()).add(i.gav); @@ -193,12 +197,23 @@ public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IO contaminatedPaths.computeIfAbsent("", s -> new HashSet<>()).add(i.gav); } toRemove.add(file.getParent()); - } else { - Log.debugf("Ignoring contaminant for %s", file.getFileName()); } - } else { + gav.ifPresent(g -> contaminatedGavs.computeIfAbsent(i.gav, s -> { + Contaminates contaminates = new Contaminates(); + contaminates.setGav(i.gav); + contaminates.setAllowed(allowed); + contaminates.setSource(i.source); + contaminates.setBuildId(i.getAttributes().get(BUILD_ID)); + contaminates.setContaminatedArtifacts(new ArrayList<>()); + return contaminates; + }) + .getContaminatedArtifacts() + .add(g.getGroupId() + ":" + g.getArtifactId() + ":" + g.getVersion())); + } else { + Log.debugf("Ignoring contaminant for %s", file.getFileName()); } + } if (gav.isPresent()) { //now add our own tracking data @@ -221,7 +236,7 @@ public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IO + gav.getVersion(), "rebuilt", Map.of("scm-uri", scmUri, "scm-commit", commit, "hermetic", - Boolean.toString(hermetic), "build-id", buildId)), + Boolean.toString(hermetic), BUILD_ID, buildId)), Files.newOutputStream(temp), false); Files.delete(file); Files.move(temp, file); @@ -265,14 +280,12 @@ public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) { }); throw new RuntimeException("deploy failed"); } - //we still deploy, but without the contaminates - // This means the build failed to produce any deployable output. - // If everything is contaminated we still need the task to succeed so we can resolve the contamination. for (var i : contaminatedGavs.entrySet()) { - gavs.removeAll(i.getValue()); + if (!i.getValue().getAllowed()) { + gavs.removeAll(i.getValue().getContaminatedArtifacts()); + } } generateBuildSbom(); - if (isNotEmpty(mvnRepo) && mvnPassword.isEmpty()) { Log.infof("Maven repository specified as %s and no password specified", mvnRepo); URL url = new URL(mvnRepo); @@ -300,6 +313,10 @@ public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) { .getAuthorizationToken()); } } + + //we still deploy, but without the contaminates + // This means the build failed to produce any deployable output. + // If everything is contaminated we still need the task to succeed so we can resolve the contamination. if (!gavs.isEmpty()) { try { cleanBrokenSymlinks(sourcePath); @@ -319,19 +336,18 @@ public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) { List newContaminates = new ArrayList<>(); for (var i : contaminatedGavs.entrySet()) { - Contaminates contaminates = new Contaminates(); - contaminates.setContaminatedArtifacts(new ArrayList<>(i.getValue())); - contaminates.setGav(i.getKey()); - newContaminates.add(contaminates); + newContaminates.add(i.getValue()); } String serialisedContaminants = ResultsUpdater.MAPPER.writeValueAsString(newContaminates); + String serialisedGitArchive = ResultsUpdater.MAPPER.writeValueAsString(archivedSourceTags); Log.infof("Updating results %s with contaminants %s and deployed resources %s", taskRun, serialisedContaminants, gavs); resultsUpdater.updateResults(taskRun, Map.of( "CONTAMINANTS", serialisedContaminants, "DEPLOYED_RESOURCES", String.join(",", gavs), "IMAGE_URL", imageName == null ? "" : imageName, - "IMAGE_DIGEST", imageDigest == null ? "" : "sha256:" + imageDigest)); + "IMAGE_DIGEST", imageDigest == null ? "" : "sha256:" + imageDigest, + "GIT_ARCHIVE", serialisedGitArchive)); } } catch (Exception e) { Log.error("Deployment failed", e); diff --git a/java-components/build-request-processor/src/main/java/com/redhat/hacbs/container/analyser/deploy/git/Git.java b/java-components/build-request-processor/src/main/java/com/redhat/hacbs/container/analyser/deploy/git/Git.java index b70f6684b..1628480c3 100644 --- a/java-components/build-request-processor/src/main/java/com/redhat/hacbs/container/analyser/deploy/git/Git.java +++ b/java-components/build-request-processor/src/main/java/com/redhat/hacbs/container/analyser/deploy/git/Git.java @@ -8,6 +8,7 @@ import org.eclipse.jgit.api.errors.GitAPIException; import org.eclipse.jgit.lib.Ref; +import org.eclipse.jgit.lib.Repository; import org.eclipse.jgit.lib.StoredConfig; import org.eclipse.jgit.transport.CredentialsProvider; import org.eclipse.jgit.transport.PushResult; @@ -19,10 +20,12 @@ public abstract class Git { protected CredentialsProvider credentialsProvider; + protected boolean disableSSLVerification; + public abstract void create(String name) throws IOException, URISyntaxException; - public abstract void add(Path path, String commit, String imageId) + public abstract GitStatus add(Path path, String commit, String imageId) throws IOException; /** @@ -35,18 +38,32 @@ public abstract void add(Path path, String commit, String imageId) */ public static Git builder(String endpoint, String identity, String token) throws IOException { + return builder(endpoint, identity, token, true); + } + + /** + * + * @param endpoint URL of the GitHub or GitLab instance. + * @param identity Might be user or organisation name. + * @param token Authorisation token. + * @param disableSSLVerification Whether to enable SSLVerification (Default: true). + * @return Valid Git instance + * @throws IOException if an error occurs + */ + public static Git builder(String endpoint, String identity, String token, boolean disableSSLVerification) + throws IOException { // TODO: This could be a bit presumptuous to assume // an on-premise installation will always contain some determinable // information. Alternative would be forcing the user to configure // endpoint, token, AND type [gitlab|github] if (endpoint != null && endpoint.contains("gitlab")) { - return new GitLab(endpoint, identity, token); + return new GitLab(endpoint, identity, token, disableSSLVerification); } else { - return new GitHub(endpoint, identity, token); + return new GitHub(endpoint, identity, token, disableSSLVerification); } } - protected void pushRepository(Path path, String httpTransportUrl, String commit, String imageId) { + protected GitStatus pushRepository(Path path, String httpTransportUrl, String commit, String imageId) { try (var jGit = org.eclipse.jgit.api.Git.init().setDirectory(path.toFile()).call()) { // Find the tag name associated with the commit. Then append the unique imageId. This is from the Go code // and is a hash of abr.Status.SCMInfo.SCMURL + abr.Status.SCMInfo.Tag + abr.Status.SCMInfo.Path @@ -61,6 +78,9 @@ protected void pushRepository(Path path, String httpTransportUrl, String commit, Log.infof("Updating current origin of %s to %s", jConfig.getString("remote", "origin", "url"), httpTransportUrl); jConfig.setString("remote", "origin", "url", httpTransportUrl); + if (disableSSLVerification) { + jConfig.setBoolean("http", null, "sslVerify", false); + } jConfig.save(); Log.infof("Pushing to %s with content from %s (branch %s, commit %s, tag %s)", httpTransportUrl, path, jRepo.getBranch(), commit, tagName); @@ -68,6 +88,7 @@ protected void pushRepository(Path path, String httpTransportUrl, String commit, Ref tagRefStable = jGit.tag().setAnnotated(true).setName(tagName + "-" + imageId).setForceUpdate(true).call(); Ref tagRefUnique = jGit.tag().setAnnotated(true).setName(tagName + "-" + UUID.randomUUID()).setForceUpdate(true) .call(); + Iterable results = jGit.push().setForce(true).setRemote("origin") .add(jRepo.getBranch()) // Push the default branch else GitHub doesn't show the code. .add(tagRefStable) @@ -76,7 +97,8 @@ protected void pushRepository(Path path, String httpTransportUrl, String commit, for (PushResult result : results) { result.getRemoteUpdates().forEach(r -> { - if (!r.getStatus().equals(RemoteRefUpdate.Status.OK)) { + if (!r.getStatus().equals(RemoteRefUpdate.Status.OK) + && !r.getStatus().equals(RemoteRefUpdate.Status.UP_TO_DATE)) { Log.errorf("Push failure " + r); throw new RuntimeException("Failed to push updates due to " + r.getMessage()); } @@ -84,6 +106,9 @@ protected void pushRepository(Path path, String httpTransportUrl, String commit, Log.debugf("Pushed " + result.getMessages() + " " + result.getURI() + " updates: " + result.getRemoteUpdates()); } + + return new GitStatus(httpTransportUrl, Repository.shortenRefName(tagRefUnique.getName()), + jRepo.getRefDatabase().peel(tagRefUnique).getPeeledObjectId().getName()); } catch (GitAPIException | IOException e) { throw new RuntimeException(e); } @@ -97,11 +122,33 @@ protected void pushRepository(Path path, String httpTransportUrl, String commit, * @return a reformatted name to use as the new repository name. * @throws URISyntaxException if an error occurs. */ - protected static String parseScmURI(String scmUri) + protected String parseScmURI(String scmUri) throws URISyntaxException { String path = new URI(scmUri).getPath().substring(1); String group = path.substring(0, path.lastIndexOf("/")); String name = (path.endsWith(".git") ? path.substring(0, path.length() - 4) : path).substring(group.length() + 1); - return group + "--" + name; + return group + groupSplit() + name; + } + + abstract String groupSplit(); + + public static class GitStatus { + public String url; + public String tag; + public String sha; + + public GitStatus() { + } + + public GitStatus(String url, String tag, String sha) { + this.url = url; + this.tag = tag; + this.sha = sha; + } + + @Override + public String toString() { + return "GitStatus{url='" + url + "', tag='" + tag + "', sha='" + sha + "'}"; + } } } diff --git a/java-components/build-request-processor/src/main/java/com/redhat/hacbs/container/analyser/deploy/git/GitHub.java b/java-components/build-request-processor/src/main/java/com/redhat/hacbs/container/analyser/deploy/git/GitHub.java index ded42ed24..92fe9dc0f 100644 --- a/java-components/build-request-processor/src/main/java/com/redhat/hacbs/container/analyser/deploy/git/GitHub.java +++ b/java-components/build-request-processor/src/main/java/com/redhat/hacbs/container/analyser/deploy/git/GitHub.java @@ -29,7 +29,7 @@ enum Type { private GHRepository repository; - public GitHub(String endpoint, String identity, String token) + public GitHub(String endpoint, String identity, String token, boolean ssl) throws IOException { if (isNotEmpty(token)) { github = new GitHubBuilder().withEndpoint(endpoint == null ? GITHUB_URL : endpoint) @@ -41,6 +41,7 @@ public GitHub(String endpoint, String identity, String token) } owner = identity; credentialsProvider = new UsernamePasswordCredentialsProvider(token, ""); + disableSSLVerification = ssl; switch (github.getUser(identity).getType()) { case "User" -> type = Type.USER; @@ -49,6 +50,11 @@ public GitHub(String endpoint, String identity, String token) Log.infof("Type %s", type); } + GitHub() { + owner = null; + github = null; + } + @Override public void create(String scmUri) throws IOException, URISyntaxException { @@ -56,6 +62,7 @@ public void create(String scmUri) if (type == Type.USER) { repository = github.getUser(owner).getRepository(name); if (repository == null) { + Log.infof("Creating repository with name %s", name); repository = github.createRepository(name) .wiki(false) .defaultBranch("main") @@ -67,6 +74,7 @@ public void create(String scmUri) } else { repository = github.getOrganization(owner).getRepository(name); if (repository == null) { + Log.infof("Creating repository with name %s", name); repository = github.getOrganization(owner).createRepository(name) .wiki(false) .defaultBranch("main") @@ -79,10 +87,15 @@ public void create(String scmUri) } @Override - public void add(Path path, String commit, String imageId) { + public GitStatus add(Path path, String commit, String imageId) { if (repository == null) { throw new RuntimeException("Call create first"); } - pushRepository(path, repository.getHttpTransportUrl(), commit, imageId); + return pushRepository(path, repository.getHttpTransportUrl(), commit, imageId); + } + + @Override + String groupSplit() { + return "--"; } } diff --git a/java-components/build-request-processor/src/main/java/com/redhat/hacbs/container/analyser/deploy/git/GitLab.java b/java-components/build-request-processor/src/main/java/com/redhat/hacbs/container/analyser/deploy/git/GitLab.java index 6624e55e4..206f0e475 100644 --- a/java-components/build-request-processor/src/main/java/com/redhat/hacbs/container/analyser/deploy/git/GitLab.java +++ b/java-components/build-request-processor/src/main/java/com/redhat/hacbs/container/analyser/deploy/git/GitLab.java @@ -20,10 +20,20 @@ public class GitLab extends Git { private Project project; - public GitLab(String endpoint, String identity, String token) { + public GitLab(String endpoint, String identity, String token, boolean ssl) { gitLabApi = new GitLabApi(endpoint, token); owner = identity; credentialsProvider = new UsernamePasswordCredentialsProvider("", token); + disableSSLVerification = ssl; + + if (disableSSLVerification) { + gitLabApi.setIgnoreCertificateErrors(true); + } + } + + GitLab() { + owner = null; + gitLabApi = null; } @Override @@ -38,6 +48,7 @@ public void create(String scmUri) Log.warnf("Repository %s already exists", name); } else { // Can't set public visibility after creation for some reason with this API. + Log.infof("Creating repository with name %s", name); project = gitLabApi.getProjectApi().createProject(name, null, null, @@ -55,11 +66,16 @@ public void create(String scmUri) } @Override - public void add(Path path, String commit, String imageId) + public GitStatus add(Path path, String commit, String imageId) throws IOException { if (project == null) { throw new RuntimeException("Call create first"); } - pushRepository(path, project.getHttpUrlToRepo(), commit, imageId); + return pushRepository(path, project.getHttpUrlToRepo(), commit, imageId); + } + + @Override + String groupSplit() { + return "-"; } } diff --git a/java-components/build-request-processor/src/main/java/com/redhat/hacbs/container/analyser/deploy/mavenrepository/MavenRepositoryDeployer.java b/java-components/build-request-processor/src/main/java/com/redhat/hacbs/container/analyser/deploy/mavenrepository/MavenRepositoryDeployer.java index 08d87c127..aa17eb66e 100644 --- a/java-components/build-request-processor/src/main/java/com/redhat/hacbs/container/analyser/deploy/mavenrepository/MavenRepositoryDeployer.java +++ b/java-components/build-request-processor/src/main/java/com/redhat/hacbs/container/analyser/deploy/mavenrepository/MavenRepositoryDeployer.java @@ -49,6 +49,8 @@ public MavenRepositoryDeployer(BootstrapMavenContext mvnCtx, String username, St this.system = mvnCtx.getRepositorySystem(); this.session = MavenRepositorySystemUtils.newSession(); + Log.infof("Maven credentials are username '%s' and repository '%s'", username, repository); + // https://maven.apache.org/resolver/third-party-integrations.html states a local repository manager should be added. session.setLocalRepositoryManager(system.newLocalRepositoryManager(session, new LocalRepository(artifacts.toFile()))); } diff --git a/java-components/build-request-processor/src/main/java/com/redhat/hacbs/container/verifier/DiffUtils.java b/java-components/build-request-processor/src/main/java/com/redhat/hacbs/container/verifier/DiffUtils.java index 8a3587edc..3ba20fb04 100644 --- a/java-components/build-request-processor/src/main/java/com/redhat/hacbs/container/verifier/DiffUtils.java +++ b/java-components/build-request-processor/src/main/java/com/redhat/hacbs/container/verifier/DiffUtils.java @@ -10,12 +10,10 @@ import java.util.stream.Collectors; import org.apache.commons.lang3.builder.DiffResult; -import org.jboss.logging.Logger; import com.redhat.hacbs.container.verifier.asm.AsmDiffable; public class DiffUtils { - private static final Logger Log = Logger.getLogger(DiffUtils.class); public record DiffResults(Set shared, Set added, Set deleted, Map> diffResults, diff --git a/java-components/build-request-processor/src/main/java/com/redhat/hacbs/container/verifier/MavenUtils.java b/java-components/build-request-processor/src/main/java/com/redhat/hacbs/container/verifier/MavenUtils.java index d381957e1..4b59231e1 100644 --- a/java-components/build-request-processor/src/main/java/com/redhat/hacbs/container/verifier/MavenUtils.java +++ b/java-components/build-request-processor/src/main/java/com/redhat/hacbs/container/verifier/MavenUtils.java @@ -28,10 +28,10 @@ import org.eclipse.aether.util.repository.AuthenticationBuilder; import org.eclipse.aether.util.repository.DefaultAuthenticationSelector; import org.eclipse.aether.util.repository.DefaultMirrorSelector; -import org.jboss.logging.Logger; + +import io.quarkus.logging.Log; public class MavenUtils { - private static final Logger Log = Logger.getLogger(MavenUtils.class); public static Settings newSettings(Path globalSettingsFile, Path settingsFile) { try { diff --git a/java-components/build-request-processor/src/main/java/com/redhat/hacbs/container/verifier/VerifyBuiltArtifactsCommand.java b/java-components/build-request-processor/src/main/java/com/redhat/hacbs/container/verifier/VerifyBuiltArtifactsCommand.java index e8dec461b..b5434f920 100644 --- a/java-components/build-request-processor/src/main/java/com/redhat/hacbs/container/verifier/VerifyBuiltArtifactsCommand.java +++ b/java-components/build-request-processor/src/main/java/com/redhat/hacbs/container/verifier/VerifyBuiltArtifactsCommand.java @@ -36,20 +36,19 @@ import org.eclipse.aether.repository.LocalRepository; import org.eclipse.aether.repository.RemoteRepository; import org.eclipse.aether.repository.RemoteRepository.Builder; -import org.jboss.logging.Logger; import com.redhat.hacbs.container.results.ResultsUpdater; import com.redhat.hacbs.container.verifier.asm.JarInfo; import io.quarkus.bootstrap.resolver.maven.BootstrapMavenContext; import io.quarkus.bootstrap.resolver.maven.BootstrapMavenException; +import io.quarkus.logging.Log; import picocli.CommandLine; import picocli.CommandLine.Command; import picocli.CommandLine.Option; @Command(name = "verify-built-artifacts") public class VerifyBuiltArtifactsCommand implements Callable { - private static final Logger Log = Logger.getLogger(VerifyBuiltArtifactsCommand.class); static class LocalOptions { @Option(required = true, names = { "-of", "--original-file" }) @@ -139,7 +138,6 @@ public Integer call() { session.setReadOnly(); } - Log.debugf("Deploy path: %s", options.mavenOptions.deployPath); var futureResults = new HashMap>>(); Files.walkFileTree(options.mavenOptions.deployPath, new SimpleFileVisitor<>() { @@ -191,7 +189,7 @@ public List call() throws Exception { } if (taskRunName != null) { var json = ResultsUpdater.MAPPER.writeValueAsString(verificationResults); - io.quarkus.logging.Log.infof("Writing verification results %s", json); + Log.infof("Writing verification results %s", json); resultsUpdater.get().updateResults(taskRunName, Map.of("VERIFICATION_RESULTS", json)); } return (failed && !reportOnly ? 1 : 0); diff --git a/java-components/build-request-processor/src/main/java/com/redhat/hacbs/container/verifier/asm/JarInfo.java b/java-components/build-request-processor/src/main/java/com/redhat/hacbs/container/verifier/asm/JarInfo.java index a90bd4ee4..94cf4fe3e 100644 --- a/java-components/build-request-processor/src/main/java/com/redhat/hacbs/container/verifier/asm/JarInfo.java +++ b/java-components/build-request-processor/src/main/java/com/redhat/hacbs/container/verifier/asm/JarInfo.java @@ -16,15 +16,14 @@ import java.util.regex.Pattern; import org.apache.commons.lang3.StringUtils; -import org.jboss.logging.Logger; import org.objectweb.asm.ClassReader; import org.objectweb.asm.tree.ClassNode; import com.redhat.hacbs.container.verifier.DiffUtils; -public record JarInfo(String name, Map classes) implements AsmDiffable { +import io.quarkus.logging.Log; - private static final Logger Log = Logger.getLogger(JarInfo.class); +public record JarInfo(String name, Map classes) implements AsmDiffable { // diffClass excluding name diff --git a/java-components/build-request-processor/src/test/java/com/redhat/hacbs/container/analyser/deploy/git/GitTest.java b/java-components/build-request-processor/src/test/java/com/redhat/hacbs/container/analyser/deploy/git/GitTest.java index 590210392..c36340777 100644 --- a/java-components/build-request-processor/src/test/java/com/redhat/hacbs/container/analyser/deploy/git/GitTest.java +++ b/java-components/build-request-processor/src/test/java/com/redhat/hacbs/container/analyser/deploy/git/GitTest.java @@ -15,6 +15,8 @@ import org.apache.commons.io.FileUtils; import org.eclipse.jgit.api.errors.GitAPIException; +import org.eclipse.jgit.lib.Ref; +import org.eclipse.jgit.lib.Repository; import org.eclipse.jgit.transport.TagOpt; import org.eclipse.jgit.transport.URIish; import org.junit.jupiter.api.BeforeEach; @@ -37,12 +39,15 @@ public void clearLogs() { @Test public void parseScmURI() throws URISyntaxException { - String result = Git.parseScmURI("https://github.com/apache/commons-codec.git"); + + String result = new GitHub().parseScmURI("https://github.com/apache/commons-codec.git"); assertEquals("apache--commons-codec", result); - result = Git.parseScmURI("https://gitlab.com/rnc/testRepo"); + result = new GitHub().parseScmURI("https://gitlab.com/rnc/testRepo"); assertEquals("rnc--testRepo", result); - result = Git.parseScmURI("file:///rnc/testRepo"); + result = new GitHub().parseScmURI("file:///rnc/testRepo"); assertEquals("rnc--testRepo", result); + result = new GitLab().parseScmURI("https://gitlab.com/rnc/testRepo"); + assertEquals("rnc-testRepo", result); } @Test @@ -51,6 +56,7 @@ public void testPush() Path initialRepo = Files.createTempDirectory("initial-repo"); Path testRepo = Files.createTempDirectory("test-repo"); String testRepoURI = "file://" + testRepo; + String imageID = "75ecd81c7a2b384151c990975eb1dd10"; try (var testRepository = org.eclipse.jgit.api.Git.init().setDirectory(testRepo.toFile()).call(); var initialRepository = org.eclipse.jgit.api.Git.init().setDirectory(initialRepo.toFile()).call()) { Path repoRoot = Paths.get(Objects.requireNonNull(getClass().getResource("/")).toURI()).getParent().getParent() @@ -71,14 +77,20 @@ public void create(String name) { } @Override - public void add(Path path, String commit, String imageId) { + public GitStatus add(Path path, String commit, String imageId) { + return null; + } + + @Override + public String groupSplit() { + return null; } }; - test.pushRepository( + Git.GitStatus tagResults = test.pushRepository( initialRepo, testRepoURI, "c396268fb90335bde5c9272b9a194c3d4302bf24", - "75ecd81c7a2b384151c990975eb1dd10"); + imageID); List logRecords = LogCollectingTestResource.current().getRecords(); @@ -90,19 +102,28 @@ public void add(Path path, String commit, String imageId) { .anyMatch( r -> LogCollectingTestResource.format(r).matches("Updating current origin of.*to " + testRepoURI))); - assertEquals(2, testRepository.tagList().call().size()); - assertTrue(testRepository.tagList().call().stream() - .anyMatch(r -> r.getName().equals("refs/tags/0.1-75ecd81c7a2b384151c990975eb1dd10"))); + List tags = testRepository.tagList().call(); + assertEquals(2, tags.size()); + assertTrue(tags.stream().anyMatch(r -> r.getName().equals("refs/tags/0.1-75ecd81c7a2b384151c990975eb1dd10"))); + + var found = tags.stream().filter(t -> Repository.shortenRefName(t.getName()).matches(tagResults.tag)).findFirst(); + assertTrue(found.isPresent()); + assertTrue(tagResults.url.contains(testRepoURI)); + assertTrue(tagResults.sha.matches(testRepository.getRepository() + .getRefDatabase() + .peel(found.get()) + .getPeeledObjectId() + .getName())); } } @Test public void testIdentity() throws IOException { - new GitHub(null, "cekit", null); + new GitHub(null, "cekit", null, true); List logRecords = LogCollectingTestResource.current().getRecords(); assertTrue(logRecords.stream().anyMatch(r -> LogCollectingTestResource.format(r).matches("Type ORGANISATION"))); LogCollectingTestResource.current().clear(); - new GitHub(null, "rnc", null); + new GitHub(null, "rnc", null, true); logRecords = LogCollectingTestResource.current().getRecords(); assertTrue(logRecords.stream().anyMatch(r -> LogCollectingTestResource.format(r).matches("Type USER"))); } diff --git a/java-components/cache/src/main/java/com/redhat/hacbs/artifactcache/services/CachePomScmLocator.java b/java-components/cache/src/main/java/com/redhat/hacbs/artifactcache/services/CachePomScmLocator.java index 62ce2cadb..d207ba078 100644 --- a/java-components/cache/src/main/java/com/redhat/hacbs/artifactcache/services/CachePomScmLocator.java +++ b/java-components/cache/src/main/java/com/redhat/hacbs/artifactcache/services/CachePomScmLocator.java @@ -9,14 +9,14 @@ import org.apache.maven.model.Model; import org.apache.maven.model.io.xpp3.MavenXpp3Reader; -import org.jboss.logging.Logger; import com.redhat.hacbs.recipies.scm.AbstractPomScmLocator; +import io.quarkus.logging.Log; + @ApplicationScoped public class CachePomScmLocator extends AbstractPomScmLocator { - private static final Logger log = Logger.getLogger(CachePomScmLocator.class); @Inject CacheFacade cache; @@ -27,7 +27,7 @@ protected AbstractPomScmLocator.PomClient createPomClient() { public Optional getPom(String group, String artifact, String version) { var response = cache.getArtifactFile("default", group.replace(".", "/"), artifact, version, artifact + "-" + version + ".pom", false); - if (!response.isPresent()) { + if (response.isEmpty()) { return Optional.empty(); } try { @@ -38,7 +38,7 @@ public Optional getPom(String group, String artifact, String version) { } } catch (Exception e) { - log.errorf(e, "Failed to get pom for %s:%s:%s", group, artifact, version); + Log.errorf(e, "Failed to get pom for %s:%s:%s", group, artifact, version); return Optional.empty(); } finally { try { diff --git a/java-components/management-console/src/main/resources/application.properties b/java-components/management-console/src/main/resources/application.properties index 6f2acf1be..c7c1300b3 100644 --- a/java-components/management-console/src/main/resources/application.properties +++ b/java-components/management-console/src/main/resources/application.properties @@ -33,4 +33,4 @@ quarkus.quinoa.build-dir=dist quarkus.quinoa.enable-spa-routing=true quarkus.resteasy-reactive.path=/api/ -%dev.sbom-discovery.enabled=false +%dev.sbom-discovery.enabled=true diff --git a/java-components/pom.xml b/java-components/pom.xml index d49a8c4d7..b524d8b4a 100644 --- a/java-components/pom.xml +++ b/java-components/pom.xml @@ -42,7 +42,7 @@ UTF-8 quarkus-bom io.quarkus.platform - 3.5.2 + 3.5.3 3.2.2 @@ -51,16 +51,17 @@ 6.9.2 3.24.2 - 1.12.592 + 1.12.598 3.0.5 8.4 - 1.316 + 1.318 ${github-api.version}.0 8.0.3 9.6 7.1.0 - 1.9.16 + + 1.9.18 redhat-appstudio-1 @@ -271,6 +272,14 @@ + + + io.quarkus + quarkus-jacoco + test + + + @@ -314,6 +323,27 @@ + + + + org.jacoco + jacoco-maven-plugin + 0.8.10 + + + default-prepare-agent + + prepare-agent + + + *QuarkusClassLoader + ${project.build.directory}/jacoco-quarkus.exec + true + + + + + diff --git a/java-components/resource-model/src/main/resources/crds/jvmbuildservice.io_dependencybuilds.yaml b/java-components/resource-model/src/main/resources/crds/jvmbuildservice.io_dependencybuilds.yaml index 6fa3f883c..19aa3b279 100644 --- a/java-components/resource-model/src/main/resources/crds/jvmbuildservice.io_dependencybuilds.yaml +++ b/java-components/resource-model/src/main/resources/crds/jvmbuildservice.io_dependencybuilds.yaml @@ -86,6 +86,16 @@ spec: items: type: string type: array + gitArchive: + description: The git archive source information + properties: + sha: + type: string + tag: + type: string + url: + type: string + type: object hermeticBuildImage: description: The hermetic build image produced by the build @@ -262,12 +272,20 @@ spec: contaminates: items: properties: + allowed: + type: boolean + buildId: + type: string contaminatedArtifacts: items: type: string type: array gav: type: string + rebuildAvailable: + type: boolean + source: + type: string type: object type: array deployedArtifacts: diff --git a/java-components/resource-model/src/main/resources/crds/jvmbuildservice.io_jbsconfigs.yaml b/java-components/resource-model/src/main/resources/crds/jvmbuildservice.io_jbsconfigs.yaml index dfadb6a02..edb422d83 100644 --- a/java-components/resource-model/src/main/resources/crds/jvmbuildservice.io_jbsconfigs.yaml +++ b/java-components/resource-model/src/main/resources/crds/jvmbuildservice.io_jbsconfigs.yaml @@ -89,6 +89,8 @@ spec: type: boolean gitSourceArchive: properties: + disableSSLVerification: + type: boolean identity: type: string url: diff --git a/pkg/apis/jvmbuildservice/v1alpha1/dependencybuild_types.go b/pkg/apis/jvmbuildservice/v1alpha1/dependencybuild_types.go index 46e3a338a..16514bfcc 100644 --- a/pkg/apis/jvmbuildservice/v1alpha1/dependencybuild_types.go +++ b/pkg/apis/jvmbuildservice/v1alpha1/dependencybuild_types.go @@ -28,7 +28,7 @@ type DependencyBuildStatus struct { Conditions []metav1.Condition `json:"conditions,omitempty"` State string `json:"state,omitempty"` Message string `json:"message,omitempty"` - Contaminants []Contaminant `json:"contaminates,omitempty"` + Contaminants []*Contaminant `json:"contaminates,omitempty"` // PotentialBuildRecipes additional recipes to try if the current recipe fails PotentialBuildRecipes []*BuildRecipe `json:"potentialBuildRecipes,omitempty"` CommitTime int64 `json:"commitTime,omitempty"` @@ -92,11 +92,19 @@ type BuildPipelineRunResults struct { Gavs []string `json:"gavs,omitempty"` // The hermetic build image produced by the build HermeticBuildImage string `json:"hermeticBuildImage,omitempty"` + // The git archive source information + GitArchive GitArchive `json:"gitArchive,omitempty"` // The Tekton results PipelineResults *PipelineResults `json:"pipelineResults,omitempty"` } +type GitArchive struct { + URL string `json:"url,omitempty"` + Tag string `json:"tag,omitempty"` + SHA string `json:"sha,omitempty"` +} + func (r *DependencyBuildStatus) GetBuildPipelineRun(pipeline string) *BuildAttempt { for i := range r.BuildAttempts { ba := r.BuildAttempts[i] @@ -115,6 +123,16 @@ func (r *DependencyBuildStatus) CurrentBuildAttempt() *BuildAttempt { return r.BuildAttempts[len(r.BuildAttempts)-1] } +func (r *DependencyBuildStatus) ProblemContaminates() []*Contaminant { + problemContaminates := []*Contaminant{} + for _, i := range r.Contaminants { + if !i.Allowed { + problemContaminates = append(problemContaminates, i) + } + } + return problemContaminates +} + type BuildRecipe struct { //Deprecated Pipeline string `json:"pipeline,omitempty"` @@ -136,6 +154,10 @@ type BuildRecipe struct { type Contaminant struct { GAV string `json:"gav,omitempty"` ContaminatedArtifacts []string `json:"contaminatedArtifacts,omitempty"` + BuildId string `json:"buildId,omitempty"` + Source string `json:"source,omitempty"` + Allowed bool `json:"allowed,omitempty"` + RebuildAvailable bool `json:"rebuildAvailable,omitempty"` } type AdditionalDownload struct { Uri string `json:"uri,omitempty"` diff --git a/pkg/apis/jvmbuildservice/v1alpha1/jbsconfig_types.go b/pkg/apis/jvmbuildservice/v1alpha1/jbsconfig_types.go index 2e44361ce..7fdb28631 100644 --- a/pkg/apis/jvmbuildservice/v1alpha1/jbsconfig_types.go +++ b/pkg/apis/jvmbuildservice/v1alpha1/jbsconfig_types.go @@ -111,8 +111,9 @@ type MavenDeployment struct { } type GitSourceArchive struct { - Identity string `json:"identity,omitempty"` - URL string `json:"url,omitempty"` + Identity string `json:"identity,omitempty"` + URL string `json:"url,omitempty"` + DisableSSLVerification bool `json:"disableSSLVerification,omitempty"` } type RelocationPatternElement struct { diff --git a/pkg/apis/jvmbuildservice/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/jvmbuildservice/v1alpha1/zz_generated.deepcopy.go index ec22b012b..f2651e6ad 100644 --- a/pkg/apis/jvmbuildservice/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/apis/jvmbuildservice/v1alpha1/zz_generated.deepcopy.go @@ -410,9 +410,13 @@ func (in *DependencyBuildStatus) DeepCopyInto(out *DependencyBuildStatus) { } if in.Contaminants != nil { in, out := &in.Contaminants, &out.Contaminants - *out = make([]Contaminant, len(*in)) + *out = make([]*Contaminant, len(*in)) for i := range *in { - (*in)[i].DeepCopyInto(&(*out)[i]) + if (*in)[i] != nil { + in, out := &(*in)[i], &(*out)[i] + *out = new(Contaminant) + (*in).DeepCopyInto(*out) + } } } if in.PotentialBuildRecipes != nil { diff --git a/pkg/reconciler/artifactbuild/artifactbuild.go b/pkg/reconciler/artifactbuild/artifactbuild.go index 0644a4fd4..91816bea9 100644 --- a/pkg/reconciler/artifactbuild/artifactbuild.go +++ b/pkg/reconciler/artifactbuild/artifactbuild.go @@ -36,14 +36,6 @@ const ( DependencyBuildIdLabel = "jvmbuildservice.io/dependencybuild-id" PipelineRunLabel = "jvmbuildservice.io/pipelinerun" - PipelineResultScmUrl = "scm-url" - PipelineResultScmTag = "scm-tag" - PipelineResultScmHash = "scm-hash" - PipelineResultScmType = "scm-type" - PipelineResultContextPath = "context" - PipelineResultMessage = "message" - PipelineResultPrivate = "private" - PreBuildTaskName = "pre-build" BuildTaskName = "build" HermeticBuildTaskName = "hermetic-build" @@ -54,12 +46,13 @@ const ( PipelineResultVerificationResult = "VERIFICATION_RESULTS" PipelineResultPassedVerification = "PASSED_VERIFICATION" //#nosec PipelineResultHermeticBuildImage = "HERMETIC_BUILD_IMAGE" + PipelineResultGitArchive = "GIT_ARCHIVE" PipelineResultGavs = "GAVS" RebuildAnnotation = "jvmbuildservice.io/rebuild" - //annotation that is applied after a rebuild, it will affect the dependencybuild behaviour + // RebuiltAnnotation annotation that is applied after a rebuild, it will affect the dependencybuild behaviour RebuiltAnnotation = "jvmbuildservice.io/rebuilt" - //if this annotation is present it will be deleted after a set time to live + // HoursToLive if this annotation is present it will be deleted after a set time to live //useful when doing builds that are being deployed to maven, and you don't want to accumulate them in the cluster HoursToLive = "jvmbuildservice.io/hours-to-live" ) @@ -383,15 +376,17 @@ func (r *ReconcileArtifactBuild) handleStateComplete(ctx context.Context, log lo if db.Status.State != v1alpha1.DependencyBuildStateContaminated { continue } - var newContaminates []v1alpha1.Contaminant - for _, contaminant := range db.Status.Contaminants { - if contaminant.GAV != abr.Spec.GAV { - newContaminates = append(newContaminates, contaminant) + allOk := true + for i := range db.Status.Contaminants { + contaminant := db.Status.Contaminants[i] + if contaminant.GAV == abr.Spec.GAV { + contaminant.RebuildAvailable = true + } else if !contaminant.Allowed && !contaminant.RebuildAvailable { + allOk = false } } - log.Info("Attempting to resolve contamination for dependencybuild", "dependencybuild", db.Name+"-"+db.Spec.ScmInfo.SCMURL+"-"+db.Spec.ScmInfo.Tag, "old", db.Status.Contaminants, "new", newContaminates) - db.Status.Contaminants = newContaminates - if len(db.Status.Contaminants) == 0 { + if allOk { + log.Info("Attempting to resolve contamination for dependencybuild as all contaminates are ready", "dependencybuild", db.Name+"-"+db.Spec.ScmInfo.SCMURL+"-"+db.Spec.ScmInfo.Tag) //TODO: we could have a situation where there are still some contamination, but not for artifacts that we care about //kick off the build again log.Info("Contamination resolved, moving to state new", "dependencybuild", db.Name+"-"+db.Spec.ScmInfo.SCMURL+"-"+db.Spec.ScmInfo.Tag) diff --git a/pkg/reconciler/artifactbuild/artifactbuild_test.go b/pkg/reconciler/artifactbuild/artifactbuild_test.go index 875ade879..275403e69 100644 --- a/pkg/reconciler/artifactbuild/artifactbuild_test.go +++ b/pkg/reconciler/artifactbuild/artifactbuild_test.go @@ -311,7 +311,7 @@ func TestStateBuilding(t *testing.T) { Labels: map[string]string{DependencyBuildIdLabel: util.HashString("")}, }, Spec: v1alpha1.DependencyBuildSpec{}, - Status: v1alpha1.DependencyBuildStatus{State: v1alpha1.DependencyBuildStateContaminated, Contaminants: []v1alpha1.Contaminant{{GAV: "com.test:test:1.0", ContaminatedArtifacts: []string{"a:b:1"}}}}, + Status: v1alpha1.DependencyBuildStatus{State: v1alpha1.DependencyBuildStateContaminated, Contaminants: []*v1alpha1.Contaminant{{GAV: "com.test:test:1.0", ContaminatedArtifacts: []string{"a:b:1"}}}}, } g.Expect(controllerutil.SetOwnerReference(abr, db, reconciler.scheme)) g.Expect(client.Create(ctx, db)) @@ -357,7 +357,7 @@ func TestStateCompleteFixingContamination(t *testing.T) { Labels: map[string]string{DependencyBuildIdLabel: util.HashString("")}, }, Spec: v1alpha1.DependencyBuildSpec{}, - Status: v1alpha1.DependencyBuildStatus{State: v1alpha1.DependencyBuildStateContaminated, Contaminants: []v1alpha1.Contaminant{{GAV: "com.test:test:1.0", ContaminatedArtifacts: []string{"a:b:1"}}}}, + Status: v1alpha1.DependencyBuildStatus{State: v1alpha1.DependencyBuildStateContaminated, Contaminants: []*v1alpha1.Contaminant{{GAV: "com.test:test:1.0", ContaminatedArtifacts: []string{"a:b:1"}}}}, } client, reconciler = setupClientAndReconciler(abr, contaiminated) } @@ -367,6 +367,12 @@ func TestStateCompleteFixingContamination(t *testing.T) { g.Expect(reconciler.Reconcile(ctx, reconcile.Request{NamespacedName: types.NamespacedName{Namespace: metav1.NamespaceDefault, Name: "test"}})) db := v1alpha1.DependencyBuild{} g.Expect(client.Get(ctx, types.NamespacedName{Namespace: metav1.NamespaceDefault, Name: contaminatedName}, &db)) - g.Expect(db.Status.Contaminants).Should(BeEmpty()) + allOk := true + for _, i := range db.Status.Contaminants { + if !i.Allowed && !i.RebuildAvailable { + allOk = false + } + } + g.Expect(allOk).Should(BeTrue()) }) } diff --git a/pkg/reconciler/dependencybuild/buildrecipeyaml.go b/pkg/reconciler/dependencybuild/buildrecipeyaml.go index 47f3eee82..bf05dcc3e 100644 --- a/pkg/reconciler/dependencybuild/buildrecipeyaml.go +++ b/pkg/reconciler/dependencybuild/buildrecipeyaml.go @@ -248,6 +248,7 @@ func createPipelineSpec(tool string, commitTime int64, jbsConfig *v1alpha12.JBSC {Name: PipelineResultImageDigest}, {Name: artifactbuild.PipelineResultPassedVerification}, {Name: artifactbuild.PipelineResultVerificationResult}, + {Name: artifactbuild.PipelineResultGitArchive}, }...), Steps: []pipelinev1beta1.Step{ { @@ -296,6 +297,7 @@ func createPipelineSpec(tool string, commitTime int64, jbsConfig *v1alpha12.JBSC {Name: PipelineResultImageDigest}, {Name: artifactbuild.PipelineResultPassedVerification}, {Name: artifactbuild.PipelineResultVerificationResult}, + {Name: artifactbuild.PipelineResultGitArchive}, }, Steps: []pipelinev1beta1.Step{ { @@ -666,6 +668,9 @@ func imageRegistryCommands(imageId string, recipe *v1alpha12.BuildRecipe, db *v1 if jbsConfig.Spec.GitSourceArchive.URL != "" { mavenArgs = append(mavenArgs, "--git-url="+jbsConfig.Spec.GitSourceArchive.URL) } + if jbsConfig.Spec.GitSourceArchive.DisableSSLVerification { + mavenArgs = append(mavenArgs, "--git-disable-ssl-verification") + } deployArgs = append(deployArgs, mavenArgs...) hermeticPreBuildImageArgs := []string{ diff --git a/pkg/reconciler/dependencybuild/dependencybuild.go b/pkg/reconciler/dependencybuild/dependencybuild.go index 04a44302a..10bb1adf6 100644 --- a/pkg/reconciler/dependencybuild/dependencybuild.go +++ b/pkg/reconciler/dependencybuild/dependencybuild.go @@ -355,11 +355,9 @@ func (r *ReconcileDependencyBuild) handleAnalyzeBuildPipelineRunReceived(ctx con if len(unmarshalled.Image) > 0 { log.Info(fmt.Sprintf("Found preexisting shared build with deployed GAVs %#v from image %#v", unmarshalled.Gavs, unmarshalled.Image)) db.Status.State = v1alpha1.DependencyBuildStateComplete - con, err := r.createRebuiltArtifacts(ctx, log, pr, &db, unmarshalled.Image, unmarshalled.Digest, unmarshalled.Gavs) + err := r.createRebuiltArtifacts(ctx, log, pr, &db, unmarshalled.Image, unmarshalled.Digest, unmarshalled.Gavs) if err != nil { return reconcile.Result{}, err - } else if !con { - return reconcile.Result{}, nil } } else { db.Status.State = v1alpha1.DependencyBuildStateSubmitBuild @@ -652,16 +650,17 @@ func (r *ReconcileDependencyBuild) handleBuildPipelineRunReceived(ctx context.Co var digest string var passedVerification bool var verificationResults string - var gavs []string var hermeticBuildImage string + var deployed []string + var gitArchive v1alpha1.GitArchive + for _, i := range pr.Status.Results { if i.Name == PipelineResultImage { image = i.Value.StringVal } else if i.Name == PipelineResultImageDigest { digest = i.Value.StringVal } else if i.Name == artifactbuild.PipelineResultContaminants { - - db.Status.Contaminants = []v1alpha1.Contaminant{} + db.Status.Contaminants = []*v1alpha1.Contaminant{} //unmarshal directly into the contaminants field err := json.Unmarshal([]byte(i.Value.StringVal), &db.Status.Contaminants) if err != nil { @@ -670,49 +669,45 @@ func (r *ReconcileDependencyBuild) handleBuildPipelineRunReceived(ctx context.Co } else if i.Name == artifactbuild.PipelineResultPassedVerification { parseBool, _ := strconv.ParseBool(i.Value.StringVal) passedVerification = !parseBool + } else if i.Name == artifactbuild.PipelineResultVerificationResult { + // Note: The TaskRun stores this as + // VERIFICATION_RESULTS {"commons-lang:commons-lang:jar:2.5":[]} + // But this is now stored as + // "verificationFailures": "{\"commons-lang:commons-lang:jar:2.5\":[]}" + verificationResults = i.Value.StringVal } else if i.Name == artifactbuild.PipelineResultGavs { + // TODO: What is the difference between this and PipelineResultDeployedResources? deployed := strings.Split(i.Value.StringVal, ",") db.Status.DeployedArtifacts = deployed - } else if i.Name == artifactbuild.PipelineResultVerificationResult { - verificationResults = i.Value.StringVal + } else if i.Name == artifactbuild.PipelineResultDeployedResources && len(i.Value.StringVal) > 0 { + //we need to create 'DeployedArtifact' resources for the objects that were deployed + deployed = strings.Split(i.Value.StringVal, ",") + } else if i.Name == artifactbuild.PipelineResultHermeticBuildImage { + hermeticBuildImage = i.Value.StringVal + } else if i.Name == artifactbuild.PipelineResultGitArchive { + err := json.Unmarshal([]byte(i.Value.StringVal), &gitArchive) + if err != nil { + return reconcile.Result{}, err + } } } + err = r.createRebuiltArtifacts(ctx, log, pr, db, image, digest, deployed) + if err != nil { + return reconcile.Result{}, err + } + run.Results = &v1alpha1.BuildPipelineRunResults{ Image: image, ImageDigest: digest, Verified: passedVerification, VerificationResults: verificationResults, - Gavs: gavs, + Gavs: deployed, + GitArchive: gitArchive, HermeticBuildImage: hermeticBuildImage, } - for _, i := range pr.Status.Results { - if i.Name == artifactbuild.PipelineResultContaminants { - - db.Status.Contaminants = []v1alpha1.Contaminant{} - //unmarshal directly into the contaminants field - err := json.Unmarshal([]byte(i.Value.StringVal), &db.Status.Contaminants) - if err != nil { - return reconcile.Result{}, err - } - } else if i.Name == artifactbuild.PipelineResultDeployedResources && len(i.Value.StringVal) > 0 { - //we need to create 'DeployedArtifact' resources for the objects that were deployed - deployed := strings.Split(i.Value.StringVal, ",") - - con, err := r.createRebuiltArtifacts(ctx, log, pr, db, image, digest, deployed) - - if err != nil { - return reconcile.Result{}, err - } else if !con { - return reconcile.Result{}, nil - } - } else if i.Name == artifactbuild.PipelineResultPassedVerification { - parseBool, _ := strconv.ParseBool(i.Value.StringVal) - db.Status.FailedVerification = !parseBool - } - } - - if len(db.Status.Contaminants) == 0 { + problemContaminates := db.Status.ProblemContaminates() + if len(problemContaminates) == 0 { db.Status.State = v1alpha1.DependencyBuildStateComplete } else { r.eventRecorder.Eventf(db, v1.EventTypeWarning, "BuildContaminated", "The DependencyBuild %s/%s was contaminated with community dependencies", db.Namespace, db.Name) @@ -720,7 +715,7 @@ func (r *ReconcileDependencyBuild) handleBuildPipelineRunReceived(ctx context.Co //most likely shaded in //we don't need to update the status here, it will be handled by the handleStateComplete method //even though there are contaminates they may not be in artifacts we care about - err := r.handleBuildCompletedWithContaminants(ctx, db, log) + err := r.handleBuildCompletedWithContaminants(ctx, db, log, problemContaminates) if err != nil { return reconcile.Result{}, err } @@ -815,7 +810,7 @@ func (r *ReconcileDependencyBuild) dependencyBuildForPipelineRun(ctx context.Con // no actual request for these artifacts. This can change if new artifacts are requested, so even when complete // we still need to verify that hte build is ok // this method will always update the status if it does not return an error -func (r *ReconcileDependencyBuild) handleBuildCompletedWithContaminants(ctx context.Context, db *v1alpha1.DependencyBuild, l logr.Logger) error { +func (r *ReconcileDependencyBuild) handleBuildCompletedWithContaminants(ctx context.Context, db *v1alpha1.DependencyBuild, l logr.Logger, problemContaminates []*v1alpha1.Contaminant) error { ownerGavs := map[string]bool{} db.Status.State = v1alpha1.DependencyBuildStateComplete @@ -838,7 +833,7 @@ func (r *ReconcileDependencyBuild) handleBuildCompletedWithContaminants(ctx cont ownerGavs[ab.Spec.GAV] = true } } - for _, contaminant := range db.Status.Contaminants { + for _, contaminant := range problemContaminates { for _, artifact := range contaminant.ContaminatedArtifacts { if ownerGavs[artifact] { db.Status.State = v1alpha1.DependencyBuildStateContaminated @@ -883,7 +878,14 @@ func (r *ReconcileDependencyBuild) handleBuildCompletedWithContaminants(ctx cont } func (r *ReconcileDependencyBuild) handleStateContaminated(ctx context.Context, db *v1alpha1.DependencyBuild) (reconcile.Result, error) { contaminants := db.Status.Contaminants - if len(contaminants) == 0 { + allOk := true + for _, i := range contaminants { + if !i.Allowed && !i.RebuildAvailable { + allOk = false + break + } + } + if allOk { //all fixed, just set the state back to building and try again //this is triggered when contaminants are removed by the ABR controller //setting it back to building should re-try the recipe that actually worked @@ -894,7 +896,7 @@ func (r *ReconcileDependencyBuild) handleStateContaminated(ctx context.Context, } func (r *ReconcileDependencyBuild) createRebuiltArtifacts(ctx context.Context, log logr.Logger, pr *pipelinev1beta1.PipelineRun, db *v1alpha1.DependencyBuild, - image string, digest string, deployed []string) (bool, error) { + image string, digest string, deployed []string) error { db.Status.DeployedArtifacts = deployed for _, i := range deployed { @@ -903,7 +905,7 @@ func (r *ReconcileDependencyBuild) createRebuiltArtifacts(ctx context.Context, l ra.Namespace = pr.Namespace ra.Name = artifactbuild.CreateABRName(i) if err := controllerutil.SetOwnerReference(db, &ra, r.scheme); err != nil { - return false, err + return err } ra.Spec.GAV = i ra.Spec.Image = image @@ -911,29 +913,24 @@ func (r *ReconcileDependencyBuild) createRebuiltArtifacts(ctx context.Context, l err := r.client.Create(ctx, &ra) if err != nil { if !errors.IsAlreadyExists(err) { - return false, err + return err } else { //if it already exists we update the image field - err := r.client.Get(ctx, types.NamespacedName{Namespace: ra.Namespace, Name: ra.Name}, &ra) + err = r.client.Get(ctx, types.NamespacedName{Namespace: ra.Namespace, Name: ra.Name}, &ra) if err != nil { - if !errors.IsNotFound(err) { - return false, err - } - //on not found we don't return the error - //no need to retry it would just result in an infinite loop - return false, nil + return err } ra.Spec.Image = image ra.Spec.Digest = digest log.Info(fmt.Sprintf("Updating existing RebuiltArtifact %s to reference image %s", ra.Name, ra.Spec.Image), "action", "UPDATE") err = r.client.Update(ctx, &ra) if err != nil { - return false, err + return err } } } } - return true, nil + return nil } func (r *ReconcileDependencyBuild) createLookupBuildInfoPipeline(ctx context.Context, log logr.Logger, db *v1alpha1.DependencyBuild, jbsConfig *v1alpha1.JBSConfig, additionalMemory int, systemConfig *v1alpha1.SystemConfig) (*pipelinev1beta1.PipelineSpec, error) { diff --git a/pkg/reconciler/dependencybuild/dependencybuild_test.go b/pkg/reconciler/dependencybuild/dependencybuild_test.go index 055ab6035..d27d3f8af 100644 --- a/pkg/reconciler/dependencybuild/dependencybuild_test.go +++ b/pkg/reconciler/dependencybuild/dependencybuild_test.go @@ -491,7 +491,7 @@ func TestStateBuilding(t *testing.T) { g.Expect(client.Update(ctx, pr)).Should(BeNil()) db := getBuild(client, g) g.Expect(controllerutil.SetOwnerReference(&ab, db, reconciler.scheme)).Should(BeNil()) - db.Status.Contaminants = []v1alpha1.Contaminant{{GAV: "com.acme:foo:1.0", ContaminatedArtifacts: []string{TestArtifact}}} + db.Status.Contaminants = []*v1alpha1.Contaminant{{GAV: "com.acme:foo:1.0", ContaminatedArtifacts: []string{TestArtifact}}} g.Expect(client.Update(ctx, db)) g.Expect(reconciler.Reconcile(ctx, reconcile.Request{NamespacedName: taskRunName})) g.Expect(reconciler.Reconcile(ctx, reconcile.Request{NamespacedName: buildName}))