From 97a75e0b271888159594747e419f7cdbf821aa94 Mon Sep 17 00:00:00 2001 From: Karol Lassak Date: Thu, 1 Aug 2024 08:00:09 +0200 Subject: [PATCH] [JENKINS-65493] Allow excluding by topics (#789) * Allow exluding by topics * Fix tests * Fix format --- .../GitHubSCMNavigator.java | 101 ++++++++---------- .../TopicsTrait/help-topicList.html | 4 +- .../GitHubSCMNavigatorTest.java | 26 +++++ ...based-on-topics-exclude-awesome-O8W78.json | 38 +++++++ ...-topics-octocat-exclude-awesome-O8W78.json | 38 +++++++ 5 files changed, 148 insertions(+), 59 deletions(-) create mode 100644 src/test/resources/api/mappings/mapping-search-repositories-based-on-topics-exclude-awesome-O8W78.json create mode 100644 src/test/resources/api/mappings/mapping-search-repositories-based-on-topics-octocat-exclude-awesome-O8W78.json diff --git a/src/main/java/org/jenkinsci/plugins/github_branch_source/GitHubSCMNavigator.java b/src/main/java/org/jenkinsci/plugins/github_branch_source/GitHubSCMNavigator.java index 13e4c3a17..26af55efc 100644 --- a/src/main/java/org/jenkinsci/plugins/github_branch_source/GitHubSCMNavigator.java +++ b/src/main/java/org/jenkinsci/plugins/github_branch_source/GitHubSCMNavigator.java @@ -51,6 +51,7 @@ import hudson.util.ListBoxModel; import java.io.FileNotFoundException; import java.io.IOException; +import java.io.PrintStream; import java.net.MalformedURLException; import java.time.Duration; import java.time.format.DateTimeParseException; @@ -99,13 +100,7 @@ import org.kohsuke.accmod.Restricted; import org.kohsuke.accmod.restrictions.DoNotUse; import org.kohsuke.accmod.restrictions.NoExternalUse; -import org.kohsuke.github.GHMyself; -import org.kohsuke.github.GHOrganization; -import org.kohsuke.github.GHRepository; -import org.kohsuke.github.GHRepositorySearchBuilder; -import org.kohsuke.github.GHUser; -import org.kohsuke.github.GitHub; -import org.kohsuke.github.HttpException; +import org.kohsuke.github.*; import org.kohsuke.stapler.AncestorInPath; import org.kohsuke.stapler.DataBoundConstructor; import org.kohsuke.stapler.DataBoundSetter; @@ -1038,16 +1033,9 @@ public void visitSources(SCMSourceObserver observer) throws IOException, Interru "Skipping repository %s because it is archived", repo.getName()))); - } else if (!gitHubSCMNavigatorContext.getTopics().isEmpty() - && !repo.listTopics().containsAll(gitHubSCMNavigatorContext.getTopics())) { + } else if (!topicMatches(gitHubSCMNavigatorContext, repo, listener.getLogger())) { // exclude repositories which are missing one or more of the specified topics witness.record(repo.getName(), false); - listener.getLogger() - .println(GitHubConsoleNote.create( - System.currentTimeMillis(), - String.format( - "Skipping repository %s because it is missing one or more of the following topics: '%s'", - repo.getName(), gitHubSCMNavigatorContext.getTopics()))); } else if (!repo.isPrivate() && gitHubSCMNavigatorContext.isExcludePublicRepositories()) { witness.record(repo.getName(), false); listener.getLogger() @@ -1127,17 +1115,9 @@ public void visitSources(SCMSourceObserver observer) throws IOException, Interru System.currentTimeMillis(), String.format( "Skipping repository %s because it is archived", repo.getName()))); - } else if (!gitHubSCMNavigatorContext.getTopics().isEmpty() - && !repo.listTopics().containsAll(gitHubSCMNavigatorContext.getTopics())) { + } else if (!topicMatches(gitHubSCMNavigatorContext, repo, listener.getLogger())) { // exclude repositories which are missing one or more of the specified topics witness.record(repo.getName(), false); - listener.getLogger() - .println(GitHubConsoleNote.create( - System.currentTimeMillis(), - String.format( - "Skipping repository %s because it is missing one or more of the following topics: '%s'", - repo.getName(), gitHubSCMNavigatorContext.getTopics()))); - } else if (!repo.isPrivate() && gitHubSCMNavigatorContext.isExcludePublicRepositories()) { witness.record(repo.getName(), false); listener.getLogger() @@ -1197,16 +1177,9 @@ public void visitSources(SCMSourceObserver observer) throws IOException, Interru String.format( "Skipping repository %s because it is archived", repo.getName()))); - } else if (!gitHubSCMNavigatorContext.getTopics().isEmpty() - && !repo.listTopics().containsAll(gitHubSCMNavigatorContext.getTopics())) { + } else if (!topicMatches(gitHubSCMNavigatorContext, repo, listener.getLogger())) { // exclude repositories which are missing one or more of the specified topics witness.record(repo.getName(), false); - listener.getLogger() - .println(GitHubConsoleNote.create( - System.currentTimeMillis(), - String.format( - "Skipping repository %s because it is missing one or more of the following topics: '%s'", - repo.getName(), gitHubSCMNavigatorContext.getTopics()))); } else if (gitHubSCMNavigatorContext.isExcludeForkedRepositories() && repo.getSource() != null) { witness.record(repo.getName(), false); @@ -1239,9 +1212,44 @@ public void visitSources(SCMSourceObserver observer) throws IOException, Interru } } + private boolean topicMatches(final GitHubSCMNavigatorContext context, final GHRepository repo, PrintStream logger) + throws IOException { + if (context.getTopics().isEmpty()) return true; + + final List topics = repo.listTopics(); + return context.getTopics().stream().allMatch(topic -> { + if (topic.startsWith("-")) { + boolean contains = topics.contains(topic.substring(1)); + if (contains) { + logger.println(GitHubConsoleNote.create( + System.currentTimeMillis(), + String.format( + "Skipping repository %s because it contains excluded topic: '%s'", + repo.getName(), topic))); + return false; + } + return true; + } else { + boolean contains = topics.contains(topic); + if (!contains) { + logger.println(GitHubConsoleNote.create( + System.currentTimeMillis(), + String.format( + "Skipping repository %s because it does not contain topic: '%s'", + repo.getName(), topic))); + return false; + } + return true; + } + }); + } + private Iterable searchRepositories(final GitHub github, final GitHubSCMNavigatorContext context) { final GHRepositorySearchBuilder ghRepositorySearchBuilder = github.searchRepositories(); - context.getTopics().forEach(ghRepositorySearchBuilder::topic); + context.getTopics().forEach(topic -> { + if (topic.startsWith("-")) ghRepositorySearchBuilder.q("-topic:" + topic.substring(1)); + else ghRepositorySearchBuilder.topic(topic); + }); ghRepositorySearchBuilder.org(getRepoOwner()); if (!context.isExcludeForkedRepositories()) { ghRepositorySearchBuilder.q("fork:true"); @@ -1330,16 +1338,9 @@ public void visitSource(String sourceName, SCMSourceObserver observer) throws IO "Skipping repository %s because it is archived", repo.getName()))); - } else if (!gitHubSCMNavigatorContext.getTopics().isEmpty() - && !repo.listTopics().containsAll(gitHubSCMNavigatorContext.getTopics())) { + } else if (!topicMatches(gitHubSCMNavigatorContext, repo, listener.getLogger())) { // exclude repositories which are missing one or more of the specified topics witness.record(repo.getName(), false); - listener.getLogger() - .println(GitHubConsoleNote.create( - System.currentTimeMillis(), - String.format( - "Skipping repository %s because it is missing one or more of the following topics: '%s'", - repo.getName(), gitHubSCMNavigatorContext.getTopics()))); } else if (!repo.isPrivate() && gitHubSCMNavigatorContext.isExcludePublicRepositories()) { witness.record(repo.getName(), false); listener.getLogger() @@ -1398,16 +1399,9 @@ public void visitSource(String sourceName, SCMSourceObserver observer) throws IO String.format( "Skipping repository %s because it is archived", repo.getName()))); - } else if (!gitHubSCMNavigatorContext.getTopics().isEmpty() - && !repo.listTopics().containsAll(gitHubSCMNavigatorContext.getTopics())) { + } else if (!topicMatches(gitHubSCMNavigatorContext, repo, listener.getLogger())) { // exclude repositories which are missing one or more of the specified topics witness.record(repo.getName(), false); - listener.getLogger() - .println(GitHubConsoleNote.create( - System.currentTimeMillis(), - String.format( - "Skipping repository %s because it is missing one or more of the following topics: '%s'", - repo.getName(), gitHubSCMNavigatorContext.getTopics()))); } else if (StringUtils.isNotBlank(gitHubSCMNavigatorContext.getTeamSlug()) && !isRepositoryVisibleToTeam(org, repo, gitHubSCMNavigatorContext.getTeamSlug())) { listener.getLogger() @@ -1476,16 +1470,9 @@ public void visitSource(String sourceName, SCMSourceObserver observer) throws IO String.format( "Skipping repository %s because it is archived", repo.getName()))); - } else if (!gitHubSCMNavigatorContext.getTopics().isEmpty() - && !repo.listTopics().containsAll(gitHubSCMNavigatorContext.getTopics())) { + } else if (!topicMatches(gitHubSCMNavigatorContext, repo, listener.getLogger())) { // exclude repositories which are missing one or more of the specified topics witness.record(repo.getName(), false); - listener.getLogger() - .println(GitHubConsoleNote.create( - System.currentTimeMillis(), - String.format( - "Skipping repository %s because it is missing one or more of the following topics: '%s'", - repo.getName(), gitHubSCMNavigatorContext.getTopics()))); } else if (!repo.isPrivate() && gitHubSCMNavigatorContext.isExcludePublicRepositories()) { witness.record(repo.getName(), false); listener.getLogger() diff --git a/src/main/resources/org/jenkinsci/plugins/github_branch_source/TopicsTrait/help-topicList.html b/src/main/resources/org/jenkinsci/plugins/github_branch_source/TopicsTrait/help-topicList.html index cc825f2b3..b3d6f95ad 100644 --- a/src/main/resources/org/jenkinsci/plugins/github_branch_source/TopicsTrait/help-topicList.html +++ b/src/main/resources/org/jenkinsci/plugins/github_branch_source/TopicsTrait/help-topicList.html @@ -1,3 +1,3 @@
-

Specify a comma-separated list of topics to filter for repositories that have all of them.

-
\ No newline at end of file +

Specify a comma-separated list of topics to filter for repositories that have (you can prefix topic with `-` to have it excluded) all of them.

+ diff --git a/src/test/java/org/jenkinsci/plugins/github_branch_source/GitHubSCMNavigatorTest.java b/src/test/java/org/jenkinsci/plugins/github_branch_source/GitHubSCMNavigatorTest.java index 957099b0a..4a501b6a0 100644 --- a/src/test/java/org/jenkinsci/plugins/github_branch_source/GitHubSCMNavigatorTest.java +++ b/src/test/java/org/jenkinsci/plugins/github_branch_source/GitHubSCMNavigatorTest.java @@ -227,6 +227,32 @@ public void fetchRepos_BelongingToAuthenticatedUser_FilteredByMultipleTopics() t assertEquals(projectNames, Collections.singleton("yolo-archived")); } + @Test + public void fetchRepos_BelongingToAuthenticatedUser_ExcludeByTopic() throws Exception { + setCredentials(Collections.singletonList(credentials)); + navigator = navigatorForRepoOwner("stephenc", credentials.getId()); + navigator.setTraits(Collections.singletonList(new TopicsTrait("-awesome"))); + final Set projectNames = new HashSet<>(); + final SCMSourceObserver observer = getObserver(projectNames); + + navigator.visitSources(observer); + + assertEquals(projectNames, Collections.singleton("yolo-archived")); + } + + @Test + public void fetchRepos_BelongingToAuthenticatedUser_ExcludeAndFilterByTopic() throws Exception { + setCredentials(Collections.singletonList(credentials)); + navigator = navigatorForRepoOwner("stephenc", credentials.getId()); + navigator.setTraits(Collections.singletonList(new TopicsTrait("-awesome,octocat"))); + final Set projectNames = new HashSet<>(); + final SCMSourceObserver observer = getObserver(projectNames); + + navigator.visitSources(observer); + + assertEquals(projectNames, Collections.singleton("yolo-archived")); + } + @Test public void fetchOneRepo_BelongingToAuthenticatedUser_ExcludingArchived() throws Exception { setCredentials(Collections.singletonList(credentials)); diff --git a/src/test/resources/api/mappings/mapping-search-repositories-based-on-topics-exclude-awesome-O8W78.json b/src/test/resources/api/mappings/mapping-search-repositories-based-on-topics-exclude-awesome-O8W78.json new file mode 100644 index 000000000..ecdf6da29 --- /dev/null +++ b/src/test/resources/api/mappings/mapping-search-repositories-based-on-topics-exclude-awesome-O8W78.json @@ -0,0 +1,38 @@ +{ + "request" : { + "urlPath" : "/search/repositories", + "method" : "GET", + "basicAuth": { + "username": "git-user", + "password": "git-secret" + }, + "queryParameters" : { + "q" : { + "equalTo" : "-topic:awesome org:stephenc fork:true" + } + } + }, + "response" : { + "status" : 200, + "bodyFileName" : "body-search-user-repos-O8W78.json", + "headers" : { + "Date" : "Fri, 31 Jan 2020 10:49:58 GMT", + "Content-Type" : "application/json; charset=utf-8", + "Server" : "GitHub.com", + "Status" : "200 OK", + "X-GitHub-Media-Type" : "github.v3; format=json", + "X-RateLimit-Limit" : "60", + "X-RateLimit-Remaining" : "54", + "X-RateLimit-Reset" : "1580469786", + "Access-Control-Expose-Headers" : "ETag, Link, Location, Retry-After, X-GitHub-OTP, X-RateLimit-Limit, X-RateLimit-Remaining, X-RateLimit-Reset, X-OAuth-Scopes, X-Accepted-OAuth-Scopes, X-Poll-Interval, X-GitHub-Media-Type", + "Access-Control-Allow-Origin" : "*", + "Strict-Transport-Security" : "max-age=31536000; includeSubdomains; preload", + "X-Frame-Options" : "deny", + "X-Content-Type-Options" : "nosniff", + "X-XSS-Protection" : "1; mode=block", + "Referrer-Policy" : "origin-when-cross-origin, strict-origin-when-cross-origin", + "Content-Security-Policy" : "default-src 'none'", + "X-GitHub-Request-Id" : "C1F9:C8CD:5FFC805:7325FB9:5E340656" + } + } +} diff --git a/src/test/resources/api/mappings/mapping-search-repositories-based-on-topics-octocat-exclude-awesome-O8W78.json b/src/test/resources/api/mappings/mapping-search-repositories-based-on-topics-octocat-exclude-awesome-O8W78.json new file mode 100644 index 000000000..a4d0cc243 --- /dev/null +++ b/src/test/resources/api/mappings/mapping-search-repositories-based-on-topics-octocat-exclude-awesome-O8W78.json @@ -0,0 +1,38 @@ +{ + "request" : { + "urlPath" : "/search/repositories", + "method" : "GET", + "basicAuth": { + "username": "git-user", + "password": "git-secret" + }, + "queryParameters" : { + "q" : { + "equalTo" : "-topic:awesome topic:octocat org:stephenc fork:true" + } + } + }, + "response" : { + "status" : 200, + "bodyFileName" : "body-search-user-repos-O8W78.json", + "headers" : { + "Date" : "Fri, 31 Jan 2020 10:49:58 GMT", + "Content-Type" : "application/json; charset=utf-8", + "Server" : "GitHub.com", + "Status" : "200 OK", + "X-GitHub-Media-Type" : "github.v3; format=json", + "X-RateLimit-Limit" : "60", + "X-RateLimit-Remaining" : "54", + "X-RateLimit-Reset" : "1580469786", + "Access-Control-Expose-Headers" : "ETag, Link, Location, Retry-After, X-GitHub-OTP, X-RateLimit-Limit, X-RateLimit-Remaining, X-RateLimit-Reset, X-OAuth-Scopes, X-Accepted-OAuth-Scopes, X-Poll-Interval, X-GitHub-Media-Type", + "Access-Control-Allow-Origin" : "*", + "Strict-Transport-Security" : "max-age=31536000; includeSubdomains; preload", + "X-Frame-Options" : "deny", + "X-Content-Type-Options" : "nosniff", + "X-XSS-Protection" : "1; mode=block", + "Referrer-Policy" : "origin-when-cross-origin, strict-origin-when-cross-origin", + "Content-Security-Policy" : "default-src 'none'", + "X-GitHub-Request-Id" : "C1F9:C8CD:5FFC805:7325FB9:5E340656" + } + } +}