Read repositories on a per project level (#31)

* read repos * deps * use repo explorer * require cache key * include a default * spotless * convert to set * Add generated changelog entries * make lifecycle clearer * use streams * remove unneeded comment * working cache in need of fixing * fix tests * listen for refresh * tidy up code * fix log comment * markups --------- Co-authored-by: Finlay Williams <[email protected]> Co-authored-by: svc-changelog <[email protected]>
palantir · Oct 10, 2024 · 2974dc4 · 2974dc4
1 parent 82d2631
commit 2974dc4
Show file tree

Hide file tree

Showing 12 changed files with 239 additions and 74 deletions.
diff --git a/changelog/@unreleased/pr-31.v2.yml b/changelog/@unreleased/pr-31.v2.yml
@@ -0,0 +1,5 @@
+type: improvement
+improvement:
+  description: Read repositories on a per project level
+  links:
+  - https://github.com/palantir/gradle-consistent-versions-idea-plugin/pull/31
diff --git a/gradle-consistent-versions-idea-plugin/build.gradle b/gradle-consistent-versions-idea-plugin/build.gradle
@@ -25,6 +25,7 @@ sourceSets.main.java.srcDirs 'build/generated/sources/intellij-lexer/main/java'
 
 dependencies {
     implementation 'com.fasterxml.jackson.dataformat:jackson-dataformat-xml'
+    implementation 'com.fasterxml.jackson.datatype:jackson-datatype-guava'
     implementation 'com.github.ben-manes.caffeine:caffeine'
     annotationProcessor 'org.immutables:value'
     compileOnly 'org.immutables:value::annotations'

diff --git a/...ersions-idea-plugin/src/main/java/com/palantir/gradle/versions/intellij/ContentsUtil.java b/...ersions-idea-plugin/src/main/java/com/palantir/gradle/versions/intellij/ContentsUtil.java
@@ -34,10 +34,6 @@
 public final class ContentsUtil {
     private static final Logger log = LoggerFactory.getLogger(ContentsUtil.class);
 
-    private ContentsUtil() {
-        // Utility class; prevent instantiation
-    }
-
     public static Optional<String> fetchPageContents(URL pageUrl) {
         ProgressIndicator indicator = ProgressManager.getInstance().getProgressIndicator();
 
@@ -86,4 +82,6 @@ private static Callable<String> fetchContentTask(URL pageUrl, ProgressIndicator
             return result.toString();
         };
     }
+
+    private ContentsUtil() {}
 }
diff --git a/...ugin/src/main/java/com/palantir/gradle/versions/intellij/FolderCompletionContributor.java b/...ugin/src/main/java/com/palantir/gradle/versions/intellij/FolderCompletionContributor.java
@@ -22,19 +22,24 @@
 import com.intellij.codeInsight.completion.CompletionResultSet;
 import com.intellij.codeInsight.completion.CompletionType;
 import com.intellij.codeInsight.lookup.LookupElementBuilder;
+import com.intellij.openapi.externalSystem.service.notification.ExternalSystemProgressNotificationManager;
+import com.intellij.openapi.project.Project;
 import com.intellij.patterns.PlatformPatterns;
 import com.intellij.psi.PsiElement;
 import com.intellij.psi.tree.IElementType;
 import com.intellij.util.ProcessingContext;
 import com.palantir.gradle.versions.intellij.psi.VersionPropsTypes;
-import java.util.List;
 
 public class FolderCompletionContributor extends CompletionContributor {
 
-    private final GradleCacheExplorer gradleCacheExplorer =
-            new GradleCacheExplorer(List.of("https://repo.maven.apache.org/maven2/"));
+    private final GradleCacheExplorer gradleCacheExplorer = new GradleCacheExplorer();
+
+    private final RepositoryExplorer repositoryExplorer = new RepositoryExplorer();
 
     public FolderCompletionContributor() {
+        // We add listener at this stage so that we can invalidate the cache when the gradle project refreshed
+        ExternalSystemProgressNotificationManager.getInstance()
+                .addNotificationListener(new InvalidateCacheOnGradleProjectRefresh(gradleCacheExplorer));
         cacheCompletion(VersionPropsTypes.GROUP_PART);
         cacheCompletion(VersionPropsTypes.NAME_KEY);
         remoteCompletion(VersionPropsTypes.GROUP_PART);
@@ -47,13 +52,12 @@ private void remoteCompletion(IElementType elementType) {
             protected void addCompletions(
                     CompletionParameters parameters, ProcessingContext context, CompletionResultSet resultSet) {
 
-                List<String> repositories = List.of("https://repo.maven.apache.org/maven2/");
-
                 DependencyGroup group = DependencyGroup.groupFromParameters(parameters);
 
-                repositories.stream()
-                        .map(RepositoryExplorer::new)
-                        .flatMap(repositoryExplorer -> repositoryExplorer.getFolders(group).stream())
+                Project project = parameters.getOriginalFile().getProject();
+
+                RepositoryLoader.loadRepositories(project).stream()
+                        .flatMap(url -> repositoryExplorer.getGroupPartOrPackageName(group, url).stream())
                         .map(LookupElementBuilder::create)
                         .forEach(resultSet::addElement);
             }
@@ -68,8 +72,10 @@ protected void addCompletions(
 
                 DependencyGroup group = DependencyGroup.groupFromParameters(parameters);
 
-                gradleCacheExplorer.getCompletions(group).stream()
-                        .map(suggestion -> LookupElementBuilder.create(Folder.of(suggestion)))
+                Project project = parameters.getOriginalFile().getProject();
+
+                gradleCacheExplorer.getCompletions(RepositoryLoader.loadRepositories(project), group).stream()
+                        .map(suggestion -> LookupElementBuilder.create(GroupPartOrPackageName.of(suggestion)))
                         .forEach(resultSet::addElement);
             }
         });

diff --git a/...-idea-plugin/src/main/java/com/palantir/gradle/versions/intellij/GradleCacheExplorer.java b/...-idea-plugin/src/main/java/com/palantir/gradle/versions/intellij/GradleCacheExplorer.java
@@ -29,7 +29,6 @@
 import java.nio.file.Paths;
 import java.util.Collections;
 import java.util.HashSet;
-import java.util.List;
 import java.util.Optional;
 import java.util.Set;
 import java.util.concurrent.Callable;
@@ -47,21 +46,19 @@ public class GradleCacheExplorer {
     private final Cache<String, Set<String>> cache =
             Caffeine.newBuilder().expireAfterWrite(10, TimeUnit.MINUTES).build();
 
-    private final List<String> projectUrls;
-
-    public GradleCacheExplorer(List<String> projectUrls) {
-        this.projectUrls = projectUrls;
+    public final void invalidateCache() {
+        cache.invalidateAll();
     }
 
-    public final Set<String> getCompletions(DependencyGroup input) {
+    public final Set<String> getCompletions(Set<String> repoUrls, DependencyGroup input) {
         String parsedInput = String.join(".", input.parts());
         ProgressIndicator indicator = ProgressManager.getInstance().getProgressIndicator();
         if (indicator == null) {
             return Collections.emptySet();
         }
 
         try {
-            Callable<Set<String>> task = () -> extractStrings(indicator);
+            Callable<Set<String>> task = () -> extractStrings(repoUrls, indicator);
             Future<Set<String>> future = ApplicationManager.getApplication().executeOnPooledThread(task);
             Set<String> results =
                     com.intellij.openapi.application.ex.ApplicationUtil.runWithCheckCanceled(future::get, indicator);
@@ -82,7 +79,7 @@ public final Set<String> getCompletions(DependencyGroup input) {
         return Collections.emptySet();
     }
 
-    private Set<String> extractStrings(ProgressIndicator indicator) {
+    private Set<String> extractStrings(Set<String> repoUrls, ProgressIndicator indicator) {
         return cache.get("metadata", key -> {
             try (Stream<Path> allFolders = Files.list(Paths.get(GRADLE_CACHE_PATH))) {
 
@@ -99,8 +96,8 @@ private Set<String> extractStrings(ProgressIndicator indicator) {
                         .map(metadataFolder -> metadataFolder.resolve("resource-at-url.bin"))
                         .filter(Files::exists)
                         .flatMap(this::extractStringsFromBinFile)
-                        .filter(this::isValidResourceUrl)
-                        .map(this::extractGroupAndArtifactFromUrl)
+                        .filter(url -> isValidResourceUrl(repoUrls, url))
+                        .map(url -> extractGroupAndArtifactFromUrl(repoUrls, url))
                         .flatMap(Optional::stream)
                         .collect(Collectors.toSet());
             } catch (IOException e) {
@@ -116,8 +113,8 @@ private Set<String> extractStrings(ProgressIndicator indicator) {
         });
     }
 
-    final boolean isValidResourceUrl(String url) {
-        return projectUrls.stream().anyMatch(url::startsWith) && (url.endsWith(".pom") || url.endsWith(".jar"));
+    final boolean isValidResourceUrl(Set<String> repoUrls, String url) {
+        return repoUrls.stream().anyMatch(url::startsWith) && (url.endsWith(".pom") || url.endsWith(".jar"));
     }
 
     final Stream<String> extractStringsFromBinFile(Path binFile) {
@@ -160,8 +157,8 @@ final Stream<String> extractStringsFromBinFile(Path binFile) {
      * @return an {@link Optional} containing a string in the format "group:artifact" if extraction is successful,
      *         or {@link Optional#empty()} if no matching project URL is found or the URL does not have the expected structure.
      */
-    Optional<String> extractGroupAndArtifactFromUrl(String url) {
-        return projectUrls.stream().filter(url::startsWith).findFirst().flatMap(projectUrl -> {
+    Optional<String> extractGroupAndArtifactFromUrl(Set<String> repoUrls, String url) {
+        return repoUrls.stream().filter(url::startsWith).findFirst().flatMap(projectUrl -> {
             String mavenLayout = url.substring(projectUrl.length());
 
             int lastSlashIndex = mavenLayout.lastIndexOf('/');

diff --git a/...ntir/gradle/versions/intellij/Folder.java → ...ions/intellij/GroupPartOrPackageName.java b/...ntir/gradle/versions/intellij/Folder.java → ...ions/intellij/GroupPartOrPackageName.java
@@ -19,11 +19,11 @@
 import org.immutables.value.Value;
 
 @Value.Immutable
-public abstract class Folder {
+public abstract class GroupPartOrPackageName {
     protected abstract String name();
 
-    public static Folder of(String name) {
-        return ImmutableFolder.builder().name(name).build();
+    public static GroupPartOrPackageName of(String name) {
+        return ImmutableGroupPartOrPackageName.builder().name(name).build();
     }
 
     @Override

diff --git a/...ain/java/com/palantir/gradle/versions/intellij/InvalidateCacheOnGradleProjectRefresh.java b/...ain/java/com/palantir/gradle/versions/intellij/InvalidateCacheOnGradleProjectRefresh.java
@@ -0,0 +1,65 @@
+/*
+ * (c) Copyright 2024 Palantir Technologies Inc. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.palantir.gradle.versions.intellij;
+
+import com.intellij.openapi.externalSystem.model.task.ExternalSystemTaskId;
+import com.intellij.openapi.externalSystem.model.task.ExternalSystemTaskNotificationEvent;
+import com.intellij.openapi.externalSystem.model.task.ExternalSystemTaskNotificationListener;
+import com.intellij.openapi.externalSystem.model.task.ExternalSystemTaskType;
+import org.jetbrains.plugins.gradle.util.GradleConstants;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class InvalidateCacheOnGradleProjectRefresh implements ExternalSystemTaskNotificationListener {
+    private static final Logger log = LoggerFactory.getLogger(InvalidateCacheOnGradleProjectRefresh.class);
+
+    private final GradleCacheExplorer gradleCacheExplorer;
+
+    public InvalidateCacheOnGradleProjectRefresh(GradleCacheExplorer gradleCacheExplorer) {
+        this.gradleCacheExplorer = gradleCacheExplorer;
+    }
+
+    @Override
+    public final void onSuccess(ExternalSystemTaskId id) {
+        if (GradleConstants.SYSTEM_ID.equals(id.getProjectSystemId())
+                && id.getType() == ExternalSystemTaskType.RESOLVE_PROJECT) {
+            log.info("Gradle project refresh finished");
+            gradleCacheExplorer.invalidateCache();
+        }
+    }
+
+    @Override
+    public void onStart(ExternalSystemTaskId id, String workingDir) {}
+
+    @Override
+    public void onFailure(ExternalSystemTaskId id, Exception exception) {}
+
+    @Override
+    public void beforeCancel(ExternalSystemTaskId id) {}
+
+    @Override
+    public void onCancel(ExternalSystemTaskId id) {}
+
+    @Override
+    public void onStatusChange(ExternalSystemTaskNotificationEvent event) {}
+
+    @Override
+    public void onTaskOutput(ExternalSystemTaskId id, String text, boolean stdOut) {}
+
+    @Override
+    public void onEnd(ExternalSystemTaskId id) {}
+}
diff --git a/...s-idea-plugin/src/main/java/com/palantir/gradle/versions/intellij/RepositoryExplorer.java b/...s-idea-plugin/src/main/java/com/palantir/gradle/versions/intellij/RepositoryExplorer.java
@@ -21,11 +21,12 @@
 import com.github.benmanes.caffeine.cache.Caffeine;
 import java.net.MalformedURLException;
 import java.net.URL;
-import java.util.ArrayList;
 import java.util.Collections;
-import java.util.List;
+import java.util.HashSet;
 import java.util.Optional;
+import java.util.Set;
 import java.util.concurrent.TimeUnit;
+import org.immutables.value.Value;
 import org.jsoup.Jsoup;
 import org.jsoup.nodes.Document;
 import org.jsoup.nodes.Element;
@@ -36,44 +37,41 @@
 public class RepositoryExplorer {
     private static final Logger log = LoggerFactory.getLogger(RepositoryExplorer.class);
 
-    private final String baseUrl;
-    private static final Cache<DependencyGroup, List<Folder>> folderCache = Caffeine.newBuilder()
+    private final Cache<CacheKey, Set<GroupPartOrPackageName>> folderCache = Caffeine.newBuilder()
             .expireAfterWrite(10, TimeUnit.MINUTES)
             .maximumSize(100)
             .build();
 
-    public RepositoryExplorer(String baseUrl) {
-        this.baseUrl = baseUrl;
-    }
-
-    public final List<Folder> getFolders(DependencyGroup group) {
-        List<Folder> folders = folderCache.get(group, key -> {
-            List<Folder> loadedFolders = loadFolders(key);
+    public final Set<GroupPartOrPackageName> getGroupPartOrPackageName(DependencyGroup group, String url) {
+        CacheKey cacheKey = CacheKey.of(url, group);
+        Set<GroupPartOrPackageName> folders = folderCache.get(cacheKey, key -> {
+            Set<GroupPartOrPackageName> loadedFolders = loadFolders(key.group(), url);
             return loadedFolders.isEmpty() ? null : loadedFolders;
         });
 
-        return folders != null ? folders : Collections.emptyList();
+        return folders != null ? folders : Collections.emptySet();
     }
 
-    private List<Folder> loadFolders(DependencyGroup group) {
-        String urlString = baseUrl + group.asUrlString();
+    private Set<GroupPartOrPackageName> loadFolders(DependencyGroup group, String url) {
+        String urlString = url + group.asUrlString();
         Optional<String> content = fetchContent(urlString);
 
         if (content.isEmpty()) {
             log.debug("Page does not exist");
-            return new ArrayList<>();
+            return Collections.emptySet();
         }
 
         return fetchFoldersFromContent(content.get());
     }
 
-    public final List<DependencyVersion> getVersions(DependencyGroup group, DependencyName dependencyPackage) {
-        String urlString = baseUrl + group.asUrlString() + dependencyPackage.name() + "/maven-metadata.xml";
+    public final Set<DependencyVersion> getVersions(
+            DependencyGroup group, DependencyName dependencyPackage, String url) {
+        String urlString = url + group.asUrlString() + dependencyPackage.name() + "/maven-metadata.xml";
         Optional<String> content = fetchContent(urlString);
 
         if (content.isEmpty()) {
             log.debug("Empty metadata content received");
-            return new ArrayList<>();
+            return Collections.emptySet();
         }
 
         return parseVersionsFromContent(content.get());
@@ -89,23 +87,23 @@ private Optional<String> fetchContent(String urlString) {
         }
     }
 
-    private List<Folder> fetchFoldersFromContent(String contents) {
-        List<Folder> folders = new ArrayList<>();
+    private Set<GroupPartOrPackageName> fetchFoldersFromContent(String contents) {
+        Set<GroupPartOrPackageName> folders = new HashSet<>();
 
         Document doc = Jsoup.parse(contents);
         Elements links = doc.select("a[href]");
 
         for (Element link : links) {
             String href = link.attr("href");
             if (href.endsWith("/") && !href.contains(".")) {
-                folders.add(Folder.of(href.substring(0, href.length() - 1)));
+                folders.add(GroupPartOrPackageName.of(href.substring(0, href.length() - 1)));
             }
         }
         return folders;
     }
 
-    private List<DependencyVersion> parseVersionsFromContent(String content) {
-        List<DependencyVersion> versions = new ArrayList<>();
+    private Set<DependencyVersion> parseVersionsFromContent(String content) {
+        Set<DependencyVersion> versions = new HashSet<>();
         try {
             XmlMapper xmlMapper = new XmlMapper();
 
@@ -121,4 +119,15 @@ private List<DependencyVersion> parseVersionsFromContent(String content) {
         }
         return versions;
     }
+
+    @Value.Immutable
+    interface CacheKey {
+        String url();
+
+        DependencyGroup group();
+
+        static CacheKey of(String url, DependencyGroup group) {
+            return ImmutableCacheKey.builder().url(url).group(group).build();
+        }
+    }
 }