SLI-1387 Avoid loading too many files in the file system

common/src/main/java/org/sonarlint/intellij/common/util/FileUtils.kt

@@ -21,6 +21,7 @@ package org.sonarlint.intellij.common.util
 
 import com.intellij.openapi.application.ApplicationManager
 import com.intellij.openapi.project.Project
+import com.intellij.openapi.project.ProjectCoreUtil
 import com.intellij.openapi.roots.GeneratedSourcesFilter.isGeneratedSourceByAnyFilter
 import com.intellij.openapi.roots.ProjectFileIndex
 import com.intellij.openapi.util.io.FileUtilRt
@@ -34,10 +35,13 @@ class FileUtils {
         fun isFileValidForSonarLint(file: VirtualFile, project: Project): Boolean {
             try {
                 val toSkip = computeReadActionSafely(file, project) {
-                    isGeneratedSourceByAnyFilter(file, project)
+                    project.isDisposed
+                        || (!ApplicationManager.getApplication().isUnitTestMode && !file.isDirectory && FileUtilRt.isTooLarge(file.length))
+                        || ProjectCoreUtil.isProjectOrWorkspaceFile(file)
                         || ProjectFileIndex.getInstance(project).isExcluded(file)
+                        || !ProjectFileIndex.getInstance(project).isInContent(file)
                         || ProjectFileIndex.getInstance(project).isInLibrary(file)
-                        || (!ApplicationManager.getApplication().isUnitTestMode && FileUtilRt.isTooLarge(file.length))
+                        || isGeneratedSourceByAnyFilter(file, project)
                 }
                 return false == toSkip
             } catch (e: Exception) {

src/main/java/org/sonarlint/intellij/SonarLintIntelliJClient.kt

@@ -672,6 +672,7 @@ object SonarLintIntelliJClient : SonarLintRpcClientDelegate {
     }
 
     override fun listFiles(configScopeId: String): List<ClientFileDto> {
+        val timeStart = System.currentTimeMillis()
         val listClientFiles = BackendService.findModule(configScopeId)?.let { module ->
             listModuleFiles(module, configScopeId)
         } ?: findProject(configScopeId)?.let { project ->
@@ -682,11 +683,15 @@ object SonarLintIntelliJClient : SonarLintRpcClientDelegate {
                     listProjectFiles.add(it)
                 }
             }
-
+            
             listProjectFiles
         }
         ?: emptyList()
-
+        val timeEnd = System.currentTimeMillis()
+        GlobalLogOutput.get().log(
+            "Listed ${listClientFiles.size} files for $configScopeId in ${(timeEnd - timeStart)} ms",
+            ClientLogOutput.Level.DEBUG
+        )
         return listClientFiles
     }
 
@@ -798,7 +803,7 @@ object SonarLintIntelliJClient : SonarLintRpcClientDelegate {
             if (module.isDisposed) {
                 return@forEach
             }
-            files.addAll(visitAndAddFiles(contentRoot, module))
+            files.addAll(visitAndAddFiles(contentRoot, module.project))
         }
         return files.toMutableSet()
     }

src/main/java/org/sonarlint/intellij/actions/SonarAnalyzeAllFilesAction.java

@@ -36,7 +36,6 @@
 import org.sonarlint.intellij.core.BackendService;
 
 import static org.sonarlint.intellij.common.util.SonarLintUtils.getService;
-import static org.sonarlint.intellij.util.ProjectUtils.hasFiles;
 import static org.sonarlint.intellij.util.ThreadUtilsKt.runOnPooledThread;
 
 public class SonarAnalyzeAllFilesAction extends AbstractSonarAction {
@@ -55,7 +54,7 @@ public SonarAnalyzeAllFilesAction(@Nullable String text, @Nullable String descri
   @Override
   protected boolean isEnabled(AnActionEvent e, Project project, AnalysisStatus status) {
     var backendIsAlive = getService(BackendService.class).isAlive();
-    return !status.isRunning() && hasFiles(project) && backendIsAlive;
+    return !status.isRunning() && backendIsAlive;
   }
 
   @Override

src/main/java/org/sonarlint/intellij/actions/SonarAnalyzeFilesAction.java

@@ -22,19 +22,13 @@
 import com.intellij.openapi.actionSystem.AnActionEvent;
 import com.intellij.openapi.actionSystem.CommonDataKeys;
 import com.intellij.openapi.project.Project;
-import com.intellij.openapi.project.ProjectCoreUtil;
-import com.intellij.openapi.vfs.VfsUtilCore;
-import com.intellij.openapi.vfs.VirtualFile;
-import com.intellij.openapi.vfs.VirtualFileVisitor;
-import java.util.LinkedHashSet;
-import java.util.Set;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 import javax.swing.Icon;
-import org.jetbrains.annotations.NotNull;
 import org.sonarlint.intellij.analysis.AnalysisStatus;
 import org.sonarlint.intellij.analysis.AnalysisSubmitter;
 import org.sonarlint.intellij.core.BackendService;
+import org.sonarlint.intellij.util.SonarLintAppUtils;
 
 import static org.sonarlint.intellij.common.util.SonarLintUtils.getService;
 import static org.sonarlint.intellij.util.ThreadUtilsKt.runOnPooledThread;
@@ -80,9 +74,8 @@ public void actionPerformed(AnActionEvent e) {
     var fileSet = Stream.of(files)
       .flatMap(f -> {
         if (f.isDirectory()) {
-          var visitor = new CollectFilesVisitor();
-          VfsUtilCore.visitChildrenRecursively(f, visitor);
-          return visitor.files.stream();
+          var listFiles = SonarLintAppUtils.visitAndAddFiles(f, project);
+          return listFiles.stream();
         } else {
           return Stream.of(f);
         }
@@ -92,20 +85,4 @@ public void actionPerformed(AnActionEvent e) {
     runOnPooledThread(project, () -> getService(project, AnalysisSubmitter.class).analyzeFilesOnUserAction(fileSet, e));
   }
 
-  private static class CollectFilesVisitor extends VirtualFileVisitor {
-    private final Set<VirtualFile> files = new LinkedHashSet<>();
-
-    public CollectFilesVisitor() {
-      super(VirtualFileVisitor.NO_FOLLOW_SYMLINKS);
-    }
-
-    @Override
-    public boolean visitFile(@NotNull VirtualFile file) {
-      var projectFile = ProjectCoreUtil.isProjectOrWorkspaceFile(file, file.getFileType());
-      if (!file.isDirectory() && !file.getFileType().isBinary() && !projectFile) {
-        files.add(file);
-      }
-      return !projectFile && !".git".equals(file.getName());
-    }
-  }
 }

src/main/java/org/sonarlint/intellij/core/BackendService.kt

@@ -35,6 +35,7 @@ import com.intellij.openapi.project.Project
 import com.intellij.openapi.project.ProjectManager
 import com.intellij.openapi.project.ProjectManagerListener
 import com.intellij.openapi.roots.TestSourcesFilter.isTestSources
+import com.intellij.openapi.util.io.FileUtilRt
 import com.intellij.openapi.vfs.VirtualFile
 import com.intellij.serviceContainer.NonInjectable
 import com.intellij.ui.jcef.JBCefApp
@@ -918,7 +919,11 @@ class BackendService : Disposable {
                         computeReadActionSafely(module.project) { isTestSources(it.virtualFile, module.project) },
                         it.getEncoding(module.project).toString(),
                         Paths.get(it.virtualFile.path),
-                        computeReadActionSafely(module.project) { getFileContent(it.virtualFile) },
+                        if (FileUtilRt.isTooLarge(it.virtualFile.length)) null else computeReadActionSafely(module.project) {
+                            getFileContent(
+                                it.virtualFile
+                            )
+                        },
                         forcedLanguage,
                         true
                     )

src/main/java/org/sonarlint/intellij/fs/DefaultVirtualFileSystemEventsHandler.kt

@@ -30,6 +30,7 @@ import com.intellij.openapi.vfs.newvfs.events.VFileEvent
 import com.intellij.serviceContainer.NonInjectable
 import java.util.concurrent.ExecutorService
 import java.util.concurrent.Executors
+import org.sonarlint.intellij.common.util.FileUtils
 import org.sonarlint.intellij.common.util.SonarLintUtils.getService
 import org.sonarlint.intellij.core.BackendService
 import org.sonarlint.intellij.util.SonarLintAppUtils.findModuleForFile
@@ -67,8 +68,9 @@ open class DefaultVirtualFileSystemEventsHandler @NonInjectable constructor(priv
         for (event in events) {
             // call event.file only once as it can be hurting performance
             val file = event.file ?: continue
-            if (ProjectCoreUtil.isProjectOrWorkspaceFile(file, file.fileType)) continue
+            if (ProjectCoreUtil.isProjectOrWorkspaceFile(file)) continue
             val fileModule = findModule(file, openProjects) ?: continue
+            if (!FileUtils.Companion.isFileValidForSonarLint(file, fileModule.project)) continue
             val fileInvolved = if (event is VFileCopyEvent) event.findCreatedFile() else file
             fileInvolved ?: continue
             val type = eventTypeConverter(event) ?: continue
@@ -83,7 +85,7 @@ open class DefaultVirtualFileSystemEventsHandler @NonInjectable constructor(priv
         fileModule: Module,
         type: ModuleFileEvent.Type,
     ): List<VirtualFileEvent> {
-        return visitAndAddFiles(file, fileModule).mapNotNull { VirtualFileEvent(type, it) }
+        return visitAndAddFiles(file, fileModule.project).mapNotNull { VirtualFileEvent(type, it) }
     }
 
     private fun findModule(file: VirtualFile?, openProjects: List<Project>): Module? {

src/main/java/org/sonarlint/intellij/util/ProjectUtils.java

@@ -21,7 +21,6 @@
 
 import com.intellij.openapi.application.ApplicationManager;
 import com.intellij.openapi.project.Project;
-import com.intellij.openapi.project.ProjectCoreUtil;
 import com.intellij.openapi.project.ProjectUtil;
 import com.intellij.openapi.roots.ProjectRootManager;
 import com.intellij.openapi.vfs.VirtualFile;
@@ -34,9 +33,8 @@
 import java.util.HashMap;
 import java.util.LinkedHashSet;
 import java.util.Map;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.function.Predicate;
 import javax.annotation.CheckForNull;
+import org.sonarlint.intellij.common.util.FileUtils;
 import org.sonarlint.intellij.finding.TextRangeMatcher;
 
 import static org.sonarlint.intellij.common.ui.ReadActionUtils.computeReadActionSafely;
@@ -45,36 +43,19 @@ public class ProjectUtils {
 
   public static Collection<VirtualFile> getAllFiles(Project project) {
     var fileSet = new LinkedHashSet<VirtualFile>();
-    iterateFilesToAnalyze(project, vFile -> {
-      fileSet.add(vFile);
-      // Continue collecting other files
-      return true;
-    });
-    return fileSet;
-  }
-
-  public static boolean hasFiles(Project project) {
-    var result = new AtomicBoolean(false);
-    iterateFilesToAnalyze(project, vFile -> {
-      result.set(true);
-      // No need to iterate other files/folders
-      return false;
-    });
-    return result.get();
-  }
-
-  private static void iterateFilesToAnalyze(Project project, Predicate<VirtualFile> fileProcessor) {
     var fileIndex = ProjectRootManager.getInstance(project).getFileIndex();
     fileIndex.iterateContent(vFile -> {
       if (project.isDisposed()) {
         return false;
       }
-      if (!vFile.isDirectory() && !ProjectCoreUtil.isProjectOrWorkspaceFile(vFile)) {
-        return fileProcessor.test(vFile);
+      if (!vFile.isDirectory() && vFile.isValid() && FileUtils.Companion.isFileValidForSonarLint(vFile, project)) {
+        fileSet.add(vFile);
+        return true;
       }
       // Continue iteration
       return true;
     });
+    return fileSet;
   }
 
   public static PsiFile toPsiFile(Project project, VirtualFile file) throws TextRangeMatcher.NoMatchException {

src/main/java/org/sonarlint/intellij/util/SonarLintAppUtils.java

@@ -157,18 +157,18 @@ static String getPathRelativeToModuleBaseDir(Module module, VirtualFile file) {
     return null;
   }
 
-  public static List<VirtualFile> visitAndAddFiles(VirtualFile file, Module module) {
+  public static List<VirtualFile> visitAndAddFiles(VirtualFile file, Project project) {
     var filesToAdd = new ArrayList<VirtualFile>();
     VfsUtilCore.visitChildrenRecursively(file, new VirtualFileVisitor<>(NO_FOLLOW_SYMLINKS) {
       @Override
       public boolean visitFile(VirtualFile file) {
-        if (!FileUtils.Companion.isFileValidForSonarLint(file, module.getProject())) {
+        if (!FileUtils.Companion.isFileValidForSonarLint(file, project)) {
           return false;
         }
         if (!file.isDirectory() && file.isValid()) {
           filesToAdd.add(file);
         }
-        return true;
+        return !".git".equals(file.getName());
       }
     });
     return filesToAdd;

src/test/java/org/sonarlint/intellij/ui/nodes/FileNodeTests.java

@@ -30,8 +30,8 @@
 import static org.mockito.Mockito.when;
 
 class FileNodeTests {
-  private VirtualFile file = mock(VirtualFile.class);
-  private FileNode node = new FileNode(file, false);
+  private final VirtualFile file = mock(VirtualFile.class);
+  private final FileNode node = new FileNode(file, false);
 
   @BeforeEach
   void setUp() {