SLI-1638 Optimize loading of files

common/src/main/java/org/sonarlint/intellij/common/util/FileUtils.kt

@@ -33,12 +33,29 @@ import org.sonarlint.intellij.common.ui.SonarLintConsole
 class FileUtils {
 
     companion object {
+        // To be used with iterateContent, because it already skips ignored and excluded files
         fun isFileValidForSonarLint(file: VirtualFile, project: Project): Boolean {
+            try {
+                val toSkip = computeReadActionSafely(file, project) {
+                    (!ApplicationManager.getApplication().isUnitTestMode && !file.isDirectory && FileUtilRt.isTooLarge(file.length))
+                        || FileElement.isArchive(file)
+                        || ProjectCoreUtil.isProjectOrWorkspaceFile(file)
+                        || isGeneratedSourceByAnyFilter(file, project)
+                }
+
+                return false == toSkip
+            } catch (e: Exception) {
+                SonarLintConsole.get(project).error("Error while visiting a file, reason: " + e.message)
+                return false
+            }
+        }
+
+        // To be used when using iterating over all children
+        fun isFileValidForSonarLintWithExtensiveChecks(file: VirtualFile, project: Project): Boolean {
             try {
                 val fileIndex = ProjectRootManager.getInstance(project).fileIndex
                 val toSkip = computeReadActionSafely(file, project) {
-                    project.isDisposed
-                        || (!ApplicationManager.getApplication().isUnitTestMode && !file.isDirectory && FileUtilRt.isTooLarge(file.length))
+                    (!ApplicationManager.getApplication().isUnitTestMode && !file.isDirectory && FileUtilRt.isTooLarge(file.length))
                         || FileElement.isArchive(file)
                         || !fileIndex.isInContent(file)
                         || fileIndex.isInLibrarySource(file)

rider/src/main/java/org/sonarlint/intellij/rider/RiderFilesContributor.kt

@@ -20,14 +20,13 @@
 package org.sonarlint.intellij.rider
 
 import com.intellij.openapi.module.Module
-import com.intellij.openapi.project.guessProjectDir
 import com.intellij.openapi.vfs.VirtualFile
 import com.jetbrains.rider.projectView.workspace.ProjectModelEntity
 import com.jetbrains.rider.projectView.workspace.ProjectModelEntityVisitor
 import com.jetbrains.rider.projectView.workspace.getVirtualFileAsContentRoot
 import com.jetbrains.rider.projectView.workspace.isProjectFile
 import org.sonarlint.intellij.common.analysis.FilesContributor
-import org.sonarlint.intellij.common.util.FileUtils.Companion.isFileValidForSonarLint
+import org.sonarlint.intellij.common.util.FileUtils.Companion.isFileValidForSonarLintWithExtensiveChecks
 
 class RiderFilesContributor : FilesContributor {
 
@@ -37,11 +36,6 @@ class RiderFilesContributor : FilesContributor {
         // List files in Solution
         filesInContentRoots.addAll(listFilesInSolution(module))
 
-        // List files in project dir, sometimes not all files are found via the previous method
-        filesInContentRoots.addAll(
-            module.project.guessProjectDir()?.children?.filter { !it.isDirectory && it.isValid }?.toSet() ?: emptySet()
-        )
-
         return filesInContentRoots
     }
 
@@ -55,7 +49,7 @@ class RiderFilesContributor : FilesContributor {
 
                 if (entity.isProjectFile()) {
                     entity.getVirtualFileAsContentRoot()?.let {
-                        if (!it.isDirectory && it.isValid && isFileValidForSonarLint(it, module.project)) {
+                        if (!it.isDirectory && isFileValidForSonarLintWithExtensiveChecks(it, module.project)) {
                             filesInSolution.add(it)
                         }
                     }

rider/src/main/java/org/sonarlint/intellij/rider/RiderSolutionGitRepoProvider.kt

@@ -28,7 +28,7 @@ import com.jetbrains.rider.projectView.workspace.isProjectFile
 import git4idea.repo.GitRepository
 import git4idea.repo.GitRepositoryManager
 import org.sonarlint.intellij.common.ui.SonarLintConsole
-import org.sonarlint.intellij.common.util.FileUtils.Companion.isFileValidForSonarLint
+import org.sonarlint.intellij.common.util.FileUtils.Companion.isFileValidForSonarLintWithExtensiveChecks
 import org.sonarlint.intellij.common.vcs.VcsRepo
 import org.sonarlint.intellij.common.vcs.VcsRepoProvider
 import org.sonarlint.intellij.git.GitRepo
@@ -78,7 +78,7 @@ class RiderSolutionGitRepoProvider : VcsRepoProvider {
 
                 if (entity.isProjectFile()) {
                     entity.getVirtualFileAsContentRoot()?.let {
-                        if (!it.isDirectory && it.isValid && isFileValidForSonarLint(it, project)) {
+                        if (!it.isDirectory && isFileValidForSonarLintWithExtensiveChecks(it, project)) {
                             repositoryManager.getRepositoryForFile(it)?.let { repo -> moduleRepositories.add(repo) }
                         }
                     }

src/main/java/org/sonarlint/intellij/SonarLintIntelliJClient.kt

@@ -34,7 +34,6 @@ import com.intellij.openapi.project.Project
 import com.intellij.openapi.project.ProjectManager
 import com.intellij.openapi.project.guessProjectDir
 import com.intellij.openapi.project.modules
-import com.intellij.openapi.roots.ModuleRootManager
 import com.intellij.openapi.roots.TestSourcesFilter.isTestSources
 import com.intellij.openapi.ui.MessageDialogBuilder
 import com.intellij.openapi.util.io.FileUtilRt
@@ -70,6 +69,7 @@ import org.sonarlint.intellij.analysis.RunningAnalysesTracker
 import org.sonarlint.intellij.common.analysis.FilesContributor
 import org.sonarlint.intellij.common.ui.ReadActionUtils.Companion.computeReadActionSafely
 import org.sonarlint.intellij.common.ui.SonarLintConsole
+import org.sonarlint.intellij.common.util.FileUtils.Companion.isFileValidForSonarLintWithExtensiveChecks
 import org.sonarlint.intellij.common.util.SonarLintUtils.getService
 import org.sonarlint.intellij.common.util.SonarLintUtils.isRider
 import org.sonarlint.intellij.common.vcs.VcsRepo
@@ -110,7 +110,7 @@ import org.sonarlint.intellij.util.GlobalLogOutput
 import org.sonarlint.intellij.util.ProjectUtils.tryFindFile
 import org.sonarlint.intellij.util.SonarLintAppUtils.findModuleForFile
 import org.sonarlint.intellij.util.SonarLintAppUtils.getRelativePathForAnalysis
-import org.sonarlint.intellij.util.SonarLintAppUtils.visitAndAddFiles
+import org.sonarlint.intellij.util.SonarLintAppUtils.visitAndAddAllFilesForModule
 import org.sonarlint.intellij.util.VirtualFileUtils
 import org.sonarlint.intellij.util.VirtualFileUtils.getFileContent
 import org.sonarlint.intellij.util.computeInEDT
@@ -674,17 +674,19 @@ object SonarLintIntelliJClient : SonarLintRpcClientDelegate {
     override fun listFiles(configScopeId: String): List<ClientFileDto> {
         val timeStart = System.currentTimeMillis()
         val listClientFiles = BackendService.findModule(configScopeId)?.let { module ->
-            listModuleFiles(module, configScopeId)
+            computeOnPooledThread(module.project, "Listing All Module Files") { listModuleFiles(module, configScopeId) }
         } ?: findProject(configScopeId)?.let { project ->
-            val listProjectFiles = listProjectFiles(project, configScopeId)
+            computeOnPooledThread(project, "Listing All Project Files") {
+                val listProjectFiles = listProjectFiles(project, configScopeId)
 
-            if (isRider()) {
-                computeRiderSharedConfiguration(project, configScopeId)?.let {
-                    listProjectFiles.add(it)
+                if (isRider()) {
+                    computeRiderSharedConfiguration(project, configScopeId)?.let {
+                        listProjectFiles.add(it)
+                    }
                 }
+
+                listProjectFiles
             }
-
-            listProjectFiles
         }
         ?: emptyList()
         val timeEnd = System.currentTimeMillis()
@@ -720,7 +722,7 @@ object SonarLintIntelliJClient : SonarLintRpcClientDelegate {
     }
 
     private fun listModuleFiles(module: Module, configScopeId: String): MutableList<ClientFileDto> {
-        val filesInContentRoots = listFilesInContentRoots(module)
+        val filesInContentRoots = visitAndAddAllFilesForModule(module)
 
         FilesContributor.EP_NAME.extensionList.forEach {
             filesInContentRoots.addAll(it.listFiles(module))
@@ -764,6 +766,16 @@ object SonarLintIntelliJClient : SonarLintRpcClientDelegate {
         }.toMutableList()
     }
 
+    // useful for Rider where the files to find are not located in content roots
+    private fun listFilesInProjectBaseDir(project: Project): Set<VirtualFile> {
+        return project.guessProjectDir()?.children?.filter {
+            !it.isDirectory && isFileValidForSonarLintWithExtensiveChecks(
+                it,
+                project
+            )
+        }?.toSet() ?: return emptySet()
+    }
+
     private fun toClientFileDto(
         project: Project,
         configScopeId: String,
@@ -795,24 +807,6 @@ object SonarLintIntelliJClient : SonarLintRpcClientDelegate {
         }
     }
 
-    private fun listFilesInContentRoots(
-        module: Module,
-    ): MutableSet<VirtualFile> {
-        val files = mutableListOf<VirtualFile>()
-        ModuleRootManager.getInstance(module).contentRoots.forEach { contentRoot ->
-            if (module.isDisposed) {
-                return@forEach
-            }
-            files.addAll(visitAndAddFiles(contentRoot, module.project))
-        }
-        return files.toMutableSet()
-    }
-
-    // useful for Rider where the files to find are not located in content roots
-    private fun listFilesInProjectBaseDir(project: Project): Set<VirtualFile> {
-        return project.guessProjectDir()?.children?.filter { !it.isDirectory && it.isValid }?.toSet() ?: return emptySet()
-    }
-
     override fun didChangeTaintVulnerabilities(
         configurationScopeId: String, closedTaintVulnerabilityIds: Set<UUID>, addedTaintVulnerabilities: List<TaintVulnerabilityDto>,
         updatedTaintVulnerabilities: List<TaintVulnerabilityDto>,

src/main/java/org/sonarlint/intellij/actions/ExcludeFileAction.kt

@@ -32,7 +32,7 @@ import org.sonarlint.intellij.config.Settings
 import org.sonarlint.intellij.config.project.ExclusionItem
 import org.sonarlint.intellij.messages.ProjectConfigurationListener
 import org.sonarlint.intellij.trigger.TriggerType
-import org.sonarlint.intellij.util.SonarLintAppUtils
+import org.sonarlint.intellij.util.SonarLintAppUtils.getRelativePathForAnalysis
 import org.sonarlint.intellij.util.runOnPooledThread
 
 class ExcludeFileAction : AbstractSonarAction {
@@ -77,7 +77,7 @@ class ExcludeFileAction : AbstractSonarAction {
     }
 
     private fun toExclusion(project: Project, virtualFile: VirtualFile): ExclusionItem? {
-        val relativeFilePath = SonarLintAppUtils.getRelativePathForAnalysis(project, virtualFile) ?: return null
+        val relativeFilePath = getRelativePathForAnalysis(project, virtualFile) ?: return null
         return if (virtualFile.isDirectory) {
             ExclusionItem(ExclusionItem.Type.DIRECTORY, relativeFilePath)
         } else {

src/main/java/org/sonarlint/intellij/actions/SonarAnalyzeFilesAction.java

@@ -28,9 +28,9 @@
 import org.sonarlint.intellij.analysis.AnalysisStatus;
 import org.sonarlint.intellij.analysis.AnalysisSubmitter;
 import org.sonarlint.intellij.core.BackendService;
-import org.sonarlint.intellij.util.SonarLintAppUtils;
 
 import static org.sonarlint.intellij.common.util.SonarLintUtils.getService;
+import static org.sonarlint.intellij.util.SonarLintAppUtils.visitAndAddAllChildren;
 import static org.sonarlint.intellij.util.ThreadUtilsKt.runOnPooledThread;
 
 public class SonarAnalyzeFilesAction extends AbstractSonarAction {
@@ -74,7 +74,7 @@ public void actionPerformed(AnActionEvent e) {
     var fileSet = Stream.of(files)
       .flatMap(f -> {
         if (f.isDirectory()) {
-          var listFiles = SonarLintAppUtils.visitAndAddFiles(f, project);
+          var listFiles = visitAndAddAllChildren(f, project);
           return listFiles.stream();
         } else {
           return Stream.of(f);

src/main/java/org/sonarlint/intellij/analysis/AnalysisSubmitter.java

@@ -51,11 +51,11 @@
 import org.sonarlint.intellij.tasks.TaskRunnerKt;
 import org.sonarlint.intellij.trigger.TriggerType;
 import org.sonarlint.intellij.ui.SonarLintToolWindowFactory;
-import org.sonarlint.intellij.util.SonarLintAppUtils;
 
 import static org.sonarlint.intellij.common.util.SonarLintUtils.getService;
 import static org.sonarlint.intellij.config.Settings.getGlobalSettings;
-import static org.sonarlint.intellij.util.ProjectUtils.getAllFiles;
+import static org.sonarlint.intellij.util.SonarLintAppUtils.findModuleForFile;
+import static org.sonarlint.intellij.util.SonarLintAppUtils.visitAndAddAllFilesForProject;
 
 @Service(Service.Level.PROJECT)
 public final class AnalysisSubmitter {
@@ -81,7 +81,7 @@ public void cancelCurrentManualAnalysis() {
   }
 
   public void analyzeAllFiles() {
-    var allFiles = getAllFiles(project);
+    var allFiles = visitAndAddAllFilesForProject(project);
     var callback = new ShowReportCallable(project);
     var analysis = new Analysis(project, allFiles, TriggerType.ALL, callback);
     TaskRunnerKt.startBackgroundableModalTask(project, ANALYSIS_TASK_TITLE, analysis::run);
@@ -102,7 +102,7 @@ public void autoAnalyzeOpenFilesForModule(TriggerType triggerType, @Nullable Mod
 
     var openFiles = FileEditorManager.getInstance(project).getOpenFiles();
     var filesToAnalyze = Arrays.stream(openFiles)
-      .filter(file -> module.equals(SonarLintAppUtils.findModuleForFile(file, project)))
+      .filter(file -> module.equals(findModuleForFile(file, project)))
       .toList();
 
     if (!filesToAnalyze.isEmpty()) {

src/main/java/org/sonarlint/intellij/analysis/LocalFileExclusions.java

@@ -47,12 +47,13 @@
 import org.sonarlint.intellij.config.project.SonarLintProjectSettings;
 import org.sonarlint.intellij.messages.GlobalConfigurationListener;
 import org.sonarlint.intellij.messages.ProjectConfigurationListener;
-import org.sonarlint.intellij.util.SonarLintAppUtils;
 import org.sonarsource.sonarlint.core.client.utils.ClientFileExclusions;
 
 import static org.sonarlint.intellij.common.ui.ReadActionUtils.computeReadActionSafely;
 import static org.sonarlint.intellij.config.Settings.getGlobalSettings;
 import static org.sonarlint.intellij.config.Settings.getSettingsFor;
+import static org.sonarlint.intellij.util.SonarLintAppUtils.findModuleForFile;
+import static org.sonarlint.intellij.util.SonarLintAppUtils.getRelativePathForAnalysis;
 
 @Service(Service.Level.PROJECT)
 public final class LocalFileExclusions {
@@ -107,7 +108,7 @@ public void applied(SonarLintGlobalSettings previousSettings, SonarLintGlobalSet
    * Checks if a file is excluded from analysis based on locally configured exclusions.
    */
   private ExcludeResult checkExclusionsFromSonarLintSettings(VirtualFile file, Module module) {
-    var relativePath = SonarLintAppUtils.getRelativePathForAnalysis(module, file);
+    var relativePath = getRelativePathForAnalysis(module, file);
     if (relativePath == null) {
       return ExcludeResult.excluded("Could not create a relative path");
     }
@@ -159,7 +160,7 @@ public Map<Module, Collection<VirtualFile>> retainNonExcludedFilesByModules(Coll
 
   private void checkExclusionsFileByFile(boolean forcedAnalysis, BiConsumer<VirtualFile, ExcludeResult> excludedFileHandler,
     Map<Module, Collection<VirtualFile>> filesByModule, VirtualFile file) {
-    var module = SonarLintAppUtils.findModuleForFile(file, myProject);
+    var module = findModuleForFile(file, myProject);
     // Handle this case first, so that later we are guaranteed module is not null
     if (module == null) {
       excludedFileHandler.accept(file, ExcludeResult.excluded("file is not part of any module in IntelliJ's project structure"));

src/main/java/org/sonarlint/intellij/analysis/SonarLintAnalyzer.java

@@ -40,13 +40,13 @@
 import org.sonarlint.intellij.common.ui.SonarLintConsole;
 import org.sonarlint.intellij.config.Settings;
 import org.sonarlint.intellij.core.BackendService;
-import org.sonarlint.intellij.util.SonarLintAppUtils;
 import org.sonarlint.intellij.util.VirtualFileUtils;
 import org.sonarsource.sonarlint.core.rpc.protocol.backend.analysis.AnalyzeFilesResponse;
 
 import static org.sonarlint.intellij.common.ui.ReadActionUtils.computeReadActionSafely;
 import static org.sonarlint.intellij.common.util.SonarLintUtils.getService;
 import static org.sonarlint.intellij.util.ProgressUtils.waitForFuture;
+import static org.sonarlint.intellij.util.SonarLintAppUtils.getRelativePathForAnalysis;
 
 @Service(Service.Level.PROJECT)
 public final class SonarLintAnalyzer {
@@ -148,7 +148,7 @@ private static List<URI> getInputFiles(Module module, Collection<VirtualFile> fi
 
   @CheckForNull
   private static URI createClientInputFile(Module module, VirtualFile virtualFile) {
-    var relativePath = SonarLintAppUtils.getRelativePathForAnalysis(module, virtualFile);
+    var relativePath = getRelativePathForAnalysis(module, virtualFile);
     if (relativePath != null) {
       return createURI(virtualFile);
     }

src/main/java/org/sonarlint/intellij/core/BackendService.kt

@@ -77,7 +77,7 @@ import org.sonarlint.intellij.messages.GlobalConfigurationListener
 import org.sonarlint.intellij.notifications.SonarLintProjectNotifications.Companion.projectLessNotification
 import org.sonarlint.intellij.ui.UiUtils.Companion.runOnUiThread
 import org.sonarlint.intellij.util.GlobalLogOutput
-import org.sonarlint.intellij.util.SonarLintAppUtils
+import org.sonarlint.intellij.util.SonarLintAppUtils.getRelativePathForAnalysis
 import org.sonarlint.intellij.util.VirtualFileUtils
 import org.sonarlint.intellij.util.VirtualFileUtils.getFileContent
 import org.sonarlint.intellij.util.runOnPooledThread
@@ -917,7 +917,7 @@ class BackendService : Disposable {
 
             event.filter { it.type != ModuleFileEvent.Type.DELETED }
                 .mapNotNull {
-                    val relativePath = SonarLintAppUtils.getRelativePathForAnalysis(module, it.virtualFile) ?: return@mapNotNull null
+                    val relativePath = getRelativePathForAnalysis(module, it.virtualFile) ?: return@mapNotNull null
                     val forcedLanguage = contributedLanguages[it.virtualFile]?.let { fl -> Language.valueOf(fl.name) }
                     val uri = VirtualFileUtils.toURI(it.virtualFile)
                     ClientFileDto(

src/main/java/org/sonarlint/intellij/fs/DefaultVirtualFileSystemEventsHandler.kt

@@ -34,7 +34,7 @@ import org.sonarlint.intellij.common.util.FileUtils
 import org.sonarlint.intellij.common.util.SonarLintUtils.getService
 import org.sonarlint.intellij.core.BackendService
 import org.sonarlint.intellij.util.SonarLintAppUtils.findModuleForFile
-import org.sonarlint.intellij.util.SonarLintAppUtils.visitAndAddFiles
+import org.sonarlint.intellij.util.SonarLintAppUtils.visitAndAddAllChildren
 import org.sonarsource.sonarlint.plugin.api.module.file.ModuleFileEvent
 
 open class DefaultVirtualFileSystemEventsHandler @NonInjectable constructor(private val executorService: ExecutorService) : VirtualFileSystemEventsHandler, Disposable {
@@ -70,7 +70,7 @@ open class DefaultVirtualFileSystemEventsHandler @NonInjectable constructor(priv
             val file = event.file ?: continue
             if (ProjectCoreUtil.isProjectOrWorkspaceFile(file)) continue
             val fileModule = findModule(file, openProjects) ?: continue
-            if (!FileUtils.Companion.isFileValidForSonarLint(file, fileModule.project)) continue
+            if (!FileUtils.Companion.isFileValidForSonarLintWithExtensiveChecks(file, fileModule.project)) continue
             val fileInvolved = if (event is VFileCopyEvent) event.findCreatedFile() else file
             fileInvolved ?: continue
             val type = eventTypeConverter(event) ?: continue
@@ -85,7 +85,7 @@ open class DefaultVirtualFileSystemEventsHandler @NonInjectable constructor(priv
         fileModule: Module,
         type: ModuleFileEvent.Type,
     ): List<VirtualFileEvent> {
-        return visitAndAddFiles(file, fileModule.project).mapNotNull { VirtualFileEvent(type, it) }
+        return visitAndAddAllChildren(file, fileModule.project).mapNotNull { VirtualFileEvent(type, it) }
     }
 
     private fun findModule(file: VirtualFile?, openProjects: List<Project>): Module? {