Skip to content

Commit

Permalink
SLI-1638 Optimize loading of files
Browse files Browse the repository at this point in the history
  • Loading branch information
nquinquenel committed Oct 3, 2024
1 parent 676e567 commit eaaf0fa
Show file tree
Hide file tree
Showing 20 changed files with 126 additions and 99 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -33,12 +33,29 @@ import org.sonarlint.intellij.common.ui.SonarLintConsole
class FileUtils {

companion object {
// To be used with iterateContent, because it already skips ignored and excluded files
fun isFileValidForSonarLint(file: VirtualFile, project: Project): Boolean {
try {
val toSkip = computeReadActionSafely(file, project) {
(!ApplicationManager.getApplication().isUnitTestMode && !file.isDirectory && FileUtilRt.isTooLarge(file.length))
|| FileElement.isArchive(file)
|| ProjectCoreUtil.isProjectOrWorkspaceFile(file)
|| isGeneratedSourceByAnyFilter(file, project)
}

return false == toSkip
} catch (e: Exception) {
SonarLintConsole.get(project).error("Error while visiting a file, reason: " + e.message)
return false
}
}

// To be used when using iterating over all children
fun isFileValidForSonarLintWithExtensiveChecks(file: VirtualFile, project: Project): Boolean {
try {
val fileIndex = ProjectRootManager.getInstance(project).fileIndex
val toSkip = computeReadActionSafely(file, project) {
project.isDisposed
|| (!ApplicationManager.getApplication().isUnitTestMode && !file.isDirectory && FileUtilRt.isTooLarge(file.length))
(!ApplicationManager.getApplication().isUnitTestMode && !file.isDirectory && FileUtilRt.isTooLarge(file.length))
|| FileElement.isArchive(file)
|| !fileIndex.isInContent(file)
|| fileIndex.isInLibrarySource(file)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,14 +20,13 @@
package org.sonarlint.intellij.rider

import com.intellij.openapi.module.Module
import com.intellij.openapi.project.guessProjectDir
import com.intellij.openapi.vfs.VirtualFile
import com.jetbrains.rider.projectView.workspace.ProjectModelEntity
import com.jetbrains.rider.projectView.workspace.ProjectModelEntityVisitor
import com.jetbrains.rider.projectView.workspace.getVirtualFileAsContentRoot
import com.jetbrains.rider.projectView.workspace.isProjectFile
import org.sonarlint.intellij.common.analysis.FilesContributor
import org.sonarlint.intellij.common.util.FileUtils.Companion.isFileValidForSonarLint
import org.sonarlint.intellij.common.util.FileUtils.Companion.isFileValidForSonarLintWithExtensiveChecks

class RiderFilesContributor : FilesContributor {

Expand All @@ -37,11 +36,6 @@ class RiderFilesContributor : FilesContributor {
// List files in Solution
filesInContentRoots.addAll(listFilesInSolution(module))

// List files in project dir, sometimes not all files are found via the previous method
filesInContentRoots.addAll(
module.project.guessProjectDir()?.children?.filter { !it.isDirectory && it.isValid }?.toSet() ?: emptySet()
)

return filesInContentRoots
}

Expand All @@ -55,7 +49,7 @@ class RiderFilesContributor : FilesContributor {

if (entity.isProjectFile()) {
entity.getVirtualFileAsContentRoot()?.let {
if (!it.isDirectory && it.isValid && isFileValidForSonarLint(it, module.project)) {
if (!it.isDirectory && isFileValidForSonarLintWithExtensiveChecks(it, module.project)) {
filesInSolution.add(it)
}
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ import com.jetbrains.rider.projectView.workspace.isProjectFile
import git4idea.repo.GitRepository
import git4idea.repo.GitRepositoryManager
import org.sonarlint.intellij.common.ui.SonarLintConsole
import org.sonarlint.intellij.common.util.FileUtils.Companion.isFileValidForSonarLint
import org.sonarlint.intellij.common.util.FileUtils.Companion.isFileValidForSonarLintWithExtensiveChecks
import org.sonarlint.intellij.common.vcs.VcsRepo
import org.sonarlint.intellij.common.vcs.VcsRepoProvider
import org.sonarlint.intellij.git.GitRepo
Expand Down Expand Up @@ -78,7 +78,7 @@ class RiderSolutionGitRepoProvider : VcsRepoProvider {

if (entity.isProjectFile()) {
entity.getVirtualFileAsContentRoot()?.let {
if (!it.isDirectory && it.isValid && isFileValidForSonarLint(it, project)) {
if (!it.isDirectory && isFileValidForSonarLintWithExtensiveChecks(it, project)) {
repositoryManager.getRepositoryForFile(it)?.let { repo -> moduleRepositories.add(repo) }
}
}
Expand Down
50 changes: 22 additions & 28 deletions src/main/java/org/sonarlint/intellij/SonarLintIntelliJClient.kt
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,6 @@ import com.intellij.openapi.project.Project
import com.intellij.openapi.project.ProjectManager
import com.intellij.openapi.project.guessProjectDir
import com.intellij.openapi.project.modules
import com.intellij.openapi.roots.ModuleRootManager
import com.intellij.openapi.roots.TestSourcesFilter.isTestSources
import com.intellij.openapi.ui.MessageDialogBuilder
import com.intellij.openapi.util.io.FileUtilRt
Expand Down Expand Up @@ -70,6 +69,7 @@ import org.sonarlint.intellij.analysis.RunningAnalysesTracker
import org.sonarlint.intellij.common.analysis.FilesContributor
import org.sonarlint.intellij.common.ui.ReadActionUtils.Companion.computeReadActionSafely
import org.sonarlint.intellij.common.ui.SonarLintConsole
import org.sonarlint.intellij.common.util.FileUtils.Companion.isFileValidForSonarLintWithExtensiveChecks
import org.sonarlint.intellij.common.util.SonarLintUtils.getService
import org.sonarlint.intellij.common.util.SonarLintUtils.isRider
import org.sonarlint.intellij.common.vcs.VcsRepo
Expand Down Expand Up @@ -110,7 +110,7 @@ import org.sonarlint.intellij.util.GlobalLogOutput
import org.sonarlint.intellij.util.ProjectUtils.tryFindFile
import org.sonarlint.intellij.util.SonarLintAppUtils.findModuleForFile
import org.sonarlint.intellij.util.SonarLintAppUtils.getRelativePathForAnalysis
import org.sonarlint.intellij.util.SonarLintAppUtils.visitAndAddFiles
import org.sonarlint.intellij.util.SonarLintAppUtils.visitAndAddAllFilesForModule
import org.sonarlint.intellij.util.VirtualFileUtils
import org.sonarlint.intellij.util.VirtualFileUtils.getFileContent
import org.sonarlint.intellij.util.computeInEDT
Expand Down Expand Up @@ -674,17 +674,19 @@ object SonarLintIntelliJClient : SonarLintRpcClientDelegate {
override fun listFiles(configScopeId: String): List<ClientFileDto> {
val timeStart = System.currentTimeMillis()
val listClientFiles = BackendService.findModule(configScopeId)?.let { module ->
listModuleFiles(module, configScopeId)
computeOnPooledThread(module.project, "Listing All Module Files") { listModuleFiles(module, configScopeId) }
} ?: findProject(configScopeId)?.let { project ->
val listProjectFiles = listProjectFiles(project, configScopeId)
computeOnPooledThread(project, "Listing All Project Files") {
val listProjectFiles = listProjectFiles(project, configScopeId)

if (isRider()) {
computeRiderSharedConfiguration(project, configScopeId)?.let {
listProjectFiles.add(it)
if (isRider()) {
computeRiderSharedConfiguration(project, configScopeId)?.let {
listProjectFiles.add(it)
}
}

listProjectFiles
}

listProjectFiles
}
?: emptyList()
val timeEnd = System.currentTimeMillis()
Expand Down Expand Up @@ -720,7 +722,7 @@ object SonarLintIntelliJClient : SonarLintRpcClientDelegate {
}

private fun listModuleFiles(module: Module, configScopeId: String): MutableList<ClientFileDto> {
val filesInContentRoots = listFilesInContentRoots(module)
val filesInContentRoots = visitAndAddAllFilesForModule(module)

FilesContributor.EP_NAME.extensionList.forEach {
filesInContentRoots.addAll(it.listFiles(module))
Expand Down Expand Up @@ -764,6 +766,16 @@ object SonarLintIntelliJClient : SonarLintRpcClientDelegate {
}.toMutableList()
}

// useful for Rider where the files to find are not located in content roots
private fun listFilesInProjectBaseDir(project: Project): Set<VirtualFile> {
return project.guessProjectDir()?.children?.filter {
!it.isDirectory && isFileValidForSonarLintWithExtensiveChecks(
it,
project
)
}?.toSet() ?: return emptySet()
}

private fun toClientFileDto(
project: Project,
configScopeId: String,
Expand Down Expand Up @@ -795,24 +807,6 @@ object SonarLintIntelliJClient : SonarLintRpcClientDelegate {
}
}

private fun listFilesInContentRoots(
module: Module,
): MutableSet<VirtualFile> {
val files = mutableListOf<VirtualFile>()
ModuleRootManager.getInstance(module).contentRoots.forEach { contentRoot ->
if (module.isDisposed) {
return@forEach
}
files.addAll(visitAndAddFiles(contentRoot, module.project))
}
return files.toMutableSet()
}

// useful for Rider where the files to find are not located in content roots
private fun listFilesInProjectBaseDir(project: Project): Set<VirtualFile> {
return project.guessProjectDir()?.children?.filter { !it.isDirectory && it.isValid }?.toSet() ?: return emptySet()
}

override fun didChangeTaintVulnerabilities(
configurationScopeId: String, closedTaintVulnerabilityIds: Set<UUID>, addedTaintVulnerabilities: List<TaintVulnerabilityDto>,
updatedTaintVulnerabilities: List<TaintVulnerabilityDto>,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ import org.sonarlint.intellij.config.Settings
import org.sonarlint.intellij.config.project.ExclusionItem
import org.sonarlint.intellij.messages.ProjectConfigurationListener
import org.sonarlint.intellij.trigger.TriggerType
import org.sonarlint.intellij.util.SonarLintAppUtils
import org.sonarlint.intellij.util.SonarLintAppUtils.getRelativePathForAnalysis
import org.sonarlint.intellij.util.runOnPooledThread

class ExcludeFileAction : AbstractSonarAction {
Expand Down Expand Up @@ -77,7 +77,7 @@ class ExcludeFileAction : AbstractSonarAction {
}

private fun toExclusion(project: Project, virtualFile: VirtualFile): ExclusionItem? {
val relativeFilePath = SonarLintAppUtils.getRelativePathForAnalysis(project, virtualFile) ?: return null
val relativeFilePath = getRelativePathForAnalysis(project, virtualFile) ?: return null
return if (virtualFile.isDirectory) {
ExclusionItem(ExclusionItem.Type.DIRECTORY, relativeFilePath)
} else {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -28,9 +28,9 @@
import org.sonarlint.intellij.analysis.AnalysisStatus;
import org.sonarlint.intellij.analysis.AnalysisSubmitter;
import org.sonarlint.intellij.core.BackendService;
import org.sonarlint.intellij.util.SonarLintAppUtils;

import static org.sonarlint.intellij.common.util.SonarLintUtils.getService;
import static org.sonarlint.intellij.util.SonarLintAppUtils.visitAndAddAllChildren;
import static org.sonarlint.intellij.util.ThreadUtilsKt.runOnPooledThread;

public class SonarAnalyzeFilesAction extends AbstractSonarAction {
Expand Down Expand Up @@ -74,7 +74,7 @@ public void actionPerformed(AnActionEvent e) {
var fileSet = Stream.of(files)
.flatMap(f -> {
if (f.isDirectory()) {
var listFiles = SonarLintAppUtils.visitAndAddFiles(f, project);
var listFiles = visitAndAddAllChildren(f, project);
return listFiles.stream();
} else {
return Stream.of(f);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -51,11 +51,11 @@
import org.sonarlint.intellij.tasks.TaskRunnerKt;
import org.sonarlint.intellij.trigger.TriggerType;
import org.sonarlint.intellij.ui.SonarLintToolWindowFactory;
import org.sonarlint.intellij.util.SonarLintAppUtils;

import static org.sonarlint.intellij.common.util.SonarLintUtils.getService;
import static org.sonarlint.intellij.config.Settings.getGlobalSettings;
import static org.sonarlint.intellij.util.ProjectUtils.getAllFiles;
import static org.sonarlint.intellij.util.SonarLintAppUtils.findModuleForFile;
import static org.sonarlint.intellij.util.SonarLintAppUtils.visitAndAddAllFilesForProject;

@Service(Service.Level.PROJECT)
public final class AnalysisSubmitter {
Expand All @@ -81,7 +81,7 @@ public void cancelCurrentManualAnalysis() {
}

public void analyzeAllFiles() {
var allFiles = getAllFiles(project);
var allFiles = visitAndAddAllFilesForProject(project);
var callback = new ShowReportCallable(project);
var analysis = new Analysis(project, allFiles, TriggerType.ALL, callback);
TaskRunnerKt.startBackgroundableModalTask(project, ANALYSIS_TASK_TITLE, analysis::run);
Expand All @@ -102,7 +102,7 @@ public void autoAnalyzeOpenFilesForModule(TriggerType triggerType, @Nullable Mod

var openFiles = FileEditorManager.getInstance(project).getOpenFiles();
var filesToAnalyze = Arrays.stream(openFiles)
.filter(file -> module.equals(SonarLintAppUtils.findModuleForFile(file, project)))
.filter(file -> module.equals(findModuleForFile(file, project)))
.toList();

if (!filesToAnalyze.isEmpty()) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -47,12 +47,13 @@
import org.sonarlint.intellij.config.project.SonarLintProjectSettings;
import org.sonarlint.intellij.messages.GlobalConfigurationListener;
import org.sonarlint.intellij.messages.ProjectConfigurationListener;
import org.sonarlint.intellij.util.SonarLintAppUtils;
import org.sonarsource.sonarlint.core.client.utils.ClientFileExclusions;

import static org.sonarlint.intellij.common.ui.ReadActionUtils.computeReadActionSafely;
import static org.sonarlint.intellij.config.Settings.getGlobalSettings;
import static org.sonarlint.intellij.config.Settings.getSettingsFor;
import static org.sonarlint.intellij.util.SonarLintAppUtils.findModuleForFile;
import static org.sonarlint.intellij.util.SonarLintAppUtils.getRelativePathForAnalysis;

@Service(Service.Level.PROJECT)
public final class LocalFileExclusions {
Expand Down Expand Up @@ -107,7 +108,7 @@ public void applied(SonarLintGlobalSettings previousSettings, SonarLintGlobalSet
* Checks if a file is excluded from analysis based on locally configured exclusions.
*/
private ExcludeResult checkExclusionsFromSonarLintSettings(VirtualFile file, Module module) {
var relativePath = SonarLintAppUtils.getRelativePathForAnalysis(module, file);
var relativePath = getRelativePathForAnalysis(module, file);
if (relativePath == null) {
return ExcludeResult.excluded("Could not create a relative path");
}
Expand Down Expand Up @@ -159,7 +160,7 @@ public Map<Module, Collection<VirtualFile>> retainNonExcludedFilesByModules(Coll

private void checkExclusionsFileByFile(boolean forcedAnalysis, BiConsumer<VirtualFile, ExcludeResult> excludedFileHandler,
Map<Module, Collection<VirtualFile>> filesByModule, VirtualFile file) {
var module = SonarLintAppUtils.findModuleForFile(file, myProject);
var module = findModuleForFile(file, myProject);
// Handle this case first, so that later we are guaranteed module is not null
if (module == null) {
excludedFileHandler.accept(file, ExcludeResult.excluded("file is not part of any module in IntelliJ's project structure"));
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -40,13 +40,13 @@
import org.sonarlint.intellij.common.ui.SonarLintConsole;
import org.sonarlint.intellij.config.Settings;
import org.sonarlint.intellij.core.BackendService;
import org.sonarlint.intellij.util.SonarLintAppUtils;
import org.sonarlint.intellij.util.VirtualFileUtils;
import org.sonarsource.sonarlint.core.rpc.protocol.backend.analysis.AnalyzeFilesResponse;

import static org.sonarlint.intellij.common.ui.ReadActionUtils.computeReadActionSafely;
import static org.sonarlint.intellij.common.util.SonarLintUtils.getService;
import static org.sonarlint.intellij.util.ProgressUtils.waitForFuture;
import static org.sonarlint.intellij.util.SonarLintAppUtils.getRelativePathForAnalysis;

@Service(Service.Level.PROJECT)
public final class SonarLintAnalyzer {
Expand Down Expand Up @@ -148,7 +148,7 @@ private static List<URI> getInputFiles(Module module, Collection<VirtualFile> fi

@CheckForNull
private static URI createClientInputFile(Module module, VirtualFile virtualFile) {
var relativePath = SonarLintAppUtils.getRelativePathForAnalysis(module, virtualFile);
var relativePath = getRelativePathForAnalysis(module, virtualFile);
if (relativePath != null) {
return createURI(virtualFile);
}
Expand Down
4 changes: 2 additions & 2 deletions src/main/java/org/sonarlint/intellij/core/BackendService.kt
Original file line number Diff line number Diff line change
Expand Up @@ -77,7 +77,7 @@ import org.sonarlint.intellij.messages.GlobalConfigurationListener
import org.sonarlint.intellij.notifications.SonarLintProjectNotifications.Companion.projectLessNotification
import org.sonarlint.intellij.ui.UiUtils.Companion.runOnUiThread
import org.sonarlint.intellij.util.GlobalLogOutput
import org.sonarlint.intellij.util.SonarLintAppUtils
import org.sonarlint.intellij.util.SonarLintAppUtils.getRelativePathForAnalysis
import org.sonarlint.intellij.util.VirtualFileUtils
import org.sonarlint.intellij.util.VirtualFileUtils.getFileContent
import org.sonarlint.intellij.util.runOnPooledThread
Expand Down Expand Up @@ -917,7 +917,7 @@ class BackendService : Disposable {

event.filter { it.type != ModuleFileEvent.Type.DELETED }
.mapNotNull {
val relativePath = SonarLintAppUtils.getRelativePathForAnalysis(module, it.virtualFile) ?: return@mapNotNull null
val relativePath = getRelativePathForAnalysis(module, it.virtualFile) ?: return@mapNotNull null
val forcedLanguage = contributedLanguages[it.virtualFile]?.let { fl -> Language.valueOf(fl.name) }
val uri = VirtualFileUtils.toURI(it.virtualFile)
ClientFileDto(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ import org.sonarlint.intellij.common.util.FileUtils
import org.sonarlint.intellij.common.util.SonarLintUtils.getService
import org.sonarlint.intellij.core.BackendService
import org.sonarlint.intellij.util.SonarLintAppUtils.findModuleForFile
import org.sonarlint.intellij.util.SonarLintAppUtils.visitAndAddFiles
import org.sonarlint.intellij.util.SonarLintAppUtils.visitAndAddAllChildren
import org.sonarsource.sonarlint.plugin.api.module.file.ModuleFileEvent

open class DefaultVirtualFileSystemEventsHandler @NonInjectable constructor(private val executorService: ExecutorService) : VirtualFileSystemEventsHandler, Disposable {
Expand Down Expand Up @@ -70,7 +70,7 @@ open class DefaultVirtualFileSystemEventsHandler @NonInjectable constructor(priv
val file = event.file ?: continue
if (ProjectCoreUtil.isProjectOrWorkspaceFile(file)) continue
val fileModule = findModule(file, openProjects) ?: continue
if (!FileUtils.Companion.isFileValidForSonarLint(file, fileModule.project)) continue
if (!FileUtils.Companion.isFileValidForSonarLintWithExtensiveChecks(file, fileModule.project)) continue
val fileInvolved = if (event is VFileCopyEvent) event.findCreatedFile() else file
fileInvolved ?: continue
val type = eventTypeConverter(event) ?: continue
Expand All @@ -85,7 +85,7 @@ open class DefaultVirtualFileSystemEventsHandler @NonInjectable constructor(priv
fileModule: Module,
type: ModuleFileEvent.Type,
): List<VirtualFileEvent> {
return visitAndAddFiles(file, fileModule.project).mapNotNull { VirtualFileEvent(type, it) }
return visitAndAddAllChildren(file, fileModule.project).mapNotNull { VirtualFileEvent(type, it) }
}

private fun findModule(file: VirtualFile?, openProjects: List<Project>): Module? {
Expand Down
Loading

0 comments on commit eaaf0fa

Please sign in to comment.