Perform initial indexing and synchronization

.indexrc

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+RubyIndexer.configure do |config|
+  config.excluded_patterns.push("**/test/fixtures/**/*.rb")
+end

Gemfile

@@ -7,21 +7,23 @@ gemspec
 # sorbet-static is not available on Windows. We also skip Tapioca since it depends on sorbet-static-and-runtime
 NON_WINDOWS_PLATFORMS = [:ruby] # C Ruby (MRI), Rubinius or TruffleRuby, but NOT Windows
 
-gem "bundler", "~> 2.4.2"
-gem "debug", "~> 1.8", require: false
-gem "minitest", "~> 5.19"
-gem "minitest-reporters", "~> 1.6"
-gem "mocha", "~> 2.1"
-gem "rake", "~> 13.0"
-gem "rubocop", "~> 1.55"
-gem "rubocop-shopify", "~> 2.14", require: false
-gem "rubocop-minitest", "~> 0.31.0", require: false
-gem "rubocop-rake", "~> 0.6.0", require: false
-gem "rubocop-sorbet", "~> 0.7", require: false
-gem "sorbet-static-and-runtime", platforms: NON_WINDOWS_PLATFORMS
-gem "tapioca", "~> 0.11", require: false, platforms: NON_WINDOWS_PLATFORMS
-gem "rdoc", require: false
-gem "psych", "~> 5.1", require: false
+group :development do
+  gem "bundler", "~> 2.4.2"
+  gem "debug", "~> 1.8", require: false
+  gem "minitest", "~> 5.19"
+  gem "minitest-reporters", "~> 1.6"
+  gem "mocha", "~> 2.1"
+  gem "rake", "~> 13.0"
+  gem "rubocop", "~> 1.55"
+  gem "rubocop-shopify", "~> 2.14", require: false
+  gem "rubocop-minitest", "~> 0.31.0", require: false
+  gem "rubocop-rake", "~> 0.6.0", require: false
+  gem "rubocop-sorbet", "~> 0.7", require: false
+  gem "sorbet-static-and-runtime", platforms: NON_WINDOWS_PLATFORMS
+  gem "tapioca", "~> 0.11", require: false, platforms: NON_WINDOWS_PLATFORMS
+  gem "rdoc", require: false
+  gem "psych", "~> 5.1", require: false
 
-# The Rails documentation link only activates when railties is detected.
-gem "railties", "~> 7.0", require: false
+  # The Rails documentation link only activates when railties is detected.
+  gem "railties", "~> 7.0", require: false
+end

lib/ruby_indexer/lib/ruby_indexer/configuration.rb

@@ -6,15 +6,49 @@ class Configuration
     extend T::Sig
 
     sig { returns(T::Array[String]) }
-    attr_accessor :files_to_index
+    attr_accessor :excluded_gems
+
+    sig { returns(T::Array[String]) }
+    attr_accessor :excluded_patterns
+
+    sig { returns(T::Array[String]) }
+    attr_accessor :included_patterns
+
+    sig { returns(T::Array[String]) }
+    attr_accessor :included_gems
 
     sig { void }
     def initialize
+      development_only_dependencies = Bundler.definition.dependencies.filter_map do |dependency|
+        dependency.name if dependency.groups == [:development]
+      end
+
+      @excluded_gems = T.let(development_only_dependencies, T::Array[String])
+      @included_gems = T.let([], T::Array[String])
+      @excluded_patterns = T.let(["*_test.rb"], T::Array[String])
+      @included_patterns = T.let(["#{Dir.pwd}/**/*.rb"], T::Array[String])
+    end
+
+    sig { returns(T::Array[String]) }
+    def files_to_index
       files_to_index = $LOAD_PATH.flat_map { |p| Dir.glob("#{p}/**/*.rb", base: p) }
-      files_to_index.concat(Dir.glob("#{Dir.pwd}/**/*.rb"))
-      files_to_index.reject! { |path| path.end_with?("_test.rb") }
 
-      @files_to_index = T.let(files_to_index, T::Array[String])
+      @included_patterns.each do |pattern|
+        files_to_index.concat(Dir.glob(pattern, File::FNM_PATHNAME | File::FNM_EXTGLOB))
+      end
+
+      gem_paths = (@excluded_gems - @included_gems).filter_map do |gem_name|
+        Gem::Specification.find_by_name(gem_name).full_gem_path
+      rescue Gem::MissingSpecError
+        warn("Gem #{gem_name} is excluded in .indexrc, but that gem was not found in the bundle")
+      end
+
+      files_to_index.reject! do |path|
+        @excluded_patterns.any? { |pattern| File.fnmatch?(pattern, path, File::FNM_PATHNAME | File::FNM_EXTGLOB) } ||
+          gem_paths.any? { |gem_path| File.fnmatch?("#{gem_path}/**/*.rb", path) }
+      end
+      files_to_index.uniq!
+      files_to_index
     end
   end
 end

lib/ruby_indexer/lib/ruby_indexer/index.rb

@@ -70,11 +70,28 @@ def resolve(name, nesting)
       nil
     end
 
+    sig { returns(T::Boolean) }
+    def empty?
+      @entries.empty?
+    end
+
     sig { void }
     def clear
       @entries.clear
     end
 
+    sig { params(paths: T::Array[String]).void }
+    def index_all(paths: RubyIndexer.configuration.files_to_index)
+      paths.each { |path| index_single(path) }
+    end
+
+    sig { params(path: String, source: T.nilable(String)).void }
+    def index_single(path, source = nil)
+      content = source || File.read(path)
+      visitor = IndexVisitor.new(self, YARP.parse(content), path)
+      visitor.run
+    end
+
     class Entry
       extend T::Sig
 

lib/ruby_indexer/ruby_indexer.rb

@@ -9,6 +9,14 @@ module RubyIndexer
   class << self
     extend T::Sig
 
+    sig { void }
+    def load_configuration_file
+      return unless File.exist?(".indexrc")
+
+      config_file = File.read(".indexrc")
+      instance_eval(config_file)
+    end
+
     sig { params(block: T.proc.params(configuration: Configuration).void).void }
     def configure(&block)
       block.call(configuration)
@@ -18,19 +26,5 @@ def configure(&block)
     def configuration
       @configuration ||= T.let(Configuration.new, T.nilable(Configuration))
     end
-
-    sig { params(paths: T::Array[String]).returns(Index) }
-    def index(paths = configuration.files_to_index)
-      index = Index.new
-      paths.each { |path| index_single(index, path) }
-      index
-    end
-
-    sig { params(index: Index, path: String, source: T.nilable(String)).void }
-    def index_single(index, path, source = nil)
-      content = source || File.read(path)
-      visitor = IndexVisitor.new(index, YARP.parse(content), path)
-      visitor.run
-    end
   end
 end

lib/ruby_indexer/test/classes_and_modules_test.rb

@@ -212,7 +212,7 @@ class Foo; end
     private
 
     def index(source)
-      RubyIndexer.index_single(@index, "/fake/path/foo.rb", source)
+      @index.index_single("/fake/path/foo.rb", source)
     end
 
     def assert_entry(expected_name, type, expected_location)

lib/ruby_indexer/test/index_test.rb

@@ -14,11 +14,11 @@ def teardown
     end
 
     def test_deleting_one_entry_for_a_class
-      RubyIndexer.index_single(@index, "/fake/path/foo.rb", <<~RUBY)
+      @index.index_single("/fake/path/foo.rb", <<~RUBY)
         class Foo
         end
       RUBY
-      RubyIndexer.index_single(@index, "/fake/path/other_foo.rb", <<~RUBY)
+      @index.index_single("/fake/path/other_foo.rb", <<~RUBY)
         class Foo
         end
       RUBY
@@ -32,7 +32,7 @@ class Foo
     end
 
     def test_deleting_all_entries_for_a_class
-      RubyIndexer.index_single(@index, "/fake/path/foo.rb", <<~RUBY)
+      @index.index_single("/fake/path/foo.rb", <<~RUBY)
         class Foo
         end
       RUBY
@@ -46,7 +46,7 @@ class Foo
     end
 
     def test_index_resolve
-      RubyIndexer.index_single(@index, "/fake/path/foo.rb", <<~RUBY)
+      @index.index_single("/fake/path/foo.rb", <<~RUBY)
         class Bar; end
 
         module Foo
@@ -80,7 +80,7 @@ class Something
     end
 
     def test_accessing_with_colon_colon_prefix
-      RubyIndexer.index_single(@index, "/fake/path/foo.rb", <<~RUBY)
+      @index.index_single("/fake/path/foo.rb", <<~RUBY)
         class Bar; end
 
         module Foo

lib/ruby_indexer/test/ruby_indexer_test.rb

@@ -0,0 +1,23 @@
+# typed: true
+# frozen_string_literal: true
+
+require "test_helper"
+
+module RubyIndexer
+  class RubyIndexerTest < Minitest::Test
+    def test_load_configuration_executes_configure_block
+      RubyIndexer.load_configuration_file
+      files_to_index = RubyIndexer.configuration.files_to_index
+
+      assert(files_to_index.none? { |path| path.include?("test/fixtures") })
+      assert(files_to_index.none? { |path| path.include?("minitest-reporters") })
+    end
+
+    def test_paths_are_unique
+      RubyIndexer.load_configuration_file
+      files_to_index = RubyIndexer.configuration.files_to_index
+
+      assert_equal(files_to_index.uniq.length, files_to_index.length)
+    end
+  end
+end

lib/ruby_lsp/executor.rb

@@ -15,6 +15,7 @@ def initialize(store, message_queue)
       @store = store
       @test_library = T.let(DependencyDetector.detected_test_library, String)
       @message_queue = message_queue
+      @index = T.let(RubyIndexer::Index.new, RubyIndexer::Index)
     end
 
     sig { params(request: T::Hash[Symbol, T.untyped]).returns(Result) }
@@ -57,6 +58,14 @@ def run(request)
           warn(errored_extensions.map(&:backtraces).join("\n\n"))
         end
 
+        if @store.experimental_features
+          # The begin progress invocation happens during `initialize`, so that the notification is sent before we are
+          # stuck indexing files
+          RubyIndexer.load_configuration_file
+          @index.index_all
+          end_progress("indexing-progress")
+        end
+
         check_formatter_is_available
 
         warn("Ruby LSP is ready")
@@ -169,11 +178,35 @@ def run(request)
         completion(uri, request.dig(:params, :position))
       when "textDocument/definition"
         definition(uri, request.dig(:params, :position))
+      when "workspace/didChangeWatchedFiles"
+        did_change_watched_files(request.dig(:params, :changes))
       when "rubyLsp/textDocument/showSyntaxTree"
         show_syntax_tree(uri, request.dig(:params, :range))
       end
     end
 
+    sig { params(changes: T::Array[{ uri: String, type: Integer }]).returns(Object) }
+    def did_change_watched_files(changes)
+      changes.each do |change|
+        # File change events include folders, but we're only interested in files
+        uri = URI(change[:uri])
+        file_path = uri.to_standardized_path
+        next if file_path.nil? || File.directory?(file_path)
+
+        case change[:type]
+        when Constant::FileChangeType::CREATED
+          @index.index_single(file_path)
+        when Constant::FileChangeType::CHANGED
+          @index.delete(file_path)
+          @index.index_single(file_path)
+        when Constant::FileChangeType::DELETED
+          @index.delete(file_path)
+        end
+      end
+
+      VOID
+    end
+
     sig { params(uri: URI::Generic, range: T.nilable(Document::RangeShape)).returns({ ast: String }) }
     def show_syntax_tree(uri, range)
       { ast: Requests::ShowSyntaxTree.new(@store.get(uri), range).run }
@@ -436,6 +469,37 @@ def completion(uri, position)
       listener.response
     end
 
+    sig { params(id: String, title: String).void }
+    def begin_progress(id, title)
+      return unless @store.supports_progress
+
+      @message_queue << Request.new(
+        message: "window/workDoneProgress/create",
+        params: Interface::WorkDoneProgressCreateParams.new(token: id),
+      )
+
+      @message_queue << Notification.new(
+        message: "$/progress",
+        params: Interface::ProgressParams.new(
+          token: id,
+          value: Interface::WorkDoneProgressBegin.new(kind: "begin", title: title),
+        ),
+      )
+    end
+
+    sig { params(id: String).void }
+    def end_progress(id)
+      return unless @store.supports_progress
+
+      @message_queue << Notification.new(
+        message: "$/progress",
+        params: Interface::ProgressParams.new(
+          token: id,
+          value: Interface::WorkDoneProgressEnd.new(kind: "end"),
+        ),
+      )
+    end
+
     sig { params(options: T::Hash[Symbol, T.untyped]).returns(Interface::InitializeResult) }
     def initialize_request(options)
       @store.clear
@@ -449,6 +513,7 @@ def initialize_request(options)
         encodings.first
       end
 
+      @store.supports_progress = options.dig(:capabilities, :window, :workDoneProgress) || true
       formatter = options.dig(:initializationOptions, :formatter) || "auto"
       @store.formatter = if formatter == "auto"
         DependencyDetector.detected_formatter
@@ -457,9 +522,7 @@ def initialize_request(options)
       end
 
       configured_features = options.dig(:initializationOptions, :enabledFeatures)
-
-      # Uncomment the line below and use the variable to gate features behind the experimental flag
-      # experimental_features = options.dig(:initializationOptions, :experimentalFeaturesEnabled)
+      @store.experimental_features = options.dig(:initializationOptions, :experimentalFeaturesEnabled) || false
 
       enabled_features = case configured_features
       when Array
@@ -541,6 +604,37 @@ def initialize_request(options)
         )
       end
 
+      if @store.experimental_features
+        # Dynamically registered capabilities
+        file_watching_caps = options.dig(:capabilities, :workspace, :didChangeWatchedFiles)
+
+        # Not every client supports dynamic registration or file watching
+        if file_watching_caps&.dig(:dynamicRegistration) && file_watching_caps&.dig(:relativePatternSupport)
+          @message_queue << Request.new(
+            message: "client/registerCapability",
+            params: Interface::RegistrationParams.new(
+              registrations: [
+                # Register watching Ruby files
+                Interface::Registration.new(
+                  id: "workspace/didChangeWatchedFiles",
+                  method: "workspace/didChangeWatchedFiles",
+                  register_options: Interface::DidChangeWatchedFilesRegistrationOptions.new(
+                    watchers: [
+                      Interface::FileSystemWatcher.new(
+                        glob_pattern: "**/*.rb",
+                        kind: Constant::WatchKind::CREATE | Constant::WatchKind::CHANGE | Constant::WatchKind::DELETE,
+                      ),
+                    ],
+                  ),
+                ),
+              ],
+            ),
+          )
+        end
+
+        begin_progress("indexing-progress", "Ruby LSP: indexing files")
+      end
+
       Interface::InitializeResult.new(
         capabilities: Interface::ServerCapabilities.new(
           text_document_sync: Interface::TextDocumentSyncOptions.new(