Use selected encoding for computing semantic tokens

lib/ruby_lsp/requests/semantic_highlighting.rb

@@ -40,10 +40,13 @@ def provider
         end
       end
 
-      sig { params(dispatcher: Prism::Dispatcher, range: T.nilable(T::Range[Integer])).void }
-      def initialize(dispatcher, range: nil)
+      sig { params(global_state: GlobalState, dispatcher: Prism::Dispatcher, range: T.nilable(T::Range[Integer])).void }
+      def initialize(global_state, dispatcher, range: nil)
         super()
-        @response_builder = T.let(ResponseBuilders::SemanticHighlighting.new, ResponseBuilders::SemanticHighlighting)
+        @response_builder = T.let(
+          ResponseBuilders::SemanticHighlighting.new(global_state.encoding),
+          ResponseBuilders::SemanticHighlighting,
+        )
         Listeners::SemanticHighlighting.new(dispatcher, @response_builder, range: range)
 
         Addon.addons.each do |addon|

lib/ruby_lsp/response_builders/semantic_highlighting.rb

@@ -55,26 +55,37 @@ class UndefinedTokenType < StandardError; end
 
       ResponseType = type_member { { fixed: Interface::SemanticTokens } }
 
-      sig { void }
-      def initialize
-        super
+      sig { params(encoding: Encoding).void }
+      def initialize(encoding)
+        super()
+        @encoding = encoding
         @stack = T.let([], T::Array[SemanticToken])
       end
 
       sig { params(location: Prism::Location, type: Symbol, modifiers: T::Array[Symbol]).void }
       def add_token(location, type, modifiers = [])
-        length = location.end_offset - location.start_offset
+        length = location.end_code_units_offset(@encoding) - location.start_code_units_offset(@encoding)
         modifiers_indices = modifiers.filter_map { |modifier| TOKEN_MODIFIERS[modifier] }
         @stack.push(
           SemanticToken.new(
-            location: location,
+            start_line: location.start_line,
+            start_code_unit_column: location.start_code_units_column(@encoding),
             length: length,
             type: T.must(TOKEN_TYPES[type]),
             modifier: modifiers_indices,
           ),
         )
       end
 
+      sig { params(location: Prism::Location).returns(T::Boolean) }
+      def last_token_matches?(location)
+        token = @stack.last
+        return false unless token
+
+        token.start_line == location.start_line &&
+          token.start_code_unit_column == location.start_code_units_column(@encoding)
+      end
+
       sig { returns(T.nilable(SemanticToken)) }
       def last
         @stack.last
@@ -88,8 +99,11 @@ def response
       class SemanticToken
         extend T::Sig
 
-        sig { returns(Prism::Location) }
-        attr_reader :location
+        sig { returns(Integer) }
+        attr_reader :start_line
+
+        sig { returns(Integer) }
+        attr_reader :start_code_unit_column
 
         sig { returns(Integer) }
         attr_reader :length
@@ -100,9 +114,18 @@ class SemanticToken
         sig { returns(T::Array[Integer]) }
         attr_reader :modifier
 
-        sig { params(location: Prism::Location, length: Integer, type: Integer, modifier: T::Array[Integer]).void }
-        def initialize(location:, length:, type:, modifier:)
-          @location = location
+        sig do
+          params(
+            start_line: Integer,
+            start_code_unit_column: Integer,
+            length: Integer,
+            type: Integer,
+            modifier: T::Array[Integer],
+          ).void
+        end
+        def initialize(start_line:, start_code_unit_column:, length:, type:, modifier:)
+          @start_line = start_line
+          @start_code_unit_column = start_code_unit_column
           @length = length
           @type = type
           @modifier = modifier
@@ -146,7 +169,7 @@ def encode(tokens)
             # Enumerable#sort_by is not deterministic when the compared values are equal.
             # When that happens, we need to use the index as a tie breaker to ensure
             # that the order of the tokens is always the same.
-            [token.location.start_line, token.location.start_column, index]
+            [token.start_line, token.start_code_unit_column, index]
           end
 
           delta = sorted_tokens.flat_map do |token|
@@ -167,8 +190,8 @@ def encode(tokens)
         # https://microsoft.github.io/language-server-protocol/specifications/specification-current/#textDocument_semanticTokens
         sig { params(token: SemanticToken).returns(T::Array[Integer]) }
         def compute_delta(token)
-          row = token.location.start_line - 1
-          column = token.location.start_column
+          row = token.start_line - 1
+          column = token.start_code_unit_column
 
           begin
             delta_line = row - @current_row

lib/ruby_lsp/server.rb

@@ -336,7 +336,7 @@ def run_combined_requests(message)
       document_link = Requests::DocumentLink.new(uri, document.comments, dispatcher)
       code_lens = Requests::CodeLens.new(@global_state, uri, dispatcher)
 
-      semantic_highlighting = Requests::SemanticHighlighting.new(dispatcher)
+      semantic_highlighting = Requests::SemanticHighlighting.new(@global_state, dispatcher)
       dispatcher.dispatch(document.tree)
 
       # Store all responses retrieve in this round of visits in the cache and then return the response for the request
@@ -368,7 +368,7 @@ def text_document_semantic_tokens_range(message)
       end_line = range.dig(:end, :line)
 
       dispatcher = Prism::Dispatcher.new
-      request = Requests::SemanticHighlighting.new(dispatcher, range: start_line..end_line)
+      request = Requests::SemanticHighlighting.new(@global_state, dispatcher, range: start_line..end_line)
       dispatcher.visit(document.tree)
 
       response = request.perform

test/expectations/semantic_highlighting/multibyte_characters.exp.json

@@ -0,0 +1,74 @@
+{
+  "result": [
+    {
+      "delta_line": 1,
+      "delta_start_char": 7,
+      "length": 3,
+      "token_type": 0,
+      "token_modifiers": 1
+    },
+    {
+      "delta_line": 0,
+      "delta_start_char": 0,
+      "length": 3,
+      "token_type": 0,
+      "token_modifiers": 0
+    },
+    {
+      "delta_line": 1,
+      "delta_start_char": 8,
+      "length": 3,
+      "token_type": 2,
+      "token_modifiers": 1
+    },
+    {
+      "delta_line": 0,
+      "delta_start_char": 0,
+      "length": 3,
+      "token_type": 0,
+      "token_modifiers": 0
+    },
+    {
+      "delta_line": 1,
+      "delta_start_char": 8,
+      "length": 1,
+      "token_type": 13,
+      "token_modifiers": 1
+    },
+    {
+      "delta_line": 1,
+      "delta_start_char": 6,
+      "length": 2,
+      "token_type": 8,
+      "token_modifiers": 0
+    },
+    {
+      "delta_line": 1,
+      "delta_start_char": 6,
+      "length": 2,
+      "token_type": 8,
+      "token_modifiers": 0
+    },
+    {
+      "delta_line": 0,
+      "delta_start_char": 5,
+      "length": 2,
+      "token_type": 8,
+      "token_modifiers": 0
+    },
+    {
+      "delta_line": 0,
+      "delta_start_char": 3,
+      "length": 6,
+      "token_type": 13,
+      "token_modifiers": 0
+    },
+    {
+      "delta_line": 1,
+      "delta_start_char": 11,
+      "length": 2,
+      "token_type": 8,
+      "token_modifiers": 0
+    }
+  ]
+}

test/fixtures/multibyte_characters.rb

@@ -0,0 +1,10 @@
+# 測試註解
+module A動物
+  class C貓咪
+    def 叫
+      叫聲 = "喵"
+      👋 = 叫聲.squish
+      puts 👋
+    end
+  end
+end

test/requests/semantic_highlighting_expectations_test.rb

@@ -18,7 +18,9 @@ def run_expectations(source)
     end
 
     dispatcher = Prism::Dispatcher.new
-    listener = RubyLsp::Requests::SemanticHighlighting.new(dispatcher, range: processed_range)
+    global_state = RubyLsp::GlobalState.new
+    global_state.apply_options({})
+    listener = RubyLsp::Requests::SemanticHighlighting.new(global_state, dispatcher, range: processed_range)
 
     dispatcher.dispatch(document.tree)
     listener.perform
@@ -89,7 +91,7 @@ def initialize(response_builder, dispatcher)
 
           def on_call_node_enter(node)
             current_token = @response_builder.last
-            if node.message == "before_create" && node.message_loc == current_token.location
+            if node.message == "before_create" && @response_builder.last_token_matches?(node.message_loc)
               current_token.replace_type(:keyword)
               current_token.replace_modifier([:declaration])
             end

test/requests/support/semantic_token_encoder_test.rb

@@ -94,12 +94,9 @@ def test_encoded_modifiers_with_some_modifiers
   private
 
   def stub_token(start_line, start_column, length, type, modifier)
-    location = Prism::Location.new(Prism::Source.new(""), 123, 123)
-    location.expects(:start_line).returns(start_line).at_least_once
-    location.expects(:start_column).returns(start_column).at_least_once
-
     RubyLsp::ResponseBuilders::SemanticHighlighting::SemanticToken.new(
-      location: location,
+      start_line: start_line,
+      start_code_unit_column: start_column,
       length: length,
       type: type,
       modifier: modifier,