Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Handle multibyte characters in indexing #2619

Merged
merged 4 commits into from
Sep 30, 2024
Merged

Handle multibyte characters in indexing #2619

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
c54b0fb
Add test for indexing multibyte characters
NotFounds Sep 24, 2024
63df3c1
Modify to consider encoding in indexing
NotFounds Sep 24, 2024
57f4f9f
Fix type error
NotFounds Sep 26, 2024
d15ede6
Format code
NotFounds Sep 26, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions lib/ruby_indexer/lib/ruby_indexer/configuration.rb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,9 +19,13 @@ class Configuration
sig { params(workspace_path: String).void }
attr_writer :workspace_path

sig { returns(Encoding) }
attr_accessor :encoding

sig { void }
def initialize
@workspace_path = T.let(Dir.pwd, String)
@encoding = T.let(Encoding::UTF_8, Encoding)
@excluded_gems = T.let(initial_excluded_gems, T::Array[String])
@included_gems = T.let([], T::Array[String])
@excluded_patterns = T.let([File.join("**", "*_test.rb"), File.join("tmp", "**", "*")], T::Array[String])
Expand Down
74 changes: 66 additions & 8 deletions lib/ruby_indexer/lib/ruby_indexer/declaration_listener.rb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -109,6 +109,7 @@ def on_class_node_enter(node)
node.location,
constant_path.location,
comments,
@index.configuration.encoding,
vinistock marked this conversation as resolved.
Show resolved Hide resolved
parent_class,
)

Expand All @@ -132,7 +133,14 @@ def on_module_node_enter(node)

comments = collect_comments(node)

entry = Entry::Module.new(actual_nesting(name), @file_path, node.location, constant_path.location, comments)
entry = Entry::Module.new(
actual_nesting(name),
@file_path,
node.location,
constant_path.location,
comments,
@index.configuration.encoding,
)

@owner_stack << entry
@index.add(entry)
Expand Down Expand Up @@ -161,14 +169,20 @@ def on_singleton_class_node_enter(node)

if existing_entries
entry = T.must(existing_entries.first)
entry.update_singleton_information(node.location, expression.location, collect_comments(node))
entry.update_singleton_information(
node.location,
expression.location,
collect_comments(node),
@index.configuration.encoding,
)
else
entry = Entry::SingletonClass.new(
real_nesting,
@file_path,
node.location,
expression.location,
collect_comments(node),
@index.configuration.encoding,
nil,
)
@index.add(entry, skip_prefix_tree: true)
Expand Down Expand Up @@ -329,6 +343,7 @@ def on_def_node_enter(node)
node.location,
node.name_loc,
comments,
@index.configuration.encoding,
[Entry::Signature.new(list_params(node.parameters))],
current_visibility,
@owner_stack.last,
Expand All @@ -345,6 +360,7 @@ def on_def_node_enter(node)
node.location,
node.name_loc,
comments,
@index.configuration.encoding,
[Entry::Signature.new(list_params(node.parameters))],
current_visibility,
singleton,
Expand Down Expand Up @@ -403,6 +419,7 @@ def on_alias_method_node_enter(node)
@file_path,
node.new_name.location,
comments,
@index.configuration.encoding,
),
)
end
Expand Down Expand Up @@ -433,7 +450,14 @@ def handle_instance_variable(node, loc)
owner = @index.existing_or_new_singleton_class(owner.name)
end

@index.add(Entry::InstanceVariable.new(name, @file_path, loc, collect_comments(node), owner))
@index.add(Entry::InstanceVariable.new(
name,
@file_path,
loc,
collect_comments(node),
@index.configuration.encoding,
owner,
))
end

sig { params(node: Prism::CallNode).void }
Expand Down Expand Up @@ -496,6 +520,7 @@ def handle_alias_method(node)
@file_path,
new_name.location,
comments,
@index.configuration.encoding,
),
)
end
Expand Down Expand Up @@ -525,19 +550,43 @@ def add_constant(node, name, value = nil)
@index.add(
case value
when Prism::ConstantReadNode, Prism::ConstantPathNode
Entry::UnresolvedConstantAlias.new(value.slice, @stack.dup, name, @file_path, node.location, comments)
Entry::UnresolvedConstantAlias.new(
value.slice,
@stack.dup,
name,
@file_path,
node.location,
comments,
@index.configuration.encoding,
)
when Prism::ConstantWriteNode, Prism::ConstantAndWriteNode, Prism::ConstantOrWriteNode,
Prism::ConstantOperatorWriteNode

# If the right hand side is another constant assignment, we need to visit it because that constant has to be
# indexed too
Entry::UnresolvedConstantAlias.new(value.name.to_s, @stack.dup, name, @file_path, node.location, comments)
Entry::UnresolvedConstantAlias.new(
value.name.to_s,
@stack.dup,
name,
@file_path,
node.location,
comments,
@index.configuration.encoding,
)
when Prism::ConstantPathWriteNode, Prism::ConstantPathOrWriteNode, Prism::ConstantPathOperatorWriteNode,
Prism::ConstantPathAndWriteNode

Entry::UnresolvedConstantAlias.new(value.target.slice, @stack.dup, name, @file_path, node.location, comments)
Entry::UnresolvedConstantAlias.new(
value.target.slice,
@stack.dup,
name,
@file_path,
node.location,
comments,
@index.configuration.encoding,
)
else
Entry::Constant.new(name, @file_path, node.location, comments)
Entry::Constant.new(name, @file_path, node.location, comments, @index.configuration.encoding)
end,
)
end
Expand Down Expand Up @@ -600,7 +649,15 @@ def handle_attribute(node, reader:, writer:)
next unless name && loc

if reader
@index.add(Entry::Accessor.new(name, @file_path, loc, comments, current_visibility, @owner_stack.last))
@index.add(Entry::Accessor.new(
name,
@file_path,
loc,
comments,
@index.configuration.encoding,
current_visibility,
@owner_stack.last,
))
end

next unless writer
Expand All @@ -610,6 +667,7 @@ def handle_attribute(node, reader:, writer:)
@file_path,
loc,
comments,
@index.configuration.encoding,
current_visibility,
@owner_stack.last,
))
Expand Down
Loading
Loading