[Backport 5.2] SCIP ctags: improve kind output (#58051)

* SCIP syntax: add more kinds for go ctags (#57806)

This change adds fine-grained kinds to the Go ctags output. Specific changes:
* Split `type` into `interface`, `struct`, and `typealias`
* Map struct members to `field` instead of `variable`

Universal ctags does not have a clear spec, and some languages use different
names for the same kind. So my strategy is not to match universal ctags
exactly, but just to capture the correct SCIP kinds. Clients need to handle the
fact that the kind names can be different.

* SCIP ctags: add kinds for C#, Python, Ruby (#57879)

Added more fine-grained kinds for C#, Python, and Ruby to better match the universal-ctags output.

Addresses https://github.com/sourcegraph/sourcegraph/issues/57659

* Add scip-ctags kinds for js, ts, and rust (#57899)

* SCIP ctags: use MethodSpec kind for Go (#57929)

Now that MethodSpec is available in SCIP, we can use it in the Go SCIP ctags
output.

* SCIP ctags: add kinds for Kotlin (#57998)

Improved ctags kind output for Kotlin:
* Split type into class, interface, object, and enum
* Split variable into enumMember, constant, and property
* Add type alias

* Fix snapshot tests by adding back trailing whitespace

Our precommit hook removes trailing whitespace, but the generated snapshots
include it.

---------

Co-authored-by: Auguste Rame <[email protected]>

docker-images/syntax-highlighter/Cargo.toml

@@ -61,7 +61,7 @@ tree-sitter = "0.20.9"
 tree-sitter-highlight = "0.20.1"
 
 # Since there is no version tag, we pin the dependency to a specific revision
-scip = { git = "https://github.com/sourcegraph/scip", rev="1bf111e1cf175569970e95566b379fb2a548d70a" }
+scip = "0.3.1"
 protobuf = "3"
 
 

docker-images/syntax-highlighter/crates/scip-syntax/queries/c_sharp/scip-tags.scm

@@ -1,20 +1,18 @@
-(using_directive (qualified_name) @descriptor.type)
-
-(class_declaration name: (_) @descriptor.type) @scope
-(interface_declaration name: (_) @descriptor.type) @scope
-(enum_declaration name: (_) @descriptor.type) @scope
-(struct_declaration name: (_) @descriptor.type) @scope
-(namespace_declaration name: (_) @descriptor.namespace) @scope
+(class_declaration name: (_) @descriptor.type @kind.class) @scope
+(interface_declaration name: (_) @descriptor.type @kind.interface) @scope
+(enum_declaration name: (_) @descriptor.type @kind.enum) @scope
+(struct_declaration name: (_) @descriptor.type @kind.struct) @scope
+(namespace_declaration name: (_) @descriptor.namespace @kind.namespace) @scope
 
 ; Counter-intuitive name; it can actually be global
-(local_function_statement name: (_) @descriptor.method)
-(method_declaration name: (_) @descriptor.method)
-(constructor_declaration name: (_) @descriptor.method)
+(local_function_statement name: (_) @descriptor.method @kind.function)
+(method_declaration name: (_) @descriptor.method @kind.method)
+(constructor_declaration name: (_) @descriptor.method @kind.constructor)
 
 (block) @local
 
-(field_declaration (variable_declaration (variable_declarator (identifier) @descriptor.term)))
-(event_field_declaration (variable_declaration (variable_declarator (identifier) @descriptor.term)))
-(property_declaration name: (identifier) @descriptor.term)
-(enum_member_declaration name: (_) @descriptor.term)
-(delegate_declaration name: (identifier) @descriptor.method)
+(field_declaration (variable_declaration (variable_declarator (identifier) @descriptor.term @kind.field)))
+(event_field_declaration (variable_declaration (variable_declarator (identifier) @kind.event @descriptor.term)))
+(property_declaration name: (identifier) @descriptor.term @kind.property)
+(enum_member_declaration name: (_) @descriptor.term @kind.enummember)
+(delegate_declaration name: (identifier) @descriptor.method @kind.delegate)

docker-images/syntax-highlighter/crates/scip-syntax/queries/go/scip-tags.scm

@@ -11,25 +11,47 @@
             (parameter_declaration
              type: [(pointer_type (type_identifier) @descriptor.type)
                     (type_identifier) @descriptor.type]))
- name: (field_identifier) @descriptor.method @enclosing
+ name: (field_identifier) @descriptor.method @kind.method @enclosing
  body: (_) @local)
 
-(type_declaration (type_spec name: (type_identifier) @descriptor.type)) @scope
+(type_declaration
+  (type_spec
+    name: (type_identifier) @descriptor.type @kind.struct
+    type: (struct_type))) @scope
+
+(type_declaration
+  (type_spec
+    name: (type_identifier) @descriptor.type @kind.interface
+    type: (interface_type))) @scope
+
+(type_declaration
+  (type_spec
+    name: (type_identifier) @descriptor.type @kind.typealias
+    type: (type_identifier)))
+
+(type_declaration
+  (type_alias
+    name: (type_identifier) @descriptor.type @kind.typealias))
 
 ;; For fields, we have nested struct definitions.
 ;;   To get the scope properly
 ((field_declaration_list
    (field_declaration
-     name: (_) @descriptor.term
+     name: (_) @descriptor.term @kind.field
      type: (_) @_type) @enclosing)
  (#filter! @_type "interface_type" "struct_type"))
 
 (field_declaration_list
   (field_declaration
-    name: (_) @descriptor.type
-    type: [(interface_type) (struct_type)] @scope))
+    name: (_) @descriptor.type @kind.interface
+    type: (interface_type) @scope))
+
+(field_declaration_list
+  (field_declaration
+    name: (_) @descriptor.type @kind.struct
+    type: (struct_type) @scope))
 
 (const_spec name: (_) @descriptor.term) @kind.constant @enclosing
 (import_spec name: (_) @descriptor.term) @enclosing
-(method_spec name: (_) @descriptor.method) @enclosing
-(var_spec name: (_) @descriptor.term) @enclosing
+(method_spec name: (_) @descriptor.method @kind.methodspec) @enclosing
+(var_spec name: (_) @descriptor.term @kind.variable) @enclosing

docker-images/syntax-highlighter/crates/scip-syntax/queries/javascript/scip-tags.scm

@@ -1,15 +1,16 @@
-(namespace_import (identifier) @descriptor.term)
+(namespace_import (identifier) @descriptor.term @kind.variable)
 (named_imports
- [(import_specifier alias: (_) @descriptor.term)
-  (import_specifier name: (_) @descriptor.term !alias)])
+  (import_specifier alias: (_) @descriptor.term @kind.variable))
+(named_imports
+  (import_specifier name: (_) @descriptor.term @kind.variable !alias))
 
 ;; Function / Generator declaration.
 ;;   Don't think there is any reason to expose anything from within the body of the functions
-(function_declaration (identifier) @descriptor.method body: (_) @local)
-(generator_function_declaration  (identifier) @descriptor.method body: (_) @local)
+(function_declaration (identifier) @descriptor.method @kind.function body: (_) @local)
+(generator_function_declaration  (identifier) @descriptor.method @kind.function body: (_) @local)
 
-(lexical_declaration (variable_declarator name: (identifier) @descriptor.term)) @scope
-(variable_declaration (variable_declarator name: (identifier) @descriptor.term)) @scope
+(lexical_declaration (variable_declarator name: (identifier) @descriptor.term @kind.variable)) @scope
+(variable_declaration (variable_declarator name: (identifier) @descriptor.term @kind.variable)) @scope
 
 ;; {{{ Handle multiple scenarios of literal objects at top level
 ;; var X = { key: value }
@@ -21,25 +22,31 @@
 ;;
 (object
   (pair
-    key: (property_identifier) @descriptor.method
+    key: (property_identifier) @descriptor.method @kind.function
     value: [(function) (arrow_function)]))
 
 ((object
    (pair
-     key: (property_identifier) @descriptor.term
+     key: (property_identifier) @descriptor.term @kind.property
      value: (_) @_value_type))
  (#filter! @_value_type "function" "arrow_function"))
 ;; }}}
 
 ;; class X { ... }
 (class_declaration
-  name: (_) @descriptor.type
+  name: (_) @descriptor.type @kind.class
   body: (_) @scope)
 
 (class_declaration
  (class_body
   [(method_definition
-     name: (_) @descriptor.method
+     name: (_) @descriptor.method @kind.method (#not-eq? @descriptor.method "constructor")
+     body: (_) @local)]))
+
+(class_declaration
+ (class_body
+  [(method_definition
+     name: (_) @descriptor.method @kind.constructor (#eq? @descriptor.method "constructor")
      body: (_) @local)]))
 
 [(if_statement) (while_statement) (for_statement) (do_statement) (call_expression)] @local

docker-images/syntax-highlighter/crates/scip-syntax/queries/kotlin/scip-tags.scm

@@ -1,20 +1,44 @@
 (source_file
     (package_header
         (identifier)
-        @descriptor.namespace)) @scope
+        @descriptor.namespace @kind.package)) @scope
 
 (function_declaration
-  (simple_identifier) @descriptor.method
-  (function_body) @local)
+  (simple_identifier) @descriptor.method @kind.method
+  (function_body)? @local)
 
-(anonymous_function (_ (type_identifier) @descriptor.type . (type_identifier) @descriptor.method)) @local
-(class_declaration (type_identifier) @descriptor.type) @scope
-(object_declaration (type_identifier) @descriptor.type) @scope
-(class_parameter (simple_identifier) @descriptor.term)
-(enum_entry (simple_identifier) @descriptor.term)
-(property_declaration (variable_declaration (simple_identifier) @descriptor.term))
+(anonymous_function) @local
 
-(multi_variable_declaration (variable_declaration (simple_identifier) @descriptor.term))
+(class_declaration "interface" (type_identifier) @descriptor.type @kind.interface) @scope
+(class_declaration "enum" "class" (type_identifier) @descriptor.type @kind.enum) @scope
+
+;; Exclude enums from the 'class' kind
+((class_declaration
+    ("enum")? @_enum "class"
+    (type_identifier) @descriptor.type @kind.class)
+  (#filter! @_enum "enum")) @scope
+
+(object_declaration (type_identifier) @descriptor.type @kind.object) @scope
+(companion_object (type_identifier) @descriptor.type @kind.object) @scope
+
+(type_alias (type_identifier) @descriptor.type @kind.typealias)
+
+(class_parameter (simple_identifier) @descriptor.term @kind.property)
+(enum_entry (simple_identifier) @descriptor.term @kind.enummember)
+
+;; In the grammar, property_modifier always represents 'const'
+(property_declaration
+    (modifiers (property_modifier))
+    (variable_declaration (simple_identifier) @descriptor.term @kind.constant))
+
+;; Exclude constants from the 'property' kind
+((property_declaration
+    (modifiers (property_modifier) @_const)?
+    (variable_declaration (simple_identifier) @descriptor.term @kind.property))
+  (#filter! @_const "property_modifier"))
+
+(property_declaration
+  (multi_variable_declaration (variable_declaration (simple_identifier) @descriptor.term @kind.property)))
 
 ;; Future TODOs:
 ;; - Should probably unescape `Escaped` simple identifiers

docker-images/syntax-highlighter/crates/scip-syntax/queries/python/scip-tags.scm

@@ -8,23 +8,22 @@
 ; (import_statement name: (_) @descriptor.term)
 ; (import_from_statement name: (_) @descriptor.term)
 
-(class_definition name: (_) @descriptor.type body: (_) @scope)
+(class_definition name: (_) @descriptor.type @kind.class body: (_) @scope)
 (class_definition
   body: (block
           [(function_definition
-             name: (_) @descriptor.method @kind.field
+             name: (_) @descriptor.method @kind.method
              body: (_) @local)
            (decorated_definition
              definition: (function_definition
-                          name: (_) @descriptor.method @kind.field
+                          name: (_) @descriptor.method @kind.method
                           body: (_) @local))]))
 
-
-(module (function_definition name: (_) @descriptor.method body: (_) @local))
-(module (decorated_definition (function_definition name: (_) @descriptor.method body: (_) @local)))
+(module (function_definition name: (_) @descriptor.method @kind.function body: (_) @local))
+(module (decorated_definition (function_definition name: (_) @descriptor.method @kind.function body: (_) @local)))
 
 ;; foo = 1
-(expression_statement (assignment left: (identifier) @descriptor.term))
+(expression_statement (assignment left: (identifier) @descriptor.term @kind.variable))
 
 ;; foo, bar, baz = 1, 2, 3
-(expression_statement (assignment left: (pattern_list (identifier) @descriptor.term)))
+(expression_statement (assignment left: (pattern_list (identifier) @descriptor.term @kind.variable)))