From 9106cde7127e8fa18a2cff0c9b08c46b221fb01a Mon Sep 17 00:00:00 2001
From: tompng <tomoyapenguin@gmail.com>
Date: Fri, 2 Aug 2024 02:45:31 +0900
Subject: [PATCH] Change comment directive parsing

---
 lib/rdoc/comment.rb                      | 166 ++++++++++++++++
 lib/rdoc/markup/pre_process.rb           |  45 ++++-
 lib/rdoc/parser/c.rb                     |  28 +--
 lib/rdoc/parser/prism_ruby.rb            | 225 +++++++++++-----------
 lib/rdoc/parser/simple.rb                |  24 +--
 lib/rdoc/tom_doc.rb                      |   2 +-
 test/rdoc/test_rdoc_comment.rb           | 230 +++++++++++++++++++++++
 test/rdoc/test_rdoc_parser.rb            |   2 +-
 test/rdoc/test_rdoc_parser_c.rb          |  23 +--
 test/rdoc/test_rdoc_parser_prism_ruby.rb |  52 +++++
 test/rdoc/test_rdoc_parser_simple.rb     |   4 +-
 11 files changed, 620 insertions(+), 181 deletions(-)

diff --git a/lib/rdoc/comment.rb b/lib/rdoc/comment.rb
index 04ec226436..983bf9349b 100644
--- a/lib/rdoc/comment.rb
+++ b/lib/rdoc/comment.rb
@@ -165,6 +165,12 @@ def normalize
     self
   end
 
+  # Change normalized, when creating already normalized comment.
+
+  def normalized=(value)
+    @normalized = value
+  end
+
   ##
   # Was this text normalized?
 
@@ -226,4 +232,164 @@ def tomdoc?
     @format == 'tomdoc'
   end
 
+  MULTILINE_DIRECTIVES = %w[call-seq].freeze # :nodoc:
+  private_constant :MULTILINE_DIRECTIVES
+
+  class << self
+
+    # Parse comment, collect directives as an attribute and return [normalized_comment_text, directives_hash]
+    # This method expands include and removes everything not needed in the document text, such as
+    # private section, directive line, comment characters `# /* * */` and indent spaces.
+    #
+    # RDoc comment consists of include, directive, multiline directive, private section and comment text.
+    #
+    # Include
+    #   # :include: filename
+    #
+    # Directive
+    #   # :directive-without-value:
+    #   # :directive-with-value: value
+    #
+    # Multiline directive (only :call-seq:)
+    #   # :multiline-directive:
+    #   #   value1
+    #   #   value2
+    #
+    # Private section
+    #   #--
+    #   # private comment
+    #   #++
+
+    def parse(text, filename, line_no, type)
+      case type
+      when :ruby
+        private_start_regexp = /^#?-{2,}$/
+        private_end_regexp = /^#?\+{2}$/
+        indent_regexp = /^#?\s*/
+        text = text.sub(/\A#\#$/, '')
+      when :c
+        private_start_regexp = /^(\s*\*)?-{2,}$/
+        private_end_regexp = /^(\s*\*)?\+{2}$/
+        indent_regexp = /^\s*(\/\*+|\*)?\s*/
+        text = text.sub(/\s*\*+\/\s*\z/, '')
+        # TODO: should not be here. Looks like another type of directive
+        text = text.gsub %r%Document-method:\s+[\w:.#=!?|^&<>~+\-/*\%@`\[\]]+%, ''
+      when :simple
+        # Unlike other types, this implementation only looks for two
+        # dashes at the beginning of the line.  Three or more dashes are considered
+        # to be a rule and ignored.
+        private_start_regexp = /^-{2}$/
+        private_end_regexp = /^\+{2}$/
+        indent_regexp = /^\s*/
+      end
+
+      directives = {}
+      lines = text.split("\n")
+      in_private = false
+      comment_lines = []
+      until lines.empty?
+        line = lines.shift
+        read_lines = 1
+        if in_private
+          in_private = false if line.match?(private_end_regexp)
+          line_no += read_lines
+          next
+        elsif line.match?(private_start_regexp)
+          in_private = true
+          line_no += read_lines
+          next
+        end
+
+        prefix = line[indent_regexp]
+        prefix_indent = ' ' * prefix.size
+        line = line.byteslice(prefix.bytesize..)
+        /\A(?<colon>\\?:|:?)(?<directive>[\w-]+):(?<param>.*)/ =~ line
+        directive = directive&.downcase
+
+        if colon == '\\:'
+          # unescape if escaped
+          comment_lines << prefix_indent + line.sub('\\:', ':')
+        elsif !directive || param.start_with?(':') || (colon.empty? && directive != 'call-seq')
+          # Something like `:toto::` is not a directive
+          # directive without colon prefix is only allowed for call-seq
+          comment_lines << prefix_indent + line
+        elsif directive == 'include'
+          filename_to_include = param.strip
+          yield(filename_to_include, prefix_indent).lines.each { |l| comment_lines << l.chomp }
+        elsif MULTILINE_DIRECTIVES.include?(directive)
+          param = param.strip
+          value_lines = take_multiline_directive_value_lines(directive, filename, line_no, lines, prefix_indent, indent_regexp, private_start_regexp)
+          read_lines += value_lines.size
+          lines.shift(value_lines.size)
+          unless param.empty?
+            # Accept `:call-seq: first-line\n  second-line` for now
+            value_lines.unshift(param)
+          end
+          value = value_lines.join("\n")
+          directives[directive] = [value.empty? ? nil : value, line_no]
+        else
+          value = param.strip
+          directives[directive] = [value.empty? ? nil : value, line_no]
+        end
+        line_no += read_lines
+      end
+      # normalize comment
+      min_spaces = nil
+      comment_lines.each do |l|
+        next if l.match?(/\A\s*\z/)
+        n = l[/\A */].size
+        min_spaces = n if !min_spaces || n < min_spaces
+      end
+      comment_lines.map! { |l| l[min_spaces..] || '' } if min_spaces
+      comment_lines.shift while comment_lines.first&.empty?
+
+      [String.new(encoding: text.encoding) << comment_lines.join("\n"), directives]
+    end
+
+    # Take value lines of multiline directive
+
+    private def take_multiline_directive_value_lines(directive, filename, line_no, lines, base_indent, indent_regexp, private_start_regexp)
+      return [] if lines.empty?
+
+      base_indent_size = base_indent.size
+
+      if lines.first[indent_regexp].size <= base_indent_size && !lines.first.sub(indent_regexp, '').chomp.empty?
+        # Unindented invalid block directive
+        warn "#{filename}:#{line_no} Multiline directive :#{directive}: should be indented."
+        # Take until next blank line, directive line or private section start to accept this invalid format
+        #   # :multiline-directive:
+        #   # line1
+        #   # line2
+        #   # :other-directive:
+        value_lines = lines.take_while do |l|
+          next false if l.match?(private_start_regexp)
+          l = l.sub(indent_regexp, '').chomp
+          !l.empty? && !l.start_with?(/:[\w-]+:/)
+        end.map do |l|
+          l.sub(indent_regexp, '').chomp
+        end
+      else
+        # Take indented lines accepting blank lines between them
+        first_indent = nil
+        value_lines = lines.take_while do |l|
+          l = l.rstrip
+          indent = l[indent_regexp]
+          if indent == l || (first_indent && indent.size >= first_indent)
+            true
+          elsif first_indent.nil? && indent.size > base_indent_size
+            first_indent = indent.size
+            true
+          end
+        end
+        first_indent ||= base_indent_size
+        value_lines.map! { |l| (l[first_indent..] || '').chomp }
+
+        if value_lines.size != lines.size && !value_lines.last.empty?
+          warn "#{filename}:#{line_no} Multiline directive :#{directive}: should end with a blank line."
+        end
+        value_lines.pop while value_lines.last&.empty?
+      end
+      value_lines
+    end
+  end
 end
diff --git a/lib/rdoc/markup/pre_process.rb b/lib/rdoc/markup/pre_process.rb
index 979f2eadae..79648ef6be 100644
--- a/lib/rdoc/markup/pre_process.rb
+++ b/lib/rdoc/markup/pre_process.rb
@@ -97,18 +97,15 @@ def initialize(input_file_name, include_path)
   # RDoc::CodeObject#metadata for details.
 
   def handle text, code_object = nil, &block
-    first_line = 1
     if RDoc::Comment === text then
       comment = text
       text = text.text
-      first_line = comment.line || 1
     end
 
     # regexp helper (square brackets for optional)
     # $1      $2  $3        $4      $5
     # [prefix][\]:directive:[spaces][param]newline
-    text = text.lines.map.with_index(first_line) do |line, num|
-      next line unless line =~ /\A([ \t]*(?:#|\/?\*)?[ \t]*)(\\?):([\w-]+):([ \t]*)(.+)?(\r?\n|$)/
+    text = text.gsub(/^([ \t]*(?:#|\/?\*)?[ \t]*)(\\?):([\w-]+):([ \t]*)(.+)?(\r?\n|$)/) do
       # skip something like ':toto::'
       next $& if $4.empty? and $5 and $5[0, 1] == ':'
 
@@ -122,9 +119,8 @@ def handle text, code_object = nil, &block
         comment.format = $5.downcase
         next "#{$1.strip}\n"
       end
-
-      handle_directive $1, $3, $5, code_object, text.encoding, num, &block
-    end.join
+      handle_directive $1, $3, $5, code_object, text.encoding, &block
+    end
 
     if comment then
       comment.text = text
@@ -132,11 +128,40 @@ def handle text, code_object = nil, &block
       comment = text
     end
 
+    run_post_processes(comment, code_object)
+
+    text
+  end
+
+  # Apply directives to a code object
+
+  def run_pre_processes(comment_text, code_object, start_line_no, type)
+    comment_text, directives = parse_comment(comment_text, start_line_no, type)
+    directives.each do |directive, (param, line_no)|
+      handle_directive('', directive, param, code_object)
+    end
+    if code_object.is_a?(RDoc::AnyMethod) && (call_seq, = directives['call-seq']) && call_seq
+      code_object.call_seq = call_seq.lines.map(&:chomp).reject(&:empty?).join("\n") if call_seq
+    end
+    format, = directives['markup']
+    [comment_text, format]
+  end
+
+
+  # Perform post preocesses to a code object
+
+  def run_post_processes(comment, code_object)
     self.class.post_processors.each do |handler|
       handler.call comment, code_object
     end
+  end
 
-    text
+  # Parse comment and return [normalized_comment_text, directives_hash]
+
+  def parse_comment(text, line_no, type)
+    RDoc::Comment.parse(text, @input_file_name, line_no, type) do |filename, prefix_indent|
+      include_file(filename, prefix_indent, text.encoding)
+    end
   end
 
   ##
@@ -151,7 +176,7 @@ def handle text, code_object = nil, &block
   # When 1.8.7 support is ditched prefix can be defaulted to ''
 
   def handle_directive prefix, directive, param, code_object = nil,
-                       encoding = nil, line = nil
+                       encoding = nil
     blankline = "#{prefix.strip}\n"
     directive = directive.downcase
 
@@ -227,7 +252,7 @@ def handle_directive prefix, directive, param, code_object = nil,
 
       blankline
     else
-      result = yield directive, param, line if block_given?
+      result = yield directive, param if block_given?
 
       case result
       when nil then
diff --git a/lib/rdoc/parser/c.rb b/lib/rdoc/parser/c.rb
index 4050d7aa49..6766fa8589 100644
--- a/lib/rdoc/parser/c.rb
+++ b/lib/rdoc/parser/c.rb
@@ -608,8 +608,6 @@ def find_body class_name, meth_name, meth_obj, file_content, quiet = false
       body = args[1]
       offset, = args[2]
 
-      comment.remove_private if comment
-
       # try to find the whole body
       body = $& if /#{Regexp.escape body}[^(]*?\{.*?^\}/m =~ file_content
 
@@ -622,7 +620,6 @@ def find_body class_name, meth_name, meth_obj, file_content, quiet = false
       override_comment = find_override_comment class_name, meth_obj
       comment = override_comment if override_comment
 
-      comment.normalize
       find_modifiers comment, meth_obj if comment
 
       #meth_obj.params = params
@@ -640,7 +637,6 @@ def find_body class_name, meth_name, meth_obj, file_content, quiet = false
 
       find_body class_name, args[3], meth_obj, file_content, true
 
-      comment.normalize
       find_modifiers comment, meth_obj
 
       meth_obj.start_collecting_tokens
@@ -664,7 +660,6 @@ def find_body class_name, meth_name, meth_obj, file_content, quiet = false
       comment = find_override_comment class_name, meth_obj
 
       if comment then
-        comment.normalize
         find_modifiers comment, meth_obj
         meth_obj.comment = comment
 
@@ -743,7 +738,6 @@ def find_class_comment class_name, class_mod
     end
 
     comment = new_comment comment, @top_level, :c
-    comment.normalize
 
     look_for_directives_in class_mod, comment
 
@@ -804,9 +798,6 @@ def find_const_comment(type, const_name, class_name = nil)
   # Handles modifiers in +comment+ and updates +meth_obj+ as appropriate.
 
   def find_modifiers comment, meth_obj
-    comment.normalize
-    comment.extract_call_seq meth_obj
-
     look_for_directives_in meth_obj, comment
   end
 
@@ -820,10 +811,10 @@ def find_override_comment class_name, meth_obj
     comment = if @content =~ %r%Document-method:
                                 \s+#{class_name}#{prefix}#{name}
                                 \s*?\n((?>.*?\*/))%xm then
-                "/*#{$1}"
+                "/*\n#{$1}"
               elsif @content =~ %r%Document-method:
                                    \s#{name}\s*?\n((?>.*?\*/))%xm then
-                "/*#{$1}"
+                "/*\n#{$1}"
               end
 
     return unless comment
@@ -1099,17 +1090,10 @@ def load_variable_map map_name
   # This method modifies the +comment+
 
   def look_for_directives_in context, comment
-    @preprocess.handle comment, context do |directive, param|
-      case directive
-      when 'main' then
-        @options.main_page = param
-        ''
-      when 'title' then
-        @options.default_title = param if @options.respond_to? :default_title=
-        ''
-      end
-    end
-
+    comment.text, format = @preprocess.run_pre_processes(comment.text, context, comment.line || 1, :c)
+    comment.format = format if format
+    @preprocess.run_post_processes(comment, context)
+    comment.normalized = true
     comment
   end
 
diff --git a/lib/rdoc/parser/prism_ruby.rb b/lib/rdoc/parser/prism_ruby.rb
index 05e98ad6c4..c56bb413d6 100644
--- a/lib/rdoc/parser/prism_ruby.rb
+++ b/lib/rdoc/parser/prism_ruby.rb
@@ -85,9 +85,9 @@ def scan
     prepare_comments(result.comments)
     return if @top_level.done_documenting
 
-    @first_non_meta_comment = nil
-    if (_line_no, start_line, rdoc_comment = @unprocessed_comments.first)
-      @first_non_meta_comment = rdoc_comment if start_line < @program_node.location.start_line
+    @first_non_meta_comment_start_line = nil
+    if (_line_no, start_line = @unprocessed_comments.first)
+      @first_non_meta_comment_start_line = start_line if start_line < @program_node.location.start_line
     end
 
     @program_node.accept(RDocVisitor.new(self, @top_level, @store))
@@ -137,7 +137,9 @@ def prepare_comments(comments)
       if comment.is_a? Prism::EmbDocComment
         consecutive_comments << [comment] << (current = [])
       elsif comment.location.start_line_slice.match?(/\S/)
-        @modifier_comments[comment.location.start_line] = RDoc::Comment.new(comment.slice, @top_level, :ruby)
+        text = comment.slice
+        text = RDoc::Encoding.change_encoding(text, @encoding) if @encoding
+        @modifier_comments[comment.location.start_line] = text
       elsif current.empty? || current.last.location.end_line + 1 == comment.location.start_line
         current << comment
       else
@@ -159,22 +161,18 @@ def prepare_comments(comments)
       texts = comments.map do |c|
         c.is_a?(Prism::EmbDocComment) ? c.slice.lines[1...-1].join : c.slice
       end
-      text = RDoc::Encoding.change_encoding(texts.join("\n"), @encoding) if @encoding
+      text = texts.join("\n")
+      text = RDoc::Encoding.change_encoding(text, @encoding) if @encoding
       line_no += 1 while @lines[line_no - 1]&.match?(/\A\s*$/)
-      comment = RDoc::Comment.new(text, @top_level, :ruby)
-      comment.line = start_line
-      [line_no, start_line, comment]
+      [line_no, start_line, text]
     end
 
     # The first comment is special. It defines markup for the rest of the comments.
     _, first_comment_start_line, first_comment_text = @unprocessed_comments.first
     if first_comment_text && @lines[0...first_comment_start_line - 1].all? { |l| l.match?(/\A\s*$/) }
-      comment = RDoc::Comment.new(first_comment_text.text, @top_level, :ruby)
-      handle_consecutive_comment_directive(@container, comment)
-      @markup = comment.format
-    end
-    @unprocessed_comments.each do |_, _, comment|
-      comment.format = @markup
+      _text, directives = @preprocess.parse_comment(first_comment_text, first_comment_start_line, :ruby)
+      markup, = directives['markup']
+      @markup = markup.downcase if markup
     end
   end
 
@@ -198,32 +196,15 @@ def parse_comment_tomdoc(container, comment, line_no, start_line)
     tokens.each { |token| meth.token_stream << token }
 
     container.add_method meth
-    comment.remove_private
-    comment.normalize
     meth.comment = comment
     @stats.add_method meth
   end
 
   def handle_modifier_directive(code_object, line_no) # :nodoc:
-    comment = @modifier_comments[line_no]
-    @preprocess.handle(comment.text, code_object) if comment
-  end
-
-  def handle_consecutive_comment_directive(code_object, comment) # :nodoc:
-    return unless comment
-    @preprocess.handle(comment, code_object) do |directive, param|
-      case directive
-      when 'method', 'singleton-method',
-           'attr', 'attr_accessor', 'attr_reader', 'attr_writer' then
-        # handled elsewhere
-        ''
-      when 'section' then
-        @container.set_current_section(param, comment.dup)
-        comment.text = ''
-        break
-      end
+    if (comment_text = @modifier_comments[line_no])
+      _text, directives = @preprocess.parse_comment(comment_text, line_no, :ruby)
+      handle_code_object_directives(code_object, directives)
     end
-    comment.remove_private
   end
 
   def call_node_name_arguments(call_node) # :nodoc:
@@ -240,39 +221,32 @@ def call_node_name_arguments(call_node) # :nodoc:
 
   # Handles meta method comments
 
-  def handle_meta_method_comment(comment, node)
+  def handle_meta_method_comment(comment, directives, node)
+    handle_code_object_directives(@container, directives)
     is_call_node = node.is_a?(Prism::CallNode)
     singleton_method = false
     visibility = @visibility
     attributes = rw = line_no = method_name = nil
-
-    processed_comment = comment.dup
-    @preprocess.handle(processed_comment, @container) do |directive, param, line|
+    directives.each do |directive, (param, line)|
       case directive
       when 'attr', 'attr_reader', 'attr_writer', 'attr_accessor'
         attributes = [param] if param
         attributes ||= call_node_name_arguments(node) if is_call_node
         rw = directive == 'attr_writer' ? 'W' : directive == 'attr_accessor' ? 'RW' : 'R'
-        ''
       when 'method'
-        method_name = param
+        method_name = param if param
         line_no = line
-        ''
       when 'singleton-method'
-        method_name = param
+        method_name = param if param
         line_no = line
         singleton_method = true
         visibility = :public
-        ''
-      when 'section' then
-        @container.set_current_section(param, comment.dup)
-        return # If the comment contains :section:, it is not a meta method comment
       end
     end
 
     if attributes
       attributes.each do |attr|
-        a = RDoc::Attr.new(@container, attr, rw, processed_comment)
+        a = RDoc::Attr.new(@container, attr, rw, comment)
         a.store = @store
         a.line = line_no
         a.singleton = @singleton
@@ -282,12 +256,6 @@ def handle_meta_method_comment(comment, node)
       end
     elsif line_no || node
       method_name ||= call_node_name_arguments(node).first if is_call_node
-      meth = RDoc::AnyMethod.new(@container, method_name)
-      meth.singleton = @singleton || singleton_method
-      handle_consecutive_comment_directive(meth, comment)
-      comment.normalize
-      comment.extract_call_seq(meth)
-      meth.comment = comment
       if node
         tokens = visible_tokens_from_location(node.location)
         line_no = node.location.start_line
@@ -295,38 +263,41 @@ def handle_meta_method_comment(comment, node)
         tokens = [file_line_comment_token(line_no)]
       end
       internal_add_method(
+        method_name,
         @container,
-        meth,
+        comment: comment,
+        directives: directives,
+        dont_rename_initialize: false,
         line_no: line_no,
         visibility: visibility,
         singleton: @singleton || singleton_method,
-        params: '()',
+        params: nil,
         calls_super: false,
         block_params: nil,
-        tokens: tokens
+        tokens: tokens,
       )
     end
   end
 
-  def normal_comment_treat_as_ghost_method_for_now?(comment_text, line_no) # :nodoc:
+  INVALID_GHOST_METHOD_ACCEPT_DIRECTIVE_LIST = %w[
+    method singleton-method attr attr_reader attr_writer attr_accessor
+  ].freeze
+  private_constant :INVALID_GHOST_METHOD_ACCEPT_DIRECTIVE_LIST
+
+  def normal_comment_treat_as_ghost_method_for_now?(directives, line_no) # :nodoc:
     # Meta method comment should start with `##` but some comments does not follow this rule.
     # For now, RDoc accepts them as a meta method comment if there is no node linked to it.
-    !@line_nodes[line_no] && comment_text.match?(/^#\s+:(method|singleton-method|attr|attr_reader|attr_writer|attr_accessor):/)
+    !@line_nodes[line_no] && INVALID_GHOST_METHOD_ACCEPT_DIRECTIVE_LIST.any? { |directive| directives.has_key?(directive) }
   end
 
-  def handle_standalone_consecutive_comment_directive(comment, line_no, start_line) # :nodoc:
-    if @markup == 'tomdoc'
-      parse_comment_tomdoc(@container, comment, line_no, start_line)
-      return
-    end
-
-    if comment.text =~ /\A#\#$/ && comment != @first_non_meta_comment
+  def handle_standalone_consecutive_comment_directive(comment, directives, start_with_sharp_sharp, line_no, start_line) # :nodoc:
+    if start_with_sharp_sharp && start_line != @first_non_meta_comment_start_line
       node = @line_nodes[line_no]
-      handle_meta_method_comment(comment, node)
-    elsif normal_comment_treat_as_ghost_method_for_now?(comment.text, line_no) && comment != @first_non_meta_comment
-      handle_meta_method_comment(comment, nil)
+      handle_meta_method_comment(comment, directives, node)
+    elsif normal_comment_treat_as_ghost_method_for_now?(directives, line_no) && start_line != @first_non_meta_comment_start_line
+      handle_meta_method_comment(comment, directives, nil)
     else
-      handle_consecutive_comment_directive(@container, comment)
+      handle_code_object_directives(@container, directives)
     end
   end
 
@@ -334,8 +305,15 @@ def handle_standalone_consecutive_comment_directive(comment, line_no, start_line
 
   def process_comments_until(line_no_until)
     while !@unprocessed_comments.empty? && @unprocessed_comments.first[0] <= line_no_until
-      line_no, start_line, rdoc_comment = @unprocessed_comments.shift
-      handle_standalone_consecutive_comment_directive(rdoc_comment, line_no, start_line)
+      line_no, start_line, text = @unprocessed_comments.shift
+      if @markup == 'tomdoc'
+        comment = RDoc::Comment.new(text, @top_level, :ruby)
+        comment.format = 'tomdoc'
+        parse_comment_tomdoc(@container, comment, line_no, start_line)
+        @preprocess.run_post_processes(comment, @container)
+      elsif (comment_text, directives = parse_comment_text_to_directives(text, start_line))
+        handle_standalone_consecutive_comment_directive(comment_text, directives, text.start_with?(/#\#$/), line_no, start_line)
+      end
     end
   end
 
@@ -351,9 +329,27 @@ def skip_comments_until(line_no_until)
   # Returns consecutive comment linked to the given line number
 
   def consecutive_comment(line_no)
-    if @unprocessed_comments.first&.first == line_no
-      @unprocessed_comments.shift.last
+    return unless @unprocessed_comments.first&.first == line_no
+    _line_no, start_line, text = @unprocessed_comments.shift
+    parse_comment_text_to_directives(text, start_line)
+  end
+
+  # Parses comment text and retuns a pair of RDoc::Comment and directives
+
+  def parse_comment_text_to_directives(comment_text, start_line) # :nodoc:
+    comment_text, directives = @preprocess.parse_comment(comment_text, start_line, :ruby)
+    comment = RDoc::Comment.new(comment_text, @top_level, :ruby)
+    comment.normalized = true
+    comment.line = start_line
+    markup, = directives['markup']
+    comment.format = markup&.downcase || @markup
+    if (section, = directives['section'])
+      # If comment has :section:, it is not a documentable comment for a code object
+      @container.set_current_section(section, comment.dup)
+      return
     end
+    @preprocess.run_post_processes(comment, @container)
+    [comment, directives]
   end
 
   def slice_tokens(start_pos, end_pos) # :nodoc:
@@ -429,11 +425,17 @@ def change_method_to_module_function(names)
     end
   end
 
+  def handle_code_object_directives(code_object, directives) # :nodoc:
+    directives.each do |directive, (param)|
+      @preprocess.handle_directive('', directive, param, code_object)
+    end
+  end
+
   # Handles `alias foo bar` and `alias_method :foo, :bar`
 
   def add_alias_method(old_name, new_name, line_no)
-    comment = consecutive_comment(line_no)
-    handle_consecutive_comment_directive(@container, comment)
+    comment, directives = consecutive_comment(line_no)
+    handle_code_object_directives(@container, directives) if directives
     visibility = @container.find_method(old_name, @singleton)&.visibility || :public
     a = RDoc::Alias.new(nil, old_name, new_name, comment, @singleton)
     a.comment = comment
@@ -450,8 +452,8 @@ def add_alias_method(old_name, new_name, line_no)
   # Handles `attr :a, :b`, `attr_reader :a, :b`, `attr_writer :a, :b` and `attr_accessor :a, :b`
 
   def add_attributes(names, rw, line_no)
-    comment = consecutive_comment(line_no)
-    handle_consecutive_comment_directive(@container, comment)
+    comment, directives = consecutive_comment(line_no)
+    handle_code_object_directives(@container, directives) if directives
     return unless @container.document_children
 
     names.each do |symbol|
@@ -467,8 +469,8 @@ def add_attributes(names, rw, line_no)
   end
 
   def add_includes_extends(names, rdoc_class, line_no) # :nodoc:
-    comment = consecutive_comment(line_no)
-    handle_consecutive_comment_directive(@container, comment)
+    comment, directives = consecutive_comment(line_no)
+    handle_code_object_directives(@container, directives) if directives
     names.each do |name|
       ie = @container.add(rdoc_class, name, '')
       ie.store = @store
@@ -492,35 +494,17 @@ def add_extends(names, line_no) # :nodoc:
 
   # Adds a method defined by `def` syntax
 
-  def add_method(name, receiver_name:, receiver_fallback_type:, visibility:, singleton:, params:, calls_super:, block_params:, tokens:, start_line:, end_line:)
+  def add_method(method_name, receiver_name:, receiver_fallback_type:, visibility:, singleton:, params:, calls_super:, block_params:, tokens:, start_line:, end_line:)
     receiver = receiver_name ? find_or_create_module_path(receiver_name, receiver_fallback_type) : @container
-    meth = RDoc::AnyMethod.new(nil, name)
-    if (comment = consecutive_comment(start_line))
-      handle_consecutive_comment_directive(@container, comment)
-      handle_consecutive_comment_directive(meth, comment)
-
-      comment.normalize
-      comment.extract_call_seq(meth)
-      meth.comment = comment
-    end
-    handle_modifier_directive(meth, start_line)
-    handle_modifier_directive(meth, end_line)
-    return unless should_document?(meth)
-
-
-    if meth.name == 'initialize' && !singleton
-      if meth.dont_rename_initialize
-        visibility = :protected
-      else
-        meth.name = 'new'
-        singleton = true
-        visibility = :public
-      end
-    end
+    comment, directives = consecutive_comment(start_line)
+    handle_code_object_directives(@container, directives) if directives
 
     internal_add_method(
+      method_name,
       receiver,
-      meth,
+      comment: comment,
+      directives: directives,
+      modifier_comment_lines: [start_line, end_line],
       line_no: start_line,
       visibility: visibility,
       singleton: singleton,
@@ -531,7 +515,28 @@ def add_method(name, receiver_name:, receiver_fallback_type:, visibility:, singl
     )
   end
 
-  private def internal_add_method(container, meth, line_no:, visibility:, singleton:, params:, calls_super:, block_params:, tokens:) # :nodoc:
+  private def internal_add_method(method_name, container, comment:, dont_rename_initialize: false, directives:, modifier_comment_lines: nil, line_no:, visibility:, singleton:, params:, calls_super:, block_params:, tokens:) # :nodoc:
+    meth = RDoc::AnyMethod.new(nil, method_name)
+    meth.comment = comment
+    handle_code_object_directives(meth, directives) if directives
+    modifier_comment_lines&.each do |line|
+      handle_modifier_directive(meth, line)
+    end
+    return unless should_document?(meth)
+
+    if !dont_rename_initialize && method_name == 'initialize' && !singleton
+      if meth.dont_rename_initialize
+        visibility = :protected
+      else
+        meth.name = 'new'
+        singleton = true
+        visibility = :public
+      end
+    end
+
+    if directives && (call_seq, = directives['call-seq'])
+      meth.call_seq = call_seq.lines.map(&:chomp).reject(&:empty?).join("\n")
+    end
     meth.name ||= meth.call_seq[/\A[^()\s]+/] if meth.call_seq
     meth.name ||= 'unknown'
     meth.store = @store
@@ -539,7 +544,7 @@ def add_method(name, receiver_name:, receiver_fallback_type:, visibility:, singl
     meth.singleton = singleton
     container.add_method(meth) # should add after setting singleton and before setting visibility
     meth.visibility = visibility
-    meth.params ||= params
+    meth.params ||= params || '()'
     meth.calls_super = calls_super
     meth.block_params ||= block_params if block_params
     record_location(meth)
@@ -609,8 +614,8 @@ def find_or_create_constant_owner_name(constant_path)
   # Adds a constant
 
   def add_constant(constant_name, rhs_name, start_line, end_line)
-    comment = consecutive_comment(start_line)
-    handle_consecutive_comment_directive(@container, comment)
+    comment, directives = consecutive_comment(start_line)
+    handle_code_object_directives(@container, directives) if directives
     owner, name = find_or_create_constant_owner_name(constant_name)
     constant = RDoc::Constant.new(name, rhs_name, comment)
     constant.store = @store
@@ -636,8 +641,8 @@ def add_constant(constant_name, rhs_name, start_line, end_line)
   # Adds module or class
 
   def add_module_or_class(module_name, start_line, end_line, is_class: false, superclass_name: nil)
-    comment = consecutive_comment(start_line)
-    handle_consecutive_comment_directive(@container, comment)
+    comment, directives = consecutive_comment(start_line)
+    handle_code_object_directives(@container, directives) if directives
     return unless @container.document_children
 
     owner, name = find_or_create_constant_owner_name(module_name)
diff --git a/lib/rdoc/parser/simple.rb b/lib/rdoc/parser/simple.rb
index b1dabad0f8..7313221c17 100644
--- a/lib/rdoc/parser/simple.rb
+++ b/lib/rdoc/parser/simple.rb
@@ -19,17 +19,16 @@ def initialize(top_level, file_name, content, options, stats)
 
     preprocess = RDoc::Markup::PreProcess.new @file_name, @options.rdoc_include
 
-    @content = preprocess.handle @content, @top_level
+    @content, = preprocess.run_pre_processes(@content, @top_level, 1, :simple)
   end
 
   ##
   # Extract the file contents and attach them to the TopLevel as a comment
 
   def scan
-    comment = remove_coding_comment @content
-    comment = remove_private_comment comment
+    content = remove_coding_comment @content
 
-    comment = RDoc::Comment.new comment, @top_level
+    comment = RDoc::Comment.new content, @top_level
 
     @top_level.comment = comment
     @top_level
@@ -41,21 +40,4 @@ def scan
   def remove_coding_comment text
     text.sub(/\A# .*coding[=:].*$/, '')
   end
-
-  ##
-  # Removes private comments.
-  #
-  # Unlike RDoc::Comment#remove_private this implementation only looks for two
-  # dashes at the beginning of the line.  Three or more dashes are considered
-  # to be a rule and ignored.
-
-  def remove_private_comment comment
-    # Workaround for gsub encoding for Ruby 1.9.2 and earlier
-    empty = ''
-    empty = RDoc::Encoding.change_encoding empty, comment.encoding
-
-    comment = comment.gsub(%r%^--\n.*?^\+\+\n?%m, empty)
-    comment.sub(%r%^--\n.*%m, empty)
-  end
-
 end
diff --git a/lib/rdoc/tom_doc.rb b/lib/rdoc/tom_doc.rb
index d10f024f70..52b2d3556a 100644
--- a/lib/rdoc/tom_doc.rb
+++ b/lib/rdoc/tom_doc.rb
@@ -49,7 +49,7 @@ def self.add_post_processor # :nodoc:
       next unless code_object and
                   RDoc::Comment === comment and comment.format == 'tomdoc'
 
-      comment.text.gsub!(/(\A\s*# )(Public|Internal|Deprecated):\s+/) do
+      comment.text.gsub!(/\A(\s*# |)(Public|Internal|Deprecated):\s+/) do
         section = code_object.add_section $2
         code_object.temporary_section = section
 
diff --git a/test/rdoc/test_rdoc_comment.rb b/test/rdoc/test_rdoc_comment.rb
index 28e8cf76b4..8569a2211b 100644
--- a/test/rdoc/test_rdoc_comment.rb
+++ b/test/rdoc/test_rdoc_comment.rb
@@ -503,4 +503,234 @@ def test_remove_private_toggle_encoding_ruby_bug?
     assert_equal Encoding::IBM437, comment.text.encoding
   end
 
+  def test_parse_directives
+    comment = <<~COMMENT
+      comment1
+      :foo:
+      comment2
+      :bar: baz
+      comment3
+    COMMENT
+    text, directives = RDoc::Comment.parse(comment.chomp, 'file', 1, :simple)
+    assert_equal({ 'foo' => [nil, 2], 'bar' => ['baz', 4] }, directives)
+
+    comment = <<~COMMENT
+      # comment1
+      # :foo:
+      # comment2
+      # :bar: baz
+      # comment3
+    COMMENT
+    text, directives = RDoc::Comment.parse(comment.chomp, 'file', 10, :ruby)
+    assert_equal "comment1\ncomment2\ncomment3", text
+    assert_equal({ 'foo' => [nil, 11], 'bar' => ['baz', 13] }, directives)
+
+    comment = <<~COMMENT
+      /* comment1
+       * :foo:
+       * comment2
+       * :bar: baz
+       * comment3
+       */
+    COMMENT
+    text, directives = RDoc::Comment.parse(comment.chomp, 'file', 100, :c)
+    assert_equal "comment1\ncomment2\ncomment3", text
+    assert_equal({ 'foo' => [nil, 101], 'bar' => ['baz', 103] }, directives)
+  end
+
+  def test_parse_escaped_directives
+    comment = <<~'COMMENT'
+      :foo: a\a
+      \:bar: b\b
+      \:baz\: c\c
+    COMMENT
+    text, directives = RDoc::Comment.parse(comment.chomp, 'file', 1, :simple)
+    assert_equal ":bar: b\\b\n\\:baz\\: c\\c", text
+    assert_equal({ 'foo' => ['a\\a', 1] }, directives)
+  end
+
+  def test_parse_multiline_directive
+    comment = <<~COMMENT
+      # comment1
+      # :call-seq:
+      #   a
+      #    b
+      #   c
+      #
+      #    d
+      #
+      # comment2
+    COMMENT
+    text, directives = RDoc::Comment.parse(comment.chomp, 'file', 1, :ruby)
+    assert_equal "comment1\n\ncomment2", text.chomp
+    assert_equal({ 'call-seq' => ["a\n b\nc\n\n d", 2] }, directives)
+
+    # Some c code contains this kind of call-seq
+    comment = <<~COMMENT
+      * comment1
+      * :call-seq:
+      *
+      *   a
+      *
+      *   b
+      *
+      *  comment2
+    COMMENT
+    text, directives = RDoc::Comment.parse(comment.chomp, 'file', 1, :c)
+    assert_equal "comment1\n\n comment2", text
+    assert_equal({ 'call-seq' => ["\na\n\nb", 2] }, directives)
+
+    comment = <<~COMMENT
+      # comment1
+      # :call-seq: a
+      #
+      # comment2
+    COMMENT
+    text, directives = RDoc::Comment.parse(comment.chomp, 'file', 1, :ruby)
+    assert_equal "comment1\n\ncomment2", text
+    assert_equal({ 'call-seq' => ['a', 2] }, directives)
+
+    comment = <<~COMMENT
+      # comment1
+      # :call-seq:
+      #
+      # comment2
+    COMMENT
+    text, directives = RDoc::Comment.parse(comment.chomp, 'file', 1, :ruby)
+    assert_equal "comment1\n\ncomment2", text
+    assert_equal({ 'call-seq' => [nil, 2] }, directives)
+  end
+
+  def test_parse_directives_with_include
+    comment = <<~COMMENT
+      # comment1
+      # :include: file.txt
+      # comment2
+      #   :include: file.txt
+      # :foo: bar
+      # comment3
+    COMMENT
+    text, directives = RDoc::Comment.parse(comment.chomp, 'file', 1, :ruby) do |file, prefix_indent|
+      2.times.map { |i| "#{prefix_indent}#{file}:#{i}\n" }.join
+    end
+    assert_equal "comment1\nfile.txt:0\nfile.txt:1\ncomment2\n  file.txt:0\n  file.txt:1\ncomment3", text
+    assert_equal({ 'foo' => ['bar', 5] }, directives)
+  end
+
+  def test_parse_c_comment_with_double_asterisk
+    # Used in ruby/ruby c files
+    comment = <<~COMMENT
+      /**
+       * :foo: bar
+       * comment
+       */
+    COMMENT
+    text, directives = RDoc::Comment.parse(comment.chomp, 'file', 1, :c)
+    assert_equal "comment", text
+    assert_equal({ 'foo' => ['bar', 2] }, directives)
+  end
+
+  def test_parse_confusing_comment
+    verbose, $VERBOSE = $VERBOSE, nil
+    comment = <<~COMMENT
+      /* :foo: value1
+       * :call-seq:
+       * :include: file.txt
+       * :bar: value2 */
+    COMMENT
+    text, directives = RDoc::Comment.parse(comment.chomp, 'file', 1, :c) do |file, prefix_indent|
+      # callseq-content, comment end and directive inside included file with no newline at the end
+      "#{prefix_indent}  f(x,y)\n#{prefix_indent}*/\n#{prefix_indent}:baz: blah\n#{prefix_indent}blah"
+    end
+    assert_equal "  f(x,y)\n*/\n:baz: blah\nblah", text
+    assert_equal({ 'call-seq' => [nil, 2], 'foo' => ['value1', 1], 'bar' => ['value2', 4] }, directives)
+  ensure
+    $VERBOSE = verbose
+  end
+
+  def test_parse_directives_with_skipping_private_section_ruby
+    verbose, $VERBOSE = $VERBOSE, nil
+    comment = <<~COMMENT
+      # :foo: foo-value
+      #--
+      # private1
+      #++
+      #-
+      # comment1
+      #+
+      # :call-seq:
+      #   a(x)
+      #---
+      # private2
+      # :bar: bar-value
+      #++
+      #   a(x, y)
+      #
+      #----
+      # private3
+    COMMENT
+    text, directives = RDoc::Comment.parse(comment.chomp, 'file', 1, :ruby)
+    assert_equal "-\n comment1\n+\n   a(x, y)", text.chomp
+    assert_equal({ 'foo' => ['foo-value', 1], 'call-seq' => ["a(x)", 8] }, directives)
+
+    # block comment `=begin\n=end` does not start with `#`
+    comment = <<~COMMENT
+      comment1
+      --
+      private1
+      :foo: foo-value
+      ++
+      :bar: bar-value
+      comment2
+      --
+    COMMENT
+    text, directives = RDoc::Comment.parse(comment.chomp, 'file', 1, :ruby)
+    assert_equal "comment1\ncomment2", text.chomp
+    assert_equal({ 'bar' => ['bar-value', 6] }, directives)
+  ensure
+    $VERBOSE = verbose
+  end
+
+  def test_parse_directives_with_skipping_private_section_c
+    comment = <<~COMMENT
+      /*
+       * comment1
+       *--
+       * private1
+       * :foo: foo-value
+       *++
+       * :bar: bar-value
+       * comment2
+       *---
+       * private2
+       */
+    COMMENT
+    text, directives = RDoc::Comment.parse(comment.chomp, 'file', 1, :c)
+    assert_equal "comment1\ncomment2", text.chomp
+    assert_equal({ 'bar' => ['bar-value', 7] }, directives)
+  end
+
+  def test_parse_directives_with_skipping_private_section_simple
+    comment = <<~COMMENT
+      comment1
+      --
+      private1
+      :foo: foo-value
+      ++
+      -
+      comment2
+      +
+       --
+       comment3
+       ++
+      ---
+      comment4
+      :bar: bar-value
+      --
+      private2
+    COMMENT
+    text, directives = RDoc::Comment.parse(comment.chomp, 'file', 1, :simple)
+    assert_equal "comment1\n-\ncomment2\n+\n --\n comment3\n ++\n---\ncomment4", text.chomp
+    assert_equal({ 'bar' => ['bar-value', 14] }, directives)
+  end
 end
diff --git a/test/rdoc/test_rdoc_parser.rb b/test/rdoc/test_rdoc_parser.rb
index 51a16ce361..e730ba4f57 100644
--- a/test/rdoc/test_rdoc_parser.rb
+++ b/test/rdoc/test_rdoc_parser.rb
@@ -120,7 +120,7 @@ def test_class_for_modeline
 
       assert_kind_of RDoc::Parser::Simple, parser
 
-      assert_equal "= NEWS\n", parser.content
+      assert_equal "= NEWS", parser.content.chomp
     end
   end
 
diff --git a/test/rdoc/test_rdoc_parser_c.rb b/test/rdoc/test_rdoc_parser_c.rb
index ab4f149869..ed1b265a84 100644
--- a/test/rdoc/test_rdoc_parser_c.rb
+++ b/test/rdoc/test_rdoc_parser_c.rb
@@ -1420,12 +1420,7 @@ def test_find_modifiers_call_seq
 
     parser.find_modifiers comment, method_obj
 
-    expected = <<-CALL_SEQ.chomp
-commercial() -> Date <br />
-
-    CALL_SEQ
-
-    assert_equal expected, method_obj.call_seq
+    assert_equal "commercial() -> Date <br />", method_obj.call_seq.chomp
   end
 
   def test_find_modifiers_nodoc
@@ -1461,7 +1456,7 @@ def test_find_modifiers_yields
 
     assert_equal 'a, b', method_obj.block_params
 
-    assert_equal "\n\nBlah", comment.text
+    assert_equal "Blah", comment.text
   end
 
   def test_handle_method_args_minus_1
@@ -1583,11 +1578,11 @@ def test_handle_singleton
   def test_look_for_directives_in
     parser = util_parser
 
-    comment = RDoc::Comment.new "# :other: not_handled\n"
+    comment = RDoc::Comment.new "* :other: not_handled\n"
 
     parser.look_for_directives_in @top_level, comment
 
-    assert_equal "# :other: not_handled\n", comment.text
+    assert_equal "", comment.text
     assert_equal 'not_handled', @top_level.metadata['other']
   end
 
@@ -1690,7 +1685,7 @@ def test_define_method
     klass = util_get_class content, 'rb_cIO'
     read_method = klass.method_list.first
     assert_equal "read", read_method.name
-    assert_equal "Method Comment!   ", read_method.comment.text
+    assert_equal "Method Comment!", read_method.comment.text
     assert_equal "rb_io_s_read", read_method.c_function
     assert read_method.singleton
     assert_nil read_method.section.title
@@ -1767,7 +1762,7 @@ def test_define_method_with_prototype
     klass = util_get_class content, 'rb_cIO'
     read_method = klass.method_list.first
     assert_equal "read", read_method.name
-    assert_equal "Method Comment!   ", read_method.comment.text
+    assert_equal "Method Comment!", read_method.comment.text
     assert_equal "rb_io_s_read", read_method.c_function
     assert read_method.singleton
   end
@@ -1797,7 +1792,7 @@ def test_define_method_private
     read_method = klass.method_list.first
     assert_equal 'IO#read', read_method.full_name
     assert_equal :private, read_method.visibility
-    assert_equal "Method Comment!   ", read_method.comment.text
+    assert_equal "Method Comment!", read_method.comment.text
   end
 
   def test_define_method_private_singleton
@@ -1825,7 +1820,7 @@ def test_define_method_private_singleton
     klass = util_get_class content, 'rb_cIO'
     read_method = klass.method_list.first
     assert_equal "read", read_method.name
-    assert_equal "Method Comment!   ", read_method.comment.text
+    assert_equal "Method Comment!", read_method.comment.text
     assert_equal :private, read_method.visibility
     assert read_method.singleton
   end
@@ -1855,7 +1850,7 @@ def test_define_method_singleton
     klass = util_get_class content, 'rb_cIO'
     read_method = klass.method_list.first
     assert_equal "read", read_method.name
-    assert_equal "Method Comment!   ", read_method.comment.text
+    assert_equal "Method Comment!", read_method.comment.text
     assert read_method.singleton
   end
 
diff --git a/test/rdoc/test_rdoc_parser_prism_ruby.rb b/test/rdoc/test_rdoc_parser_prism_ruby.rb
index 2ff11bb1a7..77a444e093 100644
--- a/test/rdoc/test_rdoc_parser_prism_ruby.rb
+++ b/test/rdoc/test_rdoc_parser_prism_ruby.rb
@@ -1967,6 +1967,58 @@ class C
 
     assert_equal expected, m.comment.parse
   end
+
+  def test_tomdoc_postprocess
+    RDoc::TomDoc.add_post_processor
+    util_parser <<~RUBY
+      # :markup: tomdoc
+
+      class C
+        # Public: foo
+        # bar
+        def m1; end
+
+        # Internal: baz
+        # blah
+        def m2; end
+      end
+    RUBY
+    klass = @top_level.classes.first
+    m1, m2 = klass.method_list
+    assert_equal 'Public', m1.section.title
+    assert_equal 'Internal', m2.section.title
+    assert_equal "foo\nbar", m1.comment.text.chomp
+    assert_equal "baz\nblah", m2.comment.text.chomp
+  end
+
+  def test_various_callseq
+    util_parser <<~RUBY
+      class Foo
+        # Undocumented form, maybe we should treat it as a single line call-seq
+        # :call-seq: foo1
+        #   bar1
+        #
+        # comment
+        def m1; end
+
+        # Blank line between
+        # :call-seq:
+        #   ARGF.readlines(a)
+        #   ARGF.readlines(b)
+        #
+        #   ARGF.readlines(c)
+        #
+        #   ARGF.readlines(d)
+        #
+        # comment
+        def m2; end
+      end
+    RUBY
+
+    m1, m2 = @top_level.classes.first.method_list
+    assert_equal "foo1\nbar1", m1.call_seq.chomp
+    assert_equal "ARGF.readlines(a)\nARGF.readlines(b)\nARGF.readlines(c)\nARGF.readlines(d)", m2.call_seq.chomp
+  end
 end
 
 class TestRDocParserPrismRuby < RDoc::TestCase
diff --git a/test/rdoc/test_rdoc_parser_simple.rb b/test/rdoc/test_rdoc_parser_simple.rb
index 4255cb97d8..e00eab064b 100644
--- a/test/rdoc/test_rdoc_parser_simple.rb
+++ b/test/rdoc/test_rdoc_parser_simple.rb
@@ -26,7 +26,7 @@ def test_initialize_metadata
 
     assert_includes @top_level.metadata, 'unhandled'
 
-    assert_equal ":unhandled: \n", parser.content
+    assert_equal "", parser.content
   end
 
   def test_remove_coding_comment
@@ -68,7 +68,7 @@ def test_remove_coding_comment
   #   # this is a comment
   #   #---
   #   # private text
-  #   #+++
+  #   #++
   #   # this is a rule:
   #   # ---