From 440cf93a708f45c619b57006573fe7451c8a9690 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Tue, 29 Aug 2023 16:18:14 -0400 Subject: [PATCH 1/2] Nest all of the YARP tests under the YARP namespace --- rakelib/lex.rake | 30 +- test/yarp/bom_test.rb | 96 +- test/yarp/comments_test.rb | 96 +- test/yarp/desugar_visitor_test.rb | 98 +- test/yarp/encoding_test.rb | 166 +-- test/yarp/errors_test.rb | 2088 ++++++++++++++--------------- test/yarp/heredoc_dedent_test.rb | 2 +- test/yarp/library_symbols_test.rb | 6 +- test/yarp/locals_test.rb | 178 +-- test/yarp/location_test.rb | 2 +- test/yarp/memsize_test.rb | 14 +- test/yarp/newline_test.rb | 138 +- test/yarp/parse_serialize_test.rb | 28 +- test/yarp/parse_test.rb | 324 ++--- test/yarp/regexp_test.rb | 338 ++--- test/yarp/ripper_compat_test.rb | 2 +- test/yarp/ruby_api_test.rb | 88 +- test/yarp/test_helper.rb | 30 +- test/yarp/unescape_test.rb | 14 +- test/yarp/version_test.rb | 8 +- 20 files changed, 1885 insertions(+), 1861 deletions(-) diff --git a/rakelib/lex.rake b/rakelib/lex.rake index 457289e0a07..91faa233625 100644 --- a/rakelib/lex.rake +++ b/rakelib/lex.rake @@ -303,21 +303,23 @@ task "parse:topgems": ["download:topgems", :compile] do require "yarp" require_relative "../test/yarp/test_helper" - class ParseTop100GemsTest < Test::Unit::TestCase - Dir["#{TOP_100_GEMS_DIR}/**/*.rb"].each do |filepath| - test filepath do - result = YARP.parse_file(filepath) - - if TOP_100_GEMS_INVALID_SYNTAX_PREFIXES.any? { |prefix| filepath.start_with?(prefix) } - assert_false result.success? - # ensure it is actually invalid syntax - assert_false system(RbConfig.ruby, "-c", filepath, out: File::NULL, err: File::NULL) - next - end + module YARP + class ParseTop100GemsTest < TestCase + Dir["#{TOP_100_GEMS_DIR}/**/*.rb"].each do |filepath| + test filepath do + result = YARP.parse_file(filepath) + + if TOP_100_GEMS_INVALID_SYNTAX_PREFIXES.any? { |prefix| filepath.start_with?(prefix) } + assert_false result.success? + # ensure it is actually invalid syntax + assert_false system(RbConfig.ruby, "-c", filepath, out: File::NULL, err: File::NULL) + next + end - assert result.success?, "failed to parse #{filepath}" - value = result.value - assert_valid_locations(value) + assert result.success?, "failed to parse #{filepath}" + value = result.value + assert_valid_locations(value) + end end end end diff --git a/test/yarp/bom_test.rb b/test/yarp/bom_test.rb index b386a5d9a33..3a4e04a900a 100644 --- a/test/yarp/bom_test.rb +++ b/test/yarp/bom_test.rb @@ -6,52 +6,54 @@ require_relative "test_helper" -class BOMTest < Test::Unit::TestCase - def test_ident - assert_bom("foo") - end - - def test_back_reference - assert_bom("$+") - end - - def test_instance_variable - assert_bom("@foo") - end - - def test_class_variable - assert_bom("@@foo") - end - - def test_global_variable - assert_bom("$foo") - end - - def test_numbered_reference - assert_bom("$1") - end - - def test_percents - assert_bom("%i[]") - assert_bom("%r[]") - assert_bom("%s[]") - assert_bom("%q{}") - assert_bom("%w[]") - assert_bom("%x[]") - assert_bom("%I[]") - assert_bom("%W[]") - assert_bom("%Q{}") - end - - def test_string - assert_bom("\"\"") - assert_bom("''") - end - - private - - def assert_bom(source) - bommed = "\xEF\xBB\xBF#{source}" - assert_equal YARP.lex_ripper(bommed), YARP.lex_compat(bommed).value +module YARP + class BOMTest < TestCase + def test_ident + assert_bom("foo") + end + + def test_back_reference + assert_bom("$+") + end + + def test_instance_variable + assert_bom("@foo") + end + + def test_class_variable + assert_bom("@@foo") + end + + def test_global_variable + assert_bom("$foo") + end + + def test_numbered_reference + assert_bom("$1") + end + + def test_percents + assert_bom("%i[]") + assert_bom("%r[]") + assert_bom("%s[]") + assert_bom("%q{}") + assert_bom("%w[]") + assert_bom("%x[]") + assert_bom("%I[]") + assert_bom("%W[]") + assert_bom("%Q{}") + end + + def test_string + assert_bom("\"\"") + assert_bom("''") + end + + private + + def assert_bom(source) + bommed = "\xEF\xBB\xBF#{source}" + assert_equal YARP.lex_ripper(bommed), YARP.lex_compat(bommed).value + end end end diff --git a/test/yarp/comments_test.rb b/test/yarp/comments_test.rb index 13bf819a4eb..ac91eab4ac5 100644 --- a/test/yarp/comments_test.rb +++ b/test/yarp/comments_test.rb @@ -2,67 +2,67 @@ require_relative "test_helper" -class CommentsTest < Test::Unit::TestCase - include ::YARP::DSL +module YARP + class CommentsTest < TestCase + def test_comment_inline + source = "# comment" - def test_comment_inline - source = "# comment" + assert_comment source, :inline, 0..9 + assert_equal [0], Debug.newlines(source) + end - assert_comment source, :inline, 0..9 - assert_equal [0], YARP.const_get(:Debug).newlines(source) - end + def test_comment_inline_def + source = <<~RUBY + def foo + # a comment + end + RUBY - def test_comment_inline_def - source = <<~RUBY - def foo - # a comment + assert_comment source, :inline, 10..22 end - RUBY - - assert_comment source, :inline, 10..22 - end - def test_comment___END__ - source = <<~RUBY - __END__ - comment - RUBY + def test_comment___END__ + source = <<~RUBY + __END__ + comment + RUBY - assert_comment source, :__END__, 0..16 - end + assert_comment source, :__END__, 0..16 + end - def test_comment___END__crlf - source = "__END__\r\ncomment\r\n" + def test_comment___END__crlf + source = "__END__\r\ncomment\r\n" - assert_comment source, :__END__, 0..18 - end + assert_comment source, :__END__, 0..18 + end - def test_comment_embedded_document - source = <<~RUBY - =begin - comment - =end - RUBY + def test_comment_embedded_document + source = <<~RUBY + =begin + comment + =end + RUBY - assert_comment source, :embdoc, 0..20 - end + assert_comment source, :embdoc, 0..20 + end - def test_comment_embedded_document_with_content_on_same_line - source = <<~RUBY - =begin other stuff - =end - RUBY + def test_comment_embedded_document_with_content_on_same_line + source = <<~RUBY + =begin other stuff + =end + RUBY - assert_comment source, :embdoc, 0..24 - end + assert_comment source, :embdoc, 0..24 + end - private + private - def assert_comment(source, type, location) - result = YARP.parse(source) - assert result.errors.empty?, result.errors.map(&:message).join("\n") - assert_equal result.comments.first.type, type - assert_equal result.comments.first.location.start_offset, location.begin - assert_equal result.comments.first.location.end_offset, location.end + def assert_comment(source, type, location) + result = YARP.parse(source) + assert result.errors.empty?, result.errors.map(&:message).join("\n") + assert_equal result.comments.first.type, type + assert_equal result.comments.first.location.start_offset, location.begin + assert_equal result.comments.first.location.end_offset, location.end + end end end diff --git a/test/yarp/desugar_visitor_test.rb b/test/yarp/desugar_visitor_test.rb index de635966a1d..4dee182ef1b 100644 --- a/test/yarp/desugar_visitor_test.rb +++ b/test/yarp/desugar_visitor_test.rb @@ -2,56 +2,58 @@ require_relative "test_helper" -class DesugarVisitorTest < Test::Unit::TestCase - def test_and_write - assert_desugars("(AndNode (ClassVariableReadNode) (ClassVariableWriteNode (CallNode)))", "@@foo &&= bar") - assert_desugars("(AndNode (ConstantPathNode (ConstantReadNode) (ConstantReadNode)) (ConstantPathWriteNode (ConstantPathNode (ConstantReadNode) (ConstantReadNode)) (CallNode)))", "Foo::Bar &&= baz") - assert_desugars("(AndNode (ConstantReadNode) (ConstantWriteNode (CallNode)))", "Foo &&= bar") - assert_desugars("(AndNode (GlobalVariableReadNode) (GlobalVariableWriteNode (CallNode)))", "$foo &&= bar") - assert_desugars("(AndNode (InstanceVariableReadNode) (InstanceVariableWriteNode (CallNode)))", "@foo &&= bar") - assert_desugars("(AndNode (LocalVariableReadNode) (LocalVariableWriteNode (CallNode)))", "foo &&= bar") - assert_desugars("(AndNode (LocalVariableReadNode) (LocalVariableWriteNode (CallNode)))", "foo = 1; foo &&= bar") - end - - def test_or_write - assert_desugars("(IfNode (DefinedNode (ClassVariableReadNode)) (StatementsNode (ClassVariableReadNode)) (ElseNode (StatementsNode (ClassVariableWriteNode (CallNode)))))", "@@foo ||= bar") - assert_desugars("(IfNode (DefinedNode (ConstantPathNode (ConstantReadNode) (ConstantReadNode))) (StatementsNode (ConstantPathNode (ConstantReadNode) (ConstantReadNode))) (ElseNode (StatementsNode (ConstantPathWriteNode (ConstantPathNode (ConstantReadNode) (ConstantReadNode)) (CallNode)))))", "Foo::Bar ||= baz") - assert_desugars("(IfNode (DefinedNode (ConstantReadNode)) (StatementsNode (ConstantReadNode)) (ElseNode (StatementsNode (ConstantWriteNode (CallNode)))))", "Foo ||= bar") - assert_desugars("(IfNode (DefinedNode (GlobalVariableReadNode)) (StatementsNode (GlobalVariableReadNode)) (ElseNode (StatementsNode (GlobalVariableWriteNode (CallNode)))))", "$foo ||= bar") - assert_desugars("(OrNode (InstanceVariableReadNode) (InstanceVariableWriteNode (CallNode)))", "@foo ||= bar") - assert_desugars("(OrNode (LocalVariableReadNode) (LocalVariableWriteNode (CallNode)))", "foo ||= bar") - assert_desugars("(OrNode (LocalVariableReadNode) (LocalVariableWriteNode (CallNode)))", "foo = 1; foo ||= bar") - end - - def test_operator_write - assert_desugars("(ClassVariableWriteNode (CallNode (ClassVariableReadNode) (ArgumentsNode (CallNode))))", "@@foo += bar") - assert_desugars("(ConstantPathWriteNode (ConstantPathNode (ConstantReadNode) (ConstantReadNode)) (CallNode (ConstantPathNode (ConstantReadNode) (ConstantReadNode)) (ArgumentsNode (CallNode))))", "Foo::Bar += baz") - assert_desugars("(ConstantWriteNode (CallNode (ConstantReadNode) (ArgumentsNode (CallNode))))", "Foo += bar") - assert_desugars("(GlobalVariableWriteNode (CallNode (GlobalVariableReadNode) (ArgumentsNode (CallNode))))", "$foo += bar") - assert_desugars("(InstanceVariableWriteNode (CallNode (InstanceVariableReadNode) (ArgumentsNode (CallNode))))", "@foo += bar") - assert_desugars("(LocalVariableWriteNode (CallNode (LocalVariableReadNode) (ArgumentsNode (CallNode))))", "foo += bar") - assert_desugars("(LocalVariableWriteNode (CallNode (LocalVariableReadNode) (ArgumentsNode (CallNode))))", "foo = 1; foo += bar") - end - - private - - def ast_inspect(node) - parts = [node.class.name.split("::").last] - - node.deconstruct_keys(nil).each do |_, value| - case value - when YARP::Node - parts << ast_inspect(value) - when Array - parts.concat(value.map { |element| ast_inspect(element) }) - end +module YARP + class DesugarVisitorTest < TestCase + def test_and_write + assert_desugars("(AndNode (ClassVariableReadNode) (ClassVariableWriteNode (CallNode)))", "@@foo &&= bar") + assert_desugars("(AndNode (ConstantPathNode (ConstantReadNode) (ConstantReadNode)) (ConstantPathWriteNode (ConstantPathNode (ConstantReadNode) (ConstantReadNode)) (CallNode)))", "Foo::Bar &&= baz") + assert_desugars("(AndNode (ConstantReadNode) (ConstantWriteNode (CallNode)))", "Foo &&= bar") + assert_desugars("(AndNode (GlobalVariableReadNode) (GlobalVariableWriteNode (CallNode)))", "$foo &&= bar") + assert_desugars("(AndNode (InstanceVariableReadNode) (InstanceVariableWriteNode (CallNode)))", "@foo &&= bar") + assert_desugars("(AndNode (LocalVariableReadNode) (LocalVariableWriteNode (CallNode)))", "foo &&= bar") + assert_desugars("(AndNode (LocalVariableReadNode) (LocalVariableWriteNode (CallNode)))", "foo = 1; foo &&= bar") + end + + def test_or_write + assert_desugars("(IfNode (DefinedNode (ClassVariableReadNode)) (StatementsNode (ClassVariableReadNode)) (ElseNode (StatementsNode (ClassVariableWriteNode (CallNode)))))", "@@foo ||= bar") + assert_desugars("(IfNode (DefinedNode (ConstantPathNode (ConstantReadNode) (ConstantReadNode))) (StatementsNode (ConstantPathNode (ConstantReadNode) (ConstantReadNode))) (ElseNode (StatementsNode (ConstantPathWriteNode (ConstantPathNode (ConstantReadNode) (ConstantReadNode)) (CallNode)))))", "Foo::Bar ||= baz") + assert_desugars("(IfNode (DefinedNode (ConstantReadNode)) (StatementsNode (ConstantReadNode)) (ElseNode (StatementsNode (ConstantWriteNode (CallNode)))))", "Foo ||= bar") + assert_desugars("(IfNode (DefinedNode (GlobalVariableReadNode)) (StatementsNode (GlobalVariableReadNode)) (ElseNode (StatementsNode (GlobalVariableWriteNode (CallNode)))))", "$foo ||= bar") + assert_desugars("(OrNode (InstanceVariableReadNode) (InstanceVariableWriteNode (CallNode)))", "@foo ||= bar") + assert_desugars("(OrNode (LocalVariableReadNode) (LocalVariableWriteNode (CallNode)))", "foo ||= bar") + assert_desugars("(OrNode (LocalVariableReadNode) (LocalVariableWriteNode (CallNode)))", "foo = 1; foo ||= bar") end + + def test_operator_write + assert_desugars("(ClassVariableWriteNode (CallNode (ClassVariableReadNode) (ArgumentsNode (CallNode))))", "@@foo += bar") + assert_desugars("(ConstantPathWriteNode (ConstantPathNode (ConstantReadNode) (ConstantReadNode)) (CallNode (ConstantPathNode (ConstantReadNode) (ConstantReadNode)) (ArgumentsNode (CallNode))))", "Foo::Bar += baz") + assert_desugars("(ConstantWriteNode (CallNode (ConstantReadNode) (ArgumentsNode (CallNode))))", "Foo += bar") + assert_desugars("(GlobalVariableWriteNode (CallNode (GlobalVariableReadNode) (ArgumentsNode (CallNode))))", "$foo += bar") + assert_desugars("(InstanceVariableWriteNode (CallNode (InstanceVariableReadNode) (ArgumentsNode (CallNode))))", "@foo += bar") + assert_desugars("(LocalVariableWriteNode (CallNode (LocalVariableReadNode) (ArgumentsNode (CallNode))))", "foo += bar") + assert_desugars("(LocalVariableWriteNode (CallNode (LocalVariableReadNode) (ArgumentsNode (CallNode))))", "foo = 1; foo += bar") + end + + private + + def ast_inspect(node) + parts = [node.class.name.split("::").last] + + node.deconstruct_keys(nil).each do |_, value| + case value + when Node + parts << ast_inspect(value) + when Array + parts.concat(value.map { |element| ast_inspect(element) }) + end + end - "(#{parts.join(" ")})" - end + "(#{parts.join(" ")})" + end - def assert_desugars(expected, source) - ast = YARP.parse(source).value.accept(YARP::DesugarVisitor.new) - assert_equal expected, ast_inspect(ast.statements.body.last) + def assert_desugars(expected, source) + ast = YARP.parse(source).value.accept(DesugarVisitor.new) + assert_equal expected, ast_inspect(ast.statements.body.last) + end end end diff --git a/test/yarp/encoding_test.rb b/test/yarp/encoding_test.rb index 2ee084cd380..828b45be730 100644 --- a/test/yarp/encoding_test.rb +++ b/test/yarp/encoding_test.rb @@ -2,97 +2,99 @@ require_relative "test_helper" -class EncodingTest < Test::Unit::TestCase - %w[ - ascii - ascii-8bit - big5 - binary - euc-jp - gbk - iso-8859-1 - iso-8859-2 - iso-8859-3 - iso-8859-4 - iso-8859-5 - iso-8859-6 - iso-8859-7 - iso-8859-8 - iso-8859-9 - iso-8859-10 - iso-8859-11 - iso-8859-13 - iso-8859-14 - iso-8859-15 - iso-8859-16 - koi8-r - shift_jis - sjis - us-ascii - utf-8 - utf8-mac - windows-31j - windows-1251 - windows-1252 - CP1251 - CP1252 - ].each do |encoding| - define_method "test_encoding_#{encoding}" do - result = YARP.parse("# encoding: #{encoding}\nident") - actual = result.value.statements.body.first.name.encoding - assert_equal Encoding.find(encoding), actual +module YARP + class EncodingTest < TestCase + %w[ + ascii + ascii-8bit + big5 + binary + euc-jp + gbk + iso-8859-1 + iso-8859-2 + iso-8859-3 + iso-8859-4 + iso-8859-5 + iso-8859-6 + iso-8859-7 + iso-8859-8 + iso-8859-9 + iso-8859-10 + iso-8859-11 + iso-8859-13 + iso-8859-14 + iso-8859-15 + iso-8859-16 + koi8-r + shift_jis + sjis + us-ascii + utf-8 + utf8-mac + windows-31j + windows-1251 + windows-1252 + CP1251 + CP1252 + ].each do |encoding| + define_method "test_encoding_#{encoding}" do + result = YARP.parse("# encoding: #{encoding}\nident") + actual = result.value.statements.body.first.name.encoding + assert_equal Encoding.find(encoding), actual + end end - end - def test_coding - result = YARP.parse("# coding: utf-8\nident") - actual = result.value.statements.body.first.name.encoding - assert_equal Encoding.find("utf-8"), actual - end + def test_coding + result = YARP.parse("# coding: utf-8\nident") + actual = result.value.statements.body.first.name.encoding + assert_equal Encoding.find("utf-8"), actual + end - def test_coding_with_whitespace - result = YARP.parse("# coding \t \r \v : \t \v \r ascii-8bit \nident") - actual = result.value.statements.body.first.name.encoding - assert_equal Encoding.find("ascii-8bit"), actual - end + def test_coding_with_whitespace + result = YARP.parse("# coding \t \r \v : \t \v \r ascii-8bit \nident") + actual = result.value.statements.body.first.name.encoding + assert_equal Encoding.find("ascii-8bit"), actual + end - def test_emacs_style - result = YARP.parse("# -*- coding: utf-8 -*-\nident") - actual = result.value.statements.body.first.name.encoding - assert_equal Encoding.find("utf-8"), actual - end + def test_emacs_style + result = YARP.parse("# -*- coding: utf-8 -*-\nident") + actual = result.value.statements.body.first.name.encoding + assert_equal Encoding.find("utf-8"), actual + end - # This test may be a little confusing. Basically when we use our strpbrk, it - # takes into account the encoding of the file. - def test_strpbrk_multibyte - result = YARP.parse(<<~RUBY) - # encoding: Shift_JIS - %w[\x81\x5c] - RUBY + # This test may be a little confusing. Basically when we use our strpbrk, it + # takes into account the encoding of the file. + def test_strpbrk_multibyte + result = YARP.parse(<<~RUBY) + # encoding: Shift_JIS + %w[\x81\x5c] + RUBY - assert(result.errors.empty?) - assert_equal( - (+"\x81\x5c").force_encoding(Encoding::Shift_JIS), - result.value.statements.body.first.elements.first.unescaped - ) - end + assert(result.errors.empty?) + assert_equal( + (+"\x81\x5c").force_encoding(Encoding::Shift_JIS), + result.value.statements.body.first.elements.first.unescaped + ) + end - def test_utf_8_variations - %w[ - utf-8-unix - utf-8-dos - utf-8-mac - utf-8-* - ].each do |encoding| - result = YARP.parse("# coding: #{encoding}\nident") - actual = result.value.statements.body.first.name.encoding - assert_equal Encoding.find("utf-8"), actual + def test_utf_8_variations + %w[ + utf-8-unix + utf-8-dos + utf-8-mac + utf-8-* + ].each do |encoding| + result = YARP.parse("# coding: #{encoding}\nident") + actual = result.value.statements.body.first.name.encoding + assert_equal Encoding.find("utf-8"), actual + end end - end - def test_first_lexed_token - encoding = YARP.lex("# encoding: ascii-8bit").value[0][0].value.encoding - assert_equal Encoding.find("ascii-8bit"), encoding + def test_first_lexed_token + encoding = YARP.lex("# encoding: ascii-8bit").value[0][0].value.encoding + assert_equal Encoding.find("ascii-8bit"), encoding + end end end diff --git a/test/yarp/errors_test.rb b/test/yarp/errors_test.rb index e461afbbd26..0e1931ec6ac 100644 --- a/test/yarp/errors_test.rb +++ b/test/yarp/errors_test.rb @@ -2,1136 +2,1136 @@ require_relative "test_helper" -class ErrorsTest < Test::Unit::TestCase - include ::YARP::DSL - - def test_constant_path_with_invalid_token_after - assert_error_messages "A::$b", [ - "Expected identifier or constant after '::'", - "Expected a newline or semicolon after statement." - ] - end - - def test_module_name_recoverable - expected = ModuleNode( - [], - Location(), - ConstantReadNode(), - StatementsNode( - [ModuleNode([], Location(), MissingNode(), nil, Location(), "")] - ), - Location(), - "Parent" - ) - - assert_errors expected, "module Parent module end", [ - ["Expected to find a module name after `module`.", 20..20] - ] - end - - def test_for_loops_index_missing - expected = ForNode( - MissingNode(), - expression("1..10"), - StatementsNode([expression("i")]), - Location(), - Location(), - nil, - Location() - ) - - assert_errors expected, "for in 1..10\ni\nend", [ - ["Expected index after for.", 0..0] - ] - end - - def test_for_loops_only_end - expected = ForNode( - MissingNode(), - MissingNode(), - nil, - Location(), - Location(), - nil, - Location() - ) - - assert_errors expected, "for end", [ - ["Expected index after for.", 0..0], - ["Expected keyword in.", 3..3], - ["Expected collection.", 3..3] - ] - end - - def test_pre_execution_missing_brace - expected = PreExecutionNode( - StatementsNode([expression("1")]), - Location(), - Location(), - Location() - ) - - assert_errors expected, "BEGIN 1 }", [ - ["Expected '{' after 'BEGIN'.", 5..5] - ] - end +module YARP + class ErrorsTest < TestCase + include DSL + + def test_constant_path_with_invalid_token_after + assert_error_messages "A::$b", [ + "Expected identifier or constant after '::'", + "Expected a newline or semicolon after statement." + ] + end - def test_pre_execution_context - expected = PreExecutionNode( - StatementsNode([ - CallNode( - expression("1"), - nil, - Location(), - nil, - ArgumentsNode([MissingNode()]), - nil, - nil, - 0, - "+" - ) - ]), - Location(), - Location(), - Location() - ) - - assert_errors expected, "BEGIN { 1 + }", [ - ["Expected a value after the operator.", 11..11] - ] - end + def test_module_name_recoverable + expected = ModuleNode( + [], + Location(), + ConstantReadNode(), + StatementsNode( + [ModuleNode([], Location(), MissingNode(), nil, Location(), "")] + ), + Location(), + "Parent" + ) - def test_unterminated_embdoc - assert_errors expression("1"), "1\n=begin\n", [ - ["Unterminated embdoc", 2..9] - ] - end + assert_errors expected, "module Parent module end", [ + ["Expected to find a module name after `module`.", 20..20] + ] + end - def test_unterminated_i_list - assert_errors expression("%i["), "%i[", [ - ["Expected a closing delimiter for a `%i` list.", 3..3] - ] - end + def test_for_loops_index_missing + expected = ForNode( + MissingNode(), + expression("1..10"), + StatementsNode([expression("i")]), + Location(), + Location(), + nil, + Location() + ) - def test_unterminated_w_list - assert_errors expression("%w["), "%w[", [ - ["Expected a closing delimiter for a `%w` list.", 3..3] - ] - end + assert_errors expected, "for in 1..10\ni\nend", [ + ["Expected index after for.", 0..0] + ] + end - def test_unterminated_W_list - assert_errors expression("%W["), "%W[", [ - ["Expected a closing delimiter for a `%W` list.", 3..3] - ] - end + def test_for_loops_only_end + expected = ForNode( + MissingNode(), + MissingNode(), + nil, + Location(), + Location(), + nil, + Location() + ) - def test_unterminated_regular_expression - assert_errors expression("/hello"), "/hello", [ - ["Expected a closing delimiter for a regular expression.", 1..1] - ] - end + assert_errors expected, "for end", [ + ["Expected index after for.", 0..0], + ["Expected keyword in.", 3..3], + ["Expected collection.", 3..3] + ] + end - def test_unterminated_regular_expression_with_heredoc - source = "<<-END + /b\nEND\n" + def test_pre_execution_missing_brace + expected = PreExecutionNode( + StatementsNode([expression("1")]), + Location(), + Location(), + Location() + ) - assert_errors expression(source), source, [ - ["Expected a closing delimiter for a regular expression.", 10..10] - ] - end + assert_errors expected, "BEGIN 1 }", [ + ["Expected '{' after 'BEGIN'.", 5..5] + ] + end - def test_unterminated_xstring - assert_errors expression("`hello"), "`hello", [ - ["Expected a closing delimiter for an xstring.", 1..1] - ] - end + def test_pre_execution_context + expected = PreExecutionNode( + StatementsNode([ + CallNode( + expression("1"), + nil, + Location(), + nil, + ArgumentsNode([MissingNode()]), + nil, + nil, + 0, + "+" + ) + ]), + Location(), + Location(), + Location() + ) - def test_unterminated_string - assert_errors expression('"hello'), '"hello', [ - ["Expected a closing delimiter for an interpolated string.", 1..1] - ] - end + assert_errors expected, "BEGIN { 1 + }", [ + ["Expected a value after the operator.", 11..11] + ] + end - def test_unterminated_s_symbol - assert_errors expression("%s[abc"), "%s[abc", [ - ["Expected a closing delimiter for a dynamic symbol.", 3..3] - ] - end + def test_unterminated_embdoc + assert_errors expression("1"), "1\n=begin\n", [ + ["Unterminated embdoc", 2..9] + ] + end - def test_unterminated_parenthesized_expression - assert_errors expression('(1 + 2'), '(1 + 2', [ - ["Expected to be able to parse an expression.", 6..6], - ["Expected a closing parenthesis.", 6..6] - ] - end + def test_unterminated_i_list + assert_errors expression("%i["), "%i[", [ + ["Expected a closing delimiter for a `%i` list.", 3..3] + ] + end - def test_unterminated_argument_expression - assert_errors expression('a %'), 'a %', [ - ["Unexpected end of input", 2..3], - ["Expected a value after the operator.", 3..3], - ] - end + def test_unterminated_w_list + assert_errors expression("%w["), "%w[", [ + ["Expected a closing delimiter for a `%w` list.", 3..3] + ] + end - def test_cr_without_lf_in_percent_expression - assert_errors expression("%\r"), "%\r", [ - ["Invalid %% token", 0..2], - ] - end + def test_unterminated_W_list + assert_errors expression("%W["), "%W[", [ + ["Expected a closing delimiter for a `%W` list.", 3..3] + ] + end - def test_1_2_3 - assert_errors expression("(1, 2, 3)"), "(1, 2, 3)", [ - ["Expected to be able to parse an expression.", 2..2], - ["Expected a closing parenthesis.", 2..2], - ["Expected a newline or semicolon after statement.", 2..2], - ["Expected to be able to parse an expression.", 2..2], - ["Expected a newline or semicolon after statement.", 5..5], - ["Expected to be able to parse an expression.", 5..5], - ["Expected a newline or semicolon after statement.", 8..8], - ["Expected to be able to parse an expression.", 8..8], - ] - end + def test_unterminated_regular_expression + assert_errors expression("/hello"), "/hello", [ + ["Expected a closing delimiter for a regular expression.", 1..1] + ] + end - def test_return_1_2_3 - assert_error_messages "return(1, 2, 3)", [ - "Expected to be able to parse an expression.", - "Expected a closing parenthesis.", - "Expected a newline or semicolon after statement.", - "Expected to be able to parse an expression." - ] - end + def test_unterminated_regular_expression_with_heredoc + source = "<<-END + /b\nEND\n" - def test_return_1 - assert_errors expression("return 1,;"), "return 1,;", [ - ["Expected to be able to parse an argument.", 9..9] - ] - end + assert_errors expression(source), source, [ + ["Expected a closing delimiter for a regular expression.", 10..10] + ] + end - def test_next_1_2_3 - assert_errors expression("next(1, 2, 3)"), "next(1, 2, 3)", [ - ["Expected to be able to parse an expression.", 6..6], - ["Expected a closing parenthesis.", 6..6], - ["Expected a newline or semicolon after statement.", 12..12], - ["Expected to be able to parse an expression.", 12..12] - ] - end + def test_unterminated_xstring + assert_errors expression("`hello"), "`hello", [ + ["Expected a closing delimiter for an xstring.", 1..1] + ] + end - def test_next_1 - assert_errors expression("next 1,;"), "next 1,;", [ - ["Expected to be able to parse an argument.", 7..7] - ] - end + def test_unterminated_string + assert_errors expression('"hello'), '"hello', [ + ["Expected a closing delimiter for an interpolated string.", 1..1] + ] + end - def test_break_1_2_3 - assert_errors expression("break(1, 2, 3)"), "break(1, 2, 3)", [ - ["Expected to be able to parse an expression.", 7..7], - ["Expected a closing parenthesis.", 7..7], - ["Expected a newline or semicolon after statement.", 13..13], - ["Expected to be able to parse an expression.", 13..13], - ] - end + def test_unterminated_s_symbol + assert_errors expression("%s[abc"), "%s[abc", [ + ["Expected a closing delimiter for a dynamic symbol.", 3..3] + ] + end - def test_break_1 - assert_errors expression("break 1,;"), "break 1,;", [ - ["Expected to be able to parse an argument.", 8..8] - ] - end + def test_unterminated_parenthesized_expression + assert_errors expression('(1 + 2'), '(1 + 2', [ + ["Expected to be able to parse an expression.", 6..6], + ["Expected a closing parenthesis.", 6..6] + ] + end - def test_argument_forwarding_when_parent_is_not_forwarding - assert_errors expression('def a(x, y, z); b(...); end'), 'def a(x, y, z); b(...); end', [ - ["unexpected ... when parent method is not forwarding.", 18..21] - ] - end + def test_unterminated_argument_expression + assert_errors expression('a %'), 'a %', [ + ["Unexpected end of input", 2..3], + ["Expected a value after the operator.", 3..3], + ] + end - def test_argument_forwarding_only_effects_its_own_internals - assert_errors expression('def a(...); b(...); end; def c(x, y, z); b(...); end'), - 'def a(...); b(...); end; def c(x, y, z); b(...); end', [ - ["unexpected ... when parent method is not forwarding.", 43..46] + def test_cr_without_lf_in_percent_expression + assert_errors expression("%\r"), "%\r", [ + ["Invalid %% token", 0..2], ] - end + end - def test_top_level_constant_with_downcased_identifier - assert_error_messages "::foo", [ - "Expected a constant after ::.", - "Expected a newline or semicolon after statement." - ] - end + def test_1_2_3 + assert_errors expression("(1, 2, 3)"), "(1, 2, 3)", [ + ["Expected to be able to parse an expression.", 2..2], + ["Expected a closing parenthesis.", 2..2], + ["Expected a newline or semicolon after statement.", 2..2], + ["Expected to be able to parse an expression.", 2..2], + ["Expected a newline or semicolon after statement.", 5..5], + ["Expected to be able to parse an expression.", 5..5], + ["Expected a newline or semicolon after statement.", 8..8], + ["Expected to be able to parse an expression.", 8..8], + ] + end - def test_top_level_constant_starting_with_downcased_identifier - assert_error_messages "::foo::A", [ - "Expected a constant after ::.", - "Expected a newline or semicolon after statement." - ] - end + def test_return_1_2_3 + assert_error_messages "return(1, 2, 3)", [ + "Expected to be able to parse an expression.", + "Expected a closing parenthesis.", + "Expected a newline or semicolon after statement.", + "Expected to be able to parse an expression." + ] + end - def test_aliasing_global_variable_with_non_global_variable - assert_errors expression("alias $a b"), "alias $a b", [ - ["Expected a global variable.", 9..10] - ] - end + def test_return_1 + assert_errors expression("return 1,;"), "return 1,;", [ + ["Expected to be able to parse an argument.", 9..9] + ] + end - def test_aliasing_non_global_variable_with_global_variable - assert_errors expression("alias a $b"), "alias a $b", [ - ["Expected a bare word or symbol argument.", 8..10] - ] - end + def test_next_1_2_3 + assert_errors expression("next(1, 2, 3)"), "next(1, 2, 3)", [ + ["Expected to be able to parse an expression.", 6..6], + ["Expected a closing parenthesis.", 6..6], + ["Expected a newline or semicolon after statement.", 12..12], + ["Expected to be able to parse an expression.", 12..12] + ] + end - def test_aliasing_global_variable_with_global_number_variable - assert_errors expression("alias $a $1"), "alias $a $1", [ - ["Can't make alias for number variables.", 9..11] - ] - end + def test_next_1 + assert_errors expression("next 1,;"), "next 1,;", [ + ["Expected to be able to parse an argument.", 7..7] + ] + end - def test_def_with_expression_receiver_and_no_identifier - assert_errors expression("def (a); end"), "def (a); end", [ - ["Expected '.' or '::' after receiver", 7..7] - ] - end + def test_break_1_2_3 + assert_errors expression("break(1, 2, 3)"), "break(1, 2, 3)", [ + ["Expected to be able to parse an expression.", 7..7], + ["Expected a closing parenthesis.", 7..7], + ["Expected a newline or semicolon after statement.", 13..13], + ["Expected to be able to parse an expression.", 13..13], + ] + end - def test_def_with_multiple_statements_receiver - assert_errors expression("def (\na\nb\n).c; end"), "def (\na\nb\n).c; end", [ - ["Expected closing ')' for receiver.", 7..7], - ["Expected '.' or '::' after receiver", 7..7], - ["Expected to be able to parse an expression.", 10..10], - ["Expected to be able to parse an expression.", 11..11] - ] - end + def test_break_1 + assert_errors expression("break 1,;"), "break 1,;", [ + ["Expected to be able to parse an argument.", 8..8] + ] + end - def test_def_with_empty_expression_receiver - assert_errors expression("def ().a; end"), "def ().a; end", [ - ["Expected to be able to parse receiver.", 5..5] - ] - end + def test_argument_forwarding_when_parent_is_not_forwarding + assert_errors expression('def a(x, y, z); b(...); end'), 'def a(x, y, z); b(...); end', [ + ["unexpected ... when parent method is not forwarding.", 18..21] + ] + end - def test_block_beginning_with_brace_and_ending_with_end - assert_error_messages "x.each { x end", [ - "Expected a newline or semicolon after statement.", - "Expected to be able to parse an expression.", - "Expected to be able to parse an expression.", - "Expected block beginning with '{' to end with '}'." - ] - end + def test_argument_forwarding_only_effects_its_own_internals + assert_errors expression('def a(...); b(...); end; def c(x, y, z); b(...); end'), + 'def a(...); b(...); end; def c(x, y, z); b(...); end', [ + ["unexpected ... when parent method is not forwarding.", 43..46] + ] + end - def test_double_splat_followed_by_splat_argument - expected = CallNode( - nil, - nil, - Location(), - Location(), - ArgumentsNode([ - KeywordHashNode([AssocSplatNode(expression("kwargs"), Location())]), - SplatNode(Location(), expression("args")) - ]), - Location(), - nil, - 0, - "a" - ) - - assert_errors expected, "a(**kwargs, *args)", [ - ["Unexpected splat argument after double splat.", 12..17] - ] - end + def test_top_level_constant_with_downcased_identifier + assert_error_messages "::foo", [ + "Expected a constant after ::.", + "Expected a newline or semicolon after statement." + ] + end - def test_arguments_after_block - expected = CallNode( - nil, - nil, - Location(), - Location(), - ArgumentsNode([ - BlockArgumentNode(expression("block"), Location()), - expression("foo") - ]), - Location(), - nil, - 0, - "a" - ) - - assert_errors expected, "a(&block, foo)", [ - ["Unexpected argument after block argument.", 10..13] - ] - end + def test_top_level_constant_starting_with_downcased_identifier + assert_error_messages "::foo::A", [ + "Expected a constant after ::.", + "Expected a newline or semicolon after statement." + ] + end - def test_arguments_binding_power_for_and - assert_error_messages "foo(*bar and baz)", [ - "Expected a ')' to close the argument list.", - "Expected a newline or semicolon after statement.", - "Expected to be able to parse an expression." - ] - end + def test_aliasing_global_variable_with_non_global_variable + assert_errors expression("alias $a b"), "alias $a b", [ + ["Expected a global variable.", 9..10] + ] + end - def test_splat_argument_after_keyword_argument - expected = CallNode( - nil, - nil, - Location(), - Location(), - ArgumentsNode([ - KeywordHashNode( - [AssocNode( - SymbolNode(nil, Location(), Location(), "foo"), - expression("bar"), - nil - )] - ), - SplatNode(Location(), expression("args")) - ]), - Location(), - nil, - 0, - "a" - ) - - assert_errors expected, "a(foo: bar, *args)", [ - ["Unexpected splat argument after double splat.", 12..17] - ] - end + def test_aliasing_non_global_variable_with_global_variable + assert_errors expression("alias a $b"), "alias a $b", [ + ["Expected a bare word or symbol argument.", 8..10] + ] + end - def test_module_definition_in_method_body - expected = DefNode( - Location(), - nil, - nil, - StatementsNode([ModuleNode([], Location(), ConstantReadNode(), nil, Location(), "A")]), - [], - Location(), - nil, - nil, - nil, - nil, - Location() - ) - - assert_errors expected, "def foo;module A;end;end", [ - ["Module definition in method body", 8..14] - ] - end + def test_aliasing_global_variable_with_global_number_variable + assert_errors expression("alias $a $1"), "alias $a $1", [ + ["Can't make alias for number variables.", 9..11] + ] + end - def test_module_definition_in_method_body_within_block - expected = DefNode( - Location(), - nil, - nil, - StatementsNode( - [CallNode( - nil, - nil, - Location(), - nil, - nil, - nil, - BlockNode( - [], - nil, - StatementsNode([ModuleNode([], Location(), ConstantReadNode(), nil, Location(), "Foo")]), - Location(), - Location() - ), - 0, - "bar" - )] - ), - [], - Location(), - nil, - nil, - nil, - nil, - Location() - ) - - assert_errors expected, " - def foo - bar do - module Foo;end - end - end - ", [ - ["Module definition in method body", 40..46] - ] - end + def test_def_with_expression_receiver_and_no_identifier + assert_errors expression("def (a); end"), "def (a); end", [ + ["Expected '.' or '::' after receiver", 7..7] + ] + end - def test_class_definition_in_method_body - expected = DefNode( - Location(), - nil, - nil, - StatementsNode( - [ClassNode( - [], - Location(), - ConstantReadNode(), - nil, - nil, - nil, - Location(), - "A" - )] - ), - [], - Location(), - nil, - nil, - nil, - nil, - Location() - ) - - assert_errors expected, "def foo;class A;end;end", [ - ["Class definition in method body", 8..13] - ] - end + def test_def_with_multiple_statements_receiver + assert_errors expression("def (\na\nb\n).c; end"), "def (\na\nb\n).c; end", [ + ["Expected closing ')' for receiver.", 7..7], + ["Expected '.' or '::' after receiver", 7..7], + ["Expected to be able to parse an expression.", 10..10], + ["Expected to be able to parse an expression.", 11..11] + ] + end - def test_bad_arguments - expected = DefNode( - Location(), - nil, - ParametersNode([ - RequiredParameterNode(:A), - RequiredParameterNode(:@a), - RequiredParameterNode(:$A), - RequiredParameterNode(:@@a), - ], [], [], nil, [], nil, nil), - nil, - [:A, :@a, :$A, :@@a], - Location(), - nil, - Location(), - Location(), - nil, - Location() - ) - - assert_errors expected, "def foo(A, @a, $A, @@a);end", [ - ["Formal argument cannot be a constant", 8..9], - ["Formal argument cannot be an instance variable", 11..13], - ["Formal argument cannot be a global variable", 15..17], - ["Formal argument cannot be a class variable", 19..22], - ] - end + def test_def_with_empty_expression_receiver + assert_errors expression("def ().a; end"), "def ().a; end", [ + ["Expected to be able to parse receiver.", 5..5] + ] + end - def test_cannot_assign_to_a_reserved_numbered_parameter - expected = BeginNode( - Location(), - StatementsNode([ - LocalVariableWriteNode(:_1, 0, SymbolNode(Location(), Location(), nil, "a"), Location(), Location()), - LocalVariableWriteNode(:_2, 0, SymbolNode(Location(), Location(), nil, "a"), Location(), Location()), - LocalVariableWriteNode(:_3, 0, SymbolNode(Location(), Location(), nil, "a"), Location(), Location()), - LocalVariableWriteNode(:_4, 0, SymbolNode(Location(), Location(), nil, "a"), Location(), Location()), - LocalVariableWriteNode(:_5, 0, SymbolNode(Location(), Location(), nil, "a"), Location(), Location()), - LocalVariableWriteNode(:_6, 0, SymbolNode(Location(), Location(), nil, "a"), Location(), Location()), - LocalVariableWriteNode(:_7, 0, SymbolNode(Location(), Location(), nil, "a"), Location(), Location()), - LocalVariableWriteNode(:_8, 0, SymbolNode(Location(), Location(), nil, "a"), Location(), Location()), - LocalVariableWriteNode(:_9, 0, SymbolNode(Location(), Location(), nil, "a"), Location(), Location()), - LocalVariableWriteNode(:_10, 0, SymbolNode(Location(), Location(), nil, "a"), Location(), Location()) - ]), - nil, - nil, - nil, - Location() - ) - source = <<~RUBY - begin - _1=:a;_2=:a;_3=:a;_4=:a;_5=:a - _6=:a;_7=:a;_8=:a;_9=:a;_10=:a - end - RUBY - assert_errors expected, source, [ - ["reserved for numbered parameter", 8..10], - ["reserved for numbered parameter", 14..16], - ["reserved for numbered parameter", 20..22], - ["reserved for numbered parameter", 26..28], - ["reserved for numbered parameter", 32..34], - ["reserved for numbered parameter", 40..42], - ["reserved for numbered parameter", 46..48], - ["reserved for numbered parameter", 52..54], - ["reserved for numbered parameter", 58..60], - ] - end + def test_block_beginning_with_brace_and_ending_with_end + assert_error_messages "x.each { x end", [ + "Expected a newline or semicolon after statement.", + "Expected to be able to parse an expression.", + "Expected to be able to parse an expression.", + "Expected block beginning with '{' to end with '}'." + ] + end - def test_do_not_allow_trailing_commas_in_method_parameters - expected = DefNode( - Location(), - nil, - ParametersNode( - [RequiredParameterNode(:a), RequiredParameterNode(:b), RequiredParameterNode(:c)], - [], - [], + def test_double_splat_followed_by_splat_argument + expected = CallNode( nil, - [], nil, - nil - ), - nil, - [:a, :b, :c], - Location(), - nil, - Location(), - Location(), - nil, - Location() - ) - - assert_errors expected, "def foo(a,b,c,);end", [ - ["Unexpected ','.", 13..14] - ] - end - - def test_do_not_allow_trailing_commas_in_lambda_parameters - expected = LambdaNode( - [:a, :b], - Location(), - Location(), - Location(), - BlockParametersNode( - ParametersNode([RequiredParameterNode(:a), RequiredParameterNode(:b)], [], [], nil, [], nil, nil), - [], Location(), - Location() - ), - nil - ) - assert_errors expected, "-> (a, b, ) {}", [ - ["Unexpected ','.", 8..9] - ] - end - - def test_do_not_allow_multiple_codepoints_in_a_single_character_literal - expected = StringNode(Location(), Location(), nil, "\u0001\u0002") - - assert_errors expected, '?\u{0001 0002}', [ - ["Multiple codepoints at single character literal", 9..12] - ] - end - - def test_do_not_allow_more_than_6_hexadecimal_digits_in_u_Unicode_character_notation - expected = StringNode(Location(), Location(), Location(), "\u0001") - - assert_errors expected, '"\u{0000001}"', [ - ["invalid Unicode escape.", 4..11], - ["invalid Unicode escape.", 4..11] - ] - end - - def test_do_not_allow_characters_other_than_0_9_a_f_and_A_F_in_u_Unicode_character_notation - expected = StringNode(Location(), Location(), Location(), "\u0000z}") + Location(), + ArgumentsNode([ + KeywordHashNode([AssocSplatNode(expression("kwargs"), Location())]), + SplatNode(Location(), expression("args")) + ]), + Location(), + nil, + 0, + "a" + ) - assert_errors expected, '"\u{000z}"', [ - ["unterminated Unicode escape", 7..7], - ["unterminated Unicode escape", 7..7] - ] - end + assert_errors expected, "a(**kwargs, *args)", [ + ["Unexpected splat argument after double splat.", 12..17] + ] + end - def test_method_parameters_after_block - expected = DefNode( - Location(), - nil, - ParametersNode( - [], - [], - [RequiredParameterNode(:a)], + def test_arguments_after_block + expected = CallNode( nil, - [], nil, - BlockParameterNode(Location(), Location()) - ), - nil, - [:block, :a], - Location(), - nil, - Location(), - Location(), - nil, - Location() - ) - assert_errors expected, "def foo(&block, a)\nend", [ - ["Unexpected parameter order", 16..17] - ] - end + Location(), + Location(), + ArgumentsNode([ + BlockArgumentNode(expression("block"), Location()), + expression("foo") + ]), + Location(), + nil, + 0, + "a" + ) - def test_method_with_arguments_after_anonymous_block - expected = DefNode( - Location(), - nil, - ParametersNode([], [], [RequiredParameterNode(:a)], nil, [], nil, BlockParameterNode(nil, Location())), - nil, - [:&, :a], - Location(), - nil, - Location(), - Location(), - nil, - Location() - ) - - assert_errors expected, "def foo(&, a)\nend", [ - ["Unexpected parameter order", 11..12] - ] - end + assert_errors expected, "a(&block, foo)", [ + ["Unexpected argument after block argument.", 10..13] + ] + end - def test_method_parameters_after_arguments_forwarding - expected = DefNode( - Location(), - nil, - ParametersNode( - [], - [], - [RequiredParameterNode(:a)], - nil, - [], - ForwardingParameterNode(), - nil - ), - nil, - [:"...", :a], - Location(), - nil, - Location(), - Location(), - nil, - Location() - ) - assert_errors expected, "def foo(..., a)\nend", [ - ["Unexpected parameter order", 13..14] - ] - end + def test_arguments_binding_power_for_and + assert_error_messages "foo(*bar and baz)", [ + "Expected a ')' to close the argument list.", + "Expected a newline or semicolon after statement.", + "Expected to be able to parse an expression." + ] + end - def test_keywords_parameters_before_required_parameters - expected = DefNode( - Location(), - nil, - ParametersNode( - [], - [], - [RequiredParameterNode(:a)], + def test_splat_argument_after_keyword_argument + expected = CallNode( nil, - [KeywordParameterNode(Location(), nil)], nil, - nil - ), - nil, - [:b, :a], - Location(), - nil, - Location(), - Location(), - nil, - Location() - ) - assert_errors expected, "def foo(b:, a)\nend", [ - ["Unexpected parameter order", 12..13] - ] - end - - def test_rest_keywords_parameters_before_required_parameters - expected = DefNode( - Location(), - nil, - ParametersNode( - [], - [], - [], + Location(), + Location(), + ArgumentsNode([ + KeywordHashNode( + [AssocNode( + SymbolNode(nil, Location(), Location(), "foo"), + expression("bar"), + nil + )] + ), + SplatNode(Location(), expression("args")) + ]), + Location(), nil, - [KeywordParameterNode(Location(), nil)], - KeywordRestParameterNode(Location(), Location()), - nil - ), - nil, - [:rest, :b], - Location(), - nil, - Location(), - Location(), - nil, - Location() - ) - assert_errors expected, "def foo(**rest, b:)\nend", [ - ["Unexpected parameter order", 16..18] - ] - end + 0, + "a" + ) - def test_double_arguments_forwarding - expected = DefNode( - Location(), - nil, - ParametersNode([], [], [], nil, [], ForwardingParameterNode(), nil), - nil, - [:"..."], - Location(), - nil, - Location(), - Location(), - nil, - Location() - ) - - assert_errors expected, "def foo(..., ...)\nend", [ - ["Unexpected parameter order", 13..16] - ] - end + assert_errors expected, "a(foo: bar, *args)", [ + ["Unexpected splat argument after double splat.", 12..17] + ] + end - def test_multiple_error_in_parameters_order - expected = DefNode( - Location(), - nil, - ParametersNode( - [], - [], - [RequiredParameterNode(:a)], + def test_module_definition_in_method_body + expected = DefNode( + Location(), nil, - [KeywordParameterNode(Location(), nil)], - KeywordRestParameterNode(Location(), Location()), - nil - ), - nil, - [:args, :a, :b], - Location(), - nil, - Location(), - Location(), - nil, - Location() - ) - - assert_errors expected, "def foo(**args, a, b:)\nend", [ - ["Unexpected parameter order", 16..17], - ["Unexpected parameter order", 19..21] - ] - end - - def test_switching_to_optional_arguments_twice - expected = DefNode( - Location(), - nil, - ParametersNode( - [], - [], - [RequiredParameterNode(:a)], nil, - [KeywordParameterNode(Location(), nil)], - KeywordRestParameterNode(Location(), Location()), - nil - ), - nil, - [:args, :a, :b], - Location(), - nil, - Location(), - Location(), - nil, - Location(), - ) - - assert_errors expected, "def foo(**args, a, b:)\nend", [ - ["Unexpected parameter order", 16..17], - ["Unexpected parameter order", 19..21] - ] - end - - def test_switching_to_named_arguments_twice - expected = DefNode( - Location(), - nil, - ParametersNode( - [], + StatementsNode([ModuleNode([], Location(), ConstantReadNode(), nil, Location(), "A")]), [], - [RequiredParameterNode(:a)], + Location(), nil, - [KeywordParameterNode(Location(), nil)], - KeywordRestParameterNode(Location(), Location()), - nil - ), - nil, - [:args, :a, :b], - Location(), - nil, - Location(), - Location(), - nil, - Location(), - ) - - assert_errors expected, "def foo(**args, a, b:)\nend", [ - ["Unexpected parameter order", 16..17], - ["Unexpected parameter order", 19..21] - ] - end - - def test_returning_to_optional_parameters_multiple_times - expected = DefNode( - Location(), - nil, - ParametersNode( - [RequiredParameterNode(:a)], - [ - OptionalParameterNode(:b, Location(), Location(), IntegerNode()), - OptionalParameterNode(:d, Location(), Location(), IntegerNode()) - ], - [RequiredParameterNode(:c), RequiredParameterNode(:e)], nil, - [], nil, - nil - ), - nil, - [:a, :b, :c, :d, :e], - Location(), - nil, - Location(), - Location(), - nil, - Location(), - ) - - assert_errors expected, "def foo(a, b = 1, c, d = 2, e)\nend", [ - ["Unexpected parameter order", 23..24] - ] - end - - def test_case_without_when_clauses_errors_on_else_clause - expected = CaseNode( - SymbolNode(Location(), Location(), nil, "a"), - [], - ElseNode(Location(), nil, Location()), - Location(), - Location() - ) - - assert_errors expected, "case :a\nelse\nend", [ - ["Unexpected else without no when clauses in case statement.", 8..12] - ] - end - - def test_setter_method_cannot_be_defined_in_an_endless_method_definition - expected = DefNode( - Location(), - nil, - nil, - StatementsNode([IntegerNode()]), - [], - Location(), - nil, - Location(), - Location(), - Location(), - nil - ) - - assert_errors expected, "def a=() = 42", [ - ["Setter method cannot be defined in an endless method definition", 4..6] - ] - end - - def test_do_not_allow_forward_arguments_in_lambda_literals - expected = LambdaNode( - [:"..."], - Location(), - Location(), - Location(), - BlockParametersNode(ParametersNode([], [], [], nil, [], ForwardingParameterNode(), nil), [], Location(), Location()), - nil - ) - - assert_errors expected, "->(...) {}", [ - ["Unexpected ...", 3..6] - ] - end - - def test_do_not_allow_forward_arguments_in_blocks - expected = CallNode( - nil, - nil, - Location(), - nil, - nil, - nil, - BlockNode( - [:"..."], - BlockParametersNode(ParametersNode([], [], [], nil, [], ForwardingParameterNode(), nil), [], Location(), Location()), nil, - Location(), Location() - ), - 0, - "a" - ) - - assert_errors expected, "a {|...|}", [ - ["Unexpected ...", 4..7] - ] - end - - def test_dont_allow_return_inside_class_body - expected = ClassNode( - [], - Location(), - ConstantReadNode(), - nil, - nil, - StatementsNode([ReturnNode(Location(), nil)]), - Location(), - "A" - ) - - assert_errors expected, "class A; return; end", [ - ["Invalid return in class/module body", 15..16] - ] - end - - def test_dont_allow_return_inside_module_body - expected = ModuleNode( - [], - Location(), - ConstantReadNode(), - StatementsNode([ReturnNode(Location(), nil)]), - Location(), - "A" - ) - - assert_errors expected, "module A; return; end", [ - ["Invalid return in class/module body", 16..17] - ] - end + ) - def test_dont_allow_setting_to_back_and_nth_reference - expected = BeginNode( - Location(), - StatementsNode([ - GlobalVariableWriteNode(Location(), Location(), NilNode()), - GlobalVariableWriteNode(Location(), Location(), NilNode()) - ]), - nil, - nil, - nil, - Location() - ) - - assert_errors expected, "begin\n$+ = nil\n$1466 = nil\nend", [ - ["Can't set variable", 6..8], - ["Can't set variable", 15..20] - ] - end + assert_errors expected, "def foo;module A;end;end", [ + ["Module definition in method body", 8..14] + ] + end - def test_duplicated_parameter_names - # For some reason, Ripper reports no error for Ruby 3.0 when you have - # duplicated parameter names for positional parameters. - unless RUBY_VERSION < "3.1.0" + def test_module_definition_in_method_body_within_block expected = DefNode( Location(), nil, - ParametersNode([RequiredParameterNode(:a), RequiredParameterNode(:b), RequiredParameterNode(:a)], [], [], nil, [], nil, nil), nil, - [:a, :b], + StatementsNode( + [CallNode( + nil, + nil, + Location(), + nil, + nil, + nil, + BlockNode( + [], + nil, + StatementsNode([ModuleNode([], Location(), ConstantReadNode(), nil, Location(), "Foo")]), + Location(), + Location() + ), + 0, + "bar" + )] + ), + [], Location(), nil, + nil, + nil, + nil, + Location() + ) + + assert_errors expected, <<~RUBY, [["Module definition in method body", 21..27]] + def foo + bar do + module Foo;end + end + end + RUBY + end + + def test_class_definition_in_method_body + expected = DefNode( Location(), + nil, + nil, + StatementsNode( + [ClassNode( + [], + Location(), + ConstantReadNode(), + nil, + nil, + nil, + Location(), + "A" + )] + ), + [], Location(), nil, + nil, + nil, + nil, Location() ) - assert_errors expected, "def foo(a,b,a);end", [ - ["Duplicated parameter name.", 12..13] - ] - end - - expected = DefNode( - Location(), - nil, - ParametersNode([RequiredParameterNode(:a), RequiredParameterNode(:b)], [], [], RestParameterNode(Location(), Location()), [], nil, nil), - nil, - [:a, :b], - Location(), - nil, - Location(), - Location(), - nil, - Location() - ) - - assert_errors expected, "def foo(a,b,*a);end", [ - ["Duplicated parameter name.", 13..14] - ] - - expected = DefNode( - Location(), - nil, - ParametersNode([RequiredParameterNode(:a), RequiredParameterNode(:b)], [], [], nil, [], KeywordRestParameterNode(Location(), Location()), nil), - nil, - [:a, :b], - Location(), - nil, - Location(), - Location(), - nil, - Location() - ) - - assert_errors expected, "def foo(a,b,**a);end", [ - ["Duplicated parameter name.", 14..15] - ] - - expected = DefNode( - Location(), - nil, - ParametersNode([RequiredParameterNode(:a), RequiredParameterNode(:b)], [], [], nil, [], nil, BlockParameterNode(Location(), Location())), - nil, - [:a, :b], - Location(), - nil, - Location(), - Location(), - nil, - Location() - ) - - assert_errors expected, "def foo(a,b,&a);end", [ - ["Duplicated parameter name.", 13..14] - ] - - expected = DefNode( - Location(), - nil, - ParametersNode([], [OptionalParameterNode(:a, Location(), Location(), IntegerNode())], [RequiredParameterNode(:b)], RestParameterNode(Location(), Location()), [], nil, nil), - nil, - [:a, :b, :c], - Location(), - nil, - Location(), - Location(), - nil, - Location() - ) - - assert_errors expected, "def foo(a = 1,b,*c);end", [["Unexpected parameter *", 16..17]] - end + assert_errors expected, "def foo;class A;end;end", [ + ["Class definition in method body", 8..13] + ] + end - def test_unterminated_global_variable - assert_errors expression("$"), "$", [ - ["Invalid global variable.", 0..1] - ] - end + def test_bad_arguments + expected = DefNode( + Location(), + nil, + ParametersNode([ + RequiredParameterNode(:A), + RequiredParameterNode(:@a), + RequiredParameterNode(:$A), + RequiredParameterNode(:@@a), + ], [], [], nil, [], nil, nil), + nil, + [:A, :@a, :$A, :@@a], + Location(), + nil, + Location(), + Location(), + nil, + Location() + ) - private + assert_errors expected, "def foo(A, @a, $A, @@a);end", [ + ["Formal argument cannot be a constant", 8..9], + ["Formal argument cannot be an instance variable", 11..13], + ["Formal argument cannot be a global variable", 15..17], + ["Formal argument cannot be a class variable", 19..22], + ] + end - def assert_errors(expected, source, errors) - # Ripper behaves differently on JRuby/TruffleRuby, so only check this on CRuby - assert_nil Ripper.sexp_raw(source) if RUBY_ENGINE == "ruby" + def test_cannot_assign_to_a_reserved_numbered_parameter + expected = BeginNode( + Location(), + StatementsNode([ + LocalVariableWriteNode(:_1, 0, SymbolNode(Location(), Location(), nil, "a"), Location(), Location()), + LocalVariableWriteNode(:_2, 0, SymbolNode(Location(), Location(), nil, "a"), Location(), Location()), + LocalVariableWriteNode(:_3, 0, SymbolNode(Location(), Location(), nil, "a"), Location(), Location()), + LocalVariableWriteNode(:_4, 0, SymbolNode(Location(), Location(), nil, "a"), Location(), Location()), + LocalVariableWriteNode(:_5, 0, SymbolNode(Location(), Location(), nil, "a"), Location(), Location()), + LocalVariableWriteNode(:_6, 0, SymbolNode(Location(), Location(), nil, "a"), Location(), Location()), + LocalVariableWriteNode(:_7, 0, SymbolNode(Location(), Location(), nil, "a"), Location(), Location()), + LocalVariableWriteNode(:_8, 0, SymbolNode(Location(), Location(), nil, "a"), Location(), Location()), + LocalVariableWriteNode(:_9, 0, SymbolNode(Location(), Location(), nil, "a"), Location(), Location()), + LocalVariableWriteNode(:_10, 0, SymbolNode(Location(), Location(), nil, "a"), Location(), Location()) + ]), + nil, + nil, + nil, + Location() + ) + source = <<~RUBY + begin + _1=:a;_2=:a;_3=:a;_4=:a;_5=:a + _6=:a;_7=:a;_8=:a;_9=:a;_10=:a + end + RUBY + assert_errors expected, source, [ + ["reserved for numbered parameter", 8..10], + ["reserved for numbered parameter", 14..16], + ["reserved for numbered parameter", 20..22], + ["reserved for numbered parameter", 26..28], + ["reserved for numbered parameter", 32..34], + ["reserved for numbered parameter", 40..42], + ["reserved for numbered parameter", 46..48], + ["reserved for numbered parameter", 52..54], + ["reserved for numbered parameter", 58..60], + ] + end - result = YARP.parse(source) - node = result.value.statements.body.last + def test_do_not_allow_trailing_commas_in_method_parameters + expected = DefNode( + Location(), + nil, + ParametersNode( + [RequiredParameterNode(:a), RequiredParameterNode(:b), RequiredParameterNode(:c)], + [], + [], + nil, + [], + nil, + nil + ), + nil, + [:a, :b, :c], + Location(), + nil, + Location(), + Location(), + nil, + Location() + ) - assert_equal_nodes(expected, node, compare_location: false) - assert_equal(errors, result.errors.map { |e| [e.message, e.location.start_offset..e.location.end_offset] }) - end + assert_errors expected, "def foo(a,b,c,);end", [ + ["Unexpected ','.", 13..14] + ] + end - def assert_error_messages(source, errors) - assert_nil Ripper.sexp_raw(source) - result = YARP.parse(source) - assert_equal(errors, result.errors.map(&:message)) - end + def test_do_not_allow_trailing_commas_in_lambda_parameters + expected = LambdaNode( + [:a, :b], + Location(), + Location(), + Location(), + BlockParametersNode( + ParametersNode([RequiredParameterNode(:a), RequiredParameterNode(:b)], [], [], nil, [], nil, nil), + [], + Location(), + Location() + ), + nil + ) + assert_errors expected, "-> (a, b, ) {}", [ + ["Unexpected ','.", 8..9] + ] + end - def expression(source) - YARP.parse(source).value.statements.body.last + def test_do_not_allow_multiple_codepoints_in_a_single_character_literal + expected = StringNode(Location(), Location(), nil, "\u0001\u0002") + + assert_errors expected, '?\u{0001 0002}', [ + ["Multiple codepoints at single character literal", 9..12] + ] + end + + def test_do_not_allow_more_than_6_hexadecimal_digits_in_u_Unicode_character_notation + expected = StringNode(Location(), Location(), Location(), "\u0001") + + assert_errors expected, '"\u{0000001}"', [ + ["invalid Unicode escape.", 4..11], + ["invalid Unicode escape.", 4..11] + ] + end + + def test_do_not_allow_characters_other_than_0_9_a_f_and_A_F_in_u_Unicode_character_notation + expected = StringNode(Location(), Location(), Location(), "\u0000z}") + + assert_errors expected, '"\u{000z}"', [ + ["unterminated Unicode escape", 7..7], + ["unterminated Unicode escape", 7..7] + ] + end + + def test_method_parameters_after_block + expected = DefNode( + Location(), + nil, + ParametersNode( + [], + [], + [RequiredParameterNode(:a)], + nil, + [], + nil, + BlockParameterNode(Location(), Location()) + ), + nil, + [:block, :a], + Location(), + nil, + Location(), + Location(), + nil, + Location() + ) + assert_errors expected, "def foo(&block, a)\nend", [ + ["Unexpected parameter order", 16..17] + ] + end + + def test_method_with_arguments_after_anonymous_block + expected = DefNode( + Location(), + nil, + ParametersNode([], [], [RequiredParameterNode(:a)], nil, [], nil, BlockParameterNode(nil, Location())), + nil, + [:&, :a], + Location(), + nil, + Location(), + Location(), + nil, + Location() + ) + + assert_errors expected, "def foo(&, a)\nend", [ + ["Unexpected parameter order", 11..12] + ] + end + + def test_method_parameters_after_arguments_forwarding + expected = DefNode( + Location(), + nil, + ParametersNode( + [], + [], + [RequiredParameterNode(:a)], + nil, + [], + ForwardingParameterNode(), + nil + ), + nil, + [:"...", :a], + Location(), + nil, + Location(), + Location(), + nil, + Location() + ) + assert_errors expected, "def foo(..., a)\nend", [ + ["Unexpected parameter order", 13..14] + ] + end + + def test_keywords_parameters_before_required_parameters + expected = DefNode( + Location(), + nil, + ParametersNode( + [], + [], + [RequiredParameterNode(:a)], + nil, + [KeywordParameterNode(Location(), nil)], + nil, + nil + ), + nil, + [:b, :a], + Location(), + nil, + Location(), + Location(), + nil, + Location() + ) + assert_errors expected, "def foo(b:, a)\nend", [ + ["Unexpected parameter order", 12..13] + ] + end + + def test_rest_keywords_parameters_before_required_parameters + expected = DefNode( + Location(), + nil, + ParametersNode( + [], + [], + [], + nil, + [KeywordParameterNode(Location(), nil)], + KeywordRestParameterNode(Location(), Location()), + nil + ), + nil, + [:rest, :b], + Location(), + nil, + Location(), + Location(), + nil, + Location() + ) + assert_errors expected, "def foo(**rest, b:)\nend", [ + ["Unexpected parameter order", 16..18] + ] + end + + def test_double_arguments_forwarding + expected = DefNode( + Location(), + nil, + ParametersNode([], [], [], nil, [], ForwardingParameterNode(), nil), + nil, + [:"..."], + Location(), + nil, + Location(), + Location(), + nil, + Location() + ) + + assert_errors expected, "def foo(..., ...)\nend", [ + ["Unexpected parameter order", 13..16] + ] + end + + def test_multiple_error_in_parameters_order + expected = DefNode( + Location(), + nil, + ParametersNode( + [], + [], + [RequiredParameterNode(:a)], + nil, + [KeywordParameterNode(Location(), nil)], + KeywordRestParameterNode(Location(), Location()), + nil + ), + nil, + [:args, :a, :b], + Location(), + nil, + Location(), + Location(), + nil, + Location() + ) + + assert_errors expected, "def foo(**args, a, b:)\nend", [ + ["Unexpected parameter order", 16..17], + ["Unexpected parameter order", 19..21] + ] + end + + def test_switching_to_optional_arguments_twice + expected = DefNode( + Location(), + nil, + ParametersNode( + [], + [], + [RequiredParameterNode(:a)], + nil, + [KeywordParameterNode(Location(), nil)], + KeywordRestParameterNode(Location(), Location()), + nil + ), + nil, + [:args, :a, :b], + Location(), + nil, + Location(), + Location(), + nil, + Location(), + ) + + assert_errors expected, "def foo(**args, a, b:)\nend", [ + ["Unexpected parameter order", 16..17], + ["Unexpected parameter order", 19..21] + ] + end + + def test_switching_to_named_arguments_twice + expected = DefNode( + Location(), + nil, + ParametersNode( + [], + [], + [RequiredParameterNode(:a)], + nil, + [KeywordParameterNode(Location(), nil)], + KeywordRestParameterNode(Location(), Location()), + nil + ), + nil, + [:args, :a, :b], + Location(), + nil, + Location(), + Location(), + nil, + Location(), + ) + + assert_errors expected, "def foo(**args, a, b:)\nend", [ + ["Unexpected parameter order", 16..17], + ["Unexpected parameter order", 19..21] + ] + end + + def test_returning_to_optional_parameters_multiple_times + expected = DefNode( + Location(), + nil, + ParametersNode( + [RequiredParameterNode(:a)], + [ + OptionalParameterNode(:b, Location(), Location(), IntegerNode()), + OptionalParameterNode(:d, Location(), Location(), IntegerNode()) + ], + [RequiredParameterNode(:c), RequiredParameterNode(:e)], + nil, + [], + nil, + nil + ), + nil, + [:a, :b, :c, :d, :e], + Location(), + nil, + Location(), + Location(), + nil, + Location(), + ) + + assert_errors expected, "def foo(a, b = 1, c, d = 2, e)\nend", [ + ["Unexpected parameter order", 23..24] + ] + end + + def test_case_without_when_clauses_errors_on_else_clause + expected = CaseNode( + SymbolNode(Location(), Location(), nil, "a"), + [], + ElseNode(Location(), nil, Location()), + Location(), + Location() + ) + + assert_errors expected, "case :a\nelse\nend", [ + ["Unexpected else without no when clauses in case statement.", 8..12] + ] + end + + def test_setter_method_cannot_be_defined_in_an_endless_method_definition + expected = DefNode( + Location(), + nil, + nil, + StatementsNode([IntegerNode()]), + [], + Location(), + nil, + Location(), + Location(), + Location(), + nil + ) + + assert_errors expected, "def a=() = 42", [ + ["Setter method cannot be defined in an endless method definition", 4..6] + ] + end + + def test_do_not_allow_forward_arguments_in_lambda_literals + expected = LambdaNode( + [:"..."], + Location(), + Location(), + Location(), + BlockParametersNode(ParametersNode([], [], [], nil, [], ForwardingParameterNode(), nil), [], Location(), Location()), + nil + ) + + assert_errors expected, "->(...) {}", [ + ["Unexpected ...", 3..6] + ] + end + + def test_do_not_allow_forward_arguments_in_blocks + expected = CallNode( + nil, + nil, + Location(), + nil, + nil, + nil, + BlockNode( + [:"..."], + BlockParametersNode(ParametersNode([], [], [], nil, [], ForwardingParameterNode(), nil), [], Location(), Location()), + nil, + Location(), + Location() + ), + 0, + "a" + ) + + assert_errors expected, "a {|...|}", [ + ["Unexpected ...", 4..7] + ] + end + + def test_dont_allow_return_inside_class_body + expected = ClassNode( + [], + Location(), + ConstantReadNode(), + nil, + nil, + StatementsNode([ReturnNode(Location(), nil)]), + Location(), + "A" + ) + + assert_errors expected, "class A; return; end", [ + ["Invalid return in class/module body", 15..16] + ] + end + + def test_dont_allow_return_inside_module_body + expected = ModuleNode( + [], + Location(), + ConstantReadNode(), + StatementsNode([ReturnNode(Location(), nil)]), + Location(), + "A" + ) + + assert_errors expected, "module A; return; end", [ + ["Invalid return in class/module body", 16..17] + ] + end + + def test_dont_allow_setting_to_back_and_nth_reference + expected = BeginNode( + Location(), + StatementsNode([ + GlobalVariableWriteNode(Location(), Location(), NilNode()), + GlobalVariableWriteNode(Location(), Location(), NilNode()) + ]), + nil, + nil, + nil, + Location() + ) + + assert_errors expected, "begin\n$+ = nil\n$1466 = nil\nend", [ + ["Can't set variable", 6..8], + ["Can't set variable", 15..20] + ] + end + + def test_duplicated_parameter_names + # For some reason, Ripper reports no error for Ruby 3.0 when you have + # duplicated parameter names for positional parameters. + unless RUBY_VERSION < "3.1.0" + expected = DefNode( + Location(), + nil, + ParametersNode([RequiredParameterNode(:a), RequiredParameterNode(:b), RequiredParameterNode(:a)], [], [], nil, [], nil, nil), + nil, + [:a, :b], + Location(), + nil, + Location(), + Location(), + nil, + Location() + ) + + assert_errors expected, "def foo(a,b,a);end", [ + ["Duplicated parameter name.", 12..13] + ] + end + + expected = DefNode( + Location(), + nil, + ParametersNode([RequiredParameterNode(:a), RequiredParameterNode(:b)], [], [], RestParameterNode(Location(), Location()), [], nil, nil), + nil, + [:a, :b], + Location(), + nil, + Location(), + Location(), + nil, + Location() + ) + + assert_errors expected, "def foo(a,b,*a);end", [ + ["Duplicated parameter name.", 13..14] + ] + + expected = DefNode( + Location(), + nil, + ParametersNode([RequiredParameterNode(:a), RequiredParameterNode(:b)], [], [], nil, [], KeywordRestParameterNode(Location(), Location()), nil), + nil, + [:a, :b], + Location(), + nil, + Location(), + Location(), + nil, + Location() + ) + + assert_errors expected, "def foo(a,b,**a);end", [ + ["Duplicated parameter name.", 14..15] + ] + + expected = DefNode( + Location(), + nil, + ParametersNode([RequiredParameterNode(:a), RequiredParameterNode(:b)], [], [], nil, [], nil, BlockParameterNode(Location(), Location())), + nil, + [:a, :b], + Location(), + nil, + Location(), + Location(), + nil, + Location() + ) + + assert_errors expected, "def foo(a,b,&a);end", [ + ["Duplicated parameter name.", 13..14] + ] + + expected = DefNode( + Location(), + nil, + ParametersNode([], [OptionalParameterNode(:a, Location(), Location(), IntegerNode())], [RequiredParameterNode(:b)], RestParameterNode(Location(), Location()), [], nil, nil), + nil, + [:a, :b, :c], + Location(), + nil, + Location(), + Location(), + nil, + Location() + ) + + assert_errors expected, "def foo(a = 1,b,*c);end", [["Unexpected parameter *", 16..17]] + end + + def test_unterminated_global_variable + assert_errors expression("$"), "$", [ + ["Invalid global variable.", 0..1] + ] + end + + private + + def assert_errors(expected, source, errors) + # Ripper behaves differently on JRuby/TruffleRuby, so only check this on CRuby + assert_nil Ripper.sexp_raw(source) if RUBY_ENGINE == "ruby" + + result = YARP.parse(source) + node = result.value.statements.body.last + + assert_equal_nodes(expected, node, compare_location: false) + assert_equal(errors, result.errors.map { |e| [e.message, e.location.start_offset..e.location.end_offset] }) + end + + def assert_error_messages(source, errors) + assert_nil Ripper.sexp_raw(source) + result = YARP.parse(source) + assert_equal(errors, result.errors.map(&:message)) + end + + def expression(source) + YARP.parse(source).value.statements.body.last + end end end diff --git a/test/yarp/heredoc_dedent_test.rb b/test/yarp/heredoc_dedent_test.rb index 2744b930ed3..975232889f4 100644 --- a/test/yarp/heredoc_dedent_test.rb +++ b/test/yarp/heredoc_dedent_test.rb @@ -3,7 +3,7 @@ require_relative "test_helper" module YARP - class HeredocDedentTest < Test::Unit::TestCase + class HeredocDedentTest < TestCase filepath = File.expand_path("fixtures/tilde_heredocs.txt", __dir__) File.read(filepath).split(/(?=\n)\n(?=<)/).each_with_index do |heredoc, index| diff --git a/test/yarp/library_symbols_test.rb b/test/yarp/library_symbols_test.rb index 9a52d189baf..53f56d9bfa6 100644 --- a/test/yarp/library_symbols_test.rb +++ b/test/yarp/library_symbols_test.rb @@ -2,12 +2,14 @@ require_relative "test_helper" -if RUBY_PLATFORM =~ /linux/ +return if RUBY_PLATFORM !~ /linux/ + +module YARP # # examine a yarp dll or static archive for expected external symbols. # these tests only work on a linux system right now. # - class LibrarySymbolsTest < Test::Unit::TestCase + class LibrarySymbolsTest < TestCase def setup super diff --git a/test/yarp/locals_test.rb b/test/yarp/locals_test.rb index 42fb72df789..45aecdcaf7e 100644 --- a/test/yarp/locals_test.rb +++ b/test/yarp/locals_test.rb @@ -15,93 +15,95 @@ require_relative "test_helper" -class LocalsTest < Test::Unit::TestCase - invalid = [] - todos = [] - - # Invalid break - invalid << "break.txt" - invalid << "if.txt" - invalid << "rescue.txt" - invalid << "seattlerb/block_break.txt" - invalid << "unless.txt" - invalid << "whitequark/break.txt" - invalid << "whitequark/break_block.txt" - - # Invalid next - invalid << "next.txt" - invalid << "seattlerb/block_next.txt" - invalid << "unparser/corpus/literal/control.txt" - invalid << "whitequark/next.txt" - invalid << "whitequark/next_block.txt" - - # Invalid redo - invalid << "keywords.txt" - invalid << "whitequark/redo.txt" - - # Invalid retry - invalid << "whitequark/retry.txt" - - # Invalid yield - invalid << "seattlerb/dasgn_icky2.txt" - invalid << "seattlerb/yield_arg.txt" - invalid << "seattlerb/yield_call_assocs.txt" - invalid << "seattlerb/yield_empty_parens.txt" - invalid << "unparser/corpus/literal/yield.txt" - invalid << "whitequark/args_assocs.txt" - invalid << "whitequark/args_assocs_legacy.txt" - invalid << "whitequark/yield.txt" - invalid << "yield.txt" - - # Dead code eliminated - invalid << "whitequark/ruby_bug_10653.txt" - - # case :a - # in Symbol(*lhs, x, *rhs) - # end - todos << "seattlerb/case_in.txt" - - # <<~HERE - # #{<<~THERE} - # THERE - # HERE - todos << "seattlerb/heredoc_nested.txt" - - base = File.join(__dir__, "fixtures") - skips = invalid | todos - - Dir["**/*.txt", base: base].each do |relative| - next if skips.include?(relative) - - filepath = File.join(base, relative) - define_method("test_#{relative}") { assert_locals(filepath) } - end - - def setup - @previous_default_external = Encoding.default_external - ignore_warnings { Encoding.default_external = Encoding::UTF_8 } - end - - def teardown - ignore_warnings { Encoding.default_external = @previous_default_external } - end - - private - - def assert_locals(filepath) - source = File.read(filepath) - - expected = YARP.const_get(:Debug).cruby_locals(source) - actual = YARP.const_get(:Debug).yarp_locals(source) - - assert_equal(expected, actual) - end - - def ignore_warnings - previous_verbosity = $VERBOSE - $VERBOSE = nil - yield - ensure - $VERBOSE = previous_verbosity +module YARP + class LocalsTest < TestCase + invalid = [] + todos = [] + + # Invalid break + invalid << "break.txt" + invalid << "if.txt" + invalid << "rescue.txt" + invalid << "seattlerb/block_break.txt" + invalid << "unless.txt" + invalid << "whitequark/break.txt" + invalid << "whitequark/break_block.txt" + + # Invalid next + invalid << "next.txt" + invalid << "seattlerb/block_next.txt" + invalid << "unparser/corpus/literal/control.txt" + invalid << "whitequark/next.txt" + invalid << "whitequark/next_block.txt" + + # Invalid redo + invalid << "keywords.txt" + invalid << "whitequark/redo.txt" + + # Invalid retry + invalid << "whitequark/retry.txt" + + # Invalid yield + invalid << "seattlerb/dasgn_icky2.txt" + invalid << "seattlerb/yield_arg.txt" + invalid << "seattlerb/yield_call_assocs.txt" + invalid << "seattlerb/yield_empty_parens.txt" + invalid << "unparser/corpus/literal/yield.txt" + invalid << "whitequark/args_assocs.txt" + invalid << "whitequark/args_assocs_legacy.txt" + invalid << "whitequark/yield.txt" + invalid << "yield.txt" + + # Dead code eliminated + invalid << "whitequark/ruby_bug_10653.txt" + + # case :a + # in Symbol(*lhs, x, *rhs) + # end + todos << "seattlerb/case_in.txt" + + # <<~HERE + # #{<<~THERE} + # THERE + # HERE + todos << "seattlerb/heredoc_nested.txt" + + base = File.join(__dir__, "fixtures") + skips = invalid | todos + + Dir["**/*.txt", base: base].each do |relative| + next if skips.include?(relative) + + filepath = File.join(base, relative) + define_method("test_#{relative}") { assert_locals(filepath) } + end + + def setup + @previous_default_external = Encoding.default_external + ignore_warnings { Encoding.default_external = Encoding::UTF_8 } + end + + def teardown + ignore_warnings { Encoding.default_external = @previous_default_external } + end + + private + + def assert_locals(filepath) + source = File.read(filepath) + + expected = Debug.cruby_locals(source) + actual = Debug.yarp_locals(source) + + assert_equal(expected, actual) + end + + def ignore_warnings + previous_verbosity = $VERBOSE + $VERBOSE = nil + yield + ensure + $VERBOSE = previous_verbosity + end end end diff --git a/test/yarp/location_test.rb b/test/yarp/location_test.rb index 8e357fa193c..192cfdd716c 100644 --- a/test/yarp/location_test.rb +++ b/test/yarp/location_test.rb @@ -3,7 +3,7 @@ require_relative "test_helper" module YARP - class LocationTest < Test::Unit::TestCase + class LocationTest < TestCase def test_AliasNode assert_location(AliasNode, "alias foo bar") end diff --git a/test/yarp/memsize_test.rb b/test/yarp/memsize_test.rb index 9ff670c1186..07c85ce3295 100644 --- a/test/yarp/memsize_test.rb +++ b/test/yarp/memsize_test.rb @@ -4,12 +4,14 @@ return if YARP::BACKEND == :FFI -class MemsizeTest < Test::Unit::TestCase - def test_memsize - result = YARP.const_get(:Debug).memsize("2 + 3") +module YARP + class MemsizeTest < TestCase + def test_memsize + result = Debug.memsize("2 + 3") - assert_equal 5, result[:length] - assert_kind_of Integer, result[:memsize] - assert_equal 6, result[:node_count] + assert_equal 5, result[:length] + assert_kind_of Integer, result[:memsize] + assert_equal 6, result[:node_count] + end end end diff --git a/test/yarp/newline_test.rb b/test/yarp/newline_test.rb index 2eaaefc61ea..5a85f856f39 100644 --- a/test/yarp/newline_test.rb +++ b/test/yarp/newline_test.rb @@ -4,97 +4,93 @@ return unless defined?(RubyVM::InstructionSequence) -# It is useful to have a diff even if the strings to compare are big -# However, ruby/ruby does not have a version of Test::Unit with access to -# max_diff_target_string_size -if defined?(Test::Unit::Assertions::AssertionMessage) - Test::Unit::Assertions::AssertionMessage.max_diff_target_string_size = 5000 -end - -class NewlineTest < Test::Unit::TestCase - class NewlineVisitor < YARP::Visitor - attr_reader :source, :newlines - - def initialize(source) - @source = source - @newlines = [] - end +module YARP + class NewlineTest < TestCase + class NewlineVisitor < Visitor + attr_reader :source, :newlines + + def initialize(source) + @source = source + @newlines = [] + end - def visit(node) - newlines << source.line(node.location.start_offset) if node&.newline? - super(node) + def visit(node) + newlines << source.line(node.location.start_offset) if node&.newline? + super(node) + end end - end - base = File.dirname(__dir__) - Dir["{lib,test}/**/*.rb", base: base].each do |relative| - define_method("test_newline_flags_#{relative}") do - assert_newlines(base, relative) + base = File.dirname(__dir__) + Dir["{lib,test}/**/*.rb", base: base].each do |relative| + define_method("test_newline_flags_#{relative}") do + assert_newlines(base, relative) + end end - end - private + private - def assert_newlines(base, relative) - filepath = File.join(base, relative) - source = File.read(filepath, binmode: true, external_encoding: Encoding::UTF_8) - expected = rubyvm_lines(source) + def assert_newlines(base, relative) + filepath = File.join(base, relative) + source = File.read(filepath, binmode: true, external_encoding: Encoding::UTF_8) + expected = rubyvm_lines(source) - result = YARP.parse_file(filepath) - assert_empty result.errors + result = YARP.parse_file(filepath) + assert_empty result.errors - result.mark_newlines - visitor = NewlineVisitor.new(result.source) + result.mark_newlines + visitor = NewlineVisitor.new(result.source) - result.value.accept(visitor) - actual = visitor.newlines + result.value.accept(visitor) + actual = visitor.newlines - source.each_line.with_index(1) do |line, line_number| - # Lines like `while (foo = bar)` result in two line flags in the bytecode - # but only one newline flag in the AST. We need to remove the extra line - # flag from the bytecode to make the test pass. - if line.match?(/while \(/) - index = expected.index(line_number) - expected.delete_at(index) if index - end + source.each_line.with_index(1) do |line, line_number| + # Lines like `while (foo = bar)` result in two line flags in the + # bytecode but only one newline flag in the AST. We need to remove the + # extra line flag from the bytecode to make the test pass. + if line.match?(/while \(/) + index = expected.index(line_number) + expected.delete_at(index) if index + end - # Lines like `foo =` where the value is on the next line result in another - # line flag in the bytecode but only one newline flag in the AST. - if line.match?(/^\s+\w+ =$/) - if source.lines[line_number].match?(/^\s+case/) - actual[actual.index(line_number)] += 1 - else - actual.delete_at(actual.index(line_number)) + # Lines like `foo =` where the value is on the next line result in + # another line flag in the bytecode but only one newline flag in the + # AST. + if line.match?(/^\s+\w+ =$/) + if source.lines[line_number].match?(/^\s+case/) + actual[actual.index(line_number)] += 1 + else + actual.delete_at(actual.index(line_number)) + end end - end - if line.match?(/^\s+\w+ = \[$/) - if !expected.include?(line_number) && !expected.include?(line_number + 2) - actual[actual.index(line_number)] += 1 + if line.match?(/^\s+\w+ = \[$/) + if !expected.include?(line_number) && !expected.include?(line_number + 2) + actual[actual.index(line_number)] += 1 + end end end + + assert_equal expected, actual end - assert_equal expected, actual - end + def ignore_warnings + previous_verbosity = $VERBOSE + $VERBOSE = nil + yield + ensure + $VERBOSE = previous_verbosity + end - def ignore_warnings - previous_verbosity = $VERBOSE - $VERBOSE = nil - yield - ensure - $VERBOSE = previous_verbosity - end + def rubyvm_lines(source) + queue = [ignore_warnings { RubyVM::InstructionSequence.compile(source) }] + lines = [] - def rubyvm_lines(source) - queue = [ignore_warnings { RubyVM::InstructionSequence.compile(source) }] - lines = [] + while iseq = queue.shift + lines.concat(iseq.trace_points.filter_map { |line, event| line if event == :line }) + iseq.each_child { |insn| queue << insn unless insn.label.start_with?("ensure in ") } + end - while iseq = queue.shift - lines.concat(iseq.trace_points.filter_map { |line, event| line if event == :line }) - iseq.each_child { |insn| queue << insn unless insn.label.start_with?("ensure in ") } + lines.sort end - - lines.sort end end diff --git a/test/yarp/parse_serialize_test.rb b/test/yarp/parse_serialize_test.rb index daecbe1488d..82a1c29d48a 100644 --- a/test/yarp/parse_serialize_test.rb +++ b/test/yarp/parse_serialize_test.rb @@ -4,23 +4,25 @@ return if YARP::BACKEND == :FFI -class ParseSerializeTest < Test::Unit::TestCase - def test_parse_serialize - dumped = YARP.const_get(:Debug).parse_serialize_file(__FILE__) - result = YARP.load(File.read(__FILE__), dumped) +module YARP + class ParseSerializeTest < TestCase + def test_parse_serialize + dumped = Debug.parse_serialize_file(__FILE__) + result = YARP.load(File.read(__FILE__), dumped) - assert_kind_of YARP::ParseResult, result, "Expected the return value to be a ParseResult" - assert_equal __FILE__, find_file_node(result)&.filepath, "Expected the filepath to be set correctly" - end + assert_kind_of ParseResult, result, "Expected the return value to be a ParseResult" + assert_equal __FILE__, find_file_node(result)&.filepath, "Expected the filepath to be set correctly" + end - private + private - def find_file_node(result) - queue = [result.value] + def find_file_node(result) + queue = [result.value] - while (node = queue.shift) - return node if node.is_a?(YARP::SourceFileNode) - queue.concat(node.child_nodes.compact) + while (node = queue.shift) + return node if node.is_a?(SourceFileNode) + queue.concat(node.child_nodes.compact) + end end end end diff --git a/test/yarp/parse_test.rb b/test/yarp/parse_test.rb index b288d597b23..5299cfd7b11 100644 --- a/test/yarp/parse_test.rb +++ b/test/yarp/parse_test.rb @@ -2,219 +2,221 @@ require_relative "test_helper" -class ParseTest < Test::Unit::TestCase - # When we pretty-print the trees to compare against the snapshots, we want to - # be certain that we print with the same external encoding. This is because - # methods like Symbol#inspect take into account external encoding and it could - # change how the snapshot is generated. On machines with certain settings - # (like LANG=C or -Eascii-8bit) this could have been changed. So here we're - # going to force it to be UTF-8 to keep the snapshots consistent. - def setup - @previous_default_external = Encoding.default_external - ignore_warnings { Encoding.default_external = Encoding::UTF_8 } - end +module YARP + class ParseTest < TestCase + # When we pretty-print the trees to compare against the snapshots, we want to + # be certain that we print with the same external encoding. This is because + # methods like Symbol#inspect take into account external encoding and it could + # change how the snapshot is generated. On machines with certain settings + # (like LANG=C or -Eascii-8bit) this could have been changed. So here we're + # going to force it to be UTF-8 to keep the snapshots consistent. + def setup + @previous_default_external = Encoding.default_external + ignore_warnings { Encoding.default_external = Encoding::UTF_8 } + end - def teardown - ignore_warnings { Encoding.default_external = @previous_default_external } - end + def teardown + ignore_warnings { Encoding.default_external = @previous_default_external } + end - def test_empty_string - result = YARP.parse("") - assert_equal [], result.value.statements.body - end + def test_empty_string + result = YARP.parse("") + assert_equal [], result.value.statements.body + end - def test_parse_takes_file_path - filepath = "filepath.rb" - result = YARP.parse("def foo; __FILE__; end", filepath) + def test_parse_takes_file_path + filepath = "filepath.rb" + result = YARP.parse("def foo; __FILE__; end", filepath) - assert_equal filepath, find_source_file_node(result.value).filepath - end + assert_equal filepath, find_source_file_node(result.value).filepath + end - def test_parse_lex - node, tokens = YARP.parse_lex("def foo; end").value + def test_parse_lex + node, tokens = YARP.parse_lex("def foo; end").value - assert_kind_of YARP::ProgramNode, node - assert_equal 5, tokens.length - end + assert_kind_of ProgramNode, node + assert_equal 5, tokens.length + end - def test_parse_lex_file - node, tokens = YARP.parse_lex_file(__FILE__).value + def test_parse_lex_file + node, tokens = YARP.parse_lex_file(__FILE__).value - assert_kind_of YARP::ProgramNode, node - refute_empty tokens - end + assert_kind_of ProgramNode, node + refute_empty tokens + end - # To accurately compare against Ripper, we need to make sure that we're - # running on Ruby 3.2+. - ripper_enabled = RUBY_VERSION >= "3.2.0" + # To accurately compare against Ripper, we need to make sure that we're + # running on Ruby 3.2+. + ripper_enabled = RUBY_VERSION >= "3.2.0" - # The FOCUS environment variable allows you to specify one particular fixture - # to test, instead of all of them. - base = File.join(__dir__, "fixtures") - relatives = ENV["FOCUS"] ? [ENV["FOCUS"]] : Dir["**/*.txt", base: base] + # The FOCUS environment variable allows you to specify one particular fixture + # to test, instead of all of them. + base = File.join(__dir__, "fixtures") + relatives = ENV["FOCUS"] ? [ENV["FOCUS"]] : Dir["**/*.txt", base: base] - relatives.each do |relative| - # These fail on TruffleRuby due to a difference in Symbol#inspect: :测试 vs :"测试" - next if RUBY_ENGINE == "truffleruby" and %w[seattlerb/bug202.txt seattlerb/magic_encoding_comment.txt].include?(relative) + relatives.each do |relative| + # These fail on TruffleRuby due to a difference in Symbol#inspect: :测试 vs :"测试" + next if RUBY_ENGINE == "truffleruby" and %w[seattlerb/bug202.txt seattlerb/magic_encoding_comment.txt].include?(relative) - filepath = File.join(base, relative) - snapshot = File.expand_path(File.join("snapshots", relative), __dir__) + filepath = File.join(base, relative) + snapshot = File.expand_path(File.join("snapshots", relative), __dir__) + + directory = File.dirname(snapshot) + FileUtils.mkdir_p(directory) unless File.directory?(directory) - directory = File.dirname(snapshot) - FileUtils.mkdir_p(directory) unless File.directory?(directory) + ripper_should_parse = ripper_should_match = ripper_enabled - ripper_should_parse = ripper_should_match = ripper_enabled + # This file has changed behavior in Ripper in Ruby 3.3, so we skip it if + # we're on an earlier version. + ripper_should_match = false if relative == "seattlerb/pct_w_heredoc_interp_nested.txt" && RUBY_VERSION < "3.3.0" - # This file has changed behavior in Ripper in Ruby 3.3, so we skip it if - # we're on an earlier version. - ripper_should_match = false if relative == "seattlerb/pct_w_heredoc_interp_nested.txt" && RUBY_VERSION < "3.3.0" + # It seems like there are some oddities with nested heredocs and ripper. + # Waiting for feedback on https://bugs.ruby-lang.org/issues/19838. + ripper_should_match = false if relative == "seattlerb/heredoc_nested.txt" - # It seems like there are some oddities with nested heredocs and ripper. - # Waiting for feedback on https://bugs.ruby-lang.org/issues/19838. - ripper_should_match = false if relative == "seattlerb/heredoc_nested.txt" + # Ripper seems to have a bug that the regex portions before and after the heredoc are combined + # into a single token. See https://bugs.ruby-lang.org/issues/19838. + # + # Additionally, Ripper cannot parse the %w[] fixture in this file, so set ripper_should_parse to false. + ripper_should_parse = false if relative == "spanning_heredoc.txt" - # Ripper seems to have a bug that the regex portions before and after the heredoc are combined - # into a single token. See https://bugs.ruby-lang.org/issues/19838. - # - # Additionally, Ripper cannot parse the %w[] fixture in this file, so set ripper_should_parse to false. - ripper_should_parse = false if relative == "spanning_heredoc.txt" + define_method "test_filepath_#{relative}" do + # First, read the source from the filepath. Use binmode to avoid converting CRLF on Windows, + # and explicitly set the external encoding to UTF-8 to override the binmode default. + source = File.read(filepath, binmode: true, external_encoding: Encoding::UTF_8) - define_method "test_filepath_#{relative}" do - # First, read the source from the filepath. Use binmode to avoid converting CRLF on Windows, - # and explicitly set the external encoding to UTF-8 to override the binmode default. - source = File.read(filepath, binmode: true, external_encoding: Encoding::UTF_8) + # Make sure that it can be correctly parsed by Ripper. If it can't, then we have a fixture + # that is invalid Ruby. + refute_nil(Ripper.sexp_raw(source), "Ripper failed to parse") if ripper_should_parse - # Make sure that it can be correctly parsed by Ripper. If it can't, then we have a fixture - # that is invalid Ruby. - refute_nil(Ripper.sexp_raw(source), "Ripper failed to parse") if ripper_should_parse + # Next, assert that there were no errors during parsing. + result = YARP.parse(source, relative) + assert_empty result.errors - # Next, assert that there were no errors during parsing. - result = YARP.parse(source, relative) - assert_empty result.errors + # Next, pretty print the source. + printed = PP.pp(result.value, +"", 79) - # Next, pretty print the source. - printed = PP.pp(result.value, +"", 79) + if File.exist?(snapshot) + saved = File.read(snapshot) - if File.exist?(snapshot) - saved = File.read(snapshot) + # If the snapshot file exists, but the printed value does not match the + # snapshot, then update the snapshot file. + if printed != saved + File.write(snapshot, printed) + warn("Updated snapshot at #{snapshot}.") + end - # If the snapshot file exists, but the printed value does not match the - # snapshot, then update the snapshot file. - if printed != saved + # If the snapshot file exists, then assert that the printed value + # matches the snapshot. + assert_equal(saved, printed) + else + # If the snapshot file does not yet exist, then write it out now. File.write(snapshot, printed) - warn("Updated snapshot at #{snapshot}.") + warn("Created snapshot at #{snapshot}.") end - # If the snapshot file exists, then assert that the printed value - # matches the snapshot. - assert_equal(saved, printed) - else - # If the snapshot file does not yet exist, then write it out now. - File.write(snapshot, printed) - warn("Created snapshot at #{snapshot}.") - end - - # Next, assert that the value can be serialized and deserialized without - # changing the shape of the tree. - assert_equal_nodes(result.value, YARP.load(source, YARP.dump(source, relative)).value) - - # Next, check that the location ranges of each node in the tree are a - # superset of their respective child nodes. - assert_non_overlapping_locations(result.value) - - # Next, assert that the newlines are in the expected places. - expected_newlines = [0] - source.b.scan("\n") { expected_newlines << $~.offset(0)[0] + 1 } + # Next, assert that the value can be serialized and deserialized without + # changing the shape of the tree. + assert_equal_nodes(result.value, YARP.load(source, YARP.dump(source, relative)).value) - # If there's a __END__, then we should trip out those newlines because we - # don't actually scan them during parsing (because we don't need to). - if found = result.comments.find { |comment| comment.type == :__END__ } - expected_newlines = expected_newlines[...found.location.start_line] - end + # Next, check that the location ranges of each node in the tree are a + # superset of their respective child nodes. + assert_non_overlapping_locations(result.value) - assert_equal expected_newlines, YARP.const_get(:Debug).newlines(source) + # Next, assert that the newlines are in the expected places. + expected_newlines = [0] + source.b.scan("\n") { expected_newlines << $~.offset(0)[0] + 1 } - if ripper_should_parse && ripper_should_match - # Finally, assert that we can lex the source and get the same tokens as - # Ripper. - lex_result = YARP.lex_compat(source) - assert_equal [], lex_result.errors - tokens = lex_result.value + # If there's a __END__, then we should trip out those newlines because we + # don't actually scan them during parsing (because we don't need to). + if found = result.comments.find { |comment| comment.type == :__END__ } + expected_newlines = expected_newlines[...found.location.start_line] + end - begin - YARP.lex_ripper(source).zip(tokens).each do |(ripper, yarp)| - assert_equal ripper, yarp + assert_equal expected_newlines, Debug.newlines(source) + + if ripper_should_parse && ripper_should_match + # Finally, assert that we can lex the source and get the same tokens as + # Ripper. + lex_result = YARP.lex_compat(source) + assert_equal [], lex_result.errors + tokens = lex_result.value + + begin + YARP.lex_ripper(source).zip(tokens).each do |(ripper, yarp)| + assert_equal ripper, yarp + end + rescue SyntaxError + raise ArgumentError, "Test file has invalid syntax #{filepath}" end - rescue SyntaxError - raise ArgumentError, "Test file has invalid syntax #{filepath}" end end end - end - Dir["*.txt", base: base].each do |relative| - next if relative == "newline_terminated.txt" + Dir["*.txt", base: base].each do |relative| + next if relative == "newline_terminated.txt" - # We test every snippet (separated by \n\n) in isolation - # to ensure the parser does not try to read bytes further than the end of each snippet - define_method "test_individual_snippets_#{relative}" do - filepath = File.join(base, relative) + # We test every snippet (separated by \n\n) in isolation + # to ensure the parser does not try to read bytes further than the end of each snippet + define_method "test_individual_snippets_#{relative}" do + filepath = File.join(base, relative) - # First, read the source from the filepath. Use binmode to avoid converting CRLF on Windows, - # and explicitly set the external encoding to UTF-8 to override the binmode default. - file_contents = File.read(filepath, binmode: true, external_encoding: Encoding::UTF_8) + # First, read the source from the filepath. Use binmode to avoid converting CRLF on Windows, + # and explicitly set the external encoding to UTF-8 to override the binmode default. + file_contents = File.read(filepath, binmode: true, external_encoding: Encoding::UTF_8) - file_contents.split(/(?<=\S)\n\n(?=\S)/).each do |snippet| - snippet = snippet.rstrip - result = YARP.parse(snippet, relative) - assert_empty result.errors + file_contents.split(/(?<=\S)\n\n(?=\S)/).each do |snippet| + snippet = snippet.rstrip + result = YARP.parse(snippet, relative) + assert_empty result.errors - assert_equal_nodes(result.value, YARP.load(snippet, YARP.dump(snippet, relative)).value) + assert_equal_nodes(result.value, YARP.load(snippet, YARP.dump(snippet, relative)).value) + end end end - end - private + private - # Check that the location ranges of each node in the tree are a superset of - # their respective child nodes. - def assert_non_overlapping_locations(node) - queue = [node] + # Check that the location ranges of each node in the tree are a superset of + # their respective child nodes. + def assert_non_overlapping_locations(node) + queue = [node] - while (current = queue.shift) - # We only want to compare parent/child location overlap in the case that - # we are not looking at a heredoc. That's because heredoc locations are - # special in that they only use the declaration of the heredoc. - compare = !(current.is_a?(YARP::InterpolatedStringNode) || current.is_a?(YARP::InterpolatedXStringNode)) || !current.opening&.start_with?("<<") + while (current = queue.shift) + # We only want to compare parent/child location overlap in the case that + # we are not looking at a heredoc. That's because heredoc locations are + # special in that they only use the declaration of the heredoc. + compare = !(current.is_a?(InterpolatedStringNode) || current.is_a?(InterpolatedXStringNode)) || !current.opening&.start_with?("<<") - current.child_nodes.each do |child| - # child_nodes can return nil values, so we need to skip those. - next unless child + current.child_nodes.each do |child| + # child_nodes can return nil values, so we need to skip those. + next unless child - # Now that we know we have a child node, add that to the queue. - queue << child + # Now that we know we have a child node, add that to the queue. + queue << child - if compare - assert_operator current.location.start_offset, :<=, child.location.start_offset - assert_operator current.location.end_offset, :>=, child.location.end_offset + if compare + assert_operator current.location.start_offset, :<=, child.location.start_offset + assert_operator current.location.end_offset, :>=, child.location.end_offset + end end end end - end - def find_source_file_node(program) - queue = [program] - while (node = queue.shift) - return node if node.is_a?(YARP::SourceFileNode) - queue.concat(node.child_nodes.compact) + def find_source_file_node(program) + queue = [program] + while (node = queue.shift) + return node if node.is_a?(SourceFileNode) + queue.concat(node.child_nodes.compact) + end end - end - def ignore_warnings - previous_verbosity = $VERBOSE - $VERBOSE = nil - yield - ensure - $VERBOSE = previous_verbosity + def ignore_warnings + previous_verbosity = $VERBOSE + $VERBOSE = nil + yield + ensure + $VERBOSE = previous_verbosity + end end end diff --git a/test/yarp/regexp_test.rb b/test/yarp/regexp_test.rb index 241fcc862f9..9863a54758a 100644 --- a/test/yarp/regexp_test.rb +++ b/test/yarp/regexp_test.rb @@ -4,196 +4,198 @@ return if YARP::BACKEND == :FFI -class RegexpTest < Test::Unit::TestCase - ############################################################################## - # These tests test the actual use case of extracting named capture groups - ############################################################################## +module YARP + class RegexpTest < TestCase + ############################################################################## + # These tests test the actual use case of extracting named capture groups + ############################################################################## + + def test_named_captures_with_arrows + assert_equal(["foo"], named_captures("(?bar)")) + end + + def test_named_captures_with_single_quotes + assert_equal(["foo"], named_captures("(?'foo'bar)")) + end + + def test_nested_named_captures_with_arrows + assert_equal(["foo", "bar"], named_captures("(?(?baz))")) + end + + def test_nested_named_captures_with_single_quotes + assert_equal(["foo", "bar"], named_captures("(?'foo'(?'bar'baz))")) + end + + def test_allows_duplicate_named_captures + assert_equal(["foo", "foo"], named_captures("(?bar)(?baz)")) + end + + def test_named_capture_inside_fake_range_quantifier + assert_equal(["foo"], named_captures("foo{1, (?2)}")) + end + + ############################################################################## + # These tests test the rest of the AST. They are not exhaustive, but they + # should cover the most common cases. We test these to make sure we don't + # accidentally regress and stop being able to extract named captures. + ############################################################################## + + def test_alternation + refute_nil(named_captures("foo|bar")) + end + + def test_anchors + refute_nil(named_captures("^foo$")) + end + + def test_any + refute_nil(named_captures(".")) + end + + def test_posix_character_classes + refute_nil(named_captures("[[:digit:]]")) + end + + def test_negated_posix_character_classes + refute_nil(named_captures("[[:^digit:]]")) + end + + def test_invalid_posix_character_classes_should_fall_back_to_regular_classes + refute_nil(named_captures("[[:foo]]")) + end + + def test_character_sets + refute_nil(named_captures("[abc]")) + end + + def test_nested_character_sets + refute_nil(named_captures("[[abc]]")) + end + + def test_nested_character_sets_with_operators + refute_nil(named_captures("[[abc] && [def]]")) + end + + def test_named_capture_inside_nested_character_set + assert_equal([], named_captures("[foo (?bar)]")) + end + + def test_negated_character_sets + refute_nil(named_captures("[^abc]")) + end + + def test_character_ranges + refute_nil(named_captures("[a-z]")) + end + + def test_negated_character_ranges + refute_nil(named_captures("[^a-z]")) + end + + def test_fake_named_captures_inside_character_sets + assert_equal([], named_captures("[a-z(?)]")) + end + + def test_fake_named_capture_inside_character_set_with_escaped_ending + assert_equal([], named_captures("[a-z\\](?)]")) + end + + def test_comments + refute_nil(named_captures("(?#foo)")) + end + + def test_comments_with_escaped_parentheses + refute_nil(named_captures("(?#foo\\)\\))")) + end - def test_named_captures_with_arrows - assert_equal(["foo"], named_captures("(?bar)")) - end - - def test_named_captures_with_single_quotes - assert_equal(["foo"], named_captures("(?'foo'bar)")) - end - - def test_nested_named_captures_with_arrows - assert_equal(["foo", "bar"], named_captures("(?(?baz))")) - end - - def test_nested_named_captures_with_single_quotes - assert_equal(["foo", "bar"], named_captures("(?'foo'(?'bar'baz))")) - end - - def test_allows_duplicate_named_captures - assert_equal(["foo", "foo"], named_captures("(?bar)(?baz)")) - end - - def test_named_capture_inside_fake_range_quantifier - assert_equal(["foo"], named_captures("foo{1, (?2)}")) - end - - ############################################################################## - # These tests test the rest of the AST. They are not exhaustive, but they - # should cover the most common cases. We test these to make sure we don't - # accidentally regress and stop being able to extract named captures. - ############################################################################## - - def test_alternation - refute_nil(named_captures("foo|bar")) - end - - def test_anchors - refute_nil(named_captures("^foo$")) - end - - def test_any - refute_nil(named_captures(".")) - end - - def test_posix_character_classes - refute_nil(named_captures("[[:digit:]]")) - end - - def test_negated_posix_character_classes - refute_nil(named_captures("[[:^digit:]]")) - end - - def test_invalid_posix_character_classes_should_fall_back_to_regular_classes - refute_nil(named_captures("[[:foo]]")) - end - - def test_character_sets - refute_nil(named_captures("[abc]")) - end - - def test_nested_character_sets - refute_nil(named_captures("[[abc]]")) - end - - def test_nested_character_sets_with_operators - refute_nil(named_captures("[[abc] && [def]]")) - end - - def test_named_capture_inside_nested_character_set - assert_equal([], named_captures("[foo (?bar)]")) - end - - def test_negated_character_sets - refute_nil(named_captures("[^abc]")) - end + def test_non_capturing_groups + refute_nil(named_captures("(?:foo)")) + end - def test_character_ranges - refute_nil(named_captures("[a-z]")) - end + def test_positive_lookaheads + refute_nil(named_captures("(?=foo)")) + end - def test_negated_character_ranges - refute_nil(named_captures("[^a-z]")) - end + def test_negative_lookaheads + refute_nil(named_captures("(?!foo)")) + end - def test_fake_named_captures_inside_character_sets - assert_equal([], named_captures("[a-z(?)]")) - end - - def test_fake_named_capture_inside_character_set_with_escaped_ending - assert_equal([], named_captures("[a-z\\](?)]")) - end - - def test_comments - refute_nil(named_captures("(?#foo)")) - end - - def test_comments_with_escaped_parentheses - refute_nil(named_captures("(?#foo\\)\\))")) - end - - def test_non_capturing_groups - refute_nil(named_captures("(?:foo)")) - end - - def test_positive_lookaheads - refute_nil(named_captures("(?=foo)")) - end - - def test_negative_lookaheads - refute_nil(named_captures("(?!foo)")) - end - - def test_positive_lookbehinds - refute_nil(named_captures("(?<=foo)")) - end + def test_positive_lookbehinds + refute_nil(named_captures("(?<=foo)")) + end - def test_negative_lookbehinds - refute_nil(named_captures("(?foo)")) - end + def test_atomic_groups + refute_nil(named_captures("(?>foo)")) + end - def test_absence_operator - refute_nil(named_captures("(?~foo)")) - end + def test_absence_operator + refute_nil(named_captures("(?~foo)")) + end - def test_conditional_expression_with_index - refute_nil(named_captures("(?(1)foo)")) - end + def test_conditional_expression_with_index + refute_nil(named_captures("(?(1)foo)")) + end - def test_conditional_expression_with_name - refute_nil(named_captures("(?(foo)bar)")) - end + def test_conditional_expression_with_name + refute_nil(named_captures("(?(foo)bar)")) + end - def test_conditional_expression_with_group - refute_nil(named_captures("(?()bar)")) - end + def test_conditional_expression_with_group + refute_nil(named_captures("(?()bar)")) + end - def test_options_on_groups - refute_nil(named_captures("(?imxdau:foo)")) - end + def test_options_on_groups + refute_nil(named_captures("(?imxdau:foo)")) + end - def test_options_on_groups_with_invalid_options - assert_nil(named_captures("(?z:bar)")) - end + def test_options_on_groups_with_invalid_options + assert_nil(named_captures("(?z:bar)")) + end - def test_options_on_groups_getting_turned_off - refute_nil(named_captures("(?-imx:foo)")) - end + def test_options_on_groups_getting_turned_off + refute_nil(named_captures("(?-imx:foo)")) + end - def test_options_on_groups_some_getting_turned_on_some_getting_turned_off - refute_nil(named_captures("(?im-x:foo)")) - end + def test_options_on_groups_some_getting_turned_on_some_getting_turned_off + refute_nil(named_captures("(?im-x:foo)")) + end - def test_star_quantifier - refute_nil(named_captures("foo*")) - end + def test_star_quantifier + refute_nil(named_captures("foo*")) + end - def test_plus_quantifier - refute_nil(named_captures("foo+")) - end + def test_plus_quantifier + refute_nil(named_captures("foo+")) + end - def test_question_mark_quantifier - refute_nil(named_captures("foo?")) - end + def test_question_mark_quantifier + refute_nil(named_captures("foo?")) + end - def test_endless_range_quantifier - refute_nil(named_captures("foo{1,}")) - end + def test_endless_range_quantifier + refute_nil(named_captures("foo{1,}")) + end - def test_beginless_range_quantifier - refute_nil(named_captures("foo{,1}")) - end + def test_beginless_range_quantifier + refute_nil(named_captures("foo{,1}")) + end - def test_range_quantifier - refute_nil(named_captures("foo{1,2}")) - end + def test_range_quantifier + refute_nil(named_captures("foo{1,2}")) + end - def test_fake_range_quantifier_because_of_spaces - refute_nil(named_captures("foo{1, 2}")) - end + def test_fake_range_quantifier_because_of_spaces + refute_nil(named_captures("foo{1, 2}")) + end - private + private - def named_captures(source) - YARP.const_get(:Debug).named_captures(source) + def named_captures(source) + Debug.named_captures(source) + end end end diff --git a/test/yarp/ripper_compat_test.rb b/test/yarp/ripper_compat_test.rb index e13cef08b1f..9fcdfe63c68 100644 --- a/test/yarp/ripper_compat_test.rb +++ b/test/yarp/ripper_compat_test.rb @@ -3,7 +3,7 @@ require_relative "test_helper" module YARP - class RipperCompatTest < Test::Unit::TestCase + class RipperCompatTest < TestCase def test_1_plus_2 assert_equivalent("1 + 2") end diff --git a/test/yarp/ruby_api_test.rb b/test/yarp/ruby_api_test.rb index 31b3f0fdec3..dc12012f44a 100644 --- a/test/yarp/ruby_api_test.rb +++ b/test/yarp/ruby_api_test.rb @@ -2,61 +2,63 @@ require_relative "test_helper" -class YARPRubyAPITest < Test::Unit::TestCase - def test_ruby_api - filepath = __FILE__ - source = File.read(filepath, binmode: true, external_encoding: Encoding::UTF_8) +module YARP + class RubyAPITest < TestCase + def test_ruby_api + filepath = __FILE__ + source = File.read(filepath, binmode: true, external_encoding: Encoding::UTF_8) - assert_equal YARP.lex(source, filepath).value, YARP.lex_file(filepath).value - assert_equal YARP.dump(source, filepath), YARP.dump_file(filepath) + assert_equal YARP.lex(source, filepath).value, YARP.lex_file(filepath).value + assert_equal YARP.dump(source, filepath), YARP.dump_file(filepath) - serialized = YARP.dump(source, filepath) - ast1 = YARP.load(source, serialized).value - ast2 = YARP.parse(source, filepath).value - ast3 = YARP.parse_file(filepath).value + serialized = YARP.dump(source, filepath) + ast1 = YARP.load(source, serialized).value + ast2 = YARP.parse(source, filepath).value + ast3 = YARP.parse_file(filepath).value - assert_equal_nodes ast1, ast2 - assert_equal_nodes ast2, ast3 - end + assert_equal_nodes ast1, ast2 + assert_equal_nodes ast2, ast3 + end - def test_literal_value_method - assert_equal 123, parse_expression("123").value - assert_equal 3.14, parse_expression("3.14").value - assert_equal 42i, parse_expression("42i").value - assert_equal 42.1ri, parse_expression("42.1ri").value - assert_equal 3.14i, parse_expression("3.14i").value - assert_equal 42r, parse_expression("42r").value - assert_equal 0.5r, parse_expression("0.5r").value - assert_equal 42ri, parse_expression("42ri").value - assert_equal 0.5ri, parse_expression("0.5ri").value - end + def test_literal_value_method + assert_equal 123, parse_expression("123").value + assert_equal 3.14, parse_expression("3.14").value + assert_equal 42i, parse_expression("42i").value + assert_equal 42.1ri, parse_expression("42.1ri").value + assert_equal 3.14i, parse_expression("3.14i").value + assert_equal 42r, parse_expression("42r").value + assert_equal 0.5r, parse_expression("0.5r").value + assert_equal 42ri, parse_expression("42ri").value + assert_equal 0.5ri, parse_expression("0.5ri").value + end - def test_location_join - recv, args_node, _ = parse_expression("1234 + 567").child_nodes - arg = args_node.arguments[0] + def test_location_join + recv, args_node, _ = parse_expression("1234 + 567").child_nodes + arg = args_node.arguments[0] - joined = recv.location.join(arg.location) - assert_equal 0, joined.start_offset - assert_equal 10, joined.length + joined = recv.location.join(arg.location) + assert_equal 0, joined.start_offset + assert_equal 10, joined.length - assert_raise RuntimeError, "Incompatible locations" do - arg.location.join(recv.location) - end + assert_raise RuntimeError, "Incompatible locations" do + arg.location.join(recv.location) + end - other_arg = parse_expression("1234 + 567").arguments.arguments[0] + other_arg = parse_expression("1234 + 567").arguments.arguments[0] - assert_raise RuntimeError, "Incompatible sources" do - other_arg.location.join(recv.location) - end + assert_raise RuntimeError, "Incompatible sources" do + other_arg.location.join(recv.location) + end - assert_raise RuntimeError, "Incompatible sources" do - recv.location.join(other_arg.location) + assert_raise RuntimeError, "Incompatible sources" do + recv.location.join(other_arg.location) + end end - end - private + private - def parse_expression(source) - YARP.parse(source).value.statements.body.first + def parse_expression(source) + YARP.parse(source).value.statements.body.first + end end end diff --git a/test/yarp/test_helper.rb b/test/yarp/test_helper.rb index 0be0f51651f..49587e3b03b 100644 --- a/test/yarp/test_helper.rb +++ b/test/yarp/test_helper.rb @@ -8,8 +8,15 @@ puts "Using YARP backend: #{YARP::BACKEND}" if ENV["YARP_FFI_BACKEND"] +# It is useful to have a diff even if the strings to compare are big +# However, ruby/ruby does not have a version of Test::Unit with access to +# max_diff_target_string_size +if defined?(Test::Unit::Assertions::AssertionMessage) + Test::Unit::Assertions::AssertionMessage.max_diff_target_string_size = 5000 +end + module YARP - module Assertions + class TestCase < ::Test::Unit::TestCase private def assert_equal_nodes(expected, actual, compare_location: true, parent: nil) @@ -31,18 +38,17 @@ def assert_equal_nodes(expected, actual, compare_location: true, parent: nil) parent: actual ) end - when YARP::SourceFileNode + when SourceFileNode deconstructed_expected = expected.deconstruct_keys(nil) deconstructed_actual = actual.deconstruct_keys(nil) assert_equal deconstructed_expected.keys, deconstructed_actual.keys - # Filepaths can be different if test suites were run - # on different machines. - # We accommodate for this by comparing the basenames, - # and not the absolute filepaths + # Filepaths can be different if test suites were run on different + # machines. We accommodate for this by comparing the basenames, and not + # the absolute filepaths. assert_equal deconstructed_expected.except(:filepath), deconstructed_actual.except(:filepath) assert_equal File.basename(deconstructed_expected[:filepath]), File.basename(deconstructed_actual[:filepath]) - when YARP::Node + when Node deconstructed_expected = expected.deconstruct_keys(nil) deconstructed_actual = actual.deconstruct_keys(nil) assert_equal deconstructed_expected.keys, deconstructed_actual.keys @@ -55,10 +61,11 @@ def assert_equal_nodes(expected, actual, compare_location: true, parent: nil) parent: actual ) end - when YARP::Location + when Location assert_operator actual.start_offset, :<=, actual.end_offset, -> { "start_offset > end_offset for #{actual.inspect}, parent is #{parent.pretty_inspect}" } + if compare_location assert_equal( expected.start_offset, @@ -71,7 +78,6 @@ def assert_equal_nodes(expected, actual, compare_location: true, parent: nil) actual.end_offset, -> { "End locations were different. Parent: #{parent.pretty_inspect}" } ) - end else assert_equal expected, actual @@ -84,11 +90,11 @@ def assert_valid_locations(value, parent: nil) value.each do |element| assert_valid_locations(element, parent: value) end - when YARP::Node + when Node value.deconstruct_keys(nil).each_value do |field| assert_valid_locations(field, parent: value) end - when YARP::Location + when Location assert_operator value.start_offset, :<=, value.end_offset, -> { "start_offset > end_offset for #{value.inspect}, parent is #{parent.pretty_inspect}" } @@ -96,5 +102,3 @@ def assert_valid_locations(value, parent: nil) end end end - -Test::Unit::TestCase.include(YARP::Assertions) diff --git a/test/yarp/unescape_test.rb b/test/yarp/unescape_test.rb index eef989ad23a..f39bdd0e394 100644 --- a/test/yarp/unescape_test.rb +++ b/test/yarp/unescape_test.rb @@ -4,8 +4,8 @@ return if YARP::BACKEND == :FFI -module UnescapeTest - class UnescapeNoneTest < Test::Unit::TestCase +module YARP + class UnescapeNoneTest < TestCase def test_backslash assert_unescape_none("\\") end @@ -17,11 +17,11 @@ def test_single_quote private def assert_unescape_none(source) - assert_equal(source, YARP.const_get(:Debug).unescape_none(source)) + assert_equal(source, Debug.unescape_none(source)) end end - class UnescapeMinimalTest < Test::Unit::TestCase + class UnescapeMinimalTest < TestCase def test_backslash assert_unescape_minimal("\\", "\\\\") end @@ -37,11 +37,11 @@ def test_single_char private def assert_unescape_minimal(expected, source) - assert_equal(expected, YARP.const_get(:Debug).unescape_minimal(source)) + assert_equal(expected, Debug.unescape_minimal(source)) end end - class UnescapeAllTest < Test::Unit::TestCase + class UnescapeAllTest < TestCase def test_backslash assert_unescape_all("\\", "\\\\") end @@ -139,7 +139,7 @@ def test_escaping_normal_characters private def unescape_all(source) - YARP.const_get(:Debug).unescape_all(source) + Debug.unescape_all(source) end def assert_unescape_all(expected, source, forced_encoding = nil) diff --git a/test/yarp/version_test.rb b/test/yarp/version_test.rb index aaace0aa89a..6011eb695d7 100644 --- a/test/yarp/version_test.rb +++ b/test/yarp/version_test.rb @@ -2,8 +2,10 @@ require_relative "test_helper" -class VersionTest < Test::Unit::TestCase - def test_version_is_set - refute_nil YARP::VERSION +module YARP + class VersionTest < TestCase + def test_version_is_set + refute_nil VERSION + end end end From 4c76f4a0c04a7245a395c415228e599312f28c67 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Wed, 30 Aug 2023 10:40:29 -0400 Subject: [PATCH 2/2] Revisit lex.rake, make lex:rubygems more useable --- bin/lex | 2 +- rakelib/lex.rake | 343 +++++++++++++----------------- test/yarp/desugar_visitor_test.rb | 6 +- test/yarp/fuzzer_test.rb | 2 +- test/yarp/test_helper.rb | 17 -- 5 files changed, 154 insertions(+), 216 deletions(-) diff --git a/bin/lex b/bin/lex index 9534b207834..e2f15067ec5 100755 --- a/bin/lex +++ b/bin/lex @@ -22,7 +22,7 @@ pattern = "%-70s %-70s" ripper = begin YARP.lex_ripper(source) - rescue SyntaxError + rescue ArgumentError, SyntaxError # If Ripper raises a syntax error, we want to continue as if it didn't # return any tokens at all. YARP won't raise a syntax error, so it's nicer # to still be able to see the tokens that YARP generated. diff --git a/rakelib/lex.rake b/rakelib/lex.rake index 91faa233625..694be840063 100644 --- a/rakelib/lex.rake +++ b/rakelib/lex.rake @@ -1,34 +1,47 @@ # frozen_string_literal: true module YARP + # This class is responsible for lexing files with both lex_compat and + # lex_ripper and ensuring they match up. It keeps track of the files which + # failed to match up, and the files which passed. class LexTask - attr_reader :previous_todos, :todos, :passing_file_count + attr_reader :failing_files, :passing_file_count - def initialize(previous_todos) - @previous_todos = previous_todos + def initialize @passing_file_count = 0 - @todos = [] + @failing_files = [] end def compare(filepath) - if lex(filepath) + # If we can't read the file, then ignore it. + return true if !File.file?(filepath) || !File.readable?(filepath) + + # Read the file into memory. + source = File.read(filepath) + + # If the filepath contains invalid Ruby code, then ignore it. + begin + RubyVM::AbstractSyntaxTree.parse(source) + rescue ArgumentError, SyntaxError + return true + end + + result = YARP.lex_compat(source) + if result.errors.empty? && YARP.lex_ripper(source) == result.value @passing_file_count += 1 true else - @todos << filepath + @failing_files << filepath false end end def failing_file_count - @todos.length + failing_files.length end - # ENV["TODOS"] was toggled and there are failing files - # or new failures were introduced def failed? - (ENV["TODOS"] && failing_file_count > 0) || - (@todos - @previous_todos).any? + failing_files.any? end def summary @@ -38,22 +51,32 @@ module YARP PERCENT=#{(passing_file_count.to_f / (passing_file_count + failing_file_count) * 100).round(2)}% RESULTS end + end + + class << self + # This method is responsible for iterating through a list of items and running + # each item in a separate thread. It will block until all items have been + # processed. This is particularly useful for tasks that are IO-bound like + # downloading files or reading files from disk. + def parallelize(items, &block) + Thread.abort_on_exception = true + + queue = Queue.new + items.each { |item| queue << item } + + workers = + ENV.fetch("WORKERS") { 16 }.to_i.times.map do + parallelize_thread(queue, &block) + end + + workers.map(&:join) + end private - # For the given filepath, read it and lex it with both lex_compat and - # lex_ripper. Compare the output of both and ensure they match. - def lex(filepath) - source = File.read(filepath) - lexed = YARP.lex_compat(source) - begin - lexed_ripper = YARP.lex_ripper(source) - rescue SyntaxError - return true # If the file is invalid, we say the output is equivalent for comparison purposes - end - lexed.errors.empty? && lexed_ripper == lexed.value - rescue - false + # Create a new thread with a minimal number of locals that it can access. + def parallelize_thread(queue, &block) + Thread.new { block.call(queue.shift) until queue.empty? } end end end @@ -96,7 +119,7 @@ TARGETS.each do |name, target| plain_text = ENV.fetch("CI", false) warn_failing = ENV.fetch("VERBOSE", false) - lex_task = YARP::LexTask.new(target.fetch(:todos, []).map { |todo| File.join(dirpath, todo) }) + lex_task = YARP::LexTask.new filepaths = Dir[File.join(dirpath, "**", "*.rb")] if excludes = target[:excludes] @@ -118,20 +141,9 @@ TARGETS.each do |name, target| puts("\n\n") - previous_todos = lex_task.previous_todos.map { _1.gsub(/tmp\/targets\/[a-zA-Z0-9_]*\//, "") } - current_todos = lex_task.todos.map { _1.gsub(/tmp\/targets\/[a-zA-Z0-9_]*\//, "") } - current_minus_previous = current_todos - previous_todos - previous_minus_current = previous_todos - current_todos - - if current_minus_previous.any? + if lex_task.failing_files.any? puts("Uh oh, there are some files which were previously passing but are now failing. Here are the files:") - puts(" - #{current_minus_previous.join("\n - ")}") - elsif (previous_minus_current).any? - puts("Some files listed as todo are now passing:") - puts(" - #{previous_minus_current.join("\n - ")}") - - puts("This is the new list which can be copied into lex.rake:") - puts(" - #{current_todos.join("\n - ")}") + puts(" - #{lex_task.failing_files.map { _1.gsub(/tmp\/targets\/[a-zA-Z0-9_]*\//, "") }.join("\n - ")}") else puts("The todos list in lex.rake is up to date") end @@ -150,7 +162,7 @@ task lex: :compile do plain_text = ENV.fetch("CI", false) warn_failing = ENV.fetch("VERBOSE", false) - lex_task = YARP::LexTask.new([]) + lex_task = YARP::LexTask.new filepaths = Dir[ENV.fetch("FILEPATHS")] filepaths.each do |filepath| @@ -170,8 +182,10 @@ task lex: :compile do exit(1) if lex_task.failed? end +directory "tmp/failing" + desc "Lex against the most recent version of various rubygems" -task "lex:rubygems": :compile do +task "lex:rubygems": [:compile, "tmp/failing"] do $:.unshift(File.expand_path("../lib", __dir__)) require "net/http" require "ripper" @@ -179,12 +193,12 @@ task "lex:rubygems": :compile do require "tmpdir" require "yarp" - queue = Queue.new + items = [] Gem::SpecFetcher.new.available_specs(:latest).first.each do |source, gems| gems.each do |tuple| gem_name = File.basename(tuple.spec_name, ".gemspec") - gem_url = source.uri.merge("/gems/#{gem_name}.gem") - queue << [gem_name, gem_url] + gem_uri = source.uri.merge("/gems/#{gem_name}.gem") + items << [gem_name, gem_uri] end end @@ -192,48 +206,40 @@ task "lex:rubygems": :compile do passing_gem_count = 0 failing_gem_count = 0 - workers = - ENV.fetch("WORKERS", 16).times.map do - Thread.new do - Net::HTTP.start("rubygems.org", 443, use_ssl: true) do |http| - until queue.empty? - (gem_name, gem_url) = queue.shift - - http.request(Net::HTTP::Get.new(gem_url)) do |response| - # Skip unexpected responses - next unless response.is_a?(Net::HTTPSuccess) - - Dir.mktmpdir do |directory| - filepath = File.join(directory, "#{gem_name}.gem") - File.write(filepath, response.body) - - begin - Gem::Package.new(filepath).extract_files(directory, "[!~]*") - - lex_task = YARP::LexTask.new([]) - Dir[File.join(directory, "**", "*.rb")].each do |filepath| - lex_task.compare(filepath) - end - - if lex_task.failed? - failing_gem_count += 1 - print("\033[31mE\033[0m") - else - passing_gem_count += 1 - warn(gem_name) if warn_failing - print("\033[32m.\033[0m") - end - rescue - # If the gem fails to extract, we'll just skip it - end - end - end - end + YARP.parallelize(items) do |(gem_name, gem_uri)| + # items.each do |(gem_name, gem_uri)| + response = Net::HTTP.get_response(gem_uri) + next unless response.is_a?(Net::HTTPSuccess) + + Dir.mktmpdir do |directory| + filepath = File.join(directory, "#{gem_name}.gem") + File.write(filepath, response.body) + + begin + Gem::Package.new(filepath).extract_files(directory, "[!~]*") + rescue + # If the gem fails to extract, we'll just skip it + next + end + + lex_task = YARP::LexTask.new + Dir[File.join(directory, "**", "*.rb")].each do |filepath| + unless lex_task.compare(filepath) + cp filepath, "tmp/failing/#{SecureRandom.hex}-#{File.basename(filepath)}" end end + + if lex_task.failed? + failing_gem_count += 1 + print("\033[31mE\033[0m") + else + passing_gem_count += 1 + warn(gem_name) if warn_failing + print("\033[32m.\033[0m") + end end + end - workers.each(&:join) puts(<<~RESULTS) PASSING=#{passing_gem_count} FAILING=#{failing_gem_count} @@ -250,84 +256,66 @@ TOP_100_GEMS_INVALID_SYNTAX_PREFIXES = %w[ top-100-gems/fastlane-2.212.1/fastlane/lib/assets/custom_action_template.rb ] -desc "Download the top 100 rubygems under #{TOP_100_GEMS_DIR}/" -task "download:topgems" do - $:.unshift(File.expand_path("../lib", __dir__)) - require "net/http" - require "rubygems/package" - require "tmpdir" +namespace :download do + directory TOP_100_GEMS_DIR - queue = Queue.new - YAML.safe_load_file(TOP_100_GEM_FILENAME).each do |gem_name| - gem_url = "https://rubygems.org/gems/#{gem_name}.gem" - queue << [gem_name, gem_url] - end + desc "Download the top 100 rubygems under #{TOP_100_GEMS_DIR}/" + task topgems: TOP_100_GEMS_DIR do + $:.unshift(File.expand_path("../lib", __dir__)) + require "net/http" + require "rubygems/package" + require "tmpdir" - Dir.mkdir TOP_100_GEMS_DIR unless File.directory? TOP_100_GEMS_DIR - - workers = - ENV.fetch("WORKERS", 16).times.map do - Thread.new do - Net::HTTP.start("rubygems.org", 443, use_ssl: true) do |http| - until queue.empty? - (gem_name, gem_url) = queue.shift - directory = File.expand_path("#{TOP_100_GEMS_DIR}/#{gem_name}") - unless File.directory? directory - puts "Downloading #{gem_name}" - - http.request(Net::HTTP::Get.new(gem_url)) do |response| - # Skip unexpected responses - raise gem_url unless response.is_a?(Net::HTTPSuccess) - - Dir.mktmpdir do |tmpdir| - filepath = File.join(tmpdir, "#{gem_name}.gem") - File.write(filepath, response.body) - - Gem::Package.new(filepath).extract_files(directory, "**/*.rb") - end - end - end - end - end + YARP.parallelize(YAML.safe_load_file(TOP_100_GEM_FILENAME)) do |gem_name| + directory = File.expand_path("#{TOP_100_GEMS_DIR}/#{gem_name}") + next if File.directory?(directory) + + puts "Downloading #{gem_name}" + + uri = URI.parse("https://rubygems.org/gems/#{gem_name}.gem") + response = Net::HTTP.get_response(uri) + raise gem_name unless response.is_a?(Net::HTTPSuccess) + + Dir.mktmpdir do |tmpdir| + filepath = File.join(tmpdir, "#{gem_name}.gem") + File.write(filepath, response.body) + Gem::Package.new(filepath).extract_files(directory, "**/*.rb") end end - - workers.each(&:join) + end end -# This task parses each .rb file of the top 100 gems with YARP and ensures they parse -# successfully (unless they are invalid syntax as confirmed by "ruby -c"). -# It also does some sanity check for every location recorded in the AST. +# This task parses each .rb file of the top 100 gems with YARP and ensures they +# parse successfully (unless they are invalid syntax as confirmed by "ruby -c"). desc "Parse the top 100 rubygems" task "parse:topgems": ["download:topgems", :compile] do + $:.unshift(File.expand_path("../lib", __dir__)) require "yarp" - require_relative "../test/yarp/test_helper" - - module YARP - class ParseTop100GemsTest < TestCase - Dir["#{TOP_100_GEMS_DIR}/**/*.rb"].each do |filepath| - test filepath do - result = YARP.parse_file(filepath) - - if TOP_100_GEMS_INVALID_SYNTAX_PREFIXES.any? { |prefix| filepath.start_with?(prefix) } - assert_false result.success? - # ensure it is actually invalid syntax - assert_false system(RbConfig.ruby, "-c", filepath, out: File::NULL, err: File::NULL) - next - end - - assert result.success?, "failed to parse #{filepath}" - value = result.value - assert_valid_locations(value) - end - end + + incorrect = [] + YARP.parallelize(Dir["#{TOP_100_GEMS_DIR}/**/*.rb"]) do |filepath| + result = YARP.parse_file(filepath) + + if TOP_100_GEMS_INVALID_SYNTAX_PREFIXES.any? { |prefix| filepath.start_with?(prefix) } + # Check that the file failed to parse and that it actually contains + # invalid syntax. + incorrect << filepath if result.success? || system(RbConfig.ruby, "-c", filepath, out: File::NULL, err: File::NULL) + else + incorrect << filepath unless result.success? end end + + if incorrect.any? + warn("The following files failed to parse:") + warn(" - #{incorrect.join("\n - ")}") + exit(1) + else + puts("All files parsed successfully.") + end end -# This task lexes against the top 100 gems, and will exit(1) -# if the existing failures in rakelib/top-100-gems.yml are no -# longer correct. +# This task lexes against the top 100 gems, and will exit(1) if any files fail +# to lex properly. desc "Lex against the top 100 rubygems" task "lex:topgems": ["download:topgems", :compile] do $:.unshift(File.expand_path("../lib", __dir__)) @@ -337,67 +325,34 @@ task "lex:topgems": ["download:topgems", :compile] do require "tmpdir" require "yarp" - previous_todos_by_gem_name = {} + gem_names = YAML.safe_load_file(TOP_100_GEM_FILENAME) + failing_files = {} - YAML.safe_load_file(TOP_100_GEM_FILENAME).each do |gem_info| - if gem_info.class == Hash - previous_todos_by_gem_name.merge!(gem_info) - else - previous_todos_by_gem_name[gem_info] = [] - end - end - - warn_failing = ENV.fetch("VERBOSE", false) - - # An array of hashes with gem_name => [new_failing_files] for any new - # failures that we didn't previously have. If there is anything in - # this list, the task will report it and exit(1) - new_failing_files_by_gem = [] - - # An array of hashes with gem_name => [all_failing_files] for all failures - # (including pre-existing ones). If there are fewer failures than before, - # this list will be used to generate the new yml file before exit(1) - updated_todos_by_gem_name = {} - - previous_todos_by_gem_name.keys.each do |gem_name| + YARP.parallelize(gem_names) do |gem_name| puts "Lexing #{gem_name}" - directory = "#{TOP_100_GEMS_DIR}/#{gem_name}" + directory = File.expand_path("#{TOP_100_GEMS_DIR}/#{gem_name}") - todos = previous_todos_by_gem_name[gem_name].map do |todo_filepath| - File.join(directory, todo_filepath) - end - lex_task = YARP::LexTask.new(todos) + lex_task = YARP::LexTask.new Dir[File.join(directory, "**", "*.rb")].each do |filepath| lex_task.compare(filepath) end - todos = lex_task.todos.map { _1.gsub("#{directory}/", "") } - - updated_todos_by_gem_name.merge!({ gem_name => todos }) - - if lex_task.failed? - new_failing_files_by_gem << { gem_name => todos } - end + gem_failing_files = lex_task.failing_files.map { |todo| todo.delete_prefix("#{directory}/") } + failing_files[gem_name] = gem_failing_files if gem_failing_files.any? end - failing_gem_count, passing_gem_count = updated_todos_by_gem_name.partition { |_, todos| todos.any? }.map(&:size) + failing = failing_files.length + passing = gem_names.length - failing puts(<<~RESULTS) - PASSING=#{passing_gem_count} - FAILING=#{failing_gem_count} - PERCENT=#{(passing_gem_count.to_f / (passing_gem_count + failing_gem_count) * 100).round(2)}% + PASSING=#{passing} + FAILING=#{failing} + PERCENT=#{(passing.to_f / gem_names.length * 100).round(2)}% RESULTS - if new_failing_files_by_gem.any? - puts "Oh no! There were new failures:" - puts new_failing_files_by_gem.to_yaml - exit(1) - elsif (updated_todos_by_gem_name != previous_todos_by_gem_name) - puts "There are files that were previously failing but are no longer failing:" - puts "Please update #{TOP_100_GEM_FILENAME} with the following" - puts (updated_todos_by_gem_name.sort_by(&:first).map do |k, v| - v.any? ? { k => v } : k - end).to_yaml + if failing > 0 + warn("The following files failed to lex:") + warn(failing_files.to_yaml) exit(1) end end diff --git a/test/yarp/desugar_visitor_test.rb b/test/yarp/desugar_visitor_test.rb index 4dee182ef1b..c03b02e67a0 100644 --- a/test/yarp/desugar_visitor_test.rb +++ b/test/yarp/desugar_visitor_test.rb @@ -13,7 +13,7 @@ def test_and_write assert_desugars("(AndNode (LocalVariableReadNode) (LocalVariableWriteNode (CallNode)))", "foo &&= bar") assert_desugars("(AndNode (LocalVariableReadNode) (LocalVariableWriteNode (CallNode)))", "foo = 1; foo &&= bar") end - + def test_or_write assert_desugars("(IfNode (DefinedNode (ClassVariableReadNode)) (StatementsNode (ClassVariableReadNode)) (ElseNode (StatementsNode (ClassVariableWriteNode (CallNode)))))", "@@foo ||= bar") assert_desugars("(IfNode (DefinedNode (ConstantPathNode (ConstantReadNode) (ConstantReadNode))) (StatementsNode (ConstantPathNode (ConstantReadNode) (ConstantReadNode))) (ElseNode (StatementsNode (ConstantPathWriteNode (ConstantPathNode (ConstantReadNode) (ConstantReadNode)) (CallNode)))))", "Foo::Bar ||= baz") @@ -23,7 +23,7 @@ def test_or_write assert_desugars("(OrNode (LocalVariableReadNode) (LocalVariableWriteNode (CallNode)))", "foo ||= bar") assert_desugars("(OrNode (LocalVariableReadNode) (LocalVariableWriteNode (CallNode)))", "foo = 1; foo ||= bar") end - + def test_operator_write assert_desugars("(ClassVariableWriteNode (CallNode (ClassVariableReadNode) (ArgumentsNode (CallNode))))", "@@foo += bar") assert_desugars("(ConstantPathWriteNode (ConstantPathNode (ConstantReadNode) (ConstantReadNode)) (CallNode (ConstantPathNode (ConstantReadNode) (ConstantReadNode)) (ArgumentsNode (CallNode))))", "Foo::Bar += baz") @@ -33,7 +33,7 @@ def test_operator_write assert_desugars("(LocalVariableWriteNode (CallNode (LocalVariableReadNode) (ArgumentsNode (CallNode))))", "foo += bar") assert_desugars("(LocalVariableWriteNode (CallNode (LocalVariableReadNode) (ArgumentsNode (CallNode))))", "foo = 1; foo += bar") end - + private def ast_inspect(node) diff --git a/test/yarp/fuzzer_test.rb b/test/yarp/fuzzer_test.rb index 61845b91f76..384f3ff0d20 100644 --- a/test/yarp/fuzzer_test.rb +++ b/test/yarp/fuzzer_test.rb @@ -4,7 +4,7 @@ module YARP # These tests are simply to exercise snippets found by the fuzzer that caused invalid memory access. - class FuzzerTest < Test::Unit::TestCase + class FuzzerTest < TestCase def self.snippet(name, source) define_method(:"test_fuzzer_#{name}") { YARP.dump(source) } end diff --git a/test/yarp/test_helper.rb b/test/yarp/test_helper.rb index 49587e3b03b..b79adf4b166 100644 --- a/test/yarp/test_helper.rb +++ b/test/yarp/test_helper.rb @@ -83,22 +83,5 @@ def assert_equal_nodes(expected, actual, compare_location: true, parent: nil) assert_equal expected, actual end end - - def assert_valid_locations(value, parent: nil) - case value - when Array - value.each do |element| - assert_valid_locations(element, parent: value) - end - when Node - value.deconstruct_keys(nil).each_value do |field| - assert_valid_locations(field, parent: value) - end - when Location - assert_operator value.start_offset, :<=, value.end_offset, -> { - "start_offset > end_offset for #{value.inspect}, parent is #{parent.pretty_inspect}" - } - end - end end end