From 965ca2433a0d4c1c0c59e1ba42c80decd7ce1bec Mon Sep 17 00:00:00 2001 From: sjanusz-r7 Date: Tue, 15 Oct 2024 11:20:34 +0100 Subject: [PATCH] wip: Reline behind a feature flag --- lib/metasploit/framework/command/console.rb | 1 - .../framework/parsed_options/console.rb | 7 +- .../framework/parsed_options/remote_db.rb | 1 - lib/msf/core/feature_manager.rb | 8 ++ lib/msf/ui/console/command_dispatcher/core.rb | 13 +-- .../console/command_dispatcher/developer.rb | 7 ++ lib/msf/ui/console/driver.rb | 96 +++++++++---------- .../console/module_option_tab_completion.rb | 7 +- lib/msf/ui/debug.rb | 5 +- .../console/command_dispatcher/stdapi/fs.rb | 5 +- lib/rex/ui/text/dispatcher_shell.rb | 20 ++-- lib/rex/ui/text/input/readline.rb | 80 +++++----------- lib/rex/ui/text/irb_shell.rb | 19 ++-- lib/rex/ui/text/output/stdio.rb | 1 + 14 files changed, 123 insertions(+), 147 deletions(-) diff --git a/lib/metasploit/framework/command/console.rb b/lib/metasploit/framework/command/console.rb index 3c476574289c9..d104a042d33be 100644 --- a/lib/metasploit/framework/command/console.rb +++ b/lib/metasploit/framework/command/console.rb @@ -92,7 +92,6 @@ def driver_options driver_options['ModulePath'] = options.modules.path driver_options['Plugins'] = options.console.plugins driver_options['Readline'] = options.console.readline - driver_options['RealReadline'] = options.console.real_readline driver_options['Resource'] = options.console.resources driver_options['XCommands'] = options.console.commands diff --git a/lib/metasploit/framework/parsed_options/console.rb b/lib/metasploit/framework/parsed_options/console.rb index 34aa126633aa3..9e9161e0f89ce 100644 --- a/lib/metasploit/framework/parsed_options/console.rb +++ b/lib/metasploit/framework/parsed_options/console.rb @@ -16,7 +16,6 @@ def options options.console.plugins = [] options.console.quiet = false options.console.readline = true - options.console.real_readline = false options.console.resources = [] options.console.subcommand = :run } @@ -54,7 +53,11 @@ def option_parser end option_parser.on('-L', '--real-readline', 'Use the system Readline library instead of RbReadline') do - options.console.real_readline = true + message = "The RealReadline option has been marked as deprecated, and is currently a noop.\n" + message << "Metasploit Framework now uses Reline exclusively as the input handling library.\n" + message << "If you require this functionality, please use the following link to tell us:\n" + message << ' https://github.com/rapid7/metasploit-framework/issues/19399' + warn message end option_parser.on('-o', '--output FILE', 'Output to the specified file') do |file| diff --git a/lib/metasploit/framework/parsed_options/remote_db.rb b/lib/metasploit/framework/parsed_options/remote_db.rb index 49dd9c466644d..5344539bfd42c 100644 --- a/lib/metasploit/framework/parsed_options/remote_db.rb +++ b/lib/metasploit/framework/parsed_options/remote_db.rb @@ -13,7 +13,6 @@ def options options.console.local_output = nil options.console.plugins = [] options.console.quiet = false - options.console.real_readline = false options.console.resources = [] options.console.subcommand = :run } diff --git a/lib/msf/core/feature_manager.rb b/lib/msf/core/feature_manager.rb index 3ab3865ae771e..fdf27545b9dd1 100644 --- a/lib/msf/core/feature_manager.rb +++ b/lib/msf/core/feature_manager.rb @@ -28,6 +28,7 @@ class FeatureManager MSSQL_SESSION_TYPE = 'mssql_session_type' LDAP_SESSION_TYPE = 'ldap_session_type' SHOW_SUCCESSFUL_LOGINS = 'show_successful_logins' + USE_RELINE = 'use_reline' DEFAULTS = [ { @@ -124,6 +125,13 @@ class FeatureManager requires_restart: false, default_value: true, developer_notes: 'Enabled in Metasploit 6.4.x' + }.freeze, + { + name: USE_RELINE, + description: 'When enabled, the new Reline library will be used instead of the legacy Readline library for input/output. New UI sessions will be using Reline, but sessions that existed before this feature has been set will continue to be Readline-based.', + requires_restart: false, + default_value: false, + developer_notes: 'To be enabled by default after sufficient testing and Reline fixes the issues raised here: https://github.com/ruby/reline/issues/created_by/sjanusz-r7' }.freeze ].freeze diff --git a/lib/msf/ui/console/command_dispatcher/core.rb b/lib/msf/ui/console/command_dispatcher/core.rb index 5599879afad6a..990b8468ee348 100644 --- a/lib/msf/ui/console/command_dispatcher/core.rb +++ b/lib/msf/ui/console/command_dispatcher/core.rb @@ -760,7 +760,8 @@ def cmd_features_tabs(_str, words) end def cmd_history(*args) - length = Readline::HISTORY.length + history = Msf::Ui::Console::Driver.using_reline? ? Reline::HISTORY : Readline::HISTORY + length = history.length if length < @history_limit limit = length @@ -780,10 +781,10 @@ def cmd_history(*args) limit = val.to_i end when '-c' - if Readline::HISTORY.respond_to?(:clear) - Readline::HISTORY.clear - elsif defined?(RbReadline) - RbReadline.clear_history + if history.respond_to?(:clear) + history.clear + elsif history.respond_to?(:pop) && history.respond_to?(:length) + history.length.times { |_i| history.pop } else print_error('Could not clear history, skipping file') return false @@ -808,7 +809,7 @@ def cmd_history(*args) (start..length-1).each do |pos| cmd_num = (pos + 1).to_s - print_line "#{cmd_num.ljust(pad_len)} #{Readline::HISTORY[pos]}" + print_line "#{cmd_num.ljust(pad_len)} #{history[pos]}" end end diff --git a/lib/msf/ui/console/command_dispatcher/developer.rb b/lib/msf/ui/console/command_dispatcher/developer.rb index 33c2c39a3f1f6..5ad376822a237 100644 --- a/lib/msf/ui/console/command_dispatcher/developer.rb +++ b/lib/msf/ui/console/command_dispatcher/developer.rb @@ -131,6 +131,7 @@ def cmd_irb(*args) framework.history_manager.with_context(name: :irb) do begin + reline_autocomplete = Reline.autocompletion if defined?(Reline) if active_module print_status("You are in #{active_module.fullname}\n") Rex::Ui::Text::IrbShell.new(active_module).run @@ -140,6 +141,8 @@ def cmd_irb(*args) end rescue print_error("Error during IRB: #{$!}\n\n#{$@.join("\n")}") + ensure + Reline.autocompletion = reline_autocomplete if defined?(Reline) end end @@ -515,6 +518,10 @@ def cmd_time_help private def modified_files + # Temporary work-around until Open3 gets fixed on Windows 11: + # https://github.com/ruby/open3/issues/9 + return [] if Rex::Compat.is_cygwin || Rex::Compat.is_windows + # Using an array avoids shelling out, so we avoid escaping/quoting changed_files = %w[git diff --name-only] begin diff --git a/lib/msf/ui/console/driver.rb b/lib/msf/ui/console/driver.rb index bce665b24eb9f..d5d905edee47f 100644 --- a/lib/msf/ui/console/driver.rb +++ b/lib/msf/ui/console/driver.rb @@ -53,8 +53,6 @@ class Driver < Msf::Ui::Driver # @option opts [Boolean] 'AllowCommandPassthru' (true) Whether to allow # unrecognized commands to be executed by the system shell # @option opts [Boolean] 'Readline' (true) Whether to use the readline or not - # @option opts [Boolean] 'RealReadline' (false) Whether to use the system's - # readline library instead of RBReadline # @option opts [String] 'HistFile' (Msf::Config.history_file) Path to a file # where we can store command history # @option opts [Array] 'Resources' ([]) A list of resource files to @@ -64,8 +62,6 @@ class Driver < Msf::Ui::Driver # @option opts [Boolean] 'SkipDatabaseInit' (false) Whether to skip # connecting to the database and running migrations def initialize(prompt = DefaultPrompt, prompt_char = DefaultPromptChar, opts = {}) - choose_readline(opts) - histfile = opts['HistFile'] || Msf::Config.history_file begin @@ -132,14 +128,6 @@ def initialize(prompt = DefaultPrompt, prompt_char = DefaultPromptChar, opts = { # stack enstack_dispatcher(CommandDispatcher::Core) - # Report readline error if there was one.. - if !@rl_err.nil? - print_error("***") - print_error("* Unable to load readline: #{@rl_err}") - print_error("* Falling back to RbReadLine") - print_error("***") - end - # Load the other "core" command dispatchers CommandDispatchers.each do |dispatcher_class| dispatcher = enstack_dispatcher(dispatcher_class) @@ -234,6 +222,45 @@ def initialize(prompt = DefaultPrompt, prompt_char = DefaultPromptChar, opts = { end end + def self.windows_legacy_console_deprecated? + # We can't need a Windows-specific workaround if we're not on Windows. + return false unless Rex::Compat::is_windows || Rex::Compat::is_cygwin + + kernel32_path = Pathname.new(File.join('C:', 'Windows', 'System32', 'Kernel32.dll')).expand_path + # This should never really happen. If it does, we have a bigger problem than legacy console support. + return false unless kernel32_path.exist? + + @@windows_calls_mutex ||= Mutex.new + @@windows_calls_mutex.synchronize do + # Windows 11 has deprecated legacy console support. + # Check if some relevant functions return an error, or an invalid value. + @@kernel32_handle ||= Fiddle.dlopen(kernel32_path) + @@get_std_handle ||= Fiddle::Function.new(@@kernel32_handle['GetStdHandle'], [Fiddle::TYPE_LONG], Fiddle::TYPE_LONG) + std_input_handle = -10 + # It is not required to call CloseHandle on this handle: https://learn.microsoft.com/en-us/windows/console/getstdhandle + @@console_input_handle = @@get_std_handle.Call(std_input_handle) + @@get_console_mode ||= Fiddle::Function.new(@@kernel32_handle['GetConsoleMode'], [Fiddle::TYPE_LONG, Fiddle::TYPE_INTPTR_T], Fiddle::TYPE_LONG) + + # The Fiddle::Pointer object will be freed at the end of this block, so we don't leak memory. + Fiddle::Pointer.malloc(4) do |console_mode_ptr| + console_mode_result = @@get_console_mode.Call(@@console_input_handle, console_mode_ptr) + + # The function returns 0 on error, non-zero on success: https://learn.microsoft.com/en-us/windows/console/getconsolemode + # We know that if this fails, the legacy console support is not available, e.g. Windows 11, so use Reline. + return true if console_mode_result == 0 || console_mode_ptr.to_s(2) == "\x00\x00" + end + + false + end + end + + def self.using_reline? + return true if Msf::FeatureManager.instance.enabled?(Msf::FeatureManager::USE_RELINE) + return true if self.windows_legacy_console_deprecated? + + false + end + # # Loads configuration that needs to be analyzed before the framework # instance is created. @@ -323,11 +350,12 @@ def save_config # Saves the recent history to the specified file # def save_recent_history(path) - num = Readline::HISTORY.length - hist_last_saved - 1 + history = self.using_reline? ? ::Reline::HISTORY : ::Readline::HISTORY + num = history.length - hist_last_saved - 1 tmprc = "" num.times { |x| - tmprc << Readline::HISTORY[hist_last_saved + x] + "\n" + tmprc << history[hist_last_saved + x] + "\n" } if tmprc.length > 0 @@ -339,7 +367,7 @@ def save_recent_history(path) # Always update this, even if we didn't save anything. We do this # so that we don't end up saving the "makerc" command itself. - self.hist_last_saved = Readline::HISTORY.length + self.hist_last_saved = history.length end # @@ -702,44 +730,6 @@ def handle_session_tlv_logging(val) false end - - # Require the appropriate readline library based on the user's preference. - # - # @return [void] - def choose_readline(opts) - # Choose a readline library before calling the parent - @rl_err = nil - if opts['RealReadline'] - # Remove the gem version from load path to be sure we're getting the - # stdlib readline. - gem_dir = Gem::Specification.find_all_by_name('rb-readline').first.gem_dir - rb_readline_path = File.join(gem_dir, "lib") - index = $LOAD_PATH.index(rb_readline_path) - # Bundler guarantees that the gem will be there, so it should be safe to - # assume we found it in the load path, but check to be on the safe side. - if index - $LOAD_PATH.delete_at(index) - end - end - - begin - require 'readline' - rescue ::LoadError => e - if @rl_err.nil? && index - # Then this is the first time the require failed and we have an index - # for the gem version as a fallback. - @rl_err = e - # Put the gem back and see if that works - $LOAD_PATH.insert(index, rb_readline_path) - index = rb_readline_path = nil - retry - else - # Either we didn't have the gem to fall back on, or we failed twice. - # Nothing more we can do here. - raise e - end - end - end end end diff --git a/lib/msf/ui/console/module_option_tab_completion.rb b/lib/msf/ui/console/module_option_tab_completion.rb index f8ecd07216e3d..99b2e45821f10 100644 --- a/lib/msf/ui/console/module_option_tab_completion.rb +++ b/lib/msf/ui/console/module_option_tab_completion.rb @@ -49,22 +49,23 @@ def tab_complete_module_datastore_names(mod, str, words) # Tab completion options values # def tab_complete_option(mod, str, words) + lib = Msf::Ui::Console::Driver.using_reline? ? ::Reline : ::Readline if str.end_with?('=') option_name = str.chop option_value = '' - ::Readline.completion_append_character = ' ' + lib.completion_append_character = ' ' return tab_complete_option_values(mod, option_value, words, opt: option_name).map { |value| "#{str}#{value}" } elsif str.include?('=') str_split = str.split('=') option_name = str_split[0].strip option_value = str_split[1].strip - ::Readline.completion_append_character = ' ' + lib.completion_append_character = ' ' return tab_complete_option_values(mod, option_value, words, opt: option_name).map { |value| "#{option_name}=#{value}" } end - ::Readline.completion_append_character = '' + lib.completion_append_character = '' tab_complete_option_names(mod, str, words).map { |name| "#{name}=" } end diff --git a/lib/msf/ui/debug.rb b/lib/msf/ui/debug.rb index 6bf8f8e3b1385..bb88b7f8bb416 100644 --- a/lib/msf/ui/debug.rb +++ b/lib/msf/ui/debug.rb @@ -220,13 +220,14 @@ def self.framework_config(framework) end def self.history(driver) - end_pos = Readline::HISTORY.length - 1 + lib = Msf::Ui::Console::Driver.using_reline? ? ::Reline : ::Readline + end_pos = lib::HISTORY.length - 1 start_pos = end_pos - COMMAND_HISTORY_TOTAL > driver.hist_last_saved ? end_pos - (COMMAND_HISTORY_TOTAL - 1) : driver.hist_last_saved commands = '' while start_pos <= end_pos # Formats command position in history to 6 characters in length - commands += "#{'%-6.6s' % start_pos.to_s} #{Readline::HISTORY[start_pos]}\n" + commands += "#{'%-6.6s' % start_pos.to_s} #{lib::HISTORY[start_pos]}\n" start_pos += 1 end diff --git a/lib/rex/post/meterpreter/ui/console/command_dispatcher/stdapi/fs.rb b/lib/rex/post/meterpreter/ui/console/command_dispatcher/stdapi/fs.rb index 676be4ace17e8..868d5b6c664f6 100644 --- a/lib/rex/post/meterpreter/ui/console/command_dispatcher/stdapi/fs.rb +++ b/lib/rex/post/meterpreter/ui/console/command_dispatcher/stdapi/fs.rb @@ -901,8 +901,9 @@ def tab_complete_cdirectory(str, words) end def tab_complete_path(str, words, dir_only) + lib = Msf::Ui::Console::Driver.using_reline? ? ::Reline : ::Readline if client.platform == 'windows' - ::Readline.completion_case_fold = true + lib.completion_case_fold = true end if client.commands.include?(COMMAND_ID_STDAPI_FS_LS) expanded = str @@ -915,7 +916,7 @@ def tab_complete_path(str, words, dir_only) # This is annoying if we're recursively tab-traversing our way through subdirectories - # we may want to continue traversing, but MSF will add a space, requiring us to back up to continue # tab-completing our way through successive subdirectories. - ::Readline.completion_append_character = nil + lib.completion_append_character = nil end results else diff --git a/lib/rex/ui/text/dispatcher_shell.rb b/lib/rex/ui/text/dispatcher_shell.rb index 2131cc363292d..79cdd95b3c035 100644 --- a/lib/rex/ui/text/dispatcher_shell.rb +++ b/lib/rex/ui/text/dispatcher_shell.rb @@ -311,7 +311,8 @@ def tab_complete_directory(str, words) # This is annoying if we're recursively tab-traversing our way through subdirectories - # we may want to continue traversing, but MSF will add a space, requiring us to back up to continue # tab-completing our way through successive subdirectories. - ::Readline.completion_append_character = nil + lib = Msf::Ui::Console::Driver.using_reline? ? ::Reline : ::Readline + lib.completion_append_character = nil end if dirs.length == 0 && File.directory?(str) @@ -408,9 +409,14 @@ def initialize(prompt, prompt_char = '>', histfile = nil, framework = nil, name # a design problem in the Readline module and depends on the # Readline.basic_word_break_characters variable being set to \x00 # - def tab_complete(str) - ::Readline.completion_append_character = ' ' - ::Readline.completion_case_fold = false + def tab_complete(str, opts: {}) + lib = Msf::Ui::Console::Driver.using_reline? ? ::Reline : ::Readline + lib.completion_append_character = ' ' + lib.completion_case_fold = false + + if opts[:preposing] && Msf::Ui::Console::Driver.using_reline? + str = "#{opts[:preposing]}#{str}" + end # Check trailing whitespace so we can tell 'x' from 'x ' str_match = str.match(/[^\\]([\\]{2})*\s+$/) @@ -424,11 +430,7 @@ def tab_complete(str) # Pop the last word and pass it to the real method result = tab_complete_stub(str, split_str) - if result - result.uniq - else - result - end + result&.uniq end # Performs tab completion of a command, if supported diff --git a/lib/rex/ui/text/input/readline.rb b/lib/rex/ui/text/input/readline.rb index 85ca9d1218056..eeb3dcfa30d05 100644 --- a/lib/rex/ui/text/input/readline.rb +++ b/lib/rex/ui/text/input/readline.rb @@ -4,8 +4,6 @@ module Rex module Ui module Text -begin - ### # # This class implements standard input using readline against @@ -18,16 +16,15 @@ class Input::Readline < Rex::Ui::Text::Input # Initializes the readline-aware Input instance for text. # def initialize(tab_complete_proc = nil) - if(not Object.const_defined?('Readline')) - require 'readline' - end - - self.extend(::Readline) + @input_library = ::Reline if Msf::Ui::Console::Driver.using_reline? + require 'readline' unless @input_library + @input_library ||= ::Readline + self.extend(@input_library) if tab_complete_proc - ::Readline.basic_word_break_characters = "" + @input_library.basic_word_break_characters = "" @rl_saved_proc = with_error_handling(tab_complete_proc) - ::Readline.completion_proc = @rl_saved_proc + @input_library.completion_proc = @rl_saved_proc end end @@ -35,8 +32,8 @@ def initialize(tab_complete_proc = nil) # Reattach the original completion proc # def reset_tab_completion(tab_complete_proc = nil) - ::Readline.basic_word_break_characters = "\x00" - ::Readline.completion_proc = tab_complete_proc ? with_error_handling(tab_complete_proc) : @rl_saved_proc + @input_library.basic_word_break_characters = "\x00" + @input_library.completion_proc = tab_complete_proc ? with_error_handling(tab_complete_proc) : @rl_saved_proc end @@ -44,11 +41,9 @@ def reset_tab_completion(tab_complete_proc = nil) # Retrieve the line buffer # def line_buffer - if defined? RbReadline - RbReadline.rl_line_buffer - else - ::Readline.line_buffer - end + return @input_library.line_buffer if @input_library.respond_to?(:line_buffer) + + nil end attr_accessor :prompt @@ -97,9 +92,10 @@ def pgets output.prompting line = readline_with_output(prompt, true) - ::Readline::HISTORY.pop if (line and line.empty?) + @input_library::HISTORY.pop if (line and line.empty?) ensure Thread.current.priority = orig || 0 + output.prompting(false) end line @@ -132,13 +128,6 @@ def intrinsic_shell? private def readline_with_output(prompt, add_history=false) - # rb-readlines's Readline.readline hardcodes the input and output to - # $stdin and $stdout, which means setting `Readline.input` or - # `Readline.ouput` has no effect when running `Readline.readline` with - # rb-readline, so need to reimplement - # []`Readline.readline`](https://github.com/luislavena/rb-readline/blob/ce4908dae45dbcae90a6e42e3710b8c3a1f2cd64/lib/readline.rb#L36-L58) - # for rb-readline to support setting input and output. Output needs to - # be set so that colorization works for the prompt on Windows. self.prompt = prompt # TODO: there are unhandled quirks in async output buffering that @@ -153,38 +142,17 @@ def readline_with_output(prompt, add_history=false) =end reset_sequence = "" - if defined? RbReadline - RbReadline.rl_instream = fd - RbReadline.rl_outstream = output - - begin - line = RbReadline.readline(reset_sequence + prompt) - rescue ::Exception => exception - RbReadline.rl_cleanup_after_signal() - RbReadline.rl_deprep_terminal() - - raise exception - end - - if add_history && line && !line.start_with?(' ') - # Don't add duplicate lines to history - if ::Readline::HISTORY.empty? || line.strip != ::Readline::HISTORY[-1] - RbReadline.add_history(line.strip) - end - end - - line.try(:dup) - else - # The line that's read is immediately added to history - line = ::Readline.readline(reset_sequence + prompt, true) - - # Don't add duplicate lines to history - if ::Readline::HISTORY.length > 1 && line == ::Readline::HISTORY[-2] - ::Readline::HISTORY.pop - end - - line + @input_library.input = fd + @input_library.output = output + + line = @input_library.readline(reset_sequence + prompt, add_history) + + # Don't add duplicate lines to history + if @input_library::HISTORY.length > 1 && line == @input_library::HISTORY[-2] + @input_library::HISTORY.pop end + + line.dup end private @@ -199,9 +167,7 @@ def with_error_handling(proc) end end -rescue LoadError end end end -end diff --git a/lib/rex/ui/text/irb_shell.rb b/lib/rex/ui/text/irb_shell.rb index 956523cb8c799..95aabb69401f3 100644 --- a/lib/rex/ui/text/irb_shell.rb +++ b/lib/rex/ui/text/irb_shell.rb @@ -39,21 +39,18 @@ def run # commands will work. IRB.conf[:MAIN_CONTEXT] = irb.context - # Trap interrupt - old_sigint = trap("SIGINT") do - begin + begin + old_sigint = trap("SIGINT") do irb.signal_handle - rescue RubyLex::TerminateLineInput - irb.eval_input end - end - # Keep processing input until the cows come home... - catch(:IRB_EXIT) do - irb.eval_input + # Keep processing input until the cows come home... + catch(:IRB_EXIT) do + irb.eval_input + end + ensure + trap("SIGINT", old_sigint) if old_sigint end - - trap("SIGINT", old_sigint) end end diff --git a/lib/rex/ui/text/output/stdio.rb b/lib/rex/ui/text/output/stdio.rb index 085c912a60335..7d53fa0af8088 100644 --- a/lib/rex/ui/text/output/stdio.rb +++ b/lib/rex/ui/text/output/stdio.rb @@ -94,6 +94,7 @@ def print_raw(msg = '') msg end alias_method :write, :print_raw + alias_method :<<, :write def supports_color? case config[:color]