wip: Fix Reline ASCII-8BIT <-> UTF-8 issues

lib/msf/ui/console/driver.rb

@@ -105,7 +105,7 @@ def initialize(prompt = DefaultPrompt, prompt_char = DefaultPromptChar, opts = {
     # Initialize the user interface to use a different input and output
     # handle if one is supplied
     input = opts['LocalInput']
-    input ||= Rex::Ui::Text::Input::Stdio.new
+    input ||= Rex::Ui::Text::Input::Utf8Stdio.new
 
     if !opts['Readline']
       input.disable_readline
@@ -118,7 +118,7 @@ def initialize(prompt = DefaultPrompt, prompt_char = DefaultPromptChar, opts = {
         output = opts['LocalOutput']
       end
     else
-      output = Rex::Ui::Text::Output::Stdio.new
+      output = Rex::Ui::Text::Output::Utf8Stdio.new
     end
 
     init_ui(input, output)

lib/rex/ui/text/dispatcher_shell.rb

@@ -459,13 +459,17 @@ def tab_complete_stub(original_str, split_str)
         res = tab_complete_helper(dispatcher, current_word, tab_words)
       end
 
+      res.map! { |item| item.dup.force_encoding(::Encoding::UTF_8) } if self.input.respond_to?(:utf8?) && self.input.utf8?
+
       if res.nil?
         # A nil response indicates no optional arguments
         return [''] if items.empty?
       else
         if res.second == :override_completions
           return res.first
         else
+          next if res.empty?
+
           # Otherwise we add the completion items to the list
           items.concat(res)
         end

lib/rex/ui/text/input/readline.rb

@@ -92,6 +92,7 @@ def pgets
         ::Reline::HISTORY.pop if (line and line.empty?)
       ensure
         Thread.current.priority = orig || 0
+        output.prompting(false)
       end
 
       line

lib/rex/ui/text/input/utf8_buffer.rb

@@ -0,0 +1,38 @@
+# -*- coding: binary -*-
+
+require 'rex/ui/text/input/utf8_common'
+
+module Rex
+module Ui
+module Text
+
+require 'rex/io/stream_abstraction'
+
+###
+#
+# This class implements input against a socket and forces UTF-8 encoding.
+#
+###
+class Input::Utf8Buffer < Rex::Ui::Text::Input::Buffer
+
+  # Array of methods that will be used to override methods from the base class to wrap them with forced UTF-8 encoding.
+  METHODS_TO_WRAP_WITH_UTF8_ENCODING = %i[sysread gets put].freeze
+
+  METHODS_TO_WRAP_WITH_UTF8_ENCODING.each do |method|
+    class_eval %{
+      def #{method} (*args, &block)
+        Rex::Ui::Text::Input::Utf8Common.with_utf8_encoding do
+          super
+        end
+      end
+    }
+  end
+
+  def utf8?
+    true
+  end
+end
+
+end
+end
+end

lib/rex/ui/text/input/utf8_common.rb

@@ -0,0 +1,26 @@
+# -*- coding: binary -*-
+
+module Rex
+module Ui
+module Text
+class Input::Utf8Common
+  def self.with_utf8_encoding(&block)
+    external_encoding_on_entry = ::Encoding.default_external
+    ::Encoding.default_external = ::Encoding::UTF_8
+
+    internal_encoding_on_entry = ::Encoding.default_internal
+    ::Encoding.default_internal = ::Encoding::UTF_8
+
+    begin
+      return block.call
+    rescue StandardError => e
+      elog("Failed to call block with UTF8 encoding. Backtrace: #{e.backtrace}", error: e)
+    ensure
+      ::Encoding.default_external = external_encoding_on_entry
+      ::Encoding.default_internal = internal_encoding_on_entry
+    end
+  end
+end
+end
+end
+end

lib/rex/ui/text/input/utf8_readline.rb

@@ -0,0 +1,37 @@
+# -*- coding: binary -*-
+
+require 'rex/ui/text/input/utf8_common'
+
+module Rex
+module Ui
+module Text
+
+###
+#
+# This class implements standard input using Reline against
+# standard input, and forces UTF-8 encoding. It supports tab completion.
+#
+###
+class Input::Utf8Readline < Input::Readline
+
+  # Array of methods that will be used to override methods from the base class to wrap them with forced UTF-8 encoding.
+  METHODS_TO_WRAP_WITH_UTF8_ENCODING = %i[pgets gets sysread readline_with_output update_prompt].freeze
+
+  METHODS_TO_WRAP_WITH_UTF8_ENCODING.each do |method|
+    class_eval %{
+      def #{method} (*args, &block)
+        Rex::Ui::Text::Input::Utf8Common.with_utf8_encoding do
+          super
+        end
+      end
+    }
+  end
+
+  def utf8?
+    true
+  end
+end
+end
+
+end
+end

lib/rex/ui/text/input/utf8_socket.rb

@@ -0,0 +1,36 @@
+# -*- coding: binary -*-
+
+require 'rex/ui/text/input/utf8_common'
+
+module Rex
+module Ui
+module Text
+
+###
+#
+# This class implements input against a socket and forces UTF-8 encoding.
+#
+###
+class Input::Utf8Socket < Rex::Ui::Text::Input::Socket
+
+  # Array of methods that will be used to override methods from the base class to wrap them with forced UTF-8 encoding.
+  METHODS_TO_WRAP_WITH_UTF8_ENCODING = %i[sysread gets].freeze
+
+  METHODS_TO_WRAP_WITH_UTF8_ENCODING.each do |method|
+    class_eval %{
+      def #{method} (*args, &block)
+        Rex::Ui::Text::Input::Utf8Common.with_utf8_encoding do
+          super
+        end
+      end
+    }
+  end
+
+  def utf8?
+    true
+  end
+end
+
+end
+end
+end

lib/rex/ui/text/input/utf8_stdio.rb

@@ -0,0 +1,36 @@
+# -*- coding: binary -*-
+
+require 'rex/ui/text/input/utf8_common'
+
+module Rex
+module Ui
+module Text
+
+###
+#
+# This class implements input against standard in and forces UTF-8 encoding.
+#
+###
+class Input::Utf8Stdio < Rex::Ui::Text::Input::Stdio
+
+  # Array of methods that will be used to override methods from the base class to wrap them with forced UTF-8 encoding.
+  METHODS_TO_WRAP_WITH_UTF8_ENCODING = %i[sysread gets].freeze
+
+  METHODS_TO_WRAP_WITH_UTF8_ENCODING.each do |method|
+    class_eval %{
+      def #{method} (*args, &block)
+        Rex::Ui::Text::Input::Utf8Common.with_utf8_encoding do
+          super
+        end
+      end
+    }
+  end
+
+  def utf8?
+    true
+  end
+end
+
+end
+end
+end

lib/rex/ui/text/output/utf8_common.rb

@@ -0,0 +1,28 @@
+# -*- coding: binary -*-
+
+module Rex
+module Ui
+module Text
+class Output::Utf8Common
+  def self.with_utf8_encoding(&block)
+    external_encoding_on_entry = ::Encoding.default_external
+    ::Encoding.default_external = ::Encoding::UTF_8
+
+    internal_encoding_on_entry = ::Encoding.default_internal
+    ::Encoding.default_internal = ::Encoding::UTF_8
+
+    begin
+      return block.call
+    rescue ::StandardError => e
+      puts 'Output'
+      puts caller
+      elog('Failed to call block with UTF8 encoding', error: e)
+    ensure
+      ::Encoding.default_external = external_encoding_on_entry
+      ::Encoding.default_internal = internal_encoding_on_entry
+    end
+  end
+end
+end
+end
+end

lib/rex/ui/text/output/utf8_stdio.rb

@@ -0,0 +1,37 @@
+# -*- coding: binary -*-
+
+require 'rex/ui/text/output/utf8_common'
+
+module Rex
+module Ui
+module Text
+
+###
+#
+# This class implements output against standard out and forces UTF-8 encoding.
+#
+###
+class Output::Utf8Stdio < Rex::Ui::Text::Output::Stdio
+
+  # Array of methods that will be used to override methods from the base class to wrap them with forced UTF-8 encoding.
+  METHODS_TO_WRAP_WITH_UTF8_ENCODING = %i[print_line print_raw].freeze
+
+  METHODS_TO_WRAP_WITH_UTF8_ENCODING.each do |method|
+    class_eval %{
+      def #{method} (*args, &block)
+        Rex::Ui::Text::Input::Utf8Common.with_utf8_encoding do
+          super
+        end
+      end
+    }
+  end
+
+  def utf8?
+    true
+  end
+end
+
+end
+end
+end
+

lib/rex/ui/text/shell.rb

@@ -1,5 +1,6 @@
 # -*- coding: binary -*-
 require 'rex/text/color'
+require 'rex/ui/text/input/utf8_readline'
 
 module Rex
 module Ui
@@ -63,13 +64,21 @@ def initialize(prompt, prompt_char = '>', histfile = nil, framework = nil, name
     self.framework = framework
   end
 
+  def default_tab_complete_lambda
+    # Unless cont_flag because there's no tab complete for continuation lines
+    proc do |str, preposing = nil, postposing = nil|
+      next nil if cont_flag
+
+      values = tab_complete(str, opts: { preposing: preposing, postposing: postposing }).map { |result| result[preposing.to_s.length..] }
+      values.map! { |val| val.dup.encode(::Encoding::UTF_8) } if self.input.respond_to?(:utf8?) && self.input.utf8?
+
+      next values
+    end
+  end
+
   def init_tab_complete
     if (self.input and self.input.supports_readline)
-      # Unless cont_flag because there's no tab complete for continuation lines
-      tab_complete_lambda = proc do |str, preposing = nil, postposing = nil|
-        next tab_complete(str, opts: { preposing: preposing, postposing: postposing }).map { |result| result[preposing.to_s.length..] } unless cont_flag
-      end
-      self.input = Input::Readline.new(tab_complete_lambda)
+      self.input = Input::Utf8Readline.new(default_tab_complete_lambda)
       self.input.output = self.output
     end
   end