diff --git a/.github/workflows/acceptance.yml b/.github/workflows/acceptance.yml index 49908e02dde9..83640245d526 100644 --- a/.github/workflows/acceptance.yml +++ b/.github/workflows/acceptance.yml @@ -36,6 +36,7 @@ on: - 'modules/payloads/**' - 'lib/msf/core/payload/**' - 'lib/msf/core/**' + - 'tools/dev/**' - 'spec/acceptance/**' - 'spec/acceptance_spec_helper.rb' # Example of running as a cron, to weed out flaky tests @@ -170,6 +171,28 @@ jobs: if: always() steps: + - name: Checkout code + uses: actions/checkout@v3 + if: always() + + - name: Install system dependencies (Linux) + if: always() + run: sudo apt-get -y --no-install-recommends install libpcap-dev graphviz + + - name: Setup Ruby + if: always() + env: + BUNDLE_WITHOUT: "coverage development" + BUNDLE_FORCE_RUBY_PLATFORM: true + uses: ruby/setup-ruby@v1 + with: + ruby-version: 3.0.2 + bundler-cache: true + cache-version: 4 + # Github actions with Ruby requires Bundler 2.2.18+ + # https://github.com/ruby/setup-ruby/tree/d2b39ad0b52eca07d23f3aa14fdf2a3fcc1f411c#windows + bundler: 2.2.33 + - uses: actions/download-artifact@v3 id: download if: always() @@ -185,8 +208,12 @@ jobs: curl -o allure-$VERSION.tgz -Ls https://github.com/allure-framework/allure2/releases/download/$VERSION/allure-$VERSION.tgz tar -zxvf allure-$VERSION.tgz -C . + ls -la ${{steps.download.outputs.download-path}} ./allure-$VERSION/bin/allure generate ${{steps.download.outputs.download-path}}/* -o ./allure-report + find ${{steps.download.outputs.download-path}} + bundle exec ruby tools/dev/report_generation/support_matrix/generate.rb --allure-data ${{steps.download.outputs.download-path}} > ./allure-report/support_matrix.html + - name: archive results if: always() uses: actions/upload-artifact@v3 diff --git a/scripts/resource/meterpreter_compatibility.rc b/scripts/resource/meterpreter_compatibility.rc index 40c2cefd7027..a87eb1ed3a45 100644 --- a/scripts/resource/meterpreter_compatibility.rc +++ b/scripts/resource/meterpreter_compatibility.rc @@ -1,4 +1,4 @@ -# Outputs the currently supported Meterpreter commands as JSON for the currently opened Meterpreter sessions +# Outputs to STDOUT the currently supported Meterpreter commands as JSON for the currently opened Meterpreter sessions # Usage: # msf> resource scripts/resource/meterpreter_compatibility.rc diff --git a/spec/acceptance/README.md b/spec/acceptance/README.md index ac4cfc7e4039..c57c6b08598a 100644 --- a/spec/acceptance/README.md +++ b/spec/acceptance/README.md @@ -4,6 +4,9 @@ A slower test suite that ensures high level functionality works as expected, such as verifying msfconsole opens successfully, and can generate Meterpreter payloads, create handlers, etc. +The test suite runs on the current host, so the Meterpreter runtimes should be available. +There is no remote host support currently. + ### Examples Useful environment variables: @@ -17,7 +20,7 @@ Running Meterpreter test suite: SPEC_OPTS='--tag acceptance' bundle exec rspec './spec/acceptance/meterpreter_spec.rb' ``` -Skip loading of Rails/Metasplotit with: +Skip loading of Rails/Metasploit with: ``` SPEC_OPTS='--tag acceptance' SPEC_HELPER_LOAD_METASPLOIT=false bundle exec rspec ./spec/acceptance @@ -30,6 +33,8 @@ SPEC_OPTS='--tag acceptance' METERPRETER=php METERPRETER_MODULE_TEST=test/unix b $env:SPEC_OPTS='--tag acceptance'; $env:SPEC_HELPER_LOAD_METASPLOIT=$false; $env:METERPRETER = 'php'; bundle exec rspec './spec/acceptance/meterpreter_spec.rb' ``` +#### Allure reports + Generate allure reports locally: ``` @@ -57,6 +62,20 @@ cd allure-report ruby -run -e httpd . -p 8000 ``` +#### Support Matrix generation + +You can download the data from an existing Github job run: + +``` +ids=(6099944525); for id in $ids; do echo $id; gh run download $id --repo rapid7/metasploit-framework --dir gh-actions-$id ; done +``` + +Then generate the report using the allure data: + +``` +bundle exec ruby tools/dev/report_generation/support_matrix/generate.rb --allure-data /path/to/gh-actions-$id > ./support_matrix.html +``` + ### Debugging If a test has failed you can enter into an interactive breakpoint with: diff --git a/spec/acceptance/meterpreter_spec.rb b/spec/acceptance/meterpreter_spec.rb index e060b0473975..6ad4c5e00486 100644 --- a/spec/acceptance/meterpreter_spec.rb +++ b/spec/acceptance/meterpreter_spec.rb @@ -53,7 +53,7 @@ meterpreter_runtime_name = "#{meterpreter_name}#{ENV.fetch('METERPRETER_RUNTIME_VERSION', '')}" describe meterpreter_runtime_name, focus: meterpreter_config[:focus] do - meterpreter_config[:payloads].each do |payload_config| + meterpreter_config[:payloads].each.with_index do |payload_config, payload_config_index| describe( Acceptance::Meterpreter.human_name_for_payload(payload_config).to_s, if: ( @@ -184,6 +184,57 @@ def get_file_attachment_contents(path) end context "#{Acceptance::Meterpreter.current_platform}" do + describe "compatibility" do + it( + "exposes available metasploit commands", + if: ( + # Assume that regardless of payload, staged/unstaged/etc, the Meterpreter will have the same commands available + # So only run this test when config_index == 0 + payload_config_index == 0 && Acceptance::Meterpreter.supported_platform?(payload_config) + # Run if ENV['METERPRETER'] = 'java php' etc + Acceptance::Meterpreter.run_meterpreter?(meterpreter_config) && + # Only run payloads / tests, if the host machine can run them + Acceptance::Meterpreter.supported_platform?(payload_config) + ) + ) do + # Ensure we have a valid session id; We intentionally omit this from a `before(:each)` to ensure the allure attachments are generated if the session dies + payload_process, _session_id = payload_process_and_session_id + expect(payload_process).to(be_alive, proc do + current_payload_status = "Expected Payload process to be running. Instead got: payload process exited with #{payload_process.wait_thread.value} - when running the command #{payload_process.cmd.inspect}" + + Allure.add_attachment( + name: 'Failed payload blob', + source: Base64.strict_encode64(File.binread(payload_process.payload_path)), + type: Allure::ContentType::TXT + ) + + current_payload_status + end) + + console.sendline("resource scripts/resource/meterpreter_compatibility.rc") + result = console.recvuntil(Acceptance::Console.prompt) + + available_commands = result.lines(chomp: true).find do |line| + line.start_with?("{") && line.end_with?("}") && JSON.parse(line) + rescue JSON::ParserError => _e + next + end + expect(available_commands).to_not be_nil + + available_commands_json = JSON.parse(available_commands, symbolize_names: true) + expect(available_commands_json[:sessions].length).to be 1 + expect(available_commands_json[:sessions].first[:commands]).to_not be_empty + ensure + # Generate an allure attachment, a report can be generated afterwards + Allure.add_attachment( + name: 'available commands', + source: JSON.pretty_generate(available_commands_json), + type: Allure::ContentType::JSON, + test_case: false + ) + end + end + meterpreter_config[:module_tests].each do |module_test| describe module_test[:name].to_s, focus: module_test[:focus] do it( diff --git a/spec/tools/dev/report_generation/support_matrix/generate_spec.rb b/spec/tools/dev/report_generation/support_matrix/generate_spec.rb new file mode 100644 index 000000000000..567447a37a56 --- /dev/null +++ b/spec/tools/dev/report_generation/support_matrix/generate_spec.rb @@ -0,0 +1,57 @@ +require 'spec_helper' + +require Metasploit::Framework.root.join('tools/dev/report_generation/support_matrix/generate.rb').to_path + +RSpec.describe ReportGeneration::SupportMatrix do + let(:data) { {} } + subject { described_class.new(data) } + + describe '#all_commands' do + it 'equals the list of available Meterpreter commands' do + expect(subject.all_commands).to eq(Rex::Post::Meterpreter::CommandMapper.get_command_names) + end + end + + describe '#table' do + # Results generated by scripts/resource/meterpreter_compatibility.rc + let(:data) do + { + sessions: [ + { + session_type: 'php/linux', + metadata: { foo: 10 }, + commands: [ + { id: 4, name: 'core_channel_open' }, + { id: 2, name: 'core_channel_eof' }, + { id: 5, name: 'core_channel_read' }, + { id: 8, name: 'core_channel_write' } + ] + }, + { + session_type: 'x64/linux', + metadata: { foo: 20 }, + commands: [ + { id: 10, name: 'core_enumextcmd' }, + { id: 13, name: 'core_machine_id' }, + { id: 22, name: 'core_set_uuid' }, + { id: 11, name: 'core_get_session_guid' } + ] + } + ] + } + end + + it 'returns the matrix as a table' do + expected_table = { + columns: [{ heading: '' }, { heading: 'php/linux', metadata: { foo: 10 } }, { heading: 'x64/linux', metadata: { foo: 20 } }], + rows: array_including([ + { heading: ['core', '11%', '11%'], values: array_including([['core_channel_open', true, false], ['core_enumextcmd', false, true]]) }, + { heading: ['stdapi', '0%', '0%'], values: array_including([['stdapi_sys_eventlog_read', false, false]]) }, + { heading: ['bofloader', '0%', '0%'], values: [['bofloader_execute', false, false]] } + ]) + } + + expect(subject.table).to include(expected_table) + end + end +end diff --git a/tools/dev/report_generation/support_matrix/generate.rb b/tools/dev/report_generation/support_matrix/generate.rb new file mode 100644 index 000000000000..1589f56d23f6 --- /dev/null +++ b/tools/dev/report_generation/support_matrix/generate.rb @@ -0,0 +1,207 @@ +# frozen_string_literal: true + +$LOAD_PATH.unshift(File.join(__dir__, '..', '..', '..', '..', 'spec')) +$LOAD_PATH.unshift(File.join(__dir__, '..', '..', '..', '..', 'lib')) + +require 'active_support' +require 'active_support/core_ext' +require 'allure_config' +require 'json' +require 'erb' +require 'optparse' +require 'msfenv' +require 'rex' +require 'rex/post' + +module ReportGeneration + class SupportMatrix + def initialize(data) + @data = data + end + + def generation_date + @generation_date ||= Time.now.strftime('%FT%T') + end + + def all_commands + Rex::Post::Meterpreter::CommandMapper.get_command_names + end + + def table + sorted_sessions = @data.fetch(:sessions, []).sort_by { |session| session[:session_type] } + + # Group into buckets, and prioritize sort order + extension_names = [ + # 'Required' Meterpreter extensions + 'core', + 'stdapi', + + 'sniffer', + 'extapi', + 'kiwi', + 'python', + 'unhook', + 'appapi', + 'winpmem', + 'powershell', + 'lanattacks', + 'priv', + 'incognito', + 'peinjector', + 'espia', + 'android', + + # any missing new/missing extensions will added to the end lexicographically + ] + + # Add any new extension names that aren't currently known about + extension_names += all_commands.each_with_object([]) do |command, unknown_extensions| + command_prefix = command.split('_').first + next if extension_names.include?(command_prefix) + + unknown_extensions << command_prefix + end.sort + + ordered_commands = all_commands.sort_by do |command| + command_prefix = command.split('_').first + sort_index = extension_names.index(command_prefix) + sort_index + end + + # Map session type to supported commands. i.e. { osx: { command_name_1: true } } + sessions_to_supported_commands_hash = sorted_sessions.each_with_object({}) do |session, hash| + session_type = session[:session_type] + # Map command name to its availability + supported_command_map = session[:commands].each_with_object({}) do |command, map| + command_name = command[:name] + map[command_name] = true + end + hash[session_type] = supported_command_map + end + + columns = [{ heading: '' }] + sorted_sessions.map do |session| + { heading: session[:session_type], metadata: session[:metadata] } + end + + rows = extension_names.map do |extension_name| + extension_commands = ordered_commands.select { |command| command.start_with?(extension_name) } + + command_rows = extension_commands.map do |command| + session_supported_cells = sessions_to_supported_commands_hash.map do |(_session, compatibility)| + compatibility.include?(command) + end + + [command] + session_supported_cells + end + extension_coverage = sessions_to_supported_commands_hash.map do |(_session, compatibility)| + implemented_count = extension_commands.select { |command| compatibility.include?(command) }.size + total_count = extension_commands.size + percentage = ((implemented_count.to_f / total_count) * 100).to_i + + "#{percentage}%" + end + + { + heading: [extension_name] + extension_coverage, + values: command_rows + } + end + + { + columns: columns, + rows: rows + } + end + + def get_binding + binding + end + end + + def self.extract_data(options) + if options[:allure_data] + results_directory = options[:allure_data] + + test_result_files = Dir['**/*-result.json', base: results_directory] + meterpreter_compatibility_results = test_result_files.filter_map do |test_result_file| + path = File.join(results_directory, test_result_file) + test_result_json = JSON.parse(File.read(path), symbolize_names: true) + + compatibility_attachment = test_result_json.fetch(:attachments, []) + .find { |attachment| attachment[:name] == 'available commands' } + next unless compatibility_attachment + + compatibility_attachment_path = File.join(File.dirname(path), compatibility_attachment[:source]) + compatibility_json = JSON.parse(File.read(compatibility_attachment_path), symbolize_names: true) + compatibility_json[:sessions].each do |session| + session[:metadata] = test_result_json[:parameters].each_with_object({}) do |param, acc| + acc[param[:name]] = param[:value] + end + end + + compatibility_json + end + + sessions = meterpreter_compatibility_results.flat_map { |results| results[:sessions] } + sorted_sessions = sessions.sort_by do |session| + [session[:session_type], session[:metadata]['host_runner_image'], session[:metadata]['meterpreter_runtime_version'].to_s] + end + + unique_sessions = sorted_sessions.each_with_object({}) do |session, acc| + acc[session[:session_type]] = session + end.values + + aggregated_data = { + sessions: unique_sessions + } + + aggregated_data + else + data_path = options.fetch(:data_path) + JSON.parse(File.read(data_path), symbolize_names: true) + end + end + + def self.generate(options) + data = extract_data(options) + support_matrix = SupportMatrix.new(data) + + if options[:format] == :json + $stdout.write JSON.pretty_generate(support_matrix.data) + else + template = File.read(File.join(File.dirname(__FILE__), 'template.erb')) + renderer = ERB.new(template, trim_mode: '-') + + html = renderer.result(support_matrix.get_binding) + $stdout.write(html) + end + end +end + +if $PROGRAM_NAME == __FILE__ + options = {} + options_parser = OptionParser.new do |opts| + opts.banner = "Usage: #{File.basename(__FILE__)} [options]" + + opts.on '-h', '--help', 'Help banner.' do + return print(opts.help) + end + + opts.on('--allure-data path', 'Use allure as the data source') do |allure_data| + allure_data ||= AllureRspec.configuration.results_directory + options[:allure_data] = allure_data + end + + opts.on('--data-path path', + 'The path to the report generated by scripts/resource/meterpreter_compatibility.rc') do |data_path| + options[:data_path] = data_path + end + + opts.on('--format value', %i[json html], 'Render in a given format') do |format| + options[:format] = format + end + end + options_parser.parse! + + ReportGeneration.generate(options) +end diff --git a/tools/dev/report_generation/support_matrix/template.erb b/tools/dev/report_generation/support_matrix/template.erb new file mode 100644 index 000000000000..aa68efef3c12 --- /dev/null +++ b/tools/dev/report_generation/support_matrix/template.erb @@ -0,0 +1,201 @@ + + + + + Meterpreter Support matrix + + + + + + + + + +
+ + + + + + <%- largest_column_length = table[:rows].flat_map { |row| row[:values] }.map { |values| values[0].size }.max %> + <%- table[:columns].each.with_index do |column, column_index| -%> + <%- if column_index == 0 -%> + + <%- else -%> + + <%- end -%> + <%- end -%> + + + + <%- table[:rows].each.with_index do |row, _row_index| -%> + + + <%- row[:heading].each.with_index do |value, value_index| -%> + <%- if value_index == 0 -%> + + <%- else -%> + + <%- end -%> + <%- end -%> + + <%- row[:values].each do |row| -%> + + <%- row.each.with_index do |value, cell_index| -%> + <%- if cell_index == 0 -%> + + <%- elsif value.is_a?(TrueClass) || value.is_a?(FalseClass) -%> + + <%- else -%> + + <%- end -%> + <%- end -%> + + <%- end -%> + + <%- end -%> +
Meterpreter Feature + <%= column[:heading] -%> + + ') %>"> + + +
<%= value -%> +
+
+
+ <%= value -%> +
<%= value -%><%= value ? ' ' : ' ' -%><%= value -%>
+
+ +