Land #18349, Add Meterpreter compatibility matrix generation

.github/workflows/acceptance.yml

@@ -36,6 +36,7 @@ on:
       - 'modules/payloads/**'
       - 'lib/msf/core/payload/**'
       - 'lib/msf/core/**'
+      - 'tools/dev/**'
       - 'spec/acceptance/**'
       - 'spec/acceptance_spec_helper.rb'
 #   Example of running as a cron, to weed out flaky tests
@@ -170,6 +171,28 @@ jobs:
     if: always()
 
     steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+        if: always()
+
+      - name: Install system dependencies (Linux)
+        if: always()
+        run: sudo apt-get -y --no-install-recommends install libpcap-dev graphviz
+
+      - name: Setup Ruby
+        if: always()
+        env:
+          BUNDLE_WITHOUT: "coverage development"
+          BUNDLE_FORCE_RUBY_PLATFORM: true
+        uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: 3.0.2
+          bundler-cache: true
+          cache-version: 4
+          # Github actions with Ruby requires Bundler 2.2.18+
+          # https://github.com/ruby/setup-ruby/tree/d2b39ad0b52eca07d23f3aa14fdf2a3fcc1f411c#windows
+          bundler: 2.2.33
+
       - uses: actions/download-artifact@v3
         id: download
         if: always()
@@ -185,8 +208,12 @@ jobs:
           curl -o allure-$VERSION.tgz -Ls https://github.com/allure-framework/allure2/releases/download/$VERSION/allure-$VERSION.tgz
           tar -zxvf allure-$VERSION.tgz -C .
 
+          ls -la ${{steps.download.outputs.download-path}}
           ./allure-$VERSION/bin/allure generate ${{steps.download.outputs.download-path}}/* -o ./allure-report
 
+          find ${{steps.download.outputs.download-path}}
+          bundle exec ruby tools/dev/report_generation/support_matrix/generate.rb --allure-data ${{steps.download.outputs.download-path}} > ./allure-report/support_matrix.html
+
       - name: archive results
         if: always()
         uses: actions/upload-artifact@v3

scripts/resource/meterpreter_compatibility.rc

@@ -1,4 +1,4 @@
-# Outputs the currently supported Meterpreter commands as JSON for the currently opened Meterpreter sessions
+# Outputs to STDOUT the currently supported Meterpreter commands as JSON for the currently opened Meterpreter sessions
 # Usage:
 # msf> resource scripts/resource/meterpreter_compatibility.rc
 

spec/acceptance/README.md

@@ -4,6 +4,9 @@ A slower test suite that ensures high level functionality works as expected,
 such as verifying msfconsole opens successfully, and can generate Meterpreter payloads,
 create handlers, etc.
 
+The test suite runs on the current host, so the Meterpreter runtimes should be available.
+There is no remote host support currently.
+
 ### Examples
 
 Useful environment variables:
@@ -17,7 +20,7 @@ Running Meterpreter test suite:
 SPEC_OPTS='--tag acceptance' bundle exec rspec './spec/acceptance/meterpreter_spec.rb'
 ```
 
-Skip loading of Rails/Metasplotit with:
+Skip loading of Rails/Metasploit with:
 
 ```
 SPEC_OPTS='--tag acceptance' SPEC_HELPER_LOAD_METASPLOIT=false bundle exec rspec ./spec/acceptance
@@ -30,6 +33,8 @@ SPEC_OPTS='--tag acceptance' METERPRETER=php METERPRETER_MODULE_TEST=test/unix b
 $env:SPEC_OPTS='--tag acceptance'; $env:SPEC_HELPER_LOAD_METASPLOIT=$false; $env:METERPRETER = 'php'; bundle exec rspec './spec/acceptance/meterpreter_spec.rb'
 ```
 
+#### Allure reports
+
 Generate allure reports locally:
 
 ```
@@ -57,6 +62,20 @@ cd allure-report
 ruby -run -e httpd . -p 8000
 ```
 
+#### Support Matrix generation
+
+You can download the data from an existing Github job run:
+
+```
+ids=(6099944525); for id in $ids; do echo $id; gh run download $id --repo rapid7/metasploit-framework --dir gh-actions-$id ; done
+```
+
+Then generate the report using the allure data:
+
+```
+bundle exec ruby tools/dev/report_generation/support_matrix/generate.rb --allure-data /path/to/gh-actions-$id > ./support_matrix.html
+```
+
 ### Debugging
 
 If a test has failed you can enter into an interactive breakpoint with:

spec/acceptance/meterpreter_spec.rb

@@ -53,7 +53,7 @@
     meterpreter_runtime_name = "#{meterpreter_name}#{ENV.fetch('METERPRETER_RUNTIME_VERSION', '')}"
 
     describe meterpreter_runtime_name, focus: meterpreter_config[:focus] do
-      meterpreter_config[:payloads].each do |payload_config|
+      meterpreter_config[:payloads].each.with_index do |payload_config, payload_config_index|
         describe(
           Acceptance::Meterpreter.human_name_for_payload(payload_config).to_s,
           if: (
@@ -184,6 +184,57 @@ def get_file_attachment_contents(path)
           end
 
           context "#{Acceptance::Meterpreter.current_platform}" do
+            describe "compatibility" do
+              it(
+                "exposes available metasploit commands",
+                if: (
+                  # Assume that regardless of payload, staged/unstaged/etc, the Meterpreter will have the same commands available
+                  # So only run this test when config_index == 0
+                  payload_config_index == 0 && Acceptance::Meterpreter.supported_platform?(payload_config)
+                  # Run if ENV['METERPRETER'] = 'java php' etc
+                  Acceptance::Meterpreter.run_meterpreter?(meterpreter_config) &&
+                  # Only run payloads / tests, if the host machine can run them
+                  Acceptance::Meterpreter.supported_platform?(payload_config)
+                )
+              ) do
+                # Ensure we have a valid session id; We intentionally omit this from a `before(:each)` to ensure the allure attachments are generated if the session dies
+                payload_process, _session_id = payload_process_and_session_id
+                expect(payload_process).to(be_alive, proc do
+                  current_payload_status = "Expected Payload process to be running. Instead got: payload process exited with #{payload_process.wait_thread.value} - when running the command #{payload_process.cmd.inspect}"
+
+                  Allure.add_attachment(
+                    name: 'Failed payload blob',
+                    source: Base64.strict_encode64(File.binread(payload_process.payload_path)),
+                    type: Allure::ContentType::TXT
+                  )
+
+                  current_payload_status
+                end)
+
+                console.sendline("resource scripts/resource/meterpreter_compatibility.rc")
+                result = console.recvuntil(Acceptance::Console.prompt)
+
+                available_commands = result.lines(chomp: true).find do |line|
+                  line.start_with?("{") && line.end_with?("}") && JSON.parse(line)
+                rescue JSON::ParserError => _e
+                  next
+                end
+                expect(available_commands).to_not be_nil
+
+                available_commands_json = JSON.parse(available_commands, symbolize_names: true)
+                expect(available_commands_json[:sessions].length).to be 1
+                expect(available_commands_json[:sessions].first[:commands]).to_not be_empty
+              ensure
+                # Generate an allure attachment, a report can be generated afterwards
+                Allure.add_attachment(
+                  name: 'available commands',
+                  source: JSON.pretty_generate(available_commands_json),
+                  type: Allure::ContentType::JSON,
+                  test_case: false
+                )
+              end
+            end
+
             meterpreter_config[:module_tests].each do |module_test|
               describe module_test[:name].to_s, focus: module_test[:focus] do
                 it(

spec/tools/dev/report_generation/support_matrix/generate_spec.rb

@@ -0,0 +1,57 @@
+require 'spec_helper'
+
+require Metasploit::Framework.root.join('tools/dev/report_generation/support_matrix/generate.rb').to_path
+
+RSpec.describe ReportGeneration::SupportMatrix do
+  let(:data) { {} }
+  subject { described_class.new(data) }
+
+  describe '#all_commands' do
+    it 'equals the list of available Meterpreter commands' do
+      expect(subject.all_commands).to eq(Rex::Post::Meterpreter::CommandMapper.get_command_names)
+    end
+  end
+
+  describe '#table' do
+    # Results generated by scripts/resource/meterpreter_compatibility.rc
+    let(:data) do
+      {
+        sessions: [
+          {
+            session_type: 'php/linux',
+            metadata: { foo: 10 },
+            commands: [
+              { id: 4, name: 'core_channel_open' },
+              { id: 2, name: 'core_channel_eof' },
+              { id: 5, name: 'core_channel_read' },
+              { id: 8, name: 'core_channel_write' }
+            ]
+          },
+          {
+            session_type: 'x64/linux',
+            metadata: { foo: 20 },
+            commands: [
+              { id: 10, name: 'core_enumextcmd' },
+              { id: 13, name: 'core_machine_id' },
+              { id: 22, name: 'core_set_uuid' },
+              { id: 11, name: 'core_get_session_guid' }
+            ]
+          }
+        ]
+      }
+    end
+
+    it 'returns the matrix as a table' do
+      expected_table = {
+        columns: [{ heading: '' }, { heading: 'php/linux', metadata: { foo: 10 } }, { heading: 'x64/linux', metadata: { foo: 20 } }],
+        rows: array_including([
+          { heading: ['core', '11%', '11%'], values: array_including([['core_channel_open', true, false], ['core_enumextcmd', false, true]]) },
+          { heading: ['stdapi', '0%', '0%'], values: array_including([['stdapi_sys_eventlog_read', false, false]]) },
+          { heading: ['bofloader', '0%', '0%'], values: [['bofloader_execute', false, false]] }
+        ])
+      }
+
+      expect(subject.table).to include(expected_table)
+    end
+  end
+end