diff --git a/lib/inferno/apps/cli/execute.rb b/lib/inferno/apps/cli/execute.rb index a57049b59..3a03d10e8 100644 --- a/lib/inferno/apps/cli/execute.rb +++ b/lib/inferno/apps/cli/execute.rb @@ -1,9 +1,8 @@ -require 'pastel' require 'active_support' require_relative '../../utils/verify_runnable' require_relative '../../utils/persist_inputs' -require_relative 'execute/console_outputter' -require_relative '../../result_summarizer' + +Dir[File.join(__dir__, 'execute', '*_outputter.rb')].each { |outputter| require outputter } module Inferno module CLI @@ -11,6 +10,13 @@ class Execute include ::Inferno::Utils::VerifyRunnable include ::Inferno::Utils::PersistInputs + OUTPUTTERS = { + 'console' => Inferno::CLI::Execute::ConsoleOutputter, + 'plain' => Inferno::CLI::Execute::PlainOutputter, + 'json' => Inferno::CLI::Execute::JSONOutputter, + 'quiet' => Inferno::CLI::Execute::QuietOutputter + }.freeze + attr_accessor :options def self.suppress_output @@ -88,8 +94,12 @@ def print_help_and_exit end def outputter - # TODO: swap outputter based on options - @outputter ||= Inferno::CLI::Execute::ConsoleOutputter.new + unless OUTPUTTERS.key? options[:outputter] + raise StandardError, + "Unrecognized outputter #{options[:outputter]}" + end + + @outputter ||= OUTPUTTERS[options[:outputter]].new end def all_selected_groups_and_tests @@ -164,7 +174,6 @@ def create_params(test_session, runnable) end def dispatch_job(test_run) - # TODO: move suppression to outputter? better suppression? if options[:verbose] Jobs.perform(Jobs::ExecuteTestRun, test_run.id, force_synchronous: true) else diff --git a/lib/inferno/apps/cli/execute/console_outputter.rb b/lib/inferno/apps/cli/execute/console_outputter.rb index 66658579e..6235f1abb 100644 --- a/lib/inferno/apps/cli/execute/console_outputter.rb +++ b/lib/inferno/apps/cli/execute/console_outputter.rb @@ -1,16 +1,16 @@ require 'pastel' -require_relative '../../web/serializers/test_run' -require_relative '../../web/serializers/result' +require_relative 'serialize' module Inferno module CLI class Execute # @private class ConsoleOutputter - COLOR = Pastel.new CHECKMARK = "\u2713".freeze BAR = ('=' * 80).freeze + include Serialize + def print_start_message(options) puts '' puts BAR @@ -23,7 +23,6 @@ def print_start_message(options) def print_around_run(_options) puts 'Running tests. This may take a while...' - # TODO: spinner/progress bar yield end @@ -46,15 +45,18 @@ def print_results(options, results) def print_end_message(options); end - def print_error(options, exception) - puts COLOR.red "Error: #{exception.full_message}" - verbose_print(options, exception.backtrace&.join('\n')) + def print_error(_options, exception) + puts color.red "Error: #{exception.full_message}" end # private def verbose_print(options, *args) - print(COLOR.dim(*args)) if options[:verbose] + print(color.dim(*args)) if options[:verbose] + end + + def color + @color ||= Pastel.new(enabled: $stdout.tty?) end def verbose_puts(options, *args) @@ -106,21 +108,21 @@ def format_outputs(result) def format_result(result) # rubocop:disable Metrics/CyclomaticComplexity case result.result when 'pass' - COLOR.bold.green(CHECKMARK, ' pass') + color.bold.green(CHECKMARK, ' pass') when 'fail' - COLOR.bold.red 'X fail' + color.bold.red 'X fail' when 'skip' - COLOR.yellow '* skip' + color.yellow '* skip' when 'omit' - COLOR.blue '* omit' + color.blue '* omit' when 'error' - COLOR.magenta 'X error' + color.magenta 'X error' when 'wait' - COLOR.bold '. wait' + color.bold '. wait' when 'cancel' - COLOR.red 'X cancel' + color.red 'X cancel' when 'running' - COLOR.bold '- running' + color.bold '- running' else raise StandardError.new, "Unrecognized result #{result.result}" end @@ -133,19 +135,6 @@ def verbose_print_json_results(options, results) verbose_puts(options, serialize(results)) verbose_puts(options, BAR) end - - def serialize(entity) - case entity.class.to_s - when 'Array' - JSON.pretty_generate(entity.map { |item| JSON.parse serialize(item) }) - when lambda { |x| - defined?(x.constantize) && defined?("Inferno::Web::Serializers::#{x.split('::').last}".constantize) - } - "Inferno::Web::Serializers::#{entity.class.to_s.split('::').last}".constantize.render(entity) - else - raise StandardError, "CLI does not know how to serialize #{entity.class}" - end - end end end end diff --git a/lib/inferno/apps/cli/execute/json_outputter.rb b/lib/inferno/apps/cli/execute/json_outputter.rb new file mode 100644 index 000000000..4691bd660 --- /dev/null +++ b/lib/inferno/apps/cli/execute/json_outputter.rb @@ -0,0 +1,28 @@ +require_relative 'serialize' + +module Inferno + module CLI + class Execute + # @private + class JSONOutputter + include Serialize + + def print_start_message(_options); end + + def print_around_run(_options, &) + yield + end + + def print_results(_options, results) + puts serialize(results) + end + + def print_end_message(_options); end + + def print_error(_options, exception) + puts exception.to_json + end + end + end + end +end diff --git a/lib/inferno/apps/cli/execute/plain_outputter.rb b/lib/inferno/apps/cli/execute/plain_outputter.rb new file mode 100644 index 000000000..eb0b2f94d --- /dev/null +++ b/lib/inferno/apps/cli/execute/plain_outputter.rb @@ -0,0 +1,18 @@ +require_relative 'console_outputter' + +module Inferno + module CLI + class Execute + # @private + class PlainOutputter < ConsoleOutputter + def print_error(_options, exception) + puts "Error: #{exception.full_message(highlight: false)}" + end + + def color + @color ||= Pastel.new(enabled: false) + end + end + end + end +end diff --git a/lib/inferno/apps/cli/execute/quiet_outputter.rb b/lib/inferno/apps/cli/execute/quiet_outputter.rb new file mode 100644 index 000000000..a76f76ae5 --- /dev/null +++ b/lib/inferno/apps/cli/execute/quiet_outputter.rb @@ -0,0 +1,22 @@ +module Inferno + module CLI + class Execute + # @private + class QuietOutputter + def print_start_message(_options); end + + def print_around_run(_options, &) + yield + end + + def print_results(_options, _results); end + + def print_end_message(_options); end + + def print_error(options, exception) + puts "Error: #{exception.full_message}" if options[:verbose] + end + end + end + end +end diff --git a/lib/inferno/apps/cli/execute/serialize.rb b/lib/inferno/apps/cli/execute/serialize.rb new file mode 100644 index 000000000..f843fb071 --- /dev/null +++ b/lib/inferno/apps/cli/execute/serialize.rb @@ -0,0 +1,21 @@ +require 'active_support' +require_relative '../../web/serializers/test_run' +require_relative '../../web/serializers/result' + +module Inferno + module CLI + class Execute + # @private + module Serialize + def serialize(entity) + case entity.class.to_s + when 'Array' + JSON.pretty_generate(entity.map { |item| JSON.parse serialize(item) }) + else + Inferno::Web::Serializers.const_get(entity.class.to_s.demodulize).render(entity) + end + end + end + end + end +end diff --git a/lib/inferno/apps/cli/main.rb b/lib/inferno/apps/cli/main.rb index 1a08db1c4..5b926b28c 100644 --- a/lib/inferno/apps/cli/main.rb +++ b/lib/inferno/apps/cli/main.rb @@ -74,6 +74,10 @@ def version You can view suite ids with: `bundle exec inferno suites` + You can select an output format with the `--outputter` option. Current outputters + are console (default), plain, quiet, and json. JSON-formatted output will copy + Inferno's REST API: https://inferno-framework.github.io/inferno-core/api-docs/#/Result. + Examples: (These examples only work from within the inferno_core directory). @@ -88,6 +92,12 @@ def version patient_id:1234321 \ --tests 1.01 1.02` => Run specific tests from suite + + `bundle exec inferno execute --suite dev_validator \ + --inputs "url:https://hapi.fhir.org/baseR4" \ + patient_id:1234321 \ + --outputter json` + => Outputs test results in JSON END_OF_HELP desc 'execute', 'Run Inferno tests in command line' long_desc EXECUTE_HELP, wrap: false @@ -97,7 +107,7 @@ def version desc: 'Test suite id to run or to select groups and tests from', banner: 'id' option :suite_options, - aliases: ['-u'], # NOTE: -o will be for outputter + aliases: ['-u'], type: :hash, desc: 'Suite options' option :groups, @@ -116,6 +126,10 @@ def version aliases: ['-i'], type: :hash, desc: 'Inputs (i.e: --inputs=foo:bar goo:baz)' + option :outputter, + aliases: ['-o'], + default: 'console', + desc: 'Select an outputter format: console | plain | json | quiet' option :verbose, aliases: ['-v'], type: :boolean, diff --git a/spec/inferno/cli/execute/console_outputter_spec.rb b/spec/inferno/cli/execute/console_outputter_spec.rb index b82d95b38..7fda528bc 100644 --- a/spec/inferno/cli/execute/console_outputter_spec.rb +++ b/spec/inferno/cli/execute/console_outputter_spec.rb @@ -1,20 +1,11 @@ require_relative '../../../../lib/inferno/apps/cli/execute/console_outputter' +require_relative 'outputter_spec' RSpec.describe Inferno::CLI::Execute::ConsoleOutputter do let(:instance) { described_class.new } let(:options) { { verbose: true } } - describe '#serialize' do - let(:test_results) { create_list(:result, 2) } - - it 'handles an array of test results without raising exception' do - expect { instance.serialize(test_results) }.to_not raise_error(StandardError) - end - - it 'returns valid JSON' do - expect { JSON.parse(instance.serialize(test_results)) }.to_not raise_error(StandardError) - end - end + include_examples 'outputter_spec', described_class describe '#verbose_print' do it 'outputs when verbose is true' do diff --git a/spec/inferno/cli/execute/json_outputter_spec.rb b/spec/inferno/cli/execute/json_outputter_spec.rb new file mode 100644 index 000000000..5b0127b1e --- /dev/null +++ b/spec/inferno/cli/execute/json_outputter_spec.rb @@ -0,0 +1,6 @@ +require_relative '../../../../lib/inferno/apps/cli/execute/json_outputter' +require_relative 'outputter_spec' + +RSpec.describe Inferno::CLI::Execute::JSONOutputter do + include_examples 'outputter_spec', described_class +end diff --git a/spec/inferno/cli/execute/outputter_spec.rb b/spec/inferno/cli/execute/outputter_spec.rb new file mode 100644 index 000000000..25243b5cf --- /dev/null +++ b/spec/inferno/cli/execute/outputter_spec.rb @@ -0,0 +1,33 @@ +RSpec.shared_examples 'outputter_spec' do |outputter_class| + it 'responds to print_start_message' do + expect(outputter_class.new).to respond_to(:print_start_message) + end + + it 'responds to print_around_run' do + expect(outputter_class.new).to respond_to(:print_around_run) + end + + it 'method print_around_run yields' do + expect do + expect { |b| outputter_class.new.print_around_run({}, &b) }.to yield_control + end.to output(/.?/).to_stdout_from_any_process # required to prevent output in rspec + end + + it 'responds to print_results' do + expect(outputter_class.new).to respond_to(:print_results) + end + + it 'responds to print_end_message' do + expect(outputter_class.new).to respond_to(:print_end_message) + end + + it 'responds to print_error' do + expect(outputter_class.new).to respond_to(:print_error) + end + + it 'returns an object whose print_error does not raise exception nor exit' do + expect do + expect { outputter_class.new.print_error({}, StandardError.new('my error')) }.to_not raise_error + end.to output(/.?/).to_stdout # required to prevent output in rspec + end +end diff --git a/spec/inferno/cli/execute/plain_outputter_spec.rb b/spec/inferno/cli/execute/plain_outputter_spec.rb new file mode 100644 index 000000000..5384214af --- /dev/null +++ b/spec/inferno/cli/execute/plain_outputter_spec.rb @@ -0,0 +1,20 @@ +require_relative '../../../../lib/inferno/apps/cli/execute/plain_outputter' +require_relative 'outputter_spec' + +RSpec.describe Inferno::CLI::Execute::PlainOutputter do + let(:instance) { described_class.new } + let(:results) { create_list(:result, 2) } + let(:options) { { outputter: 'plain', verbose: true } } + + include_examples 'outputter_spec', described_class + + it 'never outputs a color code' do + expect do + instance.print_start_message(options) + instance.print_around_run(options) { ' ' } + instance.print_results(options, results) + instance.print_end_message(options) + # instance.print_error(options, StandardError.new('Mock Error')) + end.to_not output(/\033/).to_stdout + end +end diff --git a/spec/inferno/cli/execute/quiet_outputter_spec.rb b/spec/inferno/cli/execute/quiet_outputter_spec.rb new file mode 100644 index 000000000..439688d7c --- /dev/null +++ b/spec/inferno/cli/execute/quiet_outputter_spec.rb @@ -0,0 +1,26 @@ +require_relative '../../../../lib/inferno/apps/cli/execute/quiet_outputter' +require_relative 'outputter_spec' + +RSpec.describe Inferno::CLI::Execute::QuietOutputter do + let(:instance) { described_class.new } + let(:results) { create_list(:result, 2) } + + include_examples 'outputter_spec', described_class + + it 'never outputs when verbose is false' do + options = { outputter: 'quiet', verbose: false } + + expect do + instance.print_start_message(options) + instance.print_around_run(options) { ' ' } + instance.print_results(options, results) + instance.print_end_message(options) + instance.print_error(options, StandardError.new('Mock Error')) + end.to_not output(/./).to_stdout + end + + it 'outputs error when verbose is true' do + options = { outputter: 'quiet', verbose: true } + expect { instance.print_error(options, StandardError.new('Mock Error')) }.to output.to_stdout + end +end diff --git a/spec/inferno/cli/execute/serialize_spec.rb b/spec/inferno/cli/execute/serialize_spec.rb new file mode 100644 index 000000000..097bae132 --- /dev/null +++ b/spec/inferno/cli/execute/serialize_spec.rb @@ -0,0 +1,18 @@ +require_relative '../../../../lib/inferno/apps/cli/execute/serialize' + +RSpec.describe Inferno::CLI::Execute::Serialize do + let(:dummy_class) { Class.new { include Inferno::CLI::Execute::Serialize } } + let(:instance) { dummy_class.new } + + describe '#serialize' do + let(:test_results) { create_list(:result, 2) } + + it 'handles an array of test results without raising exception' do + expect { instance.serialize(test_results) }.to_not raise_error(StandardError) + end + + it 'returns JSON' do + expect { JSON.parse(instance.serialize(test_results)) }.to_not raise_error(StandardError) + end + end +end diff --git a/spec/inferno/cli/execute_spec.rb b/spec/inferno/cli/execute_spec.rb index 9460fd4da..37969ce64 100644 --- a/spec/inferno/cli/execute_spec.rb +++ b/spec/inferno/cli/execute_spec.rb @@ -26,37 +26,16 @@ end describe '#outputter' do - it 'returns an object that responds to print_start_message' do - expect(instance.outputter).to respond_to(:print_start_message) - end - - it 'returns an object that responds to print_around_run' do - expect(instance.outputter).to respond_to(:print_around_run) - end - - it 'returns an object whose print_around_run yields' do - expect do - expect { |b| instance.outputter.print_around_run({}, &b) }.to yield_control - end.to output(/.?/).to_stdout # required to prevent output in rspec - end - - it 'returns an object that responds to print_results' do - expect(instance.outputter).to respond_to(:print_results) - end - - it 'returns an object that responds to print_end_message' do - expect(instance.outputter).to respond_to(:print_end_message) - end - - it 'returns an object that responds to print_error' do - expect(instance.outputter).to respond_to(:print_error) - end - - it 'returns an object whose print_error does not raise exception nor exit' do - allow(instance).to receive(:options).and_return({}) - expect do - expect { instance.outputter.print_error({}, StandardError.new('my error')) }.to_not raise_error - end.to output(/.?/).to_stdout # required to prevent output in rspec + { + 'console' => Inferno::CLI::Execute::ConsoleOutputter, + 'plain' => Inferno::CLI::Execute::PlainOutputter, + 'json' => Inferno::CLI::Execute::JSONOutputter, + 'quiet' => Inferno::CLI::Execute::QuietOutputter + }.each do |selected_outputter, expected_class| + it "returns #{expected_class} given '#{selected_outputter}'" do + allow(instance).to receive(:options).and_return({ outputter: selected_outputter }) + expect(instance.outputter).to be_an_instance_of expected_class + end end end @@ -268,7 +247,7 @@ .to_return(status: 200, body: FHIR::Patient.new({ name: { given: 'Smith' } }).to_json) expect do - expect { instance.run({ suite:, inputs:, verbose: true }) } + expect { instance.run({ suite:, inputs:, outputter: 'plain', verbose: true }) } .to raise_error(an_instance_of(SystemExit).and(having_attributes(status: 0))) end.to output(/.+/).to_stdout end