From 2071f6dd159c83fc2a7a526f984aef1dee5b7ed3 Mon Sep 17 00:00:00 2001
From: Karl Naden <knaden@mitre.org>
Date: Sat, 9 Nov 2024 23:34:19 -0500
Subject: [PATCH 1/9] interaction tests

---
 .../suite_spec_context.rb                     | 34 +++++++++
 .../workflow/interaction_test_spec.rb         | 74 +++++++++++++++++++
 2 files changed, 108 insertions(+)
 create mode 100644 spec/subscriptions_test_kit/suite_spec_context.rb
 create mode 100644 spec/subscriptions_test_kit/suites/subscription_r5_backport_r4_client/workflow/interaction_test_spec.rb

diff --git a/spec/subscriptions_test_kit/suite_spec_context.rb b/spec/subscriptions_test_kit/suite_spec_context.rb
new file mode 100644
index 0000000..9a52bbe
--- /dev/null
+++ b/spec/subscriptions_test_kit/suite_spec_context.rb
@@ -0,0 +1,34 @@
+RSpec.shared_context('when testing a suite') do |suite_id|
+  let(:suite) { Inferno::Repositories::TestSuites.new.find(suite_id) }
+  let(:session_data_repo) { Inferno::Repositories::SessionData.new }
+  let(:validation_url) { "#{ENV.fetch('FHIR_RESOURCE_VALIDATOR_URL')}/validate" }
+  let(:test_session) { repo_create(:test_session, test_suite_id: suite_id) }
+
+  def run(runnable, inputs = {})
+    test_run_params = { test_session_id: test_session.id }.merge(runnable.reference_hash)
+    test_run = Inferno::Repositories::TestRuns.new.create(test_run_params)
+    inputs.each do |name, value|
+      session_data_repo.save(
+        test_session_id: test_session.id,
+        name:,
+        value:,
+        type: runnable.config.input_type(name)
+      )
+    end
+
+    Inferno::TestRunner.new(test_session:, test_run:).run(runnable)
+  end
+
+  def find_test(runnable, id)
+    # target has the search id as a suffix of the parent's id
+    target_id = runnable.parent.nil? ? id : "#{runnable.parent.id}-#{id}"
+    return runnable if runnable.id == target_id
+
+    runnable.children.each do |entity|
+      found = find_test(entity, id)
+      return found unless found.nil?
+    end
+
+    nil
+  end
+end
diff --git a/spec/subscriptions_test_kit/suites/subscription_r5_backport_r4_client/workflow/interaction_test_spec.rb b/spec/subscriptions_test_kit/suites/subscription_r5_backport_r4_client/workflow/interaction_test_spec.rb
new file mode 100644
index 0000000..21fa87c
--- /dev/null
+++ b/spec/subscriptions_test_kit/suites/subscription_r5_backport_r4_client/workflow/interaction_test_spec.rb
@@ -0,0 +1,74 @@
+require_relative '../../../suite_spec_context'
+
+RSpec.describe SubscriptionsTestKit::SubscriptionsR5BackportR4Client::InteractionTest do
+  include Rack::Test::Methods
+
+  def app
+    Inferno::Web.app
+  end
+  include_context('when testing a suite', 'subscriptions_r5_backport_r4_client')
+
+  describe 'performing interactions with the client under test' do
+    let(:suite_id) { 'subscriptions_r5_backport_r4_client' }
+    let(:access_token) { '1234' }
+    let(:test) { find_test(suite, described_class.id) }
+
+    describe 'when the tester-provided notification bundle is valid' do
+      let(:valid_notification_json) do
+        File.read(File.join(__dir__, '../../../..', 'fixtures', 'empty_notification_bundle_example.json'))
+      end
+      let(:resume_pass_url) { "/custom/#{suite_id}/resume_pass" }
+      let(:resume_fail_url) { "/custom/#{suite_id}/resume_fail" }
+      let(:results_repo) { Inferno::Repositories::Results.new }
+
+      it 'passes when the tester chooses to complete the tests' do
+        result = run(test, access_token:, notification_bundle: valid_notification_json)
+        expect(result.result).to eq('wait')
+
+        get("#{resume_pass_url}?test_run_identifier=#{access_token}")
+
+        result = results_repo.find(result.id)
+        expect(result.result).to eq('pass')
+      end
+
+      it 'fails when the tester chooses to fail the tests' do
+        result = run(test, access_token:, notification_bundle: valid_notification_json)
+        expect(result.result).to eq('wait')
+
+        get("#{resume_fail_url}?test_run_identifier=#{access_token}")
+
+        result = results_repo.find(result.id)
+        expect(result.result).to eq('fail')
+      end
+    end
+
+    describe 'when the tester-provided notification bundle is not valid' do
+      it 'fails when the notification bundle is not json' do
+        result = run(test, access_token:, notification_bundle: 'not json')
+        expect(result.result).to eq('fail')
+      end
+
+      it 'fails when the notification bundle is not FHIR' do
+        result = run(test, access_token:, notification_bundle: '{"not":"FHIR"}')
+        expect(result.result).to eq('fail')
+      end
+
+      it 'fails when the notification bundle is not a Bundle' do
+        result = run(test, access_token:, notification_bundle: '{"resourceType":"Patient"}')
+        expect(result.result).to eq('fail')
+      end
+
+      it 'fails when the notification bundle does not contain a Parameters instance' do
+        result = run(test, access_token:, notification_bundle: '{"resourceType":"Bundle"}')
+        expect(result.result).to eq('fail')
+      end
+
+      it 'fails when the notification bundle Parameters instance does not contain a subscription parameter entry' do
+        result = run(test, access_token:,
+                           notification_bundle:
+                            '{"resourceType":"Bundle", "entry":[{"resource":{"resourceType":"Parameters"}}]}')
+        expect(result.result).to eq('fail')
+      end
+    end
+  end
+end

From 3a950ffd9e3e4506cf1b428d235da38b3f0a656c Mon Sep 17 00:00:00 2001
From: Karl Naden <knaden@mitre.org>
Date: Mon, 11 Nov 2024 09:05:38 -0500
Subject: [PATCH 2/9] add some documentation

---
 .../subscriptions_test_kit/suite_spec_context.rb | 16 ++++++++++------
 .../workflow/interaction_test_spec.rb            | 10 ++++++----
 2 files changed, 16 insertions(+), 10 deletions(-)

diff --git a/spec/subscriptions_test_kit/suite_spec_context.rb b/spec/subscriptions_test_kit/suite_spec_context.rb
index 9a52bbe..f618c0f 100644
--- a/spec/subscriptions_test_kit/suite_spec_context.rb
+++ b/spec/subscriptions_test_kit/suite_spec_context.rb
@@ -1,5 +1,6 @@
-RSpec.shared_context('when testing a suite') do |suite_id|
+RSpec.shared_context('when testing this suite') do |suite_id|
   let(:suite) { Inferno::Repositories::TestSuites.new.find(suite_id) }
+  let(:suite_id) { suite_id }
   let(:session_data_repo) { Inferno::Repositories::SessionData.new }
   let(:validation_url) { "#{ENV.fetch('FHIR_RESOURCE_VALIDATOR_URL')}/validate" }
   let(:test_session) { repo_create(:test_session, test_suite_id: suite_id) }
@@ -19,13 +20,16 @@ def run(runnable, inputs = {})
     Inferno::TestRunner.new(test_session:, test_run:).run(runnable)
   end
 
-  def find_test(runnable, id)
-    # target has the search id as a suffix of the parent's id
-    target_id = runnable.parent.nil? ? id : "#{runnable.parent.id}-#{id}"
-    return runnable if runnable.id == target_id
+  # depth-first search looking for a runnable with a runtime id
+  # (prefixed with the ancestor suite / group ids) that ends
+  # with the provided suffix. It can be the test's id if unique, or
+  # can include some ancestor context if needed to identify the
+  # correct test. The first matching test found will be returned.
+  def find_test(runnable, id_suffix)
+    return runnable if runnable.id.ends_with?(id_suffix)
 
     runnable.children.each do |entity|
-      found = find_test(entity, id)
+      found = find_test(entity, id_suffix)
       return found unless found.nil?
     end
 
diff --git a/spec/subscriptions_test_kit/suites/subscription_r5_backport_r4_client/workflow/interaction_test_spec.rb b/spec/subscriptions_test_kit/suites/subscription_r5_backport_r4_client/workflow/interaction_test_spec.rb
index 21fa87c..9b90c17 100644
--- a/spec/subscriptions_test_kit/suites/subscription_r5_backport_r4_client/workflow/interaction_test_spec.rb
+++ b/spec/subscriptions_test_kit/suites/subscription_r5_backport_r4_client/workflow/interaction_test_spec.rb
@@ -1,15 +1,17 @@
 require_relative '../../../suite_spec_context'
 
 RSpec.describe SubscriptionsTestKit::SubscriptionsR5BackportR4Client::InteractionTest do
+  # ----- shared setup -----
+  # 1. enables http methods and requests to suite_endpoints
   include Rack::Test::Methods
-
   def app
     Inferno::Web.app
   end
-  include_context('when testing a suite', 'subscriptions_r5_backport_r4_client')
-
+  # 2. defines
+  # - variables: suite_id, suite, session_data_repo, validation_url, and test_session
+  # - methods: run, find_test
+  include_context('when testing this suite', 'subscriptions_r5_backport_r4_client')
   describe 'performing interactions with the client under test' do
-    let(:suite_id) { 'subscriptions_r5_backport_r4_client' }
     let(:access_token) { '1234' }
     let(:test) { find_test(suite, described_class.id) }
 

From b304717cbec6386fcb8b74082782f89345597984 Mon Sep 17 00:00:00 2001
From: Karl Naden <knaden@mitre.org>
Date: Mon, 11 Nov 2024 09:25:42 -0500
Subject: [PATCH 3/9] additional wait unit test pattern documentation

---
 .../workflow/interaction_test_spec.rb                       | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/spec/subscriptions_test_kit/suites/subscription_r5_backport_r4_client/workflow/interaction_test_spec.rb b/spec/subscriptions_test_kit/suites/subscription_r5_backport_r4_client/workflow/interaction_test_spec.rb
index 9b90c17..3dc14a2 100644
--- a/spec/subscriptions_test_kit/suites/subscription_r5_backport_r4_client/workflow/interaction_test_spec.rb
+++ b/spec/subscriptions_test_kit/suites/subscription_r5_backport_r4_client/workflow/interaction_test_spec.rb
@@ -23,6 +23,12 @@ def app
       let(:resume_fail_url) { "/custom/#{suite_id}/resume_fail" }
       let(:results_repo) { Inferno::Repositories::Results.new }
 
+      # pattern for wait testing
+      # 1. execute the test, e.g., result = run(test, ...)
+      # 2. verify the test is waiting, e.g., expect(result.result).to eq('wait')
+      # 3. perform an action that cause the wait to end, e.g., get(...)
+      # 4. find the updated result, e.g., result = results_repo.find(result.id)
+      # 5. verify it is no longer waiting, e.g., expect(result.result).to eq('pass')
       it 'passes when the tester chooses to complete the tests' do
         result = run(test, access_token:, notification_bundle: valid_notification_json)
         expect(result.result).to eq('wait')

From 8f17ac6ba84c168f6ebf29df9667f8d17b255ac9 Mon Sep 17 00:00:00 2001
From: Karl Naden <knaden@mitre.org>
Date: Mon, 11 Nov 2024 09:34:33 -0500
Subject: [PATCH 4/9] clean up run execution and document

---
 .../workflow/interaction_test_spec.rb         | 31 +++++++++++++------
 1 file changed, 21 insertions(+), 10 deletions(-)

diff --git a/spec/subscriptions_test_kit/suites/subscription_r5_backport_r4_client/workflow/interaction_test_spec.rb b/spec/subscriptions_test_kit/suites/subscription_r5_backport_r4_client/workflow/interaction_test_spec.rb
index 3dc14a2..b2a31cb 100644
--- a/spec/subscriptions_test_kit/suites/subscription_r5_backport_r4_client/workflow/interaction_test_spec.rb
+++ b/spec/subscriptions_test_kit/suites/subscription_r5_backport_r4_client/workflow/interaction_test_spec.rb
@@ -23,14 +23,15 @@ def app
       let(:resume_fail_url) { "/custom/#{suite_id}/resume_fail" }
       let(:results_repo) { Inferno::Repositories::Results.new }
 
-      # pattern for wait testing
+      # Pattern for wait testing
       # 1. execute the test, e.g., result = run(test, ...)
       # 2. verify the test is waiting, e.g., expect(result.result).to eq('wait')
       # 3. perform an action that cause the wait to end, e.g., get(...)
       # 4. find the updated result, e.g., result = results_repo.find(result.id)
       # 5. verify it is no longer waiting, e.g., expect(result.result).to eq('pass')
       it 'passes when the tester chooses to complete the tests' do
-        result = run(test, access_token:, notification_bundle: valid_notification_json)
+        inputs = { access_token:, notification_bundle: valid_notification_json }
+        result = run(test, inputs)
         expect(result.result).to eq('wait')
 
         get("#{resume_pass_url}?test_run_identifier=#{access_token}")
@@ -40,7 +41,8 @@ def app
       end
 
       it 'fails when the tester chooses to fail the tests' do
-        result = run(test, access_token:, notification_bundle: valid_notification_json)
+        inputs = { access_token:, notification_bundle: valid_notification_json }
+        result = run(test, inputs)
         expect(result.result).to eq('wait')
 
         get("#{resume_fail_url}?test_run_identifier=#{access_token}")
@@ -50,31 +52,40 @@ def app
       end
     end
 
+    # Pattern for execution with tester inputs
+    # 1. create input hash, e.g., inputs = { ... }
+    # 2. pass to the run method (defined in the shared context), e.g., result = run(test, inputs)
+
     describe 'when the tester-provided notification bundle is not valid' do
       it 'fails when the notification bundle is not json' do
-        result = run(test, access_token:, notification_bundle: 'not json')
+        inputs = { access_token:, notification_bundle: 'not json' }
+        result = run(test, inputs)
         expect(result.result).to eq('fail')
       end
 
       it 'fails when the notification bundle is not FHIR' do
-        result = run(test, access_token:, notification_bundle: '{"not":"FHIR"}')
+        inputs = { access_token:, notification_bundle: '{"not":"FHIR"}' }
+        result = run(test, inputs)
         expect(result.result).to eq('fail')
       end
 
       it 'fails when the notification bundle is not a Bundle' do
-        result = run(test, access_token:, notification_bundle: '{"resourceType":"Patient"}')
+        inputs = { access_token:, notification_bundle: '{"resourceType":"Patient"}' }
+        result = run(test, inputs)
         expect(result.result).to eq('fail')
       end
 
       it 'fails when the notification bundle does not contain a Parameters instance' do
-        result = run(test, access_token:, notification_bundle: '{"resourceType":"Bundle"}')
+        inputs = { access_token:, notification_bundle: '{"resourceType":"Bundle"}' }
+        result = run(test, inputs)
         expect(result.result).to eq('fail')
       end
 
       it 'fails when the notification bundle Parameters instance does not contain a subscription parameter entry' do
-        result = run(test, access_token:,
-                           notification_bundle:
-                            '{"resourceType":"Bundle", "entry":[{"resource":{"resourceType":"Parameters"}}]}')
+        inputs = { access_token:,
+                   notification_bundle:
+                    '{"resourceType":"Bundle", "entry":[{"resource":{"resourceType":"Parameters"}}]}' }
+        result = run(test, inputs)
         expect(result.result).to eq('fail')
       end
     end

From cf4e8eccdfe8c3c78d1bae9517b90acd9d18bdc9 Mon Sep 17 00:00:00 2001
From: Karl Naden <knaden@mitre.org>
Date: Mon, 11 Nov 2024 09:40:59 -0500
Subject: [PATCH 5/9] documentation re-org

---
 .../workflow/interaction_test_spec.rb                  | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/spec/subscriptions_test_kit/suites/subscription_r5_backport_r4_client/workflow/interaction_test_spec.rb b/spec/subscriptions_test_kit/suites/subscription_r5_backport_r4_client/workflow/interaction_test_spec.rb
index b2a31cb..03ccb7a 100644
--- a/spec/subscriptions_test_kit/suites/subscription_r5_backport_r4_client/workflow/interaction_test_spec.rb
+++ b/spec/subscriptions_test_kit/suites/subscription_r5_backport_r4_client/workflow/interaction_test_spec.rb
@@ -13,6 +13,12 @@ def app
   include_context('when testing this suite', 'subscriptions_r5_backport_r4_client')
   describe 'performing interactions with the client under test' do
     let(:access_token) { '1234' }
+
+    # Pattern for execution with tester inputs
+    # 1. get the runnable into the `test` variable using the find_test function,
+    #    e.g., let(:test) { find_test(suite, described_class.id) }
+    # 2. create input hash, e.g., inputs = { ... }
+    # 3. pass to the run method (defined in the shared context), e.g., result = run(test, inputs)
     let(:test) { find_test(suite, described_class.id) }
 
     describe 'when the tester-provided notification bundle is valid' do
@@ -52,10 +58,6 @@ def app
       end
     end
 
-    # Pattern for execution with tester inputs
-    # 1. create input hash, e.g., inputs = { ... }
-    # 2. pass to the run method (defined in the shared context), e.g., result = run(test, inputs)
-
     describe 'when the tester-provided notification bundle is not valid' do
       it 'fails when the notification bundle is not json' do
         inputs = { access_token:, notification_bundle: 'not json' }

From 40919f4509d73b82d77ba361ed95443e0754efce Mon Sep 17 00:00:00 2001
From: Stephen MacVicar <smacvicar@mitre.org>
Date: Mon, 18 Nov 2024 10:04:31 -0500
Subject: [PATCH 6/9] wip

---
 Gemfile                                        |  2 ++
 Gemfile.lock                                   |  8 ++++----
 .../workflow/interaction_test.rb               |  1 +
 spec/request_helper.rb                         |  6 +++++-
 ...ite_spec_context.rb => runnable_context.rb} |  9 +++++++--
 spec/runnable_helper.rb                        | 11 +++++++++++
 spec/spec_helper.rb                            |  3 +++
 .../workflow/interaction_test_spec.rb          | 18 +++++-------------
 8 files changed, 38 insertions(+), 20 deletions(-)
 rename spec/{subscriptions_test_kit/suite_spec_context.rb => runnable_context.rb} (88%)
 create mode 100644 spec/runnable_helper.rb

diff --git a/Gemfile b/Gemfile
index d965669..b400f65 100644
--- a/Gemfile
+++ b/Gemfile
@@ -4,6 +4,8 @@ source 'https://rubygems.org'
 
 gemspec
 
+# gem 'inferno_core', path: '../inferno'
+
 group :development, :test do
   gem 'debug'
   gem 'rubocop', '~> 1.56'
diff --git a/Gemfile.lock b/Gemfile.lock
index a0b4ba9..a7c42fe 100644
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -167,15 +167,15 @@ GEM
     json (2.7.2)
     jwt (2.9.3)
       base64
-    kramdown (2.4.0)
-      rexml
+    kramdown (2.5.0)
+      rexml (>= 3.3.6)
     language_server-protocol (3.17.0.3)
     logger (1.6.1)
     method_source (1.1.0)
     mime-types (3.6.0)
       logger
       mime-types-data (~> 3.2015)
-    mime-types-data (3.2024.1001)
+    mime-types-data (3.2024.1105)
     mini_portile2 (2.8.7)
     minitest (5.25.1)
     multi_json (1.15.0)
@@ -250,7 +250,7 @@ GEM
     roo (2.10.1)
       nokogiri (~> 1)
       rubyzip (>= 1.3.0, < 3.0.0)
-    rouge (4.4.0)
+    rouge (4.5.1)
     rspec (3.13.0)
       rspec-core (~> 3.13.0)
       rspec-expectations (~> 3.13.0)
diff --git a/lib/subscriptions_test_kit/suites/subscriptions_r5_backport_r4_client/workflow/interaction_test.rb b/lib/subscriptions_test_kit/suites/subscriptions_r5_backport_r4_client/workflow/interaction_test.rb
index ee3ff18..842a7fd 100644
--- a/lib/subscriptions_test_kit/suites/subscriptions_r5_backport_r4_client/workflow/interaction_test.rb
+++ b/lib/subscriptions_test_kit/suites/subscriptions_r5_backport_r4_client/workflow/interaction_test.rb
@@ -3,6 +3,7 @@
 module SubscriptionsTestKit
   module SubscriptionsR5BackportR4Client
     class InteractionTest < Inferno::Test
+      include URLs
       id :subscriptions_r4_client_interaction
       description %(
         During this test, the client under test will interact with Inferno following the Subscription
diff --git a/spec/request_helper.rb b/spec/request_helper.rb
index 30e3320..5ceb14d 100644
--- a/spec/request_helper.rb
+++ b/spec/request_helper.rb
@@ -1,10 +1,14 @@
 require 'spec_helper'
 require 'rack/test'
 require 'inferno/apps/web/application'
+require 'inferno/utils/middleware/request_logger'
 
 module RequestHelpers
   def app
-    Inferno::Web.app
+    Rack::Builder.new do
+      use Inferno::Utils::Middleware::RequestLogger
+      run Inferno::Web.app
+    end
   end
 
   def post_json(path, data)
diff --git a/spec/subscriptions_test_kit/suite_spec_context.rb b/spec/runnable_context.rb
similarity index 88%
rename from spec/subscriptions_test_kit/suite_spec_context.rb
rename to spec/runnable_context.rb
index f618c0f..6289b6e 100644
--- a/spec/subscriptions_test_kit/suite_spec_context.rb
+++ b/spec/runnable_context.rb
@@ -1,10 +1,15 @@
-RSpec.shared_context('when testing this suite') do |suite_id|
+RSpec.shared_context('when testing a runnable') do
   let(:suite) { Inferno::Repositories::TestSuites.new.find(suite_id) }
-  let(:suite_id) { suite_id }
   let(:session_data_repo) { Inferno::Repositories::SessionData.new }
   let(:validation_url) { "#{ENV.fetch('FHIR_RESOURCE_VALIDATOR_URL')}/validate" }
   let(:test_session) { repo_create(:test_session, test_suite_id: suite_id) }
 
+  before do
+    if (described_class.parent.nil?)
+      allow(described_class).to receive(:suite).and_return(suite)
+    end
+  end
+
   def run(runnable, inputs = {})
     test_run_params = { test_session_id: test_session.id }.merge(runnable.reference_hash)
     test_run = Inferno::Repositories::TestRuns.new.create(test_run_params)
diff --git a/spec/runnable_helper.rb b/spec/runnable_helper.rb
new file mode 100644
index 0000000..cbe6ae6
--- /dev/null
+++ b/spec/runnable_helper.rb
@@ -0,0 +1,11 @@
+require_relative 'runnable_context'
+
+RSpec.configure do |config|
+  config.define_derived_metadata do |metadata|
+    if metadata[:described_class].present? && metadata[:described_class].is_a?(Inferno::DSL::Runnable)
+      metadata[:runnable] = true
+    end
+  end
+
+  config.include_context 'when testing a runnable', runnable: true
+end
diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb
index f30b83f..d4d4598 100644
--- a/spec/spec_helper.rb
+++ b/spec/spec_helper.rb
@@ -133,6 +133,9 @@
   Inferno::SpecSupport::FACTORY_PATH
 ]
 
+require_relative 'runnable_helper'
+require_relative 'request_helper'
+
 RSpec::Matchers.define_negated_matcher :exclude, :include
 
 FHIR.logger = Inferno::Application['logger']
diff --git a/spec/subscriptions_test_kit/suites/subscription_r5_backport_r4_client/workflow/interaction_test_spec.rb b/spec/subscriptions_test_kit/suites/subscription_r5_backport_r4_client/workflow/interaction_test_spec.rb
index 03ccb7a..a831c2c 100644
--- a/spec/subscriptions_test_kit/suites/subscription_r5_backport_r4_client/workflow/interaction_test_spec.rb
+++ b/spec/subscriptions_test_kit/suites/subscription_r5_backport_r4_client/workflow/interaction_test_spec.rb
@@ -1,16 +1,6 @@
-require_relative '../../../suite_spec_context'
+RSpec.describe SubscriptionsTestKit::SubscriptionsR5BackportR4Client::InteractionTest, :request do
+  let(:suite_id) { 'subscriptions_r5_backport_r4_client' }
 
-RSpec.describe SubscriptionsTestKit::SubscriptionsR5BackportR4Client::InteractionTest do
-  # ----- shared setup -----
-  # 1. enables http methods and requests to suite_endpoints
-  include Rack::Test::Methods
-  def app
-    Inferno::Web.app
-  end
-  # 2. defines
-  # - variables: suite_id, suite, session_data_repo, validation_url, and test_session
-  # - methods: run, find_test
-  include_context('when testing this suite', 'subscriptions_r5_backport_r4_client')
   describe 'performing interactions with the client under test' do
     let(:access_token) { '1234' }
 
@@ -19,7 +9,8 @@ def app
     #    e.g., let(:test) { find_test(suite, described_class.id) }
     # 2. create input hash, e.g., inputs = { ... }
     # 3. pass to the run method (defined in the shared context), e.g., result = run(test, inputs)
-    let(:test) { find_test(suite, described_class.id) }
+    # let(:test) { find_test(suite, described_class.id) }
+    let(:test) { described_class }
 
     describe 'when the tester-provided notification bundle is valid' do
       let(:valid_notification_json) do
@@ -37,6 +28,7 @@ def app
       # 5. verify it is no longer waiting, e.g., expect(result.result).to eq('pass')
       it 'passes when the tester chooses to complete the tests' do
         inputs = { access_token:, notification_bundle: valid_notification_json }
+        binding.pry
         result = run(test, inputs)
         expect(result.result).to eq('wait')
 

From 99cdbc14619484c140484268368434b79c98a514 Mon Sep 17 00:00:00 2001
From: Stephen MacVicar <smacvicar@mitre.org>
Date: Mon, 18 Nov 2024 10:16:41 -0500
Subject: [PATCH 7/9] use core implementation

---
 Gemfile                                       |  4 +-
 Gemfile.lock                                  | 62 ++++++++++---------
 spec/request_helper.rb                        | 30 ---------
 spec/runnable_context.rb                      | 43 -------------
 spec/runnable_helper.rb                       | 11 ----
 spec/spec_helper.rb                           |  7 +--
 .../notification_delivery_test_spec.rb        |  5 +-
 .../workflow/interaction_test_spec.rb         |  1 -
 subscriptions_test_kit.gemspec                |  2 +-
 9 files changed, 42 insertions(+), 123 deletions(-)
 delete mode 100644 spec/request_helper.rb
 delete mode 100644 spec/runnable_context.rb
 delete mode 100644 spec/runnable_helper.rb

diff --git a/Gemfile b/Gemfile
index b400f65..0abd345 100644
--- a/Gemfile
+++ b/Gemfile
@@ -4,7 +4,9 @@ source 'https://rubygems.org'
 
 gemspec
 
-# gem 'inferno_core', path: '../inferno'
+gem 'inferno_core',
+    git: 'https://github.com/inferno-framework/inferno-core.git',
+    branch: 'fi-3475-unit-testing-improvements'
 
 group :development, :test do
   gem 'debug'
diff --git a/Gemfile.lock b/Gemfile.lock
index a7c42fe..982cbda 100644
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -1,8 +1,40 @@
+GIT
+  remote: https://github.com/inferno-framework/inferno-core.git
+  revision: cac216ad45293f8459ac0edfc9bc32ed081fe44b
+  branch: fi-3475-unit-testing-improvements
+  specs:
+    inferno_core (0.5.0)
+      activesupport (~> 6.1.7.5)
+      base62-rb (= 0.3.1)
+      blueprinter (= 0.25.2)
+      dotenv (~> 2.7)
+      dry-configurable (= 1.0.0)
+      dry-container (= 0.10.0)
+      dry-core (= 1.0.0)
+      dry-inflector (= 1.0.0)
+      dry-system (= 1.0.0)
+      faraday (~> 1.2)
+      faraday_middleware (~> 1.2)
+      fhir_client (>= 5.0.3)
+      fhir_models (>= 4.2.2)
+      hanami-controller (= 2.0.0)
+      hanami-router (= 2.0.0)
+      oj (= 3.11.0)
+      pastel (~> 0.8.0)
+      pry
+      pry-byebug
+      puma (~> 5.6.7)
+      rake (~> 13.0)
+      sequel (~> 5.42.0)
+      sidekiq (~> 7.2.4)
+      sqlite3 (~> 1.4)
+      thor (~> 1.2.1)
+      tty-markdown (~> 0.7.1)
+
 PATH
   remote: .
   specs:
     subscriptions_test_kit (0.9.3)
-      inferno_core (~> 0.5.0)
 
 GEM
   remote: https://rubygems.org/
@@ -133,33 +165,6 @@ GEM
       domain_name (~> 0.5)
     i18n (1.14.6)
       concurrent-ruby (~> 1.0)
-    inferno_core (0.5.0)
-      activesupport (~> 6.1.7.5)
-      base62-rb (= 0.3.1)
-      blueprinter (= 0.25.2)
-      dotenv (~> 2.7)
-      dry-configurable (= 1.0.0)
-      dry-container (= 0.10.0)
-      dry-core (= 1.0.0)
-      dry-inflector (= 1.0.0)
-      dry-system (= 1.0.0)
-      faraday (~> 1.2)
-      faraday_middleware (~> 1.2)
-      fhir_client (>= 5.0.3)
-      fhir_models (>= 4.2.2)
-      hanami-controller (= 2.0.0)
-      hanami-router (= 2.0.0)
-      oj (= 3.11.0)
-      pastel (~> 0.8.0)
-      pry
-      pry-byebug
-      puma (~> 5.6.7)
-      rake (~> 13.0)
-      sequel (~> 5.42.0)
-      sidekiq (~> 7.2.4)
-      sqlite3 (~> 1.4)
-      thor (~> 1.2.1)
-      tty-markdown (~> 0.7.1)
     io-console (0.7.2)
     irb (1.13.1)
       rdoc (>= 4.0.0)
@@ -336,6 +341,7 @@ DEPENDENCIES
   database_cleaner-sequel (~> 1.8)
   debug
   factory_bot (~> 6.1)
+  inferno_core!
   rack-test
   roo (~> 2.10.1)
   rspec (~> 3.10)
diff --git a/spec/request_helper.rb b/spec/request_helper.rb
deleted file mode 100644
index 5ceb14d..0000000
--- a/spec/request_helper.rb
+++ /dev/null
@@ -1,30 +0,0 @@
-require 'spec_helper'
-require 'rack/test'
-require 'inferno/apps/web/application'
-require 'inferno/utils/middleware/request_logger'
-
-module RequestHelpers
-  def app
-    Rack::Builder.new do
-      use Inferno::Utils::Middleware::RequestLogger
-      run Inferno::Web.app
-    end
-  end
-
-  def post_json(path, data)
-    post path, data.to_json, 'CONTENT_TYPE' => 'application/json'
-  end
-
-  def parsed_body
-    JSON.parse(last_response.body)
-  end
-end
-
-RSpec.configure do |config|
-  config.define_derived_metadata(file_path: %r{/routes/}) do |metadata|
-    metadata[:request] = true
-  end
-
-  config.include Rack::Test::Methods, request: true
-  config.include RequestHelpers, request: true
-end
diff --git a/spec/runnable_context.rb b/spec/runnable_context.rb
deleted file mode 100644
index 6289b6e..0000000
--- a/spec/runnable_context.rb
+++ /dev/null
@@ -1,43 +0,0 @@
-RSpec.shared_context('when testing a runnable') do
-  let(:suite) { Inferno::Repositories::TestSuites.new.find(suite_id) }
-  let(:session_data_repo) { Inferno::Repositories::SessionData.new }
-  let(:validation_url) { "#{ENV.fetch('FHIR_RESOURCE_VALIDATOR_URL')}/validate" }
-  let(:test_session) { repo_create(:test_session, test_suite_id: suite_id) }
-
-  before do
-    if (described_class.parent.nil?)
-      allow(described_class).to receive(:suite).and_return(suite)
-    end
-  end
-
-  def run(runnable, inputs = {})
-    test_run_params = { test_session_id: test_session.id }.merge(runnable.reference_hash)
-    test_run = Inferno::Repositories::TestRuns.new.create(test_run_params)
-    inputs.each do |name, value|
-      session_data_repo.save(
-        test_session_id: test_session.id,
-        name:,
-        value:,
-        type: runnable.config.input_type(name)
-      )
-    end
-
-    Inferno::TestRunner.new(test_session:, test_run:).run(runnable)
-  end
-
-  # depth-first search looking for a runnable with a runtime id
-  # (prefixed with the ancestor suite / group ids) that ends
-  # with the provided suffix. It can be the test's id if unique, or
-  # can include some ancestor context if needed to identify the
-  # correct test. The first matching test found will be returned.
-  def find_test(runnable, id_suffix)
-    return runnable if runnable.id.ends_with?(id_suffix)
-
-    runnable.children.each do |entity|
-      found = find_test(entity, id_suffix)
-      return found unless found.nil?
-    end
-
-    nil
-  end
-end
diff --git a/spec/runnable_helper.rb b/spec/runnable_helper.rb
deleted file mode 100644
index cbe6ae6..0000000
--- a/spec/runnable_helper.rb
+++ /dev/null
@@ -1,11 +0,0 @@
-require_relative 'runnable_context'
-
-RSpec.configure do |config|
-  config.define_derived_metadata do |metadata|
-    if metadata[:described_class].present? && metadata[:described_class].is_a?(Inferno::DSL::Runnable)
-      metadata[:runnable] = true
-    end
-  end
-
-  config.include_context 'when testing a runnable', runnable: true
-end
diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb
index d4d4598..65dac30 100644
--- a/spec/spec_helper.rb
+++ b/spec/spec_helper.rb
@@ -118,6 +118,8 @@
   config.before(:suite) do
     FactoryBot.find_definitions
   end
+
+  config.shared_context_metadata_behavior = :apply_to_host_groups
 end
 
 require 'inferno/config/application'
@@ -127,15 +129,12 @@
 require 'inferno'
 Inferno::Application.finalize!
 
-require Inferno::SpecSupport::FACTORY_BOT_SUPPORT_PATH
+Inferno::SpecSupport.require_helpers
 
 FactoryBot.definition_file_paths = [
   Inferno::SpecSupport::FACTORY_PATH
 ]
 
-require_relative 'runnable_helper'
-require_relative 'request_helper'
-
 RSpec::Matchers.define_negated_matcher :exclude, :include
 
 FHIR.logger = Inferno::Application['logger']
diff --git a/spec/subscriptions_test_kit/subscription_r4_server/common/interaction/notification_delivery_test_spec.rb b/spec/subscriptions_test_kit/subscription_r4_server/common/interaction/notification_delivery_test_spec.rb
index 65717ed..ffc1abc 100644
--- a/spec/subscriptions_test_kit/subscription_r4_server/common/interaction/notification_delivery_test_spec.rb
+++ b/spec/subscriptions_test_kit/subscription_r4_server/common/interaction/notification_delivery_test_spec.rb
@@ -1,10 +1,7 @@
 require_relative '../../../../../lib/subscriptions_test_kit/suites/subscriptions_r5_backport_r4_server/' \
                  'common/interaction/notification_delivery_test'
-require_relative '../../../../request_helper'
 
-RSpec.describe SubscriptionsTestKit::SubscriptionsR5BackportR4Server::NotificationDeliveryTest do
-  include Rack::Test::Methods
-  include RequestHelpers
+RSpec.describe SubscriptionsTestKit::SubscriptionsR5BackportR4Server::NotificationDeliveryTest, :request do
 
   let(:suite) { Inferno::Repositories::TestSuites.new.find('subscriptions_r5_backport_r4_server') }
   let(:test) { Inferno::Repositories::Tests.new.find('subscriptions_r4_server_notification_delivery') }
diff --git a/spec/subscriptions_test_kit/suites/subscription_r5_backport_r4_client/workflow/interaction_test_spec.rb b/spec/subscriptions_test_kit/suites/subscription_r5_backport_r4_client/workflow/interaction_test_spec.rb
index a831c2c..a0f2ab9 100644
--- a/spec/subscriptions_test_kit/suites/subscription_r5_backport_r4_client/workflow/interaction_test_spec.rb
+++ b/spec/subscriptions_test_kit/suites/subscription_r5_backport_r4_client/workflow/interaction_test_spec.rb
@@ -28,7 +28,6 @@
       # 5. verify it is no longer waiting, e.g., expect(result.result).to eq('pass')
       it 'passes when the tester chooses to complete the tests' do
         inputs = { access_token:, notification_bundle: valid_notification_json }
-        binding.pry
         result = run(test, inputs)
         expect(result.result).to eq('wait')
 
diff --git a/subscriptions_test_kit.gemspec b/subscriptions_test_kit.gemspec
index 0bcda7f..d8e0c5a 100644
--- a/subscriptions_test_kit.gemspec
+++ b/subscriptions_test_kit.gemspec
@@ -9,7 +9,7 @@ Gem::Specification.new do |spec|
   spec.description   = 'Inferno test kit for FHIR R5-style Subscriptions'
   spec.homepage      = 'https://github.com/inferno-framework/subscriptions-test-kit'
   spec.license       = 'Apache-2.0'
-  spec.add_dependency 'inferno_core', '~> 0.5.0'
+  # spec.add_dependency 'inferno_core', '~> 0.5.0'
   spec.required_ruby_version = Gem::Requirement.new('>= 3.1.2')
   spec.metadata['homepage_uri'] = spec.homepage
   spec.metadata['source_code_uri'] = spec.homepage

From 2b52ea1141796aae0ee121c4956806dba1374f7a Mon Sep 17 00:00:00 2001
From: Stephen MacVicar <smacvicar@mitre.org>
Date: Mon, 18 Nov 2024 10:25:04 -0500
Subject: [PATCH 8/9] fix linting error

---
 .../common/interaction/notification_delivery_test_spec.rb        | 1 -
 1 file changed, 1 deletion(-)

diff --git a/spec/subscriptions_test_kit/subscription_r4_server/common/interaction/notification_delivery_test_spec.rb b/spec/subscriptions_test_kit/subscription_r4_server/common/interaction/notification_delivery_test_spec.rb
index ffc1abc..63226e9 100644
--- a/spec/subscriptions_test_kit/subscription_r4_server/common/interaction/notification_delivery_test_spec.rb
+++ b/spec/subscriptions_test_kit/subscription_r4_server/common/interaction/notification_delivery_test_spec.rb
@@ -2,7 +2,6 @@
                  'common/interaction/notification_delivery_test'
 
 RSpec.describe SubscriptionsTestKit::SubscriptionsR5BackportR4Server::NotificationDeliveryTest, :request do
-
   let(:suite) { Inferno::Repositories::TestSuites.new.find('subscriptions_r5_backport_r4_server') }
   let(:test) { Inferno::Repositories::Tests.new.find('subscriptions_r4_server_notification_delivery') }
   let(:test_group) { Inferno::Repositories::TestGroups.new.find('subscriptions_r4_server_interaction') }

From edda7f7e299c0e4d6fa3b8dad9fa590cd2f97a93 Mon Sep 17 00:00:00 2001
From: Stephen MacVicar <smacvicar@mitre.org>
Date: Mon, 2 Dec 2024 13:26:51 -0500
Subject: [PATCH 9/9] use new core release

---
 Gemfile                        |  4 --
 Gemfile.lock                   | 72 ++++++++++++++++------------------
 subscriptions_test_kit.gemspec |  2 +-
 3 files changed, 34 insertions(+), 44 deletions(-)

diff --git a/Gemfile b/Gemfile
index 0abd345..d965669 100644
--- a/Gemfile
+++ b/Gemfile
@@ -4,10 +4,6 @@ source 'https://rubygems.org'
 
 gemspec
 
-gem 'inferno_core',
-    git: 'https://github.com/inferno-framework/inferno-core.git',
-    branch: 'fi-3475-unit-testing-improvements'
-
 group :development, :test do
   gem 'debug'
   gem 'rubocop', '~> 1.56'
diff --git a/Gemfile.lock b/Gemfile.lock
index 982cbda..52ce86f 100644
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -1,40 +1,8 @@
-GIT
-  remote: https://github.com/inferno-framework/inferno-core.git
-  revision: cac216ad45293f8459ac0edfc9bc32ed081fe44b
-  branch: fi-3475-unit-testing-improvements
-  specs:
-    inferno_core (0.5.0)
-      activesupport (~> 6.1.7.5)
-      base62-rb (= 0.3.1)
-      blueprinter (= 0.25.2)
-      dotenv (~> 2.7)
-      dry-configurable (= 1.0.0)
-      dry-container (= 0.10.0)
-      dry-core (= 1.0.0)
-      dry-inflector (= 1.0.0)
-      dry-system (= 1.0.0)
-      faraday (~> 1.2)
-      faraday_middleware (~> 1.2)
-      fhir_client (>= 5.0.3)
-      fhir_models (>= 4.2.2)
-      hanami-controller (= 2.0.0)
-      hanami-router (= 2.0.0)
-      oj (= 3.11.0)
-      pastel (~> 0.8.0)
-      pry
-      pry-byebug
-      puma (~> 5.6.7)
-      rake (~> 13.0)
-      sequel (~> 5.42.0)
-      sidekiq (~> 7.2.4)
-      sqlite3 (~> 1.4)
-      thor (~> 1.2.1)
-      tty-markdown (~> 0.7.1)
-
 PATH
   remote: .
   specs:
     subscriptions_test_kit (0.9.3)
+      inferno_core (~> 0.5.0)
 
 GEM
   remote: https://rubygems.org/
@@ -165,6 +133,33 @@ GEM
       domain_name (~> 0.5)
     i18n (1.14.6)
       concurrent-ruby (~> 1.0)
+    inferno_core (0.5.1)
+      activesupport (~> 6.1.7.5)
+      base62-rb (= 0.3.1)
+      blueprinter (= 0.25.2)
+      dotenv (~> 2.7)
+      dry-configurable (= 1.0.0)
+      dry-container (= 0.10.0)
+      dry-core (= 1.0.0)
+      dry-inflector (= 1.0.0)
+      dry-system (= 1.0.0)
+      faraday (~> 1.2)
+      faraday_middleware (~> 1.2)
+      fhir_client (>= 5.0.3)
+      fhir_models (>= 4.2.2)
+      hanami-controller (= 2.0.0)
+      hanami-router (= 2.0.0)
+      oj (= 3.11.0)
+      pastel (~> 0.8.0)
+      pry
+      pry-byebug
+      puma (~> 5.6.7)
+      rake (~> 13.0)
+      sequel (~> 5.42.0)
+      sidekiq (~> 7.2.4)
+      sqlite3 (~> 1.4)
+      thor (~> 1.2.1)
+      tty-markdown (~> 0.7.1)
     io-console (0.7.2)
     irb (1.13.1)
       rdoc (>= 4.0.0)
@@ -172,17 +167,17 @@ GEM
     json (2.7.2)
     jwt (2.9.3)
       base64
-    kramdown (2.5.0)
-      rexml (>= 3.3.6)
+    kramdown (2.5.1)
+      rexml (>= 3.3.9)
     language_server-protocol (3.17.0.3)
-    logger (1.6.1)
+    logger (1.6.2)
     method_source (1.1.0)
     mime-types (3.6.0)
       logger
       mime-types-data (~> 3.2015)
     mime-types-data (3.2024.1105)
-    mini_portile2 (2.8.7)
-    minitest (5.25.1)
+    mini_portile2 (2.8.8)
+    minitest (5.25.2)
     multi_json (1.15.0)
     multi_xml (0.7.1)
       bigdecimal (~> 3.1)
@@ -341,7 +336,6 @@ DEPENDENCIES
   database_cleaner-sequel (~> 1.8)
   debug
   factory_bot (~> 6.1)
-  inferno_core!
   rack-test
   roo (~> 2.10.1)
   rspec (~> 3.10)
diff --git a/subscriptions_test_kit.gemspec b/subscriptions_test_kit.gemspec
index d8e0c5a..0bcda7f 100644
--- a/subscriptions_test_kit.gemspec
+++ b/subscriptions_test_kit.gemspec
@@ -9,7 +9,7 @@ Gem::Specification.new do |spec|
   spec.description   = 'Inferno test kit for FHIR R5-style Subscriptions'
   spec.homepage      = 'https://github.com/inferno-framework/subscriptions-test-kit'
   spec.license       = 'Apache-2.0'
-  # spec.add_dependency 'inferno_core', '~> 0.5.0'
+  spec.add_dependency 'inferno_core', '~> 0.5.0'
   spec.required_ruby_version = Gem::Requirement.new('>= 3.1.2')
   spec.metadata['homepage_uri'] = spec.homepage
   spec.metadata['source_code_uri'] = spec.homepage