FI-2946: Limit Endpoint Availability Validation Input

lib/service_base_url_test_kit.rb

@@ -26,7 +26,7 @@ class ServiceBaseURLTestSuite < Inferno::TestSuite
       [GitHub](https://github.com/inferno-framework/service-base-url-test-kit/issues),
       or by reaching out to the team on the [Inferno FHIR Zulip
       channel](https://chat.fhir.org/#narrow/stream/179309-inferno).
-      
+
       Relevant requirements from the [Conditions and Maintenance of
       Certification - Application programming
       interfaces](https://www.ecfr.gov/current/title-45/subtitle-A/subchapter-D/part-170/subpart-D/section-170.404#p-170.404(b)(2)):
@@ -62,10 +62,12 @@ class ServiceBaseURLTestSuite < Inferno::TestSuite
 
     input_instructions <<~INSTRUCTIONS
       For systems that make their Service Base URL Bundle available at a public endpoint, please input
-      the Service Base URL Publication URL to retreive the Bundle from there in order to perform validation.
+      the Service Base URL Publication URL to retrieve the Bundle from there in order to perform validation, and leave
+      the Service Base URL Publication Bundle input blank.
 
       For systems that do not have a Service Base URL Bundle served at a public endpoint, testers can validate by
-      providing the Service Base URL Publication Bundle as an input.
+      providing the Service Base URL Publication Bundle as an input and leaving the Service Base URL Publication URL
+      input blank.
     INSTRUCTIONS
 
     links [
@@ -96,7 +98,7 @@ class ServiceBaseURLTestSuite < Inferno::TestSuite
     VALIDATION_MESSAGE_FILTERS = [
       /A resource should have narrative for robust management/,
       /\A\S+: \S+: URL value '.*' does not resolve/
-    ]
+    ].freeze
 
     # All FHIR validation requests will use this FHIR validator
     fhir_resource_validator :default do

lib/service_base_url_test_kit/service_base_url_test_group.rb

@@ -6,27 +6,27 @@ class ServiceBaseURLGroup < Inferno::TestGroup
     id :service_base_url_test_group
     title 'Validate Service Base URL Publication'
     description %(
-    Verify that the developer makes its Service Base URL publication publicly available
-    in the Bundle resource format with valid Endpoint and Organization entries.
-    This test group will issue a HTTP GET request against the supplied URL to
-    retrieve the developer's Service Base URL publication and ensure the list is
-    publicly accessible. It will then ensure that the returned service base URL
-    publication is in the Bundle resource format containing its service base URLs and
-    related organizational details in valid Endpoint and Organization resources
-    that follow the specifications detailed in the HTI-1 rule and the API
-    Condition and Maintenance of Certification.
-
-    For systems that provide the service base URL Bundle at a URL, please run
-    this test with the Service Base URL Publication URL input populated. While it is the expectation of the
-    specification for the service base URL Bundle to be served at a
-    public-facing endpoint, testers can validate a Service Base URL Bundle not
-    served at a public endpoint by running these tests with the Service Base URL Publication Bundle input populated
-    and the Service Base URL Publication URL input left blank.
+      Verify that the developer makes its Service Base URL publication publicly available
+      in the Bundle resource format with valid Endpoint and Organization entries.
+      This test group will issue a HTTP GET request against the supplied URL to
+      retrieve the developer's Service Base URL publication and ensure the list is
+      publicly accessible. It will then ensure that the returned service base URL
+      publication is in the Bundle resource format containing its service base URLs and
+      related organizational details in valid Endpoint and Organization resources
+      that follow the specifications detailed in the HTI-1 rule and the API
+      Condition and Maintenance of Certification.
+
+      For systems that provide the service base URL Bundle at a URL, please run
+      this test with the Service Base URL Publication URL input populated and the Service Base URL Publication Bundle
+      input left blank. While it is the expectation of the specification for the service base URL Bundle to be served at a
+      public-facing endpoint, testers can validate a Service Base URL Bundle not served at a public endpoint by running
+      these tests with the Service Base URL Publication Bundle input populated and the Service Base URL Publication URL
+      input left blank.
     )
 
     input_instructions <<~INSTRUCTIONS
       For systems that make their Service Base URL Bundle available at a public endpoint, please input
-      the Service Base URL Publication URL to retreive the Bundle from there in order to perform validation.
+      the Service Base URL Publication URL to retrieve the Bundle from there in order to perform validation.
 
       For systems that do not have a Service Base URL Bundle served at a public endpoint, testers can validate by
       providing the Service Base URL Publication Bundle as an input and leaving the Service Base URL Publication URL

lib/service_base_url_test_kit/service_base_url_validate_group.rb

@@ -20,6 +20,40 @@ class ServiceBaseURLBundleTestGroup < Inferno::TestGroup
     input :service_base_url_bundle,
           optional: true
 
+    input :endpoint_availability_limit,
+          title: 'Endpoint Availability Limit',
+          description: %(
+            In the case where the Endpoint Availability Success Rate is 'All', input a number to cap the number of
+            Endpoints that Inferno will send requests to check for availability. This can help speed up validation when
+            there are large number of endpoints in the Service Base URL Bundle.
+          ),
+          optional: true
+
+    input :endpoint_availability_success_rate,
+          title: 'Endpoint Availability Success Rate',
+          description: %(
+            Select an option to choose how many Endpoints have to be available and send back a valid capability
+            statement for the Endpoint validation test to pass.
+          ),
+          type: 'radio',
+          options: {
+            list_options: [
+              {
+                label: 'All',
+                value: 'all'
+              },
+              {
+                label: 'At Least One',
+                value: 'at_least_1'
+              },
+              {
+                label: 'None',
+                value: 'none'
+              }
+            ]
+          },
+          default: 'all'
+
     # @private
     def find_referenced_org(bundle_resource, endpoint_id)
       bundle_resource
@@ -142,8 +176,6 @@ def skip_message
         and available.
       )
 
-      output :testing
-
       run do
         bundle_response = if service_base_url_bundle.blank?
                             load_tagged_requests('service_base_url_bundle')
@@ -159,20 +191,60 @@ def skip_message
 
         skip_if bundle_resource.entry.empty?, 'The given Bundle does not contain any resources'
 
-        bundle_resource
+        endpoint_list = bundle_resource
           .entry
           .map(&:resource)
           .select { |resource| resource.resourceType == 'Endpoint' }
           .map(&:address)
           .uniq
-          .each do |address|
+
+        if endpoint_availability_limit.present? && endpoint_availability_limit.to_i < endpoint_list.count
+          info %(
+            Only the first #{endpoint_availability_limit.to_i} endpoints of #{endpoint_list.count} total will be
+            checked.
+          )
+        end
+
+        one_endpoint_valid = false
+        endpoint_list.each_with_index do |address, index|
           assert_valid_http_uri(address)
 
+          next if endpoint_availability_success_rate == 'none' ||
+                  (endpoint_availability_limit.present? && endpoint_availability_limit.to_i <= index)
+
           address = address.delete_suffix('/')
-          get("#{address}/metadata", client: nil, headers: { Accept: 'application/fhir+json' })
-          assert_response_status(200)
-          assert resource.present?, 'The content received does not appear to be a valid FHIR resource'
-          assert_resource_type(:capability_statement)
+
+          response = nil
+          warning do
+            response = get("#{address}/metadata", client: nil, headers: { Accept: 'application/fhir+json' })
+          end
+
+          if endpoint_availability_success_rate == 'all'
+            assert response.present?, "Encountered issues while trying to make a request to #{address}/metadata."
+            assert_response_status(200)
+            assert resource.present?, 'The content received does not appear to be a valid FHIR resource'
+            assert_resource_type(:capability_statement)
+          else
+            if response.present?
+              warning do
+                assert_response_status(200)
+                assert resource.present?, 'The content received does not appear to be a valid FHIR resource'
+                assert_resource_type(:capability_statement)
+              end
+            end
+
+            if !one_endpoint_valid && response.present? && response.status == 200 && resource.present? &&
+               resource.resourceType == 'CapabilityStatement'
+              one_endpoint_valid = true
+            end
+          end
+        end
+
+        if endpoint_availability_success_rate == 'at_least_1'
+          assert(one_endpoint_valid, %(
+            There were no Endpoints that were available and returned a valid Capability Statement in the Service Base
+            URL Bundle'
+          ))
         end
       end
     end

spec/service_base_url/service_base_url_spec.rb

@@ -9,7 +9,8 @@
 
   let(:input) do
     {
-      service_base_url_publication_url:
+      service_base_url_publication_url:,
+      endpoint_availability_success_rate: 'all'
     }
   end
   let(:validator_response_success) do
@@ -99,7 +100,7 @@ def run(runnable, inputs = {})
         .with(query: hash_including({}))
         .to_return(status: 200, body: validator_response_success.to_json)
 
-      result = run(test, service_base_url_bundle: bundle_resource.to_json)
+      result = run(test, service_base_url_bundle: bundle_resource.to_json, endpoint_availability_success_rate: 'all')
 
       expect(result.result).to eq('pass'), %(
         Expected a valid inputted service base url Bundle to pass
@@ -169,6 +170,72 @@ def run(runnable, inputs = {})
       expect(validation_request).to have_been_made.times(7)
     end
 
+    it 'passes and only checks the availability of number of endpoints equal to the endpoint availability limit' do
+      stub_request(:get, service_base_url_publication_url)
+        .to_return(status: 200, body: bundle_resource.to_json, headers: {})
+
+      uri_template = Addressable::Template.new "#{base_url}/{id}/metadata"
+      capability_statement_request_success = stub_request(:get, uri_template)
+        .to_return(status: 200, body: capability_statement.to_json, headers: {})
+
+      validation_request = stub_request(:post, "#{validator_url}/validate")
+        .with(query: hash_including({}))
+        .to_return(status: 200, body: validator_response_success.to_json)
+
+      result = run(test, service_base_url_publication_url:, endpoint_availability_success_rate: 'all',
+                         endpoint_availability_limit: 2)
+
+      expect(result.result).to eq('pass')
+      expect(capability_statement_request_success).to have_been_made.times(2)
+      expect(validation_request).to have_been_made.times(7)
+    end
+
+    it 'passes if at least 1 endpoint is available when success rate input is set to at least 1' do
+      bundle_resource.entry[4].resource.address = "#{base_url}/fake/address/3"
+      bundle_resource.entry[0].resource.address = "#{base_url}/fake/address/1"
+
+      stub_request(:get, service_base_url_publication_url)
+        .to_return(status: 200, body: bundle_resource.to_json, headers: {})
+
+      fake_uri_template = Addressable::Template.new "#{base_url}/fake/address/{id}/metadata"
+      capability_statement_request_fail = stub_request(:get, fake_uri_template)
+        .to_return(status: 404, body: '', headers: {})
+
+      uri_template = Addressable::Template.new "#{base_url}/{id}/metadata"
+      capability_statement_request_success = stub_request(:get, uri_template)
+        .to_return(status: 200, body: capability_statement.to_json, headers: {})
+
+      validation_request = stub_request(:post, "#{validator_url}/validate")
+        .with(query: hash_including({}))
+        .to_return(status: 200, body: validator_response_success.to_json)
+
+      result = run(test, service_base_url_publication_url:, endpoint_availability_success_rate: 'at_least_1')
+
+      expect(result.result).to eq('pass')
+      expect(capability_statement_request_fail).to have_been_made.times(2)
+      expect(capability_statement_request_success).to have_been_made
+      expect(validation_request).to have_been_made.times(7)
+    end
+
+    it 'passes and does not retrieve any capability statements if success rate input set to none' do
+      stub_request(:get, service_base_url_publication_url)
+        .to_return(status: 200, body: bundle_resource.to_json, headers: {})
+
+      uri_template = Addressable::Template.new "#{base_url}/{id}/metadata"
+      capability_statement_request_success = stub_request(:get, uri_template)
+        .to_return(status: 200, body: capability_statement.to_json, headers: {})
+
+      validation_request = stub_request(:post, "#{validator_url}/validate")
+        .with(query: hash_including({}))
+        .to_return(status: 200, body: validator_response_success.to_json)
+
+      result = run(test, service_base_url_publication_url:, endpoint_availability_success_rate: 'none')
+
+      expect(result.result).to eq('pass')
+      expect(capability_statement_request_success).to have_been_made.times(0)
+      expect(validation_request).to have_been_made.times(7)
+    end
+
     it 'fails if Bundle contains endpoint that has an invalid URL in the address field' do
       bundle_resource.entry[4].resource.address = 'invalid_url%.com'