FI-2429: Migrate to HL7 validator wrapper (#6)

* FI-2429: Migrate to HL7 validator wrapper

* Add README to igs folder

.env.development

@@ -1,2 +1,2 @@
-V110_VALIDATOR_URL=http://localhost/validatorapi
+FHIR_RESOURCE_VALIDATOR_URL=http://localhost/hl7validatorapi
 REDIS_URL=redis://localhost:6379/0

.env.production

@@ -1,2 +1,2 @@
 REDIS_URL=redis://redis:6379/0
-V110_VALIDATOR_URL=http://validator_service:4567
+FHIR_RESOURCE_VALIDATOR_URL=http://hl7_validator_service:3500

.env.test

@@ -1,2 +1,2 @@
-VALIDATOR_URL=https://example.com/validatorapi
+FHIR_RESOURCE_VALIDATOR_URL=https://example.com/validatorapi
 ASYNC_JOBS=false

Gemfile.lock

@@ -146,7 +146,7 @@ GEM
     i18n (1.14.4)
       concurrent-ruby (~> 1.0)
     ice_nine (0.11.2)
-    inferno_core (0.4.33)
+    inferno_core (0.4.35)
       activesupport (~> 6.1.7.5)
       base62-rb (= 0.3.1)
       blueprinter (= 0.25.2)
@@ -180,14 +180,15 @@ GEM
       base64
     kramdown (2.4.0)
       rexml
-    method_source (1.0.0)
+    method_source (1.1.0)
     mime-types (3.5.2)
       mime-types-data (~> 3.2015)
     mime-types-data (3.2024.0305)
-    mini_portile2 (2.8.5)
+    mini_portile2 (2.8.6)
     minitest (5.22.3)
     multi_json (1.15.0)
-    multi_xml (0.6.0)
+    multi_xml (0.7.1)
+      bigdecimal (~> 3.1)
     multipart-post (2.4.0)
     mustermann (1.1.2)
       ruby2_keywords (~> 0.0.1)
@@ -196,14 +197,14 @@ GEM
       mustermann (= 1.1.2)
     netrc (0.11.0)
     nio4r (2.7.1)
-    nokogiri (1.16.3)
+    nokogiri (1.16.4)
       mini_portile2 (~> 2.8.2)
       racc (~> 1.4)
-    nokogiri (1.16.3-arm64-darwin)
+    nokogiri (1.16.4-arm64-darwin)
       racc (~> 1.4)
-    nokogiri (1.16.3-x86_64-darwin)
+    nokogiri (1.16.4-x86_64-darwin)
       racc (~> 1.4)
-    nokogiri (1.16.3-x86_64-linux)
+    nokogiri (1.16.4-x86_64-linux)
       racc (~> 1.4)
     oauth2 (1.4.11)
       faraday (>= 0.17.3, < 3.0)

config/nginx.background.conf

@@ -53,22 +53,38 @@ http {
     # the server will close connections after this time
     keepalive_timeout 600;
 
-    location /validator {
-      proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
-      proxy_set_header Host $http_host;
-      proxy_set_header X-Forwarded-Proto $scheme;
-      proxy_set_header X-Forwarded-Port $server_port;
-      proxy_redirect off;
-      proxy_set_header Connection '';
-      proxy_http_version 1.1;
-      chunked_transfer_encoding off;
-      proxy_buffering off;
-      proxy_cache off;
+#    location /validator {
+#      proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+#      proxy_set_header Host $http_host;
+#      proxy_set_header X-Forwarded-Proto $scheme;
+#      proxy_set_header X-Forwarded-Port $server_port;
+#      proxy_redirect off;
+#      proxy_set_header Connection '';
+#      proxy_http_version 1.1;
+#      chunked_transfer_encoding off;
+#      proxy_buffering off;
+#      proxy_cache off;
+#
+#      proxy_pass http://fhir_validator_app;
+#    }
 
-      proxy_pass http://fhir_validator_app;
-    }
+#    location /validatorapi/ {
+#      proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+#      proxy_set_header Host $http_host;
+#      proxy_set_header X-Forwarded-Proto $scheme;
+#      proxy_set_header X-Forwarded-Port $server_port;
+#      proxy_redirect off;
+#      proxy_set_header Connection '';
+#      proxy_http_version 1.1;
+#      chunked_transfer_encoding off;
+#      proxy_buffering off;
+#      proxy_cache off;
+#
+#      proxy_pass http://validator_service:4567/;
+#    }
+#  }
 
-    location /validatorapi/ {
+    location /hl7validatorapi/ {
       proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
       proxy_set_header Host $http_host;
       proxy_set_header X-Forwarded-Proto $scheme;
@@ -79,8 +95,9 @@ http {
       chunked_transfer_encoding off;
       proxy_buffering off;
       proxy_cache off;
+      proxy_read_timeout 600s;
 
-      proxy_pass http://validator_service:4567/;
+      proxy_pass http://hl7_validator_service:3500/;
     }
   }
 }

config/nginx.conf

@@ -68,7 +68,37 @@ http {
       proxy_pass http://inferno:4567;
     }
 
-    location /validator {
+#    location /validator {
+#      proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+#      proxy_set_header Host $http_host;
+#      proxy_set_header X-Forwarded-Proto $scheme;
+#      proxy_set_header X-Forwarded-Port $server_port;
+#      proxy_redirect off;
+#      proxy_set_header Connection '';
+#      proxy_http_version 1.1;
+#      chunked_transfer_encoding off;
+#      proxy_buffering off;
+#      proxy_cache off;
+#
+#      proxy_pass http://fhir_validator_app;
+#    }
+
+#    location /validatorapi/ {
+#      proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+#      proxy_set_header Host $http_host;
+#      proxy_set_header X-Forwarded-Proto $scheme;
+#      proxy_set_header X-Forwarded-Port $server_port;
+#      proxy_redirect off;
+#      proxy_set_header Connection '';
+#      proxy_http_version 1.1;
+#      chunked_transfer_encoding off;
+#      proxy_buffering off;
+#      proxy_cache off;
+#
+#      proxy_pass http://validator_service:4567/;
+#    }
+
+    location /hl7validatorapi/ {
       proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
       proxy_set_header Host $http_host;
       proxy_set_header X-Forwarded-Proto $scheme;
@@ -79,23 +109,9 @@ http {
       chunked_transfer_encoding off;
       proxy_buffering off;
       proxy_cache off;
+      proxy_read_timeout 600s;
 
-      proxy_pass http://fhir_validator_app;
-    }
-
-    location /validatorapi/ {
-      proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
-      proxy_set_header Host $http_host;
-      proxy_set_header X-Forwarded-Proto $scheme;
-      proxy_set_header X-Forwarded-Port $server_port;
-      proxy_redirect off;
-      proxy_set_header Connection '';
-      proxy_http_version 1.1;
-      chunked_transfer_encoding off;
-      proxy_buffering off;
-      proxy_cache off;
-
-      proxy_pass http://validator_service:4567/;
+      proxy_pass http://hl7_validator_service:3500/;
     }
   }
 }

docker-compose.background.yml

@@ -1,26 +1,37 @@
 version: '3'
 services:
-  validator_service:
-    image: infernocommunity/fhir-validator-service
-    # Update this path to match your directory structure
+  hl7_validator_service:
+    image: infernocommunity/inferno-resource-validator
+    environment:
+      # Defines how long validator sessions last if unused, in minutes:
+      # Negative values mean sessions never expire, 0 means sessions immediately expire
+      SESSION_CACHE_DURATION: -1
     volumes:
       - ./lib/davinci_plan_net_test_kit/igs:/home/igs
-  fhir_validator_app:
-    image: infernocommunity/fhir-validator-app
-    depends_on:
-      - validator_service
-    environment:
-      EXTERNAL_VALIDATOR_URL: http://localhost/validatorapi
-      VALIDATOR_BASE_PATH: /validator
+      # To let the service share your local FHIR package cache,
+      # uncomment the below line
+      # - ~/.fhir:/home/ktor/.fhir
+  # validator_service:
+  #   image: infernocommunity/fhir-validator-service
+  #   # Update this path to match your directory structure
+  #   volumes:
+  #     - ./lib/davinci_plan_net_test_kit/igs:/home/igs
+  # fhir_validator_app:
+  #   image: infernocommunity/fhir-validator-app
+  #   depends_on:
+  #     - validator_service
+  #   environment:
+  #     EXTERNAL_VALIDATOR_URL: http://localhost/validatorapi
+  #     VALIDATOR_BASE_PATH: /validator
   nginx:
     image: nginx
     volumes:
       - ./config/nginx.background.conf:/etc/nginx/nginx.conf
     ports:
       - "80:80"
     command: [nginx, '-g', 'daemon off;']
-    depends_on:
-      - fhir_validator_app
+    # depends_on:
+    #   - fhir_validator_app
   redis:
     image: redis
     ports:

docker-compose.yml

@@ -6,7 +6,8 @@ services:
     volumes:
       - ./data:/opt/inferno/data
     depends_on:
-      - validator_service
+      - hl7_validator_service
+      # - validator_service
   worker:
     build:
       context: ./
@@ -15,14 +16,18 @@ services:
     command: bundle exec sidekiq -r ./worker.rb
     depends_on:
       - redis
-  validator_service:
+  hl7_validator_service:
     extends:
       file: docker-compose.background.yml
-      service: validator_service
-  fhir_validator_app:
-    extends:
-      file: docker-compose.background.yml
-      service: fhir_validator_app
+      service: hl7_validator_service
+  # validator_service:
+  #   extends:
+  #     file: docker-compose.background.yml
+  #     service: validator_service
+  # fhir_validator_app:
+  #   extends:
+  #     file: docker-compose.background.yml
+  #     service: fhir_validator_app
   nginx:
     extends:
       file: docker-compose.background.yml

lib/davinci_plan_net_test_kit/generated/v1.1.0/davinci_plan_net_test_suite.rb

@@ -141,6 +141,7 @@ class DaVinciPlanNetServerTestSuite < Inferno::TestSuite
         %r{Sub-extension url 'introspect' is not defined by the Extension http://fhir-registry\.smarthealthit\.org/StructureDefinition/oauth-uris},
         %r{Sub-extension url 'revoke' is not defined by the Extension http://fhir-registry\.smarthealthit\.org/StructureDefinition/oauth-uris},
         /Observation\.effective\.ofType\(Period\): .*us-core-1:/, # Invalid invariant in US Core v3.1.1
+        /\A\S+: \S+: URL value '.*' does not resolve/,
       ].freeze
 
       VERSION_SPECIFIC_MESSAGE_FILTERS = [].freeze
@@ -151,8 +152,11 @@ def self.metadata
           end
       end
 
-      validator do
-        url ENV.fetch('V110_VALIDATOR_URL', 'http://validator_service:4567')
+      id :davinci_plan_net_server_v110
+
+      fhir_resource_validator do
+        igs 'hl7.fhir.us.davinci-pdex-plan-net#1.1.0'
+
         message_filters = VALIDATION_MESSAGE_FILTERS + VERSION_SPECIFIC_MESSAGE_FILTERS
 
         exclude_message do |message|
@@ -180,8 +184,6 @@ def self.metadata
         }
       ]
 
-      id :davinci_plan_net_server_v110
-
       input :url,
         title: 'FHIR Endpoint',
         description: 'URL of the FHIR endpoint'

lib/davinci_plan_net_test_kit/generator/suite_generator.rb

@@ -53,8 +53,9 @@ def title
         "DaVinci Plan Net #{ig_metadata.ig_version} Server Test Suite"
       end
 
-      def validator_env_name
-        "#{ig_metadata.reformatted_version.upcase}_VALIDATOR_URL"
+      def ig_identifier
+        version = ig_metadata.ig_version[1..] # Remove leading 'v'
+        "hl7.fhir.us.davinci-pdex-plan-net##{version}"
       end
 
       def ig_link

lib/davinci_plan_net_test_kit/generator/templates/suite.rb.erb

@@ -133,6 +133,7 @@ module DaVinciPlanNetTestKit
         %r{Sub-extension url 'introspect' is not defined by the Extension http://fhir-registry\.smarthealthit\.org/StructureDefinition/oauth-uris},
         %r{Sub-extension url 'revoke' is not defined by the Extension http://fhir-registry\.smarthealthit\.org/StructureDefinition/oauth-uris},
         /Observation\.effective\.ofType\(Period\): .*us-core-1:/, # Invalid invariant in US Core v3.1.1
+        /\A\S+: \S+: URL value '.*' does not resolve/,
       ].freeze
 
       VERSION_SPECIFIC_MESSAGE_FILTERS = <%=version_specific_message_filters%>.freeze
@@ -143,8 +144,11 @@ module DaVinciPlanNetTestKit
           end
       end
 
-      validator do
-        url ENV.fetch('<%= validator_env_name %>', 'http://validator_service:4567')
+      id :<%= suite_id %>
+
+      fhir_resource_validator do
+        igs '<%= ig_identifier %>'
+
         message_filters = VALIDATION_MESSAGE_FILTERS + VERSION_SPECIFIC_MESSAGE_FILTERS
 
         exclude_message do |message|
@@ -172,8 +176,6 @@ module DaVinciPlanNetTestKit
         }
       ]
 
-      id :<%= suite_id %>
-
       input :url,
         title: 'FHIR Endpoint',
         description: 'URL of the FHIR endpoint'

lib/davinci_plan_net_test_kit/igs/README.md

@@ -0,0 +1,21 @@
+# Note on this IGs folder
+
+There are three reasons why it would be necessary to put an IG package.tgz in this folder. If none of these apply, you do not need to put any files here, or can consider removing any existing files to make it clear they are unused.
+
+## 1. Generated Test Suites
+Some test kits use a "generator" to automatically generate the contents of a test suite for an IG. The IG files are required every time the test suites need to be regenerated. Examples of test kits that use this approach are the US Core Test Kit and CARIN IG for Blue Button® Test Kit.
+
+
+## 2. Non-published IG
+If your IG, or the specific version of the IG you want to test against, is not published, then the validator service needs to load the IG from file in order to be able to validate resources with it. The IG must be referenced in the `fhir_resource_validator` block in the test suite definition by filename, ie:
+
+```ruby
+      fhir_resource_validator do
+        igs 'igs/filename.tgz'
+
+        ...
+      end
+```
+
+## 3. Inferno Validator UI
+The Inferno Validator UI is configured to auto-load any IGs present in the igs folder and include them in all validations. The Inferno Validator UI is currently disabled by default, so this is only relevant if you choose to re-enable it. In general, the Inferno team is currently leaving IGs in this folder even if not otherwise necessary to make it easy to re-enable the validator UI.

lib/davinci_plan_net_test_kit/reference_resolution_test.rb

@@ -165,7 +165,8 @@ def resource_is_valid_with_target_profile?(resource, target_profile)
 
       target_profile_with_version =  target_profile.include?('|') ? target_profile : "#{target_profile}|#{metadata.profile_version}"
 
-      outcome = FHIR::OperationOutcome.new(JSON.parse(validator.validate(resource, target_profile_with_version)))
+      validator_response = validator.validate(resource, target_profile_with_version)
+      outcome = validator.operation_outcome_from_hl7_wrapped_response(validator_response)
 
       message_hashes = outcome.issue&.map { |issue| validator.message_hash_from_issue(issue, resource) } || []