Merge remote-tracking branch 'refs/remotes/origin/master' into expect…

…-columns-test

README.md

@@ -64,6 +64,10 @@ components of a test case file are:
 - **Description** (`description` attribute): A detailed explanation of what the test case aims to
   validate, including any relevant context or specifications that the test is
   based on.
+- **FHIR version**: A list of FHIR version strings like `['4.0.1', '5.0.0']`.
+  This applies to all FHIR resources in the test suite. The version numbers come
+  from [this ValueSet](https://hl7.org/fhir/R5/valueset-FHIR-version.html) and
+  can only include "Release" versions.
 - **Fixtures** (`resources` attribute): A set of FHIR resources that serve as
   input data for the test. These fixtures are essential for setting up the test
   environment and conditions.
@@ -83,26 +87,27 @@ Below is an abstract representation of what a test case file might look like:
 ```js
 {
   // unique name of test
-  title: 'title',
-  description: '...',
+  'title': 'title',
+  'description': '...',
+  'fhirVersion': ['5.0.0', '4.0.1'],
   // fixtures
-  resources: [
-    {resourceType: 'Patient', id: 'pt-1'},
-    {resourceType: 'Patient', id: 'pt-2'}
+  'resources': [
+    {'resourceType': 'Patient', 'id': 'pt-1'},
+    {'resourceType': 'Patient', 'id': 'pt-2'}
   ]
-  tests: [
+  'tests': [
     ...
     {
-      title: 'title of test case',
+      'title': 'title of test case',
       // ViewDefintion
-      view: {
-       select: [
-        {column: [{name: 'id', path: 'id'}]}
-      ]},
+      'view': {
+        'select': [
+          {'column': [{'name': 'id', 'path': 'id'}]}
+        ]},
       // expected result
-      expect: [
-        {id: 'pt-1'},
-        {id: 'pt-2'}
+      'expect': [
+        {'id': 'pt-1'},
+        {'id': 'pt-2'}
       ]
     }
     ...
@@ -150,37 +155,36 @@ runner:
 
 The test runner should produce a `test_report.json`
 file containing the results of the test executions. The structure of the test
-report should mirror that of the original test cases, with an additional
-attribute `result` added to each test object. This attribute should contain the
-set of rows returned by the implementation when evaluating the test. Ensure
-the result accurately reflects the output of your implementation for each
-test, facilitating a straightforward comparison between expected and actual
-outcomes.
+report is a map where:
+- each key is the name of a test file,
+- each value is a map with a single `tests` list,
+- each element of the `tests` list has a `name` and a `result` field, reporting
+  whether the `name` test `passed` or not. Beside `passed`, the `result` map 
+  may also have a `reason` text field describing why the test did not pass.
+Here is an example:
 
 ```js
 //example test_report.json
 {
-  "title": "Example Test Case",
-  "description": "This test case validates...",
-  "resources": [...],
-  "tests": [
-    {
-      "title": "Test Object 1",
-      "view": {...},
-      "expect": [...],
-      "result": [
-        // Actual rows returned by your implementation
-      ]
-    },
-    {
-      "title": "Test Object 2",
-      "view": {...},
-      "expect": [...],
-      "result": [
-        // Actual rows returned by your implementation
-      ]
-    }
-  ]
+  "logic.json": {
+    "tests": [
+      {
+        "name": "filtering with 'and'",
+        "result": {
+          "passed": true
+        }
+      },
+      {
+        "name": "filtering with 'or'",
+        "result": {
+          "passed": false,
+          "reason": "skipped"
+        }
+      },
+      ...
+    ]
+  },
+  ...
 }
 ```
 
@@ -231,9 +235,10 @@ the process of publishing your test report and registering your implementation.
 ### Registering Your Implementation
 
 Once your test report is published and accessible, the final step is to register
-your implementation in the `./implementations.json` file. This file serves as a
-registry of available implementations and their test results, facilitating
-discovery and comparison.
+your implementation in the
+[`test_report/public/implementations.json`](test_report/public/implementations.json)
+file. This file serves as a registry of available implementations and their test
+results, facilitating discovery and comparison.
 
 1. **Format of `implementations.json`**:
    - The `implementations.json` file is a JSON document that lists implementations along with URLs to their test reports.

input/fsh/examples.fsh

@@ -23,6 +23,43 @@ Usage:  #example
     * path = "family"
     * name = "family_name"
 
+Instance:   ShareablePatientDemographics
+InstanceOf: ViewDefinition
+Description: """
+Creates the same view as the 'PatientDemographics' example, but applies both the
+ShareableViewDefinition and TabularViewDefinition profiles.
+"""
+Usage:  #example
+* url = "http://example.com/ShareablePatientDemographics"
+* name = "patient_demographics"
+* fhirVersion[+] = #4.0
+* fhirVersion[+] = #5.0
+* status = #draft
+* resource = #Patient
+* meta
+  * profile[+] = "http://hl7.org/fhir/uv/sql-on-fhir/StructureDefinition/ShareableViewDefinition"
+  * profile[+] = "http://hl7.org/fhir/uv/sql-on-fhir/StructureDefinition/TabularViewDefinition"
+* select[+]
+  * column[+]
+    * path = "getResourceKey()"
+    * name = "id"
+    * type = "string"
+  * column[+]
+    * path = "gender"
+    * name = "gender"
+    * type = "string"
+* select[+]
+  * forEach = "name.where(use = 'official').first()"
+  * column[+]
+    * path = "given.join(' ')"
+    * name = "given_name"
+    * description = "A single given name field with all names joined together."
+    * type = "string"
+  * column[+]
+    * path = "family"
+    * name = "family_name"
+    * type = "string"
+
 Instance:   PatientAddresses
 InstanceOf: ViewDefinition
 Description: """An example of unnesting patient addresses into multiple
@@ -119,7 +156,7 @@ Usage: #example
     * path = "getResourceKey()"
     * name = "id"
   * column[+]
-    * path = "subject.getReferenceKey('Patient')"
+    * path = "subject.getReferenceKey(Patient)"
     * name = "patient_id"
     * description = "Can be used to join to patient tables created by other views."
   * column[+]
@@ -154,4 +191,4 @@ Usage: #example
     * name = "dbp_quantity_value"
     * path = "value.ofType(Quantity).value"
 * where[+]
-  * path = "code.coding.exists(system='http://loinc.org' and code=%bp_code)"
+  * path = "code.coding.exists(system='http://loinc.org' and code=%bp_code)"

input/fsh/profiles.fsh

@@ -1,7 +1,41 @@
+Invariant: no-collections
+Description: """
+Tabular view columns must not be collections.
+"""
+Severity: #error
+Expression: "collection.empty() or collection = false"
+
+Invariant: primitives-only
+Description: """
+Tabular view columns only contain primitive values.
+"""
+Severity: #error
+Expression: "type in ('base64Binary' | 'boolean' | 'canonical' | 'code' | 'dateTime' | 'decimal' | 'id' | 'instant' | 'integer' | 'integer64' | 'markdown' | 'oid' | 'string' | 'positiveInt' | 'time' | 'unsignedInt' | 'url' | 'uuid')"
+
 Profile: ShareableViewDefinition
+Title: "Shareable View Definition"
 Parent: ViewDefinition
-Description: "A Shareable View Definition"
+Description: """
+A profile for View Definitions intended to be shared between multiple systems. This requires there
+be a defined URL, name, and version. Also, each column must have specified type so consuming
+systems 
+
+Shareable View Definitions often also use the TabularViewDefinition profile, requiring
+that the view be shareable and contain only tabular values, as is common in many databases
+and analytic tools.
+"""
 * url 1..1
 * name 1..1
 * fhirVersion 1..*
 * select.column.type 1..1
+
+Profile: TabularViewDefinition
+Title: "Tabular View Definition"
+Parent: ViewDefinition
+Description: """
+A profile for View Definitions where each resulting field must contain only a simple scalar value.
+This is sometimes referred to as 'CSV Mode', but applies to any system that explicitly constrains its
+views or tables to tabluar data.
+"""
+* select.column obeys no-collections
+* select.column obeys primitives-only

input/pagecontent/StructureDefinition-ViewDefinition-notes.md

@@ -128,9 +128,9 @@ A view of  *Observation* resources would then have its own row key and a foreign
         "name": "id"
       },
       {
-        // The 'Patient' parameter is optional, but ensures the returned value
+        // The `Patient` parameter is optional, but ensures the returned value
         // will either be a patient row key or null.
-        "path": "subject.getReferenceKey('Patient')",
+        "path": "subject.getReferenceKey(Patient)",
         "name": "patient_id"
       }
     ]

input/pagecontent/index.md

@@ -64,6 +64,31 @@ Such tabular views can be created for any FHIR resource, with
 [View Definition page](StructureDefinition-ViewDefinition.html) for the full definition
 of the above structure.
 
+### View Definition non-goals
+View Definitions are intentionally constrained to a narrow set of functionality to make them easily
+and broadly implementable, while deferring higher-level capabilities to database engines or processing
+pipelines that solve those problems well. Therefore it's important to know what View Definitions
+do *not* do, by design:
+
+#### A single View Definition will not join different resources in any way
+A single View Definition defines a tabular view of exactly one resource type, like a view of `Patient`
+or a view of `Condition` resources. Any joins between resources are exclusively in downstream systems,
+like between database tables computed by view definitions. This makes it possible for a wide set of
+FHIR infrastructure to implement this spec, and lets database engines or processing pipelines join as
+needed.
+
+#### View Definitions do not have sorting, aggregation, or limit capabilities
+View Definitions define only the logical schema of views, and therefore defer sorting, aggergation or limit
+operations to engines, along with cross-view joins. *View Runners* (described below) or future FHIR
+server operations may accept limits or sort columns as part of their operations, so users at runtime can
+specify what they need dynamically and independently of the definition of the view itself.
+
+#### View Definitions are not aware of output formats
+View Definitions themselves are independent of any tech stack and therefore unaware of the output format.
+*View Runners* are the component that applies definitions to a particular stack, producing output like
+a database table, Parquet file, CSV, or another format specific to the runner.
+
+
 ### System Layers
 
 The [View Definition](StructureDefinition-ViewDefinition.html) is the central element of this
@@ -117,6 +142,18 @@ Example view runners may include:
 * A runner that queries FHIR JSON directly and creates a table in a web application
 * A runner that loads data directly into a notebook or other data analysis tool
 
+#### Generating Schemas
+The output of many runners will have technology-specific schemas, such as database table
+definitions or schema for structured files like Parquet. This will be runner- and technology-
+specific, but runner implementaitons SHOULD offer a way to compute that schema from a ViewDefinition
+when applicable.
+
+For example, a runner that produces a table in a database system could return a "CREATE TABLE" or
+"CREATE VIEW" statement based on the ViewDefinition, allowing the system to define tables prior to 
+populating them by evaluating the views over data.
+
+This would not apply to outputs that do not have common a schema specification, like CSV files. 
+
 ### The Analytics Layer
 
 Finally, users must be able to easily leverage the above views with the analytic

test_report/README.md

@@ -5,11 +5,11 @@
 Install the dependencies...
 
 ```bash
-cd svelte-app
 npm install
 ```
 
-...then start [Rollup](https://rollupjs.org):
+...then start [Rollup](https://rollupjs.org) (you need to have
+[Bun](https://bun.sh/docs/installation) installed as well) :
 
 ```bash
 npm run dev

test_report/public/implementations.json

@@ -9,7 +9,7 @@
     "name": "Aidbox",
     "url": "https://docs.aidbox.app/storage-1/sql-on-fhir",
     "description": "FHIR++ Platform by Health Samurai",
-    "testResultsUrl": "https://storage.googleapis.com/aidbox-public/sof-test-results.json"
+    "testResultsUrl": "https://storage.googleapis.com/aidbox-public/aidbox-sof-test-results.json"
   },
   {
     "name": "Pathling",
@@ -18,13 +18,20 @@
     "testResultsUrl": "https://pathling.csiro.au/test-results/refs/heads/issue/1759/sof-test-results.json"
   },
   {
-    "name": "Google",
-    "url": "TBD",
-    "description": "TBD"
+    "name": "Google (Open Health Stack)",
+    "url": "https://developers.google.com/open-health-stack/fhir-analytics/data-pipes",
+    "description": "OHS's fhir-data-pipes offers an option to use ViewDefinitions for creating 'flat tables' from FHIR resources. The actual implementation is in this package: https://github.com/google/fhir-data-pipes/tree/master/pipelines/common/src/main/java/com/google/fhir/analytics/view",
+    "testResultsUrl": "https://raw.githubusercontent.com/google/fhir-data-pipes/master/pipelines/common/src/test/resources/sql-on-fhir-v2-test-result.json"
   },
   {
     "name": "Grahame Impl",
     "url": "TBD",
     "description": "TBD"
+  },
+  {
+    "name": "NCQA Fhir Analytics",
+    "url": "https://github.com/ncqa-org/fhir-analytics/tree/main",
+    "description": "TBD",
+    "testResultsUrl": "https://raw.githubusercontent.com/ncqa-org/fhir-analytics/fca5934a937ac399237391d28a2dfeac70be8565/test_results.json"
   }
 ]

test_report/src/Impls.svelte

@@ -1,6 +1,6 @@
 <script>
  import { onMount } from "svelte";
- import implementations from '../../implementations.json';
+ import implementations from '../public/implementations.json';
  import tests from '../public/tests.json';
 
  async function load(file){

tests/basic.json

@@ -1,6 +1,7 @@
 {
   "title": "basic",
   "description": "basic view definition",
+  "fhirVersion": ["5.0.0", "4.0.1", "3.0.2"],
   "resources": [
     {
       "resourceType": "Patient",