flattened conditions, added new code table (#105)

* flattened conditions, added new code table * moved var, included all as default * docs * tweak to column name
smart-on-fhir · Aug 16, 2023 · 31a3888 · 31a3888
1 parent b78b480
commit 31a3888
Show file tree

Hide file tree

Showing 9 changed files with 224 additions and 91 deletions.
diff --git a/cumulus_library/studies/core/builder_condition_codeableconcept.py b/cumulus_library/studies/core/builder_condition_codeableconcept.py
@@ -17,18 +17,27 @@ def prepare_queries(self, cursor: object, schema: str):
         :param schema: the schema/db name, matching the cursor
 
         """
-        config = CodeableConceptConfig(
+        preferred_config = CodeableConceptConfig(
             source_table="condition",
             source_id="id",
-            cc_column={
-                "name": "code",
-                "is_array": False,
-                "code_systems": [
-                    "http://snomed.info/sct",
-                    "http://hl7.org/fhir/sid/icd-10-cm",
-                    "http://hl7.org/fhir/sid/icd-9-cm",
-                ],
-            },
-            target_table="core__condition_codable_concepts",
+            column_name="code",
+            is_array=False,
+            target_table="core__condition_codable_concepts_display",
+            filter_priority=True,
+            code_systems=[
+                "http://snomed.info/sct",
+                "http://hl7.org/fhir/sid/icd-10-cm",
+                "http://hl7.org/fhir/sid/icd-9-cm",
+            ],
         )
-        self.queries.append(get_codeable_concept_denormalize_query(config))
+        self.queries.append(get_codeable_concept_denormalize_query(preferred_config))
+
+        all_config = CodeableConceptConfig(
+            source_table="condition",
+            source_id="id",
+            column_name="code",
+            is_array=False,
+            target_table="core__condition_codable_concepts_all",
+            filter_priority=False,
+        )
+        self.queries.append(get_codeable_concept_denormalize_query(all_config))
diff --git a/cumulus_library/studies/core/builder_core_medication.py b/cumulus_library/studies/core/builder_core_medication.py
@@ -3,7 +3,6 @@
 from cumulus_library.base_table_builder import BaseTableBuilder
 from cumulus_library.helper import get_progress_bar, query_console_output
 from cumulus_library.template_sql.templates import (
-    CodeableConceptConfig,
     get_core_medication_query,
     get_is_table_not_empty_query,
     get_column_datatype_query,

diff --git a/cumulus_library/studies/core/builder_encounter_coding.py b/cumulus_library/studies/core/builder_encounter_coding.py
@@ -51,11 +51,11 @@ def _check_data_in_fields(self, code_sources: list[dict], schema, cursor) -> dic
             for code_source in code_sources:
                 if code_source["is_array"]:
                     code_source["has_data"] = is_codeable_concept_array_populated(
-                        schema, "encounter", code_source["name"], cursor
+                        schema, "encounter", code_source["column_name"], cursor
                     )
                 else:
                     code_source["has_data"] = is_codeable_concept_populated(
-                        schema, "encounter", code_source["name"], cursor
+                        schema, "encounter", code_source["column_name"], cursor
                     )
                 progress.advance(task)
         return code_sources
@@ -70,8 +70,9 @@ def prepare_queries(self, cursor: object, schema: str):
 
         code_sources = [
             {
-                "name": "type",
+                "column_name": "type",
                 "is_array": True,
+                "filter_priority": True,
                 "code_systems": [
                     "http://terminology.hl7.org/CodeSystem/encounter-type",
                     "http://terminology.hl7.org/CodeSystem/v2-0004",
@@ -81,8 +82,9 @@ def prepare_queries(self, cursor: object, schema: str):
                 "has_data": False,
             },
             {
-                "name": "servicetype",
+                "column_name": "servicetype",
                 "is_array": False,
+                "filter_priority": True,
                 "code_systems": [
                     "http://terminology.hl7.org/CodeSystem/service-type",
                     "urn:oid:2.16.840.1.113883.4.642.3.518",
@@ -91,8 +93,9 @@ def prepare_queries(self, cursor: object, schema: str):
                 "has_data": False,
             },
             {
-                "name": "priority",
+                "column_name": "priority",
                 "is_array": False,
+                "filter_priority": True,
                 "code_systems": [
                     "http://terminology.hl7.org/CodeSystem/v3-ActPriority",
                     "http://snomed.info/sct",
@@ -106,15 +109,18 @@ def prepare_queries(self, cursor: object, schema: str):
                 config = CodeableConceptConfig(
                     source_table="encounter",
                     source_id="id",
-                    cc_column=code_source,
-                    target_table=f"core__encounter_dn_{code_source['name']}",
+                    column_name=code_source["column_name"],
+                    is_array=code_source["is_array"],
+                    filter_priority=code_source["filter_priority"],
+                    code_systems=code_source["code_systems"],
+                    target_table=f"core__encounter_dn_{code_source['column_name']}",
                 )
                 self.queries.append(get_codeable_concept_denormalize_query(config))
             else:
                 self.queries.append(
                     get_ctas_empty_query(
                         schema_name=schema,
-                        table_name=f"core__encounter_dn_{code_source['name']}",
+                        table_name=f"core__encounter_dn_{code_source['column_name']}",
                         table_cols=["id", "code", "code_system", "display"],
                     )
                 )
diff --git a/cumulus_library/studies/core/condition.sql b/cumulus_library/studies/core/condition.sql
@@ -2,37 +2,43 @@
 -- Condition
 -- https://build.fhir.org/ig/HL7/US-Core/StructureDefinition-us-core-condition-encounter-diagnosis.html
 -- 
---Each Condition must have:
+-- Each Condition must have:
 --    a category code of “problem-list-item” or “health-concern”
---    a code that identifies the condition
+--    a code that identifies the condition (this is available in 
+--        core__condition_codable_concepts_all)
 --    a patient
---Each Condition must support:
+-- Each Condition must support:
 --    a clinical status of the condition (e.g., active or resolved)
 --    a verification status
 --    a category code of ‘sdoh’
 --    a date of diagnosis*
 --    abatement date (in other words, date of resolution or remission)
 --    a date when recorded
 
-
 CREATE TABLE core__condition AS
 WITH temp_condition AS (
     SELECT
         c.category,
-        c.code,
         c.clinicalstatus,
+        cca.code,
+        cca.code_system,
+        cca.display,
         c.verificationstatus,
         c.subject.reference AS subject_ref,
         c.encounter.reference AS encounter_ref,
         c.id AS condition_id,
         date(from_iso8601_timestamp(c.recordeddate)) AS recordeddate,
         concat('Condition/', c.id) AS condition_ref
     FROM condition AS c
+    INNER JOIN core__condition_codable_concepts_all AS cca ON c.id = cca.id
 )
 
 SELECT
-    t_category_coding.category_row AS category,
-    tc.code AS cond_code,
+    t_category_coding.category_row.code AS category_code,
+    t_category_coding.category_row.display AS category_display,
+    tc.code,
+    tc.code_system,
+    tc.code_display,
     tc.subject_ref,
     tc.encounter_ref,
     tc.condition_id,
@@ -42,9 +48,9 @@ SELECT
     date_trunc('month', date(tc.recordeddate)) AS recorded_month,
     date_trunc('year', date(tc.recordeddate)) AS recorded_year
 FROM temp_condition AS tc,
-    unnest(category) AS t_category (category_coding), --noqa
-    unnest(category_coding.coding) AS t_category_coding (category_row), --noqa
-    unnest(code.coding) AS t_coding (code_row) --noqa
+    unnest(category) AS t_category (category_coding),
+    unnest(category_coding.coding) AS t_category_coding (category_row)
+
 WHERE tc.recordeddate BETWEEN date('2016-01-01') AND current_date;
 
 -- ###########################################################################
@@ -53,14 +59,14 @@ WHERE tc.recordeddate BETWEEN date('2016-01-01') AND current_date;
 
 CREATE TABLE core__count_condition_month AS WITH
 concept_map AS (
-    SELECT
+    SELECT DISTINCT
         c.recorded_month AS cond_month,
         c.subject_ref,
         coalesce(c.encounter_ref, 'None') AS encounter_ref,
         coalesce(mapping.display, 'None') AS cond_code_display,
-        c.category.code AS cond_category_code
+        c.category_code AS cond_category_code
     FROM core__condition AS c
-    LEFT JOIN core__condition_codable_concepts AS mapping
+    LEFT JOIN core__condition_codable_concepts_display AS mapping
         ON c.condition_id = mapping.id
 ),
 

diff --git a/cumulus_library/studies/core/condition_codeable_concept.sql b/cumulus_library/studies/core/condition_codeable_concept.sql
@@ -7,10 +7,11 @@ in the future change the priority order of concept systems, or add
 additional systems to support other implementations if we run into
 unusual data in the wild.
 */
-CREATE TABLE core__condition_codable_concepts AS (
-    WITH
 
-    system_0 AS (
+# filtering case
+CREATE TABLE core__condition_codable_concepts_preferred AS (
+    WITH
+    system_code_0 AS (
         SELECT DISTINCT
             s.id AS id,
             '0' AS priority,
@@ -19,12 +20,11 @@ CREATE TABLE core__condition_codable_concepts AS (
             u.codeable_concept.system AS code_system
         FROM
             condition AS s,
-            UNNEST(s.code.coding) AS u (codeable_concept) --noqa: AL05
+            UNNEST(s.code.coding) AS u (codeable_concept)
         WHERE
             u.codeable_concept.system = 'http://snomed.info/sct'
     ), --noqa: LT07
-
-    system_1 AS (
+    system_code_1 AS (
         SELECT DISTINCT
             s.id AS id,
             '1' AS priority,
@@ -33,12 +33,11 @@ CREATE TABLE core__condition_codable_concepts AS (
             u.codeable_concept.system AS code_system
         FROM
             condition AS s,
-            UNNEST(s.code.coding) AS u (codeable_concept) --noqa: AL05
+            UNNEST(s.code.coding) AS u (codeable_concept)
         WHERE
             u.codeable_concept.system = 'http://hl7.org/fhir/sid/icd-10-cm'
     ), --noqa: LT07
-
-    system_2 AS (
+    system_code_2 AS (
         SELECT DISTINCT
             s.id AS id,
             '2' AS priority,
@@ -47,7 +46,7 @@ CREATE TABLE core__condition_codable_concepts AS (
             u.codeable_concept.system AS code_system
         FROM
             condition AS s,
-            UNNEST(s.code.coding) AS u (codeable_concept) --noqa: AL05
+            UNNEST(s.code.coding) AS u (codeable_concept)
         WHERE
             u.codeable_concept.system = 'http://hl7.org/fhir/sid/icd-9-cm'
     ), --noqa: LT07
@@ -59,43 +58,78 @@ CREATE TABLE core__condition_codable_concepts AS (
             code_system,
             code,
             display
-        FROM system_0
+        FROM system_code_0
         UNION
         SELECT
             id,
             priority,
             code_system,
             code,
             display
-        FROM system_1
+        FROM system_code_1
         UNION
         SELECT
             id,
             priority,
             code_system,
             code,
             display
-        FROM system_2
-        ORDER BY id, priority
-    )
+        FROM system_code_2
+    ),
 
-    SELECT
-        id,
-        code,
-        code_system,
-        display
-    FROM (
+    partitioned_table AS (
         SELECT
             id,
             code,
             code_system,
             display,
+            priority,
             ROW_NUMBER()
             OVER (
                 PARTITION BY id
             ) AS available_priority
         FROM union_table
-        GROUP BY id, code_system, code, display
+        GROUP BY id, priority, code_system, code, display
+        ORDER BY priority ASC
     )
+
+    SELECT
+        id,
+        code,
+        code_system,
+        display
+    FROM partitioned_table
     WHERE available_priority = 1
-);
+);
+
+#non-filtering case
+
+CREATE TABLE target__concepts AS (
+    WITH
+    system_code_col_0 AS (
+        SELECT DISTINCT
+            s.id AS id,
+            u.codeable_concept.code AS code,
+            u.codeable_concept.display AS display,
+            u.codeable_concept.system AS code_system
+        FROM
+            source AS s,
+            UNNEST(s.code_col) AS cc (cc_row),
+            UNNEST(cc.cc_row.coding) AS u (codeable_concept)
+    ), --noqa: LT07
+
+    union_table AS (
+        SELECT
+            id,
+            code_system,
+            code,
+            display
+        FROM system_code_col_0
+    )
+    SELECT
+        id,
+        code,
+        code_system,
+        display
+    FROM union_table
+);