Merge pull request #46 from thehyve/dataloading-fixes

Dataloading fixes
thehyve · Jul 24, 2017 · 2f0d1f0 · 2f0d1f0
2 parents 3ba799b + 493da6c
commit 2f0d1f0
Show file tree

Hide file tree

Showing 8 changed files with 53 additions and 21 deletions.
diff --git a/transmart-batch/src/main/groovy/org/transmartproject/batch/concept/ConceptTree.groovy b/transmart-batch/src/main/groovy/org/transmartproject/batch/concept/ConceptTree.groovy
@@ -28,6 +28,9 @@ class ConceptTree {
     @Value("#{jobParameters['TOP_NODE']}")
     ConceptPath topNodePath
 
+    @Value("#{jobParameters['STUDY_ID']}")
+    String studyId
+
     @Autowired
     private SequenceReserver reserver
 
@@ -162,6 +165,12 @@ class ConceptTree {
             node.conceptPath = variable.conceptPath
             node.code = variable.conceptCode
             node.ontologyNode = variable.ontologyNode
+        } else if (isStudyNode(node)) {
+            // For compatibility with transmartApp data export,
+            // there needs to be a 'study concept'
+            node.conceptPath = path
+            node.conceptName = studyId
+            node.code = studyId
         }
         log.debug("Generated new concept node: $path")
         nodeMap[path] = node

diff --git a/...art-batch/src/main/groovy/org/transmartproject/batch/concept/InsertConceptsService.groovy b/...art-batch/src/main/groovy/org/transmartproject/batch/concept/InsertConceptsService.groovy
@@ -120,7 +120,7 @@ class InsertConceptsService {
                 c_columnname: 'concept_path',
                 c_columndatatype: 'T',
                 c_operator: 'LIKE',
-                c_dimcode: node.conceptPath?.toString() ?: '',
+                c_dimcode: node.conceptPath?.toString() ?: '@',
             ]
         }
         tableAccessInsert.executeBatch(tableAccessRows as Map[])
@@ -139,6 +139,12 @@ class InsertConceptsService {
         newConcepts.each {
             String visualAttributes = visualAttributesFor(it)
             boolean isStudyNode = topNode.path == it.path.path
+            /* For compatibility with transmartApp, also intermediate nodes
+               are treated as concept nodes, in order to be able to select
+               subtrees based on a concept path prefix.
+               The c_comment column is only for compatibility with transmartApp
+               and should not have a role in security checks in the future.
+            */
             Map i2b2Row = [
                     c_hlevel          : it.level,
                     c_fullname        : it.path.toString(),
@@ -147,25 +153,27 @@ class InsertConceptsService {
                     c_synonym_cd      : 'N',
                     c_visualattributes: visualAttributes,
                     c_metadataxml     : metadataXmlFor(it),
-                    c_facttablecolumn : isStudyNode ? '@' : 'CONCEPT_CD',
-                    c_tablename       : isStudyNode ? 'STUDY' : 'CONCEPT_DIMENSION',
-                    c_columnname      : isStudyNode ? 'STUDY_ID' : 'CONCEPT_PATH',
+                    c_facttablecolumn : 'CONCEPT_CD',
+                    c_tablename       : 'CONCEPT_DIMENSION',
+                    c_columnname      : 'CONCEPT_PATH',
                     c_columndatatype  : 'T',
-                    c_operator        : isStudyNode ? '=' : 'LIKE',
-                    c_dimcode         : isStudyNode ? studyId : (it.conceptPath?.toString() ?: '@'),
+                    c_operator        : 'LIKE',
+                    c_dimcode         : it.conceptPath?.toString() ?: it.path.toString(),
                     c_tooltip         : it.path.toString(),
                     m_applied_path    : '@',
                     update_date       : now,
                     download_date     : now,
                     import_date       : now,
                     sourcesystem_cd   : conceptTree.isStudyNode(it) ? studyId : null,
+                    c_comment         : conceptTree.isStudyNode(it) ? "trial:${studyId}" : null,
             ]
 
             if (recordId) {
                 i2b2Row.record_id = -1
             }
 
             Map i2b2SecureRow = new HashMap(i2b2Row)
+            i2b2SecureRow.remove('c_comment')
             /* In this case, we use 'EXP:PUBLIC' for public nodes, to make the nodes visible in transmartApp. */
             def sot = (it.ontologyNode || secureObjectToken.public) ? 'EXP:PUBLIC' : secureObjectToken.toString()
             i2b2SecureRow.put('secure_obj_token', sot)

diff --git a/...ain/groovy/org/transmartproject/batch/highdim/beans/AbstractTypicalHdDataJobConfig.groovy b/...ain/groovy/org/transmartproject/batch/highdim/beans/AbstractTypicalHdDataJobConfig.groovy
@@ -102,8 +102,16 @@ abstract class AbstractTypicalHdDataJobConfig {
 
                 .next(deleteHdData)
                 .next(deleteCurrentAssays)
-                .next(deleteConceptCounts)
-                .next(deleteObservationFacts)
+
+                // FIXME: these lines do not only delete observations for
+                // highdim data, but also for clinical data.
+                // Deleting observations prior to inserting new observations
+                // is meant to support reloading of studies.
+                // A proper backout script should be written that deletes
+                // a selected subset of data for a study.
+                //
+                //.next(deleteConceptCounts)
+                //.next(deleteObservationFacts)
 
                 .next(insertConcepts)
                 .next(insertPseudoFacts)

diff --git a/...est-func/groovy/org/transmartproject/batch/clinical/ClinicalDataCleanScenarioTests.groovy b/...est-func/groovy/org/transmartproject/batch/clinical/ClinicalDataCleanScenarioTests.groovy
@@ -276,12 +276,13 @@ class ClinicalDataCleanScenarioTests implements JobRunningTestTrait {
 
         // i2b2_secure has one more column than i2b2, so we test
         // that we find all the columns in i2b2 against those in
-        // i2b2_secure (with two exceptions)
+        // i2b2_secure (with three exceptions)
         0..(r.size() - 1).each { i ->
             assertThat rSec[i], allOf(
                     r[i].collect { column, value ->
-                        // exclude i2b2_id/record_id columns from comparison
+                        // exclude i2b2_id/record_id/c_comment columns from comparison
                         column.equalsIgnoreCase('i2b2_id') ||
+                                column.equalsIgnoreCase('c_comment') ||
                                 column.equalsIgnoreCase('record_id') ?
                                 null :
                                 hasEntry(column, value)
@@ -474,7 +475,7 @@ class ClinicalDataCleanScenarioTests implements JobRunningTestTrait {
                 hasEntry('c_columnname', 'concept_path'),
                 hasEntry('c_columndatatype', 'T'),
                 hasEntry('c_operator', 'LIKE'),
-                hasEntry('c_dimcode', ''),
+                hasEntry('c_dimcode', '@'),
         )
     }
 

diff --git a/transmart-data/build.gradle b/transmart-data/build.gradle
@@ -74,6 +74,8 @@ downloadR.dependsOn 'createVars'
 distTar.dependsOn 'downloadR'
 
 task generateRequiredFiles << {
+    make('-C ddl/postgres drivers')
+    make('-C ddl/oracle drivers')
     make('-C data/common makefiles')
     make('-C ddl/postgres/META default_permissions.tsv')
     make('-C data/common/searchapp/plugin_modules_params ../plugin_module.tsv')

diff --git a/transmart-data/ddl/oracle/Makefile b/transmart-data/ddl/oracle/Makefile
@@ -88,6 +88,8 @@ drop: drop_users
 clean_all:
 	find $(SCHEMAS) GLOBAL -name '*.sql' -delete -o -name items.json -delete
 
+drivers: $(JDBC_DRIVER_ORA_PATH) $(JDBC_XDB_ORA_PATH)
+
 runsql.sh: $(JDBC_DRIVER_ORA_PATH)
 	printf '#!/bin/sh\n\n%s $$@\n' '$(RUN_SQL)' > $@
 	chmod +x $@

diff --git a/transmart-data/ddl/oracle/i2b2metadata/dimension_description.sql b/transmart-data/ddl/oracle/i2b2metadata/dimension_description.sql
@@ -1,22 +1,22 @@
 --
 -- Type: TABLE; Owner: I2B2METADATA; Name: DIMENSION_DESCRIPTION
 --
- CREATE TABLE "I2B2METADATA"."DIMENSION_DESCRIPTION"
-  (	"ID" NUMBER,
+CREATE TABLE "I2B2METADATA"."DIMENSION_DESCRIPTION"
+( "ID" NUMBER,
 "DENSITY" VARCHAR2(255 BYTE),
 "MODIFIER_CODE" VARCHAR2(255 BYTE),
 "VALUE_TYPE" VARCHAR2(50 BYTE),
 "NAME" VARCHAR2(255 BYTE) NOT NULL ENABLE,
 "PACKABLE" VARCHAR2(255 BYTE),
-"SIZE_CD" VARCHAR2(255 BYTE),
- PRIMARY KEY ("ID")
- CONSTRAINT "DIMENSION_DESCRIPTION_UNIQUE_NAME" UNIQUE ("NAME")
- USING INDEX
- TABLESPACE "TRANSMART"  ENABLE
-  ) SEGMENT CREATION DEFERRED
+"SIZE_CD" VARCHAR2(255 BYTE))
 NOCOMPRESS LOGGING
- TABLESPACE "TRANSMART" ;
---
+TABLESPACE "TRANSMART" ;
+
+ALTER TABLE "I2B2METADATA"."DIMENSION_DESCRIPTION"
+ADD CONSTRAINT "DIMENSION_DESC_UNIQUE_NAME" UNIQUE ("NAME");
+
+ALTER TABLE "I2B2METADATA"."DIMENSION_DESCRIPTION"
+ADD CONSTRAINT "DIMENSION_DESC_PK_ID" PRIMARY KEY ("ID") USING INDEX TABLESPACE INDX;--
 -- Type: SEQUENCE; Owner: I2B2METADATA; Name: DIMENSION_DESCRIPTION_ID_SEQ
 --
 CREATE SEQUENCE  "I2B2METADATA"."DIMENSION_DESCRIPTION_ID_SEQ"  MINVALUE 1 MAXVALUE 9999999999999999999999999999 INCREMENT BY 1 START WITH 1 CACHE 20 NOORDER  NOCYCLE  NOPARTITION ;

diff --git a/transmart-data/ddl/postgres/Makefile b/transmart-data/ddl/postgres/Makefile
@@ -102,4 +102,6 @@ load: $(LOAD_TARGETS)
 	$(MAKE) load_cross
 	$(MAKE) -C META load
 
+drivers: $(JDBC_DRIVER_PSQL_PATH)
+
 .PHONY: load load_global load_cross