From bbe42f0b1f37e9999861f4884618260b3b0878e6 Mon Sep 17 00:00:00 2001 From: Gijs Kant Date: Tue, 18 Jul 2017 11:40:26 +0200 Subject: [PATCH 1/7] Transmart-batch workaround: skip delete for highdim. Skip the delete observations step when loading highdim data, as also clinical data observations are removed. A rewrite of study backout functionality is required to properly fix the issue. --- .../beans/AbstractTypicalHdDataJobConfig.groovy | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/transmart-batch/src/main/groovy/org/transmartproject/batch/highdim/beans/AbstractTypicalHdDataJobConfig.groovy b/transmart-batch/src/main/groovy/org/transmartproject/batch/highdim/beans/AbstractTypicalHdDataJobConfig.groovy index 928c4f6d7a..3b15a2b985 100644 --- a/transmart-batch/src/main/groovy/org/transmartproject/batch/highdim/beans/AbstractTypicalHdDataJobConfig.groovy +++ b/transmart-batch/src/main/groovy/org/transmartproject/batch/highdim/beans/AbstractTypicalHdDataJobConfig.groovy @@ -102,8 +102,16 @@ abstract class AbstractTypicalHdDataJobConfig { .next(deleteHdData) .next(deleteCurrentAssays) - .next(deleteConceptCounts) - .next(deleteObservationFacts) + + // FIXME: these lines do not only delete observations for + // highdim data, but also for clinical data. + // Deleting observations prior to inserting new observations + // is meant to support reloading of studies. + // A proper backout script should be written that deletes + // a selected subset of data for a study. + // + //.next(deleteConceptCounts) + //.next(deleteObservationFacts) .next(insertConcepts) .next(insertPseudoFacts) From 8568f964e90eedff7faca20c90beda09bee51415 Mon Sep 17 00:00:00 2001 From: Gijs Kant Date: Tue, 18 Jul 2017 11:41:57 +0200 Subject: [PATCH 2/7] Fetch database drivers prior to packaging transmart-data. --- transmart-data/build.gradle | 2 ++ transmart-data/ddl/oracle/Makefile | 2 ++ transmart-data/ddl/postgres/Makefile | 2 ++ 3 files changed, 6 insertions(+) diff --git a/transmart-data/build.gradle b/transmart-data/build.gradle index a97336d8de..2fd091ac40 100644 --- a/transmart-data/build.gradle +++ b/transmart-data/build.gradle @@ -74,6 +74,8 @@ downloadR.dependsOn 'createVars' distTar.dependsOn 'downloadR' task generateRequiredFiles << { + make('-C ddl/postgres drivers') + make('-C ddl/oracle drivers') make('-C data/common makefiles') make('-C ddl/postgres/META default_permissions.tsv') make('-C data/common/searchapp/plugin_modules_params ../plugin_module.tsv') diff --git a/transmart-data/ddl/oracle/Makefile b/transmart-data/ddl/oracle/Makefile index 1ad39a058d..019b3b374e 100644 --- a/transmart-data/ddl/oracle/Makefile +++ b/transmart-data/ddl/oracle/Makefile @@ -88,6 +88,8 @@ drop: drop_users clean_all: find $(SCHEMAS) GLOBAL -name '*.sql' -delete -o -name items.json -delete +drivers: $(JDBC_DRIVER_ORA_PATH) $(JDBC_XDB_ORA_PATH) + runsql.sh: $(JDBC_DRIVER_ORA_PATH) printf '#!/bin/sh\n\n%s $$@\n' '$(RUN_SQL)' > $@ chmod +x $@ diff --git a/transmart-data/ddl/postgres/Makefile b/transmart-data/ddl/postgres/Makefile index 3fc3eb7567..8356b51ed6 100644 --- a/transmart-data/ddl/postgres/Makefile +++ b/transmart-data/ddl/postgres/Makefile @@ -102,4 +102,6 @@ load: $(LOAD_TARGETS) $(MAKE) load_cross $(MAKE) -C META load +drivers: $(JDBC_DRIVER_PSQL_PATH) + .PHONY: load load_global load_cross From da54d871afdea3316c78655bf8226b4d96403d11 Mon Sep 17 00:00:00 2001 From: Gijs Kant Date: Tue, 18 Jul 2017 18:07:45 +0200 Subject: [PATCH 3/7] Fix insertion of table access entries for Oracle. --- .../transmartproject/batch/concept/InsertConceptsService.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/transmart-batch/src/main/groovy/org/transmartproject/batch/concept/InsertConceptsService.groovy b/transmart-batch/src/main/groovy/org/transmartproject/batch/concept/InsertConceptsService.groovy index fd9f9ab4ee..2dbc7c7881 100644 --- a/transmart-batch/src/main/groovy/org/transmartproject/batch/concept/InsertConceptsService.groovy +++ b/transmart-batch/src/main/groovy/org/transmartproject/batch/concept/InsertConceptsService.groovy @@ -120,7 +120,7 @@ class InsertConceptsService { c_columnname: 'concept_path', c_columndatatype: 'T', c_operator: 'LIKE', - c_dimcode: node.conceptPath?.toString() ?: '', + c_dimcode: node.conceptPath?.toString() ?: '@', ] } tableAccessInsert.executeBatch(tableAccessRows as Map[]) From dff63390ad9845caf11bb6058e992198f9089467 Mon Sep 17 00:00:00 2001 From: Gijs Kant Date: Tue, 18 Jul 2017 18:09:03 +0200 Subject: [PATCH 4/7] Fix dimension description ddl for Oracle. --- .../i2b2metadata/dimension_description.sql | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/transmart-data/ddl/oracle/i2b2metadata/dimension_description.sql b/transmart-data/ddl/oracle/i2b2metadata/dimension_description.sql index 73fa93805e..0b8b8f4243 100644 --- a/transmart-data/ddl/oracle/i2b2metadata/dimension_description.sql +++ b/transmart-data/ddl/oracle/i2b2metadata/dimension_description.sql @@ -1,22 +1,22 @@ -- -- Type: TABLE; Owner: I2B2METADATA; Name: DIMENSION_DESCRIPTION -- - CREATE TABLE "I2B2METADATA"."DIMENSION_DESCRIPTION" - ( "ID" NUMBER, +CREATE TABLE "I2B2METADATA"."DIMENSION_DESCRIPTION" +( "ID" NUMBER, "DENSITY" VARCHAR2(255 BYTE), "MODIFIER_CODE" VARCHAR2(255 BYTE), "VALUE_TYPE" VARCHAR2(50 BYTE), "NAME" VARCHAR2(255 BYTE) NOT NULL ENABLE, "PACKABLE" VARCHAR2(255 BYTE), -"SIZE_CD" VARCHAR2(255 BYTE), - PRIMARY KEY ("ID") - CONSTRAINT "DIMENSION_DESCRIPTION_UNIQUE_NAME" UNIQUE ("NAME") - USING INDEX - TABLESPACE "TRANSMART" ENABLE - ) SEGMENT CREATION DEFERRED +"SIZE_CD" VARCHAR2(255 BYTE)) NOCOMPRESS LOGGING - TABLESPACE "TRANSMART" ; --- +TABLESPACE "TRANSMART" ; + +ALTER TABLE "I2B2METADATA"."DIMENSION_DESCRIPTION" +ADD CONSTRAINT "DIMENSION_DESC_UNIQUE_NAME" UNIQUE ("NAME"); + +ALTER TABLE "I2B2METADATA"."DIMENSION_DESCRIPTION" +ADD CONSTRAINT "DIMENSION_DESC_PK_ID" PRIMARY KEY ("ID") USING INDEX TABLESPACE INDX;-- -- Type: SEQUENCE; Owner: I2B2METADATA; Name: DIMENSION_DESCRIPTION_ID_SEQ -- CREATE SEQUENCE "I2B2METADATA"."DIMENSION_DESCRIPTION_ID_SEQ" MINVALUE 1 MAXVALUE 9999999999999999999999999999 INCREMENT BY 1 START WITH 1 CACHE 20 NOORDER NOCYCLE NOPARTITION ; From 8976ebc71c2b7a30ffdb8c289b6b01dc180232b6 Mon Sep 17 00:00:00 2001 From: Gijs Kant Date: Tue, 18 Jul 2017 22:29:50 +0200 Subject: [PATCH 5/7] Fix test for transmart-batch. --- .../batch/clinical/ClinicalDataCleanScenarioTests.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/transmart-batch/src/test-func/groovy/org/transmartproject/batch/clinical/ClinicalDataCleanScenarioTests.groovy b/transmart-batch/src/test-func/groovy/org/transmartproject/batch/clinical/ClinicalDataCleanScenarioTests.groovy index 7a3ef0183a..e0523d9c20 100644 --- a/transmart-batch/src/test-func/groovy/org/transmartproject/batch/clinical/ClinicalDataCleanScenarioTests.groovy +++ b/transmart-batch/src/test-func/groovy/org/transmartproject/batch/clinical/ClinicalDataCleanScenarioTests.groovy @@ -474,7 +474,7 @@ class ClinicalDataCleanScenarioTests implements JobRunningTestTrait { hasEntry('c_columnname', 'concept_path'), hasEntry('c_columndatatype', 'T'), hasEntry('c_operator', 'LIKE'), - hasEntry('c_dimcode', ''), + hasEntry('c_dimcode', '@'), ) } From ab408d63b81c613a84d570b59c45a20ee2aa5d27 Mon Sep 17 00:00:00 2001 From: Gijs Kant Date: Thu, 20 Jul 2017 16:03:57 +0200 Subject: [PATCH 6/7] Fix insertion of ontology nodes for compatibility with transmartApp. --- .../batch/concept/InsertConceptsService.groovy | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/transmart-batch/src/main/groovy/org/transmartproject/batch/concept/InsertConceptsService.groovy b/transmart-batch/src/main/groovy/org/transmartproject/batch/concept/InsertConceptsService.groovy index 2dbc7c7881..b6846b0f9a 100644 --- a/transmart-batch/src/main/groovy/org/transmartproject/batch/concept/InsertConceptsService.groovy +++ b/transmart-batch/src/main/groovy/org/transmartproject/batch/concept/InsertConceptsService.groovy @@ -139,6 +139,12 @@ class InsertConceptsService { newConcepts.each { String visualAttributes = visualAttributesFor(it) boolean isStudyNode = topNode.path == it.path.path + /* For compatibility with transmartApp, also intermediate nodes + are treated as concept nodes, in order to be able to select + subtrees based on a concept path prefix. + The c_comment column is only for compatibility with transmartApp + and should not have a role in security checks in the future. + */ Map i2b2Row = [ c_hlevel : it.level, c_fullname : it.path.toString(), @@ -147,18 +153,19 @@ class InsertConceptsService { c_synonym_cd : 'N', c_visualattributes: visualAttributes, c_metadataxml : metadataXmlFor(it), - c_facttablecolumn : isStudyNode ? '@' : 'CONCEPT_CD', - c_tablename : isStudyNode ? 'STUDY' : 'CONCEPT_DIMENSION', - c_columnname : isStudyNode ? 'STUDY_ID' : 'CONCEPT_PATH', + c_facttablecolumn : 'CONCEPT_CD', + c_tablename : 'CONCEPT_DIMENSION', + c_columnname : 'CONCEPT_PATH', c_columndatatype : 'T', - c_operator : isStudyNode ? '=' : 'LIKE', - c_dimcode : isStudyNode ? studyId : (it.conceptPath?.toString() ?: '@'), + c_operator : 'LIKE', + c_dimcode : it.conceptPath?.toString() ?: it.path.toString(), c_tooltip : it.path.toString(), m_applied_path : '@', update_date : now, download_date : now, import_date : now, sourcesystem_cd : conceptTree.isStudyNode(it) ? studyId : null, + c_comment : conceptTree.isStudyNode(it) ? "trial:${studyId}" : null, ] if (recordId) { @@ -166,6 +173,7 @@ class InsertConceptsService { } Map i2b2SecureRow = new HashMap(i2b2Row) + i2b2SecureRow.remove('c_comment') /* In this case, we use 'EXP:PUBLIC' for public nodes, to make the nodes visible in transmartApp. */ def sot = (it.ontologyNode || secureObjectToken.public) ? 'EXP:PUBLIC' : secureObjectToken.toString() i2b2SecureRow.put('secure_obj_token', sot) From 493da6cba3344f4100534434cd5847101ec791a8 Mon Sep 17 00:00:00 2001 From: Gijs Kant Date: Thu, 20 Jul 2017 21:31:36 +0200 Subject: [PATCH 7/7] Fix study entry loading for transmartApp compatibility. --- .../transmartproject/batch/concept/ConceptTree.groovy | 9 +++++++++ .../batch/clinical/ClinicalDataCleanScenarioTests.groovy | 5 +++-- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/transmart-batch/src/main/groovy/org/transmartproject/batch/concept/ConceptTree.groovy b/transmart-batch/src/main/groovy/org/transmartproject/batch/concept/ConceptTree.groovy index ed27e2250c..4be68d6832 100644 --- a/transmart-batch/src/main/groovy/org/transmartproject/batch/concept/ConceptTree.groovy +++ b/transmart-batch/src/main/groovy/org/transmartproject/batch/concept/ConceptTree.groovy @@ -28,6 +28,9 @@ class ConceptTree { @Value("#{jobParameters['TOP_NODE']}") ConceptPath topNodePath + @Value("#{jobParameters['STUDY_ID']}") + String studyId + @Autowired private SequenceReserver reserver @@ -162,6 +165,12 @@ class ConceptTree { node.conceptPath = variable.conceptPath node.code = variable.conceptCode node.ontologyNode = variable.ontologyNode + } else if (isStudyNode(node)) { + // For compatibility with transmartApp data export, + // there needs to be a 'study concept' + node.conceptPath = path + node.conceptName = studyId + node.code = studyId } log.debug("Generated new concept node: $path") nodeMap[path] = node diff --git a/transmart-batch/src/test-func/groovy/org/transmartproject/batch/clinical/ClinicalDataCleanScenarioTests.groovy b/transmart-batch/src/test-func/groovy/org/transmartproject/batch/clinical/ClinicalDataCleanScenarioTests.groovy index e0523d9c20..57af3ab2c5 100644 --- a/transmart-batch/src/test-func/groovy/org/transmartproject/batch/clinical/ClinicalDataCleanScenarioTests.groovy +++ b/transmart-batch/src/test-func/groovy/org/transmartproject/batch/clinical/ClinicalDataCleanScenarioTests.groovy @@ -276,12 +276,13 @@ class ClinicalDataCleanScenarioTests implements JobRunningTestTrait { // i2b2_secure has one more column than i2b2, so we test // that we find all the columns in i2b2 against those in - // i2b2_secure (with two exceptions) + // i2b2_secure (with three exceptions) 0..(r.size() - 1).each { i -> assertThat rSec[i], allOf( r[i].collect { column, value -> - // exclude i2b2_id/record_id columns from comparison + // exclude i2b2_id/record_id/c_comment columns from comparison column.equalsIgnoreCase('i2b2_id') || + column.equalsIgnoreCase('c_comment') || column.equalsIgnoreCase('record_id') ? null : hasEntry(column, value)