diff --git a/.gitignore b/.gitignore index 46ff8e91..2672d9e1 100644 --- a/.gitignore +++ b/.gitignore @@ -4,8 +4,14 @@ dist/ /Error.txt /.gitignore /Error1.txt -/ScanReport.xlsx /.classpath target/ *.iml .idea/ + +/ScanReport.xlsx +*ScanReport* +*swp +tmp/* +Try* +/examples/ diff --git a/README.md b/README.md index 21ef48dd..c77725a0 100644 --- a/README.md +++ b/README.md @@ -58,6 +58,31 @@ Rabbit-In-A-Hat Note: on releases earlier than version 0.8.0, open the respective `WhiteRabbit.jar` or `RabbitInAHat.jar` files instead. +Examples +======== +The file `examples.zip` contains a set of input and output examples for White Rabbit and Rabbit in a Hat. +These are used for testing of the main White Rabbit and Rabbit in a Hat features. + +Contents: +- `wr_input_csv` + + csv files to test scanning on different data types and long table names. +- `wr_input_sas` + + sas7bdat files to test sas input +- `wr_output` + + Scan reports created from files in `wr_input_csv`, `wr_input_sas` and [native a Synthea database loaded in Postgres](https://github.com/ohdsi/ETL-Synthea). All with default scan options. + This folder also includes fake data generated from the csv scan report. The csv scan report is used to test the opening a Scan Report in Rabbit in a Hat. + +- `riah_input` + + An example mapping file used to create the Rabbit in a Hat outputs. +- `riah_output` + + All export formats created by Rabbit in a Hat: as word, html, markdown, sql skeleton and the R TestFramework. + These are all generated from `riah_input/riah_mapping_example.gz`. + Getting Involved ============= * User guide and Help: WhiteRabbit Wiki diff --git a/examples.zip b/examples.zip new file mode 100644 index 00000000..47652161 Binary files /dev/null and b/examples.zip differ diff --git a/lib/com/simba/googlebigquery/jdbc/GoogleBigQueryJDBC/42/GoogleBigQueryJDBC-42.jar b/lib/com/simba/googlebigquery/jdbc/GoogleBigQueryJDBC/42/GoogleBigQueryJDBC-42.jar new file mode 100644 index 00000000..ac2f8b2e Binary files /dev/null and b/lib/com/simba/googlebigquery/jdbc/GoogleBigQueryJDBC/42/GoogleBigQueryJDBC-42.jar differ diff --git a/lib/com/simba/googlebigquery/jdbc/GoogleBigQueryJDBC/42/GoogleBigQueryJDBC-42.jar.md5 b/lib/com/simba/googlebigquery/jdbc/GoogleBigQueryJDBC/42/GoogleBigQueryJDBC-42.jar.md5 new file mode 100644 index 00000000..9c25027a --- /dev/null +++ b/lib/com/simba/googlebigquery/jdbc/GoogleBigQueryJDBC/42/GoogleBigQueryJDBC-42.jar.md5 @@ -0,0 +1 @@ +262cc658e65aa9fdf4282ea6714f1367 \ No newline at end of file diff --git a/lib/com/simba/googlebigquery/jdbc/GoogleBigQueryJDBC/42/GoogleBigQueryJDBC-42.jar.sha1 b/lib/com/simba/googlebigquery/jdbc/GoogleBigQueryJDBC/42/GoogleBigQueryJDBC-42.jar.sha1 new file mode 100644 index 00000000..759f7606 --- /dev/null +++ b/lib/com/simba/googlebigquery/jdbc/GoogleBigQueryJDBC/42/GoogleBigQueryJDBC-42.jar.sha1 @@ -0,0 +1 @@ +2a6fc1b481483d45f3efbc69e76c6665c11cc2ec \ No newline at end of file diff --git a/lib/com/simba/googlebigquery/jdbc/GoogleBigQueryJDBC/42/GoogleBigQueryJDBC-42.pom b/lib/com/simba/googlebigquery/jdbc/GoogleBigQueryJDBC/42/GoogleBigQueryJDBC-42.pom new file mode 100644 index 00000000..a8e80cf3 --- /dev/null +++ b/lib/com/simba/googlebigquery/jdbc/GoogleBigQueryJDBC/42/GoogleBigQueryJDBC-42.pom @@ -0,0 +1,8 @@ + + + 4.0.0 + com.simba.googlebigquery.jdbc + GoogleBigQueryJDBC + 42 + diff --git a/lib/com/simba/googlebigquery/jdbc/GoogleBigQueryJDBC/42/GoogleBigQueryJDBC-42.pom.md5 b/lib/com/simba/googlebigquery/jdbc/GoogleBigQueryJDBC/42/GoogleBigQueryJDBC-42.pom.md5 new file mode 100644 index 00000000..6ef1cc9e --- /dev/null +++ b/lib/com/simba/googlebigquery/jdbc/GoogleBigQueryJDBC/42/GoogleBigQueryJDBC-42.pom.md5 @@ -0,0 +1 @@ +fd2d1c1022b642e24fe0e790adafc8a9 \ No newline at end of file diff --git a/lib/com/simba/googlebigquery/jdbc/GoogleBigQueryJDBC/42/GoogleBigQueryJDBC-42.pom.sha1 b/lib/com/simba/googlebigquery/jdbc/GoogleBigQueryJDBC/42/GoogleBigQueryJDBC-42.pom.sha1 new file mode 100644 index 00000000..76bcd62f --- /dev/null +++ b/lib/com/simba/googlebigquery/jdbc/GoogleBigQueryJDBC/42/GoogleBigQueryJDBC-42.pom.sha1 @@ -0,0 +1 @@ +5ba2502c65e57974ed7d38bb4fffea3a5434deca \ No newline at end of file diff --git a/lib/com/simba/googlebigquery/jdbc/GoogleBigQueryJDBC/maven-metadata.xml b/lib/com/simba/googlebigquery/jdbc/GoogleBigQueryJDBC/maven-metadata.xml new file mode 100644 index 00000000..e7fcaaac --- /dev/null +++ b/lib/com/simba/googlebigquery/jdbc/GoogleBigQueryJDBC/maven-metadata.xml @@ -0,0 +1,12 @@ + + + com.simba.googlebigquery.jdbc + GoogleBigQueryJDBC + + 42 + + 42 + + 20191004152516 + + diff --git a/lib/com/simba/googlebigquery/jdbc/GoogleBigQueryJDBC/maven-metadata.xml.md5 b/lib/com/simba/googlebigquery/jdbc/GoogleBigQueryJDBC/maven-metadata.xml.md5 new file mode 100644 index 00000000..25766af7 --- /dev/null +++ b/lib/com/simba/googlebigquery/jdbc/GoogleBigQueryJDBC/maven-metadata.xml.md5 @@ -0,0 +1 @@ +8b7363e0b69e10aa775fb910f5e0dcbc \ No newline at end of file diff --git a/lib/com/simba/googlebigquery/jdbc/GoogleBigQueryJDBC/maven-metadata.xml.sha1 b/lib/com/simba/googlebigquery/jdbc/GoogleBigQueryJDBC/maven-metadata.xml.sha1 new file mode 100644 index 00000000..85e8fa6a --- /dev/null +++ b/lib/com/simba/googlebigquery/jdbc/GoogleBigQueryJDBC/maven-metadata.xml.sha1 @@ -0,0 +1 @@ +be98b4ed33c88fd2e80c10dea787c26c04cd58a7 \ No newline at end of file diff --git a/lib/com/simba/googlebigquery/jdbc/avro/1.8.2/avro-1.8.2.jar b/lib/com/simba/googlebigquery/jdbc/avro/1.8.2/avro-1.8.2.jar new file mode 100644 index 00000000..23b3958b Binary files /dev/null and b/lib/com/simba/googlebigquery/jdbc/avro/1.8.2/avro-1.8.2.jar differ diff --git a/lib/com/simba/googlebigquery/jdbc/avro/1.8.2/avro-1.8.2.jar.md5 b/lib/com/simba/googlebigquery/jdbc/avro/1.8.2/avro-1.8.2.jar.md5 new file mode 100644 index 00000000..0ccdef6b --- /dev/null +++ b/lib/com/simba/googlebigquery/jdbc/avro/1.8.2/avro-1.8.2.jar.md5 @@ -0,0 +1 @@ +10395e5a571e1a1f6113411f276d2fea \ No newline at end of file diff --git a/lib/com/simba/googlebigquery/jdbc/avro/1.8.2/avro-1.8.2.jar.sha1 b/lib/com/simba/googlebigquery/jdbc/avro/1.8.2/avro-1.8.2.jar.sha1 new file mode 100644 index 00000000..d10965a1 --- /dev/null +++ b/lib/com/simba/googlebigquery/jdbc/avro/1.8.2/avro-1.8.2.jar.sha1 @@ -0,0 +1 @@ +91e3146dfff4bd510181032c8276a3a0130c0697 \ No newline at end of file diff --git a/lib/com/simba/googlebigquery/jdbc/avro/1.8.2/avro-1.8.2.pom b/lib/com/simba/googlebigquery/jdbc/avro/1.8.2/avro-1.8.2.pom new file mode 100644 index 00000000..a5bc1158 --- /dev/null +++ b/lib/com/simba/googlebigquery/jdbc/avro/1.8.2/avro-1.8.2.pom @@ -0,0 +1,8 @@ + + + 4.0.0 + com.simba.googlebigquery.jdbc + avro + 1.8.2 + diff --git a/lib/com/simba/googlebigquery/jdbc/avro/1.8.2/avro-1.8.2.pom.md5 b/lib/com/simba/googlebigquery/jdbc/avro/1.8.2/avro-1.8.2.pom.md5 new file mode 100644 index 00000000..0508c3ee --- /dev/null +++ b/lib/com/simba/googlebigquery/jdbc/avro/1.8.2/avro-1.8.2.pom.md5 @@ -0,0 +1 @@ +3d7b0a7f1bd6803fce9fd00ba4654068 \ No newline at end of file diff --git a/lib/com/simba/googlebigquery/jdbc/avro/1.8.2/avro-1.8.2.pom.sha1 b/lib/com/simba/googlebigquery/jdbc/avro/1.8.2/avro-1.8.2.pom.sha1 new file mode 100644 index 00000000..fcfe86eb --- /dev/null +++ b/lib/com/simba/googlebigquery/jdbc/avro/1.8.2/avro-1.8.2.pom.sha1 @@ -0,0 +1 @@ +be4010b1fc46d26fac0777567bbadea0c6dc99b4 \ No newline at end of file diff --git a/lib/com/simba/googlebigquery/jdbc/avro/maven-metadata.xml b/lib/com/simba/googlebigquery/jdbc/avro/maven-metadata.xml new file mode 100644 index 00000000..d50f644a --- /dev/null +++ b/lib/com/simba/googlebigquery/jdbc/avro/maven-metadata.xml @@ -0,0 +1,12 @@ + + + com.simba.googlebigquery.jdbc + avro + + 1.8.2 + + 1.8.2 + + 20191004152634 + + diff --git a/lib/com/simba/googlebigquery/jdbc/avro/maven-metadata.xml.md5 b/lib/com/simba/googlebigquery/jdbc/avro/maven-metadata.xml.md5 new file mode 100644 index 00000000..f90c9a83 --- /dev/null +++ b/lib/com/simba/googlebigquery/jdbc/avro/maven-metadata.xml.md5 @@ -0,0 +1 @@ +31df9ef707a0b96bf9ff21c762d3b75d \ No newline at end of file diff --git a/lib/com/simba/googlebigquery/jdbc/avro/maven-metadata.xml.sha1 b/lib/com/simba/googlebigquery/jdbc/avro/maven-metadata.xml.sha1 new file mode 100644 index 00000000..e3ac5799 --- /dev/null +++ b/lib/com/simba/googlebigquery/jdbc/avro/maven-metadata.xml.sha1 @@ -0,0 +1 @@ +7cf68ae83ecad5129aaebbe3195def8355c55424 \ No newline at end of file diff --git a/lib/com/simba/googlebigquery/jdbc/gax/1.42.0/gax-1.42.0.jar b/lib/com/simba/googlebigquery/jdbc/gax/1.42.0/gax-1.42.0.jar new file mode 100644 index 00000000..c956144a Binary files /dev/null and b/lib/com/simba/googlebigquery/jdbc/gax/1.42.0/gax-1.42.0.jar differ diff --git a/lib/com/simba/googlebigquery/jdbc/gax/1.42.0/gax-1.42.0.jar.md5 b/lib/com/simba/googlebigquery/jdbc/gax/1.42.0/gax-1.42.0.jar.md5 new file mode 100644 index 00000000..d30e793c --- /dev/null +++ b/lib/com/simba/googlebigquery/jdbc/gax/1.42.0/gax-1.42.0.jar.md5 @@ -0,0 +1 @@ +bf9a95970b7aa8da9d86a452872dea0a \ No newline at end of file diff --git a/lib/com/simba/googlebigquery/jdbc/gax/1.42.0/gax-1.42.0.jar.sha1 b/lib/com/simba/googlebigquery/jdbc/gax/1.42.0/gax-1.42.0.jar.sha1 new file mode 100644 index 00000000..a9df54c9 --- /dev/null +++ b/lib/com/simba/googlebigquery/jdbc/gax/1.42.0/gax-1.42.0.jar.sha1 @@ -0,0 +1 @@ +ff099b100d2202d4bf0c0a00f14cb369e0186375 \ No newline at end of file diff --git a/lib/com/simba/googlebigquery/jdbc/gax/1.42.0/gax-1.42.0.pom b/lib/com/simba/googlebigquery/jdbc/gax/1.42.0/gax-1.42.0.pom new file mode 100644 index 00000000..34f769f3 --- /dev/null +++ b/lib/com/simba/googlebigquery/jdbc/gax/1.42.0/gax-1.42.0.pom @@ -0,0 +1,8 @@ + + + 4.0.0 + com.simba.googlebigquery.jdbc + gax + 1.42.0 + diff --git a/lib/com/simba/googlebigquery/jdbc/gax/1.42.0/gax-1.42.0.pom.md5 b/lib/com/simba/googlebigquery/jdbc/gax/1.42.0/gax-1.42.0.pom.md5 new file mode 100644 index 00000000..8049c31e --- /dev/null +++ b/lib/com/simba/googlebigquery/jdbc/gax/1.42.0/gax-1.42.0.pom.md5 @@ -0,0 +1 @@ +0917a660d6d89a9f9391c726efaaeb81 \ No newline at end of file diff --git a/lib/com/simba/googlebigquery/jdbc/gax/1.42.0/gax-1.42.0.pom.sha1 b/lib/com/simba/googlebigquery/jdbc/gax/1.42.0/gax-1.42.0.pom.sha1 new file mode 100644 index 00000000..65abedc9 --- /dev/null +++ b/lib/com/simba/googlebigquery/jdbc/gax/1.42.0/gax-1.42.0.pom.sha1 @@ -0,0 +1 @@ +044024b82e18ac19ed32b4f685e34816c427c782 \ No newline at end of file diff --git a/lib/com/simba/googlebigquery/jdbc/gax/maven-metadata.xml b/lib/com/simba/googlebigquery/jdbc/gax/maven-metadata.xml new file mode 100644 index 00000000..9c1868f4 --- /dev/null +++ b/lib/com/simba/googlebigquery/jdbc/gax/maven-metadata.xml @@ -0,0 +1,12 @@ + + + com.simba.googlebigquery.jdbc + gax + + 1.42.0 + + 1.42.0 + + 20191004152532 + + diff --git a/lib/com/simba/googlebigquery/jdbc/gax/maven-metadata.xml.md5 b/lib/com/simba/googlebigquery/jdbc/gax/maven-metadata.xml.md5 new file mode 100644 index 00000000..cf6e0882 --- /dev/null +++ b/lib/com/simba/googlebigquery/jdbc/gax/maven-metadata.xml.md5 @@ -0,0 +1 @@ +caa3a807b0d4edc0b04fe9585ab69681 \ No newline at end of file diff --git a/lib/com/simba/googlebigquery/jdbc/gax/maven-metadata.xml.sha1 b/lib/com/simba/googlebigquery/jdbc/gax/maven-metadata.xml.sha1 new file mode 100644 index 00000000..ce934a98 --- /dev/null +++ b/lib/com/simba/googlebigquery/jdbc/gax/maven-metadata.xml.sha1 @@ -0,0 +1 @@ +5620904f6d8596e1a296cb8994a5485ed9805416 \ No newline at end of file diff --git a/lib/com/simba/googlebigquery/jdbc/google-api-client/1.28.0/google-api-client-1.28.0.jar b/lib/com/simba/googlebigquery/jdbc/google-api-client/1.28.0/google-api-client-1.28.0.jar new file mode 100644 index 00000000..450fea8e Binary files /dev/null and b/lib/com/simba/googlebigquery/jdbc/google-api-client/1.28.0/google-api-client-1.28.0.jar differ diff --git a/lib/com/simba/googlebigquery/jdbc/google-api-client/1.28.0/google-api-client-1.28.0.jar.md5 b/lib/com/simba/googlebigquery/jdbc/google-api-client/1.28.0/google-api-client-1.28.0.jar.md5 new file mode 100644 index 00000000..1e73f28d --- /dev/null +++ b/lib/com/simba/googlebigquery/jdbc/google-api-client/1.28.0/google-api-client-1.28.0.jar.md5 @@ -0,0 +1 @@ +4831bcd0d30d13382b3af50acbbbf53e \ No newline at end of file diff --git a/lib/com/simba/googlebigquery/jdbc/google-api-client/1.28.0/google-api-client-1.28.0.jar.sha1 b/lib/com/simba/googlebigquery/jdbc/google-api-client/1.28.0/google-api-client-1.28.0.jar.sha1 new file mode 100644 index 00000000..c9b0efd4 --- /dev/null +++ b/lib/com/simba/googlebigquery/jdbc/google-api-client/1.28.0/google-api-client-1.28.0.jar.sha1 @@ -0,0 +1 @@ +8fe155d766ed22480939e3a9db428151e0264d9e \ No newline at end of file diff --git a/lib/com/simba/googlebigquery/jdbc/google-api-client/1.28.0/google-api-client-1.28.0.pom b/lib/com/simba/googlebigquery/jdbc/google-api-client/1.28.0/google-api-client-1.28.0.pom new file mode 100644 index 00000000..815e7a51 --- /dev/null +++ b/lib/com/simba/googlebigquery/jdbc/google-api-client/1.28.0/google-api-client-1.28.0.pom @@ -0,0 +1,8 @@ + + + 4.0.0 + com.simba.googlebigquery.jdbc + google-api-client + 1.28.0 + diff --git a/lib/com/simba/googlebigquery/jdbc/google-api-client/1.28.0/google-api-client-1.28.0.pom.md5 b/lib/com/simba/googlebigquery/jdbc/google-api-client/1.28.0/google-api-client-1.28.0.pom.md5 new file mode 100644 index 00000000..15e7d1b2 --- /dev/null +++ b/lib/com/simba/googlebigquery/jdbc/google-api-client/1.28.0/google-api-client-1.28.0.pom.md5 @@ -0,0 +1 @@ +5d233fc2bdaee2a4f1a511531eaf2395 \ No newline at end of file diff --git a/lib/com/simba/googlebigquery/jdbc/google-api-client/1.28.0/google-api-client-1.28.0.pom.sha1 b/lib/com/simba/googlebigquery/jdbc/google-api-client/1.28.0/google-api-client-1.28.0.pom.sha1 new file mode 100644 index 00000000..721f1605 --- /dev/null +++ b/lib/com/simba/googlebigquery/jdbc/google-api-client/1.28.0/google-api-client-1.28.0.pom.sha1 @@ -0,0 +1 @@ +d36628ab6b89c2f6a79c2160b10bf83492c38c63 \ No newline at end of file diff --git a/lib/com/simba/googlebigquery/jdbc/google-api-client/maven-metadata.xml b/lib/com/simba/googlebigquery/jdbc/google-api-client/maven-metadata.xml new file mode 100644 index 00000000..b5e11ec8 --- /dev/null +++ b/lib/com/simba/googlebigquery/jdbc/google-api-client/maven-metadata.xml @@ -0,0 +1,12 @@ + + + com.simba.googlebigquery.jdbc + google-api-client + + 1.28.0 + + 1.28.0 + + 20191004152523 + + diff --git a/lib/com/simba/googlebigquery/jdbc/google-api-client/maven-metadata.xml.md5 b/lib/com/simba/googlebigquery/jdbc/google-api-client/maven-metadata.xml.md5 new file mode 100644 index 00000000..07fbbc7d --- /dev/null +++ b/lib/com/simba/googlebigquery/jdbc/google-api-client/maven-metadata.xml.md5 @@ -0,0 +1 @@ +dd0548af810f22977717b141232a7087 \ No newline at end of file diff --git a/lib/com/simba/googlebigquery/jdbc/google-api-client/maven-metadata.xml.sha1 b/lib/com/simba/googlebigquery/jdbc/google-api-client/maven-metadata.xml.sha1 new file mode 100644 index 00000000..52084890 --- /dev/null +++ b/lib/com/simba/googlebigquery/jdbc/google-api-client/maven-metadata.xml.sha1 @@ -0,0 +1 @@ +f27a1719919b86b3da4dd436e4e3167808109224 \ No newline at end of file diff --git a/lib/com/simba/googlebigquery/jdbc/google-api-service-bigquery/V2-rev426-1.25.0/google-api-service-bigquery-V2-rev426-1.25.0.jar b/lib/com/simba/googlebigquery/jdbc/google-api-service-bigquery/V2-rev426-1.25.0/google-api-service-bigquery-V2-rev426-1.25.0.jar new file mode 100644 index 00000000..2af75993 Binary files /dev/null and b/lib/com/simba/googlebigquery/jdbc/google-api-service-bigquery/V2-rev426-1.25.0/google-api-service-bigquery-V2-rev426-1.25.0.jar differ diff --git a/lib/com/simba/googlebigquery/jdbc/google-api-service-bigquery/V2-rev426-1.25.0/google-api-service-bigquery-V2-rev426-1.25.0.jar.md5 b/lib/com/simba/googlebigquery/jdbc/google-api-service-bigquery/V2-rev426-1.25.0/google-api-service-bigquery-V2-rev426-1.25.0.jar.md5 new file mode 100644 index 00000000..a5b776ba --- /dev/null +++ b/lib/com/simba/googlebigquery/jdbc/google-api-service-bigquery/V2-rev426-1.25.0/google-api-service-bigquery-V2-rev426-1.25.0.jar.md5 @@ -0,0 +1 @@ +a8865c7a220728b974fe3101581f29eb \ No newline at end of file diff --git a/lib/com/simba/googlebigquery/jdbc/google-api-service-bigquery/V2-rev426-1.25.0/google-api-service-bigquery-V2-rev426-1.25.0.jar.sha1 b/lib/com/simba/googlebigquery/jdbc/google-api-service-bigquery/V2-rev426-1.25.0/google-api-service-bigquery-V2-rev426-1.25.0.jar.sha1 new file mode 100644 index 00000000..ad56647e --- /dev/null +++ b/lib/com/simba/googlebigquery/jdbc/google-api-service-bigquery/V2-rev426-1.25.0/google-api-service-bigquery-V2-rev426-1.25.0.jar.sha1 @@ -0,0 +1 @@ +d1ba9dd5c60f5fcc8ac94ce1533d81b69e0d76aa \ No newline at end of file diff --git a/lib/com/simba/googlebigquery/jdbc/google-api-service-bigquery/V2-rev426-1.25.0/google-api-service-bigquery-V2-rev426-1.25.0.pom b/lib/com/simba/googlebigquery/jdbc/google-api-service-bigquery/V2-rev426-1.25.0/google-api-service-bigquery-V2-rev426-1.25.0.pom new file mode 100644 index 00000000..61669b52 --- /dev/null +++ b/lib/com/simba/googlebigquery/jdbc/google-api-service-bigquery/V2-rev426-1.25.0/google-api-service-bigquery-V2-rev426-1.25.0.pom @@ -0,0 +1,8 @@ + + + 4.0.0 + com.simba.googlebigquery.jdbc + google-api-service-bigquery + V2-rev426-1.25.0 + diff --git a/lib/com/simba/googlebigquery/jdbc/google-api-service-bigquery/V2-rev426-1.25.0/google-api-service-bigquery-V2-rev426-1.25.0.pom.md5 b/lib/com/simba/googlebigquery/jdbc/google-api-service-bigquery/V2-rev426-1.25.0/google-api-service-bigquery-V2-rev426-1.25.0.pom.md5 new file mode 100644 index 00000000..ae044234 --- /dev/null +++ b/lib/com/simba/googlebigquery/jdbc/google-api-service-bigquery/V2-rev426-1.25.0/google-api-service-bigquery-V2-rev426-1.25.0.pom.md5 @@ -0,0 +1 @@ +c8edc39e56425571ad865b47be4a887b \ No newline at end of file diff --git a/lib/com/simba/googlebigquery/jdbc/google-api-service-bigquery/V2-rev426-1.25.0/google-api-service-bigquery-V2-rev426-1.25.0.pom.sha1 b/lib/com/simba/googlebigquery/jdbc/google-api-service-bigquery/V2-rev426-1.25.0/google-api-service-bigquery-V2-rev426-1.25.0.pom.sha1 new file mode 100644 index 00000000..ebfa40ee --- /dev/null +++ b/lib/com/simba/googlebigquery/jdbc/google-api-service-bigquery/V2-rev426-1.25.0/google-api-service-bigquery-V2-rev426-1.25.0.pom.sha1 @@ -0,0 +1 @@ +3f8eb2ad6890d6752cc949f5753612c96dbaae5d \ No newline at end of file diff --git a/lib/com/simba/googlebigquery/jdbc/google-api-service-bigquery/maven-metadata.xml b/lib/com/simba/googlebigquery/jdbc/google-api-service-bigquery/maven-metadata.xml new file mode 100644 index 00000000..b88b51a4 --- /dev/null +++ b/lib/com/simba/googlebigquery/jdbc/google-api-service-bigquery/maven-metadata.xml @@ -0,0 +1,12 @@ + + + com.simba.googlebigquery.jdbc + google-api-service-bigquery + + V2-rev426-1.25.0 + + V2-rev426-1.25.0 + + 20191004152609 + + diff --git a/lib/com/simba/googlebigquery/jdbc/google-api-service-bigquery/maven-metadata.xml.md5 b/lib/com/simba/googlebigquery/jdbc/google-api-service-bigquery/maven-metadata.xml.md5 new file mode 100644 index 00000000..86cc4720 --- /dev/null +++ b/lib/com/simba/googlebigquery/jdbc/google-api-service-bigquery/maven-metadata.xml.md5 @@ -0,0 +1 @@ +9f4c377aeecd5d6179f069a0d2b74b48 \ No newline at end of file diff --git a/lib/com/simba/googlebigquery/jdbc/google-api-service-bigquery/maven-metadata.xml.sha1 b/lib/com/simba/googlebigquery/jdbc/google-api-service-bigquery/maven-metadata.xml.sha1 new file mode 100644 index 00000000..b371952b --- /dev/null +++ b/lib/com/simba/googlebigquery/jdbc/google-api-service-bigquery/maven-metadata.xml.sha1 @@ -0,0 +1 @@ +3ca39758a654a2a1e739236ec5010e1665f9109c \ No newline at end of file diff --git a/lib/com/simba/googlebigquery/jdbc/google-auth-library-credentials/0.15.0/google-auth-library-credentials-0.15.0.jar b/lib/com/simba/googlebigquery/jdbc/google-auth-library-credentials/0.15.0/google-auth-library-credentials-0.15.0.jar new file mode 100644 index 00000000..7834fac7 Binary files /dev/null and b/lib/com/simba/googlebigquery/jdbc/google-auth-library-credentials/0.15.0/google-auth-library-credentials-0.15.0.jar differ diff --git a/lib/com/simba/googlebigquery/jdbc/google-auth-library-credentials/0.15.0/google-auth-library-credentials-0.15.0.jar.md5 b/lib/com/simba/googlebigquery/jdbc/google-auth-library-credentials/0.15.0/google-auth-library-credentials-0.15.0.jar.md5 new file mode 100644 index 00000000..4040a3d0 --- /dev/null +++ b/lib/com/simba/googlebigquery/jdbc/google-auth-library-credentials/0.15.0/google-auth-library-credentials-0.15.0.jar.md5 @@ -0,0 +1 @@ +4bd2bd355d775b0d25113d585b93328a \ No newline at end of file diff --git a/lib/com/simba/googlebigquery/jdbc/google-auth-library-credentials/0.15.0/google-auth-library-credentials-0.15.0.jar.sha1 b/lib/com/simba/googlebigquery/jdbc/google-auth-library-credentials/0.15.0/google-auth-library-credentials-0.15.0.jar.sha1 new file mode 100644 index 00000000..551508e1 --- /dev/null +++ b/lib/com/simba/googlebigquery/jdbc/google-auth-library-credentials/0.15.0/google-auth-library-credentials-0.15.0.jar.sha1 @@ -0,0 +1 @@ +29fc7f62c488cf3b28e98db71a75f47da6fb7afc \ No newline at end of file diff --git a/lib/com/simba/googlebigquery/jdbc/google-auth-library-credentials/0.15.0/google-auth-library-credentials-0.15.0.pom b/lib/com/simba/googlebigquery/jdbc/google-auth-library-credentials/0.15.0/google-auth-library-credentials-0.15.0.pom new file mode 100644 index 00000000..411dc879 --- /dev/null +++ b/lib/com/simba/googlebigquery/jdbc/google-auth-library-credentials/0.15.0/google-auth-library-credentials-0.15.0.pom @@ -0,0 +1,8 @@ + + + 4.0.0 + com.simba.googlebigquery.jdbc + google-auth-library-credentials + 0.15.0 + diff --git a/lib/com/simba/googlebigquery/jdbc/google-auth-library-credentials/0.15.0/google-auth-library-credentials-0.15.0.pom.md5 b/lib/com/simba/googlebigquery/jdbc/google-auth-library-credentials/0.15.0/google-auth-library-credentials-0.15.0.pom.md5 new file mode 100644 index 00000000..2f7c04af --- /dev/null +++ b/lib/com/simba/googlebigquery/jdbc/google-auth-library-credentials/0.15.0/google-auth-library-credentials-0.15.0.pom.md5 @@ -0,0 +1 @@ +12891163747096e78c566d4ac99c22a1 \ No newline at end of file diff --git a/lib/com/simba/googlebigquery/jdbc/google-auth-library-credentials/0.15.0/google-auth-library-credentials-0.15.0.pom.sha1 b/lib/com/simba/googlebigquery/jdbc/google-auth-library-credentials/0.15.0/google-auth-library-credentials-0.15.0.pom.sha1 new file mode 100644 index 00000000..859ece83 --- /dev/null +++ b/lib/com/simba/googlebigquery/jdbc/google-auth-library-credentials/0.15.0/google-auth-library-credentials-0.15.0.pom.sha1 @@ -0,0 +1 @@ +2d0c561aa86f0d3775fdbfc698a49470ebf47f53 \ No newline at end of file diff --git a/lib/com/simba/googlebigquery/jdbc/google-auth-library-credentials/maven-metadata.xml b/lib/com/simba/googlebigquery/jdbc/google-auth-library-credentials/maven-metadata.xml new file mode 100644 index 00000000..1138ed8c --- /dev/null +++ b/lib/com/simba/googlebigquery/jdbc/google-auth-library-credentials/maven-metadata.xml @@ -0,0 +1,12 @@ + + + com.simba.googlebigquery.jdbc + google-auth-library-credentials + + 0.15.0 + + 0.15.0 + + 20191004152553 + + diff --git a/lib/com/simba/googlebigquery/jdbc/google-auth-library-credentials/maven-metadata.xml.md5 b/lib/com/simba/googlebigquery/jdbc/google-auth-library-credentials/maven-metadata.xml.md5 new file mode 100644 index 00000000..98ca52c9 --- /dev/null +++ b/lib/com/simba/googlebigquery/jdbc/google-auth-library-credentials/maven-metadata.xml.md5 @@ -0,0 +1 @@ +a0788d857e74a4b4b713ad5db35c4d6a \ No newline at end of file diff --git a/lib/com/simba/googlebigquery/jdbc/google-auth-library-credentials/maven-metadata.xml.sha1 b/lib/com/simba/googlebigquery/jdbc/google-auth-library-credentials/maven-metadata.xml.sha1 new file mode 100644 index 00000000..e206bd65 --- /dev/null +++ b/lib/com/simba/googlebigquery/jdbc/google-auth-library-credentials/maven-metadata.xml.sha1 @@ -0,0 +1 @@ +7bc08992abfa673469e1d68c2caa16e634ca3785 \ No newline at end of file diff --git a/lib/com/simba/googlebigquery/jdbc/google-auth-library-oauth2-http/0.13.0/google-auth-library-oauth2-http-0.13.0.jar b/lib/com/simba/googlebigquery/jdbc/google-auth-library-oauth2-http/0.13.0/google-auth-library-oauth2-http-0.13.0.jar new file mode 100644 index 00000000..2f22be95 Binary files /dev/null and b/lib/com/simba/googlebigquery/jdbc/google-auth-library-oauth2-http/0.13.0/google-auth-library-oauth2-http-0.13.0.jar differ diff --git a/lib/com/simba/googlebigquery/jdbc/google-auth-library-oauth2-http/0.13.0/google-auth-library-oauth2-http-0.13.0.jar.md5 b/lib/com/simba/googlebigquery/jdbc/google-auth-library-oauth2-http/0.13.0/google-auth-library-oauth2-http-0.13.0.jar.md5 new file mode 100644 index 00000000..e1cd2311 --- /dev/null +++ b/lib/com/simba/googlebigquery/jdbc/google-auth-library-oauth2-http/0.13.0/google-auth-library-oauth2-http-0.13.0.jar.md5 @@ -0,0 +1 @@ +b9dd0f4c45881f8eceaed743eb33967e \ No newline at end of file diff --git a/lib/com/simba/googlebigquery/jdbc/google-auth-library-oauth2-http/0.13.0/google-auth-library-oauth2-http-0.13.0.jar.sha1 b/lib/com/simba/googlebigquery/jdbc/google-auth-library-oauth2-http/0.13.0/google-auth-library-oauth2-http-0.13.0.jar.sha1 new file mode 100644 index 00000000..4c664705 --- /dev/null +++ b/lib/com/simba/googlebigquery/jdbc/google-auth-library-oauth2-http/0.13.0/google-auth-library-oauth2-http-0.13.0.jar.sha1 @@ -0,0 +1 @@ +c93cc4307226014ab6b7554af71308ba1a8f4f54 \ No newline at end of file diff --git a/lib/com/simba/googlebigquery/jdbc/google-auth-library-oauth2-http/0.13.0/google-auth-library-oauth2-http-0.13.0.pom b/lib/com/simba/googlebigquery/jdbc/google-auth-library-oauth2-http/0.13.0/google-auth-library-oauth2-http-0.13.0.pom new file mode 100644 index 00000000..0a6cbd57 --- /dev/null +++ b/lib/com/simba/googlebigquery/jdbc/google-auth-library-oauth2-http/0.13.0/google-auth-library-oauth2-http-0.13.0.pom @@ -0,0 +1,8 @@ + + + 4.0.0 + com.simba.googlebigquery.jdbc + google-auth-library-oauth2-http + 0.13.0 + diff --git a/lib/com/simba/googlebigquery/jdbc/google-auth-library-oauth2-http/0.13.0/google-auth-library-oauth2-http-0.13.0.pom.md5 b/lib/com/simba/googlebigquery/jdbc/google-auth-library-oauth2-http/0.13.0/google-auth-library-oauth2-http-0.13.0.pom.md5 new file mode 100644 index 00000000..c2408725 --- /dev/null +++ b/lib/com/simba/googlebigquery/jdbc/google-auth-library-oauth2-http/0.13.0/google-auth-library-oauth2-http-0.13.0.pom.md5 @@ -0,0 +1 @@ +e03a0e66edc6b916c59ae3fd2b2a04ce \ No newline at end of file diff --git a/lib/com/simba/googlebigquery/jdbc/google-auth-library-oauth2-http/0.13.0/google-auth-library-oauth2-http-0.13.0.pom.sha1 b/lib/com/simba/googlebigquery/jdbc/google-auth-library-oauth2-http/0.13.0/google-auth-library-oauth2-http-0.13.0.pom.sha1 new file mode 100644 index 00000000..05a338b9 --- /dev/null +++ b/lib/com/simba/googlebigquery/jdbc/google-auth-library-oauth2-http/0.13.0/google-auth-library-oauth2-http-0.13.0.pom.sha1 @@ -0,0 +1 @@ +b6b48751748ce1b872cc142a77db12d3794b4691 \ No newline at end of file diff --git a/lib/com/simba/googlebigquery/jdbc/google-auth-library-oauth2-http/maven-metadata.xml b/lib/com/simba/googlebigquery/jdbc/google-auth-library-oauth2-http/maven-metadata.xml new file mode 100644 index 00000000..1389fc70 --- /dev/null +++ b/lib/com/simba/googlebigquery/jdbc/google-auth-library-oauth2-http/maven-metadata.xml @@ -0,0 +1,12 @@ + + + com.simba.googlebigquery.jdbc + google-auth-library-oauth2-http + + 0.13.0 + + 0.13.0 + + 20191004152547 + + diff --git a/lib/com/simba/googlebigquery/jdbc/google-auth-library-oauth2-http/maven-metadata.xml.md5 b/lib/com/simba/googlebigquery/jdbc/google-auth-library-oauth2-http/maven-metadata.xml.md5 new file mode 100644 index 00000000..6f2b70d0 --- /dev/null +++ b/lib/com/simba/googlebigquery/jdbc/google-auth-library-oauth2-http/maven-metadata.xml.md5 @@ -0,0 +1 @@ +42079d356cc0c0e04cd444efec112777 \ No newline at end of file diff --git a/lib/com/simba/googlebigquery/jdbc/google-auth-library-oauth2-http/maven-metadata.xml.sha1 b/lib/com/simba/googlebigquery/jdbc/google-auth-library-oauth2-http/maven-metadata.xml.sha1 new file mode 100644 index 00000000..caaa246d --- /dev/null +++ b/lib/com/simba/googlebigquery/jdbc/google-auth-library-oauth2-http/maven-metadata.xml.sha1 @@ -0,0 +1 @@ +9077335e7edaa24c30a0beb01d981a11394fa028 \ No newline at end of file diff --git a/lib/com/simba/googlebigquery/jdbc/google-http-client-jackson2/1.28.0/google-http-client-jackson2-1.28.0.jar b/lib/com/simba/googlebigquery/jdbc/google-http-client-jackson2/1.28.0/google-http-client-jackson2-1.28.0.jar new file mode 100644 index 00000000..608e2282 Binary files /dev/null and b/lib/com/simba/googlebigquery/jdbc/google-http-client-jackson2/1.28.0/google-http-client-jackson2-1.28.0.jar differ diff --git a/lib/com/simba/googlebigquery/jdbc/google-http-client-jackson2/1.28.0/google-http-client-jackson2-1.28.0.jar.md5 b/lib/com/simba/googlebigquery/jdbc/google-http-client-jackson2/1.28.0/google-http-client-jackson2-1.28.0.jar.md5 new file mode 100644 index 00000000..baebd1b1 --- /dev/null +++ b/lib/com/simba/googlebigquery/jdbc/google-http-client-jackson2/1.28.0/google-http-client-jackson2-1.28.0.jar.md5 @@ -0,0 +1 @@ +632ba8fd2363561ffdcdf6e7b054ebec \ No newline at end of file diff --git a/lib/com/simba/googlebigquery/jdbc/google-http-client-jackson2/1.28.0/google-http-client-jackson2-1.28.0.jar.sha1 b/lib/com/simba/googlebigquery/jdbc/google-http-client-jackson2/1.28.0/google-http-client-jackson2-1.28.0.jar.sha1 new file mode 100644 index 00000000..88c31d80 --- /dev/null +++ b/lib/com/simba/googlebigquery/jdbc/google-http-client-jackson2/1.28.0/google-http-client-jackson2-1.28.0.jar.sha1 @@ -0,0 +1 @@ +ff58559c28960a7988aeab3d66dbe3a07e983187 \ No newline at end of file diff --git a/lib/com/simba/googlebigquery/jdbc/google-http-client-jackson2/1.28.0/google-http-client-jackson2-1.28.0.pom b/lib/com/simba/googlebigquery/jdbc/google-http-client-jackson2/1.28.0/google-http-client-jackson2-1.28.0.pom new file mode 100644 index 00000000..c2786f20 --- /dev/null +++ b/lib/com/simba/googlebigquery/jdbc/google-http-client-jackson2/1.28.0/google-http-client-jackson2-1.28.0.pom @@ -0,0 +1,8 @@ + + + 4.0.0 + com.simba.googlebigquery.jdbc + google-http-client-jackson2 + 1.28.0 + diff --git a/lib/com/simba/googlebigquery/jdbc/google-http-client-jackson2/1.28.0/google-http-client-jackson2-1.28.0.pom.md5 b/lib/com/simba/googlebigquery/jdbc/google-http-client-jackson2/1.28.0/google-http-client-jackson2-1.28.0.pom.md5 new file mode 100644 index 00000000..9a55e289 --- /dev/null +++ b/lib/com/simba/googlebigquery/jdbc/google-http-client-jackson2/1.28.0/google-http-client-jackson2-1.28.0.pom.md5 @@ -0,0 +1 @@ +528fb9febf66db75c5392a96551b73da \ No newline at end of file diff --git a/lib/com/simba/googlebigquery/jdbc/google-http-client-jackson2/1.28.0/google-http-client-jackson2-1.28.0.pom.sha1 b/lib/com/simba/googlebigquery/jdbc/google-http-client-jackson2/1.28.0/google-http-client-jackson2-1.28.0.pom.sha1 new file mode 100644 index 00000000..fe95ab92 --- /dev/null +++ b/lib/com/simba/googlebigquery/jdbc/google-http-client-jackson2/1.28.0/google-http-client-jackson2-1.28.0.pom.sha1 @@ -0,0 +1 @@ +084f871c2bb4861fea8c824a173744969bc65bf2 \ No newline at end of file diff --git a/lib/com/simba/googlebigquery/jdbc/google-http-client-jackson2/maven-metadata.xml b/lib/com/simba/googlebigquery/jdbc/google-http-client-jackson2/maven-metadata.xml new file mode 100644 index 00000000..452ec425 --- /dev/null +++ b/lib/com/simba/googlebigquery/jdbc/google-http-client-jackson2/maven-metadata.xml @@ -0,0 +1,12 @@ + + + com.simba.googlebigquery.jdbc + google-http-client-jackson2 + + 1.28.0 + + 1.28.0 + + 20191004152537 + + diff --git a/lib/com/simba/googlebigquery/jdbc/google-http-client-jackson2/maven-metadata.xml.md5 b/lib/com/simba/googlebigquery/jdbc/google-http-client-jackson2/maven-metadata.xml.md5 new file mode 100644 index 00000000..97626529 --- /dev/null +++ b/lib/com/simba/googlebigquery/jdbc/google-http-client-jackson2/maven-metadata.xml.md5 @@ -0,0 +1 @@ +a9fe2fc7af86dddae6ad03a896966a79 \ No newline at end of file diff --git a/lib/com/simba/googlebigquery/jdbc/google-http-client-jackson2/maven-metadata.xml.sha1 b/lib/com/simba/googlebigquery/jdbc/google-http-client-jackson2/maven-metadata.xml.sha1 new file mode 100644 index 00000000..df72a0dd --- /dev/null +++ b/lib/com/simba/googlebigquery/jdbc/google-http-client-jackson2/maven-metadata.xml.sha1 @@ -0,0 +1 @@ +7e194994c04133fa7a7db7a1bce3b97d42f87cd2 \ No newline at end of file diff --git a/lib/com/simba/googlebigquery/jdbc/google-http-client/1.29.0/google-http-client-1.29.0.jar b/lib/com/simba/googlebigquery/jdbc/google-http-client/1.29.0/google-http-client-1.29.0.jar new file mode 100644 index 00000000..7313db4f Binary files /dev/null and b/lib/com/simba/googlebigquery/jdbc/google-http-client/1.29.0/google-http-client-1.29.0.jar differ diff --git a/lib/com/simba/googlebigquery/jdbc/google-http-client/1.29.0/google-http-client-1.29.0.jar.md5 b/lib/com/simba/googlebigquery/jdbc/google-http-client/1.29.0/google-http-client-1.29.0.jar.md5 new file mode 100644 index 00000000..38bdda6e --- /dev/null +++ b/lib/com/simba/googlebigquery/jdbc/google-http-client/1.29.0/google-http-client-1.29.0.jar.md5 @@ -0,0 +1 @@ +ca383f9f24fb83cc773f962f52a95e6e \ No newline at end of file diff --git a/lib/com/simba/googlebigquery/jdbc/google-http-client/1.29.0/google-http-client-1.29.0.jar.sha1 b/lib/com/simba/googlebigquery/jdbc/google-http-client/1.29.0/google-http-client-1.29.0.jar.sha1 new file mode 100644 index 00000000..8d499926 --- /dev/null +++ b/lib/com/simba/googlebigquery/jdbc/google-http-client/1.29.0/google-http-client-1.29.0.jar.sha1 @@ -0,0 +1 @@ +ab265acd8d9c13ac73c644a50cf5a8c746aa77c6 \ No newline at end of file diff --git a/lib/com/simba/googlebigquery/jdbc/google-http-client/1.29.0/google-http-client-1.29.0.pom b/lib/com/simba/googlebigquery/jdbc/google-http-client/1.29.0/google-http-client-1.29.0.pom new file mode 100644 index 00000000..fde65869 --- /dev/null +++ b/lib/com/simba/googlebigquery/jdbc/google-http-client/1.29.0/google-http-client-1.29.0.pom @@ -0,0 +1,8 @@ + + + 4.0.0 + com.simba.googlebigquery.jdbc + google-http-client + 1.29.0 + diff --git a/lib/com/simba/googlebigquery/jdbc/google-http-client/1.29.0/google-http-client-1.29.0.pom.md5 b/lib/com/simba/googlebigquery/jdbc/google-http-client/1.29.0/google-http-client-1.29.0.pom.md5 new file mode 100644 index 00000000..80dc1bf5 --- /dev/null +++ b/lib/com/simba/googlebigquery/jdbc/google-http-client/1.29.0/google-http-client-1.29.0.pom.md5 @@ -0,0 +1 @@ +4f1749f1c588cbea0013ba6072501833 \ No newline at end of file diff --git a/lib/com/simba/googlebigquery/jdbc/google-http-client/1.29.0/google-http-client-1.29.0.pom.sha1 b/lib/com/simba/googlebigquery/jdbc/google-http-client/1.29.0/google-http-client-1.29.0.pom.sha1 new file mode 100644 index 00000000..39b4b231 --- /dev/null +++ b/lib/com/simba/googlebigquery/jdbc/google-http-client/1.29.0/google-http-client-1.29.0.pom.sha1 @@ -0,0 +1 @@ +969f8eb4405b3fc9f9f0a3a5dc9b612d870dea76 \ No newline at end of file diff --git a/lib/com/simba/googlebigquery/jdbc/google-http-client/maven-metadata.xml b/lib/com/simba/googlebigquery/jdbc/google-http-client/maven-metadata.xml new file mode 100644 index 00000000..1ecd19e1 --- /dev/null +++ b/lib/com/simba/googlebigquery/jdbc/google-http-client/maven-metadata.xml @@ -0,0 +1,12 @@ + + + com.simba.googlebigquery.jdbc + google-http-client + + 1.29.0 + + 1.29.0 + + 20191004152527 + + diff --git a/lib/com/simba/googlebigquery/jdbc/google-http-client/maven-metadata.xml.md5 b/lib/com/simba/googlebigquery/jdbc/google-http-client/maven-metadata.xml.md5 new file mode 100644 index 00000000..4225412f --- /dev/null +++ b/lib/com/simba/googlebigquery/jdbc/google-http-client/maven-metadata.xml.md5 @@ -0,0 +1 @@ +e3c666be43639cb9757f7d54891ec396 \ No newline at end of file diff --git a/lib/com/simba/googlebigquery/jdbc/google-http-client/maven-metadata.xml.sha1 b/lib/com/simba/googlebigquery/jdbc/google-http-client/maven-metadata.xml.sha1 new file mode 100644 index 00000000..97a0644c --- /dev/null +++ b/lib/com/simba/googlebigquery/jdbc/google-http-client/maven-metadata.xml.sha1 @@ -0,0 +1 @@ +83fc77d599430d01c9a955465a80d91d0284e1d4 \ No newline at end of file diff --git a/lib/com/simba/googlebigquery/jdbc/google-oauth-client/1.28.0/google-oauth-client-1.28.0.jar b/lib/com/simba/googlebigquery/jdbc/google-oauth-client/1.28.0/google-oauth-client-1.28.0.jar new file mode 100644 index 00000000..b7a40f9b Binary files /dev/null and b/lib/com/simba/googlebigquery/jdbc/google-oauth-client/1.28.0/google-oauth-client-1.28.0.jar differ diff --git a/lib/com/simba/googlebigquery/jdbc/google-oauth-client/1.28.0/google-oauth-client-1.28.0.jar.md5 b/lib/com/simba/googlebigquery/jdbc/google-oauth-client/1.28.0/google-oauth-client-1.28.0.jar.md5 new file mode 100644 index 00000000..233440fb --- /dev/null +++ b/lib/com/simba/googlebigquery/jdbc/google-oauth-client/1.28.0/google-oauth-client-1.28.0.jar.md5 @@ -0,0 +1 @@ +bb04bc90cce670b9720532760406fbca \ No newline at end of file diff --git a/lib/com/simba/googlebigquery/jdbc/google-oauth-client/1.28.0/google-oauth-client-1.28.0.jar.sha1 b/lib/com/simba/googlebigquery/jdbc/google-oauth-client/1.28.0/google-oauth-client-1.28.0.jar.sha1 new file mode 100644 index 00000000..474df6e0 --- /dev/null +++ b/lib/com/simba/googlebigquery/jdbc/google-oauth-client/1.28.0/google-oauth-client-1.28.0.jar.sha1 @@ -0,0 +1 @@ +9a9e5d0c33b663d6475c96ce79b2949545a113af \ No newline at end of file diff --git a/lib/com/simba/googlebigquery/jdbc/google-oauth-client/1.28.0/google-oauth-client-1.28.0.pom b/lib/com/simba/googlebigquery/jdbc/google-oauth-client/1.28.0/google-oauth-client-1.28.0.pom new file mode 100644 index 00000000..a8b98608 --- /dev/null +++ b/lib/com/simba/googlebigquery/jdbc/google-oauth-client/1.28.0/google-oauth-client-1.28.0.pom @@ -0,0 +1,8 @@ + + + 4.0.0 + com.simba.googlebigquery.jdbc + google-oauth-client + 1.28.0 + diff --git a/lib/com/simba/googlebigquery/jdbc/google-oauth-client/1.28.0/google-oauth-client-1.28.0.pom.md5 b/lib/com/simba/googlebigquery/jdbc/google-oauth-client/1.28.0/google-oauth-client-1.28.0.pom.md5 new file mode 100644 index 00000000..6ef807d9 --- /dev/null +++ b/lib/com/simba/googlebigquery/jdbc/google-oauth-client/1.28.0/google-oauth-client-1.28.0.pom.md5 @@ -0,0 +1 @@ +8e022693d47b0c9f409a4eff5141e286 \ No newline at end of file diff --git a/lib/com/simba/googlebigquery/jdbc/google-oauth-client/1.28.0/google-oauth-client-1.28.0.pom.sha1 b/lib/com/simba/googlebigquery/jdbc/google-oauth-client/1.28.0/google-oauth-client-1.28.0.pom.sha1 new file mode 100644 index 00000000..2554aa2c --- /dev/null +++ b/lib/com/simba/googlebigquery/jdbc/google-oauth-client/1.28.0/google-oauth-client-1.28.0.pom.sha1 @@ -0,0 +1 @@ +b73198fbc6900e4b4a2046076e37f06c805c7004 \ No newline at end of file diff --git a/lib/com/simba/googlebigquery/jdbc/google-oauth-client/maven-metadata.xml b/lib/com/simba/googlebigquery/jdbc/google-oauth-client/maven-metadata.xml new file mode 100644 index 00000000..2a36a470 --- /dev/null +++ b/lib/com/simba/googlebigquery/jdbc/google-oauth-client/maven-metadata.xml @@ -0,0 +1,12 @@ + + + com.simba.googlebigquery.jdbc + google-oauth-client + + 1.28.0 + + 1.28.0 + + 20191004152543 + + diff --git a/lib/com/simba/googlebigquery/jdbc/google-oauth-client/maven-metadata.xml.md5 b/lib/com/simba/googlebigquery/jdbc/google-oauth-client/maven-metadata.xml.md5 new file mode 100644 index 00000000..f03b6eca --- /dev/null +++ b/lib/com/simba/googlebigquery/jdbc/google-oauth-client/maven-metadata.xml.md5 @@ -0,0 +1 @@ +1f85bba0521ac3f917c32ad63aa6475d \ No newline at end of file diff --git a/lib/com/simba/googlebigquery/jdbc/google-oauth-client/maven-metadata.xml.sha1 b/lib/com/simba/googlebigquery/jdbc/google-oauth-client/maven-metadata.xml.sha1 new file mode 100644 index 00000000..78b3e2a0 --- /dev/null +++ b/lib/com/simba/googlebigquery/jdbc/google-oauth-client/maven-metadata.xml.sha1 @@ -0,0 +1 @@ +fbddd160cec3ad094fcb3314a2df6b0f324acf99 \ No newline at end of file diff --git a/lib/com/simba/googlebigquery/jdbc/grpc-context/1.18.0/grpc-context-1.18.0.jar b/lib/com/simba/googlebigquery/jdbc/grpc-context/1.18.0/grpc-context-1.18.0.jar new file mode 100644 index 00000000..4115054d Binary files /dev/null and b/lib/com/simba/googlebigquery/jdbc/grpc-context/1.18.0/grpc-context-1.18.0.jar differ diff --git a/lib/com/simba/googlebigquery/jdbc/grpc-context/1.18.0/grpc-context-1.18.0.jar.md5 b/lib/com/simba/googlebigquery/jdbc/grpc-context/1.18.0/grpc-context-1.18.0.jar.md5 new file mode 100644 index 00000000..eb976db9 --- /dev/null +++ b/lib/com/simba/googlebigquery/jdbc/grpc-context/1.18.0/grpc-context-1.18.0.jar.md5 @@ -0,0 +1 @@ +6007cf1ef04daa0219c52efa89923155 \ No newline at end of file diff --git a/lib/com/simba/googlebigquery/jdbc/grpc-context/1.18.0/grpc-context-1.18.0.jar.sha1 b/lib/com/simba/googlebigquery/jdbc/grpc-context/1.18.0/grpc-context-1.18.0.jar.sha1 new file mode 100644 index 00000000..4664e492 --- /dev/null +++ b/lib/com/simba/googlebigquery/jdbc/grpc-context/1.18.0/grpc-context-1.18.0.jar.sha1 @@ -0,0 +1 @@ +c63e8b86af0fb16b5696480dc14f48e6eaa7193b \ No newline at end of file diff --git a/lib/com/simba/googlebigquery/jdbc/grpc-context/1.18.0/grpc-context-1.18.0.pom b/lib/com/simba/googlebigquery/jdbc/grpc-context/1.18.0/grpc-context-1.18.0.pom new file mode 100644 index 00000000..0eac5490 --- /dev/null +++ b/lib/com/simba/googlebigquery/jdbc/grpc-context/1.18.0/grpc-context-1.18.0.pom @@ -0,0 +1,8 @@ + + + 4.0.0 + com.simba.googlebigquery.jdbc + grpc-context + 1.18.0 + diff --git a/lib/com/simba/googlebigquery/jdbc/grpc-context/1.18.0/grpc-context-1.18.0.pom.md5 b/lib/com/simba/googlebigquery/jdbc/grpc-context/1.18.0/grpc-context-1.18.0.pom.md5 new file mode 100644 index 00000000..27ae6246 --- /dev/null +++ b/lib/com/simba/googlebigquery/jdbc/grpc-context/1.18.0/grpc-context-1.18.0.pom.md5 @@ -0,0 +1 @@ +d67028022ed204d394f79e1055b568d7 \ No newline at end of file diff --git a/lib/com/simba/googlebigquery/jdbc/grpc-context/1.18.0/grpc-context-1.18.0.pom.sha1 b/lib/com/simba/googlebigquery/jdbc/grpc-context/1.18.0/grpc-context-1.18.0.pom.sha1 new file mode 100644 index 00000000..7a2bb1d0 --- /dev/null +++ b/lib/com/simba/googlebigquery/jdbc/grpc-context/1.18.0/grpc-context-1.18.0.pom.sha1 @@ -0,0 +1 @@ +13bc6bd8cd8cba817c7e70f767567a83d9ce42be \ No newline at end of file diff --git a/lib/com/simba/googlebigquery/jdbc/grpc-context/maven-metadata.xml b/lib/com/simba/googlebigquery/jdbc/grpc-context/maven-metadata.xml new file mode 100644 index 00000000..937217a5 --- /dev/null +++ b/lib/com/simba/googlebigquery/jdbc/grpc-context/maven-metadata.xml @@ -0,0 +1,12 @@ + + + com.simba.googlebigquery.jdbc + grpc-context + + 1.18.0 + + 1.18.0 + + 20191004152623 + + diff --git a/lib/com/simba/googlebigquery/jdbc/grpc-context/maven-metadata.xml.md5 b/lib/com/simba/googlebigquery/jdbc/grpc-context/maven-metadata.xml.md5 new file mode 100644 index 00000000..efbdebbb --- /dev/null +++ b/lib/com/simba/googlebigquery/jdbc/grpc-context/maven-metadata.xml.md5 @@ -0,0 +1 @@ +174ee142a09fca924612c9708d51aaf7 \ No newline at end of file diff --git a/lib/com/simba/googlebigquery/jdbc/grpc-context/maven-metadata.xml.sha1 b/lib/com/simba/googlebigquery/jdbc/grpc-context/maven-metadata.xml.sha1 new file mode 100644 index 00000000..83753cd2 --- /dev/null +++ b/lib/com/simba/googlebigquery/jdbc/grpc-context/maven-metadata.xml.sha1 @@ -0,0 +1 @@ +4233e48171c60d58a4d472c9f448737b1aea6753 \ No newline at end of file diff --git a/lib/com/simba/googlebigquery/jdbc/guava/26.0/guava-26.0.jar b/lib/com/simba/googlebigquery/jdbc/guava/26.0/guava-26.0.jar new file mode 100644 index 00000000..3db9b6c4 Binary files /dev/null and b/lib/com/simba/googlebigquery/jdbc/guava/26.0/guava-26.0.jar differ diff --git a/lib/com/simba/googlebigquery/jdbc/guava/26.0/guava-26.0.jar.md5 b/lib/com/simba/googlebigquery/jdbc/guava/26.0/guava-26.0.jar.md5 new file mode 100644 index 00000000..fb2a9b7e --- /dev/null +++ b/lib/com/simba/googlebigquery/jdbc/guava/26.0/guava-26.0.jar.md5 @@ -0,0 +1 @@ +4ba075ab44dcb3c9874d56ceaa602c49 \ No newline at end of file diff --git a/lib/com/simba/googlebigquery/jdbc/guava/26.0/guava-26.0.jar.sha1 b/lib/com/simba/googlebigquery/jdbc/guava/26.0/guava-26.0.jar.sha1 new file mode 100644 index 00000000..9b840139 --- /dev/null +++ b/lib/com/simba/googlebigquery/jdbc/guava/26.0/guava-26.0.jar.sha1 @@ -0,0 +1 @@ +ef69663836b339db335fde0df06fb3cd84e3742b \ No newline at end of file diff --git a/lib/com/simba/googlebigquery/jdbc/guava/26.0/guava-26.0.pom b/lib/com/simba/googlebigquery/jdbc/guava/26.0/guava-26.0.pom new file mode 100644 index 00000000..ddddaccf --- /dev/null +++ b/lib/com/simba/googlebigquery/jdbc/guava/26.0/guava-26.0.pom @@ -0,0 +1,8 @@ + + + 4.0.0 + com.simba.googlebigquery.jdbc + guava + 26.0 + diff --git a/lib/com/simba/googlebigquery/jdbc/guava/26.0/guava-26.0.pom.md5 b/lib/com/simba/googlebigquery/jdbc/guava/26.0/guava-26.0.pom.md5 new file mode 100644 index 00000000..c918cd33 --- /dev/null +++ b/lib/com/simba/googlebigquery/jdbc/guava/26.0/guava-26.0.pom.md5 @@ -0,0 +1 @@ +e0520577f39f3343fa4de414eea33f5c \ No newline at end of file diff --git a/lib/com/simba/googlebigquery/jdbc/guava/26.0/guava-26.0.pom.sha1 b/lib/com/simba/googlebigquery/jdbc/guava/26.0/guava-26.0.pom.sha1 new file mode 100644 index 00000000..9dd5808a --- /dev/null +++ b/lib/com/simba/googlebigquery/jdbc/guava/26.0/guava-26.0.pom.sha1 @@ -0,0 +1 @@ +f3a61b6eaf859dcc255f8d959199dc3a9316c0a4 \ No newline at end of file diff --git a/lib/com/simba/googlebigquery/jdbc/guava/maven-metadata.xml b/lib/com/simba/googlebigquery/jdbc/guava/maven-metadata.xml new file mode 100644 index 00000000..532c70d6 --- /dev/null +++ b/lib/com/simba/googlebigquery/jdbc/guava/maven-metadata.xml @@ -0,0 +1,12 @@ + + + com.simba.googlebigquery.jdbc + guava + + 26.0 + + 26.0 + + 20191004152605 + + diff --git a/lib/com/simba/googlebigquery/jdbc/guava/maven-metadata.xml.md5 b/lib/com/simba/googlebigquery/jdbc/guava/maven-metadata.xml.md5 new file mode 100644 index 00000000..01a402e6 --- /dev/null +++ b/lib/com/simba/googlebigquery/jdbc/guava/maven-metadata.xml.md5 @@ -0,0 +1 @@ +03364fd08dc3326dfc0d8f881b9455e2 \ No newline at end of file diff --git a/lib/com/simba/googlebigquery/jdbc/guava/maven-metadata.xml.sha1 b/lib/com/simba/googlebigquery/jdbc/guava/maven-metadata.xml.sha1 new file mode 100644 index 00000000..e41cbf70 --- /dev/null +++ b/lib/com/simba/googlebigquery/jdbc/guava/maven-metadata.xml.sha1 @@ -0,0 +1 @@ +b618173c2e5000725b61c8af2b346449a91ce47d \ No newline at end of file diff --git a/lib/com/simba/googlebigquery/jdbc/jackson-core/2.9.6/jackson-core-2.9.6.jar b/lib/com/simba/googlebigquery/jdbc/jackson-core/2.9.6/jackson-core-2.9.6.jar new file mode 100644 index 00000000..09e7dd20 Binary files /dev/null and b/lib/com/simba/googlebigquery/jdbc/jackson-core/2.9.6/jackson-core-2.9.6.jar differ diff --git a/lib/com/simba/googlebigquery/jdbc/jackson-core/2.9.6/jackson-core-2.9.6.jar.md5 b/lib/com/simba/googlebigquery/jdbc/jackson-core/2.9.6/jackson-core-2.9.6.jar.md5 new file mode 100644 index 00000000..5cb1c46a --- /dev/null +++ b/lib/com/simba/googlebigquery/jdbc/jackson-core/2.9.6/jackson-core-2.9.6.jar.md5 @@ -0,0 +1 @@ +f3cf83b839fac92307cad542c2ded5c4 \ No newline at end of file diff --git a/lib/com/simba/googlebigquery/jdbc/jackson-core/2.9.6/jackson-core-2.9.6.jar.sha1 b/lib/com/simba/googlebigquery/jdbc/jackson-core/2.9.6/jackson-core-2.9.6.jar.sha1 new file mode 100644 index 00000000..bd5af1a1 --- /dev/null +++ b/lib/com/simba/googlebigquery/jdbc/jackson-core/2.9.6/jackson-core-2.9.6.jar.sha1 @@ -0,0 +1 @@ +4e393793c37c77e042ccc7be5a914ae39251b365 \ No newline at end of file diff --git a/lib/com/simba/googlebigquery/jdbc/jackson-core/2.9.6/jackson-core-2.9.6.pom b/lib/com/simba/googlebigquery/jdbc/jackson-core/2.9.6/jackson-core-2.9.6.pom new file mode 100644 index 00000000..dd9d3bad --- /dev/null +++ b/lib/com/simba/googlebigquery/jdbc/jackson-core/2.9.6/jackson-core-2.9.6.pom @@ -0,0 +1,8 @@ + + + 4.0.0 + com.simba.googlebigquery.jdbc + jackson-core + 2.9.6 + diff --git a/lib/com/simba/googlebigquery/jdbc/jackson-core/2.9.6/jackson-core-2.9.6.pom.md5 b/lib/com/simba/googlebigquery/jdbc/jackson-core/2.9.6/jackson-core-2.9.6.pom.md5 new file mode 100644 index 00000000..84af7b48 --- /dev/null +++ b/lib/com/simba/googlebigquery/jdbc/jackson-core/2.9.6/jackson-core-2.9.6.pom.md5 @@ -0,0 +1 @@ +72dc1837a3710503360c52031f6dbc23 \ No newline at end of file diff --git a/lib/com/simba/googlebigquery/jdbc/jackson-core/2.9.6/jackson-core-2.9.6.pom.sha1 b/lib/com/simba/googlebigquery/jdbc/jackson-core/2.9.6/jackson-core-2.9.6.pom.sha1 new file mode 100644 index 00000000..8cfbfc1e --- /dev/null +++ b/lib/com/simba/googlebigquery/jdbc/jackson-core/2.9.6/jackson-core-2.9.6.pom.sha1 @@ -0,0 +1 @@ +b02bef0d597fdacdd5914ac5ef74524a4b724c28 \ No newline at end of file diff --git a/lib/com/simba/googlebigquery/jdbc/jackson-core/maven-metadata.xml b/lib/com/simba/googlebigquery/jdbc/jackson-core/maven-metadata.xml new file mode 100644 index 00000000..612d794b --- /dev/null +++ b/lib/com/simba/googlebigquery/jdbc/jackson-core/maven-metadata.xml @@ -0,0 +1,12 @@ + + + com.simba.googlebigquery.jdbc + jackson-core + + 2.9.6 + + 2.9.6 + + 20191004152558 + + diff --git a/lib/com/simba/googlebigquery/jdbc/jackson-core/maven-metadata.xml.md5 b/lib/com/simba/googlebigquery/jdbc/jackson-core/maven-metadata.xml.md5 new file mode 100644 index 00000000..90d1664c --- /dev/null +++ b/lib/com/simba/googlebigquery/jdbc/jackson-core/maven-metadata.xml.md5 @@ -0,0 +1 @@ +87a86e41987aea0ffbb11727da16299b \ No newline at end of file diff --git a/lib/com/simba/googlebigquery/jdbc/jackson-core/maven-metadata.xml.sha1 b/lib/com/simba/googlebigquery/jdbc/jackson-core/maven-metadata.xml.sha1 new file mode 100644 index 00000000..5978bc50 --- /dev/null +++ b/lib/com/simba/googlebigquery/jdbc/jackson-core/maven-metadata.xml.sha1 @@ -0,0 +1 @@ +94aad5a628bbbf4f9397b123bb28922cae44635e \ No newline at end of file diff --git a/lib/com/simba/googlebigquery/jdbc/joda-time/2.10.1/joda-time-2.10.1.jar b/lib/com/simba/googlebigquery/jdbc/joda-time/2.10.1/joda-time-2.10.1.jar new file mode 100644 index 00000000..aa55ad2b Binary files /dev/null and b/lib/com/simba/googlebigquery/jdbc/joda-time/2.10.1/joda-time-2.10.1.jar differ diff --git a/lib/com/simba/googlebigquery/jdbc/joda-time/2.10.1/joda-time-2.10.1.jar.md5 b/lib/com/simba/googlebigquery/jdbc/joda-time/2.10.1/joda-time-2.10.1.jar.md5 new file mode 100644 index 00000000..19cb816a --- /dev/null +++ b/lib/com/simba/googlebigquery/jdbc/joda-time/2.10.1/joda-time-2.10.1.jar.md5 @@ -0,0 +1 @@ +488e6b287cde4fe6142c0da65495ab63 \ No newline at end of file diff --git a/lib/com/simba/googlebigquery/jdbc/joda-time/2.10.1/joda-time-2.10.1.jar.sha1 b/lib/com/simba/googlebigquery/jdbc/joda-time/2.10.1/joda-time-2.10.1.jar.sha1 new file mode 100644 index 00000000..75e80975 --- /dev/null +++ b/lib/com/simba/googlebigquery/jdbc/joda-time/2.10.1/joda-time-2.10.1.jar.sha1 @@ -0,0 +1 @@ +9ac3dbf89dbf2ee385185dd0cd3064fe789efee0 \ No newline at end of file diff --git a/lib/com/simba/googlebigquery/jdbc/joda-time/2.10.1/joda-time-2.10.1.pom b/lib/com/simba/googlebigquery/jdbc/joda-time/2.10.1/joda-time-2.10.1.pom new file mode 100644 index 00000000..4a10f368 --- /dev/null +++ b/lib/com/simba/googlebigquery/jdbc/joda-time/2.10.1/joda-time-2.10.1.pom @@ -0,0 +1,8 @@ + + + 4.0.0 + com.simba.googlebigquery.jdbc + joda-time + 2.10.1 + diff --git a/lib/com/simba/googlebigquery/jdbc/joda-time/2.10.1/joda-time-2.10.1.pom.md5 b/lib/com/simba/googlebigquery/jdbc/joda-time/2.10.1/joda-time-2.10.1.pom.md5 new file mode 100644 index 00000000..71e0ff6d --- /dev/null +++ b/lib/com/simba/googlebigquery/jdbc/joda-time/2.10.1/joda-time-2.10.1.pom.md5 @@ -0,0 +1 @@ +a6d7baa3198ee02d19aae03870284b3f \ No newline at end of file diff --git a/lib/com/simba/googlebigquery/jdbc/joda-time/2.10.1/joda-time-2.10.1.pom.sha1 b/lib/com/simba/googlebigquery/jdbc/joda-time/2.10.1/joda-time-2.10.1.pom.sha1 new file mode 100644 index 00000000..29de1c26 --- /dev/null +++ b/lib/com/simba/googlebigquery/jdbc/joda-time/2.10.1/joda-time-2.10.1.pom.sha1 @@ -0,0 +1 @@ +c1889f4f49a433c5dfd037ce74ed4cd8f8e824e7 \ No newline at end of file diff --git a/lib/com/simba/googlebigquery/jdbc/joda-time/maven-metadata.xml b/lib/com/simba/googlebigquery/jdbc/joda-time/maven-metadata.xml new file mode 100644 index 00000000..14d1b43c --- /dev/null +++ b/lib/com/simba/googlebigquery/jdbc/joda-time/maven-metadata.xml @@ -0,0 +1,12 @@ + + + com.simba.googlebigquery.jdbc + joda-time + + 2.10.1 + + 2.10.1 + + 20191004152629 + + diff --git a/lib/com/simba/googlebigquery/jdbc/joda-time/maven-metadata.xml.md5 b/lib/com/simba/googlebigquery/jdbc/joda-time/maven-metadata.xml.md5 new file mode 100644 index 00000000..2f6c3104 --- /dev/null +++ b/lib/com/simba/googlebigquery/jdbc/joda-time/maven-metadata.xml.md5 @@ -0,0 +1 @@ +14e689e392d3a0562536d1866783e488 \ No newline at end of file diff --git a/lib/com/simba/googlebigquery/jdbc/joda-time/maven-metadata.xml.sha1 b/lib/com/simba/googlebigquery/jdbc/joda-time/maven-metadata.xml.sha1 new file mode 100644 index 00000000..ca18c564 --- /dev/null +++ b/lib/com/simba/googlebigquery/jdbc/joda-time/maven-metadata.xml.sha1 @@ -0,0 +1 @@ +bd84247a75e9f0ad4518ba4e31634a630cc2d10e \ No newline at end of file diff --git a/lib/com/simba/googlebigquery/jdbc/opencensus-api/0.18.0/opencensus-api-0.18.0.jar b/lib/com/simba/googlebigquery/jdbc/opencensus-api/0.18.0/opencensus-api-0.18.0.jar new file mode 100644 index 00000000..5c9afd32 Binary files /dev/null and b/lib/com/simba/googlebigquery/jdbc/opencensus-api/0.18.0/opencensus-api-0.18.0.jar differ diff --git a/lib/com/simba/googlebigquery/jdbc/opencensus-api/0.18.0/opencensus-api-0.18.0.jar.md5 b/lib/com/simba/googlebigquery/jdbc/opencensus-api/0.18.0/opencensus-api-0.18.0.jar.md5 new file mode 100644 index 00000000..170894ed --- /dev/null +++ b/lib/com/simba/googlebigquery/jdbc/opencensus-api/0.18.0/opencensus-api-0.18.0.jar.md5 @@ -0,0 +1 @@ +059407e5c0e8a4b9d815c3bc6448a769 \ No newline at end of file diff --git a/lib/com/simba/googlebigquery/jdbc/opencensus-api/0.18.0/opencensus-api-0.18.0.jar.sha1 b/lib/com/simba/googlebigquery/jdbc/opencensus-api/0.18.0/opencensus-api-0.18.0.jar.sha1 new file mode 100644 index 00000000..8b95ab4e --- /dev/null +++ b/lib/com/simba/googlebigquery/jdbc/opencensus-api/0.18.0/opencensus-api-0.18.0.jar.sha1 @@ -0,0 +1 @@ +b89a8f8dfd1e1e0d68d83c82a855624814b19a6e \ No newline at end of file diff --git a/lib/com/simba/googlebigquery/jdbc/opencensus-api/0.18.0/opencensus-api-0.18.0.pom b/lib/com/simba/googlebigquery/jdbc/opencensus-api/0.18.0/opencensus-api-0.18.0.pom new file mode 100644 index 00000000..4e1851a7 --- /dev/null +++ b/lib/com/simba/googlebigquery/jdbc/opencensus-api/0.18.0/opencensus-api-0.18.0.pom @@ -0,0 +1,8 @@ + + + 4.0.0 + com.simba.googlebigquery.jdbc + opencensus-api + 0.18.0 + diff --git a/lib/com/simba/googlebigquery/jdbc/opencensus-api/0.18.0/opencensus-api-0.18.0.pom.md5 b/lib/com/simba/googlebigquery/jdbc/opencensus-api/0.18.0/opencensus-api-0.18.0.pom.md5 new file mode 100644 index 00000000..2b387a1e --- /dev/null +++ b/lib/com/simba/googlebigquery/jdbc/opencensus-api/0.18.0/opencensus-api-0.18.0.pom.md5 @@ -0,0 +1 @@ +21c0fda2f9905439d5d04ad86c89294a \ No newline at end of file diff --git a/lib/com/simba/googlebigquery/jdbc/opencensus-api/0.18.0/opencensus-api-0.18.0.pom.sha1 b/lib/com/simba/googlebigquery/jdbc/opencensus-api/0.18.0/opencensus-api-0.18.0.pom.sha1 new file mode 100644 index 00000000..4692552f --- /dev/null +++ b/lib/com/simba/googlebigquery/jdbc/opencensus-api/0.18.0/opencensus-api-0.18.0.pom.sha1 @@ -0,0 +1 @@ +80ec0e0d4cf4b47fbf545fcdc63610d5f5fbae41 \ No newline at end of file diff --git a/lib/com/simba/googlebigquery/jdbc/opencensus-api/maven-metadata.xml b/lib/com/simba/googlebigquery/jdbc/opencensus-api/maven-metadata.xml new file mode 100644 index 00000000..ec6d05c6 --- /dev/null +++ b/lib/com/simba/googlebigquery/jdbc/opencensus-api/maven-metadata.xml @@ -0,0 +1,12 @@ + + + com.simba.googlebigquery.jdbc + opencensus-api + + 0.18.0 + + 0.18.0 + + 20191004152613 + + diff --git a/lib/com/simba/googlebigquery/jdbc/opencensus-api/maven-metadata.xml.md5 b/lib/com/simba/googlebigquery/jdbc/opencensus-api/maven-metadata.xml.md5 new file mode 100644 index 00000000..fb15102b --- /dev/null +++ b/lib/com/simba/googlebigquery/jdbc/opencensus-api/maven-metadata.xml.md5 @@ -0,0 +1 @@ +63c89c19121cd01bfb4c591fbffe17d9 \ No newline at end of file diff --git a/lib/com/simba/googlebigquery/jdbc/opencensus-api/maven-metadata.xml.sha1 b/lib/com/simba/googlebigquery/jdbc/opencensus-api/maven-metadata.xml.sha1 new file mode 100644 index 00000000..efd053ba --- /dev/null +++ b/lib/com/simba/googlebigquery/jdbc/opencensus-api/maven-metadata.xml.sha1 @@ -0,0 +1 @@ +d1cbe8acedaa95b0cc1930bebbec3ecca1b2ab94 \ No newline at end of file diff --git a/lib/com/simba/googlebigquery/jdbc/opencensus-contrib-http-util/0.18.0/opencensus-contrib-http-util-0.18.0.jar b/lib/com/simba/googlebigquery/jdbc/opencensus-contrib-http-util/0.18.0/opencensus-contrib-http-util-0.18.0.jar new file mode 100644 index 00000000..48aa45cc Binary files /dev/null and b/lib/com/simba/googlebigquery/jdbc/opencensus-contrib-http-util/0.18.0/opencensus-contrib-http-util-0.18.0.jar differ diff --git a/lib/com/simba/googlebigquery/jdbc/opencensus-contrib-http-util/0.18.0/opencensus-contrib-http-util-0.18.0.jar.md5 b/lib/com/simba/googlebigquery/jdbc/opencensus-contrib-http-util/0.18.0/opencensus-contrib-http-util-0.18.0.jar.md5 new file mode 100644 index 00000000..71d0f2b5 --- /dev/null +++ b/lib/com/simba/googlebigquery/jdbc/opencensus-contrib-http-util/0.18.0/opencensus-contrib-http-util-0.18.0.jar.md5 @@ -0,0 +1 @@ +ae15cdc3ea04218c9ec14708cad2db94 \ No newline at end of file diff --git a/lib/com/simba/googlebigquery/jdbc/opencensus-contrib-http-util/0.18.0/opencensus-contrib-http-util-0.18.0.jar.sha1 b/lib/com/simba/googlebigquery/jdbc/opencensus-contrib-http-util/0.18.0/opencensus-contrib-http-util-0.18.0.jar.sha1 new file mode 100644 index 00000000..1757e005 --- /dev/null +++ b/lib/com/simba/googlebigquery/jdbc/opencensus-contrib-http-util/0.18.0/opencensus-contrib-http-util-0.18.0.jar.sha1 @@ -0,0 +1 @@ +76a37e4a931d5801a9e25b0c0353e5f37c4d1e8e \ No newline at end of file diff --git a/lib/com/simba/googlebigquery/jdbc/opencensus-contrib-http-util/0.18.0/opencensus-contrib-http-util-0.18.0.pom b/lib/com/simba/googlebigquery/jdbc/opencensus-contrib-http-util/0.18.0/opencensus-contrib-http-util-0.18.0.pom new file mode 100644 index 00000000..21f172aa --- /dev/null +++ b/lib/com/simba/googlebigquery/jdbc/opencensus-contrib-http-util/0.18.0/opencensus-contrib-http-util-0.18.0.pom @@ -0,0 +1,8 @@ + + + 4.0.0 + com.simba.googlebigquery.jdbc + opencensus-contrib-http-util + 0.18.0 + diff --git a/lib/com/simba/googlebigquery/jdbc/opencensus-contrib-http-util/0.18.0/opencensus-contrib-http-util-0.18.0.pom.md5 b/lib/com/simba/googlebigquery/jdbc/opencensus-contrib-http-util/0.18.0/opencensus-contrib-http-util-0.18.0.pom.md5 new file mode 100644 index 00000000..9c9331ec --- /dev/null +++ b/lib/com/simba/googlebigquery/jdbc/opencensus-contrib-http-util/0.18.0/opencensus-contrib-http-util-0.18.0.pom.md5 @@ -0,0 +1 @@ +109dec0822afe75c7a5e4b880f311fc7 \ No newline at end of file diff --git a/lib/com/simba/googlebigquery/jdbc/opencensus-contrib-http-util/0.18.0/opencensus-contrib-http-util-0.18.0.pom.sha1 b/lib/com/simba/googlebigquery/jdbc/opencensus-contrib-http-util/0.18.0/opencensus-contrib-http-util-0.18.0.pom.sha1 new file mode 100644 index 00000000..4befb082 --- /dev/null +++ b/lib/com/simba/googlebigquery/jdbc/opencensus-contrib-http-util/0.18.0/opencensus-contrib-http-util-0.18.0.pom.sha1 @@ -0,0 +1 @@ +6159772dc464e2435ce5d806162593409293394a \ No newline at end of file diff --git a/lib/com/simba/googlebigquery/jdbc/opencensus-contrib-http-util/maven-metadata.xml b/lib/com/simba/googlebigquery/jdbc/opencensus-contrib-http-util/maven-metadata.xml new file mode 100644 index 00000000..9f356dfa --- /dev/null +++ b/lib/com/simba/googlebigquery/jdbc/opencensus-contrib-http-util/maven-metadata.xml @@ -0,0 +1,12 @@ + + + com.simba.googlebigquery.jdbc + opencensus-contrib-http-util + + 0.18.0 + + 0.18.0 + + 20191004152618 + + diff --git a/lib/com/simba/googlebigquery/jdbc/opencensus-contrib-http-util/maven-metadata.xml.md5 b/lib/com/simba/googlebigquery/jdbc/opencensus-contrib-http-util/maven-metadata.xml.md5 new file mode 100644 index 00000000..7f0c4d13 --- /dev/null +++ b/lib/com/simba/googlebigquery/jdbc/opencensus-contrib-http-util/maven-metadata.xml.md5 @@ -0,0 +1 @@ +a8bf8f5b88a5e5976d6b1c86a233a415 \ No newline at end of file diff --git a/lib/com/simba/googlebigquery/jdbc/opencensus-contrib-http-util/maven-metadata.xml.sha1 b/lib/com/simba/googlebigquery/jdbc/opencensus-contrib-http-util/maven-metadata.xml.sha1 new file mode 100644 index 00000000..d8a3e4d1 --- /dev/null +++ b/lib/com/simba/googlebigquery/jdbc/opencensus-contrib-http-util/maven-metadata.xml.sha1 @@ -0,0 +1 @@ +fc44a8f5a3b71431f543f71a67e2034d5f60cc89 \ No newline at end of file diff --git a/pom.xml b/pom.xml index 667b9947..d3b9d658 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ org.ohdsi leporidae pom - 0.8.0-SNAPSHOT + 0.9.0 rabbitinahat whiterabbit diff --git a/rabbit-core/pom.xml b/rabbit-core/pom.xml index 41d29da7..c8735e02 100644 --- a/rabbit-core/pom.xml +++ b/rabbit-core/pom.xml @@ -5,7 +5,7 @@ leporidae org.ohdsi - 0.8.0-SNAPSHOT + 0.9.0 4.0.0 @@ -118,5 +118,90 @@ tdgssconfig 16.0.0.0 + + com.simba.googlebigquery.jdbc + GoogleBigQueryJDBC + 42 + + + com.simba.googlebigquery.jdbc + google-api-client + 1.28.0 + + + com.simba.googlebigquery.jdbc + google-http-client + 1.29.0 + + + com.simba.googlebigquery.jdbc + gax + 1.42.0 + + + com.simba.googlebigquery.jdbc + google-http-client-jackson2 + 1.28.0 + + + com.simba.googlebigquery.jdbc + google-oauth-client + 1.28.0 + + + com.simba.googlebigquery.jdbc + google-auth-library-oauth2-http + 0.13.0 + + + com.simba.googlebigquery.jdbc + google-auth-library-credentials + 0.15.0 + + + com.simba.googlebigquery.jdbc + jackson-core + 2.9.6 + + + com.simba.googlebigquery.jdbc + guava + 26.0 + + + com.simba.googlebigquery.jdbc + google-api-service-bigquery + V2-rev426-1.25.0 + + + com.simba.googlebigquery.jdbc + opencensus-api + 0.18.0 + + + com.simba.googlebigquery.jdbc + opencensus-contrib-http-util + 0.18.0 + + + com.simba.googlebigquery.jdbc + grpc-context + 1.18.0 + + + com.simba.googlebigquery.jdbc + joda-time + 2.10.1 + + + com.simba.googlebigquery.jdbc + avro + 1.8.2 + + + com.epam + parso + 2.0 + - \ No newline at end of file + diff --git a/rabbit-core/src/main/java/org/ohdsi/databases/DBConnector.java b/rabbit-core/src/main/java/org/ohdsi/databases/DBConnector.java index e44b07a5..837ba490 100644 --- a/rabbit-core/src/main/java/org/ohdsi/databases/DBConnector.java +++ b/rabbit-core/src/main/java/org/ohdsi/databases/DBConnector.java @@ -20,14 +20,18 @@ import java.sql.Connection; import java.sql.DriverManager; import java.sql.SQLException; +import java.util.regex.Matcher; +import java.util.regex.Pattern; import oracle.jdbc.pool.OracleDataSource; +import org.apache.tools.ant.types.selectors.SelectSelector; public class DBConnector { public static void main(String[] args) { } + // If dbType.BIGQUERY: domain field has been replaced with database field public static Connection connect(String server, String domain, String user, String password, DbType dbType) { if (dbType.equals(DbType.MYSQL)) return DBConnector.connectToMySQL(server, user, password); @@ -43,6 +47,8 @@ else if (dbType.equals(DbType.REDSHIFT)) return DBConnector.connectToRedshift(server, user, password); else if (dbType.equals(DbType.TERADATA)) return DBConnector.connectToTeradata(server, user, password); + else if (dbType.equals(DbType.BIGQUERY)) + return DBConnector.connectToBigQuery(server, domain, user, password); else return null; } @@ -216,4 +222,34 @@ public static Connection connectToOracle(String server, String domain, String us } return null; } + + public static Connection connectToBigQuery(String server, String domain, String user, String password) { + try { + Class.forName("com.simba.googlebigquery.jdbc42.Driver"); + } catch (ClassNotFoundException e1) { + throw new RuntimeException("Cannot find Simba GoogleBigQuery JDBC Driver class"); + } + /* See http://howtodojava.com/regex/java-regex-validate-email.address/ */ + String email_regex = "^[\\w!#$%&'*+/=?`{|}~^-]+(?:\\.[\\w!#$%&'*+/=?`{|}~^-]+)*@(?:[a-zA-Z0-9-]+\\.)+[a-zA-Z]{2,6}$"; + Pattern pattern = Pattern.compile(email_regex); + Matcher matcher = pattern.matcher(user); + Integer timeout = 3600; + String url = ""; + if (matcher.matches()) { + /* use Service Account authentication (less secure - no auditing events to stackdriver) */ + url = "jdbc:bigquery://https://www.googleapis.com/bigquery/v2:443;ProjectID=" + server + ";OAuthType=0;OAuthServiceAcctEmail=" + user + ";OAuthPvtKeyPath=" + password + ";DefaultDataset=" + domain + ";Timeout=" + timeout + ";"; + } else { + /* use application default credentials (more secure - writes auditing events to stackdriver) */ + /* requires user to run: 'gcloud auth application-default login' */ + /* only once on each computer. Writes application key to ~/.config/gcloud/application_default_credentials.json */ + /* See https://cloud.google.com/sdk/gcloud/reference/auth/application-default/ for documentation */ + url = "jdbc:bigquery://https://www.googleapis.com/bigquery/v2:443;ProjectID=" + server + ";OAuthType=3;DefaultDataset=" + domain + ";Timeout=" + timeout + ";"; + }; + try { + return DriverManager.getConnection(url); + } catch (SQLException e1) { + throw new RuntimeException("Simba URL failed: Cannot connect to DB server: " + e1.getMessage()); + } + } } + diff --git a/rabbit-core/src/main/java/org/ohdsi/databases/DbType.java b/rabbit-core/src/main/java/org/ohdsi/databases/DbType.java index 4d995c84..e58ef673 100644 --- a/rabbit-core/src/main/java/org/ohdsi/databases/DbType.java +++ b/rabbit-core/src/main/java/org/ohdsi/databases/DbType.java @@ -26,9 +26,10 @@ public class DbType { public static DbType MSACCESS = new DbType("msaccess"); public static DbType REDSHIFT = new DbType("redshift"); public static DbType TERADATA = new DbType("teradata"); + public static DbType BIGQUERY = new DbType("bigquery"); private enum Type { - MYSQL, MSSQL, PDW, ORACLE, POSTGRESQL, MSACCESS, REDSHIFT, TERADATA + MYSQL, MSSQL, PDW, ORACLE, POSTGRESQL, MSACCESS, REDSHIFT, TERADATA, BIGQUERY }; private Type type; diff --git a/rabbit-core/src/main/java/org/ohdsi/databases/RichConnection.java b/rabbit-core/src/main/java/org/ohdsi/databases/RichConnection.java index a390dc5e..1f5b5533 100644 --- a/rabbit-core/src/main/java/org/ohdsi/databases/RichConnection.java +++ b/rabbit-core/src/main/java/org/ohdsi/databases/RichConnection.java @@ -17,6 +17,7 @@ ******************************************************************************/ package org.ohdsi.databases; +import java.io.Closeable; import java.sql.BatchUpdateException; import java.sql.Connection; import java.sql.DatabaseMetaData; @@ -38,7 +39,7 @@ import org.ohdsi.utilities.files.Row; import org.ohdsi.utilities.files.WriteCSVFileWithHeader; -public class RichConnection { +public class RichConnection implements Closeable { public static int INSERT_BATCH_SIZE = 100000; private Connection connection; private boolean verbose = false; @@ -131,7 +132,7 @@ public void use(String database) { execute("ALTER SESSION SET current_schema = " + database); else if (dbType == DbType.POSTGRESQL || dbType == DbType.REDSHIFT) execute("SET search_path TO " + database); - else if (dbType == DbType.MSACCESS) + else if (dbType == DbType.MSACCESS || dbType == DbType.BIGQUERY) ; // NOOP else if (dbType == DbType.TERADATA) { execute("database " + database); @@ -154,6 +155,8 @@ public List getTableNames(String database) { query = "SELECT Name FROM sys.MSysObjects WHERE Type=1 AND Flags=0;"; } else if (dbType == DbType.TERADATA) { query = "SELECT TableName from dbc.tables WHERE tablekind = 'T' and databasename='" + database + "'"; + } else if (dbType == DbType.BIGQUERY) { + query = "SELECT table_name from " + database + ".INFORMATION_SCHEMA.TABLES ORDER BY table_name;"; } for (Row row : query(query)) diff --git a/rabbit-core/src/main/java/org/ohdsi/rabbitInAHat/dataModel/ConceptsMap.java b/rabbit-core/src/main/java/org/ohdsi/rabbitInAHat/dataModel/ConceptsMap.java new file mode 100644 index 00000000..1fe88278 --- /dev/null +++ b/rabbit-core/src/main/java/org/ohdsi/rabbitInAHat/dataModel/ConceptsMap.java @@ -0,0 +1,103 @@ +/******************************************************************************* + * Copyright 2019 Observational Health Data Sciences and Informatics & The Hyve + * + * This file is part of WhiteRabbit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + ******************************************************************************/ +package org.ohdsi.rabbitInAHat.dataModel; + +import org.apache.commons.csv.CSVFormat; +import org.apache.commons.csv.CSVRecord; + +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.util.*; + +public class ConceptsMap { + + private Map>> conceptMap; + + private ConceptsMap() { + this.conceptMap = new HashMap<>(); + } + + ConceptsMap(String filename) throws IOException{ + this(); + this.load(filename); + } + + private void load(String filename) throws IOException{ + try (InputStream conceptStream = Database.class.getResourceAsStream(filename)) { + for (CSVRecord conceptRow : CSVFormat.RFC4180.withHeader().parse(new InputStreamReader(conceptStream))) { + String tableName = conceptRow.get("omop_cdm_table"); + String fieldName = conceptRow.get("omop_cdm_field"); + + Concept concept = new Concept(); + concept.setConceptId(conceptRow.get("concept_id")); + concept.setConceptName(conceptRow.get("concept_name")); + concept.setStandardConcept(conceptRow.get("standard_concept")); + + this.put(tableName, fieldName, concept); + } + } catch (IOException e) { + throw new IOException("Could not load concept_id hints: " + e.getMessage()); + } + } + + public void put(String targetTableName, String targetFieldName, Concept concept) { + this.conceptMap + .computeIfAbsent(targetTableName, t -> new HashMap<>()) + .computeIfAbsent(targetFieldName, t -> new ArrayList<>()) + .add(concept); + } + + public List get(String targetTable, String targetField) { + return conceptMap.getOrDefault(targetTable, Collections.emptyMap()).get(targetField); + } + + public static class Concept { + private String conceptId; + private String conceptName; + private String standardConcept; + + public String getConceptId() { + return conceptId; + } + + void setConceptId(String conceptId) { + this.conceptId = conceptId; + } + + public String getConceptName() { + return conceptName; + } + + void setConceptName(String conceptName) { + this.conceptName = conceptName; + } + + public String getStandardConcept() { + return standardConcept; + } + + void setStandardConcept(String standardConcept) { + this.standardConcept = standardConcept; + } + + public String toString() { + return this.conceptId + " -- " + this.conceptName; + } + } +} diff --git a/rabbit-core/src/main/java/org/ohdsi/rabbitInAHat/dataModel/Database.java b/rabbit-core/src/main/java/org/ohdsi/rabbitInAHat/dataModel/Database.java index 2dddb93c..364643d7 100644 --- a/rabbit-core/src/main/java/org/ohdsi/rabbitInAHat/dataModel/Database.java +++ b/rabbit-core/src/main/java/org/ohdsi/rabbitInAHat/dataModel/Database.java @@ -21,11 +21,7 @@ import java.io.InputStream; import java.io.InputStreamReader; import java.io.Serializable; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.Iterator; -import java.util.List; -import java.util.Map; +import java.util.*; import org.apache.commons.csv.CSVFormat; import org.apache.commons.csv.CSVRecord; @@ -47,6 +43,7 @@ public enum CDMVersion { private List tables = new ArrayList
(); private static final long serialVersionUID = -3912166654601191039L; private String dbName = ""; + private static String CONCEPT_ID_HINTS_FILE_NAME = "CDMConceptIDHints_v5.0_MAR-18.csv"; public List
getTables() { return tables; @@ -76,8 +73,9 @@ public static Database generateModelFromCSV(InputStream stream, String dbName) { database.dbName = dbName.substring(0, dbName.lastIndexOf(".")); - Map nameToTable = new HashMap(); + Map nameToTable = new HashMap<>(); try { + ConceptsMap conceptsMap = new ConceptsMap(CONCEPT_ID_HINTS_FILE_NAME); for (CSVRecord row : CSVFormat.RFC4180.withHeader().parse(new InputStreamReader(stream))) { String tableNameColumn; @@ -114,17 +112,13 @@ public static Database generateModelFromCSV(InputStream stream, String dbName) { field.setNullable(row.get(isNullableColumn).equals(nullableValue)); field.setType(row.get(dataTypeColumn)); field.setDescription(row.get(descriptionColumn)); + field.setConceptIdHints(conceptsMap.get(table.getName(), field.getName())); + table.getFields().add(field); } } catch (IOException e) { throw new RuntimeException(e.getMessage()); } - // Collections.sort(database.tables, new Comparator
() { - // - // @Override - // public int compare(Table o1, Table o2) { - // return o1.getName().compareTo(o2.getName()); - // }}); return database; } @@ -147,6 +141,7 @@ public static Database generateModelFromScanReport(String filename) { table = new Table(); table.setName(tableName.toLowerCase()); table.setRowCount((int) Double.parseDouble(row.get(fieldName2ColumnIndex.get("N rows")))); + table.setRowsCheckedCount((int) Double.parseDouble(row.get(fieldName2ColumnIndex.get("N rows checked")))); nameToTable.put(tableName, table); database.tables.add(table); } diff --git a/rabbit-core/src/main/java/org/ohdsi/rabbitInAHat/dataModel/Field.java b/rabbit-core/src/main/java/org/ohdsi/rabbitInAHat/dataModel/Field.java index becf3657..ed28fd8b 100644 --- a/rabbit-core/src/main/java/org/ohdsi/rabbitInAHat/dataModel/Field.java +++ b/rabbit-core/src/main/java/org/ohdsi/rabbitInAHat/dataModel/Field.java @@ -17,6 +17,8 @@ ******************************************************************************/ package org.ohdsi.rabbitInAHat.dataModel; +import java.util.List; + public class Field implements MappableItem { private static final long serialVersionUID = 3687778470032619497L; @@ -29,6 +31,7 @@ public class Field implements MappableItem { private String description = ""; private int maxLength; private boolean isStem; + private List conceptIdHints; public Field(String name, Table table) { this.table = table; @@ -71,6 +74,10 @@ public void setValueCounts(String[][] valueCounts) { this.valueCounts = valueCounts; } + public int getRowsCheckedCount() { + return this.table.getRowsCheckedCount(); + } + public boolean isNullable() { return isNullable; } @@ -122,4 +129,12 @@ public boolean isStem() { public void setStem(boolean isStem) { this.isStem = isStem; } + + public List getConceptIdHints() { + return conceptIdHints; + } + + public void setConceptIdHints(List conceptIdHints) { + this.conceptIdHints = conceptIdHints; + } } diff --git a/rabbit-core/src/main/java/org/ohdsi/rabbitInAHat/dataModel/Mapping.java b/rabbit-core/src/main/java/org/ohdsi/rabbitInAHat/dataModel/Mapping.java index 11225d06..a2e3a299 100644 --- a/rabbit-core/src/main/java/org/ohdsi/rabbitInAHat/dataModel/Mapping.java +++ b/rabbit-core/src/main/java/org/ohdsi/rabbitInAHat/dataModel/Mapping.java @@ -18,7 +18,6 @@ package org.ohdsi.rabbitInAHat.dataModel; import java.util.ArrayList; -import java.util.Iterator; import java.util.List; public class Mapping { @@ -73,19 +72,37 @@ public List getSourceToTargetMaps() { return sourceToCdmMaps; } - public void removeSourceToTargetMap(MappableItem sourceItem, MappableItem targetItem) { - Iterator iterator = sourceToCdmMaps.iterator(); - while (iterator.hasNext()) { - ItemToItemMap sourceToTargetMap = iterator.next(); - if (sourceToTargetMap.getSourceItem().equals(sourceItem) && sourceToTargetMap.getTargetItem().equals(targetItem)) - iterator.remove(); + public List getSourceToTargetMapsOrderedByCdmItems() { + List result = new ArrayList<>(); + for (MappableItem targetItem : cdmItems) { + boolean sourceFound = false; + for (MappableItem sourceItem : sourceItems) { + ItemToItemMap mapping = getSourceToTargetMap(sourceItem, targetItem); + if (mapping != null) { + result.add(mapping); + sourceFound = true; + } + } + if (!sourceFound) + result.add(null); } + +// result.removeAll(Collections.singleton(null)); + return result; + } + + public void removeSourceToTargetMap(MappableItem sourceItem, MappableItem targetItem) { + sourceToCdmMaps.removeIf(sourceToTargetMap -> + sourceToTargetMap.getSourceItem().equals(sourceItem) && sourceToTargetMap.getTargetItem().equals(targetItem) + ); + } + + public void removeAllSourceToTargetMaps() { + sourceToCdmMaps.clear(); } public ItemToItemMap getSourceToTargetMap(MappableItem sourceItem, MappableItem targetItem) { - Iterator iterator = sourceToCdmMaps.iterator(); - while (iterator.hasNext()) { - ItemToItemMap sourceToTargetMap = iterator.next(); + for (ItemToItemMap sourceToTargetMap : sourceToCdmMaps) { if (sourceToTargetMap.getSourceItem().equals(sourceItem) && sourceToTargetMap.getTargetItem().equals(targetItem)) return sourceToTargetMap; } @@ -93,10 +110,7 @@ public ItemToItemMap getSourceToTargetMap(MappableItem sourceItem, MappableItem } public ItemToItemMap getSourceToTargetMapByName(MappableItem sourceItem, MappableItem targetItem) { - Iterator iterator = sourceToCdmMaps.iterator(); - - while (iterator.hasNext()) { - ItemToItemMap sourceToTargetMap = iterator.next(); + for (ItemToItemMap sourceToTargetMap : sourceToCdmMaps) { if (sourceToTargetMap.getSourceItem().getName().equals(sourceItem.getName()) && sourceToTargetMap.getTargetItem().getName().equals(targetItem.getName())) return sourceToTargetMap; diff --git a/rabbit-core/src/main/java/org/ohdsi/rabbitInAHat/dataModel/Table.java b/rabbit-core/src/main/java/org/ohdsi/rabbitInAHat/dataModel/Table.java index 07efe38e..9bc2ccd1 100644 --- a/rabbit-core/src/main/java/org/ohdsi/rabbitInAHat/dataModel/Table.java +++ b/rabbit-core/src/main/java/org/ohdsi/rabbitInAHat/dataModel/Table.java @@ -25,6 +25,7 @@ public class Table implements MappableItem { private Database db; private String name; private int rowCount; + private int rowsCheckedCount; private String comment = ""; private List fields = new ArrayList(); private boolean isStem = false; @@ -39,6 +40,7 @@ public Table(Table table) { db = table.db; name = table.name; rowCount = table.rowCount; + rowsCheckedCount = table.rowsCheckedCount; comment = table.comment; fields = new ArrayList(table.fields); isStem = table.isStem; @@ -72,6 +74,14 @@ public void setRowCount(int rowCount) { this.rowCount = rowCount; } + public int getRowsCheckedCount() { + return rowsCheckedCount; + } + + public void setRowsCheckedCount(int rowsCheckedCount) { + this.rowsCheckedCount = rowsCheckedCount; + } + public List getFields() { return fields; } diff --git a/rabbit-core/src/main/java/org/ohdsi/utilities/collections/CountingSet.java b/rabbit-core/src/main/java/org/ohdsi/utilities/collections/CountingSet.java index 5935417d..a70241c5 100644 --- a/rabbit-core/src/main/java/org/ohdsi/utilities/collections/CountingSet.java +++ b/rabbit-core/src/main/java/org/ohdsi/utilities/collections/CountingSet.java @@ -17,16 +17,13 @@ ******************************************************************************/ package org.ohdsi.utilities.collections; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; +import java.util.AbstractSet; import java.util.Comparator; import java.util.HashMap; import java.util.Iterator; -import java.util.List; import java.util.Map; -import java.util.Map.Entry; -import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.Stream; /** * Class for counting recurring objects. @@ -34,24 +31,27 @@ * @author schuemie * @param */ -public class CountingSet implements Set { +public class CountingSet extends AbstractSet { public Map key2count; public CountingSet() { - key2count = new HashMap(); + key2count = new HashMap<>(); } public CountingSet(int capacity) { - key2count = new HashMap(capacity); + key2count = new HashMap<>(capacity); } public CountingSet(CountingSet set) { - key2count = new HashMap(set.key2count.size()); - for (Map.Entry entry : set.key2count.entrySet()) - key2count.put(entry.getKey(), new Count(entry.getValue().count)); + key2count = set.key2count.entrySet().stream() + .collect(Collectors.toMap( + Map.Entry::getKey, + e -> new Count(e.getValue().count), + (v1, v2) -> v1, + HashMap::new)); } - + public int getCount(T key) { Count count = key2count.get(key); if (count == null) @@ -66,23 +66,21 @@ public int getCount(T key) { * @return */ public int getSum() { - int sum = 0; - for (Count count : key2count.values()) - sum += count.count; - return sum; + return key2count.values().parallelStream() + .mapToInt(c -> c.count) + .sum(); } - + /** * Returns the maximum count * * @return */ public int getMax() { - int max = 0; - for (Count count : key2count.values()) - if (count.count > max) - max = count.count; - return max; + return key2count.values().parallelStream() + .mapToInt(c -> c.count) + .max() + .orElse(0); } /** @@ -100,25 +98,21 @@ public double getMean() { * @return */ public double getSD() { - double mean = getMean(); - double sum = 0; - for (Count count : key2count.values()) - sum += sqr(count.count - mean); - return Math.sqrt(sum / (double) key2count.size()); + final double mean = getMean(); + double sqSum = key2count.values().parallelStream() + .mapToDouble(c -> sqr(c.count - mean)) + .sum(); + return Math.sqrt(sqSum / key2count.size()); } - private double sqr(double d) { + private static double sqr(double d) { return d * d; } public int size() { return key2count.size(); } - - public boolean isEmpty() { - return key2count.isEmpty(); - } - + public boolean contains(Object arg0) { return key2count.containsKey(arg0); } @@ -126,24 +120,14 @@ public boolean contains(Object arg0) { public Iterator iterator() { return key2count.keySet().iterator(); } - - public Object[] toArray() { - return key2count.keySet().toArray(); - } - - @SuppressWarnings("unchecked") - public Object[] toArray(Object[] arg0) { - return key2count.keySet().toArray(arg0); - } - + public boolean add(T arg0) { Count count = key2count.get(arg0); if (count == null) { - count = new Count(); - key2count.put(arg0, count); + key2count.put(arg0, new Count(1)); return true; } else { - count.count++; + count.increment(); return false; } } @@ -151,42 +135,18 @@ public boolean add(T arg0) { public boolean add(T arg0, int inc) { Count count = key2count.get(arg0); if (count == null) { - count = new Count(); - count.count = inc; - key2count.put(arg0, count); + key2count.put(arg0, new Count(inc)); return true; } else { - count.count += inc; + count.add(inc); return false; } } public boolean remove(Object arg0) { - return (key2count.remove(arg0) != null); } - public boolean containsAll(Collection arg0) { - return key2count.keySet().containsAll(arg0); - } - - public boolean addAll(Collection arg0) { - boolean changed = false; - for (T object : arg0) { - if (add(object)) - changed = true; - } - return changed; - } - - public boolean retainAll(Collection arg0) { - return key2count.keySet().retainAll(arg0); - } - - public boolean removeAll(Collection arg0) { - return key2count.keySet().removeAll(arg0); - } - public void clear() { key2count.clear(); } @@ -199,42 +159,40 @@ public void clear() { public void keepTopN(int n) { if (size() < n) return; - List> list = new ArrayList>(key2count.entrySet()); - Collections.sort(list, new Comparator>() { - - @Override - public int compare(Entry arg0, Entry arg1) { - return IntegerComparator.compare(arg1.getValue().count, arg0.getValue().count); - } - }); - - Map newMap = new HashMap(n); - for (int i = 0; i < n; i++) - newMap.put(list.get(i).getKey(), list.get(i).getValue()); - - key2count = newMap; + + key2count = decliningCountStream() + .limit(n) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue, (v1, v2) -> v1, HashMap::new)); } - - public static class Count { - public int count = 1; - - public Count() { - } + + public static class Count implements Comparable { + public int count; public Count(int count) { this.count = count; } - + + public void increment() { + count++; + } + + public void add(int count) { + this.count += count; + } + + @Override + public int compareTo(Count o) { + return count - o.count; + } } public void printCounts() { - List> result = new ArrayList>(key2count.entrySet()); - Collections.sort(result, new Comparator>() { - public int compare(Entry o1, Entry o2) { - return IntegerComparator.compare(o2.getValue().count, o1.getValue().count); - } - }); - for (Map.Entry entry : result) - System.out.println(entry.getKey() + "\t" + entry.getValue().count); + decliningCountStream() + .forEach(entry -> System.out.println(entry.getKey() + "\t" + entry.getValue().count)); + } + + private Stream> decliningCountStream() { + return key2count.entrySet().stream() + .sorted(Comparator., Count>comparing(Map.Entry::getValue).reversed()); } } diff --git a/rabbitinahat/pom.xml b/rabbitinahat/pom.xml index 8a302145..c7820d9e 100644 --- a/rabbitinahat/pom.xml +++ b/rabbitinahat/pom.xml @@ -5,7 +5,7 @@ leporidae org.ohdsi - 0.8.0-SNAPSHOT + 0.9.0 4.0.0 diff --git a/rabbitinahat/src/main/java/org/ohdsi/rabbitInAHat/DetailsListener.java b/rabbitinahat/src/main/java/org/ohdsi/rabbitInAHat/DetailsListener.java index 533c1859..b4cd4149 100644 --- a/rabbitinahat/src/main/java/org/ohdsi/rabbitInAHat/DetailsListener.java +++ b/rabbitinahat/src/main/java/org/ohdsi/rabbitInAHat/DetailsListener.java @@ -19,4 +19,5 @@ public interface DetailsListener { public void showDetails(Object object); + public void showDetails(Object object, boolean isObject); } diff --git a/rabbitinahat/src/main/java/org/ohdsi/rabbitInAHat/DetailsPanel.java b/rabbitinahat/src/main/java/org/ohdsi/rabbitInAHat/DetailsPanel.java index 2f38c079..b7977997 100644 --- a/rabbitinahat/src/main/java/org/ohdsi/rabbitInAHat/DetailsPanel.java +++ b/rabbitinahat/src/main/java/org/ohdsi/rabbitInAHat/DetailsPanel.java @@ -27,22 +27,11 @@ import java.awt.Toolkit; import java.awt.event.ActionEvent; import java.awt.event.KeyEvent; -import java.text.DecimalFormat; +import java.text.NumberFormat; import java.util.ArrayList; import java.util.List; -import javax.swing.AbstractAction; -import javax.swing.ActionMap; -import javax.swing.BorderFactory; -import javax.swing.InputMap; -import javax.swing.JComponent; -import javax.swing.JLabel; -import javax.swing.JPanel; -import javax.swing.JScrollPane; -import javax.swing.JTable; -import javax.swing.JTextArea; -import javax.swing.KeyStroke; -import javax.swing.UIManager; +import javax.swing.*; import javax.swing.border.MatteBorder; import javax.swing.event.ChangeEvent; import javax.swing.event.DocumentEvent; @@ -54,16 +43,15 @@ import javax.swing.event.TableModelListener; import javax.swing.event.UndoableEditEvent; import javax.swing.event.UndoableEditListener; +import javax.swing.table.TableColumn; +import javax.swing.table.DefaultTableCellRenderer; import javax.swing.table.TableModel; import javax.swing.text.Document; import javax.swing.undo.CannotUndoException; import javax.swing.undo.UndoManager; -import org.ohdsi.rabbitInAHat.dataModel.Field; -import org.ohdsi.rabbitInAHat.dataModel.ItemToItemMap; -import org.ohdsi.rabbitInAHat.dataModel.Table; +import org.ohdsi.rabbitInAHat.dataModel.*; import org.ohdsi.utilities.StringUtilities; -import org.ohdsi.rabbitInAHat.dataModel.TableCellLongTextRenderer; public class DetailsPanel extends JPanel implements DetailsListener { @@ -73,8 +61,11 @@ public class DetailsPanel extends JPanel implements DetailsListener { private Object object; private TablePanel tablePanel; private FieldPanel fieldPanel; + private TargetFieldPanel targetFieldPanel; private ItemToItemMapPanel itemToItemMapPanel; private CardLayout cardLayout = new CardLayout(); + private NumberFormat numberFormat = NumberFormat.getNumberInstance(); + private NumberFormat percentageFormat = NumberFormat.getPercentInstance(); private UndoManager undoManager; @@ -89,6 +80,9 @@ public DetailsPanel() { fieldPanel = new FieldPanel(); add(fieldPanel, Field.class.getName()); + targetFieldPanel = new TargetFieldPanel(); + add(targetFieldPanel, "target" + Field.class.getName()); + itemToItemMapPanel = new ItemToItemMapPanel(); add(itemToItemMapPanel, ItemToItemMap.class.getName()); @@ -98,29 +92,41 @@ public DetailsPanel() { cardLayout.show(this, ""); undoManager = new UndoManager(); + + percentageFormat.setMinimumFractionDigits(1); } @Override - public void showDetails(Object object) { + public void showDetails(Object object, boolean isSource) { this.object = object; if (object instanceof Table) { tablePanel.showTable((Table) object); tablePanel.updateRowHeights(); cardLayout.show(this, Table.class.getName()); } else if (object instanceof Field) { - fieldPanel.showField((Field) object); - cardLayout.show(this, Field.class.getName()); + if (isSource) { + fieldPanel.showField((Field) object); + cardLayout.show(this, Field.class.getName()); + } else { + targetFieldPanel.showField((Field) object); + cardLayout.show(this, "target" + Field.class.getName()); + } } else if (object instanceof ItemToItemMap) { itemToItemMapPanel.showItemToItemMap((ItemToItemMap) object); cardLayout.show(this, ItemToItemMap.class.getName()); } else cardLayout.show(this, ""); - + // Discard edits made by showing a new details view undoManager.discardAllEdits(); } + @Override + public void showDetails(Object object) { + showDetails(object, true); + } + public void refresh() { showDetails(object); } @@ -286,9 +292,14 @@ private void updateRowHeights() { public void showTable(Table table) { this.table = table; nameLabel.setText(table.getName()); - DecimalFormat formatter = new DecimalFormat("#,###"); - rowCountLabel.setText(formatter.format(table.getRowCount())); - fieldTable.clear(); + + if (table.getRowCount() > 0) { + rowCountLabel.setText(numberFormat.format(table.getRowCount())); + } else { + rowCountLabel.setText(">= " + numberFormat.format(table.getRowsCheckedCount())); + } + + fieldTable.clear(); for (Field field : table.getFields()){ fieldTable.add(field.outputName(), field.getType(),field.getDescription()); @@ -315,15 +326,26 @@ public void removeUpdate(DocumentEvent e) { private class FieldPanel extends JPanel implements DocumentListener { - private static final long serialVersionUID = -4393026616049677944L; - private JLabel nameLabel = new JLabel(""); - private JLabel rowCountLabel = new JLabel(""); - private DescriptionTextArea description = new DescriptionTextArea (""); - private SimpleTableModel valueTable = new SimpleTableModel("Value", "Frequency", "Percent of Total (%)"); - private JTextArea commentsArea = new JTextArea(); - private Field field; + private static final long serialVersionUID = -4393026616049677944L; + JLabel nameLabel; + JLabel rowCountLabel; + DescriptionTextArea description; + SimpleTableModel valueTable; + JTextArea commentsArea; + Boolean isTargetFieldPanel; + private Field field; public FieldPanel() { + nameLabel = new JLabel(""); + rowCountLabel = new JLabel(""); + description = new DescriptionTextArea (""); + valueTable = new SimpleTableModel("Value", "Frequency", "Percentage"); + commentsArea = new JTextArea(); + isTargetFieldPanel = false; + initialise(); + } + + public void initialise() { setLayout(new BorderLayout()); JPanel generalInfoPanel = new JPanel(); @@ -357,8 +379,30 @@ public FieldPanel() { table.setFont(font); table.setRowHeight(24); table.setBorder(new MatteBorder(1, 0, 1, 0, Color.BLACK)); - - fieldListPanel.setBorder(BorderFactory.createTitledBorder("Fields")); + table.setCellSelectionEnabled(true); + + // Make first column wider if source panel or second column if this is a target field panel + int wideColumnIndex = isTargetFieldPanel ? 1 : 0; + TableColumn column; + for (int i = 0; i < table.getColumnCount(); i++) { + column = table.getColumnModel().getColumn(i); + if (i == wideColumnIndex) { + column.setPreferredWidth(500); + } else { + column.setPreferredWidth(50); + } + } + + if (!isTargetFieldPanel) { + // Right align the frequency and percentage + DefaultTableCellRenderer rightRenderer = new DefaultTableCellRenderer(); + rightRenderer.setHorizontalAlignment(SwingConstants.RIGHT); + table.getColumnModel().getColumn(1).setCellRenderer(rightRenderer); + table.getColumnModel().getColumn(2).setCellRenderer(rightRenderer); + } + + String title = isTargetFieldPanel ? "Concept ID Hints" : "Value Counts"; + fieldListPanel.setBorder(BorderFactory.createTitledBorder(title)); add(fieldListPanel, BorderLayout.CENTER); JScrollPane commentsPanel = new JScrollPane(commentsArea); @@ -376,48 +420,43 @@ public FieldPanel() { public void showField(Field field) { this.field = field; - + nameLabel.setText(field.getName()); rowCountLabel.setText(field.getType()); description.setText(field.getDescription()); - + // Hide description if it's empty description.getParent().setVisible(!description.getText().isEmpty()); + this.createValueList(field); + commentsArea.setText(field.getComment()); + } + + public void createValueList(Field field) { valueTable.clear(); - if (field.getValueCounts() != null) { - double valueCountTotal = 0.0; - for (String[] total : field.getValueCounts()) { - String temp = total[1]; - if (StringUtilities.isNumber(temp)) { - double valueCountTemp = Double.parseDouble(temp); - valueCountTotal += valueCountTemp; - } - } - DecimalFormat formatter = new DecimalFormat("#,###"); - DecimalFormat formatterPercent = new DecimalFormat("#,##0.0"); + int valueCountTotal = field.getRowsCheckedCount(); + for (String[] valueCount : field.getValueCounts()) { - String nr = valueCount[1]; - String vp = ""; - if (StringUtilities.isNumber(nr)) { - double number = Double.parseDouble(nr); - nr = formatter.format(number); - double valueCountPercent = number / valueCountTotal * 100; - if (valueCountPercent < 0.1) { - vp = "< 0.1"; + String valueNumber = valueCount[1]; + String valuePercent = ""; + if (StringUtilities.isNumber(valueNumber)) { + double number = Double.parseDouble(valueNumber); + valueNumber = numberFormat.format(number); + double valueCountPercent = number / (double) valueCountTotal; + if (valueCountPercent < 0.001) { + valuePercent = "<" + percentageFormat.format(0.001); } - else if (valueCountPercent > 99) { - vp = "> 99.0"; + else if (valueCountPercent > 0.99) { + valuePercent = ">" + percentageFormat.format(0.99); } else { - vp = formatterPercent.format(valueCountPercent); + valuePercent = percentageFormat.format(valueCountPercent); } } - valueTable.add(valueCount[0], nr, vp); + valueTable.add(valueCount[0], valueNumber, valuePercent); } } - commentsArea.setText(field.getComment()); } @Override @@ -437,6 +476,28 @@ public void removeUpdate(DocumentEvent e) { } + private class TargetFieldPanel extends FieldPanel { + public TargetFieldPanel() { + nameLabel = new JLabel(""); + rowCountLabel = new JLabel(""); + description = new DescriptionTextArea (""); + valueTable = new SimpleTableModel("Concept ID", "Concept Name", "Standard?"); + commentsArea = new JTextArea(); + isTargetFieldPanel = true; + super.initialise(); + } + + @Override + public void createValueList(Field field) { + valueTable.clear(); + if (field.getConceptIdHints() != null) { + for (ConceptsMap.Concept conceptIdHint : field.getConceptIdHints()) { + valueTable.add(conceptIdHint.getConceptId(), conceptIdHint.getConceptName(), conceptIdHint.getStandardConcept()); + } + } + } + } + private class ItemToItemMapPanel extends JPanel implements DocumentListener { private static final long serialVersionUID = -4393026616049677944L; diff --git a/rabbitinahat/src/main/java/org/ohdsi/rabbitInAHat/ETLMarkupDocumentGenerator.java b/rabbitinahat/src/main/java/org/ohdsi/rabbitInAHat/ETLMarkupDocumentGenerator.java index 48a4ab36..0eb3fc32 100644 --- a/rabbitinahat/src/main/java/org/ohdsi/rabbitInAHat/ETLMarkupDocumentGenerator.java +++ b/rabbitinahat/src/main/java/org/ohdsi/rabbitInAHat/ETLMarkupDocumentGenerator.java @@ -19,15 +19,16 @@ import java.awt.Color; import java.awt.image.BufferedImage; +import java.io.BufferedWriter; import java.io.File; -import java.io.FileNotFoundException; import java.io.IOException; +import java.io.PrintWriter; +import java.nio.file.Files; import java.util.ArrayList; import java.util.List; import javax.imageio.ImageIO; -import org.apache.poi.openxml4j.exceptions.InvalidFormatException; import org.ohdsi.rabbitInAHat.dataModel.ETL; import org.ohdsi.rabbitInAHat.dataModel.ETL.FileFormat; import org.ohdsi.rabbitInAHat.dataModel.Field; @@ -37,54 +38,84 @@ import org.ohdsi.rabbitInAHat.dataModel.Table; import org.ohdsi.utilities.StringUtilities; import org.ohdsi.utilities.files.Row; -import org.ohdsi.utilities.files.WriteTextFile; public class ETLMarkupDocumentGenerator { - private MarkupDocument document; - private ETL etl; + private MarkupDocument document; + private List targetTablesWritten; + private ETL etl; public enum DocumentType { MARKDOWN, HTML - }; + } public static void main(String[] args) { ETL etl = ETL.fromFile("c:/temp/markdown/exampleEtl.json.gz", FileFormat.GzipJson); ETLMarkupDocumentGenerator generator = new ETLMarkupDocumentGenerator(etl); generator.generate("c:/temp/markdown/index.html", DocumentType.HTML); - } - public ETLMarkupDocumentGenerator(ETL etl) { + ETLMarkupDocumentGenerator(ETL etl) { this.etl = etl; + this.targetTablesWritten = new ArrayList<>(); } - public void generate(String fileName, DocumentType documentType) { - try { - if (documentType == DocumentType.HTML) - document = new HtmlDocument(fileName); - else - document = new MarkdownDocument(fileName); - addTableLevelSection(); - - for (Table targetTable : etl.getTargetDatabase().getTables()) - addTargetTableSection(targetTable); - - document.close(); - } catch (FileNotFoundException e) { - e.printStackTrace(); - } catch (IOException e) { - e.printStackTrace(); - } catch (InvalidFormatException e) { - e.printStackTrace(); + void generate(String fileName, DocumentType documentType) { + if (documentType == DocumentType.HTML) { + document = new HtmlDocument(fileName); + } else { + document = new MarkdownDocument(fileName); + } + + for (Table targetTable : etl.getTargetDatabase().getTables()) { + boolean targetTableHasMapping = addTargetTableSection(targetTable); + + if (targetTableHasMapping) { + document.write(targetTable.getName()); + targetTablesWritten.add(targetTable.getName()); + } else { + document.reset(); + }; } + + document.addHeader1("Appendix: source tables"); + for (Table sourceTable : etl.getSourceDatabase().getTables()) { + addSourceTablesAppendix(sourceTable); + } + document.write("source_appendix"); + targetTablesWritten.add("source_appendix"); + + addTableLevelSection(); + document.write("index"); } - private void addTargetTableSection(Table targetTable) throws InvalidFormatException, IOException { + private void addTableLevelSection() { + MappingPanel mappingPanel = new MappingPanel(etl.getTableToTableMapping()); + mappingPanel.setShowOnlyConnectedItems(true); + int height = mappingPanel.getMinimumSize().height; + mappingPanel.setSize(800, height); + + document.addHeader1(mappingPanel.getSourceDbName() + " Data Mapping Approach to " + mappingPanel.getTargetDbName()); + + BufferedImage image = new BufferedImage(800, height, BufferedImage.TYPE_INT_ARGB); + image.getGraphics().setColor(Color.WHITE); + image.getGraphics().fillRect(0, 0, image.getWidth(), image.getHeight()); + mappingPanel.paint(image.getGraphics()); + document.addImage(image, "Table mapping"); + + document.addHeader2("Contents"); + for (String targetTableName : targetTablesWritten) { + document.addFileLink(targetTableName); + } + } + + private boolean addTargetTableSection(Table targetTable) { document.addHeader2("Table name: " + targetTable.getName()); + boolean hasMappings = false; for (ItemToItemMap tableToTableMap : etl.getTableToTableMapping().getSourceToTargetMaps()) if (tableToTableMap.getTargetItem() == targetTable) { + hasMappings = true; Table sourceTable = (Table) tableToTableMap.getSourceItem(); Mapping fieldtoFieldMapping = etl.getFieldToFieldMapping(sourceTable, targetTable); @@ -110,7 +141,7 @@ private void addTargetTableSection(Table targetTable) throws InvalidFormatExcept document.addImage(image, "Field mapping"); // Add table of field to field mapping - List rows = new ArrayList(); + List rows = new ArrayList<>(); for (MappableItem targetField : fieldtoFieldMapping.getTargetItems()) { Row row = new Row(); row.add("Destination Field", targetField.getName()); @@ -121,22 +152,22 @@ private void addTargetTableSection(Table targetTable) throws InvalidFormatExcept for (ItemToItemMap fieldToFieldMap : fieldtoFieldMapping.getSourceToTargetMaps()) { if (fieldToFieldMap.getTargetItem() == targetField) { if (source.length() != 0) - source.append("\n"); + source.append("
"); source.append(fieldToFieldMap.getSourceItem().getName().trim()); if (logic.length() != 0) - logic.append("\n"); + logic.append("
"); logic.append(fieldToFieldMap.getLogic().trim()); if (comment.length() != 0) - comment.append("\n"); + comment.append("
"); comment.append(fieldToFieldMap.getComment().trim()); } } for (Field field : targetTable.getFields()) { if (field.getName().equals(targetField.getName())) { if (comment.length() != 0) - comment.append("\n"); + comment.append("
"); comment.append(field.getComment().trim()); } } @@ -145,60 +176,67 @@ private void addTargetTableSection(Table targetTable) throws InvalidFormatExcept row.add("Comment field", comment.toString().trim()); rows.add(row); } - - + document.addTable(rows); } + return hasMappings; } - private void addTableLevelSection() throws InvalidFormatException, IOException { - MappingPanel mappingPanel = new MappingPanel(etl.getTableToTableMapping()); - mappingPanel.setShowOnlyConnectedItems(true); - int height = mappingPanel.getMinimumSize().height; - mappingPanel.setSize(800, height); + private void addSourceTablesAppendix(Table sourceTable) { + document.addHeader3("Table: " + sourceTable.getName()); - document.addHeader1(mappingPanel.getSourceDbName() + " Data Mapping Approach to " + mappingPanel.getTargetDbName()); + List rows = new ArrayList<>(); + for (Field field : sourceTable.getFields()) { + String mostFrequentValue = ""; + if (field.getValueCounts() != null && field.getValueCounts().length != 0) { + mostFrequentValue = field.getValueCounts()[0][0]; + } - BufferedImage image = new BufferedImage(800, height, BufferedImage.TYPE_INT_ARGB); - image.getGraphics().setColor(Color.WHITE); - image.getGraphics().fillRect(0, 0, image.getWidth(), image.getHeight()); - mappingPanel.paint(image.getGraphics()); - document.addImage(image, "Table mapping"); + Row row = new Row(); + row.add("Field", field.getName()); + row.add("Type", field.getType()); + row.add("Most freq. value", mostFrequentValue); + row.add("Comment", field.getComment()); + rows.add(row); + } + document.addTable(rows); } private interface MarkupDocument { - public void addHeader1(String header); + void addHeader1(String header); - public void addHeader2(String header); + void addHeader2(String header); - public void addHeader3(String header); + void addHeader3(String header); - public void addParagraph(String text); + void addParagraph(String text); - public void addImage(BufferedImage image, String alternative); + void addImage(BufferedImage image, String alternative); - public void addTable(List rows); + void addFileLink(String targetName); - public void close(); + void addTable(List rows); + + void write(String targetName); + + void reset(); } - private class MarkdownDocument implements MarkupDocument { - private List lines = new ArrayList(); - private int imageIndex = 0; - private String fileName; - private String filesFolder; - private String mainFolder; - + private static class MarkdownDocument implements MarkupDocument { + private List lines = new ArrayList<>(); + private int imageIndex = 0; + private File mainFolder; + private File filesFolder; + /** * - * @param fileName Full path of the markdown document to create + * @param directoryName Full path of the markdown document to create */ - public MarkdownDocument(String fileName) { - this.fileName = fileName; - mainFolder = new File(fileName).getParent(); - filesFolder = new File(fileName).getName().replaceAll("(\\.md)|(\\.MD)", "_files"); + MarkdownDocument(String directoryName) { + this.mainFolder = new File(directoryName); + this.filesFolder = new File(directoryName, "_files"); } - + @Override public void addHeader1(String header) { lines.add("# " + header); @@ -225,71 +263,84 @@ public void addParagraph(String text) { @Override public void addImage(BufferedImage image, String alternative) { - if (imageIndex == 0) { - File folder = new File(mainFolder + "/"+ filesFolder); - if (!folder.exists()) - folder.mkdirs(); + if (imageIndex == 0 && !filesFolder.exists()) { + filesFolder.mkdirs(); } imageIndex++; - String imageFile = filesFolder + "/image" + imageIndex + ".png"; + File imageFile = new File(filesFolder, "image" + imageIndex + ".png"); try { - ImageIO.write(image, "png", new File(mainFolder + "/" + imageFile)); + ImageIO.write(image, "png", imageFile); } catch (IOException e) { throw new RuntimeException(e); } - lines.add("![](" + imageFile + ")"); + lines.add("![](" + filesFolder.getName() + "/" + imageFile.getName() + ")"); + lines.add(""); + } + + public void addFileLink(String targetName) { + lines.add(String.format("[%1$s](%1$s.md)", targetName)); lines.add(""); } @Override - public void close() { - WriteTextFile out = new WriteTextFile(fileName); - for (String line : lines) - out.writeln(line); - out.close(); + public void write(String targetName) { + File outFile = new File(mainFolder, targetName + ".md"); + try (BufferedWriter bw = Files.newBufferedWriter(outFile.toPath()); + PrintWriter out = new PrintWriter(bw)) { + for (String line : lines) { + out.println(line); + } + } catch (IOException e) { + e.printStackTrace(); + } + this.reset(); + } + + @Override + public void reset() { + lines = new ArrayList<>(); } @Override public void addTable(List rows) { - if (rows.size() > 0) { - String header = "| " + StringUtilities.join(rows.get(0).getFieldNames(), " | ") + " |"; - header = header.replaceAll("\n", " "); - lines.add(header); - StringBuilder line = new StringBuilder(); - for (int i = 0; i < rows.get(0).getFieldNames().size(); i++) - line.append("| --- "); + if (rows.size() == 0) { + return; + } + + String header = "| " + StringUtilities.join(rows.get(0).getFieldNames(), " | ") + " |"; + header = header.replaceAll("\n", " "); + lines.add(header); + StringBuilder line = new StringBuilder(); + for (int i = 0; i < rows.get(0).getFieldNames().size(); i++) + line.append("| --- "); + line.append("|"); + lines.add(line.toString()); + + for (Row row : rows) { + line = new StringBuilder(); + for (String value : row.getCells()) + line.append("| ").append(value.replaceAll("\n", " ")).append(" "); line.append("|"); lines.add(line.toString()); - - for (Row row : rows) { - line = new StringBuilder(); - for (String value : row.getCells()) - line.append("| " + value.replaceAll("\n", " ") + " "); - line.append("|"); - lines.add(line.toString()); - } - lines.add(""); } + lines.add(""); } } - private class HtmlDocument implements MarkupDocument { - private List lines = new ArrayList(); - private int imageIndex = 0; - private String fileName; - private String filesFolder; - private String mainFolder; - + private static class HtmlDocument implements MarkupDocument { + private List lines = new ArrayList<>(); + private int imageIndex = 0; + private File mainFolder; + private File filesFolder; + /** * - * @param fileName Full path of the HTML file to create. + * @param directoryName Full path of the HTML file to create. */ - public HtmlDocument(String fileName) { - this.fileName = fileName; - mainFolder = new File(fileName).getParent(); - filesFolder = new File(fileName).getName().replaceAll("(\\.html?)|(\\.html?)", "_files"); + HtmlDocument(String directoryName) { + this.mainFolder = new File(directoryName); + this.filesFolder = new File(directoryName, "_files"); } - @Override public void addHeader1(String header) { @@ -317,28 +368,43 @@ public void addParagraph(String text) { @Override public void addImage(BufferedImage image, String alternative) { - if (imageIndex == 0) { - File folder = new File(mainFolder + "/"+ filesFolder); - if (!folder.exists()) - folder.mkdirs(); + if (imageIndex == 0 && !filesFolder.exists()) { + filesFolder.mkdirs(); } imageIndex++; - String imageFile = filesFolder + "/image" + imageIndex + ".png"; + File imageFile = new File(filesFolder, "image" + imageIndex + ".png"); try { - ImageIO.write(image, "png", new File(mainFolder + "/" + imageFile)); + ImageIO.write(image, "png", imageFile); } catch (IOException e) { throw new RuntimeException(e); } - lines.add("\"""); + lines.add("\"""); + lines.add(""); + } + + public void addFileLink(String targetName) { + lines.add(String.format("", targetName)); lines.add(""); } @Override - public void close() { - WriteTextFile out = new WriteTextFile(fileName); - for (String line : lines) - out.writeln(line); - out.close(); + public void write(String targetName) { + File outFile = new File(mainFolder, targetName + ".html"); + try (BufferedWriter bw = Files.newBufferedWriter(outFile.toPath()); + PrintWriter out = new PrintWriter(bw)) { + for (String line : lines) { + out.println(line); + } + } catch (IOException e) { + e.printStackTrace(); + } + + this.reset(); + } + + @Override + public void reset() { + lines = new ArrayList<>(); } @Override diff --git a/rabbitinahat/src/main/java/org/ohdsi/rabbitInAHat/ETLTestFrameWorkGenerator.java b/rabbitinahat/src/main/java/org/ohdsi/rabbitInAHat/ETLTestFrameWorkGenerator.java index 78c35a9f..eaa70398 100644 --- a/rabbitinahat/src/main/java/org/ohdsi/rabbitInAHat/ETLTestFrameWorkGenerator.java +++ b/rabbitinahat/src/main/java/org/ohdsi/rabbitInAHat/ETLTestFrameWorkGenerator.java @@ -17,43 +17,42 @@ ******************************************************************************/ package org.ohdsi.rabbitInAHat; -import java.util.ArrayList; -import java.util.HashSet; -import java.util.List; -import java.util.Set; +import java.io.BufferedWriter; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.PrintWriter; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.*; +import java.util.stream.Collectors; -import org.ohdsi.rabbitInAHat.dataModel.Database; -import org.ohdsi.rabbitInAHat.dataModel.ETL; +import org.ohdsi.rabbitInAHat.dataModel.*; import org.ohdsi.rabbitInAHat.dataModel.ETL.FileFormat; -import org.ohdsi.rabbitInAHat.dataModel.Field; -import org.ohdsi.rabbitInAHat.dataModel.Table; -import org.ohdsi.utilities.StringUtilities; -import org.ohdsi.utilities.files.WriteTextFile; public class ETLTestFrameWorkGenerator { - private static int DEFAULT = 0; - private static int NEGATE = 1; - private static int COUNT = 2; - private String[] keywords = new String[] { "ADD", "ALL", "ALTER", "AND", "ANY", "AS", "ASC", "AUTHORIZATION", "BACKUP", "BEGIN", "BETWEEN", - "BREAK", "BROWSE", "BULK", "BY", "CASCADE", "CASE", "CHECK", "CHECKPOINT", "CLOSE", "CLUSTERED", "COALESCE", "COLLATE", "COLUMN", "COMMIT", - "COMPUTE", "CONSTRAINT", "CONTAINS", "CONTAINSTABLE", "CONTINUE", "CONVERT", "CREATE", "CROSS", "CURRENT", "CURRENT_DATE", "CURRENT_TIME", - "CURRENT_TIMESTAMP", "CURRENT_USER", "CURSOR", "DATABASE", "DBCC", "DEALLOCATE", "DECLARE", "DEFAULT", "DELETE", "DENY", "DESC", "DISK", "DISTINCT", - "DISTRIBUTED", "DOUBLE", "DROP", "DUMP", "ELSE", "END", "ERRLVL", "ESCAPE", "EXCEPT", "EXEC", "EXECUTE", "EXISTS", "EXIT", "EXTERNAL", "FETCH", - "FILE", "FILLFACTOR", "FOR", "FOREIGN", "FREETEXT", "FREETEXTTABLE", "FROM", "FULL", "FUNCTION", "GOTO", "GRANT", "GROUP", "HAVING", "HOLDLOCK", - "IDENTITY", "IDENTITY_INSERT", "IDENTITYCOL", "IF", "IN", "INDEX", "INNER", "INSERT", "INTERSECT", "INTO", "IS", "JOIN", "KEY", "KILL", "LEFT", - "LIKE", "LINENO", "LOAD", "MERGE", "NATIONAL", "NOCHECK", "NONCLUSTERED", "NOT", "NULL", "NULLIF", "OF", "OFF", "OFFSETS", "ON", "OPEN", - "OPENDATASOURCE", "OPENQUERY", "OPENROWSET", "OPENXML", "OPTION", "OR", "ORDER", "OUTER", "OVER", "PERCENT", "PIVOT", "PLAN", "PRECISION", - "PRIMARY", "PRINT", "PROC", "PROCEDURE", "PUBLIC", "RAISERROR", "READ", "READTEXT", "RECONFIGURE", "REFERENCES", "REPLICATION", "RESTORE", - "RESTRICT", "RETURN", "REVERT", "REVOKE", "RIGHT", "ROLLBACK", "ROWCOUNT", "ROWGUIDCOL", "RULE", "SAVE", "SCHEMA", "SECURITYAUDIT", "SELECT", - "SEMANTICKEYPHRASETABLE", "SEMANTICSIMILARITYDETAILSTABLE", "SEMANTICSIMILARITYTABLE", "SESSION_USER", "SET", "SETUSER", "SHUTDOWN", "SOME", - "STATISTICS", "SYSTEM_USER", "TABLE", "TABLESAMPLE", "TEXTSIZE", "THEN", "TO", "TOP", "TRAN", "TRANSACTION", "TRIGGER", "TRUNCATE", "TRY_CONVERT", - "TSEQUAL", "UNION", "UNIQUE", "UNPIVOT", "UPDATE", "UPDATETEXT", "USE", "USER", "VALUES", "VARYING", "VIEW", "WAITFOR", "WHEN", "WHERE", "WHILE", - "WITH", "WITHIN GROUP", "WRITETEXT" }; - protected Set keywordSet; - private List r; - private Database sourceDb; - private Database targetDb; + private static int DEFAULT = 0; + private static int NEGATE = 1; + private static int COUNT = 2; + private static final Set keywordSet = new HashSet<>(Arrays.asList("ADD", "ALL", "ALTER", "AND", "ANY", "AS", "ASC", "AUTHORIZATION", "BACKUP", "BEGIN", "BETWEEN", + "BREAK", "BROWSE", "BULK", "BY", "CASCADE", "CASE", "CHECK", "CHECKPOINT", "CLOSE", "CLUSTERED", "COALESCE", "COLLATE", "COLUMN", "COMMIT", + "COMPUTE", "CONSTRAINT", "CONTAINS", "CONTAINSTABLE", "CONTINUE", "CONVERT", "CREATE", "CROSS", "CURRENT", "CURRENT_DATE", "CURRENT_TIME", + "CURRENT_TIMESTAMP", "CURRENT_USER", "CURSOR", "DATABASE", "DBCC", "DEALLOCATE", "DECLARE", "DEFAULT", "DELETE", "DENY", "DESC", "DISK", "DISTINCT", + "DISTRIBUTED", "DOUBLE", "DROP", "DUMP", "ELSE", "END", "ERRLVL", "ESCAPE", "EXCEPT", "EXEC", "EXECUTE", "EXISTS", "EXIT", "EXTERNAL", "FETCH", + "FILE", "FILLFACTOR", "FOR", "FOREIGN", "FREETEXT", "FREETEXTTABLE", "FROM", "FULL", "FUNCTION", "GOTO", "GRANT", "GROUP", "HAVING", "HOLDLOCK", + "IDENTITY", "IDENTITY_INSERT", "IDENTITYCOL", "IF", "IN", "INDEX", "INNER", "INSERT", "INTERSECT", "INTO", "IS", "JOIN", "KEY", "KILL", "LEFT", + "LIKE", "LINENO", "LOAD", "MERGE", "NATIONAL", "NOCHECK", "NONCLUSTERED", "NOT", "NULL", "NULLIF", "OF", "OFF", "OFFSETS", "ON", "OPEN", + "OPENDATASOURCE", "OPENQUERY", "OPENROWSET", "OPENXML", "OPTION", "OR", "ORDER", "OUTER", "OVER", "PERCENT", "PIVOT", "PLAN", "PRECISION", + "PRIMARY", "PRINT", "PROC", "PROCEDURE", "PUBLIC", "RAISERROR", "READ", "READTEXT", "RECONFIGURE", "REFERENCES", "REPLICATION", "RESTORE", + "RESTRICT", "RETURN", "REVERT", "REVOKE", "RIGHT", "ROLLBACK", "ROWCOUNT", "ROWGUIDCOL", "RULE", "SAVE", "SCHEMA", "SECURITYAUDIT", "SELECT", + "SEMANTICKEYPHRASETABLE", "SEMANTICSIMILARITYDETAILSTABLE", "SEMANTICSIMILARITYTABLE", "SESSION_USER", "SET", "SETUSER", "SHUTDOWN", "SOME", + "STATISTICS", "SYSTEM_USER", "TABLE", "TABLESAMPLE", "TEXTSIZE", "THEN", "TO", "TOP", "TRAN", "TRANSACTION", "TRIGGER", "TRUNCATE", "TRY_CONVERT", + "TSEQUAL", "UNION", "UNIQUE", "UNPIVOT", "UPDATE", "UPDATETEXT", "USE", "USER", "VALUES", "VARYING", "VIEW", "WAITFOR", "WHEN", "WHERE", "WHILE", + "WITH", "WITHIN GROUP", "WRITETEXT")); + private PrintWriter writer; + private ETL etl; public static void main(String[] args) { ETL etl = ETL.fromFile("C:\\Home\\Research\\ETLs\\JMDC ETL\\JMDC ETL CDMv5\\JMDC to CDMv5 ETL v08.json.gz", FileFormat.GzipJson); @@ -61,24 +60,25 @@ public static void main(String[] args) { generator.generate(etl, "C:\\Home\\Research\\ETLs\\JMDC ETL\\JMDC ETL CDMv5\\JmdcTestFramework.R"); } - public ETLTestFrameWorkGenerator() { - keywordSet = new HashSet(); - for (String keyword : keywords) - keywordSet.add(keyword); + ETLTestFrameWorkGenerator() { } - public void generate(ETL etl, String filename) { - List r = generateRScript(etl); - WriteTextFile out = new WriteTextFile(filename); - for (String line : r) - out.writeln(line); - out.close(); + void generate(ETL etl, String filename) { + this.etl = etl; + + Path path = Paths.get(filename); + try (BufferedWriter bw = Files.newBufferedWriter(path, StandardCharsets.UTF_8); + PrintWriter writer = new PrintWriter(bw)) { + this.writer = writer; + generateRScript(); + } catch (FileNotFoundException ex) { + ex.printStackTrace(); + } catch (IOException ex) { + ex.printStackTrace(); + } } - private List generateRScript(ETL etl) { - this.sourceDb = etl.getSourceDatabase(); - this.targetDb = etl.getTargetDatabase(); - this.r = new ArrayList(); + private void generateRScript() { createInitFunction(); createSetDefaultFunctions(); createGetDefaultFunctions(); @@ -89,24 +89,30 @@ private List generateRScript(ETL etl) { createExpectFunctions(COUNT); createLookupFunctions(); createGenerateInsertSqlFunction(); + createSourceCsvFunction(); + createExtractTestTypeStringFunction(); createGenerateTestSqlFunction(); - return r; + createTestsOverviewFunctions(); + createSummaryFunction(); + createGetUntestedFieldsFunctions(); + createOutputTestResultsSummaryFunction(); } private void createInitFunction() { - r.add("initFramework <- function() {"); - r.add(" frameworkContext <- new.env(parent = globalenv())"); - r.add(" assign('frameworkContext', frameworkContext, envir = globalenv())"); - r.add(" frameworkContext$inserts <- list()"); - r.add(" frameworkContext$expects <- list()"); - r.add(" frameworkContext$testId <- -1"); - r.add(" frameworkContext$testDescription <- \"\""); - r.add(" frameworkContext$defaultValues <- new.env(parent = frameworkContext)"); - for (Table table : sourceDb.getTables()) { + writer.println("initFramework <- function() {"); + writer.println(" frameworkContext <- new.env(parent = globalenv())"); + writer.println(" class(frameworkContext) <- 'frameworkContext'"); + writer.println(" assign('frameworkContext', frameworkContext, envir = globalenv())"); + writer.println(" frameworkContext$inserts <- list()"); + writer.println(" frameworkContext$expects <- list()"); + writer.println(" frameworkContext$testId <- -1"); + writer.println(" frameworkContext$testDescription <- \"\""); + writer.println(" frameworkContext$defaultValues <- new.env(parent = frameworkContext)"); + for (Table table : etl.getSourceDatabase().getTables()) { if (!table.isStem()) { String rTableName = convertToRName(table.getName()); - r.add(""); - r.add(" defaults <- list()"); + writer.println(""); + writer.println(" defaults <- list()"); for (Field field : table.getFields()) { String rFieldName = convertToRName(field.getName()); String defaultValue; @@ -115,286 +121,489 @@ private void createInitFunction() { else defaultValue = field.getValueCounts()[0][0]; if (!defaultValue.equals("") && !defaultValue.equals("List truncated...")) - r.add(" defaults$" + rFieldName + " <- '" + defaultValue + "'"); + writer.println(" defaults$" + rFieldName + " <- '" + defaultValue + "'"); + else + writer.println(" defaults$" + rFieldName + " <- ''"); + } + writer.println(" assign('" + rTableName + "', defaults, envir = frameworkContext$defaultValues)"); + } + } + writer.println(""); + createFieldsMapped(); + writer.println(""); + + writer.println(" frameworkContext$sourceFieldsTested <- c()"); + writer.println(" frameworkContext$targetFieldsTested <- c()"); + + writer.println("}"); + writer.println(""); + writer.println("initFramework()"); + writer.println(""); + } + + + private void createFieldsMapped() { + // Collect all stem fields mapped to + Set stemTargets = new HashSet<>(); + if (etl.hasStemTable()) { + Optional
stemTableOptional = etl.getTargetDatabase().getTables().stream().filter(Table::isStem).findFirst(); + if (stemTableOptional.isPresent()) { + Table stemTable = stemTableOptional.get(); + for (Table table : etl.getSourceDatabase().getTables()) { + Mapping toStemMapping = etl.getFieldToFieldMapping(table, stemTable); + stemTargets.addAll(toStemMapping.getSourceToTargetMaps().stream().map(x -> x.getTargetItem().toString()).collect(Collectors.toSet())); + } + } + } + + // Collect all fields that are either mapped from or mapped to. Excluding stem table. + Set sourceFieldsMapped = new HashSet<>(); + Set targetFieldsMapped = new HashSet<>(); + for (ItemToItemMap tableToTableMap : etl.getTableToTableMapping().getSourceToTargetMaps()) { + Table sourceTable = (Table) tableToTableMap.getSourceItem(); + Table targetTable = (Table) tableToTableMap.getTargetItem(); + + Mapping fieldToFieldMapping = etl.getFieldToFieldMapping(sourceTable, targetTable); + for (ItemToItemMap fieldToFieldMap : fieldToFieldMapping.getSourceToTargetMaps()) { + if (!sourceTable.isStem()) { + sourceFieldsMapped.add((Field) fieldToFieldMap.getSourceItem()); + } + if (!targetTable.isStem()) { + // if from stem to target, only use stem fields that are mapped to from source. + if (sourceTable.isStem()) { + if (stemTargets.contains(fieldToFieldMap.getSourceItem().toString())) { + targetFieldsMapped.add((Field) fieldToFieldMap.getTargetItem()); + } + } else { + targetFieldsMapped.add((Field) fieldToFieldMap.getTargetItem()); + } } - r.add(" assign('" + rTableName + "', defaults, envir = frameworkContext$defaultValues)"); } } - r.add("}"); - r.add(""); - r.add("initFramework()"); - r.add(""); + + writer.println(" frameworkContext$sourceFieldsMapped <- c("); + boolean isFirst = true; + for (Field field : sourceFieldsMapped) { + String prefix = isFirst ? " '" : " ,'"; + writer.println(prefix + convertFieldToFullName(field) + "'"); + isFirst = false; + } + writer.println(" )"); + writer.println(""); + + writer.println(" frameworkContext$targetFieldsMapped <- c("); + isFirst = true; + for (Field field : targetFieldsMapped) { + String prefix = isFirst ? " '" : " ,'"; + writer.println(prefix + convertFieldToFullName(field) + "'"); + isFirst = false; + } + writer.println(" )"); } private void createSetDefaultFunctions() { - for (Table table : sourceDb.getTables()) { + for (Table table : etl.getSourceDatabase().getTables()) { if (!table.isStem()) { - StringBuilder line = new StringBuilder(); String rTableName = convertToRName(table.getName()); - List argDefs = new ArrayList(); - List insertLines = new ArrayList(); + + writer.print("set_defaults_" + rTableName + " <- function("); + writer.print(table.getFields().stream().map(Field::getName).map(this::convertToRName).collect(Collectors.joining(", "))); + writer.println(") {"); + writer.println(" defaults <- get('" + rTableName + "', envir = frameworkContext$defaultValues)"); for (Field field : table.getFields()) { String rFieldName = convertToRName(field.getName()); - argDefs.add(rFieldName); - insertLines.add(" if (!missing(" + rFieldName + ")) {"); - insertLines.add(" defaults$" + rFieldName + " <- " + rFieldName); - insertLines.add(" }"); + writer.println(" if (!missing(" + rFieldName + ")) {"); + writer.println(" defaults$" + rFieldName + " <- " + rFieldName); + writer.println(" }"); } - - line.append("set_defaults_" + rTableName + " <- function("); - line.append(StringUtilities.join(argDefs, ", ")); - line.append(") {"); - r.add(line.toString()); - r.add(" defaults <- get('" + rTableName + "', envir = frameworkContext$defaultValues)"); - r.addAll(insertLines); - r.add(" assign('" + rTableName + "', defaults, envir = frameworkContext$defaultValues)"); - r.add(" invisible(defaults)"); - r.add("}"); - r.add(""); + writer.println(" assign('" + rTableName + "', defaults, envir = frameworkContext$defaultValues)"); + writer.println(" invisible(defaults)"); + writer.println("}"); + writer.println(""); } } } private void createGetDefaultFunctions() { - for (Table table : sourceDb.getTables()) { + for (Table table : etl.getSourceDatabase().getTables()) { String rTableName = convertToRName(table.getName()); - r.add("get_defaults_" + rTableName + " <- function() {"); - r.add(" defaults <- get('" + rTableName + "', envir = frameworkContext$defaultValues)"); - r.add(" return(defaults)"); - r.add("}"); - r.add(""); + writer.println("get_defaults_" + rTableName + " <- function() {"); + writer.println(" defaults <- get('" + rTableName + "', envir = frameworkContext$defaultValues)"); + writer.println(" return(defaults)"); + writer.println("}"); + writer.println(""); } } private void createDeclareTestFunction() { - r.add("declareTest <- function(id, description) {"); - r.add(" frameworkContext$testId <- id"); - r.add(" frameworkContext$testDescription <- description"); - r.add("}"); - r.add(""); + writer.println("declareTest <- function(id, description) {"); + writer.println(" frameworkContext$testId <- id"); + writer.println(" frameworkContext$testDescription <- description"); + writer.println("}"); + writer.println(""); } private void createAddFunctions() { - for (Table table : sourceDb.getTables()) { + for (Table table : etl.getSourceDatabase().getTables()) { if (!table.isStem()) { - StringBuilder line = new StringBuilder(); String rTableName = convertToRName(table.getName()); String sqlTableName = convertToSqlName(table.getName()); - List argDefs = new ArrayList(); - for (Field field : table.getFields()) { - String rFieldName = convertToRName(field.getName()); - argDefs.add(rFieldName); - } - line.append("add_" + rTableName + " <- function("); - line.append(StringUtilities.join(argDefs, ", ")); - line.append(") {"); - r.add(line.toString()); - r.add(" defaults <- get('" + rTableName + "', envir = frameworkContext$defaultValues)"); - r.add(" fields <- c()"); - r.add(" values <- c()"); + writer.print("add_" + rTableName + " <- function("); + writer.print(table.getFields().stream().map(Field::getName).map(this::convertToRName).collect(Collectors.joining(", "))); + writer.println(") {"); + writer.println(" defaults <- get('" + rTableName + "', envir = frameworkContext$defaultValues)"); + writer.println(" fields <- c()"); + writer.println(" values <- c()"); for (Field field : table.getFields()) { String rFieldName = convertToRName(field.getName()); String sqlFieldName = convertToSqlName(field.getName()); - r.add(" if (missing(" + rFieldName + ")) {"); - r.add(" " + rFieldName + " <- defaults$" + rFieldName); - r.add(" }"); - r.add(" if (!is.null(" + rFieldName + ")) {"); - r.add(" fields <- c(fields, \"" + sqlFieldName + "\")"); - r.add(" values <- c(values, " + createSqlValueCode(rFieldName) + ")"); - r.add(" }"); - r.add(""); + writer.println(" if (missing(" + rFieldName + ")) {"); + writer.println(" " + rFieldName + " <- defaults$" + rFieldName); + writer.println(" } else {"); + writer.println(" frameworkContext$sourceFieldsTested <- c(frameworkContext$sourceFieldsTested, '" + convertFieldToFullName(field) + "')"); + writer.println(" }"); + writer.println(" fields <- c(fields, \"" + sqlFieldName + "\")"); + writer.println(" values <- c(values, " + createSqlValueCode(rFieldName) + ")"); + writer.println(""); } - r.add(" inserts <- list(testId = frameworkContext$testId, testDescription = frameworkContext$testDescription, table = \"" + sqlTableName + writer.println(" inserts <- list(testId = frameworkContext$testId, testDescription = frameworkContext$testDescription, table = \"" + sqlTableName + "\", fields = fields, values = values)"); - r.add(" frameworkContext$inserts[[length(frameworkContext$inserts) + 1]] <- inserts"); - r.add(" invisible(NULL)"); - r.add("}"); - r.add(""); + writer.println(" frameworkContext$inserts[[length(frameworkContext$inserts) + 1]] <- inserts"); + writer.println(" invisible(NULL)"); + writer.println("}"); + writer.println(""); } } } private void createExpectFunctions(int type) { - for (Table table : targetDb.getTables()) { + for (Table table : etl.getTargetDatabase().getTables()) { if (!table.isStem()) { - StringBuilder line = new StringBuilder(); String rTableName = convertToRName(table.getName()); String sqlTableName = convertToSqlName(table.getName()); - List argDefs = new ArrayList(); - for (Field field : table.getFields()) { - String rFieldName = convertToRName(field.getName()); - argDefs.add(rFieldName); - } if (type == DEFAULT) - line.append("expect_" + rTableName + " <- function("); + writer.print("expect_" + rTableName + " <- function("); else if (type == NEGATE) - line.append("expect_no_" + rTableName + " <- function("); + writer.print("expect_no_" + rTableName + " <- function("); else - line.append("expect_count_" + rTableName + " <- function(rowCount, "); - line.append(StringUtilities.join(argDefs, ", ")); - line.append(") {"); - r.add(line.toString()); - r.add(" fields <- c()"); - r.add(" values <- c()"); + writer.print("expect_count_" + rTableName + " <- function(rowCount, "); + writer.print(table.getFields().stream().map(Field::getName).map(this::convertToRName).collect(Collectors.joining(", "))); + writer.println(") {"); + writer.println(" fields <- c()"); + writer.println(" values <- c()"); for (Field field : table.getFields()) { String rFieldName = convertToRName(field.getName()); String sqlFieldName = convertToSqlName(field.getName()); - r.add(" if (!missing(" + rFieldName + ")) {"); - r.add(" fields <- c(fields, \"" + sqlFieldName + "\")"); - r.add(" values <- c(values, " + createSqlValueCode(rFieldName) + ")"); - r.add(" }"); - r.add(""); + writer.println(" if (!missing(" + rFieldName + ")) {"); + writer.println(" fields <- c(fields, \"" + sqlFieldName + "\")"); + writer.println(" values <- c(values, " + createSqlValueCode(rFieldName) + ")"); + if (type != NEGATE) { + writer.println(" frameworkContext$targetFieldsTested <- c(frameworkContext$targetFieldsTested, '" + convertFieldToFullName(field) + "')"); + } + writer.println(" }"); + writer.println(""); } - r.add(" expects <- list(testId = frameworkContext$testId, testDescription = frameworkContext$testDescription, type = " + type + ", table = \"" + writer.println(" expects <- list(testId = frameworkContext$testId, testDescription = frameworkContext$testDescription, type = " + type + ", table = \"" + sqlTableName + "\", fields = fields, values = values)"); if (type == COUNT) - r.add(" expects$rowCount = rowCount"); - r.add(" frameworkContext$expects[[length(frameworkContext$expects) + 1]] <- expects"); - r.add(" invisible(NULL)"); - r.add("}"); - r.add(""); + writer.println(" expects$rowCount = rowCount"); + writer.println(" frameworkContext$expects[[length(frameworkContext$expects) + 1]] <- expects"); + writer.println(" invisible(NULL)"); + writer.println("}"); + writer.println(""); } } } - protected void createLookupFunctions() { - for (Table table : targetDb.getTables()) { + private void createLookupFunctions() { + for (Table table : etl.getTargetDatabase().getTables()) { if (!table.isStem()) { - StringBuilder line = new StringBuilder(); String rTableName = convertToRName(table.getName()); String sqlTableName = convertToSqlName(table.getName()); - List argDefs = new ArrayList(); - for (Field field : table.getFields()) { - String rFieldName = convertToRName(field.getName()); - argDefs.add(rFieldName); - } - line.append("lookup_" + rTableName + " <- function(fetchField, "); - line.append(StringUtilities.join(argDefs, ", ")); - line.append(") {"); - r.add(line.toString()); - line = new StringBuilder(); - line.append(" statement <- paste0('SELECT ', fetchField , ' FROM @cdm_database_schema."); - line.append(sqlTableName); - line.append(" WHERE')"); - r.add(line.toString()); - r.add(" first <- TRUE"); + writer.print("lookup_" + rTableName + " <- function(fetchField, "); + writer.print(table.getFields().stream().map(Field::getName).map(this::convertToRName).collect(Collectors.joining(", "))); + writer.println(") {"); + + writer.print(" statement <- paste0('SELECT ', fetchField , ' FROM @cdm_database_schema."); + writer.print(sqlTableName); + writer.println(" WHERE')"); + + writer.println(" first <- TRUE"); for (Field field : table.getFields()) { String rFieldName = convertToRName(field.getName()); String sqlFieldName = convertToSqlName(field.getName()); - argDefs.add(rFieldName); - r.add(" if (!missing(" + rFieldName + ")) {"); - r.add(" if (first) {"); - r.add(" first <- FALSE"); - r.add(" } else {"); - r.add(" statement <- paste0(statement, \" AND\")"); - r.add(" }"); - r.add(" statement <- paste0(statement, \" " + sqlFieldName + " = \", " + createSqlValueCode(rFieldName) + ")"); - r.add(" }"); - r.add(""); + writer.println(" if (!missing(" + rFieldName + ")) {"); + writer.println(" if (first) {"); + writer.println(" first <- FALSE"); + writer.println(" } else {"); + writer.println(" statement <- paste0(statement, \" AND\")"); + writer.println(" }"); + writer.println(" statement <- paste0(statement, \" " + sqlFieldName + " = \", " + createSqlValueCode(rFieldName) + ")"); + writer.println(" }"); + writer.println(""); } - r.add(" class(statement) <- 'subQuery'"); - r.add(" return(statement)"); - r.add("}"); - r.add(""); + writer.println(" class(statement) <- 'subQuery'"); + writer.println(" return(statement)"); + writer.println("}"); + writer.println(""); } } } - protected void createGenerateInsertSqlFunction() { - r.add("generateInsertSql <- function(databaseSchema = NULL) {"); - r.add(" insertSql <- c()"); - for (Table table : sourceDb.getTables()) + private void createGenerateInsertSqlFunction() { + writer.println("generateInsertSql <- function(databaseSchema = NULL) {"); + writer.println(" insertSql <- c()"); + for (Table table : etl.getSourceDatabase().getTables()) { if (!table.isStem()) - r.add(" insertSql <- c(insertSql, \"TRUNCATE TABLE @cdm_database_schema." + convertToSqlName(table.getName()) + ";\")"); - r.add(" createInsertStatement <- function(insert, env) {"); - r.add(" s <- c()"); - r.add(" if (env$testId != insert$testId) {"); - r.add(" s <- c(s, paste0('-- ', insert$testId, ': ', insert$testDescription))"); - r.add(" env$testId <- insert$testId"); - r.add(" }"); - r.add(" s <- c(s, paste0(\"INSERT INTO @cdm_database_schema.\","); - r.add(" insert$table,"); - r.add(" \"(\","); - r.add(" paste(insert$fields, collapse = \", \"),"); - r.add(" \") VALUES (\","); - r.add(" paste(insert$values, collapse = \", \"), "); - r.add(" \");\"))"); - r.add(" return(s)"); - r.add(" }"); - r.add(" env <- new.env()"); - r.add(" env$testId <- -1"); - r.add(" insertSql <- c(insertSql, do.call(c, lapply(frameworkContext$inserts, createInsertStatement, env)))"); - r.add(" if (is.null(databaseSchema)) {"); - r.add(" insertSql <- gsub('@cdm_database_schema.', '', insertSql)"); - r.add(" } else {"); - r.add(" insertSql <- gsub('@cdm_database_schema', databaseSchema, insertSql)"); - r.add(" }"); - r.add(" return(insertSql)"); - r.add("}"); - r.add(""); + writer.println(" insertSql <- c(insertSql, \"TRUNCATE TABLE @cdm_database_schema." + convertToSqlName(table.getName()) + ";\")"); + } + writer.println(" createInsertStatement <- function(insert, env) {"); + writer.println(" s <- c()"); + writer.println(" if (env$testId != insert$testId) {"); + writer.println(" s <- c(s, paste0('-- ', insert$testId, ': ', insert$testDescription))"); + writer.println(" env$testId <- insert$testId"); + writer.println(" }"); + writer.println(" s <- c(s, paste0(\"INSERT INTO @cdm_database_schema.\","); + writer.println(" insert$table,"); + writer.println(" \"(\","); + writer.println(" paste(insert$fields, collapse = \", \"),"); + writer.println(" \") VALUES (\","); + writer.println(" paste(insert$values, collapse = \", \"), "); + writer.println(" \");\"))"); + writer.println(" return(s)"); + writer.println(" }"); + writer.println(" env <- new.env()"); + writer.println(" env$testId <- -1"); + writer.println(" insertSql <- c(insertSql, do.call(c, lapply(frameworkContext$inserts, createInsertStatement, env)))"); + writer.println(" if (is.null(databaseSchema)) {"); + writer.println(" insertSql <- gsub('@cdm_database_schema.', '', insertSql)"); + writer.println(" } else {"); + writer.println(" insertSql <- gsub('@cdm_database_schema', databaseSchema, insertSql)"); + writer.println(" }"); + writer.println(" return(insertSql)"); + writer.println("}"); + writer.println(""); } - protected void createGenerateTestSqlFunction() { - r.add("generateTestSql <- function(databaseSchema = NULL) {"); - r.add(" testSql <- c()"); - r.add(" testSql <- c(testSql, \"IF OBJECT_ID('@cdm_database_schema.test_results', 'U') IS NOT NULL DROP TABLE @cdm_database_schema.test_results;\")"); - r.add(" testSql <- c(testSql, \"CREATE TABLE @cdm_database_schema.test_results (id INT, description VARCHAR(512), test VARCHAR(256), status VARCHAR(5));\")"); - r.add(" createExpectStatement <- function(expect, env) {"); - r.add(" s <- c()"); - r.add(" if (env$testId != expect$testId) {"); - r.add(" s <- c(s, paste0('-- ', expect$testId, ': ', expect$testDescription))"); - r.add(" env$testId <- expect$testId"); - r.add(" }"); - r.add(" operators <- rep(\"=\", length(expect$fields))"); - r.add(" operators[expect$values == \"NULL\"] <- rep(\"IS\", sum(expect$values == \"NULL\"))"); - r.add(" s <- c(s, paste0(\"INSERT INTO @cdm_database_schema.test_results SELECT \","); - r.add(" expect$testId,"); - r.add(" \" AS id, '\","); - r.add(" expect$testDescription,"); - r.add(" \"' AS description, 'Expect \","); - r.add(" expect$table,"); - r.add(" \"' AS test, CASE WHEN (SELECT COUNT(*) FROM @cdm_database_schema.\","); - r.add(" expect$table,"); - r.add(" \" WHERE \","); - r.add(" paste(paste(expect$fields, operators, expect$values), collapse = \" AND \"),"); - r.add(" \") \","); - r.add(" if (expect$type == " + DEFAULT + ") \"= 0\" else if (expect$type == " + NEGATE + private void createSourceCsvFunction() { + writer.println("writeSourceCsv <- function(directory = NULL, separator = ',') {"); + // Function to remove artificial quotes, escape quotes and separator + writer.println(" clean_value <- function(x) {"); + writer.println(" if (x == 'NULL') {"); + writer.println(" return('')"); + writer.println(" }"); + writer.println(" value <- substring(x, 2, nchar(x)-1)"); + writer.println(" value <- gsub('\"', '\"\"', value)"); + writer.println(" if (grepl(separator, value)) {"); + writer.println(" return(paste0('\"', value, '\"'))"); + writer.println(" }"); + writer.println(" return(value)"); + writer.println(" }"); + writer.println(""); + + // Function to remove leading and trailing [], if present + writer.println(" clean_fields <- function(x) {"); + writer.println(" if (grepl(\"^\\\\[.+?\\\\]$\", x)) {"); + writer.println(" return(substring(x, 2, nchar(x)-1))"); + writer.println(" }"); + writer.println(" return(x)"); + writer.println(" }"); + writer.println(" dir.create(directory, showWarnings = F)"); + writer.println(" "); + + // Write values + // Initialize all new source files with header. Overwrites existing source files from previous runs in the directory. + writer.println(" seen_tables <- c()"); + writer.println(" for (insert in frameworkContext$inserts) {"); + writer.println(" filename <- file.path(directory, paste0(insert$table, '.csv'))"); + writer.println(" if (!(insert$table %in% seen_tables)) {"); + writer.println(" write(paste(sapply(insert$fields, clean_fields), collapse = separator), filename, append=F)"); + writer.println(" seen_tables <- c(seen_tables, insert$table)"); + writer.println(" }"); + writer.println(" write(paste(sapply(insert$values, clean_value), collapse = separator), filename, append=T)"); + writer.println(" }"); + writer.println(" "); + + // Create source files for which there are no inserts + writer.println(" for (table_name in names(frameworkContext$defaultValues)) {"); + writer.println(" if (!(table_name %in% seen_tables)) {"); + writer.println(" filename <- file.path(directory, paste0(table_name, '.csv'))"); + writer.println(" write(paste(names(frameworkContext$defaultValues[[table_name]]), collapse = separator), filename, append=F)"); + writer.println(" }"); + writer.println(" }"); + writer.println("}"); + writer.println(""); + } + + private void createGenerateTestSqlFunction() { + writer.println("generateTestSql <- function(databaseSchema = NULL) {"); + writer.println(" testSql <- c()"); + writer.println(" testSql <- c(testSql, \"IF OBJECT_ID('@cdm_database_schema.test_results', 'U') IS NOT NULL DROP TABLE @cdm_database_schema.test_results;\")"); + writer.println(" testSql <- c(testSql, \"CREATE TABLE @cdm_database_schema.test_results (id INT, description VARCHAR(512), test VARCHAR(256), status VARCHAR(5));\")"); + writer.println(" createExpectStatement <- function(expect, env) {"); + writer.println(" s <- c()"); + writer.println(" if (env$testId != expect$testId) {"); + writer.println(" s <- c(s, paste0('-- ', expect$testId, ': ', expect$testDescription))"); + writer.println(" env$testId <- expect$testId"); + writer.println(" }"); + writer.println(" operators <- rep(\"=\", length(expect$fields))"); + writer.println(" operators[expect$values == \"NULL\"] <- rep(\"IS\", sum(expect$values == \"NULL\"))"); + writer.println(" s <- c(s, paste0(\"INSERT INTO @cdm_database_schema.test_results SELECT \","); + writer.println(" expect$testId,"); + writer.println(" \" AS id, '\","); + writer.println(" expect$testDescription,"); + writer.println(" \"' AS description, '\","); + writer.println(" extractTestTypeString(expect), \" \", expect$table,"); + writer.println(" \"' AS test, CASE WHEN (SELECT COUNT(*) FROM @cdm_database_schema.\","); + writer.println(" expect$table,"); + writer.println(" \" WHERE \","); + writer.println(" paste(paste(expect$fields, operators, expect$values), collapse = \" AND \"),"); + writer.println(" \") \","); + writer.println(" if (expect$type == " + DEFAULT + ") \"= 0\" else if (expect$type == " + NEGATE + ") \"!= 0\" else paste(\"!=\", expect$rowCount),"); - r.add(" \" THEN 'FAIL' ELSE 'PASS' END AS status;\"))"); - r.add(" return(s)"); - r.add(" }"); - r.add(" env <- new.env()"); - r.add(" env$testId <- -1"); - r.add(" testSql <- c(testSql, do.call(c, lapply(frameworkContext$expects, createExpectStatement, env)))"); - r.add(" if (is.null(databaseSchema)) {"); - r.add(" testSql <- gsub('@cdm_database_schema.', '', testSql)"); - r.add(" } else {"); - r.add(" testSql <- gsub('@cdm_database_schema', databaseSchema, testSql)"); - r.add(" }"); - r.add(" return(testSql)"); - r.add("}"); - r.add(""); + writer.println(" \" THEN 'FAIL' ELSE 'PASS' END AS status;\"))"); + writer.println(" return(s)"); + writer.println(" }"); + writer.println(" env <- new.env()"); + writer.println(" env$testId <- -1"); + writer.println(" testSql <- c(testSql, do.call(c, lapply(frameworkContext$expects, createExpectStatement, env)))"); + writer.println(" if (is.null(databaseSchema)) {"); + writer.println(" testSql <- gsub('@cdm_database_schema.', '', testSql)"); + writer.println(" } else {"); + writer.println(" testSql <- gsub('@cdm_database_schema', databaseSchema, testSql)"); + writer.println(" }"); + writer.println(" return(testSql)"); + writer.println("}"); + writer.println(""); + } + + private void createExtractTestTypeStringFunction() { + writer.println("extractTestTypeString <- function(x) {"); + writer.println(" if (x$type == 0) {"); + writer.println(" return('Expect')"); + writer.println(" } else if (x$type==1) {"); + writer.println(" return('Expect No')"); + writer.println(" } else if (x$type==2) {"); + writer.println(" return(paste('Expect', x$rowCount))"); + writer.println(" }"); + writer.println("}"); + writer.println(""); + } + + private void createTestsOverviewFunctions() { + writer.println("getTestsOverview <- function() {"); + writer.println(" df <- data.frame("); + writer.println(" testId = sapply(frameworkContext$expects, function(x) x$testId),"); + writer.println(" testDescription = sapply(frameworkContext$expects, function(x) x$testDescription),"); + writer.println(" testType = sapply(frameworkContext$expects, extractTestTypeString),"); + writer.println(" testTable = sapply(frameworkContext$expects, function(x) x$table)"); + writer.println(" )"); + writer.println(" return(df)"); + writer.println("}"); + writer.println(""); + writer.println("exportTestsOverviewToFile <- function(filename) {"); + writer.println(" df <- getTestsOverview()"); + writer.println(" write.csv(unique(df), filename, row.names=F)"); + writer.println("}"); + writer.println(""); + } + + private void createSummaryFunction() { + writer.println("summary.frameworkContext <- function(object, ...) {"); + writer.println(" nSourceFieldsTested <- length(intersect(object$sourceFieldsMapped, object$sourceFieldsTested))"); + writer.println(" nTargetFieldsTested <- length(intersect(object$targetFieldsMapped, object$targetFieldsTested))"); + writer.println(" nTotalSourceFields <- length(object$sourceFieldsMapped)"); + writer.println(" nTotalTargetFields <- length(object$targetFieldsMapped)"); + writer.println(" summary <- c("); + writer.println(" length(object$expects),"); + writer.println(" length(unique(sapply(object$expects, function(x) x$testId))),"); + writer.println(" nSourceFieldsTested,"); + writer.println(" nTotalSourceFields,"); + writer.println(" round(100*nSourceFieldsTested/nTotalSourceFields, 2),"); + writer.println(" nTargetFieldsTested,"); + writer.println(" nTotalTargetFields,"); + writer.println(" round(100*nTargetFieldsTested/nTotalTargetFields, 2)"); + writer.println(" )"); + writer.println(" names(summary) <- c('n_tests', 'n_cases', 'n_source_fields_tested', 'n_source_fields_mapped_from', 'source_coverage (%)', 'n_target_fields_tested', 'n_target_fields_mapped_to', 'target_coverage (%)')"); + writer.println(" return(as.data.frame(summary))"); + writer.println("}"); + writer.println(""); + writer.println("summaryTestFramework <- function() {"); + writer.println(" return(summary(frameworkContext));"); + writer.println("}"); + writer.println(""); + } + + private void createGetUntestedFieldsFunctions() { + writer.println("getUntestedSourceFields <- function() {"); + writer.println(" sort(setdiff(frameworkContext$sourceFieldsMapped, frameworkContext$sourceFieldsTested))"); + writer.println("}"); + writer.println(""); + + writer.println("getUntestedTargetFields <- function() {"); + writer.println(" sort(setdiff(frameworkContext$targetFieldsMapped, frameworkContext$targetFieldsTested))"); + writer.println("}"); + writer.println(""); + } + + private void createOutputTestResultsSummaryFunction() { + // Suppress any errors or warnings if unable to load DatabaseConnector, as the rest of the test framework does not need it. + writer.println("outputTestResultsSummary <- function(connection, databaseSchema = NULL) {"); + writer.println(" suppressWarnings(require(DatabaseConnector, quietly = TRUE))"); + writer.println(" query = 'SELECT * FROM @cdm_database_schema.test_results;'"); + writer.println(" if (is.null(databaseSchema)) {"); + writer.println(" query <- gsub('@cdm_database_schema.', '', query)"); + writer.println(" } else {"); + writer.println(" query <- gsub('@cdm_database_schema', databaseSchema, query)"); + writer.println(" }"); + writer.println(" df_results <- DatabaseConnector::querySql(connection, query)"); + writer.println(" n_tests <- nrow(df_results)"); + writer.println(" n_failed_tests <- sum(df_results$'STATUS' == 'FAIL')"); + writer.println(" if (n_failed_tests > 0) {"); + writer.println(" write(sprintf('FAILED unit tests: %d/%d (%.1f%%)', n_failed_tests, n_tests, n_failed_tests/n_tests * 100), file='')"); + writer.println(" print(df_results[df_results$'STATUS' == 'FAIL',])"); + writer.println(" } else {"); + writer.println(" write(sprintf('All %d tests PASSED', n_tests), file='')"); + writer.println(" }"); + writer.println("}"); + writer.println(""); + } + + private String removeExtension(String name) { + return name.replaceAll("\\.\\w{3,4}$", ""); } private String convertToRName(String name) { - name = name.replaceAll(" ", "_").replaceAll("-", "_").replaceAll("^_+", ""); + name = removeExtension(name); + // Replace space, dash and brackets by an underscore. If name starts with underscore, remove. + name = name.replaceAll("[\\s-()\\[\\]{}]", "_").replaceAll("^_+", ""); + // Remove BOM + if (name.startsWith("\uFEFF")) { + name = name.substring(1); + } return name; } - protected String createSqlValueCode(String rFieldName) { - StringBuilder expression = new StringBuilder(); - expression.append("if (is.null(" + rFieldName + ")) \"NULL\" "); - expression.append("else if (is(" + rFieldName + ", \"subQuery\")) paste0(\"(\", as.character(" + rFieldName + "), \")\") "); - expression.append("else paste0(\"'\", as.character(" + rFieldName + "), \"'\")"); - return (expression.toString()); + private String createSqlValueCode(String rFieldName) { + return "if (is.null(" + rFieldName + ")) \"NULL\" " + + "else if (is(" + rFieldName + ", \"subQuery\")) " + + "paste0(\"(\", as.character(" + rFieldName + "), \")\") " + + "else paste0(\"'\", as.character(" + rFieldName + "), \"'\")"; } - protected String convertToSqlName(String name) { - if (name.startsWith("[") && name.endsWith("]")) + private String convertToSqlName(String name) { + name = removeExtension(name); + if (name.startsWith("[") && name.endsWith("]")) { return name; + } name = name.replace('[', '_').replace(']', '_'); - if (name.contains(" ") || name.contains(".") || keywordSet.contains(name.toUpperCase())) + if (name.contains(" ") || name.contains(".") || keywordSet.contains(name.toUpperCase())) { return "[" + name + "]"; + } return name; } + + private String convertFieldToFullName(Field field) { + return convertToRName(field.getTable().getName()) + "." + convertToRName(field.getName()); + } } diff --git a/rabbitinahat/src/main/java/org/ohdsi/rabbitInAHat/LabeledRectangle.java b/rabbitinahat/src/main/java/org/ohdsi/rabbitInAHat/LabeledRectangle.java index 14f6f6d7..86c21a77 100644 --- a/rabbitinahat/src/main/java/org/ohdsi/rabbitInAHat/LabeledRectangle.java +++ b/rabbitinahat/src/main/java/org/ohdsi/rabbitInAHat/LabeledRectangle.java @@ -126,21 +126,34 @@ public void paint(Graphics g) { index = nextBreakPoint(item.outputName(), index + 1); } if (breakPoint == 0) { - int textX = (this.getWidth() - (int) r.getWidth()) / 2; - int textY = (this.getHeight() - (int) r.getHeight()) / 2 + fm.getAscent(); - g2d.drawString(item.outputName(), x + textX, y + textY); + breakPoint = (int) midPoint + 1; + } else { + breakPoint++; } - breakPoint++; String line1 = item.outputName().substring(0, breakPoint); String line2 = item.outputName().substring(breakPoint); - r = fm.getStringBounds(line1, g2d); - int textX = (this.getWidth() - (int) r.getWidth()) / 2; - int textY = (this.getHeight() / 2 - (int) r.getHeight()) / 2 + fm.getAscent(); - g2d.drawString(line1, x + textX, y + textY); - r = fm.getStringBounds(line2, g2d); - textX = (this.getWidth() - (int) r.getWidth()) / 2; - textY = (int) Math.round(this.getHeight() * 1.5 - (int) r.getHeight()) / 2 + fm.getAscent(); - g2d.drawString(line2, x + textX, y + textY); + + Rectangle2D r1 = fm.getStringBounds(line1, g2d); + Rectangle2D r2 = fm.getStringBounds(line2, g2d); + if (r1.getWidth() >= width) { + line1 = item.outputName().substring(0, (int) midPoint); + line2 = item.outputName().substring((int) midPoint); + r1 = fm.getStringBounds(line1, g2d); + r2 = fm.getStringBounds(line2, g2d); + } else if (r2.getWidth() >= width) { + line1 = item.outputName().substring(0, (int) midPoint); + line2 = item.outputName().substring((int) midPoint); + r1 = fm.getStringBounds(line1, g2d); + r2 = fm.getStringBounds(line2, g2d); + } + // If both lines are too wide, then the text will still go out of bounds + int textX1 = (this.getWidth() - (int) r1.getWidth()) / 2; + int textY1 = (this.getHeight() / 2 - (int) r1.getHeight()) / 2 + fm.getAscent(); + g2d.drawString(line1, x + textX1, y + textY1); + + int textX2 = (this.getWidth() - (int) r2.getWidth()) / 2; + int textY2 = (int) Math.round(this.getHeight() * 1.5 - (int) r2.getHeight()) / 2 + fm.getAscent(); + g2d.drawString(line2, x + textX2, y + textY2); } else { int textX = (this.getWidth() - (int) r.getWidth()) / 2; int textY = (this.getHeight() - (int) r.getHeight()) / 2 + fm.getAscent(); diff --git a/rabbitinahat/src/main/java/org/ohdsi/rabbitInAHat/MappingPanel.java b/rabbitinahat/src/main/java/org/ohdsi/rabbitInAHat/MappingPanel.java index 990405b0..621e3600 100644 --- a/rabbitinahat/src/main/java/org/ohdsi/rabbitInAHat/MappingPanel.java +++ b/rabbitinahat/src/main/java/org/ohdsi/rabbitInAHat/MappingPanel.java @@ -274,7 +274,7 @@ public void setSize(int width, int height) { public void filterComponents(String searchTerm, boolean filterTarget) { List components; - if (filterTarget == true) { + if (filterTarget) { components = cdmComponents; lastTargetFilter = searchTerm; } else { @@ -730,7 +730,8 @@ private void LabeledRectangleClicked(MouseEvent event, List co lastSelectedRectangle = null; } - detailsListener.showDetails(component.getItem()); + boolean isSourceComponent = sourceComponents.contains(component); + detailsListener.showDetails(component.getItem(), isSourceComponent); repaint(); break; } diff --git a/rabbitinahat/src/main/java/org/ohdsi/rabbitInAHat/RabbitInAHatMain.java b/rabbitinahat/src/main/java/org/ohdsi/rabbitInAHat/RabbitInAHatMain.java index 505abc83..dff8c676 100644 --- a/rabbitinahat/src/main/java/org/ohdsi/rabbitInAHat/RabbitInAHatMain.java +++ b/rabbitinahat/src/main/java/org/ohdsi/rabbitInAHat/RabbitInAHatMain.java @@ -38,20 +38,7 @@ import java.util.ArrayList; import java.util.List; -import javax.swing.BoxLayout; -import javax.swing.ButtonGroup; -import javax.swing.JFileChooser; -import javax.swing.JFrame; -import javax.swing.JMenu; -import javax.swing.JMenuBar; -import javax.swing.JMenuItem; -import javax.swing.JOptionPane; -import javax.swing.JRadioButtonMenuItem; -import javax.swing.JScrollPane; -import javax.swing.JSplitPane; -import javax.swing.KeyStroke; -import javax.swing.ScrollPaneConstants; -import javax.swing.UIManager; +import javax.swing.*; import javax.swing.border.TitledBorder; import javax.swing.filechooser.FileFilter; import javax.swing.filechooser.FileNameExtensionFilter; @@ -62,7 +49,7 @@ import org.ohdsi.rabbitInAHat.dataModel.ETL; import org.ohdsi.rabbitInAHat.dataModel.Field; import org.ohdsi.rabbitInAHat.dataModel.MappableItem; -import org.ohdsi.rabbitInAHat.dataModel.StemTableAdd; +import org.ohdsi.rabbitInAHat.dataModel.StemTableFactory; import org.ohdsi.rabbitInAHat.dataModel.Table; /** @@ -78,6 +65,7 @@ public class RabbitInAHatMain implements ResizeListener, ActionListener { public final static String ACTION_GENERATE_ETL_HTML_DOCUMENT = "Generate ETL HTML Document"; public final static String ACTION_GENERATE_ETL_MD_DOCUMENT = "Generate ETL Markdown Document"; public final static String ACTION_GENERATE_TEST_FRAMEWORK = "Generate ETL Test Framework"; + public final static String ACTION_GENERATE_SQL = "Generate SQL Skeleton"; public final static String ACTION_DISCARD_COUNTS = "Discard Value Counts"; public final static String ACTION_FILTER = "Filter"; public final static String ACTION_MAKE_MAPPING = "Make Mappings"; @@ -91,6 +79,7 @@ public class RabbitInAHatMain implements ResizeListener, ActionListener { public final static String ACTION_SET_TARGET_V531 = "CDM v5.3.1"; public final static String ACTION_SET_TARGET_V60 = "CDM v6.0"; public final static String ACTION_ADD_STEM_TABLE = "Add stem table"; + public final static String ACTION_REMOVE_STEM_TABLE = "Remove stem table"; public final static String ACTION_SET_TARGET_CUSTOM = "Load Custom..."; public final static String ACTION_MARK_COMPLETED = "Mark Highlighted As Complete"; public final static String ACTION_UNMARK_COMPLETED = "Mark Highlighted As Incomplete"; @@ -132,9 +121,20 @@ public RabbitInAHatMain(String[] args) { } frame = new JFrame("Rabbit in a Hat"); + + frame.setDefaultCloseOperation(JFrame.DO_NOTHING_ON_CLOSE); frame.addWindowListener(new WindowAdapter() { public void windowClosing(WindowEvent e) { - System.exit(0); + String ObjButtons[] = {"Yes","No"}; + int PromptResult = JOptionPane.showOptionDialog( + null, + "Do you want to exit?\nPlease make sure that any work is saved", + "Rabbit In A Hat",JOptionPane.DEFAULT_OPTION,JOptionPane.WARNING_MESSAGE, + null,ObjButtons,ObjButtons[1] + ); + if (PromptResult==JOptionPane.YES_OPTION) { + System.exit(0); + } } }); frame.setPreferredSize(new Dimension(700, 600)); @@ -151,6 +151,7 @@ public void windowClosing(WindowEvent e) { scrollPane1 = new JScrollPane(tableMappingPanel); scrollPane1.setBorder(new TitledBorder("Tables")); scrollPane1.getVerticalScrollBar().setUnitIncrement(16); + scrollPane1.getViewport().setScrollMode(JViewport.BACKINGSTORE_SCROLL_MODE); scrollPane1.setAutoscrolls(true); scrollPane1.setOpaque(true); scrollPane1.setBackground(Color.WHITE); @@ -160,6 +161,7 @@ public void windowClosing(WindowEvent e) { fieldMappingPanel.addResizeListener(this); scrollPane2 = new JScrollPane(fieldMappingPanel); scrollPane2.getVerticalScrollBar().setUnitIncrement(16); + scrollPane2.getViewport().setScrollMode(JViewport.BACKINGSTORE_SCROLL_MODE); scrollPane2.setVisible(false); scrollPane2.setBorder(new TitledBorder("Fields")); @@ -258,6 +260,11 @@ private JMenuBar createMenuBar() { generateTestFrameworkItem.addActionListener(this); fileMenu.add(generateTestFrameworkItem); + JMenuItem generateSql = new JMenuItem(ACTION_GENERATE_SQL); + generateSql.addActionListener(this); + generateSql.setActionCommand(ACTION_GENERATE_SQL); + fileMenu.add(generateSql); + JMenu editMenu = new JMenu("Edit"); menuBar.add(editMenu); @@ -299,7 +306,7 @@ private JMenuBar createMenuBar() { JRadioButtonMenuItem targetCDMV531 = new JRadioButtonMenuItem(ACTION_SET_TARGET_V531); targetCDMV531.addActionListener(this); setTarget.add(targetCDMV531); - + JRadioButtonMenuItem targetCDMV60 = new JRadioButtonMenuItem(ACTION_SET_TARGET_V60, true); targetCDMV60.addActionListener(this); setTarget.add(targetCDMV60); @@ -324,6 +331,10 @@ private JMenuBar createMenuBar() { addStemTable.addActionListener(this); editMenu.add(addStemTable); + JMenuItem removeStemTable = new JMenuItem(ACTION_REMOVE_STEM_TABLE); + removeStemTable.addActionListener(this); + editMenu.add(removeStemTable); + JMenu arrowMenu = new JMenu("Arrows"); menuBar.add(arrowMenu); @@ -383,7 +394,7 @@ public void notifyResized(int height, boolean minimized, boolean maximized) { * restrict files displayed * @return if file selected, absolute path of selected file otherwise null */ - private String choosePath(boolean saveMode, FileFilter... filter) { + private String choosePath(boolean saveMode, boolean directoryMode, FileFilter... filter) { String result = null; if (chooser == null) { @@ -401,16 +412,32 @@ else if (filter[0] == FILE_FILTER_R) chooser.setSelectedFile(new File(chooser.getSelectedFile().getAbsolutePath().replaceAll("\\..*$", ".R"))); } chooser.resetChoosableFileFilters(); - chooser.setFileFilter(filter[0]); - for (int i = 1; i < filter.length; i++) - chooser.addChoosableFileFilter(filter[i]); + + if (directoryMode) { + chooser.setFileSelectionMode(JFileChooser.DIRECTORIES_ONLY); + chooser.setAcceptAllFileFilterUsed(false); + } else { + chooser.setFileSelectionMode(JFileChooser.FILES_ONLY); + chooser.setFileFilter(filter[0]); + for (int i = 1; i < filter.length; i++) + chooser.addChoosableFileFilter(filter[i]); + } int dialogResult = saveMode ? chooser.showSaveDialog(frame) : chooser.showOpenDialog(frame); - if (dialogResult == JFileChooser.APPROVE_OPTION) - result = chooser.getSelectedFile().getAbsolutePath(); + if (dialogResult == JFileChooser.APPROVE_OPTION) { + if (directoryMode) + result = chooser.getCurrentDirectory().getAbsolutePath(); + else + result = chooser.getSelectedFile().getAbsolutePath(); + } + return result; } + private String choosePath(boolean saveMode, FileFilter... filter) { + return choosePath(saveMode, false, filter); + } + private String chooseSavePath(FileFilter... fileFilter) { String path = choosePath(true, fileFilter); if (path != null && fileFilter[0] == FILE_FILTER_GZ && !path.toLowerCase().endsWith(".json.gz") && !path.toLowerCase().endsWith(".json")) @@ -430,6 +457,10 @@ private String chooseOpenPath(FileFilter... fileFilter) { return choosePath(false, fileFilter); } + private String chooseSaveDirectory() { + return choosePath(true, true, new FileNameExtensionFilter("Directories",".")); + } + @Override public void actionPerformed(ActionEvent event) { switch (event.getActionCommand()) { @@ -450,14 +481,17 @@ public void actionPerformed(ActionEvent event) { doGenerateEtlWordDoc(chooseSavePath(FILE_FILTER_DOCX)); break; case ACTION_GENERATE_ETL_HTML_DOCUMENT: - doGenerateEtlHtmlDoc(chooseSavePath(FILE_FILTER_HTML)); + doGenerateEtlHtmlDoc(chooseSaveDirectory()); break; case ACTION_GENERATE_ETL_MD_DOCUMENT: - doGenerateEtlMdDoc(chooseSavePath(FILE_FILTER_MD)); + doGenerateEtlMdDoc(chooseSaveDirectory()); break; case ACTION_GENERATE_TEST_FRAMEWORK: doGenerateTestFramework(chooseSavePath(FILE_FILTER_R)); break; + case ACTION_GENERATE_SQL: + doGenerateSql(chooseSaveDirectory()); + break; case ACTION_DISCARD_COUNTS: doDiscardCounts(); break; @@ -500,6 +534,9 @@ public void actionPerformed(ActionEvent event) { case ACTION_ADD_STEM_TABLE: doAddStemTable(); break; + case ACTION_REMOVE_STEM_TABLE: + doRemoveStemTable(); + break; case ACTION_MARK_COMPLETED: doMarkCompleted(); break; @@ -513,10 +550,27 @@ public void actionPerformed(ActionEvent event) { } private void doAddStemTable() { - ETL etl = ObjectExchange.etl; - StemTableAdd.addStemTable(etl); - ObjectExchange.etl = etl; - tableMappingPanel.setMapping(etl.getTableToTableMapping()); + if (!ObjectExchange.etl.hasStemTable()) { + StemTableFactory.addStemTable(ObjectExchange.etl); + tableMappingPanel.setMapping(ObjectExchange.etl.getTableToTableMapping()); + } + } + + private void doRemoveStemTable() { + if (ObjectExchange.etl.hasStemTable()) { + String[] ObjButtons = {"Yes","No"}; + int PromptResult = JOptionPane.showOptionDialog( + null, + "Any mappings to/from the stem table will be lost. Are you sure?", + "Rabbit In A Hat",JOptionPane.DEFAULT_OPTION,JOptionPane.WARNING_MESSAGE, + null,ObjButtons,ObjButtons[1] + ); + + if (PromptResult==JOptionPane.YES_OPTION) { + StemTableFactory.removeStemTable(ObjectExchange.etl); + tableMappingPanel.setMapping(ObjectExchange.etl.getTableToTableMapping()); + } + } } private void doGenerateTestFramework(String filename) { @@ -637,6 +691,7 @@ private void doOpenScanReport(String filename) { frame.setCursor(Cursor.getPredefinedCursor(Cursor.WAIT_CURSOR)); if (replace) { ETL etl = new ETL(); + doRemoveStemTable(); try { etl.setSourceDatabase(Database.generateModelFromScanReport(filename)); etl.setTargetDatabase(ObjectExchange.etl.getTargetDatabase()); @@ -686,20 +741,29 @@ private void doGenerateEtlWordDoc(String filename) { } } - private void doGenerateEtlHtmlDoc(String filename) { - if (filename != null) { + private void doGenerateEtlHtmlDoc(String directoryName) { + if (directoryName != null) { frame.setCursor(Cursor.getPredefinedCursor(Cursor.WAIT_CURSOR)); ETLMarkupDocumentGenerator generator = new ETLMarkupDocumentGenerator(ObjectExchange.etl); - generator.generate(filename, DocumentType.HTML); + generator.generate(directoryName, DocumentType.HTML); frame.setCursor(Cursor.getPredefinedCursor(Cursor.DEFAULT_CURSOR)); } } - private void doGenerateEtlMdDoc(String filename) { - if (filename != null) { + private void doGenerateEtlMdDoc(String directoryName) { + if (directoryName != null) { frame.setCursor(Cursor.getPredefinedCursor(Cursor.WAIT_CURSOR)); ETLMarkupDocumentGenerator generator = new ETLMarkupDocumentGenerator(ObjectExchange.etl); - generator.generate(filename, DocumentType.MARKDOWN); + generator.generate(directoryName, DocumentType.MARKDOWN); + frame.setCursor(Cursor.getPredefinedCursor(Cursor.DEFAULT_CURSOR)); + } + } + + private void doGenerateSql(String directoryName) { + if (directoryName != null) { + frame.setCursor(Cursor.getPredefinedCursor(Cursor.WAIT_CURSOR)); + SQLGenerator sqlGenerator = new SQLGenerator(ObjectExchange.etl, directoryName); + sqlGenerator.generate(); frame.setCursor(Cursor.getPredefinedCursor(Cursor.DEFAULT_CURSOR)); } } diff --git a/rabbitinahat/src/main/java/org/ohdsi/rabbitInAHat/SQLGenerator.java b/rabbitinahat/src/main/java/org/ohdsi/rabbitInAHat/SQLGenerator.java new file mode 100644 index 00000000..2457036c --- /dev/null +++ b/rabbitinahat/src/main/java/org/ohdsi/rabbitInAHat/SQLGenerator.java @@ -0,0 +1,157 @@ +package org.ohdsi.rabbitInAHat; + +import org.ohdsi.rabbitInAHat.dataModel.*; + +import java.io.*; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +/** + * Created 04-01-17. + * Generates separate SQL files for each table to table mapping in the given directory. + */ +public class SQLGenerator { + ETL etl; + String outputDirectory; + + public SQLGenerator(ETL etl, String directory) { + this.etl = etl; + this.outputDirectory = directory; + } + + public void generate() { + // Generate a sql file for each source to target table mapping + Integer counter = 0; + for (ItemToItemMap tableToTableMap : etl.getTableToTableMapping().getSourceToTargetMaps()) { + writeSqlFile(tableToTableMap); + counter++; + } + System.out.println(counter.toString() + " sql files were generated"); // TODO report in UI + } + + private void writeSqlFile(ItemToItemMap tableToTableMap){ + Table sourceTable = (Table) tableToTableMap.getSourceItem(); + Table targetTable = (Table) tableToTableMap.getTargetItem(); + Mapping fieldToFieldMapping = etl.getFieldToFieldMapping(sourceTable, targetTable); + List mappings = fieldToFieldMapping.getSourceToTargetMapsOrderedByCdmItems(); + + // Create new sql file in the selected directory + File outFile = new File( outputDirectory, sourceTable.getName() + "_to_" + targetTable.getName() + ".sql"); + System.out.println( "Writing to: " + outFile.getAbsoluteFile() ); + + int n_mappings = mappings.size(); + Set targetFieldsSeen = new HashSet<>(); + try (FileOutputStream fout = new FileOutputStream(outFile); + OutputStreamWriter fwr = new OutputStreamWriter(fout, "UTF-8"); + Writer out = new BufferedWriter(fwr)) { + + // Table specific comments + out.write(createBlockComment(sourceTable.getComment())); + out.write(createBlockComment(targetTable.getComment())); + out.write(createBlockComment(tableToTableMap.getComment())); + out.write(createBlockComment(tableToTableMap.getLogic())); + out.write('\n'); + + // To + out.write("INSERT INTO " + targetTable.getName() + "\n"); + out.write("(\n"); + List targetsWithoutSource = new ArrayList<>(); + for (int i=0;i mapping = etl.getTableToTableMapping(); + List> tablesMappingsToRemove = new ArrayList<>(); + for (ItemToItemMap sourceToTargetMap : mapping.getSourceToTargetMaps()) { + Table sourceTable = (Table) sourceToTargetMap.getSourceItem(); + Table targetTable = (Table) sourceToTargetMap.getTargetItem(); + if (sourceTable.isStem() || targetTable.isStem()) { + tablesMappingsToRemove.add(new Pair<>(sourceTable, targetTable)); + } + } + + // Remove stem table to table and field to field mappings + for (Pair tableTablePair : tablesMappingsToRemove) { + etl.getFieldToFieldMapping(tableTablePair.getItem1(), tableTablePair.getItem2()).removeAllSourceToTargetMaps(); + mapping.removeSourceToTargetMap(tableTablePair.getItem1(), tableTablePair.getItem2()); + } + + // Remove stem source table + Database sourceDatabase = etl.getSourceDatabase(); + List
newSourceTables = new ArrayList<>(); + for (Table table : sourceDatabase.getTables()) { + if (!table.isStem()) { + newSourceTables.add(table); + } + } + sourceDatabase.setTables(newSourceTables); + + // Remove stem target table + Database targetDatabase = etl.getTargetDatabase(); + List
newTargetTables = new ArrayList<>(); + for (Table table : targetDatabase.getTables()) { + if (!table.isStem()) { + newTargetTables.add(table); + } + } + targetDatabase.setTables(newTargetTables); + } + } diff --git a/rabbitinahat/src/main/resources/org/ohdsi/rabbitInAHat/dataModel/CDMConceptIDHints_v5.0_MAR-18.csv b/rabbitinahat/src/main/resources/org/ohdsi/rabbitInAHat/dataModel/CDMConceptIDHints_v5.0_MAR-18.csv new file mode 100644 index 00000000..6911a13a --- /dev/null +++ b/rabbitinahat/src/main/resources/org/ohdsi/rabbitInAHat/dataModel/CDMConceptIDHints_v5.0_MAR-18.csv @@ -0,0 +1,1441 @@ +omop_cdm_table,omop_cdm_field,concept_id,concept_name,standard_concept +condition_occurrence,condition_type_concept_id,45756852,Carrier claim detail - 10th position,S +condition_occurrence,condition_type_concept_id,45756853,Carrier claim detail - 11th position,S +condition_occurrence,condition_type_concept_id,45756854,Carrier claim detail - 12th position,S +condition_occurrence,condition_type_concept_id,45756855,Carrier claim detail - 13th position,S +condition_occurrence,condition_type_concept_id,45756843,Carrier claim detail - 1st position,S +condition_occurrence,condition_type_concept_id,45756844,Carrier claim detail - 2nd position,S +condition_occurrence,condition_type_concept_id,45756845,Carrier claim detail - 3rd position,S +condition_occurrence,condition_type_concept_id,45756846,Carrier claim detail - 4th position,S +condition_occurrence,condition_type_concept_id,45756847,Carrier claim detail - 5th position,S +condition_occurrence,condition_type_concept_id,45756848,Carrier claim detail - 6th position,S +condition_occurrence,condition_type_concept_id,45756849,Carrier claim detail - 7th position,S +condition_occurrence,condition_type_concept_id,45756850,Carrier claim detail - 8th position,S +condition_occurrence,condition_type_concept_id,45756851,Carrier claim detail - 9th position,S +condition_occurrence,condition_type_concept_id,45756835,Carrier claim header - 1st position,S +condition_occurrence,condition_type_concept_id,45756836,Carrier claim header - 2nd position,S +condition_occurrence,condition_type_concept_id,45756837,Carrier claim header - 3rd position,S +condition_occurrence,condition_type_concept_id,45756838,Carrier claim header - 4th position,S +condition_occurrence,condition_type_concept_id,45756839,Carrier claim header - 5th position,S +condition_occurrence,condition_type_concept_id,45756840,Carrier claim header - 6th position,S +condition_occurrence,condition_type_concept_id,45756841,Carrier claim header - 7th position,S +condition_occurrence,condition_type_concept_id,45756842,Carrier claim header - 8th position,S +condition_occurrence,condition_type_concept_id,38000246,Condition era - 0 days persistence window,S +condition_occurrence,condition_type_concept_id,38000247,Condition era - 30 days persistence window,S +condition_occurrence,condition_type_concept_id,5086,Condition tested for by diagnostic procedure,S +condition_occurrence,condition_type_concept_id,42894222,EHR Chief Complaint,S +condition_occurrence,condition_type_concept_id,45754805,EHR Episode Entry,S +condition_occurrence,condition_type_concept_id,32019,EHR billing diagnosis,S +condition_occurrence,condition_type_concept_id,32020,EHR encounter diagnosis,S +condition_occurrence,condition_type_concept_id,38000245,EHR problem list entry,S +condition_occurrence,condition_type_concept_id,44786628,First Position Condition,S +condition_occurrence,condition_type_concept_id,38000193,Inpatient detail - 10th position,S +condition_occurrence,condition_type_concept_id,38000194,Inpatient detail - 11th position,S +condition_occurrence,condition_type_concept_id,38000195,Inpatient detail - 12th position,S +condition_occurrence,condition_type_concept_id,38000196,Inpatient detail - 13th position,S +condition_occurrence,condition_type_concept_id,38000197,Inpatient detail - 14th position,S +condition_occurrence,condition_type_concept_id,38000198,Inpatient detail - 15th position,S +condition_occurrence,condition_type_concept_id,44818709,Inpatient detail - 16th position,S +condition_occurrence,condition_type_concept_id,44818710,Inpatient detail - 17th position,S +condition_occurrence,condition_type_concept_id,44818711,Inpatient detail - 18th position,S +condition_occurrence,condition_type_concept_id,44818712,Inpatient detail - 19th position,S +condition_occurrence,condition_type_concept_id,38000184,Inpatient detail - 1st position,S +condition_occurrence,condition_type_concept_id,44818713,Inpatient detail - 20th position,S +condition_occurrence,condition_type_concept_id,38000185,Inpatient detail - 2nd position,S +condition_occurrence,condition_type_concept_id,38000186,Inpatient detail - 3rd position,S +condition_occurrence,condition_type_concept_id,38000187,Inpatient detail - 4th position,S +condition_occurrence,condition_type_concept_id,38000188,Inpatient detail - 5th position,S +condition_occurrence,condition_type_concept_id,38000189,Inpatient detail - 6th position,S +condition_occurrence,condition_type_concept_id,38000190,Inpatient detail - 7th position,S +condition_occurrence,condition_type_concept_id,38000191,Inpatient detail - 8th position,S +condition_occurrence,condition_type_concept_id,38000192,Inpatient detail - 9th position,S +condition_occurrence,condition_type_concept_id,38000183,Inpatient detail - primary,S +condition_occurrence,condition_type_concept_id,38000209,Inpatient header - 10th position,S +condition_occurrence,condition_type_concept_id,38000210,Inpatient header - 11th position,S +condition_occurrence,condition_type_concept_id,38000211,Inpatient header - 12th position,S +condition_occurrence,condition_type_concept_id,38000212,Inpatient header - 13th position,S +condition_occurrence,condition_type_concept_id,38000213,Inpatient header - 14th position,S +condition_occurrence,condition_type_concept_id,38000214,Inpatient header - 15th position,S +condition_occurrence,condition_type_concept_id,38000200,Inpatient header - 1st position,S +condition_occurrence,condition_type_concept_id,38000201,Inpatient header - 2nd position,S +condition_occurrence,condition_type_concept_id,38000202,Inpatient header - 3rd position,S +condition_occurrence,condition_type_concept_id,38000203,Inpatient header - 4th position,S +condition_occurrence,condition_type_concept_id,38000204,Inpatient header - 5th position,S +condition_occurrence,condition_type_concept_id,38000205,Inpatient header - 6th position,S +condition_occurrence,condition_type_concept_id,38000206,Inpatient header - 7th position,S +condition_occurrence,condition_type_concept_id,38000207,Inpatient header - 8th position,S +condition_occurrence,condition_type_concept_id,38000208,Inpatient header - 9th position,S +condition_occurrence,condition_type_concept_id,38000199,Inpatient header - primary,S +condition_occurrence,condition_type_concept_id,43542353,Observation recorded from EHR,S +condition_occurrence,condition_type_concept_id,38000224,Outpatient detail - 10th position,S +condition_occurrence,condition_type_concept_id,38000225,Outpatient detail - 11th position,S +condition_occurrence,condition_type_concept_id,38000226,Outpatient detail - 12th position,S +condition_occurrence,condition_type_concept_id,38000227,Outpatient detail - 13th position,S +condition_occurrence,condition_type_concept_id,38000228,Outpatient detail - 14th position,S +condition_occurrence,condition_type_concept_id,38000229,Outpatient detail - 15th position,S +condition_occurrence,condition_type_concept_id,38000215,Outpatient detail - 1st position,S +condition_occurrence,condition_type_concept_id,38000216,Outpatient detail - 2nd position,S +condition_occurrence,condition_type_concept_id,38000217,Outpatient detail - 3rd position,S +condition_occurrence,condition_type_concept_id,38000218,Outpatient detail - 4th position,S +condition_occurrence,condition_type_concept_id,38000219,Outpatient detail - 5th position,S +condition_occurrence,condition_type_concept_id,38000220,Outpatient detail - 6th position,S +condition_occurrence,condition_type_concept_id,38000221,Outpatient detail - 7th position,S +condition_occurrence,condition_type_concept_id,38000222,Outpatient detail - 8th position,S +condition_occurrence,condition_type_concept_id,38000223,Outpatient detail - 9th position,S +condition_occurrence,condition_type_concept_id,38000239,Outpatient header - 10th position,S +condition_occurrence,condition_type_concept_id,38000240,Outpatient header - 11th position,S +condition_occurrence,condition_type_concept_id,38000241,Outpatient header - 12th position,S +condition_occurrence,condition_type_concept_id,38000242,Outpatient header - 13th position,S +condition_occurrence,condition_type_concept_id,38000243,Outpatient header - 14th position,S +condition_occurrence,condition_type_concept_id,38000244,Outpatient header - 15th position,S +condition_occurrence,condition_type_concept_id,38000230,Outpatient header - 1st position,S +condition_occurrence,condition_type_concept_id,38000231,Outpatient header - 2nd position,S +condition_occurrence,condition_type_concept_id,38000232,Outpatient header - 3rd position,S +condition_occurrence,condition_type_concept_id,38000233,Outpatient header - 4th position,S +condition_occurrence,condition_type_concept_id,38000234,Outpatient header - 5th position,S +condition_occurrence,condition_type_concept_id,38000235,Outpatient header - 6th position,S +condition_occurrence,condition_type_concept_id,38000236,Outpatient header - 7th position,S +condition_occurrence,condition_type_concept_id,38000237,Outpatient header - 8th position,S +condition_occurrence,condition_type_concept_id,38000238,Outpatient header - 9th position,S +condition_occurrence,condition_type_concept_id,45905770,Patient Self-Reported Condition,S +condition_occurrence,condition_type_concept_id,44786627,Primary Condition,S +condition_occurrence,condition_type_concept_id,42898140,Referral record,S +condition_occurrence,condition_type_concept_id,44786629,Secondary Condition,S +cost,cost_concept_id,32017,Accumulated Deductible,S +cost,cost_concept_id,32000,Allowed,S +cost,cost_concept_id,31978,Allowed,S +cost,cost_concept_id,31979,Allowed (calculated),S +cost,cost_concept_id,32001,Allowed (calculated),S +cost,cost_concept_id,31984,Average Wholesale,S +cost,cost_concept_id,32006,Average Wholesale,S +cost,cost_concept_id,32009,Balance bill,S +cost,cost_concept_id,31987,Balance bill,S +cost,cost_concept_id,31993,Bundled payment,S +cost,cost_concept_id,32015,Bundled payment,S +cost,cost_concept_id,32011,Capitation,S +cost,cost_concept_id,31989,Capitation,S +cost,cost_concept_id,32014,Care Coordination Fee,S +cost,cost_concept_id,31992,Care Coordination Fee,S +cost,cost_concept_id,31973,Charged,S +cost,cost_concept_id,31995,Charged,S +cost,cost_concept_id,31997,Coinsurance,S +cost,cost_concept_id,31975,Coinsurance,S +cost,cost_concept_id,31996,Copayment,S +cost,cost_concept_id,31974,Copayment,S +cost,cost_concept_id,31985,Cost,S +cost,cost_concept_id,32007,Cost,S +cost,cost_concept_id,31976,Deductible,S +cost,cost_concept_id,31998,Deductible,S +cost,cost_concept_id,31982,Fee,S +cost,cost_concept_id,32004,Fee,S +cost,cost_concept_id,32010,Incentive,S +cost,cost_concept_id,31988,Incentive,S +cost,cost_concept_id,32012,Lifetime limit,S +cost,cost_concept_id,31990,Lifetime limit,S +cost,cost_concept_id,32013,Net costs,S +cost,cost_concept_id,31991,Net costs,S +cost,cost_concept_id,32003,Out of pocket,S +cost,cost_concept_id,31981,Out of pocket,S +cost,cost_concept_id,31980,Paid,S +cost,cost_concept_id,32002,Paid,S +cost,cost_concept_id,31983,Pharmacy ingredient,S +cost,cost_concept_id,32005,Pharmacy ingredient,S +cost,cost_concept_id,31972,"Premium, employer contribution",S +cost,cost_concept_id,31971,"Premium, member contribution",S +cost,cost_concept_id,31994,Rebate payment,S +cost,cost_concept_id,32016,Rebate payment,S +cost,cost_concept_id,31977,Recovered,S +cost,cost_concept_id,31999,Recovered,S +cost,cost_concept_id,31986,"Usual, customary and reasonable",S +cost,cost_concept_id,32008,"Usual, customary and reasonable",S +cost,cost_type_concept_id,5032,"Amount charged to the patient or the payer by the provider, list price", +cost,cost_type_concept_id,5031,Amount paid by the patient or reimbursed by the payer, +cost,cost_type_concept_id,5033,Cost incurred by the provider, +death,death_type_concept_id,38003617,Death Certificate contributory cause,S +death,death_type_concept_id,38003570,Death Certificate immediate cause,S +death,death_type_concept_id,38003618,Death Certificate underlying cause,S +death,death_type_concept_id,255,EHR Record contributory cause,S +death,death_type_concept_id,254,EHR Record immediate cause,S +death,death_type_concept_id,256,EHR Record underlying cause,S +death,death_type_concept_id,44818516,"EHR discharge status ""Expired""",S +death,death_type_concept_id,38003569,"EHR record patient status ""Deceased""",S +death,death_type_concept_id,38003568,Medical claim DRG code indicating death,S +death,death_type_concept_id,38003567,Medical claim diagnostic code indicating death,S +death,death_type_concept_id,38003566,"Medical claim discharge status ""Died""",S +death,death_type_concept_id,242,Other government reported or identified death,S +death,death_type_concept_id,38003565,"Payer enrollment status ""Deceased""",S +death,death_type_concept_id,261,US Social Security Death Master File record,S +device_exposure,device_type_concept_id,44818707,EHR Detail,S +device_exposure,device_type_concept_id,44818705,Inferred from procedure claim,S +device_exposure,device_type_concept_id,44818706,Patient reported device,S +drug_exposure,drug_type_concept_id,581452,Dispensed in Outpatient office,S +drug_exposure,drug_type_concept_id,38000181,Drug era - 0 days persistence window,S +drug_exposure,drug_type_concept_id,38000182,Drug era - 30 days persistence window,S +drug_exposure,drug_type_concept_id,38000180,Inpatient administration,S +drug_exposure,drug_type_concept_id,38000178,Medication list entry,S +drug_exposure,drug_type_concept_id,44787730,Patient Self-Reported Medication,S +drug_exposure,drug_type_concept_id,38000179,Physician administered drug (identified as procedure),S +drug_exposure,drug_type_concept_id,43542358,Physician administered drug (identified from EHR observation),S +drug_exposure,drug_type_concept_id,581373,Physician administered drug (identified from EHR order),S +drug_exposure,drug_type_concept_id,43542356,Physician administered drug (identified from EHR problem list),S +drug_exposure,drug_type_concept_id,43542357,Physician administered drug (identified from referral record),S +drug_exposure,drug_type_concept_id,38000175,Prescription dispensed in pharmacy,S +drug_exposure,drug_type_concept_id,38000176,Prescription dispensed through mail order,S +drug_exposure,drug_type_concept_id,38000177,Prescription written,S +drug_exposure,drug_type_concept_id,44777970,Randomized Drug,S +drug_exposure,route_concept_id,44783786,Arteriovenous fistula route,S +drug_exposure,route_concept_id,762840,Arteriovenous graft route,S +drug_exposure,route_concept_id,4023156,Auricular,S +drug_exposure,route_concept_id,4222254,Body cavity use,S +drug_exposure,route_concept_id,4181897,Buccal,S +drug_exposure,route_concept_id,4220455,Caudal route,S +drug_exposure,route_concept_id,4168047,Colostomy route,S +drug_exposure,route_concept_id,40486444,Conjunctival route,S +drug_exposure,route_concept_id,40490507,Cutaneous route,S +drug_exposure,route_concept_id,4163765,Dental,S +drug_exposure,route_concept_id,40487501,Digestive tract route,S +drug_exposure,route_concept_id,4186831,Endocervical,S +drug_exposure,route_concept_id,4157756,Endosinusial,S +drug_exposure,route_concept_id,4186832,Endotracheopulmonary,S +drug_exposure,route_concept_id,4167540,Enteral route,S +drug_exposure,route_concept_id,4225555,Epidural,S +drug_exposure,route_concept_id,45956880,Epilesional,S +drug_exposure,route_concept_id,4172191,Esophagostomy route,S +drug_exposure,route_concept_id,4186833,Extraamniotic,S +drug_exposure,route_concept_id,37018288,Extracorporeal route,S +drug_exposure,route_concept_id,4304277,Fistula route,S +drug_exposure,route_concept_id,4168665,Gastro-intestinal stoma route,S +drug_exposure,route_concept_id,4186834,Gastroenteral,S +drug_exposure,route_concept_id,4132254,Gastrostomy route,S +drug_exposure,route_concept_id,4156704,Gingival,S +drug_exposure,route_concept_id,44801748,Haemodiafiltration route,S +drug_exposure,route_concept_id,45956871,Haemodialysis,S +drug_exposure,route_concept_id,45956877,Haemofiltration,S +drug_exposure,route_concept_id,4305679,Ileostomy route,S +drug_exposure,route_concept_id,37397638,Infiltration route,S +drug_exposure,route_concept_id,45956874,Inhalation,S +drug_exposure,route_concept_id,4327128,Interstitial route,S +drug_exposure,route_concept_id,37103746,Intestinal use,S +drug_exposure,route_concept_id,4304882,Intraabdominal route,S +drug_exposure,route_concept_id,4163767,Intraamniotic,S +drug_exposure,route_concept_id,4240824,Intraarterial,S +drug_exposure,route_concept_id,4006860,Intraarticular,S +drug_exposure,route_concept_id,4223965,Intrabiliary route,S +drug_exposure,route_concept_id,4303263,Intrabronchial route,S +drug_exposure,route_concept_id,4163768,Intrabursal,S +drug_exposure,route_concept_id,4303409,Intracameral,S +drug_exposure,route_concept_id,4156705,Intracardiac,S +drug_exposure,route_concept_id,4303676,Intracartilaginous route,S +drug_exposure,route_concept_id,4157757,Intracavernous,S +drug_exposure,route_concept_id,40488317,Intracerebral route,S +drug_exposure,route_concept_id,4224886,Intracerebroventricular,S +drug_exposure,route_concept_id,45956872,Intracervical,S +drug_exposure,route_concept_id,4305993,Intracisternal route,S +drug_exposure,route_concept_id,40489990,Intracolonic route,S +drug_exposure,route_concept_id,4305690,Intracorneal route,S +drug_exposure,route_concept_id,4303667,Intracoronal route,S +drug_exposure,route_concept_id,4186836,Intracoronary,S +drug_exposure,route_concept_id,40492302,Intracorpus cavernosum route,S +drug_exposure,route_concept_id,4171079,Intracranial route,S +drug_exposure,route_concept_id,4156706,Intradermal,S +drug_exposure,route_concept_id,4163769,Intradiscal,S +drug_exposure,route_concept_id,4170083,Intraductal route,S +drug_exposure,route_concept_id,4302354,Intraduodenal route,S +drug_exposure,route_concept_id,40492288,Intradural route,S +drug_exposure,route_concept_id,40487473,Intraepicardial route,S +drug_exposure,route_concept_id,40487983,Intraepidermal,S +drug_exposure,route_concept_id,40492284,Intraesophageal route,S +drug_exposure,route_concept_id,40492301,Intragastric route,S +drug_exposure,route_concept_id,40492286,Intragingival route,S +drug_exposure,route_concept_id,40493258,Intrahepatic route,S +drug_exposure,route_concept_id,40490837,Intraileal route,S +drug_exposure,route_concept_id,40489989,Intrajejunal route,S +drug_exposure,route_concept_id,4157758,Intralesional,S +drug_exposure,route_concept_id,40493227,Intralingual route,S +drug_exposure,route_concept_id,4292410,Intraluminal route,S +drug_exposure,route_concept_id,4157759,Intralymphatic,S +drug_exposure,route_concept_id,40491321,Intramammary route,S +drug_exposure,route_concept_id,4246511,Intramedullary route,S +drug_exposure,route_concept_id,40492300,Intrameningeal route,S +drug_exposure,route_concept_id,46272926,Intramural route,S +drug_exposure,route_concept_id,4302612,Intramuscular,S +drug_exposure,route_concept_id,4168038,Intramyometrial route,S +drug_exposure,route_concept_id,46272911,Intraneural route,S +drug_exposure,route_concept_id,4157760,Intraocular,S +drug_exposure,route_concept_id,4213522,Intraosseous,S +drug_exposure,route_concept_id,4306657,Intraovarian route,S +drug_exposure,route_concept_id,40492305,Intrapericardial route,S +drug_exposure,route_concept_id,4243022,Intraperitoneal,S +drug_exposure,route_concept_id,4156707,Intrapleural,S +drug_exposure,route_concept_id,4171725,Intraprostatic route,S +drug_exposure,route_concept_id,4169270,Intrapulmonary route,S +drug_exposure,route_concept_id,4169440,Intrasinal route,S +drug_exposure,route_concept_id,4302788,Intraspinal route,S +drug_exposure,route_concept_id,4186837,Intrasternal,S +drug_exposure,route_concept_id,4302352,Intrasynovial route,S +drug_exposure,route_concept_id,4303939,Intratendinous route,S +drug_exposure,route_concept_id,4171067,Intratesticular route,S +drug_exposure,route_concept_id,4217202,Intrathecal,S +drug_exposure,route_concept_id,4167393,Intrathoracic route,S +drug_exposure,route_concept_id,4229543,Intratracheal route,S +drug_exposure,route_concept_id,40491322,Intratumoral,S +drug_exposure,route_concept_id,4168656,Intratympanic route,S +drug_exposure,route_concept_id,4269621,Intrauterine,S +drug_exposure,route_concept_id,40492287,Intravascular route,S +drug_exposure,route_concept_id,4171047,Intravenous,S +drug_exposure,route_concept_id,4170113,Intravenous central route,S +drug_exposure,route_concept_id,4171884,Intravenous peripheral route,S +drug_exposure,route_concept_id,4222259,Intraventricular cardiac,S +drug_exposure,route_concept_id,4186838,Intravesical,S +drug_exposure,route_concept_id,4302785,Intravitreal,S +drug_exposure,route_concept_id,45956881,Iontophoresis,S +drug_exposure,route_concept_id,4133177,Jejunostomy route,S +drug_exposure,route_concept_id,4170440,Laryngeal route,S +drug_exposure,route_concept_id,40490898,Lower respiratory tract route,S +drug_exposure,route_concept_id,4171243,Mucous fistula route,S +drug_exposure,route_concept_id,4262914,Nasal,S +drug_exposure,route_concept_id,4172316,Nasoduodenal route,S +drug_exposure,route_concept_id,4132711,Nasogastric route,S +drug_exposure,route_concept_id,4305834,Nasojejunal route,S +drug_exposure,route_concept_id,4184451,Ocular,S +drug_exposure,route_concept_id,4132161,Oral,S +drug_exposure,route_concept_id,4303795,Orogastric route,S +drug_exposure,route_concept_id,4186839,Oromucosal,S +drug_exposure,route_concept_id,4303277,Oropharyngeal route,S +drug_exposure,route_concept_id,4303515,Paracervical route,S +drug_exposure,route_concept_id,4170267,Paravertebral route,S +drug_exposure,route_concept_id,4177987,Percutaneous route,S +drug_exposure,route_concept_id,4156708,Periarticular,S +drug_exposure,route_concept_id,4304274,Peribulbar route,S +drug_exposure,route_concept_id,40490896,Peridural route,S +drug_exposure,route_concept_id,4157761,Perineural,S +drug_exposure,route_concept_id,40490866,Periodontal route,S +drug_exposure,route_concept_id,4171893,Periosteal route,S +drug_exposure,route_concept_id,4305564,Peritendinous route,S +drug_exposure,route_concept_id,4303646,Periurethral route,S +drug_exposure,route_concept_id,4290759,Rectal,S +drug_exposure,route_concept_id,45956879,Regional perfusion,S +drug_exposure,route_concept_id,40486069,Respiratory tract route,S +drug_exposure,route_concept_id,4303673,Retrobulbar route,S +drug_exposure,route_concept_id,45956875,Route of administration not applicable,S +drug_exposure,route_concept_id,4106215,Route of administration value,S +drug_exposure,route_concept_id,4163770,Subconjunctival,S +drug_exposure,route_concept_id,4142048,Subcutaneous,S +drug_exposure,route_concept_id,4306649,Subgingival route,S +drug_exposure,route_concept_id,46270168,Sublesional route,S +drug_exposure,route_concept_id,4292110,Sublingual,S +drug_exposure,route_concept_id,45956878,Submucosal rectal,S +drug_exposure,route_concept_id,4169634,Submucosal route,S +drug_exposure,route_concept_id,4166865,Suborbital route,S +drug_exposure,route_concept_id,36703536,Subretinal,S +drug_exposure,route_concept_id,4302493,Subtendinous route,S +drug_exposure,route_concept_id,4170771,Surgical cavity route,S +drug_exposure,route_concept_id,4304412,Surgical drain route,S +drug_exposure,route_concept_id,4263689,Topical route,S +drug_exposure,route_concept_id,4304730,Transcervical route,S +drug_exposure,route_concept_id,4262099,Transdermal,S +drug_exposure,route_concept_id,40487850,Transendocardial route,S +drug_exposure,route_concept_id,4232601,Transmucosal route,S +drug_exposure,route_concept_id,40487858,Transplacental route,S +drug_exposure,route_concept_id,40491832,Transtracheal route,S +drug_exposure,route_concept_id,40491830,Transtympanic route,S +drug_exposure,route_concept_id,4305382,Transurethral route,S +drug_exposure,route_concept_id,4169472,Tumor cavity route,S +drug_exposure,route_concept_id,4304571,Ureteral route,S +drug_exposure,route_concept_id,4233974,Urethral,S +drug_exposure,route_concept_id,4170435,Urostomy route,S +drug_exposure,route_concept_id,4057765,Vaginal,S +drug_exposure,route_concept_id,45956883,Body cavity use, +drug_exposure,route_concept_id,45956882,Epidural, +drug_exposure,route_concept_id,4228125,Haemodialysis, +drug_exposure,route_concept_id,4011083,Inhalation, +drug_exposure,route_concept_id,45956885,Intracerebroventricular, +drug_exposure,route_concept_id,4186835,Intracervical, +drug_exposure,route_concept_id,45956886,Intraepidermal, +drug_exposure,route_concept_id,45956884,Intraventricular cardiac, +drug_exposure,route_concept_id,4302956,Iontophoresis, +drug_exposure,route_concept_id,45956876,Obsolete-Intraventricular, +drug_exposure,route_concept_id,45956873,Obsolete-Oromucosal other, +drug_exposure,route_concept_id,4227373,Obsolete-Oromucosal other, +drug_exposure,route_concept_id,40549429,Ocular, +measurement,measurement_type_concept_id,45754907,Derived value,S +measurement,measurement_type_concept_id,44818701,From physical examination,S +measurement,measurement_type_concept_id,44818702,Lab result,S +measurement,measurement_type_concept_id,44818703,Pathology finding,S +measurement,measurement_type_concept_id,44818704,Patient reported value,S +measurement,measurement_type_concept_id,5001,Test ordered through EHR,S +measurement,operator_concept_id,4171756,<,S +measurement,operator_concept_id,4171754,<=,S +measurement,operator_concept_id,4172703,=,S +measurement,operator_concept_id,4172704,>,S +measurement,operator_concept_id,4171755,>=,S +note,note_type_concept_id,44814638,Admission note,S +note,note_type_concept_id,44814643,Ancillary report,S +note,note_type_concept_id,44814637,Discharge summary,S +note,note_type_concept_id,44814646,Emergency department note,S +note,note_type_concept_id,44814639,Inpatient note,S +note,note_type_concept_id,44814645,Note,S +note,note_type_concept_id,44814644,Nursing report,S +note,note_type_concept_id,44814640,Outpatient note,S +note,note_type_concept_id,44814642,Pathology report,S +note,note_type_concept_id,44814641,Radiology report,S +observation,observation_type_concept_id,38000282,Chief complaint,S +observation,observation_type_concept_id,44786633,HRA Observation Numeric Result,S +observation,observation_type_concept_id,44786634,HRA Observation Text,S +observation,observation_type_concept_id,38000279,Lab observation concept code result,S +observation,observation_type_concept_id,38000277,Lab observation numeric result,S +observation,observation_type_concept_id,38000278,Lab observation text,S +observation,observation_type_concept_id,45905771,Observation Recorded from a Survey,S +observation,observation_type_concept_id,581413,Observation from Measurement,S +observation,observation_type_concept_id,38000280,Observation recorded from EHR,S +observation,observation_type_concept_id,38000281,Observation recorded from EHR with text result,S +observation,observation_type_concept_id,44814721,Patient reported,S +observation,observation_type_concept_id,38000276,Problem list from EHR,S +observation,observation_type_concept_id,43542355,Referral Record,S +observation_period,period_type_concept_id,44814724,Period covering healthcare encounters,S +observation_period,period_type_concept_id,44814725,Period inferred by algorithm,S +observation_period,period_type_concept_id,45890994,Period of complete data capture based on geographic isolation,S +observation_period,period_type_concept_id,44814722,Period while enrolled in insurance,S +observation_period,period_type_concept_id,44814723,Period while enrolled in study,S +observation_period,period_type_concept_id,45747656,Pre-qualification time period,S +person,ethnicity_concept_id,38003563,Hispanic or Latino,S +person,ethnicity_concept_id,38003564,Not Hispanic or Latino,S +person,gender_concept_id,8532,FEMALE,S +person,gender_concept_id,8507,MALE,S +person,gender_concept_id,8570,AMBIGUOUS, +person,gender_concept_id,45766035,Feminine gender, +person,gender_concept_id,4268709,Gender finding, +person,gender_concept_id,45518388,Gender unknown, +person,gender_concept_id,4214687,Gender unknown, +person,gender_concept_id,45454912,Gender unspecified, +person,gender_concept_id,4215271,Gender unspecified, +person,gender_concept_id,45438358,Male, +person,gender_concept_id,45766034,Masculine gender, +person,gender_concept_id,42689678,Non-binary gender, +person,gender_concept_id,8521,OTHER, +person,gender_concept_id,4234363,Surgically transgendered transsexual, +person,gender_concept_id,4231242,"Surgically transgendered transsexual, female-to-male", +person,gender_concept_id,4251434,"Surgically transgendered transsexual, male-to-female", +person,gender_concept_id,1585842,Transgender, +person,gender_concept_id,36712829,Transgender identity, +person,gender_concept_id,8551,UNKNOWN, +person,race_concept_id,38003600,African,S +person,race_concept_id,38003599,African American,S +person,race_concept_id,38003573,Alaska Native,S +person,race_concept_id,38003572,American Indian,S +person,race_concept_id,8657,American Indian or Alaska Native,S +person,race_concept_id,38003616,Arab,S +person,race_concept_id,8515,Asian,S +person,race_concept_id,38003574,Asian Indian,S +person,race_concept_id,38003601,Bahamian,S +person,race_concept_id,38003575,Bangladeshi,S +person,race_concept_id,38003602,Barbadian,S +person,race_concept_id,38003576,Bhutanese,S +person,race_concept_id,38003598,Black,S +person,race_concept_id,8516,Black or African American,S +person,race_concept_id,38003577,Burmese,S +person,race_concept_id,38003578,Cambodian,S +person,race_concept_id,38003579,Chinese,S +person,race_concept_id,38003604,Dominica Islander,S +person,race_concept_id,38003603,Dominican,S +person,race_concept_id,38003614,European,S +person,race_concept_id,38003581,Filipino,S +person,race_concept_id,38003605,Haitian,S +person,race_concept_id,38003582,Hmong,S +person,race_concept_id,38003583,Indonesian,S +person,race_concept_id,38003593,Iwo Jiman,S +person,race_concept_id,38003606,Jamaican,S +person,race_concept_id,38003584,Japanese,S +person,race_concept_id,38003585,Korean,S +person,race_concept_id,38003586,Laotian,S +person,race_concept_id,38003597,Madagascar,S +person,race_concept_id,38003587,Malaysian,S +person,race_concept_id,38003594,Maldivian,S +person,race_concept_id,38003612,Melanesian,S +person,race_concept_id,38003611,Micronesian,S +person,race_concept_id,38003615,Middle Eastern or North African,S +person,race_concept_id,8557,Native Hawaiian or Other Pacific Islander,S +person,race_concept_id,38003595,Nepalese,S +person,race_concept_id,38003588,Okinawan,S +person,race_concept_id,38003613,Other Pacific Islander,S +person,race_concept_id,38003589,Pakistani,S +person,race_concept_id,38003610,Polynesian,S +person,race_concept_id,38003596,Singaporean,S +person,race_concept_id,38003590,Sri Lankan,S +person,race_concept_id,38003580,Taiwanese,S +person,race_concept_id,38003591,Thai,S +person,race_concept_id,38003607,Tobagoan,S +person,race_concept_id,38003608,Trinidadian,S +person,race_concept_id,38003592,Vietnamese,S +person,race_concept_id,38003609,West Indian,S +person,race_concept_id,8527,White,S +person,race_concept_id,4228069,Abyssinians, +person,race_concept_id,4119705,Admiralty Islanders, +person,race_concept_id,4035165,African American, +person,race_concept_id,4211218,African race, +person,race_concept_id,4184962,Afro-Caribbean, +person,race_concept_id,4187747,Afro-Caucasian, +person,race_concept_id,4100645,Ainu, +person,race_concept_id,4231129,Alacaluf, +person,race_concept_id,4050100,Aleuts, +person,race_concept_id,4184966,American Indian or Alaska native, +person,race_concept_id,4211331,American Indian race, +person,race_concept_id,4281687,Amerind, +person,race_concept_id,4175829,Andamanese, +person,race_concept_id,4251760,Apache, +person,race_concept_id,4231846,Arabs, +person,race_concept_id,4014421,Armenians, +person,race_concept_id,4201937,Asian - ethnic group, +person,race_concept_id,4184984,Asian or Pacific islander, +person,race_concept_id,4211353,Asian race, +person,race_concept_id,4298833,Atacamenos, +person,race_concept_id,4242388,Athabascans, +person,race_concept_id,4183595,Australian Aborigines, +person,race_concept_id,4186705,Australian aborigine, +person,race_concept_id,4285116,Austrians, +person,race_concept_id,4014099,Aymara, +person,race_concept_id,4195705,Aztec, +person,race_concept_id,4263311,Badagas, +person,race_concept_id,45771426,Bajau, +person,race_concept_id,45452880,Bangladeshi, +person,race_concept_id,4091018,Bangladeshi, +person,race_concept_id,4219735,Bantu, +person,race_concept_id,4240732,Barundi, +person,race_concept_id,4163527,Basques, +person,race_concept_id,4034654,Batutsi, +person,race_concept_id,4210302,Belgians, +person,race_concept_id,4102303,Bhutanese, +person,race_concept_id,45767090,Bidayuh, +person,race_concept_id,4201929,Black - ethnic group, +person,race_concept_id,45496299,Black - other African country, +person,race_concept_id,4091011,Black - other African country, +person,race_concept_id,45452879,Black - other Asian, +person,race_concept_id,4091012,Black - other Asian, +person,race_concept_id,4091013,"Black - other, mixed", +person,race_concept_id,45502955,"Black - other, mixed", +person,race_concept_id,4057568,Black African, +person,race_concept_id,45469811,Black African, +person,race_concept_id,4153158,Black African and White, +person,race_concept_id,45499627,Black African and White, +person,race_concept_id,4076901,Black Arab, +person,race_concept_id,45476450,Black Arab, +person,race_concept_id,45429648,Black Black - other, +person,race_concept_id,45462994,Black British, +person,race_concept_id,4090389,Black British, +person,race_concept_id,45496298,Black Caribbean, +person,race_concept_id,4087917,Black Caribbean, +person,race_concept_id,45522914,Black Caribbean, +person,race_concept_id,45456221,Black Caribbean and White, +person,race_concept_id,4151776,Black Caribbean and White, +person,race_concept_id,4152832,Black Caribbean/W.I./Guyana, +person,race_concept_id,45522913,Black Caribbean/W.I./Guyana, +person,race_concept_id,45436284,Black E Afric Asia/Indo-Caribb, +person,race_concept_id,45479787,Black East African Asian, +person,race_concept_id,4078123,Black East African Asian, +person,race_concept_id,4151577,Black East African Asian/Indo-Caribbean, +person,race_concept_id,4205380,Black Guyana, +person,race_concept_id,45446238,Black Guyana, +person,race_concept_id,45512960,Black Indian sub-continent, +person,race_concept_id,4085483,Black Indian sub-continent, +person,race_concept_id,45462995,Black Indo-Caribbean, +person,race_concept_id,4078124,Black Indo-Caribbean, +person,race_concept_id,45456219,Black Iranian, +person,race_concept_id,4077358,Black Iranian, +person,race_concept_id,4305595,Black Jews, +person,race_concept_id,45519603,Black N African/Arab/Iranian, +person,race_concept_id,4151576,Black N African/Arab/Iranian, +person,race_concept_id,45429647,Black North African, +person,race_concept_id,4078122,Black North African, +person,race_concept_id,4203120,Black West Indian, +person,race_concept_id,45429646,Black West Indian, +person,race_concept_id,4211480,"Black, not of hispanic origin", +person,race_concept_id,45516366,"Black, other, non-mixed origin", +person,race_concept_id,4091010,"Black, other, non-mixed origin", +person,race_concept_id,4277901,Blackfeet, +person,race_concept_id,4139924,Bloods, +person,race_concept_id,4169399,Bogobos, +person,race_concept_id,4301306,Bororo, +person,race_concept_id,4218447,Brazilian Indians, +person,race_concept_id,45436285,Brit. ethnic minor. spec.(NMO), +person,race_concept_id,45439570,Brit. ethnic minor. unsp (NMO), +person,race_concept_id,4090390,British ethnic minority specified, +person,race_concept_id,4091019,British ethnic minority unspecified, +person,race_concept_id,4264197,Bruneians, +person,race_concept_id,4167621,Bulgarian, +person,race_concept_id,45479793,Bulgarian, +person,race_concept_id,1397910,Bulgarian Roma, +person,race_concept_id,37394642,Bulgarian Roma, +person,race_concept_id,36717066,Bulgarian Roma, +person,race_concept_id,4221071,Buriats, +person,race_concept_id,4309789,Bushmen, +person,race_concept_id,4249154,Caingang, +person,race_concept_id,37116650,Canadian, +person,race_concept_id,4151578,Caribbean I./W.I./Guyana, +person,race_concept_id,45479788,Caribbean I./W.I./Guyana (NMO), +person,race_concept_id,4077359,Caribbean Island, +person,race_concept_id,45479789,Caribbean Island (NMO), +person,race_concept_id,4216433,Caroline Islanders, +person,race_concept_id,4030411,Caucasian, +person,race_concept_id,4185154,Caucasian, +person,race_concept_id,4185155,"Caucasian, not of hispanic origin", +person,race_concept_id,4211849,Chenchu, +person,race_concept_id,4144377,Chinese, +person,race_concept_id,45419859,Chinese, +person,race_concept_id,45476453,Chinese, +person,race_concept_id,4215810,Chippewa, +person,race_concept_id,4270899,Choco, +person,race_concept_id,4178142,Congolese, +person,race_concept_id,4087928,Cook Island Maori, +person,race_concept_id,45522915,Cook Island Maori, +person,race_concept_id,4012612,Coushatta, +person,race_concept_id,4182121,Cuna, +person,race_concept_id,44803061,Czech, +person,race_concept_id,45473117,Czech, +person,race_concept_id,4102622,Czech, +person,race_concept_id,37394638,Czech Roma, +person,race_concept_id,1397642,Czech Roma, +person,race_concept_id,36713937,Czech Roma, +person,race_concept_id,4031845,Danes, +person,race_concept_id,4185580,Dieguenos, +person,race_concept_id,4317261,Dutch, +person,race_concept_id,4137597,Dyaks, +person,race_concept_id,4148888,E Afric Asian/Indo-Carib, +person,race_concept_id,45439571,E Afric Asian/Indo-Carib (NMO), +person,race_concept_id,4077381,East African Asian, +person,race_concept_id,45516367,East African Asian (NMO), +person,race_concept_id,4103076,Easter Islanders, +person,race_concept_id,4000051,Egyptians, +person,race_concept_id,4247162,Ellice Islanders, +person,race_concept_id,4093769,English, +person,race_concept_id,4309952,Eskimo, +person,race_concept_id,4090368,Estonians, +person,race_concept_id,759814,Ethnic category unknown, +person,race_concept_id,4155301,Ethnic group, +person,race_concept_id,4273145,Ethnic groups (1991 census), +person,race_concept_id,4212990,European, +person,race_concept_id,4002414,Ewe, +person,race_concept_id,4321400,Fijian, +person,race_concept_id,45479791,Fijian, +person,race_concept_id,4296018,Filipinos, +person,race_concept_id,4005076,Finns, +person,race_concept_id,4027841,Flathead, +person,race_concept_id,4024294,French, +person,race_concept_id,4292111,Fulani, +person,race_concept_id,4236027,Gambians, +person,race_concept_id,4138072,Georgians, +person,race_concept_id,4297319,Germans, +person,race_concept_id,4292272,Ghanaians, +person,race_concept_id,4233496,Ghashgai, +person,race_concept_id,4171181,Gilbertese, +person,race_concept_id,4076904,Greek, +person,race_concept_id,45423089,Greek (NMO), +person,race_concept_id,4077361,Greek Cypriot, +person,race_concept_id,45436287,Greek Cypriot (NMO), +person,race_concept_id,4152834,Greek/Greek Cypriot, +person,race_concept_id,45462996,Greek/Greek Cypriot (NMO), +person,race_concept_id,4032635,Greeks, +person,race_concept_id,4012103,Guamians, +person,race_concept_id,4077360,Guyana, +person,race_concept_id,45442862,Guyana (NMO), +person,race_concept_id,4221073,Gypsies, +person,race_concept_id,4065367,Hawaiians, +person,race_concept_id,4188159,Hispanic, +person,race_concept_id,4190391,"Hispanic, black", +person,race_concept_id,4188161,"Hispanic, color unknown", +person,race_concept_id,4190392,"Hispanic, white", +person,race_concept_id,4218288,Hobe, +person,race_concept_id,4198993,Hottentot, +person,race_concept_id,4279480,Huasteco, +person,race_concept_id,4297991,Huichol, +person,race_concept_id,4273384,Hungarian, +person,race_concept_id,1398020,Hungarian Roma, +person,race_concept_id,37394639,Hungarian Roma, +person,race_concept_id,36713940,Hungarian Roma, +person,race_concept_id,4098866,Hututu, +person,race_concept_id,45767091,Iban, +person,race_concept_id,4245024,Ibo, +person,race_concept_id,4248713,Icelanders, +person,race_concept_id,4210186,Inca, +person,race_concept_id,4091016,Indian, +person,race_concept_id,45483083,Indian, +person,race_concept_id,45446239,Indian, +person,race_concept_id,4185920,Indian, +person,race_concept_id,4090392,Indian sub-continent, +person,race_concept_id,45476451,Indian sub-continent (NMO), +person,race_concept_id,4276043,Indians, +person,race_concept_id,4076903,Indo-Caribbean, +person,race_concept_id,45436286,Indo-Caribbean (NMO), +person,race_concept_id,4318647,Indonesians, +person,race_concept_id,4289159,Irani, +person,race_concept_id,4078125,Iranian, +person,race_concept_id,45496301,Iranian (NMO), +person,race_concept_id,4100470,Iraqi, +person,race_concept_id,4090393,Irish, +person,race_concept_id,45419861,Irish (NMO), +person,race_concept_id,45512963,Irish traveller, +person,race_concept_id,4201938,Irish traveller, +person,race_concept_id,4221856,Irula, +person,race_concept_id,4135806,Italians, +person,race_concept_id,4185935,Japanese, +person,race_concept_id,4068834,Javanese, +person,race_concept_id,45771425,Kadazan, +person,race_concept_id,4048709,Kapingas, +person,race_concept_id,4036536,Kenyans, +person,race_concept_id,4218957,Kikuyu, +person,race_concept_id,4180571,Kirghiz, +person,race_concept_id,4243164,Koreans, +person,race_concept_id,4298426,Kwakiutl, +person,race_concept_id,4326740,Labradors, +person,race_concept_id,4069284,Lacandon, +person,race_concept_id,4282477,Lapps, +person,race_concept_id,4240299,Liberians, +person,race_concept_id,4308897,Luo, +person,race_concept_id,4071025,Madagascans, +person,race_concept_id,4028336,Malays, +person,race_concept_id,4249883,Maori, +person,race_concept_id,4327656,Mapuche, +person,race_concept_id,4180729,Marathas, +person,race_concept_id,4245756,Marshallese, +person,race_concept_id,4269942,Maya, +person,race_concept_id,45767088,Melanau, +person,race_concept_id,4189474,Melanesian, +person,race_concept_id,4053172,Melanesians, +person,race_concept_id,4250319,Melanuans, +person,race_concept_id,4186453,Mexican Indians, +person,race_concept_id,4283895,Micronesians, +person,race_concept_id,4196429,Mixed ethnic census group, +person,race_concept_id,4212311,Mixed racial group, +person,race_concept_id,4087322,Mongol, +person,race_concept_id,4165388,Mozambiquans, +person,race_concept_id,4283366,Msutu, +person,race_concept_id,45767089,Murut, +person,race_concept_id,4152833,N African Arab/Iranian, +person,race_concept_id,45419860,N African Arab/Iranian (NMO), +person,race_concept_id,4217600,Naiars, +person,race_concept_id,4186402,National surgical quality improvement program defined race unknown, +person,race_concept_id,4212088,Navaho, +person,race_concept_id,44803976,Nepali, +person,race_concept_id,45459567,Nepali, +person,race_concept_id,4237214,New Britons, +person,race_concept_id,4145762,New Caledonians, +person,race_concept_id,4273538,New Hebrideans, +person,race_concept_id,4090521,New Zealand European, +person,race_concept_id,45486422,New Zealand European, +person,race_concept_id,45496302,New Zealand Maori, +person,race_concept_id,4091022,New Zealand Maori, +person,race_concept_id,45499628,New Zealand ethnic group NOS, +person,race_concept_id,4087926,New Zealand ethnic groups, +person,race_concept_id,45442863,New Zealand ethnic groups, +person,race_concept_id,4337671,Nez Percé, +person,race_concept_id,4229603,Nigerians, +person,race_concept_id,4085489,Niuean, +person,race_concept_id,45509567,Niuean, +person,race_concept_id,9178,Non-white, +person,race_concept_id,4077346,North African Arab, +person,race_concept_id,45449500,North African Arab (NMO), +person,race_concept_id,4063377,Norwegians, +person,race_concept_id,4085322,Oceanian, +person,race_concept_id,4017926,Onge, +person,race_concept_id,45767087,Orang asli, +person,race_concept_id,4235464,Oraons, +person,race_concept_id,4189785,Oriental, +person,race_concept_id,4199011,Oriental Jews, +person,race_concept_id,4090391,Other African countries, +person,race_concept_id,45469812,Other African countries (NMO), +person,race_concept_id,4087922,Other Asian, +person,race_concept_id,45473116,Other Asian, +person,race_concept_id,45466387,Other Asian (NMO), +person,race_concept_id,45512962,Other Asian ethnic group, +person,race_concept_id,4199940,Other Asian ethnic group, +person,race_concept_id,45496300,Other Black - Black/Asian orig, +person,race_concept_id,4087920,Other Black - Black/Asian orig, +person,race_concept_id,45506204,Other Black - Black/White orig, +person,race_concept_id,4087919,Other Black - Black/White orig, +person,race_concept_id,4090516,Other European, +person,race_concept_id,45459564,Other European (NMO), +person,race_concept_id,4085488,Other European in New Zealand, +person,race_concept_id,45506206,Other European in New Zealand, +person,race_concept_id,45442864,Other New Zealand ethnic group, +person,race_concept_id,45502956,Other Pacific ethnic group, +person,race_concept_id,8522,Other Race, +person,race_concept_id,45462997,Other black ethnic group, +person,race_concept_id,4202555,Other black ethnic group, +person,race_concept_id,45449501,Other ethnic NEC (NMO), +person,race_concept_id,45483082,Other ethnic group, +person,race_concept_id,4087921,Other ethnic non-mixed, +person,race_concept_id,45466386,Other ethnic non-mixed (NMO), +person,race_concept_id,45459565,"Other ethnic, Asian/White orig", +person,race_concept_id,4091021,"Other ethnic, Asian/White origin", +person,race_concept_id,45429649,"Other ethnic, Black/White orig", +person,race_concept_id,4085487,"Other ethnic, Black/White origin", +person,race_concept_id,4091020,"Other ethnic, mixed origin", +person,race_concept_id,45456220,"Other ethnic, mixed origin", +person,race_concept_id,45423090,"Other ethnic, mixed white orig", +person,race_concept_id,4087924,"Other ethnic, mixed white origin", +person,race_concept_id,45506205,"Other ethnic, other mixed orig", +person,race_concept_id,4090518,"Other ethnic, other mixed origin", +person,race_concept_id,45436283,Other white British ethnic group, +person,race_concept_id,4268433,Other white British ethnic group, +person,race_concept_id,45459563,Other white ethnic group, +person,race_concept_id,4099457,Paez, +person,race_concept_id,45492990,Pakeha, +person,race_concept_id,45512961,Pakistani, +person,race_concept_id,4091017,Pakistani, +person,race_concept_id,4216519,Pakistani, +person,race_concept_id,4234343,Palauans, +person,race_concept_id,4073535,Papuans, +person,race_concept_id,4217467,Pehuenches, +person,race_concept_id,4102599,Poles, +person,race_concept_id,1397643,Polish Roma, +person,race_concept_id,37394640,Polish Roma, +person,race_concept_id,36713941,Polish Roma, +person,race_concept_id,4053167,Polynesians, +person,race_concept_id,4213463,Portuguese, +person,race_concept_id,45469813,Portuguese, +person,race_concept_id,4140692,Pueblo, +person,race_concept_id,45767086,Punjabi, +person,race_concept_id,4323551,Pygmies, +person,race_concept_id,4194076,Quechua, +person,race_concept_id,45528619,RACE AFRICAN, +person,race_concept_id,45529590,RACE ASIAN, +person,race_concept_id,45526661,RACE WEST INDIAN, +person,race_concept_id,45532670,RACE WHITE, +person,race_concept_id,45428285,RACE: Afro-caribbean, +person,race_concept_id,45501542,RACE: Afro-caucasian, +person,race_concept_id,45451482,RACE: Arab, +person,race_concept_id,45458163,RACE: Bangladeshi, +person,race_concept_id,45514934,RACE: Caucasian, +person,race_concept_id,45421717,RACE: Chinese, +person,race_concept_id,45441530,RACE: Japanese, +person,race_concept_id,45504866,RACE: Korean, +person,race_concept_id,45501541,RACE: Mixed, +person,race_concept_id,45518249,RACE: Not stated, +person,race_concept_id,45421718,RACE: Oriental, +person,race_concept_id,45511540,RACE: Pakistani, +person,race_concept_id,45431577,RACE: Unknown, +person,race_concept_id,45511539,RACE: West indian, +person,race_concept_id,45478359,RACE: White, +person,race_concept_id,4190758,Race not stated, +person,race_concept_id,4051279,Race: West indian, +person,race_concept_id,4216292,Racial group, +person,race_concept_id,36717683,Roma, +person,race_concept_id,1397856,Roma ethnic group, +person,race_concept_id,37398747,Roma ethnic group, +person,race_concept_id,40483304,Romanian, +person,race_concept_id,45439572,Romanian, +person,race_concept_id,37394641,Romanian Roma, +person,race_concept_id,36713938,Romanian Roma, +person,race_concept_id,1397881,Romanian Roma, +person,race_concept_id,4309511,Russians, +person,race_concept_id,4077735,Saipanese, +person,race_concept_id,45516368,Samoan, +person,race_concept_id,4313587,Samoan, +person,race_concept_id,42536269,Scandinavian, +person,race_concept_id,4170372,Seminole, +person,race_concept_id,4244365,Senegalese, +person,race_concept_id,4198417,Senoi, +person,race_concept_id,4201594,Serbs, +person,race_concept_id,4219475,Shona, +person,race_concept_id,4317523,Shoshone, +person,race_concept_id,4103243,Siamese, +person,race_concept_id,45483084,Slovak, +person,race_concept_id,4267217,Slovak, +person,race_concept_id,44813053,Slovak, +person,race_concept_id,1397882,Slovak Roma, +person,race_concept_id,37394635,Slovak Roma, +person,race_concept_id,36713939,Slovak Roma, +person,race_concept_id,4167520,Solomon Islanders, +person,race_concept_id,4263713,Somalis, +person,race_concept_id,4030302,South Asian AND/OR Australian aborigine, +person,race_concept_id,4095109,South Asian Aborigine, +person,race_concept_id,4087930,South East Asian, +person,race_concept_id,45479792,South East Asian, +person,race_concept_id,4152781,Spaniards, +person,race_concept_id,4229453,Sudanese, +person,race_concept_id,4216818,Swedes, +person,race_concept_id,4297790,Swiss, +person,race_concept_id,4241969,Syrians, +person,race_concept_id,4270900,Taiwanese, +person,race_concept_id,4053816,Tamils, +person,race_concept_id,4028346,Tanganyikans, +person,race_concept_id,4235433,Tatars, +person,race_concept_id,4308112,Thais, +person,race_concept_id,4192497,Toba, +person,race_concept_id,4216036,Todas, +person,race_concept_id,45449503,Tokelauan, +person,race_concept_id,4091023,Tokelauan, +person,race_concept_id,4304542,Tongan, +person,race_concept_id,45459566,Tongan, +person,race_concept_id,45432962,Traveller - gypsy, +person,race_concept_id,4097690,Tristan da Cunhans, +person,race_concept_id,4141462,Trukese, +person,race_concept_id,4076905,Turkish, +person,race_concept_id,45419862,Turkish (NMO), +person,race_concept_id,4078126,Turkish Cypriot, +person,race_concept_id,45479790,Turkish Cypriot (NMO), +person,race_concept_id,4155521,Turkish/Turkish Cypriot, +person,race_concept_id,45509566,Turkish/Turkish Cypriot (NMO), +person,race_concept_id,4195322,Turks, +person,race_concept_id,4030735,Tutsi, +person,race_concept_id,4296606,Ugandans, +person,race_concept_id,37116384,Ukrainian, +person,race_concept_id,8552,Unknown, +person,race_concept_id,4218674,Unknown racial group, +person,race_concept_id,4148384,Utes, +person,race_concept_id,4269914,Venezuelan Indians, +person,race_concept_id,45489704,Vietnamese, +person,race_concept_id,4210116,Vietnamese, +person,race_concept_id,4033391,Welsh, +person,race_concept_id,4216026,West Africans, +person,race_concept_id,4076902,West Indian, +person,race_concept_id,45446240,West Indian (NMO), +person,race_concept_id,45442861,White, +person,race_concept_id,4090388,White - ethnic group, +person,race_concept_id,45519602,White British, +person,race_concept_id,4196428,White British, +person,race_concept_id,4201928,White Irish, +person,race_concept_id,45479786,White Irish, +person,race_concept_id,4268432,White Scottish, +person,race_concept_id,45462993,White Scottish, +person,race_concept_id,4092681,Xavante, +person,race_concept_id,4214735,Xosa, +person,race_concept_id,4295144,Yanomama, +person,race_concept_id,4170060,Yapese, +person,race_concept_id,44791722,Yemeni, +person,race_concept_id,45456222,Yemeni, +person,race_concept_id,4296997,Zulu, +procedure_occurrence,procedure_type_concept_id,45756909,Carrier claim detail - 10th position,S +procedure_occurrence,procedure_type_concept_id,45756910,Carrier claim detail - 11th position,S +procedure_occurrence,procedure_type_concept_id,45756911,Carrier claim detail - 12th position,S +procedure_occurrence,procedure_type_concept_id,45756912,Carrier claim detail - 13th position,S +procedure_occurrence,procedure_type_concept_id,45756900,Carrier claim detail - 1st position,S +procedure_occurrence,procedure_type_concept_id,45756901,Carrier claim detail - 2nd position,S +procedure_occurrence,procedure_type_concept_id,45756902,Carrier claim detail - 3rd position,S +procedure_occurrence,procedure_type_concept_id,45756903,Carrier claim detail - 4th position,S +procedure_occurrence,procedure_type_concept_id,45756904,Carrier claim detail - 5th position,S +procedure_occurrence,procedure_type_concept_id,45756905,Carrier claim detail - 6th position,S +procedure_occurrence,procedure_type_concept_id,45756906,Carrier claim detail - 7th position,S +procedure_occurrence,procedure_type_concept_id,45756907,Carrier claim detail - 8th position,S +procedure_occurrence,procedure_type_concept_id,45756908,Carrier claim detail - 9th position,S +procedure_occurrence,procedure_type_concept_id,42865906,Condition Procedure,S +procedure_occurrence,procedure_type_concept_id,38000275,EHR order list entry,S +procedure_occurrence,procedure_type_concept_id,42865905,Facility header,S +procedure_occurrence,procedure_type_concept_id,257,Hospitalization Cost Record,S +procedure_occurrence,procedure_type_concept_id,38000249,Inpatient detail - 1st position,S +procedure_occurrence,procedure_type_concept_id,38000248,Inpatient detail - primary position,S +procedure_occurrence,procedure_type_concept_id,38000260,Inpatient header - 10th position,S +procedure_occurrence,procedure_type_concept_id,38000261,Inpatient header - 11th position,S +procedure_occurrence,procedure_type_concept_id,38000262,Inpatient header - 12th position,S +procedure_occurrence,procedure_type_concept_id,38000263,Inpatient header - 13th position,S +procedure_occurrence,procedure_type_concept_id,38000264,Inpatient header - 14th position,S +procedure_occurrence,procedure_type_concept_id,38000265,Inpatient header - 15th position,S +procedure_occurrence,procedure_type_concept_id,38000251,Inpatient header - 1st position,S +procedure_occurrence,procedure_type_concept_id,38000252,Inpatient header - 2nd position,S +procedure_occurrence,procedure_type_concept_id,38000253,Inpatient header - 3rd position,S +procedure_occurrence,procedure_type_concept_id,38000254,Inpatient header - 4th position,S +procedure_occurrence,procedure_type_concept_id,38000255,Inpatient header - 5th position,S +procedure_occurrence,procedure_type_concept_id,38000256,Inpatient header - 6th position,S +procedure_occurrence,procedure_type_concept_id,38000257,Inpatient header - 7th position,S +procedure_occurrence,procedure_type_concept_id,38000258,Inpatient header - 8th position,S +procedure_occurrence,procedure_type_concept_id,38000259,Inpatient header - 9th position,S +procedure_occurrence,procedure_type_concept_id,38000250,Inpatient header - primary position,S +procedure_occurrence,procedure_type_concept_id,45756864,Outpatient detail - 10th position,S +procedure_occurrence,procedure_type_concept_id,45756865,Outpatient detail - 11th position,S +procedure_occurrence,procedure_type_concept_id,45756866,Outpatient detail - 12th position,S +procedure_occurrence,procedure_type_concept_id,45756867,Outpatient detail - 13th position,S +procedure_occurrence,procedure_type_concept_id,45756868,Outpatient detail - 14th position,S +procedure_occurrence,procedure_type_concept_id,45756869,Outpatient detail - 15th position,S +procedure_occurrence,procedure_type_concept_id,45756870,Outpatient detail - 16th position,S +procedure_occurrence,procedure_type_concept_id,45756871,Outpatient detail - 17th position,S +procedure_occurrence,procedure_type_concept_id,45756872,Outpatient detail - 18th position,S +procedure_occurrence,procedure_type_concept_id,45756873,Outpatient detail - 19th position,S +procedure_occurrence,procedure_type_concept_id,38000267,Outpatient detail - 1st position,S +procedure_occurrence,procedure_type_concept_id,45756874,Outpatient detail - 20th position,S +procedure_occurrence,procedure_type_concept_id,45756875,Outpatient detail - 21th position,S +procedure_occurrence,procedure_type_concept_id,45756876,Outpatient detail - 22th position,S +procedure_occurrence,procedure_type_concept_id,45756877,Outpatient detail - 23th position,S +procedure_occurrence,procedure_type_concept_id,45756878,Outpatient detail - 24th position,S +procedure_occurrence,procedure_type_concept_id,45756879,Outpatient detail - 25th position,S +procedure_occurrence,procedure_type_concept_id,45756880,Outpatient detail - 26th position,S +procedure_occurrence,procedure_type_concept_id,45756881,Outpatient detail - 27th position,S +procedure_occurrence,procedure_type_concept_id,45756882,Outpatient detail - 28th position,S +procedure_occurrence,procedure_type_concept_id,45756883,Outpatient detail - 29th position,S +procedure_occurrence,procedure_type_concept_id,45756856,Outpatient detail - 2nd position,S +procedure_occurrence,procedure_type_concept_id,45756884,Outpatient detail - 30th position,S +procedure_occurrence,procedure_type_concept_id,45756885,Outpatient detail - 31th position,S +procedure_occurrence,procedure_type_concept_id,45756886,Outpatient detail - 32th position,S +procedure_occurrence,procedure_type_concept_id,45756887,Outpatient detail - 33th position,S +procedure_occurrence,procedure_type_concept_id,45756888,Outpatient detail - 34th position,S +procedure_occurrence,procedure_type_concept_id,45756889,Outpatient detail - 35th position,S +procedure_occurrence,procedure_type_concept_id,45756890,Outpatient detail - 36th position,S +procedure_occurrence,procedure_type_concept_id,45756891,Outpatient detail - 37th position,S +procedure_occurrence,procedure_type_concept_id,45756892,Outpatient detail - 38th position,S +procedure_occurrence,procedure_type_concept_id,45756893,Outpatient detail - 39th position,S +procedure_occurrence,procedure_type_concept_id,45756857,Outpatient detail - 3rd position,S +procedure_occurrence,procedure_type_concept_id,45756894,Outpatient detail - 40th position,S +procedure_occurrence,procedure_type_concept_id,45756895,Outpatient detail - 41th position,S +procedure_occurrence,procedure_type_concept_id,45756896,Outpatient detail - 42th position,S +procedure_occurrence,procedure_type_concept_id,45756897,Outpatient detail - 43th position,S +procedure_occurrence,procedure_type_concept_id,45756898,Outpatient detail - 44th position,S +procedure_occurrence,procedure_type_concept_id,45756899,Outpatient detail - 45th position,S +procedure_occurrence,procedure_type_concept_id,45756858,Outpatient detail - 4th position,S +procedure_occurrence,procedure_type_concept_id,45756859,Outpatient detail - 5th position,S +procedure_occurrence,procedure_type_concept_id,45756860,Outpatient detail - 6th position,S +procedure_occurrence,procedure_type_concept_id,45756861,Outpatient detail - 7th position,S +procedure_occurrence,procedure_type_concept_id,45756862,Outpatient detail - 8th position,S +procedure_occurrence,procedure_type_concept_id,45756863,Outpatient detail - 9th position,S +procedure_occurrence,procedure_type_concept_id,38000266,Outpatient detail - primary position,S +procedure_occurrence,procedure_type_concept_id,38000269,Outpatient header - 1st position,S +procedure_occurrence,procedure_type_concept_id,38000270,Outpatient header - 2nd position,S +procedure_occurrence,procedure_type_concept_id,38000271,Outpatient header - 3rd position,S +procedure_occurrence,procedure_type_concept_id,38000272,Outpatient header - 4th position,S +procedure_occurrence,procedure_type_concept_id,38000273,Outpatient header - 5th position,S +procedure_occurrence,procedure_type_concept_id,38000274,Outpatient header - 6th position,S +procedure_occurrence,procedure_type_concept_id,38000268,Outpatient header - primary position,S +procedure_occurrence,procedure_type_concept_id,43542354,Physician administered drug (identified as procedure),S +procedure_occurrence,procedure_type_concept_id,44786630,Primary Procedure,S +procedure_occurrence,procedure_type_concept_id,581412,Procedure Recorded from a Survey,S +procedure_occurrence,procedure_type_concept_id,38003622,Procedure recorded as diagnostic code,S +procedure_occurrence,procedure_type_concept_id,38003621,Procedure recorded as lab test,S +procedure_occurrence,procedure_type_concept_id,42898141,Referral record,S +procedure_occurrence,procedure_type_concept_id,44786631,Secondary Procedure,S +specimen,specimen_type_concept_id,581378,EHR Detail,S +visit_occurrence,visit_concept_id,32217,Admitted as an inpatient to this hospital,S +visit_occurrence,visit_concept_id,8882,Adult Living Care Facility,S +visit_occurrence,visit_concept_id,8850,Ambulance - Air or Water,S +visit_occurrence,visit_concept_id,8668,Ambulance - Land,S +visit_occurrence,visit_concept_id,581478,Ambulance Visit,S +visit_occurrence,visit_concept_id,8883,Ambulatory Surgical Center,S +visit_occurrence,visit_concept_id,8615,Assisted Living Facility,S +visit_occurrence,visit_concept_id,8650,Birthing Center,S +visit_occurrence,visit_concept_id,32194,Clinic or physicians office,S +visit_occurrence,visit_concept_id,8964,Community Mental Health Center,S +visit_occurrence,visit_concept_id,8920,Comprehensive Inpatient Rehabilitation Facility,S +visit_occurrence,visit_concept_id,8947,Comprehensive Outpatient Rehabilitation Facility,S +visit_occurrence,visit_concept_id,32198,Court/Law enforcement,S +visit_occurrence,visit_concept_id,8827,Custodial Care Facility,S +visit_occurrence,visit_concept_id,32235,Discharged to home or self-care with a planned acute care hospital readmission,S +visit_occurrence,visit_concept_id,32210,Discharged to home/self care (routine charge).,S +visit_occurrence,visit_concept_id,32231,Discharged/Transferred to a psychiatric hospital or psychiatric distinct unit of a hospital,S +visit_occurrence,visit_concept_id,32219,Discharged/transferred to Court/Law Enforcement,S +visit_occurrence,visit_concept_id,32232,Discharged/transferred to a Critical Access Hospital (CAH),S +visit_occurrence,visit_concept_id,32245,Discharged/transferred to a Medicare certified long term care hospital (LTCH) with a planned acute care hospital inpatient readmission,S +visit_occurrence,visit_concept_id,32248,Discharged/transferred to a critical access hospital (CAH) with a planned acute care hospital inpatient readmission,S +visit_occurrence,visit_concept_id,32239,Discharged/transferred to a designated cancer center or children?s hospital with a planned acute care hospital inpatient readmission,S +visit_occurrence,visit_concept_id,32233,Discharged/transferred to a designated disaster alternative care site,S +visit_occurrence,visit_concept_id,32238,Discharged/transferred to a facility that provides custodial or supportive care with a planned acute care hospital inpatient readmission,S +visit_occurrence,visit_concept_id,32242,Discharged/transferred to a federal health care facility with a planned acute care hospital inpatient readmission,S +visit_occurrence,visit_concept_id,32224,Discharged/transferred to a federal hospital,S +visit_occurrence,visit_concept_id,32243,Discharged/transferred to a hospital-based Medicare approved swing bed with a planned acute care hospital inpatient readmission,S +visit_occurrence,visit_concept_id,32229,Discharged/transferred to a long term care hospitals,S +visit_occurrence,visit_concept_id,32230,Discharged/transferred to a nursing facility certified under Medicaid but not under Medicare,S +visit_occurrence,visit_concept_id,32247,Discharged/transferred to a psychiatric hospital/distinct part unit of a hospital with a planned acute care hospital inpatient readmission,S +visit_occurrence,visit_concept_id,32236,Discharged/transferred to a short term general hospital for inpatient care with a planned acute care hospital inpatient readmission,S +visit_occurrence,visit_concept_id,32237,Discharged/transferred to a skilled nursing facility (SNF) with Medicare certification with a planned acute care hospital inpatient readmission,S +visit_occurrence,visit_concept_id,32244,Discharged/transferred to an inpatient rehabilitation facility (IRF) including rehabilitation distinct part units of a hospital with a planned acute care hospital inpatient readmission,S +visit_occurrence,visit_concept_id,32228,Discharged/transferred to an inpatient rehabilitation facility including distinct parts units of a hospital,S +visit_occurrence,visit_concept_id,32234,Discharged/transferred to another type of health care institution not defined elsewhere in code list.,S +visit_occurrence,visit_concept_id,32249,Discharged/transferred to another type of health care institution not defined elsewhere in this code list with a planned acute care hospital inpatient readmission,S +visit_occurrence,visit_concept_id,32214,Discharged/transferred to another type of institution for inpatient care (designated cancer center or childrens hospital),S +visit_occurrence,visit_concept_id,32241,Discharged/transferred to court/law enforcement with a planned acute care hospital inpatient readmission,S +visit_occurrence,visit_concept_id,32215,Discharged/transferred to home care of organized home health service organization.,S +visit_occurrence,visit_concept_id,32240,Discharged/transferred to home under care of organized home health service organization with a planned acute care hospital inpatient readmission,S +visit_occurrence,visit_concept_id,32213,Discharged/transferred to intermediate care facility (ICF).,S +visit_occurrence,visit_concept_id,32246,Discharged/transferred to nursing facility certified under Medicaid but not certified under Medicare with a planned acute care hospital inpatient readmission,S +visit_occurrence,visit_concept_id,32211,Discharged/transferred to other short term general hospital for inpatient care.,S +visit_occurrence,visit_concept_id,32212,Discharged/transferred to skilled nursing facility (SNF) with Medicare certification in anticipation of covered skilled care,S +visit_occurrence,visit_concept_id,32227,Discharged/transferred within this institution to a hospital-based Medicare approved swing bed,S +visit_occurrence,visit_concept_id,8870,Emergency Room - Hospital,S +visit_occurrence,visit_concept_id,581381,Emergency Room Critical Care Facility,S +visit_occurrence,visit_concept_id,9203,Emergency Room Visit,S +visit_occurrence,visit_concept_id,262,Emergency Room and Inpatient Visit,S +visit_occurrence,visit_concept_id,8949,End-Stage Renal Disease Treatment Facility,S +visit_occurrence,visit_concept_id,32218,Expired,S +visit_occurrence,visit_concept_id,32223,Expired - place unknown (Hospice claims only),S +visit_occurrence,visit_concept_id,32221,Expired at home (hospice claims only),S +visit_occurrence,visit_concept_id,32222,"Expired in a medical facility such as hospital, SNF, ICF, or freestanding hospice. (Hospice claims only)",S +visit_occurrence,visit_concept_id,8966,Federally Qualified Health Center,S +visit_occurrence,visit_concept_id,8851,Group Home,S +visit_occurrence,visit_concept_id,8536,Home,S +visit_occurrence,visit_concept_id,581476,Home Visit,S +visit_occurrence,visit_concept_id,32193,Home/non health care facility point of origin,S +visit_occurrence,visit_concept_id,8672,Homeless Shelter,S +visit_occurrence,visit_concept_id,8546,Hospice,S +visit_occurrence,visit_concept_id,32225,Hospice - home,S +visit_occurrence,visit_concept_id,32226,Hospice - medical facility,S +visit_occurrence,visit_concept_id,8716,Independent Clinic,S +visit_occurrence,visit_concept_id,8809,Independent Laboratory,S +visit_occurrence,visit_concept_id,8968,Indian Health Service Free-standing Facility,S +visit_occurrence,visit_concept_id,8969,Indian Health Service Provider-based Facility,S +visit_occurrence,visit_concept_id,32199,Information not available,S +visit_occurrence,visit_concept_id,581383,Inpatient Cardiac Care Facility,S +visit_occurrence,visit_concept_id,581379,Inpatient Critical Care Facility,S +visit_occurrence,visit_concept_id,8717,Inpatient Hospital,S +visit_occurrence,visit_concept_id,581382,Inpatient Intensive Care Facility,S +visit_occurrence,visit_concept_id,8970,Inpatient Long-term Care,S +visit_occurrence,visit_concept_id,581384,Inpatient Nursery,S +visit_occurrence,visit_concept_id,8971,Inpatient Psychiatric Facility,S +visit_occurrence,visit_concept_id,9201,Inpatient Visit,S +visit_occurrence,visit_concept_id,32037,Intensive Care,S +visit_occurrence,visit_concept_id,581399,Interactive Telemedicine Service,S +visit_occurrence,visit_concept_id,8951,Intermediate Care Facility/Mentally Retarded,S +visit_occurrence,visit_concept_id,32036,Laboratory Visit,S +visit_occurrence,visit_concept_id,32216,Left against medical advice or discontinued care.,S +visit_occurrence,visit_concept_id,42898160,Long Term Care Visit,S +visit_occurrence,visit_concept_id,8858,Mass Immunization Center,S +visit_occurrence,visit_concept_id,8905,Military Treatment Facility,S +visit_occurrence,visit_concept_id,8584,Mobile Unit,S +visit_occurrence,visit_concept_id,8974,Non-residential Substance Abuse Treatment Facility,S +visit_occurrence,visit_concept_id,8676,Nursing Facility,S +visit_occurrence,visit_concept_id,581385,Observation Room,S +visit_occurrence,visit_concept_id,5084,Off Campus-Outpatient Hospital,S +visit_occurrence,visit_concept_id,8940,Office,S +visit_occurrence,visit_concept_id,581477,Office Visit,S +visit_occurrence,visit_concept_id,8892,Other Inpatient Care,S +visit_occurrence,visit_concept_id,8844,Other Place of Service,S +visit_occurrence,visit_concept_id,581380,Outpatient Critical Care Facility,S +visit_occurrence,visit_concept_id,8756,Outpatient Hospital,S +visit_occurrence,visit_concept_id,8677,Outpatient NEC,S +visit_occurrence,visit_concept_id,9202,Outpatient Visit,S +visit_occurrence,visit_concept_id,8562,Pharmacy,S +visit_occurrence,visit_concept_id,581458,Pharmacy visit,S +visit_occurrence,visit_concept_id,581475,Place of Employment-Worksite,S +visit_occurrence,visit_concept_id,38003619,Prison/Correctional Facility,S +visit_occurrence,visit_concept_id,8913,Psychiatric Facility-Partial Hospitalization,S +visit_occurrence,visit_concept_id,8976,Psychiatric Residential Treatment Center,S +visit_occurrence,visit_concept_id,8977,Public Health Clinic,S +visit_occurrence,visit_concept_id,581479,Rehabilitation Visit,S +visit_occurrence,visit_concept_id,8957,Residential Substance Abuse Treatment Facility,S +visit_occurrence,visit_concept_id,8761,Rural Health Clinic,S +visit_occurrence,visit_concept_id,8537,School,S +visit_occurrence,visit_concept_id,8863,Skilled Nursing Facility,S +visit_occurrence,visit_concept_id,32220,Still patient,S +visit_occurrence,visit_concept_id,5083,Telehealth,S +visit_occurrence,visit_concept_id,8602,Temporary Lodging,S +visit_occurrence,visit_concept_id,32196,"Transfer from Skilled nursing facility, Intermediate care facility, Assisted living facility",S +visit_occurrence,visit_concept_id,32195,"Transfer from a Hospital, different facility",S +visit_occurrence,visit_concept_id,32201,Transfer from ambulatory care center,S +visit_occurrence,visit_concept_id,32197,Transfer from another Healthcare facility,S +visit_occurrence,visit_concept_id,32202,Transfer from hospice,S +visit_occurrence,visit_concept_id,32200,"Transfer within hospital, but seperate claim",S +visit_occurrence,visit_concept_id,8941,Tribal 638 Free-standing Facility,S +visit_occurrence,visit_concept_id,8960,Tribal 638 Provider-based Facility,S +visit_occurrence,visit_concept_id,32209,Unknown Value (but present in data),S +visit_occurrence,visit_concept_id,8782,Urgent Care Facility,S +visit_occurrence,visit_concept_id,38003620,Walk-in Retail Health Clinic,S +visit_occurrence,visit_concept_id,32288,Admission/Election Notice, +visit_occurrence,visit_concept_id,32280,Admit through Discharge Claim, +visit_occurrence,visit_concept_id,32274,Ambulatory Surgery Center, +visit_occurrence,visit_concept_id,32156,Ambulatory Surgery Center@Admit through Discharge Claim, +visit_occurrence,visit_concept_id,32158,Ambulatory Surgery Center@Interim-Cont claim (b), +visit_occurrence,visit_concept_id,32157,Ambulatory Surgery Center@Interim-first claim, +visit_occurrence,visit_concept_id,32159,Ambulatory Surgery Center@Interim-last claim (b), +visit_occurrence,visit_concept_id,32160,Ambulatory Surgery Center@Late Charges Only, +visit_occurrence,visit_concept_id,32155,Ambulatory Surgery Center@Non-Payment/Zero, +visit_occurrence,visit_concept_id,32161,Ambulatory Surgery Center@Replacement of Prior Claim, +visit_occurrence,visit_concept_id,32162,Ambulatory Surgery Center@Void/ Cancel of Prior claim (a), +visit_occurrence,visit_concept_id,32292,CMS Initiated Adjustment, +visit_occurrence,visit_concept_id,32291,CWF Initiated Adjustment Claim, +visit_occurrence,visit_concept_id,32264,Clinic- Hosp Based or Indep Renal Dialysis Center, +visit_occurrence,visit_concept_id,32090,Clinic- Hosp Based or Indep Renal Dialysis Center@Admit through Discharge Claim, +visit_occurrence,visit_concept_id,32097,Clinic- Hosp Based or Indep Renal Dialysis Center@CWF Initiated Adjustment Claim, +visit_occurrence,visit_concept_id,32099,Clinic- Hosp Based or Indep Renal Dialysis Center@Initiated Adjustment Claim-other, +visit_occurrence,visit_concept_id,32092,Clinic- Hosp Based or Indep Renal Dialysis Center@Interim-Cont claim (b), +visit_occurrence,visit_concept_id,32091,Clinic- Hosp Based or Indep Renal Dialysis Center@Interim-first claim, +visit_occurrence,visit_concept_id,32093,Clinic- Hosp Based or Indep Renal Dialysis Center@Interim-last claim (b), +visit_occurrence,visit_concept_id,32098,Clinic- Hosp Based or Indep Renal Dialysis Center@Intermediary Adjustment Claim (Other than QIO or Provider), +visit_occurrence,visit_concept_id,32094,Clinic- Hosp Based or Indep Renal Dialysis Center@Late Charges Only, +visit_occurrence,visit_concept_id,32089,Clinic- Hosp Based or Indep Renal Dialysis Center@Non-Payment/Zero, +visit_occurrence,visit_concept_id,32095,Clinic- Hosp Based or Indep Renal Dialysis Center@Replacement of Prior Claim, +visit_occurrence,visit_concept_id,32096,Clinic- Hosp Based or Indep Renal Dialysis Center@Void/ Cancel of Prior claim (a), +visit_occurrence,visit_concept_id,32268,Clinic-Community Mental Health Center, +visit_occurrence,visit_concept_id,32124,Clinic-Community Mental Health Center@Admit through Discharge Claim, +visit_occurrence,visit_concept_id,32129,Clinic-Community Mental Health Center@Final Claim for a Home Health PPS Episode, +visit_occurrence,visit_concept_id,32126,Clinic-Community Mental Health Center@Interim-Cont claim (b), +visit_occurrence,visit_concept_id,32125,Clinic-Community Mental Health Center@Interim-first claim, +visit_occurrence,visit_concept_id,32127,Clinic-Community Mental Health Center@Interim-last claim (b), +visit_occurrence,visit_concept_id,32123,Clinic-Community Mental Health Center@Non-Payment/Zero, +visit_occurrence,visit_concept_id,32128,Clinic-Community Mental Health Center@Replacement of Prior Claim, +visit_occurrence,visit_concept_id,32267,Clinic-Comprehensive Outpatient Rehab (CORF), +visit_occurrence,visit_concept_id,32116,Clinic-Comprehensive Outpatient Rehab (CORF)@Admit through Discharge Claim, +visit_occurrence,visit_concept_id,32121,Clinic-Comprehensive Outpatient Rehab (CORF)@CWF Initiated Adjustment Claim, +visit_occurrence,visit_concept_id,32118,Clinic-Comprehensive Outpatient Rehab (CORF)@Interim-Cont claim (b), +visit_occurrence,visit_concept_id,32117,Clinic-Comprehensive Outpatient Rehab (CORF)@Interim-first claim, +visit_occurrence,visit_concept_id,32119,Clinic-Comprehensive Outpatient Rehab (CORF)@Interim-last claim (b), +visit_occurrence,visit_concept_id,32122,Clinic-Comprehensive Outpatient Rehab (CORF)@Intermediary Adjustment Claim (Other than QIO or Provider), +visit_occurrence,visit_concept_id,32120,Clinic-Comprehensive Outpatient Rehab (CORF)@Replacement of Prior Claim, +visit_occurrence,visit_concept_id,32269,Clinic-Federally Qualified Health Center (FQHC), +visit_occurrence,visit_concept_id,32130,Clinic-Federally Qualified Health Center (FQHC)@Admit through Discharge Claim, +visit_occurrence,visit_concept_id,32134,Clinic-Federally Qualified Health Center (FQHC)@CWF Initiated Adjustment Claim, +visit_occurrence,visit_concept_id,32131,Clinic-Federally Qualified Health Center (FQHC)@Interim-Cont claim (b), +visit_occurrence,visit_concept_id,32132,Clinic-Federally Qualified Health Center (FQHC)@Replacement of Prior Claim, +visit_occurrence,visit_concept_id,32133,Clinic-Federally Qualified Health Center (FQHC)@Void/ Cancel of Prior claim (a), +visit_occurrence,visit_concept_id,32265,Clinic-Freestanding, +visit_occurrence,visit_concept_id,32100,Clinic-Freestanding@Admit through Discharge Claim, +visit_occurrence,visit_concept_id,32102,Clinic-Freestanding@Interim-Cont claim (b), +visit_occurrence,visit_concept_id,32101,Clinic-Freestanding@Interim-first claim, +visit_occurrence,visit_concept_id,32103,Clinic-Freestanding@Interim-last claim (b), +visit_occurrence,visit_concept_id,32104,Clinic-Freestanding@Replacement of Prior Claim, +visit_occurrence,visit_concept_id,32271,Clinic-Other, +visit_occurrence,visit_concept_id,32137,Clinic-Other@Admit through Discharge Claim, +visit_occurrence,visit_concept_id,32139,Clinic-Other@Interim-Cont claim (b), +visit_occurrence,visit_concept_id,32138,Clinic-Other@Interim-first claim, +visit_occurrence,visit_concept_id,32140,Clinic-Other@Late Charges Only, +visit_occurrence,visit_concept_id,32136,Clinic-Other@Non-Payment/Zero, +visit_occurrence,visit_concept_id,32141,Clinic-Other@Replacement of Prior Claim, +visit_occurrence,visit_concept_id,32266,Clinic-Outpatient Rehab Facility (ORF), +visit_occurrence,visit_concept_id,32106,Clinic-Outpatient Rehab Facility (ORF)@Admit through Discharge Claim, +visit_occurrence,visit_concept_id,32114,Clinic-Outpatient Rehab Facility (ORF)@CWF Initiated Adjustment Claim, +visit_occurrence,visit_concept_id,32113,Clinic-Outpatient Rehab Facility (ORF)@Final Claim for a Home Health PPS Episode, +visit_occurrence,visit_concept_id,32108,Clinic-Outpatient Rehab Facility (ORF)@Interim-Cont claim (b), +visit_occurrence,visit_concept_id,32107,Clinic-Outpatient Rehab Facility (ORF)@Interim-first claim, +visit_occurrence,visit_concept_id,32109,Clinic-Outpatient Rehab Facility (ORF)@Interim-last claim (b), +visit_occurrence,visit_concept_id,32115,Clinic-Outpatient Rehab Facility (ORF)@Intermediary Adjustment Claim (Other than QIO or Provider), +visit_occurrence,visit_concept_id,32110,Clinic-Outpatient Rehab Facility (ORF)@Late Charges Only, +visit_occurrence,visit_concept_id,32105,Clinic-Outpatient Rehab Facility (ORF)@Non-Payment/Zero, +visit_occurrence,visit_concept_id,32111,Clinic-Outpatient Rehab Facility (ORF)@Replacement of Prior Claim, +visit_occurrence,visit_concept_id,32112,Clinic-Outpatient Rehab Facility (ORF)@Void/ Cancel of Prior claim (a), +visit_occurrence,visit_concept_id,32263,Clinic-Rural Health, +visit_occurrence,visit_concept_id,32081,Clinic-Rural Health@Admit through Discharge Claim, +visit_occurrence,visit_concept_id,32086,Clinic-Rural Health@CWF Initiated Adjustment Claim, +visit_occurrence,visit_concept_id,32083,Clinic-Rural Health@Interim-Cont claim (b), +visit_occurrence,visit_concept_id,32082,Clinic-Rural Health@Interim-first claim, +visit_occurrence,visit_concept_id,32087,Clinic-Rural Health@Intermediary Adjustment Claim (Other than QIO or Provider), +visit_occurrence,visit_concept_id,32088,Clinic-Rural Health@MSP Initiated Adjustment Claim, +visit_occurrence,visit_concept_id,32080,Clinic-Rural Health@Non-Payment/Zero, +visit_occurrence,visit_concept_id,32084,Clinic-Rural Health@Replacement of Prior Claim, +visit_occurrence,visit_concept_id,32085,Clinic-Rural Health@Void/ Cancel of Prior claim (a), +visit_occurrence,visit_concept_id,32276,Critical Access Hospital, +visit_occurrence,visit_concept_id,32165,Critical Access Hospital@Admit through Discharge Claim, +visit_occurrence,visit_concept_id,32171,Critical Access Hospital@CWF Initiated Adjustment Claim, +visit_occurrence,visit_concept_id,32167,Critical Access Hospital@Interim-Cont claim (b), +visit_occurrence,visit_concept_id,32166,Critical Access Hospital@Interim-first claim, +visit_occurrence,visit_concept_id,32168,Critical Access Hospital@Interim-last claim (b), +visit_occurrence,visit_concept_id,32172,Critical Access Hospital@Intermediary Adjustment Claim (Other than QIO or Provider), +visit_occurrence,visit_concept_id,32169,Critical Access Hospital@Late Charges Only, +visit_occurrence,visit_concept_id,32164,Critical Access Hospital@Non-Payment/Zero, +visit_occurrence,visit_concept_id,32170,Critical Access Hospital@Replacement of Prior Claim, +visit_occurrence,visit_concept_id,32287,Final Claim for a Home Health PPS Episode, +visit_occurrence,visit_concept_id,32275,Freestanding Birthing Center, +visit_occurrence,visit_concept_id,32163,Freestanding Birthing Center@Admit through Discharge Claim, +visit_occurrence,visit_concept_id,32258,Home Health-Inpatient (treatment Part B only), +visit_occurrence,visit_concept_id,32376,Home Health-Inpatient (treatment Part B only)@Admit through Discharge Claim, +visit_occurrence,visit_concept_id,32383,Home Health-Inpatient (treatment Part B only)@Final Claim for a Home Health PPS Episode, +visit_occurrence,visit_concept_id,32384,Home Health-Inpatient (treatment Part B only)@Initiated Adjustment Claim-other, +visit_occurrence,visit_concept_id,32378,Home Health-Inpatient (treatment Part B only)@Interim-Cont claim (b), +visit_occurrence,visit_concept_id,32377,Home Health-Inpatient (treatment Part B only)@Interim-first claim, +visit_occurrence,visit_concept_id,32379,Home Health-Inpatient (treatment Part B only)@Interim-last claim (b), +visit_occurrence,visit_concept_id,32380,Home Health-Inpatient (treatment Part B only)@Late Charges Only, +visit_occurrence,visit_concept_id,32375,Home Health-Inpatient (treatment Part B only)@Non-Payment/Zero, +visit_occurrence,visit_concept_id,32381,Home Health-Inpatient (treatment Part B only)@Replacement of Prior Claim, +visit_occurrence,visit_concept_id,32382,Home Health-Inpatient (treatment Part B only)@Void/ Cancel of Prior claim (a), +visit_occurrence,visit_concept_id,32260,Home Health-Other (Med & Surg under plan of treatment), +visit_occurrence,visit_concept_id,32396,Home Health-Other (Med & Surg under plan of treatment)@Admit through Discharge Claim, +visit_occurrence,visit_concept_id,32052,Home Health-Other (Med & Surg under plan of treatment)@Final Claim for a Home Health PPS Episode, +visit_occurrence,visit_concept_id,32049,Home Health-Other (Med & Surg under plan of treatment)@Interim-Cont claim (b), +visit_occurrence,visit_concept_id,32048,Home Health-Other (Med & Surg under plan of treatment)@Interim-first claim, +visit_occurrence,visit_concept_id,32050,Home Health-Other (Med & Surg under plan of treatment)@Interim-last claim (b), +visit_occurrence,visit_concept_id,32053,Home Health-Other (Med & Surg under plan of treatment)@Intermediary Adjustment Claim (Other than QIO or Provider), +visit_occurrence,visit_concept_id,32395,Home Health-Other (Med & Surg under plan of treatment)@Non-Payment/Zero, +visit_occurrence,visit_concept_id,32051,Home Health-Other (Med & Surg under plan of treatment)@Replacement of Prior Claim, +visit_occurrence,visit_concept_id,32054,Home Health-Other (Med & Surg under plan of treatment)@Void/Cance a Prior Abreviated Encounter Submission, +visit_occurrence,visit_concept_id,32259,"Home Health-Outpatient (Part A, Includes DME Part A)", +visit_occurrence,visit_concept_id,32386,"Home Health-Outpatient (Part A, Includes DME Part A)@Admit through Discharge Claim", +visit_occurrence,visit_concept_id,32393,"Home Health-Outpatient (Part A, Includes DME Part A)@Final Claim for a Home Health PPS Episode", +visit_occurrence,visit_concept_id,32388,"Home Health-Outpatient (Part A, Includes DME Part A)@Interim-Cont claim (b)", +visit_occurrence,visit_concept_id,32387,"Home Health-Outpatient (Part A, Includes DME Part A)@Interim-first claim", +visit_occurrence,visit_concept_id,32389,"Home Health-Outpatient (Part A, Includes DME Part A)@Interim-last claim (b)", +visit_occurrence,visit_concept_id,32390,"Home Health-Outpatient (Part A, Includes DME Part A)@Late Charges Only", +visit_occurrence,visit_concept_id,32394,"Home Health-Outpatient (Part A, Includes DME Part A)@MSP Initiated Adjustment Claim", +visit_occurrence,visit_concept_id,32385,"Home Health-Outpatient (Part A, Includes DME Part A)@Non-Payment/Zero", +visit_occurrence,visit_concept_id,32391,"Home Health-Outpatient (Part A, Includes DME Part A)@Replacement of Prior Claim", +visit_occurrence,visit_concept_id,32392,"Home Health-Outpatient (Part A, Includes DME Part A)@Void/ Cancel of Prior claim (a)", +visit_occurrence,visit_concept_id,32273,Hospice (hospital based), +visit_occurrence,visit_concept_id,32149,Hospice (hospital based)@Admit through Discharge Claim, +visit_occurrence,visit_concept_id,32151,Hospice (hospital based)@Interim-Cont claim (b), +visit_occurrence,visit_concept_id,32150,Hospice (hospital based)@Interim-first claim, +visit_occurrence,visit_concept_id,32152,Hospice (hospital based)@Interim-last claim (b), +visit_occurrence,visit_concept_id,32153,Hospice (hospital based)@Late Charges Only, +visit_occurrence,visit_concept_id,32154,Hospice (hospital based)@Replacement of Prior Claim, +visit_occurrence,visit_concept_id,32289,Hospice/CMS Coord Care/Religious NonMed/Cent of ExcellenceDemonstration, +visit_occurrence,visit_concept_id,32272,Hospital (non-hospital based), +visit_occurrence,visit_concept_id,32142,Hospital (non-hospital based)@Admit through Discharge Claim, +visit_occurrence,visit_concept_id,32148,Hospital (non-hospital based)@Hospice/CMS Coord Care/Religious NonMed/Cent of ExcellenceDemonstration, +visit_occurrence,visit_concept_id,32144,Hospital (non-hospital based)@Interim-Cont claim (b), +visit_occurrence,visit_concept_id,32143,Hospital (non-hospital based)@Interim-first claim, +visit_occurrence,visit_concept_id,32145,Hospital (non-hospital based)@Interim-last claim (b), +visit_occurrence,visit_concept_id,32146,Hospital (non-hospital based)@Late Charges Only, +visit_occurrence,visit_concept_id,32147,Hospital (non-hospital based)@Replacement of Prior Claim, +visit_occurrence,visit_concept_id,32253,Hospital -Laboratory Services provided to non-pt, +visit_occurrence,visit_concept_id,32337,Hospital -Laboratory Services provided to non-pt@Admit through Discharge Claim, +visit_occurrence,visit_concept_id,32339,Hospital -Laboratory Services provided to non-pt@Interim-Cont claim (b), +visit_occurrence,visit_concept_id,32338,Hospital -Laboratory Services provided to non-pt@Interim-first claim, +visit_occurrence,visit_concept_id,32340,Hospital -Laboratory Services provided to non-pt@Interim-last claim (b), +visit_occurrence,visit_concept_id,32341,Hospital -Laboratory Services provided to non-pt@Late Charges Only, +visit_occurrence,visit_concept_id,32336,Hospital -Laboratory Services provided to non-pt@Non-Payment/Zero, +visit_occurrence,visit_concept_id,32342,Hospital -Laboratory Services provided to non-pt@Replacement of Prior Claim, +visit_occurrence,visit_concept_id,32343,Hospital -Laboratory Services provided to non-pt@Void/ Cancel of Prior claim (a), +visit_occurrence,visit_concept_id,32250,Hospital Inpatient (Including Medicare Part A), +visit_occurrence,visit_concept_id,32300,Hospital Inpatient (Including Medicare Part A)@Admit through Discharge Claim, +visit_occurrence,visit_concept_id,32307,Hospital Inpatient (Including Medicare Part A)@CWF Initiated Adjustment Claim, +visit_occurrence,visit_concept_id,32302,Hospital Inpatient (Including Medicare Part A)@Interim-Cont claim (b), +visit_occurrence,visit_concept_id,32301,Hospital Inpatient (Including Medicare Part A)@Interim-first claim, +visit_occurrence,visit_concept_id,32303,Hospital Inpatient (Including Medicare Part A)@Interim-last claim (b), +visit_occurrence,visit_concept_id,32308,Hospital Inpatient (Including Medicare Part A)@Intermediary Adjustment Claim (Other than QIO or Provider), +visit_occurrence,visit_concept_id,32304,Hospital Inpatient (Including Medicare Part A)@Late Charges Only, +visit_occurrence,visit_concept_id,32299,Hospital Inpatient (Including Medicare Part A)@Non-Payment/Zero, +visit_occurrence,visit_concept_id,32310,Hospital Inpatient (Including Medicare Part A)@Replacement of Prior Abbreviated Encounter Submission, +visit_occurrence,visit_concept_id,32305,Hospital Inpatient (Including Medicare Part A)@Replacement of Prior Claim, +visit_occurrence,visit_concept_id,32306,Hospital Inpatient (Including Medicare Part A)@Void/ Cancel of Prior claim (a), +visit_occurrence,visit_concept_id,32309,Hospital Inpatient (Including Medicare Part A)@Void/Cance a Prior Abreviated Encounter Submission, +visit_occurrence,visit_concept_id,32251,Hospital Inpatient (Medicare Part B only), +visit_occurrence,visit_concept_id,32312,Hospital Inpatient (Medicare Part B only)@Admit through Discharge Claim, +visit_occurrence,visit_concept_id,32314,Hospital Inpatient (Medicare Part B only)@Interim-Cont claim (b), +visit_occurrence,visit_concept_id,32313,Hospital Inpatient (Medicare Part B only)@Interim-first claim, +visit_occurrence,visit_concept_id,32315,Hospital Inpatient (Medicare Part B only)@Interim-last claim (b), +visit_occurrence,visit_concept_id,32317,Hospital Inpatient (Medicare Part B only)@Intermediary Adjustment Claim (Other than QIO or Provider), +visit_occurrence,visit_concept_id,32311,Hospital Inpatient (Medicare Part B only)@Non-Payment/Zero, +visit_occurrence,visit_concept_id,32316,Hospital Inpatient (Medicare Part B only)@Replacement of Prior Claim, +visit_occurrence,visit_concept_id,32252,Hospital Outpatient, +visit_occurrence,visit_concept_id,32319,Hospital Outpatient@Admit through Discharge Claim, +visit_occurrence,visit_concept_id,32329,Hospital Outpatient@CMS Initiated Adjustment, +visit_occurrence,visit_concept_id,32328,Hospital Outpatient@CWF Initiated Adjustment Claim, +visit_occurrence,visit_concept_id,32326,Hospital Outpatient@Final Claim for a Home Health PPS Episode, +visit_occurrence,visit_concept_id,32331,Hospital Outpatient@Initiated Adjustment Claim-other, +visit_occurrence,visit_concept_id,32321,Hospital Outpatient@Interim-Cont claim (b), +visit_occurrence,visit_concept_id,32320,Hospital Outpatient@Interim-first claim, +visit_occurrence,visit_concept_id,32322,Hospital Outpatient@Interim-last claim (b), +visit_occurrence,visit_concept_id,32330,Hospital Outpatient@Intermediary Adjustment Claim (Other than QIO or Provider), +visit_occurrence,visit_concept_id,32323,Hospital Outpatient@Late Charges Only, +visit_occurrence,visit_concept_id,32333,Hospital Outpatient@MSP Initiated Adjustment Claim, +visit_occurrence,visit_concept_id,32318,Hospital Outpatient@Non-Payment/Zero, +visit_occurrence,visit_concept_id,32332,Hospital Outpatient@OIG Initiated Adjustment Claim, +visit_occurrence,visit_concept_id,32335,Hospital Outpatient@Replacement of Prior Abbreviated Encounter Submission, +visit_occurrence,visit_concept_id,32324,Hospital Outpatient@Replacement of Prior Claim, +visit_occurrence,visit_concept_id,32325,Hospital Outpatient@Void/ Cancel of Prior claim (a), +visit_occurrence,visit_concept_id,32334,Hospital Outpatient@Void/Cance a Prior Abreviated Encounter Submission, +visit_occurrence,visit_concept_id,32327,Hospital Outpatient@Void/Cancel Hospice/CMS Coor Care/Non-med Religious/Cent for Excellence, +visit_occurrence,visit_concept_id,32254,Hospital-Swing Beds, +visit_occurrence,visit_concept_id,32344,Hospital-Swing Beds@Admit through Discharge Claim, +visit_occurrence,visit_concept_id,32346,Hospital-Swing Beds@Interim-Cont claim (b), +visit_occurrence,visit_concept_id,32345,Hospital-Swing Beds@Interim-first claim, +visit_occurrence,visit_concept_id,32347,Hospital-Swing Beds@Interim-last claim (b), +visit_occurrence,visit_concept_id,32348,Hospital-Swing Beds@Replacement of Prior Claim, +visit_occurrence,visit_concept_id,32294,Initiated Adjustment Claim-other, +visit_occurrence,visit_concept_id,32282,Interim-Cont claim (b), +visit_occurrence,visit_concept_id,32281,Interim-first claim, +visit_occurrence,visit_concept_id,32283,Interim-last claim (b), +visit_occurrence,visit_concept_id,32293,Intermediary Adjustment Claim (Other than QIO or Provider), +visit_occurrence,visit_concept_id,32401,Intermediate Care - Level 1 (Including Medicare Part A), +visit_occurrence,visit_concept_id,32057,Intermediate Care - Level 1 (Including Medicare Part A)@Admit through Discharge Claim, +visit_occurrence,visit_concept_id,32064,Intermediate Care - Level 1 (Including Medicare Part A)@CWF Initiated Adjustment Claim, +visit_occurrence,visit_concept_id,32059,Intermediate Care - Level 1 (Including Medicare Part A)@Interim-Cont claim (b), +visit_occurrence,visit_concept_id,32058,Intermediate Care - Level 1 (Including Medicare Part A)@Interim-first claim, +visit_occurrence,visit_concept_id,32060,Intermediate Care - Level 1 (Including Medicare Part A)@Interim-last claim (b), +visit_occurrence,visit_concept_id,32065,Intermediate Care - Level 1 (Including Medicare Part A)@Intermediary Adjustment Claim (Other than QIO or Provider), +visit_occurrence,visit_concept_id,32061,Intermediate Care - Level 1 (Including Medicare Part A)@Late Charges Only, +visit_occurrence,visit_concept_id,32056,Intermediate Care - Level 1 (Including Medicare Part A)@Non-Payment/Zero, +visit_occurrence,visit_concept_id,32067,Intermediate Care - Level 1 (Including Medicare Part A)@Replacement of Prior Abbreviated Encounter Submission, +visit_occurrence,visit_concept_id,32062,Intermediate Care - Level 1 (Including Medicare Part A)@Replacement of Prior Claim, +visit_occurrence,visit_concept_id,32063,Intermediate Care - Level 1 (Including Medicare Part A)@Void/ Cancel of Prior claim (a), +visit_occurrence,visit_concept_id,32066,Intermediate Care - Level 1 (Including Medicare Part A)@Void/Cance a Prior Abreviated Encounter Submission, +visit_occurrence,visit_concept_id,32402,Intermediate Care - Level 2 (Including Medicare Part A), +visit_occurrence,visit_concept_id,32069,Intermediate Care - Level 2 (Including Medicare Part A)@Admit through Discharge Claim, +visit_occurrence,visit_concept_id,32076,Intermediate Care - Level 2 (Including Medicare Part A)@CWF Initiated Adjustment Claim, +visit_occurrence,visit_concept_id,32071,Intermediate Care - Level 2 (Including Medicare Part A)@Interim-Cont claim (b), +visit_occurrence,visit_concept_id,32070,Intermediate Care - Level 2 (Including Medicare Part A)@Interim-first claim, +visit_occurrence,visit_concept_id,32072,Intermediate Care - Level 2 (Including Medicare Part A)@Interim-last claim (b), +visit_occurrence,visit_concept_id,32077,Intermediate Care - Level 2 (Including Medicare Part A)@Intermediary Adjustment Claim (Other than QIO or Provider), +visit_occurrence,visit_concept_id,32073,Intermediate Care - Level 2 (Including Medicare Part A)@Late Charges Only, +visit_occurrence,visit_concept_id,32068,Intermediate Care - Level 2 (Including Medicare Part A)@Non-Payment/Zero, +visit_occurrence,visit_concept_id,32079,Intermediate Care - Level 2 (Including Medicare Part A)@Replacement of Prior Abbreviated Encounter Submission, +visit_occurrence,visit_concept_id,32074,Intermediate Care - Level 2 (Including Medicare Part A)@Replacement of Prior Claim, +visit_occurrence,visit_concept_id,32075,Intermediate Care - Level 2 (Including Medicare Part A)@Void/ Cancel of Prior claim (a), +visit_occurrence,visit_concept_id,32078,Intermediate Care - Level 2 (Including Medicare Part A)@Void/Cance a Prior Abreviated Encounter Submission, +visit_occurrence,visit_concept_id,32284,Late Charges Only, +visit_occurrence,visit_concept_id,32270,Licensed Freestanding Emergency Medical Facility, +visit_occurrence,visit_concept_id,32135,Licensed Freestanding Emergency Medical Facility@Admit through Discharge Claim, +visit_occurrence,visit_concept_id,32296,MSP Initiated Adjustment Claim, +visit_occurrence,visit_concept_id,32279,Non-Payment/Zero, +visit_occurrence,visit_concept_id,32295,OIG Initiated Adjustment Claim, +visit_occurrence,visit_concept_id,32261,Religious Non-medical Health Care Inst-Outpatient Services, +visit_occurrence,visit_concept_id,32055,Religious Non-medical Health Care Inst-Outpatient Services@Admit through Discharge Claim, +visit_occurrence,visit_concept_id,32298,Replacement of Prior Abbreviated Encounter Submission, +visit_occurrence,visit_concept_id,32285,Replacement of Prior Claim, +visit_occurrence,visit_concept_id,32262,Reserved for NUBC, +visit_occurrence,visit_concept_id,32174,Residential Facility@Admit through Discharge Claim, +visit_occurrence,visit_concept_id,32181,Residential Facility@CWF Initiated Adjustment Claim, +visit_occurrence,visit_concept_id,32176,Residential Facility@Interim-Cont claim (b), +visit_occurrence,visit_concept_id,32175,Residential Facility@Interim-first claim, +visit_occurrence,visit_concept_id,32177,Residential Facility@Interim-last claim (b), +visit_occurrence,visit_concept_id,32182,Residential Facility@Intermediary Adjustment Claim (Other than QIO or Provider), +visit_occurrence,visit_concept_id,32178,Residential Facility@Late Charges Only, +visit_occurrence,visit_concept_id,32173,Residential Facility@Non-Payment/Zero, +visit_occurrence,visit_concept_id,32184,Residential Facility@Replacement of Prior Abbreviated Encounter Submission, +visit_occurrence,visit_concept_id,32179,Residential Facility@Replacement of Prior Claim, +visit_occurrence,visit_concept_id,32180,Residential Facility@Void/ Cancel of Prior claim (a), +visit_occurrence,visit_concept_id,32183,Residential Facility@Void/Cance a Prior Abreviated Encounter Submission, +visit_occurrence,visit_concept_id,32255,Skilled Nursing-Inpatient (Includes Medicare Part A), +visit_occurrence,visit_concept_id,32350,Skilled Nursing-Inpatient (Includes Medicare Part A)@Admit through Discharge Claim, +visit_occurrence,visit_concept_id,32356,Skilled Nursing-Inpatient (Includes Medicare Part A)@CWF Initiated Adjustment Claim, +visit_occurrence,visit_concept_id,32352,Skilled Nursing-Inpatient (Includes Medicare Part A)@Interim-Cont claim (b), +visit_occurrence,visit_concept_id,32351,Skilled Nursing-Inpatient (Includes Medicare Part A)@Interim-first claim, +visit_occurrence,visit_concept_id,32353,Skilled Nursing-Inpatient (Includes Medicare Part A)@Interim-last claim (b), +visit_occurrence,visit_concept_id,32357,Skilled Nursing-Inpatient (Includes Medicare Part A)@Intermediary Adjustment Claim (Other than QIO or Provider), +visit_occurrence,visit_concept_id,32349,Skilled Nursing-Inpatient (Includes Medicare Part A)@Non-Payment/Zero, +visit_occurrence,visit_concept_id,32354,Skilled Nursing-Inpatient (Includes Medicare Part A)@Replacement of Prior Claim, +visit_occurrence,visit_concept_id,32355,Skilled Nursing-Inpatient (Includes Medicare Part A)@Void/ Cancel of Prior claim (a), +visit_occurrence,visit_concept_id,32256,Skilled Nursing-Inpatient (Medicare Part B Only), +visit_occurrence,visit_concept_id,32358,Skilled Nursing-Inpatient (Medicare Part B Only)@Admit through Discharge Claim, +visit_occurrence,visit_concept_id,32360,Skilled Nursing-Inpatient (Medicare Part B Only)@Interim-Cont claim (b), +visit_occurrence,visit_concept_id,32359,Skilled Nursing-Inpatient (Medicare Part B Only)@Interim-first claim, +visit_occurrence,visit_concept_id,32361,Skilled Nursing-Inpatient (Medicare Part B Only)@Interim-last claim (b), +visit_occurrence,visit_concept_id,32364,Skilled Nursing-Inpatient (Medicare Part B Only)@Intermediary Adjustment Claim (Other than QIO or Provider), +visit_occurrence,visit_concept_id,32362,Skilled Nursing-Inpatient (Medicare Part B Only)@Replacement of Prior Claim, +visit_occurrence,visit_concept_id,32363,Skilled Nursing-Inpatient (Medicare Part B Only)@Void/ Cancel of Prior claim (a), +visit_occurrence,visit_concept_id,32257,Skilled Nursing-Outpatient, +visit_occurrence,visit_concept_id,32366,Skilled Nursing-Outpatient@Admit through Discharge Claim, +visit_occurrence,visit_concept_id,32372,Skilled Nursing-Outpatient@CWF Initiated Adjustment Claim, +visit_occurrence,visit_concept_id,32371,Skilled Nursing-Outpatient@Final Claim for a Home Health PPS Episode, +visit_occurrence,visit_concept_id,32368,Skilled Nursing-Outpatient@Interim-Cont claim (b), +visit_occurrence,visit_concept_id,32367,Skilled Nursing-Outpatient@Interim-first claim, +visit_occurrence,visit_concept_id,32369,Skilled Nursing-Outpatient@Interim-last claim (b), +visit_occurrence,visit_concept_id,32373,Skilled Nursing-Outpatient@Intermediary Adjustment Claim (Other than QIO or Provider), +visit_occurrence,visit_concept_id,32365,Skilled Nursing-Outpatient@Non-Payment/Zero, +visit_occurrence,visit_concept_id,32370,Skilled Nursing-Outpatient@Replacement of Prior Claim, +visit_occurrence,visit_concept_id,32374,Skilled Nursing-Outpatient@Void/Cance a Prior Abreviated Encounter Submission, +visit_occurrence,visit_concept_id,32186,Special Facility - Other@Admit through Discharge Claim, +visit_occurrence,visit_concept_id,32397,Special Facility - Other@CWF Initiated Adjustment Claim, +visit_occurrence,visit_concept_id,32188,Special Facility - Other@Interim-Cont claim (b), +visit_occurrence,visit_concept_id,32187,Special Facility - Other@Interim-first claim, +visit_occurrence,visit_concept_id,32189,Special Facility - Other@Interim-last claim (b), +visit_occurrence,visit_concept_id,32398,Special Facility - Other@Intermediary Adjustment Claim (Other than QIO or Provider), +visit_occurrence,visit_concept_id,32190,Special Facility - Other@Late Charges Only, +visit_occurrence,visit_concept_id,32185,Special Facility - Other@Non-Payment/Zero, +visit_occurrence,visit_concept_id,32400,Special Facility - Other@Replacement of Prior Abbreviated Encounter Submission, +visit_occurrence,visit_concept_id,32191,Special Facility - Other@Replacement of Prior Claim, +visit_occurrence,visit_concept_id,32192,Special Facility - Other@Void/ Cancel of Prior claim (a), +visit_occurrence,visit_concept_id,32399,Special Facility - Other@Void/Cance a Prior Abreviated Encounter Submission, +visit_occurrence,visit_concept_id,32277,Special Facility-Other, +visit_occurrence,visit_concept_id,32278,Special Facility-Other, +visit_occurrence,visit_concept_id,32286,Void/ Cancel of Prior claim (a), +visit_occurrence,visit_concept_id,32297,Void/Cance a Prior Abreviated Encounter Submission, +visit_occurrence,visit_concept_id,32290,Void/Cancel Hospice/CMS Coor Care/Non-med Religious/Cent for Excellence, +visit_occurrence,visit_type_concept_id,44818519,Clinical Study visit,S +visit_occurrence,visit_type_concept_id,32034,Visit derived from EHR billing record,S +visit_occurrence,visit_type_concept_id,32035,Visit derived from EHR encounter record,S +visit_occurrence,visit_type_concept_id,44818518,Visit derived from EHR record,S +visit_occurrence,visit_type_concept_id,44818517,Visit derived from encounter on claim,S +visit_occurrence,visit_type_concept_id,32031,Visit derived from encounter on claim authorization,S +visit_occurrence,visit_type_concept_id,32033,Visit derived from encounter on dental claim,S +visit_occurrence,visit_type_concept_id,32021,Visit derived from encounter on medical claim,S +visit_occurrence,visit_type_concept_id,32023,Visit derived from encounter on medical facility claim,S +visit_occurrence,visit_type_concept_id,32027,Visit derived from encounter on medical facility claim deferred,S +visit_occurrence,visit_type_concept_id,32026,Visit derived from encounter on medical facility claim denied,S +visit_occurrence,visit_type_concept_id,32025,Visit derived from encounter on medical facility claim paid,S +visit_occurrence,visit_type_concept_id,32024,Visit derived from encounter on medical professional claim,S +visit_occurrence,visit_type_concept_id,32030,Visit derived from encounter on medical professional claim deferred,S +visit_occurrence,visit_type_concept_id,32029,Visit derived from encounter on medical professional claim denied,S +visit_occurrence,visit_type_concept_id,32028,Visit derived from encounter on medical professional claim paid,S +visit_occurrence,visit_type_concept_id,32022,Visit derived from encounter on pharmacy claim,S +visit_occurrence,visit_type_concept_id,32032,Visit derived from encounter on vision claim,S diff --git a/rabbitinahat/src/main/resources/org/ohdsi/rabbitInAHat/dataModel/CDMV5.2.0.csv b/rabbitinahat/src/main/resources/org/ohdsi/rabbitInAHat/dataModel/CDMV5.2.0.csv index 685ccdd4..afb5aeae 100644 --- a/rabbitinahat/src/main/resources/org/ohdsi/rabbitInAHat/dataModel/CDMV5.2.0.csv +++ b/rabbitinahat/src/main/resources/org/ohdsi/rabbitInAHat/dataModel/CDMV5.2.0.csv @@ -183,7 +183,7 @@ SPECIMEN,ANATOMIC_SITE_SOURCE_VALUE,YES,CHARACTER VARYING,"The information about SPECIMEN,DISEASE_STATUS_SOURCE_VALUE,YES,CHARACTER VARYING,"The information about the disease status as detailed in the source." VISIT_DETAIL,VISIT_DETAIL_ID,NO,INTEGER,"A unique identifier for each Person's visit or encounter at a healthcare provider." VISIT_DETAIL,PERSON_ID,NO,INTEGER,"A foreign key identifier to the Person for whom the visit is recorded. The demographic details of that Person are stored in the PERSON table." -VISIT_DETAIL,VISIT_CONCEPT_ID,NO,INTEGER,"A foreign key that refers to a visit Concept identifier in the Standardized Vocabularies." +VISIT_DETAIL,VISIT_DETAIL_CONCEPT_ID,NO,INTEGER,"A foreign key that refers to a visit Concept identifier in the Standardized Vocabularies." VISIT_DETAIL,VISIT_START_DATE,NO,DATE,"The start date of the visit." VISIT_DETAIL,VISIT_START_DATETIME,YES,DATETIME,"The date and time of the visit started." VISIT_DETAIL,VISIT_END_DATE,NO,DATE,"The end date of the visit. If this is a one-day visit the end date should match the start date." diff --git a/rabbitinahat/src/main/resources/org/ohdsi/rabbitInAHat/dataModel/CDMV5.3.0.csv b/rabbitinahat/src/main/resources/org/ohdsi/rabbitInAHat/dataModel/CDMV5.3.0.csv index 838a2295..03ab97cf 100644 --- a/rabbitinahat/src/main/resources/org/ohdsi/rabbitInAHat/dataModel/CDMV5.3.0.csv +++ b/rabbitinahat/src/main/resources/org/ohdsi/rabbitInAHat/dataModel/CDMV5.3.0.csv @@ -110,8 +110,8 @@ "nlp_date","Yes","DATE","The date of the note processing.Useful for data provenance.","note_nlp" "nlp_date_time","No","DATETIME","The date and time of the note processing. Useful for data provenance.","note_nlp" "term_exists","No","VARCHAR(1)","A summary modifier that signifies presence or absence of the term for a given patient. Useful for quick querying. *","note_nlp" -"term_temporal","No","VARCHAR(50)","An optional time modifier associated with the extracted term. (for now “past” or “present” only). Standardize it later.","note_nlp" -"term_modifiers","No","VARCHAR(2000)","A compact description of all the modifiers of the specific term extracted by the NLP system. (e.g. “son has rash” ? “negated=no,subject=family, certainty=undef,conditional=false,general=false”).","note_nlp" +"term_temporal","No","VARCHAR(50)","An optional time modifier associated with the extracted term. (for now �past� or �present� only). Standardize it later.","note_nlp" +"term_modifiers","No","VARCHAR(2000)","A compact description of all the modifiers of the specific term extracted by the NLP system. (e.g. �son has rash� ? �negated=no,subject=family, certainty=undef,conditional=false,general=false�).","note_nlp" "observation_id","Yes","INTEGER","A unique identifier for each observation.","observation" "person_id","Yes","INTEGER","A foreign key identifier to the Person about whom the observation was recorded. The demographic details of that Person are stored in the PERSON table.","observation" "observation_concept_id","Yes","INTEGER","A foreign key to the standard observation concept identifier in the Standardized Vocabularies.","observation" @@ -184,7 +184,7 @@ "disease_status_source_value","No","VARCHAR(50)","The information about the disease status as detailed in the source.","specimen" "visit_detail_id","Yes","INTEGER","A unique identifier for each Person's visit or encounter at a healthcare provider.","visit_detail" "person_id","Yes","INTEGER","A foreign key identifier to the Person for whom the visit is recorded. The demographic details of that Person are stored in the PERSON table.","visit_detail" -"visit_concept_id","Yes","INTEGER","A foreign key that refers to a visit Concept identifier in the Standardized Vocabularies.","visit_detail" +"visit_detail_concept_id","Yes","INTEGER","A foreign key that refers to a visit Concept identifier in the Standardized Vocabularies.","visit_detail" "visit_start_date","Yes","DATE","The start date of the visit.","visit_detail" "visit_start_datetime","No","DATETIME","The date and time of the visit started.","visit_detail" "visit_end_date","Yes","DATE","The end date of the visit. If this is a one-day visit the end date should match the start date.","visit_detail" diff --git a/rabbitinahat/src/main/resources/org/ohdsi/rabbitInAHat/dataModel/CDMV5.3.1.csv b/rabbitinahat/src/main/resources/org/ohdsi/rabbitInAHat/dataModel/CDMV5.3.1.csv index bc05babb..a6af87f0 100644 --- a/rabbitinahat/src/main/resources/org/ohdsi/rabbitInAHat/dataModel/CDMV5.3.1.csv +++ b/rabbitinahat/src/main/resources/org/ohdsi/rabbitInAHat/dataModel/CDMV5.3.1.csv @@ -185,16 +185,16 @@ anatomic_site_source_value,No,VARCHAR(50),The information about the anatomic sit disease_status_source_value,No,VARCHAR(50),The information about the disease status as detailed in the source.,specimen,cdm visit_detail_id,Yes,INTEGER,A unique identifier for each Person's visit or encounter at a healthcare provider.,visit_detail,cdm person_id,Yes,INTEGER,A foreign key identifier to the Person for whom the visit is recorded. The demographic details of that Person are stored in the PERSON table.,visit_detail,cdm -visit_concept_id,Yes,INTEGER,A foreign key that refers to a visit Concept identifier in the Standardized Vocabularies.,visit_detail,cdm -visit_start_date,Yes,DATE,The start date of the visit.,visit_detail,cdm -visit_start_datetime,No,DATETIME,The date and time of the visit started.,visit_detail,cdm -visit_end_date,Yes,DATE,The end date of the visit. If this is a one-day visit the end date should match the start date.,visit_detail,cdm -visit_end_datetime,No,DATETIME,The date and time of the visit end.,visit_detail,cdm -visit_type_concept_id,Yes,INTEGER,A foreign key to the predefined Concept identifier in the Standardized Vocabularies reflecting the type of source data from which the visit record is derived.,visit_detail,cdm +visit_detail_concept_id,Yes,INTEGER,A foreign key that refers to a visit Concept identifier in the Standardized Vocabularies.,visit_detail,cdm +visit_detail_start_date,Yes,DATE,The start date of the visit.,visit_detail,cdm +visit_detail_start_datetime,No,DATETIME,The date and time of the visit started.,visit_detail,cdm +visit_detail_end_date,Yes,DATE,The end date of the visit. If this is a one-day visit the end date should match the start date.,visit_detail,cdm +visit_detail_end_datetime,No,DATETIME,The date and time of the visit end.,visit_detail,cdm +visit_detail_type_concept_id,Yes,INTEGER,A foreign key to the predefined Concept identifier in the Standardized Vocabularies reflecting the type of source data from which the visit record is derived.,visit_detail,cdm provider_id,No,INTEGER,A foreign key to the provider in the provider table who was associated with the visit.,visit_detail,cdm care_site_id,No,INTEGER,A foreign key to the care site in the care site table that was visited.,visit_detail,cdm -visit_source_value,No,STRING(50),The source code for the visit as it appears in the source data.,visit_detail,cdm -visit_source_concept_id,No,INTEGER,A foreign key to a Concept that refers to the code used in the source.,visit_detail,cdm +visit_detail_source_value,No,STRING(50),The source code for the visit as it appears in the source data.,visit_detail,cdm +visit_detail_source_concept_id,No,INTEGER,A foreign key to a Concept that refers to the code used in the source.,visit_detail,cdm admitting_source_value,No,VARCHAR(50),The source code for the admitting source as it appears in the source data.,visit_detail,cdm admitting_source_concept_id,No,INTEGER,A foreign key to the predefined concept in the Place of Service Vocabulary reflecting the admitting source for a visit.,visit_detail,cdm discharge_to_source_value,No,VARCHAR(50),The source code for the discharge disposition as it appears in the source data.,visit_detail,cdm @@ -284,7 +284,7 @@ plan_source_value,No,VARCHAR(50),The source code for the Person's health benefit plan_source_concept_id,No,INTEGER,A foreign key to a plan concept that refers to the code used in the source.,payer_plan_period,cdm sponsor_concept_id,No,INTEGER,A foreign key that refers to a Standard plan that represents the sponsor in the Standardized Vocabularies,payer_plan_period,cdm sponsor_source_value,No,VARCHAR(50),The source code for the Person's sponsor of the health plan as it appears in the source data.,payer_plan_period,cdm -sponsor_source_concept_id*,No,INTEGER,A foreign key to a sponsor concept that refers to the code used in the source.,payer_plan_period,cdm +sponsor_source_concept_id,No,INTEGER,A foreign key to a sponsor concept that refers to the code used in the source.,payer_plan_period,cdm family_source_value,No,VARCHAR(50),The source code for the Person's family as it appears in the source data.,payer_plan_period,cdm stop_reason_concept_id,No,INTEGER,A foreign key that refers to a Standard termination reason that represents the reason for the termination in the Standardized Vocabularies.,payer_plan_period,cdm stop_reason_source_value,No,VARCHAR(50),The reason for stop-coverage of the record.,payer_plan_period,cdm diff --git a/rabbitinahat/src/main/resources/org/ohdsi/rabbitInAHat/dataModel/CDMV6.0.csv b/rabbitinahat/src/main/resources/org/ohdsi/rabbitInAHat/dataModel/CDMV6.0.csv index ae737e7b..2d5d02d6 100644 --- a/rabbitinahat/src/main/resources/org/ohdsi/rabbitInAHat/dataModel/CDMV6.0.csv +++ b/rabbitinahat/src/main/resources/org/ohdsi/rabbitInAHat/dataModel/CDMV6.0.csv @@ -1,4 +1,45 @@ field,required,type,description,table,schema +person_id,Yes,INTEGER,A unique identifier for each person.,person,cdm +gender_concept_id,Yes,INTEGER,A foreign key that refers to an identifier in the CONCEPT table for the unique gender of the person.,person,cdm +year_of_birth,Yes,INTEGER,"The year of birth of the person. For data sources with date of birth, the year is extracted. For data sources where the year of birth is not available, the approximate year of birth is derived based on any age group categorization available.",person,cdm +month_of_birth,No,INTEGER,"The month of birth of the person. For data sources that provide the precise date of birth, the month is extracted and stored in this field.",person,cdm +day_of_birth,No,INTEGER,"The day of the month of birth of the person. For data sources that provide the precise date of birth, the day is extracted and stored in this field.",person,cdm +birth_datetime,No,DATETIME,The date and time of birth of the person.,person,cdm +death_datetime,No,DATETIME,The date and time of death of the person.,person,cdm +race_concept_id,Yes,INTEGER,A foreign key that refers to an identifier in the CONCEPT table for the unique race of the person.,person,cdm +ethnicity_concept_id,Yes,INTEGER,A foreign key that refers to the standard concept identifier in the Standardized Vocabularies for the ethnicity of the person.,person,cdm +location_id,No,INTEGER,"A foreign key to the place of residency for the person in the location table, where the detailed address information is stored.",person,cdm +provider_id,No,INTEGER,A foreign key to the primary care provider the person is seeing in the provider table.,person,cdm +care_site_id,No,INTEGER,"A foreign key to the site of primary care in the care_site table, where the details of the care site are stored.",person,cdm +person_source_value,No,VARCHAR(50),An (encrypted) key derived from the person identifier in the source data. This is necessary when a use case requires a link back to the person data at the source dataset.,person,cdm +gender_source_value,No,VARCHAR(50),The source code for the gender of the person as it appears in the source data. The person’s gender is mapped to a standard gender concept in the Standardized Vocabularies; the original value is stored here for reference.,person,cdm +gender_source_concept_id,No,INTEGER,A foreign key to the gender concept that refers to the code used in the source.,person,cdm +race_source_value,No,VARCHAR(50),The source code for the race of the person as it appears in the source data. The person race is mapped to a standard race concept in the Standardized Vocabularies and the original value is stored here for reference.,person,cdm +race_source_concept_id,No,INTEGER,A foreign key to the race concept that refers to the code used in the source.,person,cdm +ethnicity_source_value,No,VARCHAR(50),"The source code for the ethnicity of the person as it appears in the source data. The person ethnicity is mapped to a standard ethnicity concept in the Standardized Vocabularies and the original code is, stored here for reference.",person,cdm +ethnicity_source_concept_id,No,INTEGER,A foreign key to the ethnicity concept that refers to the code used in the source.,person,cdm +observation_period_id,Yes,INTEGER,A unique identifier for each observation period.,observation_period,cdm +person_id,Yes,INTEGER,A foreign key identifier to the person for whom the observation period is defined. The demographic details of that person are stored in the person table.,observation_period,cdm +observation_period_start_date,Yes,DATE,The start date of the observation period for which data are available from the data source.,observation_period,cdm +observation_period_end_date,Yes,DATE,The end date of the observation period for which data are available from the data source.,observation_period,cdm +period_type_concept_id,Yes,INTEGER,A foreign key identifier to the predefined concept in the Standardized Vocabularies reflecting the source of the observation period information,observation_period,cdm +visit_occurrence_id,Yes,INTEGER,A unique identifier for each Person's visit or encounter at a healthcare provider.,visit_occurrence,cdm +person_id,Yes,INTEGER,A foreign key identifier to the Person for whom the visit is recorded. The demographic details of that Person are stored in the PERSON table.,visit_occurrence,cdm +visit_concept_id,Yes,INTEGER,A foreign key that refers to a visit Concept identifier in the Standardized Vocabularies.,visit_occurrence,cdm +visit_start_date,Yes,DATE,The start date of the visit.,visit_occurrence,cdm +visit_start_datetime,No,DATETIME,The date and time of the visit started.,visit_occurrence,cdm +visit_end_date,Yes,DATE,The end date of the visit. If this is a one-day visit the end date should match the start date.,visit_occurrence,cdm +visit_end_datetime,No,DATETIME,The date and time of the visit end.,visit_occurrence,cdm +visit_type_concept_id,Yes,INTEGER,A foreign key to the predefined Concept identifier in the Standardized Vocabularies reflecting the type of source data from which the visit record is derived.,visit_occurrence,cdm +provider_id,No,INTEGER,A foreign key to the provider in the provider table who was associated with the visit.,visit_occurrence,cdm +care_site_id,No,INTEGER,A foreign key to the care site in the care site table that was visited.,visit_occurrence,cdm +visit_source_value,No,VARCHAR(50),The source code for the visit as it appears in the source data.,visit_occurrence,cdm +visit_source_concept_id,No,INTEGER,A foreign key to a Concept that refers to the code used in the source.,visit_occurrence,cdm +admitted_from_concept_id,No,INTEGER,A foreign key to the predefined concept in the Place of Service Vocabulary reflecting the admitting source for a visit.,visit_occurrence,cdm +admitted_from_source_value,No,VARCHAR(50),The source code for the admitting source as it appears in the source data.,visit_occurrence,cdm +discharge_to_concept_id,No,INTEGER,A foreign key to the predefined concept in the Place of Service Vocabulary reflecting the discharge disposition for a visit.,visit_occurrence,cdm +discharge_to_source_value,No,VARCHAR(50),The source code for the discharge disposition as it appears in the source data.,visit_occurrence,cdm +preceding_visit_occurrence_id,No,INTEGER,A foreign key to the VISIT_OCCURRENCE table of the visit immediately preceding this visit,visit_occurrence,cdm condition_occurrence_id,Yes,INTEGER,A unique identifier for each Condition Occurrence event.,condition_occurrence,cdm person_id,Yes,INTEGER,A foreign key identifier to the Person who is experiencing the condition. The demographic details of that Person are stored in the PERSON table.,condition_occurrence,cdm condition_concept_id,Yes,INTEGER,A foreign key that refers to a Standard Condition Concept identifier in the Standardized Vocabularies.,condition_occurrence,cdm @@ -129,30 +170,6 @@ qualifier_source_value,No,VARCHAR(50),The source value associated with a qualifi observation_event_id,No,INTEGER,"A foreign key to an event table (e.g., PROCEDURE_OCCURRENCE_ID).",observation,cdm obs_event_field_concept_id,No,INTEGER,A foreign key that refers to a Standard Concept identifier in the Standardized Vocabularies referring to the field represented in the OBSERVATION_EVENT_ID.,observation,cdm value_as_datetime,No,DATETIME,The observation result stored as a datetime value. This is applicable to observations where the result is expressed as a point in time.,observation,cdm -observation_period_id,Yes,INTEGER,A unique identifier for each observation period.,observation_period,cdm -person_id,Yes,INTEGER,A foreign key identifier to the person for whom the observation period is defined. The demographic details of that person are stored in the person table.,observation_period,cdm -observation_period_start_date,Yes,DATE,The start date of the observation period for which data are available from the data source.,observation_period,cdm -observation_period_end_date,Yes,DATE,The end date of the observation period for which data are available from the data source.,observation_period,cdm -period_type_concept_id,Yes,INTEGER,A foreign key identifier to the predefined concept in the Standardized Vocabularies reflecting the source of the observation period information,observation_period,cdm -person_id,Yes,INTEGER,A unique identifier for each person.,person,cdm -gender_concept_id,Yes,INTEGER,A foreign key that refers to an identifier in the CONCEPT table for the unique gender of the person.,person,cdm -year_of_birth,Yes,INTEGER,"The year of birth of the person. For data sources with date of birth, the year is extracted. For data sources where the year of birth is not available, the approximate year of birth is derived based on any age group categorization available.",person,cdm -month_of_birth,No,INTEGER,"The month of birth of the person. For data sources that provide the precise date of birth, the month is extracted and stored in this field.",person,cdm -day_of_birth,No,INTEGER,"The day of the month of birth of the person. For data sources that provide the precise date of birth, the day is extracted and stored in this field.",person,cdm -birth_datetime,No,DATETIME,The date and time of birth of the person.,person,cdm -death_datetime,No,DATETIME,The date and time of death of the person.,person,cdm -race_concept_id,Yes,INTEGER,A foreign key that refers to an identifier in the CONCEPT table for the unique race of the person.,person,cdm -ethnicity_concept_id,Yes,INTEGER,A foreign key that refers to the standard concept identifier in the Standardized Vocabularies for the ethnicity of the person.,person,cdm -location_id,No,INTEGER,"A foreign key to the place of residency for the person in the location table, where the detailed address information is stored.",person,cdm -provider_id,No,INTEGER,A foreign key to the primary care provider the person is seeing in the provider table.,person,cdm -care_site_id,No,INTEGER,"A foreign key to the site of primary care in the care_site table, where the details of the care site are stored.",person,cdm -person_source_value,No,VARCHAR(50),An (encrypted) key derived from the person identifier in the source data. This is necessary when a use case requires a link back to the person data at the source dataset.,person,cdm -gender_source_value,No,VARCHAR(50),The source code for the gender of the person as it appears in the source data. The person’s gender is mapped to a standard gender concept in the Standardized Vocabularies; the original value is stored here for reference.,person,cdm -gender_source_concept_id,No,INTEGER,A foreign key to the gender concept that refers to the code used in the source.,person,cdm -race_source_value,No,VARCHAR(50),The source code for the race of the person as it appears in the source data. The person race is mapped to a standard race concept in the Standardized Vocabularies and the original value is stored here for reference.,person,cdm -race_source_concept_id,No,INTEGER,A foreign key to the race concept that refers to the code used in the source.,person,cdm -ethnicity_source_value,No,VARCHAR(50),"The source code for the ethnicity of the person as it appears in the source data. The person ethnicity is mapped to a standard ethnicity concept in the Standardized Vocabularies and the original code is, stored here for reference.",person,cdm -ethnicity_source_concept_id,No,INTEGER,A foreign key to the ethnicity concept that refers to the code used in the source.,person,cdm procedure_occurrence_id,Yes,INTEGER,A system-generated unique identifier for each Procedure Occurrence.,procedure_occurrence,cdm person_id,Yes,INTEGER,A foreign key identifier to the Person who is subjected to the Procedure. The demographic details of that Person are stored in the PERSON table.,procedure_occurrence,cdm procedure_concept_id,Yes,INTEGER,A foreign key that refers to a standard procedure Concept identifier in the Standardized Vocabularies.,procedure_occurrence,cdm @@ -184,16 +201,16 @@ anatomic_site_source_value,No,VARCHAR(50),The information about the anatomic sit disease_status_source_value,No,VARCHAR(50),The information about the disease status as detailed in the source.,specimen,cdm visit_detail_id,Yes,INTEGER,A unique identifier for each Person's visit or encounter at a healthcare provider.,visit_detail,cdm person_id,Yes,INTEGER,A foreign key identifier to the Person for whom the visit is recorded. The demographic details of that Person are stored in the PERSON table.,visit_detail,cdm -visit_concept_id,Yes,INTEGER,A foreign key that refers to a visit Concept identifier in the Standardized Vocabularies.,visit_detail,cdm -visit_start_date,Yes,DATE,The start date of the visit.,visit_detail,cdm -visit_start_datetime,No,DATETIME,The date and time of the visit started.,visit_detail,cdm -visit_end_date,Yes,DATE,The end date of the visit. If this is a one-day visit the end date should match the start date.,visit_detail,cdm -visit_end_datetime,No,DATETIME,The date and time of the visit end.,visit_detail,cdm -visit_type_concept_id,Yes,INTEGER,A foreign key to the predefined Concept identifier in the Standardized Vocabularies reflecting the type of source data from which the visit record is derived.,visit_detail,cdm +visit_detail_concept_id,Yes,INTEGER,A foreign key that refers to a visit Concept identifier in the Standardized Vocabularies.,visit_detail,cdm +visit_detail_start_date,Yes,DATE,The start date of the visit.,visit_detail,cdm +visit_detail_start_datetime,No,DATETIME,The date and time of the visit started.,visit_detail,cdm +visit_detail_end_date,Yes,DATE,The end date of the visit. If this is a one-day visit the end date should match the start date.,visit_detail,cdm +visit_detail_end_datetime,No,DATETIME,The date and time of the visit end.,visit_detail,cdm +visit_detail_type_concept_id,Yes,INTEGER,A foreign key to the predefined Concept identifier in the Standardized Vocabularies reflecting the type of source data from which the visit record is derived.,visit_detail,cdm provider_id,No,INTEGER,A foreign key to the provider in the provider table who was associated with the visit.,visit_detail,cdm care_site_id,No,INTEGER,A foreign key to the care site in the care site table that was visited.,visit_detail,cdm -visit_source_value,No,STRING(50),The source code for the visit as it appears in the source data.,visit_detail,cdm -visit_source_concept_id,No,INTEGER,A foreign key to a Concept that refers to the code used in the source.,visit_detail,cdm +visit_detail_source_value,No,STRING(50),The source code for the visit as it appears in the source data.,visit_detail,cdm +visit_detail_source_concept_id,No,INTEGER,A foreign key to a Concept that refers to the code used in the source.,visit_detail,cdm admitting_source_value,No,VARCHAR(50),The source code for the admitting source as it appears in the source data.,visit_detail,cdm admitting_source_concept_id,No,INTEGER,A foreign key to the predefined concept in the Place of Service Vocabulary reflecting the admitting source for a visit.,visit_detail,cdm admitted_from_source_value,No,VARCHAR(50),The source code for the discharge disposition as it appears in the source data.,visit_detail,cdm @@ -201,23 +218,6 @@ admitted_from_concept_id,No,INTEGER,A foreign key to the predefined concept in t preceding_visit_detail_id,No,INTEGER,A foreign key to the VISIT_DETAIL table of the visit immediately preceding this visit,visit_detail,cdm visit_detail_parent_id,No,INTEGER,A foreign key to the VISIT_DETAIL table record to represent the immediate parent visit-detail record.,visit_detail,cdm visit_occurrence_id,Yes,INTEGER,"A foreign key that refers to the record in the VISIT_OCCURRENCE table. This is a required field, because for every visit_detail is a child of visit_occurrence and cannot exist without a corresponding parent record in visit_occurrence.",visit_detail,cdm -visit_occurrence_id,Yes,INTEGER,A unique identifier for each Person's visit or encounter at a healthcare provider.,visit_occurrence,cdm -person_id,Yes,INTEGER,A foreign key identifier to the Person for whom the visit is recorded. The demographic details of that Person are stored in the PERSON table.,visit_occurrence,cdm -visit_concept_id,Yes,INTEGER,A foreign key that refers to a visit Concept identifier in the Standardized Vocabularies.,visit_occurrence,cdm -visit_start_date,Yes,DATE,The start date of the visit.,visit_occurrence,cdm -visit_start_datetime,No,DATETIME,The date and time of the visit started.,visit_occurrence,cdm -visit_end_date,Yes,DATE,The end date of the visit. If this is a one-day visit the end date should match the start date.,visit_occurrence,cdm -visit_end_datetime,No,DATETIME,The date and time of the visit end.,visit_occurrence,cdm -visit_type_concept_id,Yes,INTEGER,A foreign key to the predefined Concept identifier in the Standardized Vocabularies reflecting the type of source data from which the visit record is derived.,visit_occurrence,cdm -provider_id,No,INTEGER,A foreign key to the provider in the provider table who was associated with the visit.,visit_occurrence,cdm -care_site_id,No,INTEGER,A foreign key to the care site in the care site table that was visited.,visit_occurrence,cdm -visit_source_value,No,VARCHAR(50),The source code for the visit as it appears in the source data.,visit_occurrence,cdm -visit_source_concept_id,No,INTEGER,A foreign key to a Concept that refers to the code used in the source.,visit_occurrence,cdm -admitted_from_concept_id,No,INTEGER,A foreign key to the predefined concept in the Place of Service Vocabulary reflecting the admitting source for a visit.,visit_occurrence,cdm -admitted_from_source_value,No,VARCHAR(50),The source code for the admitting source as it appears in the source data.,visit_occurrence,cdm -discharge_to_concept_id,No,INTEGER,A foreign key to the predefined concept in the Place of Service Vocabulary reflecting the discharge disposition for a visit.,visit_occurrence,cdm -discharge_to_source_value,No,VARCHAR(50),The source code for the discharge disposition as it appears in the source data.,visit_occurrence,cdm -preceding_visit_occurrence_id,No,INTEGER,A foreign key to the VISIT_OCCURRENCE table of the visit immediately preceding this visit,visit_occurrence,cdm cohort_definition_id,Yes,INTEGER,A foreign key to a record in the COHORT_DEFINITION table containing relevant Cohort Definition information.,cohort,results subject_id,Yes,INTEGER,"A foreign key to the subject in the cohort. These could be referring to records in the PERSON, PROVIDER, VISIT_OCCURRENCE table.",cohort,results cohort_start_date,Yes,DATE,"The date when the Cohort Definition criteria for the Person, Provider or Visit first match.",cohort,results @@ -271,12 +271,12 @@ payer_source_concept_id,No,INTEGER,A foreign key to a payer concept that refers plan_concept_id,No,INTEGER,A foreign key that refers to a Standard plan that represents the health benefit plan in the Standardized Vocabularies,payer_plan_period,cdm plan_source_value,No,VARCHAR(50),The source code for the Person's health benefit plan as it appears in the source data.,payer_plan_period,cdm plan_source_concept_id,No,INTEGER,A foreign key to a plan concept that refers to the code used in the source.,payer_plan_period,cdm -contract_person_id,No,INTEGER,"A foreign key identifier to the person_id in person table, for the person who is the primary subscriber/contract owner for the record in the payer_plan_period table. Maybe the same person or different person, depending on who is the primary subscriber/contract owner.",payer_plan_period,cdm -contract_person_id,No,INTEGER,"A foreign key identifier to the person_id in person table, for the person who is the primary subscriber/contract owner for the record in the payer_plan_period table. Maybe the same person or different person, depending on who is the primary subscriber/contract owner.",payer_plan_period,cdm -contract_person_id,No,INTEGER,"A foreign key identifier to the person_id in person table, for the person who is the primary subscriber/contract owner for the record in the payer_plan_period table. Maybe the same person or different person, depending on who is the primary subscriber/contract owner.",payer_plan_period,cdm +contract_concept_id,Yes,INTEGER,A foreign key to a standard concept representing the reason justifying the contract between person_id and contract_person_id.,payer_plan_period,cdm +contract_source_value,No,INTEGER,"The source code representing the reason justifying the contract. Usually it is family relationship like a spouse, domestic partner, child etc.",payer_plan_period,cdm +contract_source_concept_id,Yes,INTEGER,A foreign key to a concept that refers to the code used in the source as the reason justifying the contract.,payer_plan_period,cdm sponsor_concept_id,No,INTEGER,A foreign key that refers to a Standard plan that represents the sponsor in the Standardized Vocabularies,payer_plan_period,cdm sponsor_source_value,No,VARCHAR(50),The source code for the Person's sponsor of the health plan as it appears in the source data.,payer_plan_period,cdm -sponsor_source_concept_id*,No,INTEGER,A foreign key to a sponsor concept that refers to the code used in the source.,payer_plan_period,cdm +sponsor_source_concept_id,No,INTEGER,A foreign key to a sponsor concept that refers to the code used in the source.,payer_plan_period,cdm family_source_value,No,VARCHAR(50),The source code for the Person's family as it appears in the source data.,payer_plan_period,cdm stop_reason_concept_id,No,INTEGER,A foreign key that refers to a Standard termination reason that represents the reason for the termination in the Standardized Vocabularies.,payer_plan_period,cdm stop_reason_source_value,No,VARCHAR(50),The reason for stop-coverage of the record.,payer_plan_period,cdm @@ -342,12 +342,12 @@ survey_start_date,No,DATE,Date on which the survey was started.,survey_conduct,c survey_start_datetime,No,DATETIME,Date and time the survey was started.,survey_conduct,cdm survey_end_date,No,DATE,Date on which the survey was completed.,survey_conduct,cdm survey_end_datetime,Yes,DATETIME,Date and time the survey was completed.,survey_conduct,cdm -provider_id,No ,INTEGER ,A foreign key to the provider in the provider table who was associated with the survey completion.,survey_conduct,cdm +provider_id,No,INTEGER,A foreign key to the provider in the provider table who was associated with the survey completion.,survey_conduct,cdm assisted_concept_id,Yes,INTEGER,A foreign key to the predefined Concept identifier in the Standardized Vocabularies indicating whether the survey was completed with assistance.,survey_conduct,cdm respondent_type_concept_id,Yes,INTEGER,"A foreign key to the predefined Concept identifier in the Standardized Vocabularies reflecting the respondent type. Example: Research Associate, Patient.",survey_conduct,cdm timing_concept_id,Yes,INTEGER,"A foreign key to the predefined Concept identifier in the Standardized Vocabularies that refers to a certain timing. Example: 3 month follow-up, 6 month follow-up.",survey_conduct,cdm collection_method_concept_id,Yes,INTEGER,"A foreign key to the predefined Concept identifier in the Standardized Vocabularies reflecting the data collection method (e.g. Paper, Telephone, Electronic Questionnaire).",survey_conduct,cdm -assisted_source_value,No,VARCHAR(50),"Source value representing whether patient required assistance to complete the survey. Example: “Completed without assistance”, ”Completed with assistance”.",survey_conduct,cdm +assisted_source_value,No,VARCHAR(50),"Source value representing whether patient required assistance to complete the survey. Example: ""Completed without assistance"", ""Completed with assistance"".",survey_conduct,cdm respondent_type_source_value,No,VARCHAR(100),Source code representing role of person who completed the survey.,survey_conduct,cdm timing_source_value,No,VARCHAR(100),"Text string representing the timing of the survey. Example: Baseline, 6-month follow-up.",survey_conduct,cdm collection_method_source_value,No,VARCHAR(100),The collection method as it appears in the source data.,survey_conduct,cdm @@ -357,11 +357,11 @@ survey_source_identifier,No,VARCHAR(100),Unique identifier for each completed su validated_survey_concept_id,Yes,INTEGER,A foreign key to the predefined Concept identifier in the Standardized Vocabularies reflecting the validation status of the survey.,survey_conduct,cdm validated_survey_source_value,No,INTEGER,Source value representing the validation status of the survey.,survey_conduct,cdm survey_version_number,No,VARCHAR(20),Version number of the questionnaire or survey used.,survey_conduct,cdm -visit_occurrence_id,No,INTEGER,A foreign key to the VISIT_OCCURRENCE table during which the survey was completed,survey_conduct,cdm -response_visit_occurrence_id,No,INTEGER ,A foreign key to the visit in the VISIT_OCCURRENCE table during which treatment was carried out that relates to this survey.,survey_conduct,cdm -location_id,Yes,INTEGER ,A foreign key to the location table.,location_history,cdm +visit_occurrence_id,No,INTEGER,A foreign key to the VISIT_OCCURRENCE table during which the survey was completed,survey_conduct,cdm +response_visit_occurrence_id,No,INTEGER,A foreign key to the visit in the VISIT_OCCURRENCE table during which treatment was carried out that relates to this survey.,survey_conduct,cdm +location_id,Yes,INTEGER,A foreign key to the location table.,location_history,cdm relationship_type_concept_id,Yes,VARCHAR(50),The type of relationship between location and entity.,location_history,cdm domain_id,Yes,VARCHAR(50),"The domain of the entity that is related to the location. Either PERSON, PROVIDER, or CARE_SITE.",location_history,cdm -entity_id,Yes,INTEGER ,"The unique identifier for the entity. References either person_id, provider_id, or care_site_id, depending on domain_id.",location_history,cdm +entity_id,Yes,INTEGER,"The unique identifier for the entity. References either person_id, provider_id, or care_site_id, depending on domain_id.",location_history,cdm start_date,Yes,DATE,The date the relationship started.,location_history,cdm -end_date,No,DATE,The date the relationship ended.,location_history,cdm +end_date,No,DATE,The date the relationship ended.,location_history,cdm \ No newline at end of file diff --git a/rabbitinahat/src/main/resources/org/ohdsi/rabbitInAHat/dataModel/StemTableDefaultMappingV5.0.1.csv b/rabbitinahat/src/main/resources/org/ohdsi/rabbitInAHat/dataModel/StemTableDefaultMappingV5.0.1.csv index 3c1110cf..ed08e1fe 100644 --- a/rabbitinahat/src/main/resources/org/ohdsi/rabbitInAHat/dataModel/StemTableDefaultMappingV5.0.1.csv +++ b/rabbitinahat/src/main/resources/org/ohdsi/rabbitInAHat/dataModel/StemTableDefaultMappingV5.0.1.csv @@ -57,7 +57,7 @@ STEM_TABLE,RANGE_LOW,MEASUREMENT,RANGE_LOW STEM_TABLE,RANGE_HIGH,MEASUREMENT,RANGE_HIGH STEM_TABLE,PROVIDER_ID,MEASUREMENT,PROVIDER_ID STEM_TABLE,VISIT_OCCURRENCE_ID,MEASUREMENT,VISIT_OCCURRENCE_ID -STEM_TABLE,SOURCE_VALUE,MEASUREMENT,MEAUSREMENT_SOURCE_VALUE +STEM_TABLE,SOURCE_VALUE,MEASUREMENT,MEASUREMENT_SOURCE_VALUE STEM_TABLE,SOURCE_CONCEPT_ID,MEASUREMENT,MEASUREMENT_SOURCE_CONCEPT_ID STEM_TABLE,UNIT_SOURCE_VALUE,MEASUREMENT,UNIT_SOURCE_VALUE STEM_TABLE,VALUE_SOURCE_VALUE,MEASUREMENT,VALUE_SOURCE_VALUE diff --git a/rabbitinahat/src/main/resources/org/ohdsi/rabbitInAHat/dataModel/StemTableDefaultMappingV5.1.0.csv b/rabbitinahat/src/main/resources/org/ohdsi/rabbitInAHat/dataModel/StemTableDefaultMappingV5.1.0.csv index e82056f8..4d682dea 100644 --- a/rabbitinahat/src/main/resources/org/ohdsi/rabbitInAHat/dataModel/StemTableDefaultMappingV5.1.0.csv +++ b/rabbitinahat/src/main/resources/org/ohdsi/rabbitInAHat/dataModel/StemTableDefaultMappingV5.1.0.csv @@ -63,7 +63,7 @@ STEM_TABLE,RANGE_LOW,MEASUREMENT,RANGE_LOW STEM_TABLE,RANGE_HIGH,MEASUREMENT,RANGE_HIGH STEM_TABLE,PROVIDER_ID,MEASUREMENT,PROVIDER_ID STEM_TABLE,VISIT_OCCURRENCE_ID,MEASUREMENT,VISIT_OCCURRENCE_ID -STEM_TABLE,SOURCE_VALUE,MEASUREMENT,MEAUSREMENT_SOURCE_VALUE +STEM_TABLE,SOURCE_VALUE,MEASUREMENT,MEASUREMENT_SOURCE_VALUE STEM_TABLE,SOURCE_CONCEPT_ID,MEASUREMENT,MEASUREMENT_SOURCE_CONCEPT_ID STEM_TABLE,UNIT_SOURCE_VALUE,MEASUREMENT,UNIT_SOURCE_VALUE STEM_TABLE,VALUE_SOURCE_VALUE,MEASUREMENT,VALUE_SOURCE_VALUE diff --git a/rabbitinahat/src/main/resources/org/ohdsi/rabbitInAHat/dataModel/StemTableDefaultMappingV5.2.0.csv b/rabbitinahat/src/main/resources/org/ohdsi/rabbitInAHat/dataModel/StemTableDefaultMappingV5.2.0.csv index f764548a..ebd91f51 100644 --- a/rabbitinahat/src/main/resources/org/ohdsi/rabbitInAHat/dataModel/StemTableDefaultMappingV5.2.0.csv +++ b/rabbitinahat/src/main/resources/org/ohdsi/rabbitInAHat/dataModel/StemTableDefaultMappingV5.2.0.csv @@ -63,7 +63,7 @@ STEM_TABLE,RANGE_LOW,MEASUREMENT,RANGE_LOW STEM_TABLE,RANGE_HIGH,MEASUREMENT,RANGE_HIGH STEM_TABLE,PROVIDER_ID,MEASUREMENT,PROVIDER_ID STEM_TABLE,VISIT_OCCURRENCE_ID,MEASUREMENT,VISIT_OCCURRENCE_ID -STEM_TABLE,SOURCE_VALUE,MEASUREMENT,MEAUSREMENT_SOURCE_VALUE +STEM_TABLE,SOURCE_VALUE,MEASUREMENT,MEASUREMENT_SOURCE_VALUE STEM_TABLE,SOURCE_CONCEPT_ID,MEASUREMENT,MEASUREMENT_SOURCE_CONCEPT_ID STEM_TABLE,UNIT_SOURCE_VALUE,MEASUREMENT,UNIT_SOURCE_VALUE STEM_TABLE,VALUE_SOURCE_VALUE,MEASUREMENT,VALUE_SOURCE_VALUE diff --git a/rabbitinahat/src/main/resources/org/ohdsi/rabbitInAHat/dataModel/StemTableDefaultMappingV5.3.0.csv b/rabbitinahat/src/main/resources/org/ohdsi/rabbitInAHat/dataModel/StemTableDefaultMappingV5.3.0.csv index 83782349..fb5ef707 100644 --- a/rabbitinahat/src/main/resources/org/ohdsi/rabbitInAHat/dataModel/StemTableDefaultMappingV5.3.0.csv +++ b/rabbitinahat/src/main/resources/org/ohdsi/rabbitInAHat/dataModel/StemTableDefaultMappingV5.3.0.csv @@ -63,7 +63,7 @@ STEM_TABLE,RANGE_LOW,MEASUREMENT,RANGE_LOW STEM_TABLE,RANGE_HIGH,MEASUREMENT,RANGE_HIGH STEM_TABLE,PROVIDER_ID,MEASUREMENT,PROVIDER_ID STEM_TABLE,VISIT_OCCURRENCE_ID,MEASUREMENT,VISIT_OCCURRENCE_ID -STEM_TABLE,SOURCE_VALUE,MEASUREMENT,MEAUSREMENT_SOURCE_VALUE +STEM_TABLE,SOURCE_VALUE,MEASUREMENT,MEASUREMENT_SOURCE_VALUE STEM_TABLE,SOURCE_CONCEPT_ID,MEASUREMENT,MEASUREMENT_SOURCE_CONCEPT_ID STEM_TABLE,UNIT_SOURCE_VALUE,MEASUREMENT,UNIT_SOURCE_VALUE STEM_TABLE,VALUE_SOURCE_VALUE,MEASUREMENT,VALUE_SOURCE_VALUE diff --git a/rabbitinahat/src/main/resources/org/ohdsi/rabbitInAHat/dataModel/StemTableDefaultMappingV5.3.1.csv b/rabbitinahat/src/main/resources/org/ohdsi/rabbitInAHat/dataModel/StemTableDefaultMappingV5.3.1.csv index 83782349..fb5ef707 100644 --- a/rabbitinahat/src/main/resources/org/ohdsi/rabbitInAHat/dataModel/StemTableDefaultMappingV5.3.1.csv +++ b/rabbitinahat/src/main/resources/org/ohdsi/rabbitInAHat/dataModel/StemTableDefaultMappingV5.3.1.csv @@ -63,7 +63,7 @@ STEM_TABLE,RANGE_LOW,MEASUREMENT,RANGE_LOW STEM_TABLE,RANGE_HIGH,MEASUREMENT,RANGE_HIGH STEM_TABLE,PROVIDER_ID,MEASUREMENT,PROVIDER_ID STEM_TABLE,VISIT_OCCURRENCE_ID,MEASUREMENT,VISIT_OCCURRENCE_ID -STEM_TABLE,SOURCE_VALUE,MEASUREMENT,MEAUSREMENT_SOURCE_VALUE +STEM_TABLE,SOURCE_VALUE,MEASUREMENT,MEASUREMENT_SOURCE_VALUE STEM_TABLE,SOURCE_CONCEPT_ID,MEASUREMENT,MEASUREMENT_SOURCE_CONCEPT_ID STEM_TABLE,UNIT_SOURCE_VALUE,MEASUREMENT,UNIT_SOURCE_VALUE STEM_TABLE,VALUE_SOURCE_VALUE,MEASUREMENT,VALUE_SOURCE_VALUE diff --git a/rabbitinahat/src/main/resources/org/ohdsi/rabbitInAHat/dataModel/StemTableDefaultMappingV6.0.csv b/rabbitinahat/src/main/resources/org/ohdsi/rabbitInAHat/dataModel/StemTableDefaultMappingV6.0.csv index cbddc588..07a22beb 100644 --- a/rabbitinahat/src/main/resources/org/ohdsi/rabbitInAHat/dataModel/StemTableDefaultMappingV6.0.csv +++ b/rabbitinahat/src/main/resources/org/ohdsi/rabbitInAHat/dataModel/StemTableDefaultMappingV6.0.csv @@ -68,7 +68,7 @@ STEM_TABLE,RANGE_HIGH,MEASUREMENT,RANGE_HIGH STEM_TABLE,PROVIDER_ID,MEASUREMENT,PROVIDER_ID STEM_TABLE,VISIT_OCCURRENCE_ID,MEASUREMENT,VISIT_OCCURRENCE_ID STEM_TABLE,VISIT_DETAIL_ID,MEASUREMENT,VISIT_DETAIL_ID -STEM_TABLE,SOURCE_VALUE,MEASUREMENT,MEAUSREMENT_SOURCE_VALUE +STEM_TABLE,SOURCE_VALUE,MEASUREMENT,MEASUREMENT_SOURCE_VALUE STEM_TABLE,SOURCE_CONCEPT_ID,MEASUREMENT,MEASUREMENT_SOURCE_CONCEPT_ID STEM_TABLE,UNIT_SOURCE_VALUE,MEASUREMENT,UNIT_SOURCE_VALUE STEM_TABLE,VALUE_SOURCE_VALUE,MEASUREMENT,VALUE_SOURCE_VALUE diff --git a/rabbitinahat/src/main/resources/org/ohdsi/rabbitInAHat/dataModel/StemTableV5.3.0.csv b/rabbitinahat/src/main/resources/org/ohdsi/rabbitInAHat/dataModel/StemTableV5.3.0.csv index 5526c3d2..987f7cf5 100644 --- a/rabbitinahat/src/main/resources/org/ohdsi/rabbitInAHat/dataModel/StemTableV5.3.0.csv +++ b/rabbitinahat/src/main/resources/org/ohdsi/rabbitInAHat/dataModel/StemTableV5.3.0.csv @@ -16,9 +16,9 @@ STEM_TABLE,END_DATETIME,YES,DATETIME,The end datetime of the event. STEM_TABLE,DAYS_SUPPLY,YES,INTEGER,The number of days of supply of the medication as recorded in the original prescription or dispensing record. STEM_TABLE,DOSE_UNIT_SOURCE_VALUE,YES,CHARACTER VARYING,The information about the dose unit as detailed in the source. STEM_TABLE,LOT_NUMBER,YES,CHARACTER VARYING,An identifier assigned to a particular quantity or lot of Drug product from the manufacturer. -STEM_TABLE,MODIFIER_CONCEPT_ID,YES,INTEGER,A foreign key to a Standard Concept identifier for a modifier to the Procedure (e.g. bilateral) +STEM_TABLE,MODIFIER_CONCEPT_ID,YES,INTEGER,A foreign key to a Standard Concept identifier for a modifier (e.g. bilateral) +STEM_TABLE,MODIFIER_SOURCE_VALUE,YES,CHARACTER VARYING,The source code for the modifier as it appears in the source data. STEM_TABLE,OPERATOR_CONCEPT_ID,YES,INTEGER,"A foreign key identifier to the predefined Concept in the Standardized Vocabularies reflecting the mathematical operator that is applied to the value_as_number. Operators are <, ?, =, ?, >." -STEM_TABLE,MODIFIER_CONCEPT_ID,YES,INTEGER,"A foreign key to a Standard Concept ID for a modifier (e.g., severity of drug-drug interaction alert)" STEM_TABLE,MODIFIER_SOURCE_VALUE,YES,CHARACTER VARYING,The source code for the modifier as it appears in the source data. STEM_TABLE,QUANTITY,YES,INTEGER,The number of individual Devices/Procedures/Drugs used for the exposure. STEM_TABLE,RANGE_HIGH,YES,FLOAT,The upper limit of the normal range of the Measurement. The upper range is assumed to be of the same unit of measure as the Measurement value. @@ -26,7 +26,7 @@ STEM_TABLE,RANGE_LOW,YES,FLOAT,The lower limit of the normal range of the Measur STEM_TABLE,REFILLS,YES,INTEGER,"The number of refills after the initial prescription. The initial prescription is not counted, values start with 0." STEM_TABLE,ROUTE_CONCEPT_ID,YES,INTEGER,A foreign key to a predefined concept in the Standardized Vocabularies reflecting the route of administration. STEM_TABLE,ROUTE_SOURCE_VALUE,YES,CHARACTER VARYING,The information about the route of administration as detailed in the source. -STEM_TABLE,SIG,YES,CHARACTER VARYING,The directions (“signetur”) on the Drug prescription as recorded in the original prescription (and printed on the container) or dispensing record. +STEM_TABLE,SIG,YES,CHARACTER VARYING,The directions (�signetur�) on the Drug prescription as recorded in the original prescription (and printed on the container) or dispensing record. STEM_TABLE,STOP_REASON,YES,CHARACTER VARYING,"The reason that the condition was no longer present or the drug exposure stopped, as indicated in the source data." STEM_TABLE,UNIQUE_DEVICE_ID,YES,CHARACTER VARYING,A UDI or equivalent identifying the instance of the Device used in the Person. STEM_TABLE,UNIT_CONCEPT_ID,YES,INTEGER,A foreign key to a predefined concept in the Standardized Vocabularies reflecting the unit the effective_drug_dose/measurement/observation/specimen value is expressed. diff --git a/rabbitinahat/src/main/resources/org/ohdsi/rabbitInAHat/dataModel/StemTableV5.3.1.csv b/rabbitinahat/src/main/resources/org/ohdsi/rabbitInAHat/dataModel/StemTableV5.3.1.csv index c4ba2c81..2fa1b793 100644 --- a/rabbitinahat/src/main/resources/org/ohdsi/rabbitInAHat/dataModel/StemTableV5.3.1.csv +++ b/rabbitinahat/src/main/resources/org/ohdsi/rabbitInAHat/dataModel/StemTableV5.3.1.csv @@ -16,9 +16,9 @@ STEM_TABLE,VERBATIM_END_DATE,YES,DATE,"The verbatim end date (not derived) of th STEM_TABLE,DAYS_SUPPLY,YES,INTEGER,"The number of days of supply of the medication as recorded in the original prescription or dispensing record." STEM_TABLE,DOSE_UNIT_SOURCE_VALUE,YES,CHARACTER VARYING,"The information about the dose unit as detailed in the source." STEM_TABLE,LOT_NUMBER,YES,CHARACTER VARYING,"An identifier assigned to a particular quantity or lot of Drug product from the manufacturer." -STEM_TABLE,MODIFIER_CONCEPT_ID,YES,INTEGER,"A foreign key to a Standard Concept identifier for a modifier to the Procedure (e.g. bilateral)" +STEM_TABLE,MODIFIER_CONCEPT_ID,YES,INTEGER,A foreign key to a Standard Concept identifier for a modifier (e.g. bilateral) +STEM_TABLE,MODIFIER_SOURCE_VALUE,YES,CHARACTER VARYING,The source code for the modifier as it appears in the source data. STEM_TABLE,OPERATOR_CONCEPT_ID,YES,INTEGER,"A foreign key identifier to the predefined Concept in the Standardized Vocabularies reflecting the mathematical operator that is applied to the value_as_number. Operators are <, ?, =, ?, >." -STEM_TABLE,MODIFIER_CONCEPT_ID,YES,INTEGER,"A foreign key to a Standard Concept ID for a modifier (e.g., severity of drug-drug interaction alert)" STEM_TABLE,MODIFIER_SOURCE_VALUE,YES,CHARACTER VARYING,"The source code for the modifier as it appears in the source data." STEM_TABLE,QUANTITY,YES,INTEGER,"The number of individual Devices/Procedures/Drugs used for the exposure." STEM_TABLE,RANGE_HIGH,YES,FLOAT,"The upper limit of the normal range of the Measurement. The upper range is assumed to be of the same unit of measure as the Measurement value." @@ -26,7 +26,7 @@ STEM_TABLE,RANGE_LOW,YES,FLOAT,"The lower limit of the normal range of the Measu STEM_TABLE,REFILLS,YES,INTEGER,"The number of refills after the initial prescription. The initial prescription is not counted, values start with 0." STEM_TABLE,ROUTE_CONCEPT_ID,YES,INTEGER,"A foreign key to a predefined concept in the Standardized Vocabularies reflecting the route of administration." STEM_TABLE,ROUTE_SOURCE_VALUE,YES,CHARACTER VARYING,"The information about the route of administration as detailed in the source." -STEM_TABLE,SIG,YES,CHARACTER VARYING,"The directions (“signetur”) on the Drug prescription as recorded in the original prescription (and printed on the container) or dispensing record." +STEM_TABLE,SIG,YES,CHARACTER VARYING,"The directions (�signetur�) on the Drug prescription as recorded in the original prescription (and printed on the container) or dispensing record." STEM_TABLE,STOP_REASON,YES,CHARACTER VARYING,"The reason that the condition was no longer present or the drug exposure stopped, as indicated in the source data." STEM_TABLE,UNIQUE_DEVICE_ID,YES,CHARACTER VARYING,"A UDI or equivalent identifying the instance of the Device used in the Person." STEM_TABLE,UNIT_CONCEPT_ID,YES,INTEGER,"A foreign key to a predefined concept in the Standardized Vocabularies reflecting the unit the effective_drug_dose/measurement/observation/specimen value is expressed." diff --git a/rabbitinahat/src/main/resources/org/ohdsi/rabbitInAHat/dataModel/StemTableV6.0.csv b/rabbitinahat/src/main/resources/org/ohdsi/rabbitInAHat/dataModel/StemTableV6.0.csv index 33830b3b..d1d7e948 100644 --- a/rabbitinahat/src/main/resources/org/ohdsi/rabbitInAHat/dataModel/StemTableV6.0.csv +++ b/rabbitinahat/src/main/resources/org/ohdsi/rabbitInAHat/dataModel/StemTableV6.0.csv @@ -17,17 +17,16 @@ STEM_TABLE,VERBATIM_END_DATE,YES,DATE,The verbatim end date (not derived) of the STEM_TABLE,DAYS_SUPPLY,YES,INTEGER,The number of days of supply of the medication as recorded in the original prescription or dispensing record. STEM_TABLE,DOSE_UNIT_SOURCE_VALUE,YES,CHARACTER VARYING,The information about the dose unit as detailed in the source. STEM_TABLE,LOT_NUMBER,YES,CHARACTER VARYING,An identifier assigned to a particular quantity or lot of Drug product from the manufacturer. -STEM_TABLE,MODIFIER_CONCEPT_ID,YES,INTEGER,A foreign key to a Standard Concept identifier for a modifier to the Procedure (e.g. bilateral) -STEM_TABLE,OPERATOR_CONCEPT_ID,YES,INTEGER,"A foreign key identifier to the predefined Concept in the Standardized Vocabularies reflecting the mathematical operator that is applied to the value_as_number. Operators are <, ?, =, ?, >." -STEM_TABLE,MODIFIER_CONCEPT_ID,YES,INTEGER,"A foreign key to a Standard Concept ID for a modifier (e.g., severity of drug-drug interaction alert)" +STEM_TABLE,MODIFIER_CONCEPT_ID,YES,INTEGER,A foreign key to a Standard Concept identifier for a modifier (e.g. bilateral) STEM_TABLE,MODIFIER_SOURCE_VALUE,YES,CHARACTER VARYING,The source code for the modifier as it appears in the source data. +STEM_TABLE,OPERATOR_CONCEPT_ID,YES,INTEGER,"A foreign key identifier to the predefined Concept in the Standardized Vocabularies reflecting the mathematical operator that is applied to the value_as_number. Operators are <, ?, =, ?, >." STEM_TABLE,QUANTITY,YES,INTEGER,The number of individual Devices/Procedures/Drugs used for the exposure. STEM_TABLE,RANGE_HIGH,YES,FLOAT,The upper limit of the normal range of the Measurement. The upper range is assumed to be of the same unit of measure as the Measurement value. STEM_TABLE,RANGE_LOW,YES,FLOAT,The lower limit of the normal range of the Measurement result. The lower range is assumed to be of the same unit of measure as the Measurement value. STEM_TABLE,REFILLS,YES,INTEGER,"The number of refills after the initial prescription. The initial prescription is not counted, values start with 0." STEM_TABLE,ROUTE_CONCEPT_ID,YES,INTEGER,A foreign key to a predefined concept in the Standardized Vocabularies reflecting the route of administration. STEM_TABLE,ROUTE_SOURCE_VALUE,YES,CHARACTER VARYING,The information about the route of administration as detailed in the source. -STEM_TABLE,SIG,YES,CHARACTER VARYING,The directions (“signetur”) on the Drug prescription as recorded in the original prescription (and printed on the container) or dispensing record. +STEM_TABLE,SIG,YES,CHARACTER VARYING,The directions (�signetur�) on the Drug prescription as recorded in the original prescription (and printed on the container) or dispensing record. STEM_TABLE,STOP_REASON,YES,CHARACTER VARYING,"The reason that the condition was no longer present or the drug exposure stopped, as indicated in the source data." STEM_TABLE,UNIQUE_DEVICE_ID,YES,CHARACTER VARYING,A UDI or equivalent identifying the instance of the Device used in the Person. STEM_TABLE,UNIT_CONCEPT_ID,YES,INTEGER,A foreign key to a predefined concept in the Standardized Vocabularies reflecting the unit the effective_drug_dose/measurement/observation/specimen value is expressed. diff --git a/rabbitinahat/src/main/resources/org/ohdsi/rabbitInAHat/dataModel/concept_id_hint_select.sql b/rabbitinahat/src/main/resources/org/ohdsi/rabbitInAHat/dataModel/concept_id_hint_select.sql new file mode 100644 index 00000000..c84958fe --- /dev/null +++ b/rabbitinahat/src/main/resources/org/ohdsi/rabbitInAHat/dataModel/concept_id_hint_select.sql @@ -0,0 +1,87 @@ +SELECT vocabulary_version from @vocab.vocabulary WHERE vocabulary_id = 'None'; + +SELECT 'person' AS omop_cdm_table, 'gender_concept_id' AS omop_cdm_field, concept_id, concept_name, standard_concept, 'v5.0 12-MAR-18' AS vocabulary_version +FROM @vocab.concept +WHERE domain_id = 'Gender' +UNION ALL +SELECT 'person' AS omop_cdm_table, 'ethnicity_concept_id' AS omop_cdm_field, concept_id, concept_name, standard_concept, 'v5.0 12-MAR-18' AS vocabulary_version +FROM @vocab.concept +WHERE domain_id = 'Ethnicity' +UNION ALL +SELECT 'person' AS omop_cdm_table, 'race_concept_id' AS omop_cdm_field, concept_id, concept_name, standard_concept, 'v5.0 12-MAR-18' AS vocabulary_version +FROM @vocab.concept +WHERE domain_id = 'Race' +UNION ALL +SELECT 'condition_occurrence' AS omop_cdm_table, 'condition_type_concept_id' AS omop_cdm_field, concept_id, concept_name, standard_concept, 'v5.0 12-MAR-18' AS vocabulary_version +FROM @vocab.concept +WHERE domain_id = 'Type Concept' AND concept_class_id = 'Condition Type' +UNION ALL +SELECT 'cost' AS omop_cdm_table, 'cost_type_concept_id' AS omop_cdm_field, concept_id, concept_name, standard_concept, 'v5.0 12-MAR-18' AS vocabulary_version +FROM @vocab.concept +WHERE domain_id = 'Type Concept' AND concept_class_id = 'Cost Type' +UNION ALL +SELECT 'death' AS omop_cdm_table, 'death_type_concept_id' AS omop_cdm_field, concept_id, concept_name, standard_concept, 'v5.0 12-MAR-18' AS vocabulary_version +FROM @vocab.concept +WHERE domain_id = 'Type Concept' AND concept_class_id = 'Death Type' +UNION ALL +SELECT 'device_exposure' AS omop_cdm_table, 'device_type_concept_id' AS omop_cdm_field, concept_id, concept_name, standard_concept, 'v5.0 12-MAR-18' AS vocabulary_version +FROM @vocab.concept +WHERE domain_id = 'Type Concept' AND concept_class_id = 'Device Type' +UNION ALL +SELECT 'drug_exposure' AS omop_cdm_table, 'drug_type_concept_id' AS omop_cdm_field, concept_id, concept_name, standard_concept, 'v5.0 12-MAR-18' AS vocabulary_version +FROM @vocab.concept +WHERE domain_id = 'Type Concept' AND concept_class_id = 'Drug Type' +UNION ALL +SELECT 'episode' AS omop_cdm_table, 'episode_type_concept_id' AS omop_cdm_field, concept_id, concept_name, standard_concept, 'v5.0 12-MAR-18' AS vocabulary_version +FROM @vocab.concept +WHERE domain_id = 'Type Concept' AND concept_class_id = 'Episode Type' +UNION ALL +SELECT 'measurement' AS omop_cdm_table, 'measurement_type_concept_id' AS omop_cdm_field, concept_id, concept_name, standard_concept, 'v5.0 12-MAR-18' AS vocabulary_version +FROM @vocab.concept +WHERE domain_id = 'Type Concept' AND concept_class_id = 'Meas Type' +UNION ALL +SELECT 'note' AS omop_cdm_table, 'note_type_concept_id' AS omop_cdm_field, concept_id, concept_name, standard_concept, 'v5.0 12-MAR-18' AS vocabulary_version +FROM @vocab.concept +WHERE domain_id = 'Type Concept' AND concept_class_id = 'Note Type' +UNION ALL +SELECT 'observation_period' AS omop_cdm_table, 'period_type_concept_id' AS omop_cdm_field, concept_id, concept_name, standard_concept, 'v5.0 12-MAR-18' AS vocabulary_version +FROM @vocab.concept +WHERE domain_id = 'Type Concept' AND concept_class_id = 'Obs Period Type' +UNION ALL +SELECT 'observation' AS omop_cdm_table, 'observation_type_concept_id' AS omop_cdm_field, concept_id, concept_name, standard_concept, 'v5.0 12-MAR-18' AS vocabulary_version +FROM @vocab.concept +WHERE domain_id = 'Type Concept' AND concept_class_id = 'Observation Type' +UNION ALL +SELECT 'procedure_occurrence' AS omop_cdm_table, 'procedure_type_concept_id' AS omop_cdm_field, concept_id, concept_name, standard_concept, 'v5.0 12-MAR-18' AS vocabulary_version +FROM @vocab.concept +WHERE domain_id = 'Type Concept' AND concept_class_id = 'Procedure Type' +UNION ALL +SELECT 'specimen' AS omop_cdm_table, 'specimen_type_concept_id' AS omop_cdm_field, concept_id, concept_name, standard_concept, 'v5.0 12-MAR-18' AS vocabulary_version +FROM @vocab.concept +WHERE domain_id = 'Type Concept' AND concept_class_id = 'Specimen Type' +UNION ALL +SELECT 'visit_occurrence' AS omop_cdm_table, 'visit_type_concept_id' AS omop_cdm_field, concept_id, concept_name, standard_concept, 'v5.0 12-MAR-18' AS vocabulary_version +FROM @vocab.concept +WHERE domain_id = 'Type Concept' AND concept_class_id = 'Visit Type' +UNION ALL +SELECT 'visit_occurrence' AS omop_cdm_table, 'visit_concept_id' AS omop_cdm_field, concept_id, concept_name, standard_concept, 'v5.0 12-MAR-18' AS vocabulary_version +FROM @vocab.concept +WHERE domain_id = 'Visit' +UNION ALL +SELECT 'cost' AS omop_cdm_table, 'cost_concept_id' AS omop_cdm_field, concept_id, concept_name, standard_concept, 'v5.0 12-MAR-18' AS vocabulary_version +FROM @vocab.concept +WHERE domain_id = 'Cost' +UNION ALL +SELECT 'payer_plan_period' AS omop_cdm_table, 'plan_concept_id' AS omop_cdm_field, concept_id, concept_name, standard_concept, 'v5.0 12-MAR-18' AS vocabulary_version +FROM @vocab.concept +WHERE domain_id = 'Plan' +UNION ALL +SELECT 'drug_exposure' AS omop_cdm_table, 'route_concept_id' AS omop_cdm_field, concept_id, concept_name, standard_concept, 'v5.0 12-MAR-18' AS vocabulary_version +FROM @vocab.concept +WHERE domain_id = 'Route' +UNION ALL +SELECT 'measurement' AS omop_cdm_table, 'operator_concept_id' AS omop_cdm_field, concept_id, concept_name, standard_concept, 'v5.0 12-MAR-18' AS vocabulary_version +FROM @vocab.concept +WHERE domain_id = 'Meas Value Operator' +ORDER BY omop_cdm_table, omop_cdm_field, standard_concept, concept_name +; diff --git a/whiterabbit/pom.xml b/whiterabbit/pom.xml index 931dff95..24a63e4d 100644 --- a/whiterabbit/pom.xml +++ b/whiterabbit/pom.xml @@ -5,7 +5,7 @@ leporidae org.ohdsi - 0.8.0-SNAPSHOT + 0.9.0 4.0.0 @@ -20,7 +20,8 @@ appassembler-maven-plugin ${project.parent.basedir}/dist - -Xmx1200m + + -Xmx2400m org.ohdsi.whiteRabbit.WhiteRabbitMain @@ -39,4 +40,4 @@ ${project.version} - \ No newline at end of file + diff --git a/whiterabbit/src/main/java/org/ohdsi/whiteRabbit/DbSettings.java b/whiterabbit/src/main/java/org/ohdsi/whiteRabbit/DbSettings.java index b40433aa..aca2d7c2 100644 --- a/whiterabbit/src/main/java/org/ohdsi/whiteRabbit/DbSettings.java +++ b/whiterabbit/src/main/java/org/ohdsi/whiteRabbit/DbSettings.java @@ -26,7 +26,8 @@ public class DbSettings { public static int DATABASE = 1; public static int CSVFILES = 2; - + public static int SASFILES = 3; + public int dataType; public List tables = new ArrayList(); diff --git a/whiterabbit/src/main/java/org/ohdsi/whiteRabbit/WhiteRabbitMain.java b/whiterabbit/src/main/java/org/ohdsi/whiteRabbit/WhiteRabbitMain.java index e2ed4ecc..66ad57dd 100644 --- a/whiterabbit/src/main/java/org/ohdsi/whiteRabbit/WhiteRabbitMain.java +++ b/whiterabbit/src/main/java/org/ohdsi/whiteRabbit/WhiteRabbitMain.java @@ -112,6 +112,7 @@ public class WhiteRabbitMain implements ActionListener { private JList tableList; private Vector tables = new Vector(); private boolean sourceIsFiles = true; + private boolean sourceIsSas = false; private boolean targetIsFiles = false; private List componentsToDisableWhenRunning = new ArrayList(); @@ -148,6 +149,7 @@ public void windowClosing(WindowEvent e) { } private void launchCommandLine(String iniFileName) { + // TODO: add option to scan sas7bdat from command line, using ini file IniFile iniFile = new IniFile(iniFileName); DbSettings dbSettings = new DbSettings(); if (iniFile.get("DATA_TYPE").equalsIgnoreCase("Delimited text files")) { @@ -192,12 +194,16 @@ else if (iniFile.get("DATA_TYPE").equalsIgnoreCase("SQL Server")) { dbSettings.dbType = DbType.MSACCESS; else if (iniFile.get("DATA_TYPE").equalsIgnoreCase("Teradata")) dbSettings.dbType = DbType.TERADATA; + else if (iniFile.get("DATA_TYPE").equalsIgnoreCase("BigQuery")) { + dbSettings.dbType = DbType.BIGQUERY; + /* GBQ requires database. Putting database into domain var for connect() */ + dbSettings.domain = dbSettings.database; + } } if (iniFile.get("TABLES_TO_SCAN").equalsIgnoreCase("*")) { - RichConnection connection = new RichConnection(dbSettings.server, dbSettings.domain, dbSettings.user, dbSettings.password, dbSettings.dbType); - for (String table : connection.getTableNames(dbSettings.database)) - dbSettings.tables.add(table); - connection.close(); + try (RichConnection connection = new RichConnection(dbSettings.server, dbSettings.domain, dbSettings.user, dbSettings.password, dbSettings.dbType)) { + dbSettings.tables.addAll(connection.getTableNames(dbSettings.database)); + } } else { for (String table : iniFile.get("TABLES_TO_SCAN").split(",")) { if (dbSettings.dataType == DbSettings.CSVFILES) @@ -262,42 +268,45 @@ public void actionPerformed(ActionEvent e) { sourcePanel.setLayout(new GridLayout(0, 2)); sourcePanel.setBorder(BorderFactory.createTitledBorder("Source data location")); sourcePanel.add(new JLabel("Data type")); - sourceType = new JComboBox(new String[] { "Delimited text files", "MySQL", "Oracle", "SQL Server", "PostgreSQL", "MS Access", "PDW", "Redshift", "Teradata" }); + sourceType = new JComboBox<>(new String[] { "Delimited text files", "SAS7bdat", "MySQL", "Oracle", "SQL Server", "PostgreSQL", "MS Access", "PDW", "Redshift", "Teradata", "BigQuery" }); sourceType.setToolTipText("Select the type of source data available"); - sourceType.addItemListener(new ItemListener() { - - @Override - public void itemStateChanged(ItemEvent arg0) { - sourceIsFiles = arg0.getItem().toString().equals("Delimited text files"); - sourceServerField.setEnabled(!sourceIsFiles); - sourceUserField.setEnabled(!sourceIsFiles); - sourcePasswordField.setEnabled(!sourceIsFiles); - sourceDatabaseField.setEnabled(!sourceIsFiles); - sourceDelimiterField.setEnabled(sourceIsFiles); - addAllButton.setEnabled(!sourceIsFiles); - - if (!sourceIsFiles && arg0.getItem().toString().equals("Oracle")) { - sourceServerField - .setToolTipText("For Oracle servers this field contains the SID, servicename, and optionally the port: '/', ':/', '/', or ':/'"); - sourceUserField.setToolTipText("For Oracle servers this field contains the name of the user used to log in"); - sourcePasswordField.setToolTipText("For Oracle servers this field contains the password corresponding to the user"); - sourceDatabaseField - .setToolTipText("For Oracle servers this field contains the schema (i.e. 'user' in Oracle terms) containing the source tables"); - } else if (!sourceIsFiles && arg0.getItem().toString().equals("PostgreSQL")) { - sourceServerField.setToolTipText("For PostgreSQL servers this field contains the host name and database name (/)"); + sourceType.addItemListener(itemEvent -> { + String selectedSourceType = itemEvent.getItem().toString(); + sourceIsFiles = selectedSourceType.equals("Delimited text files"); + sourceIsSas = selectedSourceType.equals("SAS7bdat"); + boolean sourceIsDatabase = !(sourceIsFiles || sourceIsSas); + + sourceServerField.setEnabled(sourceIsDatabase); + sourceUserField.setEnabled(sourceIsDatabase); + sourcePasswordField.setEnabled(sourceIsDatabase); + sourceDatabaseField.setEnabled(sourceIsDatabase); + sourceDelimiterField.setEnabled(sourceIsFiles); + addAllButton.setEnabled(sourceIsDatabase); + + if (sourceIsDatabase && selectedSourceType.equals("Oracle")) { + sourceServerField.setToolTipText("For Oracle servers this field contains the SID, servicename, and optionally the port: '/', ':/', '/', or ':/'"); + sourceUserField.setToolTipText("For Oracle servers this field contains the name of the user used to log in"); + sourcePasswordField.setToolTipText("For Oracle servers this field contains the password corresponding to the user"); + sourceDatabaseField.setToolTipText("For Oracle servers this field contains the schema (i.e. 'user' in Oracle terms) containing the source tables"); + } else if (sourceIsDatabase && selectedSourceType.equals("PostgreSQL")) { + sourceServerField.setToolTipText("For PostgreSQL servers this field contains the host name and database name (/)"); + sourceUserField.setToolTipText("The user used to log in to the server"); + sourcePasswordField.setToolTipText("The password used to log in to the server"); + sourceDatabaseField.setToolTipText("For PostgreSQL servers this field contains the schema containing the source tables"); + } else if (sourceIsDatabase && selectedSourceType.equals("BigQuery")) { + sourceServerField.setToolTipText("GBQ SA & UA: ProjectID"); + sourceUserField.setToolTipText("GBQ SA only: OAuthServiceAccountEMAIL"); + sourcePasswordField.setToolTipText("GBQ SA only: OAuthPvtKeyPath"); + sourceDatabaseField.setToolTipText("GBQ SA & UA: Data Set within ProjectID"); + } else if (sourceIsDatabase) { + sourceServerField.setToolTipText("This field contains the name or IP address of the database server"); + if (selectedSourceType.equals("SQL Server")) { + sourceUserField.setToolTipText("The user used to log in to the server. Optionally, the domain can be specified as / (e.g. 'MyDomain/Joe')"); + } else { sourceUserField.setToolTipText("The user used to log in to the server"); - sourcePasswordField.setToolTipText("The password used to log in to the server"); - sourceDatabaseField.setToolTipText("For PostgreSQL servers this field contains the schema containing the source tables"); - } else if (!sourceIsFiles) { - sourceServerField.setToolTipText("This field contains the name or IP address of the database server"); - if (arg0.getItem().toString().equals("SQL Server")) - sourceUserField - .setToolTipText("The user used to log in to the server. Optionally, the domain can be specified as / (e.g. 'MyDomain/Joe')"); - else - sourceUserField.setToolTipText("The user used to log in to the server"); - sourcePasswordField.setToolTipText("The password used to log in to the server"); - sourceDatabaseField.setToolTipText("The name of the database containing the source tables"); } + sourcePasswordField.setToolTipText("The password used to log in to the server"); + sourceDatabaseField.setToolTipText("The name of the database containing the source tables"); } }); sourcePanel.add(sourceType); @@ -460,6 +469,7 @@ public void actionPerformed(ActionEvent e) { } private JPanel createFakeDataPanel() { + // TODO: add sas7bdat as target for fake data. JPanel panel = new JPanel(); panel.setLayout(new GridBagLayout()); @@ -518,6 +528,11 @@ public void itemStateChanged(ItemEvent arg0) { targetUserField.setToolTipText("The user used to log in to the server"); targetPasswordField.setToolTipText("The password used to log in to the server"); targetDatabaseField.setToolTipText("For PostgreSQL servers this field contains the schema containing the source tables"); + } else if (!targetIsFiles && arg0.getItem().toString().equals("BigQuery")) { + targetServerField.setToolTipText("GBQ ProjectID"); + targetUserField.setToolTipText("GBQ OAuthServiceAccountEmail"); + targetPasswordField.setToolTipText("GBQ OAuthPvtKeyPath"); + targetDatabaseField.setToolTipText("GBQ Data Set within ProjectID"); } else if (!targetIsFiles) { targetServerField.setToolTipText("This field contains the name or IP address of the database server"); if (arg0.getItem().toString().equals("SQL Server")) @@ -680,12 +695,16 @@ private void addAllTables() { private void pickTables() { DbSettings sourceDbSettings = getSourceDbSettings(); if (sourceDbSettings != null) { - if (sourceDbSettings.dataType == DbSettings.CSVFILES) { + if (sourceDbSettings.dataType == DbSettings.CSVFILES || sourceDbSettings.dataType == DbSettings.SASFILES) { JFileChooser fileChooser = new JFileChooser(new File(folderField.getText())); fileChooser.setMultiSelectionEnabled(true); fileChooser.setFileSelectionMode(JFileChooser.FILES_ONLY); - FileNameExtensionFilter filter = new FileNameExtensionFilter("Delimited text files", "csv", "txt"); - fileChooser.setFileFilter(filter); + + if (sourceDbSettings.dataType == DbSettings.CSVFILES) { + fileChooser.setFileFilter(new FileNameExtensionFilter("Delimited text files", "csv", "txt")); + } else if (sourceDbSettings.dataType == DbSettings.SASFILES) { + fileChooser.setFileFilter(new FileNameExtensionFilter("SAS Data Files", "sas7bdat")); + } int returnVal = fileChooser.showDialog(frame, "Select tables"); if (returnVal == JFileChooser.APPROVE_OPTION) { @@ -732,6 +751,8 @@ private DbSettings getSourceDbSettings() { dbSettings.delimiter = '\t'; else dbSettings.delimiter = sourceDelimiterField.getText().charAt(0); + } else if (sourceType.getSelectedItem().equals("SAS7bdat")) { + dbSettings.dataType = DbSettings.SASFILES; } else { dbSettings.dataType = DbSettings.DATABASE; dbSettings.user = sourceUserField.getText(); @@ -744,6 +765,8 @@ else if (sourceType.getSelectedItem().toString().equals("Oracle")) dbSettings.dbType = DbType.ORACLE; else if (sourceType.getSelectedItem().toString().equals("PostgreSQL")) dbSettings.dbType = DbType.POSTGRESQL; + else if (sourceType.getSelectedItem().toString().equals("BigQuery")) + dbSettings.dbType = DbType.BIGQUERY; else if (sourceType.getSelectedItem().toString().equals("Redshift")) dbSettings.dbType = DbType.REDSHIFT; else if (sourceType.getSelectedItem().toString().equals("SQL Server")) { @@ -773,7 +796,7 @@ else if (sourceType.getSelectedItem().toString().equals("Teradata")) } private void testConnection(DbSettings dbSettings) { - if (dbSettings.dataType == DbSettings.CSVFILES) { + if (dbSettings.dataType == DbSettings.CSVFILES || dbSettings.dataType == DbSettings.SASFILES) { if (new File(folderField.getText()).exists()) { String message = "Folder " + folderField.getText() + " found"; JOptionPane.showMessageDialog(frame, StringUtilities.wordWrap(message, 80), "Working folder found", JOptionPane.INFORMATION_MESSAGE); @@ -795,13 +818,12 @@ private void testConnection(DbSettings dbSettings) { RichConnection connection; try { - connection = new RichConnection(dbSettings.server, dbSettings.domain, dbSettings.user, dbSettings.password, dbSettings.dbType); + connection = new RichConnection(dbSettings.server, dbSettings.domain, dbSettings.user, dbSettings.password, dbSettings.dbType); } catch (Exception e) { String message = "Could not connect: " + e.getMessage(); JOptionPane.showMessageDialog(frame, StringUtilities.wordWrap(message, 80), "Error connecting to server", JOptionPane.ERROR_MESSAGE); return; } - try { List tableNames = connection.getTableNames(dbSettings.database); if (tableNames.size() == 0) @@ -887,7 +909,7 @@ else if (sourceType.getSelectedItem().toString().equals("SQL Server")) { private void scanRun() { if (tables.size() == 0) { - if (sourceIsFiles) { + if (sourceIsFiles || sourceIsSas) { String message = "No files selected for scanning"; JOptionPane.showMessageDialog(frame, StringUtilities.wordWrap(message, 80), "No files selected", JOptionPane.ERROR_MESSAGE); return; @@ -956,7 +978,7 @@ public void run() { DbSettings dbSettings = getSourceDbSettings(); if (dbSettings != null) { for (String table : tables) { - if (dbSettings.dataType == DbSettings.CSVFILES) + if (dbSettings.dataType == DbSettings.CSVFILES || dbSettings.dataType == DbSettings.SASFILES) table = folderField.getText() + "/" + table; dbSettings.tables.add(table); } diff --git a/whiterabbit/src/main/java/org/ohdsi/whiteRabbit/scan/SourceDataScan.java b/whiterabbit/src/main/java/org/ohdsi/whiteRabbit/scan/SourceDataScan.java index 3eaef3b9..7d6a4d2a 100644 --- a/whiterabbit/src/main/java/org/ohdsi/whiteRabbit/scan/SourceDataScan.java +++ b/whiterabbit/src/main/java/org/ohdsi/whiteRabbit/scan/SourceDataScan.java @@ -1,495 +1,583 @@ -/******************************************************************************* - * Copyright 2019 Observational Health Data Sciences and Informatics - * - * This file is part of WhiteRabbit - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - ******************************************************************************/ -package org.ohdsi.whiteRabbit.scan; - -import java.io.File; -import java.io.FileOutputStream; -import java.io.IOException; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.util.ArrayList; -import java.util.Collections; -import java.util.Comparator; -import java.util.HashMap; -import java.util.Iterator; -import java.util.List; -import java.util.Map; - -import org.apache.poi.ss.usermodel.Cell; -import org.apache.poi.ss.usermodel.Row; -import org.apache.poi.ss.usermodel.Sheet; -import org.apache.poi.xssf.streaming.SXSSFWorkbook; -import org.ohdsi.databases.DbType; -import org.ohdsi.databases.RichConnection; -import org.ohdsi.databases.RichConnection.QueryResult; -import org.ohdsi.rabbitInAHat.dataModel.Table; -import org.ohdsi.utilities.StringUtilities; -import org.ohdsi.utilities.collections.CountingSet; -import org.ohdsi.utilities.collections.CountingSet.Count; -import org.ohdsi.utilities.collections.Pair; -import org.ohdsi.utilities.files.ReadTextFile; -import org.ohdsi.whiteRabbit.DbSettings; - -public class SourceDataScan { - - public static int MAX_VALUES_IN_MEMORY = 100000; - public static int MIN_CELL_COUNT_FOR_CSV = 1000000; - public static int N_FOR_FREE_TEXT_CHECK = 1000; - public static int MIN_AVERAGE_LENGTH_FOR_FREE_TEXT = 100; - - private char delimiter = ','; - private int sampleSize; - private boolean scanValues; - private int minCellCount; - private int maxValues; - private DbType dbType; - private String database; - - public void process(DbSettings dbSettings, int sampleSize, boolean scanValues, int minCellCount, int maxValues, String filename) { - this.sampleSize = sampleSize; - this.scanValues = scanValues; - this.minCellCount = minCellCount; - this.maxValues = maxValues; - Map> tableToFieldInfos; - if (dbSettings.dataType == DbSettings.CSVFILES) { - if (!scanValues) - minCellCount = Math.max(minCellCount, MIN_CELL_COUNT_FOR_CSV); - tableToFieldInfos = processCsvFiles(dbSettings); - } else - tableToFieldInfos = processDatabase(dbSettings); - generateReport(tableToFieldInfos, filename); - } - - private Map> processDatabase(DbSettings dbSettings) { - Map> tableToFieldInfos = new HashMap>(); - RichConnection connection = new RichConnection(dbSettings.server, dbSettings.domain, dbSettings.user, dbSettings.password, dbSettings.dbType); - connection.setVerbose(false); - connection.use(dbSettings.database); - - dbType = dbSettings.dbType; - database = dbSettings.database; - - for (String table : dbSettings.tables) { - List fieldInfos = processDatabaseTable(table, connection); - tableToFieldInfos.put(table, fieldInfos); - } - - connection.close(); - return tableToFieldInfos; - } - - private Map> processCsvFiles(DbSettings dbSettings) { - delimiter = dbSettings.delimiter; - Map> tableToFieldInfos = new HashMap>(); - for (String table : dbSettings.tables) { - List fieldInfos = processCsvFile(table); - String tableName = new File(table).getName(); - if (!tableToFieldInfos.containsKey(tableName)) { - tableToFieldInfos.put(tableName, fieldInfos); - } else { - tableToFieldInfos.put(table, fieldInfos); - } - - } - return tableToFieldInfos; - } - - private void generateReport(Map> tableToFieldInfos, String filename) { - System.out.println("Generating scan report"); - removeEmptyTables(tableToFieldInfos); - List tables = new ArrayList(tableToFieldInfos.keySet()); - Collections.sort(tables); - - SXSSFWorkbook workbook = new SXSSFWorkbook(100); // keep 100 rows in memory, exceeding rows will be flushed to disk - - // Create overview sheet - Sheet overviewSheet = workbook.createSheet("Overview"); - if (!scanValues) { - addRow(overviewSheet, "Table", "Field", "Type", "N rows"); - for (String table : tables) { - for (FieldInfo fieldInfo : tableToFieldInfos.get(table)) - addRow(overviewSheet, table, fieldInfo.name, fieldInfo.getTypeDescription(), Long.valueOf(fieldInfo.rowCount)); - addRow(overviewSheet, ""); - } - } else { - addRow(overviewSheet, "Table", "Field", "Type", "Max length", "N rows", "N rows checked", "Fraction empty"); - int sheetIndex = 0; - Map sheetNameLookup = new HashMap<>(); - for (String tableName : tables) { - // Make tablename unique - String tableNameIndexed = Table.indexTableNameForSheet(tableName, sheetIndex); - - String sheetName = Table.createSheetNameFromTableName(tableNameIndexed); - sheetNameLookup.put(tableName, sheetName); - - for (FieldInfo fieldInfo : tableToFieldInfos.get(tableName)) - addRow(overviewSheet, tableNameIndexed, fieldInfo.name, fieldInfo.getTypeDescription(), Integer.valueOf(fieldInfo.maxLength), Long.valueOf(fieldInfo.rowCount), - Long.valueOf(fieldInfo.nProcessed), fieldInfo.getFractionEmpty()); - addRow(overviewSheet, ""); - sheetIndex += 1; - } - - // Create per table scan values - for (String tableName : tables) { - Sheet valueSheet = workbook.createSheet(sheetNameLookup.get(tableName)); - - List fieldInfos = tableToFieldInfos.get(tableName); - List>> valueCounts = new ArrayList<>(); - Object[] header = new Object[fieldInfos.size() * 2]; - int maxCount = 0; - for (int i = 0; i < fieldInfos.size(); i++) { - FieldInfo fieldInfo = fieldInfos.get(i); - header[i * 2] = fieldInfo.name; - if (fieldInfo.isFreeText) - header[(i * 2) + 1] = "Word count"; - else - header[(i * 2) + 1] = "Frequency"; - List> counts = fieldInfo.getSortedValuesWithoutSmallValues(); - valueCounts.add(counts); - if (counts.size() > maxCount) - maxCount = counts.size(); - } - addRow(valueSheet, header); - for (int i = 0; i < maxCount; i++) { - Object[] row = new Object[fieldInfos.size() * 2]; - for (int j = 0; j < fieldInfos.size(); j++) { - List> counts = valueCounts.get(j); - if (counts.size() > i) { - row[j * 2] = counts.get(i).getItem1(); - row[(j * 2) + 1] = counts.get(i).getItem2() == -1 ? "" : counts.get(i).getItem2(); - } else { - row[j * 2] = ""; - row[(j * 2) + 1] = ""; - } - } - addRow(valueSheet, row); - } - // Save some memory by derefencing tables already included in the report: - tableToFieldInfos.remove(tableName); - } - } - - try { - FileOutputStream out = new FileOutputStream(new File(filename)); - workbook.write(out); - out.close(); - StringUtilities.outputWithTime("Scan report generated: " + filename); - } catch (IOException e) { - throw new RuntimeException(e.getMessage()); - } - } - - private void removeEmptyTables(Map> tableToFieldInfos) { - Iterator>> iterator = tableToFieldInfos.entrySet().iterator(); - while (iterator.hasNext()) { - if (iterator.next().getValue().size() == 0) - iterator.remove(); - } - } - - private List processDatabaseTable(String table, RichConnection connection) { - StringUtilities.outputWithTime("Scanning table " + table); - - long rowCount = connection.getTableSize(table); - List fieldInfos = fetchTableStructure(connection, table); - if (scanValues) { - int actualCount = 0; - QueryResult queryResult = null; - try { - queryResult = fetchRowsFromTable(connection, table, rowCount); - for (org.ohdsi.utilities.files.Row row : queryResult) { - for (int i = 0; i < fieldInfos.size(); i++) - fieldInfos.get(i).processValue(row.get(fieldInfos.get(i).name)); - actualCount++; - if (sampleSize != -1 && actualCount >= sampleSize) { - System.out.println("Stopped after " + actualCount + " rows"); - break; - } - } - for (FieldInfo fieldInfo : fieldInfos) - fieldInfo.trim(); - } catch (Exception e) { - System.out.println("Error: " + e.getMessage()); - } finally { - if (queryResult != null) { - queryResult.close(); - } - } - } - - return fieldInfos; - } - - private QueryResult fetchRowsFromTable(RichConnection connection, String table, long rowCount) { - String query = null; - - if (sampleSize == -1) { - if (dbType == DbType.MSACCESS) - query = "SELECT * FROM [" + table + "]"; - else if (dbType == DbType.MSSQL || dbType == DbType.PDW) - query = "SELECT * FROM [" + table.replaceAll("\\.", "].[") + "]"; - else - query = "SELECT * FROM " + table; - } else { - if (dbType == DbType.MSSQL) - query = "SELECT * FROM [" + table.replaceAll("\\.", "].[") + "] TABLESAMPLE (" + sampleSize + " ROWS)"; - else if (dbType == DbType.MYSQL) - query = "SELECT * FROM " + table + " ORDER BY RAND() LIMIT " + sampleSize; - else if (dbType == DbType.PDW) - query = "SELECT TOP " + sampleSize + " * FROM [" + table.replaceAll("\\.", "].[") + "] ORDER BY RAND()"; - else if (dbType == DbType.ORACLE) { - if (sampleSize < rowCount) { - double percentage = 100 * sampleSize / (double) rowCount; - if (percentage < 100) - query = "SELECT * FROM " + table + " SAMPLE(" + percentage + ")"; - } else { - query = "SELECT * FROM " + table; - } - } else if (dbType == DbType.POSTGRESQL || dbType == DbType.REDSHIFT) - query = "SELECT * FROM " + table + " ORDER BY RANDOM() LIMIT " + sampleSize; - else if (dbType == DbType.MSACCESS) - query = "SELECT " + "TOP " + sampleSize + " * FROM [" + table + "]"; - } - // System.out.println("SQL: " + query); - return connection.query(query); - - } - - private List fetchTableStructure(RichConnection connection, String table) { - List fieldInfos = new ArrayList(); - - if (dbType == DbType.MSACCESS) { - ResultSet rs = connection.getMsAccessFieldNames(table); - try { - while (rs.next()) { - FieldInfo fieldInfo = new FieldInfo(rs.getString("COLUMN_NAME")); - fieldInfo.type = rs.getString("TYPE_NAME"); - fieldInfo.rowCount = connection.getTableSize(table); - fieldInfos.add(fieldInfo); - } - } catch (SQLException e) { - throw new RuntimeException(e.getMessage()); - } - } else { - String query = null; - if (dbType == DbType.ORACLE) - query = "SELECT COLUMN_NAME,DATA_TYPE FROM ALL_TAB_COLUMNS WHERE table_name = '" + table + "' AND owner = '" + database.toUpperCase() + "'"; - else if (dbType == DbType.MSSQL || dbType == DbType.PDW) { - String trimmedDatabase = database; - if (database.startsWith("[") && database.endsWith("]")) - trimmedDatabase = database.substring(1, database.length() - 1); - String[] parts = table.split("\\."); - query = "SELECT COLUMN_NAME,DATA_TYPE FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_CATALOG='" + trimmedDatabase + "' AND TABLE_SCHEMA='" + parts[0] + - "' AND TABLE_NAME='" + parts[1] + "';"; - } else if (dbType == DbType.MYSQL) - query = "SELECT COLUMN_NAME,DATA_TYPE FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_SCHEMA = '" + database + "' AND TABLE_NAME = '" + table - + "';"; - else if (dbType == DbType.POSTGRESQL || dbType == DbType.REDSHIFT) - query = "SELECT COLUMN_NAME,DATA_TYPE FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_SCHEMA = '" + database.toLowerCase() + "' AND TABLE_NAME = '" - + table.toLowerCase() + "' ORDER BY ordinal_position;"; - else if (dbType == DbType.TERADATA) { - query = "SELECT ColumnName, ColumnType FROM dbc.columns WHERE DatabaseName= '" + database.toLowerCase() + "' AND TableName = '" - + table.toLowerCase() + "';"; - } - - for (org.ohdsi.utilities.files.Row row : connection.query(query)) { - row.upperCaseFieldNames(); - FieldInfo fieldInfo; - if (dbType == DbType.TERADATA) { - fieldInfo = new FieldInfo(row.get("COLUMNNAME")); - } else { - fieldInfo = new FieldInfo(row.get("COLUMN_NAME")); - } - if (dbType == DbType.TERADATA) { - fieldInfo.type = row.get("COLUMNTYPE"); - } else { - fieldInfo.type = row.get("DATA_TYPE"); - } - fieldInfo.rowCount = connection.getTableSize(table); - ; - fieldInfos.add(fieldInfo); - } - } - return fieldInfos; - } - - private List processCsvFile(String filename) { - StringUtilities.outputWithTime("Scanning table " + filename); - List fieldInfos = new ArrayList(); - int lineNr = 0; - for (String line : new ReadTextFile(filename)) { - lineNr++; - List row = StringUtilities.safeSplit(line, delimiter); - for (int i = 0; i < row.size(); i++) { - String column = row.get(i); - if (column.startsWith("\"") && column.endsWith("\"") && column.length() > 1) - column = column.substring(1, column.length() - 1); - column = column.replace("\\\"", "\""); - row.set(i, column); - } - if (lineNr == 1) { - for (String cell : row) - fieldInfos.add(new FieldInfo(cell)); - } else { - if (row.size() == fieldInfos.size()) { // Else there appears to be a formatting error, so skip - for (int i = 0; i < row.size(); i++) - fieldInfos.get(i).processValue(row.get(i)); - } - } - if (sampleSize != -1 && lineNr == sampleSize) - break; - } - for (FieldInfo fieldInfo : fieldInfos) - fieldInfo.trim(); - - return fieldInfos; - } - - private class FieldInfo { - public String type; - public String name; - public CountingSet valueCounts = new CountingSet(); - public long sumLength = 0; - public int maxLength = 0; - public long nProcessed = 0; - public long emptyCount = 0; - public long rowCount = -1; - public boolean isInteger = true; - public boolean isReal = true; - public boolean isDate = true; - public boolean isFreeText = false; - public boolean tooManyValues = false; - - public FieldInfo(String name) { - this.name = name; - } - - public void trim() { - if (valueCounts.size() > maxValues) - valueCounts.keepTopN(maxValues); - } - - public Double getFractionEmpty() { - if (nProcessed == 0) - return 0d; - else - return emptyCount / (double) nProcessed; - } - - public String getTypeDescription() { - if (type != null) - return type; - else if (nProcessed == emptyCount) - return "empty"; - else if (isFreeText) - return "text"; - else if (isDate) - return "date"; - else if (isInteger) - return "int"; - else if (isReal) - return "real"; - else - return "varchar"; - } - - public void processValue(String value) { - String trimValue = value.trim(); - nProcessed++; - sumLength += value.length(); - if (value.length() > maxLength) - maxLength = value.length(); - - if (trimValue.length() == 0) - emptyCount++; - - if (!isFreeText) { - valueCounts.add(value); - - if (trimValue.length() != 0) { - if (isReal && !StringUtilities.isNumber(trimValue)) - isReal = false; - if (isInteger && !StringUtilities.isLong(trimValue)) - isInteger = false; - if (isDate && !StringUtilities.isDate(trimValue)) - isDate = false; - } - if (nProcessed == N_FOR_FREE_TEXT_CHECK) { - if (!isInteger && !isReal && !isDate) { - double averageLength = sumLength / (double) (nProcessed - emptyCount); - if (averageLength >= MIN_AVERAGE_LENGTH_FOR_FREE_TEXT) { - isFreeText = true; - CountingSet wordCounts = new CountingSet(); - for (Map.Entry entry : valueCounts.key2count.entrySet()) - for (String word : StringUtilities.mapToWords(entry.getKey().toLowerCase())) - wordCounts.add(word, entry.getValue().count); - valueCounts = wordCounts; - } - } - } - } else { - for (String word : StringUtilities.mapToWords(trimValue.toLowerCase())) - valueCounts.add(word); - } - - if (!tooManyValues && valueCounts.size() > MAX_VALUES_IN_MEMORY) { - tooManyValues = true; - valueCounts.keepTopN(maxValues); - } - } - - public List> getSortedValuesWithoutSmallValues() { - boolean truncated = false; - List> result = new ArrayList>(); - - for (Map.Entry entry : valueCounts.key2count.entrySet()) { - if (entry.getValue().count < minCellCount) - truncated = true; - else { - result.add(new Pair(entry.getKey(), entry.getValue().count)); - if (result.size() > maxValues) { - truncated = true; - break; - } - } - } - - Collections.sort(result, new Comparator>() { - public int compare(Pair o1, Pair o2) { - return o2.getItem2().compareTo(o1.getItem2()); - } - }); - if (truncated) - result.add(new Pair("List truncated...", -1)); - return result; - } - } - - private void addRow(Sheet sheet, Object... values) { - Row row = sheet.createRow(sheet.getPhysicalNumberOfRows()); - for (Object value : values) { - Cell cell = row.createCell(row.getPhysicalNumberOfCells()); - - if (value instanceof Integer || value instanceof Long || value instanceof Double) - cell.setCellValue(Double.parseDouble(value.toString())); - else - cell.setCellValue(value.toString()); - - } - } -} +/******************************************************************************* + * Copyright 2019 Observational Health Data Sciences and Informatics + * + * This file is part of WhiteRabbit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + ******************************************************************************/ +package org.ohdsi.whiteRabbit.scan; + +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.function.Function; +import java.util.stream.Collectors; + +import com.epam.parso.Column; +import com.epam.parso.SasFileProperties; +import com.epam.parso.SasFileReader; +import com.epam.parso.impl.SasFileReaderImpl; +import org.apache.poi.ss.usermodel.Cell; +import org.apache.poi.ss.usermodel.CellStyle; +import org.apache.poi.ss.usermodel.Row; +import org.apache.poi.ss.usermodel.Sheet; +import org.apache.poi.xssf.streaming.SXSSFWorkbook; +import org.ohdsi.databases.DbType; +import org.ohdsi.databases.RichConnection; +import org.ohdsi.databases.RichConnection.QueryResult; +import org.ohdsi.rabbitInAHat.dataModel.Table; +import org.ohdsi.utilities.StringUtilities; +import org.ohdsi.utilities.collections.CountingSet; +import org.ohdsi.utilities.collections.CountingSet.Count; +import org.ohdsi.utilities.collections.Pair; +import org.ohdsi.utilities.files.ReadTextFile; +import org.ohdsi.whiteRabbit.DbSettings; + +public class SourceDataScan { + + public static int MAX_VALUES_IN_MEMORY = 100000; + public static int MIN_CELL_COUNT_FOR_CSV = 1000000; + public static int N_FOR_FREE_TEXT_CHECK = 1000; + public static int MIN_AVERAGE_LENGTH_FOR_FREE_TEXT = 100; + + private char delimiter = ','; + private int sampleSize; + private boolean scanValues; + private int minCellCount; + private int maxValues; + private DbType dbType; + private String database; + + public void process(DbSettings dbSettings, int sampleSize, boolean scanValues, int minCellCount, int maxValues, String filename) { + this.sampleSize = sampleSize; + this.scanValues = scanValues; + this.minCellCount = minCellCount; + this.maxValues = maxValues; + Map> tableToFieldInfos; + if (dbSettings.dataType == DbSettings.CSVFILES) { + if (!scanValues) + this.minCellCount = Math.max(minCellCount, MIN_CELL_COUNT_FOR_CSV); + tableToFieldInfos = processCsvFiles(dbSettings); + } else if (dbSettings.dataType == DbSettings.SASFILES) { + tableToFieldInfos = processSasFiles(dbSettings); + } else { + tableToFieldInfos = processDatabase(dbSettings); + } + generateReport(tableToFieldInfos, filename); + } + + private Map> processDatabase(DbSettings dbSettings) { + // GBQ requires database. Put database value into domain var + if (dbSettings.dbType == DbType.BIGQUERY) { + dbSettings.domain = dbSettings.database; + }; + + try (RichConnection connection = new RichConnection(dbSettings.server, dbSettings.domain, dbSettings.user, dbSettings.password, dbSettings.dbType)) { + connection.setVerbose(false); + connection.use(dbSettings.database); + + dbType = dbSettings.dbType; + database = dbSettings.database; + + return dbSettings.tables.stream() + .collect(Collectors.toMap(Function.identity(), table -> processDatabaseTable(table, connection))); + } + } + + private Map> processCsvFiles(DbSettings dbSettings) { + delimiter = dbSettings.delimiter; + Map> tableToFieldInfos = new HashMap<>(); + for (String table : dbSettings.tables) { + List fieldInfos = processCsvFile(table); + String tableName = new File(table).getName(); + if (!tableToFieldInfos.containsKey(tableName)) { + tableToFieldInfos.put(tableName, fieldInfos); + } else { + tableToFieldInfos.put(table, fieldInfos); + } + } + return tableToFieldInfos; + } + + private Map> processSasFiles(DbSettings dbSettings) { + Map> tableToFieldInfos = new HashMap<>(); + for (String fileName : dbSettings.tables) { + List fieldInfos = processSasFile(fileName); + String tableName = new File(fileName).getName(); + if (!tableToFieldInfos.containsKey(tableName)) { + tableToFieldInfos.put(tableName, fieldInfos); + } else { + tableToFieldInfos.put(fileName, fieldInfos); + } + } + return tableToFieldInfos; + } + + private void generateReport(Map> tableToFieldInfos, String filename) { + System.out.println("Generating scan report"); + removeEmptyTables(tableToFieldInfos); + List tables = new ArrayList(tableToFieldInfos.keySet()); + Collections.sort(tables); + + SXSSFWorkbook workbook = new SXSSFWorkbook(100); // keep 100 rows in memory, exceeding rows will be flushed to disk + CellStyle percentageStyle = workbook.createCellStyle(); + percentageStyle.setDataFormat(workbook.createDataFormat().getFormat("0%")); + + // Create overview sheet + Sheet overviewSheet = workbook.createSheet("Overview"); + if (!scanValues) { + addRow(overviewSheet, "Table", "Field", "Type", "N rows"); + for (String table : tables) { + for (FieldInfo fieldInfo : tableToFieldInfos.get(table)) { + addRow(overviewSheet, table, fieldInfo.name, fieldInfo.getTypeDescription(), Long.valueOf(fieldInfo.rowCount)); + } + addRow(overviewSheet, ""); + } + } else { + addRow(overviewSheet, "Table", "Field", "Type", "Max length", "N rows", "N rows checked", "Fraction empty", "N unique values", "Fraction unique values"); + int sheetIndex = 0; + Map sheetNameLookup = new HashMap<>(); + for (String tableName : tables) { + // Make tablename unique + String tableNameIndexed = Table.indexTableNameForSheet(tableName, sheetIndex); + + String sheetName = Table.createSheetNameFromTableName(tableNameIndexed); + sheetNameLookup.put(tableName, sheetName); + + for (FieldInfo fieldInfo : tableToFieldInfos.get(tableName)) { + Long uniqueCount = fieldInfo.uniqueCount; + Double fractionUnique = fieldInfo.getFractionUnique(); + addRow(overviewSheet, tableNameIndexed, fieldInfo.name, fieldInfo.getTypeDescription(), + Integer.valueOf(fieldInfo.maxLength), + Long.valueOf(fieldInfo.rowCount), + Long.valueOf(fieldInfo.nProcessed), + fieldInfo.getFractionEmpty(), + fieldInfo.hasValuesTrimmed() ? String.format("<= %d", uniqueCount) : uniqueCount, + fieldInfo.hasValuesTrimmed() ? String.format("<= %.3f", fractionUnique) : fractionUnique + ); + this.setCellStyles(overviewSheet, percentageStyle, 6, 8); + } + addRow(overviewSheet, ""); + sheetIndex += 1; + } + + // Create per table scan values + for (String tableName : tables) { + Sheet valueSheet = workbook.createSheet(sheetNameLookup.get(tableName)); + + List fieldInfos = tableToFieldInfos.get(tableName); + List>> valueCounts = new ArrayList<>(); + Object[] header = new Object[fieldInfos.size() * 2]; + int maxCount = 0; + for (int i = 0; i < fieldInfos.size(); i++) { + FieldInfo fieldInfo = fieldInfos.get(i); + header[i * 2] = fieldInfo.name; + if (fieldInfo.isFreeText) + header[(i * 2) + 1] = "Word count"; + else + header[(i * 2) + 1] = "Frequency"; + List> counts = fieldInfo.getSortedValuesWithoutSmallValues(); + valueCounts.add(counts); + if (counts.size() > maxCount) + maxCount = counts.size(); + } + addRow(valueSheet, header); + for (int i = 0; i < maxCount; i++) { + Object[] row = new Object[fieldInfos.size() * 2]; + for (int j = 0; j < fieldInfos.size(); j++) { + List> counts = valueCounts.get(j); + if (counts.size() > i) { + row[j * 2] = counts.get(i).getItem1(); + row[(j * 2) + 1] = counts.get(i).getItem2() == -1 ? "" : counts.get(i).getItem2(); + } else { + row[j * 2] = ""; + row[(j * 2) + 1] = ""; + } + } + addRow(valueSheet, row); + } + // Save some memory by derefencing tables already included in the report: + tableToFieldInfos.remove(tableName); + } + } + + try { + FileOutputStream out = new FileOutputStream(new File(filename)); + workbook.write(out); + out.close(); + StringUtilities.outputWithTime("Scan report generated: " + filename); + } catch (IOException e) { + throw new RuntimeException(e.getMessage()); + } + } + + private void removeEmptyTables(Map> tableToFieldInfos) { + tableToFieldInfos.entrySet() + .removeIf(stringListEntry -> stringListEntry.getValue().size() == 0); + } + + private List processDatabaseTable(String table, RichConnection connection) { + StringUtilities.outputWithTime("Scanning table " + table); + + long rowCount = connection.getTableSize(table); + List fieldInfos = fetchTableStructure(connection, table); + if (scanValues) { + int actualCount = 0; + QueryResult queryResult = null; + try { + queryResult = fetchRowsFromTable(connection, table, rowCount); + for (org.ohdsi.utilities.files.Row row : queryResult) { + for (FieldInfo fieldInfo : fieldInfos) { + fieldInfo.processValue(row.get(fieldInfo.name)); + } + actualCount++; + if (sampleSize != -1 && actualCount >= sampleSize) { + System.out.println("Stopped after " + actualCount + " rows"); + break; + } + } + for (FieldInfo fieldInfo : fieldInfos) { + fieldInfo.trim(); + } + } catch (Exception e) { + System.out.println("Error: " + e.getMessage()); + } finally { + if (queryResult != null) { + queryResult.close(); + } + } + } + + return fieldInfos; + } + + private QueryResult fetchRowsFromTable(RichConnection connection, String table, long rowCount) { + String query = null; + + if (sampleSize == -1) { + if (dbType == DbType.MSACCESS) + query = "SELECT * FROM [" + table + "]"; + else if (dbType == DbType.MSSQL || dbType == DbType.PDW) + query = "SELECT * FROM [" + table.replaceAll("\\.", "].[") + "]"; + else + query = "SELECT * FROM " + table; + } else { + if (dbType == DbType.MSSQL) + query = "SELECT * FROM [" + table.replaceAll("\\.", "].[") + "] TABLESAMPLE (" + sampleSize + " ROWS)"; + else if (dbType == DbType.MYSQL) + query = "SELECT * FROM " + table + " ORDER BY RAND() LIMIT " + sampleSize; + else if (dbType == DbType.PDW) + query = "SELECT TOP " + sampleSize + " * FROM [" + table.replaceAll("\\.", "].[") + "] ORDER BY RAND()"; + else if (dbType == DbType.ORACLE) { + if (sampleSize < rowCount) { + double percentage = 100 * sampleSize / (double) rowCount; + if (percentage < 100) + query = "SELECT * FROM " + table + " SAMPLE(" + percentage + ")"; + } else { + query = "SELECT * FROM " + table; + } + } else if (dbType == DbType.POSTGRESQL || dbType == DbType.REDSHIFT) + query = "SELECT * FROM " + table + " ORDER BY RANDOM() LIMIT " + sampleSize; + else if (dbType == DbType.MSACCESS) + query = "SELECT " + "TOP " + sampleSize + " * FROM [" + table + "]"; + else if (dbType == DbType.BIGQUERY) + query = "SELECT * FROM " + table + " ORDER BY RAND() LIMIT " + sampleSize; + } + // System.out.println("SQL: " + query); + return connection.query(query); + + } + + private List fetchTableStructure(RichConnection connection, String table) { + List fieldInfos = new ArrayList(); + + if (dbType == DbType.MSACCESS) { + ResultSet rs = connection.getMsAccessFieldNames(table); + try { + while (rs.next()) { + FieldInfo fieldInfo = new FieldInfo(rs.getString("COLUMN_NAME")); + fieldInfo.type = rs.getString("TYPE_NAME"); + fieldInfo.rowCount = connection.getTableSize(table); + fieldInfos.add(fieldInfo); + } + } catch (SQLException e) { + throw new RuntimeException(e.getMessage()); + } + } else { + String query = null; + if (dbType == DbType.ORACLE) + query = "SELECT COLUMN_NAME,DATA_TYPE FROM ALL_TAB_COLUMNS WHERE table_name = '" + table + "' AND owner = '" + database.toUpperCase() + "'"; + else if (dbType == DbType.MSSQL || dbType == DbType.PDW) { + String trimmedDatabase = database; + if (database.startsWith("[") && database.endsWith("]")) + trimmedDatabase = database.substring(1, database.length() - 1); + String[] parts = table.split("\\."); + query = "SELECT COLUMN_NAME,DATA_TYPE FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_CATALOG='" + trimmedDatabase + "' AND TABLE_SCHEMA='" + parts[0] + + "' AND TABLE_NAME='" + parts[1] + "';"; + } else if (dbType == DbType.MYSQL) + query = "SELECT COLUMN_NAME,DATA_TYPE FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_SCHEMA = '" + database + "' AND TABLE_NAME = '" + table + + "';"; + else if (dbType == DbType.POSTGRESQL || dbType == DbType.REDSHIFT) + query = "SELECT COLUMN_NAME,DATA_TYPE FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_SCHEMA = '" + database.toLowerCase() + "' AND TABLE_NAME = '" + + table.toLowerCase() + "' ORDER BY ordinal_position;"; + else if (dbType == DbType.TERADATA) { + query = "SELECT ColumnName, ColumnType FROM dbc.columns WHERE DatabaseName= '" + database.toLowerCase() + "' AND TableName = '" + + table.toLowerCase() + "';"; + } + else if (dbType == DbType.BIGQUERY) { + query = "SELECT column_name AS COLUMN_NAME, data_type as DATA_TYPE FROM " + database + ".INFORMATION_SCHEMA.COLUMNS WHERE table_name = \"" + table + "\";"; + } + for (org.ohdsi.utilities.files.Row row : connection.query(query)) { + row.upperCaseFieldNames(); + FieldInfo fieldInfo; + if (dbType == DbType.TERADATA) { + fieldInfo = new FieldInfo(row.get("COLUMNNAME")); + } else { + fieldInfo = new FieldInfo(row.get("COLUMN_NAME")); + } + if (dbType == DbType.TERADATA) { + fieldInfo.type = row.get("COLUMNTYPE"); + } else { + fieldInfo.type = row.get("DATA_TYPE"); + } + fieldInfo.rowCount = connection.getTableSize(table); + ; + fieldInfos.add(fieldInfo); + } + } + return fieldInfos; + } + + private List processCsvFile(String filename) { + StringUtilities.outputWithTime("Scanning table " + filename); + List fieldInfos = new ArrayList<>(); + int lineNr = 0; + for (String line : new ReadTextFile(filename)) { + lineNr++; + List row = StringUtilities.safeSplit(line, delimiter); + for (int i = 0; i < row.size(); i++) { + String column = row.get(i); + if (column.startsWith("\"") && column.endsWith("\"") && column.length() > 1) + column = column.substring(1, column.length() - 1); + column = column.replace("\\\"", "\""); + row.set(i, column); + } + if (lineNr == 1) { + for (String cell : row) + fieldInfos.add(new FieldInfo(cell)); + } else { + if (row.size() == fieldInfos.size()) { // Else there appears to be a formatting error, so skip + for (int i = 0; i < row.size(); i++) + fieldInfos.get(i).processValue(row.get(i)); + } + } + if (lineNr == sampleSize) + break; + } + for (FieldInfo fieldInfo : fieldInfos) + fieldInfo.trim(); + + return fieldInfos; + } + + private List processSasFile(String filename) { + StringUtilities.outputWithTime("Scanning table " + filename); + List fieldInfos = new ArrayList<>(); + + try(FileInputStream inputStream = new FileInputStream(new File(filename))) { + SasFileReader sasFileReader = new SasFileReaderImpl(inputStream); + + // It is possible to retrieve more information from the sasFileProperties, like data type and length. + SasFileProperties sasFileProperties = sasFileReader.getSasFileProperties(); + for (Column column : sasFileReader.getColumns()) { + fieldInfos.add(new FieldInfo(column.getName())); + } + + for (int lineNr = 0; lineNr < sasFileProperties.getRowCount(); lineNr++) { + Object[] row = sasFileReader.readNext(); + + if (row.length == fieldInfos.size()) { // Else there appears to be a formatting error, so skip + for (int i = 0; i < row.length; i++) { + fieldInfos.get(i).processValue(row[i] == null ? "" : row[i].toString()); + } + } + if (lineNr == sampleSize) + break; + } + inputStream.close(); + } catch (IOException e) { + e.printStackTrace(); + } + + for (FieldInfo fieldInfo : fieldInfos) { + fieldInfo.trim(); + } + + return fieldInfos; + } + + private class FieldInfo { + public String type; + public String name; + public CountingSet valueCounts = new CountingSet<>(); + public long sumLength = 0; + public int maxLength = 0; + public long nProcessed = 0; + public long emptyCount = 0; + public long uniqueCount = 0; + public long rowCount = -1; + public boolean isInteger = true; + public boolean isReal = true; + public boolean isDate = true; + public boolean isFreeText = false; + public boolean tooManyValues = false; + + public FieldInfo(String name) { + this.name = name; + } + + public void trim() { + if (valueCounts.size() > maxValues) { + valueCounts.keepTopN(maxValues); + } + } + + public boolean hasValuesTrimmed() { + return tooManyValues; + } + + public Double getFractionEmpty() { + if (nProcessed == 0) + return 0d; + else + return emptyCount / (double) nProcessed; + } + + public String getTypeDescription() { + if (type != null) + return type; + else if (nProcessed == emptyCount) + return "empty"; + else if (isFreeText) + return "text"; + else if (isDate) + return "date"; + else if (isInteger) + return "int"; + else if (isReal) + return "real"; + else + return "varchar"; + } + + public Double getFractionUnique() { + if (nProcessed == 0 || uniqueCount == 1) { + return 0d; + } + else { + return uniqueCount / (double) nProcessed; + } + + } + + public void processValue(String value) { + String trimValue = value.trim(); + nProcessed++; + sumLength += value.length(); + if (value.length() > maxLength) + maxLength = value.length(); + + if (trimValue.length() == 0) + emptyCount++; + + if (!isFreeText) { + boolean newlyAdded = valueCounts.add(value); + if (newlyAdded) uniqueCount++; + + if (trimValue.length() != 0) { + if (isReal && !StringUtilities.isNumber(trimValue)) + isReal = false; + if (isInteger && !StringUtilities.isLong(trimValue)) + isInteger = false; + if (isDate && !StringUtilities.isDate(trimValue)) + isDate = false; + } + if (nProcessed == N_FOR_FREE_TEXT_CHECK) { + if (!isInteger && !isReal && !isDate) { + double averageLength = sumLength / (double) (nProcessed - emptyCount); + if (averageLength >= MIN_AVERAGE_LENGTH_FOR_FREE_TEXT) { + isFreeText = true; + CountingSet wordCounts = new CountingSet(); + for (Map.Entry entry : valueCounts.key2count.entrySet()) + for (String word : StringUtilities.mapToWords(entry.getKey().toLowerCase())) + wordCounts.add(word, entry.getValue().count); + valueCounts = wordCounts; + } + } + } + } else { + valueCounts.addAll(StringUtilities.mapToWords(trimValue.toLowerCase())); + } + + if (!tooManyValues && valueCounts.size() > MAX_VALUES_IN_MEMORY) { + tooManyValues = true; + this.trim(); + } + } + + public List> getSortedValuesWithoutSmallValues() { + List> result = valueCounts.key2count.entrySet().stream() + .filter(e -> e.getValue().count >= minCellCount) + .sorted(Comparator.>comparingInt(e -> e.getValue().count).reversed()) + .limit(maxValues) + .map(e -> new Pair<>(e.getKey(), e.getValue().count)) + .collect(Collectors.toCollection(ArrayList::new)); + + if (result.size() < valueCounts.key2count.size()) { + result.add(new Pair<>("List truncated...", -1)); + } + return result; + } + } + + private void addRow(Sheet sheet, Object... values) { + Row row = sheet.createRow(sheet.getPhysicalNumberOfRows()); + for (Object value : values) { + Cell cell = row.createCell(row.getPhysicalNumberOfCells()); + + if (value instanceof Integer || value instanceof Long || value instanceof Double) + cell.setCellValue(Double.parseDouble(value.toString())); + else + cell.setCellValue(value.toString()); + + } + } + + private void setCellStyles(Sheet sheet, CellStyle style, int... colNums) { + Row row = sheet.getRow(sheet.getLastRowNum()); + for(int i : colNums) { + Cell cell = row.getCell(i); + if (cell != null) + cell.setCellStyle(style); + } + } +}