diff --git a/.classpath b/.classpath deleted file mode 100644 index f2c0afa0..00000000 --- a/.classpath +++ /dev/null @@ -1,27 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/.gitignore b/.gitignore index 76ef4f89..46ff8e91 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,6 @@ dist/ /Error1.txt /ScanReport.xlsx /.classpath +target/ +*.iml +.idea/ diff --git a/.project b/.project index ebaa3b16..1f855c0f 100644 --- a/.project +++ b/.project @@ -1,17 +1,17 @@ - WhiteRabbit + leporidae - org.eclipse.jdt.core.javabuilder + org.eclipse.m2e.core.maven2Builder - org.eclipse.jdt.core.javanature + org.eclipse.m2e.core.maven2Nature diff --git a/.settings/org.eclipse.m2e.core.prefs b/.settings/org.eclipse.m2e.core.prefs new file mode 100644 index 00000000..f897a7f1 --- /dev/null +++ b/.settings/org.eclipse.m2e.core.prefs @@ -0,0 +1,4 @@ +activeProfiles= +eclipse.preferences.version=1 +resolveWorkspaceProjects=true +version=1 diff --git a/README.md b/README.md index defc9b98..21ef48dd 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -![alt text](https://github.com/OHDSI/WhiteRabbit/blob/master/src/org/ohdsi/whiteRabbit/WhiteRabbit64.png) WhiteRabbit +![alt text](https://github.com/OHDSI/WhiteRabbit/blob/master/whiterabbit/src/main/resources/org/ohdsi/whiteRabbit/WhiteRabbit64.png) WhiteRabbit =========== Introduction @@ -35,32 +35,35 @@ White Rabbit and Rabbit in a Hat are pure Java applications. Both applications u System Requirements ============ -Requires Java 1.7 or higher, and read access to the database to be scanned. Java can be downloaded from +Requires Java 1.8 or higher, and read access to the database to be scanned. Java can be downloaded from http://www.java.com. Dependencies ============ - * There are no dependencies. +For the distributable packages, the only requirement is Java 8. For building the package, also Maven is needed. Getting Started =============== WhiteRabbit -1. Under the [Releases](https://github.com/OHDSI/WhiteRabbit/releases) tab, download WhiteRabbit*.zip +1. Under the [Releases](https://github.com/OHDSI/WhiteRabbit/releases) tab, download `WhiteRabbit*.zip` 2. Unzip the download -3. Double-click on WhiteRabbit.jar to start White Rabbit. +3. Double-click on `bin/whiteRabbit.bat` on Windows to start White Rabbit, and `bin/whiteRabbit` on macOS and Linux. (See the [Wiki](http://www.ohdsi.org/web/wiki/doku.php?id=documentation:software:whiterabbit#running_from_the_command_line) for details on how to run from the command prompt instead) Rabbit-In-A-Hat -1. Using the files downloaded for WhiteRabbit, double-click on RabbitInAHat.jar to start Rabbit-In-A-Hat. +1. Using the files downloaded for WhiteRabbit, double-click on `bin/rabbitInAHat.bat` to start Rabbit-In-A-Hat on Windows, and `bin/rabbitInAHat` on macOS and Linux. + +Note: on releases earlier than version 0.8.0, open the respective `WhiteRabbit.jar` or `RabbitInAHat.jar` files instead. Getting Involved ============= * User guide and Help: WhiteRabbit Wiki * Developer questions/comments/feedback: OHDSI Forum * We use the GitHub issue tracker for all bugs/issues/enhancements +* Historically, all files have CRLF line endings. Please configure your IDE and local git to keep line endings as is. This avoids merge conflicts. License ======= @@ -68,7 +71,9 @@ WhiteRabbit is licensed under Apache License 2.0 Development =========== -White Rabbit and Rabbit in a Hat are being developed in Eclipse. Contributions are welcome. +White Rabbit and Rabbit in a Hat are structured as a Maven package and can be developed in Eclipse. Contributions are welcome. + +To generate the files ready for distribution, run `mvn install`. ### Development status diff --git a/build.xml b/build.xml deleted file mode 100644 index 733c19e9..00000000 --- a/build.xml +++ /dev/null @@ -1,66 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/lib/RedshiftJDBC4-1.1.10.1010.jar b/lib/RedshiftJDBC4-1.1.10.1010.jar deleted file mode 100644 index 52b1cf81..00000000 Binary files a/lib/RedshiftJDBC4-1.1.10.1010.jar and /dev/null differ diff --git a/lib/terajdbc4.jar b/lib/com/teradata/jdbc/terajdbc4/16.0.0.32/terajdbc4-16.0.0.32.jar old mode 100755 new mode 100644 similarity index 100% rename from lib/terajdbc4.jar rename to lib/com/teradata/jdbc/terajdbc4/16.0.0.32/terajdbc4-16.0.0.32.jar diff --git a/lib/com/teradata/jdbc/terajdbc4/16.0.0.32/terajdbc4-16.0.0.32.pom b/lib/com/teradata/jdbc/terajdbc4/16.0.0.32/terajdbc4-16.0.0.32.pom new file mode 100644 index 00000000..19b6b4a2 --- /dev/null +++ b/lib/com/teradata/jdbc/terajdbc4/16.0.0.32/terajdbc4-16.0.0.32.pom @@ -0,0 +1,9 @@ + + + 4.0.0 + com.teradata.jdbc + terajdbc4 + 16.0.0.32 + POM was created from install:install-file + diff --git a/lib/com/teradata/jdbc/terajdbc4/maven-metadata-local.xml b/lib/com/teradata/jdbc/terajdbc4/maven-metadata-local.xml new file mode 100644 index 00000000..61416a7e --- /dev/null +++ b/lib/com/teradata/jdbc/terajdbc4/maven-metadata-local.xml @@ -0,0 +1,12 @@ + + + com.teradata.jdbc + terajdbc4 + + 16.0.0.32 + + 16.0.0.32 + + 20190507142405 + + diff --git a/lib/tdgssconfig.jar b/lib/com/teradata/tdgss/tdgssconfig/16.0.0.0/tdgssconfig-16.0.0.0.jar old mode 100755 new mode 100644 similarity index 100% rename from lib/tdgssconfig.jar rename to lib/com/teradata/tdgss/tdgssconfig/16.0.0.0/tdgssconfig-16.0.0.0.jar diff --git a/lib/com/teradata/tdgss/tdgssconfig/16.0.0.0/tdgssconfig-16.0.0.0.pom b/lib/com/teradata/tdgss/tdgssconfig/16.0.0.0/tdgssconfig-16.0.0.0.pom new file mode 100644 index 00000000..50cfbf09 --- /dev/null +++ b/lib/com/teradata/tdgss/tdgssconfig/16.0.0.0/tdgssconfig-16.0.0.0.pom @@ -0,0 +1,9 @@ + + + 4.0.0 + com.teradata.tdgss + tdgssconfig + 16.0.0.0 + POM was created from install:install-file + diff --git a/lib/com/teradata/tdgss/tdgssconfig/maven-metadata-local.xml b/lib/com/teradata/tdgss/tdgssconfig/maven-metadata-local.xml new file mode 100644 index 00000000..279b440c --- /dev/null +++ b/lib/com/teradata/tdgss/tdgssconfig/maven-metadata-local.xml @@ -0,0 +1,12 @@ + + + com.teradata.tdgss + tdgssconfig + + 16.0.0.0 + + 16.0.0.0 + + 20190507142309 + + diff --git a/lib/commons-csv-1.1.jar b/lib/commons-csv-1.1.jar deleted file mode 100644 index 89bd547f..00000000 Binary files a/lib/commons-csv-1.1.jar and /dev/null differ diff --git a/lib/commons-lang-2.0.jar b/lib/commons-lang-2.0.jar deleted file mode 100644 index c8a28701..00000000 Binary files a/lib/commons-lang-2.0.jar and /dev/null differ diff --git a/lib/commons-logging-1.2.jar b/lib/commons-logging-1.2.jar deleted file mode 100644 index 93a3b9f6..00000000 Binary files a/lib/commons-logging-1.2.jar and /dev/null differ diff --git a/lib/dom4j-1.6.1.jar b/lib/dom4j-1.6.1.jar deleted file mode 100644 index c8c4dbb9..00000000 Binary files a/lib/dom4j-1.6.1.jar and /dev/null differ diff --git a/lib/hsqldb-2.2.5.jar b/lib/hsqldb-2.2.5.jar deleted file mode 100644 index d5bc2bb6..00000000 Binary files a/lib/hsqldb-2.2.5.jar and /dev/null differ diff --git a/lib/jackcess-2.1.3.jar b/lib/jackcess-2.1.3.jar deleted file mode 100644 index 848aad3e..00000000 Binary files a/lib/jackcess-2.1.3.jar and /dev/null differ diff --git a/lib/json-io-3.0.1.jar b/lib/json-io-3.0.1.jar deleted file mode 100644 index 3ac6d696..00000000 Binary files a/lib/json-io-3.0.1.jar and /dev/null differ diff --git a/lib/mysql-connector-java-5.1.37-bin.jar b/lib/mysql-connector-java-5.1.37-bin.jar deleted file mode 100644 index 465af670..00000000 Binary files a/lib/mysql-connector-java-5.1.37-bin.jar and /dev/null differ diff --git a/lib/ojdbc14.jar b/lib/ojdbc14.jar deleted file mode 100644 index 05adfefa..00000000 Binary files a/lib/ojdbc14.jar and /dev/null differ diff --git a/lib/poi-3.9-20121203.jar b/lib/poi-3.9-20121203.jar deleted file mode 100644 index 0f462880..00000000 Binary files a/lib/poi-3.9-20121203.jar and /dev/null differ diff --git a/lib/poi-excelant-3.9-20121203.jar b/lib/poi-excelant-3.9-20121203.jar deleted file mode 100644 index cb403fa6..00000000 Binary files a/lib/poi-excelant-3.9-20121203.jar and /dev/null differ diff --git a/lib/poi-ooxml-3.9-20121203.jar b/lib/poi-ooxml-3.9-20121203.jar deleted file mode 100644 index 8792d463..00000000 Binary files a/lib/poi-ooxml-3.9-20121203.jar and /dev/null differ diff --git a/lib/poi-ooxml-schemas-3.9-20121203.jar b/lib/poi-ooxml-schemas-3.9-20121203.jar deleted file mode 100644 index eda4ef49..00000000 Binary files a/lib/poi-ooxml-schemas-3.9-20121203.jar and /dev/null differ diff --git a/lib/postgresql-9.4-1201.jdbc4.jar b/lib/postgresql-9.4-1201.jdbc4.jar deleted file mode 100644 index 0cd5ce20..00000000 Binary files a/lib/postgresql-9.4-1201.jdbc4.jar and /dev/null differ diff --git a/lib/postgresql-9.4-1201.jdbc41.jar b/lib/postgresql-9.4-1201.jdbc41.jar deleted file mode 100644 index a3f19593..00000000 Binary files a/lib/postgresql-9.4-1201.jdbc41.jar and /dev/null differ diff --git a/lib/sqljdbc4.jar b/lib/sqljdbc4.jar deleted file mode 100644 index d6b7f6da..00000000 Binary files a/lib/sqljdbc4.jar and /dev/null differ diff --git a/lib/stax-api-1.0.1.jar b/lib/stax-api-1.0.1.jar deleted file mode 100644 index d9a16651..00000000 Binary files a/lib/stax-api-1.0.1.jar and /dev/null differ diff --git a/lib/ucanaccess-3.0.3.1.jar b/lib/ucanaccess-3.0.3.1.jar deleted file mode 100644 index 3d7ddd51..00000000 Binary files a/lib/ucanaccess-3.0.3.1.jar and /dev/null differ diff --git a/lib/xmlbeans-2.3.0.jar b/lib/xmlbeans-2.3.0.jar deleted file mode 100644 index ccd81634..00000000 Binary files a/lib/xmlbeans-2.3.0.jar and /dev/null differ diff --git a/pom.xml b/pom.xml new file mode 100644 index 00000000..667b9947 --- /dev/null +++ b/pom.xml @@ -0,0 +1,133 @@ + + 4.0.0 + org.ohdsi + leporidae + pom + 0.8.0-SNAPSHOT + + rabbitinahat + whiterabbit + rabbit-core + + Leporidae + https://www.ohdsi.org/analytic-tools/whiterabbit-for-etl-design/ + + + + central + Central Repository + https://repo.maven.apache.org/maven2 + default + + false + + + + clojars + Clojars Repository + http://clojars.org/repo/ + default + + false + + + + icm + ICM Repository + https://maven.ceon.pl/artifactory/ + default + + false + + + + mulesoft + Mulesoft Repository + https://repository.mulesoft.org/nexus/content/repositories/public/ + default + + false + + + + jahia + Geomajas Repository + http://maven.geomajas.org/ + default + + false + + + + local-maven-repo + Local Repository + default + file:///${project.basedir}/../lib + + + + + + central + Central Repository + https://repo.maven.apache.org/maven2 + default + + false + + + never + + + + + + 1.8 + 1.8 + + + + + + + org.codehaus.mojo + appassembler-maven-plugin + 1.10 + + + + + + + jar:jar + + assemble + + + + + + + + + + maven-clean-plugin + 3.1.0 + + + + dist + + ** + + false + + + + + + + diff --git a/rabbit-core/.classpath b/rabbit-core/.classpath new file mode 100644 index 00000000..af1430be --- /dev/null +++ b/rabbit-core/.classpath @@ -0,0 +1,26 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/rabbit-core/.project b/rabbit-core/.project new file mode 100644 index 00000000..8a6ded5e --- /dev/null +++ b/rabbit-core/.project @@ -0,0 +1,23 @@ + + + rabbit-core + + + + + + org.eclipse.jdt.core.javabuilder + + + + + org.eclipse.m2e.core.maven2Builder + + + + + + org.eclipse.jdt.core.javanature + org.eclipse.m2e.core.maven2Nature + + diff --git a/rabbit-core/.settings/org.eclipse.jdt.core.prefs b/rabbit-core/.settings/org.eclipse.jdt.core.prefs new file mode 100644 index 00000000..714351ae --- /dev/null +++ b/rabbit-core/.settings/org.eclipse.jdt.core.prefs @@ -0,0 +1,5 @@ +eclipse.preferences.version=1 +org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.8 +org.eclipse.jdt.core.compiler.compliance=1.8 +org.eclipse.jdt.core.compiler.problem.forbiddenReference=warning +org.eclipse.jdt.core.compiler.source=1.8 diff --git a/rabbit-core/.settings/org.eclipse.m2e.core.prefs b/rabbit-core/.settings/org.eclipse.m2e.core.prefs new file mode 100644 index 00000000..f897a7f1 --- /dev/null +++ b/rabbit-core/.settings/org.eclipse.m2e.core.prefs @@ -0,0 +1,4 @@ +activeProfiles= +eclipse.preferences.version=1 +resolveWorkspaceProjects=true +version=1 diff --git a/rabbit-core/pom.xml b/rabbit-core/pom.xml new file mode 100644 index 00000000..41d29da7 --- /dev/null +++ b/rabbit-core/pom.xml @@ -0,0 +1,122 @@ + + + + leporidae + org.ohdsi + 0.8.0-SNAPSHOT + + 4.0.0 + + rabbit-core + jar + + + + com.oracle.jdbc + ojdbc14 + 10.2.0.1.0 + + + com.microsoft.sqlserver + sqljdbc4 + 4.0 + + + mysql + mysql-connector-java + 5.1.37 + + + dom4j + dom4j + 1.6.1 + + + org.apache.poi + poi + 3.9 + + + org.apache.poi + poi-ooxml + 3.9 + + + org.apache.poi + poi-excelant + 3.9 + + + org.apache.poi + poi-ooxml-schemas + 3.9 + + + stax + stax-api + 1.0.1 + + + org.apache.xmlbeans + xmlbeans + 2.3.0 + + + org.postgresql + postgresql + 9.4-1204-jdbc41 + + + com.cedarsoftware + json-io + 3.0.1 + + + org.apache.commons + commons-csv + 1.1 + + + commons-lang + commons-lang + 2.0 + + + commons-logging + commons-logging + 1.2 + + + org.hsqldb + hsqldb + 2.2.6 + + + com.healthmarketscience.jackcess + jackcess + 2.1.3 + + + net.sf.ucanaccess + ucanaccess + 3.0.3.1 + + + com.amazon.redshift + redshift-jdbc41 + 1.1.10.1010 + + + com.teradata.jdbc + terajdbc4 + 16.0.0.32 + + + com.teradata.tdgss + tdgssconfig + 16.0.0.0 + + + \ No newline at end of file diff --git a/src/org/ohdsi/databases/DBConnector.java b/rabbit-core/src/main/java/org/ohdsi/databases/DBConnector.java similarity index 96% rename from src/org/ohdsi/databases/DBConnector.java rename to rabbit-core/src/main/java/org/ohdsi/databases/DBConnector.java index 5689a69b..e44b07a5 100644 --- a/src/org/ohdsi/databases/DBConnector.java +++ b/rabbit-core/src/main/java/org/ohdsi/databases/DBConnector.java @@ -1,219 +1,219 @@ -/******************************************************************************* - * Copyright 2017 Observational Health Data Sciences and Informatics - * - * This file is part of WhiteRabbit - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - ******************************************************************************/ -package org.ohdsi.databases; - -import java.sql.Connection; -import java.sql.DriverManager; -import java.sql.SQLException; - -import oracle.jdbc.pool.OracleDataSource; - -public class DBConnector { - - public static void main(String[] args) { - } - - public static Connection connect(String server, String domain, String user, String password, DbType dbType) { - if (dbType.equals(DbType.MYSQL)) - return DBConnector.connectToMySQL(server, user, password); - else if (dbType.equals(DbType.MSSQL) || dbType.equals(DbType.PDW)) - return DBConnector.connectToMSSQL(server, domain, user, password); - else if (dbType.equals(DbType.ORACLE)) - return DBConnector.connectToOracle(server, domain, user, password); - else if (dbType.equals(DbType.POSTGRESQL)) - return DBConnector.connectToPostgreSQL(server, user, password); - else if (dbType.equals(DbType.MSACCESS)) - return DBConnector.connectToMsAccess(server, user, password); - else if (dbType.equals(DbType.REDSHIFT)) - return DBConnector.connectToRedshift(server, user, password); - else if (dbType.equals(DbType.TERADATA)) - return DBConnector.connectToTeradata(server, user, password); - else - return null; - } - - public static Connection connectToTeradata(String server, String user, String password) { - try { - Class.forName("com.teradata.jdbc.TeraDriver"); - } catch(ClassNotFoundException e) { - throw new RuntimeException("Cannot find JDBC driver. Make sure the terajdbc4.jar and tdgssconfig.jar are in the path"); - } - String url = "jdbc:teradata://" + server; - try { - return DriverManager.getConnection(url, user, password); - } catch (SQLException e1) { - throw new RuntimeException("Cannot connect to DB server: " + e1.getMessage()); - } - } - - public static Connection connectToRedshift(String server, String user, String password) { - if (!server.contains("/")) - throw new RuntimeException("For Redshift, database name must be specified in the server field (:/?)"); - try { - Class.forName("com.amazon.redshift.jdbc4.Driver"); - } catch (ClassNotFoundException e1) { - throw new RuntimeException("Cannot find JDBC driver. Make sure the file RedshiftJDBCx-x.x.xx.xxxx.jar is in the path"); - } - String url = "jdbc:redshift://" + server; - try { - return DriverManager.getConnection(url, user, password); - } catch (SQLException e1) { - throw new RuntimeException("Cannot connect to DB server: " + e1.getMessage()); - } - } - - public static Connection connectToMsAccess(String server, String user, String password) { - try { - Class.forName("net.ucanaccess.jdbc.UcanaccessDriver"); - } catch (ClassNotFoundException e) { - throw new RuntimeException("Cannot find ucanaccess driver. Make sure the file ucanaccess-3.0.3.1.jar is in the path"); - } - String url = "jdbc:ucanaccess://" + server + ";sysschema=true"; - try { - return DriverManager.getConnection(url, user, password); - } catch (SQLException e) { - throw new RuntimeException("Cannot connect to DB server: " + e.getMessage()); - } - } - - public static Connection connectToPostgreSQL(String server, String user, String password) { - if (!server.contains("/")) - throw new RuntimeException("For PostgreSQL, database name must be specified in the server field (/)"); - if (!server.contains(":")) - server = server.replace("/", ":5432/"); - try { - Class.forName("org.postgresql.Driver"); - } catch (ClassNotFoundException e1) { - throw new RuntimeException("Cannot find JDBC driver. Make sure the file postgresql-x.x-xxxx.jdbcx.jar is in the path"); - } - String url = "jdbc:postgresql://" + server; - try { - return DriverManager.getConnection(url, user, password); - } catch (SQLException e1) { - throw new RuntimeException("Cannot connect to DB server: " + e1.getMessage()); - } - } - - public static Connection connectToMySQL(String server, String user, String password) { - try { - Class.forName("com.mysql.jdbc.Driver"); - } catch (ClassNotFoundException e1) { - throw new RuntimeException("Cannot find JDBC driver. Make sure the file mysql-connector-java-x.x.xx-bin.jar is in the path"); - } - - String url = "jdbc:mysql://" + server + ":3306/?useCursorFetch=true&zeroDateTimeBehavior=convertToNull"; - - try { - return DriverManager.getConnection(url, user, password); - } catch (SQLException e1) { - throw new RuntimeException("Cannot connect to DB server: " + e1.getMessage()); - } - } - - public static Connection connectToODBC(String server, String user, String password) { - try { - Class.forName("sun.jdbc.odbc.JdbcOdbcDriver"); - } catch (ClassNotFoundException e1) { - throw new RuntimeException("Cannot find ODBC driver"); - } - - String url = "jdbc:odbc:" + server; - - try { - Connection connection = DriverManager.getConnection(url, user, password); - - return connection; - } catch (SQLException e1) { - throw new RuntimeException("Cannot connect to DB server: " + e1.getMessage()); - } - } - - /* - * public static Connection connectToMSSQL(String server, String domain, String user, String password) { try { - * Class.forName("net.sourceforge.jtds.jdbc.Driver"); - * - * } catch (ClassNotFoundException e1) { throw new RuntimeException("Cannot find JDBC driver. Make sure the file sqljdbc4.jar is in the path"); } - * - * String url = "jdbc:jtds:sqlserver://"+server+(domain.length()==0?"":";domain="+domain); - * - * try { return DriverManager.getConnection(url,user, password); } catch (SQLException e1) { throw new RuntimeException("Cannot connect to DB server: " + - * e1.getMessage()); } } - */ - public static Connection connectToMSSQL(String server, String domain, String user, String password) { - try { - Class.forName("com.microsoft.sqlserver.jdbc.SQLServerDriver"); - } catch (ClassNotFoundException e1) { - throw new RuntimeException("Cannot find JDBC driver. Make sure the file sqljdbc4.jar is in the path"); - } - String url = "jdbc:sqlserver://" + server; - if (user == null || user.length() == 0) { // Use Windows integrated security - url = url + ";integratedSecurity=true"; - } - try { - return DriverManager.getConnection(url, user, password); - } catch (SQLException e1) { - throw new RuntimeException("Cannot connect to DB server: " + e1.getMessage()); - } - } - - public static Connection connectToOracle(String server, String domain, String user, String password) { - try { - Class.forName("oracle.jdbc.driver.OracleDriver"); - } catch (ClassNotFoundException e) { - throw new RuntimeException("Class not found exception: " + e.getMessage()); - } - // First try OCI driver: - String error = null; - try { - OracleDataSource ods; - ods = new OracleDataSource(); - ods.setURL("jdbc:oracle:oci8:@" + server); - ods.setUser(user); - ods.setPassword(password); - return ods.getConnection(); - } catch (UnsatisfiedLinkError e) { - error = e.getMessage(); - } catch (SQLException e) { - error = e.getMessage(); - } - // If fails, try THIN driver: - if (error != null) - try { - String host = "127.0.0.1"; - String sid = server; - String port = "1521"; - if (server.contains("/")) { - host = server.split("/")[0]; - if (host.contains(":")) { - port = host.split(":")[1]; - host = host.split(":")[0]; - } - sid = server.split("/")[1]; - } - OracleDataSource ods; - ods = new OracleDataSource(); - ods.setURL("jdbc:oracle:thin:@" + host + ":" + port + ":" + sid); - ods.setUser(user); - ods.setPassword(password); - return ods.getConnection(); - } catch (SQLException e) { - throw new RuntimeException("Cannot connect to DB server:\n- When using OCI: " + error + "\n- When using THIN: " + e.getMessage()); - } - return null; - } -} +/******************************************************************************* + * Copyright 2019 Observational Health Data Sciences and Informatics + * + * This file is part of WhiteRabbit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + ******************************************************************************/ +package org.ohdsi.databases; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; + +import oracle.jdbc.pool.OracleDataSource; + +public class DBConnector { + + public static void main(String[] args) { + } + + public static Connection connect(String server, String domain, String user, String password, DbType dbType) { + if (dbType.equals(DbType.MYSQL)) + return DBConnector.connectToMySQL(server, user, password); + else if (dbType.equals(DbType.MSSQL) || dbType.equals(DbType.PDW)) + return DBConnector.connectToMSSQL(server, domain, user, password); + else if (dbType.equals(DbType.ORACLE)) + return DBConnector.connectToOracle(server, domain, user, password); + else if (dbType.equals(DbType.POSTGRESQL)) + return DBConnector.connectToPostgreSQL(server, user, password); + else if (dbType.equals(DbType.MSACCESS)) + return DBConnector.connectToMsAccess(server, user, password); + else if (dbType.equals(DbType.REDSHIFT)) + return DBConnector.connectToRedshift(server, user, password); + else if (dbType.equals(DbType.TERADATA)) + return DBConnector.connectToTeradata(server, user, password); + else + return null; + } + + public static Connection connectToTeradata(String server, String user, String password) { + try { + Class.forName("com.teradata.jdbc.TeraDriver"); + } catch(ClassNotFoundException e) { + throw new RuntimeException("Cannot find JDBC driver. Make sure the terajdbc4.jar and tdgssconfig.jar are in the path"); + } + String url = "jdbc:teradata://" + server; + try { + return DriverManager.getConnection(url, user, password); + } catch (SQLException e1) { + throw new RuntimeException("Cannot connect to DB server: " + e1.getMessage()); + } + } + + public static Connection connectToRedshift(String server, String user, String password) { + if (!server.contains("/")) + throw new RuntimeException("For Redshift, database name must be specified in the server field (:/?)"); + try { + Class.forName("com.amazon.redshift.jdbc41.Driver"); + } catch (ClassNotFoundException e1) { + throw new RuntimeException("Cannot find JDBC driver. Make sure the file RedshiftJDBCx-x.x.xx.xxxx.jar is in the path"); + } + String url = "jdbc:redshift://" + server; + try { + return DriverManager.getConnection(url, user, password); + } catch (SQLException e1) { + throw new RuntimeException("Cannot connect to DB server: " + e1.getMessage()); + } + } + + public static Connection connectToMsAccess(String server, String user, String password) { + try { + Class.forName("net.ucanaccess.jdbc.UcanaccessDriver"); + } catch (ClassNotFoundException e) { + throw new RuntimeException("Cannot find ucanaccess driver. Make sure the file ucanaccess-3.0.3.1.jar is in the path"); + } + String url = "jdbc:ucanaccess://" + server + ";sysschema=true"; + try { + return DriverManager.getConnection(url, user, password); + } catch (SQLException e) { + throw new RuntimeException("Cannot connect to DB server: " + e.getMessage()); + } + } + + public static Connection connectToPostgreSQL(String server, String user, String password) { + if (!server.contains("/")) + throw new RuntimeException("For PostgreSQL, database name must be specified in the server field (/)"); + if (!server.contains(":")) + server = server.replace("/", ":5432/"); + try { + Class.forName("org.postgresql.Driver"); + } catch (ClassNotFoundException e1) { + throw new RuntimeException("Cannot find JDBC driver. Make sure the file postgresql-x.x-xxxx.jdbcx.jar is in the path"); + } + String url = "jdbc:postgresql://" + server; + try { + return DriverManager.getConnection(url, user, password); + } catch (SQLException e1) { + throw new RuntimeException("Cannot connect to DB server: " + e1.getMessage()); + } + } + + public static Connection connectToMySQL(String server, String user, String password) { + try { + Class.forName("com.mysql.jdbc.Driver"); + } catch (ClassNotFoundException e1) { + throw new RuntimeException("Cannot find JDBC driver. Make sure the file mysql-connector-java-x.x.xx-bin.jar is in the path"); + } + + String url = "jdbc:mysql://" + server + ":3306/?useCursorFetch=true&zeroDateTimeBehavior=convertToNull"; + + try { + return DriverManager.getConnection(url, user, password); + } catch (SQLException e1) { + throw new RuntimeException("Cannot connect to DB server: " + e1.getMessage()); + } + } + + public static Connection connectToODBC(String server, String user, String password) { + try { + Class.forName("sun.jdbc.odbc.JdbcOdbcDriver"); + } catch (ClassNotFoundException e1) { + throw new RuntimeException("Cannot find ODBC driver"); + } + + String url = "jdbc:odbc:" + server; + + try { + Connection connection = DriverManager.getConnection(url, user, password); + + return connection; + } catch (SQLException e1) { + throw new RuntimeException("Cannot connect to DB server: " + e1.getMessage()); + } + } + + /* + * public static Connection connectToMSSQL(String server, String domain, String user, String password) { try { + * Class.forName("net.sourceforge.jtds.jdbc.Driver"); + * + * } catch (ClassNotFoundException e1) { throw new RuntimeException("Cannot find JDBC driver. Make sure the file sqljdbc4.jar is in the path"); } + * + * String url = "jdbc:jtds:sqlserver://"+server+(domain.length()==0?"":";domain="+domain); + * + * try { return DriverManager.getConnection(url,user, password); } catch (SQLException e1) { throw new RuntimeException("Cannot connect to DB server: " + + * e1.getMessage()); } } + */ + public static Connection connectToMSSQL(String server, String domain, String user, String password) { + try { + Class.forName("com.microsoft.sqlserver.jdbc.SQLServerDriver"); + } catch (ClassNotFoundException e1) { + throw new RuntimeException("Cannot find JDBC driver. Make sure the file sqljdbc4.jar is in the path"); + } + String url = "jdbc:sqlserver://" + server; + if (user == null || user.length() == 0) { // Use Windows integrated security + url = url + ";integratedSecurity=true"; + } + try { + return DriverManager.getConnection(url, user, password); + } catch (SQLException e1) { + throw new RuntimeException("Cannot connect to DB server: " + e1.getMessage()); + } + } + + public static Connection connectToOracle(String server, String domain, String user, String password) { + try { + Class.forName("oracle.jdbc.driver.OracleDriver"); + } catch (ClassNotFoundException e) { + throw new RuntimeException("Class not found exception: " + e.getMessage()); + } + // First try OCI driver: + String error = null; + try { + OracleDataSource ods; + ods = new OracleDataSource(); + ods.setURL("jdbc:oracle:oci8:@" + server); + ods.setUser(user); + ods.setPassword(password); + return ods.getConnection(); + } catch (UnsatisfiedLinkError e) { + error = e.getMessage(); + } catch (SQLException e) { + error = e.getMessage(); + } + // If fails, try THIN driver: + if (error != null) + try { + String host = "127.0.0.1"; + String sid = server; + String port = "1521"; + if (server.contains("/")) { + host = server.split("/")[0]; + if (host.contains(":")) { + port = host.split(":")[1]; + host = host.split(":")[0]; + } + sid = server.split("/")[1]; + } + OracleDataSource ods; + ods = new OracleDataSource(); + ods.setURL("jdbc:oracle:thin:@" + host + ":" + port + ":" + sid); + ods.setUser(user); + ods.setPassword(password); + return ods.getConnection(); + } catch (SQLException e) { + throw new RuntimeException("Cannot connect to DB server:\n- When using OCI: " + error + "\n- When using THIN: " + e.getMessage()); + } + return null; + } +} diff --git a/src/org/ohdsi/databases/DbType.java b/rabbit-core/src/main/java/org/ohdsi/databases/DbType.java similarity index 93% rename from src/org/ohdsi/databases/DbType.java rename to rabbit-core/src/main/java/org/ohdsi/databases/DbType.java index 236dd74e..4d995c84 100644 --- a/src/org/ohdsi/databases/DbType.java +++ b/rabbit-core/src/main/java/org/ohdsi/databases/DbType.java @@ -1,5 +1,5 @@ /******************************************************************************* - * Copyright 2017 Observational Health Data Sciences and Informatics + * Copyright 2019 Observational Health Data Sciences and Informatics * * This file is part of WhiteRabbit * diff --git a/src/org/ohdsi/databases/RichConnection.java b/rabbit-core/src/main/java/org/ohdsi/databases/RichConnection.java similarity index 90% rename from src/org/ohdsi/databases/RichConnection.java rename to rabbit-core/src/main/java/org/ohdsi/databases/RichConnection.java index 48984acf..65b26f16 100644 --- a/src/org/ohdsi/databases/RichConnection.java +++ b/rabbit-core/src/main/java/org/ohdsi/databases/RichConnection.java @@ -1,525 +1,526 @@ -/******************************************************************************* - * Copyright 2017 Observational Health Data Sciences and Informatics - * - * This file is part of WhiteRabbit - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - ******************************************************************************/ -package org.ohdsi.databases; - -import java.sql.BatchUpdateException; -import java.sql.Connection; -import java.sql.DatabaseMetaData; -import java.sql.PreparedStatement; -import java.sql.ResultSet; -import java.sql.ResultSetMetaData; -import java.sql.SQLException; -import java.sql.Statement; -import java.sql.Types; -import java.text.DecimalFormat; -import java.util.ArrayList; -import java.util.HashSet; -import java.util.Iterator; -import java.util.List; -import java.util.Set; - -import org.ohdsi.utilities.SimpleCounter; -import org.ohdsi.utilities.StringUtilities; -import org.ohdsi.utilities.files.Row; -import org.ohdsi.utilities.files.WriteCSVFileWithHeader; - -public class RichConnection { - public static int INSERT_BATCH_SIZE = 100000; - private Connection connection; - private boolean verbose = false; - private static DecimalFormat decimalFormat = new DecimalFormat("#.#"); - private DbType dbType; - - public RichConnection(String server, String domain, String user, String password, DbType dbType) { - this.connection = DBConnector.connect(server, domain, user, password, dbType); - this.dbType = dbType; - } - - /** - * Execute the given SQL statement. - * - * @param sql - */ - public void execute(String sql) { - Statement statement = null; - try { - if (sql.length() == 0) - return; - - statement = connection.createStatement(ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY); - for (String subQuery : sql.split(";")) { - if (verbose) { - String abbrSQL = subQuery.replace('\n', ' ').replace('\t', ' ').trim(); - if (abbrSQL.length() > 100) - abbrSQL = abbrSQL.substring(0, 100).trim() + "..."; - System.out.println("Adding query to batch: " + abbrSQL); - } - - statement.addBatch(subQuery); - } - long start = System.currentTimeMillis(); - if (verbose) - System.out.println("Executing batch"); - statement.executeBatch(); - if (verbose) - outputQueryStats(statement, System.currentTimeMillis() - start); - } catch (SQLException e) { - System.err.println(sql); - e.printStackTrace(); - } finally { - if (statement != null) { - try { - statement.close(); - } catch (SQLException e) { - // TODO Auto-generated catch block - System.err.println(e.getMessage()); - } - } - } - } - - private void outputQueryStats(Statement statement, long ms) throws SQLException { - Throwable warning = statement.getWarnings(); - if (warning != null) - System.out.println("- SERVER: " + warning.getMessage()); - String timeString; - if (ms < 1000) - timeString = ms + " ms"; - else if (ms < 60000) - timeString = decimalFormat.format(ms / 1000d) + " seconds"; - else if (ms < 3600000) - timeString = decimalFormat.format(ms / 60000d) + " minutes"; - else - timeString = decimalFormat.format(ms / 3600000d) + " hours"; - System.out.println("- Query completed in " + timeString); - } - - /** - * Query the database using the provided SQL statement. - * - * @param sql - * @return - */ - public QueryResult query(String sql) { - return new QueryResult(sql); - } - - /** - * Switch the database to use. - * - * @param database - */ - public void use(String database) { - if (database == null) - return; - if (dbType == DbType.ORACLE) - execute("ALTER SESSION SET current_schema = " + database); - else if (dbType == DbType.POSTGRESQL || dbType == DbType.REDSHIFT) - execute("SET search_path TO " + database); - else if (dbType == DbType.MSACCESS) - ; // NOOP - else if (dbType == DbType.TERADATA) { - execute("database " + database); - } else - execute("USE " + database); - } - - public List getTableNames(String database) { - List names = new ArrayList(); - String query = null; - if (dbType == DbType.MYSQL) { - query = "SHOW TABLES IN " + database; - } else if (dbType == DbType.MSSQL || dbType == DbType.PDW) { - query = "SELECT name FROM " + database + ".sys.tables ORDER BY name"; - } else if (dbType == DbType.ORACLE) { - query = "SELECT table_name FROM all_tables WHERE owner='" + database.toUpperCase() + "'"; - } else if (dbType == DbType.POSTGRESQL || dbType == DbType.REDSHIFT) { - query = "SELECT table_name FROM information_schema.tables WHERE table_schema = '" + database.toLowerCase() + "' ORDER BY table_name"; - } else if (dbType == DbType.MSACCESS) { - query = "SELECT Name FROM sys.MSysObjects WHERE Type=1 AND Flags=0;"; - } else if (dbType == DbType.TERADATA) { - query = "SELECT TableName from dbc.tables WHERE tablekind = 'T' and databasename='" + database + "'"; - } - - for (Row row : query(query)) - names.add(row.get(row.getFieldNames().get(0))); - return names; - } - - public List getFieldNames(String table) { - List names = new ArrayList(); - if (dbType == DbType.MSSQL || dbType == DbType.PDW) { - for (Row row : query("SELECT name FROM syscolumns WHERE id=OBJECT_ID('" + table + "')")) - names.add(row.get("name")); - } else if (dbType == DbType.MYSQL) - for (Row row : query("SHOW COLUMNS FROM " + table)) - names.add(row.get("COLUMN_NAME")); - else - throw new RuntimeException("DB type not supported"); - - return names; - } - - public ResultSet getMsAccessFieldNames(String table) { - if (dbType == DbType.MSACCESS) { - try { - DatabaseMetaData metadata = connection.getMetaData(); - return metadata.getColumns(null, null, table, null); - } catch (SQLException e) { - throw new RuntimeException(e.getMessage()); - } - } else - throw new RuntimeException("DB is not of type MS Access"); - } - - /** - * Returns the row count of the specified table. - * - * @param tableName - * @return - */ - public long getTableSize(String tableName) { - QueryResult qr = null; - Long returnVal = null; - if (dbType == DbType.MSSQL || dbType == DbType.PDW) - qr = query("SELECT COUNT_BIG(*) FROM [" + tableName + "];"); - else if (dbType == DbType.MSACCESS) - qr = query("SELECT COUNT(*) FROM [" + tableName + "];"); - else - qr = query("SELECT COUNT(*) FROM " + tableName + ";"); - try { - returnVal = Long.parseLong(qr.iterator().next().getCells().get(0)); - } catch (Exception e) { - throw new RuntimeException(e); - } finally { - if (qr != null) { - qr.close(); - } - } - return returnVal; - } - - /** - * Close the connection to the database. - */ - public void close() { - try { - connection.close(); - } catch (SQLException e) { - e.printStackTrace(); - } - } - - public boolean isVerbose() { - return verbose; - } - - public void setVerbose(boolean verbose) { - this.verbose = verbose; - } - - public class QueryResult implements Iterable { - private String sql; - - private List iterators = new ArrayList(); - - public QueryResult(String sql) { - this.sql = sql; - } - - @Override - public Iterator iterator() { - DBRowIterator iterator = new DBRowIterator(sql); - iterators.add(iterator); - return iterator; - } - - public void close() { - for (DBRowIterator iterator : iterators) { - iterator.close(); - } - } - } - - /** - * Writes the results of a query to the specified file in CSV format. - * - * @param queryResult - * @param filename - */ - public void writeToFile(QueryResult queryResult, String filename) { - WriteCSVFileWithHeader out = new WriteCSVFileWithHeader(filename); - for (Row row : queryResult) - out.write(row); - out.close(); - } - - /** - * Inserts the rows into a table in the database. - * - * @param iterator - * @param tableName - * @param create - * If true, the data format is determined based on the first batch of rows and used to create the table structure. - */ - public void insertIntoTable(Iterator iterator, String table, boolean create) { - List batch = new ArrayList(INSERT_BATCH_SIZE); - - boolean first = true; - SimpleCounter counter = new SimpleCounter(1000000, true); - while (iterator.hasNext()) { - if (batch.size() == INSERT_BATCH_SIZE) { - if (first && create) - createTable(table, batch); - insert(table, batch); - batch.clear(); - first = false; - } - batch.add(iterator.next()); - counter.count(); - } - if (batch.size() != 0) { - if (first && create) - createTable(table, batch); - insert(table, batch); - } - } - - private void insert(String tableName, List rows) { - List columns = null; - columns = rows.get(0).getFieldNames(); - for (int i = 0; i < columns.size(); i++) - columns.set(i, columnNameToSqlName(columns.get(i))); - - String sql = "INSERT INTO " + tableName; - sql = sql + " (" + StringUtilities.join(columns, ",") + ")"; - sql = sql + " VALUES (?"; - for (int i = 1; i < columns.size(); i++) - sql = sql + ",?"; - sql = sql + ")"; - try { - connection.setAutoCommit(false); - PreparedStatement statement = connection.prepareStatement(sql); - for (Row row : rows) { - for (int i = 0; i < columns.size(); i++) { - String value = row.get(columns.get(i)); - if (value == null) - System.out.println(row.toString()); - if (value.length() == 0) - value = null; - // System.out.println(value); - if (dbType == DbType.POSTGRESQL || dbType == DbType.REDSHIFT) // PostgreSQL does not allow unspecified types - statement.setObject(i + 1, value, Types.OTHER); - else if (dbType == DbType.ORACLE) { - if (isDate(value)) { - // System.out.println(value); - statement.setDate(i + 1, java.sql.Date.valueOf(value)); - - } else - statement.setString(i + 1, value); - } else - statement.setString(i + 1, value); - } - statement.addBatch(); - } - statement.executeBatch(); - connection.commit(); - statement.close(); - connection.setAutoCommit(true); - connection.clearWarnings(); - } catch (SQLException e) { - e.printStackTrace(); - if (e instanceof BatchUpdateException) { - System.err.println(((BatchUpdateException) e).getNextException().getMessage()); - } - } - } - - private static boolean isDate(String string) { - if (string != null && string.length() == 10 && string.charAt(4) == '-' && string.charAt(7) == '-') - try { - int year = Integer.parseInt(string.substring(0, 4)); - if (year < 1700 || year > 2200) - return false; - int month = Integer.parseInt(string.substring(5, 7)); - if (month < 1 || month > 12) - return false; - int day = Integer.parseInt(string.substring(8, 10)); - if (day < 1 || day > 31) - return false; - return true; - } catch (Exception e) { - return false; - } - return false; - } - - private Set createTable(String tableName, List rows) { - Set numericFields = new HashSet(); - Row firstRow = rows.get(0); - List fields = new ArrayList(rows.size()); - for (String field : firstRow.getFieldNames()) - fields.add(new FieldInfo(field)); - for (Row row : rows) { - for (FieldInfo fieldInfo : fields) { - String value = row.get(fieldInfo.name); - if (fieldInfo.isNumeric && !StringUtilities.isInteger(value)) - fieldInfo.isNumeric = false; - if (value.length() > fieldInfo.maxLength) - fieldInfo.maxLength = value.length(); - } - } - - StringBuilder sql = new StringBuilder(); - sql.append("CREATE TABLE " + tableName + " (\n"); - for (FieldInfo fieldInfo : fields) { - sql.append(" " + fieldInfo.toString() + ",\n"); - if (fieldInfo.isNumeric) - numericFields.add(fieldInfo.name); - } - sql.append(");"); - execute(sql.toString()); - return numericFields; - } - - private String columnNameToSqlName(String name) { - return name.replaceAll(" ", "_").replace("-", "_").replace(",", "_").replaceAll("_+", "_"); - } - - private class FieldInfo { - public String name; - public boolean isNumeric = true; - public int maxLength = 0; - - public FieldInfo(String name) { - this.name = name; - } - - public String toString() { - if (dbType == DbType.MYSQL) { - if (isNumeric) - return columnNameToSqlName(name) + " int(" + maxLength + ")"; - else if (maxLength > 255) - return columnNameToSqlName(name) + " text"; - else - return columnNameToSqlName(name) + " varchar(255)"; - } else if (dbType == DbType.MSSQL || dbType == DbType.PDW) { - if (isNumeric) { - if (maxLength < 10) - return columnNameToSqlName(name) + " int"; - else - return columnNameToSqlName(name) + " bigint"; - } else if (maxLength > 255) - return columnNameToSqlName(name) + " varchar(max)"; - else - return columnNameToSqlName(name) + " varchar(255)"; - } else - throw new RuntimeException("Create table syntax not specified for type " + dbType); - } - } - - private class DBRowIterator implements Iterator { - - private ResultSet resultSet; - - private boolean hasNext; - - private Set columnNames = new HashSet(); - - public DBRowIterator(String sql) { - Statement statement = null; - try { - sql.trim(); - if (sql.endsWith(";")) - sql = sql.substring(0, sql.length() - 1); - if (verbose) { - String abbrSQL = sql.replace('\n', ' ').replace('\t', ' ').trim(); - if (abbrSQL.length() > 100) - abbrSQL = abbrSQL.substring(0, 100).trim() + "..."; - System.out.println("Executing query: " + abbrSQL); - } - long start = System.currentTimeMillis(); - statement = connection.createStatement(ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY); - resultSet = statement.executeQuery(sql.toString()); - hasNext = resultSet.next(); - if (verbose) - outputQueryStats(statement, System.currentTimeMillis() - start); - } catch (SQLException e) { - System.err.println(sql.toString()); - System.err.println(e.getMessage()); - throw new RuntimeException(e); - } - } - - public void close() { - if (resultSet != null) { - try { - resultSet.close(); - } catch (SQLException e) { - e.printStackTrace(); - } - resultSet = null; - hasNext = false; - } - } - - @Override - public boolean hasNext() { - return hasNext; - } - - @Override - public Row next() { - try { - Row row = new Row(); - ResultSetMetaData metaData; - metaData = resultSet.getMetaData(); - columnNames.clear(); - - for (int i = 1; i < metaData.getColumnCount() + 1; i++) { - String columnName = metaData.getColumnName(i); - if (columnNames.add(columnName)) { - String value; - try { - value = resultSet.getString(i); - } catch (Exception e) { - value = ""; - } - if (value == null) - value = ""; - - row.add(columnName, value.replace(" 00:00:00", "")); - } - } - hasNext = resultSet.next(); - if (!hasNext) { - resultSet.close(); - resultSet = null; - } - return row; - } catch (SQLException e) { - e.printStackTrace(); - throw new RuntimeException(e); - } - } - - @Override - public void remove() { - } - } -} +/******************************************************************************* + * Copyright 2019 Observational Health Data Sciences and Informatics + * + * This file is part of WhiteRabbit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + ******************************************************************************/ +package org.ohdsi.databases; + +import java.io.Closeable; +import java.sql.BatchUpdateException; +import java.sql.Connection; +import java.sql.DatabaseMetaData; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.ResultSetMetaData; +import java.sql.SQLException; +import java.sql.Statement; +import java.sql.Types; +import java.text.DecimalFormat; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Set; + +import org.ohdsi.utilities.SimpleCounter; +import org.ohdsi.utilities.StringUtilities; +import org.ohdsi.utilities.files.Row; +import org.ohdsi.utilities.files.WriteCSVFileWithHeader; + +public class RichConnection implements Closeable { + public static int INSERT_BATCH_SIZE = 100000; + private Connection connection; + private boolean verbose = false; + private static DecimalFormat decimalFormat = new DecimalFormat("#.#"); + private DbType dbType; + + public RichConnection(String server, String domain, String user, String password, DbType dbType) { + this.connection = DBConnector.connect(server, domain, user, password, dbType); + this.dbType = dbType; + } + + /** + * Execute the given SQL statement. + * + * @param sql + */ + public void execute(String sql) { + Statement statement = null; + try { + if (sql.length() == 0) + return; + + statement = connection.createStatement(ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY); + for (String subQuery : sql.split(";")) { + if (verbose) { + String abbrSQL = subQuery.replace('\n', ' ').replace('\t', ' ').trim(); + if (abbrSQL.length() > 100) + abbrSQL = abbrSQL.substring(0, 100).trim() + "..."; + System.out.println("Adding query to batch: " + abbrSQL); + } + + statement.addBatch(subQuery); + } + long start = System.currentTimeMillis(); + if (verbose) + System.out.println("Executing batch"); + statement.executeBatch(); + if (verbose) + outputQueryStats(statement, System.currentTimeMillis() - start); + } catch (SQLException e) { + System.err.println(sql); + e.printStackTrace(); + } finally { + if (statement != null) { + try { + statement.close(); + } catch (SQLException e) { + // TODO Auto-generated catch block + System.err.println(e.getMessage()); + } + } + } + } + + private void outputQueryStats(Statement statement, long ms) throws SQLException { + Throwable warning = statement.getWarnings(); + if (warning != null) + System.out.println("- SERVER: " + warning.getMessage()); + String timeString; + if (ms < 1000) + timeString = ms + " ms"; + else if (ms < 60000) + timeString = decimalFormat.format(ms / 1000d) + " seconds"; + else if (ms < 3600000) + timeString = decimalFormat.format(ms / 60000d) + " minutes"; + else + timeString = decimalFormat.format(ms / 3600000d) + " hours"; + System.out.println("- Query completed in " + timeString); + } + + /** + * Query the database using the provided SQL statement. + * + * @param sql + * @return + */ + public QueryResult query(String sql) { + return new QueryResult(sql); + } + + /** + * Switch the database to use. + * + * @param database + */ + public void use(String database) { + if (database == null) + return; + if (dbType == DbType.ORACLE) + execute("ALTER SESSION SET current_schema = " + database); + else if (dbType == DbType.POSTGRESQL || dbType == DbType.REDSHIFT) + execute("SET search_path TO " + database); + else if (dbType == DbType.MSACCESS) + ; // NOOP + else if (dbType == DbType.TERADATA) { + execute("database " + database); + } else + execute("USE " + database); + } + + public List getTableNames(String database) { + List names = new ArrayList(); + String query = null; + if (dbType == DbType.MYSQL) { + query = "SHOW TABLES IN " + database; + } else if (dbType == DbType.MSSQL || dbType == DbType.PDW) { + query = "SELECT CONCAT(schemas.name, '.', tables.name) FROM " + database + ".sys.tables INNER JOIN " + database + ".sys.schemas ON tables.schema_id = schemas.schema_id ORDER BY schemas.name, tables.name"; + } else if (dbType == DbType.ORACLE) { + query = "SELECT table_name FROM all_tables WHERE owner='" + database.toUpperCase() + "'"; + } else if (dbType == DbType.POSTGRESQL || dbType == DbType.REDSHIFT) { + query = "SELECT table_name FROM information_schema.tables WHERE table_schema = '" + database.toLowerCase() + "' ORDER BY table_name"; + } else if (dbType == DbType.MSACCESS) { + query = "SELECT Name FROM sys.MSysObjects WHERE Type=1 AND Flags=0;"; + } else if (dbType == DbType.TERADATA) { + query = "SELECT TableName from dbc.tables WHERE tablekind = 'T' and databasename='" + database + "'"; + } + + for (Row row : query(query)) + names.add(row.get(row.getFieldNames().get(0))); + return names; + } + +// public List getFieldNames(String table) { +// List names = new ArrayList(); +// if (dbType == DbType.MSSQL || dbType == DbType.PDW) { +// for (Row row : query("SELECT name FROM syscolumns WHERE id=OBJECT_ID('" + table + "')")) +// names.add(row.get("name")); +// } else if (dbType == DbType.MYSQL) +// for (Row row : query("SHOW COLUMNS FROM " + table)) +// names.add(row.get("COLUMN_NAME")); +// else +// throw new RuntimeException("DB type not supported"); +// +// return names; +// } + + public ResultSet getMsAccessFieldNames(String table) { + if (dbType == DbType.MSACCESS) { + try { + DatabaseMetaData metadata = connection.getMetaData(); + return metadata.getColumns(null, null, table, null); + } catch (SQLException e) { + throw new RuntimeException(e.getMessage()); + } + } else + throw new RuntimeException("DB is not of type MS Access"); + } + + /** + * Returns the row count of the specified table. + * + * @param tableName + * @return + */ + public long getTableSize(String tableName) { + QueryResult qr = null; + Long returnVal = null; + if (dbType == DbType.MSSQL || dbType == DbType.PDW) + qr = query("SELECT COUNT_BIG(*) FROM [" + tableName.replaceAll("\\.", "].[") + "];"); + else if (dbType == DbType.MSACCESS) + qr = query("SELECT COUNT(*) FROM [" + tableName + "];"); + else + qr = query("SELECT COUNT(*) FROM " + tableName + ";"); + try { + returnVal = Long.parseLong(qr.iterator().next().getCells().get(0)); + } catch (Exception e) { + throw new RuntimeException(e); + } finally { + if (qr != null) { + qr.close(); + } + } + return returnVal; + } + + /** + * Close the connection to the database. + */ + public void close() { + try { + connection.close(); + } catch (SQLException e) { + e.printStackTrace(); + } + } + + public boolean isVerbose() { + return verbose; + } + + public void setVerbose(boolean verbose) { + this.verbose = verbose; + } + + public class QueryResult implements Iterable { + private String sql; + + private List iterators = new ArrayList(); + + public QueryResult(String sql) { + this.sql = sql; + } + + @Override + public Iterator iterator() { + DBRowIterator iterator = new DBRowIterator(sql); + iterators.add(iterator); + return iterator; + } + + public void close() { + for (DBRowIterator iterator : iterators) { + iterator.close(); + } + } + } + + /** + * Writes the results of a query to the specified file in CSV format. + * + * @param queryResult + * @param filename + */ + public void writeToFile(QueryResult queryResult, String filename) { + WriteCSVFileWithHeader out = new WriteCSVFileWithHeader(filename); + for (Row row : queryResult) + out.write(row); + out.close(); + } + + /** + * Inserts the rows into a table in the database. + * + * @param iterator + * @param tableName + * @param create + * If true, the data format is determined based on the first batch of rows and used to create the table structure. + */ + public void insertIntoTable(Iterator iterator, String table, boolean create) { + List batch = new ArrayList(INSERT_BATCH_SIZE); + + boolean first = true; + SimpleCounter counter = new SimpleCounter(1000000, true); + while (iterator.hasNext()) { + if (batch.size() == INSERT_BATCH_SIZE) { + if (first && create) + createTable(table, batch); + insert(table, batch); + batch.clear(); + first = false; + } + batch.add(iterator.next()); + counter.count(); + } + if (batch.size() != 0) { + if (first && create) + createTable(table, batch); + insert(table, batch); + } + } + + private void insert(String tableName, List rows) { + List columns = null; + columns = rows.get(0).getFieldNames(); + for (int i = 0; i < columns.size(); i++) + columns.set(i, columnNameToSqlName(columns.get(i))); + + String sql = "INSERT INTO " + tableName; + sql = sql + " (" + StringUtilities.join(columns, ",") + ")"; + sql = sql + " VALUES (?"; + for (int i = 1; i < columns.size(); i++) + sql = sql + ",?"; + sql = sql + ")"; + try { + connection.setAutoCommit(false); + PreparedStatement statement = connection.prepareStatement(sql); + for (Row row : rows) { + for (int i = 0; i < columns.size(); i++) { + String value = row.get(columns.get(i)); + if (value == null) + System.out.println(row.toString()); + if (value.length() == 0) + value = null; + // System.out.println(value); + if (dbType == DbType.POSTGRESQL || dbType == DbType.REDSHIFT) // PostgreSQL does not allow unspecified types + statement.setObject(i + 1, value, Types.OTHER); + else if (dbType == DbType.ORACLE) { + if (isDate(value)) { + // System.out.println(value); + statement.setDate(i + 1, java.sql.Date.valueOf(value)); + + } else + statement.setString(i + 1, value); + } else + statement.setString(i + 1, value); + } + statement.addBatch(); + } + statement.executeBatch(); + connection.commit(); + statement.close(); + connection.setAutoCommit(true); + connection.clearWarnings(); + } catch (SQLException e) { + e.printStackTrace(); + if (e instanceof BatchUpdateException) { + System.err.println(((BatchUpdateException) e).getNextException().getMessage()); + } + } + } + + private static boolean isDate(String string) { + if (string != null && string.length() == 10 && string.charAt(4) == '-' && string.charAt(7) == '-') + try { + int year = Integer.parseInt(string.substring(0, 4)); + if (year < 1700 || year > 2200) + return false; + int month = Integer.parseInt(string.substring(5, 7)); + if (month < 1 || month > 12) + return false; + int day = Integer.parseInt(string.substring(8, 10)); + if (day < 1 || day > 31) + return false; + return true; + } catch (Exception e) { + return false; + } + return false; + } + + private Set createTable(String tableName, List rows) { + Set numericFields = new HashSet(); + Row firstRow = rows.get(0); + List fields = new ArrayList(rows.size()); + for (String field : firstRow.getFieldNames()) + fields.add(new FieldInfo(field)); + for (Row row : rows) { + for (FieldInfo fieldInfo : fields) { + String value = row.get(fieldInfo.name); + if (fieldInfo.isNumeric && !StringUtilities.isInteger(value)) + fieldInfo.isNumeric = false; + if (value.length() > fieldInfo.maxLength) + fieldInfo.maxLength = value.length(); + } + } + + StringBuilder sql = new StringBuilder(); + sql.append("CREATE TABLE " + tableName + " (\n"); + for (FieldInfo fieldInfo : fields) { + sql.append(" " + fieldInfo.toString() + ",\n"); + if (fieldInfo.isNumeric) + numericFields.add(fieldInfo.name); + } + sql.append(");"); + execute(sql.toString()); + return numericFields; + } + + private String columnNameToSqlName(String name) { + return name.replaceAll(" ", "_").replace("-", "_").replace(",", "_").replaceAll("_+", "_"); + } + + private class FieldInfo { + public String name; + public boolean isNumeric = true; + public int maxLength = 0; + + public FieldInfo(String name) { + this.name = name; + } + + public String toString() { + if (dbType == DbType.MYSQL) { + if (isNumeric) + return columnNameToSqlName(name) + " int(" + maxLength + ")"; + else if (maxLength > 255) + return columnNameToSqlName(name) + " text"; + else + return columnNameToSqlName(name) + " varchar(255)"; + } else if (dbType == DbType.MSSQL || dbType == DbType.PDW) { + if (isNumeric) { + if (maxLength < 10) + return columnNameToSqlName(name) + " int"; + else + return columnNameToSqlName(name) + " bigint"; + } else if (maxLength > 255) + return columnNameToSqlName(name) + " varchar(max)"; + else + return columnNameToSqlName(name) + " varchar(255)"; + } else + throw new RuntimeException("Create table syntax not specified for type " + dbType); + } + } + + private class DBRowIterator implements Iterator { + + private ResultSet resultSet; + + private boolean hasNext; + + private Set columnNames = new HashSet(); + + public DBRowIterator(String sql) { + Statement statement = null; + try { + sql.trim(); + if (sql.endsWith(";")) + sql = sql.substring(0, sql.length() - 1); + if (verbose) { + String abbrSQL = sql.replace('\n', ' ').replace('\t', ' ').trim(); + if (abbrSQL.length() > 100) + abbrSQL = abbrSQL.substring(0, 100).trim() + "..."; + System.out.println("Executing query: " + abbrSQL); + } + long start = System.currentTimeMillis(); + statement = connection.createStatement(ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY); + resultSet = statement.executeQuery(sql.toString()); + hasNext = resultSet.next(); + if (verbose) + outputQueryStats(statement, System.currentTimeMillis() - start); + } catch (SQLException e) { + System.err.println(sql.toString()); + System.err.println(e.getMessage()); + throw new RuntimeException(e); + } + } + + public void close() { + if (resultSet != null) { + try { + resultSet.close(); + } catch (SQLException e) { + e.printStackTrace(); + } + resultSet = null; + hasNext = false; + } + } + + @Override + public boolean hasNext() { + return hasNext; + } + + @Override + public Row next() { + try { + Row row = new Row(); + ResultSetMetaData metaData; + metaData = resultSet.getMetaData(); + columnNames.clear(); + + for (int i = 1; i < metaData.getColumnCount() + 1; i++) { + String columnName = metaData.getColumnName(i); + if (columnNames.add(columnName)) { + String value; + try { + value = resultSet.getString(i); + } catch (Exception e) { + value = ""; + } + if (value == null) + value = ""; + + row.add(columnName, value.replace(" 00:00:00", "")); + } + } + hasNext = resultSet.next(); + if (!hasNext) { + resultSet.close(); + resultSet = null; + } + return row; + } catch (SQLException e) { + e.printStackTrace(); + throw new RuntimeException(e); + } + } + + @Override + public void remove() { + } + } +} diff --git a/src/org/ohdsi/ooxml/CustomXWPFDocument.java b/rabbit-core/src/main/java/org/ohdsi/ooxml/CustomXWPFDocument.java similarity index 95% rename from src/org/ohdsi/ooxml/CustomXWPFDocument.java rename to rabbit-core/src/main/java/org/ohdsi/ooxml/CustomXWPFDocument.java index c01ead9c..03fada75 100644 --- a/src/org/ohdsi/ooxml/CustomXWPFDocument.java +++ b/rabbit-core/src/main/java/org/ohdsi/ooxml/CustomXWPFDocument.java @@ -1,5 +1,5 @@ /******************************************************************************* - * Copyright 2017 Observational Health Data Sciences and Informatics + * Copyright 2019 Observational Health Data Sciences and Informatics * * This file is part of WhiteRabbit * diff --git a/src/org/ohdsi/ooxml/ReadXlsxFileWithHeader.java b/rabbit-core/src/main/java/org/ohdsi/ooxml/ReadXlsxFileWithHeader.java similarity index 94% rename from src/org/ohdsi/ooxml/ReadXlsxFileWithHeader.java rename to rabbit-core/src/main/java/org/ohdsi/ooxml/ReadXlsxFileWithHeader.java index 75052900..2cb7018f 100644 --- a/src/org/ohdsi/ooxml/ReadXlsxFileWithHeader.java +++ b/rabbit-core/src/main/java/org/ohdsi/ooxml/ReadXlsxFileWithHeader.java @@ -1,5 +1,5 @@ /******************************************************************************* - * Copyright 2017 Observational Health Data Sciences and Informatics + * Copyright 2019 Observational Health Data Sciences and Informatics * * This file is part of WhiteRabbit * diff --git a/src/org/ohdsi/rabbitInAHat/dataModel/Database.java b/rabbit-core/src/main/java/org/ohdsi/rabbitInAHat/dataModel/Database.java similarity index 92% rename from src/org/ohdsi/rabbitInAHat/dataModel/Database.java rename to rabbit-core/src/main/java/org/ohdsi/rabbitInAHat/dataModel/Database.java index d595f2d0..a9db5f71 100644 --- a/src/org/ohdsi/rabbitInAHat/dataModel/Database.java +++ b/rabbit-core/src/main/java/org/ohdsi/rabbitInAHat/dataModel/Database.java @@ -1,209 +1,212 @@ -/******************************************************************************* - * Copyright 2017 Observational Health Data Sciences and Informatics - * - * This file is part of WhiteRabbit - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - ******************************************************************************/ -package org.ohdsi.rabbitInAHat.dataModel; - -import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.io.Serializable; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.Iterator; -import java.util.List; -import java.util.Map; - -import org.apache.commons.csv.CSVFormat; -import org.apache.commons.csv.CSVRecord; -import org.ohdsi.utilities.files.QuickAndDirtyXlsxReader; -import org.ohdsi.utilities.files.QuickAndDirtyXlsxReader.Sheet; - -public class Database implements Serializable { - - public enum CDMVersion { - CDMV4("CDMV4.csv"), CDMV5("CDMV5.csv"), CDMV501("CDMV5.0.1.csv"), CDMV510("CDMV5.1.0.csv"), CDMV520("CDMV5.2.0.csv"), CDMV530("CDMV5.3.0.csv"), CDMV531("CDMV5.3.1.csv"); - - private final String fileName; - - CDMVersion(String fileName) { - this.fileName = fileName; - } - } - - private List tables = new ArrayList
(); - private static final long serialVersionUID = -3912166654601191039L; - private String dbName = ""; - - public List
getTables() { - return tables; - } - - public Table getTableByName(String name) { - for (Table table : tables) - if (table.getName().toLowerCase().equals(name.toLowerCase())) - return table; - return null; - } - - public void setTables(List
tables) { - this.tables = tables; - } - - public String getDbName() { - return dbName; - } - - public static Database generateCDMModel(CDMVersion cdmVersion) { - return Database.generateModelFromCSV(Database.class.getResourceAsStream(cdmVersion.fileName), cdmVersion.fileName); - } - - public static Database generateModelFromCSV(InputStream stream, String dbName) { - Database database = new Database(); - - database.dbName = dbName.substring(0, dbName.lastIndexOf(".")); - - Map nameToTable = new HashMap(); - try { - - for (CSVRecord row : CSVFormat.RFC4180.withHeader().parse(new InputStreamReader(stream))) { - String tableNameColumn; - String fieldNameColumn; - String isNullableColumn; - String nullableValue; - String dataTypeColumn; - String descriptionColumn; - if (row.isSet("TABLE_NAME")) { - tableNameColumn = "TABLE_NAME"; - fieldNameColumn = "COLUMN_NAME"; - isNullableColumn = "IS_NULLABLE"; - nullableValue = "YES"; - dataTypeColumn = "DATA_TYPE"; - descriptionColumn = "DESCRIPTION"; - } else { - tableNameColumn = "table"; - fieldNameColumn = "field"; - isNullableColumn = "required"; - nullableValue = "No"; - dataTypeColumn = "type"; - descriptionColumn = "description"; - } - Table table = nameToTable.get(row.get(tableNameColumn).toLowerCase()); - - if (table == null) { - table = new Table(); - table.setDb(database); - table.setName(row.get(tableNameColumn).toLowerCase()); - nameToTable.put(row.get(tableNameColumn).toLowerCase(), table); - database.tables.add(table); - } - Field field = new Field(row.get(fieldNameColumn).toLowerCase(), table); - field.setNullable(row.get(isNullableColumn).equals(nullableValue)); - field.setType(row.get(dataTypeColumn)); - field.setDescription(row.get(descriptionColumn)); - table.getFields().add(field); - } - } catch (IOException e) { - throw new RuntimeException(e.getMessage()); - } - // Collections.sort(database.tables, new Comparator
() { - // - // @Override - // public int compare(Table o1, Table o2) { - // return o1.getName().compareTo(o2.getName()); - // }}); - return database; - } - - public static Database generateModelFromScanReport(String filename) { - Database database = new Database(); - Map nameToTable = new HashMap(); - QuickAndDirtyXlsxReader workbook = new QuickAndDirtyXlsxReader(filename); - Sheet sheet = workbook.get(0); - Iterator iterator = sheet.iterator(); - Map fieldName2ColumnIndex = new HashMap(); - for (String header : iterator.next()) - fieldName2ColumnIndex.put(header, fieldName2ColumnIndex.size()); - - while (iterator.hasNext()) { - org.ohdsi.utilities.files.QuickAndDirtyXlsxReader.Row row = iterator.next(); - String tableName = row.get(fieldName2ColumnIndex.get("Table")); - if (tableName.length() != 0) { - Table table = nameToTable.get(tableName); - if (table == null) { - table = new Table(); - table.setName(tableName.toLowerCase()); - table.setRowCount((int) Double.parseDouble(row.get(fieldName2ColumnIndex.get("N rows")))); - nameToTable.put(tableName, table); - database.tables.add(table); - } - String fieldName = row.get(fieldName2ColumnIndex.get("Field")); - Field field = new Field(fieldName.toLowerCase(), table); - Integer index; - // Someone may have manually deleted data, so can't assume this - // is always there: - index = fieldName2ColumnIndex.get("Fraction empty"); - if (index != null && index < row.size()) - field.setNullable(!row.get(index).equals("0")); - - index = fieldName2ColumnIndex.get("Type"); - if (index != null && index < row.size()) - field.setType(row.get(index)); - - index = fieldName2ColumnIndex.get("Max length"); - if (index != null && index >= 0 && index < row.size()) - field.setMaxLength((int) (Double.parseDouble(row.get(index)))); - field.setValueCounts(getValueCounts(workbook, tableName, fieldName)); - table.getFields().add(field); - } - } - // database.defaultOrdering = new ArrayList
(database.tables); - return database; - } - - private static String[][] getValueCounts(QuickAndDirtyXlsxReader workbook, String tableName, String fieldName) { - Sheet tableSheet = null; - for (Sheet sheet : workbook) - if (sheet.getName().equals(tableName)) { - tableSheet = sheet; - break; - } - if (tableSheet == null) // Sheet not found for table, return empty array - return new String[0][0]; - - Iterator iterator = tableSheet.iterator(); - org.ohdsi.utilities.files.QuickAndDirtyXlsxReader.Row header = iterator.next(); - int index = header.indexOf(fieldName); - List list = new ArrayList(); - if (index != -1) // Could happen when people manually delete columns - while (iterator.hasNext()) { - org.ohdsi.utilities.files.QuickAndDirtyXlsxReader.Row row = iterator.next(); - if (row.size() > index) { - String value = row.get(index); - String count; - if (row.size() > index + 1) - count = row.get(index + 1); - else - count = ""; - if (value.equals("") && count.equals("")) - break; - list.add(new String[] { value, count }); - } - } - return list.toArray(new String[list.size()][2]); - } - -} +/******************************************************************************* + * Copyright 2019 Observational Health Data Sciences and Informatics + * + * This file is part of WhiteRabbit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + ******************************************************************************/ +package org.ohdsi.rabbitInAHat.dataModel; + +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.Serializable; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; + +import org.apache.commons.csv.CSVFormat; +import org.apache.commons.csv.CSVRecord; +import org.ohdsi.utilities.files.QuickAndDirtyXlsxReader; +import org.ohdsi.utilities.files.QuickAndDirtyXlsxReader.Sheet; + +public class Database implements Serializable { + + public enum CDMVersion { + CDMV4("CDMV4.csv"), CDMV5("CDMV5.csv"), CDMV501("CDMV5.0.1.csv"), CDMV510("CDMV5.1.0.csv"), CDMV520("CDMV5.2.0.csv"), CDMV530("CDMV5.3.0.csv"), CDMV531("CDMV5.3.1.csv"), CDMV60("CDMV6.0.csv"); + + private final String fileName; + + CDMVersion(String fileName) { + this.fileName = fileName; + } + } + + private List
tables = new ArrayList
(); + private static final long serialVersionUID = -3912166654601191039L; + private String dbName = ""; + + public List
getTables() { + return tables; + } + + public Table getTableByName(String name) { + for (Table table : tables) + if (table.getName().toLowerCase().equals(name.toLowerCase())) + return table; + return null; + } + + public void setTables(List
tables) { + this.tables = tables; + } + + public String getDbName() { + return dbName; + } + + public static Database generateCDMModel(CDMVersion cdmVersion) { + return Database.generateModelFromCSV(Database.class.getResourceAsStream(cdmVersion.fileName), cdmVersion.fileName); + } + + public static Database generateModelFromCSV(InputStream stream, String dbName) { + Database database = new Database(); + + database.dbName = dbName.substring(0, dbName.lastIndexOf(".")); + + Map nameToTable = new HashMap(); + try { + + for (CSVRecord row : CSVFormat.RFC4180.withHeader().parse(new InputStreamReader(stream))) { + String tableNameColumn; + String fieldNameColumn; + String isNullableColumn; + String nullableValue; + String dataTypeColumn; + String descriptionColumn; + if (row.isSet("TABLE_NAME")) { + tableNameColumn = "TABLE_NAME"; + fieldNameColumn = "COLUMN_NAME"; + isNullableColumn = "IS_NULLABLE"; + nullableValue = "YES"; + dataTypeColumn = "DATA_TYPE"; + descriptionColumn = "DESCRIPTION"; + } else { + tableNameColumn = "table"; + fieldNameColumn = "field"; + isNullableColumn = "required"; + nullableValue = "No"; + dataTypeColumn = "type"; + descriptionColumn = "description"; + } + Table table = nameToTable.get(row.get(tableNameColumn).toLowerCase()); + + if (table == null) { + table = new Table(); + table.setDb(database); + table.setName(row.get(tableNameColumn).toLowerCase()); + nameToTable.put(row.get(tableNameColumn).toLowerCase(), table); + database.tables.add(table); + } + Field field = new Field(row.get(fieldNameColumn).toLowerCase(), table); + field.setNullable(row.get(isNullableColumn).equals(nullableValue)); + field.setType(row.get(dataTypeColumn)); + field.setDescription(row.get(descriptionColumn)); + table.getFields().add(field); + } + } catch (IOException e) { + throw new RuntimeException(e.getMessage()); + } + // Collections.sort(database.tables, new Comparator
() { + // + // @Override + // public int compare(Table o1, Table o2) { + // return o1.getName().compareTo(o2.getName()); + // }}); + return database; + } + + public static Database generateModelFromScanReport(String filename) { + Database database = new Database(); + Map nameToTable = new HashMap(); + QuickAndDirtyXlsxReader workbook = new QuickAndDirtyXlsxReader(filename); + Sheet sheet = workbook.get(0); + Iterator iterator = sheet.iterator(); + Map fieldName2ColumnIndex = new HashMap(); + for (String header : iterator.next()) + fieldName2ColumnIndex.put(header, fieldName2ColumnIndex.size()); + + while (iterator.hasNext()) { + org.ohdsi.utilities.files.QuickAndDirtyXlsxReader.Row row = iterator.next(); + String tableName = row.get(fieldName2ColumnIndex.get("Table")); + if (tableName.length() != 0) { + Table table = nameToTable.get(tableName); + if (table == null) { + table = new Table(); + table.setName(tableName.toLowerCase()); + table.setRowCount((int) Double.parseDouble(row.get(fieldName2ColumnIndex.get("N rows")))); + table.setRowsCheckedCount((int) Double.parseDouble(row.get(fieldName2ColumnIndex.get("N rows checked")))); + nameToTable.put(tableName, table); + database.tables.add(table); + } + String fieldName = row.get(fieldName2ColumnIndex.get("Field")); + Field field = new Field(fieldName.toLowerCase(), table); + Integer index; + // Someone may have manually deleted data, so can't assume this + // is always there: + index = fieldName2ColumnIndex.get("Fraction empty"); + if (index != null && index < row.size()) + field.setNullable(!row.get(index).equals("0")); + + index = fieldName2ColumnIndex.get("Type"); + if (index != null && index < row.size()) + field.setType(row.get(index)); + + index = fieldName2ColumnIndex.get("Max length"); + if (index != null && index >= 0 && index < row.size()) + field.setMaxLength((int) (Double.parseDouble(row.get(index)))); + field.setValueCounts(getValueCounts(workbook, tableName, fieldName)); + table.getFields().add(field); + } + } + // database.defaultOrdering = new ArrayList
(database.tables); + return database; + } + + private static String[][] getValueCounts(QuickAndDirtyXlsxReader workbook, String tableName, String fieldName) { + Sheet tableSheet = null; + String targetSheetName = Table.createSheetNameFromTableName(tableName); + for (Sheet sheet : workbook) { + if (sheet.getName().equals(targetSheetName)) { + tableSheet = sheet; + break; + } + } + if (tableSheet == null) // Sheet not found for table, return empty array + return new String[0][0]; + + Iterator iterator = tableSheet.iterator(); + org.ohdsi.utilities.files.QuickAndDirtyXlsxReader.Row header = iterator.next(); + int index = header.indexOf(fieldName); + List list = new ArrayList(); + if (index != -1) // Could happen when people manually delete columns + while (iterator.hasNext()) { + org.ohdsi.utilities.files.QuickAndDirtyXlsxReader.Row row = iterator.next(); + if (row.size() > index) { + String value = row.get(index); + String count; + if (row.size() > index + 1) + count = row.get(index + 1); + else + count = ""; + if (value.equals("") && count.equals("")) + break; + list.add(new String[] { value, count }); + } + } + return list.toArray(new String[list.size()][2]); + } + +} diff --git a/src/org/ohdsi/rabbitInAHat/dataModel/Field.java b/rabbit-core/src/main/java/org/ohdsi/rabbitInAHat/dataModel/Field.java similarity index 90% rename from src/org/ohdsi/rabbitInAHat/dataModel/Field.java rename to rabbit-core/src/main/java/org/ohdsi/rabbitInAHat/dataModel/Field.java index 54fe9da4..4cc843aa 100644 --- a/src/org/ohdsi/rabbitInAHat/dataModel/Field.java +++ b/rabbit-core/src/main/java/org/ohdsi/rabbitInAHat/dataModel/Field.java @@ -1,125 +1,129 @@ -/******************************************************************************* - * Copyright 2017 Observational Health Data Sciences and Informatics - * - * This file is part of WhiteRabbit - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - ******************************************************************************/ -package org.ohdsi.rabbitInAHat.dataModel; - -public class Field implements MappableItem { - - private static final long serialVersionUID = 3687778470032619497L; - private Table table; - private String name; - private String comment = ""; - private String[][] valueCounts; - private boolean isNullable; - private String type; - private String description = ""; - private int maxLength; - private boolean isStem; - - public Field(String name, Table table) { - this.table = table; - this.name = name; - } - - public Database getDb() { - return this.table.getDb(); - } - - public Table getTable() { - return table; - } - - public void setTable(Table table) { - this.table = table; - } - - public String getName() { - return name; - } - - public String outputName() { - if (!isNullable) { - return "*" + name; - } else { - return name; - } - } - - public void setName(String name) { - this.name = name; - } - - public String[][] getValueCounts() { - return valueCounts; - } - - public void setValueCounts(String[][] valueCounts) { - this.valueCounts = valueCounts; - } - - public boolean isNullable() { - return isNullable; - } - - public void setNullable(boolean isNullable) { - this.isNullable = isNullable; - } - - public String getType() { - return type; - } - - public void setType(String type) { - this.type = type; - } - - public String getDescription() { - return this.description; - } - - public void setDescription(String description) { - this.description = description; - } - - public String toString() { - return table.getName() + "." + name; - } - - public void setComment(String comment) { - this.comment = comment; - } - - public String getComment() { - return comment; - } - - public int getMaxLength() { - return maxLength; - } - - public void setMaxLength(int maxLength) { - this.maxLength = maxLength; - } - - public boolean isStem() { - return isStem; - } - - public void setStem(boolean isStem) { - this.isStem = isStem; - } -} +/******************************************************************************* + * Copyright 2019 Observational Health Data Sciences and Informatics + * + * This file is part of WhiteRabbit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + ******************************************************************************/ +package org.ohdsi.rabbitInAHat.dataModel; + +public class Field implements MappableItem { + + private static final long serialVersionUID = 3687778470032619497L; + private Table table; + private String name; + private String comment = ""; + private String[][] valueCounts; + private boolean isNullable; + private String type; + private String description = ""; + private int maxLength; + private boolean isStem; + + public Field(String name, Table table) { + this.table = table; + this.name = name; + } + + public Database getDb() { + return this.table.getDb(); + } + + public Table getTable() { + return table; + } + + public void setTable(Table table) { + this.table = table; + } + + public String getName() { + return name; + } + + public String outputName() { + if (!isNullable) { + return "*" + name; + } else { + return name; + } + } + + public void setName(String name) { + this.name = name; + } + + public String[][] getValueCounts() { + return valueCounts; + } + + public void setValueCounts(String[][] valueCounts) { + this.valueCounts = valueCounts; + } + + public int getRowsCheckedCount() { + return this.table.getRowsCheckedCount(); + } + + public boolean isNullable() { + return isNullable; + } + + public void setNullable(boolean isNullable) { + this.isNullable = isNullable; + } + + public String getType() { + return type; + } + + public void setType(String type) { + this.type = type; + } + + public String getDescription() { + return this.description; + } + + public void setDescription(String description) { + this.description = description; + } + + public String toString() { + return table.getName() + "." + name; + } + + public void setComment(String comment) { + this.comment = comment; + } + + public String getComment() { + return comment; + } + + public int getMaxLength() { + return maxLength; + } + + public void setMaxLength(int maxLength) { + this.maxLength = maxLength; + } + + public boolean isStem() { + return isStem; + } + + public void setStem(boolean isStem) { + this.isStem = isStem; + } +} diff --git a/src/org/ohdsi/rabbitInAHat/dataModel/ItemToItemMap.java b/rabbit-core/src/main/java/org/ohdsi/rabbitInAHat/dataModel/ItemToItemMap.java similarity index 94% rename from src/org/ohdsi/rabbitInAHat/dataModel/ItemToItemMap.java rename to rabbit-core/src/main/java/org/ohdsi/rabbitInAHat/dataModel/ItemToItemMap.java index 3ac7ee00..9537d3c5 100644 --- a/src/org/ohdsi/rabbitInAHat/dataModel/ItemToItemMap.java +++ b/rabbit-core/src/main/java/org/ohdsi/rabbitInAHat/dataModel/ItemToItemMap.java @@ -1,5 +1,5 @@ /******************************************************************************* - * Copyright 2017 Observational Health Data Sciences and Informatics + * Copyright 2019 Observational Health Data Sciences and Informatics * * This file is part of WhiteRabbit * diff --git a/src/org/ohdsi/rabbitInAHat/dataModel/MappableItem.java b/rabbit-core/src/main/java/org/ohdsi/rabbitInAHat/dataModel/MappableItem.java similarity index 91% rename from src/org/ohdsi/rabbitInAHat/dataModel/MappableItem.java rename to rabbit-core/src/main/java/org/ohdsi/rabbitInAHat/dataModel/MappableItem.java index 3baeb329..4f0268e3 100644 --- a/src/org/ohdsi/rabbitInAHat/dataModel/MappableItem.java +++ b/rabbit-core/src/main/java/org/ohdsi/rabbitInAHat/dataModel/MappableItem.java @@ -1,5 +1,5 @@ /******************************************************************************* - * Copyright 2017 Observational Health Data Sciences and Informatics + * Copyright 2019 Observational Health Data Sciences and Informatics * * This file is part of WhiteRabbit * diff --git a/src/org/ohdsi/rabbitInAHat/dataModel/Mapping.java b/rabbit-core/src/main/java/org/ohdsi/rabbitInAHat/dataModel/Mapping.java similarity index 73% rename from src/org/ohdsi/rabbitInAHat/dataModel/Mapping.java rename to rabbit-core/src/main/java/org/ohdsi/rabbitInAHat/dataModel/Mapping.java index 6999b759..a2e3a299 100644 --- a/src/org/ohdsi/rabbitInAHat/dataModel/Mapping.java +++ b/rabbit-core/src/main/java/org/ohdsi/rabbitInAHat/dataModel/Mapping.java @@ -1,5 +1,5 @@ /******************************************************************************* - * Copyright 2017 Observational Health Data Sciences and Informatics + * Copyright 2019 Observational Health Data Sciences and Informatics * * This file is part of WhiteRabbit * @@ -18,7 +18,6 @@ package org.ohdsi.rabbitInAHat.dataModel; import java.util.ArrayList; -import java.util.Iterator; import java.util.List; public class Mapping { @@ -73,19 +72,37 @@ public List getSourceToTargetMaps() { return sourceToCdmMaps; } - public void removeSourceToTargetMap(MappableItem sourceItem, MappableItem targetItem) { - Iterator iterator = sourceToCdmMaps.iterator(); - while (iterator.hasNext()) { - ItemToItemMap sourceToTargetMap = iterator.next(); - if (sourceToTargetMap.getSourceItem().equals(sourceItem) && sourceToTargetMap.getTargetItem().equals(targetItem)) - iterator.remove(); + public List getSourceToTargetMapsOrderedByCdmItems() { + List result = new ArrayList<>(); + for (MappableItem targetItem : cdmItems) { + boolean sourceFound = false; + for (MappableItem sourceItem : sourceItems) { + ItemToItemMap mapping = getSourceToTargetMap(sourceItem, targetItem); + if (mapping != null) { + result.add(mapping); + sourceFound = true; + } + } + if (!sourceFound) + result.add(null); } + +// result.removeAll(Collections.singleton(null)); + return result; + } + + public void removeSourceToTargetMap(MappableItem sourceItem, MappableItem targetItem) { + sourceToCdmMaps.removeIf(sourceToTargetMap -> + sourceToTargetMap.getSourceItem().equals(sourceItem) && sourceToTargetMap.getTargetItem().equals(targetItem) + ); + } + + public void removeAllSourceToTargetMaps() { + sourceToCdmMaps.clear(); } public ItemToItemMap getSourceToTargetMap(MappableItem sourceItem, MappableItem targetItem) { - Iterator iterator = sourceToCdmMaps.iterator(); - while (iterator.hasNext()) { - ItemToItemMap sourceToTargetMap = iterator.next(); + for (ItemToItemMap sourceToTargetMap : sourceToCdmMaps) { if (sourceToTargetMap.getSourceItem().equals(sourceItem) && sourceToTargetMap.getTargetItem().equals(targetItem)) return sourceToTargetMap; } @@ -93,10 +110,7 @@ public ItemToItemMap getSourceToTargetMap(MappableItem sourceItem, MappableItem } public ItemToItemMap getSourceToTargetMapByName(MappableItem sourceItem, MappableItem targetItem) { - Iterator iterator = sourceToCdmMaps.iterator(); - - while (iterator.hasNext()) { - ItemToItemMap sourceToTargetMap = iterator.next(); + for (ItemToItemMap sourceToTargetMap : sourceToCdmMaps) { if (sourceToTargetMap.getSourceItem().getName().equals(sourceItem.getName()) && sourceToTargetMap.getTargetItem().getName().equals(targetItem.getName())) return sourceToTargetMap; diff --git a/src/org/ohdsi/rabbitInAHat/dataModel/Table.java b/rabbit-core/src/main/java/org/ohdsi/rabbitInAHat/dataModel/Table.java similarity index 70% rename from src/org/ohdsi/rabbitInAHat/dataModel/Table.java rename to rabbit-core/src/main/java/org/ohdsi/rabbitInAHat/dataModel/Table.java index 7d5df590..be037154 100644 --- a/src/org/ohdsi/rabbitInAHat/dataModel/Table.java +++ b/rabbit-core/src/main/java/org/ohdsi/rabbitInAHat/dataModel/Table.java @@ -1,5 +1,5 @@ /******************************************************************************* - * Copyright 2017 Observational Health Data Sciences and Informatics + * Copyright 2019 Observational Health Data Sciences and Informatics * * This file is part of WhiteRabbit * @@ -25,6 +25,7 @@ public class Table implements MappableItem { private Database db; private String name; private int rowCount; + private int rowsCheckedCount; private String comment = ""; private List fields = new ArrayList(); private boolean isStem = false; @@ -72,6 +73,14 @@ public void setRowCount(int rowCount) { this.rowCount = rowCount; } + public int getRowsCheckedCount() { + return rowsCheckedCount; + } + + public void setRowsCheckedCount(int rowsCheckedCount) { + this.rowsCheckedCount = rowsCheckedCount; + } + public List getFields() { return fields; } @@ -102,4 +111,26 @@ public boolean isStem() { public void setStem(boolean isStem) { this.isStem = isStem; } + + public static String indexTableNameForSheet(String tableName, int index) { + String name = tableName; + // Prepend index for long table names (to make sheet name unique later) + if (name.length() > 31) { + name = Integer.toString(index) + '_' + name; + } + return name; + } + + public static String createSheetNameFromTableName(String tableName) { + String name = tableName; + + // Excel sheet names have a maximum of 31 characters + if (name.length() > 31) { + name = name.substring(0, 31); + } + + // Backslash causes issues in excel + name = name.replace('/','_'); + return name; + } } diff --git a/src/org/ohdsi/rabbitInAHat/dataModel/TableCellLongTextRenderer.java b/rabbit-core/src/main/java/org/ohdsi/rabbitInAHat/dataModel/TableCellLongTextRenderer.java similarity index 96% rename from src/org/ohdsi/rabbitInAHat/dataModel/TableCellLongTextRenderer.java rename to rabbit-core/src/main/java/org/ohdsi/rabbitInAHat/dataModel/TableCellLongTextRenderer.java index 3c81bf2a..9e49e383 100644 --- a/src/org/ohdsi/rabbitInAHat/dataModel/TableCellLongTextRenderer.java +++ b/rabbit-core/src/main/java/org/ohdsi/rabbitInAHat/dataModel/TableCellLongTextRenderer.java @@ -1,41 +1,41 @@ -package org.ohdsi.rabbitInAHat.dataModel; - -import java.awt.Component; -import java.awt.Insets; - -import javax.swing.JTable; -import javax.swing.JTextArea; -import javax.swing.table.DefaultTableCellRenderer; -import javax.swing.table.TableCellRenderer; - - -/** - * Setups the table to properly wrap in the table - * - * - * @author Paul Zepernick - */ -public class TableCellLongTextRenderer extends DefaultTableCellRenderer implements TableCellRenderer{ - - /** - * - */ - private static final long serialVersionUID = 7920163334647774178L; - - @Override - public Component getTableCellRendererComponent(JTable table, Object value, boolean isSelected, boolean hasFocus, int row, int column) { - final JTextArea jtext = new JTextArea(); - jtext.setText((String)value); - jtext.setWrapStyleWord(true); - jtext.setLineWrap(true); - jtext.setFont(table.getFont()); - jtext.setSize(table.getColumn(table.getColumnName(column)).getWidth(), (int)jtext.getPreferredSize().getHeight()); - - jtext.setMargin(new Insets(10,5,10,5)); - - return jtext; - } - - - -} +package org.ohdsi.rabbitInAHat.dataModel; + +import java.awt.Component; +import java.awt.Insets; + +import javax.swing.JTable; +import javax.swing.JTextArea; +import javax.swing.table.DefaultTableCellRenderer; +import javax.swing.table.TableCellRenderer; + + +/** + * Setups the table to properly wrap in the table + * + * + * @author Paul Zepernick + */ +public class TableCellLongTextRenderer extends DefaultTableCellRenderer implements TableCellRenderer{ + + /** + * + */ + private static final long serialVersionUID = 7920163334647774178L; + + @Override + public Component getTableCellRendererComponent(JTable table, Object value, boolean isSelected, boolean hasFocus, int row, int column) { + final JTextArea jtext = new JTextArea(); + jtext.setText((String)value); + jtext.setWrapStyleWord(true); + jtext.setLineWrap(true); + jtext.setFont(table.getFont()); + jtext.setSize(table.getColumn(table.getColumnName(column)).getWidth(), (int)jtext.getPreferredSize().getHeight()); + + jtext.setMargin(new Insets(10,5,10,5)); + + return jtext; + } + + + +} diff --git a/src/org/ohdsi/utilities/DirectoryUtilities.java b/rabbit-core/src/main/java/org/ohdsi/utilities/DirectoryUtilities.java similarity index 94% rename from src/org/ohdsi/utilities/DirectoryUtilities.java rename to rabbit-core/src/main/java/org/ohdsi/utilities/DirectoryUtilities.java index da5e49f2..b9f2adb9 100644 --- a/src/org/ohdsi/utilities/DirectoryUtilities.java +++ b/rabbit-core/src/main/java/org/ohdsi/utilities/DirectoryUtilities.java @@ -1,5 +1,5 @@ /******************************************************************************* - * Copyright 2017 Observational Health Data Sciences and Informatics + * Copyright 2019 Observational Health Data Sciences and Informatics * * This file is part of WhiteRabbit * diff --git a/src/org/ohdsi/utilities/RandomUtilities.java b/rabbit-core/src/main/java/org/ohdsi/utilities/RandomUtilities.java similarity index 94% rename from src/org/ohdsi/utilities/RandomUtilities.java rename to rabbit-core/src/main/java/org/ohdsi/utilities/RandomUtilities.java index baa73e16..04a05d66 100644 --- a/src/org/ohdsi/utilities/RandomUtilities.java +++ b/rabbit-core/src/main/java/org/ohdsi/utilities/RandomUtilities.java @@ -1,5 +1,5 @@ /******************************************************************************* - * Copyright 2017 Observational Health Data Sciences and Informatics + * Copyright 2019 Observational Health Data Sciences and Informatics * * This file is part of WhiteRabbit * diff --git a/src/org/ohdsi/utilities/SimpleCounter.java b/rabbit-core/src/main/java/org/ohdsi/utilities/SimpleCounter.java similarity index 93% rename from src/org/ohdsi/utilities/SimpleCounter.java rename to rabbit-core/src/main/java/org/ohdsi/utilities/SimpleCounter.java index e3f35c03..8a066408 100644 --- a/src/org/ohdsi/utilities/SimpleCounter.java +++ b/rabbit-core/src/main/java/org/ohdsi/utilities/SimpleCounter.java @@ -1,5 +1,5 @@ /******************************************************************************* - * Copyright 2017 Observational Health Data Sciences and Informatics + * Copyright 2019 Observational Health Data Sciences and Informatics * * This file is part of WhiteRabbit * diff --git a/src/org/ohdsi/utilities/StringUtilities.java b/rabbit-core/src/main/java/org/ohdsi/utilities/StringUtilities.java similarity index 96% rename from src/org/ohdsi/utilities/StringUtilities.java rename to rabbit-core/src/main/java/org/ohdsi/utilities/StringUtilities.java index 1403574a..ff2cfeda 100644 --- a/src/org/ohdsi/utilities/StringUtilities.java +++ b/rabbit-core/src/main/java/org/ohdsi/utilities/StringUtilities.java @@ -1,5 +1,5 @@ /******************************************************************************* - * Copyright 2017 Observational Health Data Sciences and Informatics + * Copyright 2019 Observational Health Data Sciences and Informatics * * This file is part of WhiteRabbit * diff --git a/rabbit-core/src/main/java/org/ohdsi/utilities/collections/CountingSet.java b/rabbit-core/src/main/java/org/ohdsi/utilities/collections/CountingSet.java new file mode 100644 index 00000000..a70241c5 --- /dev/null +++ b/rabbit-core/src/main/java/org/ohdsi/utilities/collections/CountingSet.java @@ -0,0 +1,198 @@ +/******************************************************************************* + * Copyright 2019 Observational Health Data Sciences and Informatics + * + * This file is part of WhiteRabbit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + ******************************************************************************/ +package org.ohdsi.utilities.collections; + +import java.util.AbstractSet; +import java.util.Comparator; +import java.util.HashMap; +import java.util.Iterator; +import java.util.Map; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +/** + * Class for counting recurring objects. + * + * @author schuemie + * @param + */ +public class CountingSet extends AbstractSet { + + public Map key2count; + + public CountingSet() { + key2count = new HashMap<>(); + } + + public CountingSet(int capacity) { + key2count = new HashMap<>(capacity); + } + + public CountingSet(CountingSet set) { + key2count = set.key2count.entrySet().stream() + .collect(Collectors.toMap( + Map.Entry::getKey, + e -> new Count(e.getValue().count), + (v1, v2) -> v1, + HashMap::new)); + } + + public int getCount(T key) { + Count count = key2count.get(key); + if (count == null) + return 0; + else + return count.count; + } + + /** + * Computes the sum of the counts + * + * @return + */ + public int getSum() { + return key2count.values().parallelStream() + .mapToInt(c -> c.count) + .sum(); + } + + /** + * Returns the maximum count + * + * @return + */ + public int getMax() { + return key2count.values().parallelStream() + .mapToInt(c -> c.count) + .max() + .orElse(0); + } + + /** + * Computes the mean of the counts + * + * @return + */ + public double getMean() { + return (getSum() / (double) key2count.size()); + } + + /** + * Computes the standard deviations of the counts + * + * @return + */ + public double getSD() { + final double mean = getMean(); + double sqSum = key2count.values().parallelStream() + .mapToDouble(c -> sqr(c.count - mean)) + .sum(); + return Math.sqrt(sqSum / key2count.size()); + } + + private static double sqr(double d) { + return d * d; + } + + public int size() { + return key2count.size(); + } + + public boolean contains(Object arg0) { + return key2count.containsKey(arg0); + } + + public Iterator iterator() { + return key2count.keySet().iterator(); + } + + public boolean add(T arg0) { + Count count = key2count.get(arg0); + if (count == null) { + key2count.put(arg0, new Count(1)); + return true; + } else { + count.increment(); + return false; + } + } + + public boolean add(T arg0, int inc) { + Count count = key2count.get(arg0); + if (count == null) { + key2count.put(arg0, new Count(inc)); + return true; + } else { + count.add(inc); + return false; + } + } + + public boolean remove(Object arg0) { + return (key2count.remove(arg0) != null); + } + + public void clear() { + key2count.clear(); + } + + /** + * Keep the n most frequent values, remove the rest + * + * @param n + */ + public void keepTopN(int n) { + if (size() < n) + return; + + key2count = decliningCountStream() + .limit(n) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue, (v1, v2) -> v1, HashMap::new)); + } + + public static class Count implements Comparable { + public int count; + + public Count(int count) { + this.count = count; + } + + public void increment() { + count++; + } + + public void add(int count) { + this.count += count; + } + + @Override + public int compareTo(Count o) { + return count - o.count; + } + } + + public void printCounts() { + decliningCountStream() + .forEach(entry -> System.out.println(entry.getKey() + "\t" + entry.getValue().count)); + } + + private Stream> decliningCountStream() { + return key2count.entrySet().stream() + .sorted(Comparator., Count>comparing(Map.Entry::getValue).reversed()); + } +} diff --git a/src/org/ohdsi/utilities/collections/IntegerComparator.java b/rabbit-core/src/main/java/org/ohdsi/utilities/collections/IntegerComparator.java similarity index 91% rename from src/org/ohdsi/utilities/collections/IntegerComparator.java rename to rabbit-core/src/main/java/org/ohdsi/utilities/collections/IntegerComparator.java index 0f08e407..5c0a17cd 100644 --- a/src/org/ohdsi/utilities/collections/IntegerComparator.java +++ b/rabbit-core/src/main/java/org/ohdsi/utilities/collections/IntegerComparator.java @@ -1,5 +1,5 @@ /******************************************************************************* - * Copyright 2017 Observational Health Data Sciences and Informatics + * Copyright 2019 Observational Health Data Sciences and Informatics * * This file is part of WhiteRabbit * diff --git a/src/org/ohdsi/utilities/collections/LongComparator.java b/rabbit-core/src/main/java/org/ohdsi/utilities/collections/LongComparator.java similarity index 91% rename from src/org/ohdsi/utilities/collections/LongComparator.java rename to rabbit-core/src/main/java/org/ohdsi/utilities/collections/LongComparator.java index 65af2265..1b8ca57e 100644 --- a/src/org/ohdsi/utilities/collections/LongComparator.java +++ b/rabbit-core/src/main/java/org/ohdsi/utilities/collections/LongComparator.java @@ -1,5 +1,5 @@ /******************************************************************************* - * Copyright 2017 Observational Health Data Sciences and Informatics + * Copyright 2019 Observational Health Data Sciences and Informatics * * This file is part of WhiteRabbit * diff --git a/src/org/ohdsi/utilities/collections/OneToManyList.java b/rabbit-core/src/main/java/org/ohdsi/utilities/collections/OneToManyList.java similarity index 93% rename from src/org/ohdsi/utilities/collections/OneToManyList.java rename to rabbit-core/src/main/java/org/ohdsi/utilities/collections/OneToManyList.java index c8d71876..dc81c3f0 100644 --- a/src/org/ohdsi/utilities/collections/OneToManyList.java +++ b/rabbit-core/src/main/java/org/ohdsi/utilities/collections/OneToManyList.java @@ -1,5 +1,5 @@ /******************************************************************************* - * Copyright 2017 Observational Health Data Sciences and Informatics + * Copyright 2019 Observational Health Data Sciences and Informatics * * This file is part of WhiteRabbit * diff --git a/src/org/ohdsi/utilities/collections/OneToManySet.java b/rabbit-core/src/main/java/org/ohdsi/utilities/collections/OneToManySet.java similarity index 92% rename from src/org/ohdsi/utilities/collections/OneToManySet.java rename to rabbit-core/src/main/java/org/ohdsi/utilities/collections/OneToManySet.java index 71e742cb..94394043 100644 --- a/src/org/ohdsi/utilities/collections/OneToManySet.java +++ b/rabbit-core/src/main/java/org/ohdsi/utilities/collections/OneToManySet.java @@ -1,5 +1,5 @@ /******************************************************************************* - * Copyright 2017 Observational Health Data Sciences and Informatics + * Copyright 2019 Observational Health Data Sciences and Informatics * * This file is part of WhiteRabbit * diff --git a/src/org/ohdsi/utilities/collections/Pair.java b/rabbit-core/src/main/java/org/ohdsi/utilities/collections/Pair.java similarity index 93% rename from src/org/ohdsi/utilities/collections/Pair.java rename to rabbit-core/src/main/java/org/ohdsi/utilities/collections/Pair.java index c0de32a7..03a3b91b 100644 --- a/src/org/ohdsi/utilities/collections/Pair.java +++ b/rabbit-core/src/main/java/org/ohdsi/utilities/collections/Pair.java @@ -1,5 +1,5 @@ /******************************************************************************* - * Copyright 2017 Observational Health Data Sciences and Informatics + * Copyright 2019 Observational Health Data Sciences and Informatics * * This file is part of WhiteRabbit * diff --git a/src/org/ohdsi/utilities/files/IniFile.java b/rabbit-core/src/main/java/org/ohdsi/utilities/files/IniFile.java similarity index 93% rename from src/org/ohdsi/utilities/files/IniFile.java rename to rabbit-core/src/main/java/org/ohdsi/utilities/files/IniFile.java index 6a854770..e2605d76 100644 --- a/src/org/ohdsi/utilities/files/IniFile.java +++ b/rabbit-core/src/main/java/org/ohdsi/utilities/files/IniFile.java @@ -1,46 +1,46 @@ -/******************************************************************************* - * Copyright 2017 Observational Health Data Sciences and Informatics - * - * This file is part of WhiteRabbit - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - ******************************************************************************/ -package org.ohdsi.utilities.files; - -import java.util.HashMap; -import java.util.Map; - -public class IniFile { - private Map settings = new HashMap(); - - public IniFile(String filename){ - for (String line : new ReadTextFile(filename)){ - int indexOfHash = line.lastIndexOf('#'); - if (indexOfHash != -1) - line = line.substring(0,indexOfHash); - - int indexOfEqualsSigns = line.indexOf('='); - - if (indexOfEqualsSigns != -1) - settings.put(line.substring(0,indexOfEqualsSigns).trim().toLowerCase(), line.substring(indexOfEqualsSigns+1).trim()); - } - } - - public String get(String fieldName){ - String value = settings.get(fieldName.toLowerCase()); - if (value == null) - return ""; - else - return value; - } -} +/******************************************************************************* + * Copyright 2019 Observational Health Data Sciences and Informatics + * + * This file is part of WhiteRabbit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + ******************************************************************************/ +package org.ohdsi.utilities.files; + +import java.util.HashMap; +import java.util.Map; + +public class IniFile { + private Map settings = new HashMap(); + + public IniFile(String filename){ + for (String line : new ReadTextFile(filename)){ + int indexOfHash = line.lastIndexOf('#'); + if (indexOfHash != -1) + line = line.substring(0,indexOfHash); + + int indexOfEqualsSigns = line.indexOf('='); + + if (indexOfEqualsSigns != -1) + settings.put(line.substring(0,indexOfEqualsSigns).trim().toLowerCase(), line.substring(indexOfEqualsSigns+1).trim()); + } + } + + public String get(String fieldName){ + String value = settings.get(fieldName.toLowerCase()); + if (value == null) + return ""; + else + return value; + } +} diff --git a/src/org/ohdsi/utilities/files/MultiRowIterator.java b/rabbit-core/src/main/java/org/ohdsi/utilities/files/MultiRowIterator.java similarity index 95% rename from src/org/ohdsi/utilities/files/MultiRowIterator.java rename to rabbit-core/src/main/java/org/ohdsi/utilities/files/MultiRowIterator.java index 5bb7a16d..3996cbe6 100644 --- a/src/org/ohdsi/utilities/files/MultiRowIterator.java +++ b/rabbit-core/src/main/java/org/ohdsi/utilities/files/MultiRowIterator.java @@ -1,159 +1,159 @@ -/******************************************************************************* - * Copyright 2017 Observational Health Data Sciences and Informatics - * - * This file is part of WhiteRabbit - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - ******************************************************************************/ -package org.ohdsi.utilities.files; - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.Iterator; -import java.util.List; - -import org.ohdsi.utilities.files.MultiRowIterator.MultiRowSet; - -/** - * Allows iteration over multiple tables (as Iterator) simultaneously, synchronized by the value of the [linkingColumn]. Assumes all tables are sorted by - * the [linkingColumn]. - * - * @author MSCHUEMI - */ -public class MultiRowIterator implements Iterator { - - private Iterator[] iterators; - private String[] tableNames; - private Row[] nextRows; - private MultiRowSet buffer; - private String linkingColumn; - private boolean sortedNumerically; - - @SafeVarargs - public MultiRowIterator(String linkingColumn, String[] tableNames, Iterator... tableIterators) { - this(linkingColumn, false, tableNames, tableIterators); - } - - public MultiRowIterator(String linkingColumn, boolean sortedNumerically, String[] tableNames, Iterator[] tableIterators) { - this.iterators = tableIterators; - this.linkingColumn = linkingColumn; - this.tableNames = tableNames; - this.sortedNumerically = sortedNumerically; - startRead(); - } - - private void startRead() { - nextRows = new Row[iterators.length]; - for (int i = 0; i < iterators.length; i++) - if (iterators[i].hasNext()) - nextRows[i] = iterators[i].next(); - else - nextRows[i] = null; - readNext(); - } - - @Override - public boolean hasNext() { - return (buffer != null); - } - - @Override - public MultiRowSet next() { - MultiRowSet result = buffer; - readNext(); - return result; - } - - private void readNext() { - String lowestLinkingColumn = findLowestLinkingColumn(nextRows); - if (lowestLinkingColumn == null) { - buffer = null; - return; - } - buffer = new MultiRowSet(tableNames); - buffer.linkingId = lowestLinkingColumn; - for (int i = 0; i < iterators.length; i++) { - Iterator iterator = iterators[i]; - while (nextRows[i] != null && nextRows[i].get(linkingColumn).equals(lowestLinkingColumn)) { - buffer.get(tableNames[i]).add(nextRows[i]); - if (iterator.hasNext()) - nextRows[i] = iterator.next(); - else - nextRows[i] = null; - } - } - } - - private String findLowestLinkingColumn(Row[] rows) { - String linkingId = null; - for (Row row : rows) - if (row != null && (linkingId == null || compare(row.get(linkingColumn), linkingId) < 0)) - linkingId = row.get(linkingColumn); - return linkingId; - } - - private int compare(String value1, String value2) { - if (sortedNumerically) - return efficientLongCompare(value1, value2); - else - return value1.compareTo(value2); - } - - private int efficientLongCompare(String value1, String value2) { - if (value1.length() > value2.length()) - return 1; - else if (value1.length() < value2.length()) - return -1; - else - return value1.compareTo(value2); - } - - @Override - public void remove() { - System.err.println("Calling unimplemented remove method in class " + this.getClass().getName()); - } - - public static class MultiRowSet extends HashMap> { - private static final long serialVersionUID = 1164317535150664720L; - - public String linkingId; - - public MultiRowSet(String[] tableNames) { - for (String tableName : tableNames) { - put(tableName, new ArrayList()); - } - } - - public List getNonEmptyTableNames() { - List result = new ArrayList(); - for (String tableName : keySet()) - if (get(tableName).size() != 0) - result.add(tableName); - return result; - } - - /** - * returns the total number of rows (summed across the tables) - * - * @return - */ - public int totalSize() { - int size = 0; - for (List rows : values()) - size += rows.size(); - return size; - } - - } - -} +/******************************************************************************* + * Copyright 2019 Observational Health Data Sciences and Informatics + * + * This file is part of WhiteRabbit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + ******************************************************************************/ +package org.ohdsi.utilities.files; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; + +import org.ohdsi.utilities.files.MultiRowIterator.MultiRowSet; + +/** + * Allows iteration over multiple tables (as Iterator) simultaneously, synchronized by the value of the [linkingColumn]. Assumes all tables are sorted by + * the [linkingColumn]. + * + * @author MSCHUEMI + */ +public class MultiRowIterator implements Iterator { + + private Iterator[] iterators; + private String[] tableNames; + private Row[] nextRows; + private MultiRowSet buffer; + private String linkingColumn; + private boolean sortedNumerically; + + @SafeVarargs + public MultiRowIterator(String linkingColumn, String[] tableNames, Iterator... tableIterators) { + this(linkingColumn, false, tableNames, tableIterators); + } + + public MultiRowIterator(String linkingColumn, boolean sortedNumerically, String[] tableNames, Iterator[] tableIterators) { + this.iterators = tableIterators; + this.linkingColumn = linkingColumn; + this.tableNames = tableNames; + this.sortedNumerically = sortedNumerically; + startRead(); + } + + private void startRead() { + nextRows = new Row[iterators.length]; + for (int i = 0; i < iterators.length; i++) + if (iterators[i].hasNext()) + nextRows[i] = iterators[i].next(); + else + nextRows[i] = null; + readNext(); + } + + @Override + public boolean hasNext() { + return (buffer != null); + } + + @Override + public MultiRowSet next() { + MultiRowSet result = buffer; + readNext(); + return result; + } + + private void readNext() { + String lowestLinkingColumn = findLowestLinkingColumn(nextRows); + if (lowestLinkingColumn == null) { + buffer = null; + return; + } + buffer = new MultiRowSet(tableNames); + buffer.linkingId = lowestLinkingColumn; + for (int i = 0; i < iterators.length; i++) { + Iterator iterator = iterators[i]; + while (nextRows[i] != null && nextRows[i].get(linkingColumn).equals(lowestLinkingColumn)) { + buffer.get(tableNames[i]).add(nextRows[i]); + if (iterator.hasNext()) + nextRows[i] = iterator.next(); + else + nextRows[i] = null; + } + } + } + + private String findLowestLinkingColumn(Row[] rows) { + String linkingId = null; + for (Row row : rows) + if (row != null && (linkingId == null || compare(row.get(linkingColumn), linkingId) < 0)) + linkingId = row.get(linkingColumn); + return linkingId; + } + + private int compare(String value1, String value2) { + if (sortedNumerically) + return efficientLongCompare(value1, value2); + else + return value1.compareTo(value2); + } + + private int efficientLongCompare(String value1, String value2) { + if (value1.length() > value2.length()) + return 1; + else if (value1.length() < value2.length()) + return -1; + else + return value1.compareTo(value2); + } + + @Override + public void remove() { + System.err.println("Calling unimplemented remove method in class " + this.getClass().getName()); + } + + public static class MultiRowSet extends HashMap> { + private static final long serialVersionUID = 1164317535150664720L; + + public String linkingId; + + public MultiRowSet(String[] tableNames) { + for (String tableName : tableNames) { + put(tableName, new ArrayList()); + } + } + + public List getNonEmptyTableNames() { + List result = new ArrayList(); + for (String tableName : keySet()) + if (get(tableName).size() != 0) + result.add(tableName); + return result; + } + + /** + * returns the total number of rows (summed across the tables) + * + * @return + */ + public int totalSize() { + int size = 0; + for (List rows : values()) + size += rows.size(); + return size; + } + + } + +} diff --git a/src/org/ohdsi/utilities/files/QuickAndDirtyXlsxReader.java b/rabbit-core/src/main/java/org/ohdsi/utilities/files/QuickAndDirtyXlsxReader.java similarity index 97% rename from src/org/ohdsi/utilities/files/QuickAndDirtyXlsxReader.java rename to rabbit-core/src/main/java/org/ohdsi/utilities/files/QuickAndDirtyXlsxReader.java index c47fcd74..8dd34b71 100644 --- a/src/org/ohdsi/utilities/files/QuickAndDirtyXlsxReader.java +++ b/rabbit-core/src/main/java/org/ohdsi/utilities/files/QuickAndDirtyXlsxReader.java @@ -1,547 +1,547 @@ -/******************************************************************************* - * Copyright 2017 Observational Health Data Sciences and Informatics - * - * This file is part of WhiteRabbit - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - ******************************************************************************/ -package org.ohdsi.utilities.files; - -import java.io.BufferedReader; -import java.io.FileInputStream; -import java.io.FileNotFoundException; -import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.util.ArrayList; -import java.util.Collections; -import java.util.Comparator; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.zip.ZipEntry; -import java.util.zip.ZipInputStream; - -import org.ohdsi.utilities.StringUtilities; -import org.ohdsi.utilities.collections.IntegerComparator; -import org.ohdsi.utilities.files.QuickAndDirtyXlsxReader.Sheet; - -public class QuickAndDirtyXlsxReader extends ArrayList { - - private static final long serialVersionUID = 25124428448185386L; - - private List sharedStrings = new ArrayList(); - - private Map rIdToSheet = new HashMap(); - private Map filenameToSheet = new HashMap(); - - public QuickAndDirtyXlsxReader(String filename) { - try { - // Step 1: load the shared strings (if any), and the rels - FileInputStream inputStream = new FileInputStream(filename); - loadSharedStringsAndRels(inputStream); - - // Step 2: load the data: - inputStream = new FileInputStream(filename); - readFromStream(inputStream); - - // Step 3: order the sheets: - Collections.sort(this, new Comparator() { - - @Override - public int compare(Sheet o1, Sheet o2) { - return IntegerComparator.compare(o1.order, o2.order); - } - }); - } catch (FileNotFoundException e) { - e.printStackTrace(); - } - - } - - private void loadSharedStringsAndRels(FileInputStream inputStream) { - try { - int tasksComplete = 0; - ZipInputStream zipInputStream = new ZipInputStream(inputStream); - - ZipEntry zipEntry = null; - while ((zipEntry = zipInputStream.getNextEntry()) != null) { - String filename = zipEntry.getName(); - if (filename.equals("xl/sharedStrings.xml")) { - processSharedStrings(zipInputStream); - tasksComplete++; - } else if (filename.equals("xl/_rels/workbook.xml.rels")) { - processRels(zipInputStream); - tasksComplete++; - } - - if (tasksComplete == 2) { - zipInputStream.close(); - break; - } - } - } catch (IOException e) { - e.printStackTrace(); - } - } - - private void processRels(ZipInputStream inputStream) throws IOException { - BufferedReader bufferedReader = new BufferedReader(new InputStreamReader(inputStream, "UTF-8")); - String line = null; - while ((line = bufferedReader.readLine()) != null) - for (String tag : StringUtilities.multiFindBetween(line, "")) { - String rId = StringUtilities.findBetween(tag, "Id=\"", "\""); - String filename = "xl/" + StringUtilities.findBetween(tag, "Target=\"", "\""); - if (filename.contains("/sheet")) { - Sheet sheet = new Sheet(); - add(sheet); - rIdToSheet.put(rId, sheet); - filenameToSheet.put(filename, sheet); - } - } - } - - public QuickAndDirtyXlsxReader(InputStream stream) { - readFromStream(stream); - } - - private void readFromStream(InputStream inputStream) { - try { - ZipInputStream zipInputStream = new ZipInputStream(inputStream); - - ZipEntry zipEntry = null; - while ((zipEntry = zipInputStream.getNextEntry()) != null) { - String filename = zipEntry.getName(); - if (filename.equals("xl/workbook.xml")) - processWorkBook(zipInputStream); - else if (filename.startsWith("xl/worksheets/sheet")) - processSheet(filename, zipInputStream); - } - } catch (IOException e) { - e.printStackTrace(); - } - - } - - private void processSharedStrings(ZipInputStream inputStream) throws IOException { - BufferedReader bufferedReader = new BufferedReader(new InputStreamReader(inputStream, "UTF-8")); - StringBuilder fullFile = new StringBuilder(); - String line; - while ((line = bufferedReader.readLine()) != null) - fullFile.append(line); - - for (String string : StringUtilities.multiFindBetween(fullFile.toString(), "", "")) - if (string.trim().equals("")) // Empty string - sharedStrings.add(""); - else { - string = StringUtilities.findBetween(string, ">", "<"); - sharedStrings.add(string); - } - } - - private void processSheet(String filename, ZipInputStream inputStream) throws IOException { - Sheet sheet = filenameToSheet.get(filename); - //System.out.println(filename + "\t" + sheet.name); - BufferedReader bufferedReader = new BufferedReader(new InputStreamReader(inputStream, "UTF-8")); - String line; - StringBuilder fullSheet = new StringBuilder(); - while ((line = bufferedReader.readLine()) != null) - fullSheet.append(line); - - for (String rowLine : StringUtilities.multiFindBetween(fullSheet.toString(), "")) { - Row row = new Row(); - row.addAll(findCellValues(rowLine)); - if (row.size() != 0) - sheet.add(row); - } - - } - - public List findCellValues(String string) { - List result = new ArrayList(); - int tagStart = -1; - int stringStart = -1; - int column = -1; - boolean sharedString = false; - for (int i = 0; i < string.length(); i++) { - char ch = string.charAt(i); - if (ch == '<') - tagStart = i + 1; - else if (ch == '>') { - if (tagStart != -1 && i > tagStart) { - String tag = string.substring(tagStart, i); - if (tag.charAt(0) == 'c') { - sharedString = tag.contains("t=\"s\""); - column = parseColumn(StringUtilities.findBetween(tag, "r=\"", "\"")); - } else if (tag.startsWith("v") || tag.startsWith("t")) - stringStart = i + 1; - else if (tag.equals("/v") || tag.equals("/t")) { - if (stringStart != -1 && i > stringStart) { - for (int j = result.size(); j <= column; j++) - result.add(""); - if (sharedString) { - int index = Integer.parseInt(string.substring(stringStart, tagStart - 1)); - result.set(column, sharedStrings.get(index)); - } else - result.set(column, decode(string.substring(stringStart, tagStart - 1))); - } - stringStart = -1; - column = -1; - } - - } - } - - } - return result; - } - - private int parseColumn(String cellIdString) { - int column = 0; - for (int i = 0; i < cellIdString.length(); i++) { - char ch = cellIdString.charAt(i); - if (Character.isDigit(cellIdString.charAt(i))) { - return column - 1; - } else { - column *= 26; - column += (int) ch - 64; - } - } - return -1; - } - - private static HashMap htmlEntities = new HashMap(); - static { - htmlEntities.put("nbsp", new Integer(160)); - htmlEntities.put("iexcl", new Integer(161)); - htmlEntities.put("cent", new Integer(162)); - htmlEntities.put("pound", new Integer(163)); - htmlEntities.put("curren", new Integer(164)); - htmlEntities.put("yen", new Integer(165)); - htmlEntities.put("brvbar", new Integer(166)); - htmlEntities.put("sect", new Integer(167)); - htmlEntities.put("uml", new Integer(168)); - htmlEntities.put("copy", new Integer(169)); - htmlEntities.put("ordf", new Integer(170)); - htmlEntities.put("laquo", new Integer(171)); - htmlEntities.put("not", new Integer(172)); - htmlEntities.put("shy", new Integer(173)); - htmlEntities.put("reg", new Integer(174)); - htmlEntities.put("macr", new Integer(175)); - htmlEntities.put("deg", new Integer(176)); - htmlEntities.put("plusmn", new Integer(177)); - htmlEntities.put("sup2", new Integer(178)); - htmlEntities.put("sup3", new Integer(179)); - htmlEntities.put("acute", new Integer(180)); - htmlEntities.put("micro", new Integer(181)); - htmlEntities.put("para", new Integer(182)); - htmlEntities.put("middot", new Integer(183)); - htmlEntities.put("cedil", new Integer(184)); - htmlEntities.put("sup1", new Integer(185)); - htmlEntities.put("ordm", new Integer(186)); - htmlEntities.put("raquo", new Integer(187)); - htmlEntities.put("frac14", new Integer(188)); - htmlEntities.put("frac12", new Integer(189)); - htmlEntities.put("frac34", new Integer(190)); - htmlEntities.put("iquest", new Integer(191)); - htmlEntities.put("Agrave", new Integer(192)); - htmlEntities.put("Aacute", new Integer(193)); - htmlEntities.put("Acirc", new Integer(194)); - htmlEntities.put("Atilde", new Integer(195)); - htmlEntities.put("Auml", new Integer(196)); - htmlEntities.put("Aring", new Integer(197)); - htmlEntities.put("AElig", new Integer(198)); - htmlEntities.put("Ccedil", new Integer(199)); - htmlEntities.put("Egrave", new Integer(200)); - htmlEntities.put("Eacute", new Integer(201)); - htmlEntities.put("Ecirc", new Integer(202)); - htmlEntities.put("Euml", new Integer(203)); - htmlEntities.put("Igrave", new Integer(204)); - htmlEntities.put("Iacute", new Integer(205)); - htmlEntities.put("Icirc", new Integer(206)); - htmlEntities.put("Iuml", new Integer(207)); - htmlEntities.put("ETH", new Integer(208)); - htmlEntities.put("Ntilde", new Integer(209)); - htmlEntities.put("Ograve", new Integer(210)); - htmlEntities.put("Oacute", new Integer(211)); - htmlEntities.put("Ocirc", new Integer(212)); - htmlEntities.put("Otilde", new Integer(213)); - htmlEntities.put("Ouml", new Integer(214)); - htmlEntities.put("times", new Integer(215)); - htmlEntities.put("Oslash", new Integer(216)); - htmlEntities.put("Ugrave", new Integer(217)); - htmlEntities.put("Uacute", new Integer(218)); - htmlEntities.put("Ucirc", new Integer(219)); - htmlEntities.put("Uuml", new Integer(220)); - htmlEntities.put("Yacute", new Integer(221)); - htmlEntities.put("THORN", new Integer(222)); - htmlEntities.put("szlig", new Integer(223)); - htmlEntities.put("agrave", new Integer(224)); - htmlEntities.put("aacute", new Integer(225)); - htmlEntities.put("acirc", new Integer(226)); - htmlEntities.put("atilde", new Integer(227)); - htmlEntities.put("auml", new Integer(228)); - htmlEntities.put("aring", new Integer(229)); - htmlEntities.put("aelig", new Integer(230)); - htmlEntities.put("ccedil", new Integer(231)); - htmlEntities.put("egrave", new Integer(232)); - htmlEntities.put("eacute", new Integer(233)); - htmlEntities.put("ecirc", new Integer(234)); - htmlEntities.put("euml", new Integer(235)); - htmlEntities.put("igrave", new Integer(236)); - htmlEntities.put("iacute", new Integer(237)); - htmlEntities.put("icirc", new Integer(238)); - htmlEntities.put("iuml", new Integer(239)); - htmlEntities.put("eth", new Integer(240)); - htmlEntities.put("ntilde", new Integer(241)); - htmlEntities.put("ograve", new Integer(242)); - htmlEntities.put("oacute", new Integer(243)); - htmlEntities.put("ocirc", new Integer(244)); - htmlEntities.put("otilde", new Integer(245)); - htmlEntities.put("ouml", new Integer(246)); - htmlEntities.put("divide", new Integer(247)); - htmlEntities.put("oslash", new Integer(248)); - htmlEntities.put("ugrave", new Integer(249)); - htmlEntities.put("uacute", new Integer(250)); - htmlEntities.put("ucirc", new Integer(251)); - htmlEntities.put("uuml", new Integer(252)); - htmlEntities.put("yacute", new Integer(253)); - htmlEntities.put("thorn", new Integer(254)); - htmlEntities.put("yuml", new Integer(255)); - htmlEntities.put("fnof", new Integer(402)); - htmlEntities.put("Alpha", new Integer(913)); - htmlEntities.put("Beta", new Integer(914)); - htmlEntities.put("Gamma", new Integer(915)); - htmlEntities.put("Delta", new Integer(916)); - htmlEntities.put("Epsilon", new Integer(917)); - htmlEntities.put("Zeta", new Integer(918)); - htmlEntities.put("Eta", new Integer(919)); - htmlEntities.put("Theta", new Integer(920)); - htmlEntities.put("Iota", new Integer(921)); - htmlEntities.put("Kappa", new Integer(922)); - htmlEntities.put("Lambda", new Integer(923)); - htmlEntities.put("Mu", new Integer(924)); - htmlEntities.put("Nu", new Integer(925)); - htmlEntities.put("Xi", new Integer(926)); - htmlEntities.put("Omicron", new Integer(927)); - htmlEntities.put("Pi", new Integer(928)); - htmlEntities.put("Rho", new Integer(929)); - htmlEntities.put("Sigma", new Integer(931)); - htmlEntities.put("Tau", new Integer(932)); - htmlEntities.put("Upsilon", new Integer(933)); - htmlEntities.put("Phi", new Integer(934)); - htmlEntities.put("Chi", new Integer(935)); - htmlEntities.put("Psi", new Integer(936)); - htmlEntities.put("Omega", new Integer(937)); - htmlEntities.put("alpha", new Integer(945)); - htmlEntities.put("beta", new Integer(946)); - htmlEntities.put("gamma", new Integer(947)); - htmlEntities.put("delta", new Integer(948)); - htmlEntities.put("epsilon", new Integer(949)); - htmlEntities.put("zeta", new Integer(950)); - htmlEntities.put("eta", new Integer(951)); - htmlEntities.put("theta", new Integer(952)); - htmlEntities.put("iota", new Integer(953)); - htmlEntities.put("kappa", new Integer(954)); - htmlEntities.put("lambda", new Integer(955)); - htmlEntities.put("mu", new Integer(956)); - htmlEntities.put("nu", new Integer(957)); - htmlEntities.put("xi", new Integer(958)); - htmlEntities.put("omicron", new Integer(959)); - htmlEntities.put("pi", new Integer(960)); - htmlEntities.put("rho", new Integer(961)); - htmlEntities.put("sigmaf", new Integer(962)); - htmlEntities.put("sigma", new Integer(963)); - htmlEntities.put("tau", new Integer(964)); - htmlEntities.put("upsilon", new Integer(965)); - htmlEntities.put("phi", new Integer(966)); - htmlEntities.put("chi", new Integer(967)); - htmlEntities.put("psi", new Integer(968)); - htmlEntities.put("omega", new Integer(969)); - htmlEntities.put("thetasym", new Integer(977)); - htmlEntities.put("upsih", new Integer(978)); - htmlEntities.put("piv", new Integer(982)); - htmlEntities.put("bull", new Integer(8226)); - htmlEntities.put("hellip", new Integer(8230)); - htmlEntities.put("prime", new Integer(8242)); - htmlEntities.put("Prime", new Integer(8243)); - htmlEntities.put("oline", new Integer(8254)); - htmlEntities.put("frasl", new Integer(8260)); - htmlEntities.put("weierp", new Integer(8472)); - htmlEntities.put("image", new Integer(8465)); - htmlEntities.put("real", new Integer(8476)); - htmlEntities.put("trade", new Integer(8482)); - htmlEntities.put("alefsym", new Integer(8501)); - htmlEntities.put("larr", new Integer(8592)); - htmlEntities.put("uarr", new Integer(8593)); - htmlEntities.put("rarr", new Integer(8594)); - htmlEntities.put("darr", new Integer(8595)); - htmlEntities.put("harr", new Integer(8596)); - htmlEntities.put("crarr", new Integer(8629)); - htmlEntities.put("lArr", new Integer(8656)); - htmlEntities.put("uArr", new Integer(8657)); - htmlEntities.put("rArr", new Integer(8658)); - htmlEntities.put("dArr", new Integer(8659)); - htmlEntities.put("hArr", new Integer(8660)); - htmlEntities.put("forall", new Integer(8704)); - htmlEntities.put("part", new Integer(8706)); - htmlEntities.put("exist", new Integer(8707)); - htmlEntities.put("empty", new Integer(8709)); - htmlEntities.put("nabla", new Integer(8711)); - htmlEntities.put("isin", new Integer(8712)); - htmlEntities.put("notin", new Integer(8713)); - htmlEntities.put("ni", new Integer(8715)); - htmlEntities.put("prod", new Integer(8719)); - htmlEntities.put("sum", new Integer(8721)); - htmlEntities.put("minus", new Integer(8722)); - htmlEntities.put("lowast", new Integer(8727)); - htmlEntities.put("radic", new Integer(8730)); - htmlEntities.put("prop", new Integer(8733)); - htmlEntities.put("infin", new Integer(8734)); - htmlEntities.put("ang", new Integer(8736)); - htmlEntities.put("and", new Integer(8743)); - htmlEntities.put("or", new Integer(8744)); - htmlEntities.put("cap", new Integer(8745)); - htmlEntities.put("cup", new Integer(8746)); - htmlEntities.put("int", new Integer(8747)); - htmlEntities.put("there4", new Integer(8756)); - htmlEntities.put("sim", new Integer(8764)); - htmlEntities.put("cong", new Integer(8773)); - htmlEntities.put("asymp", new Integer(8776)); - htmlEntities.put("ne", new Integer(8800)); - htmlEntities.put("equiv", new Integer(8801)); - htmlEntities.put("le", new Integer(8804)); - htmlEntities.put("ge", new Integer(8805)); - htmlEntities.put("sub", new Integer(8834)); - htmlEntities.put("sup", new Integer(8835)); - htmlEntities.put("nsub", new Integer(8836)); - htmlEntities.put("sube", new Integer(8838)); - htmlEntities.put("supe", new Integer(8839)); - htmlEntities.put("oplus", new Integer(8853)); - htmlEntities.put("otimes", new Integer(8855)); - htmlEntities.put("perp", new Integer(8869)); - htmlEntities.put("sdot", new Integer(8901)); - htmlEntities.put("lceil", new Integer(8968)); - htmlEntities.put("rceil", new Integer(8969)); - htmlEntities.put("lfloor", new Integer(8970)); - htmlEntities.put("rfloor", new Integer(8971)); - htmlEntities.put("lang", new Integer(9001)); - htmlEntities.put("rang", new Integer(9002)); - htmlEntities.put("loz", new Integer(9674)); - htmlEntities.put("spades", new Integer(9824)); - htmlEntities.put("clubs", new Integer(9827)); - htmlEntities.put("hearts", new Integer(9829)); - htmlEntities.put("diams", new Integer(9830)); - htmlEntities.put("quot", new Integer(34)); - htmlEntities.put("amp", new Integer(38)); - htmlEntities.put("lt", new Integer(60)); - htmlEntities.put("gt", new Integer(62)); - htmlEntities.put("OElig", new Integer(338)); - htmlEntities.put("oelig", new Integer(339)); - htmlEntities.put("Scaron", new Integer(352)); - htmlEntities.put("scaron", new Integer(353)); - htmlEntities.put("Yuml", new Integer(376)); - htmlEntities.put("circ", new Integer(710)); - htmlEntities.put("tilde", new Integer(732)); - htmlEntities.put("ensp", new Integer(8194)); - htmlEntities.put("emsp", new Integer(8195)); - htmlEntities.put("thinsp", new Integer(8201)); - htmlEntities.put("zwnj", new Integer(8204)); - htmlEntities.put("zwj", new Integer(8205)); - htmlEntities.put("lrm", new Integer(8206)); - htmlEntities.put("rlm", new Integer(8207)); - htmlEntities.put("ndash", new Integer(8211)); - htmlEntities.put("mdash", new Integer(8212)); - htmlEntities.put("lsquo", new Integer(8216)); - htmlEntities.put("rsquo", new Integer(8217)); - htmlEntities.put("sbquo", new Integer(8218)); - htmlEntities.put("ldquo", new Integer(8220)); - htmlEntities.put("rdquo", new Integer(8221)); - htmlEntities.put("bdquo", new Integer(8222)); - htmlEntities.put("dagger", new Integer(8224)); - htmlEntities.put("Dagger", new Integer(8225)); - htmlEntities.put("permil", new Integer(8240)); - htmlEntities.put("lsaquo", new Integer(8249)); - htmlEntities.put("rsaquo", new Integer(8250)); - htmlEntities.put("euro", new Integer(8364)); - } - - public static String decode(String s) { - StringBuffer result = new StringBuffer(s.length()); - int ampInd = s.indexOf("&"); - int lastEnd = 0; - while (ampInd >= 0) { - int nextAmp = s.indexOf("&", ampInd + 1); - int nextSemi = s.indexOf(";", ampInd + 1); - if (nextSemi != -1 && (nextAmp == -1 || nextSemi < nextAmp)) { - int value = -1; - String escape = s.substring(ampInd + 1, nextSemi); - try { - if (escape.startsWith("#")) { - value = Integer.parseInt(escape.substring(1), 10); - } else { - if (htmlEntities.containsKey(escape)) { - value = ((Integer) (htmlEntities.get(escape))).intValue(); - } - } - } catch (NumberFormatException x) { - } - result.append(s.substring(lastEnd, ampInd)); - lastEnd = nextSemi + 1; - if (value >= 0 && value <= 0xffff) { - result.append((char) value); - } else { - result.append("&").append(escape).append(";"); - } - } - ampInd = nextAmp; - } - result.append(s.substring(lastEnd)); - return result.toString(); - } - - private void processWorkBook(InputStream inputStream) throws NumberFormatException, IOException { - BufferedReader bufferedReader = new BufferedReader(new InputStreamReader(inputStream, "UTF-8")); - String line = null; - while ((line = bufferedReader.readLine()) != null) { - for (String sheetTag : StringUtilities.multiFindBetween(line, "")) { - String name = StringUtilities.findBetween(sheetTag, "name=\"", "\""); - String order = StringUtilities.findBetween(sheetTag, "sheetId=\"", "\""); - String rId = StringUtilities.findBetween(sheetTag, "r:id=\"", "\""); - Sheet sheet = rIdToSheet.get(rId); - sheet.setName(name); - sheet.order = Integer.parseInt(order); - } - } - } - - public class Sheet extends ArrayList { - private static final long serialVersionUID = -8597151681911998153L; - private String name; - private int order; - - public String getName() { - return name; - } - - public void setName(String name) { - this.name = name; - } - - } - - public class Row extends ArrayList { - private static final long serialVersionUID = -6391290892840364766L; - - } -} +/******************************************************************************* + * Copyright 2019 Observational Health Data Sciences and Informatics + * + * This file is part of WhiteRabbit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + ******************************************************************************/ +package org.ohdsi.utilities.files; + +import java.io.BufferedReader; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.zip.ZipEntry; +import java.util.zip.ZipInputStream; + +import org.ohdsi.utilities.StringUtilities; +import org.ohdsi.utilities.collections.IntegerComparator; +import org.ohdsi.utilities.files.QuickAndDirtyXlsxReader.Sheet; + +public class QuickAndDirtyXlsxReader extends ArrayList { + + private static final long serialVersionUID = 25124428448185386L; + + private List sharedStrings = new ArrayList(); + + private Map rIdToSheet = new HashMap(); + private Map filenameToSheet = new HashMap(); + + public QuickAndDirtyXlsxReader(String filename) { + try { + // Step 1: load the shared strings (if any), and the rels + FileInputStream inputStream = new FileInputStream(filename); + loadSharedStringsAndRels(inputStream); + + // Step 2: load the data: + inputStream = new FileInputStream(filename); + readFromStream(inputStream); + + // Step 3: order the sheets: + Collections.sort(this, new Comparator() { + + @Override + public int compare(Sheet o1, Sheet o2) { + return IntegerComparator.compare(o1.order, o2.order); + } + }); + } catch (FileNotFoundException e) { + e.printStackTrace(); + } + + } + + private void loadSharedStringsAndRels(FileInputStream inputStream) { + try { + int tasksComplete = 0; + ZipInputStream zipInputStream = new ZipInputStream(inputStream); + + ZipEntry zipEntry = null; + while ((zipEntry = zipInputStream.getNextEntry()) != null) { + String filename = zipEntry.getName(); + if (filename.equals("xl/sharedStrings.xml")) { + processSharedStrings(zipInputStream); + tasksComplete++; + } else if (filename.equals("xl/_rels/workbook.xml.rels")) { + processRels(zipInputStream); + tasksComplete++; + } + + if (tasksComplete == 2) { + zipInputStream.close(); + break; + } + } + } catch (IOException e) { + e.printStackTrace(); + } + } + + private void processRels(ZipInputStream inputStream) throws IOException { + BufferedReader bufferedReader = new BufferedReader(new InputStreamReader(inputStream, "UTF-8")); + String line = null; + while ((line = bufferedReader.readLine()) != null) + for (String tag : StringUtilities.multiFindBetween(line, "")) { + String rId = StringUtilities.findBetween(tag, "Id=\"", "\""); + String filename = "xl/" + StringUtilities.findBetween(tag, "Target=\"", "\""); + if (filename.contains("/sheet")) { + Sheet sheet = new Sheet(); + add(sheet); + rIdToSheet.put(rId, sheet); + filenameToSheet.put(filename, sheet); + } + } + } + + public QuickAndDirtyXlsxReader(InputStream stream) { + readFromStream(stream); + } + + private void readFromStream(InputStream inputStream) { + try { + ZipInputStream zipInputStream = new ZipInputStream(inputStream); + + ZipEntry zipEntry = null; + while ((zipEntry = zipInputStream.getNextEntry()) != null) { + String filename = zipEntry.getName(); + if (filename.equals("xl/workbook.xml")) + processWorkBook(zipInputStream); + else if (filename.startsWith("xl/worksheets/sheet")) + processSheet(filename, zipInputStream); + } + } catch (IOException e) { + e.printStackTrace(); + } + + } + + private void processSharedStrings(ZipInputStream inputStream) throws IOException { + BufferedReader bufferedReader = new BufferedReader(new InputStreamReader(inputStream, "UTF-8")); + StringBuilder fullFile = new StringBuilder(); + String line; + while ((line = bufferedReader.readLine()) != null) + fullFile.append(line); + + for (String string : StringUtilities.multiFindBetween(fullFile.toString(), "", "")) + if (string.trim().equals("")) // Empty string + sharedStrings.add(""); + else { + string = StringUtilities.findBetween(string, ">", "<"); + sharedStrings.add(string); + } + } + + private void processSheet(String filename, ZipInputStream inputStream) throws IOException { + Sheet sheet = filenameToSheet.get(filename); + //System.out.println(filename + "\t" + sheet.name); + BufferedReader bufferedReader = new BufferedReader(new InputStreamReader(inputStream, "UTF-8")); + String line; + StringBuilder fullSheet = new StringBuilder(); + while ((line = bufferedReader.readLine()) != null) + fullSheet.append(line); + + for (String rowLine : StringUtilities.multiFindBetween(fullSheet.toString(), "")) { + Row row = new Row(); + row.addAll(findCellValues(rowLine)); + if (row.size() != 0) + sheet.add(row); + } + + } + + public List findCellValues(String string) { + List result = new ArrayList(); + int tagStart = -1; + int stringStart = -1; + int column = -1; + boolean sharedString = false; + for (int i = 0; i < string.length(); i++) { + char ch = string.charAt(i); + if (ch == '<') + tagStart = i + 1; + else if (ch == '>') { + if (tagStart != -1 && i > tagStart) { + String tag = string.substring(tagStart, i); + if (tag.charAt(0) == 'c') { + sharedString = tag.contains("t=\"s\""); + column = parseColumn(StringUtilities.findBetween(tag, "r=\"", "\"")); + } else if (tag.startsWith("v") || tag.startsWith("t")) + stringStart = i + 1; + else if (tag.equals("/v") || tag.equals("/t")) { + if (stringStart != -1 && i > stringStart) { + for (int j = result.size(); j <= column; j++) + result.add(""); + if (sharedString) { + int index = Integer.parseInt(string.substring(stringStart, tagStart - 1)); + result.set(column, sharedStrings.get(index)); + } else + result.set(column, decode(string.substring(stringStart, tagStart - 1))); + } + stringStart = -1; + column = -1; + } + + } + } + + } + return result; + } + + private int parseColumn(String cellIdString) { + int column = 0; + for (int i = 0; i < cellIdString.length(); i++) { + char ch = cellIdString.charAt(i); + if (Character.isDigit(cellIdString.charAt(i))) { + return column - 1; + } else { + column *= 26; + column += (int) ch - 64; + } + } + return -1; + } + + private static HashMap htmlEntities = new HashMap(); + static { + htmlEntities.put("nbsp", new Integer(160)); + htmlEntities.put("iexcl", new Integer(161)); + htmlEntities.put("cent", new Integer(162)); + htmlEntities.put("pound", new Integer(163)); + htmlEntities.put("curren", new Integer(164)); + htmlEntities.put("yen", new Integer(165)); + htmlEntities.put("brvbar", new Integer(166)); + htmlEntities.put("sect", new Integer(167)); + htmlEntities.put("uml", new Integer(168)); + htmlEntities.put("copy", new Integer(169)); + htmlEntities.put("ordf", new Integer(170)); + htmlEntities.put("laquo", new Integer(171)); + htmlEntities.put("not", new Integer(172)); + htmlEntities.put("shy", new Integer(173)); + htmlEntities.put("reg", new Integer(174)); + htmlEntities.put("macr", new Integer(175)); + htmlEntities.put("deg", new Integer(176)); + htmlEntities.put("plusmn", new Integer(177)); + htmlEntities.put("sup2", new Integer(178)); + htmlEntities.put("sup3", new Integer(179)); + htmlEntities.put("acute", new Integer(180)); + htmlEntities.put("micro", new Integer(181)); + htmlEntities.put("para", new Integer(182)); + htmlEntities.put("middot", new Integer(183)); + htmlEntities.put("cedil", new Integer(184)); + htmlEntities.put("sup1", new Integer(185)); + htmlEntities.put("ordm", new Integer(186)); + htmlEntities.put("raquo", new Integer(187)); + htmlEntities.put("frac14", new Integer(188)); + htmlEntities.put("frac12", new Integer(189)); + htmlEntities.put("frac34", new Integer(190)); + htmlEntities.put("iquest", new Integer(191)); + htmlEntities.put("Agrave", new Integer(192)); + htmlEntities.put("Aacute", new Integer(193)); + htmlEntities.put("Acirc", new Integer(194)); + htmlEntities.put("Atilde", new Integer(195)); + htmlEntities.put("Auml", new Integer(196)); + htmlEntities.put("Aring", new Integer(197)); + htmlEntities.put("AElig", new Integer(198)); + htmlEntities.put("Ccedil", new Integer(199)); + htmlEntities.put("Egrave", new Integer(200)); + htmlEntities.put("Eacute", new Integer(201)); + htmlEntities.put("Ecirc", new Integer(202)); + htmlEntities.put("Euml", new Integer(203)); + htmlEntities.put("Igrave", new Integer(204)); + htmlEntities.put("Iacute", new Integer(205)); + htmlEntities.put("Icirc", new Integer(206)); + htmlEntities.put("Iuml", new Integer(207)); + htmlEntities.put("ETH", new Integer(208)); + htmlEntities.put("Ntilde", new Integer(209)); + htmlEntities.put("Ograve", new Integer(210)); + htmlEntities.put("Oacute", new Integer(211)); + htmlEntities.put("Ocirc", new Integer(212)); + htmlEntities.put("Otilde", new Integer(213)); + htmlEntities.put("Ouml", new Integer(214)); + htmlEntities.put("times", new Integer(215)); + htmlEntities.put("Oslash", new Integer(216)); + htmlEntities.put("Ugrave", new Integer(217)); + htmlEntities.put("Uacute", new Integer(218)); + htmlEntities.put("Ucirc", new Integer(219)); + htmlEntities.put("Uuml", new Integer(220)); + htmlEntities.put("Yacute", new Integer(221)); + htmlEntities.put("THORN", new Integer(222)); + htmlEntities.put("szlig", new Integer(223)); + htmlEntities.put("agrave", new Integer(224)); + htmlEntities.put("aacute", new Integer(225)); + htmlEntities.put("acirc", new Integer(226)); + htmlEntities.put("atilde", new Integer(227)); + htmlEntities.put("auml", new Integer(228)); + htmlEntities.put("aring", new Integer(229)); + htmlEntities.put("aelig", new Integer(230)); + htmlEntities.put("ccedil", new Integer(231)); + htmlEntities.put("egrave", new Integer(232)); + htmlEntities.put("eacute", new Integer(233)); + htmlEntities.put("ecirc", new Integer(234)); + htmlEntities.put("euml", new Integer(235)); + htmlEntities.put("igrave", new Integer(236)); + htmlEntities.put("iacute", new Integer(237)); + htmlEntities.put("icirc", new Integer(238)); + htmlEntities.put("iuml", new Integer(239)); + htmlEntities.put("eth", new Integer(240)); + htmlEntities.put("ntilde", new Integer(241)); + htmlEntities.put("ograve", new Integer(242)); + htmlEntities.put("oacute", new Integer(243)); + htmlEntities.put("ocirc", new Integer(244)); + htmlEntities.put("otilde", new Integer(245)); + htmlEntities.put("ouml", new Integer(246)); + htmlEntities.put("divide", new Integer(247)); + htmlEntities.put("oslash", new Integer(248)); + htmlEntities.put("ugrave", new Integer(249)); + htmlEntities.put("uacute", new Integer(250)); + htmlEntities.put("ucirc", new Integer(251)); + htmlEntities.put("uuml", new Integer(252)); + htmlEntities.put("yacute", new Integer(253)); + htmlEntities.put("thorn", new Integer(254)); + htmlEntities.put("yuml", new Integer(255)); + htmlEntities.put("fnof", new Integer(402)); + htmlEntities.put("Alpha", new Integer(913)); + htmlEntities.put("Beta", new Integer(914)); + htmlEntities.put("Gamma", new Integer(915)); + htmlEntities.put("Delta", new Integer(916)); + htmlEntities.put("Epsilon", new Integer(917)); + htmlEntities.put("Zeta", new Integer(918)); + htmlEntities.put("Eta", new Integer(919)); + htmlEntities.put("Theta", new Integer(920)); + htmlEntities.put("Iota", new Integer(921)); + htmlEntities.put("Kappa", new Integer(922)); + htmlEntities.put("Lambda", new Integer(923)); + htmlEntities.put("Mu", new Integer(924)); + htmlEntities.put("Nu", new Integer(925)); + htmlEntities.put("Xi", new Integer(926)); + htmlEntities.put("Omicron", new Integer(927)); + htmlEntities.put("Pi", new Integer(928)); + htmlEntities.put("Rho", new Integer(929)); + htmlEntities.put("Sigma", new Integer(931)); + htmlEntities.put("Tau", new Integer(932)); + htmlEntities.put("Upsilon", new Integer(933)); + htmlEntities.put("Phi", new Integer(934)); + htmlEntities.put("Chi", new Integer(935)); + htmlEntities.put("Psi", new Integer(936)); + htmlEntities.put("Omega", new Integer(937)); + htmlEntities.put("alpha", new Integer(945)); + htmlEntities.put("beta", new Integer(946)); + htmlEntities.put("gamma", new Integer(947)); + htmlEntities.put("delta", new Integer(948)); + htmlEntities.put("epsilon", new Integer(949)); + htmlEntities.put("zeta", new Integer(950)); + htmlEntities.put("eta", new Integer(951)); + htmlEntities.put("theta", new Integer(952)); + htmlEntities.put("iota", new Integer(953)); + htmlEntities.put("kappa", new Integer(954)); + htmlEntities.put("lambda", new Integer(955)); + htmlEntities.put("mu", new Integer(956)); + htmlEntities.put("nu", new Integer(957)); + htmlEntities.put("xi", new Integer(958)); + htmlEntities.put("omicron", new Integer(959)); + htmlEntities.put("pi", new Integer(960)); + htmlEntities.put("rho", new Integer(961)); + htmlEntities.put("sigmaf", new Integer(962)); + htmlEntities.put("sigma", new Integer(963)); + htmlEntities.put("tau", new Integer(964)); + htmlEntities.put("upsilon", new Integer(965)); + htmlEntities.put("phi", new Integer(966)); + htmlEntities.put("chi", new Integer(967)); + htmlEntities.put("psi", new Integer(968)); + htmlEntities.put("omega", new Integer(969)); + htmlEntities.put("thetasym", new Integer(977)); + htmlEntities.put("upsih", new Integer(978)); + htmlEntities.put("piv", new Integer(982)); + htmlEntities.put("bull", new Integer(8226)); + htmlEntities.put("hellip", new Integer(8230)); + htmlEntities.put("prime", new Integer(8242)); + htmlEntities.put("Prime", new Integer(8243)); + htmlEntities.put("oline", new Integer(8254)); + htmlEntities.put("frasl", new Integer(8260)); + htmlEntities.put("weierp", new Integer(8472)); + htmlEntities.put("image", new Integer(8465)); + htmlEntities.put("real", new Integer(8476)); + htmlEntities.put("trade", new Integer(8482)); + htmlEntities.put("alefsym", new Integer(8501)); + htmlEntities.put("larr", new Integer(8592)); + htmlEntities.put("uarr", new Integer(8593)); + htmlEntities.put("rarr", new Integer(8594)); + htmlEntities.put("darr", new Integer(8595)); + htmlEntities.put("harr", new Integer(8596)); + htmlEntities.put("crarr", new Integer(8629)); + htmlEntities.put("lArr", new Integer(8656)); + htmlEntities.put("uArr", new Integer(8657)); + htmlEntities.put("rArr", new Integer(8658)); + htmlEntities.put("dArr", new Integer(8659)); + htmlEntities.put("hArr", new Integer(8660)); + htmlEntities.put("forall", new Integer(8704)); + htmlEntities.put("part", new Integer(8706)); + htmlEntities.put("exist", new Integer(8707)); + htmlEntities.put("empty", new Integer(8709)); + htmlEntities.put("nabla", new Integer(8711)); + htmlEntities.put("isin", new Integer(8712)); + htmlEntities.put("notin", new Integer(8713)); + htmlEntities.put("ni", new Integer(8715)); + htmlEntities.put("prod", new Integer(8719)); + htmlEntities.put("sum", new Integer(8721)); + htmlEntities.put("minus", new Integer(8722)); + htmlEntities.put("lowast", new Integer(8727)); + htmlEntities.put("radic", new Integer(8730)); + htmlEntities.put("prop", new Integer(8733)); + htmlEntities.put("infin", new Integer(8734)); + htmlEntities.put("ang", new Integer(8736)); + htmlEntities.put("and", new Integer(8743)); + htmlEntities.put("or", new Integer(8744)); + htmlEntities.put("cap", new Integer(8745)); + htmlEntities.put("cup", new Integer(8746)); + htmlEntities.put("int", new Integer(8747)); + htmlEntities.put("there4", new Integer(8756)); + htmlEntities.put("sim", new Integer(8764)); + htmlEntities.put("cong", new Integer(8773)); + htmlEntities.put("asymp", new Integer(8776)); + htmlEntities.put("ne", new Integer(8800)); + htmlEntities.put("equiv", new Integer(8801)); + htmlEntities.put("le", new Integer(8804)); + htmlEntities.put("ge", new Integer(8805)); + htmlEntities.put("sub", new Integer(8834)); + htmlEntities.put("sup", new Integer(8835)); + htmlEntities.put("nsub", new Integer(8836)); + htmlEntities.put("sube", new Integer(8838)); + htmlEntities.put("supe", new Integer(8839)); + htmlEntities.put("oplus", new Integer(8853)); + htmlEntities.put("otimes", new Integer(8855)); + htmlEntities.put("perp", new Integer(8869)); + htmlEntities.put("sdot", new Integer(8901)); + htmlEntities.put("lceil", new Integer(8968)); + htmlEntities.put("rceil", new Integer(8969)); + htmlEntities.put("lfloor", new Integer(8970)); + htmlEntities.put("rfloor", new Integer(8971)); + htmlEntities.put("lang", new Integer(9001)); + htmlEntities.put("rang", new Integer(9002)); + htmlEntities.put("loz", new Integer(9674)); + htmlEntities.put("spades", new Integer(9824)); + htmlEntities.put("clubs", new Integer(9827)); + htmlEntities.put("hearts", new Integer(9829)); + htmlEntities.put("diams", new Integer(9830)); + htmlEntities.put("quot", new Integer(34)); + htmlEntities.put("amp", new Integer(38)); + htmlEntities.put("lt", new Integer(60)); + htmlEntities.put("gt", new Integer(62)); + htmlEntities.put("OElig", new Integer(338)); + htmlEntities.put("oelig", new Integer(339)); + htmlEntities.put("Scaron", new Integer(352)); + htmlEntities.put("scaron", new Integer(353)); + htmlEntities.put("Yuml", new Integer(376)); + htmlEntities.put("circ", new Integer(710)); + htmlEntities.put("tilde", new Integer(732)); + htmlEntities.put("ensp", new Integer(8194)); + htmlEntities.put("emsp", new Integer(8195)); + htmlEntities.put("thinsp", new Integer(8201)); + htmlEntities.put("zwnj", new Integer(8204)); + htmlEntities.put("zwj", new Integer(8205)); + htmlEntities.put("lrm", new Integer(8206)); + htmlEntities.put("rlm", new Integer(8207)); + htmlEntities.put("ndash", new Integer(8211)); + htmlEntities.put("mdash", new Integer(8212)); + htmlEntities.put("lsquo", new Integer(8216)); + htmlEntities.put("rsquo", new Integer(8217)); + htmlEntities.put("sbquo", new Integer(8218)); + htmlEntities.put("ldquo", new Integer(8220)); + htmlEntities.put("rdquo", new Integer(8221)); + htmlEntities.put("bdquo", new Integer(8222)); + htmlEntities.put("dagger", new Integer(8224)); + htmlEntities.put("Dagger", new Integer(8225)); + htmlEntities.put("permil", new Integer(8240)); + htmlEntities.put("lsaquo", new Integer(8249)); + htmlEntities.put("rsaquo", new Integer(8250)); + htmlEntities.put("euro", new Integer(8364)); + } + + public static String decode(String s) { + StringBuffer result = new StringBuffer(s.length()); + int ampInd = s.indexOf("&"); + int lastEnd = 0; + while (ampInd >= 0) { + int nextAmp = s.indexOf("&", ampInd + 1); + int nextSemi = s.indexOf(";", ampInd + 1); + if (nextSemi != -1 && (nextAmp == -1 || nextSemi < nextAmp)) { + int value = -1; + String escape = s.substring(ampInd + 1, nextSemi); + try { + if (escape.startsWith("#")) { + value = Integer.parseInt(escape.substring(1), 10); + } else { + if (htmlEntities.containsKey(escape)) { + value = ((Integer) (htmlEntities.get(escape))).intValue(); + } + } + } catch (NumberFormatException x) { + } + result.append(s.substring(lastEnd, ampInd)); + lastEnd = nextSemi + 1; + if (value >= 0 && value <= 0xffff) { + result.append((char) value); + } else { + result.append("&").append(escape).append(";"); + } + } + ampInd = nextAmp; + } + result.append(s.substring(lastEnd)); + return result.toString(); + } + + private void processWorkBook(InputStream inputStream) throws NumberFormatException, IOException { + BufferedReader bufferedReader = new BufferedReader(new InputStreamReader(inputStream, "UTF-8")); + String line = null; + while ((line = bufferedReader.readLine()) != null) { + for (String sheetTag : StringUtilities.multiFindBetween(line, "")) { + String name = StringUtilities.findBetween(sheetTag, "name=\"", "\""); + String order = StringUtilities.findBetween(sheetTag, "sheetId=\"", "\""); + String rId = StringUtilities.findBetween(sheetTag, "r:id=\"", "\""); + Sheet sheet = rIdToSheet.get(rId); + sheet.setName(name); + sheet.order = Integer.parseInt(order); + } + } + } + + public class Sheet extends ArrayList { + private static final long serialVersionUID = -8597151681911998153L; + private String name; + private int order; + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + } + + public class Row extends ArrayList { + private static final long serialVersionUID = -6391290892840364766L; + + } +} diff --git a/src/org/ohdsi/utilities/files/ReadTextFile.java b/rabbit-core/src/main/java/org/ohdsi/utilities/files/ReadTextFile.java similarity index 94% rename from src/org/ohdsi/utilities/files/ReadTextFile.java rename to rabbit-core/src/main/java/org/ohdsi/utilities/files/ReadTextFile.java index 4afc3db9..5dbcce40 100644 --- a/src/org/ohdsi/utilities/files/ReadTextFile.java +++ b/rabbit-core/src/main/java/org/ohdsi/utilities/files/ReadTextFile.java @@ -1,5 +1,5 @@ /******************************************************************************* - * Copyright 2017 Observational Health Data Sciences and Informatics + * Copyright 2019 Observational Health Data Sciences and Informatics * * This file is part of WhiteRabbit * diff --git a/src/org/ohdsi/utilities/files/Row.java b/rabbit-core/src/main/java/org/ohdsi/utilities/files/Row.java similarity index 95% rename from src/org/ohdsi/utilities/files/Row.java rename to rabbit-core/src/main/java/org/ohdsi/utilities/files/Row.java index 77f384fa..10689451 100644 --- a/src/org/ohdsi/utilities/files/Row.java +++ b/rabbit-core/src/main/java/org/ohdsi/utilities/files/Row.java @@ -1,5 +1,5 @@ /******************************************************************************* - * Copyright 2017 Observational Health Data Sciences and Informatics + * Copyright 2019 Observational Health Data Sciences and Informatics * * This file is part of WhiteRabbit * diff --git a/src/org/ohdsi/utilities/files/RowUtilities.java b/rabbit-core/src/main/java/org/ohdsi/utilities/files/RowUtilities.java similarity index 94% rename from src/org/ohdsi/utilities/files/RowUtilities.java rename to rabbit-core/src/main/java/org/ohdsi/utilities/files/RowUtilities.java index 5b8e67d2..05a0499b 100644 --- a/src/org/ohdsi/utilities/files/RowUtilities.java +++ b/rabbit-core/src/main/java/org/ohdsi/utilities/files/RowUtilities.java @@ -1,5 +1,5 @@ /******************************************************************************* - * Copyright 2017 Observational Health Data Sciences and Informatics + * Copyright 2019 Observational Health Data Sciences and Informatics * * This file is part of WhiteRabbit * diff --git a/src/org/ohdsi/utilities/files/StringIdComparator.java b/rabbit-core/src/main/java/org/ohdsi/utilities/files/StringIdComparator.java similarity index 93% rename from src/org/ohdsi/utilities/files/StringIdComparator.java rename to rabbit-core/src/main/java/org/ohdsi/utilities/files/StringIdComparator.java index 1bf2fffc..7da66def 100644 --- a/src/org/ohdsi/utilities/files/StringIdComparator.java +++ b/rabbit-core/src/main/java/org/ohdsi/utilities/files/StringIdComparator.java @@ -1,5 +1,5 @@ /******************************************************************************* - * Copyright 2017 Observational Health Data Sciences and Informatics + * Copyright 2019 Observational Health Data Sciences and Informatics * * This file is part of WhiteRabbit * diff --git a/src/org/ohdsi/utilities/files/WriteCSVFileWithHeader.java b/rabbit-core/src/main/java/org/ohdsi/utilities/files/WriteCSVFileWithHeader.java similarity index 96% rename from src/org/ohdsi/utilities/files/WriteCSVFileWithHeader.java rename to rabbit-core/src/main/java/org/ohdsi/utilities/files/WriteCSVFileWithHeader.java index dd1ea499..f6329154 100644 --- a/src/org/ohdsi/utilities/files/WriteCSVFileWithHeader.java +++ b/rabbit-core/src/main/java/org/ohdsi/utilities/files/WriteCSVFileWithHeader.java @@ -1,59 +1,59 @@ -package org.ohdsi.utilities.files; - -import java.io.FileWriter; -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; - -import org.apache.commons.csv.CSVFormat; -import org.apache.commons.csv.CSVPrinter; - -public class WriteCSVFileWithHeader { - - private CSVPrinter printer; - private boolean headerWritten = false; - - public WriteCSVFileWithHeader(String fileName) { - this(fileName, CSVFormat.RFC4180); - } - - public WriteCSVFileWithHeader(String fileName, CSVFormat format) { - try { - printer = new CSVPrinter(new FileWriter(fileName), format); - } catch (IOException e) { - throw new RuntimeException(e.getMessage()); - } - } - - public void write(Row row) { - try { - if (!headerWritten) - writeHeader(row); - printer.printRecord(row.getCells()); - } catch (IOException e) { - throw new RuntimeException(e.getMessage()); - } - - } - - private void writeHeader(Row row) throws IOException { - headerWritten = true; - Map fieldName2ColumnIndex = row.getfieldName2ColumnIndex(); - int size = fieldName2ColumnIndex.size(); - List header = new ArrayList(size); - for (int i = 0; i < size; i++) - header.add(null); - for (Map.Entry entry : fieldName2ColumnIndex.entrySet()) - header.set(entry.getValue(), entry.getKey()); - printer.printRecord(header); - } - - public void close() { - try { - printer.close(); - } catch (IOException e) { - throw new RuntimeException(e.getMessage()); - } - } -} +package org.ohdsi.utilities.files; + +import java.io.FileWriter; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +import org.apache.commons.csv.CSVFormat; +import org.apache.commons.csv.CSVPrinter; + +public class WriteCSVFileWithHeader { + + private CSVPrinter printer; + private boolean headerWritten = false; + + public WriteCSVFileWithHeader(String fileName) { + this(fileName, CSVFormat.RFC4180); + } + + public WriteCSVFileWithHeader(String fileName, CSVFormat format) { + try { + printer = new CSVPrinter(new FileWriter(fileName), format); + } catch (IOException e) { + throw new RuntimeException(e.getMessage()); + } + } + + public void write(Row row) { + try { + if (!headerWritten) + writeHeader(row); + printer.printRecord(row.getCells()); + } catch (IOException e) { + throw new RuntimeException(e.getMessage()); + } + + } + + private void writeHeader(Row row) throws IOException { + headerWritten = true; + Map fieldName2ColumnIndex = row.getfieldName2ColumnIndex(); + int size = fieldName2ColumnIndex.size(); + List header = new ArrayList(size); + for (int i = 0; i < size; i++) + header.add(null); + for (Map.Entry entry : fieldName2ColumnIndex.entrySet()) + header.set(entry.getValue(), entry.getKey()); + printer.printRecord(header); + } + + public void close() { + try { + printer.close(); + } catch (IOException e) { + throw new RuntimeException(e.getMessage()); + } + } +} diff --git a/src/org/ohdsi/utilities/files/WriteTextFile.java b/rabbit-core/src/main/java/org/ohdsi/utilities/files/WriteTextFile.java similarity index 93% rename from src/org/ohdsi/utilities/files/WriteTextFile.java rename to rabbit-core/src/main/java/org/ohdsi/utilities/files/WriteTextFile.java index bcf34cb3..88e67a51 100644 --- a/src/org/ohdsi/utilities/files/WriteTextFile.java +++ b/rabbit-core/src/main/java/org/ohdsi/utilities/files/WriteTextFile.java @@ -1,5 +1,5 @@ /******************************************************************************* - * Copyright 2017 Observational Health Data Sciences and Informatics + * Copyright 2019 Observational Health Data Sciences and Informatics * * This file is part of WhiteRabbit * diff --git a/rabbitinahat/.classpath b/rabbitinahat/.classpath new file mode 100644 index 00000000..6d7587a8 --- /dev/null +++ b/rabbitinahat/.classpath @@ -0,0 +1,31 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/rabbitinahat/.project b/rabbitinahat/.project new file mode 100644 index 00000000..cd9a2bfb --- /dev/null +++ b/rabbitinahat/.project @@ -0,0 +1,23 @@ + + + rabbitinahat + + + + + + org.eclipse.jdt.core.javabuilder + + + + + org.eclipse.m2e.core.maven2Builder + + + + + + org.eclipse.jdt.core.javanature + org.eclipse.m2e.core.maven2Nature + + diff --git a/rabbitinahat/.settings/org.eclipse.jdt.core.prefs b/rabbitinahat/.settings/org.eclipse.jdt.core.prefs new file mode 100644 index 00000000..714351ae --- /dev/null +++ b/rabbitinahat/.settings/org.eclipse.jdt.core.prefs @@ -0,0 +1,5 @@ +eclipse.preferences.version=1 +org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.8 +org.eclipse.jdt.core.compiler.compliance=1.8 +org.eclipse.jdt.core.compiler.problem.forbiddenReference=warning +org.eclipse.jdt.core.compiler.source=1.8 diff --git a/rabbitinahat/.settings/org.eclipse.m2e.core.prefs b/rabbitinahat/.settings/org.eclipse.m2e.core.prefs new file mode 100644 index 00000000..f897a7f1 --- /dev/null +++ b/rabbitinahat/.settings/org.eclipse.m2e.core.prefs @@ -0,0 +1,4 @@ +activeProfiles= +eclipse.preferences.version=1 +resolveWorkspaceProjects=true +version=1 diff --git a/rabbitinahat/pom.xml b/rabbitinahat/pom.xml new file mode 100644 index 00000000..8a302145 --- /dev/null +++ b/rabbitinahat/pom.xml @@ -0,0 +1,42 @@ + + + + leporidae + org.ohdsi + 0.8.0-SNAPSHOT + + 4.0.0 + + rabbitinahat + RabbitInAHat + jar + + + + + org.codehaus.mojo + appassembler-maven-plugin + + ${project.parent.basedir}/dist + -Xmx1200m + + + org.ohdsi.rabbitInAHat.RabbitInAHatMain + rabbitInAHat + + + + + + + + + + org.ohdsi + rabbit-core + ${project.version} + + + \ No newline at end of file diff --git a/src/org/ohdsi/rabbitInAHat/Arrow.java b/rabbitinahat/src/main/java/org/ohdsi/rabbitInAHat/Arrow.java similarity index 95% rename from src/org/ohdsi/rabbitInAHat/Arrow.java rename to rabbitinahat/src/main/java/org/ohdsi/rabbitInAHat/Arrow.java index a9715614..d3a7e349 100644 --- a/src/org/ohdsi/rabbitInAHat/Arrow.java +++ b/rabbitinahat/src/main/java/org/ohdsi/rabbitInAHat/Arrow.java @@ -1,261 +1,261 @@ -/******************************************************************************* - * Copyright 2017 Observational Health Data Sciences and Informatics - * - * This file is part of WhiteRabbit - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - ******************************************************************************/ -package org.ohdsi.rabbitInAHat; - -import java.awt.BasicStroke; -import java.awt.Color; -import java.awt.Graphics; -import java.awt.Graphics2D; -import java.awt.Point; -import java.awt.Polygon; -import org.ohdsi.rabbitInAHat.dataModel.ItemToItemMap; - -public class Arrow implements MappingComponent { - - public enum HighlightStatus { - IS_SELECTED (new Color(128, 128, 128, 192)), - BOTH_SELECTED (new Color(255, 255, 0, 192)), - SOURCE_SELECTED (new Color(255, 128, 0, 192)), - TARGET_SELECTED (new Color(0, 0, 255, 192)), - NONE_SELECTED (new Color(128, 128, 128, 192)), - IS_COMPLETED (new Color(128, 128, 128, 50)); - - private final Color color; - - HighlightStatus(Color color) { - this.color = color; - } - } - - public static float thickness = 5; - public static int headThickness = 15; - public static Color color = HighlightStatus.NONE_SELECTED.color; - private static BasicStroke dashed = new BasicStroke(2.0f, BasicStroke.CAP_BUTT, BasicStroke.JOIN_MITER, 10.0f, new float[] { 10.f }, 0.0f); - - private int x1; - private int y1; - private int x2; - private int y2; - private LabeledRectangle source = null; - private LabeledRectangle target = null; - private ItemToItemMap itemToItemMap; - - private int width; - private int height; - - private Polygon polygon; - - private boolean isSelected = false; - private boolean isVisible = true; - - public Arrow(LabeledRectangle source) { - this.source = source; - } - - public Arrow(int x1, int y1, int x2, int y2) { - this.x1 = x1; - this.y1 = y1; - this.x2 = x2; - this.y2 = y2; - - this.width = Math.abs(x1 - x2); - this.height = Math.abs(y1 - y2); - } - - public Arrow(LabeledRectangle source, LabeledRectangle target) { - this.source = source; - this.target = target; - } - - public Arrow(LabeledRectangle source, LabeledRectangle target, ItemToItemMap itemToItemMap) { - this.source = source; - this.target = target; - this.itemToItemMap = itemToItemMap; - } - - public ItemToItemMap getItemToItemMap() { - return itemToItemMap; - } - - public void setItemToItemMap(ItemToItemMap itemToItemMap) { - this.itemToItemMap = itemToItemMap; - } - - public int getWidth() { - return width; - } - - public int getHeight() { - return height; - } - - public boolean isVisible(){ - return isVisible; - } - public LabeledRectangle getSource() { - return source; - } - - public void setTargetPoint(Point point) { - if (point == null) { - x2 = source.getX() + source.getWidth() + Arrow.headThickness; - y2 = source.getY() + source.getHeight() / 2; - } else { - x2 = point.x; - y2 = point.y; - } - } - - public void paint(Graphics g) { - if (!isVisible) - return; - - if( source != null && target != null){ - if(!source.isVisible() || !target.isVisible()){ - return; - } - } - Graphics2D g2d = (Graphics2D) g; - - if (source != null) { - x1 = source.getX() + source.getWidth(); - y1 = source.getY() + source.getHeight() / 2; - width = Math.abs(x1 - x2); - height = Math.abs(y1 - y2); - } - if (target != null) { - x2 = target.getX(); - y2 = target.getY() + target.getHeight() / 2; - width = Math.abs(x1 - x2); - height = Math.abs(y1 - y2); - } - int nPoints = 25; - int[] xPoints = new int[nPoints * 2 + 3]; - int[] yPoints = new int[nPoints * 2 + 3]; - float widthMinHead = getWidth() - headThickness; - float stepSize = widthMinHead / (float) (nPoints - 1); - - for (int i = 0; i < nPoints; i++) { - float x = x1 + stepSize * i; - float y = (float) (y1 + (Math.cos(Math.PI * i / (float) nPoints) / 2d - 0.5) * (y1 - y2)); - xPoints[i] = Math.round(x); - yPoints[i] = Math.round(y - thickness); - xPoints[nPoints * 2 + 3 - i - 1] = Math.round(x); - yPoints[nPoints * 2 + 3 - i - 1] = Math.round(y + thickness); - } - xPoints[nPoints] = x2 - headThickness; - yPoints[nPoints] = y2 - headThickness; - xPoints[nPoints + 1] = x2; - yPoints[nPoints + 1] = y2; - xPoints[nPoints + 2] = x2 - headThickness; - yPoints[nPoints + 2] = y2 + headThickness; - polygon = new Polygon(xPoints, yPoints, nPoints * 2 + 3); - - g2d.setColor(fillColor()); - g2d.fillPolygon(polygon); - - if (isSelected) { - g2d.setColor(Color.BLACK); - g2d.setStroke(dashed); - g2d.drawPolygon(polygon); - } - } - - public Color fillColor() { - return getHighlightStatus().color; - } - - private boolean isTargetSelected() { - return target != null && target.isSelected(); - } - - private boolean isSourceSelected() { - return source != null && source.isSelected(); - } - - public static void drawArrowHead(Graphics2D g2d, int x, int y) { - int nPoints = 3; - int[] xPoints = new int[nPoints]; - int[] yPoints = new int[nPoints]; - xPoints[0] = x - headThickness; - yPoints[0] = y - headThickness; - xPoints[1] = x; - yPoints[1] = y; - xPoints[2] = x - headThickness; - yPoints[2] = y + headThickness; - g2d.setColor(color); - g2d.fillPolygon(xPoints, yPoints, nPoints); - } - - public void setTarget(LabeledRectangle target) { - this.target = target; - } - - public LabeledRectangle getTarget() { - return target; - } - - public HighlightStatus getHighlightStatus() { - if (isSelected()) { - return HighlightStatus.IS_SELECTED; - } else if (isSourceSelected() && isTargetSelected()) { - return HighlightStatus.BOTH_SELECTED; - } else if (isSourceSelected()) { - return HighlightStatus.SOURCE_SELECTED; - } else if (isTargetSelected()) { - return HighlightStatus.TARGET_SELECTED; - } else if (isCompleted()) { - return HighlightStatus.IS_COMPLETED; - } else { - return HighlightStatus.NONE_SELECTED; - } - } - - public boolean isSelected() { - return isSelected; - } - - public void setSelected(boolean isSelected) { - this.isSelected = isSelected; - } - - public boolean isCompleted() { - if (getItemToItemMap() != null) { - return getItemToItemMap().isCompleted(); - } - else { - return false; - } - } - - public boolean contains(Point point) { - return polygon.contains(point); - } - - public void setVisible(boolean value) { - isVisible = value; - } - - public boolean isSourceAndTargetVisible(){ - return source.isVisible() && target.isVisible(); - } - - public boolean isConnected(){ - return source != null && target != null; - } -} +/******************************************************************************* + * Copyright 2019 Observational Health Data Sciences and Informatics + * + * This file is part of WhiteRabbit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + ******************************************************************************/ +package org.ohdsi.rabbitInAHat; + +import java.awt.BasicStroke; +import java.awt.Color; +import java.awt.Graphics; +import java.awt.Graphics2D; +import java.awt.Point; +import java.awt.Polygon; +import org.ohdsi.rabbitInAHat.dataModel.ItemToItemMap; + +public class Arrow implements MappingComponent { + + public enum HighlightStatus { + IS_SELECTED (new Color(128, 128, 128, 192)), + BOTH_SELECTED (new Color(255, 255, 0, 192)), + SOURCE_SELECTED (new Color(255, 128, 0, 192)), + TARGET_SELECTED (new Color(0, 0, 255, 192)), + NONE_SELECTED (new Color(128, 128, 128, 192)), + IS_COMPLETED (new Color(128, 128, 128, 50)); + + private final Color color; + + HighlightStatus(Color color) { + this.color = color; + } + } + + public static float thickness = 5; + public static int headThickness = 15; + public static Color color = HighlightStatus.NONE_SELECTED.color; + private static BasicStroke dashed = new BasicStroke(2.0f, BasicStroke.CAP_BUTT, BasicStroke.JOIN_MITER, 10.0f, new float[] { 10.f }, 0.0f); + + private int x1; + private int y1; + private int x2; + private int y2; + private LabeledRectangle source = null; + private LabeledRectangle target = null; + private ItemToItemMap itemToItemMap; + + private int width; + private int height; + + private Polygon polygon; + + private boolean isSelected = false; + private boolean isVisible = true; + + public Arrow(LabeledRectangle source) { + this.source = source; + } + + public Arrow(int x1, int y1, int x2, int y2) { + this.x1 = x1; + this.y1 = y1; + this.x2 = x2; + this.y2 = y2; + + this.width = Math.abs(x1 - x2); + this.height = Math.abs(y1 - y2); + } + + public Arrow(LabeledRectangle source, LabeledRectangle target) { + this.source = source; + this.target = target; + } + + public Arrow(LabeledRectangle source, LabeledRectangle target, ItemToItemMap itemToItemMap) { + this.source = source; + this.target = target; + this.itemToItemMap = itemToItemMap; + } + + public ItemToItemMap getItemToItemMap() { + return itemToItemMap; + } + + public void setItemToItemMap(ItemToItemMap itemToItemMap) { + this.itemToItemMap = itemToItemMap; + } + + public int getWidth() { + return width; + } + + public int getHeight() { + return height; + } + + public boolean isVisible(){ + return isVisible; + } + public LabeledRectangle getSource() { + return source; + } + + public void setTargetPoint(Point point) { + if (point == null) { + x2 = source.getX() + source.getWidth() + Arrow.headThickness; + y2 = source.getY() + source.getHeight() / 2; + } else { + x2 = point.x; + y2 = point.y; + } + } + + public void paint(Graphics g) { + if (!isVisible) + return; + + if( source != null && target != null){ + if(!source.isVisible() || !target.isVisible()){ + return; + } + } + Graphics2D g2d = (Graphics2D) g; + + if (source != null) { + x1 = source.getX() + source.getWidth(); + y1 = source.getY() + source.getHeight() / 2; + width = Math.abs(x1 - x2); + height = Math.abs(y1 - y2); + } + if (target != null) { + x2 = target.getX(); + y2 = target.getY() + target.getHeight() / 2; + width = Math.abs(x1 - x2); + height = Math.abs(y1 - y2); + } + int nPoints = 25; + int[] xPoints = new int[nPoints * 2 + 3]; + int[] yPoints = new int[nPoints * 2 + 3]; + float widthMinHead = getWidth() - headThickness; + float stepSize = widthMinHead / (float) (nPoints - 1); + + for (int i = 0; i < nPoints; i++) { + float x = x1 + stepSize * i; + float y = (float) (y1 + (Math.cos(Math.PI * i / (float) nPoints) / 2d - 0.5) * (y1 - y2)); + xPoints[i] = Math.round(x); + yPoints[i] = Math.round(y - thickness); + xPoints[nPoints * 2 + 3 - i - 1] = Math.round(x); + yPoints[nPoints * 2 + 3 - i - 1] = Math.round(y + thickness); + } + xPoints[nPoints] = x2 - headThickness; + yPoints[nPoints] = y2 - headThickness; + xPoints[nPoints + 1] = x2; + yPoints[nPoints + 1] = y2; + xPoints[nPoints + 2] = x2 - headThickness; + yPoints[nPoints + 2] = y2 + headThickness; + polygon = new Polygon(xPoints, yPoints, nPoints * 2 + 3); + + g2d.setColor(fillColor()); + g2d.fillPolygon(polygon); + + if (isSelected) { + g2d.setColor(Color.BLACK); + g2d.setStroke(dashed); + g2d.drawPolygon(polygon); + } + } + + public Color fillColor() { + return getHighlightStatus().color; + } + + private boolean isTargetSelected() { + return target != null && target.isSelected(); + } + + private boolean isSourceSelected() { + return source != null && source.isSelected(); + } + + public static void drawArrowHead(Graphics2D g2d, int x, int y) { + int nPoints = 3; + int[] xPoints = new int[nPoints]; + int[] yPoints = new int[nPoints]; + xPoints[0] = x - headThickness; + yPoints[0] = y - headThickness; + xPoints[1] = x; + yPoints[1] = y; + xPoints[2] = x - headThickness; + yPoints[2] = y + headThickness; + g2d.setColor(color); + g2d.fillPolygon(xPoints, yPoints, nPoints); + } + + public void setTarget(LabeledRectangle target) { + this.target = target; + } + + public LabeledRectangle getTarget() { + return target; + } + + public HighlightStatus getHighlightStatus() { + if (isSelected()) { + return HighlightStatus.IS_SELECTED; + } else if (isSourceSelected() && isTargetSelected()) { + return HighlightStatus.BOTH_SELECTED; + } else if (isSourceSelected()) { + return HighlightStatus.SOURCE_SELECTED; + } else if (isTargetSelected()) { + return HighlightStatus.TARGET_SELECTED; + } else if (isCompleted()) { + return HighlightStatus.IS_COMPLETED; + } else { + return HighlightStatus.NONE_SELECTED; + } + } + + public boolean isSelected() { + return isSelected; + } + + public void setSelected(boolean isSelected) { + this.isSelected = isSelected; + } + + public boolean isCompleted() { + if (getItemToItemMap() != null) { + return getItemToItemMap().isCompleted(); + } + else { + return false; + } + } + + public boolean contains(Point point) { + return polygon.contains(point); + } + + public void setVisible(boolean value) { + isVisible = value; + } + + public boolean isSourceAndTargetVisible(){ + return source.isVisible() && target.isVisible(); + } + + public boolean isConnected(){ + return source != null && target != null; + } +} diff --git a/src/org/ohdsi/rabbitInAHat/DescriptionTextArea.java b/rabbitinahat/src/main/java/org/ohdsi/rabbitInAHat/DescriptionTextArea.java similarity index 95% rename from src/org/ohdsi/rabbitInAHat/DescriptionTextArea.java rename to rabbitinahat/src/main/java/org/ohdsi/rabbitInAHat/DescriptionTextArea.java index d742c9ca..4d2db70e 100644 --- a/src/org/ohdsi/rabbitInAHat/DescriptionTextArea.java +++ b/rabbitinahat/src/main/java/org/ohdsi/rabbitInAHat/DescriptionTextArea.java @@ -1,23 +1,23 @@ -package org.ohdsi.rabbitInAHat; - -import javax.swing.JTextArea; -import javax.swing.UIManager; - -public class DescriptionTextArea extends JTextArea { - - private static final long serialVersionUID = -3065135241375027552L; - - public DescriptionTextArea(String text) { - super(text); - - setWrapStyleWord(true); - setLineWrap(true); - setOpaque(false); - setEditable(false); - setFocusable(false); - setBackground(UIManager.getColor("Label.background")); - setFont(UIManager.getFont("Label.font")); - setBorder(UIManager.getBorder("Label.border")); - } - -} +package org.ohdsi.rabbitInAHat; + +import javax.swing.JTextArea; +import javax.swing.UIManager; + +public class DescriptionTextArea extends JTextArea { + + private static final long serialVersionUID = -3065135241375027552L; + + public DescriptionTextArea(String text) { + super(text); + + setWrapStyleWord(true); + setLineWrap(true); + setOpaque(false); + setEditable(false); + setFocusable(false); + setBackground(UIManager.getColor("Label.background")); + setFont(UIManager.getFont("Label.font")); + setBorder(UIManager.getBorder("Label.border")); + } + +} diff --git a/src/org/ohdsi/rabbitInAHat/DetailsListener.java b/rabbitinahat/src/main/java/org/ohdsi/rabbitInAHat/DetailsListener.java similarity index 91% rename from src/org/ohdsi/rabbitInAHat/DetailsListener.java rename to rabbitinahat/src/main/java/org/ohdsi/rabbitInAHat/DetailsListener.java index 2b46ce68..533c1859 100644 --- a/src/org/ohdsi/rabbitInAHat/DetailsListener.java +++ b/rabbitinahat/src/main/java/org/ohdsi/rabbitInAHat/DetailsListener.java @@ -1,5 +1,5 @@ /******************************************************************************* - * Copyright 2017 Observational Health Data Sciences and Informatics + * Copyright 2019 Observational Health Data Sciences and Informatics * * This file is part of WhiteRabbit * diff --git a/src/org/ohdsi/rabbitInAHat/DetailsPanel.java b/rabbitinahat/src/main/java/org/ohdsi/rabbitInAHat/DetailsPanel.java similarity index 87% rename from src/org/ohdsi/rabbitInAHat/DetailsPanel.java rename to rabbitinahat/src/main/java/org/ohdsi/rabbitInAHat/DetailsPanel.java index 2a244716..4630276e 100644 --- a/src/org/ohdsi/rabbitInAHat/DetailsPanel.java +++ b/rabbitinahat/src/main/java/org/ohdsi/rabbitInAHat/DetailsPanel.java @@ -1,602 +1,598 @@ -/******************************************************************************* - * Copyright 2017 Observational Health Data Sciences and Informatics - * - * This file is part of WhiteRabbit - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - ******************************************************************************/ -package org.ohdsi.rabbitInAHat; - -import java.awt.BorderLayout; -import java.awt.CardLayout; -import java.awt.Color; -import java.awt.Component; -import java.awt.Dimension; -import java.awt.Font; -import java.awt.GridLayout; -import java.awt.Toolkit; -import java.awt.event.ActionEvent; -import java.awt.event.KeyEvent; -import java.text.DecimalFormat; -import java.util.ArrayList; -import java.util.List; - -import javax.swing.AbstractAction; -import javax.swing.ActionMap; -import javax.swing.BorderFactory; -import javax.swing.InputMap; -import javax.swing.JComponent; -import javax.swing.JLabel; -import javax.swing.JPanel; -import javax.swing.JScrollPane; -import javax.swing.JTable; -import javax.swing.JTextArea; -import javax.swing.KeyStroke; -import javax.swing.UIManager; -import javax.swing.border.MatteBorder; -import javax.swing.event.ChangeEvent; -import javax.swing.event.DocumentEvent; -import javax.swing.event.DocumentListener; -import javax.swing.event.ListSelectionEvent; -import javax.swing.event.TableColumnModelEvent; -import javax.swing.event.TableColumnModelListener; -import javax.swing.event.TableModelEvent; -import javax.swing.event.TableModelListener; -import javax.swing.event.UndoableEditEvent; -import javax.swing.event.UndoableEditListener; -import javax.swing.table.TableModel; -import javax.swing.text.Document; -import javax.swing.undo.CannotUndoException; -import javax.swing.undo.UndoManager; - -import org.ohdsi.rabbitInAHat.dataModel.Field; -import org.ohdsi.rabbitInAHat.dataModel.ItemToItemMap; -import org.ohdsi.rabbitInAHat.dataModel.Table; -import org.ohdsi.utilities.StringUtilities; -import org.ohdsi.rabbitInAHat.dataModel.TableCellLongTextRenderer; - -public class DetailsPanel extends JPanel implements DetailsListener { - - public static Font font = new Font("default", Font.PLAIN, 18); - - private static final long serialVersionUID = 4477553676983048468L; - private Object object; - private TablePanel tablePanel; - private FieldPanel fieldPanel; - private ItemToItemMapPanel itemToItemMapPanel; - private CardLayout cardLayout = new CardLayout(); - - private UndoManager undoManager; - - public DetailsPanel() { - UIManager.put("Label.font", font); - - setLayout(cardLayout); - - tablePanel = new TablePanel(); - add(tablePanel, Table.class.getName()); - - fieldPanel = new FieldPanel(); - add(fieldPanel, Field.class.getName()); - - itemToItemMapPanel = new ItemToItemMapPanel(); - add(itemToItemMapPanel, ItemToItemMap.class.getName()); - - JPanel nullPanel = new JPanel(); - add(nullPanel, ""); - - cardLayout.show(this, ""); - - undoManager = new UndoManager(); - - } - - @Override - public void showDetails(Object object) { - this.object = object; - if (object instanceof Table) { - tablePanel.showTable((Table) object); - tablePanel.updateRowHeights(); - cardLayout.show(this, Table.class.getName()); - } else if (object instanceof Field) { - fieldPanel.showField((Field) object); - cardLayout.show(this, Field.class.getName()); - } else if (object instanceof ItemToItemMap) { - itemToItemMapPanel.showItemToItemMap((ItemToItemMap) object); - cardLayout.show(this, ItemToItemMap.class.getName()); - } else - cardLayout.show(this, ""); - - // Discard edits made by showing a new details view - undoManager.discardAllEdits(); - } - - public void refresh() { - showDetails(object); - } - - private void addUndoToTextArea(JTextArea jta){ - Document doc = jta.getDocument(); - doc.addUndoableEditListener(new UndoableEditListener() { - @Override - public void undoableEditHappened(UndoableEditEvent e) { - undoManager.addEdit(e.getEdit()); - } - }); - - InputMap im = jta.getInputMap(JComponent.WHEN_FOCUSED); - ActionMap am = jta.getActionMap(); - - im.put(KeyStroke.getKeyStroke(KeyEvent.VK_Z, Toolkit.getDefaultToolkit().getMenuShortcutKeyMask()), "Undo"); - im.put(KeyStroke.getKeyStroke(KeyEvent.VK_Y, Toolkit.getDefaultToolkit().getMenuShortcutKeyMask()), "Redo"); - - am.put("Undo", new AbstractAction() { - /** - * - */ - private static final long serialVersionUID = -3363877112423623107L; - - @Override - public void actionPerformed(ActionEvent e) { - try { - if (undoManager.canUndo()) { - undoManager.undo(); - } - } catch (CannotUndoException exp) { - exp.printStackTrace(); - } - } - }); - - am.put("Redo", new AbstractAction() { - /** - * - */ - private static final long serialVersionUID = -5581878642285644039L; - - @Override - public void actionPerformed(ActionEvent e) { - try { - if (undoManager.canRedo()) { - undoManager.redo(); - } - } catch (CannotUndoException exp) { - exp.printStackTrace(); - } - } - }); - } - private class TablePanel extends JPanel implements DocumentListener { - - private static final long serialVersionUID = -4393026616049677944L; - private Table table; - private JLabel nameLabel = new JLabel(""); - private JLabel rowCountLabel = new JLabel(""); - private SimpleTableModel fieldTable = new SimpleTableModel("Field", "Type","Description"); - private JTextArea commentsArea = new JTextArea(); - private JTable displayTable = new JTable(fieldTable); - - - public TablePanel() { - setLayout(new BorderLayout()); - - JPanel generalInfoPanel = new JPanel(); - generalInfoPanel.setLayout(new GridLayout(0, 2)); - generalInfoPanel.setBorder(BorderFactory.createTitledBorder("General information")); - - generalInfoPanel.add(new JLabel("Table name: ")); - generalInfoPanel.add(nameLabel); - - generalInfoPanel.add(new JLabel("Number of rows: ")); - generalInfoPanel.add(rowCountLabel); - add(generalInfoPanel, BorderLayout.NORTH); - - - JScrollPane fieldListPanel = new JScrollPane(displayTable); - - // Updates row heights when column widths change - displayTable.getColumnModel().addColumnModelListener(new TableColumnModelListener(){ - - @Override - public void columnMarginChanged(ChangeEvent e) { - updateRowHeights(); - - } - - @Override - public void columnMoved(TableColumnModelEvent e){ - - } - - @Override - public void columnAdded(TableColumnModelEvent e) { - // TODO Auto-generated method stub - - } - - @Override - public void columnRemoved(TableColumnModelEvent e) { - // TODO Auto-generated method stub - - } - - - - @Override - public void columnSelectionChanged(ListSelectionEvent e) { - // TODO Auto-generated method stub - - } - }); - - displayTable.setFont(font); - - // Set cell renderer that wraps content - for (int c = 0; c < displayTable.getColumnCount(); c++){ - displayTable.getColumnModel().getColumn(c).setCellRenderer(new TableCellLongTextRenderer()); - } - - fieldListPanel.setBorder(BorderFactory.createTitledBorder("Fields")); - add(fieldListPanel, BorderLayout.CENTER); - - JScrollPane commentsPanel = new JScrollPane(commentsArea); - commentsArea.setFont(font); - commentsArea.getDocument().addDocumentListener(this); - commentsArea.setWrapStyleWord(true); - commentsArea.setLineWrap(true); - - commentsPanel.setBorder(BorderFactory.createTitledBorder("Comments")); - commentsPanel.setPreferredSize(new Dimension(100, 200)); - add(commentsPanel, BorderLayout.SOUTH); - - } - - private void updateRowHeights() { - - /* - * Auto adjust the height of rows in a JTable. - * The only way to know the row height for sure is to render each cell - * to determine the rendered height. After your table is populated with - * data you can do: - * - */ - for (int row = 0; row < displayTable.getRowCount(); row++) { - int rowHeight = displayTable.getRowHeight(); - for (int column = 0; column < displayTable.getColumnCount(); column++) - { - Component comp = displayTable.prepareRenderer(displayTable.getCellRenderer(row, column), row, column); - - rowHeight = Math.max(rowHeight, comp.getPreferredSize().height); - - } - displayTable.setRowHeight(row, rowHeight); - } - } - - public void showTable(Table table) { - this.table = table; - nameLabel.setText(table.getName()); - DecimalFormat formatter = new DecimalFormat("#,###"); - rowCountLabel.setText(formatter.format(table.getRowCount())); - fieldTable.clear(); - - for (Field field : table.getFields()){ - fieldTable.add(field.outputName(), field.getType(),field.getDescription()); - } - - commentsArea.setText(table.getComment()); - } - - @Override - public void changedUpdate(DocumentEvent e) { - table.setComment(commentsArea.getText()); - } - - @Override - public void insertUpdate(DocumentEvent e) { - table.setComment(commentsArea.getText()); - } - - @Override - public void removeUpdate(DocumentEvent e) { - table.setComment(commentsArea.getText()); - } - } - - private class FieldPanel extends JPanel implements DocumentListener { - - private static final long serialVersionUID = -4393026616049677944L; - private JLabel nameLabel = new JLabel(""); - private JLabel rowCountLabel = new JLabel(""); - private DescriptionTextArea description = new DescriptionTextArea (""); - private SimpleTableModel valueTable = new SimpleTableModel("Value", "Frequency", "Percent of Total (%)"); - private JTextArea commentsArea = new JTextArea(); - private Field field; - - public FieldPanel() { - setLayout(new BorderLayout()); - - JPanel generalInfoPanel = new JPanel(); - - generalInfoPanel.setLayout(new BorderLayout(5,5)); - - generalInfoPanel.setBorder(BorderFactory.createTitledBorder("General information")); - - JPanel fieldInfo = new JPanel(); - fieldInfo.setLayout(new GridLayout(0,2)); - - fieldInfo.add(new JLabel("Field name: ")); - fieldInfo.add(nameLabel); - - fieldInfo.add(new JLabel("Field type: ")); - fieldInfo.add(rowCountLabel); - - generalInfoPanel.add(fieldInfo,BorderLayout.NORTH); - - JPanel descriptionInfo = new JPanel(); - descriptionInfo.setLayout(new GridLayout(0,2)); - descriptionInfo.add(new JLabel("Description: ")); - descriptionInfo.add(description); - - generalInfoPanel.add(descriptionInfo,BorderLayout.SOUTH); - - add(generalInfoPanel, BorderLayout.NORTH); - - JTable table = new JTable(valueTable); - JScrollPane fieldListPanel = new JScrollPane(table); - table.setFont(font); - table.setRowHeight(24); - table.setBorder(new MatteBorder(1, 0, 1, 0, Color.BLACK)); - - fieldListPanel.setBorder(BorderFactory.createTitledBorder("Fields")); - add(fieldListPanel, BorderLayout.CENTER); - - JScrollPane commentsPanel = new JScrollPane(commentsArea); - commentsArea.setFont(font); - commentsArea.getDocument().addDocumentListener(this); - commentsArea.setWrapStyleWord(true); - commentsArea.setLineWrap(true); - addUndoToTextArea(commentsArea); - - commentsPanel.setBorder(BorderFactory.createTitledBorder("Comments")); - commentsPanel.setPreferredSize(new Dimension(100, 200)); - - add(commentsPanel, BorderLayout.SOUTH); - } - - public void showField(Field field) { - this.field = field; - - nameLabel.setText(field.getName()); - rowCountLabel.setText(field.getType()); - description.setText(field.getDescription()); - - // Hide description if it's empty - description.getParent().setVisible(!description.getText().isEmpty()); - - valueTable.clear(); - - if (field.getValueCounts() != null) { - double valueCountTotal = 0.0; - for (String[] total : field.getValueCounts()) { - String temp = total[1]; - if (StringUtilities.isNumber(temp)) { - double valueCountTemp = Double.parseDouble(temp); - valueCountTotal += valueCountTemp; - } - } - DecimalFormat formatter = new DecimalFormat("#,###"); - DecimalFormat formatterPercent = new DecimalFormat("#,##0.0"); - for (String[] valueCount : field.getValueCounts()) { - String nr = valueCount[1]; - String vp = ""; - if (StringUtilities.isNumber(nr)) { - double number = Double.parseDouble(nr); - nr = formatter.format(number); - double valueCountPercent = number / valueCountTotal * 100; - if (valueCountPercent < 0.1) { - vp = "< 0.1"; - } - else if (valueCountPercent > 99) { - vp = "> 99.0"; - } - else { - vp = formatterPercent.format(valueCountPercent); - } - } - valueTable.add(valueCount[0], nr, vp); - } - } - commentsArea.setText(field.getComment()); - } - - @Override - public void changedUpdate(DocumentEvent e) { - field.setComment(commentsArea.getText()); - } - - @Override - public void insertUpdate(DocumentEvent e) { - field.setComment(commentsArea.getText()); - } - - @Override - public void removeUpdate(DocumentEvent e) { - field.setComment(commentsArea.getText()); - } - - } - - private class ItemToItemMapPanel extends JPanel implements DocumentListener { - - private static final long serialVersionUID = -4393026616049677944L; - private JLabel sourceLabel = new JLabel(""); - private JLabel targetLabel = new JLabel(""); - private JTextArea logicArea = new JTextArea(); - private JTextArea commentsArea = new JTextArea(); - // private FlexTable testTable = new FlexTable(); - private ItemToItemMap itemToItemMap; - - public ItemToItemMapPanel() { - setLayout(new BorderLayout()); - - JPanel generalInfoPanel = new JPanel(); - generalInfoPanel.setLayout(new GridLayout(0, 2)); - generalInfoPanel.setBorder(BorderFactory.createTitledBorder("General information")); - - generalInfoPanel.add(new JLabel("Source: ")); - generalInfoPanel.add(sourceLabel); - - generalInfoPanel.add(new JLabel("Target: ")); - generalInfoPanel.add(targetLabel); - - add(generalInfoPanel, BorderLayout.NORTH); - - JScrollPane logicPanel = new JScrollPane(logicArea); - logicArea.setFont(font); - logicArea.getDocument().addDocumentListener(this); - logicArea.setWrapStyleWord(true); - logicArea.setLineWrap(true); - addUndoToTextArea(logicArea); - - logicPanel.setBorder(BorderFactory.createTitledBorder("Logic")); - logicPanel.setPreferredSize(new Dimension(100, 200)); - - add(logicPanel, BorderLayout.CENTER); - - JScrollPane commentsPanel = new JScrollPane(commentsArea); - commentsArea.setFont(font); - commentsArea.getDocument().addDocumentListener(this); - commentsArea.setWrapStyleWord(true); - commentsArea.setLineWrap(true); - addUndoToTextArea(commentsArea); - - commentsPanel.setBorder(BorderFactory.createTitledBorder("Comments")); - commentsPanel.setPreferredSize(new Dimension(100, 200)); - - add(commentsPanel, BorderLayout.SOUTH); - - // testTable.setMinimumSize(new Dimension(200, 200)); - // JScrollPane testPanel = new JScrollPane(testTable); - // testPanel.setHorizontalScrollBarPolicy(JScrollPane.HORIZONTAL_SCROLLBAR_ALWAYS); - // add(testPanel, BorderLayout.SOUTH); - } - - public void showItemToItemMap(ItemToItemMap itemToItemMap) { - this.itemToItemMap = itemToItemMap; - sourceLabel.setText(itemToItemMap.getSourceItem().toString()); - targetLabel.setText(itemToItemMap.getTargetItem().toString()); - logicArea.setText(itemToItemMap.getLogic()); - commentsArea.setText(itemToItemMap.getComment()); - } - - @Override - public void changedUpdate(DocumentEvent e) { - if (e.getDocument() == logicArea.getDocument()) - itemToItemMap.setLogic(logicArea.getText()); - else - itemToItemMap.setComment(commentsArea.getText()); - } - - @Override - public void insertUpdate(DocumentEvent e) { - if (e.getDocument() == logicArea.getDocument()) - itemToItemMap.setLogic(logicArea.getText()); - else - itemToItemMap.setComment(commentsArea.getText()); - } - - @Override - public void removeUpdate(DocumentEvent e) { - if (e.getDocument() == logicArea.getDocument()) - itemToItemMap.setLogic(logicArea.getText()); - else - itemToItemMap.setComment(commentsArea.getText()); - } - - } - - private class SimpleTableModel implements TableModel { - - private List listeners = new ArrayList(); - private List> data = new ArrayList>(); - private String[] columnNames; - - public void clear() { - data.clear(); - notifyListeners(); - } - - private void notifyListeners() { - for (TableModelListener listener : listeners) - listener.tableChanged(new TableModelEvent(this)); - } - - public void add(String... values) { - List row = new ArrayList(values.length); - for (int i = 0; i < values.length; i++) - row.add(values[i]); - data.add(row); - notifyListeners(); - } - - public SimpleTableModel(String... columnNames) { - this.columnNames = columnNames; - } - - @Override - public void addTableModelListener(TableModelListener l) { - listeners.add(l); - } - - @Override - public Class getColumnClass(int columnIndex) { - return String.class; - } - - @Override - public int getColumnCount() { - return columnNames.length; - } - - @Override - public String getColumnName(int columnIndex) { - return columnNames[columnIndex]; - } - - @Override - public int getRowCount() { - return data.size(); - } - - @Override - public Object getValueAt(int rowIndex, int columnIndex) { - return data.get(rowIndex).get(columnIndex); - } - - @Override - public boolean isCellEditable(int rowIndex, int columnIndex) { - return false; - } - - @Override - public void removeTableModelListener(TableModelListener l) { - listeners.remove(l); - } - - @Override - public void setValueAt(Object aValue, int rowIndex, int columnIndex) { - } - - } -} +/******************************************************************************* + * Copyright 2019 Observational Health Data Sciences and Informatics + * + * This file is part of WhiteRabbit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + ******************************************************************************/ +package org.ohdsi.rabbitInAHat; + +import java.awt.BorderLayout; +import java.awt.CardLayout; +import java.awt.Color; +import java.awt.Component; +import java.awt.Dimension; +import java.awt.Font; +import java.awt.GridLayout; +import java.awt.Toolkit; +import java.awt.event.ActionEvent; +import java.awt.event.KeyEvent; +import java.text.NumberFormat; +import java.util.ArrayList; +import java.util.List; + +import javax.swing.*; +import javax.swing.border.MatteBorder; +import javax.swing.event.ChangeEvent; +import javax.swing.event.DocumentEvent; +import javax.swing.event.DocumentListener; +import javax.swing.event.ListSelectionEvent; +import javax.swing.event.TableColumnModelEvent; +import javax.swing.event.TableColumnModelListener; +import javax.swing.event.TableModelEvent; +import javax.swing.event.TableModelListener; +import javax.swing.event.UndoableEditEvent; +import javax.swing.event.UndoableEditListener; +import javax.swing.table.DefaultTableCellRenderer; +import javax.swing.table.TableModel; +import javax.swing.text.Document; +import javax.swing.undo.CannotUndoException; +import javax.swing.undo.UndoManager; + +import org.ohdsi.rabbitInAHat.dataModel.Field; +import org.ohdsi.rabbitInAHat.dataModel.ItemToItemMap; +import org.ohdsi.rabbitInAHat.dataModel.Table; +import org.ohdsi.utilities.StringUtilities; +import org.ohdsi.rabbitInAHat.dataModel.TableCellLongTextRenderer; + +public class DetailsPanel extends JPanel implements DetailsListener { + + public static Font font = new Font("default", Font.PLAIN, 18); + + private static final long serialVersionUID = 4477553676983048468L; + private Object object; + private TablePanel tablePanel; + private FieldPanel fieldPanel; + private ItemToItemMapPanel itemToItemMapPanel; + private CardLayout cardLayout = new CardLayout(); + private NumberFormat numberFormat = NumberFormat.getNumberInstance(); + private NumberFormat percentageFormat = NumberFormat.getPercentInstance(); + + private UndoManager undoManager; + + public DetailsPanel() { + UIManager.put("Label.font", font); + + setLayout(cardLayout); + + tablePanel = new TablePanel(); + add(tablePanel, Table.class.getName()); + + fieldPanel = new FieldPanel(); + add(fieldPanel, Field.class.getName()); + + itemToItemMapPanel = new ItemToItemMapPanel(); + add(itemToItemMapPanel, ItemToItemMap.class.getName()); + + JPanel nullPanel = new JPanel(); + add(nullPanel, ""); + + cardLayout.show(this, ""); + + undoManager = new UndoManager(); + + percentageFormat.setMinimumFractionDigits(1); + + } + + @Override + public void showDetails(Object object) { + this.object = object; + if (object instanceof Table) { + tablePanel.showTable((Table) object); + tablePanel.updateRowHeights(); + cardLayout.show(this, Table.class.getName()); + } else if (object instanceof Field) { + fieldPanel.showField((Field) object); + cardLayout.show(this, Field.class.getName()); + } else if (object instanceof ItemToItemMap) { + itemToItemMapPanel.showItemToItemMap((ItemToItemMap) object); + cardLayout.show(this, ItemToItemMap.class.getName()); + } else + cardLayout.show(this, ""); + + // Discard edits made by showing a new details view + undoManager.discardAllEdits(); + } + + public void refresh() { + showDetails(object); + } + + private void addUndoToTextArea(JTextArea jta){ + Document doc = jta.getDocument(); + doc.addUndoableEditListener(new UndoableEditListener() { + @Override + public void undoableEditHappened(UndoableEditEvent e) { + undoManager.addEdit(e.getEdit()); + } + }); + + InputMap im = jta.getInputMap(JComponent.WHEN_FOCUSED); + ActionMap am = jta.getActionMap(); + + im.put(KeyStroke.getKeyStroke(KeyEvent.VK_Z, Toolkit.getDefaultToolkit().getMenuShortcutKeyMask()), "Undo"); + im.put(KeyStroke.getKeyStroke(KeyEvent.VK_Y, Toolkit.getDefaultToolkit().getMenuShortcutKeyMask()), "Redo"); + + am.put("Undo", new AbstractAction() { + /** + * + */ + private static final long serialVersionUID = -3363877112423623107L; + + @Override + public void actionPerformed(ActionEvent e) { + try { + if (undoManager.canUndo()) { + undoManager.undo(); + } + } catch (CannotUndoException exp) { + exp.printStackTrace(); + } + } + }); + + am.put("Redo", new AbstractAction() { + /** + * + */ + private static final long serialVersionUID = -5581878642285644039L; + + @Override + public void actionPerformed(ActionEvent e) { + try { + if (undoManager.canRedo()) { + undoManager.redo(); + } + } catch (CannotUndoException exp) { + exp.printStackTrace(); + } + } + }); + } + private class TablePanel extends JPanel implements DocumentListener { + + private static final long serialVersionUID = -4393026616049677944L; + private Table table; + private JLabel nameLabel = new JLabel(""); + private JLabel rowCountLabel = new JLabel(""); + private SimpleTableModel fieldTable = new SimpleTableModel("Field", "Type","Description"); + private JTextArea commentsArea = new JTextArea(); + private JTable displayTable = new JTable(fieldTable); + + + public TablePanel() { + setLayout(new BorderLayout()); + + JPanel generalInfoPanel = new JPanel(); + generalInfoPanel.setLayout(new GridLayout(0, 2)); + generalInfoPanel.setBorder(BorderFactory.createTitledBorder("General information")); + + generalInfoPanel.add(new JLabel("Table name: ")); + generalInfoPanel.add(nameLabel); + + generalInfoPanel.add(new JLabel("Number of rows: ")); + generalInfoPanel.add(rowCountLabel); + add(generalInfoPanel, BorderLayout.NORTH); + + + JScrollPane fieldListPanel = new JScrollPane(displayTable); + + // Updates row heights when column widths change + displayTable.getColumnModel().addColumnModelListener(new TableColumnModelListener(){ + + @Override + public void columnMarginChanged(ChangeEvent e) { + updateRowHeights(); + + } + + @Override + public void columnMoved(TableColumnModelEvent e){ + + } + + @Override + public void columnAdded(TableColumnModelEvent e) { + // TODO Auto-generated method stub + + } + + @Override + public void columnRemoved(TableColumnModelEvent e) { + // TODO Auto-generated method stub + + } + + + + @Override + public void columnSelectionChanged(ListSelectionEvent e) { + // TODO Auto-generated method stub + + } + }); + + displayTable.setFont(font); + + // Set cell renderer that wraps content + for (int c = 0; c < displayTable.getColumnCount(); c++){ + displayTable.getColumnModel().getColumn(c).setCellRenderer(new TableCellLongTextRenderer()); + } + + fieldListPanel.setBorder(BorderFactory.createTitledBorder("Fields")); + add(fieldListPanel, BorderLayout.CENTER); + + JScrollPane commentsPanel = new JScrollPane(commentsArea); + commentsArea.setFont(font); + commentsArea.getDocument().addDocumentListener(this); + commentsArea.setWrapStyleWord(true); + commentsArea.setLineWrap(true); + + commentsPanel.setBorder(BorderFactory.createTitledBorder("Comments")); + commentsPanel.setPreferredSize(new Dimension(100, 200)); + add(commentsPanel, BorderLayout.SOUTH); + + } + + private void updateRowHeights() { + + /* + * Auto adjust the height of rows in a JTable. + * The only way to know the row height for sure is to render each cell + * to determine the rendered height. After your table is populated with + * data you can do: + * + */ + for (int row = 0; row < displayTable.getRowCount(); row++) { + int rowHeight = displayTable.getRowHeight(); + for (int column = 0; column < displayTable.getColumnCount(); column++) + { + Component comp = displayTable.prepareRenderer(displayTable.getCellRenderer(row, column), row, column); + + rowHeight = Math.max(rowHeight, comp.getPreferredSize().height); + + } + displayTable.setRowHeight(row, rowHeight); + } + } + + public void showTable(Table table) { + this.table = table; + nameLabel.setText(table.getName()); + + if (table.getRowCount() > 0) { + rowCountLabel.setText(numberFormat.format(table.getRowCount())); + } else { + rowCountLabel.setText(">= " + numberFormat.format(table.getRowsCheckedCount())); + } + + fieldTable.clear(); + + for (Field field : table.getFields()){ + fieldTable.add(field.outputName(), field.getType(),field.getDescription()); + } + + commentsArea.setText(table.getComment()); + } + + @Override + public void changedUpdate(DocumentEvent e) { + table.setComment(commentsArea.getText()); + } + + @Override + public void insertUpdate(DocumentEvent e) { + table.setComment(commentsArea.getText()); + } + + @Override + public void removeUpdate(DocumentEvent e) { + table.setComment(commentsArea.getText()); + } + } + + private class FieldPanel extends JPanel implements DocumentListener { + + private static final long serialVersionUID = -4393026616049677944L; + private JLabel nameLabel = new JLabel(""); + private JLabel rowCountLabel = new JLabel(""); + private DescriptionTextArea description = new DescriptionTextArea (""); + private SimpleTableModel valueTable = new SimpleTableModel("Value", "Frequency", "Fraction"); + private JTextArea commentsArea = new JTextArea(); + private Field field; + + public FieldPanel() { + setLayout(new BorderLayout()); + + JPanel generalInfoPanel = new JPanel(); + + generalInfoPanel.setLayout(new BorderLayout(5,5)); + + generalInfoPanel.setBorder(BorderFactory.createTitledBorder("General information")); + + JPanel fieldInfo = new JPanel(); + fieldInfo.setLayout(new GridLayout(0,2)); + + fieldInfo.add(new JLabel("Field name: ")); + fieldInfo.add(nameLabel); + + fieldInfo.add(new JLabel("Field type: ")); + fieldInfo.add(rowCountLabel); + + generalInfoPanel.add(fieldInfo,BorderLayout.NORTH); + + JPanel descriptionInfo = new JPanel(); + descriptionInfo.setLayout(new GridLayout(0,2)); + descriptionInfo.add(new JLabel("Description: ")); + descriptionInfo.add(description); + + generalInfoPanel.add(descriptionInfo,BorderLayout.SOUTH); + + add(generalInfoPanel, BorderLayout.NORTH); + + JTable table = new JTable(valueTable); + JScrollPane fieldListPanel = new JScrollPane(table); + table.setFont(font); + table.setRowHeight(24); + table.setBorder(new MatteBorder(1, 0, 1, 0, Color.BLACK)); + // Right align the frequency and percentage + DefaultTableCellRenderer rightRenderer = new DefaultTableCellRenderer(); + rightRenderer.setHorizontalAlignment(SwingConstants.RIGHT); + table.getColumnModel().getColumn(1).setCellRenderer(rightRenderer); + table.getColumnModel().getColumn(2).setCellRenderer(rightRenderer); + + fieldListPanel.setBorder(BorderFactory.createTitledBorder("Fields")); + add(fieldListPanel, BorderLayout.CENTER); + + JScrollPane commentsPanel = new JScrollPane(commentsArea); + commentsArea.setFont(font); + commentsArea.getDocument().addDocumentListener(this); + commentsArea.setWrapStyleWord(true); + commentsArea.setLineWrap(true); + addUndoToTextArea(commentsArea); + + commentsPanel.setBorder(BorderFactory.createTitledBorder("Comments")); + commentsPanel.setPreferredSize(new Dimension(100, 200)); + + add(commentsPanel, BorderLayout.SOUTH); + } + + public void showField(Field field) { + this.field = field; + + nameLabel.setText(field.getName()); + rowCountLabel.setText(field.getType()); + description.setText(field.getDescription()); + + // Hide description if it's empty + description.getParent().setVisible(!description.getText().isEmpty()); + + valueTable.clear(); + + if (field.getValueCounts() != null) { + int valueCountTotal = field.getRowsCheckedCount(); + + for (String[] valueCount : field.getValueCounts()) { + String valueNumber = valueCount[1]; + String valuePercent = ""; + if (StringUtilities.isNumber(valueNumber)) { + double number = Double.parseDouble(valueNumber); + valueNumber = numberFormat.format(number); + double valueCountPercent = number / (double) valueCountTotal; + if (valueCountPercent < 0.001) { + valuePercent = "<" + percentageFormat.format(0.001); + } + else if (valueCountPercent > 0.99) { + valuePercent = ">" + percentageFormat.format(0.99); + } + else { + valuePercent = percentageFormat.format(valueCountPercent); + } + } + valueTable.add(valueCount[0], valueNumber, valuePercent); + } + } + commentsArea.setText(field.getComment()); + } + + @Override + public void changedUpdate(DocumentEvent e) { + field.setComment(commentsArea.getText()); + } + + @Override + public void insertUpdate(DocumentEvent e) { + field.setComment(commentsArea.getText()); + } + + @Override + public void removeUpdate(DocumentEvent e) { + field.setComment(commentsArea.getText()); + } + + } + + private class ItemToItemMapPanel extends JPanel implements DocumentListener { + + private static final long serialVersionUID = -4393026616049677944L; + private JLabel sourceLabel = new JLabel(""); + private JLabel targetLabel = new JLabel(""); + private JTextArea logicArea = new JTextArea(); + private JTextArea commentsArea = new JTextArea(); + // private FlexTable testTable = new FlexTable(); + private ItemToItemMap itemToItemMap; + + public ItemToItemMapPanel() { + setLayout(new BorderLayout()); + + JPanel generalInfoPanel = new JPanel(); + generalInfoPanel.setLayout(new GridLayout(0, 2)); + generalInfoPanel.setBorder(BorderFactory.createTitledBorder("General information")); + + generalInfoPanel.add(new JLabel("Source: ")); + generalInfoPanel.add(sourceLabel); + + generalInfoPanel.add(new JLabel("Target: ")); + generalInfoPanel.add(targetLabel); + + add(generalInfoPanel, BorderLayout.NORTH); + + JScrollPane logicPanel = new JScrollPane(logicArea); + logicArea.setFont(font); + logicArea.getDocument().addDocumentListener(this); + logicArea.setWrapStyleWord(true); + logicArea.setLineWrap(true); + addUndoToTextArea(logicArea); + + logicPanel.setBorder(BorderFactory.createTitledBorder("Logic")); + logicPanel.setPreferredSize(new Dimension(100, 200)); + + add(logicPanel, BorderLayout.CENTER); + + JScrollPane commentsPanel = new JScrollPane(commentsArea); + commentsArea.setFont(font); + commentsArea.getDocument().addDocumentListener(this); + commentsArea.setWrapStyleWord(true); + commentsArea.setLineWrap(true); + addUndoToTextArea(commentsArea); + + commentsPanel.setBorder(BorderFactory.createTitledBorder("Comments")); + commentsPanel.setPreferredSize(new Dimension(100, 200)); + + add(commentsPanel, BorderLayout.SOUTH); + + // testTable.setMinimumSize(new Dimension(200, 200)); + // JScrollPane testPanel = new JScrollPane(testTable); + // testPanel.setHorizontalScrollBarPolicy(JScrollPane.HORIZONTAL_SCROLLBAR_ALWAYS); + // add(testPanel, BorderLayout.SOUTH); + } + + public void showItemToItemMap(ItemToItemMap itemToItemMap) { + this.itemToItemMap = itemToItemMap; + sourceLabel.setText(itemToItemMap.getSourceItem().toString()); + targetLabel.setText(itemToItemMap.getTargetItem().toString()); + logicArea.setText(itemToItemMap.getLogic()); + commentsArea.setText(itemToItemMap.getComment()); + } + + @Override + public void changedUpdate(DocumentEvent e) { + if (e.getDocument() == logicArea.getDocument()) + itemToItemMap.setLogic(logicArea.getText()); + else + itemToItemMap.setComment(commentsArea.getText()); + } + + @Override + public void insertUpdate(DocumentEvent e) { + if (e.getDocument() == logicArea.getDocument()) + itemToItemMap.setLogic(logicArea.getText()); + else + itemToItemMap.setComment(commentsArea.getText()); + } + + @Override + public void removeUpdate(DocumentEvent e) { + if (e.getDocument() == logicArea.getDocument()) + itemToItemMap.setLogic(logicArea.getText()); + else + itemToItemMap.setComment(commentsArea.getText()); + } + + } + + private class SimpleTableModel implements TableModel { + + private List listeners = new ArrayList(); + private List> data = new ArrayList>(); + private String[] columnNames; + + public void clear() { + data.clear(); + notifyListeners(); + } + + private void notifyListeners() { + for (TableModelListener listener : listeners) + listener.tableChanged(new TableModelEvent(this)); + } + + public void add(String... values) { + List row = new ArrayList(values.length); + for (int i = 0; i < values.length; i++) + row.add(values[i]); + data.add(row); + notifyListeners(); + } + + public SimpleTableModel(String... columnNames) { + this.columnNames = columnNames; + } + + @Override + public void addTableModelListener(TableModelListener l) { + listeners.add(l); + } + + @Override + public Class getColumnClass(int columnIndex) { + return String.class; + } + + @Override + public int getColumnCount() { + return columnNames.length; + } + + @Override + public String getColumnName(int columnIndex) { + return columnNames[columnIndex]; + } + + @Override + public int getRowCount() { + return data.size(); + } + + @Override + public Object getValueAt(int rowIndex, int columnIndex) { + return data.get(rowIndex).get(columnIndex); + } + + @Override + public boolean isCellEditable(int rowIndex, int columnIndex) { + return false; + } + + @Override + public void removeTableModelListener(TableModelListener l) { + listeners.remove(l); + } + + @Override + public void setValueAt(Object aValue, int rowIndex, int columnIndex) { + } + + } +} diff --git a/rabbitinahat/src/main/java/org/ohdsi/rabbitInAHat/ETLMarkupDocumentGenerator.java b/rabbitinahat/src/main/java/org/ohdsi/rabbitInAHat/ETLMarkupDocumentGenerator.java new file mode 100644 index 00000000..48a4ab36 --- /dev/null +++ b/rabbitinahat/src/main/java/org/ohdsi/rabbitInAHat/ETLMarkupDocumentGenerator.java @@ -0,0 +1,364 @@ +/******************************************************************************* + * Copyright 2019 Observational Health Data Sciences and Informatics + * + * This file is part of WhiteRabbit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + ******************************************************************************/ +package org.ohdsi.rabbitInAHat; + +import java.awt.Color; +import java.awt.image.BufferedImage; +import java.io.File; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import javax.imageio.ImageIO; + +import org.apache.poi.openxml4j.exceptions.InvalidFormatException; +import org.ohdsi.rabbitInAHat.dataModel.ETL; +import org.ohdsi.rabbitInAHat.dataModel.ETL.FileFormat; +import org.ohdsi.rabbitInAHat.dataModel.Field; +import org.ohdsi.rabbitInAHat.dataModel.ItemToItemMap; +import org.ohdsi.rabbitInAHat.dataModel.MappableItem; +import org.ohdsi.rabbitInAHat.dataModel.Mapping; +import org.ohdsi.rabbitInAHat.dataModel.Table; +import org.ohdsi.utilities.StringUtilities; +import org.ohdsi.utilities.files.Row; +import org.ohdsi.utilities.files.WriteTextFile; + +public class ETLMarkupDocumentGenerator { + + private MarkupDocument document; + private ETL etl; + + public enum DocumentType { + MARKDOWN, HTML + }; + + public static void main(String[] args) { + ETL etl = ETL.fromFile("c:/temp/markdown/exampleEtl.json.gz", FileFormat.GzipJson); + ETLMarkupDocumentGenerator generator = new ETLMarkupDocumentGenerator(etl); + generator.generate("c:/temp/markdown/index.html", DocumentType.HTML); + + } + + public ETLMarkupDocumentGenerator(ETL etl) { + this.etl = etl; + } + + public void generate(String fileName, DocumentType documentType) { + try { + if (documentType == DocumentType.HTML) + document = new HtmlDocument(fileName); + else + document = new MarkdownDocument(fileName); + addTableLevelSection(); + + for (Table targetTable : etl.getTargetDatabase().getTables()) + addTargetTableSection(targetTable); + + document.close(); + } catch (FileNotFoundException e) { + e.printStackTrace(); + } catch (IOException e) { + e.printStackTrace(); + } catch (InvalidFormatException e) { + e.printStackTrace(); + } + } + + private void addTargetTableSection(Table targetTable) throws InvalidFormatException, IOException { + document.addHeader2("Table name: " + targetTable.getName()); + + for (ItemToItemMap tableToTableMap : etl.getTableToTableMapping().getSourceToTargetMaps()) + if (tableToTableMap.getTargetItem() == targetTable) { + Table sourceTable = (Table) tableToTableMap.getSourceItem(); + Mapping fieldtoFieldMapping = etl.getFieldToFieldMapping(sourceTable, targetTable); + + document.addHeader3("Reading from " + tableToTableMap.getSourceItem()); + + if (!tableToTableMap.getLogic().equals("")) { + document.addParagraph(tableToTableMap.getLogic()); + } + + if (!tableToTableMap.getComment().equals("")) { + document.addParagraph(tableToTableMap.getComment()); + } + + // Add image of field to field mapping + MappingPanel mappingPanel = new MappingPanel(fieldtoFieldMapping); + mappingPanel.setShowOnlyConnectedItems(true); + int height = mappingPanel.getMinimumSize().height; + mappingPanel.setSize(800, height); + BufferedImage image = new BufferedImage(800, height, BufferedImage.TYPE_INT_ARGB); + image.getGraphics().setColor(Color.WHITE); + image.getGraphics().fillRect(0, 0, image.getWidth(), image.getHeight()); + mappingPanel.paint(image.getGraphics()); + document.addImage(image, "Field mapping"); + + // Add table of field to field mapping + List rows = new ArrayList(); + for (MappableItem targetField : fieldtoFieldMapping.getTargetItems()) { + Row row = new Row(); + row.add("Destination Field", targetField.getName()); + + StringBuilder source = new StringBuilder(); + StringBuilder logic = new StringBuilder(); + StringBuilder comment = new StringBuilder(); + for (ItemToItemMap fieldToFieldMap : fieldtoFieldMapping.getSourceToTargetMaps()) { + if (fieldToFieldMap.getTargetItem() == targetField) { + if (source.length() != 0) + source.append("\n"); + source.append(fieldToFieldMap.getSourceItem().getName().trim()); + + if (logic.length() != 0) + logic.append("\n"); + logic.append(fieldToFieldMap.getLogic().trim()); + + if (comment.length() != 0) + comment.append("\n"); + comment.append(fieldToFieldMap.getComment().trim()); + } + } + for (Field field : targetTable.getFields()) { + if (field.getName().equals(targetField.getName())) { + if (comment.length() != 0) + comment.append("\n"); + comment.append(field.getComment().trim()); + } + } + row.add("Source field", source.toString().trim()); + row.add("Logic", logic.toString().trim()); + row.add("Comment field", comment.toString().trim()); + rows.add(row); + } + + + document.addTable(rows); + } + } + + private void addTableLevelSection() throws InvalidFormatException, IOException { + MappingPanel mappingPanel = new MappingPanel(etl.getTableToTableMapping()); + mappingPanel.setShowOnlyConnectedItems(true); + int height = mappingPanel.getMinimumSize().height; + mappingPanel.setSize(800, height); + + document.addHeader1(mappingPanel.getSourceDbName() + " Data Mapping Approach to " + mappingPanel.getTargetDbName()); + + BufferedImage image = new BufferedImage(800, height, BufferedImage.TYPE_INT_ARGB); + image.getGraphics().setColor(Color.WHITE); + image.getGraphics().fillRect(0, 0, image.getWidth(), image.getHeight()); + mappingPanel.paint(image.getGraphics()); + document.addImage(image, "Table mapping"); + } + + private interface MarkupDocument { + public void addHeader1(String header); + + public void addHeader2(String header); + + public void addHeader3(String header); + + public void addParagraph(String text); + + public void addImage(BufferedImage image, String alternative); + + public void addTable(List rows); + + public void close(); + } + + private class MarkdownDocument implements MarkupDocument { + private List lines = new ArrayList(); + private int imageIndex = 0; + private String fileName; + private String filesFolder; + private String mainFolder; + + /** + * + * @param fileName Full path of the markdown document to create + */ + public MarkdownDocument(String fileName) { + this.fileName = fileName; + mainFolder = new File(fileName).getParent(); + filesFolder = new File(fileName).getName().replaceAll("(\\.md)|(\\.MD)", "_files"); + } + + @Override + public void addHeader1(String header) { + lines.add("# " + header); + lines.add(""); + } + + @Override + public void addHeader2(String header) { + lines.add("## " + header); + lines.add(""); + } + + @Override + public void addHeader3(String header) { + lines.add("### " + header); + lines.add(""); + } + + @Override + public void addParagraph(String text) { + lines.add(text); + lines.add(""); + } + + @Override + public void addImage(BufferedImage image, String alternative) { + if (imageIndex == 0) { + File folder = new File(mainFolder + "/"+ filesFolder); + if (!folder.exists()) + folder.mkdirs(); + } + imageIndex++; + String imageFile = filesFolder + "/image" + imageIndex + ".png"; + try { + ImageIO.write(image, "png", new File(mainFolder + "/" + imageFile)); + } catch (IOException e) { + throw new RuntimeException(e); + } + lines.add("![](" + imageFile + ")"); + lines.add(""); + } + + @Override + public void close() { + WriteTextFile out = new WriteTextFile(fileName); + for (String line : lines) + out.writeln(line); + out.close(); + } + + @Override + public void addTable(List rows) { + if (rows.size() > 0) { + String header = "| " + StringUtilities.join(rows.get(0).getFieldNames(), " | ") + " |"; + header = header.replaceAll("\n", " "); + lines.add(header); + StringBuilder line = new StringBuilder(); + for (int i = 0; i < rows.get(0).getFieldNames().size(); i++) + line.append("| --- "); + line.append("|"); + lines.add(line.toString()); + + for (Row row : rows) { + line = new StringBuilder(); + for (String value : row.getCells()) + line.append("| " + value.replaceAll("\n", " ") + " "); + line.append("|"); + lines.add(line.toString()); + } + lines.add(""); + } + } + } + + private class HtmlDocument implements MarkupDocument { + private List lines = new ArrayList(); + private int imageIndex = 0; + private String fileName; + private String filesFolder; + private String mainFolder; + + /** + * + * @param fileName Full path of the HTML file to create. + */ + public HtmlDocument(String fileName) { + this.fileName = fileName; + mainFolder = new File(fileName).getParent(); + filesFolder = new File(fileName).getName().replaceAll("(\\.html?)|(\\.html?)", "_files"); + } + + + @Override + public void addHeader1(String header) { + lines.add("

" + header + "

"); + lines.add(""); + } + + @Override + public void addHeader2(String header) { + lines.add("

" + header + "

"); + lines.add(""); + } + + @Override + public void addHeader3(String header) { + lines.add("

" + header + "

"); + lines.add(""); + } + + @Override + public void addParagraph(String text) { + lines.add("

" + text + "

"); + lines.add(""); + } + + @Override + public void addImage(BufferedImage image, String alternative) { + if (imageIndex == 0) { + File folder = new File(mainFolder + "/"+ filesFolder); + if (!folder.exists()) + folder.mkdirs(); + } + imageIndex++; + String imageFile = filesFolder + "/image" + imageIndex + ".png"; + try { + ImageIO.write(image, "png", new File(mainFolder + "/" + imageFile)); + } catch (IOException e) { + throw new RuntimeException(e); + } + lines.add("\"""); + lines.add(""); + } + + @Override + public void close() { + WriteTextFile out = new WriteTextFile(fileName); + for (String line : lines) + out.writeln(line); + out.close(); + } + + @Override + public void addTable(List rows) { + if (rows.size() > 0) { + lines.add("
"); + lines.add("\t"); + for (String fieldName : rows.get(0).getFieldNames()) + lines.add("\t\t"); + lines.add("\t"); + + for (Row row : rows) { + lines.add("\t"); + for (String cell : row.getCells()) + lines.add("\t\t"); + lines.add("\t"); + } + lines.add("
" + fieldName + "
" + cell + "
"); + lines.add(""); + } + } + } +} diff --git a/src/org/ohdsi/rabbitInAHat/ETLTestFrameWorkGenerator.java b/rabbitinahat/src/main/java/org/ohdsi/rabbitInAHat/ETLTestFrameWorkGenerator.java similarity index 97% rename from src/org/ohdsi/rabbitInAHat/ETLTestFrameWorkGenerator.java rename to rabbitinahat/src/main/java/org/ohdsi/rabbitInAHat/ETLTestFrameWorkGenerator.java index 21b622ba..78c35a9f 100644 --- a/src/org/ohdsi/rabbitInAHat/ETLTestFrameWorkGenerator.java +++ b/rabbitinahat/src/main/java/org/ohdsi/rabbitInAHat/ETLTestFrameWorkGenerator.java @@ -1,400 +1,400 @@ -/******************************************************************************* - * Copyright 2017 Observational Health Data Sciences and Informatics - * - * This file is part of WhiteRabbit - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - ******************************************************************************/ -package org.ohdsi.rabbitInAHat; - -import java.util.ArrayList; -import java.util.HashSet; -import java.util.List; -import java.util.Set; - -import org.ohdsi.rabbitInAHat.dataModel.Database; -import org.ohdsi.rabbitInAHat.dataModel.ETL; -import org.ohdsi.rabbitInAHat.dataModel.ETL.FileFormat; -import org.ohdsi.rabbitInAHat.dataModel.Field; -import org.ohdsi.rabbitInAHat.dataModel.Table; -import org.ohdsi.utilities.StringUtilities; -import org.ohdsi.utilities.files.WriteTextFile; - -public class ETLTestFrameWorkGenerator { - - private static int DEFAULT = 0; - private static int NEGATE = 1; - private static int COUNT = 2; - private String[] keywords = new String[] { "ADD", "ALL", "ALTER", "AND", "ANY", "AS", "ASC", "AUTHORIZATION", "BACKUP", "BEGIN", "BETWEEN", - "BREAK", "BROWSE", "BULK", "BY", "CASCADE", "CASE", "CHECK", "CHECKPOINT", "CLOSE", "CLUSTERED", "COALESCE", "COLLATE", "COLUMN", "COMMIT", - "COMPUTE", "CONSTRAINT", "CONTAINS", "CONTAINSTABLE", "CONTINUE", "CONVERT", "CREATE", "CROSS", "CURRENT", "CURRENT_DATE", "CURRENT_TIME", - "CURRENT_TIMESTAMP", "CURRENT_USER", "CURSOR", "DATABASE", "DBCC", "DEALLOCATE", "DECLARE", "DEFAULT", "DELETE", "DENY", "DESC", "DISK", "DISTINCT", - "DISTRIBUTED", "DOUBLE", "DROP", "DUMP", "ELSE", "END", "ERRLVL", "ESCAPE", "EXCEPT", "EXEC", "EXECUTE", "EXISTS", "EXIT", "EXTERNAL", "FETCH", - "FILE", "FILLFACTOR", "FOR", "FOREIGN", "FREETEXT", "FREETEXTTABLE", "FROM", "FULL", "FUNCTION", "GOTO", "GRANT", "GROUP", "HAVING", "HOLDLOCK", - "IDENTITY", "IDENTITY_INSERT", "IDENTITYCOL", "IF", "IN", "INDEX", "INNER", "INSERT", "INTERSECT", "INTO", "IS", "JOIN", "KEY", "KILL", "LEFT", - "LIKE", "LINENO", "LOAD", "MERGE", "NATIONAL", "NOCHECK", "NONCLUSTERED", "NOT", "NULL", "NULLIF", "OF", "OFF", "OFFSETS", "ON", "OPEN", - "OPENDATASOURCE", "OPENQUERY", "OPENROWSET", "OPENXML", "OPTION", "OR", "ORDER", "OUTER", "OVER", "PERCENT", "PIVOT", "PLAN", "PRECISION", - "PRIMARY", "PRINT", "PROC", "PROCEDURE", "PUBLIC", "RAISERROR", "READ", "READTEXT", "RECONFIGURE", "REFERENCES", "REPLICATION", "RESTORE", - "RESTRICT", "RETURN", "REVERT", "REVOKE", "RIGHT", "ROLLBACK", "ROWCOUNT", "ROWGUIDCOL", "RULE", "SAVE", "SCHEMA", "SECURITYAUDIT", "SELECT", - "SEMANTICKEYPHRASETABLE", "SEMANTICSIMILARITYDETAILSTABLE", "SEMANTICSIMILARITYTABLE", "SESSION_USER", "SET", "SETUSER", "SHUTDOWN", "SOME", - "STATISTICS", "SYSTEM_USER", "TABLE", "TABLESAMPLE", "TEXTSIZE", "THEN", "TO", "TOP", "TRAN", "TRANSACTION", "TRIGGER", "TRUNCATE", "TRY_CONVERT", - "TSEQUAL", "UNION", "UNIQUE", "UNPIVOT", "UPDATE", "UPDATETEXT", "USE", "USER", "VALUES", "VARYING", "VIEW", "WAITFOR", "WHEN", "WHERE", "WHILE", - "WITH", "WITHIN GROUP", "WRITETEXT" }; - protected Set keywordSet; - private List r; - private Database sourceDb; - private Database targetDb; - - public static void main(String[] args) { - ETL etl = ETL.fromFile("C:\\Home\\Research\\ETLs\\JMDC ETL\\JMDC ETL CDMv5\\JMDC to CDMv5 ETL v08.json.gz", FileFormat.GzipJson); - ETLTestFrameWorkGenerator generator = new ETLTestFrameWorkGenerator(); - generator.generate(etl, "C:\\Home\\Research\\ETLs\\JMDC ETL\\JMDC ETL CDMv5\\JmdcTestFramework.R"); - } - - public ETLTestFrameWorkGenerator() { - keywordSet = new HashSet(); - for (String keyword : keywords) - keywordSet.add(keyword); - } - - public void generate(ETL etl, String filename) { - List r = generateRScript(etl); - WriteTextFile out = new WriteTextFile(filename); - for (String line : r) - out.writeln(line); - out.close(); - } - - private List generateRScript(ETL etl) { - this.sourceDb = etl.getSourceDatabase(); - this.targetDb = etl.getTargetDatabase(); - this.r = new ArrayList(); - createInitFunction(); - createSetDefaultFunctions(); - createGetDefaultFunctions(); - createDeclareTestFunction(); - createAddFunctions(); - createExpectFunctions(DEFAULT); - createExpectFunctions(NEGATE); - createExpectFunctions(COUNT); - createLookupFunctions(); - createGenerateInsertSqlFunction(); - createGenerateTestSqlFunction(); - return r; - } - - private void createInitFunction() { - r.add("initFramework <- function() {"); - r.add(" frameworkContext <- new.env(parent = globalenv())"); - r.add(" assign('frameworkContext', frameworkContext, envir = globalenv())"); - r.add(" frameworkContext$inserts <- list()"); - r.add(" frameworkContext$expects <- list()"); - r.add(" frameworkContext$testId <- -1"); - r.add(" frameworkContext$testDescription <- \"\""); - r.add(" frameworkContext$defaultValues <- new.env(parent = frameworkContext)"); - for (Table table : sourceDb.getTables()) { - if (!table.isStem()) { - String rTableName = convertToRName(table.getName()); - r.add(""); - r.add(" defaults <- list()"); - for (Field field : table.getFields()) { - String rFieldName = convertToRName(field.getName()); - String defaultValue; - if (field.getValueCounts().length == 0) - defaultValue = ""; - else - defaultValue = field.getValueCounts()[0][0]; - if (!defaultValue.equals("") && !defaultValue.equals("List truncated...")) - r.add(" defaults$" + rFieldName + " <- '" + defaultValue + "'"); - } - r.add(" assign('" + rTableName + "', defaults, envir = frameworkContext$defaultValues)"); - } - } - r.add("}"); - r.add(""); - r.add("initFramework()"); - r.add(""); - } - - private void createSetDefaultFunctions() { - for (Table table : sourceDb.getTables()) { - if (!table.isStem()) { - StringBuilder line = new StringBuilder(); - String rTableName = convertToRName(table.getName()); - List argDefs = new ArrayList(); - List insertLines = new ArrayList(); - for (Field field : table.getFields()) { - String rFieldName = convertToRName(field.getName()); - argDefs.add(rFieldName); - insertLines.add(" if (!missing(" + rFieldName + ")) {"); - insertLines.add(" defaults$" + rFieldName + " <- " + rFieldName); - insertLines.add(" }"); - } - - line.append("set_defaults_" + rTableName + " <- function("); - line.append(StringUtilities.join(argDefs, ", ")); - line.append(") {"); - r.add(line.toString()); - r.add(" defaults <- get('" + rTableName + "', envir = frameworkContext$defaultValues)"); - r.addAll(insertLines); - r.add(" assign('" + rTableName + "', defaults, envir = frameworkContext$defaultValues)"); - r.add(" invisible(defaults)"); - r.add("}"); - r.add(""); - } - } - } - - private void createGetDefaultFunctions() { - for (Table table : sourceDb.getTables()) { - String rTableName = convertToRName(table.getName()); - r.add("get_defaults_" + rTableName + " <- function() {"); - r.add(" defaults <- get('" + rTableName + "', envir = frameworkContext$defaultValues)"); - r.add(" return(defaults)"); - r.add("}"); - r.add(""); - } - } - - private void createDeclareTestFunction() { - r.add("declareTest <- function(id, description) {"); - r.add(" frameworkContext$testId <- id"); - r.add(" frameworkContext$testDescription <- description"); - r.add("}"); - r.add(""); - } - - private void createAddFunctions() { - for (Table table : sourceDb.getTables()) { - if (!table.isStem()) { - StringBuilder line = new StringBuilder(); - String rTableName = convertToRName(table.getName()); - String sqlTableName = convertToSqlName(table.getName()); - List argDefs = new ArrayList(); - for (Field field : table.getFields()) { - String rFieldName = convertToRName(field.getName()); - argDefs.add(rFieldName); - } - line.append("add_" + rTableName + " <- function("); - line.append(StringUtilities.join(argDefs, ", ")); - line.append(") {"); - r.add(line.toString()); - r.add(" defaults <- get('" + rTableName + "', envir = frameworkContext$defaultValues)"); - r.add(" fields <- c()"); - r.add(" values <- c()"); - for (Field field : table.getFields()) { - String rFieldName = convertToRName(field.getName()); - String sqlFieldName = convertToSqlName(field.getName()); - r.add(" if (missing(" + rFieldName + ")) {"); - r.add(" " + rFieldName + " <- defaults$" + rFieldName); - r.add(" }"); - r.add(" if (!is.null(" + rFieldName + ")) {"); - r.add(" fields <- c(fields, \"" + sqlFieldName + "\")"); - r.add(" values <- c(values, " + createSqlValueCode(rFieldName) + ")"); - r.add(" }"); - r.add(""); - } - r.add(" inserts <- list(testId = frameworkContext$testId, testDescription = frameworkContext$testDescription, table = \"" + sqlTableName - + "\", fields = fields, values = values)"); - r.add(" frameworkContext$inserts[[length(frameworkContext$inserts) + 1]] <- inserts"); - r.add(" invisible(NULL)"); - r.add("}"); - r.add(""); - } - } - } - - private void createExpectFunctions(int type) { - for (Table table : targetDb.getTables()) { - if (!table.isStem()) { - StringBuilder line = new StringBuilder(); - String rTableName = convertToRName(table.getName()); - String sqlTableName = convertToSqlName(table.getName()); - List argDefs = new ArrayList(); - for (Field field : table.getFields()) { - String rFieldName = convertToRName(field.getName()); - argDefs.add(rFieldName); - } - if (type == DEFAULT) - line.append("expect_" + rTableName + " <- function("); - else if (type == NEGATE) - line.append("expect_no_" + rTableName + " <- function("); - else - line.append("expect_count_" + rTableName + " <- function(rowCount, "); - line.append(StringUtilities.join(argDefs, ", ")); - line.append(") {"); - r.add(line.toString()); - r.add(" fields <- c()"); - r.add(" values <- c()"); - for (Field field : table.getFields()) { - String rFieldName = convertToRName(field.getName()); - String sqlFieldName = convertToSqlName(field.getName()); - r.add(" if (!missing(" + rFieldName + ")) {"); - r.add(" fields <- c(fields, \"" + sqlFieldName + "\")"); - r.add(" values <- c(values, " + createSqlValueCode(rFieldName) + ")"); - r.add(" }"); - r.add(""); - } - r.add(" expects <- list(testId = frameworkContext$testId, testDescription = frameworkContext$testDescription, type = " + type + ", table = \"" - + sqlTableName + "\", fields = fields, values = values)"); - if (type == COUNT) - r.add(" expects$rowCount = rowCount"); - r.add(" frameworkContext$expects[[length(frameworkContext$expects) + 1]] <- expects"); - r.add(" invisible(NULL)"); - r.add("}"); - r.add(""); - } - } - } - - protected void createLookupFunctions() { - for (Table table : targetDb.getTables()) { - if (!table.isStem()) { - StringBuilder line = new StringBuilder(); - String rTableName = convertToRName(table.getName()); - String sqlTableName = convertToSqlName(table.getName()); - List argDefs = new ArrayList(); - for (Field field : table.getFields()) { - String rFieldName = convertToRName(field.getName()); - argDefs.add(rFieldName); - } - line.append("lookup_" + rTableName + " <- function(fetchField, "); - line.append(StringUtilities.join(argDefs, ", ")); - line.append(") {"); - r.add(line.toString()); - line = new StringBuilder(); - line.append(" statement <- paste0('SELECT ', fetchField , ' FROM @cdm_database_schema."); - line.append(sqlTableName); - line.append(" WHERE')"); - r.add(line.toString()); - r.add(" first <- TRUE"); - for (Field field : table.getFields()) { - String rFieldName = convertToRName(field.getName()); - String sqlFieldName = convertToSqlName(field.getName()); - argDefs.add(rFieldName); - r.add(" if (!missing(" + rFieldName + ")) {"); - r.add(" if (first) {"); - r.add(" first <- FALSE"); - r.add(" } else {"); - r.add(" statement <- paste0(statement, \" AND\")"); - r.add(" }"); - r.add(" statement <- paste0(statement, \" " + sqlFieldName + " = \", " + createSqlValueCode(rFieldName) + ")"); - r.add(" }"); - r.add(""); - } - r.add(" class(statement) <- 'subQuery'"); - r.add(" return(statement)"); - r.add("}"); - r.add(""); - } - } - } - - protected void createGenerateInsertSqlFunction() { - r.add("generateInsertSql <- function(databaseSchema = NULL) {"); - r.add(" insertSql <- c()"); - for (Table table : sourceDb.getTables()) - if (!table.isStem()) - r.add(" insertSql <- c(insertSql, \"TRUNCATE TABLE @cdm_database_schema." + convertToSqlName(table.getName()) + ";\")"); - r.add(" createInsertStatement <- function(insert, env) {"); - r.add(" s <- c()"); - r.add(" if (env$testId != insert$testId) {"); - r.add(" s <- c(s, paste0('-- ', insert$testId, ': ', insert$testDescription))"); - r.add(" env$testId <- insert$testId"); - r.add(" }"); - r.add(" s <- c(s, paste0(\"INSERT INTO @cdm_database_schema.\","); - r.add(" insert$table,"); - r.add(" \"(\","); - r.add(" paste(insert$fields, collapse = \", \"),"); - r.add(" \") VALUES (\","); - r.add(" paste(insert$values, collapse = \", \"), "); - r.add(" \");\"))"); - r.add(" return(s)"); - r.add(" }"); - r.add(" env <- new.env()"); - r.add(" env$testId <- -1"); - r.add(" insertSql <- c(insertSql, do.call(c, lapply(frameworkContext$inserts, createInsertStatement, env)))"); - r.add(" if (is.null(databaseSchema)) {"); - r.add(" insertSql <- gsub('@cdm_database_schema.', '', insertSql)"); - r.add(" } else {"); - r.add(" insertSql <- gsub('@cdm_database_schema', databaseSchema, insertSql)"); - r.add(" }"); - r.add(" return(insertSql)"); - r.add("}"); - r.add(""); - } - - protected void createGenerateTestSqlFunction() { - r.add("generateTestSql <- function(databaseSchema = NULL) {"); - r.add(" testSql <- c()"); - r.add(" testSql <- c(testSql, \"IF OBJECT_ID('@cdm_database_schema.test_results', 'U') IS NOT NULL DROP TABLE @cdm_database_schema.test_results;\")"); - r.add(" testSql <- c(testSql, \"CREATE TABLE @cdm_database_schema.test_results (id INT, description VARCHAR(512), test VARCHAR(256), status VARCHAR(5));\")"); - r.add(" createExpectStatement <- function(expect, env) {"); - r.add(" s <- c()"); - r.add(" if (env$testId != expect$testId) {"); - r.add(" s <- c(s, paste0('-- ', expect$testId, ': ', expect$testDescription))"); - r.add(" env$testId <- expect$testId"); - r.add(" }"); - r.add(" operators <- rep(\"=\", length(expect$fields))"); - r.add(" operators[expect$values == \"NULL\"] <- rep(\"IS\", sum(expect$values == \"NULL\"))"); - r.add(" s <- c(s, paste0(\"INSERT INTO @cdm_database_schema.test_results SELECT \","); - r.add(" expect$testId,"); - r.add(" \" AS id, '\","); - r.add(" expect$testDescription,"); - r.add(" \"' AS description, 'Expect \","); - r.add(" expect$table,"); - r.add(" \"' AS test, CASE WHEN (SELECT COUNT(*) FROM @cdm_database_schema.\","); - r.add(" expect$table,"); - r.add(" \" WHERE \","); - r.add(" paste(paste(expect$fields, operators, expect$values), collapse = \" AND \"),"); - r.add(" \") \","); - r.add(" if (expect$type == " + DEFAULT + ") \"= 0\" else if (expect$type == " + NEGATE - + ") \"!= 0\" else paste(\"!=\", expect$rowCount),"); - r.add(" \" THEN 'FAIL' ELSE 'PASS' END AS status;\"))"); - r.add(" return(s)"); - r.add(" }"); - r.add(" env <- new.env()"); - r.add(" env$testId <- -1"); - r.add(" testSql <- c(testSql, do.call(c, lapply(frameworkContext$expects, createExpectStatement, env)))"); - r.add(" if (is.null(databaseSchema)) {"); - r.add(" testSql <- gsub('@cdm_database_schema.', '', testSql)"); - r.add(" } else {"); - r.add(" testSql <- gsub('@cdm_database_schema', databaseSchema, testSql)"); - r.add(" }"); - r.add(" return(testSql)"); - r.add("}"); - r.add(""); - } - - private String convertToRName(String name) { - name = name.replaceAll(" ", "_").replaceAll("-", "_").replaceAll("^_+", ""); - return name; - } - - protected String createSqlValueCode(String rFieldName) { - StringBuilder expression = new StringBuilder(); - expression.append("if (is.null(" + rFieldName + ")) \"NULL\" "); - expression.append("else if (is(" + rFieldName + ", \"subQuery\")) paste0(\"(\", as.character(" + rFieldName + "), \")\") "); - expression.append("else paste0(\"'\", as.character(" + rFieldName + "), \"'\")"); - return (expression.toString()); - } - - protected String convertToSqlName(String name) { - if (name.startsWith("[") && name.endsWith("]")) - return name; - name = name.replace('[', '_').replace(']', '_'); - if (name.contains(" ") || name.contains(".") || keywordSet.contains(name.toUpperCase())) - return "[" + name + "]"; - return name; - } -} +/******************************************************************************* + * Copyright 2019 Observational Health Data Sciences and Informatics + * + * This file is part of WhiteRabbit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + ******************************************************************************/ +package org.ohdsi.rabbitInAHat; + +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +import org.ohdsi.rabbitInAHat.dataModel.Database; +import org.ohdsi.rabbitInAHat.dataModel.ETL; +import org.ohdsi.rabbitInAHat.dataModel.ETL.FileFormat; +import org.ohdsi.rabbitInAHat.dataModel.Field; +import org.ohdsi.rabbitInAHat.dataModel.Table; +import org.ohdsi.utilities.StringUtilities; +import org.ohdsi.utilities.files.WriteTextFile; + +public class ETLTestFrameWorkGenerator { + + private static int DEFAULT = 0; + private static int NEGATE = 1; + private static int COUNT = 2; + private String[] keywords = new String[] { "ADD", "ALL", "ALTER", "AND", "ANY", "AS", "ASC", "AUTHORIZATION", "BACKUP", "BEGIN", "BETWEEN", + "BREAK", "BROWSE", "BULK", "BY", "CASCADE", "CASE", "CHECK", "CHECKPOINT", "CLOSE", "CLUSTERED", "COALESCE", "COLLATE", "COLUMN", "COMMIT", + "COMPUTE", "CONSTRAINT", "CONTAINS", "CONTAINSTABLE", "CONTINUE", "CONVERT", "CREATE", "CROSS", "CURRENT", "CURRENT_DATE", "CURRENT_TIME", + "CURRENT_TIMESTAMP", "CURRENT_USER", "CURSOR", "DATABASE", "DBCC", "DEALLOCATE", "DECLARE", "DEFAULT", "DELETE", "DENY", "DESC", "DISK", "DISTINCT", + "DISTRIBUTED", "DOUBLE", "DROP", "DUMP", "ELSE", "END", "ERRLVL", "ESCAPE", "EXCEPT", "EXEC", "EXECUTE", "EXISTS", "EXIT", "EXTERNAL", "FETCH", + "FILE", "FILLFACTOR", "FOR", "FOREIGN", "FREETEXT", "FREETEXTTABLE", "FROM", "FULL", "FUNCTION", "GOTO", "GRANT", "GROUP", "HAVING", "HOLDLOCK", + "IDENTITY", "IDENTITY_INSERT", "IDENTITYCOL", "IF", "IN", "INDEX", "INNER", "INSERT", "INTERSECT", "INTO", "IS", "JOIN", "KEY", "KILL", "LEFT", + "LIKE", "LINENO", "LOAD", "MERGE", "NATIONAL", "NOCHECK", "NONCLUSTERED", "NOT", "NULL", "NULLIF", "OF", "OFF", "OFFSETS", "ON", "OPEN", + "OPENDATASOURCE", "OPENQUERY", "OPENROWSET", "OPENXML", "OPTION", "OR", "ORDER", "OUTER", "OVER", "PERCENT", "PIVOT", "PLAN", "PRECISION", + "PRIMARY", "PRINT", "PROC", "PROCEDURE", "PUBLIC", "RAISERROR", "READ", "READTEXT", "RECONFIGURE", "REFERENCES", "REPLICATION", "RESTORE", + "RESTRICT", "RETURN", "REVERT", "REVOKE", "RIGHT", "ROLLBACK", "ROWCOUNT", "ROWGUIDCOL", "RULE", "SAVE", "SCHEMA", "SECURITYAUDIT", "SELECT", + "SEMANTICKEYPHRASETABLE", "SEMANTICSIMILARITYDETAILSTABLE", "SEMANTICSIMILARITYTABLE", "SESSION_USER", "SET", "SETUSER", "SHUTDOWN", "SOME", + "STATISTICS", "SYSTEM_USER", "TABLE", "TABLESAMPLE", "TEXTSIZE", "THEN", "TO", "TOP", "TRAN", "TRANSACTION", "TRIGGER", "TRUNCATE", "TRY_CONVERT", + "TSEQUAL", "UNION", "UNIQUE", "UNPIVOT", "UPDATE", "UPDATETEXT", "USE", "USER", "VALUES", "VARYING", "VIEW", "WAITFOR", "WHEN", "WHERE", "WHILE", + "WITH", "WITHIN GROUP", "WRITETEXT" }; + protected Set keywordSet; + private List r; + private Database sourceDb; + private Database targetDb; + + public static void main(String[] args) { + ETL etl = ETL.fromFile("C:\\Home\\Research\\ETLs\\JMDC ETL\\JMDC ETL CDMv5\\JMDC to CDMv5 ETL v08.json.gz", FileFormat.GzipJson); + ETLTestFrameWorkGenerator generator = new ETLTestFrameWorkGenerator(); + generator.generate(etl, "C:\\Home\\Research\\ETLs\\JMDC ETL\\JMDC ETL CDMv5\\JmdcTestFramework.R"); + } + + public ETLTestFrameWorkGenerator() { + keywordSet = new HashSet(); + for (String keyword : keywords) + keywordSet.add(keyword); + } + + public void generate(ETL etl, String filename) { + List r = generateRScript(etl); + WriteTextFile out = new WriteTextFile(filename); + for (String line : r) + out.writeln(line); + out.close(); + } + + private List generateRScript(ETL etl) { + this.sourceDb = etl.getSourceDatabase(); + this.targetDb = etl.getTargetDatabase(); + this.r = new ArrayList(); + createInitFunction(); + createSetDefaultFunctions(); + createGetDefaultFunctions(); + createDeclareTestFunction(); + createAddFunctions(); + createExpectFunctions(DEFAULT); + createExpectFunctions(NEGATE); + createExpectFunctions(COUNT); + createLookupFunctions(); + createGenerateInsertSqlFunction(); + createGenerateTestSqlFunction(); + return r; + } + + private void createInitFunction() { + r.add("initFramework <- function() {"); + r.add(" frameworkContext <- new.env(parent = globalenv())"); + r.add(" assign('frameworkContext', frameworkContext, envir = globalenv())"); + r.add(" frameworkContext$inserts <- list()"); + r.add(" frameworkContext$expects <- list()"); + r.add(" frameworkContext$testId <- -1"); + r.add(" frameworkContext$testDescription <- \"\""); + r.add(" frameworkContext$defaultValues <- new.env(parent = frameworkContext)"); + for (Table table : sourceDb.getTables()) { + if (!table.isStem()) { + String rTableName = convertToRName(table.getName()); + r.add(""); + r.add(" defaults <- list()"); + for (Field field : table.getFields()) { + String rFieldName = convertToRName(field.getName()); + String defaultValue; + if (field.getValueCounts().length == 0) + defaultValue = ""; + else + defaultValue = field.getValueCounts()[0][0]; + if (!defaultValue.equals("") && !defaultValue.equals("List truncated...")) + r.add(" defaults$" + rFieldName + " <- '" + defaultValue + "'"); + } + r.add(" assign('" + rTableName + "', defaults, envir = frameworkContext$defaultValues)"); + } + } + r.add("}"); + r.add(""); + r.add("initFramework()"); + r.add(""); + } + + private void createSetDefaultFunctions() { + for (Table table : sourceDb.getTables()) { + if (!table.isStem()) { + StringBuilder line = new StringBuilder(); + String rTableName = convertToRName(table.getName()); + List argDefs = new ArrayList(); + List insertLines = new ArrayList(); + for (Field field : table.getFields()) { + String rFieldName = convertToRName(field.getName()); + argDefs.add(rFieldName); + insertLines.add(" if (!missing(" + rFieldName + ")) {"); + insertLines.add(" defaults$" + rFieldName + " <- " + rFieldName); + insertLines.add(" }"); + } + + line.append("set_defaults_" + rTableName + " <- function("); + line.append(StringUtilities.join(argDefs, ", ")); + line.append(") {"); + r.add(line.toString()); + r.add(" defaults <- get('" + rTableName + "', envir = frameworkContext$defaultValues)"); + r.addAll(insertLines); + r.add(" assign('" + rTableName + "', defaults, envir = frameworkContext$defaultValues)"); + r.add(" invisible(defaults)"); + r.add("}"); + r.add(""); + } + } + } + + private void createGetDefaultFunctions() { + for (Table table : sourceDb.getTables()) { + String rTableName = convertToRName(table.getName()); + r.add("get_defaults_" + rTableName + " <- function() {"); + r.add(" defaults <- get('" + rTableName + "', envir = frameworkContext$defaultValues)"); + r.add(" return(defaults)"); + r.add("}"); + r.add(""); + } + } + + private void createDeclareTestFunction() { + r.add("declareTest <- function(id, description) {"); + r.add(" frameworkContext$testId <- id"); + r.add(" frameworkContext$testDescription <- description"); + r.add("}"); + r.add(""); + } + + private void createAddFunctions() { + for (Table table : sourceDb.getTables()) { + if (!table.isStem()) { + StringBuilder line = new StringBuilder(); + String rTableName = convertToRName(table.getName()); + String sqlTableName = convertToSqlName(table.getName()); + List argDefs = new ArrayList(); + for (Field field : table.getFields()) { + String rFieldName = convertToRName(field.getName()); + argDefs.add(rFieldName); + } + line.append("add_" + rTableName + " <- function("); + line.append(StringUtilities.join(argDefs, ", ")); + line.append(") {"); + r.add(line.toString()); + r.add(" defaults <- get('" + rTableName + "', envir = frameworkContext$defaultValues)"); + r.add(" fields <- c()"); + r.add(" values <- c()"); + for (Field field : table.getFields()) { + String rFieldName = convertToRName(field.getName()); + String sqlFieldName = convertToSqlName(field.getName()); + r.add(" if (missing(" + rFieldName + ")) {"); + r.add(" " + rFieldName + " <- defaults$" + rFieldName); + r.add(" }"); + r.add(" if (!is.null(" + rFieldName + ")) {"); + r.add(" fields <- c(fields, \"" + sqlFieldName + "\")"); + r.add(" values <- c(values, " + createSqlValueCode(rFieldName) + ")"); + r.add(" }"); + r.add(""); + } + r.add(" inserts <- list(testId = frameworkContext$testId, testDescription = frameworkContext$testDescription, table = \"" + sqlTableName + + "\", fields = fields, values = values)"); + r.add(" frameworkContext$inserts[[length(frameworkContext$inserts) + 1]] <- inserts"); + r.add(" invisible(NULL)"); + r.add("}"); + r.add(""); + } + } + } + + private void createExpectFunctions(int type) { + for (Table table : targetDb.getTables()) { + if (!table.isStem()) { + StringBuilder line = new StringBuilder(); + String rTableName = convertToRName(table.getName()); + String sqlTableName = convertToSqlName(table.getName()); + List argDefs = new ArrayList(); + for (Field field : table.getFields()) { + String rFieldName = convertToRName(field.getName()); + argDefs.add(rFieldName); + } + if (type == DEFAULT) + line.append("expect_" + rTableName + " <- function("); + else if (type == NEGATE) + line.append("expect_no_" + rTableName + " <- function("); + else + line.append("expect_count_" + rTableName + " <- function(rowCount, "); + line.append(StringUtilities.join(argDefs, ", ")); + line.append(") {"); + r.add(line.toString()); + r.add(" fields <- c()"); + r.add(" values <- c()"); + for (Field field : table.getFields()) { + String rFieldName = convertToRName(field.getName()); + String sqlFieldName = convertToSqlName(field.getName()); + r.add(" if (!missing(" + rFieldName + ")) {"); + r.add(" fields <- c(fields, \"" + sqlFieldName + "\")"); + r.add(" values <- c(values, " + createSqlValueCode(rFieldName) + ")"); + r.add(" }"); + r.add(""); + } + r.add(" expects <- list(testId = frameworkContext$testId, testDescription = frameworkContext$testDescription, type = " + type + ", table = \"" + + sqlTableName + "\", fields = fields, values = values)"); + if (type == COUNT) + r.add(" expects$rowCount = rowCount"); + r.add(" frameworkContext$expects[[length(frameworkContext$expects) + 1]] <- expects"); + r.add(" invisible(NULL)"); + r.add("}"); + r.add(""); + } + } + } + + protected void createLookupFunctions() { + for (Table table : targetDb.getTables()) { + if (!table.isStem()) { + StringBuilder line = new StringBuilder(); + String rTableName = convertToRName(table.getName()); + String sqlTableName = convertToSqlName(table.getName()); + List argDefs = new ArrayList(); + for (Field field : table.getFields()) { + String rFieldName = convertToRName(field.getName()); + argDefs.add(rFieldName); + } + line.append("lookup_" + rTableName + " <- function(fetchField, "); + line.append(StringUtilities.join(argDefs, ", ")); + line.append(") {"); + r.add(line.toString()); + line = new StringBuilder(); + line.append(" statement <- paste0('SELECT ', fetchField , ' FROM @cdm_database_schema."); + line.append(sqlTableName); + line.append(" WHERE')"); + r.add(line.toString()); + r.add(" first <- TRUE"); + for (Field field : table.getFields()) { + String rFieldName = convertToRName(field.getName()); + String sqlFieldName = convertToSqlName(field.getName()); + argDefs.add(rFieldName); + r.add(" if (!missing(" + rFieldName + ")) {"); + r.add(" if (first) {"); + r.add(" first <- FALSE"); + r.add(" } else {"); + r.add(" statement <- paste0(statement, \" AND\")"); + r.add(" }"); + r.add(" statement <- paste0(statement, \" " + sqlFieldName + " = \", " + createSqlValueCode(rFieldName) + ")"); + r.add(" }"); + r.add(""); + } + r.add(" class(statement) <- 'subQuery'"); + r.add(" return(statement)"); + r.add("}"); + r.add(""); + } + } + } + + protected void createGenerateInsertSqlFunction() { + r.add("generateInsertSql <- function(databaseSchema = NULL) {"); + r.add(" insertSql <- c()"); + for (Table table : sourceDb.getTables()) + if (!table.isStem()) + r.add(" insertSql <- c(insertSql, \"TRUNCATE TABLE @cdm_database_schema." + convertToSqlName(table.getName()) + ";\")"); + r.add(" createInsertStatement <- function(insert, env) {"); + r.add(" s <- c()"); + r.add(" if (env$testId != insert$testId) {"); + r.add(" s <- c(s, paste0('-- ', insert$testId, ': ', insert$testDescription))"); + r.add(" env$testId <- insert$testId"); + r.add(" }"); + r.add(" s <- c(s, paste0(\"INSERT INTO @cdm_database_schema.\","); + r.add(" insert$table,"); + r.add(" \"(\","); + r.add(" paste(insert$fields, collapse = \", \"),"); + r.add(" \") VALUES (\","); + r.add(" paste(insert$values, collapse = \", \"), "); + r.add(" \");\"))"); + r.add(" return(s)"); + r.add(" }"); + r.add(" env <- new.env()"); + r.add(" env$testId <- -1"); + r.add(" insertSql <- c(insertSql, do.call(c, lapply(frameworkContext$inserts, createInsertStatement, env)))"); + r.add(" if (is.null(databaseSchema)) {"); + r.add(" insertSql <- gsub('@cdm_database_schema.', '', insertSql)"); + r.add(" } else {"); + r.add(" insertSql <- gsub('@cdm_database_schema', databaseSchema, insertSql)"); + r.add(" }"); + r.add(" return(insertSql)"); + r.add("}"); + r.add(""); + } + + protected void createGenerateTestSqlFunction() { + r.add("generateTestSql <- function(databaseSchema = NULL) {"); + r.add(" testSql <- c()"); + r.add(" testSql <- c(testSql, \"IF OBJECT_ID('@cdm_database_schema.test_results', 'U') IS NOT NULL DROP TABLE @cdm_database_schema.test_results;\")"); + r.add(" testSql <- c(testSql, \"CREATE TABLE @cdm_database_schema.test_results (id INT, description VARCHAR(512), test VARCHAR(256), status VARCHAR(5));\")"); + r.add(" createExpectStatement <- function(expect, env) {"); + r.add(" s <- c()"); + r.add(" if (env$testId != expect$testId) {"); + r.add(" s <- c(s, paste0('-- ', expect$testId, ': ', expect$testDescription))"); + r.add(" env$testId <- expect$testId"); + r.add(" }"); + r.add(" operators <- rep(\"=\", length(expect$fields))"); + r.add(" operators[expect$values == \"NULL\"] <- rep(\"IS\", sum(expect$values == \"NULL\"))"); + r.add(" s <- c(s, paste0(\"INSERT INTO @cdm_database_schema.test_results SELECT \","); + r.add(" expect$testId,"); + r.add(" \" AS id, '\","); + r.add(" expect$testDescription,"); + r.add(" \"' AS description, 'Expect \","); + r.add(" expect$table,"); + r.add(" \"' AS test, CASE WHEN (SELECT COUNT(*) FROM @cdm_database_schema.\","); + r.add(" expect$table,"); + r.add(" \" WHERE \","); + r.add(" paste(paste(expect$fields, operators, expect$values), collapse = \" AND \"),"); + r.add(" \") \","); + r.add(" if (expect$type == " + DEFAULT + ") \"= 0\" else if (expect$type == " + NEGATE + + ") \"!= 0\" else paste(\"!=\", expect$rowCount),"); + r.add(" \" THEN 'FAIL' ELSE 'PASS' END AS status;\"))"); + r.add(" return(s)"); + r.add(" }"); + r.add(" env <- new.env()"); + r.add(" env$testId <- -1"); + r.add(" testSql <- c(testSql, do.call(c, lapply(frameworkContext$expects, createExpectStatement, env)))"); + r.add(" if (is.null(databaseSchema)) {"); + r.add(" testSql <- gsub('@cdm_database_schema.', '', testSql)"); + r.add(" } else {"); + r.add(" testSql <- gsub('@cdm_database_schema', databaseSchema, testSql)"); + r.add(" }"); + r.add(" return(testSql)"); + r.add("}"); + r.add(""); + } + + private String convertToRName(String name) { + name = name.replaceAll(" ", "_").replaceAll("-", "_").replaceAll("^_+", ""); + return name; + } + + protected String createSqlValueCode(String rFieldName) { + StringBuilder expression = new StringBuilder(); + expression.append("if (is.null(" + rFieldName + ")) \"NULL\" "); + expression.append("else if (is(" + rFieldName + ", \"subQuery\")) paste0(\"(\", as.character(" + rFieldName + "), \")\") "); + expression.append("else paste0(\"'\", as.character(" + rFieldName + "), \"'\")"); + return (expression.toString()); + } + + protected String convertToSqlName(String name) { + if (name.startsWith("[") && name.endsWith("]")) + return name; + name = name.replace('[', '_').replace(']', '_'); + if (name.contains(" ") || name.contains(".") || keywordSet.contains(name.toUpperCase())) + return "[" + name + "]"; + return name; + } +} diff --git a/src/org/ohdsi/rabbitInAHat/ETLDocumentGenerator.java b/rabbitinahat/src/main/java/org/ohdsi/rabbitInAHat/ETLWordDocumentGenerator.java similarity index 96% rename from src/org/ohdsi/rabbitInAHat/ETLDocumentGenerator.java rename to rabbitinahat/src/main/java/org/ohdsi/rabbitInAHat/ETLWordDocumentGenerator.java index 0980841c..cb015b03 100644 --- a/src/org/ohdsi/rabbitInAHat/ETLDocumentGenerator.java +++ b/rabbitinahat/src/main/java/org/ohdsi/rabbitInAHat/ETLWordDocumentGenerator.java @@ -1,5 +1,5 @@ /******************************************************************************* - * Copyright 2017 Observational Health Data Sciences and Informatics + * Copyright 2019 Observational Health Data Sciences and Informatics * * This file is part of WhiteRabbit * @@ -39,7 +39,7 @@ import org.ohdsi.rabbitInAHat.dataModel.Mapping; import org.ohdsi.rabbitInAHat.dataModel.Table; -public class ETLDocumentGenerator { +public class ETLWordDocumentGenerator { public static void generate(ETL etl, String filename, boolean includeCounts) { try { diff --git a/src/org/ohdsi/rabbitInAHat/FetchCDMModelFromServer.java b/rabbitinahat/src/main/java/org/ohdsi/rabbitInAHat/FetchCDMModelFromServer.java similarity index 94% rename from src/org/ohdsi/rabbitInAHat/FetchCDMModelFromServer.java rename to rabbitinahat/src/main/java/org/ohdsi/rabbitInAHat/FetchCDMModelFromServer.java index 1c139b33..7dc47aee 100644 --- a/src/org/ohdsi/rabbitInAHat/FetchCDMModelFromServer.java +++ b/rabbitinahat/src/main/java/org/ohdsi/rabbitInAHat/FetchCDMModelFromServer.java @@ -1,5 +1,5 @@ /******************************************************************************* - * Copyright 2017 Observational Health Data Sciences and Informatics + * Copyright 2019 Observational Health Data Sciences and Informatics * * This file is part of WhiteRabbit * diff --git a/src/org/ohdsi/rabbitInAHat/FilterDialog.java b/rabbitinahat/src/main/java/org/ohdsi/rabbitInAHat/FilterDialog.java similarity index 96% rename from src/org/ohdsi/rabbitInAHat/FilterDialog.java rename to rabbitinahat/src/main/java/org/ohdsi/rabbitInAHat/FilterDialog.java index dfca4bea..259803b6 100644 --- a/src/org/ohdsi/rabbitInAHat/FilterDialog.java +++ b/rabbitinahat/src/main/java/org/ohdsi/rabbitInAHat/FilterDialog.java @@ -1,209 +1,209 @@ -package org.ohdsi.rabbitInAHat; - -import java.awt.Container; -import java.awt.Window; -import java.awt.event.ActionEvent; -import java.awt.event.ActionListener; -import java.awt.event.FocusEvent; -import java.awt.event.FocusListener; -import java.awt.event.KeyEvent; -import java.awt.event.KeyListener; - -import org.ohdsi.rabbitInAHat.ResizeListener; - -import javax.swing.JButton; -import javax.swing.JDialog; -import javax.swing.JLabel; -import javax.swing.JTextField; -import javax.swing.SpringLayout; - -public class FilterDialog extends JDialog implements ActionListener, ResizeListener { - - - /** - * - */ - private static final long serialVersionUID = 7009265246652874341L; - - - private JTextField sourceSearchField; - private JTextField targetSearchField; - private MappingPanel filterPanel; - - SpringLayout layout = new SpringLayout(); - - Container contentPane = this.getContentPane(); - - public FilterDialog(Window parentWindow){ - - super(parentWindow,"Filter",ModalityType.MODELESS); - this.setResizable(false); - this.setLocation(parentWindow.getX()+parentWindow.getWidth()/2, parentWindow.getY()+100); - - contentPane.setLayout(layout); - sourceSearchField = new JTextField(30); - sourceSearchField.setName("Source"); - - targetSearchField = new JTextField(30); - targetSearchField.setName("Target"); - - - // Add key listener to send search string as it's being typed - sourceSearchField.addKeyListener(new SearchListener() ); - sourceSearchField.addFocusListener(new SearchFocusListener()); - JLabel sourceLabel = new JLabel("Filter Source:",JLabel.TRAILING); - JButton sourceClearBtn = new JButton("Clear"); - contentPane.add(sourceLabel); - contentPane.add(sourceSearchField); - sourceClearBtn.addActionListener(this); - sourceClearBtn.setActionCommand("Clear Source"); - sourceClearBtn.setFocusable(false); - contentPane.add(sourceClearBtn); - - targetSearchField.addKeyListener(new SearchListener() ); - targetSearchField.addFocusListener(new SearchFocusListener()); - JLabel targetLabel = new JLabel("Filter Target:",JLabel.TRAILING); - JButton targetClearBtn = new JButton("Clear"); - contentPane.add(targetLabel); - contentPane.add(targetSearchField); - targetClearBtn.addActionListener(this); - targetClearBtn.setActionCommand("Clear Target"); - targetClearBtn.setFocusable(false); - contentPane.add(targetClearBtn); - - layout.putConstraint(SpringLayout.WEST, sourceLabel, 5, SpringLayout.WEST, contentPane); - layout.putConstraint(SpringLayout.NORTH, sourceLabel, 5, SpringLayout.NORTH, contentPane); - - layout.putConstraint(SpringLayout.WEST, sourceSearchField, 5, SpringLayout.EAST, sourceLabel); - layout.putConstraint(SpringLayout.NORTH, sourceSearchField, 5, SpringLayout.NORTH, contentPane); - - layout.putConstraint(SpringLayout.WEST, sourceClearBtn, 5, SpringLayout.EAST, sourceSearchField); - layout.putConstraint(SpringLayout.NORTH, sourceClearBtn, 5, SpringLayout.NORTH, contentPane); - - layout.putConstraint(SpringLayout.WEST, targetLabel, 5, SpringLayout.WEST, contentPane); - layout.putConstraint(SpringLayout.NORTH, targetLabel, 10, SpringLayout.SOUTH, sourceLabel); - - layout.putConstraint(SpringLayout.WEST, targetSearchField, 0, SpringLayout.WEST, sourceSearchField); - layout.putConstraint(SpringLayout.NORTH, targetSearchField, 0, SpringLayout.NORTH, targetLabel); - - layout.putConstraint(SpringLayout.WEST, targetClearBtn, 5, SpringLayout.EAST, targetSearchField); - layout.putConstraint(SpringLayout.NORTH, targetClearBtn, 0, SpringLayout.NORTH, targetSearchField); - - - layout.putConstraint(SpringLayout.SOUTH, contentPane, 5, SpringLayout.SOUTH, targetLabel); - layout.putConstraint(SpringLayout.NORTH, contentPane, 5, SpringLayout.NORTH, sourceLabel); - layout.putConstraint(SpringLayout.WEST, contentPane, 5, SpringLayout.WEST, sourceLabel); - layout.putConstraint(SpringLayout.EAST, contentPane, 5, SpringLayout.EAST, targetClearBtn); - - this.pack(); - }; - - public void setFilterPanel(MappingPanel aFilterPanel){ - if (filterPanel != null) { - filterPanel.removeResizeListener(this); - } - - filterPanel = aFilterPanel; - - if (filterPanel != null) { - aFilterPanel.addResizeListener(this); - } - - setSearchFieldsToLastSearch(); - } - - public MappingPanel getFilterPanel(){ - if(filterPanel.isMinimized()){ - return filterPanel.getSlaveMappingPanel(); - }else{ - return filterPanel; - } - } - - public void doFilterPanel(String str,String panelName){ - getFilterPanel().filterComponents(str,panelName=="Target"); - } - - @Override - public void actionPerformed(ActionEvent event) { - switch (event.getActionCommand()) { - case "Clear Source": - clearSourceFilter(); - break; - - case "Clear Target": - clearTargetFilter(); - break; - - } - - } - - private void clearTargetFilter() { - targetSearchField.setText(""); - doFilterPanel("","Target"); - } - - private void clearSourceFilter() { - sourceSearchField.setText(""); - doFilterPanel("","Source"); - } - - private void setSearchFieldsToLastSearch(){ - sourceSearchField.setText(getFilterPanel().getLastSourceFilter()); - targetSearchField.setText(getFilterPanel().getLastTargetFilter()); - } - public void notifyResized(int height, boolean minimized, boolean maximized) { - setSearchFieldsToLastSearch(); - } - - class SearchListener implements KeyListener{ - private String searchStrBuffer = ""; - - @Override - public void keyPressed(KeyEvent event) { - - } - - @Override - public void keyReleased(KeyEvent event) { - - switch (event.getComponent().getName()) { - case "Source": - searchStrBuffer = sourceSearchField.getText(); - break; - case "Target": - searchStrBuffer = targetSearchField.getText(); - break; - } - - doFilterPanel(searchStrBuffer,event.getComponent().getName()); - } - - @Override - public void keyTyped(KeyEvent event) { - - } - - } - - public class SearchFocusListener implements FocusListener { - - @Override - public void focusGained(FocusEvent e) { - // TODO Auto-generated method stub - JTextField jtf = (JTextField) e.getComponent(); - jtf.selectAll(); - } - - @Override - public void focusLost(FocusEvent e) { - // TODO Auto-generated method stub - - } - - } - - -} - +package org.ohdsi.rabbitInAHat; + +import java.awt.Container; +import java.awt.Window; +import java.awt.event.ActionEvent; +import java.awt.event.ActionListener; +import java.awt.event.FocusEvent; +import java.awt.event.FocusListener; +import java.awt.event.KeyEvent; +import java.awt.event.KeyListener; + +import org.ohdsi.rabbitInAHat.ResizeListener; + +import javax.swing.JButton; +import javax.swing.JDialog; +import javax.swing.JLabel; +import javax.swing.JTextField; +import javax.swing.SpringLayout; + +public class FilterDialog extends JDialog implements ActionListener, ResizeListener { + + + /** + * + */ + private static final long serialVersionUID = 7009265246652874341L; + + + private JTextField sourceSearchField; + private JTextField targetSearchField; + private MappingPanel filterPanel; + + SpringLayout layout = new SpringLayout(); + + Container contentPane = this.getContentPane(); + + public FilterDialog(Window parentWindow){ + + super(parentWindow,"Filter",ModalityType.MODELESS); + this.setResizable(false); + this.setLocation(parentWindow.getX()+parentWindow.getWidth()/2, parentWindow.getY()+100); + + contentPane.setLayout(layout); + sourceSearchField = new JTextField(30); + sourceSearchField.setName("Source"); + + targetSearchField = new JTextField(30); + targetSearchField.setName("Target"); + + + // Add key listener to send search string as it's being typed + sourceSearchField.addKeyListener(new SearchListener() ); + sourceSearchField.addFocusListener(new SearchFocusListener()); + JLabel sourceLabel = new JLabel("Filter Source:",JLabel.TRAILING); + JButton sourceClearBtn = new JButton("Clear"); + contentPane.add(sourceLabel); + contentPane.add(sourceSearchField); + sourceClearBtn.addActionListener(this); + sourceClearBtn.setActionCommand("Clear Source"); + sourceClearBtn.setFocusable(false); + contentPane.add(sourceClearBtn); + + targetSearchField.addKeyListener(new SearchListener() ); + targetSearchField.addFocusListener(new SearchFocusListener()); + JLabel targetLabel = new JLabel("Filter Target:",JLabel.TRAILING); + JButton targetClearBtn = new JButton("Clear"); + contentPane.add(targetLabel); + contentPane.add(targetSearchField); + targetClearBtn.addActionListener(this); + targetClearBtn.setActionCommand("Clear Target"); + targetClearBtn.setFocusable(false); + contentPane.add(targetClearBtn); + + layout.putConstraint(SpringLayout.WEST, sourceLabel, 5, SpringLayout.WEST, contentPane); + layout.putConstraint(SpringLayout.NORTH, sourceLabel, 5, SpringLayout.NORTH, contentPane); + + layout.putConstraint(SpringLayout.WEST, sourceSearchField, 5, SpringLayout.EAST, sourceLabel); + layout.putConstraint(SpringLayout.NORTH, sourceSearchField, 5, SpringLayout.NORTH, contentPane); + + layout.putConstraint(SpringLayout.WEST, sourceClearBtn, 5, SpringLayout.EAST, sourceSearchField); + layout.putConstraint(SpringLayout.NORTH, sourceClearBtn, 5, SpringLayout.NORTH, contentPane); + + layout.putConstraint(SpringLayout.WEST, targetLabel, 5, SpringLayout.WEST, contentPane); + layout.putConstraint(SpringLayout.NORTH, targetLabel, 10, SpringLayout.SOUTH, sourceLabel); + + layout.putConstraint(SpringLayout.WEST, targetSearchField, 0, SpringLayout.WEST, sourceSearchField); + layout.putConstraint(SpringLayout.NORTH, targetSearchField, 0, SpringLayout.NORTH, targetLabel); + + layout.putConstraint(SpringLayout.WEST, targetClearBtn, 5, SpringLayout.EAST, targetSearchField); + layout.putConstraint(SpringLayout.NORTH, targetClearBtn, 0, SpringLayout.NORTH, targetSearchField); + + + layout.putConstraint(SpringLayout.SOUTH, contentPane, 5, SpringLayout.SOUTH, targetLabel); + layout.putConstraint(SpringLayout.NORTH, contentPane, 5, SpringLayout.NORTH, sourceLabel); + layout.putConstraint(SpringLayout.WEST, contentPane, 5, SpringLayout.WEST, sourceLabel); + layout.putConstraint(SpringLayout.EAST, contentPane, 5, SpringLayout.EAST, targetClearBtn); + + this.pack(); + }; + + public void setFilterPanel(MappingPanel aFilterPanel){ + if (filterPanel != null) { + filterPanel.removeResizeListener(this); + } + + filterPanel = aFilterPanel; + + if (filterPanel != null) { + aFilterPanel.addResizeListener(this); + } + + setSearchFieldsToLastSearch(); + } + + public MappingPanel getFilterPanel(){ + if(filterPanel.isMinimized()){ + return filterPanel.getSlaveMappingPanel(); + }else{ + return filterPanel; + } + } + + public void doFilterPanel(String str,String panelName){ + getFilterPanel().filterComponents(str,panelName=="Target"); + } + + @Override + public void actionPerformed(ActionEvent event) { + switch (event.getActionCommand()) { + case "Clear Source": + clearSourceFilter(); + break; + + case "Clear Target": + clearTargetFilter(); + break; + + } + + } + + private void clearTargetFilter() { + targetSearchField.setText(""); + doFilterPanel("","Target"); + } + + private void clearSourceFilter() { + sourceSearchField.setText(""); + doFilterPanel("","Source"); + } + + private void setSearchFieldsToLastSearch(){ + sourceSearchField.setText(getFilterPanel().getLastSourceFilter()); + targetSearchField.setText(getFilterPanel().getLastTargetFilter()); + } + public void notifyResized(int height, boolean minimized, boolean maximized) { + setSearchFieldsToLastSearch(); + } + + class SearchListener implements KeyListener{ + private String searchStrBuffer = ""; + + @Override + public void keyPressed(KeyEvent event) { + + } + + @Override + public void keyReleased(KeyEvent event) { + + switch (event.getComponent().getName()) { + case "Source": + searchStrBuffer = sourceSearchField.getText(); + break; + case "Target": + searchStrBuffer = targetSearchField.getText(); + break; + } + + doFilterPanel(searchStrBuffer,event.getComponent().getName()); + } + + @Override + public void keyTyped(KeyEvent event) { + + } + + } + + public class SearchFocusListener implements FocusListener { + + @Override + public void focusGained(FocusEvent e) { + // TODO Auto-generated method stub + JTextField jtf = (JTextField) e.getComponent(); + jtf.selectAll(); + } + + @Override + public void focusLost(FocusEvent e) { + // TODO Auto-generated method stub + + } + + } + + +} + diff --git a/src/org/ohdsi/rabbitInAHat/FlexTable.java b/rabbitinahat/src/main/java/org/ohdsi/rabbitInAHat/FlexTable.java similarity index 96% rename from src/org/ohdsi/rabbitInAHat/FlexTable.java rename to rabbitinahat/src/main/java/org/ohdsi/rabbitInAHat/FlexTable.java index 3fb5bcfe..558fde2f 100644 --- a/src/org/ohdsi/rabbitInAHat/FlexTable.java +++ b/rabbitinahat/src/main/java/org/ohdsi/rabbitInAHat/FlexTable.java @@ -1,5 +1,5 @@ /******************************************************************************* - * Copyright 2017 Observational Health Data Sciences and Informatics + * Copyright 2019 Observational Health Data Sciences and Informatics * * This file is part of WhiteRabbit * diff --git a/src/org/ohdsi/rabbitInAHat/LabeledRectangle.java b/rabbitinahat/src/main/java/org/ohdsi/rabbitInAHat/LabeledRectangle.java similarity index 95% rename from src/org/ohdsi/rabbitInAHat/LabeledRectangle.java rename to rabbitinahat/src/main/java/org/ohdsi/rabbitInAHat/LabeledRectangle.java index 184f83b5..14f6f6d7 100644 --- a/src/org/ohdsi/rabbitInAHat/LabeledRectangle.java +++ b/rabbitinahat/src/main/java/org/ohdsi/rabbitInAHat/LabeledRectangle.java @@ -1,200 +1,200 @@ -/******************************************************************************* - * Copyright 2017 Observational Health Data Sciences and Informatics - * - * This file is part of WhiteRabbit - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - ******************************************************************************/ -package org.ohdsi.rabbitInAHat; - -import java.awt.BasicStroke; -import java.awt.Color; -import java.awt.Font; -import java.awt.FontMetrics; -import java.awt.Graphics; -import java.awt.Graphics2D; -import java.awt.Point; -import java.awt.Stroke; -import java.awt.geom.Rectangle2D; -import java.util.ArrayList; -import java.util.List; - -import javax.swing.event.ChangeListener; - -import org.ohdsi.rabbitInAHat.dataModel.MappableItem; - -public class LabeledRectangle implements MappingComponent { - - public static int FONT_SIZE = 18; - - private static Stroke stroke = new BasicStroke(2); - private static BasicStroke dashed = new BasicStroke(2.0f, BasicStroke.CAP_BUTT, BasicStroke.JOIN_MITER, 10.0f, new float[] { 10.f }, 0.0f); - private List changeListeners = new ArrayList(); - private int x; - private int y; - private int width; - private int height; - private MappableItem item; - private Color baseColor; - private Color transparentColor; - private boolean isVisible = true; - private boolean isSelected = false; - - public void addChangeListener(ChangeListener x) { - changeListeners.add(x); - } - - public void removeChangeListener(ChangeListener x) { - changeListeners.remove(x); - } - - public LabeledRectangle(int x, int y, int width, int height, MappableItem item, Color baseColor) { - this.x = x; - this.y = y; - this.width = width; - this.height = height; - this.item = item; - this.baseColor = baseColor; - this.transparentColor = new Color(baseColor.getRed(), baseColor.getGreen(), baseColor.getBlue(), 128); - } - - public boolean isVisible(){ - return isVisible == true; - } - - public int getWidth() { - return width; - } - - public int getHeight() { - return height; - } - - public void setLocation(int x, int y) { - this.x = x; - this.y = y; - } - - public void filter(String searchTerm){ - if (this.getItem().getName().matches(".*(" + searchTerm + ").*") || searchTerm.equals("") ){ - this.setVisible(true); - }else{ - this.setVisible(false); - this.setSelected(false); - } - } - - public void paint(Graphics g) { - if (!isVisible) - return; - - Graphics2D g2d = (Graphics2D) g.create(); - - g2d.setColor(transparentColor); - g2d.fillRect(x, y, width, height); - if (isSelected) { - g2d.setColor(Color.BLACK); - g2d.setStroke(dashed); - } else { - g2d.setColor(baseColor); - g2d.setStroke(stroke); - } - g2d.drawRect(x, y, width, height); - g2d.setColor(Color.BLACK); - - g2d.setFont(new Font("default", Font.PLAIN, FONT_SIZE)); - FontMetrics fm = g2d.getFontMetrics(); - - Rectangle2D r = fm.getStringBounds(item.outputName(), g2d); - if (r.getWidth() >= width) { - int breakPoint = 0; - int index = nextBreakPoint(item.outputName(), 0); - double midPoint = item.outputName().length() / 2d; - while (index != -1) { - if (Math.abs(index - midPoint) < Math.abs(breakPoint - midPoint)) - breakPoint = index; - index = nextBreakPoint(item.outputName(), index + 1); - } - if (breakPoint == 0) { - int textX = (this.getWidth() - (int) r.getWidth()) / 2; - int textY = (this.getHeight() - (int) r.getHeight()) / 2 + fm.getAscent(); - g2d.drawString(item.outputName(), x + textX, y + textY); - } - breakPoint++; - String line1 = item.outputName().substring(0, breakPoint); - String line2 = item.outputName().substring(breakPoint); - r = fm.getStringBounds(line1, g2d); - int textX = (this.getWidth() - (int) r.getWidth()) / 2; - int textY = (this.getHeight() / 2 - (int) r.getHeight()) / 2 + fm.getAscent(); - g2d.drawString(line1, x + textX, y + textY); - r = fm.getStringBounds(line2, g2d); - textX = (this.getWidth() - (int) r.getWidth()) / 2; - textY = (int) Math.round(this.getHeight() * 1.5 - (int) r.getHeight()) / 2 + fm.getAscent(); - g2d.drawString(line2, x + textX, y + textY); - } else { - int textX = (this.getWidth() - (int) r.getWidth()) / 2; - int textY = (this.getHeight() - (int) r.getHeight()) / 2 + fm.getAscent(); - g2d.drawString(item.outputName(), x + textX, y + textY); - } - } - - private int nextBreakPoint(String string, int start) { - int index1 = string.indexOf(' ', start); - int index2 = string.indexOf('_', start); - if (index1 == -1) - return index2; - else if (index2 == -1) - return index1; - else - return Math.min(index1, index2); - } - - public boolean contains(Point point) { - return (point.x >= x && point.x <= x + width && point.y >= y && point.y <= y + height); - } - - public boolean contains(Point point, int xOffset, int yOffset) { - Point p = new Point(point.x + xOffset, point.y + yOffset); - return contains(p); - } - - public int getX() { - return x; - } - - public int getY() { - return y; - } - - public MappableItem getItem() { - return item; - } - - public boolean isSelected() { - return isSelected; - } - - public void setSelected(boolean isSelected) { - this.isSelected = isSelected; - } - - public boolean toggleSelected(){ - this.isSelected = !this.isSelected; - return isSelected; - } - - public void setVisible(boolean value) { - isVisible = value; - } - -} +/******************************************************************************* + * Copyright 2019 Observational Health Data Sciences and Informatics + * + * This file is part of WhiteRabbit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + ******************************************************************************/ +package org.ohdsi.rabbitInAHat; + +import java.awt.BasicStroke; +import java.awt.Color; +import java.awt.Font; +import java.awt.FontMetrics; +import java.awt.Graphics; +import java.awt.Graphics2D; +import java.awt.Point; +import java.awt.Stroke; +import java.awt.geom.Rectangle2D; +import java.util.ArrayList; +import java.util.List; + +import javax.swing.event.ChangeListener; + +import org.ohdsi.rabbitInAHat.dataModel.MappableItem; + +public class LabeledRectangle implements MappingComponent { + + public static int FONT_SIZE = 18; + + private static Stroke stroke = new BasicStroke(2); + private static BasicStroke dashed = new BasicStroke(2.0f, BasicStroke.CAP_BUTT, BasicStroke.JOIN_MITER, 10.0f, new float[] { 10.f }, 0.0f); + private List changeListeners = new ArrayList(); + private int x; + private int y; + private int width; + private int height; + private MappableItem item; + private Color baseColor; + private Color transparentColor; + private boolean isVisible = true; + private boolean isSelected = false; + + public void addChangeListener(ChangeListener x) { + changeListeners.add(x); + } + + public void removeChangeListener(ChangeListener x) { + changeListeners.remove(x); + } + + public LabeledRectangle(int x, int y, int width, int height, MappableItem item, Color baseColor) { + this.x = x; + this.y = y; + this.width = width; + this.height = height; + this.item = item; + this.baseColor = baseColor; + this.transparentColor = new Color(baseColor.getRed(), baseColor.getGreen(), baseColor.getBlue(), 128); + } + + public boolean isVisible(){ + return isVisible == true; + } + + public int getWidth() { + return width; + } + + public int getHeight() { + return height; + } + + public void setLocation(int x, int y) { + this.x = x; + this.y = y; + } + + public void filter(String searchTerm){ + if (this.getItem().getName().matches(".*(" + searchTerm + ").*") || searchTerm.equals("") ){ + this.setVisible(true); + }else{ + this.setVisible(false); + this.setSelected(false); + } + } + + public void paint(Graphics g) { + if (!isVisible) + return; + + Graphics2D g2d = (Graphics2D) g.create(); + + g2d.setColor(transparentColor); + g2d.fillRect(x, y, width, height); + if (isSelected) { + g2d.setColor(Color.BLACK); + g2d.setStroke(dashed); + } else { + g2d.setColor(baseColor); + g2d.setStroke(stroke); + } + g2d.drawRect(x, y, width, height); + g2d.setColor(Color.BLACK); + + g2d.setFont(new Font("default", Font.PLAIN, FONT_SIZE)); + FontMetrics fm = g2d.getFontMetrics(); + + Rectangle2D r = fm.getStringBounds(item.outputName(), g2d); + if (r.getWidth() >= width) { + int breakPoint = 0; + int index = nextBreakPoint(item.outputName(), 0); + double midPoint = item.outputName().length() / 2d; + while (index != -1) { + if (Math.abs(index - midPoint) < Math.abs(breakPoint - midPoint)) + breakPoint = index; + index = nextBreakPoint(item.outputName(), index + 1); + } + if (breakPoint == 0) { + int textX = (this.getWidth() - (int) r.getWidth()) / 2; + int textY = (this.getHeight() - (int) r.getHeight()) / 2 + fm.getAscent(); + g2d.drawString(item.outputName(), x + textX, y + textY); + } + breakPoint++; + String line1 = item.outputName().substring(0, breakPoint); + String line2 = item.outputName().substring(breakPoint); + r = fm.getStringBounds(line1, g2d); + int textX = (this.getWidth() - (int) r.getWidth()) / 2; + int textY = (this.getHeight() / 2 - (int) r.getHeight()) / 2 + fm.getAscent(); + g2d.drawString(line1, x + textX, y + textY); + r = fm.getStringBounds(line2, g2d); + textX = (this.getWidth() - (int) r.getWidth()) / 2; + textY = (int) Math.round(this.getHeight() * 1.5 - (int) r.getHeight()) / 2 + fm.getAscent(); + g2d.drawString(line2, x + textX, y + textY); + } else { + int textX = (this.getWidth() - (int) r.getWidth()) / 2; + int textY = (this.getHeight() - (int) r.getHeight()) / 2 + fm.getAscent(); + g2d.drawString(item.outputName(), x + textX, y + textY); + } + } + + private int nextBreakPoint(String string, int start) { + int index1 = string.indexOf(' ', start); + int index2 = string.indexOf('_', start); + if (index1 == -1) + return index2; + else if (index2 == -1) + return index1; + else + return Math.min(index1, index2); + } + + public boolean contains(Point point) { + return (point.x >= x && point.x <= x + width && point.y >= y && point.y <= y + height); + } + + public boolean contains(Point point, int xOffset, int yOffset) { + Point p = new Point(point.x + xOffset, point.y + yOffset); + return contains(p); + } + + public int getX() { + return x; + } + + public int getY() { + return y; + } + + public MappableItem getItem() { + return item; + } + + public boolean isSelected() { + return isSelected; + } + + public void setSelected(boolean isSelected) { + this.isSelected = isSelected; + } + + public boolean toggleSelected(){ + this.isSelected = !this.isSelected; + return isSelected; + } + + public void setVisible(boolean value) { + isVisible = value; + } + +} diff --git a/src/org/ohdsi/rabbitInAHat/MappingComponent.java b/rabbitinahat/src/main/java/org/ohdsi/rabbitInAHat/MappingComponent.java similarity index 91% rename from src/org/ohdsi/rabbitInAHat/MappingComponent.java rename to rabbitinahat/src/main/java/org/ohdsi/rabbitInAHat/MappingComponent.java index 6170210c..7ffb29e5 100644 --- a/src/org/ohdsi/rabbitInAHat/MappingComponent.java +++ b/rabbitinahat/src/main/java/org/ohdsi/rabbitInAHat/MappingComponent.java @@ -1,5 +1,5 @@ /******************************************************************************* - * Copyright 2017 Observational Health Data Sciences and Informatics + * Copyright 2019 Observational Health Data Sciences and Informatics * * This file is part of WhiteRabbit * diff --git a/src/org/ohdsi/rabbitInAHat/MappingPanel.java b/rabbitinahat/src/main/java/org/ohdsi/rabbitInAHat/MappingPanel.java similarity index 96% rename from src/org/ohdsi/rabbitInAHat/MappingPanel.java rename to rabbitinahat/src/main/java/org/ohdsi/rabbitInAHat/MappingPanel.java index 02194870..97628516 100644 --- a/src/org/ohdsi/rabbitInAHat/MappingPanel.java +++ b/rabbitinahat/src/main/java/org/ohdsi/rabbitInAHat/MappingPanel.java @@ -1,5 +1,5 @@ /******************************************************************************* - * Copyright 2017 Observational Health Data Sciences and Informatics + * Copyright 2019 Observational Health Data Sciences and Informatics * * This file is part of WhiteRabbit * @@ -51,7 +51,6 @@ import org.ohdsi.rabbitInAHat.dataModel.Mapping; import org.ohdsi.rabbitInAHat.dataModel.Table; import org.ohdsi.utilities.collections.IntegerComparator; -import org.ohdsi.whiteRabbit.ObjectExchange; public class MappingPanel extends JPanel implements MouseListener, MouseMotionListener { @@ -275,7 +274,7 @@ public void setSize(int width, int height) { public void filterComponents(String searchTerm, boolean filterTarget) { List components; - if (filterTarget == true) { + if (filterTarget) { components = cdmComponents; lastTargetFilter = searchTerm; } else { diff --git a/rabbitinahat/src/main/java/org/ohdsi/rabbitInAHat/ObjectExchange.java b/rabbitinahat/src/main/java/org/ohdsi/rabbitInAHat/ObjectExchange.java new file mode 100644 index 00000000..9b6cc374 --- /dev/null +++ b/rabbitinahat/src/main/java/org/ohdsi/rabbitInAHat/ObjectExchange.java @@ -0,0 +1,32 @@ +/******************************************************************************* + * Copyright 2019 Observational Health Data Sciences and Informatics + * + * This file is part of WhiteRabbit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + ******************************************************************************/ +package org.ohdsi.rabbitInAHat; + +import org.ohdsi.rabbitInAHat.dataModel.ETL; + +import javax.swing.*; + +/** + * This class is used to hold global variables + */ +public final class ObjectExchange { + // utility class + private ObjectExchange() {} + + public static ETL etl; +} diff --git a/src/org/ohdsi/rabbitInAHat/RabbitInAHatMain.java b/rabbitinahat/src/main/java/org/ohdsi/rabbitInAHat/RabbitInAHatMain.java similarity index 63% rename from src/org/ohdsi/rabbitInAHat/RabbitInAHatMain.java rename to rabbitinahat/src/main/java/org/ohdsi/rabbitInAHat/RabbitInAHatMain.java index f15ca2da..6b759b3c 100644 --- a/src/org/ohdsi/rabbitInAHat/RabbitInAHatMain.java +++ b/rabbitinahat/src/main/java/org/ohdsi/rabbitInAHat/RabbitInAHatMain.java @@ -1,677 +1,779 @@ -/******************************************************************************* - * Copyright 2017 Observational Health Data Sciences and Informatics - * - * This file is part of WhiteRabbit - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - ******************************************************************************/ -package org.ohdsi.rabbitInAHat; - -import java.awt.Color; -import java.awt.Cursor; -import java.awt.Desktop; -import java.awt.Dimension; -import java.awt.Image; -import java.awt.MediaTracker; -import java.awt.Toolkit; -import java.awt.event.ActionEvent; -import java.awt.event.ActionListener; -import java.awt.event.KeyEvent; -import java.awt.event.WindowAdapter; -import java.awt.event.WindowEvent; -import java.io.File; -import java.io.FileInputStream; -import java.io.IOException; -import java.io.InputStream; -import java.net.URI; -import java.net.URISyntaxException; -import java.util.ArrayList; -import java.util.List; - -import javax.swing.BoxLayout; -import javax.swing.ButtonGroup; -import javax.swing.JFileChooser; -import javax.swing.JFrame; -import javax.swing.JMenu; -import javax.swing.JMenuBar; -import javax.swing.JMenuItem; -import javax.swing.JOptionPane; -import javax.swing.JRadioButtonMenuItem; -import javax.swing.JScrollPane; -import javax.swing.JSplitPane; -import javax.swing.KeyStroke; -import javax.swing.ScrollPaneConstants; -import javax.swing.UIManager; -import javax.swing.border.TitledBorder; -import javax.swing.filechooser.FileFilter; -import javax.swing.filechooser.FileNameExtensionFilter; - -import org.ohdsi.rabbitInAHat.dataModel.Database; -import org.ohdsi.rabbitInAHat.dataModel.Database.CDMVersion; -import org.ohdsi.rabbitInAHat.dataModel.ETL; -import org.ohdsi.rabbitInAHat.dataModel.Field; -import org.ohdsi.rabbitInAHat.dataModel.MappableItem; -import org.ohdsi.rabbitInAHat.dataModel.StemTableAdd; -import org.ohdsi.rabbitInAHat.dataModel.Table; -import org.ohdsi.whiteRabbit.ObjectExchange; - -/** - * This is the main class for the RabbitInAHat application - */ -public class RabbitInAHatMain implements ResizeListener, ActionListener { - - public final static String ACTION_CMD_SAVE = "Save"; - public final static String ACTION_CMD_SAVE_AS = "Save As"; - public final static String ACTION_CMD_OPEN_ETL_SPECS = "Open ETL Specs"; - public final static String ACTION_CMD_OPEN_SCAN_REPORT = "Open Scan Report"; - public final static String ACTION_CMD_GENERATE_ETL_DOCUMENT = "Generate ETL Document"; - public final static String ACTION_CMD_GENERATE_TEST_FRAMEWORK = "Generate ETL Test Framework"; - public final static String ACTION_CMD_DISCARD_COUNTS = "Discard Value Counts"; - public final static String ACTION_CMD_FILTER = "Filter"; - public final static String ACTION_CMD_MAKE_MAPPING = "Make Mappings"; - public final static String ACTION_CMD_REMOVE_MAPPING = "Remove Mappings"; - public final static String ACTION_CMD_SET_TARGET_V4 = "CDM v4"; - public final static String ACTION_CMD_SET_TARGET_V5 = "CDM v5.0.0"; - public final static String ACTION_CMD_SET_TARGET_V501 = "CDM v5.0.1"; - public final static String ACTION_CMD_SET_TARGET_V510 = "CDM v5.1.0"; - public final static String ACTION_CMD_SET_TARGET_V520 = "CDM v5.2.0"; - public final static String ACTION_CMD_SET_TARGET_V530 = "CDM v5.3.0"; - public final static String ACTION_CMD_SET_TARGET_V531 = "CDM v5.3.1"; - public final static String ACTION_ADD_STEM_TABLE = "Add stem table"; - public final static String ACTION_CMD_SET_TARGET_CUSTOM = "Load Custom..."; - public final static String ACTION_CMD_MARK_COMPLETED = "Mark Highlighted As Complete"; - public final static String ACTION_CMD_UNMARK_COMPLETED = "Mark Highlighted As Incomplete"; - public final static String ACTION_CMD_HELP = "Open help Wiki"; - - public final static String WIKI_URL = "http://www.ohdsi.org/web/wiki/doku.php?id=documentation:software:whiterabbit#rabbit-in-a-hat"; - private final static FileFilter FILE_FILTER_GZ = new FileNameExtensionFilter("GZIP Files (*.gz)", "gz"); - private final static FileFilter FILE_FILTER_JSON = new FileNameExtensionFilter("JSON Files (*.json)", "json"); - private final static FileFilter FILE_FILTER_DOCX = new FileNameExtensionFilter("Microsoft Word documents (*.docx)", "docx"); - private final static FileFilter FILE_FILTER_CSV = new FileNameExtensionFilter("Text Files (*.csv)", "csv"); - private final static FileFilter FILE_FILTER_R = new FileNameExtensionFilter("R script (*.r)", "r"); - private final static FileFilter FILE_FILTER_XLSX = new FileNameExtensionFilter("XLSX files (*.xlsx)", "xlsx"); - - public final static String DBMS_SQLSERVER = "SQL Server"; - public final static String DBMS_REDSHIFT = "Redshift"; - - private JFrame frame; - private JScrollPane scrollPane1; - private JScrollPane scrollPane2; - private MappingPanel tableMappingPanel; - private MappingPanel fieldMappingPanel; - private DetailsPanel detailsPanel; - private JSplitPane tableFieldSplitPane; - private JFileChooser chooser; - - public static void main(String[] args) { - new RabbitInAHatMain(args); - } - - public RabbitInAHatMain(String[] args) { - - // Set look and feel to the system look and feel - try { - UIManager.setLookAndFeel(UIManager.getSystemLookAndFeelClassName()); - } catch (Exception ex) { - ex.printStackTrace(); - } - - frame = new JFrame("Rabbit in a Hat"); - frame.addWindowListener(new WindowAdapter() { - public void windowClosing(WindowEvent e) { - System.exit(0); - } - }); - frame.setPreferredSize(new Dimension(700, 600)); - frame.setLayout(new BoxLayout(frame.getContentPane(), BoxLayout.Y_AXIS)); - frame.setJMenuBar(createMenuBar()); - - ETL etl = new ETL(); - etl.setTargetDatabase(Database.generateCDMModel(CDMVersion.CDMV531)); - - ObjectExchange.etl = etl; - - tableMappingPanel = new MappingPanel(etl.getTableToTableMapping()); - tableMappingPanel.addResizeListener(this); - scrollPane1 = new JScrollPane(tableMappingPanel); - scrollPane1.setBorder(new TitledBorder("Tables")); - scrollPane1.getVerticalScrollBar().setUnitIncrement(16); - scrollPane1.setAutoscrolls(true); - scrollPane1.setOpaque(true); - scrollPane1.setBackground(Color.WHITE); - - fieldMappingPanel = new MappingPanel(etl.getTableToTableMapping()); - tableMappingPanel.setSlaveMappingPanel(fieldMappingPanel); - fieldMappingPanel.addResizeListener(this); - scrollPane2 = new JScrollPane(fieldMappingPanel); - scrollPane2.getVerticalScrollBar().setUnitIncrement(16); - scrollPane2.setVisible(false); - scrollPane2.setBorder(new TitledBorder("Fields")); - - tableFieldSplitPane = new JSplitPane(JSplitPane.VERTICAL_SPLIT, scrollPane1, scrollPane2); - tableFieldSplitPane.setDividerLocation(600); - tableFieldSplitPane.setDividerSize(0); - - detailsPanel = new DetailsPanel(); - detailsPanel.setBorder(new TitledBorder("Details")); - detailsPanel.setPreferredSize(new Dimension(200, 500)); - detailsPanel.setMinimumSize(new Dimension(0, 0)); - tableMappingPanel.setDetailsListener(detailsPanel); - fieldMappingPanel.setDetailsListener(detailsPanel); - JSplitPane leftRightSplinePane = new JSplitPane(JSplitPane.HORIZONTAL_SPLIT, tableFieldSplitPane, detailsPanel); - leftRightSplinePane.setResizeWeight(0.40); - frame.add(leftRightSplinePane); - - loadIcons(frame); - frame.pack(); - frame.setExtendedState(java.awt.Frame.MAXIMIZED_BOTH); - frame.setVisible(true); - - if (args.length == 1) { - doOpenSpecs(args[0]); - } - if (args.length > 1 && args[0].equals("-folder")) { - chooser = new JFileChooser(args[1]); - } - } - - private void loadIcons(JFrame f) { - List icons = new ArrayList(); - icons.add(loadIcon("RabbitInAHat16.png", f)); - icons.add(loadIcon("RabbitInAHat32.png", f)); - icons.add(loadIcon("RabbitInAHat48.png", f)); - icons.add(loadIcon("RabbitInAHat64.png", f)); - icons.add(loadIcon("RabbitInAHat128.png", f)); - icons.add(loadIcon("RabbitInAHat256.png", f)); - f.setIconImages(icons); - } - - private Image loadIcon(String name, JFrame f) { - Image icon = Toolkit.getDefaultToolkit().getImage(RabbitInAHatMain.class.getResource(name)); - MediaTracker mediaTracker = new MediaTracker(f); - mediaTracker.addImage(icon, 0); - try { - mediaTracker.waitForID(0); - return icon; - } catch (Exception e1) { - e1.printStackTrace(); - } - return null; - } - - private JMenuBar createMenuBar() { - JMenuBar menuBar = new JMenuBar(); - JMenu fileMenu = new JMenu("File"); - int menuShortcutMask = Toolkit.getDefaultToolkit().getMenuShortcutKeyMask(); - - menuBar.add(fileMenu); - - JMenuItem openScanReportItem = new JMenuItem(ACTION_CMD_OPEN_SCAN_REPORT); - openScanReportItem.addActionListener(this); - openScanReportItem.setActionCommand(ACTION_CMD_OPEN_SCAN_REPORT); - fileMenu.add(openScanReportItem); - - JMenuItem openItem = new JMenuItem(ACTION_CMD_OPEN_ETL_SPECS); - openItem.addActionListener(this); - openItem.setActionCommand(ACTION_CMD_OPEN_ETL_SPECS); - openItem.setAccelerator(KeyStroke.getKeyStroke(KeyEvent.VK_O, menuShortcutMask)); - fileMenu.add(openItem); - - JMenuItem saveItem = new JMenuItem(ACTION_CMD_SAVE); - saveItem.addActionListener(this); - saveItem.setActionCommand(ACTION_CMD_SAVE); - saveItem.setAccelerator(KeyStroke.getKeyStroke(KeyEvent.VK_S, menuShortcutMask)); - fileMenu.add(saveItem); - - JMenuItem saveAsItem = new JMenuItem(ACTION_CMD_SAVE_AS); - saveAsItem.addActionListener(this); - saveAsItem.setActionCommand(ACTION_CMD_SAVE_AS); - fileMenu.add(saveAsItem); - - JMenuItem generateDocItem = new JMenuItem(ACTION_CMD_GENERATE_ETL_DOCUMENT); - generateDocItem.addActionListener(this); - generateDocItem.setActionCommand(ACTION_CMD_GENERATE_ETL_DOCUMENT); - fileMenu.add(generateDocItem); - - JMenuItem generateTestFrameworkItem = new JMenuItem(ACTION_CMD_GENERATE_TEST_FRAMEWORK); - generateTestFrameworkItem.addActionListener(this); - generateTestFrameworkItem.setActionCommand(ACTION_CMD_GENERATE_TEST_FRAMEWORK); - fileMenu.add(generateTestFrameworkItem); - - JMenu editMenu = new JMenu("Edit"); - menuBar.add(editMenu); - - JMenuItem discardCounts = new JMenuItem(ACTION_CMD_DISCARD_COUNTS); - discardCounts.addActionListener(this); - discardCounts.setActionCommand(ACTION_CMD_DISCARD_COUNTS); - editMenu.add(discardCounts); - - JMenuItem filter = new JMenuItem(ACTION_CMD_FILTER); - filter.addActionListener(this); - filter.setActionCommand(ACTION_CMD_FILTER); - filter.setAccelerator(KeyStroke.getKeyStroke(KeyEvent.VK_F, menuShortcutMask)); - editMenu.add(filter); - - JMenu setTarget = new JMenu("Set Target Database"); - - JRadioButtonMenuItem targetCDMV4 = new JRadioButtonMenuItem(ACTION_CMD_SET_TARGET_V4); - targetCDMV4.addActionListener(this); - targetCDMV4.setActionCommand(ACTION_CMD_SET_TARGET_V4); - setTarget.add(targetCDMV4); - - JRadioButtonMenuItem targetCDMV5 = new JRadioButtonMenuItem(ACTION_CMD_SET_TARGET_V5); - targetCDMV5.addActionListener(this); - targetCDMV5.setActionCommand(ACTION_CMD_SET_TARGET_V5); - setTarget.add(targetCDMV5); - - JRadioButtonMenuItem targetCDMV501 = new JRadioButtonMenuItem(ACTION_CMD_SET_TARGET_V501); - targetCDMV501.addActionListener(this); - targetCDMV501.setActionCommand(ACTION_CMD_SET_TARGET_V501); - setTarget.add(targetCDMV501); - - JRadioButtonMenuItem targetCDMV510 = new JRadioButtonMenuItem(ACTION_CMD_SET_TARGET_V510); - targetCDMV510.addActionListener(this); - targetCDMV510.setActionCommand(ACTION_CMD_SET_TARGET_V510); - setTarget.add(targetCDMV510); - - JRadioButtonMenuItem targetCDMV520 = new JRadioButtonMenuItem(ACTION_CMD_SET_TARGET_V520); - targetCDMV520.addActionListener(this); - targetCDMV520.setActionCommand(ACTION_CMD_SET_TARGET_V520); - setTarget.add(targetCDMV520); - - JRadioButtonMenuItem targetCDMV530 = new JRadioButtonMenuItem(ACTION_CMD_SET_TARGET_V530); - targetCDMV530.addActionListener(this); - targetCDMV530.setActionCommand(ACTION_CMD_SET_TARGET_V530); - setTarget.add(targetCDMV530); - - JRadioButtonMenuItem targetCDMV531 = new JRadioButtonMenuItem(ACTION_CMD_SET_TARGET_V531, true); - targetCDMV531.addActionListener(this); - targetCDMV531.setActionCommand(ACTION_CMD_SET_TARGET_V531); - setTarget.add(targetCDMV531); - - JRadioButtonMenuItem loadTarget = new JRadioButtonMenuItem(ACTION_CMD_SET_TARGET_CUSTOM); - - loadTarget.addActionListener(this); - loadTarget.setActionCommand(ACTION_CMD_SET_TARGET_CUSTOM); - setTarget.add(loadTarget); - editMenu.add(setTarget); - - ButtonGroup targetGroup = new ButtonGroup(); - targetGroup.add(targetCDMV4); - targetGroup.add(targetCDMV5); - targetGroup.add(targetCDMV501); - targetGroup.add(targetCDMV510); - targetGroup.add(targetCDMV520); - targetGroup.add(targetCDMV530); - targetGroup.add(targetCDMV531); - targetGroup.add(loadTarget); - - JMenuItem addStemTable = new JMenuItem(ACTION_ADD_STEM_TABLE); - addStemTable.addActionListener(this); - addStemTable.setActionCommand(ACTION_ADD_STEM_TABLE); - editMenu.add(addStemTable); - - JMenu arrowMenu = new JMenu("Arrows"); - menuBar.add(arrowMenu); - - JMenuItem makeMappings = new JMenuItem(ACTION_CMD_MAKE_MAPPING); - makeMappings.addActionListener(this); - makeMappings.setActionCommand(ACTION_CMD_MAKE_MAPPING); - makeMappings.setAccelerator(KeyStroke.getKeyStroke(KeyEvent.VK_M, menuShortcutMask)); - arrowMenu.add(makeMappings); - - JMenuItem removeMappings = new JMenuItem(ACTION_CMD_REMOVE_MAPPING); - removeMappings.addActionListener(this); - removeMappings.setActionCommand(ACTION_CMD_REMOVE_MAPPING); - removeMappings.setAccelerator(KeyStroke.getKeyStroke(KeyEvent.VK_R, menuShortcutMask)); - arrowMenu.add(removeMappings); - - JMenuItem markCompleted = new JMenuItem(ACTION_CMD_MARK_COMPLETED); - markCompleted.addActionListener(this); - markCompleted.setActionCommand(ACTION_CMD_MARK_COMPLETED); - markCompleted.setAccelerator(KeyStroke.getKeyStroke(KeyEvent.VK_D, menuShortcutMask)); - arrowMenu.add(markCompleted); - - JMenuItem unmarkCompleted = new JMenuItem(ACTION_CMD_UNMARK_COMPLETED); - unmarkCompleted.addActionListener(this); - unmarkCompleted.setActionCommand(ACTION_CMD_UNMARK_COMPLETED); - unmarkCompleted.setAccelerator(KeyStroke.getKeyStroke(KeyEvent.VK_I, menuShortcutMask)); - arrowMenu.add(unmarkCompleted); - - // JMenu viewMenu = new JMenu("View"); - // menuBar.add(viewMenu); - - JMenu helpMenu = new JMenu("Help"); - menuBar.add(helpMenu); - JMenuItem helpItem = new JMenuItem(ACTION_CMD_HELP); - helpItem.addActionListener(this); - helpItem.setActionCommand(ACTION_CMD_HELP); - helpMenu.add(helpItem); - - return menuBar; - } - - @Override - public void notifyResized(int height, boolean minimized, boolean maximized) { - if (scrollPane2.isVisible() == maximized) - scrollPane2.setVisible(!maximized); - - if (!maximized) - scrollPane1.setVerticalScrollBarPolicy(ScrollPaneConstants.VERTICAL_SCROLLBAR_NEVER); - else - scrollPane1.setVerticalScrollBarPolicy(ScrollPaneConstants.VERTICAL_SCROLLBAR_AS_NEEDED); - - if (!minimized) - scrollPane2.setVerticalScrollBarPolicy(ScrollPaneConstants.VERTICAL_SCROLLBAR_NEVER); - else - scrollPane2.setVerticalScrollBarPolicy(ScrollPaneConstants.VERTICAL_SCROLLBAR_AS_NEEDED); - - tableFieldSplitPane.setDividerLocation(height); - } - - /** - * Display file chooser for a path to save or open to - * - * @param saveMode - * true to display a save dialog, false for open - * @param filter - * restrict files displayed - * @return if file selected, absolute path of selected file otherwise null - */ - private String choosePath(boolean saveMode, FileFilter... filter) { - String result = null; - - if (chooser == null) { - chooser = new JFileChooser(); - } - chooser.resetChoosableFileFilters(); - chooser.setFileFilter(filter[0]); - for (int i = 1; i < filter.length; i++) - chooser.addChoosableFileFilter(filter[i]); - - int dialogResult = saveMode ? chooser.showSaveDialog(frame) : chooser.showOpenDialog(frame); - if (dialogResult == JFileChooser.APPROVE_OPTION) - result = chooser.getSelectedFile().getAbsolutePath(); - return result; - } - - private String chooseSavePath(FileFilter... fileFilter) { - String path = choosePath(true, fileFilter); - if (path != null && fileFilter[0] == FILE_FILTER_GZ && !path.toLowerCase().endsWith(".json.gz") && !path.toLowerCase().endsWith(".json")) - path += ".json.gz"; - if (path != null && fileFilter[0] == FILE_FILTER_DOCX && !path.toLowerCase().endsWith(".docx")) - path += ".docx"; - return path; - } - - private String chooseOpenPath(FileFilter... fileFilter) { - return choosePath(false, fileFilter); - } - - @Override - public void actionPerformed(ActionEvent event) { - switch (event.getActionCommand()) { - case ACTION_CMD_SAVE: - String filename = ObjectExchange.etl.getFilename(); - doSave((filename == null || !filename.toLowerCase().endsWith(".json.gz")) ? chooseSavePath(FILE_FILTER_GZ) : filename); - break; - case ACTION_CMD_SAVE_AS: - doSave(chooseSavePath(FILE_FILTER_GZ, FILE_FILTER_JSON)); - break; - case ACTION_CMD_OPEN_ETL_SPECS: - doOpenSpecs(chooseOpenPath(FILE_FILTER_GZ, FILE_FILTER_JSON)); - break; - case ACTION_CMD_OPEN_SCAN_REPORT: - doOpenScanReport(chooseOpenPath(FILE_FILTER_XLSX)); - break; - case ACTION_CMD_GENERATE_ETL_DOCUMENT: - doGenerateEtlDoc(chooseSavePath(FILE_FILTER_DOCX)); - break; - case ACTION_CMD_GENERATE_TEST_FRAMEWORK: - doGenerateTestFramework(chooseSavePath(FILE_FILTER_R)); - break; - case ACTION_CMD_DISCARD_COUNTS: - doDiscardCounts(); - break; - case ACTION_CMD_FILTER: - doOpenFilterDialog(); - break; - case ACTION_CMD_MAKE_MAPPING: - doMakeMappings(); - break; - case ACTION_CMD_REMOVE_MAPPING: - doRemoveMappings(); - break; - case ACTION_CMD_SET_TARGET_V4: - doSetTargetCDM(CDMVersion.CDMV4); - break; - case ACTION_CMD_SET_TARGET_V5: - doSetTargetCDM(CDMVersion.CDMV5); - break; - case ACTION_CMD_SET_TARGET_V501: - doSetTargetCDM(CDMVersion.CDMV501); - break; - case ACTION_CMD_SET_TARGET_V510: - doSetTargetCDM(CDMVersion.CDMV510); - break; - case ACTION_CMD_SET_TARGET_V520: - doSetTargetCDM(CDMVersion.CDMV520); - break; - case ACTION_CMD_SET_TARGET_V530: - doSetTargetCDM(CDMVersion.CDMV530); - break; - case ACTION_CMD_SET_TARGET_V531: - doSetTargetCDM(CDMVersion.CDMV531); - break; - case ACTION_CMD_SET_TARGET_CUSTOM: - doSetTargetCustom(chooseOpenPath(FILE_FILTER_CSV)); - break; - case ACTION_ADD_STEM_TABLE: - doAddStemTable(); - break; - case ACTION_CMD_MARK_COMPLETED: - doMarkCompleted(); - break; - case ACTION_CMD_UNMARK_COMPLETED: - doUnmarkCompleted(); - break; - case ACTION_CMD_HELP: - doOpenWiki(); - break; - } - } - - private void doAddStemTable() { - ETL etl = ObjectExchange.etl; - StemTableAdd.addStemTable(etl); - ObjectExchange.etl = etl; - tableMappingPanel.setMapping(etl.getTableToTableMapping()); - } - - private void doGenerateTestFramework(String filename) { - if (filename != null) { - frame.setCursor(Cursor.getPredefinedCursor(Cursor.WAIT_CURSOR)); - ETLTestFrameWorkGenerator etlTestFrameWorkGenerator = new ETLTestFrameWorkGenerator(); - etlTestFrameWorkGenerator.generate(ObjectExchange.etl, filename); - frame.setCursor(Cursor.getPredefinedCursor(Cursor.DEFAULT_CURSOR)); - } - } - - private void doOpenWiki() { - try { - Desktop desktop = Desktop.getDesktop(); - desktop.browse(new URI(WIKI_URL)); - } catch (URISyntaxException | IOException ex) { - - } - } - - private void doSetTargetCustom(String fileName) { - - if (fileName != null) { - File file = new File(fileName); - InputStream stream; - - try { - stream = new FileInputStream(file); - ETL etl = new ETL(ObjectExchange.etl.getSourceDatabase(), Database.generateModelFromCSV(stream, file.getName())); - - etl.copyETLMappings(ObjectExchange.etl); - tableMappingPanel.setMapping(etl.getTableToTableMapping()); - ObjectExchange.etl = etl; - } catch (IOException e) { - // Do nothing if error - } - } - - } - - private void doSetTargetCDM(CDMVersion cdmVersion) { - ETL etl = new ETL(ObjectExchange.etl.getSourceDatabase(), Database.generateCDMModel(cdmVersion)); - etl.copyETLMappings(ObjectExchange.etl); - tableMappingPanel.setMapping(etl.getTableToTableMapping()); - ObjectExchange.etl = etl; - } - - private void doOpenFilterDialog() { - FilterDialog filter; - filter = new FilterDialog(frame); - - filter.setFilterPanel(tableMappingPanel); - - filter.setVisible(true); - } - - private void doMakeMappings() { - if (this.tableMappingPanel.isMaximized()) { - this.tableMappingPanel.makeMapSelectedSourceAndTarget(); - } else { - this.fieldMappingPanel.makeMapSelectedSourceAndTarget(); - } - - } - - private void doRemoveMappings() { - if (this.tableMappingPanel.isMaximized()) { - this.tableMappingPanel.removeMapSelectedSourceAndTarget(); - } else { - this.fieldMappingPanel.removeMapSelectedSourceAndTarget(); - } - - } - - private void doDiscardCounts() { - ObjectExchange.etl.discardCounts(); - detailsPanel.refresh(); - } - - private void doSave(String filename) { - if (filename != null) { - frame.setCursor(Cursor.getPredefinedCursor(Cursor.WAIT_CURSOR)); - ETL.FileFormat fileFormat = filename.endsWith("json.gz") ? ETL.FileFormat.GzipJson - : filename.endsWith("json") ? ETL.FileFormat.Json : ETL.FileFormat.Binary; - ObjectExchange.etl.save(filename, fileFormat); - frame.setCursor(Cursor.getPredefinedCursor(Cursor.DEFAULT_CURSOR)); - } - } - - private void doOpenSpecs(String filename) { - if (filename != null) { - frame.setCursor(Cursor.getPredefinedCursor(Cursor.WAIT_CURSOR)); - ETL.FileFormat fileFormat = filename.endsWith(".json.gz") ? ETL.FileFormat.GzipJson - : filename.endsWith(".json") ? ETL.FileFormat.Json : ETL.FileFormat.Binary; - try { - ObjectExchange.etl = ETL.fromFile(filename, fileFormat); - tableMappingPanel.setMapping(ObjectExchange.etl.getTableToTableMapping()); - } catch (Exception e) { - e.printStackTrace(); - JOptionPane.showMessageDialog(null, "Invalid File Format", "Error", JOptionPane.ERROR_MESSAGE); - } - frame.setCursor(Cursor.getPredefinedCursor(Cursor.DEFAULT_CURSOR)); - } - } - - private void doOpenScanReport(String filename) { - if (filename != null) { - boolean replace = true; - if (ObjectExchange.etl.getSourceDatabase().getTables().size() != 0) { - Object[] options = { "Replace current data", "Load data on field values only" }; - int result = JOptionPane.showOptionDialog(frame, "You already have source data loaded. Do you want to", "Replace source data?", - JOptionPane.YES_NO_CANCEL_OPTION, JOptionPane.QUESTION_MESSAGE, null, options, options[0]); - if (result == -1) - return; - if (result == 1) - replace = false; - } - frame.setCursor(Cursor.getPredefinedCursor(Cursor.WAIT_CURSOR)); - if (replace) { - ETL etl = new ETL(); - try { - etl.setSourceDatabase(Database.generateModelFromScanReport(filename)); - etl.setTargetDatabase(ObjectExchange.etl.getTargetDatabase()); - tableMappingPanel.setMapping(etl.getTableToTableMapping()); - ObjectExchange.etl = etl; - } catch (Exception e) { - e.printStackTrace(); - JOptionPane.showMessageDialog(null, "Invalid File Format", "Error", JOptionPane.ERROR_MESSAGE); - } - } else { - try { - Database newData = Database.generateModelFromScanReport(filename); - Database oldData = ObjectExchange.etl.getSourceDatabase(); - for (Table newTable : newData.getTables()) { - Table oldTable = (Table) findByName(newTable.getName(), oldData.getTables()); - if (oldTable != null) { - for (Field newField : newTable.getFields()) { - Field oldField = (Field) findByName(newField.getName(), oldTable.getFields()); - if (oldField != null) { - oldField.setValueCounts(newField.getValueCounts()); - } - } - } - } - } catch (Exception e) { - e.printStackTrace(); - JOptionPane.showMessageDialog(null, "Invalid File Format", "Error", JOptionPane.ERROR_MESSAGE); - } - - } - frame.setCursor(Cursor.getPredefinedCursor(Cursor.DEFAULT_CURSOR)); - } - } - - private MappableItem findByName(String name, List list) { - for (MappableItem item : list) - if (item.getName().toLowerCase().equals(name.toLowerCase())) - return item; - return null; - } - - private void doGenerateEtlDoc(String filename) { - if (filename != null) { - frame.setCursor(Cursor.getPredefinedCursor(Cursor.WAIT_CURSOR)); - ETLDocumentGenerator.generate(ObjectExchange.etl, filename); - frame.setCursor(Cursor.getPredefinedCursor(Cursor.DEFAULT_CURSOR)); - } - } - - private void doMarkCompleted() { - this.tableMappingPanel.markCompleted(); - this.fieldMappingPanel.markCompleted(); - } - - private void doUnmarkCompleted() { - this.tableMappingPanel.unmarkCompleted(); - this.fieldMappingPanel.unmarkCompleted(); - } -} +/******************************************************************************* + * Copyright 2019 Observational Health Data Sciences and Informatics + * + * This file is part of WhiteRabbit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + ******************************************************************************/ +package org.ohdsi.rabbitInAHat; + +import java.awt.Color; +import java.awt.Cursor; +import java.awt.Desktop; +import java.awt.Dimension; +import java.awt.Image; +import java.awt.MediaTracker; +import java.awt.Toolkit; +import java.awt.event.ActionEvent; +import java.awt.event.ActionListener; +import java.awt.event.KeyEvent; +import java.awt.event.WindowAdapter; +import java.awt.event.WindowEvent; +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.net.URI; +import java.net.URISyntaxException; +import java.util.ArrayList; +import java.util.List; + +import javax.swing.*; +import javax.swing.border.TitledBorder; +import javax.swing.filechooser.FileFilter; +import javax.swing.filechooser.FileNameExtensionFilter; + +import org.ohdsi.rabbitInAHat.ETLMarkupDocumentGenerator.DocumentType; +import org.ohdsi.rabbitInAHat.dataModel.Database; +import org.ohdsi.rabbitInAHat.dataModel.Database.CDMVersion; +import org.ohdsi.rabbitInAHat.dataModel.ETL; +import org.ohdsi.rabbitInAHat.dataModel.Field; +import org.ohdsi.rabbitInAHat.dataModel.MappableItem; +import org.ohdsi.rabbitInAHat.dataModel.StemTableFactory; +import org.ohdsi.rabbitInAHat.dataModel.Table; + +/** + * This is the main class for the RabbitInAHat application + */ +public class RabbitInAHatMain implements ResizeListener, ActionListener { + + public final static String ACTION_SAVE = "Save"; + public final static String ACTION_SAVE_AS = "Save As"; + public final static String ACTION_OPEN_ETL_SPECS = "Open ETL Specs"; + public final static String ACTION_OPEN_SCAN_REPORT = "Open Scan Report"; + public final static String ACTION_GENERATE_ETL_WORD_DOCUMENT = "Generate ETL Word Document"; + public final static String ACTION_GENERATE_ETL_HTML_DOCUMENT = "Generate ETL HTML Document"; + public final static String ACTION_GENERATE_ETL_MD_DOCUMENT = "Generate ETL Markdown Document"; + public final static String ACTION_GENERATE_TEST_FRAMEWORK = "Generate ETL Test Framework"; + public final static String ACTION_GENERATE_SQL = "Generate SQL Skeleton"; + public final static String ACTION_DISCARD_COUNTS = "Discard Value Counts"; + public final static String ACTION_FILTER = "Filter"; + public final static String ACTION_MAKE_MAPPING = "Make Mappings"; + public final static String ACTION_REMOVE_MAPPING = "Remove Mappings"; + public final static String ACTION_SET_TARGET_V4 = "CDM v4"; + public final static String ACTION_SET_TARGET_V5 = "CDM v5.0.0"; + public final static String ACTION_SET_TARGET_V501 = "CDM v5.0.1"; + public final static String ACTION_SET_TARGET_V510 = "CDM v5.1.0"; + public final static String ACTION_SET_TARGET_V520 = "CDM v5.2.0"; + public final static String ACTION_SET_TARGET_V530 = "CDM v5.3.0"; + public final static String ACTION_SET_TARGET_V531 = "CDM v5.3.1"; + public final static String ACTION_SET_TARGET_V60 = "CDM v6.0"; + public final static String ACTION_ADD_STEM_TABLE = "Add stem table"; + public final static String ACTION_REMOVE_STEM_TABLE = "Remove stem table"; + public final static String ACTION_SET_TARGET_CUSTOM = "Load Custom..."; + public final static String ACTION_MARK_COMPLETED = "Mark Highlighted As Complete"; + public final static String ACTION_UNMARK_COMPLETED = "Mark Highlighted As Incomplete"; + public final static String ACTION_HELP = "Open help Wiki"; + + public final static String WIKI_URL = "http://www.ohdsi.org/web/wiki/doku.php?id=documentation:software:whiterabbit#rabbit-in-a-hat"; + private final static FileFilter FILE_FILTER_GZ = new FileNameExtensionFilter("GZIP Files (*.gz)", "gz"); + private final static FileFilter FILE_FILTER_JSON = new FileNameExtensionFilter("JSON Files (*.json)", "json"); + private final static FileFilter FILE_FILTER_DOCX = new FileNameExtensionFilter("Microsoft Word documents (*.docx)", "docx"); + private final static FileFilter FILE_FILTER_HTML = new FileNameExtensionFilter("HTML documents (*.html)", "html"); + private final static FileFilter FILE_FILTER_MD = new FileNameExtensionFilter("Markdown documents (*.md)", "md"); + private final static FileFilter FILE_FILTER_CSV = new FileNameExtensionFilter("Text Files (*.csv)", "csv"); + private final static FileFilter FILE_FILTER_R = new FileNameExtensionFilter("R script (*.r)", "r"); + private final static FileFilter FILE_FILTER_XLSX = new FileNameExtensionFilter("XLSX files (*.xlsx)", "xlsx"); + + public final static String DBMS_SQLSERVER = "SQL Server"; + public final static String DBMS_REDSHIFT = "Redshift"; + + private JFrame frame; + private JScrollPane scrollPane1; + private JScrollPane scrollPane2; + private MappingPanel tableMappingPanel; + private MappingPanel fieldMappingPanel; + private DetailsPanel detailsPanel; + private JSplitPane tableFieldSplitPane; + private JFileChooser chooser; + + public static void main(String[] args) { + new RabbitInAHatMain(args); + } + + public RabbitInAHatMain(String[] args) { + + // Set look and feel to the system look and feel + try { + UIManager.setLookAndFeel(UIManager.getSystemLookAndFeelClassName()); + } catch (Exception ex) { + ex.printStackTrace(); + } + + frame = new JFrame("Rabbit in a Hat"); + + frame.setDefaultCloseOperation(JFrame.DO_NOTHING_ON_CLOSE); + frame.addWindowListener(new WindowAdapter() { + public void windowClosing(WindowEvent e) { + String ObjButtons[] = {"Yes","No"}; + int PromptResult = JOptionPane.showOptionDialog( + null, + "Do you want to exit?\nPlease make sure that any work is saved", + "Rabbit In A Hat",JOptionPane.DEFAULT_OPTION,JOptionPane.WARNING_MESSAGE, + null,ObjButtons,ObjButtons[1] + ); + if (PromptResult==JOptionPane.YES_OPTION) { + System.exit(0); + } + } + }); + frame.setPreferredSize(new Dimension(700, 600)); + frame.setLayout(new BoxLayout(frame.getContentPane(), BoxLayout.Y_AXIS)); + frame.setJMenuBar(createMenuBar()); + + ETL etl = new ETL(); + etl.setTargetDatabase(Database.generateCDMModel(CDMVersion.CDMV60)); + + ObjectExchange.etl = etl; + + tableMappingPanel = new MappingPanel(etl.getTableToTableMapping()); + tableMappingPanel.addResizeListener(this); + scrollPane1 = new JScrollPane(tableMappingPanel); + scrollPane1.setBorder(new TitledBorder("Tables")); + scrollPane1.getVerticalScrollBar().setUnitIncrement(16); + scrollPane1.getViewport().setScrollMode(JViewport.BACKINGSTORE_SCROLL_MODE); + scrollPane1.setAutoscrolls(true); + scrollPane1.setOpaque(true); + scrollPane1.setBackground(Color.WHITE); + + fieldMappingPanel = new MappingPanel(etl.getTableToTableMapping()); + tableMappingPanel.setSlaveMappingPanel(fieldMappingPanel); + fieldMappingPanel.addResizeListener(this); + scrollPane2 = new JScrollPane(fieldMappingPanel); + scrollPane2.getVerticalScrollBar().setUnitIncrement(16); + scrollPane2.getViewport().setScrollMode(JViewport.BACKINGSTORE_SCROLL_MODE); + scrollPane2.setVisible(false); + scrollPane2.setBorder(new TitledBorder("Fields")); + + tableFieldSplitPane = new JSplitPane(JSplitPane.VERTICAL_SPLIT, scrollPane1, scrollPane2); + tableFieldSplitPane.setDividerLocation(600); + tableFieldSplitPane.setDividerSize(0); + + detailsPanel = new DetailsPanel(); + detailsPanel.setBorder(new TitledBorder("Details")); + detailsPanel.setPreferredSize(new Dimension(200, 500)); + detailsPanel.setMinimumSize(new Dimension(0, 0)); + tableMappingPanel.setDetailsListener(detailsPanel); + fieldMappingPanel.setDetailsListener(detailsPanel); + JSplitPane leftRightSplinePane = new JSplitPane(JSplitPane.HORIZONTAL_SPLIT, tableFieldSplitPane, detailsPanel); + leftRightSplinePane.setResizeWeight(0.40); + frame.add(leftRightSplinePane); + + loadIcons(frame); + frame.pack(); + frame.setExtendedState(java.awt.Frame.MAXIMIZED_BOTH); + frame.setVisible(true); + + if (args.length == 1) { + doOpenSpecs(args[0]); + } + if (args.length > 1 && args[0].equals("-folder")) { + chooser = new JFileChooser(args[1]); + } + } + + private void loadIcons(JFrame f) { + List icons = new ArrayList(); + icons.add(loadIcon("RabbitInAHat16.png", f)); + icons.add(loadIcon("RabbitInAHat32.png", f)); + icons.add(loadIcon("RabbitInAHat48.png", f)); + icons.add(loadIcon("RabbitInAHat64.png", f)); + icons.add(loadIcon("RabbitInAHat128.png", f)); + icons.add(loadIcon("RabbitInAHat256.png", f)); + f.setIconImages(icons); + } + + private Image loadIcon(String name, JFrame f) { + Image icon = Toolkit.getDefaultToolkit().getImage(RabbitInAHatMain.class.getResource(name)); + MediaTracker mediaTracker = new MediaTracker(f); + mediaTracker.addImage(icon, 0); + try { + mediaTracker.waitForID(0); + return icon; + } catch (Exception e1) { + e1.printStackTrace(); + } + return null; + } + + private JMenuBar createMenuBar() { + JMenuBar menuBar = new JMenuBar(); + JMenu fileMenu = new JMenu("File"); + int menuShortcutMask = Toolkit.getDefaultToolkit().getMenuShortcutKeyMask(); + + menuBar.add(fileMenu); + + JMenuItem openScanReportItem = new JMenuItem(ACTION_OPEN_SCAN_REPORT); + openScanReportItem.addActionListener(this); + fileMenu.add(openScanReportItem); + + JMenuItem openItem = new JMenuItem(ACTION_OPEN_ETL_SPECS); + openItem.addActionListener(this); + openItem.setAccelerator(KeyStroke.getKeyStroke(KeyEvent.VK_O, menuShortcutMask)); + fileMenu.add(openItem); + + JMenuItem saveItem = new JMenuItem(ACTION_SAVE); + saveItem.addActionListener(this); + saveItem.setAccelerator(KeyStroke.getKeyStroke(KeyEvent.VK_S, menuShortcutMask)); + fileMenu.add(saveItem); + + JMenuItem saveAsItem = new JMenuItem(ACTION_SAVE_AS); + saveAsItem.addActionListener(this); + fileMenu.add(saveAsItem); + + JMenu generateEtlDocument = new JMenu("Generate ETL document"); + fileMenu.add(generateEtlDocument); + + JMenuItem generateWordDocItem = new JMenuItem(ACTION_GENERATE_ETL_WORD_DOCUMENT); + generateWordDocItem.addActionListener(this); + generateEtlDocument.add(generateWordDocItem); + + JMenuItem generateHtmlDocItem = new JMenuItem(ACTION_GENERATE_ETL_HTML_DOCUMENT); + generateHtmlDocItem.addActionListener(this); + generateEtlDocument.add(generateHtmlDocItem); + + JMenuItem generateMdDocItem = new JMenuItem(ACTION_GENERATE_ETL_MD_DOCUMENT); + generateMdDocItem.addActionListener(this); + generateEtlDocument.add(generateMdDocItem); + + JMenuItem generateTestFrameworkItem = new JMenuItem(ACTION_GENERATE_TEST_FRAMEWORK); + generateTestFrameworkItem.addActionListener(this); + fileMenu.add(generateTestFrameworkItem); + + JMenuItem generateSql = new JMenuItem(ACTION_GENERATE_SQL); + generateSql.addActionListener(this); + generateSql.setActionCommand(ACTION_GENERATE_SQL); + fileMenu.add(generateSql); + + JMenu editMenu = new JMenu("Edit"); + menuBar.add(editMenu); + + JMenuItem discardCounts = new JMenuItem(ACTION_DISCARD_COUNTS); + discardCounts.addActionListener(this); + editMenu.add(discardCounts); + + JMenuItem filter = new JMenuItem(ACTION_FILTER); + filter.addActionListener(this); + filter.setAccelerator(KeyStroke.getKeyStroke(KeyEvent.VK_F, menuShortcutMask)); + editMenu.add(filter); + + JMenu setTarget = new JMenu("Set Target Database"); + + JRadioButtonMenuItem targetCDMV4 = new JRadioButtonMenuItem(ACTION_SET_TARGET_V4); + targetCDMV4.addActionListener(this); + setTarget.add(targetCDMV4); + + JRadioButtonMenuItem targetCDMV5 = new JRadioButtonMenuItem(ACTION_SET_TARGET_V5); + targetCDMV5.addActionListener(this); + setTarget.add(targetCDMV5); + + JRadioButtonMenuItem targetCDMV501 = new JRadioButtonMenuItem(ACTION_SET_TARGET_V501); + targetCDMV501.addActionListener(this); + setTarget.add(targetCDMV501); + + JRadioButtonMenuItem targetCDMV510 = new JRadioButtonMenuItem(ACTION_SET_TARGET_V510); + targetCDMV510.addActionListener(this); + setTarget.add(targetCDMV510); + + JRadioButtonMenuItem targetCDMV520 = new JRadioButtonMenuItem(ACTION_SET_TARGET_V520); + targetCDMV520.addActionListener(this); + setTarget.add(targetCDMV520); + + JRadioButtonMenuItem targetCDMV530 = new JRadioButtonMenuItem(ACTION_SET_TARGET_V530); + targetCDMV530.addActionListener(this); + setTarget.add(targetCDMV530); + + JRadioButtonMenuItem targetCDMV531 = new JRadioButtonMenuItem(ACTION_SET_TARGET_V531); + targetCDMV531.addActionListener(this); + setTarget.add(targetCDMV531); + + JRadioButtonMenuItem targetCDMV60 = new JRadioButtonMenuItem(ACTION_SET_TARGET_V60, true); + targetCDMV60.addActionListener(this); + setTarget.add(targetCDMV60); + + JRadioButtonMenuItem loadTarget = new JRadioButtonMenuItem(ACTION_SET_TARGET_CUSTOM); + loadTarget.addActionListener(this); + setTarget.add(loadTarget); + editMenu.add(setTarget); + + ButtonGroup targetGroup = new ButtonGroup(); + targetGroup.add(targetCDMV4); + targetGroup.add(targetCDMV5); + targetGroup.add(targetCDMV501); + targetGroup.add(targetCDMV510); + targetGroup.add(targetCDMV520); + targetGroup.add(targetCDMV530); + targetGroup.add(targetCDMV531); + targetGroup.add(targetCDMV60); + targetGroup.add(loadTarget); + + JMenuItem addStemTable = new JMenuItem(ACTION_ADD_STEM_TABLE); + addStemTable.addActionListener(this); + editMenu.add(addStemTable); + + JMenuItem removeStemTable = new JMenuItem(ACTION_REMOVE_STEM_TABLE); + removeStemTable.addActionListener(this); + editMenu.add(removeStemTable); + + JMenu arrowMenu = new JMenu("Arrows"); + menuBar.add(arrowMenu); + + JMenuItem makeMappings = new JMenuItem(ACTION_MAKE_MAPPING); + makeMappings.addActionListener(this); + makeMappings.setAccelerator(KeyStroke.getKeyStroke(KeyEvent.VK_M, menuShortcutMask)); + arrowMenu.add(makeMappings); + + JMenuItem removeMappings = new JMenuItem(ACTION_REMOVE_MAPPING); + removeMappings.addActionListener(this); + removeMappings.setAccelerator(KeyStroke.getKeyStroke(KeyEvent.VK_R, menuShortcutMask)); + arrowMenu.add(removeMappings); + + JMenuItem markCompleted = new JMenuItem(ACTION_MARK_COMPLETED); + markCompleted.addActionListener(this); + markCompleted.setAccelerator(KeyStroke.getKeyStroke(KeyEvent.VK_D, menuShortcutMask)); + arrowMenu.add(markCompleted); + + JMenuItem unmarkCompleted = new JMenuItem(ACTION_UNMARK_COMPLETED); + unmarkCompleted.addActionListener(this); + unmarkCompleted.setAccelerator(KeyStroke.getKeyStroke(KeyEvent.VK_I, menuShortcutMask)); + arrowMenu.add(unmarkCompleted); + + JMenu helpMenu = new JMenu("Help"); + menuBar.add(helpMenu); + JMenuItem helpItem = new JMenuItem(ACTION_HELP); + helpItem.addActionListener(this); + helpMenu.add(helpItem); + + return menuBar; + } + + @Override + public void notifyResized(int height, boolean minimized, boolean maximized) { + if (scrollPane2.isVisible() == maximized) + scrollPane2.setVisible(!maximized); + + if (!maximized) + scrollPane1.setVerticalScrollBarPolicy(ScrollPaneConstants.VERTICAL_SCROLLBAR_NEVER); + else + scrollPane1.setVerticalScrollBarPolicy(ScrollPaneConstants.VERTICAL_SCROLLBAR_AS_NEEDED); + + if (!minimized) + scrollPane2.setVerticalScrollBarPolicy(ScrollPaneConstants.VERTICAL_SCROLLBAR_NEVER); + else + scrollPane2.setVerticalScrollBarPolicy(ScrollPaneConstants.VERTICAL_SCROLLBAR_AS_NEEDED); + + tableFieldSplitPane.setDividerLocation(height); + } + + /** + * Display file chooser for a path to save or open to + * + * @param saveMode + * true to display a save dialog, false for open + * @param filter + * restrict files displayed + * @return if file selected, absolute path of selected file otherwise null + */ + private String choosePath(boolean saveMode, boolean directoryMode, FileFilter... filter) { + String result = null; + + if (chooser == null) { + chooser = new JFileChooser(); + } else if (saveMode && chooser.getSelectedFile() != null){ + if (filter[0] == FILE_FILTER_GZ) + chooser.setSelectedFile(new File(chooser.getSelectedFile().getAbsolutePath().replaceAll("\\..*$", ".json.gz"))); + else if (filter[0] == FILE_FILTER_DOCX) + chooser.setSelectedFile(new File(chooser.getSelectedFile().getAbsolutePath().replaceAll("\\..*$", ".docx"))); + else if (filter[0] == FILE_FILTER_HTML) + chooser.setSelectedFile(new File(chooser.getSelectedFile().getAbsolutePath().replaceAll("\\..*$", ".html"))); + else if (filter[0] == FILE_FILTER_MD) + chooser.setSelectedFile(new File(chooser.getSelectedFile().getAbsolutePath().replaceAll("\\..*$", ".md"))); + else if (filter[0] == FILE_FILTER_R) + chooser.setSelectedFile(new File(chooser.getSelectedFile().getAbsolutePath().replaceAll("\\..*$", ".R"))); + } + chooser.resetChoosableFileFilters(); + + if (directoryMode) { + chooser.setFileSelectionMode(JFileChooser.DIRECTORIES_ONLY); + chooser.setAcceptAllFileFilterUsed(false); + } else { + chooser.setFileSelectionMode(JFileChooser.FILES_ONLY); + chooser.setFileFilter(filter[0]); + for (int i = 1; i < filter.length; i++) + chooser.addChoosableFileFilter(filter[i]); + } + + int dialogResult = saveMode ? chooser.showSaveDialog(frame) : chooser.showOpenDialog(frame); + if (dialogResult == JFileChooser.APPROVE_OPTION) { + if (directoryMode) + result = chooser.getCurrentDirectory().getAbsolutePath(); + else + result = chooser.getSelectedFile().getAbsolutePath(); + } + + return result; + } + + private String choosePath(boolean saveMode, FileFilter... filter) { + return choosePath(saveMode, false, filter); + } + + private String chooseSavePath(FileFilter... fileFilter) { + String path = choosePath(true, fileFilter); + if (path != null && fileFilter[0] == FILE_FILTER_GZ && !path.toLowerCase().endsWith(".json.gz") && !path.toLowerCase().endsWith(".json")) + path += ".json.gz"; + if (path != null && fileFilter[0] == FILE_FILTER_DOCX && !path.toLowerCase().endsWith(".docx")) + path += ".docx"; + if (path != null && fileFilter[0] == FILE_FILTER_HTML && !path.toLowerCase().endsWith(".html")) + path += ".html"; + if (path != null && fileFilter[0] == FILE_FILTER_MD && !path.toLowerCase().endsWith(".md")) + path += ".md"; + if (path != null && fileFilter[0] == FILE_FILTER_R && !path.toLowerCase().endsWith(".R")) + path += ".R"; + return path; + } + + private String chooseOpenPath(FileFilter... fileFilter) { + return choosePath(false, fileFilter); + } + + private String chooseSaveDirectory() { + return choosePath(true, true, new FileNameExtensionFilter("Directories",".")); + } + + @Override + public void actionPerformed(ActionEvent event) { + switch (event.getActionCommand()) { + case ACTION_SAVE: + String filename = ObjectExchange.etl.getFilename(); + doSave((filename == null || !filename.toLowerCase().endsWith(".json.gz")) ? chooseSavePath(FILE_FILTER_GZ) : filename); + break; + case ACTION_SAVE_AS: + doSave(chooseSavePath(FILE_FILTER_GZ, FILE_FILTER_JSON)); + break; + case ACTION_OPEN_ETL_SPECS: + doOpenSpecs(chooseOpenPath(FILE_FILTER_GZ, FILE_FILTER_JSON)); + break; + case ACTION_OPEN_SCAN_REPORT: + doOpenScanReport(chooseOpenPath(FILE_FILTER_XLSX)); + break; + case ACTION_GENERATE_ETL_WORD_DOCUMENT: + doGenerateEtlWordDoc(chooseSavePath(FILE_FILTER_DOCX)); + break; + case ACTION_GENERATE_ETL_HTML_DOCUMENT: + doGenerateEtlHtmlDoc(chooseSavePath(FILE_FILTER_HTML)); + break; + case ACTION_GENERATE_ETL_MD_DOCUMENT: + doGenerateEtlMdDoc(chooseSavePath(FILE_FILTER_MD)); + break; + case ACTION_GENERATE_TEST_FRAMEWORK: + doGenerateTestFramework(chooseSavePath(FILE_FILTER_R)); + case ACTION_GENERATE_SQL: + doGenerateSql(chooseSaveDirectory()); + break; + case ACTION_DISCARD_COUNTS: + doDiscardCounts(); + break; + case ACTION_FILTER: + doOpenFilterDialog(); + break; + case ACTION_MAKE_MAPPING: + doMakeMappings(); + break; + case ACTION_REMOVE_MAPPING: + doRemoveMappings(); + break; + case ACTION_SET_TARGET_V4: + doSetTargetCDM(CDMVersion.CDMV4); + break; + case ACTION_SET_TARGET_V5: + doSetTargetCDM(CDMVersion.CDMV5); + break; + case ACTION_SET_TARGET_V501: + doSetTargetCDM(CDMVersion.CDMV501); + break; + case ACTION_SET_TARGET_V510: + doSetTargetCDM(CDMVersion.CDMV510); + break; + case ACTION_SET_TARGET_V520: + doSetTargetCDM(CDMVersion.CDMV520); + break; + case ACTION_SET_TARGET_V530: + doSetTargetCDM(CDMVersion.CDMV530); + break; + case ACTION_SET_TARGET_V531: + doSetTargetCDM(CDMVersion.CDMV531); + break; + case ACTION_SET_TARGET_V60: + doSetTargetCDM(CDMVersion.CDMV60); + break; + case ACTION_SET_TARGET_CUSTOM: + doSetTargetCustom(chooseOpenPath(FILE_FILTER_CSV)); + break; + case ACTION_ADD_STEM_TABLE: + doAddStemTable(); + break; + case ACTION_REMOVE_STEM_TABLE: + doRemoveStemTable(); + break; + case ACTION_MARK_COMPLETED: + doMarkCompleted(); + break; + case ACTION_UNMARK_COMPLETED: + doUnmarkCompleted(); + break; + case ACTION_HELP: + doOpenWiki(); + break; + } + } + + private void doAddStemTable() { + if (!ObjectExchange.etl.hasStemTable()) { + StemTableFactory.addStemTable(ObjectExchange.etl); + tableMappingPanel.setMapping(ObjectExchange.etl.getTableToTableMapping()); + } + } + + private void doRemoveStemTable() { + if (ObjectExchange.etl.hasStemTable()) { + String[] ObjButtons = {"Yes","No"}; + int PromptResult = JOptionPane.showOptionDialog( + null, + "Any mappings to/from the stem table will be lost. Are you sure?", + "Rabbit In A Hat",JOptionPane.DEFAULT_OPTION,JOptionPane.WARNING_MESSAGE, + null,ObjButtons,ObjButtons[1] + ); + + if (PromptResult==JOptionPane.YES_OPTION) { + StemTableFactory.removeStemTable(ObjectExchange.etl); + tableMappingPanel.setMapping(ObjectExchange.etl.getTableToTableMapping()); + } + } + } + + private void doGenerateTestFramework(String filename) { + if (filename != null) { + frame.setCursor(Cursor.getPredefinedCursor(Cursor.WAIT_CURSOR)); + ETLTestFrameWorkGenerator etlTestFrameWorkGenerator = new ETLTestFrameWorkGenerator(); + etlTestFrameWorkGenerator.generate(ObjectExchange.etl, filename); + frame.setCursor(Cursor.getPredefinedCursor(Cursor.DEFAULT_CURSOR)); + } + } + + private void doOpenWiki() { + try { + Desktop desktop = Desktop.getDesktop(); + desktop.browse(new URI(WIKI_URL)); + } catch (URISyntaxException | IOException ex) { + + } + } + + private void doSetTargetCustom(String fileName) { + + if (fileName != null) { + File file = new File(fileName); + InputStream stream; + + try { + stream = new FileInputStream(file); + ETL etl = new ETL(ObjectExchange.etl.getSourceDatabase(), Database.generateModelFromCSV(stream, file.getName())); + + etl.copyETLMappings(ObjectExchange.etl); + tableMappingPanel.setMapping(etl.getTableToTableMapping()); + ObjectExchange.etl = etl; + } catch (IOException e) { + // Do nothing if error + } + } + + } + + private void doSetTargetCDM(CDMVersion cdmVersion) { + ETL etl = new ETL(ObjectExchange.etl.getSourceDatabase(), Database.generateCDMModel(cdmVersion)); + etl.copyETLMappings(ObjectExchange.etl); + tableMappingPanel.setMapping(etl.getTableToTableMapping()); + ObjectExchange.etl = etl; + } + + private void doOpenFilterDialog() { + FilterDialog filter; + filter = new FilterDialog(frame); + + filter.setFilterPanel(tableMappingPanel); + + filter.setVisible(true); + } + + private void doMakeMappings() { + if (this.tableMappingPanel.isMaximized()) { + this.tableMappingPanel.makeMapSelectedSourceAndTarget(); + } else { + this.fieldMappingPanel.makeMapSelectedSourceAndTarget(); + } + + } + + private void doRemoveMappings() { + if (this.tableMappingPanel.isMaximized()) { + this.tableMappingPanel.removeMapSelectedSourceAndTarget(); + } else { + this.fieldMappingPanel.removeMapSelectedSourceAndTarget(); + } + + } + + private void doDiscardCounts() { + ObjectExchange.etl.discardCounts(); + detailsPanel.refresh(); + } + + private void doSave(String filename) { + if (filename != null) { + frame.setCursor(Cursor.getPredefinedCursor(Cursor.WAIT_CURSOR)); + ETL.FileFormat fileFormat = filename.endsWith("json.gz") ? ETL.FileFormat.GzipJson + : filename.endsWith("json") ? ETL.FileFormat.Json : ETL.FileFormat.Binary; + ObjectExchange.etl.save(filename, fileFormat); + frame.setCursor(Cursor.getPredefinedCursor(Cursor.DEFAULT_CURSOR)); + } + } + + private void doOpenSpecs(String filename) { + if (filename != null) { + frame.setCursor(Cursor.getPredefinedCursor(Cursor.WAIT_CURSOR)); + ETL.FileFormat fileFormat = filename.endsWith(".json.gz") ? ETL.FileFormat.GzipJson + : filename.endsWith(".json") ? ETL.FileFormat.Json : ETL.FileFormat.Binary; + try { + ObjectExchange.etl = ETL.fromFile(filename, fileFormat); + tableMappingPanel.setMapping(ObjectExchange.etl.getTableToTableMapping()); + } catch (Exception e) { + e.printStackTrace(); + JOptionPane.showMessageDialog(null, "Invalid File Format", "Error", JOptionPane.ERROR_MESSAGE); + } + frame.setCursor(Cursor.getPredefinedCursor(Cursor.DEFAULT_CURSOR)); + } + } + + private void doOpenScanReport(String filename) { + if (filename != null) { + boolean replace = true; + if (ObjectExchange.etl.getSourceDatabase().getTables().size() != 0) { + Object[] options = { "Replace current data", "Load data on field values only" }; + int result = JOptionPane.showOptionDialog(frame, "You already have source data loaded. Do you want to", "Replace source data?", + JOptionPane.YES_NO_CANCEL_OPTION, JOptionPane.QUESTION_MESSAGE, null, options, options[0]); + if (result == -1) + return; + if (result == 1) + replace = false; + } + frame.setCursor(Cursor.getPredefinedCursor(Cursor.WAIT_CURSOR)); + if (replace) { + ETL etl = new ETL(); + doRemoveStemTable(); + try { + etl.setSourceDatabase(Database.generateModelFromScanReport(filename)); + etl.setTargetDatabase(ObjectExchange.etl.getTargetDatabase()); + tableMappingPanel.setMapping(etl.getTableToTableMapping()); + ObjectExchange.etl = etl; + } catch (Exception e) { + e.printStackTrace(); + JOptionPane.showMessageDialog(null, "Invalid File Format", "Error", JOptionPane.ERROR_MESSAGE); + } + } else { + try { + Database newData = Database.generateModelFromScanReport(filename); + Database oldData = ObjectExchange.etl.getSourceDatabase(); + for (Table newTable : newData.getTables()) { + Table oldTable = (Table) findByName(newTable.getName(), oldData.getTables()); + if (oldTable != null) { + for (Field newField : newTable.getFields()) { + Field oldField = (Field) findByName(newField.getName(), oldTable.getFields()); + if (oldField != null) { + oldField.setValueCounts(newField.getValueCounts()); + } + } + } + } + } catch (Exception e) { + e.printStackTrace(); + JOptionPane.showMessageDialog(null, "Invalid File Format", "Error", JOptionPane.ERROR_MESSAGE); + } + + } + frame.setCursor(Cursor.getPredefinedCursor(Cursor.DEFAULT_CURSOR)); + } + } + + private MappableItem findByName(String name, List list) { + for (MappableItem item : list) + if (item.getName().toLowerCase().equals(name.toLowerCase())) + return item; + return null; + } + + private void doGenerateEtlWordDoc(String filename) { + if (filename != null) { + frame.setCursor(Cursor.getPredefinedCursor(Cursor.WAIT_CURSOR)); + ETLWordDocumentGenerator.generate(ObjectExchange.etl, filename); + frame.setCursor(Cursor.getPredefinedCursor(Cursor.DEFAULT_CURSOR)); + } + } + + private void doGenerateEtlHtmlDoc(String filename) { + if (filename != null) { + frame.setCursor(Cursor.getPredefinedCursor(Cursor.WAIT_CURSOR)); + ETLMarkupDocumentGenerator generator = new ETLMarkupDocumentGenerator(ObjectExchange.etl); + generator.generate(filename, DocumentType.HTML); + frame.setCursor(Cursor.getPredefinedCursor(Cursor.DEFAULT_CURSOR)); + } + } + + private void doGenerateEtlMdDoc(String filename) { + if (filename != null) { + frame.setCursor(Cursor.getPredefinedCursor(Cursor.WAIT_CURSOR)); + ETLMarkupDocumentGenerator generator = new ETLMarkupDocumentGenerator(ObjectExchange.etl); + generator.generate(filename, DocumentType.MARKDOWN); + frame.setCursor(Cursor.getPredefinedCursor(Cursor.DEFAULT_CURSOR)); + } + } + + private void doGenerateSql(String directoryName) { + if (directoryName != null) { + frame.setCursor(Cursor.getPredefinedCursor(Cursor.WAIT_CURSOR)); + SQLGenerator sqlGenerator = new SQLGenerator(ObjectExchange.etl, directoryName); + sqlGenerator.generate(); + frame.setCursor(Cursor.getPredefinedCursor(Cursor.DEFAULT_CURSOR)); + } + } + + private void doMarkCompleted() { + this.tableMappingPanel.markCompleted(); + this.fieldMappingPanel.markCompleted(); + } + + private void doUnmarkCompleted() { + this.tableMappingPanel.unmarkCompleted(); + this.fieldMappingPanel.unmarkCompleted(); + } +} diff --git a/src/org/ohdsi/rabbitInAHat/ResizeListener.java b/rabbitinahat/src/main/java/org/ohdsi/rabbitInAHat/ResizeListener.java similarity index 91% rename from src/org/ohdsi/rabbitInAHat/ResizeListener.java rename to rabbitinahat/src/main/java/org/ohdsi/rabbitInAHat/ResizeListener.java index e567e231..bbf96cef 100644 --- a/src/org/ohdsi/rabbitInAHat/ResizeListener.java +++ b/rabbitinahat/src/main/java/org/ohdsi/rabbitInAHat/ResizeListener.java @@ -1,5 +1,5 @@ /******************************************************************************* - * Copyright 2017 Observational Health Data Sciences and Informatics + * Copyright 2019 Observational Health Data Sciences and Informatics * * This file is part of WhiteRabbit * diff --git a/rabbitinahat/src/main/java/org/ohdsi/rabbitInAHat/SQLGenerator.java b/rabbitinahat/src/main/java/org/ohdsi/rabbitInAHat/SQLGenerator.java new file mode 100644 index 00000000..2457036c --- /dev/null +++ b/rabbitinahat/src/main/java/org/ohdsi/rabbitInAHat/SQLGenerator.java @@ -0,0 +1,157 @@ +package org.ohdsi.rabbitInAHat; + +import org.ohdsi.rabbitInAHat.dataModel.*; + +import java.io.*; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +/** + * Created 04-01-17. + * Generates separate SQL files for each table to table mapping in the given directory. + */ +public class SQLGenerator { + ETL etl; + String outputDirectory; + + public SQLGenerator(ETL etl, String directory) { + this.etl = etl; + this.outputDirectory = directory; + } + + public void generate() { + // Generate a sql file for each source to target table mapping + Integer counter = 0; + for (ItemToItemMap tableToTableMap : etl.getTableToTableMapping().getSourceToTargetMaps()) { + writeSqlFile(tableToTableMap); + counter++; + } + System.out.println(counter.toString() + " sql files were generated"); // TODO report in UI + } + + private void writeSqlFile(ItemToItemMap tableToTableMap){ + Table sourceTable = (Table) tableToTableMap.getSourceItem(); + Table targetTable = (Table) tableToTableMap.getTargetItem(); + Mapping fieldToFieldMapping = etl.getFieldToFieldMapping(sourceTable, targetTable); + List mappings = fieldToFieldMapping.getSourceToTargetMapsOrderedByCdmItems(); + + // Create new sql file in the selected directory + File outFile = new File( outputDirectory, sourceTable.getName() + "_to_" + targetTable.getName() + ".sql"); + System.out.println( "Writing to: " + outFile.getAbsoluteFile() ); + + int n_mappings = mappings.size(); + Set targetFieldsSeen = new HashSet<>(); + try (FileOutputStream fout = new FileOutputStream(outFile); + OutputStreamWriter fwr = new OutputStreamWriter(fout, "UTF-8"); + Writer out = new BufferedWriter(fwr)) { + + // Table specific comments + out.write(createBlockComment(sourceTable.getComment())); + out.write(createBlockComment(targetTable.getComment())); + out.write(createBlockComment(tableToTableMap.getComment())); + out.write(createBlockComment(tableToTableMap.getLogic())); + out.write('\n'); + + // To + out.write("INSERT INTO " + targetTable.getName() + "\n"); + out.write("(\n"); + List targetsWithoutSource = new ArrayList<>(); + for (int i=0;i tableToTableMaps = new ArrayList(); - private Map> tableMapToFieldToFieldMaps = new HashMap>(); - private transient String filename = null; - private static final long serialVersionUID = 8987388381751618498L; - - public ETL() { - } - - public ETL(Database sourceDB, Database targetDb) { - this.setSourceDatabase(sourceDB); - this.setTargetDatabase(targetDb); - } - - public void copyETLMappings(ETL etl) { - Mapping oldTableMapping = etl.getTableToTableMapping(); - Mapping
newTableMapping = this.getTableToTableMapping(); - - for (Table sourceTable : sourceDb.getTables()) { - for (Table targetTable : cdmDb.getTables()) { - - ItemToItemMap copyMapping = oldTableMapping.getSourceToTargetMapByName(sourceTable, targetTable); - - if (copyMapping != null) { - copyMapping.setSourceItem(sourceTable); - copyMapping.setTargetItem(targetTable); - - newTableMapping.addSourceToTargetMap(copyMapping); - - Mapping oldFieldMapping = etl.getFieldToFieldMapping(sourceTable, targetTable); - Mapping newFieldMapping = this.getFieldToFieldMapping(sourceTable, targetTable); - - for (Field sourceField : sourceTable.getFields()) { - for (Field targetField : targetTable.getFields()) { - copyMapping = oldFieldMapping.getSourceToTargetMapByName(sourceField, targetField); - - if (copyMapping != null) { - copyMapping.setSourceItem(sourceField); - copyMapping.setTargetItem(targetField); - - newFieldMapping.addSourceToTargetMap(copyMapping); - } - } - } - } - } - } - } - - public void saveCurrentState() { - - } - - public void revertToPreviousState() { - - } - - public void revertToNextState() { - - } - - public Mapping
getTableToTableMapping() { - return new Mapping
(sourceDb.getTables(), cdmDb.getTables(), tableToTableMaps); - } - - public Mapping getFieldToFieldMapping(Table sourceTable, Table targetTable) { - List fieldToFieldMaps = tableMapToFieldToFieldMaps.get(new ItemToItemMap(sourceTable, targetTable)); - if (fieldToFieldMaps == null) { - fieldToFieldMaps = new ArrayList(); - tableMapToFieldToFieldMaps.put(new ItemToItemMap(sourceTable, targetTable), fieldToFieldMaps); - } - return new Mapping(sourceTable.getFields(), targetTable.getFields(), fieldToFieldMaps); - } - - public String getFilename() { - return filename; - } - - public void setTargetDatabase(Database targetDb) { - this.cdmDb = targetDb; - } - - public void setSourceDatabase(Database sourceDb) { - this.sourceDb = sourceDb; - } - - public Database getTargetDatabase() { - return cdmDb; - } - - public Database getSourceDatabase() { - return sourceDb; - } - - public String toJson() { - return JsonWriter.formatJson(JsonWriter.objectToJson(this)); - } - - public void save(String filename, FileFormat format) { - try { - switch (format) { - case Binary: - try (FileOutputStream fileOutputStream = new FileOutputStream(filename); - GZIPOutputStream gzipOutputStream = new GZIPOutputStream(fileOutputStream); - ObjectOutputStream out = new ObjectOutputStream(gzipOutputStream)) { - out.writeObject(this); - } - break; - case GzipJson: - String json = toJson(); - try (FileOutputStream fileOutputStream = new FileOutputStream(filename); - GZIPOutputStream gzipOutputStream = new GZIPOutputStream(fileOutputStream); - OutputStreamWriter out = new OutputStreamWriter(gzipOutputStream, "UTF-8")) { - out.write(json); - } - break; - case Json: - String json2 = toJson(); - try (FileOutputStream fileOutputStream = new FileOutputStream(filename); - OutputStreamWriter out = new OutputStreamWriter(fileOutputStream, "UTF-8")) { - out.write(json2); - } - break; - } - this.filename = filename; - } catch (IOException e) { - e.printStackTrace(); - } - } - - /** - * Create an ETL instance by reading a saved file - * - * @param filename - * path of the file - * @param format - * [[FileFormat.Binary]] or [[FileFormat.Json]] - * @return resulting ETL object - */ - public static ETL fromFile(String filename, FileFormat format) { - ETL result = null; - try { - switch (format) { - case Binary: - try (FileInputStream fileInputStream = new FileInputStream(filename); - GZIPInputStream gzipInputStream = new GZIPInputStream(fileInputStream); - ObjectInputStream objectInputStream = new ObjectInputStream(gzipInputStream)) { - result = (ETL) objectInputStream.readObject(); - } - break; - case GzipJson: - try (FileInputStream fileInputStream = new FileInputStream(filename); - GZIPInputStream gzipInputStream = new GZIPInputStream(fileInputStream); - JsonReader jr = new JsonReader(gzipInputStream)) { - result = (ETL) jr.readObject(); - } - break; - case Json: - try (FileInputStream fileInputStream = new FileInputStream(filename); JsonReader jr = new JsonReader(fileInputStream)) { - result = (ETL) jr.readObject(); - } - break; - } - result.filename = filename; - } catch (Exception exception) { - exception.printStackTrace(); - } - return result; - } - - public void discardCounts() { - for (Table table : sourceDb.getTables()) - for (Field field : table.getFields()) - field.setValueCounts(new String[0][]); - } -} +/******************************************************************************* + * Copyright 2019 Observational Health Data Sciences and Informatics + * + * This file is part of WhiteRabbit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + ******************************************************************************/ +package org.ohdsi.rabbitInAHat.dataModel; + +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.ObjectInputStream; +import java.io.ObjectOutputStream; +import java.io.OutputStreamWriter; +import java.io.Serializable; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.zip.GZIPInputStream; +import java.util.zip.GZIPOutputStream; + +import com.cedarsoftware.util.io.JsonReader; +import com.cedarsoftware.util.io.JsonWriter; + +public class ETL implements Serializable { + public enum FileFormat { + Binary, Json, GzipJson + } + + private Database sourceDb = new Database(); + private Database cdmDb = new Database(); + private List tableToTableMaps = new ArrayList(); + private Map> tableMapToFieldToFieldMaps = new HashMap>(); + private transient String filename = null; + private static final long serialVersionUID = 8987388381751618498L; + + public ETL() { + } + + public ETL(Database sourceDB, Database targetDb) { + this.setSourceDatabase(sourceDB); + this.setTargetDatabase(targetDb); + } + + public void copyETLMappings(ETL etl) { + Mapping
oldTableMapping = etl.getTableToTableMapping(); + Mapping
newTableMapping = this.getTableToTableMapping(); + + for (Table sourceTable : sourceDb.getTables()) { + for (Table targetTable : cdmDb.getTables()) { + + ItemToItemMap copyMapping = oldTableMapping.getSourceToTargetMapByName(sourceTable, targetTable); + + if (copyMapping != null) { + copyMapping.setSourceItem(sourceTable); + copyMapping.setTargetItem(targetTable); + + newTableMapping.addSourceToTargetMap(copyMapping); + + Mapping oldFieldMapping = etl.getFieldToFieldMapping(sourceTable, targetTable); + Mapping newFieldMapping = this.getFieldToFieldMapping(sourceTable, targetTable); + + for (Field sourceField : sourceTable.getFields()) { + for (Field targetField : targetTable.getFields()) { + copyMapping = oldFieldMapping.getSourceToTargetMapByName(sourceField, targetField); + + if (copyMapping != null) { + copyMapping.setSourceItem(sourceField); + copyMapping.setTargetItem(targetField); + + newFieldMapping.addSourceToTargetMap(copyMapping); + } + } + } + } + } + } + } + + public void saveCurrentState() { + + } + + public void revertToPreviousState() { + + } + + public void revertToNextState() { + + } + + public Mapping
getTableToTableMapping() { + return new Mapping
(sourceDb.getTables(), cdmDb.getTables(), tableToTableMaps); + } + + public Mapping getFieldToFieldMapping(Table sourceTable, Table targetTable) { + List fieldToFieldMaps = tableMapToFieldToFieldMaps.get(new ItemToItemMap(sourceTable, targetTable)); + if (fieldToFieldMaps == null) { + fieldToFieldMaps = new ArrayList(); + tableMapToFieldToFieldMaps.put(new ItemToItemMap(sourceTable, targetTable), fieldToFieldMaps); + } + return new Mapping(sourceTable.getFields(), targetTable.getFields(), fieldToFieldMaps); + } + + public String getFilename() { + return filename; + } + + public void setTargetDatabase(Database targetDb) { + this.cdmDb = targetDb; + } + + public void setSourceDatabase(Database sourceDb) { + this.sourceDb = sourceDb; + } + + public Database getTargetDatabase() { + return cdmDb; + } + + public Database getSourceDatabase() { + return sourceDb; + } + + public String toJson() { + return JsonWriter.formatJson(JsonWriter.objectToJson(this)); + } + + public void save(String filename, FileFormat format) { + try { + switch (format) { + case Binary: + try (FileOutputStream fileOutputStream = new FileOutputStream(filename); + GZIPOutputStream gzipOutputStream = new GZIPOutputStream(fileOutputStream); + ObjectOutputStream out = new ObjectOutputStream(gzipOutputStream)) { + out.writeObject(this); + } + break; + case GzipJson: + String json = toJson(); + try (FileOutputStream fileOutputStream = new FileOutputStream(filename); + GZIPOutputStream gzipOutputStream = new GZIPOutputStream(fileOutputStream); + OutputStreamWriter out = new OutputStreamWriter(gzipOutputStream, "UTF-8")) { + out.write(json); + } + break; + case Json: + String json2 = toJson(); + try (FileOutputStream fileOutputStream = new FileOutputStream(filename); + OutputStreamWriter out = new OutputStreamWriter(fileOutputStream, "UTF-8")) { + out.write(json2); + } + break; + } + this.filename = filename; + } catch (IOException e) { + e.printStackTrace(); + } + } + + /** + * Create an ETL instance by reading a saved file + * + * @param filename + * path of the file + * @param format + * [[FileFormat.Binary]] or [[FileFormat.Json]] + * @return resulting ETL object + */ + public static ETL fromFile(String filename, FileFormat format) { + ETL result = null; + try { + switch (format) { + case Binary: + try (FileInputStream fileInputStream = new FileInputStream(filename); + GZIPInputStream gzipInputStream = new GZIPInputStream(fileInputStream); + ObjectInputStream objectInputStream = new ObjectInputStream(gzipInputStream)) { + result = (ETL) objectInputStream.readObject(); + } + break; + case GzipJson: + try (FileInputStream fileInputStream = new FileInputStream(filename); + GZIPInputStream gzipInputStream = new GZIPInputStream(fileInputStream); + JsonReader jr = new JsonReader(gzipInputStream)) { + result = (ETL) jr.readObject(); + } + break; + case Json: + try (FileInputStream fileInputStream = new FileInputStream(filename); JsonReader jr = new JsonReader(fileInputStream)) { + result = (ETL) jr.readObject(); + } + break; + } + result.filename = filename; + } catch (Exception exception) { + exception.printStackTrace(); + } + return result; + } + + public void discardCounts() { + for (Table table : sourceDb.getTables()) + for (Field field : table.getFields()) + field.setValueCounts(new String[0][]); + } + + public boolean hasStemTable() { + return getSourceDatabase().getTables().stream().anyMatch(Table::isStem); + } +} diff --git a/rabbitinahat/src/main/java/org/ohdsi/rabbitInAHat/dataModel/StemTableFactory.java b/rabbitinahat/src/main/java/org/ohdsi/rabbitInAHat/dataModel/StemTableFactory.java new file mode 100644 index 00000000..000413f3 --- /dev/null +++ b/rabbitinahat/src/main/java/org/ohdsi/rabbitInAHat/dataModel/StemTableFactory.java @@ -0,0 +1,127 @@ +package org.ohdsi.rabbitInAHat.dataModel; + +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import javax.swing.JOptionPane; + +import org.apache.commons.csv.CSVFormat; +import org.apache.commons.csv.CSVRecord; +import org.ohdsi.utilities.collections.Pair; + +public class StemTableFactory { + + public static void addStemTable(ETL etl) { + Database sourceDatabase = etl.getSourceDatabase(); + Database targetDatabase = etl.getTargetDatabase(); + InputStream tableStream; + InputStream mappingStream; + if (targetDatabase.getDbName().toLowerCase().equals("cdmv5.0.1")) { + tableStream = StemTableFactory.class.getResourceAsStream("StemTableV5.0.1.csv"); + mappingStream = StemTableFactory.class.getResourceAsStream("StemTableDefaultMappingV5.0.1.csv"); + } else if (targetDatabase.getDbName().toLowerCase().equals("cdmv5.1.0")) { + tableStream = StemTableFactory.class.getResourceAsStream("StemTableV5.1.0.csv"); + mappingStream = StemTableFactory.class.getResourceAsStream("StemTableDefaultMappingV5.1.0.csv"); + } else if (targetDatabase.getDbName().toLowerCase().equals("cdmv5.2.0")) { + tableStream = StemTableFactory.class.getResourceAsStream("StemTableV5.2.0.csv"); + mappingStream = StemTableFactory.class.getResourceAsStream("StemTableDefaultMappingV5.2.0.csv"); + } else if (targetDatabase.getDbName().toLowerCase().equals("cdmv5.3.0")) { + tableStream = StemTableFactory.class.getResourceAsStream("StemTableV5.3.0.csv"); + mappingStream = StemTableFactory.class.getResourceAsStream("StemTableDefaultMappingV5.3.0.csv"); + } else if (targetDatabase.getDbName().toLowerCase().equals("cdmv5.3.1")) { + tableStream = StemTableFactory.class.getResourceAsStream("StemTableV5.3.1.csv"); + mappingStream = StemTableFactory.class.getResourceAsStream("StemTableDefaultMappingV5.3.1.csv"); + } else if (targetDatabase.getDbName().toLowerCase().equals("cdmv6.0")) { + tableStream = StemTableFactory.class.getResourceAsStream("StemTableV6.0.csv"); + mappingStream = StemTableFactory.class.getResourceAsStream("StemTableDefaultMappingV6.0.csv"); + } else { + JOptionPane.showMessageDialog(null, "No stem table definition available for " + targetDatabase.getDbName(), "Error", JOptionPane.ERROR_MESSAGE); + return; + } + + try { + Table sourceStemTable = new Table(); + sourceStemTable.setStem(true); + for (CSVRecord row : CSVFormat.RFC4180.withHeader().parse(new InputStreamReader(tableStream))) { + if (sourceStemTable.getName() == null) + sourceStemTable.setName(row.get("TABLE_NAME").toLowerCase()); + Field field = new Field(row.get("COLUMN_NAME").toLowerCase(), sourceStemTable); + field.setNullable(row.get("IS_NULLABLE").equals("YES")); + field.setType(row.get("DATA_TYPE")); + field.setDescription(row.get("DESCRIPTION")); + field.setStem(true); + sourceStemTable.getFields().add(field); + } + sourceStemTable.setDb(sourceDatabase); + sourceDatabase.getTables().add(sourceStemTable); + Table targetStemTable = new Table(sourceStemTable); + targetStemTable.setDb(targetDatabase); + targetDatabase.getTables().add(targetStemTable); + + Mapping
mapping = etl.getTableToTableMapping(); + Map nameToTable = new HashMap(); + for (CSVRecord row : CSVFormat.RFC4180.withHeader().parse(new InputStreamReader(mappingStream))) { + String targetTableName = row.get("TARGET_TABLE"); + Table targetTable = nameToTable.get(targetTableName); + if (targetTable == null) { + targetTable = targetDatabase.getTableByName(targetTableName); + mapping.addSourceToTargetMap(sourceStemTable, targetTable); + nameToTable.put(targetTableName, targetTable); + } + Mapping fieldToFieldMapping = etl.getFieldToFieldMapping(sourceStemTable, targetTable); + Field sourceField = sourceStemTable.getFieldByName(row.get("SOURCE_FIELD")); + Field targetField = targetTable.getFieldByName(row.get("TARGET_FIELD")); + fieldToFieldMapping.addSourceToTargetMap(sourceField, targetField); + } + + } catch (IOException e) { + throw new RuntimeException(e.getMessage()); + } + + } + + public static void removeStemTable(ETL etl) { + // Find stem source and target tables + Mapping
mapping = etl.getTableToTableMapping(); + List> tablesMappingsToRemove = new ArrayList<>(); + for (ItemToItemMap sourceToTargetMap : mapping.getSourceToTargetMaps()) { + Table sourceTable = (Table) sourceToTargetMap.getSourceItem(); + Table targetTable = (Table) sourceToTargetMap.getTargetItem(); + if (sourceTable.isStem() || targetTable.isStem()) { + tablesMappingsToRemove.add(new Pair<>(sourceTable, targetTable)); + } + } + + // Remove stem table to table and field to field mappings + for (Pair tableTablePair : tablesMappingsToRemove) { + etl.getFieldToFieldMapping(tableTablePair.getItem1(), tableTablePair.getItem2()).removeAllSourceToTargetMaps(); + mapping.removeSourceToTargetMap(tableTablePair.getItem1(), tableTablePair.getItem2()); + } + + // Remove stem source table + Database sourceDatabase = etl.getSourceDatabase(); + List
newSourceTables = new ArrayList<>(); + for (Table table : sourceDatabase.getTables()) { + if (!table.isStem()) { + newSourceTables.add(table); + } + } + sourceDatabase.setTables(newSourceTables); + + // Remove stem target table + Database targetDatabase = etl.getTargetDatabase(); + List
newTargetTables = new ArrayList<>(); + for (Table table : targetDatabase.getTables()) { + if (!table.isStem()) { + newTargetTables.add(table); + } + } + targetDatabase.setTables(newTargetTables); + } + +} diff --git a/src/org/ohdsi/rabbitInAHat/RabbitInAHat128.png b/rabbitinahat/src/main/resources/org/ohdsi/rabbitInAHat/RabbitInAHat128.png similarity index 100% rename from src/org/ohdsi/rabbitInAHat/RabbitInAHat128.png rename to rabbitinahat/src/main/resources/org/ohdsi/rabbitInAHat/RabbitInAHat128.png diff --git a/src/org/ohdsi/rabbitInAHat/RabbitInAHat16.png b/rabbitinahat/src/main/resources/org/ohdsi/rabbitInAHat/RabbitInAHat16.png similarity index 100% rename from src/org/ohdsi/rabbitInAHat/RabbitInAHat16.png rename to rabbitinahat/src/main/resources/org/ohdsi/rabbitInAHat/RabbitInAHat16.png diff --git a/src/org/ohdsi/rabbitInAHat/RabbitInAHat256.png b/rabbitinahat/src/main/resources/org/ohdsi/rabbitInAHat/RabbitInAHat256.png similarity index 100% rename from src/org/ohdsi/rabbitInAHat/RabbitInAHat256.png rename to rabbitinahat/src/main/resources/org/ohdsi/rabbitInAHat/RabbitInAHat256.png diff --git a/src/org/ohdsi/rabbitInAHat/RabbitInAHat32.png b/rabbitinahat/src/main/resources/org/ohdsi/rabbitInAHat/RabbitInAHat32.png similarity index 100% rename from src/org/ohdsi/rabbitInAHat/RabbitInAHat32.png rename to rabbitinahat/src/main/resources/org/ohdsi/rabbitInAHat/RabbitInAHat32.png diff --git a/src/org/ohdsi/rabbitInAHat/RabbitInAHat48.png b/rabbitinahat/src/main/resources/org/ohdsi/rabbitInAHat/RabbitInAHat48.png similarity index 100% rename from src/org/ohdsi/rabbitInAHat/RabbitInAHat48.png rename to rabbitinahat/src/main/resources/org/ohdsi/rabbitInAHat/RabbitInAHat48.png diff --git a/src/org/ohdsi/rabbitInAHat/RabbitInAHat64.png b/rabbitinahat/src/main/resources/org/ohdsi/rabbitInAHat/RabbitInAHat64.png similarity index 100% rename from src/org/ohdsi/rabbitInAHat/RabbitInAHat64.png rename to rabbitinahat/src/main/resources/org/ohdsi/rabbitInAHat/RabbitInAHat64.png diff --git a/src/org/ohdsi/rabbitInAHat/dataModel/CDMV4.csv b/rabbitinahat/src/main/resources/org/ohdsi/rabbitInAHat/dataModel/CDMV4.csv similarity index 100% rename from src/org/ohdsi/rabbitInAHat/dataModel/CDMV4.csv rename to rabbitinahat/src/main/resources/org/ohdsi/rabbitInAHat/dataModel/CDMV4.csv diff --git a/src/org/ohdsi/rabbitInAHat/dataModel/CDMV5.0.1.csv b/rabbitinahat/src/main/resources/org/ohdsi/rabbitInAHat/dataModel/CDMV5.0.1.csv similarity index 100% rename from src/org/ohdsi/rabbitInAHat/dataModel/CDMV5.0.1.csv rename to rabbitinahat/src/main/resources/org/ohdsi/rabbitInAHat/dataModel/CDMV5.0.1.csv diff --git a/src/org/ohdsi/rabbitInAHat/dataModel/CDMV5.1.0.csv b/rabbitinahat/src/main/resources/org/ohdsi/rabbitInAHat/dataModel/CDMV5.1.0.csv similarity index 100% rename from src/org/ohdsi/rabbitInAHat/dataModel/CDMV5.1.0.csv rename to rabbitinahat/src/main/resources/org/ohdsi/rabbitInAHat/dataModel/CDMV5.1.0.csv diff --git a/src/org/ohdsi/rabbitInAHat/dataModel/CDMV5.2.0.csv b/rabbitinahat/src/main/resources/org/ohdsi/rabbitInAHat/dataModel/CDMV5.2.0.csv similarity index 100% rename from src/org/ohdsi/rabbitInAHat/dataModel/CDMV5.2.0.csv rename to rabbitinahat/src/main/resources/org/ohdsi/rabbitInAHat/dataModel/CDMV5.2.0.csv diff --git a/src/org/ohdsi/rabbitInAHat/dataModel/CDMV5.3.0.csv b/rabbitinahat/src/main/resources/org/ohdsi/rabbitInAHat/dataModel/CDMV5.3.0.csv similarity index 100% rename from src/org/ohdsi/rabbitInAHat/dataModel/CDMV5.3.0.csv rename to rabbitinahat/src/main/resources/org/ohdsi/rabbitInAHat/dataModel/CDMV5.3.0.csv diff --git a/src/org/ohdsi/rabbitInAHat/dataModel/CDMV5.3.1.csv b/rabbitinahat/src/main/resources/org/ohdsi/rabbitInAHat/dataModel/CDMV5.3.1.csv similarity index 100% rename from src/org/ohdsi/rabbitInAHat/dataModel/CDMV5.3.1.csv rename to rabbitinahat/src/main/resources/org/ohdsi/rabbitInAHat/dataModel/CDMV5.3.1.csv diff --git a/src/org/ohdsi/rabbitInAHat/dataModel/CDMV5.csv b/rabbitinahat/src/main/resources/org/ohdsi/rabbitInAHat/dataModel/CDMV5.csv similarity index 100% rename from src/org/ohdsi/rabbitInAHat/dataModel/CDMV5.csv rename to rabbitinahat/src/main/resources/org/ohdsi/rabbitInAHat/dataModel/CDMV5.csv diff --git a/rabbitinahat/src/main/resources/org/ohdsi/rabbitInAHat/dataModel/CDMV6.0.csv b/rabbitinahat/src/main/resources/org/ohdsi/rabbitInAHat/dataModel/CDMV6.0.csv new file mode 100644 index 00000000..ae737e7b --- /dev/null +++ b/rabbitinahat/src/main/resources/org/ohdsi/rabbitInAHat/dataModel/CDMV6.0.csv @@ -0,0 +1,367 @@ +field,required,type,description,table,schema +condition_occurrence_id,Yes,INTEGER,A unique identifier for each Condition Occurrence event.,condition_occurrence,cdm +person_id,Yes,INTEGER,A foreign key identifier to the Person who is experiencing the condition. The demographic details of that Person are stored in the PERSON table.,condition_occurrence,cdm +condition_concept_id,Yes,INTEGER,A foreign key that refers to a Standard Condition Concept identifier in the Standardized Vocabularies.,condition_occurrence,cdm +condition_start_date,Yes,DATE,The date when the instance of the Condition is recorded.,condition_occurrence,cdm +condition_start_datetime,No,DATETIME,The date and time when the instance of the Condition is recorded.,condition_occurrence,cdm +condition_end_date,No,DATE,The date when the instance of the Condition is considered to have ended.,condition_occurrence,cdm +condition_end_datetime,No,DATE,The date when the instance of the Condition is considered to have ended.,condition_occurrence,cdm +condition_type_concept_id,Yes,INTEGER,"A foreign key to the predefined Concept identifier in the Standardized Vocabularies reflecting the source data from which the condition was recorded, the level of standardization, and the type of occurrence.",condition_occurrence,cdm +stop_reason,No,VARCHAR(20),"The reason that the condition was no longer present, as indicated in the source data.",condition_occurrence,cdm +provider_id,No,INTEGER,A foreign key to the Provider in the PROVIDER table who was responsible for capturing (diagnosing) the Condition.,condition_occurrence,cdm +visit_occurrence_id,No,INTEGER,A foreign key to the visit in the VISIT_OCCURRENCE table during which the Condition was determined (diagnosed).,condition_occurrence,cdm +visit_detail_id,No,INTEGER,A foreign key to the visit in the VISIT_DETAIL table during which the Condition was determined (diagnosed).,condition_occurrence,cdm +condition_source_value,No,VARCHAR(50),The source code for the condition as it appears in the source data. This code is mapped to a standard condition concept in the Standardized Vocabularies and the original code is stored here for reference.,condition_occurrence,cdm +condition_source_concept_id,No,INTEGER,A foreign key to a Condition Concept that refers to the code used in the source.,condition_occurrence,cdm +condition_status_source_value,No,VARCHAR(50),The source code for the condition status as it appears in the source data.,condition_occurrence,cdm +condition_status_concept_id,No,INTEGER,A foreign key to the predefined Concept in the Standard Vocabulary reflecting the condition status,condition_occurrence,cdm +device_exposure_id,Yes,INTEGER,A system-generated unique identifier for each Device Exposure.,device_exposure,cdm +person_id,Yes,INTEGER,A foreign key identifier to the Person who is subjected to the Device. The demographic details of that person are stored in the Person table.,device_exposure,cdm +device_concept_id,Yes,INTEGER,A foreign key that refers to a Standard Concept identifier in the Standardized Vocabularies for the Device concept.,device_exposure,cdm +device_exposure_start_date,Yes,DATE,The date the Device or supply was applied or used.,device_exposure,cdm +device_exposure_start_datetime,No,DATETIME,The date and time the Device or supply was applied or used.,device_exposure,cdm +device_exposure_end_date,No,DATE,The date the Device or supply was removed from use.,device_exposure,cdm +device_exposure_end_datetime,No,DATETIME,The date and time the Device or supply was removed from use.,device_exposure,cdm +device_type_concept_id,Yes,INTEGER,A foreign key to the predefined Concept identifier in the Standardized Vocabularies reflecting the type of Device Exposure recorded. It indicates how the Device Exposure was represented in the source data.,device_exposure,cdm +unique_device_id,No,VARCHAR(50),A UDI or equivalent identifying the instance of the Device used in the Person.,device_exposure,cdm +quantity,No,INTEGER,The number of individual Devices used for the exposure.,device_exposure,cdm +provider_id,No,INTEGER,A foreign key to the provider in the PROVIDER table who initiated of administered the Device.,device_exposure,cdm +visit_occurrence_id,No,INTEGER,A foreign key to the visit in the VISIT_OCCURRENCE table during which the device was used.,device_exposure,cdm +visit_detail_id,No,INTEGER,A foreign key to the visit detail in the VISIT_DETAIL table during which the Drug Exposure was initiated.,device_exposure,cdm +device_source_value,No,VARCHAR(50),The source code for the Device as it appears in the source data. This code is mapped to a standard Device Concept in the Standardized Vocabularies and the original code is stored here for reference.,device_exposure,cdm +device_source_concept_id,No,INTEGER,A foreign key to a Device Concept that refers to the code used in the source.,device_exposure,cdm +drug_exposure_id,Yes,INTEGER,A system-generated unique identifier for each Drug utilization event.,drug_exposure,cdm +person_id,Yes,INTEGER,A foreign key identifier to the person who is subjected to the Drug. The demographic details of that person are stored in the person table.,drug_exposure,cdm +drug_concept_id,Yes,INTEGER,A foreign key that refers to a Standard Concept identifier in the Standardized Vocabularies for the Drug concept.,drug_exposure,cdm +drug_exposure_start_date,Yes,DATE,"The start date for the current instance of Drug utilization. Valid entries include a start date of a prescription, the date a prescription was filled, or the date on which a Drug administration procedure was recorded.",drug_exposure,cdm +drug_exposure_start_datetime,No,DATETIME,"The start date and time for the current instance of Drug utilization. Valid entries include a start date of a prescription, the date a prescription was filled, or the date on which a Drug administration procedure was recorded.",drug_exposure,cdm +drug_exposure_end_date,Yes,DATE,The end date for the current instance of Drug utilization. It is not available from all sources.,drug_exposure,cdm +drug_exposure_end_datetime,No,DATETIME,The end date and time for the current instance of Drug utilization. It is not available from all sources.,drug_exposure,cdm +verbatim_end_date,No,DATE,The known end date of a drug_exposure as provided by the source,drug_exposure,cdm +drug_type_concept_id,Yes,INTEGER,A foreign key to the predefined Concept identifier in the Standardized Vocabularies reflecting the type of Drug Exposure recorded. It indicates how the Drug Exposure was represented in the source data.,drug_exposure,cdm +stop_reason,No,VARCHAR(20),"The reason the Drug was stopped. Reasons include regimen completed, changed, removed, etc.",drug_exposure,cdm +refills,No,INTEGER,"The number of refills after the initial prescription. The initial prescription is not counted, values start with 0.",drug_exposure,cdm +quantity,No,FLOAT,The quantity of drug as recorded in the original prescription or dispensing record.,drug_exposure,cdm +days_supply,No,INTEGER,The number of days of supply of the medication as recorded in the original prescription or dispensing record.,drug_exposure,cdm +sig,No,VARCHAR(MAX),"The directions (""signetur"") on the Drug prescription as recorded in the original prescription (and printed on the container) or dispensing record.",drug_exposure,cdm +route_concept_id,No,INTEGER,A foreign key to a predefined concept in the Standardized Vocabularies reflecting the route of administration.,drug_exposure,cdm +lot_number,No,VARCHAR(50),An identifier assigned to a particular quantity or lot of Drug product from the manufacturer.,drug_exposure,cdm +provider_id,No,INTEGER,A foreign key to the provider in the PROVIDER table who initiated (prescribed or administered) the Drug Exposure.,drug_exposure,cdm +visit_occurrence_id,No,INTEGER,A foreign key to the Visit in the VISIT_OCCURRENCE table during which the Drug Exposure was initiated.,drug_exposure,cdm +visit_detail_id,No,INTEGER,A foreign key to the Visit Detail in the VISIT_DETAIL table during which the Drug Exposure was initiated.,drug_exposure,cdm +drug_source_value,No,VARCHAR(50),"The source code for the Drug as it appears in the source data. This code is mapped to a Standard Drug concept in the Standardized Vocabularies and the original code is, stored here for reference.",drug_exposure,cdm +drug_source_concept_id,No,INTEGER,A foreign key to a Drug Concept that refers to the code used in the source.,drug_exposure,cdm +route_source_value,No,VARCHAR(50),The information about the route of administration as detailed in the source.,drug_exposure,cdm +dose_unit_source_value,No,VARCHAR(50),The information about the dose unit as detailed in the source.,drug_exposure,cdm +domain_concept_id_1,Yes,INTEGER,"The concept representing the domain of fact one, from which the corresponding table can be inferred.",fact_relationship,cdm +fact_id_1,Yes,INTEGER,The unique identifier in the table corresponding to the domain of fact one.,fact_relationship,cdm +domain_concept_id_2,Yes,INTEGER,"The concept representing the domain of fact two, from which the corresponding table can be inferred.",fact_relationship,cdm +fact_id_2,Yes,INTEGER,The unique identifier in the table corresponding to the domain of fact two.,fact_relationship,cdm +relationship_concept_id,Yes,INTEGER,A foreign key to a Standard Concept ID of relationship in the Standardized Vocabularies.,fact_relationship,cdm +measurement_id,Yes,INTEGER,A unique identifier for each Measurement.,measurement,cdm +person_id,Yes,INTEGER,A foreign key identifier to the Person about whom the measurement was recorded. The demographic details of that Person are stored in the PERSON table.,measurement,cdm +measurement_concept_id,Yes,INTEGER,A foreign key to the standard measurement concept identifier in the Standardized Vocabularies.,measurement,cdm +measurement_date,Yes,DATE,The date of the Measurement.,measurement,cdm +measurement_datetime,No,DATETIME,"The date and time of the Measurement. Some database systems don't have a datatype of time. To accomodate all temporal analyses, datatype datetime can be used (combining measurement_date and measurement_time [forum discussion](http://forums.ohdsi.org/t/date-time-and-datetime-problem-and-the-world-of-hours-and-1day/314))",measurement,cdm +measurement_time,No,VARCHAR(10),The time of the Measurement. This is present for backwards compatibility and will deprecated in an upcoming version,measurement,cdm +measurement_type_concept_id,Yes,INTEGER,A foreign key to the predefined Concept in the Standardized Vocabularies reflecting the provenance from where the Measurement record was recorded.,measurement,cdm +operator_concept_id,No,INTEGER,"A foreign key identifier to the predefined Concept in the Standardized Vocabularies reflecting the mathematical operator that is applied to the value_as_number. Operators are <, <=, =, >=, >.",measurement,cdm +value_as_number,No,FLOAT,A Measurement result where the result is expressed as a numeric value.,measurement,cdm +value_as_concept_id,No,INTEGER,"A foreign key to a Measurement result represented as a Concept from the Standardized Vocabularies (e.g., positive/negative, present/absent, low/high, etc.).",measurement,cdm +unit_concept_id,No,INTEGER,A foreign key to a Standard Concept ID of Measurement Units in the Standardized Vocabularies.,measurement,cdm +range_low,No,FLOAT,The lower limit of the normal range of the Measurement result. The lower range is assumed to be of the same unit of measure as the Measurement value.,measurement,cdm +range_high,No,FLOAT,The upper limit of the normal range of the Measurement. The upper range is assumed to be of the same unit of measure as the Measurement value.,measurement,cdm +provider_id,No,INTEGER,A foreign key to the provider in the PROVIDER table who was responsible for initiating or obtaining the measurement.,measurement,cdm +visit_occurrence_id,No,INTEGER,A foreign key to the Visit in the VISIT_OCCURRENCE table during which the Measurement was recorded.,measurement,cdm +visit_detail_id,No,INTEGER,A foreign key to the Visit Detail in the VISIT_DETAIL table during which the Measurement was recorded.,measurement,cdm +measurement_source_value,No,VARCHAR(50),The Measurement name as it appears in the source data. This code is mapped to a Standard Concept in the Standardized Vocabularies and the original code is stored here for reference.,measurement,cdm +measurement_source_concept_id,No,INTEGER,A foreign key to a Concept in the Standard Vocabularies that refers to the code used in the source.,measurement,cdm +unit_source_value,No,VARCHAR(50),The source code for the unit as it appears in the source data. This code is mapped to a standard unit concept in the Standardized Vocabularies and the original code is stored here for reference.,measurement,cdm +value_source_value,No,VARCHAR(50),The source value associated with the content of the value_as_number or value_as_concept_id as stored in the source data.,measurement,cdm +note_id,Yes,INTEGER,A unique identifier for each note.,note,cdm +person_id,Yes,INTEGER,A foreign key identifier to the Person about whom the Note was recorded. The demographic details of that Person are stored in the PERSON table.,note,cdm +note_event_id,No,INTEGER,"A foreign key identifier to the event (e.g. Measurement, Procedure, Visit, Drug Exposure, etc) record during which the note was recorded.",note,cdm +note_event_field_concept_id,No,INTEGER,A foreign key to the predefined Concept in the Standardized Vocabularies reflecting the field to which the note_event_id is referring.,note,cdm +note_date,Yes,DATE,The date the note was recorded.,note,cdm +note_datetime,No,DATETIME,The date and time the note was recorded.,note,cdm +note_type_concept_id,Yes,INTEGER,"A foreign key to the predefined Concept in the Standardized Vocabularies reflecting the type, origin or provenance of the Note.",note,cdm +note_class_concept_id,Yes,INTEGER,A foreign key to the predefined Concept in the Standardized Vocabularies reflecting the HL7 LOINC Document Type Vocabulary classification of the note.,note,cdm +note_title,No,VARCHAR(250),The title of the Note as it appears in the source.,note,cdm +note_text,Yes,VARCHAR(MAX),The content of the Note.,note,cdm +encoding_concept_id,Yes,INTEGER,A foreign key to the predefined Concept in the Standardized Vocabularies reflecting the note character encoding type,note,cdm +language_concept_id,Yes,INTEGER,A foreign key to the predefined Concept in the Standardized Vocabularies reflecting the language of the note,note,cdm +provider_id,No,INTEGER,A foreign key to the Provider in the PROVIDER table who took the Note.,note,cdm +visit_occurrence_id,No,INTEGER,A foreign key to the Visit in the VISIT_OCCURRENCE table when the Note was taken.,note,cdm +visit_detail_id,No,INTEGER,A foreign key to the Visit in the VISIT_DETAIL table when the Note was taken.,note,cdm +note_source_value,No,VARCHAR(50),The source value associated with the origin of the Note,note,cdm +note_nlp_id,Yes,INTEGER,A unique identifier for each term extracted from a Note.,note_nlp,cdm +note_id,Yes,INTEGER,A foreign key to the note table note the term was extracted from.,note_nlp,cdm +section_concept_id,No,INTEGER,A foreign key to the predefined concept in the standardized vocabularies representing the section of the extracted term.,note_nlp,cdm +snippet,No,VARCHAR(250),A small window of text surrounding the term.,note_nlp,cdm +offset,No,VARCHAR(50),Character offset of the extracted term in the input note.,note_nlp,cdm +lexical_variant,Yes,VARCHAR(250),Raw text extracted from the NLP tool.,note_nlp,cdm +note_nlp_concept_id,No,INTEGER,A foreign key to the predefined concept in the standardized vocabularies reflecting the normalized concept for the extracted term. Domain of the term is represented as part of the concept table.,note_nlp,cdm +note_nlp_source_concept_id,No,INTEGER,A foreign key to a concept that refers to the code in the source vocabulary used by the NLP system,note_nlp,cdm +nlp_system,No,VARCHAR(250),Name and version of the NLP system that extracted the term.useful for data provenance.,note_nlp,cdm +nlp_date,Yes,DATE,The date of the note processing.useful for data provenance.,note_nlp,cdm +nlp_datetime,No,DATETIME,The date and time of the note processing. Useful for data provenance.,note_nlp,cdm +term_exists,No,VARCHAR(1),A summary modifier that signifies presence or absence of the term for a given patient. Useful for quick querying.,note_nlp,cdm +term_temporal,No,VARCHAR(50),An optional time modifier associated with the extracted term. (For now “past†or “present†only). Standardize it later.,note_nlp,cdm +term_modifiers,No,VARCHAR(2000),"A compact description of all the modifiers of the specific term extracted by the NLP system. (e.g. “son has rash†? “negated=no,subject=family, certainty=undef,conditional=false,general=falseâ€).",note_nlp,cdm +observation_id,Yes,INTEGER,A unique identifier for each observation.,observation,cdm +person_id,Yes,INTEGER,A foreign key identifier to the Person about whom the observation was recorded. The demographic details of that Person are stored in the PERSON table.,observation,cdm +observation_concept_id,Yes,INTEGER,A foreign key to the standard observation concept identifier in the Standardized Vocabularies.,observation,cdm +observation_date,Yes,DATE,The date of the observation.,observation,cdm +observation_datetime,No,DATETIME,The date and time of the observation.,observation,cdm +observation_type_concept_id,Yes,INTEGER,A foreign key to the predefined concept identifier in the Standardized Vocabularies reflecting the type of the observation.,observation,cdm +value_as_number,No,FLOAT,The observation result stored as a number. This is applicable to observations where the result is expressed as a numeric value.,observation,cdm +value_as_string,No,VARCHAR(60),The observation result stored as a string. This is applicable to observations where the result is expressed as verbatim text.,observation,cdm +value_as_concept_id,No,INTEGER,"A foreign key to an observation result stored as a Concept ID. This is applicable to observations where the result can be expressed as a Standard Concept from the Standardized Vocabularies (e.g., positive/negative, present/absent, low/high, etc.).",observation,cdm +qualifier_concept_id,No,INTEGER,"A foreign key to a Standard Concept ID for a qualifier (e.g., severity of drug-drug interaction alert)",observation,cdm +unit_concept_id,No,INTEGER,A foreign key to a Standard Concept ID of measurement units in the Standardized Vocabularies.,observation,cdm +provider_id,No,INTEGER,A foreign key to the provider in the PROVIDER table who was responsible for making the observation.,observation,cdm +visit_occurrence_id,No,INTEGER,A foreign key to the visit in the VISIT_OCCURRENCE table during which the observation was recorded.,observation,cdm +visit_detail_id,No,INTEGER,A foreign key to the visit in the VISIT_DETAIL table during which the observation was recorded.,observation,cdm +observation_source_value,No,VARCHAR(50),"The observation code as it appears in the source data. This code is mapped to a Standard Concept in the Standardized Vocabularies and the original code is, stored here for reference.",observation,cdm +observation_source_concept_id,No,INTEGER,A foreign key to a Concept that refers to the code used in the source.,observation,cdm +unit_source_value,No,VARCHAR(50),"The source code for the unit as it appears in the source data. This code is mapped to a standard unit concept in the Standardized Vocabularies and the original code is, stored here for reference.",observation,cdm +qualifier_source_value,No,VARCHAR(50),The source value associated with a qualifier to characterize the observation,observation,cdm +observation_event_id,No,INTEGER,"A foreign key to an event table (e.g., PROCEDURE_OCCURRENCE_ID).",observation,cdm +obs_event_field_concept_id,No,INTEGER,A foreign key that refers to a Standard Concept identifier in the Standardized Vocabularies referring to the field represented in the OBSERVATION_EVENT_ID.,observation,cdm +value_as_datetime,No,DATETIME,The observation result stored as a datetime value. This is applicable to observations where the result is expressed as a point in time.,observation,cdm +observation_period_id,Yes,INTEGER,A unique identifier for each observation period.,observation_period,cdm +person_id,Yes,INTEGER,A foreign key identifier to the person for whom the observation period is defined. The demographic details of that person are stored in the person table.,observation_period,cdm +observation_period_start_date,Yes,DATE,The start date of the observation period for which data are available from the data source.,observation_period,cdm +observation_period_end_date,Yes,DATE,The end date of the observation period for which data are available from the data source.,observation_period,cdm +period_type_concept_id,Yes,INTEGER,A foreign key identifier to the predefined concept in the Standardized Vocabularies reflecting the source of the observation period information,observation_period,cdm +person_id,Yes,INTEGER,A unique identifier for each person.,person,cdm +gender_concept_id,Yes,INTEGER,A foreign key that refers to an identifier in the CONCEPT table for the unique gender of the person.,person,cdm +year_of_birth,Yes,INTEGER,"The year of birth of the person. For data sources with date of birth, the year is extracted. For data sources where the year of birth is not available, the approximate year of birth is derived based on any age group categorization available.",person,cdm +month_of_birth,No,INTEGER,"The month of birth of the person. For data sources that provide the precise date of birth, the month is extracted and stored in this field.",person,cdm +day_of_birth,No,INTEGER,"The day of the month of birth of the person. For data sources that provide the precise date of birth, the day is extracted and stored in this field.",person,cdm +birth_datetime,No,DATETIME,The date and time of birth of the person.,person,cdm +death_datetime,No,DATETIME,The date and time of death of the person.,person,cdm +race_concept_id,Yes,INTEGER,A foreign key that refers to an identifier in the CONCEPT table for the unique race of the person.,person,cdm +ethnicity_concept_id,Yes,INTEGER,A foreign key that refers to the standard concept identifier in the Standardized Vocabularies for the ethnicity of the person.,person,cdm +location_id,No,INTEGER,"A foreign key to the place of residency for the person in the location table, where the detailed address information is stored.",person,cdm +provider_id,No,INTEGER,A foreign key to the primary care provider the person is seeing in the provider table.,person,cdm +care_site_id,No,INTEGER,"A foreign key to the site of primary care in the care_site table, where the details of the care site are stored.",person,cdm +person_source_value,No,VARCHAR(50),An (encrypted) key derived from the person identifier in the source data. This is necessary when a use case requires a link back to the person data at the source dataset.,person,cdm +gender_source_value,No,VARCHAR(50),The source code for the gender of the person as it appears in the source data. The person’s gender is mapped to a standard gender concept in the Standardized Vocabularies; the original value is stored here for reference.,person,cdm +gender_source_concept_id,No,INTEGER,A foreign key to the gender concept that refers to the code used in the source.,person,cdm +race_source_value,No,VARCHAR(50),The source code for the race of the person as it appears in the source data. The person race is mapped to a standard race concept in the Standardized Vocabularies and the original value is stored here for reference.,person,cdm +race_source_concept_id,No,INTEGER,A foreign key to the race concept that refers to the code used in the source.,person,cdm +ethnicity_source_value,No,VARCHAR(50),"The source code for the ethnicity of the person as it appears in the source data. The person ethnicity is mapped to a standard ethnicity concept in the Standardized Vocabularies and the original code is, stored here for reference.",person,cdm +ethnicity_source_concept_id,No,INTEGER,A foreign key to the ethnicity concept that refers to the code used in the source.,person,cdm +procedure_occurrence_id,Yes,INTEGER,A system-generated unique identifier for each Procedure Occurrence.,procedure_occurrence,cdm +person_id,Yes,INTEGER,A foreign key identifier to the Person who is subjected to the Procedure. The demographic details of that Person are stored in the PERSON table.,procedure_occurrence,cdm +procedure_concept_id,Yes,INTEGER,A foreign key that refers to a standard procedure Concept identifier in the Standardized Vocabularies.,procedure_occurrence,cdm +procedure_date,Yes,DATE,The date on which the Procedure was performed.,procedure_occurrence,cdm +procedure_datetime,No,DATETIME,The date and time on which the Procedure was performed.,procedure_occurrence,cdm +procedure_type_concept_id,Yes,INTEGER,A foreign key to the predefined Concept identifier in the Standardized Vocabularies reflecting the type of source data from which the procedure record is derived.,procedure_occurrence,cdm +modifier_concept_id,No,INTEGER,A foreign key to a Standard Concept identifier for a modifier to the Procedure (e.g. bilateral),procedure_occurrence,cdm +quantity,No,INTEGER,The quantity of procedures ordered or administered.,procedure_occurrence,cdm +provider_id,No,INTEGER,A foreign key to the provider in the PROVIDER table who was responsible for carrying out the procedure.,procedure_occurrence,cdm +visit_occurrence_id,No,INTEGER,A foreign key to the Visit in the VISIT_OCCURRENCE table during which the Procedure was carried out.,procedure_occurrence,cdm +visit_detail_id,No,INTEGER,A foreign key to the Visit Detail in the VISIT_DETAIL table during which the Procedure was carried out.,procedure_occurrence,cdm +procedure_source_value,No,VARCHAR(50),"The source code for the Procedure as it appears in the source data. This code is mapped to a standard procedure Concept in the Standardized Vocabularies and the original code is, stored here for reference. Procedure source codes are typically ICD-9-Proc, CPT-4, HCPCS or OPCS-4 codes.",procedure_occurrence,cdm +procedure_source_concept_id,No,INTEGER,A foreign key to a Procedure Concept that refers to the code used in the source.,procedure_occurrence,cdm +modifier_source_value,No,VARCHAR(50),The source code for the qualifier as it appears in the source data.,procedure_occurrence,cdm +specimen_id,Yes,INTEGER,A unique identifier for each specimen.,specimen,cdm +person_id,Yes,INTEGER,A foreign key identifier to the Person for whom the Specimen is recorded.,specimen,cdm +specimen_concept_id,Yes,INTEGER,A foreign key referring to a Standard Concept identifier in the Standardized Vocabularies for the Specimen.,specimen,cdm +specimen_type_concept_id,Yes,INTEGER,A foreign key referring to the Concept identifier in the Standardized Vocabularies reflecting the system of record from which the Specimen was represented in the source data.,specimen,cdm +specimen_date,Yes,DATE,The date the specimen was obtained from the Person.,specimen,cdm +specimen_datetime,No,DATETIME,The date and time on the date when the Specimen was obtained from the person.,specimen,cdm +quantity,No,FLOAT,The amount of specimen collection from the person during the sampling procedure.,specimen,cdm +unit_concept_id,No,INTEGER,A foreign key to a Standard Concept identifier for the Unit associated with the numeric quantity of the Specimen collection.,specimen,cdm +anatomic_site_concept_id,No,INTEGER,A foreign key to a Standard Concept identifier for the anatomic location of specimen collection.,specimen,cdm +disease_status_concept_id,No,INTEGER,A foreign key to a Standard Concept identifier for the Disease Status of specimen collection.,specimen,cdm +specimen_source_id,No,VARCHAR(50),The Specimen identifier as it appears in the source data.,specimen,cdm +specimen_source_value,No,VARCHAR(50),"The Specimen value as it appears in the source data. This value is mapped to a Standard Concept in the Standardized Vocabularies and the original code is, stored here for reference.",specimen,cdm +unit_source_value,No,VARCHAR(50),The information about the Unit as detailed in the source.,specimen,cdm +anatomic_site_source_value,No,VARCHAR(50),The information about the anatomic site as detailed in the source.,specimen,cdm +disease_status_source_value,No,VARCHAR(50),The information about the disease status as detailed in the source.,specimen,cdm +visit_detail_id,Yes,INTEGER,A unique identifier for each Person's visit or encounter at a healthcare provider.,visit_detail,cdm +person_id,Yes,INTEGER,A foreign key identifier to the Person for whom the visit is recorded. The demographic details of that Person are stored in the PERSON table.,visit_detail,cdm +visit_concept_id,Yes,INTEGER,A foreign key that refers to a visit Concept identifier in the Standardized Vocabularies.,visit_detail,cdm +visit_start_date,Yes,DATE,The start date of the visit.,visit_detail,cdm +visit_start_datetime,No,DATETIME,The date and time of the visit started.,visit_detail,cdm +visit_end_date,Yes,DATE,The end date of the visit. If this is a one-day visit the end date should match the start date.,visit_detail,cdm +visit_end_datetime,No,DATETIME,The date and time of the visit end.,visit_detail,cdm +visit_type_concept_id,Yes,INTEGER,A foreign key to the predefined Concept identifier in the Standardized Vocabularies reflecting the type of source data from which the visit record is derived.,visit_detail,cdm +provider_id,No,INTEGER,A foreign key to the provider in the provider table who was associated with the visit.,visit_detail,cdm +care_site_id,No,INTEGER,A foreign key to the care site in the care site table that was visited.,visit_detail,cdm +visit_source_value,No,STRING(50),The source code for the visit as it appears in the source data.,visit_detail,cdm +visit_source_concept_id,No,INTEGER,A foreign key to a Concept that refers to the code used in the source.,visit_detail,cdm +admitting_source_value,No,VARCHAR(50),The source code for the admitting source as it appears in the source data.,visit_detail,cdm +admitting_source_concept_id,No,INTEGER,A foreign key to the predefined concept in the Place of Service Vocabulary reflecting the admitting source for a visit.,visit_detail,cdm +admitted_from_source_value,No,VARCHAR(50),The source code for the discharge disposition as it appears in the source data.,visit_detail,cdm +admitted_from_concept_id,No,INTEGER,A foreign key to the predefined concept in the Place of Service Vocabulary reflecting the discharge disposition for a visit.,visit_detail,cdm +preceding_visit_detail_id,No,INTEGER,A foreign key to the VISIT_DETAIL table of the visit immediately preceding this visit,visit_detail,cdm +visit_detail_parent_id,No,INTEGER,A foreign key to the VISIT_DETAIL table record to represent the immediate parent visit-detail record.,visit_detail,cdm +visit_occurrence_id,Yes,INTEGER,"A foreign key that refers to the record in the VISIT_OCCURRENCE table. This is a required field, because for every visit_detail is a child of visit_occurrence and cannot exist without a corresponding parent record in visit_occurrence.",visit_detail,cdm +visit_occurrence_id,Yes,INTEGER,A unique identifier for each Person's visit or encounter at a healthcare provider.,visit_occurrence,cdm +person_id,Yes,INTEGER,A foreign key identifier to the Person for whom the visit is recorded. The demographic details of that Person are stored in the PERSON table.,visit_occurrence,cdm +visit_concept_id,Yes,INTEGER,A foreign key that refers to a visit Concept identifier in the Standardized Vocabularies.,visit_occurrence,cdm +visit_start_date,Yes,DATE,The start date of the visit.,visit_occurrence,cdm +visit_start_datetime,No,DATETIME,The date and time of the visit started.,visit_occurrence,cdm +visit_end_date,Yes,DATE,The end date of the visit. If this is a one-day visit the end date should match the start date.,visit_occurrence,cdm +visit_end_datetime,No,DATETIME,The date and time of the visit end.,visit_occurrence,cdm +visit_type_concept_id,Yes,INTEGER,A foreign key to the predefined Concept identifier in the Standardized Vocabularies reflecting the type of source data from which the visit record is derived.,visit_occurrence,cdm +provider_id,No,INTEGER,A foreign key to the provider in the provider table who was associated with the visit.,visit_occurrence,cdm +care_site_id,No,INTEGER,A foreign key to the care site in the care site table that was visited.,visit_occurrence,cdm +visit_source_value,No,VARCHAR(50),The source code for the visit as it appears in the source data.,visit_occurrence,cdm +visit_source_concept_id,No,INTEGER,A foreign key to a Concept that refers to the code used in the source.,visit_occurrence,cdm +admitted_from_concept_id,No,INTEGER,A foreign key to the predefined concept in the Place of Service Vocabulary reflecting the admitting source for a visit.,visit_occurrence,cdm +admitted_from_source_value,No,VARCHAR(50),The source code for the admitting source as it appears in the source data.,visit_occurrence,cdm +discharge_to_concept_id,No,INTEGER,A foreign key to the predefined concept in the Place of Service Vocabulary reflecting the discharge disposition for a visit.,visit_occurrence,cdm +discharge_to_source_value,No,VARCHAR(50),The source code for the discharge disposition as it appears in the source data.,visit_occurrence,cdm +preceding_visit_occurrence_id,No,INTEGER,A foreign key to the VISIT_OCCURRENCE table of the visit immediately preceding this visit,visit_occurrence,cdm +cohort_definition_id,Yes,INTEGER,A foreign key to a record in the COHORT_DEFINITION table containing relevant Cohort Definition information.,cohort,results +subject_id,Yes,INTEGER,"A foreign key to the subject in the cohort. These could be referring to records in the PERSON, PROVIDER, VISIT_OCCURRENCE table.",cohort,results +cohort_start_date,Yes,DATE,"The date when the Cohort Definition criteria for the Person, Provider or Visit first match.",cohort,results +cohort_end_date,Yes,DATE,"The date when the Cohort Definition criteria for the Person, Provider or Visit no longer match or the Cohort membership was terminated.",cohort,results +condition_era_id,Yes,INTEGER,A unique identifier for each Condition Era.,condition_era,cdm +person_id,Yes,INTEGER,A foreign key identifier to the Person who is experiencing the Condition during the Condition Era. The demographic details of that Person are stored in the PERSON table.,condition_era,cdm +condition_concept_id,Yes,INTEGER,A foreign key that refers to a standard Condition Concept identifier in the Standardized Vocabularies.,condition_era,cdm +condition_era_start_datetime,Yes,DATETIME,The start date for the Condition Era constructed from the individual instances of Condition Occurrences. It is the start date of the very first chronologically recorded instance of the condition.,condition_era,cdm +condition_era_end_datetime,Yes,DATETIME,The end date for the Condition Era constructed from the individual instances of Condition Occurrences. It is the end date of the final continuously recorded instance of the Condition.,condition_era,cdm +condition_occurrence_count,No,INTEGER,The number of individual Condition Occurrences used to construct the condition era.,condition_era,cdm +dose_era_id,Yes,INTEGER,A unique identifier for each Dose Era.,dose_era,cdm +person_id,Yes,INTEGER,A foreign key identifier to the Person who is subjected to the drug during the drug era. The demographic details of that Person are stored in the PERSON table.,dose_era,cdm +drug_concept_id,Yes,INTEGER,A foreign key that refers to a Standard Concept identifier in the Standardized Vocabularies for the active Ingredient Concept.,dose_era,cdm +unit_concept_id,Yes,INTEGER,A foreign key that refers to a Standard Concept identifier in the Standardized Vocabularies for the unit concept.,dose_era,cdm +dose_value,Yes,FLOAT,The numeric value of the dose.,dose_era,cdm +dose_era_start_datetime,Yes,DATETIME,The start date for the drug era constructed from the individual instances of drug exposures. It is the start date of the very first chronologically recorded instance of utilization of a drug.,dose_era,cdm +dose_era_end_datetime,Yes,DATETIME,The end date for the drug era constructed from the individual instance of drug exposures. It is the end date of the final continuously recorded instance of utilization of a drug.,dose_era,cdm +drug_era_id,Yes,INTEGER,A unique identifier for each Drug Era.,drug_era,cdm +person_id,Yes,INTEGER,A foreign key identifier to the Person who is subjected to the Drug during the fDrug Era. The demographic details of that Person are stored in the PERSON table.,drug_era,cdm +drug_concept_id,Yes,INTEGER,A foreign key that refers to a Standard Concept identifier in the Standardized Vocabularies for the Ingredient Concept.,drug_era,cdm +drug_era_start_datetime,Yes,DATETIME,The start date for the Drug Era constructed from the individual instances of Drug Exposures. It is the start date of the very first chronologically recorded instance of conutilization of a Drug.,drug_era,cdm +drug_era_end_datetime,Yes,DATETIME,The end date for the drug era constructed from the individual instance of drug exposures. It is the end date of the final continuously recorded instance of utilization of a drug.,drug_era,cdm +drug_exposure_count,No,INTEGER,The number of individual Drug Exposure occurrences used to construct the Drug Era.,drug_era,cdm +gap_days,No,INTEGER,The number of days that are not covered by DRUG_EXPOSURE records that were used to make up the era record.,drug_era,cdm +cost_id,Yes,INTEGER,A unique identifier for each COST record.,cost,cdm +person_id,Yes,INTEGER,A unique identifier for each PERSON.,cost,cdm +cost_event_id,Yes,INTEGER,"A foreign key identifier to the event (e.g. Measurement, Procedure, Visit, Drug Exposure, etc) record for which cost data are recorded.",cost,cdm +cost_event_field_concept_id,Yes,INTEGER,A foreign key identifier to a concept in the CONCEPT table representing the identity of the field represented by COST_EVENT_ID,cost,cdm +cost_concept_id,Yes,INTEGER,A foreign key that refers to a Standard Cost Concept identifier in the Standardized Vocabularies belonging to the 'Cost' vocabulary.,cost,cdm +cost_type_concept_id,Yes,INTEGER,A foreign key identifier to a concept in the CONCEPT table for the provenance or the source of the COST data and belonging to the 'Cost Type' vocabulary,cost,cdm +cost_source_concept_id,Yes,INTEGER,A foreign key to a Cost Concept that refers to the code used in the source.,cost,cdm +cost_source_value,No,VARCHAR(50),The source value for the cost as it appears in the source data,cost,cdm +currency_concept_id,Yes,INTEGER,"A foreign key identifier to the concept representing the 3-letter code used to delineate international currencies, such as USD for US Dollar. These belong to the 'Currency' vocabulary",cost,cdm +cost,Yes,FLOAT,The actual financial cost amount,cost,cdm +incurred_date,Yes,DATE,"The first date of service of the clinical event corresponding to the cost as in table capturing the information (e.g. date of visit, date of procedure, date of condition, date of drug etc).",cost,cdm +billed_date,No,DATE,The date a bill was generated for a service or encounter,cost,cdm +paid_date,No,DATE,The date payment was received for a service or encounter,cost,cdm +revenue_code_concept_id,Yes,INTEGER,A foreign key referring to a Standard Concept ID in the Standardized Vocabularies for Revenue codes belonging to the 'Revenue Code' vocabulary.,cost,cdm +drg_concept_id,Yes,INTEGER,A foreign key referring to a Standard Concept ID in the Standardized Vocabularies for DRG codes belonging to the 'DRG' vocabulary.,cost,cdm +revenue_code_source_value,No,VARCHAR(50),"The source value for the Revenue code as it appears in the source data, stored here for reference.",cost,cdm +drg_source_value,No,VARCHAR(50),"The source value for the 3-digit DRG source code as it appears in the source data, stored here for reference.",cost,cdm +payer_plan_period_id,No,INTEGER,"A foreign key to the PAYER_PLAN_PERIOD table, where the details of the Payer, Plan and Family are stored. Record the payer_plan_id that relates to the payer who contributed to the paid_by_payer field.",cost,cdm +payer_plan_period_id,Yes,INTEGER,"A identifier for each unique combination of payer, sponsor, plan, family code and time span.",payer_plan_period,cdm +person_id,Yes,INTEGER,A foreign key identifier to the Person covered by the payer. The demographic details of that Person are stored in the PERSON table.,payer_plan_period,cdm +contract_person_id,No,INTEGER,"A foreign key identifier to the person_id in person table, for the person who is the primary subscriber/contract owner for the record in the payer_plan_period table. Maybe the same person or different person, depending on who is the primary subscriber/contract owner.",payer_plan_period,cdm +payer_plan_period_start_date,Yes,DATE,The start date of the payer plan period.,payer_plan_period,cdm +payer_plan_period_end_date,Yes,DATE,The end date of the payer plan period.,payer_plan_period,cdm +payer_concept_id,No,INTEGER,A foreign key that refers to a Standard Payer concept identifiers in the Standardized Vocabularies,payer_plan_period,cdm +payer_source_value,No,VARCHAR(50),The source code for the payer as it appears in the source data.,payer_plan_period,cdm +payer_source_concept_id,No,INTEGER,A foreign key to a payer concept that refers to the code used in the source.,payer_plan_period,cdm +plan_concept_id,No,INTEGER,A foreign key that refers to a Standard plan that represents the health benefit plan in the Standardized Vocabularies,payer_plan_period,cdm +plan_source_value,No,VARCHAR(50),The source code for the Person's health benefit plan as it appears in the source data.,payer_plan_period,cdm +plan_source_concept_id,No,INTEGER,A foreign key to a plan concept that refers to the code used in the source.,payer_plan_period,cdm +contract_person_id,No,INTEGER,"A foreign key identifier to the person_id in person table, for the person who is the primary subscriber/contract owner for the record in the payer_plan_period table. Maybe the same person or different person, depending on who is the primary subscriber/contract owner.",payer_plan_period,cdm +contract_person_id,No,INTEGER,"A foreign key identifier to the person_id in person table, for the person who is the primary subscriber/contract owner for the record in the payer_plan_period table. Maybe the same person or different person, depending on who is the primary subscriber/contract owner.",payer_plan_period,cdm +contract_person_id,No,INTEGER,"A foreign key identifier to the person_id in person table, for the person who is the primary subscriber/contract owner for the record in the payer_plan_period table. Maybe the same person or different person, depending on who is the primary subscriber/contract owner.",payer_plan_period,cdm +sponsor_concept_id,No,INTEGER,A foreign key that refers to a Standard plan that represents the sponsor in the Standardized Vocabularies,payer_plan_period,cdm +sponsor_source_value,No,VARCHAR(50),The source code for the Person's sponsor of the health plan as it appears in the source data.,payer_plan_period,cdm +sponsor_source_concept_id*,No,INTEGER,A foreign key to a sponsor concept that refers to the code used in the source.,payer_plan_period,cdm +family_source_value,No,VARCHAR(50),The source code for the Person's family as it appears in the source data.,payer_plan_period,cdm +stop_reason_concept_id,No,INTEGER,A foreign key that refers to a Standard termination reason that represents the reason for the termination in the Standardized Vocabularies.,payer_plan_period,cdm +stop_reason_source_value,No,VARCHAR(50),The reason for stop-coverage of the record.,payer_plan_period,cdm +stop_reason_source_concept_id,No,INTEGER,A foreign key to a stop-coverage concept that refers to the code used in the source.,payer_plan_period,cdm +care_site_id,Yes,INTEGER,A unique identifier for each Care Site.,care_site,cdm +care_site_name,No,VARCHAR(255),The verbatim description or name of the Care Site as in data source,care_site,cdm +place_of_service_concept_id,No,INTEGER,A foreign key that refers to a Place of Service Concept ID in the Standardized Vocabularies.,care_site,cdm +location_id,No,INTEGER,"A foreign key to the geographic Location in the LOCATION table, where the detailed address information is stored.",care_site,cdm +care_site_source_value,No,VARCHAR(50),"The identifier for the Care Site in the source data, stored here for reference.",care_site,cdm +place_of_service_source_value,No,VARCHAR(50),"The source code for the Place of Service as it appears in the source data, stored here for reference.",care_site,cdm +location_id,Yes,INTEGER,A unique identifier for each geographic location.,location,cdm +address_1,No,VARCHAR(50),"The address field 1, typically used for the street address, as it appears in the source data.",location,cdm +address_2,No,VARCHAR(50),"The address field 2, typically used for additional detail such as buildings, suites, floors, as it appears in the source data.",location,cdm +city,No,VARCHAR(50),The city field as it appears in the source data.,location,cdm +state,No,VARCHAR(2),The state field as it appears in the source data.,location,cdm +zip,No,VARCHAR(9),The zip or postal code.,location,cdm +county,No,VARCHAR(20),The county.,location,cdm +country,No,VARCHAR(100),The country,location,cdm +location_source_value,No,VARCHAR(50),The verbatim information that is used to uniquely identify the location as it appears in the source data.,location,cdm +latitude,No,FLOAT,The geocoded latitude,location,cdm +longitude,No,FLOAT,The geocoded longitude,location,cdm +provider_id,Yes,INTEGER,A unique identifier for each Provider.,provider,cdm +provider_name,No,VARCHAR(255),A description of the Provider.,provider,cdm +npi,No,VARCHAR(20),The National Provider Identifier (NPI) of the provider.,provider,cdm +dea,No,VARCHAR(20),The Drug Enforcement Administration (DEA) number of the provider.,provider,cdm +specialty_concept_id,No,INTEGER,A foreign key to a Standard Specialty Concept ID in the Standardized Vocabularies.,provider,cdm +care_site_id,No,INTEGER,A foreign key to the main Care Site where the provider is practicing.,provider,cdm +year_of_birth,No,INTEGER,The year of birth of the Provider.,provider,cdm +gender_concept_id,No,INTEGER,The gender of the Provider.,provider,cdm +provider_source_value,No,VARCHAR(50),"The identifier used for the Provider in the source data, stored here for reference.",provider,cdm +specialty_source_value,No,VARCHAR(50),"The source code for the Provider specialty as it appears in the source data, stored here for reference.",provider,cdm +specialty_source_concept_id,No,INTEGER,A foreign key to a Concept that refers to the code used in the source.,provider,cdm +gender_source_value,No,VARCHAR(50),"The gender code for the Provider as it appears in the source data, stored here for reference.",provider,cdm +gender_source_concept_id,No,INTEGER,A foreign key to a Concept that refers to the code used in the source.,provider,cdm +cdm_source_name,Yes,VARCHAR(255),The full name of the source,cdm_source,cdm +cdm_source_abbreviation,No,VARCHAR(25),An abbreviation of the name,cdm_source,cdm +cdm_holder,No,VARCHAR(255),The name of the organization responsible for the development of the CDM instance,cdm_source,cdm +source_description,No,CLOB,A description of the source data origin and purpose for collection. The description may contain a summary of the period of time that is expected to be covered by this dataset.,cdm_source,cdm +source_documentation_reference,No,VARCHAR(255),URL or other external reference to location of source documentation,cdm_source,cdm +cdm_etl_reference,No,VARCHAR(255),URL or other external reference to location of ETL specification documentation and ETL source code,cdm_source,cdm +source_release_date,No,DATE,"The date for which the source data are most current, such as the last day of data capture",cdm_source,cdm +cdm_release_date,No,DATE,The date when the CDM was instantiated,cdm_source,cdm +cdm_version,No,VARCHAR(10),The version of CDM used,cdm_source,cdm +vocabulary_version,No,VARCHAR(20),The version of the vocabulary used,cdm_source,cdm +metadata_concept_id,Yes,INTEGER,A foreign key that refers to a Standard Metadata Concept identifier in the Standardized Vocabularies.,metadata,cdm +metadata_type_concept_id,Yes,INTEGER,A foreign key that refers to a Standard Type Concept identifier in the Standardized Vocabularies.,metadata,cdm +name,Yes,VARCHAR(250),The name of the Concept stored in metadata_concept_id or a description of the data being stored.,metadata,cdm +value_as_string,No,NVARCHAR,The metadata value stored as a string.,metadata,cdm +value_as_concept_id,No,INTEGER,A foreign key to a metadata value stored as a Concept ID.,metadata,cdm +metadata date,No,DATE,The date associated with the metadata,metadata,cdm +metadata_datetime,No,DATETIME,The date and time associated with the metadata,metadata,cdm +cohort_definition_id,Yes,INTEGER,A unique identifier for each Cohort.,cohort_definition,cdm +cohort_definition_name,Yes,VARCHAR(255),A short description of the Cohort.,cohort_definition,cdm +cohort_definition_description,No,VARCHAR(MAX),A complete description of the Cohort definition,cohort_definition,cdm +definition_type_concept_id,Yes,INTEGER,Type defining what kind of Cohort Definition the record represents and how the syntax may be executed,cohort_definition,cdm +cohort_definition_syntax,No,VARCHAR(MAX),Syntax or code to operationalize the Cohort definition,cohort_definition,cdm +subject_concept_id,Yes,INTEGER,"A foreign key to the Concept to which defines the domain of subjects that are members of the cohort (e.g., Person, Provider, Visit).",cohort_definition,cdm +cohort_initiation_date,No,DATE,A date to indicate when the Cohort was initiated in the COHORT table,cohort_definition,cdm +survey_conduct_id,Yes,INTEGER,Unique identifier for each completed survey.,survey_conduct,cdm +person_id,Yes,INTEGER,A foreign key identifier to the Person in the PERSON table about whom the survey was completed.,survey_conduct,cdm +survey_concept_id,Yes,INTEGER,A foreign key to the predefined Concept identifier in the Standardized Vocabularies reflecting the name and identity of the survey.,survey_conduct,cdm +survey_start_date,No,DATE,Date on which the survey was started.,survey_conduct,cdm +survey_start_datetime,No,DATETIME,Date and time the survey was started.,survey_conduct,cdm +survey_end_date,No,DATE,Date on which the survey was completed.,survey_conduct,cdm +survey_end_datetime,Yes,DATETIME,Date and time the survey was completed.,survey_conduct,cdm +provider_id,No ,INTEGER ,A foreign key to the provider in the provider table who was associated with the survey completion.,survey_conduct,cdm +assisted_concept_id,Yes,INTEGER,A foreign key to the predefined Concept identifier in the Standardized Vocabularies indicating whether the survey was completed with assistance.,survey_conduct,cdm +respondent_type_concept_id,Yes,INTEGER,"A foreign key to the predefined Concept identifier in the Standardized Vocabularies reflecting the respondent type. Example: Research Associate, Patient.",survey_conduct,cdm +timing_concept_id,Yes,INTEGER,"A foreign key to the predefined Concept identifier in the Standardized Vocabularies that refers to a certain timing. Example: 3 month follow-up, 6 month follow-up.",survey_conduct,cdm +collection_method_concept_id,Yes,INTEGER,"A foreign key to the predefined Concept identifier in the Standardized Vocabularies reflecting the data collection method (e.g. Paper, Telephone, Electronic Questionnaire).",survey_conduct,cdm +assisted_source_value,No,VARCHAR(50),"Source value representing whether patient required assistance to complete the survey. Example: “Completed without assistance”, ”Completed with assistance”.",survey_conduct,cdm +respondent_type_source_value,No,VARCHAR(100),Source code representing role of person who completed the survey.,survey_conduct,cdm +timing_source_value,No,VARCHAR(100),"Text string representing the timing of the survey. Example: Baseline, 6-month follow-up.",survey_conduct,cdm +collection_method_source_value,No,VARCHAR(100),The collection method as it appears in the source data.,survey_conduct,cdm +survey_source_value,No,VARCHAR(100),The survey name/title as it appears in the source data.,survey_conduct,cdm +survey_source_concept_id,Yes,INTEGER,A foreign key to a predefined Concept that refers to the code for the survey name/title used in the source.,survey_conduct,cdm +survey_source_identifier,No,VARCHAR(100),Unique identifier for each completed survey in source system.,survey_conduct,cdm +validated_survey_concept_id,Yes,INTEGER,A foreign key to the predefined Concept identifier in the Standardized Vocabularies reflecting the validation status of the survey.,survey_conduct,cdm +validated_survey_source_value,No,INTEGER,Source value representing the validation status of the survey.,survey_conduct,cdm +survey_version_number,No,VARCHAR(20),Version number of the questionnaire or survey used.,survey_conduct,cdm +visit_occurrence_id,No,INTEGER,A foreign key to the VISIT_OCCURRENCE table during which the survey was completed,survey_conduct,cdm +response_visit_occurrence_id,No,INTEGER ,A foreign key to the visit in the VISIT_OCCURRENCE table during which treatment was carried out that relates to this survey.,survey_conduct,cdm +location_id,Yes,INTEGER ,A foreign key to the location table.,location_history,cdm +relationship_type_concept_id,Yes,VARCHAR(50),The type of relationship between location and entity.,location_history,cdm +domain_id,Yes,VARCHAR(50),"The domain of the entity that is related to the location. Either PERSON, PROVIDER, or CARE_SITE.",location_history,cdm +entity_id,Yes,INTEGER ,"The unique identifier for the entity. References either person_id, provider_id, or care_site_id, depending on domain_id.",location_history,cdm +start_date,Yes,DATE,The date the relationship started.,location_history,cdm +end_date,No,DATE,The date the relationship ended.,location_history,cdm diff --git a/src/org/ohdsi/rabbitInAHat/dataModel/StemTableDefaultMappingV5.0.1.csv b/rabbitinahat/src/main/resources/org/ohdsi/rabbitInAHat/dataModel/StemTableDefaultMappingV5.0.1.csv similarity index 99% rename from src/org/ohdsi/rabbitInAHat/dataModel/StemTableDefaultMappingV5.0.1.csv rename to rabbitinahat/src/main/resources/org/ohdsi/rabbitInAHat/dataModel/StemTableDefaultMappingV5.0.1.csv index 3c1110cf..ed08e1fe 100644 --- a/src/org/ohdsi/rabbitInAHat/dataModel/StemTableDefaultMappingV5.0.1.csv +++ b/rabbitinahat/src/main/resources/org/ohdsi/rabbitInAHat/dataModel/StemTableDefaultMappingV5.0.1.csv @@ -57,7 +57,7 @@ STEM_TABLE,RANGE_LOW,MEASUREMENT,RANGE_LOW STEM_TABLE,RANGE_HIGH,MEASUREMENT,RANGE_HIGH STEM_TABLE,PROVIDER_ID,MEASUREMENT,PROVIDER_ID STEM_TABLE,VISIT_OCCURRENCE_ID,MEASUREMENT,VISIT_OCCURRENCE_ID -STEM_TABLE,SOURCE_VALUE,MEASUREMENT,MEAUSREMENT_SOURCE_VALUE +STEM_TABLE,SOURCE_VALUE,MEASUREMENT,MEASUREMENT_SOURCE_VALUE STEM_TABLE,SOURCE_CONCEPT_ID,MEASUREMENT,MEASUREMENT_SOURCE_CONCEPT_ID STEM_TABLE,UNIT_SOURCE_VALUE,MEASUREMENT,UNIT_SOURCE_VALUE STEM_TABLE,VALUE_SOURCE_VALUE,MEASUREMENT,VALUE_SOURCE_VALUE diff --git a/src/org/ohdsi/rabbitInAHat/dataModel/StemTableDefaultMappingV5.1.0.csv b/rabbitinahat/src/main/resources/org/ohdsi/rabbitInAHat/dataModel/StemTableDefaultMappingV5.1.0.csv similarity index 99% rename from src/org/ohdsi/rabbitInAHat/dataModel/StemTableDefaultMappingV5.1.0.csv rename to rabbitinahat/src/main/resources/org/ohdsi/rabbitInAHat/dataModel/StemTableDefaultMappingV5.1.0.csv index e82056f8..4d682dea 100644 --- a/src/org/ohdsi/rabbitInAHat/dataModel/StemTableDefaultMappingV5.1.0.csv +++ b/rabbitinahat/src/main/resources/org/ohdsi/rabbitInAHat/dataModel/StemTableDefaultMappingV5.1.0.csv @@ -63,7 +63,7 @@ STEM_TABLE,RANGE_LOW,MEASUREMENT,RANGE_LOW STEM_TABLE,RANGE_HIGH,MEASUREMENT,RANGE_HIGH STEM_TABLE,PROVIDER_ID,MEASUREMENT,PROVIDER_ID STEM_TABLE,VISIT_OCCURRENCE_ID,MEASUREMENT,VISIT_OCCURRENCE_ID -STEM_TABLE,SOURCE_VALUE,MEASUREMENT,MEAUSREMENT_SOURCE_VALUE +STEM_TABLE,SOURCE_VALUE,MEASUREMENT,MEASUREMENT_SOURCE_VALUE STEM_TABLE,SOURCE_CONCEPT_ID,MEASUREMENT,MEASUREMENT_SOURCE_CONCEPT_ID STEM_TABLE,UNIT_SOURCE_VALUE,MEASUREMENT,UNIT_SOURCE_VALUE STEM_TABLE,VALUE_SOURCE_VALUE,MEASUREMENT,VALUE_SOURCE_VALUE diff --git a/src/org/ohdsi/rabbitInAHat/dataModel/StemTableDefaultMappingV5.2.0.csv b/rabbitinahat/src/main/resources/org/ohdsi/rabbitInAHat/dataModel/StemTableDefaultMappingV5.2.0.csv similarity index 99% rename from src/org/ohdsi/rabbitInAHat/dataModel/StemTableDefaultMappingV5.2.0.csv rename to rabbitinahat/src/main/resources/org/ohdsi/rabbitInAHat/dataModel/StemTableDefaultMappingV5.2.0.csv index f764548a..ebd91f51 100644 --- a/src/org/ohdsi/rabbitInAHat/dataModel/StemTableDefaultMappingV5.2.0.csv +++ b/rabbitinahat/src/main/resources/org/ohdsi/rabbitInAHat/dataModel/StemTableDefaultMappingV5.2.0.csv @@ -63,7 +63,7 @@ STEM_TABLE,RANGE_LOW,MEASUREMENT,RANGE_LOW STEM_TABLE,RANGE_HIGH,MEASUREMENT,RANGE_HIGH STEM_TABLE,PROVIDER_ID,MEASUREMENT,PROVIDER_ID STEM_TABLE,VISIT_OCCURRENCE_ID,MEASUREMENT,VISIT_OCCURRENCE_ID -STEM_TABLE,SOURCE_VALUE,MEASUREMENT,MEAUSREMENT_SOURCE_VALUE +STEM_TABLE,SOURCE_VALUE,MEASUREMENT,MEASUREMENT_SOURCE_VALUE STEM_TABLE,SOURCE_CONCEPT_ID,MEASUREMENT,MEASUREMENT_SOURCE_CONCEPT_ID STEM_TABLE,UNIT_SOURCE_VALUE,MEASUREMENT,UNIT_SOURCE_VALUE STEM_TABLE,VALUE_SOURCE_VALUE,MEASUREMENT,VALUE_SOURCE_VALUE diff --git a/src/org/ohdsi/rabbitInAHat/dataModel/StemTableDefaultMappingV5.3.0.csv b/rabbitinahat/src/main/resources/org/ohdsi/rabbitInAHat/dataModel/StemTableDefaultMappingV5.3.0.csv similarity index 99% rename from src/org/ohdsi/rabbitInAHat/dataModel/StemTableDefaultMappingV5.3.0.csv rename to rabbitinahat/src/main/resources/org/ohdsi/rabbitInAHat/dataModel/StemTableDefaultMappingV5.3.0.csv index 83782349..fb5ef707 100644 --- a/src/org/ohdsi/rabbitInAHat/dataModel/StemTableDefaultMappingV5.3.0.csv +++ b/rabbitinahat/src/main/resources/org/ohdsi/rabbitInAHat/dataModel/StemTableDefaultMappingV5.3.0.csv @@ -63,7 +63,7 @@ STEM_TABLE,RANGE_LOW,MEASUREMENT,RANGE_LOW STEM_TABLE,RANGE_HIGH,MEASUREMENT,RANGE_HIGH STEM_TABLE,PROVIDER_ID,MEASUREMENT,PROVIDER_ID STEM_TABLE,VISIT_OCCURRENCE_ID,MEASUREMENT,VISIT_OCCURRENCE_ID -STEM_TABLE,SOURCE_VALUE,MEASUREMENT,MEAUSREMENT_SOURCE_VALUE +STEM_TABLE,SOURCE_VALUE,MEASUREMENT,MEASUREMENT_SOURCE_VALUE STEM_TABLE,SOURCE_CONCEPT_ID,MEASUREMENT,MEASUREMENT_SOURCE_CONCEPT_ID STEM_TABLE,UNIT_SOURCE_VALUE,MEASUREMENT,UNIT_SOURCE_VALUE STEM_TABLE,VALUE_SOURCE_VALUE,MEASUREMENT,VALUE_SOURCE_VALUE diff --git a/src/org/ohdsi/rabbitInAHat/dataModel/StemTableDefaultMappingV5.3.1.csv b/rabbitinahat/src/main/resources/org/ohdsi/rabbitInAHat/dataModel/StemTableDefaultMappingV5.3.1.csv similarity index 99% rename from src/org/ohdsi/rabbitInAHat/dataModel/StemTableDefaultMappingV5.3.1.csv rename to rabbitinahat/src/main/resources/org/ohdsi/rabbitInAHat/dataModel/StemTableDefaultMappingV5.3.1.csv index 83782349..fb5ef707 100644 --- a/src/org/ohdsi/rabbitInAHat/dataModel/StemTableDefaultMappingV5.3.1.csv +++ b/rabbitinahat/src/main/resources/org/ohdsi/rabbitInAHat/dataModel/StemTableDefaultMappingV5.3.1.csv @@ -63,7 +63,7 @@ STEM_TABLE,RANGE_LOW,MEASUREMENT,RANGE_LOW STEM_TABLE,RANGE_HIGH,MEASUREMENT,RANGE_HIGH STEM_TABLE,PROVIDER_ID,MEASUREMENT,PROVIDER_ID STEM_TABLE,VISIT_OCCURRENCE_ID,MEASUREMENT,VISIT_OCCURRENCE_ID -STEM_TABLE,SOURCE_VALUE,MEASUREMENT,MEAUSREMENT_SOURCE_VALUE +STEM_TABLE,SOURCE_VALUE,MEASUREMENT,MEASUREMENT_SOURCE_VALUE STEM_TABLE,SOURCE_CONCEPT_ID,MEASUREMENT,MEASUREMENT_SOURCE_CONCEPT_ID STEM_TABLE,UNIT_SOURCE_VALUE,MEASUREMENT,UNIT_SOURCE_VALUE STEM_TABLE,VALUE_SOURCE_VALUE,MEASUREMENT,VALUE_SOURCE_VALUE diff --git a/rabbitinahat/src/main/resources/org/ohdsi/rabbitInAHat/dataModel/StemTableDefaultMappingV6.0.csv b/rabbitinahat/src/main/resources/org/ohdsi/rabbitInAHat/dataModel/StemTableDefaultMappingV6.0.csv new file mode 100644 index 00000000..07a22beb --- /dev/null +++ b/rabbitinahat/src/main/resources/org/ohdsi/rabbitInAHat/dataModel/StemTableDefaultMappingV6.0.csv @@ -0,0 +1,124 @@ +SOURCE_TABLE,SOURCE_FIELD,TARGET_TABLE,TARGET_FIELD +STEM_TABLE,PERSON_ID,CONDITION_OCCURRENCE,PERSON_ID +STEM_TABLE,VISIT_OCCURRENCE_ID,CONDITION_OCCURRENCE,VISIT_OCCURRENCE_ID +STEM_TABLE,VISIT_DETAIL_ID,CONDITION_OCCURRENCE,VISIT_DETAIL_ID +STEM_TABLE,PROVIDER_ID,CONDITION_OCCURRENCE,PROVIDER_ID +STEM_TABLE,ID,CONDITION_OCCURRENCE,CONDITION_OCCURRENCE_ID +STEM_TABLE,CONCEPT_ID,CONDITION_OCCURRENCE,CONDITION_CONCEPT_ID +STEM_TABLE,SOURCE_VALUE,CONDITION_OCCURRENCE,CONDITION_SOURCE_VALUE +STEM_TABLE,SOURCE_CONCEPT_ID,CONDITION_OCCURRENCE,CONDITION_SOURCE_CONCEPT_ID +STEM_TABLE,TYPE_CONCEPT_ID,CONDITION_OCCURRENCE,CONDITION_TYPE_CONCEPT_ID +STEM_TABLE,START_DATE,CONDITION_OCCURRENCE,CONDITION_START_DATE +STEM_TABLE,START_DATETIME,CONDITION_OCCURRENCE,CONDITION_START_DATETIME +STEM_TABLE,END_DATE,CONDITION_OCCURRENCE,CONDITION_END_DATE +STEM_TABLE,END_DATETIME,CONDITION_OCCURRENCE,CONDITION_END_DATETIME +STEM_TABLE,STOP_REASON,CONDITION_OCCURRENCE,STOP_REASON +STEM_TABLE,CONDITION_STATUS_SOURCE_VALUE,CONDITION_OCCURRENCE,CONDITION_STATUS_SOURCE_VALUE +STEM_TABLE,CONDITION_STATUS_CONCEPT_ID,CONDITION_OCCURRENCE,CONDITION_STATUS_CONCEPT_ID +STEM_TABLE,PERSON_ID,DRUG_EXPOSURE,PERSON_ID +STEM_TABLE,ID,DRUG_EXPOSURE,DRUG_EXPOSURE_ID +STEM_TABLE,CONCEPT_ID,DRUG_EXPOSURE,DRUG_CONCEPT_ID +STEM_TABLE,START_DATE,DRUG_EXPOSURE,DRUG_EXPOSURE_START_DATE +STEM_TABLE,START_DATETIME,DRUG_EXPOSURE,DRUG_EXPOSURE_START_DATETIME +STEM_TABLE,END_DATE,DRUG_EXPOSURE,DRUG_EXPOSURE_END_DATE +STEM_TABLE,END_DATETIME,DRUG_EXPOSURE,DRUG_EXPOSURE_END_DATETIME +STEM_TABLE,VERBATIM_END_DATE,DRUG_EXPOSURE,VERBATIM_END_DATE +STEM_TABLE,TYPE_CONCEPT_ID,DRUG_EXPOSURE,DRUG_TYPE_CONCEPT_ID +STEM_TABLE,STOP_REASON,DRUG_EXPOSURE,STOP_REASON +STEM_TABLE,REFILLS,DRUG_EXPOSURE,REFILLS +STEM_TABLE,QUANTITY,DRUG_EXPOSURE,QUANTITY +STEM_TABLE,DAYS_SUPPLY,DRUG_EXPOSURE,DAYS_SUPPLY +STEM_TABLE,SIG,DRUG_EXPOSURE,SIG +STEM_TABLE,ROUTE_CONCEPT_ID,DRUG_EXPOSURE,ROUTE_CONCEPT_ID +STEM_TABLE,LOT_NUMBER,DRUG_EXPOSURE,LOT_NUMBER +STEM_TABLE,PROVIDER_ID,DRUG_EXPOSURE,PROVIDER_ID +STEM_TABLE,VISIT_OCCURRENCE_ID,DRUG_EXPOSURE,VISIT_OCCURRENCE_ID +STEM_TABLE,VISIT_DETAIL_ID,DRUG_EXPOSURE,VISIT_DETAIL_ID +STEM_TABLE,SOURCE_VALUE,DRUG_EXPOSURE,DRUG_SOURCE_VALUE +STEM_TABLE,SOURCE_CONCEPT_ID,DRUG_EXPOSURE,DRUG_SOURCE_CONCEPT_ID +STEM_TABLE,ROUTE_SOURCE_VALUE,DRUG_EXPOSURE,ROUTE_SOURCE_VALUE +STEM_TABLE,DOSE_UNIT_SOURCE_VALUE,DRUG_EXPOSURE,DOSE_UNIT_SOURCE_VALUE +STEM_TABLE,PERSON_ID,DEVICE_EXPOSURE,PERSON_ID +STEM_TABLE,ID,DEVICE_EXPOSURE,DEVICE_EXPOSURE_ID +STEM_TABLE,CONCEPT_ID,DEVICE_EXPOSURE,DEVICE_CONCEPT_ID +STEM_TABLE,START_DATE,DEVICE_EXPOSURE,DEVICE_EXPOSURE_START_DATE +STEM_TABLE,START_DATETIME,DEVICE_EXPOSURE,DEVICE_EXPOSURE_START_DATETIME +STEM_TABLE,END_DATE,DEVICE_EXPOSURE,DEVICE_EXPOSURE_END_DATE +STEM_TABLE,END_DATETIME,DEVICE_EXPOSURE,DEVICE_EXPOSURE_END_DATETIME +STEM_TABLE,TYPE_CONCEPT_ID,DEVICE_EXPOSURE,DEVICE_TYPE_CONCEPT_ID +STEM_TABLE,UNIQUE_DEVICE_ID,DEVICE_EXPOSURE,UNIQUE_DEVICE_ID +STEM_TABLE,QUANTITY,DEVICE_EXPOSURE,QUANTITY +STEM_TABLE,PROVIDER_ID,DEVICE_EXPOSURE,PROVIDER_ID +STEM_TABLE,VISIT_OCCURRENCE_ID,DEVICE_EXPOSURE,VISIT_OCCURRENCE_ID +STEM_TABLE,VISIT_DETAIL_ID,DEVICE_EXPOSURE,VISIT_DETAIL_ID +STEM_TABLE,SOURCE_VALUE,DEVICE_EXPOSURE,DEVICE_SOURCE_VALUE +STEM_TABLE,SOURCE_CONCEPT_ID,DEVICE_EXPOSURE,DEVICE_SOURCE_CONCEPT_ID +STEM_TABLE,PERSON_ID,MEASUREMENT,PERSON_ID +STEM_TABLE,ID,MEASUREMENT,MEASUREMENT_ID +STEM_TABLE,CONCEPT_ID,MEASUREMENT,MEASUREMENT_CONCEPT_ID +STEM_TABLE,START_DATE,MEASUREMENT,MEASUREMENT_DATE +STEM_TABLE,START_DATETIME,MEASUREMENT,MEASUREMENT_DATETIME +STEM_TABLE,TYPE_CONCEPT_ID,MEASUREMENT,MEASUREMENT_TYPE_CONCEPT_ID +STEM_TABLE,OPERATOR_CONCEPT_ID,MEASUREMENT,OPERATOR_CONCEPT_ID +STEM_TABLE,VALUE_AS_NUMBER,MEASUREMENT,VALUE_AS_NUMBER +STEM_TABLE,VALUE_AS_CONCEPT_ID,MEASUREMENT,VALUE_AS_CONCEPT_ID +STEM_TABLE,UNIT_CONCEPT_ID,MEASUREMENT,UNIT_CONCEPT_ID +STEM_TABLE,RANGE_LOW,MEASUREMENT,RANGE_LOW +STEM_TABLE,RANGE_HIGH,MEASUREMENT,RANGE_HIGH +STEM_TABLE,PROVIDER_ID,MEASUREMENT,PROVIDER_ID +STEM_TABLE,VISIT_OCCURRENCE_ID,MEASUREMENT,VISIT_OCCURRENCE_ID +STEM_TABLE,VISIT_DETAIL_ID,MEASUREMENT,VISIT_DETAIL_ID +STEM_TABLE,SOURCE_VALUE,MEASUREMENT,MEASUREMENT_SOURCE_VALUE +STEM_TABLE,SOURCE_CONCEPT_ID,MEASUREMENT,MEASUREMENT_SOURCE_CONCEPT_ID +STEM_TABLE,UNIT_SOURCE_VALUE,MEASUREMENT,UNIT_SOURCE_VALUE +STEM_TABLE,VALUE_SOURCE_VALUE,MEASUREMENT,VALUE_SOURCE_VALUE +STEM_TABLE,PERSON_ID,OBSERVATION,PERSON_ID +STEM_TABLE,ID,OBSERVATION,OBSERVATION_ID +STEM_TABLE,CONCEPT_ID,OBSERVATION,OBSERVATION_CONCEPT_ID +STEM_TABLE,START_DATE,OBSERVATION,OBSERVATION_DATE +STEM_TABLE,START_DATETIME,OBSERVATION,OBSERVATION_DATETIME +STEM_TABLE,TYPE_CONCEPT_ID,OBSERVATION,OBSERVATION_TYPE_CONCEPT_ID +STEM_TABLE,VALUE_AS_NUMBER,OBSERVATION,VALUE_AS_NUMBER +STEM_TABLE,VALUE_AS_STRING,OBSERVATION,VALUE_AS_STRING +STEM_TABLE,VALUE_AS_CONCEPT_ID,OBSERVATION,VALUE_AS_CONCEPT_ID +STEM_TABLE,QUALIFIER_CONCEPT_ID,OBSERVATION,QUALIFIER_CONCEPT_ID +STEM_TABLE,UNIT_CONCEPT_ID,OBSERVATION,UNIT_CONCEPT_ID +STEM_TABLE,PROVIDER_ID,OBSERVATION,PROVIDER_ID +STEM_TABLE,VISIT_OCCURRENCE_ID,OBSERVATION,VISIT_OCCURRENCE_ID +STEM_TABLE,VISIT_DETAIL_ID,OBSERVATION,VISIT_DETAIL_ID +STEM_TABLE,SOURCE_VALUE,OBSERVATION,OBSERVATION_SOURCE_VALUE +STEM_TABLE,SOURCE_CONCEPT_ID,OBSERVATION,OBSERVATION_SOURCE_CONCEPT_ID +STEM_TABLE,UNIT_SOURCE_VALUE,OBSERVATION,UNIT_SOURCE_VALUE +STEM_TABLE,QUALIFIER_SOURCE_VALUE,OBSERVATION,QUALIFIER_SOURCE_VALUE +STEM_TABLE,EVENT_ID,OBSERVATION,OBSERVATION_EVENT_ID +STEM_TABLE,EVENT_FIELD_CONCEPT_ID,OBSERVATION,OBS_EVENT_FIELD_CONCEPT_ID +STEM_TABLE,VALUE_AS_DATETIME,OBSERVATION,VALUE_AS_DATETIME +STEM_TABLE,PERSON_ID,PROCEDURE_OCCURRENCE,PERSON_ID +STEM_TABLE,ID,PROCEDURE_OCCURRENCE,PROCEDURE_OCCURRENCE_ID +STEM_TABLE,CONCEPT_ID,PROCEDURE_OCCURRENCE,PROCEDURE_CONCEPT_ID +STEM_TABLE,START_DATE,PROCEDURE_OCCURRENCE,PROCEDURE_DATE +STEM_TABLE,START_DATETIME,PROCEDURE_OCCURRENCE,PROCEDURE_DATETIME +STEM_TABLE,TYPE_CONCEPT_ID,PROCEDURE_OCCURRENCE,PROCEDURE_TYPE_CONCEPT_ID +STEM_TABLE,MODIFIER_CONCEPT_ID,PROCEDURE_OCCURRENCE,MODIFIER_CONCEPT_ID +STEM_TABLE,QUANTITY,PROCEDURE_OCCURRENCE,QUANTITY +STEM_TABLE,PROVIDER_ID,PROCEDURE_OCCURRENCE,PROVIDER_ID +STEM_TABLE,VISIT_OCCURRENCE_ID,PROCEDURE_OCCURRENCE,VISIT_OCCURRENCE_ID +STEM_TABLE,VISIT_DETAIL_ID,PROCEDURE_OCCURRENCE,VISIT_DETAIL_ID +STEM_TABLE,SOURCE_VALUE,PROCEDURE_OCCURRENCE,PROCEDURE_SOURCE_VALUE +STEM_TABLE,SOURCE_CONCEPT_ID,PROCEDURE_OCCURRENCE,PROCEDURE_SOURCE_CONCEPT_ID +STEM_TABLE,MODIFIER_SOURCE_VALUE,PROCEDURE_OCCURRENCE,MODIFIER_SOURCE_VALUE +STEM_TABLE,PERSON_ID,SPECIMEN,PERSON_ID +STEM_TABLE,ID,SPECIMEN,SPECIMEN_ID +STEM_TABLE,CONCEPT_ID,SPECIMEN,SPECIMEN_CONCEPT_ID +STEM_TABLE,TYPE_CONCEPT_ID,SPECIMEN,SPECIMEN_TYPE_CONCEPT_ID +STEM_TABLE,START_DATE,SPECIMEN,SPECIMEN_DATE +STEM_TABLE,START_DATETIME,SPECIMEN,SPECIMEN_DATETIME +STEM_TABLE,QUANTITY,SPECIMEN,QUANTITY +STEM_TABLE,UNIT_CONCEPT_ID,SPECIMEN,UNIT_CONCEPT_ID +STEM_TABLE,ANATOMIC_SITE_CONCEPT_ID,SPECIMEN,ANATOMIC_SITE_CONCEPT_ID +STEM_TABLE,DISEASE_STATUS_CONCEPT_ID,SPECIMEN,DISEASE_STATUS_CONCEPT_ID +STEM_TABLE,SPECIMEN_SOURCE_ID,SPECIMEN,SPECIMEN_SOURCE_ID +STEM_TABLE,SOURCE_VALUE,SPECIMEN,SPECIMEN_SOURCE_VALUE +STEM_TABLE,UNIT_SOURCE_VALUE,SPECIMEN,UNIT_SOURCE_VALUE +STEM_TABLE,ANATOMIC_SITE_SOURCE_VALUE,SPECIMEN,ANATOMIC_SITE_SOURCE_VALUE +STEM_TABLE,DISEASE_STATUS_SOURCE_VALUE,SPECIMEN,DISEASE_STATUS_SOURCE_VALUE diff --git a/src/org/ohdsi/rabbitInAHat/dataModel/StemTableV5.0.1.csv b/rabbitinahat/src/main/resources/org/ohdsi/rabbitInAHat/dataModel/StemTableV5.0.1.csv similarity index 100% rename from src/org/ohdsi/rabbitInAHat/dataModel/StemTableV5.0.1.csv rename to rabbitinahat/src/main/resources/org/ohdsi/rabbitInAHat/dataModel/StemTableV5.0.1.csv diff --git a/src/org/ohdsi/rabbitInAHat/dataModel/StemTableV5.1.0.csv b/rabbitinahat/src/main/resources/org/ohdsi/rabbitInAHat/dataModel/StemTableV5.1.0.csv similarity index 100% rename from src/org/ohdsi/rabbitInAHat/dataModel/StemTableV5.1.0.csv rename to rabbitinahat/src/main/resources/org/ohdsi/rabbitInAHat/dataModel/StemTableV5.1.0.csv diff --git a/src/org/ohdsi/rabbitInAHat/dataModel/StemTableV5.2.0.csv b/rabbitinahat/src/main/resources/org/ohdsi/rabbitInAHat/dataModel/StemTableV5.2.0.csv similarity index 100% rename from src/org/ohdsi/rabbitInAHat/dataModel/StemTableV5.2.0.csv rename to rabbitinahat/src/main/resources/org/ohdsi/rabbitInAHat/dataModel/StemTableV5.2.0.csv diff --git a/src/org/ohdsi/rabbitInAHat/dataModel/StemTableV5.3.0.csv b/rabbitinahat/src/main/resources/org/ohdsi/rabbitInAHat/dataModel/StemTableV5.3.0.csv similarity index 100% rename from src/org/ohdsi/rabbitInAHat/dataModel/StemTableV5.3.0.csv rename to rabbitinahat/src/main/resources/org/ohdsi/rabbitInAHat/dataModel/StemTableV5.3.0.csv diff --git a/rabbitinahat/src/main/resources/org/ohdsi/rabbitInAHat/dataModel/StemTableV5.3.1.csv b/rabbitinahat/src/main/resources/org/ohdsi/rabbitInAHat/dataModel/StemTableV5.3.1.csv new file mode 100644 index 00000000..c4ba2c81 --- /dev/null +++ b/rabbitinahat/src/main/resources/org/ohdsi/rabbitInAHat/dataModel/StemTableV5.3.1.csv @@ -0,0 +1,46 @@ +TABLE_NAME,COLUMN_NAME,IS_NULLABLE,DATA_TYPE,DESCRIPTION +STEM_TABLE,DOMAIN_ID,NO,CHARACTER VARYING,"A foreign key idenfifying the domain this event belongs to. The domain drives the target CDM table this event will be recorded in. If one is not set specify a default domain." +STEM_TABLE,PERSON_ID,NO,INTEGER,"A foreign key identifier to the Person. The demographic details of that Person are stored in the PERSON table." +STEM_TABLE,VISIT_OCCURRENCE_ID,YES,INTEGER,"A foreign key to the visit in the VISIT table." +STEM_TABLE,PROVIDER_ID,YES,INTEGER,"A foreign key to the Provider in the PROVIDER table who was responsible for capturing this event." +STEM_TABLE,ID,NO,INTEGER,"A unique identifier for identifying the event." +STEM_TABLE,CONCEPT_ID,NO,INTEGER,"A foreign key that refers to a Standard Condition Concept identifier in the Standardized Vocabularies." +STEM_TABLE,SOURCE_VALUE,YES,CHARACTER VARYING,"The source code for the Concept as it appears in the source data. This code is mapped to a Standard Concept in the Standardized Vocabularies and the original code is stored here for reference." +STEM_TABLE,SOURCE_CONCEPT_ID,YES,INTEGER,"A foreign key to a Concept that refers to the code used in the source." +STEM_TABLE,TYPE_CONCEPT_ID,NO,INTEGER,"A foreign key to the predefined Concept identifier in the Standardized Vocabularies reflecting the source data from which the event was recorded, the level of standardization, and the type of occurrence." +STEM_TABLE,START_DATE,NO,DATE,"The start date of the event." +STEM_TABLE,START_DATETIME,YES,DATETIME,"The start datetime of the event." +STEM_TABLE,END_DATE,YES,DATE,"The end date of the event." +STEM_TABLE,END_DATETIME,YES,DATETIME,"The end datetime of the event." +STEM_TABLE,VERBATIM_END_DATE,YES,DATE,"The verbatim end date (not derived) of the event." +STEM_TABLE,DAYS_SUPPLY,YES,INTEGER,"The number of days of supply of the medication as recorded in the original prescription or dispensing record." +STEM_TABLE,DOSE_UNIT_SOURCE_VALUE,YES,CHARACTER VARYING,"The information about the dose unit as detailed in the source." +STEM_TABLE,LOT_NUMBER,YES,CHARACTER VARYING,"An identifier assigned to a particular quantity or lot of Drug product from the manufacturer." +STEM_TABLE,MODIFIER_CONCEPT_ID,YES,INTEGER,"A foreign key to a Standard Concept identifier for a modifier to the Procedure (e.g. bilateral)" +STEM_TABLE,OPERATOR_CONCEPT_ID,YES,INTEGER,"A foreign key identifier to the predefined Concept in the Standardized Vocabularies reflecting the mathematical operator that is applied to the value_as_number. Operators are <, ?, =, ?, >." +STEM_TABLE,MODIFIER_CONCEPT_ID,YES,INTEGER,"A foreign key to a Standard Concept ID for a modifier (e.g., severity of drug-drug interaction alert)" +STEM_TABLE,MODIFIER_SOURCE_VALUE,YES,CHARACTER VARYING,"The source code for the modifier as it appears in the source data." +STEM_TABLE,QUANTITY,YES,INTEGER,"The number of individual Devices/Procedures/Drugs used for the exposure." +STEM_TABLE,RANGE_HIGH,YES,FLOAT,"The upper limit of the normal range of the Measurement. The upper range is assumed to be of the same unit of measure as the Measurement value." +STEM_TABLE,RANGE_LOW,YES,FLOAT,"The lower limit of the normal range of the Measurement result. The lower range is assumed to be of the same unit of measure as the Measurement value." +STEM_TABLE,REFILLS,YES,INTEGER,"The number of refills after the initial prescription. The initial prescription is not counted, values start with 0." +STEM_TABLE,ROUTE_CONCEPT_ID,YES,INTEGER,"A foreign key to a predefined concept in the Standardized Vocabularies reflecting the route of administration." +STEM_TABLE,ROUTE_SOURCE_VALUE,YES,CHARACTER VARYING,"The information about the route of administration as detailed in the source." +STEM_TABLE,SIG,YES,CHARACTER VARYING,"The directions (“signetur”) on the Drug prescription as recorded in the original prescription (and printed on the container) or dispensing record." +STEM_TABLE,STOP_REASON,YES,CHARACTER VARYING,"The reason that the condition was no longer present or the drug exposure stopped, as indicated in the source data." +STEM_TABLE,UNIQUE_DEVICE_ID,YES,CHARACTER VARYING,"A UDI or equivalent identifying the instance of the Device used in the Person." +STEM_TABLE,UNIT_CONCEPT_ID,YES,INTEGER,"A foreign key to a predefined concept in the Standardized Vocabularies reflecting the unit the effective_drug_dose/measurement/observation/specimen value is expressed." +STEM_TABLE,UNIT_SOURCE_VALUE,YES,CHARACTER VARYING,"The source code for the unit as it appears in the source data. This code is mapped to a standard unit concept in the Standardized Vocabularies and the original code is stored here for reference." +STEM_TABLE,VALUE_AS_CONCEPT_ID,YES,INTEGER,"A foreign key to a result represented as a Concept from the Standardized Vocabularies (e.g., positive/negative, present/absent, low/high, etc.)." +STEM_TABLE,VALUE_AS_NUMBER,YES,DECIMAL,"A result where the result is expressed as a numeric value." +STEM_TABLE,VALUE_AS_STRING,YES,CHARACTER VARYING,"The result stored as a string. This is applicable where the result is expressed as verbatim text." +STEM_TABLE,VALUE_SOURCE_VALUE,YES,CHARACTER VARYING,"The source value associated with the content of the value_as_number or value_as_concept as stored in the source data." +STEM_TABLE,ANATOMIC_SITE_CONCEPT_ID,YES,INTEGER,"A foreign key to a Standard Concept identifier for the anatomic location of specimen collection." +STEM_TABLE,DISEASE_STATUS_CONCEPT_ID,YES,INTEGER,"A foreign key to a Standard Concept identifier for the Disease Status of specimen collection." +STEM_TABLE,SPECIMEN_SOURCE_ID,YES,INTEGER,"The Specimen identifier as it appears in the source data." +STEM_TABLE,ANATOMIC_SITE_SOURCE_VALUE,YES,CHARACTER VARYING,"The information about the anatomic site as detailed in the source." +STEM_TABLE,DISEASE_STATUS_SOURCE_VALUE,YES,CHARACTER VARYING,"The information about the disease status as detailed in the source." +STEM_TABLE,CONDITION_STATUS_CONCEPT_ID,YES,CHARACTER VARYING,"A foreign key to the predefined Concept identifier in the Standardized Vocabularies reflecting the condition_status." +STEM_TABLE,CONDITION_STATUS_SOURCE_VALUE,YES,INTEGER,"The source code as it appears in the source data" +STEM_TABLE,QUALIFIER_CONCEPT_ID,YES,INTEGER,"A foreign key to a Standard Concept ID for a qualifier (e.g., severity of drug-drug interaction alert)" +STEM_TABLE,QUALIFIER_SOURCE_VALUE,YES,CHARACTER VARYING,"The source value associated with a qualifier to characterize the observation" diff --git a/src/org/ohdsi/rabbitInAHat/dataModel/StemTableV5.3.1.csv b/rabbitinahat/src/main/resources/org/ohdsi/rabbitInAHat/dataModel/StemTableV6.0.csv similarity index 86% rename from src/org/ohdsi/rabbitInAHat/dataModel/StemTableV5.3.1.csv rename to rabbitinahat/src/main/resources/org/ohdsi/rabbitInAHat/dataModel/StemTableV6.0.csv index 5526c3d2..33830b3b 100644 --- a/src/org/ohdsi/rabbitInAHat/dataModel/StemTableV5.3.1.csv +++ b/rabbitinahat/src/main/resources/org/ohdsi/rabbitInAHat/dataModel/StemTableV6.0.csv @@ -2,6 +2,7 @@ TABLE_NAME,COLUMN_NAME,IS_NULLABLE,DATA_TYPE,DESCRIPTION STEM_TABLE,DOMAIN_ID,NO,CHARACTER VARYING,A foreign key idenfifying the domain this event belongs to. The domain drives the target CDM table this event will be recorded in. If one is not set specify a default domain. STEM_TABLE,PERSON_ID,NO,INTEGER,A foreign key identifier to the Person. The demographic details of that Person are stored in the PERSON table. STEM_TABLE,VISIT_OCCURRENCE_ID,YES,INTEGER,A foreign key to the visit in the VISIT table. +STEM_TABLE,VISIT_DETAIL_ID,YES,INTEGER,A foreign key to the visit detail record in the VISIT_DETAIL table. STEM_TABLE,PROVIDER_ID,YES,INTEGER,A foreign key to the Provider in the PROVIDER table who was responsible for capturing this event. STEM_TABLE,ID,NO,INTEGER,A unique identifier for identifying the event. STEM_TABLE,CONCEPT_ID,NO,INTEGER,A foreign key that refers to a Standard Condition Concept identifier in the Standardized Vocabularies. @@ -12,7 +13,7 @@ STEM_TABLE,START_DATE,NO,DATE,The start date of the event. STEM_TABLE,START_DATETIME,YES,DATETIME,The start datetime of the event. STEM_TABLE,END_DATE,YES,DATE,The end date of the event. STEM_TABLE,END_DATETIME,YES,DATETIME,The end datetime of the event. -,VERBATIM_END_DATE,YES,DATE,The verbatim end date (not derived) of the event. +STEM_TABLE,VERBATIM_END_DATE,YES,DATE,The verbatim end date (not derived) of the event. STEM_TABLE,DAYS_SUPPLY,YES,INTEGER,The number of days of supply of the medication as recorded in the original prescription or dispensing record. STEM_TABLE,DOSE_UNIT_SOURCE_VALUE,YES,CHARACTER VARYING,The information about the dose unit as detailed in the source. STEM_TABLE,LOT_NUMBER,YES,CHARACTER VARYING,An identifier assigned to a particular quantity or lot of Drug product from the manufacturer. @@ -42,3 +43,8 @@ STEM_TABLE,ANATOMIC_SITE_SOURCE_VALUE,YES,CHARACTER VARYING,The information abou STEM_TABLE,DISEASE_STATUS_SOURCE_VALUE,YES,CHARACTER VARYING,The information about the disease status as detailed in the source. STEM_TABLE,CONDITION_STATUS_CONCEPT_ID,YES,CHARACTER VARYING,A foreign key to the predefined Concept identifier in the Standardized Vocabularies reflecting the condition_status. STEM_TABLE,CONDITION_STATUS_SOURCE_VALUE,YES,INTEGER,The source code as it appears in the source data +STEM_TABLE,EVENT_ID,YES,INTEGER,"A foreign key to an event table (e.g., PROCEDURE_OCCURRENCE_ID)." +STEM_TABLE,EVENT_FIELD_CONCEPT_ID,YES,INTEGER,A foreign key that refers to a Standard Concept identifier in the Standardized Vocabularies referring to the field represented in the EVENT_ID. +STEM_TABLE,VALUE_AS_DATETIME,YES,DATETIME,The observation result stored as a datetime value. This is applicable to observations where the result is expressed as a point in time. +STEM_TABLE,QUALIFIER_CONCEPT_ID,YES,INTEGER,"A foreign key to a Standard Concept ID for a qualifier (e.g., severity of drug-drug interaction alert)" +STEM_TABLE,QUALIFIER_SOURCE_VALUE,YES,CHARACTER VARYING,"The source value associated with a qualifier to characterize the observation" diff --git a/src/org/ohdsi/rabbitInAHat/dataModel/StemTableAdd.java b/src/org/ohdsi/rabbitInAHat/dataModel/StemTableAdd.java deleted file mode 100644 index 8b5b21c2..00000000 --- a/src/org/ohdsi/rabbitInAHat/dataModel/StemTableAdd.java +++ /dev/null @@ -1,82 +0,0 @@ -package org.ohdsi.rabbitInAHat.dataModel; - -import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.util.HashMap; -import java.util.Map; - -import javax.swing.JOptionPane; - -import org.apache.commons.csv.CSVFormat; -import org.apache.commons.csv.CSVRecord; - -public class StemTableAdd { - - public static void addStemTable(ETL etl) { - Database sourceDatabase = etl.getSourceDatabase(); - Database targetDatabase = etl.getTargetDatabase(); - InputStream tableStream; - InputStream mappingStream; - if (targetDatabase.getDbName().toLowerCase().equals("cdmv5.0.1")) { - tableStream = StemTableAdd.class.getResourceAsStream("StemTableV5.0.1.csv"); - mappingStream = StemTableAdd.class.getResourceAsStream("StemTableDefaultMappingV5.0.1.csv"); - } else if (targetDatabase.getDbName().toLowerCase().equals("cdmv5.1.0")) { - tableStream = StemTableAdd.class.getResourceAsStream("StemTableV5.1.0.csv"); - mappingStream = StemTableAdd.class.getResourceAsStream("StemTableDefaultMappingV5.1.0.csv"); - } else if (targetDatabase.getDbName().toLowerCase().equals("cdmv5.2.0")) { - tableStream = StemTableAdd.class.getResourceAsStream("StemTableV5.2.0.csv"); - mappingStream = StemTableAdd.class.getResourceAsStream("StemTableDefaultMappingV5.2.0.csv"); - } else if (targetDatabase.getDbName().toLowerCase().equals("cdmv5.3.0")) { - tableStream = StemTableAdd.class.getResourceAsStream("StemTableV5.3.0.csv"); - mappingStream = StemTableAdd.class.getResourceAsStream("StemTableDefaultMappingV5.3.0.csv"); - } else if (targetDatabase.getDbName().toLowerCase().equals("cdmv5.3.1")) { - tableStream = StemTableAdd.class.getResourceAsStream("StemTableV5.3.1.csv"); - mappingStream = StemTableAdd.class.getResourceAsStream("StemTableDefaultMappingV5.3.1.csv"); - } else { - JOptionPane.showMessageDialog(null, "No stem table definition available for " + targetDatabase.getDbName(), "Error", JOptionPane.ERROR_MESSAGE); - return; - } - - try { - Table sourceStemTable = new Table(); - sourceStemTable.setStem(true); - for (CSVRecord row : CSVFormat.RFC4180.withHeader().parse(new InputStreamReader(tableStream))) { - if (sourceStemTable.getName() == null) - sourceStemTable.setName(row.get("TABLE_NAME").toLowerCase()); - Field field = new Field(row.get("COLUMN_NAME").toLowerCase(), sourceStemTable); - field.setNullable(row.get("IS_NULLABLE").equals("YES")); - field.setType(row.get("DATA_TYPE")); - field.setDescription(row.get("DESCRIPTION")); - field.setStem(true); - sourceStemTable.getFields().add(field); - } - sourceStemTable.setDb(sourceDatabase); - sourceDatabase.getTables().add(sourceStemTable); - Table targetStemTable = new Table(sourceStemTable); - targetStemTable.setDb(targetDatabase); - targetDatabase.getTables().add(targetStemTable); - - Mapping
mapping = etl.getTableToTableMapping(); - Map nameToTable = new HashMap(); - for (CSVRecord row : CSVFormat.RFC4180.withHeader().parse(new InputStreamReader(mappingStream))) { - String targetTableName = row.get("TARGET_TABLE"); - Table targetTable = nameToTable.get(targetTableName); - if (targetTable == null) { - targetTable = targetDatabase.getTableByName(targetTableName); - mapping.addSourceToTargetMap(sourceStemTable, targetTable); - nameToTable.put(targetTableName, targetTable); - } - Mapping fieldToFieldMapping = etl.getFieldToFieldMapping(sourceStemTable, targetTable); - Field sourceField = sourceStemTable.getFieldByName(row.get("SOURCE_FIELD")); - Field targetField = targetTable.getFieldByName(row.get("TARGET_FIELD")); - fieldToFieldMapping.addSourceToTargetMap(sourceField, targetField); - } - - } catch (IOException e) { - throw new RuntimeException(e.getMessage()); - } - - } - -} diff --git a/src/org/ohdsi/utilities/RabbitInAHatLauncher.java b/src/org/ohdsi/utilities/RabbitInAHatLauncher.java deleted file mode 100644 index b65fae7b..00000000 --- a/src/org/ohdsi/utilities/RabbitInAHatLauncher.java +++ /dev/null @@ -1,31 +0,0 @@ -package org.ohdsi.utilities; - -import org.ohdsi.rabbitInAHat.RabbitInAHatMain; - -import java.util.Arrays; -import java.util.ArrayList; - -/* Adapted from code found on: - * http://silentdevelopment.blogspot.com/2010/03/how-to-set-or-increase-xmx-heap-memory.html - */ -public class RabbitInAHatLauncher { - private final static int MIN_HEAP = 1200; - - public static void main(String[] args) throws Exception { - - float heapSizeMegs = (Runtime.getRuntime().maxMemory() / 1024) / 1024; - - if (heapSizeMegs > MIN_HEAP) { - System.out.println("Launching with current VM"); - RabbitInAHatMain.main(args); - } else { - System.out.println("Starting new VM"); - String pathToJar = RabbitInAHatMain.class.getProtectionDomain().getCodeSource().getLocation().toURI().getPath(); - ArrayList command = new ArrayList(); - command.addAll(Arrays.asList("java", "-Xmx" + MIN_HEAP + "m", "-classpath", pathToJar, "org.ohdsi.rabbitInAHat.RabbitInAHatMain")); - command.addAll(Arrays.asList(args)); - ProcessBuilder pb = new ProcessBuilder(command); - pb.start(); - } - } -} diff --git a/src/org/ohdsi/utilities/WhiteRabbitLauncher.java b/src/org/ohdsi/utilities/WhiteRabbitLauncher.java deleted file mode 100644 index 9ac93460..00000000 --- a/src/org/ohdsi/utilities/WhiteRabbitLauncher.java +++ /dev/null @@ -1,25 +0,0 @@ -package org.ohdsi.utilities; - -import org.ohdsi.whiteRabbit.WhiteRabbitMain; - -/* Adapted from code found on: - * http://silentdevelopment.blogspot.com/2010/03/how-to-set-or-increase-xmx-heap-memory.html - */ -public class WhiteRabbitLauncher { - private final static int MIN_HEAP = 1200; - - public static void main(String[] args) throws Exception { - - float heapSizeMegs = (Runtime.getRuntime().maxMemory() / 1024) / 1024; - - if (heapSizeMegs > MIN_HEAP || args.length > 0) { - System.out.println("Launching with current VM"); - WhiteRabbitMain.main(args); - } else { - System.out.println("Starting new VM"); - String pathToJar = WhiteRabbitMain.class.getProtectionDomain().getCodeSource().getLocation().toURI().getPath(); - ProcessBuilder pb = new ProcessBuilder("java", "-Xmx" + MIN_HEAP + "m", "-classpath", pathToJar, "org.ohdsi.whiteRabbit.WhiteRabbitMain"); - pb.start(); - } - } -} diff --git a/src/org/ohdsi/utilities/collections/CountingSet.java b/src/org/ohdsi/utilities/collections/CountingSet.java deleted file mode 100644 index 731eb398..00000000 --- a/src/org/ohdsi/utilities/collections/CountingSet.java +++ /dev/null @@ -1,240 +0,0 @@ -/******************************************************************************* - * Copyright 2017 Observational Health Data Sciences and Informatics - * - * This file is part of WhiteRabbit - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - ******************************************************************************/ -package org.ohdsi.utilities.collections; - -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.Comparator; -import java.util.HashMap; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.Map.Entry; -import java.util.Set; - -/** - * Class for counting recurring objects. - * - * @author schuemie - * @param - */ -public class CountingSet implements Set { - - public Map key2count; - - public CountingSet() { - key2count = new HashMap(); - } - - public CountingSet(int capacity) { - key2count = new HashMap(capacity); - } - - public CountingSet(CountingSet set) { - key2count = new HashMap(set.key2count.size()); - for (Map.Entry entry : set.key2count.entrySet()) - key2count.put(entry.getKey(), new Count(entry.getValue().count)); - } - - public int getCount(T key) { - Count count = key2count.get(key); - if (count == null) - return 0; - else - return count.count; - } - - /** - * Computes the sum of the counts - * - * @return - */ - public int getSum() { - int sum = 0; - for (Count count : key2count.values()) - sum += count.count; - return sum; - } - - /** - * Returns the maximum count - * - * @return - */ - public int getMax() { - int max = 0; - for (Count count : key2count.values()) - if (count.count > max) - max = count.count; - return max; - } - - /** - * Computes the mean of the counts - * - * @return - */ - public double getMean() { - return (getSum() / (double) key2count.size()); - } - - /** - * Computes the standard deviations of the counts - * - * @return - */ - public double getSD() { - double mean = getMean(); - double sum = 0; - for (Count count : key2count.values()) - sum += sqr(count.count - mean); - return Math.sqrt(sum / (double) key2count.size()); - } - - private double sqr(double d) { - return d * d; - } - - public int size() { - return key2count.size(); - } - - public boolean isEmpty() { - return key2count.isEmpty(); - } - - public boolean contains(Object arg0) { - return key2count.containsKey(arg0); - } - - public Iterator iterator() { - return key2count.keySet().iterator(); - } - - public Object[] toArray() { - return key2count.keySet().toArray(); - } - - @SuppressWarnings("unchecked") - public Object[] toArray(Object[] arg0) { - return key2count.keySet().toArray(arg0); - } - - public boolean add(T arg0) { - Count count = key2count.get(arg0); - if (count == null) { - count = new Count(); - key2count.put(arg0, count); - return true; - } else { - count.count++; - return false; - } - } - - public boolean add(T arg0, int inc) { - Count count = key2count.get(arg0); - if (count == null) { - count = new Count(); - count.count = inc; - key2count.put(arg0, count); - return true; - } else { - count.count += inc; - return false; - } - } - - public boolean remove(Object arg0) { - - return (key2count.remove(arg0) != null); - } - - public boolean containsAll(Collection arg0) { - return key2count.keySet().containsAll(arg0); - } - - public boolean addAll(Collection arg0) { - boolean changed = false; - for (T object : arg0) { - if (add(object)) - changed = true; - } - return changed; - } - - public boolean retainAll(Collection arg0) { - return key2count.keySet().retainAll(arg0); - } - - public boolean removeAll(Collection arg0) { - return key2count.keySet().removeAll(arg0); - } - - public void clear() { - key2count.clear(); - } - - /** - * Keep the n most frequent values, remove the rest - * - * @param n - */ - public void keepTopN(int n) { - if (size() < n) - return; - List> list = new ArrayList>(key2count.entrySet()); - Collections.sort(list, new Comparator>() { - - @Override - public int compare(Entry arg0, Entry arg1) { - return IntegerComparator.compare(arg1.getValue().count, arg0.getValue().count); - } - }); - - Map newMap = new HashMap(n); - for (int i = 0; i < n; i++) - newMap.put(list.get(i).getKey(), list.get(i).getValue()); - - key2count = newMap; - } - - public static class Count { - public int count = 1; - - public Count() { - } - - public Count(int count) { - this.count = count; - } - - } - - public void printCounts() { - List> result = new ArrayList>(key2count.entrySet()); - Collections.sort(result, new Comparator>() { - public int compare(Entry o1, Entry o2) { - return IntegerComparator.compare(o2.getValue().count, o1.getValue().count); - } - }); - for (Map.Entry entry : result) - System.out.println(entry.getKey() + "\t" + entry.getValue().count); - } -} diff --git a/whiterabbit/.classpath b/whiterabbit/.classpath new file mode 100644 index 00000000..6d7587a8 --- /dev/null +++ b/whiterabbit/.classpath @@ -0,0 +1,31 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/whiterabbit/.project b/whiterabbit/.project new file mode 100644 index 00000000..1d176399 --- /dev/null +++ b/whiterabbit/.project @@ -0,0 +1,23 @@ + + + whiterabbit + + + + + + org.eclipse.jdt.core.javabuilder + + + + + org.eclipse.m2e.core.maven2Builder + + + + + + org.eclipse.jdt.core.javanature + org.eclipse.m2e.core.maven2Nature + + diff --git a/whiterabbit/.settings/org.eclipse.jdt.core.prefs b/whiterabbit/.settings/org.eclipse.jdt.core.prefs new file mode 100644 index 00000000..714351ae --- /dev/null +++ b/whiterabbit/.settings/org.eclipse.jdt.core.prefs @@ -0,0 +1,5 @@ +eclipse.preferences.version=1 +org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.8 +org.eclipse.jdt.core.compiler.compliance=1.8 +org.eclipse.jdt.core.compiler.problem.forbiddenReference=warning +org.eclipse.jdt.core.compiler.source=1.8 diff --git a/whiterabbit/.settings/org.eclipse.m2e.core.prefs b/whiterabbit/.settings/org.eclipse.m2e.core.prefs new file mode 100644 index 00000000..f897a7f1 --- /dev/null +++ b/whiterabbit/.settings/org.eclipse.m2e.core.prefs @@ -0,0 +1,4 @@ +activeProfiles= +eclipse.preferences.version=1 +resolveWorkspaceProjects=true +version=1 diff --git a/whiterabbit/pom.xml b/whiterabbit/pom.xml new file mode 100644 index 00000000..931dff95 --- /dev/null +++ b/whiterabbit/pom.xml @@ -0,0 +1,42 @@ + + + + leporidae + org.ohdsi + 0.8.0-SNAPSHOT + + 4.0.0 + + whiterabbit + WhiteRabbit + jar + + + + + org.codehaus.mojo + appassembler-maven-plugin + + ${project.parent.basedir}/dist + -Xmx1200m + + + org.ohdsi.whiteRabbit.WhiteRabbitMain + whiteRabbit + + + + + + + + + + org.ohdsi + rabbit-core + ${project.version} + + + \ No newline at end of file diff --git a/src/org/ohdsi/whiteRabbit/Console.java b/whiterabbit/src/main/java/org/ohdsi/whiteRabbit/Console.java similarity index 93% rename from src/org/ohdsi/whiteRabbit/Console.java rename to whiterabbit/src/main/java/org/ohdsi/whiteRabbit/Console.java index 32aa2294..374be963 100644 --- a/src/org/ohdsi/whiteRabbit/Console.java +++ b/whiterabbit/src/main/java/org/ohdsi/whiteRabbit/Console.java @@ -1,5 +1,5 @@ /******************************************************************************* - * Copyright 2017 Observational Health Data Sciences and Informatics + * Copyright 2019 Observational Health Data Sciences and Informatics * * This file is part of WhiteRabbit * diff --git a/src/org/ohdsi/whiteRabbit/DbSettings.java b/whiterabbit/src/main/java/org/ohdsi/whiteRabbit/DbSettings.java similarity index 92% rename from src/org/ohdsi/whiteRabbit/DbSettings.java rename to whiterabbit/src/main/java/org/ohdsi/whiteRabbit/DbSettings.java index d628307c..b40433aa 100644 --- a/src/org/ohdsi/whiteRabbit/DbSettings.java +++ b/whiterabbit/src/main/java/org/ohdsi/whiteRabbit/DbSettings.java @@ -1,5 +1,5 @@ /******************************************************************************* - * Copyright 2017 Observational Health Data Sciences and Informatics + * Copyright 2019 Observational Health Data Sciences and Informatics * * This file is part of WhiteRabbit * diff --git a/src/org/ohdsi/whiteRabbit/ErrorReport.java b/whiterabbit/src/main/java/org/ohdsi/whiteRabbit/ErrorReport.java similarity index 96% rename from src/org/ohdsi/whiteRabbit/ErrorReport.java rename to whiterabbit/src/main/java/org/ohdsi/whiteRabbit/ErrorReport.java index bf0da129..28fe0eea 100644 --- a/src/org/ohdsi/whiteRabbit/ErrorReport.java +++ b/whiterabbit/src/main/java/org/ohdsi/whiteRabbit/ErrorReport.java @@ -1,5 +1,5 @@ /******************************************************************************* - * Copyright 2017 Observational Health Data Sciences and Informatics + * Copyright 2019 Observational Health Data Sciences and Informatics * * This file is part of WhiteRabbit * diff --git a/src/org/ohdsi/whiteRabbit/ObjectExchange.java b/whiterabbit/src/main/java/org/ohdsi/whiteRabbit/ObjectExchange.java similarity index 83% rename from src/org/ohdsi/whiteRabbit/ObjectExchange.java rename to whiterabbit/src/main/java/org/ohdsi/whiteRabbit/ObjectExchange.java index 379a9bfa..643e87a0 100644 --- a/src/org/ohdsi/whiteRabbit/ObjectExchange.java +++ b/whiterabbit/src/main/java/org/ohdsi/whiteRabbit/ObjectExchange.java @@ -1,31 +1,31 @@ -/******************************************************************************* - * Copyright 2017 Observational Health Data Sciences and Informatics - * - * This file is part of WhiteRabbit - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - ******************************************************************************/ -package org.ohdsi.whiteRabbit; - -import javax.swing.JFrame; - -import org.ohdsi.rabbitInAHat.dataModel.ETL; - -/** - * This class is used to hold global variables - */ -public class ObjectExchange { - public static JFrame frame; - public static Console console; - public static ETL etl; -} +/******************************************************************************* + * Copyright 2019 Observational Health Data Sciences and Informatics + * + * This file is part of WhiteRabbit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + ******************************************************************************/ +package org.ohdsi.whiteRabbit; + +import javax.swing.JFrame; + +/** + * This class is used to hold global variables + */ +public final class ObjectExchange { + // utility class + private ObjectExchange() {} + + public static JFrame frame; + public static Console console; +} diff --git a/src/org/ohdsi/whiteRabbit/WhiteRabbitMain.java b/whiterabbit/src/main/java/org/ohdsi/whiteRabbit/WhiteRabbitMain.java similarity index 96% rename from src/org/ohdsi/whiteRabbit/WhiteRabbitMain.java rename to whiterabbit/src/main/java/org/ohdsi/whiteRabbit/WhiteRabbitMain.java index 4ed37015..23197c5c 100644 --- a/src/org/ohdsi/whiteRabbit/WhiteRabbitMain.java +++ b/whiterabbit/src/main/java/org/ohdsi/whiteRabbit/WhiteRabbitMain.java @@ -1,1098 +1,1097 @@ -/******************************************************************************* - * Copyright 2017 Observational Health Data Sciences and Informatics - * - * This file is part of WhiteRabbit - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - ******************************************************************************/ -package org.ohdsi.whiteRabbit; - -import java.awt.BorderLayout; -import java.awt.Color; -import java.awt.Desktop; -import java.awt.Dimension; -import java.awt.GridBagConstraints; -import java.awt.GridBagLayout; -import java.awt.GridLayout; -import java.awt.Image; -import java.awt.MediaTracker; -import java.awt.Toolkit; -import java.awt.event.ActionEvent; -import java.awt.event.ActionListener; -import java.awt.event.ItemEvent; -import java.awt.event.ItemListener; -import java.awt.event.WindowAdapter; -import java.awt.event.WindowEvent; -import java.io.File; -import java.io.IOException; -import java.io.PrintStream; -import java.net.URI; -import java.net.URISyntaxException; -import java.util.ArrayList; -import java.util.List; -import java.util.Vector; - -import javax.swing.BorderFactory; -import javax.swing.Box; -import javax.swing.BoxLayout; -import javax.swing.JButton; -import javax.swing.JCheckBox; -import javax.swing.JComboBox; -import javax.swing.JComponent; -import javax.swing.JDialog; -import javax.swing.JFileChooser; -import javax.swing.JFrame; -import javax.swing.JLabel; -import javax.swing.JList; -import javax.swing.JMenu; -import javax.swing.JMenuBar; -import javax.swing.JMenuItem; -import javax.swing.JOptionPane; -import javax.swing.JPanel; -import javax.swing.JPasswordField; -import javax.swing.JScrollPane; -import javax.swing.JSpinner; -import javax.swing.JTabbedPane; -import javax.swing.JTextArea; -import javax.swing.JTextField; -import javax.swing.border.TitledBorder; -import javax.swing.event.ChangeEvent; -import javax.swing.event.ChangeListener; -import javax.swing.filechooser.FileNameExtensionFilter; - -import org.apache.commons.csv.CSVFormat; -import org.ohdsi.databases.DbType; -import org.ohdsi.databases.RichConnection; -import org.ohdsi.utilities.DirectoryUtilities; -import org.ohdsi.utilities.StringUtilities; -import org.ohdsi.utilities.files.IniFile; -import org.ohdsi.whiteRabbit.fakeDataGenerator.FakeDataGenerator; -import org.ohdsi.whiteRabbit.scan.SourceDataScan; - -/** - * This is the WhiteRabbit main class - */ -public class WhiteRabbitMain implements ActionListener { - - public final static String WIKI_URL = "http://www.ohdsi.org/web/wiki/doku.php?id=documentation:software:whiterabbit"; - public final static String ACTION_CMD_HELP = "Open help Wiki"; - - private JFrame frame; - private JTextField folderField; - private JTextField scanReportFileField; - - private JComboBox scanRowCount; - private JComboBox scanValuesCount; - private JCheckBox scanValueScan; - private JSpinner scanMinCellCount; - private JSpinner generateRowCount; - private JComboBox sourceType; - private JComboBox targetType; - private JTextField targetUserField; - private JTextField targetPasswordField; - private JTextField targetServerField; - private JTextField targetDatabaseField; - private JTextField sourceDelimiterField; - private JComboBox targetCSVFormat; - private JTextField sourceServerField; - private JTextField sourceUserField; - private JTextField sourcePasswordField; - private JTextField sourceDatabaseField; - private JButton addAllButton; - private JList tableList; - private Vector tables = new Vector(); - private boolean sourceIsFiles = true; - private boolean targetIsFiles = false; - - private List componentsToDisableWhenRunning = new ArrayList(); - - public static void main(String[] args) { - new WhiteRabbitMain(args); - } - - public WhiteRabbitMain(String[] args) { - if (args.length == 2 && args[0].equalsIgnoreCase("-ini")) - launchCommandLine(args[1]); - else { - frame = new JFrame("White Rabbit"); - - frame.addWindowListener(new WindowAdapter() { - public void windowClosing(WindowEvent e) { - System.exit(0); - } - }); - frame.setLayout(new BorderLayout()); - frame.setJMenuBar(createMenuBar()); - - JComponent tabsPanel = createTabsPanel(); - JComponent consolePanel = createConsolePanel(); - - frame.add(consolePanel, BorderLayout.CENTER); - frame.add(tabsPanel, BorderLayout.NORTH); - - loadIcons(frame); - frame.pack(); - frame.setVisible(true); - ObjectExchange.frame = frame; - } - } - - private void launchCommandLine(String iniFileName) { - IniFile iniFile = new IniFile(iniFileName); - DbSettings dbSettings = new DbSettings(); - if (iniFile.get("DATA_TYPE").equalsIgnoreCase("Delimited text files")) { - dbSettings.dataType = DbSettings.CSVFILES; - if (iniFile.get("DELIMITER").equalsIgnoreCase("tab")) - dbSettings.delimiter = '\t'; - else - dbSettings.delimiter = iniFile.get("DELIMITER").charAt(0); - } else { - dbSettings.dataType = DbSettings.DATABASE; - dbSettings.user = iniFile.get("USER_NAME"); - dbSettings.password = iniFile.get("PASSWORD"); - dbSettings.server = iniFile.get("SERVER_LOCATION"); - dbSettings.database = iniFile.get("DATABASE_NAME"); - if (iniFile.get("DATA_TYPE").equalsIgnoreCase("MySQL")) - dbSettings.dbType = DbType.MYSQL; - else if (iniFile.get("DATA_TYPE").equalsIgnoreCase("Oracle")) - dbSettings.dbType = DbType.ORACLE; - else if (iniFile.get("DATA_TYPE").equalsIgnoreCase("PostgreSQL")) - dbSettings.dbType = DbType.POSTGRESQL; - else if (iniFile.get("DATA_TYPE").equalsIgnoreCase("Redshift")) - dbSettings.dbType = DbType.REDSHIFT; - else if (iniFile.get("DATA_TYPE").equalsIgnoreCase("SQL Server")) { - dbSettings.dbType = DbType.MSSQL; - if (iniFile.get("USER_NAME").length() != 0) { // Not using windows authentication - String[] parts = iniFile.get("USER_NAME").split("/"); - if (parts.length == 2) { - dbSettings.user = parts[1]; - dbSettings.domain = parts[0]; - } - } - } else if (iniFile.get("DATA_TYPE").equalsIgnoreCase("PDW")) { - dbSettings.dbType = DbType.PDW; - if (iniFile.get("USER_NAME").length() != 0) { // Not using windows authentication - String[] parts = iniFile.get("USER_NAME").split("/"); - if (parts.length == 2) { - dbSettings.user = parts[1]; - dbSettings.domain = parts[0]; - } - } - } else if (iniFile.get("DATA_TYPE").equalsIgnoreCase("MS Access")) - dbSettings.dbType = DbType.MSACCESS; - else if (iniFile.get("DATA_TYPE").equalsIgnoreCase("Teradata")) - dbSettings.dbType = DbType.TERADATA; - } - if (iniFile.get("TABLES_TO_SCAN").equalsIgnoreCase("*")) { - RichConnection connection = new RichConnection(dbSettings.server, dbSettings.domain, dbSettings.user, dbSettings.password, dbSettings.dbType); - for (String table : connection.getTableNames(dbSettings.database)) - dbSettings.tables.add(table); - connection.close(); - } else { - for (String table : iniFile.get("TABLES_TO_SCAN").split(",")) { - if (dbSettings.dataType == DbSettings.CSVFILES) - table = iniFile.get("WORKING_FOLDER") + "/" + table; - dbSettings.tables.add(table); - } - } - - SourceDataScan sourceDataScan = new SourceDataScan(); - int maxRows = Integer.parseInt(iniFile.get("ROWS_PER_TABLE")); - boolean scanValues = iniFile.get("SCAN_FIELD_VALUES").equalsIgnoreCase("yes"); - int minCellCount = Integer.parseInt(iniFile.get("MIN_CELL_COUNT")); - int maxValues = Integer.parseInt(iniFile.get("MAX_DISTINCT_VALUES")); - sourceDataScan.process(dbSettings, maxRows, scanValues, minCellCount, maxValues, iniFile.get("WORKING_FOLDER") + "/ScanReport.xlsx"); - } - - private JComponent createTabsPanel() { - JTabbedPane tabbedPane = new JTabbedPane(); - - JPanel locationPanel = createLocationsPanel(); - tabbedPane.addTab("Locations", null, locationPanel, "Specify the location of the source data and the working folder"); - - JPanel scanPanel = createScanPanel(); - tabbedPane.addTab("Scan", null, scanPanel, "Create a scan of the source data"); - - JPanel fakeDataPanel = createFakeDataPanel(); - tabbedPane.addTab("Fake data generation", null, fakeDataPanel, "Create fake data based on a scan report for development purposes"); - - return tabbedPane; - } - - private JPanel createLocationsPanel() { - JPanel panel = new JPanel(); - - panel.setLayout(new GridBagLayout()); - GridBagConstraints c = new GridBagConstraints(); - c.fill = GridBagConstraints.BOTH; - c.weightx = 0.5; - - JPanel folderPanel = new JPanel(); - folderPanel.setLayout(new BoxLayout(folderPanel, BoxLayout.X_AXIS)); - folderPanel.setBorder(BorderFactory.createTitledBorder("Working folder")); - folderField = new JTextField(); - folderField.setText((new File("").getAbsolutePath())); - folderField.setToolTipText("The folder where all output will be written"); - folderPanel.add(folderField); - JButton pickButton = new JButton("Pick folder"); - pickButton.setToolTipText("Pick a different working folder"); - folderPanel.add(pickButton); - pickButton.addActionListener(new ActionListener() { - public void actionPerformed(ActionEvent e) { - pickFolder(); - } - }); - componentsToDisableWhenRunning.add(pickButton); - c.gridx = 0; - c.gridy = 0; - c.gridwidth = 1; - panel.add(folderPanel, c); - - JPanel sourcePanel = new JPanel(); - sourcePanel.setLayout(new GridLayout(0, 2)); - sourcePanel.setBorder(BorderFactory.createTitledBorder("Source data location")); - sourcePanel.add(new JLabel("Data type")); - sourceType = new JComboBox(new String[] { "Delimited text files", "MySQL", "Oracle", "SQL Server", "PostgreSQL", "MS Access", "PDW", "Redshift", "Teradata" }); - sourceType.setToolTipText("Select the type of source data available"); - sourceType.addItemListener(new ItemListener() { - - @Override - public void itemStateChanged(ItemEvent arg0) { - sourceIsFiles = arg0.getItem().toString().equals("Delimited text files"); - sourceServerField.setEnabled(!sourceIsFiles); - sourceUserField.setEnabled(!sourceIsFiles); - sourcePasswordField.setEnabled(!sourceIsFiles); - sourceDatabaseField.setEnabled(!sourceIsFiles); - sourceDelimiterField.setEnabled(sourceIsFiles); - addAllButton.setEnabled(!sourceIsFiles); - - if (!sourceIsFiles && arg0.getItem().toString().equals("Oracle")) { - sourceServerField - .setToolTipText("For Oracle servers this field contains the SID, servicename, and optionally the port: '/', ':/', '/', or ':/'"); - sourceUserField.setToolTipText("For Oracle servers this field contains the name of the user used to log in"); - sourcePasswordField.setToolTipText("For Oracle servers this field contains the password corresponding to the user"); - sourceDatabaseField - .setToolTipText("For Oracle servers this field contains the schema (i.e. 'user' in Oracle terms) containing the source tables"); - } else if (!sourceIsFiles && arg0.getItem().toString().equals("PostgreSQL")) { - sourceServerField.setToolTipText("For PostgreSQL servers this field contains the host name and database name (/)"); - sourceUserField.setToolTipText("The user used to log in to the server"); - sourcePasswordField.setToolTipText("The password used to log in to the server"); - sourceDatabaseField.setToolTipText("For PostgreSQL servers this field contains the schema containing the source tables"); - } else if (!sourceIsFiles) { - sourceServerField.setToolTipText("This field contains the name or IP address of the database server"); - if (arg0.getItem().toString().equals("SQL Server")) - sourceUserField - .setToolTipText("The user used to log in to the server. Optionally, the domain can be specified as / (e.g. 'MyDomain/Joe')"); - else - sourceUserField.setToolTipText("The user used to log in to the server"); - sourcePasswordField.setToolTipText("The password used to log in to the server"); - sourceDatabaseField.setToolTipText("The name of the database containing the source tables"); - } - } - }); - sourcePanel.add(sourceType); - - sourcePanel.add(new JLabel("Server location")); - sourceServerField = new JTextField("127.0.0.1"); - sourceServerField.setEnabled(false); - sourcePanel.add(sourceServerField); - sourcePanel.add(new JLabel("User name")); - sourceUserField = new JTextField(""); - sourceUserField.setEnabled(false); - sourcePanel.add(sourceUserField); - sourcePanel.add(new JLabel("Password")); - sourcePasswordField = new JPasswordField(""); - sourcePasswordField.setEnabled(false); - sourcePanel.add(sourcePasswordField); - sourcePanel.add(new JLabel("Database name")); - sourceDatabaseField = new JTextField(""); - sourceDatabaseField.setEnabled(false); - sourcePanel.add(sourceDatabaseField); - - sourcePanel.add(new JLabel("Delimiter")); - sourceDelimiterField = new JTextField(","); - sourceDelimiterField.setToolTipText("The delimiter that separates values. Enter 'tab' for tab."); - sourcePanel.add(sourceDelimiterField); - - c.gridx = 0; - c.gridy = 1; - c.gridwidth = 1; - panel.add(sourcePanel, c); - - JPanel testConnectionButtonPanel = new JPanel(); - testConnectionButtonPanel.setLayout(new BoxLayout(testConnectionButtonPanel, BoxLayout.X_AXIS)); - testConnectionButtonPanel.add(Box.createHorizontalGlue()); - - JButton testConnectionButton = new JButton("Test connection"); - testConnectionButton.setBackground(new Color(151, 220, 141)); - testConnectionButton.setToolTipText("Test the connection"); - testConnectionButton.addActionListener(new ActionListener() { - public void actionPerformed(ActionEvent e) { - testConnection(getSourceDbSettings()); - } - }); - componentsToDisableWhenRunning.add(testConnectionButton); - testConnectionButtonPanel.add(testConnectionButton); - - c.gridx = 0; - c.gridy = 2; - c.gridwidth = 1; - panel.add(testConnectionButtonPanel, c); - - return panel; - } - - private JPanel createScanPanel() { - JPanel panel = new JPanel(); - panel.setLayout(new BorderLayout()); - - JPanel tablePanel = new JPanel(); - tablePanel.setLayout(new BorderLayout()); - tablePanel.setBorder(new TitledBorder("Tables to scan")); - tableList = new JList(); - tableList.setToolTipText("Specify the tables (or CSV files) to be scanned here"); - tablePanel.add(new JScrollPane(tableList), BorderLayout.CENTER); - - JPanel tableButtonPanel = new JPanel(); - tableButtonPanel.setLayout(new GridLayout(3, 1)); - addAllButton = new JButton("Add all in DB"); - addAllButton.setToolTipText("Add all tables in the database"); - addAllButton.addActionListener(new ActionListener() { - public void actionPerformed(ActionEvent e) { - addAllTables(); - } - }); - addAllButton.setEnabled(false); - tableButtonPanel.add(addAllButton); - JButton addButton = new JButton("Add"); - addButton.setToolTipText("Add tables to list"); - addButton.addActionListener(new ActionListener() { - public void actionPerformed(ActionEvent e) { - pickTables(); - } - }); - tableButtonPanel.add(addButton); - JButton removeButton = new JButton("Remove"); - removeButton.setToolTipText("Remove tables from list"); - removeButton.addActionListener(new ActionListener() { - public void actionPerformed(ActionEvent e) { - removeTables(); - } - }); - tableButtonPanel.add(removeButton); - tablePanel.add(tableButtonPanel, BorderLayout.EAST); - - panel.add(tablePanel, BorderLayout.CENTER); - - JPanel southPanel = new JPanel(); - southPanel.setLayout(new BoxLayout(southPanel, BoxLayout.Y_AXIS)); - - JPanel scanOptionsPanel = new JPanel(); - scanOptionsPanel.setLayout(new BoxLayout(scanOptionsPanel, BoxLayout.X_AXIS)); - - scanValueScan = new JCheckBox("Scan field values", true); - scanValueScan.setToolTipText("Include a frequency count of field values in the scan report"); - scanValueScan.addChangeListener(new ChangeListener() { - - @Override - public void stateChanged(ChangeEvent arg0) { - scanMinCellCount.setEnabled(((JCheckBox) arg0.getSource()).isSelected()); - scanRowCount.setEnabled(((JCheckBox) arg0.getSource()).isSelected()); - scanValuesCount.setEnabled(((JCheckBox) arg0.getSource()).isSelected()); - } - }); - scanOptionsPanel.add(scanValueScan); - scanOptionsPanel.add(Box.createHorizontalGlue()); - - scanOptionsPanel.add(new JLabel("Min cell count ")); - scanMinCellCount = new JSpinner(); - scanMinCellCount.setValue(5); - scanMinCellCount.setToolTipText("Minimum frequency for a field value to be included in the report"); - scanOptionsPanel.add(scanMinCellCount); - scanOptionsPanel.add(Box.createHorizontalGlue()); - - scanOptionsPanel.add(new JLabel("Max distinct values ")); - scanValuesCount = new JComboBox(new String[] { "100", "1,000", "10,000" }); - scanValuesCount.setSelectedIndex(1); - scanValuesCount.setToolTipText("Maximum number of distinct values per field to be reported"); - scanOptionsPanel.add(scanValuesCount); - scanOptionsPanel.add(Box.createHorizontalGlue()); - - scanOptionsPanel.add(new JLabel("Rows per table ")); - scanRowCount = new JComboBox(new String[] { "100,000", "500,000", "1 million", "all" }); - scanRowCount.setSelectedIndex(0); - scanRowCount.setToolTipText("Maximum number of rows per table to be scanned for field values"); - scanOptionsPanel.add(scanRowCount); - - southPanel.add(scanOptionsPanel); - - southPanel.add(Box.createVerticalStrut(3)); - - JPanel scanButtonPanel = new JPanel(); - scanButtonPanel.setLayout(new BoxLayout(scanButtonPanel, BoxLayout.X_AXIS)); - scanButtonPanel.add(Box.createHorizontalGlue()); - - JButton scanButton = new JButton("Scan tables"); - scanButton.setBackground(new Color(151, 220, 141)); - scanButton.setToolTipText("Scan the selected tables"); - scanButton.addActionListener(new ActionListener() { - public void actionPerformed(ActionEvent e) { - scanRun(); - } - }); - componentsToDisableWhenRunning.add(scanButton); - scanButtonPanel.add(scanButton); - southPanel.add(scanButtonPanel); - - panel.add(southPanel, BorderLayout.SOUTH); - - return panel; - } - - private JPanel createFakeDataPanel() { - JPanel panel = new JPanel(); - - panel.setLayout(new GridBagLayout()); - GridBagConstraints c = new GridBagConstraints(); - c.fill = GridBagConstraints.BOTH; - c.weightx = 0.5; - - JPanel folderPanel = new JPanel(); - folderPanel.setLayout(new BoxLayout(folderPanel, BoxLayout.X_AXIS)); - folderPanel.setBorder(BorderFactory.createTitledBorder("Scan report file")); - scanReportFileField = new JTextField(); - scanReportFileField.setText((new File("ScanReport.xlsx").getAbsolutePath())); - scanReportFileField.setToolTipText("The path to the scan report that will be used as a template to generate the fake data"); - folderPanel.add(scanReportFileField); - JButton pickButton = new JButton("Pick file"); - pickButton.setToolTipText("Pick a scan report file"); - folderPanel.add(pickButton); - pickButton.addActionListener(new ActionListener() { - public void actionPerformed(ActionEvent e) { - pickScanReportFile(); - } - }); - componentsToDisableWhenRunning.add(pickButton); - c.gridx = 0; - c.gridy = 0; - c.gridwidth = 1; - panel.add(folderPanel, c); - - JPanel targetPanel = new JPanel(); - targetPanel.setLayout(new GridLayout(0, 2)); - targetPanel.setBorder(BorderFactory.createTitledBorder("Target data location")); - targetPanel.add(new JLabel("Data type")); - targetType = new JComboBox(new String[] { "Delimited text files", "MySQL", "Oracle", "SQL Server", "PostgreSQL" }); - // targetType = new JComboBox(new String[] { "Delimited text files", "MySQL" }); - targetType.setToolTipText("Select the type of source data available"); - targetType.addItemListener(new ItemListener() { - - @Override - public void itemStateChanged(ItemEvent arg0) { - targetIsFiles = arg0.getItem().toString().equals("Delimited text files"); - targetServerField.setEnabled(!targetIsFiles); - targetUserField.setEnabled(!targetIsFiles); - targetPasswordField.setEnabled(!targetIsFiles); - targetDatabaseField.setEnabled(!targetIsFiles); - targetCSVFormat.setEnabled(targetIsFiles); - - if (!targetIsFiles && arg0.getItem().toString().equals("Oracle")) { - targetServerField - .setToolTipText("For Oracle servers this field contains the SID, servicename, and optionally the port: '/', ':/', '/', or ':/'"); - targetUserField.setToolTipText("For Oracle servers this field contains the name of the user used to log in"); - targetPasswordField.setToolTipText("For Oracle servers this field contains the password corresponding to the user"); - targetDatabaseField - .setToolTipText("For Oracle servers this field contains the schema (i.e. 'user' in Oracle terms) containing the source tables"); - } else if (!targetIsFiles && arg0.getItem().toString().equals("PostgreSQL")) { - targetServerField.setToolTipText("For PostgreSQL servers this field contains the host name and database name (/)"); - targetUserField.setToolTipText("The user used to log in to the server"); - targetPasswordField.setToolTipText("The password used to log in to the server"); - targetDatabaseField.setToolTipText("For PostgreSQL servers this field contains the schema containing the source tables"); - } else if (!targetIsFiles) { - targetServerField.setToolTipText("This field contains the name or IP address of the database server"); - if (arg0.getItem().toString().equals("SQL Server")) - targetUserField - .setToolTipText("The user used to log in to the server. Optionally, the domain can be specified as / (e.g. 'MyDomain/Joe')"); - else - targetUserField.setToolTipText("The user used to log in to the server"); - targetPasswordField.setToolTipText("The password used to log in to the server"); - targetDatabaseField.setToolTipText("The name of the database containing the source tables"); - } - } - }); - targetPanel.add(targetType); - - targetPanel.add(new JLabel("Server location")); - targetServerField = new JTextField("127.0.0.1"); - targetServerField.setEnabled(false); - targetPanel.add(targetServerField); - targetPanel.add(new JLabel("User name")); - targetUserField = new JTextField(""); - targetUserField.setEnabled(false); - targetPanel.add(targetUserField); - targetPanel.add(new JLabel("Password")); - targetPasswordField = new JPasswordField(""); - targetPasswordField.setEnabled(false); - targetPanel.add(targetPasswordField); - targetPanel.add(new JLabel("Database name")); - targetDatabaseField = new JTextField(""); - targetDatabaseField.setEnabled(false); - targetPanel.add(targetDatabaseField); - - targetPanel.add(new JLabel("CSV Format")); - targetCSVFormat = new JComboBox<>( - new String[] { "Default (comma, CRLF)", "TDF (tab, CRLF)", "MySQL (tab, LF)", "RFC4180", "Excel CSV" }); - targetCSVFormat.setToolTipText("The format of the output"); - targetCSVFormat.setEnabled(true); - targetPanel.add(targetCSVFormat); - - c.gridx = 0; - c.gridy = 1; - c.gridwidth = 1; - panel.add(targetPanel, c); - - JPanel fakeDataButtonPanel = new JPanel(); - fakeDataButtonPanel.setLayout(new BoxLayout(fakeDataButtonPanel, BoxLayout.X_AXIS)); - - fakeDataButtonPanel.add(new JLabel("Max rows per table")); - generateRowCount = new JSpinner(); - generateRowCount.setValue(10000); - fakeDataButtonPanel.add(generateRowCount); - fakeDataButtonPanel.add(Box.createHorizontalGlue()); - - JButton testConnectionButton = new JButton("Test connection"); - testConnectionButton.setBackground(new Color(151, 220, 141)); - testConnectionButton.setToolTipText("Test the connection"); - testConnectionButton.addActionListener(new ActionListener() { - public void actionPerformed(ActionEvent e) { - testConnection(getTargetDbSettings()); - } - }); - componentsToDisableWhenRunning.add(testConnectionButton); - fakeDataButtonPanel.add(testConnectionButton); - - JButton fakeDataButton = new JButton("Generate fake data"); - fakeDataButton.setBackground(new Color(151, 220, 141)); - fakeDataButton.setToolTipText("Generate fake data based on the scan report"); - fakeDataButton.addActionListener(new ActionListener() { - public void actionPerformed(ActionEvent e) { - fakeDataRun(); - } - }); - componentsToDisableWhenRunning.add(fakeDataButton); - fakeDataButtonPanel.add(fakeDataButton); - - c.gridx = 0; - c.gridy = 2; - c.gridwidth = 1; - panel.add(fakeDataButtonPanel, c); - - return panel; - } - - private JComponent createConsolePanel() { - JTextArea consoleArea = new JTextArea(); - consoleArea.setToolTipText("General progress information"); - consoleArea.setEditable(false); - Console console = new Console(); - console.setTextArea(consoleArea); - System.setOut(new PrintStream(console)); - System.setErr(new PrintStream(console)); - JScrollPane consoleScrollPane = new JScrollPane(consoleArea); - consoleScrollPane.setBorder(BorderFactory.createTitledBorder("Console")); - consoleScrollPane.setPreferredSize(new Dimension(800, 200)); - consoleScrollPane.setAutoscrolls(true); - ObjectExchange.console = console; - return consoleScrollPane; - } - - private void loadIcons(JFrame f) { - List icons = new ArrayList(); - icons.add(loadIcon("WhiteRabbit16.png", f)); - icons.add(loadIcon("WhiteRabbit32.png", f)); - icons.add(loadIcon("WhiteRabbit48.png", f)); - icons.add(loadIcon("WhiteRabbit64.png", f)); - icons.add(loadIcon("WhiteRabbit128.png", f)); - icons.add(loadIcon("WhiteRabbit256.png", f)); - f.setIconImages(icons); - } - - private Image loadIcon(String name, JFrame f) { - Image icon = Toolkit.getDefaultToolkit().getImage(WhiteRabbitMain.class.getResource(name)); - MediaTracker mediaTracker = new MediaTracker(f); - mediaTracker.addImage(icon, 0); - try { - mediaTracker.waitForID(0); - return icon; - } catch (Exception e1) { - e1.printStackTrace(); - } - return null; - } - - private void pickFolder() { - JFileChooser fileChooser = new JFileChooser(new File(folderField.getText())); - fileChooser.setFileSelectionMode(JFileChooser.DIRECTORIES_ONLY); - int returnVal = fileChooser.showDialog(frame, "Select folder"); - if (returnVal == JFileChooser.APPROVE_OPTION) - folderField.setText(fileChooser.getSelectedFile().getAbsolutePath()); - } - - private void pickScanReportFile() { - JFileChooser fileChooser = new JFileChooser(new File(folderField.getText())); - fileChooser.setFileSelectionMode(JFileChooser.FILES_ONLY); - int returnVal = fileChooser.showDialog(frame, "Select scan report file"); - if (returnVal == JFileChooser.APPROVE_OPTION) - scanReportFileField.setText(fileChooser.getSelectedFile().getAbsolutePath()); - } - - private void removeTables() { - for (String item : tableList.getSelectedValuesList()) { - tables.remove(item); - tableList.setListData(tables); - } - } - - private void addAllTables() { - DbSettings sourceDbSettings = getSourceDbSettings(); - if (sourceDbSettings != null) { - RichConnection connection = new RichConnection(sourceDbSettings.server, sourceDbSettings.domain, sourceDbSettings.user, sourceDbSettings.password, - sourceDbSettings.dbType); - for (String table : connection.getTableNames(sourceDbSettings.database)) { - if (!tables.contains(table)) - tables.add((String) table); - tableList.setListData(tables); - } - connection.close(); - } - } - - private void pickTables() { - DbSettings sourceDbSettings = getSourceDbSettings(); - if (sourceDbSettings != null) { - if (sourceDbSettings.dataType == DbSettings.CSVFILES) { - JFileChooser fileChooser = new JFileChooser(new File(folderField.getText())); - fileChooser.setMultiSelectionEnabled(true); - fileChooser.setFileSelectionMode(JFileChooser.FILES_ONLY); - FileNameExtensionFilter filter = new FileNameExtensionFilter("Delimited text files", "csv", "txt"); - fileChooser.setFileFilter(filter); - - int returnVal = fileChooser.showDialog(frame, "Select tables"); - if (returnVal == JFileChooser.APPROVE_OPTION) { - for (File table : fileChooser.getSelectedFiles()) { - String tableName = DirectoryUtilities.getRelativePath(new File(folderField.getText()), table); - if (!tables.contains(tableName)) - tables.add(tableName); - tableList.setListData(tables); - } - - } - } else if (sourceDbSettings.dataType == DbSettings.DATABASE) { - RichConnection connection = new RichConnection(sourceDbSettings.server, sourceDbSettings.domain, sourceDbSettings.user, - sourceDbSettings.password, sourceDbSettings.dbType); - String tableNames = StringUtilities.join(connection.getTableNames(sourceDbSettings.database), "\t"); - if (tableNames.length() == 0) { - JOptionPane.showMessageDialog(frame, "No tables found in database " + sourceDbSettings.database, "Error fetching table names", - JOptionPane.ERROR_MESSAGE); - } else { - DBTableSelectionDialog selectionDialog = new DBTableSelectionDialog(frame, true, tableNames); - if (selectionDialog.getAnswer()) { - for (Object item : selectionDialog.getSelectedItems()) { - if (!tables.contains(item)) - tables.add((String) item); - tableList.setListData(tables); - } - } - } - connection.close(); - } - } - } - - private DbSettings getSourceDbSettings() { - DbSettings dbSettings = new DbSettings(); - if (sourceType.getSelectedItem().equals("Delimited text files")) { - dbSettings.dataType = DbSettings.CSVFILES; - if (sourceDelimiterField.getText().length() == 0) { - JOptionPane.showMessageDialog(frame, "Delimiter field cannot be empty for source database", "Error connecting to server", - JOptionPane.ERROR_MESSAGE); - return null; - } - if (sourceDelimiterField.getText().toLowerCase().equals("tab")) - dbSettings.delimiter = '\t'; - else - dbSettings.delimiter = sourceDelimiterField.getText().charAt(0); - } else { - dbSettings.dataType = DbSettings.DATABASE; - dbSettings.user = sourceUserField.getText(); - dbSettings.password = sourcePasswordField.getText(); - dbSettings.server = sourceServerField.getText(); - dbSettings.database = sourceDatabaseField.getText().trim().length() == 0 ? null : sourceDatabaseField.getText(); - if (sourceType.getSelectedItem().toString().equals("MySQL")) - dbSettings.dbType = DbType.MYSQL; - else if (sourceType.getSelectedItem().toString().equals("Oracle")) - dbSettings.dbType = DbType.ORACLE; - else if (sourceType.getSelectedItem().toString().equals("PostgreSQL")) - dbSettings.dbType = DbType.POSTGRESQL; - else if (sourceType.getSelectedItem().toString().equals("Redshift")) - dbSettings.dbType = DbType.REDSHIFT; - else if (sourceType.getSelectedItem().toString().equals("SQL Server")) { - dbSettings.dbType = DbType.MSSQL; - if (sourceUserField.getText().length() != 0) { // Not using windows authentication - String[] parts = sourceUserField.getText().split("/"); - if (parts.length == 2) { - dbSettings.user = parts[1]; - dbSettings.domain = parts[0]; - } - } - } if (sourceType.getSelectedItem().toString().equals("PDW")) { - dbSettings.dbType = DbType.PDW; - if (sourceUserField.getText().length() != 0) { // Not using windows authentication - String[] parts = sourceUserField.getText().split("/"); - if (parts.length == 2) { - dbSettings.user = parts[1]; - dbSettings.domain = parts[0]; - } - } - } else if (sourceType.getSelectedItem().toString().equals("MS Access")) - dbSettings.dbType = DbType.MSACCESS; - else if (sourceType.getSelectedItem().toString().equals("Teradata")) - dbSettings.dbType = DbType.TERADATA; - } - return dbSettings; - } - - private void testConnection(DbSettings dbSettings) { - if (dbSettings.dataType == DbSettings.CSVFILES) { - if (new File(folderField.getText()).exists()) { - String message = "Folder " + folderField.getText() + " found"; - JOptionPane.showMessageDialog(frame, StringUtilities.wordWrap(message, 80), "Working folder found", JOptionPane.INFORMATION_MESSAGE); - } else { - String message = "Folder " + folderField.getText() + " not found"; - JOptionPane.showMessageDialog(frame, StringUtilities.wordWrap(message, 80), "Working folder not found", JOptionPane.ERROR_MESSAGE); - } - } else { - if (dbSettings.database == null || dbSettings.database.equals("")) { - JOptionPane.showMessageDialog(frame, StringUtilities.wordWrap("Please specify database name", 80), "Error connecting to server", - JOptionPane.ERROR_MESSAGE); - return; - } - if (dbSettings.server == null || dbSettings.server.equals("")) { - JOptionPane.showMessageDialog(frame, StringUtilities.wordWrap("Please specify the server", 80), "Error connecting to server", - JOptionPane.ERROR_MESSAGE); - return; - } - - RichConnection connection; - try { - connection = new RichConnection(dbSettings.server, dbSettings.domain, dbSettings.user, dbSettings.password, dbSettings.dbType); - } catch (Exception e) { - String message = "Could not connect: " + e.getMessage(); - JOptionPane.showMessageDialog(frame, StringUtilities.wordWrap(message, 80), "Error connecting to server", JOptionPane.ERROR_MESSAGE); - return; - } - - try { - List tableNames = connection.getTableNames(dbSettings.database); - if (tableNames.size() == 0) - throw new RuntimeException("Unable to retrieve table names for database " + dbSettings.database); - } catch (Exception e) { - String message = "Could not connect to database: " + e.getMessage(); - JOptionPane.showMessageDialog(frame, StringUtilities.wordWrap(message, 80), "Error connecting to server", JOptionPane.ERROR_MESSAGE); - return; - } - - connection.close(); - String message = "Succesfully connected to " + dbSettings.database + " on server " + dbSettings.server; - JOptionPane.showMessageDialog(frame, StringUtilities.wordWrap(message, 80), "Connection succesful", JOptionPane.INFORMATION_MESSAGE); - - } - } - - private DbSettings getTargetDbSettings() { - DbSettings dbSettings = new DbSettings(); - if (targetType.getSelectedItem().equals("Delimited text files")) { - dbSettings.dataType = DbSettings.CSVFILES; - - switch((String) targetCSVFormat.getSelectedItem()) { - case "Default (comma, CRLF)": - dbSettings.csvFormat = CSVFormat.DEFAULT; - break; - case "RFC4180": - dbSettings.csvFormat = CSVFormat.RFC4180; - break; - case "Excel CSV": - dbSettings.csvFormat = CSVFormat.EXCEL; - break; - case "TDF (tab, CRLF)": - dbSettings.csvFormat = CSVFormat.TDF; - break; - case "MySQL (tab, LF)": - dbSettings.csvFormat = CSVFormat.MYSQL; - break; - default: - dbSettings.csvFormat = CSVFormat.RFC4180; - } - - } else { - dbSettings.dataType = DbSettings.DATABASE; - dbSettings.user = targetUserField.getText(); - dbSettings.password = targetPasswordField.getText(); - dbSettings.server = targetServerField.getText(); - dbSettings.database = targetDatabaseField.getText(); - if (targetType.getSelectedItem().toString().equals("MySQL")) - dbSettings.dbType = DbType.MYSQL; - else if (targetType.getSelectedItem().toString().equals("Oracle")) - dbSettings.dbType = DbType.ORACLE; - else if (sourceType.getSelectedItem().toString().equals("PostgreSQL")) - dbSettings.dbType = DbType.POSTGRESQL; - else if (sourceType.getSelectedItem().toString().equals("SQL Server")) { - dbSettings.dbType = DbType.MSSQL; - if (sourceUserField.getText().length() != 0) { // Not using windows authentication - String[] parts = sourceUserField.getText().split("/"); - if (parts.length == 2) { - dbSettings.user = parts[1]; - dbSettings.domain = parts[0]; - } - } - } else if (sourceType.getSelectedItem().toString().equals("PDW")) { - dbSettings.dbType = DbType.PDW; - if (sourceUserField.getText().length() != 0) { // Not using windows authentication - String[] parts = sourceUserField.getText().split("/"); - if (parts.length == 2) { - dbSettings.user = parts[1]; - dbSettings.domain = parts[0]; - } - } - } - - if (dbSettings.database.trim().length() == 0) { - String message = "Please specify a name for the target database"; - JOptionPane.showMessageDialog(frame, StringUtilities.wordWrap(message, 80), "Database error", JOptionPane.ERROR_MESSAGE); - return null; - } - } - return dbSettings; - } - - private void scanRun() { - if (tables.size() == 0) { - if (sourceIsFiles) { - String message = "No files selected for scanning"; - JOptionPane.showMessageDialog(frame, StringUtilities.wordWrap(message, 80), "No files selected", JOptionPane.ERROR_MESSAGE); - return; - } else { - String message = "No tables were selected for scanning. Do you want to select all tables in the database for scanning?"; - String title = "No tables selected"; - int answer = JOptionPane.showConfirmDialog(ObjectExchange.frame, message, title, JOptionPane.YES_NO_OPTION); - if (answer == JOptionPane.YES_OPTION) { - addAllTables(); - } else - return; - } - } - int rowCount = 0; - if (scanRowCount.getSelectedItem().toString().equals("100,000")) - rowCount = 100000; - else if (scanRowCount.getSelectedItem().toString().equals("500,000")) - rowCount = 500000; - else if (scanRowCount.getSelectedItem().toString().equals("1 million")) - rowCount = 1000000; - if (scanRowCount.getSelectedItem().toString().equals("all")) - rowCount = -1; - - int valuesCount = 0; - if (scanValuesCount.getSelectedItem().toString().equals("100")) - valuesCount = 100; - else if (scanValuesCount.getSelectedItem().toString().equals("1,000")) - valuesCount = 1000; - else if (scanValuesCount.getSelectedItem().toString().equals("10,000")) - valuesCount = 10000; - - ScanThread scanThread = new ScanThread(rowCount, valuesCount, scanValueScan.isSelected(), Integer.parseInt(scanMinCellCount.getValue().toString())); - scanThread.start(); - } - - private void fakeDataRun() { - String filename = scanReportFileField.getText(); - if (!new File(filename).exists()) { - String message = "File " + filename + " not found"; - JOptionPane.showMessageDialog(frame, StringUtilities.wordWrap(message, 80), "File not found", JOptionPane.ERROR_MESSAGE); - } else { - FakeDataThread thread = new FakeDataThread(Integer.parseInt(generateRowCount.getValue().toString()), filename); - thread.start(); - } - } - - private class ScanThread extends Thread { - - private int maxRows; - private int maxValues; - private boolean scanValues; - private int minCellCount; - - public ScanThread(int maxRows, int maxValues, boolean scanValues, int minCellCount) { - this.maxRows = maxRows; - this.scanValues = scanValues; - this.minCellCount = minCellCount; - this.maxValues = maxValues; - } - - public void run() { - for (JComponent component : componentsToDisableWhenRunning) - component.setEnabled(false); - try { - SourceDataScan sourceDataScan = new SourceDataScan(); - DbSettings dbSettings = getSourceDbSettings(); - if (dbSettings != null) { - for (String table : tables) { - if (dbSettings.dataType == DbSettings.CSVFILES) - table = folderField.getText() + "/" + table; - dbSettings.tables.add(table); - } - sourceDataScan.process(dbSettings, maxRows, scanValues, minCellCount, maxValues, folderField.getText() + "/ScanReport.xlsx"); - } - } catch (Exception e) { - handleError(e); - } finally { - for (JComponent component : componentsToDisableWhenRunning) - component.setEnabled(true); - } - } - - } - - private class FakeDataThread extends Thread { - private int maxRowCount; - private String filename; - - public FakeDataThread(int maxRowCount, String filename) { - this.maxRowCount = maxRowCount; - this.filename = filename; - } - - public void run() { - for (JComponent component : componentsToDisableWhenRunning) - component.setEnabled(false); - try { - FakeDataGenerator process = new FakeDataGenerator(); - DbSettings dbSettings = getTargetDbSettings(); - if (dbSettings != null) - process.generateData(dbSettings, maxRowCount, filename, folderField.getText()); - } catch (Exception e) { - handleError(e); - } finally { - for (JComponent component : componentsToDisableWhenRunning) - component.setEnabled(true); - } - - } - } - - private class DBTableSelectionDialog extends JDialog implements ActionListener { - private static final long serialVersionUID = 4527207331482143091L; - private JButton yesButton = null; - private JButton noButton = null; - private boolean answer = false; - private JList list; - - public boolean getAnswer() { - return answer; - } - - public DBTableSelectionDialog(JFrame frame, boolean modal, String tableNames) { - super(frame, modal); - - setTitle("Select tables"); - JPanel panel = new JPanel(); - panel.setPreferredSize(new Dimension(800, 500)); - getContentPane().add(panel); - panel.setLayout(new BorderLayout()); - - JLabel message = new JLabel("Select tables"); - panel.add(message, BorderLayout.NORTH); - - list = new JList(tableNames.split("\t")); - JScrollPane scrollPane = new JScrollPane(list); - panel.add(scrollPane, BorderLayout.CENTER); - - JPanel buttonPanel = new JPanel(); - yesButton = new JButton("Select tables"); - yesButton.addActionListener(this); - buttonPanel.add(yesButton); - noButton = new JButton("Cancel"); - noButton.addActionListener(this); - buttonPanel.add(noButton); - panel.add(buttonPanel, BorderLayout.SOUTH); - - pack(); - setLocationRelativeTo(frame); - setVisible(true); - } - - public void actionPerformed(ActionEvent e) { - if (yesButton == e.getSource()) { - answer = true; - setVisible(false); - } else if (noButton == e.getSource()) { - answer = false; - setVisible(false); - } - } - - public List getSelectedItems() { - return list.getSelectedValuesList(); - } - } - - @Override - public void actionPerformed(ActionEvent event) { - switch (event.getActionCommand()) { - case ACTION_CMD_HELP: - doOpenWiki(); - break; - - } - } - - private void doOpenWiki() { - try { - Desktop desktop = Desktop.getDesktop(); - desktop.browse(new URI(WIKI_URL)); - } catch (URISyntaxException | IOException ex) { - - } - } - - private void handleError(Exception e) { - System.err.println("Error: " + e.getMessage()); - String errorReportFilename = ErrorReport.generate(folderField.getText(), e); - String message = "Error: " + e.getLocalizedMessage(); - message += "\nAn error report has been generated:\n" + errorReportFilename; - System.out.println(message); - JOptionPane.showMessageDialog(frame, StringUtilities.wordWrap(message, 80), "Error", JOptionPane.ERROR_MESSAGE); - } - - private JMenuBar createMenuBar() { - JMenuBar menuBar = new JMenuBar(); - JMenu helpMenu = new JMenu("Help"); - menuBar.add(helpMenu); - JMenuItem helpItem = new JMenuItem(ACTION_CMD_HELP); - helpItem.addActionListener(this); - helpItem.setActionCommand(ACTION_CMD_HELP); - helpMenu.add(helpItem); - - return menuBar; - } - -} +/******************************************************************************* + * Copyright 2019 Observational Health Data Sciences and Informatics + * + * This file is part of WhiteRabbit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + ******************************************************************************/ +package org.ohdsi.whiteRabbit; + +import java.awt.BorderLayout; +import java.awt.Color; +import java.awt.Desktop; +import java.awt.Dimension; +import java.awt.GridBagConstraints; +import java.awt.GridBagLayout; +import java.awt.GridLayout; +import java.awt.Image; +import java.awt.MediaTracker; +import java.awt.Toolkit; +import java.awt.event.ActionEvent; +import java.awt.event.ActionListener; +import java.awt.event.ItemEvent; +import java.awt.event.ItemListener; +import java.awt.event.WindowAdapter; +import java.awt.event.WindowEvent; +import java.io.File; +import java.io.IOException; +import java.io.PrintStream; +import java.net.URI; +import java.net.URISyntaxException; +import java.util.ArrayList; +import java.util.List; +import java.util.Vector; + +import javax.swing.BorderFactory; +import javax.swing.Box; +import javax.swing.BoxLayout; +import javax.swing.JButton; +import javax.swing.JCheckBox; +import javax.swing.JComboBox; +import javax.swing.JComponent; +import javax.swing.JDialog; +import javax.swing.JFileChooser; +import javax.swing.JFrame; +import javax.swing.JLabel; +import javax.swing.JList; +import javax.swing.JMenu; +import javax.swing.JMenuBar; +import javax.swing.JMenuItem; +import javax.swing.JOptionPane; +import javax.swing.JPanel; +import javax.swing.JPasswordField; +import javax.swing.JScrollPane; +import javax.swing.JSpinner; +import javax.swing.JTabbedPane; +import javax.swing.JTextArea; +import javax.swing.JTextField; +import javax.swing.border.TitledBorder; +import javax.swing.event.ChangeEvent; +import javax.swing.event.ChangeListener; +import javax.swing.filechooser.FileNameExtensionFilter; + +import org.apache.commons.csv.CSVFormat; +import org.ohdsi.databases.DbType; +import org.ohdsi.databases.RichConnection; +import org.ohdsi.utilities.DirectoryUtilities; +import org.ohdsi.utilities.StringUtilities; +import org.ohdsi.utilities.files.IniFile; +import org.ohdsi.whiteRabbit.fakeDataGenerator.FakeDataGenerator; +import org.ohdsi.whiteRabbit.scan.SourceDataScan; + +/** + * This is the WhiteRabbit main class + */ +public class WhiteRabbitMain implements ActionListener { + + public final static String WIKI_URL = "http://www.ohdsi.org/web/wiki/doku.php?id=documentation:software:whiterabbit"; + public final static String ACTION_CMD_HELP = "Open help Wiki"; + + private JFrame frame; + private JTextField folderField; + private JTextField scanReportFileField; + + private JComboBox scanRowCount; + private JComboBox scanValuesCount; + private JCheckBox scanValueScan; + private JSpinner scanMinCellCount; + private JSpinner generateRowCount; + private JComboBox sourceType; + private JComboBox targetType; + private JTextField targetUserField; + private JTextField targetPasswordField; + private JTextField targetServerField; + private JTextField targetDatabaseField; + private JTextField sourceDelimiterField; + private JComboBox targetCSVFormat; + private JTextField sourceServerField; + private JTextField sourceUserField; + private JTextField sourcePasswordField; + private JTextField sourceDatabaseField; + private JButton addAllButton; + private JList tableList; + private Vector tables = new Vector(); + private boolean sourceIsFiles = true; + private boolean targetIsFiles = false; + + private List componentsToDisableWhenRunning = new ArrayList(); + + public static void main(String[] args) { + new WhiteRabbitMain(args); + } + + public WhiteRabbitMain(String[] args) { + if (args.length == 2 && args[0].equalsIgnoreCase("-ini")) + launchCommandLine(args[1]); + else { + frame = new JFrame("White Rabbit"); + + frame.addWindowListener(new WindowAdapter() { + public void windowClosing(WindowEvent e) { + System.exit(0); + } + }); + frame.setLayout(new BorderLayout()); + frame.setJMenuBar(createMenuBar()); + + JComponent tabsPanel = createTabsPanel(); + JComponent consolePanel = createConsolePanel(); + + frame.add(consolePanel, BorderLayout.CENTER); + frame.add(tabsPanel, BorderLayout.NORTH); + + loadIcons(frame); + frame.pack(); + frame.setVisible(true); + ObjectExchange.frame = frame; + } + } + + private void launchCommandLine(String iniFileName) { + IniFile iniFile = new IniFile(iniFileName); + DbSettings dbSettings = new DbSettings(); + if (iniFile.get("DATA_TYPE").equalsIgnoreCase("Delimited text files")) { + dbSettings.dataType = DbSettings.CSVFILES; + if (iniFile.get("DELIMITER").equalsIgnoreCase("tab")) + dbSettings.delimiter = '\t'; + else + dbSettings.delimiter = iniFile.get("DELIMITER").charAt(0); + } else { + dbSettings.dataType = DbSettings.DATABASE; + dbSettings.user = iniFile.get("USER_NAME"); + dbSettings.password = iniFile.get("PASSWORD"); + dbSettings.server = iniFile.get("SERVER_LOCATION"); + dbSettings.database = iniFile.get("DATABASE_NAME"); + if (iniFile.get("DATA_TYPE").equalsIgnoreCase("MySQL")) + dbSettings.dbType = DbType.MYSQL; + else if (iniFile.get("DATA_TYPE").equalsIgnoreCase("Oracle")) + dbSettings.dbType = DbType.ORACLE; + else if (iniFile.get("DATA_TYPE").equalsIgnoreCase("PostgreSQL")) + dbSettings.dbType = DbType.POSTGRESQL; + else if (iniFile.get("DATA_TYPE").equalsIgnoreCase("Redshift")) + dbSettings.dbType = DbType.REDSHIFT; + else if (iniFile.get("DATA_TYPE").equalsIgnoreCase("SQL Server")) { + dbSettings.dbType = DbType.MSSQL; + if (iniFile.get("USER_NAME").length() != 0) { // Not using windows authentication + String[] parts = iniFile.get("USER_NAME").split("/"); + if (parts.length == 2) { + dbSettings.user = parts[1]; + dbSettings.domain = parts[0]; + } + } + } else if (iniFile.get("DATA_TYPE").equalsIgnoreCase("PDW")) { + dbSettings.dbType = DbType.PDW; + if (iniFile.get("USER_NAME").length() != 0) { // Not using windows authentication + String[] parts = iniFile.get("USER_NAME").split("/"); + if (parts.length == 2) { + dbSettings.user = parts[1]; + dbSettings.domain = parts[0]; + } + } + } else if (iniFile.get("DATA_TYPE").equalsIgnoreCase("MS Access")) + dbSettings.dbType = DbType.MSACCESS; + else if (iniFile.get("DATA_TYPE").equalsIgnoreCase("Teradata")) + dbSettings.dbType = DbType.TERADATA; + } + if (iniFile.get("TABLES_TO_SCAN").equalsIgnoreCase("*")) { + try (RichConnection connection = new RichConnection(dbSettings.server, dbSettings.domain, dbSettings.user, dbSettings.password, dbSettings.dbType)) { + dbSettings.tables.addAll(connection.getTableNames(dbSettings.database)); + } + } else { + for (String table : iniFile.get("TABLES_TO_SCAN").split(",")) { + if (dbSettings.dataType == DbSettings.CSVFILES) + table = iniFile.get("WORKING_FOLDER") + "/" + table; + dbSettings.tables.add(table); + } + } + + SourceDataScan sourceDataScan = new SourceDataScan(); + int maxRows = Integer.parseInt(iniFile.get("ROWS_PER_TABLE")); + boolean scanValues = iniFile.get("SCAN_FIELD_VALUES").equalsIgnoreCase("yes"); + int minCellCount = Integer.parseInt(iniFile.get("MIN_CELL_COUNT")); + int maxValues = Integer.parseInt(iniFile.get("MAX_DISTINCT_VALUES")); + sourceDataScan.process(dbSettings, maxRows, scanValues, minCellCount, maxValues, iniFile.get("WORKING_FOLDER") + "/ScanReport.xlsx"); + } + + private JComponent createTabsPanel() { + JTabbedPane tabbedPane = new JTabbedPane(); + + JPanel locationPanel = createLocationsPanel(); + tabbedPane.addTab("Locations", null, locationPanel, "Specify the location of the source data and the working folder"); + + JPanel scanPanel = createScanPanel(); + tabbedPane.addTab("Scan", null, scanPanel, "Create a scan of the source data"); + + JPanel fakeDataPanel = createFakeDataPanel(); + tabbedPane.addTab("Fake data generation", null, fakeDataPanel, "Create fake data based on a scan report for development purposes"); + + return tabbedPane; + } + + private JPanel createLocationsPanel() { + JPanel panel = new JPanel(); + + panel.setLayout(new GridBagLayout()); + GridBagConstraints c = new GridBagConstraints(); + c.fill = GridBagConstraints.BOTH; + c.weightx = 0.5; + + JPanel folderPanel = new JPanel(); + folderPanel.setLayout(new BoxLayout(folderPanel, BoxLayout.X_AXIS)); + folderPanel.setBorder(BorderFactory.createTitledBorder("Working folder")); + folderField = new JTextField(); + folderField.setText((new File("").getAbsolutePath())); + folderField.setToolTipText("The folder where all output will be written"); + folderPanel.add(folderField); + JButton pickButton = new JButton("Pick folder"); + pickButton.setToolTipText("Pick a different working folder"); + folderPanel.add(pickButton); + pickButton.addActionListener(new ActionListener() { + public void actionPerformed(ActionEvent e) { + pickFolder(); + } + }); + componentsToDisableWhenRunning.add(pickButton); + c.gridx = 0; + c.gridy = 0; + c.gridwidth = 1; + panel.add(folderPanel, c); + + JPanel sourcePanel = new JPanel(); + sourcePanel.setLayout(new GridLayout(0, 2)); + sourcePanel.setBorder(BorderFactory.createTitledBorder("Source data location")); + sourcePanel.add(new JLabel("Data type")); + sourceType = new JComboBox(new String[] { "Delimited text files", "MySQL", "Oracle", "SQL Server", "PostgreSQL", "MS Access", "PDW", "Redshift", "Teradata" }); + sourceType.setToolTipText("Select the type of source data available"); + sourceType.addItemListener(new ItemListener() { + + @Override + public void itemStateChanged(ItemEvent arg0) { + sourceIsFiles = arg0.getItem().toString().equals("Delimited text files"); + sourceServerField.setEnabled(!sourceIsFiles); + sourceUserField.setEnabled(!sourceIsFiles); + sourcePasswordField.setEnabled(!sourceIsFiles); + sourceDatabaseField.setEnabled(!sourceIsFiles); + sourceDelimiterField.setEnabled(sourceIsFiles); + addAllButton.setEnabled(!sourceIsFiles); + + if (!sourceIsFiles && arg0.getItem().toString().equals("Oracle")) { + sourceServerField + .setToolTipText("For Oracle servers this field contains the SID, servicename, and optionally the port: '/', ':/', '/', or ':/'"); + sourceUserField.setToolTipText("For Oracle servers this field contains the name of the user used to log in"); + sourcePasswordField.setToolTipText("For Oracle servers this field contains the password corresponding to the user"); + sourceDatabaseField + .setToolTipText("For Oracle servers this field contains the schema (i.e. 'user' in Oracle terms) containing the source tables"); + } else if (!sourceIsFiles && arg0.getItem().toString().equals("PostgreSQL")) { + sourceServerField.setToolTipText("For PostgreSQL servers this field contains the host name and database name (/)"); + sourceUserField.setToolTipText("The user used to log in to the server"); + sourcePasswordField.setToolTipText("The password used to log in to the server"); + sourceDatabaseField.setToolTipText("For PostgreSQL servers this field contains the schema containing the source tables"); + } else if (!sourceIsFiles) { + sourceServerField.setToolTipText("This field contains the name or IP address of the database server"); + if (arg0.getItem().toString().equals("SQL Server")) + sourceUserField + .setToolTipText("The user used to log in to the server. Optionally, the domain can be specified as / (e.g. 'MyDomain/Joe')"); + else + sourceUserField.setToolTipText("The user used to log in to the server"); + sourcePasswordField.setToolTipText("The password used to log in to the server"); + sourceDatabaseField.setToolTipText("The name of the database containing the source tables"); + } + } + }); + sourcePanel.add(sourceType); + + sourcePanel.add(new JLabel("Server location")); + sourceServerField = new JTextField("127.0.0.1"); + sourceServerField.setEnabled(false); + sourcePanel.add(sourceServerField); + sourcePanel.add(new JLabel("User name")); + sourceUserField = new JTextField(""); + sourceUserField.setEnabled(false); + sourcePanel.add(sourceUserField); + sourcePanel.add(new JLabel("Password")); + sourcePasswordField = new JPasswordField(""); + sourcePasswordField.setEnabled(false); + sourcePanel.add(sourcePasswordField); + sourcePanel.add(new JLabel("Database name")); + sourceDatabaseField = new JTextField(""); + sourceDatabaseField.setEnabled(false); + sourcePanel.add(sourceDatabaseField); + + sourcePanel.add(new JLabel("Delimiter")); + sourceDelimiterField = new JTextField(","); + sourceDelimiterField.setToolTipText("The delimiter that separates values. Enter 'tab' for tab."); + sourcePanel.add(sourceDelimiterField); + + c.gridx = 0; + c.gridy = 1; + c.gridwidth = 1; + panel.add(sourcePanel, c); + + JPanel testConnectionButtonPanel = new JPanel(); + testConnectionButtonPanel.setLayout(new BoxLayout(testConnectionButtonPanel, BoxLayout.X_AXIS)); + testConnectionButtonPanel.add(Box.createHorizontalGlue()); + + JButton testConnectionButton = new JButton("Test connection"); + testConnectionButton.setBackground(new Color(151, 220, 141)); + testConnectionButton.setToolTipText("Test the connection"); + testConnectionButton.addActionListener(new ActionListener() { + public void actionPerformed(ActionEvent e) { + testConnection(getSourceDbSettings()); + } + }); + componentsToDisableWhenRunning.add(testConnectionButton); + testConnectionButtonPanel.add(testConnectionButton); + + c.gridx = 0; + c.gridy = 2; + c.gridwidth = 1; + panel.add(testConnectionButtonPanel, c); + + return panel; + } + + private JPanel createScanPanel() { + JPanel panel = new JPanel(); + panel.setLayout(new BorderLayout()); + + JPanel tablePanel = new JPanel(); + tablePanel.setLayout(new BorderLayout()); + tablePanel.setBorder(new TitledBorder("Tables to scan")); + tableList = new JList(); + tableList.setToolTipText("Specify the tables (or CSV files) to be scanned here"); + tablePanel.add(new JScrollPane(tableList), BorderLayout.CENTER); + + JPanel tableButtonPanel = new JPanel(); + tableButtonPanel.setLayout(new GridLayout(3, 1)); + addAllButton = new JButton("Add all in DB"); + addAllButton.setToolTipText("Add all tables in the database"); + addAllButton.addActionListener(new ActionListener() { + public void actionPerformed(ActionEvent e) { + addAllTables(); + } + }); + addAllButton.setEnabled(false); + tableButtonPanel.add(addAllButton); + JButton addButton = new JButton("Add"); + addButton.setToolTipText("Add tables to list"); + addButton.addActionListener(new ActionListener() { + public void actionPerformed(ActionEvent e) { + pickTables(); + } + }); + tableButtonPanel.add(addButton); + JButton removeButton = new JButton("Remove"); + removeButton.setToolTipText("Remove tables from list"); + removeButton.addActionListener(new ActionListener() { + public void actionPerformed(ActionEvent e) { + removeTables(); + } + }); + tableButtonPanel.add(removeButton); + tablePanel.add(tableButtonPanel, BorderLayout.EAST); + + panel.add(tablePanel, BorderLayout.CENTER); + + JPanel southPanel = new JPanel(); + southPanel.setLayout(new BoxLayout(southPanel, BoxLayout.Y_AXIS)); + + JPanel scanOptionsPanel = new JPanel(); + scanOptionsPanel.setLayout(new BoxLayout(scanOptionsPanel, BoxLayout.X_AXIS)); + + scanValueScan = new JCheckBox("Scan field values", true); + scanValueScan.setToolTipText("Include a frequency count of field values in the scan report"); + scanValueScan.addChangeListener(new ChangeListener() { + + @Override + public void stateChanged(ChangeEvent arg0) { + scanMinCellCount.setEnabled(((JCheckBox) arg0.getSource()).isSelected()); + scanRowCount.setEnabled(((JCheckBox) arg0.getSource()).isSelected()); + scanValuesCount.setEnabled(((JCheckBox) arg0.getSource()).isSelected()); + } + }); + scanOptionsPanel.add(scanValueScan); + scanOptionsPanel.add(Box.createHorizontalGlue()); + + scanOptionsPanel.add(new JLabel("Min cell count ")); + scanMinCellCount = new JSpinner(); + scanMinCellCount.setValue(5); + scanMinCellCount.setToolTipText("Minimum frequency for a field value to be included in the report"); + scanOptionsPanel.add(scanMinCellCount); + scanOptionsPanel.add(Box.createHorizontalGlue()); + + scanOptionsPanel.add(new JLabel("Max distinct values ")); + scanValuesCount = new JComboBox(new String[] { "100", "1,000", "10,000" }); + scanValuesCount.setSelectedIndex(1); + scanValuesCount.setToolTipText("Maximum number of distinct values per field to be reported"); + scanOptionsPanel.add(scanValuesCount); + scanOptionsPanel.add(Box.createHorizontalGlue()); + + scanOptionsPanel.add(new JLabel("Rows per table ")); + scanRowCount = new JComboBox(new String[] { "100,000", "500,000", "1 million", "all" }); + scanRowCount.setSelectedIndex(0); + scanRowCount.setToolTipText("Maximum number of rows per table to be scanned for field values"); + scanOptionsPanel.add(scanRowCount); + + southPanel.add(scanOptionsPanel); + + southPanel.add(Box.createVerticalStrut(3)); + + JPanel scanButtonPanel = new JPanel(); + scanButtonPanel.setLayout(new BoxLayout(scanButtonPanel, BoxLayout.X_AXIS)); + scanButtonPanel.add(Box.createHorizontalGlue()); + + JButton scanButton = new JButton("Scan tables"); + scanButton.setBackground(new Color(151, 220, 141)); + scanButton.setToolTipText("Scan the selected tables"); + scanButton.addActionListener(new ActionListener() { + public void actionPerformed(ActionEvent e) { + scanRun(); + } + }); + componentsToDisableWhenRunning.add(scanButton); + scanButtonPanel.add(scanButton); + southPanel.add(scanButtonPanel); + + panel.add(southPanel, BorderLayout.SOUTH); + + return panel; + } + + private JPanel createFakeDataPanel() { + JPanel panel = new JPanel(); + + panel.setLayout(new GridBagLayout()); + GridBagConstraints c = new GridBagConstraints(); + c.fill = GridBagConstraints.BOTH; + c.weightx = 0.5; + + JPanel folderPanel = new JPanel(); + folderPanel.setLayout(new BoxLayout(folderPanel, BoxLayout.X_AXIS)); + folderPanel.setBorder(BorderFactory.createTitledBorder("Scan report file")); + scanReportFileField = new JTextField(); + scanReportFileField.setText((new File("ScanReport.xlsx").getAbsolutePath())); + scanReportFileField.setToolTipText("The path to the scan report that will be used as a template to generate the fake data"); + folderPanel.add(scanReportFileField); + JButton pickButton = new JButton("Pick file"); + pickButton.setToolTipText("Pick a scan report file"); + folderPanel.add(pickButton); + pickButton.addActionListener(new ActionListener() { + public void actionPerformed(ActionEvent e) { + pickScanReportFile(); + } + }); + componentsToDisableWhenRunning.add(pickButton); + c.gridx = 0; + c.gridy = 0; + c.gridwidth = 1; + panel.add(folderPanel, c); + + JPanel targetPanel = new JPanel(); + targetPanel.setLayout(new GridLayout(0, 2)); + targetPanel.setBorder(BorderFactory.createTitledBorder("Target data location")); + targetPanel.add(new JLabel("Data type")); + targetType = new JComboBox(new String[] { "Delimited text files", "MySQL", "Oracle", "SQL Server", "PostgreSQL" }); + // targetType = new JComboBox(new String[] { "Delimited text files", "MySQL" }); + targetType.setToolTipText("Select the type of source data available"); + targetType.addItemListener(new ItemListener() { + + @Override + public void itemStateChanged(ItemEvent arg0) { + targetIsFiles = arg0.getItem().toString().equals("Delimited text files"); + targetServerField.setEnabled(!targetIsFiles); + targetUserField.setEnabled(!targetIsFiles); + targetPasswordField.setEnabled(!targetIsFiles); + targetDatabaseField.setEnabled(!targetIsFiles); + targetCSVFormat.setEnabled(targetIsFiles); + + if (!targetIsFiles && arg0.getItem().toString().equals("Oracle")) { + targetServerField + .setToolTipText("For Oracle servers this field contains the SID, servicename, and optionally the port: '/', ':/', '/', or ':/'"); + targetUserField.setToolTipText("For Oracle servers this field contains the name of the user used to log in"); + targetPasswordField.setToolTipText("For Oracle servers this field contains the password corresponding to the user"); + targetDatabaseField + .setToolTipText("For Oracle servers this field contains the schema (i.e. 'user' in Oracle terms) containing the source tables"); + } else if (!targetIsFiles && arg0.getItem().toString().equals("PostgreSQL")) { + targetServerField.setToolTipText("For PostgreSQL servers this field contains the host name and database name (/)"); + targetUserField.setToolTipText("The user used to log in to the server"); + targetPasswordField.setToolTipText("The password used to log in to the server"); + targetDatabaseField.setToolTipText("For PostgreSQL servers this field contains the schema containing the source tables"); + } else if (!targetIsFiles) { + targetServerField.setToolTipText("This field contains the name or IP address of the database server"); + if (arg0.getItem().toString().equals("SQL Server")) + targetUserField + .setToolTipText("The user used to log in to the server. Optionally, the domain can be specified as / (e.g. 'MyDomain/Joe')"); + else + targetUserField.setToolTipText("The user used to log in to the server"); + targetPasswordField.setToolTipText("The password used to log in to the server"); + targetDatabaseField.setToolTipText("The name of the database containing the source tables"); + } + } + }); + targetPanel.add(targetType); + + targetPanel.add(new JLabel("Server location")); + targetServerField = new JTextField("127.0.0.1"); + targetServerField.setEnabled(false); + targetPanel.add(targetServerField); + targetPanel.add(new JLabel("User name")); + targetUserField = new JTextField(""); + targetUserField.setEnabled(false); + targetPanel.add(targetUserField); + targetPanel.add(new JLabel("Password")); + targetPasswordField = new JPasswordField(""); + targetPasswordField.setEnabled(false); + targetPanel.add(targetPasswordField); + targetPanel.add(new JLabel("Database name")); + targetDatabaseField = new JTextField(""); + targetDatabaseField.setEnabled(false); + targetPanel.add(targetDatabaseField); + + targetPanel.add(new JLabel("CSV Format")); + targetCSVFormat = new JComboBox<>( + new String[] { "Default (comma, CRLF)", "TDF (tab, CRLF)", "MySQL (tab, LF)", "RFC4180", "Excel CSV" }); + targetCSVFormat.setToolTipText("The format of the output"); + targetCSVFormat.setEnabled(true); + targetPanel.add(targetCSVFormat); + + c.gridx = 0; + c.gridy = 1; + c.gridwidth = 1; + panel.add(targetPanel, c); + + JPanel fakeDataButtonPanel = new JPanel(); + fakeDataButtonPanel.setLayout(new BoxLayout(fakeDataButtonPanel, BoxLayout.X_AXIS)); + + fakeDataButtonPanel.add(new JLabel("Max rows per table")); + generateRowCount = new JSpinner(); + generateRowCount.setValue(10000); + fakeDataButtonPanel.add(generateRowCount); + fakeDataButtonPanel.add(Box.createHorizontalGlue()); + + JButton testConnectionButton = new JButton("Test connection"); + testConnectionButton.setBackground(new Color(151, 220, 141)); + testConnectionButton.setToolTipText("Test the connection"); + testConnectionButton.addActionListener(new ActionListener() { + public void actionPerformed(ActionEvent e) { + testConnection(getTargetDbSettings()); + } + }); + componentsToDisableWhenRunning.add(testConnectionButton); + fakeDataButtonPanel.add(testConnectionButton); + + JButton fakeDataButton = new JButton("Generate fake data"); + fakeDataButton.setBackground(new Color(151, 220, 141)); + fakeDataButton.setToolTipText("Generate fake data based on the scan report"); + fakeDataButton.addActionListener(new ActionListener() { + public void actionPerformed(ActionEvent e) { + fakeDataRun(); + } + }); + componentsToDisableWhenRunning.add(fakeDataButton); + fakeDataButtonPanel.add(fakeDataButton); + + c.gridx = 0; + c.gridy = 2; + c.gridwidth = 1; + panel.add(fakeDataButtonPanel, c); + + return panel; + } + + private JComponent createConsolePanel() { + JTextArea consoleArea = new JTextArea(); + consoleArea.setToolTipText("General progress information"); + consoleArea.setEditable(false); + Console console = new Console(); + console.setTextArea(consoleArea); + System.setOut(new PrintStream(console)); + System.setErr(new PrintStream(console)); + JScrollPane consoleScrollPane = new JScrollPane(consoleArea); + consoleScrollPane.setBorder(BorderFactory.createTitledBorder("Console")); + consoleScrollPane.setPreferredSize(new Dimension(800, 200)); + consoleScrollPane.setAutoscrolls(true); + ObjectExchange.console = console; + return consoleScrollPane; + } + + private void loadIcons(JFrame f) { + List icons = new ArrayList(); + icons.add(loadIcon("WhiteRabbit16.png", f)); + icons.add(loadIcon("WhiteRabbit32.png", f)); + icons.add(loadIcon("WhiteRabbit48.png", f)); + icons.add(loadIcon("WhiteRabbit64.png", f)); + icons.add(loadIcon("WhiteRabbit128.png", f)); + icons.add(loadIcon("WhiteRabbit256.png", f)); + f.setIconImages(icons); + } + + private Image loadIcon(String name, JFrame f) { + Image icon = Toolkit.getDefaultToolkit().getImage(WhiteRabbitMain.class.getResource(name)); + MediaTracker mediaTracker = new MediaTracker(f); + mediaTracker.addImage(icon, 0); + try { + mediaTracker.waitForID(0); + return icon; + } catch (Exception e1) { + e1.printStackTrace(); + } + return null; + } + + private void pickFolder() { + JFileChooser fileChooser = new JFileChooser(new File(folderField.getText())); + fileChooser.setFileSelectionMode(JFileChooser.DIRECTORIES_ONLY); + int returnVal = fileChooser.showDialog(frame, "Select folder"); + if (returnVal == JFileChooser.APPROVE_OPTION) + folderField.setText(fileChooser.getSelectedFile().getAbsolutePath()); + } + + private void pickScanReportFile() { + JFileChooser fileChooser = new JFileChooser(new File(folderField.getText())); + fileChooser.setFileSelectionMode(JFileChooser.FILES_ONLY); + int returnVal = fileChooser.showDialog(frame, "Select scan report file"); + if (returnVal == JFileChooser.APPROVE_OPTION) + scanReportFileField.setText(fileChooser.getSelectedFile().getAbsolutePath()); + } + + private void removeTables() { + for (String item : tableList.getSelectedValuesList()) { + tables.remove(item); + tableList.setListData(tables); + } + } + + private void addAllTables() { + DbSettings sourceDbSettings = getSourceDbSettings(); + if (sourceDbSettings != null) { + RichConnection connection = new RichConnection(sourceDbSettings.server, sourceDbSettings.domain, sourceDbSettings.user, sourceDbSettings.password, + sourceDbSettings.dbType); + for (String table : connection.getTableNames(sourceDbSettings.database)) { + if (!tables.contains(table)) + tables.add((String) table); + tableList.setListData(tables); + } + connection.close(); + } + } + + private void pickTables() { + DbSettings sourceDbSettings = getSourceDbSettings(); + if (sourceDbSettings != null) { + if (sourceDbSettings.dataType == DbSettings.CSVFILES) { + JFileChooser fileChooser = new JFileChooser(new File(folderField.getText())); + fileChooser.setMultiSelectionEnabled(true); + fileChooser.setFileSelectionMode(JFileChooser.FILES_ONLY); + FileNameExtensionFilter filter = new FileNameExtensionFilter("Delimited text files", "csv", "txt"); + fileChooser.setFileFilter(filter); + + int returnVal = fileChooser.showDialog(frame, "Select tables"); + if (returnVal == JFileChooser.APPROVE_OPTION) { + for (File table : fileChooser.getSelectedFiles()) { + String tableName = DirectoryUtilities.getRelativePath(new File(folderField.getText()), table); + if (!tables.contains(tableName)) + tables.add(tableName); + tableList.setListData(tables); + } + + } + } else if (sourceDbSettings.dataType == DbSettings.DATABASE) { + RichConnection connection = new RichConnection(sourceDbSettings.server, sourceDbSettings.domain, sourceDbSettings.user, + sourceDbSettings.password, sourceDbSettings.dbType); + String tableNames = StringUtilities.join(connection.getTableNames(sourceDbSettings.database), "\t"); + if (tableNames.length() == 0) { + JOptionPane.showMessageDialog(frame, "No tables found in database " + sourceDbSettings.database, "Error fetching table names", + JOptionPane.ERROR_MESSAGE); + } else { + DBTableSelectionDialog selectionDialog = new DBTableSelectionDialog(frame, true, tableNames); + if (selectionDialog.getAnswer()) { + for (Object item : selectionDialog.getSelectedItems()) { + if (!tables.contains(item)) + tables.add((String) item); + tableList.setListData(tables); + } + } + } + connection.close(); + } + } + } + + private DbSettings getSourceDbSettings() { + DbSettings dbSettings = new DbSettings(); + if (sourceType.getSelectedItem().equals("Delimited text files")) { + dbSettings.dataType = DbSettings.CSVFILES; + if (sourceDelimiterField.getText().length() == 0) { + JOptionPane.showMessageDialog(frame, "Delimiter field cannot be empty for source database", "Error connecting to server", + JOptionPane.ERROR_MESSAGE); + return null; + } + if (sourceDelimiterField.getText().toLowerCase().equals("tab")) + dbSettings.delimiter = '\t'; + else + dbSettings.delimiter = sourceDelimiterField.getText().charAt(0); + } else { + dbSettings.dataType = DbSettings.DATABASE; + dbSettings.user = sourceUserField.getText(); + dbSettings.password = sourcePasswordField.getText(); + dbSettings.server = sourceServerField.getText(); + dbSettings.database = sourceDatabaseField.getText().trim().length() == 0 ? null : sourceDatabaseField.getText(); + if (sourceType.getSelectedItem().toString().equals("MySQL")) + dbSettings.dbType = DbType.MYSQL; + else if (sourceType.getSelectedItem().toString().equals("Oracle")) + dbSettings.dbType = DbType.ORACLE; + else if (sourceType.getSelectedItem().toString().equals("PostgreSQL")) + dbSettings.dbType = DbType.POSTGRESQL; + else if (sourceType.getSelectedItem().toString().equals("Redshift")) + dbSettings.dbType = DbType.REDSHIFT; + else if (sourceType.getSelectedItem().toString().equals("SQL Server")) { + dbSettings.dbType = DbType.MSSQL; + if (sourceUserField.getText().length() != 0) { // Not using windows authentication + String[] parts = sourceUserField.getText().split("/"); + if (parts.length == 2) { + dbSettings.user = parts[1]; + dbSettings.domain = parts[0]; + } + } + } if (sourceType.getSelectedItem().toString().equals("PDW")) { + dbSettings.dbType = DbType.PDW; + if (sourceUserField.getText().length() != 0) { // Not using windows authentication + String[] parts = sourceUserField.getText().split("/"); + if (parts.length == 2) { + dbSettings.user = parts[1]; + dbSettings.domain = parts[0]; + } + } + } else if (sourceType.getSelectedItem().toString().equals("MS Access")) + dbSettings.dbType = DbType.MSACCESS; + else if (sourceType.getSelectedItem().toString().equals("Teradata")) + dbSettings.dbType = DbType.TERADATA; + } + return dbSettings; + } + + private void testConnection(DbSettings dbSettings) { + if (dbSettings.dataType == DbSettings.CSVFILES) { + if (new File(folderField.getText()).exists()) { + String message = "Folder " + folderField.getText() + " found"; + JOptionPane.showMessageDialog(frame, StringUtilities.wordWrap(message, 80), "Working folder found", JOptionPane.INFORMATION_MESSAGE); + } else { + String message = "Folder " + folderField.getText() + " not found"; + JOptionPane.showMessageDialog(frame, StringUtilities.wordWrap(message, 80), "Working folder not found", JOptionPane.ERROR_MESSAGE); + } + } else { + if (dbSettings.database == null || dbSettings.database.equals("")) { + JOptionPane.showMessageDialog(frame, StringUtilities.wordWrap("Please specify database name", 80), "Error connecting to server", + JOptionPane.ERROR_MESSAGE); + return; + } + if (dbSettings.server == null || dbSettings.server.equals("")) { + JOptionPane.showMessageDialog(frame, StringUtilities.wordWrap("Please specify the server", 80), "Error connecting to server", + JOptionPane.ERROR_MESSAGE); + return; + } + + RichConnection connection; + try { + connection = new RichConnection(dbSettings.server, dbSettings.domain, dbSettings.user, dbSettings.password, dbSettings.dbType); + } catch (Exception e) { + String message = "Could not connect: " + e.getMessage(); + JOptionPane.showMessageDialog(frame, StringUtilities.wordWrap(message, 80), "Error connecting to server", JOptionPane.ERROR_MESSAGE); + return; + } + + try { + List tableNames = connection.getTableNames(dbSettings.database); + if (tableNames.size() == 0) + throw new RuntimeException("Unable to retrieve table names for database " + dbSettings.database); + } catch (Exception e) { + String message = "Could not connect to database: " + e.getMessage(); + JOptionPane.showMessageDialog(frame, StringUtilities.wordWrap(message, 80), "Error connecting to server", JOptionPane.ERROR_MESSAGE); + return; + } + + connection.close(); + String message = "Succesfully connected to " + dbSettings.database + " on server " + dbSettings.server; + JOptionPane.showMessageDialog(frame, StringUtilities.wordWrap(message, 80), "Connection succesful", JOptionPane.INFORMATION_MESSAGE); + + } + } + + private DbSettings getTargetDbSettings() { + DbSettings dbSettings = new DbSettings(); + if (targetType.getSelectedItem().equals("Delimited text files")) { + dbSettings.dataType = DbSettings.CSVFILES; + + switch((String) targetCSVFormat.getSelectedItem()) { + case "Default (comma, CRLF)": + dbSettings.csvFormat = CSVFormat.DEFAULT; + break; + case "RFC4180": + dbSettings.csvFormat = CSVFormat.RFC4180; + break; + case "Excel CSV": + dbSettings.csvFormat = CSVFormat.EXCEL; + break; + case "TDF (tab, CRLF)": + dbSettings.csvFormat = CSVFormat.TDF; + break; + case "MySQL (tab, LF)": + dbSettings.csvFormat = CSVFormat.MYSQL; + break; + default: + dbSettings.csvFormat = CSVFormat.RFC4180; + } + + } else { + dbSettings.dataType = DbSettings.DATABASE; + dbSettings.user = targetUserField.getText(); + dbSettings.password = targetPasswordField.getText(); + dbSettings.server = targetServerField.getText(); + dbSettings.database = targetDatabaseField.getText(); + if (targetType.getSelectedItem().toString().equals("MySQL")) + dbSettings.dbType = DbType.MYSQL; + else if (targetType.getSelectedItem().toString().equals("Oracle")) + dbSettings.dbType = DbType.ORACLE; + else if (sourceType.getSelectedItem().toString().equals("PostgreSQL")) + dbSettings.dbType = DbType.POSTGRESQL; + else if (sourceType.getSelectedItem().toString().equals("SQL Server")) { + dbSettings.dbType = DbType.MSSQL; + if (sourceUserField.getText().length() != 0) { // Not using windows authentication + String[] parts = sourceUserField.getText().split("/"); + if (parts.length == 2) { + dbSettings.user = parts[1]; + dbSettings.domain = parts[0]; + } + } + } else if (sourceType.getSelectedItem().toString().equals("PDW")) { + dbSettings.dbType = DbType.PDW; + if (sourceUserField.getText().length() != 0) { // Not using windows authentication + String[] parts = sourceUserField.getText().split("/"); + if (parts.length == 2) { + dbSettings.user = parts[1]; + dbSettings.domain = parts[0]; + } + } + } + + if (dbSettings.database.trim().length() == 0) { + String message = "Please specify a name for the target database"; + JOptionPane.showMessageDialog(frame, StringUtilities.wordWrap(message, 80), "Database error", JOptionPane.ERROR_MESSAGE); + return null; + } + } + return dbSettings; + } + + private void scanRun() { + if (tables.size() == 0) { + if (sourceIsFiles) { + String message = "No files selected for scanning"; + JOptionPane.showMessageDialog(frame, StringUtilities.wordWrap(message, 80), "No files selected", JOptionPane.ERROR_MESSAGE); + return; + } else { + String message = "No tables were selected for scanning. Do you want to select all tables in the database for scanning?"; + String title = "No tables selected"; + int answer = JOptionPane.showConfirmDialog(ObjectExchange.frame, message, title, JOptionPane.YES_NO_OPTION); + if (answer == JOptionPane.YES_OPTION) { + addAllTables(); + } else + return; + } + } + int rowCount = 0; + if (scanRowCount.getSelectedItem().toString().equals("100,000")) + rowCount = 100000; + else if (scanRowCount.getSelectedItem().toString().equals("500,000")) + rowCount = 500000; + else if (scanRowCount.getSelectedItem().toString().equals("1 million")) + rowCount = 1000000; + if (scanRowCount.getSelectedItem().toString().equals("all")) + rowCount = -1; + + int valuesCount = 0; + if (scanValuesCount.getSelectedItem().toString().equals("100")) + valuesCount = 100; + else if (scanValuesCount.getSelectedItem().toString().equals("1,000")) + valuesCount = 1000; + else if (scanValuesCount.getSelectedItem().toString().equals("10,000")) + valuesCount = 10000; + + ScanThread scanThread = new ScanThread(rowCount, valuesCount, scanValueScan.isSelected(), Integer.parseInt(scanMinCellCount.getValue().toString())); + scanThread.start(); + } + + private void fakeDataRun() { + String filename = scanReportFileField.getText(); + if (!new File(filename).exists()) { + String message = "File " + filename + " not found"; + JOptionPane.showMessageDialog(frame, StringUtilities.wordWrap(message, 80), "File not found", JOptionPane.ERROR_MESSAGE); + } else { + FakeDataThread thread = new FakeDataThread(Integer.parseInt(generateRowCount.getValue().toString()), filename); + thread.start(); + } + } + + private class ScanThread extends Thread { + + private int maxRows; + private int maxValues; + private boolean scanValues; + private int minCellCount; + + public ScanThread(int maxRows, int maxValues, boolean scanValues, int minCellCount) { + this.maxRows = maxRows; + this.scanValues = scanValues; + this.minCellCount = minCellCount; + this.maxValues = maxValues; + } + + public void run() { + for (JComponent component : componentsToDisableWhenRunning) + component.setEnabled(false); + try { + SourceDataScan sourceDataScan = new SourceDataScan(); + DbSettings dbSettings = getSourceDbSettings(); + if (dbSettings != null) { + for (String table : tables) { + if (dbSettings.dataType == DbSettings.CSVFILES) + table = folderField.getText() + "/" + table; + dbSettings.tables.add(table); + } + sourceDataScan.process(dbSettings, maxRows, scanValues, minCellCount, maxValues, folderField.getText() + "/ScanReport.xlsx"); + } + } catch (Exception e) { + handleError(e); + } finally { + for (JComponent component : componentsToDisableWhenRunning) + component.setEnabled(true); + } + } + + } + + private class FakeDataThread extends Thread { + private int maxRowCount; + private String filename; + + public FakeDataThread(int maxRowCount, String filename) { + this.maxRowCount = maxRowCount; + this.filename = filename; + } + + public void run() { + for (JComponent component : componentsToDisableWhenRunning) + component.setEnabled(false); + try { + FakeDataGenerator process = new FakeDataGenerator(); + DbSettings dbSettings = getTargetDbSettings(); + if (dbSettings != null) + process.generateData(dbSettings, maxRowCount, filename, folderField.getText()); + } catch (Exception e) { + handleError(e); + } finally { + for (JComponent component : componentsToDisableWhenRunning) + component.setEnabled(true); + } + + } + } + + private class DBTableSelectionDialog extends JDialog implements ActionListener { + private static final long serialVersionUID = 4527207331482143091L; + private JButton yesButton = null; + private JButton noButton = null; + private boolean answer = false; + private JList list; + + public boolean getAnswer() { + return answer; + } + + public DBTableSelectionDialog(JFrame frame, boolean modal, String tableNames) { + super(frame, modal); + + setTitle("Select tables"); + JPanel panel = new JPanel(); + panel.setPreferredSize(new Dimension(800, 500)); + getContentPane().add(panel); + panel.setLayout(new BorderLayout()); + + JLabel message = new JLabel("Select tables"); + panel.add(message, BorderLayout.NORTH); + + list = new JList(tableNames.split("\t")); + JScrollPane scrollPane = new JScrollPane(list); + panel.add(scrollPane, BorderLayout.CENTER); + + JPanel buttonPanel = new JPanel(); + yesButton = new JButton("Select tables"); + yesButton.addActionListener(this); + buttonPanel.add(yesButton); + noButton = new JButton("Cancel"); + noButton.addActionListener(this); + buttonPanel.add(noButton); + panel.add(buttonPanel, BorderLayout.SOUTH); + + pack(); + setLocationRelativeTo(frame); + setVisible(true); + } + + public void actionPerformed(ActionEvent e) { + if (yesButton == e.getSource()) { + answer = true; + setVisible(false); + } else if (noButton == e.getSource()) { + answer = false; + setVisible(false); + } + } + + public List getSelectedItems() { + return list.getSelectedValuesList(); + } + } + + @Override + public void actionPerformed(ActionEvent event) { + switch (event.getActionCommand()) { + case ACTION_CMD_HELP: + doOpenWiki(); + break; + + } + } + + private void doOpenWiki() { + try { + Desktop desktop = Desktop.getDesktop(); + desktop.browse(new URI(WIKI_URL)); + } catch (URISyntaxException | IOException ex) { + + } + } + + private void handleError(Exception e) { + System.err.println("Error: " + e.getMessage()); + String errorReportFilename = ErrorReport.generate(folderField.getText(), e); + String message = "Error: " + e.getLocalizedMessage(); + message += "\nAn error report has been generated:\n" + errorReportFilename; + System.out.println(message); + JOptionPane.showMessageDialog(frame, StringUtilities.wordWrap(message, 80), "Error", JOptionPane.ERROR_MESSAGE); + } + + private JMenuBar createMenuBar() { + JMenuBar menuBar = new JMenuBar(); + JMenu helpMenu = new JMenu("Help"); + menuBar.add(helpMenu); + JMenuItem helpItem = new JMenuItem(ACTION_CMD_HELP); + helpItem.addActionListener(this); + helpItem.setActionCommand(ACTION_CMD_HELP); + helpMenu.add(helpItem); + + return menuBar; + } + +} diff --git a/src/org/ohdsi/whiteRabbit/fakeDataGenerator/FakeDataGenerator.java b/whiterabbit/src/main/java/org/ohdsi/whiteRabbit/fakeDataGenerator/FakeDataGenerator.java similarity index 96% rename from src/org/ohdsi/whiteRabbit/fakeDataGenerator/FakeDataGenerator.java rename to whiterabbit/src/main/java/org/ohdsi/whiteRabbit/fakeDataGenerator/FakeDataGenerator.java index ccb02b9a..0f18aa4c 100644 --- a/src/org/ohdsi/whiteRabbit/fakeDataGenerator/FakeDataGenerator.java +++ b/whiterabbit/src/main/java/org/ohdsi/whiteRabbit/fakeDataGenerator/FakeDataGenerator.java @@ -1,300 +1,300 @@ -/******************************************************************************* - * Copyright 2017 Observational Health Data Sciences and Informatics - * - * This file is part of WhiteRabbit - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - ******************************************************************************/ -package org.ohdsi.whiteRabbit.fakeDataGenerator; - -import java.util.ArrayList; -import java.util.HashSet; -import java.util.List; -import java.util.Random; -import java.util.Set; - -import org.ohdsi.databases.DbType; -import org.ohdsi.databases.RichConnection; -import org.ohdsi.rabbitInAHat.dataModel.Database; -import org.ohdsi.rabbitInAHat.dataModel.Field; -import org.ohdsi.rabbitInAHat.dataModel.Table; -import org.ohdsi.utilities.StringUtilities; -import org.ohdsi.utilities.collections.OneToManySet; -import org.ohdsi.utilities.files.Row; -import org.ohdsi.utilities.files.WriteCSVFileWithHeader; -import org.ohdsi.whiteRabbit.DbSettings; - -public class FakeDataGenerator { - - private RichConnection connection; - // private DbType dbType; - private int targetType; - private OneToManySet primaryKeyToValues; - private int maxRowsPerTable = 1000; - - private static int REGULAR = 0; - private static int RANDOM = 1; - private static int PRIMARY_KEY = 2; - - public static void main(String[] args) { - FakeDataGenerator fakeDataGenerator = new FakeDataGenerator(); - - DbSettings dbSettings = new DbSettings(); - dbSettings.dataType = DbSettings.DATABASE; - dbSettings.dbType = DbType.POSTGRESQL; - dbSettings.server = "127.0.0.1/ohdsi"; - dbSettings.database = "ars"; - dbSettings.user = "postgres"; - dbSettings.password = "F1r3starter"; - - fakeDataGenerator.generateData(dbSettings, 100000, "c:/temp/ScanReport.xlsx", "c:/temp"); - // fakeDataGenerator.generateData(dbSettings, 1000, "C:/home/Research/EMIF WP12/ARS CDM loading/ScanReport.xlsx", "c:/temp"); - } - - public void generateData(DbSettings dbSettings, int maxRowsPerTable, String filename, String folder) { - this.maxRowsPerTable = maxRowsPerTable; - // this.dbType = dbSettings.dbType; - this.targetType = dbSettings.dataType; - - StringUtilities.outputWithTime("Starting creation of fake data"); - System.out.println("Loading scan report from " + filename); - Database database = Database.generateModelFromScanReport(filename); - findValuesForPrimaryKeys(database); - - if (targetType == DbSettings.DATABASE) { - connection = new RichConnection(dbSettings.server, dbSettings.domain, dbSettings.user, dbSettings.password, dbSettings.dbType); - connection.use(dbSettings.database); - for (Table table : database.getTables()) { - if (table.getName().toLowerCase().endsWith(".csv")) - table.setName(table.getName().substring(0, table.getName().length() - 4)); - System.out.println("Generating table " + table.getName()); - createTable(table); - connection.insertIntoTable(generateRows(table).iterator(), table.getName(), false); - } - connection.close(); - } else { - for (Table table : database.getTables()) { - String name = folder + "/" + table.getName(); - if (!name.toLowerCase().endsWith(".csv")) - name = name + ".csv"; - System.out.println("Generating table " + name); - WriteCSVFileWithHeader out = new WriteCSVFileWithHeader(name, dbSettings.csvFormat); - for (Row row : generateRows(table)) - out.write(row); - out.close(); - } - } - StringUtilities.outputWithTime("Done"); - } - - private void findValuesForPrimaryKeys(Database database) { - Set primaryKeys = new HashSet(); - for (Table table : database.getTables()) { - for (Field field : table.getFields()) { - if (field.getValueCounts()[0][0].equals("List truncated...")) { - primaryKeys.add(field.getName()); - } - } - } - - primaryKeyToValues = new OneToManySet(); - for (Table table : database.getTables()) { - for (Field field : table.getFields()) { - if (primaryKeys.contains(field.getName()) && !field.getValueCounts()[0][0].equals("List truncated...")) { - for (int i = 0; i < field.getValueCounts().length; i++) - if (!field.getValueCounts()[i][0].equals("") && !field.getValueCounts()[i][0].equals("List truncated...")) - primaryKeyToValues.put(field.getName(), field.getValueCounts()[i][0]); - } - } - } - } - - private List generateRows(Table table) { - String[] fieldNames = new String[table.getFields().size()]; - ValueGenerator[] valueGenerators = new ValueGenerator[table.getFields().size()]; - int size = maxRowsPerTable; - for (int i = 0; i < table.getFields().size(); i++) { - Field field = table.getFields().get(i); - fieldNames[i] = field.getName(); - ValueGenerator valueGenerator = new ValueGenerator(field); - valueGenerators[i] = valueGenerator; - if (valueGenerator.generatorType == PRIMARY_KEY && valueGenerator.values.length < size) - size = valueGenerator.values.length; - } - List rows = new ArrayList(); - for (int i = 0; i < size; i++) { - Row row = new Row(); - for (int j = 0; j < fieldNames.length; j++) - row.add(fieldNames[j], valueGenerators[j].generate()); - rows.add(row); - } - return rows; - } - - private void createTable(Table table) { - StringBuilder sql = new StringBuilder(); - sql.append("CREATE TABLE " + table.getName() + " (\n"); - for (int i = 0; i < table.getFields().size(); i++) { - Field field = table.getFields().get(i); - sql.append(" " + field.getName() + " " + field.getType().toUpperCase()); - if (i < table.getFields().size() - 1) - sql.append(",\n"); - } - sql.append("\n);"); - connection.execute(sql.toString()); - } - - // private String correctType(Field field) { - // String type = field.getType().toUpperCase(); - // if (field.getMaxLength() == 0) - // field.setMaxLength(256); - // if (dbType == DbType.MYSQL) { - // if (isVarChar(type)) - // return "VARCHAR(" + field.getMaxLength() + ")"; - // else if (isInt(type)) - // return "BIGINT"; - // else if (isNumber(type)) - // return "DOUBLE"; - // else if (isText(type)) - // return "TEXT"; - // else if (type.equals("EMPTY")) - // return "VARCHAR(255)"; - // else - // return type; - // } else if (dbType == DbType.POSTGRESQL) { - // if (isVarChar(type)) - // return "VARCHAR(" + field.getMaxLength() + ")"; - // else if (isInt(type)) - // return "BIGINT"; - // else if (isNumber(type)) - // return "DOUBLE"; - // else if (isText(type)) - // return "TEXT"; - // else if (type.equals("EMPTY")) - // return "VARCHAR(255)"; - // else - // return type; - // } - // return null; - // } - - private boolean isVarChar(String type) { - type = type.toUpperCase(); - return (type.equals("VARCHAR") || type.equals("VARCHAR2") || type.equals("CHARACTER VARYING")); - } - - private boolean isInt(String type) { - type = type.toUpperCase(); - return (type.equals("INT") || type.equals("INTEGER") || type.equals("BIGINT")); - } - - // private boolean isNumber(String type) { - // type = type.toUpperCase(); - // return (type.equals("REAL") || type.equals("DOUBLE") || type.equals("NUMBER") || type.equals("FLOAT") || type.equals("DOUBLE PRECISION")); - // } - // - // private boolean isText(String type) { - // type = type.toUpperCase(); - // return (type.equals("TEXT") || type.equals("CLOB")); - // } - - private class ValueGenerator { - - private String[] values; - private int[] cumulativeFrequency; - private int totalFrequency; - private String type; - private int length; - private int cursor; - private int generatorType = REGULAR; - private Random random = new Random(); - - public ValueGenerator(Field field) { - String[][] valueCounts = field.getValueCounts(); - type = field.getType(); - if (valueCounts[0][0].equals("List truncated...")) { - Set values = primaryKeyToValues.get(field.getName()); - if (values.size() != 0) { - this.values = convertToArray(values); - cursor = 0; - generatorType = PRIMARY_KEY; - } else { - length = field.getMaxLength(); - generatorType = RANDOM; - } - } else { - int length = valueCounts.length; - if (valueCounts[length - 1][1].equals("")) // Last value could be "List truncated..." - length--; - - values = new String[length]; - cumulativeFrequency = new int[length]; - totalFrequency = 0; - for (int i = 0; i < length; i++) { - int frequency = (int) (Double.parseDouble(valueCounts[i][1])); - totalFrequency += frequency; - - values[i] = valueCounts[i][0]; - cumulativeFrequency[i] = totalFrequency; - } - generatorType = REGULAR; - } - } - - private String[] convertToArray(Set set) { - String[] array = new String[set.size()]; - int i = 0; - for (String item : set) - array[i++] = item; - return array; - } - - public String generate() { - if (generatorType == RANDOM) { // Random generate a string: - if (isVarChar(type)) { - StringBuilder sb = new StringBuilder(); - for (int i = 0; i < length; i++) - sb.append(Character.toChars(65 + random.nextInt(26))); - return sb.toString(); - } else if (isInt(type)) { - StringBuilder sb = new StringBuilder(); - for (int i = 0; i < length; i++) - sb.append(Character.toChars(48 + random.nextInt(10))); - return sb.toString(); - } else if (type.equals("Date")) // todo: add code - return ""; - else if (type.equals("Real")) // todo: add code - return ""; - else if (type.equals("Empty")) - return ""; - else - return ""; - } else if (generatorType == PRIMARY_KEY) { // Pick the next value: - String value = values[cursor]; - cursor++; - if (cursor >= values.length) - cursor = 0; - return value; - } else { // Sample from values: - int index = random.nextInt(totalFrequency); - int i = 0; - while (i < values.length - 1 && cumulativeFrequency[i] < index) - i++; - if (!type.equals("VarChar") && values[i].trim().length() == 0) - return ""; - return values[i]; - } - } - } - -} +/******************************************************************************* + * Copyright 2019 Observational Health Data Sciences and Informatics + * + * This file is part of WhiteRabbit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + ******************************************************************************/ +package org.ohdsi.whiteRabbit.fakeDataGenerator; + +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Random; +import java.util.Set; + +import org.ohdsi.databases.DbType; +import org.ohdsi.databases.RichConnection; +import org.ohdsi.rabbitInAHat.dataModel.Database; +import org.ohdsi.rabbitInAHat.dataModel.Field; +import org.ohdsi.rabbitInAHat.dataModel.Table; +import org.ohdsi.utilities.StringUtilities; +import org.ohdsi.utilities.collections.OneToManySet; +import org.ohdsi.utilities.files.Row; +import org.ohdsi.utilities.files.WriteCSVFileWithHeader; +import org.ohdsi.whiteRabbit.DbSettings; + +public class FakeDataGenerator { + + private RichConnection connection; + // private DbType dbType; + private int targetType; + private OneToManySet primaryKeyToValues; + private int maxRowsPerTable = 1000; + + private static int REGULAR = 0; + private static int RANDOM = 1; + private static int PRIMARY_KEY = 2; + + public static void main(String[] args) { + FakeDataGenerator fakeDataGenerator = new FakeDataGenerator(); + + DbSettings dbSettings = new DbSettings(); + dbSettings.dataType = DbSettings.DATABASE; + dbSettings.dbType = DbType.POSTGRESQL; + dbSettings.server = "127.0.0.1/ohdsi"; + dbSettings.database = "ars"; + dbSettings.user = "postgres"; + dbSettings.password = "F1r3starter"; + + fakeDataGenerator.generateData(dbSettings, 100000, "c:/temp/ScanReport.xlsx", "c:/temp"); + // fakeDataGenerator.generateData(dbSettings, 1000, "C:/home/Research/EMIF WP12/ARS CDM loading/ScanReport.xlsx", "c:/temp"); + } + + public void generateData(DbSettings dbSettings, int maxRowsPerTable, String filename, String folder) { + this.maxRowsPerTable = maxRowsPerTable; + // this.dbType = dbSettings.dbType; + this.targetType = dbSettings.dataType; + + StringUtilities.outputWithTime("Starting creation of fake data"); + System.out.println("Loading scan report from " + filename); + Database database = Database.generateModelFromScanReport(filename); + findValuesForPrimaryKeys(database); + + if (targetType == DbSettings.DATABASE) { + connection = new RichConnection(dbSettings.server, dbSettings.domain, dbSettings.user, dbSettings.password, dbSettings.dbType); + connection.use(dbSettings.database); + for (Table table : database.getTables()) { + if (table.getName().toLowerCase().endsWith(".csv")) + table.setName(table.getName().substring(0, table.getName().length() - 4)); + System.out.println("Generating table " + table.getName()); + createTable(table); + connection.insertIntoTable(generateRows(table).iterator(), table.getName(), false); + } + connection.close(); + } else { + for (Table table : database.getTables()) { + String name = folder + "/" + table.getName(); + if (!name.toLowerCase().endsWith(".csv")) + name = name + ".csv"; + System.out.println("Generating table " + name); + WriteCSVFileWithHeader out = new WriteCSVFileWithHeader(name, dbSettings.csvFormat); + for (Row row : generateRows(table)) + out.write(row); + out.close(); + } + } + StringUtilities.outputWithTime("Done"); + } + + private void findValuesForPrimaryKeys(Database database) { + Set primaryKeys = new HashSet(); + for (Table table : database.getTables()) { + for (Field field : table.getFields()) { + if (field.getValueCounts()[0][0].equals("List truncated...")) { + primaryKeys.add(field.getName()); + } + } + } + + primaryKeyToValues = new OneToManySet(); + for (Table table : database.getTables()) { + for (Field field : table.getFields()) { + if (primaryKeys.contains(field.getName()) && !field.getValueCounts()[0][0].equals("List truncated...")) { + for (int i = 0; i < field.getValueCounts().length; i++) + if (!field.getValueCounts()[i][0].equals("") && !field.getValueCounts()[i][0].equals("List truncated...")) + primaryKeyToValues.put(field.getName(), field.getValueCounts()[i][0]); + } + } + } + } + + private List generateRows(Table table) { + String[] fieldNames = new String[table.getFields().size()]; + ValueGenerator[] valueGenerators = new ValueGenerator[table.getFields().size()]; + int size = maxRowsPerTable; + for (int i = 0; i < table.getFields().size(); i++) { + Field field = table.getFields().get(i); + fieldNames[i] = field.getName(); + ValueGenerator valueGenerator = new ValueGenerator(field); + valueGenerators[i] = valueGenerator; + if (valueGenerator.generatorType == PRIMARY_KEY && valueGenerator.values.length < size) + size = valueGenerator.values.length; + } + List rows = new ArrayList(); + for (int i = 0; i < size; i++) { + Row row = new Row(); + for (int j = 0; j < fieldNames.length; j++) + row.add(fieldNames[j], valueGenerators[j].generate()); + rows.add(row); + } + return rows; + } + + private void createTable(Table table) { + StringBuilder sql = new StringBuilder(); + sql.append("CREATE TABLE " + table.getName() + " (\n"); + for (int i = 0; i < table.getFields().size(); i++) { + Field field = table.getFields().get(i); + sql.append(" " + field.getName() + " " + field.getType().toUpperCase()); + if (i < table.getFields().size() - 1) + sql.append(",\n"); + } + sql.append("\n);"); + connection.execute(sql.toString()); + } + + // private String correctType(Field field) { + // String type = field.getType().toUpperCase(); + // if (field.getMaxLength() == 0) + // field.setMaxLength(256); + // if (dbType == DbType.MYSQL) { + // if (isVarChar(type)) + // return "VARCHAR(" + field.getMaxLength() + ")"; + // else if (isInt(type)) + // return "BIGINT"; + // else if (isNumber(type)) + // return "DOUBLE"; + // else if (isText(type)) + // return "TEXT"; + // else if (type.equals("EMPTY")) + // return "VARCHAR(255)"; + // else + // return type; + // } else if (dbType == DbType.POSTGRESQL) { + // if (isVarChar(type)) + // return "VARCHAR(" + field.getMaxLength() + ")"; + // else if (isInt(type)) + // return "BIGINT"; + // else if (isNumber(type)) + // return "DOUBLE"; + // else if (isText(type)) + // return "TEXT"; + // else if (type.equals("EMPTY")) + // return "VARCHAR(255)"; + // else + // return type; + // } + // return null; + // } + + private boolean isVarChar(String type) { + type = type.toUpperCase(); + return (type.equals("VARCHAR") || type.equals("VARCHAR2") || type.equals("CHARACTER VARYING")); + } + + private boolean isInt(String type) { + type = type.toUpperCase(); + return (type.equals("INT") || type.equals("INTEGER") || type.equals("BIGINT")); + } + + // private boolean isNumber(String type) { + // type = type.toUpperCase(); + // return (type.equals("REAL") || type.equals("DOUBLE") || type.equals("NUMBER") || type.equals("FLOAT") || type.equals("DOUBLE PRECISION")); + // } + // + // private boolean isText(String type) { + // type = type.toUpperCase(); + // return (type.equals("TEXT") || type.equals("CLOB")); + // } + + private class ValueGenerator { + + private String[] values; + private int[] cumulativeFrequency; + private int totalFrequency; + private String type; + private int length; + private int cursor; + private int generatorType = REGULAR; + private Random random = new Random(); + + public ValueGenerator(Field field) { + String[][] valueCounts = field.getValueCounts(); + type = field.getType(); + if (valueCounts[0][0].equals("List truncated...")) { + Set values = primaryKeyToValues.get(field.getName()); + if (values.size() != 0) { + this.values = convertToArray(values); + cursor = 0; + generatorType = PRIMARY_KEY; + } else { + length = field.getMaxLength(); + generatorType = RANDOM; + } + } else { + int length = valueCounts.length; + if (valueCounts[length - 1][1].equals("")) // Last value could be "List truncated..." + length--; + + values = new String[length]; + cumulativeFrequency = new int[length]; + totalFrequency = 0; + for (int i = 0; i < length; i++) { + int frequency = (int) (Double.parseDouble(valueCounts[i][1])); + totalFrequency += frequency; + + values[i] = valueCounts[i][0]; + cumulativeFrequency[i] = totalFrequency; + } + generatorType = REGULAR; + } + } + + private String[] convertToArray(Set set) { + String[] array = new String[set.size()]; + int i = 0; + for (String item : set) + array[i++] = item; + return array; + } + + public String generate() { + if (generatorType == RANDOM) { // Random generate a string: + if (isVarChar(type)) { + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < length; i++) + sb.append(Character.toChars(65 + random.nextInt(26))); + return sb.toString(); + } else if (isInt(type)) { + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < length; i++) + sb.append(Character.toChars(48 + random.nextInt(10))); + return sb.toString(); + } else if (type.equals("Date")) // todo: add code + return ""; + else if (type.equals("Real")) // todo: add code + return ""; + else if (type.equals("Empty")) + return ""; + else + return ""; + } else if (generatorType == PRIMARY_KEY) { // Pick the next value: + String value = values[cursor]; + cursor++; + if (cursor >= values.length) + cursor = 0; + return value; + } else { // Sample from values: + int index = random.nextInt(totalFrequency); + int i = 0; + while (i < values.length - 1 && cumulativeFrequency[i] < index) + i++; + if (!type.equals("VarChar") && values[i].trim().length() == 0) + return ""; + return values[i]; + } + } + } + +} diff --git a/whiterabbit/src/main/java/org/ohdsi/whiteRabbit/scan/SourceDataScan.java b/whiterabbit/src/main/java/org/ohdsi/whiteRabbit/scan/SourceDataScan.java new file mode 100644 index 00000000..971ee624 --- /dev/null +++ b/whiterabbit/src/main/java/org/ohdsi/whiteRabbit/scan/SourceDataScan.java @@ -0,0 +1,518 @@ +/******************************************************************************* + * Copyright 2019 Observational Health Data Sciences and Informatics + * + * This file is part of WhiteRabbit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + ******************************************************************************/ +package org.ohdsi.whiteRabbit.scan; + +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.function.Function; +import java.util.stream.Collectors; + +import org.apache.poi.ss.usermodel.Cell; +import org.apache.poi.ss.usermodel.CellStyle; +import org.apache.poi.ss.usermodel.Row; +import org.apache.poi.ss.usermodel.Sheet; +import org.apache.poi.xssf.streaming.SXSSFWorkbook; +import org.ohdsi.databases.DbType; +import org.ohdsi.databases.RichConnection; +import org.ohdsi.databases.RichConnection.QueryResult; +import org.ohdsi.rabbitInAHat.dataModel.Table; +import org.ohdsi.utilities.StringUtilities; +import org.ohdsi.utilities.collections.CountingSet; +import org.ohdsi.utilities.collections.CountingSet.Count; +import org.ohdsi.utilities.collections.Pair; +import org.ohdsi.utilities.files.ReadTextFile; +import org.ohdsi.whiteRabbit.DbSettings; + +public class SourceDataScan { + + public static int MAX_VALUES_IN_MEMORY = 100000; + public static int MIN_CELL_COUNT_FOR_CSV = 1000000; + public static int N_FOR_FREE_TEXT_CHECK = 1000; + public static int MIN_AVERAGE_LENGTH_FOR_FREE_TEXT = 100; + + private char delimiter = ','; + private int sampleSize; + private boolean scanValues; + private int minCellCount; + private int maxValues; + private DbType dbType; + private String database; + + public void process(DbSettings dbSettings, int sampleSize, boolean scanValues, int minCellCount, int maxValues, String filename) { + this.sampleSize = sampleSize; + this.scanValues = scanValues; + this.minCellCount = minCellCount; + this.maxValues = maxValues; + Map> tableToFieldInfos; + if (dbSettings.dataType == DbSettings.CSVFILES) { + if (!scanValues) + this.minCellCount = Math.max(minCellCount, MIN_CELL_COUNT_FOR_CSV); + tableToFieldInfos = processCsvFiles(dbSettings); + } else + tableToFieldInfos = processDatabase(dbSettings); + generateReport(tableToFieldInfos, filename); + } + + private Map> processDatabase(DbSettings dbSettings) { + try (RichConnection connection = new RichConnection(dbSettings.server, dbSettings.domain, dbSettings.user, dbSettings.password, dbSettings.dbType)) { + connection.setVerbose(false); + connection.use(dbSettings.database); + + dbType = dbSettings.dbType; + database = dbSettings.database; + + return dbSettings.tables.stream() + .collect(Collectors.toMap(Function.identity(), table -> processDatabaseTable(table, connection))); + + } + } + + private Map> processCsvFiles(DbSettings dbSettings) { + delimiter = dbSettings.delimiter; + Map> tableToFieldInfos = new HashMap>(); + for (String table : dbSettings.tables) { + List fieldInfos = processCsvFile(table); + String tableName = new File(table).getName(); + if (!tableToFieldInfos.containsKey(tableName)) { + tableToFieldInfos.put(tableName, fieldInfos); + } else { + tableToFieldInfos.put(table, fieldInfos); + } + + } + return tableToFieldInfos; + } + + private void generateReport(Map> tableToFieldInfos, String filename) { + System.out.println("Generating scan report"); + removeEmptyTables(tableToFieldInfos); + List tables = new ArrayList(tableToFieldInfos.keySet()); + Collections.sort(tables); + + SXSSFWorkbook workbook = new SXSSFWorkbook(100); // keep 100 rows in memory, exceeding rows will be flushed to disk + CellStyle percentageStyle = workbook.createCellStyle(); + percentageStyle.setDataFormat(workbook.createDataFormat().getFormat("0%")); + + // Create overview sheet + Sheet overviewSheet = workbook.createSheet("Overview"); + if (!scanValues) { + addRow(overviewSheet, "Table", "Field", "Type", "N rows"); + for (String table : tables) { + for (FieldInfo fieldInfo : tableToFieldInfos.get(table)) { + addRow(overviewSheet, table, fieldInfo.name, fieldInfo.getTypeDescription(), Long.valueOf(fieldInfo.rowCount)); + } + addRow(overviewSheet, ""); + } + } else { + addRow(overviewSheet, "Table", "Field", "Type", "Max length", "N rows", "N rows checked", "Fraction empty", "N unique values", "Fraction unique values"); + int sheetIndex = 0; + Map sheetNameLookup = new HashMap<>(); + for (String tableName : tables) { + // Make tablename unique + String tableNameIndexed = Table.indexTableNameForSheet(tableName, sheetIndex); + + String sheetName = Table.createSheetNameFromTableName(tableNameIndexed); + sheetNameLookup.put(tableName, sheetName); + + for (FieldInfo fieldInfo : tableToFieldInfos.get(tableName)) { + Long uniqueCount = fieldInfo.uniqueCount; + Double fractionUnique = fieldInfo.getFractionUnique(); + addRow(overviewSheet, tableNameIndexed, fieldInfo.name, fieldInfo.getTypeDescription(), + Integer.valueOf(fieldInfo.maxLength), + Long.valueOf(fieldInfo.rowCount), + Long.valueOf(fieldInfo.nProcessed), + fieldInfo.getFractionEmpty(), + fieldInfo.hasValuesTrimmed() ? String.format("<= %d", uniqueCount) : uniqueCount, + fieldInfo.hasValuesTrimmed() ? String.format("<= %.3f", fractionUnique) : fractionUnique + ); + this.setCellStyles(overviewSheet, percentageStyle, 6, 8); + } + addRow(overviewSheet, ""); + sheetIndex += 1; + } + + // Create per table scan values + for (String tableName : tables) { + Sheet valueSheet = workbook.createSheet(sheetNameLookup.get(tableName)); + + List fieldInfos = tableToFieldInfos.get(tableName); + List>> valueCounts = new ArrayList<>(); + Object[] header = new Object[fieldInfos.size() * 2]; + int maxCount = 0; + for (int i = 0; i < fieldInfos.size(); i++) { + FieldInfo fieldInfo = fieldInfos.get(i); + header[i * 2] = fieldInfo.name; + if (fieldInfo.isFreeText) + header[(i * 2) + 1] = "Word count"; + else + header[(i * 2) + 1] = "Frequency"; + List> counts = fieldInfo.getSortedValuesWithoutSmallValues(); + valueCounts.add(counts); + if (counts.size() > maxCount) + maxCount = counts.size(); + } + addRow(valueSheet, header); + for (int i = 0; i < maxCount; i++) { + Object[] row = new Object[fieldInfos.size() * 2]; + for (int j = 0; j < fieldInfos.size(); j++) { + List> counts = valueCounts.get(j); + if (counts.size() > i) { + row[j * 2] = counts.get(i).getItem1(); + row[(j * 2) + 1] = counts.get(i).getItem2() == -1 ? "" : counts.get(i).getItem2(); + } else { + row[j * 2] = ""; + row[(j * 2) + 1] = ""; + } + } + addRow(valueSheet, row); + } + // Save some memory by derefencing tables already included in the report: + tableToFieldInfos.remove(tableName); + } + } + + try { + FileOutputStream out = new FileOutputStream(new File(filename)); + workbook.write(out); + out.close(); + StringUtilities.outputWithTime("Scan report generated: " + filename); + } catch (IOException e) { + throw new RuntimeException(e.getMessage()); + } + } + + private void removeEmptyTables(Map> tableToFieldInfos) { + tableToFieldInfos.entrySet() + .removeIf(stringListEntry -> stringListEntry.getValue().size() == 0); + } + + private List processDatabaseTable(String table, RichConnection connection) { + StringUtilities.outputWithTime("Scanning table " + table); + + long rowCount = connection.getTableSize(table); + List fieldInfos = fetchTableStructure(connection, table); + if (scanValues) { + int actualCount = 0; + QueryResult queryResult = null; + try { + queryResult = fetchRowsFromTable(connection, table, rowCount); + for (org.ohdsi.utilities.files.Row row : queryResult) { + for (FieldInfo fieldInfo : fieldInfos) { + fieldInfo.processValue(row.get(fieldInfo.name)); + } + actualCount++; + if (sampleSize != -1 && actualCount >= sampleSize) { + System.out.println("Stopped after " + actualCount + " rows"); + break; + } + } + for (FieldInfo fieldInfo : fieldInfos) { + fieldInfo.trim(); + } + } catch (Exception e) { + System.out.println("Error: " + e.getMessage()); + } finally { + if (queryResult != null) { + queryResult.close(); + } + } + } + + return fieldInfos; + } + + private QueryResult fetchRowsFromTable(RichConnection connection, String table, long rowCount) { + String query = null; + + if (sampleSize == -1) { + if (dbType == DbType.MSACCESS) + query = "SELECT * FROM [" + table + "]"; + else if (dbType == DbType.MSSQL || dbType == DbType.PDW) + query = "SELECT * FROM [" + table.replaceAll("\\.", "].[") + "]"; + else + query = "SELECT * FROM " + table; + } else { + if (dbType == DbType.MSSQL) + query = "SELECT * FROM [" + table.replaceAll("\\.", "].[") + "] TABLESAMPLE (" + sampleSize + " ROWS)"; + else if (dbType == DbType.MYSQL) + query = "SELECT * FROM " + table + " ORDER BY RAND() LIMIT " + sampleSize; + else if (dbType == DbType.PDW) + query = "SELECT TOP " + sampleSize + " * FROM [" + table.replaceAll("\\.", "].[") + "] ORDER BY RAND()"; + else if (dbType == DbType.ORACLE) { + if (sampleSize < rowCount) { + double percentage = 100 * sampleSize / (double) rowCount; + if (percentage < 100) + query = "SELECT * FROM " + table + " SAMPLE(" + percentage + ")"; + } else { + query = "SELECT * FROM " + table; + } + } else if (dbType == DbType.POSTGRESQL || dbType == DbType.REDSHIFT) + query = "SELECT * FROM " + table + " ORDER BY RANDOM() LIMIT " + sampleSize; + else if (dbType == DbType.MSACCESS) + query = "SELECT " + "TOP " + sampleSize + " * FROM [" + table + "]"; + } + // System.out.println("SQL: " + query); + return connection.query(query); + + } + + private List fetchTableStructure(RichConnection connection, String table) { + List fieldInfos = new ArrayList(); + + if (dbType == DbType.MSACCESS) { + ResultSet rs = connection.getMsAccessFieldNames(table); + try { + while (rs.next()) { + FieldInfo fieldInfo = new FieldInfo(rs.getString("COLUMN_NAME")); + fieldInfo.type = rs.getString("TYPE_NAME"); + fieldInfo.rowCount = connection.getTableSize(table); + fieldInfos.add(fieldInfo); + } + } catch (SQLException e) { + throw new RuntimeException(e.getMessage()); + } + } else { + String query = null; + if (dbType == DbType.ORACLE) + query = "SELECT COLUMN_NAME,DATA_TYPE FROM ALL_TAB_COLUMNS WHERE table_name = '" + table + "' AND owner = '" + database.toUpperCase() + "'"; + else if (dbType == DbType.MSSQL || dbType == DbType.PDW) { + String trimmedDatabase = database; + if (database.startsWith("[") && database.endsWith("]")) + trimmedDatabase = database.substring(1, database.length() - 1); + String[] parts = table.split("\\."); + query = "SELECT COLUMN_NAME,DATA_TYPE FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_CATALOG='" + trimmedDatabase + "' AND TABLE_SCHEMA='" + parts[0] + + "' AND TABLE_NAME='" + parts[1] + "';"; + } else if (dbType == DbType.MYSQL) + query = "SELECT COLUMN_NAME,DATA_TYPE FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_SCHEMA = '" + database + "' AND TABLE_NAME = '" + table + + "';"; + else if (dbType == DbType.POSTGRESQL || dbType == DbType.REDSHIFT) + query = "SELECT COLUMN_NAME,DATA_TYPE FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_SCHEMA = '" + database.toLowerCase() + "' AND TABLE_NAME = '" + + table.toLowerCase() + "' ORDER BY ordinal_position;"; + else if (dbType == DbType.TERADATA) { + query = "SELECT ColumnName, ColumnType FROM dbc.columns WHERE DatabaseName= '" + database.toLowerCase() + "' AND TableName = '" + + table.toLowerCase() + "';"; + } + + for (org.ohdsi.utilities.files.Row row : connection.query(query)) { + row.upperCaseFieldNames(); + FieldInfo fieldInfo; + if (dbType == DbType.TERADATA) { + fieldInfo = new FieldInfo(row.get("COLUMNNAME")); + } else { + fieldInfo = new FieldInfo(row.get("COLUMN_NAME")); + } + if (dbType == DbType.TERADATA) { + fieldInfo.type = row.get("COLUMNTYPE"); + } else { + fieldInfo.type = row.get("DATA_TYPE"); + } + fieldInfo.rowCount = connection.getTableSize(table); + ; + fieldInfos.add(fieldInfo); + } + } + return fieldInfos; + } + + private List processCsvFile(String filename) { + StringUtilities.outputWithTime("Scanning table " + filename); + List fieldInfos = new ArrayList(); + int lineNr = 0; + for (String line : new ReadTextFile(filename)) { + lineNr++; + List row = StringUtilities.safeSplit(line, delimiter); + for (int i = 0; i < row.size(); i++) { + String column = row.get(i); + if (column.startsWith("\"") && column.endsWith("\"") && column.length() > 1) + column = column.substring(1, column.length() - 1); + column = column.replace("\\\"", "\""); + row.set(i, column); + } + if (lineNr == 1) { + for (String cell : row) + fieldInfos.add(new FieldInfo(cell)); + } else { + if (row.size() == fieldInfos.size()) { // Else there appears to be a formatting error, so skip + for (int i = 0; i < row.size(); i++) + fieldInfos.get(i).processValue(row.get(i)); + } + } + if (sampleSize != -1 && lineNr == sampleSize) + break; + } + for (FieldInfo fieldInfo : fieldInfos) + fieldInfo.trim(); + + return fieldInfos; + } + + private class FieldInfo { + public String type; + public String name; + public CountingSet valueCounts = new CountingSet(); + public long sumLength = 0; + public int maxLength = 0; + public long nProcessed = 0; + public long emptyCount = 0; + public long uniqueCount = 0; + public long rowCount = -1; + public boolean isInteger = true; + public boolean isReal = true; + public boolean isDate = true; + public boolean isFreeText = false; + public boolean tooManyValues = false; + + public FieldInfo(String name) { + this.name = name; + } + + public void trim() { + if (valueCounts.size() > maxValues) { + valueCounts.keepTopN(maxValues); + } + } + + public boolean hasValuesTrimmed() { + return tooManyValues; + } + + public Double getFractionEmpty() { + if (nProcessed == 0) + return 0d; + else + return emptyCount / (double) nProcessed; + } + + public String getTypeDescription() { + if (type != null) + return type; + else if (nProcessed == emptyCount) + return "empty"; + else if (isFreeText) + return "text"; + else if (isDate) + return "date"; + else if (isInteger) + return "int"; + else if (isReal) + return "real"; + else + return "varchar"; + } + + public Double getFractionUnique() { + if (nProcessed == 0 || uniqueCount == 1) { + return 0d; + } + else { + return uniqueCount / (double) nProcessed; + } + + } + + public void processValue(String value) { + String trimValue = value.trim(); + nProcessed++; + sumLength += value.length(); + if (value.length() > maxLength) + maxLength = value.length(); + + if (trimValue.length() == 0) + emptyCount++; + + if (!isFreeText) { + boolean newlyAdded = valueCounts.add(value); + if (newlyAdded) uniqueCount++; + + if (trimValue.length() != 0) { + if (isReal && !StringUtilities.isNumber(trimValue)) + isReal = false; + if (isInteger && !StringUtilities.isLong(trimValue)) + isInteger = false; + if (isDate && !StringUtilities.isDate(trimValue)) + isDate = false; + } + if (nProcessed == N_FOR_FREE_TEXT_CHECK) { + if (!isInteger && !isReal && !isDate) { + double averageLength = sumLength / (double) (nProcessed - emptyCount); + if (averageLength >= MIN_AVERAGE_LENGTH_FOR_FREE_TEXT) { + isFreeText = true; + CountingSet wordCounts = new CountingSet(); + for (Map.Entry entry : valueCounts.key2count.entrySet()) + for (String word : StringUtilities.mapToWords(entry.getKey().toLowerCase())) + wordCounts.add(word, entry.getValue().count); + valueCounts = wordCounts; + } + } + } + } else { + valueCounts.addAll(StringUtilities.mapToWords(trimValue.toLowerCase())); + } + + if (!tooManyValues && valueCounts.size() > MAX_VALUES_IN_MEMORY) { + tooManyValues = true; + this.trim(); + } + } + + public List> getSortedValuesWithoutSmallValues() { + List> result = valueCounts.key2count.entrySet().stream() + .filter(e -> e.getValue().count >= minCellCount) + .sorted(Comparator.>comparingInt(e -> e.getValue().count).reversed()) + .limit(maxValues) + .map(e -> new Pair<>(e.getKey(), e.getValue().count)) + .collect(Collectors.toCollection(ArrayList::new)); + + if (result.size() < valueCounts.key2count.size()) { + result.add(new Pair<>("List truncated...", -1)); + } + return result; + } + } + + private void addRow(Sheet sheet, Object... values) { + Row row = sheet.createRow(sheet.getPhysicalNumberOfRows()); + for (Object value : values) { + Cell cell = row.createCell(row.getPhysicalNumberOfCells()); + + if (value instanceof Integer || value instanceof Long || value instanceof Double) + cell.setCellValue(Double.parseDouble(value.toString())); + else + cell.setCellValue(value.toString()); + + } + } + + private void setCellStyles(Sheet sheet, CellStyle style, int... colNums) { + Row row = sheet.getRow(sheet.getLastRowNum()); + for(int i : colNums) { + Cell cell = row.getCell(i); + if (cell != null) + cell.setCellStyle(style); + } + } +} diff --git a/src/org/ohdsi/whiteRabbit/WhiteRabbit.ico b/whiterabbit/src/main/resources/org/ohdsi/whiteRabbit/WhiteRabbit.ico similarity index 100% rename from src/org/ohdsi/whiteRabbit/WhiteRabbit.ico rename to whiterabbit/src/main/resources/org/ohdsi/whiteRabbit/WhiteRabbit.ico diff --git a/src/org/ohdsi/whiteRabbit/WhiteRabbit128.png b/whiterabbit/src/main/resources/org/ohdsi/whiteRabbit/WhiteRabbit128.png similarity index 100% rename from src/org/ohdsi/whiteRabbit/WhiteRabbit128.png rename to whiterabbit/src/main/resources/org/ohdsi/whiteRabbit/WhiteRabbit128.png diff --git a/src/org/ohdsi/whiteRabbit/WhiteRabbit16.png b/whiterabbit/src/main/resources/org/ohdsi/whiteRabbit/WhiteRabbit16.png similarity index 100% rename from src/org/ohdsi/whiteRabbit/WhiteRabbit16.png rename to whiterabbit/src/main/resources/org/ohdsi/whiteRabbit/WhiteRabbit16.png diff --git a/src/org/ohdsi/whiteRabbit/WhiteRabbit256.png b/whiterabbit/src/main/resources/org/ohdsi/whiteRabbit/WhiteRabbit256.png similarity index 100% rename from src/org/ohdsi/whiteRabbit/WhiteRabbit256.png rename to whiterabbit/src/main/resources/org/ohdsi/whiteRabbit/WhiteRabbit256.png diff --git a/src/org/ohdsi/whiteRabbit/WhiteRabbit32.png b/whiterabbit/src/main/resources/org/ohdsi/whiteRabbit/WhiteRabbit32.png similarity index 100% rename from src/org/ohdsi/whiteRabbit/WhiteRabbit32.png rename to whiterabbit/src/main/resources/org/ohdsi/whiteRabbit/WhiteRabbit32.png diff --git a/src/org/ohdsi/whiteRabbit/WhiteRabbit48.png b/whiterabbit/src/main/resources/org/ohdsi/whiteRabbit/WhiteRabbit48.png similarity index 100% rename from src/org/ohdsi/whiteRabbit/WhiteRabbit48.png rename to whiterabbit/src/main/resources/org/ohdsi/whiteRabbit/WhiteRabbit48.png diff --git a/src/org/ohdsi/whiteRabbit/WhiteRabbit64.png b/whiterabbit/src/main/resources/org/ohdsi/whiteRabbit/WhiteRabbit64.png similarity index 100% rename from src/org/ohdsi/whiteRabbit/WhiteRabbit64.png rename to whiterabbit/src/main/resources/org/ohdsi/whiteRabbit/WhiteRabbit64.png