Merge branch 'riah-source-details'

thehyve · Sep 24, 2019 · 2632075 · 2632075
2 parents 79d2c5d + cbf3ce2
commit 2632075
Show file tree

Hide file tree

Showing 8 changed files with 90 additions and 36 deletions.
diff --git a/rabbit-core/src/main/java/org/ohdsi/rabbitInAHat/dataModel/Database.java b/rabbit-core/src/main/java/org/ohdsi/rabbitInAHat/dataModel/Database.java
@@ -154,10 +154,12 @@ public static Database generateModelFromScanReport(String filename) {
 				String fieldName = row.getStringByHeaderName(ScanFieldName.FIELD);
 				Field field = new Field(fieldName.toLowerCase(), table);
 
-				String fractionEmpty = row.getByHeaderName(ScanFieldName.FRACTION_EMPTY);
-				field.setNullable(fractionEmpty == null || !fractionEmpty.equals("0"));
 				field.setType(row.getByHeaderName(ScanFieldName.TYPE));
 				field.setMaxLength(row.getIntByHeaderName(ScanFieldName.MAX_LENGTH));
+				field.setFractionEmpty(row.getDoubleByHeaderName(ScanFieldName.FRACTION_EMPTY));
+				field.setUniqueCount(row.getIntByHeaderName(ScanFieldName.UNIQUE_COUNT));
+				field.setFractionUnique(row.getDoubleByHeaderName(ScanFieldName.FRACTION_UNIQUE));
+				field.setDescription(row.getStringByHeaderName(ScanFieldName.SOURCE_DESCRIPTION));
 
 				field.setValueCounts(getValueCounts(workbook, tableName, fieldName));
 				table.getFields().add(field);

diff --git a/rabbit-core/src/main/java/org/ohdsi/rabbitInAHat/dataModel/Field.java b/rabbit-core/src/main/java/org/ohdsi/rabbitInAHat/dataModel/Field.java
@@ -29,6 +29,9 @@ public class Field implements MappableItem {
 	private String				description			= "";
 	private Integer				maxLength;
 	private boolean				isStem;
+	private Double				fractionEmpty;
+	private Integer				uniqueCount;
+	private Double				fractionUnique;
 
 	public Field(String name, Table table) {
 		this.table = table;
@@ -72,7 +75,7 @@ public void setValueCounts(String[][] valueCounts) {
 	}
 
 	public int getRowsCheckedCount() {
-		return this.table.getRowsCheckedCount();
+		return this.table.getRowCheckedCount();
 	}
 
 	public boolean isNullable() {
@@ -126,4 +129,29 @@ public boolean isStem() {
 	public void setStem(boolean isStem) {
 		this.isStem = isStem;
 	}
+
+	public Double getFractionEmpty() {
+		return fractionEmpty;
+	}
+
+	public void setFractionEmpty(Double fractionEmpty) {
+		this.fractionEmpty = fractionEmpty;
+		this.setNullable(fractionEmpty == null || fractionEmpty != 0);
+	}
+
+	public Integer getUniqueCount() {
+		return uniqueCount;
+	}
+
+	public void setUniqueCount(Integer uniqueCount) {
+		this.uniqueCount = uniqueCount;
+	}
+
+	public Double getFractionUnique() {
+		return fractionUnique;
+	}
+
+	public void setFractionUnique(Double fractionUnique) {
+		this.fractionUnique = fractionUnique;
+	}
 }
diff --git a/rabbit-core/src/main/java/org/ohdsi/rabbitInAHat/dataModel/Table.java b/rabbit-core/src/main/java/org/ohdsi/rabbitInAHat/dataModel/Table.java
@@ -25,7 +25,7 @@ public class Table implements MappableItem {
 	private Database			db;
 	private String				name;
 	private int					rowCount;
-	private int					rowsCheckedCount;
+	private int rowCheckedCount;
 	private String				comment				= "";
 	private List<Field>			fields				= new ArrayList<Field>();
 	private boolean				isStem				= false;
@@ -73,12 +73,12 @@ public void setRowCount(int rowCount) {
 		this.rowCount = rowCount;
 	}
 
-	public int getRowsCheckedCount() {
-		return rowsCheckedCount;
+	public int getRowCheckedCount() {
+		return rowCheckedCount;
 	}
 
-	public void setRowsCheckedCount(int rowsCheckedCount) {
-		this.rowsCheckedCount = rowsCheckedCount;
+	public void setRowCheckedCount(int rowCheckedCount) {
+		this.rowCheckedCount = rowCheckedCount;
 	}
 
 	public List<Field> getFields() {

diff --git a/rabbit-core/src/main/java/org/ohdsi/utilities/ScanFieldName.java b/rabbit-core/src/main/java/org/ohdsi/utilities/ScanFieldName.java
@@ -8,13 +8,17 @@ public interface ScanFieldName {
     String N_ROWS = "N rows";
     String N_ROWS_CHECKED = "N rows checked";
     String FRACTION_EMPTY = "Fraction empty";
-    String AVERAGE = "Average";
-    String STDEV = "Standard Deviation";
-    String MIN = "Min";
-    String Q1 = "25%";
-    String Q2 = "Median";
-    String Q3 = "75%";
-    String MAX = "Max";
+    String UNIQUE_COUNT = "N unique values";
+    String FRACTION_UNIQUE = "Fraction unique";
+    String SOURCE_DESCRIPTION = "Description";
+    String SOURCE_ALIAS = "Alias";
+//    String AVERAGE = "Average";
+//    String STDEV = "Standard Deviation";
+//    String MIN = "Min";
+//    String Q1 = "25%";
+//    String Q2 = "Median";
+//    String Q3 = "75%";
+//    String MAX = "Max%";
 }
 
 

diff --git a/rabbit-core/src/main/java/org/ohdsi/utilities/files/QuickAndDirtyXlsxReader.java b/rabbit-core/src/main/java/org/ohdsi/utilities/files/QuickAndDirtyXlsxReader.java
@@ -589,20 +589,18 @@ public String getStringByHeaderName(String fieldName) {
 
 		public Double getDoubleByHeaderName(String fieldName) {
 			String value = getStringByHeaderName(fieldName);
-			if (value != null) {
-				return Double.parseDouble(value);
-			} else {
+			if (value == null || value.isEmpty()) {
 				return null;
 			}
+			return Double.parseDouble(value);
 		}
 
 		public Integer getIntByHeaderName(String fieldName) {
 			Double value = getDoubleByHeaderName(fieldName);
 			if (value == null) {
 				return null;
-			} else {
-				return value.intValue();
 			}
+			return value.intValue();
 		}
 	}
 }
diff --git a/rabbitinahat/src/main/java/org/ohdsi/rabbitInAHat/DetailsPanel.java b/rabbitinahat/src/main/java/org/ohdsi/rabbitInAHat/DetailsPanel.java
@@ -284,7 +284,7 @@ public void showTable(Table table) {
 			if (table.getRowCount() > 0) {
 				rowCountLabel.setText(numberFormat.format(table.getRowCount()));
 			} else {
-				rowCountLabel.setText(">= " + numberFormat.format(table.getRowsCheckedCount()));
+				rowCountLabel.setText(">= " + numberFormat.format(table.getRowCheckedCount()));
 			}
 
 			fieldTable.clear();
@@ -316,7 +316,8 @@ private class FieldPanel extends JPanel implements DocumentListener {
 
 		private static final long	serialVersionUID	= -4393026616049677944L;
 		private JLabel				nameLabel			= new JLabel("");
-		private JLabel				rowCountLabel		= new JLabel("");
+		private JLabel 				typeLabel 			= new JLabel("");
+		private JLabel				valueDetailLabel	= new JLabel("");
 		private DescriptionTextArea description			= new DescriptionTextArea ("");
 		private SimpleTableModel	valueTable			= new SimpleTableModel("Value", "Frequency", "Fraction");
 		private JTextArea			commentsArea		= new JTextArea();
@@ -326,9 +327,7 @@ public FieldPanel() {
 			setLayout(new BorderLayout());
 
 			JPanel generalInfoPanel = new JPanel();
-
 			generalInfoPanel.setLayout(new BorderLayout(5,5));
-
 			generalInfoPanel.setBorder(BorderFactory.createTitledBorder("General information"));
 
 			JPanel fieldInfo = new JPanel();
@@ -338,10 +337,18 @@ public FieldPanel() {
 			fieldInfo.add(nameLabel);
 
 			fieldInfo.add(new JLabel("Field type: "));
-			fieldInfo.add(rowCountLabel);
-
-			generalInfoPanel.add(fieldInfo,BorderLayout.NORTH);
-
+			fieldInfo.add(typeLabel);
+
+			generalInfoPanel.add(fieldInfo, BorderLayout.NORTH);
+
+			JPanel sourceDetailsPanel = new JPanel();
+			sourceDetailsPanel.setLayout(new GridLayout(0,2));
+
+			sourceDetailsPanel.add(new JLabel("Unique values: "));
+			sourceDetailsPanel.add(valueDetailLabel);
+
+			generalInfoPanel.add(sourceDetailsPanel);
+
 			JPanel descriptionInfo = new JPanel();
 			descriptionInfo.setLayout(new GridLayout(0,2));
 			descriptionInfo.add(new JLabel("Description: "));
@@ -382,10 +389,27 @@ public void showField(Field field) {
 			this.field = field;
 
 			nameLabel.setText(field.getName());
-			rowCountLabel.setText(field.getType());
+			typeLabel.setText(field.getType());
+
+			// Additional unique count and percentage empty. Hide when not given
+			StringBuilder valueDetailText = new StringBuilder();
+			if (field.getUniqueCount() != null) {
+				valueDetailText.append(numberFormat.format(field.getUniqueCount()));
+			}
+			if (field.getFractionEmpty() != null) {
+				String fractionEmptyFormatted;
+				if (field.getFractionEmpty() > 0 && field.getFractionEmpty() < 0.001) {
+					fractionEmptyFormatted = "<" + percentageFormat.format(0.001);
+				} else {
+					fractionEmptyFormatted = percentageFormat.format(field.getFractionEmpty());
+				}
+				valueDetailText.append(String.format(" (%s empty)", fractionEmptyFormatted));
+			}
+			valueDetailLabel.setText(valueDetailText.toString());
+			valueDetailLabel.getParent().setVisible(!valueDetailLabel.getText().isEmpty());
+
+			// Description. Hide when empty
 			description.setText(field.getDescription());
-
-			// Hide description if it's empty
 			description.getParent().setVisible(!description.getText().isEmpty());
 
 			valueTable.clear();

diff --git a/rabbitinahat/src/main/java/org/ohdsi/rabbitInAHat/RabbitInAHatMain.java b/rabbitinahat/src/main/java/org/ohdsi/rabbitInAHat/RabbitInAHatMain.java
@@ -338,11 +338,11 @@ private JMenuBar createMenuBar() {
 
 		JRadioButtonMenuItem explorationMode = new JRadioButtonMenuItem(ACTION_SET_SOURCE_EXPLORATION_MODE);
 		viewGroup.add(explorationMode);
-		targetMenu.add(explorationMode).addActionListener(evt -> this.setSourceExplorationMode());
+		viewMenu.add(explorationMode).addActionListener(evt -> this.setSourceExplorationMode());
 
 		JRadioButtonMenuItem mappingMode = new JRadioButtonMenuItem(ACTION_SET_MAPPING_MODE, true);
 		viewGroup.add(mappingMode);
-		targetMenu.add(mappingMode).addActionListener(evt -> this.setMappingMode());
+		viewMenu.add(mappingMode).addActionListener(evt -> this.setMappingMode());
 
 		JMenu helpMenu = new JMenu("Help");
 		menuBar.add(helpMenu);

diff --git a/whiterabbit/src/main/java/org/ohdsi/whiteRabbit/scan/SourceDataScan.java b/whiterabbit/src/main/java/org/ohdsi/whiteRabbit/scan/SourceDataScan.java
@@ -132,9 +132,7 @@ private void generateReport(Map<String, List<FieldInfo>> tableToFieldInfos, Stri
 			addRow(overviewSheet,
 					ScanFieldName.TABLE, ScanFieldName.FIELD, ScanFieldName.TYPE, ScanFieldName.MAX_LENGTH,
 					ScanFieldName.N_ROWS, ScanFieldName.N_ROWS_CHECKED, ScanFieldName.FRACTION_EMPTY,
-					"N unique values", "Fraction unique values",
-					ScanFieldName.AVERAGE, ScanFieldName.STDEV,
-					ScanFieldName.MIN, ScanFieldName.Q1, ScanFieldName.Q2, ScanFieldName.Q3, ScanFieldName.MAX
+					ScanFieldName.UNIQUE_COUNT, ScanFieldName.FRACTION_UNIQUE
 			);
 			int sheetIndex = 0;
 			Map<String, String> sheetNameLookup = new HashMap<>();