diff --git a/BookOfOhdsiPlp/PLPViewer.Rproj b/BookOfOhdsiPlp/PLPViewer.Rproj new file mode 100644 index 00000000..8e3c2ebc --- /dev/null +++ b/BookOfOhdsiPlp/PLPViewer.Rproj @@ -0,0 +1,13 @@ +Version: 1.0 + +RestoreWorkspace: Default +SaveWorkspace: Default +AlwaysSaveHistory: Default + +EnableCodeIndexing: Yes +UseSpacesForTab: Yes +NumSpacesForTab: 2 +Encoding: UTF-8 + +RnwWeave: Sweave +LaTeX: pdfLaTeX diff --git a/BookOfOhdsiPlp/data/Analysis_1/plpLog.txt b/BookOfOhdsiPlp/data/Analysis_1/plpLog.txt new file mode 100644 index 00000000..8346016e --- /dev/null +++ b/BookOfOhdsiPlp/data/Analysis_1/plpLog.txt @@ -0,0 +1,38 @@ +2019-06-03 22:53:09 [Main thread] INFO PatientLevelPrediction Patient-Level Prediction Package version 3.0.5 +2019-06-03 22:53:09 [Main thread] INFO PatientLevelPrediction AnalysisID: Analysis_1 +2019-06-03 22:53:09 [Main thread] INFO PatientLevelPrediction CohortID: 3 +2019-06-03 22:53:09 [Main thread] INFO PatientLevelPrediction OutcomeID: 1 +2019-06-03 22:53:09 [Main thread] INFO PatientLevelPrediction Cohort size: 500000 +2019-06-03 22:53:09 [Main thread] INFO PatientLevelPrediction Covariates: 74348 +2019-06-03 22:53:09 [Main thread] INFO PatientLevelPrediction Population size: 351028 +2019-06-03 22:53:09 [Main thread] INFO PatientLevelPrediction Cases: 2601 +2019-06-03 22:53:10 [Main thread] INFO PatientLevelPrediction personSplitter Creating a 25% test and 75% train (into 3 folds) stratified split by person +2019-06-03 22:53:10 [Main thread] INFO PatientLevelPrediction personSplitter Data split into 87757 test cases and 263271 train cases (87758, 87757, 87756) +2019-06-03 22:53:11 [Main thread] INFO PatientLevelPrediction Training Lasso Logistic Regression model +2019-06-03 22:56:44 [Main thread] INFO PatientLevelPrediction fitGLMModel Running Cyclops +2019-06-03 23:09:24 [Main thread] INFO PatientLevelPrediction fitGLMModel Done. +2019-06-03 23:09:24 [Main thread] INFO PatientLevelPrediction fitGLMModel GLM fit status: OK +2019-06-03 23:09:24 [Main thread] INFO PatientLevelPrediction fitGLMModel Fitting model took 13.8 mins +2019-06-03 23:10:00 [Main thread] INFO PatientLevelPrediction predictProbabilities Prediction took 11.9 secs +2019-06-03 23:10:48 [Main thread] INFO PatientLevelPrediction predictProbabilities Prediction took 4.04 secs +2019-06-03 23:10:48 [Main thread] INFO PatientLevelPrediction Train set evaluation +2019-06-03 23:10:51 [Main thread] INFO PatientLevelPrediction evaluatePlp AUC: 78.08 +2019-06-03 23:10:51 [Main thread] INFO PatientLevelPrediction evaluatePlp AUPRC: 4.35 +2019-06-03 23:10:51 [Main thread] INFO PatientLevelPrediction evaluatePlp Brier: 0.01 +2019-06-03 23:11:25 [Main thread] INFO PatientLevelPrediction evaluatePlp Calibration gradient: 1.21 intercept: -0.00 +2019-06-03 23:11:30 [Main thread] INFO PatientLevelPrediction evaluatePlp Average Precision: 0.04 +2019-06-03 23:11:30 [Main thread] INFO PatientLevelPrediction Test set evaluation +2019-06-03 23:11:31 [Main thread] INFO PatientLevelPrediction evaluatePlp AUC: 74.49 +2019-06-03 23:11:31 [Main thread] INFO PatientLevelPrediction evaluatePlp 95% lower AUC: 72.54 +2019-06-03 23:11:31 [Main thread] INFO PatientLevelPrediction evaluatePlp 95% upper AUC: 76.44 +2019-06-03 23:11:31 [Main thread] INFO PatientLevelPrediction evaluatePlp AUPRC: 3.09 +2019-06-03 23:11:31 [Main thread] INFO PatientLevelPrediction evaluatePlp Brier: 0.01 +2019-06-03 23:12:04 [Main thread] INFO PatientLevelPrediction evaluatePlp Calibration gradient: 1.07 intercept: -0.00 +2019-06-03 23:12:05 [Main thread] INFO PatientLevelPrediction evaluatePlp Average Precision: 0.03 +2019-06-03 23:12:06 [Main thread] INFO PatientLevelPrediction Calculating covariate summary @ 2019-06-03 23:12:06 +2019-06-03 23:12:06 [Main thread] INFO PatientLevelPrediction This can take a while... +2019-06-03 23:16:31 [Main thread] INFO PatientLevelPrediction Finished covariate summary @ 2019-06-03 23:16:31 +2019-06-03 23:16:31 [Main thread] INFO PatientLevelPrediction Saving PlpResult +2019-06-03 23:16:37 [Main thread] INFO PatientLevelPrediction plpResult saved to ..\T:/atlasResults/testingDocResults/Analysis_1\plpResult +2019-06-03 23:16:37 [Main thread] INFO PatientLevelPrediction Log saved to T:/atlasResults/testingDocResults/Analysis_1/plplog.txt +2019-06-03 23:16:37 [Main thread] INFO PatientLevelPrediction Run finished successfully. diff --git a/BookOfOhdsiPlp/data/Analysis_1/plpResult.rds b/BookOfOhdsiPlp/data/Analysis_1/plpResult.rds new file mode 100644 index 00000000..830241e8 Binary files /dev/null and b/BookOfOhdsiPlp/data/Analysis_1/plpResult.rds differ diff --git a/BookOfOhdsiPlp/data/Analysis_2/plpLog.txt b/BookOfOhdsiPlp/data/Analysis_2/plpLog.txt new file mode 100644 index 00000000..7cdfc2ca --- /dev/null +++ b/BookOfOhdsiPlp/data/Analysis_2/plpLog.txt @@ -0,0 +1,40 @@ +2019-06-04 00:20:16 [Main thread] INFO PatientLevelPrediction Patient-Level Prediction Package version 3.0.5 +2019-06-04 00:20:16 [Main thread] INFO PatientLevelPrediction AnalysisID: Analysis_2 +2019-06-04 00:20:16 [Main thread] INFO PatientLevelPrediction CohortID: 4 +2019-06-04 00:20:16 [Main thread] INFO PatientLevelPrediction OutcomeID: 1 +2019-06-04 00:20:16 [Main thread] INFO PatientLevelPrediction Cohort size: 216387 +2019-06-04 00:20:16 [Main thread] INFO PatientLevelPrediction Covariates: 63216 +2019-06-04 00:20:16 [Main thread] INFO PatientLevelPrediction Population size: 152994 +2019-06-04 00:20:16 [Main thread] INFO PatientLevelPrediction Cases: 788 +2019-06-04 00:20:16 [Main thread] INFO PatientLevelPrediction personSplitter Creating a 25% test and 75% train (into 3 folds) stratified split by person +2019-06-04 00:20:17 [Main thread] INFO PatientLevelPrediction personSplitter Data split into 38248 test cases and 114746 train cases (38249, 38249, 38248) +2019-06-04 00:20:17 [Main thread] INFO PatientLevelPrediction Training Lasso Logistic Regression model +2019-06-04 00:21:45 [Main thread] INFO PatientLevelPrediction fitGLMModel Running Cyclops +2019-06-04 00:25:21 [Main thread] INFO PatientLevelPrediction fitGLMModel Done. +2019-06-04 00:25:21 [Main thread] INFO PatientLevelPrediction fitGLMModel GLM fit status: OK +2019-06-04 00:25:21 [Main thread] INFO PatientLevelPrediction fitGLMModel Fitting model took 4.03 mins +2019-06-04 00:25:35 [Main thread] INFO PatientLevelPrediction predictProbabilities Prediction took 4.19 secs +2019-06-04 00:25:55 [Main thread] INFO PatientLevelPrediction predictProbabilities Prediction took 1.26 secs +2019-06-04 00:25:55 [Main thread] INFO PatientLevelPrediction Train set evaluation +2019-06-04 00:25:55 [Main thread] INFO PatientLevelPrediction evaluatePlp AUC: 79.85 +2019-06-04 00:25:55 [Main thread] INFO PatientLevelPrediction evaluatePlp 95% lower AUC: 78.05 +2019-06-04 00:25:55 [Main thread] INFO PatientLevelPrediction evaluatePlp 95% upper AUC: 81.65 +2019-06-04 00:25:55 [Main thread] INFO PatientLevelPrediction evaluatePlp AUPRC: 4.09 +2019-06-04 00:25:55 [Main thread] INFO PatientLevelPrediction evaluatePlp Brier: 0.01 +2019-06-04 00:26:11 [Main thread] INFO PatientLevelPrediction evaluatePlp Calibration gradient: 1.28 intercept: -0.00 +2019-06-04 00:26:14 [Main thread] INFO PatientLevelPrediction evaluatePlp Average Precision: 0.04 +2019-06-04 00:26:14 [Main thread] INFO PatientLevelPrediction Test set evaluation +2019-06-04 00:26:14 [Main thread] INFO PatientLevelPrediction evaluatePlp AUC: 73.51 +2019-06-04 00:26:14 [Main thread] INFO PatientLevelPrediction evaluatePlp 95% lower AUC: 70.07 +2019-06-04 00:26:14 [Main thread] INFO PatientLevelPrediction evaluatePlp 95% upper AUC: 76.94 +2019-06-04 00:26:14 [Main thread] INFO PatientLevelPrediction evaluatePlp AUPRC: 1.76 +2019-06-04 00:26:14 [Main thread] INFO PatientLevelPrediction evaluatePlp Brier: 0.01 +2019-06-04 00:26:28 [Main thread] INFO PatientLevelPrediction evaluatePlp Calibration gradient: 0.84 intercept: 0.00 +2019-06-04 00:26:29 [Main thread] INFO PatientLevelPrediction evaluatePlp Average Precision: 0.02 +2019-06-04 00:26:29 [Main thread] INFO PatientLevelPrediction Calculating covariate summary @ 2019-06-04 00:26:29 +2019-06-04 00:26:29 [Main thread] INFO PatientLevelPrediction This can take a while... +2019-06-04 00:28:18 [Main thread] INFO PatientLevelPrediction Finished covariate summary @ 2019-06-04 00:28:18 +2019-06-04 00:28:18 [Main thread] INFO PatientLevelPrediction Saving PlpResult +2019-06-04 00:28:21 [Main thread] INFO PatientLevelPrediction plpResult saved to ..\T:/atlasResults/testingDocResults/Analysis_2\plpResult +2019-06-04 00:28:21 [Main thread] INFO PatientLevelPrediction Log saved to T:/atlasResults/testingDocResults/Analysis_2/plplog.txt +2019-06-04 00:28:21 [Main thread] INFO PatientLevelPrediction Run finished successfully. diff --git a/BookOfOhdsiPlp/data/Analysis_2/plpResult.rds b/BookOfOhdsiPlp/data/Analysis_2/plpResult.rds new file mode 100644 index 00000000..0f78ebc9 Binary files /dev/null and b/BookOfOhdsiPlp/data/Analysis_2/plpResult.rds differ diff --git a/BookOfOhdsiPlp/data/Analysis_3/plpLog.txt b/BookOfOhdsiPlp/data/Analysis_3/plpLog.txt new file mode 100644 index 00000000..0b73fc5d --- /dev/null +++ b/BookOfOhdsiPlp/data/Analysis_3/plpLog.txt @@ -0,0 +1,40 @@ +2019-06-04 00:38:38 [Main thread] INFO PatientLevelPrediction Patient-Level Prediction Package version 3.0.5 +2019-06-04 00:38:38 [Main thread] INFO PatientLevelPrediction AnalysisID: Analysis_3 +2019-06-04 00:38:38 [Main thread] INFO PatientLevelPrediction CohortID: 3 +2019-06-04 00:38:38 [Main thread] INFO PatientLevelPrediction OutcomeID: 2 +2019-06-04 00:38:38 [Main thread] INFO PatientLevelPrediction Cohort size: 500000 +2019-06-04 00:38:38 [Main thread] INFO PatientLevelPrediction Covariates: 74348 +2019-06-04 00:38:38 [Main thread] INFO PatientLevelPrediction Population size: 350460 +2019-06-04 00:38:38 [Main thread] INFO PatientLevelPrediction Cases: 591 +2019-06-04 00:38:39 [Main thread] INFO PatientLevelPrediction personSplitter Creating a 25% test and 75% train (into 3 folds) stratified split by person +2019-06-04 00:38:39 [Main thread] INFO PatientLevelPrediction personSplitter Data split into 87615 test cases and 262845 train cases (87616, 87615, 87614) +2019-06-04 00:38:40 [Main thread] INFO PatientLevelPrediction Training Lasso Logistic Regression model +2019-06-04 00:42:18 [Main thread] INFO PatientLevelPrediction fitGLMModel Running Cyclops +2019-06-04 00:49:27 [Main thread] INFO PatientLevelPrediction fitGLMModel Done. +2019-06-04 00:49:27 [Main thread] INFO PatientLevelPrediction fitGLMModel GLM fit status: OK +2019-06-04 00:49:27 [Main thread] INFO PatientLevelPrediction fitGLMModel Fitting model took 8.26 mins +2019-06-04 00:49:59 [Main thread] INFO PatientLevelPrediction predictProbabilities Prediction took 7.21 secs +2019-06-04 00:50:45 [Main thread] INFO PatientLevelPrediction predictProbabilities Prediction took 2.56 secs +2019-06-04 00:50:45 [Main thread] INFO PatientLevelPrediction Train set evaluation +2019-06-04 00:50:47 [Main thread] INFO PatientLevelPrediction evaluatePlp AUC: 66.79 +2019-06-04 00:50:47 [Main thread] INFO PatientLevelPrediction evaluatePlp 95% lower AUC: 64.08 +2019-06-04 00:50:47 [Main thread] INFO PatientLevelPrediction evaluatePlp 95% upper AUC: 69.50 +2019-06-04 00:50:47 [Main thread] INFO PatientLevelPrediction evaluatePlp AUPRC: 0.44 +2019-06-04 00:50:47 [Main thread] INFO PatientLevelPrediction evaluatePlp Brier: 0.00 +2019-06-04 00:51:20 [Main thread] INFO PatientLevelPrediction evaluatePlp Calibration gradient: 1.79 intercept: -0.00 +2019-06-04 00:51:25 [Main thread] INFO PatientLevelPrediction evaluatePlp Average Precision: 0.00 +2019-06-04 00:51:25 [Main thread] INFO PatientLevelPrediction Test set evaluation +2019-06-04 00:51:26 [Main thread] INFO PatientLevelPrediction evaluatePlp AUC: 60.52 +2019-06-04 00:51:26 [Main thread] INFO PatientLevelPrediction evaluatePlp 95% lower AUC: 55.83 +2019-06-04 00:51:26 [Main thread] INFO PatientLevelPrediction evaluatePlp 95% upper AUC: 65.21 +2019-06-04 00:51:26 [Main thread] INFO PatientLevelPrediction evaluatePlp AUPRC: 0.25 +2019-06-04 00:51:26 [Main thread] INFO PatientLevelPrediction evaluatePlp Brier: 0.00 +2019-06-04 00:51:59 [Main thread] INFO PatientLevelPrediction evaluatePlp Calibration gradient: 1.02 intercept: -0.00 +2019-06-04 00:52:01 [Main thread] INFO PatientLevelPrediction evaluatePlp Average Precision: 0.00 +2019-06-04 00:52:01 [Main thread] INFO PatientLevelPrediction Calculating covariate summary @ 2019-06-04 00:52:01 +2019-06-04 00:52:01 [Main thread] INFO PatientLevelPrediction This can take a while... +2019-06-04 00:56:23 [Main thread] INFO PatientLevelPrediction Finished covariate summary @ 2019-06-04 00:56:23 +2019-06-04 00:56:23 [Main thread] INFO PatientLevelPrediction Saving PlpResult +2019-06-04 00:56:29 [Main thread] INFO PatientLevelPrediction plpResult saved to ..\T:/atlasResults/testingDocResults/Analysis_3\plpResult +2019-06-04 00:56:29 [Main thread] INFO PatientLevelPrediction Log saved to T:/atlasResults/testingDocResults/Analysis_3/plplog.txt +2019-06-04 00:56:29 [Main thread] INFO PatientLevelPrediction Run finished successfully. diff --git a/BookOfOhdsiPlp/data/Analysis_3/plpResult.rds b/BookOfOhdsiPlp/data/Analysis_3/plpResult.rds new file mode 100644 index 00000000..5ca3d205 Binary files /dev/null and b/BookOfOhdsiPlp/data/Analysis_3/plpResult.rds differ diff --git a/BookOfOhdsiPlp/data/Analysis_4/plpLog.txt b/BookOfOhdsiPlp/data/Analysis_4/plpLog.txt new file mode 100644 index 00000000..ec953eb0 --- /dev/null +++ b/BookOfOhdsiPlp/data/Analysis_4/plpLog.txt @@ -0,0 +1,40 @@ +2019-06-04 01:36:46 [Main thread] INFO PatientLevelPrediction Patient-Level Prediction Package version 3.0.5 +2019-06-04 01:36:46 [Main thread] INFO PatientLevelPrediction AnalysisID: Analysis_4 +2019-06-04 01:36:46 [Main thread] INFO PatientLevelPrediction CohortID: 4 +2019-06-04 01:36:46 [Main thread] INFO PatientLevelPrediction OutcomeID: 2 +2019-06-04 01:36:46 [Main thread] INFO PatientLevelPrediction Cohort size: 216387 +2019-06-04 01:36:46 [Main thread] INFO PatientLevelPrediction Covariates: 63216 +2019-06-04 01:36:46 [Main thread] INFO PatientLevelPrediction Population size: 152819 +2019-06-04 01:36:46 [Main thread] INFO PatientLevelPrediction Cases: 164 +2019-06-04 01:36:46 [Main thread] INFO PatientLevelPrediction personSplitter Creating a 25% test and 75% train (into 3 folds) stratified split by person +2019-06-04 01:36:46 [Main thread] INFO PatientLevelPrediction personSplitter Data split into 38205 test cases and 114614 train cases (38205, 38205, 38204) +2019-06-04 01:36:47 [Main thread] INFO PatientLevelPrediction Training Lasso Logistic Regression model +2019-06-04 01:38:10 [Main thread] INFO PatientLevelPrediction fitGLMModel Running Cyclops +2019-06-04 01:41:39 [Main thread] INFO PatientLevelPrediction fitGLMModel Done. +2019-06-04 01:41:39 [Main thread] INFO PatientLevelPrediction fitGLMModel GLM fit status: OK +2019-06-04 01:41:39 [Main thread] INFO PatientLevelPrediction fitGLMModel Fitting model took 3.89 mins +2019-06-04 01:41:51 [Main thread] INFO PatientLevelPrediction predictProbabilities Prediction took 2.85 secs +2019-06-04 01:42:12 [Main thread] INFO PatientLevelPrediction predictProbabilities Prediction took 1.09 secs +2019-06-04 01:42:12 [Main thread] INFO PatientLevelPrediction Train set evaluation +2019-06-04 01:42:12 [Main thread] INFO PatientLevelPrediction evaluatePlp AUC: 70.08 +2019-06-04 01:42:12 [Main thread] INFO PatientLevelPrediction evaluatePlp 95% lower AUC: 65.13 +2019-06-04 01:42:12 [Main thread] INFO PatientLevelPrediction evaluatePlp 95% upper AUC: 75.03 +2019-06-04 01:42:12 [Main thread] INFO PatientLevelPrediction evaluatePlp AUPRC: 0.91 +2019-06-04 01:42:12 [Main thread] INFO PatientLevelPrediction evaluatePlp Brier: 0.00 +2019-06-04 01:42:26 [Main thread] INFO PatientLevelPrediction evaluatePlp Calibration gradient: 1.81 intercept: -0.00 +2019-06-04 01:42:29 [Main thread] INFO PatientLevelPrediction evaluatePlp Average Precision: 0.01 +2019-06-04 01:42:29 [Main thread] INFO PatientLevelPrediction Test set evaluation +2019-06-04 01:42:29 [Main thread] INFO PatientLevelPrediction evaluatePlp AUC: 64.35 +2019-06-04 01:42:29 [Main thread] INFO PatientLevelPrediction evaluatePlp 95% lower AUC: 55.62 +2019-06-04 01:42:29 [Main thread] INFO PatientLevelPrediction evaluatePlp 95% upper AUC: 73.09 +2019-06-04 01:42:29 [Main thread] INFO PatientLevelPrediction evaluatePlp AUPRC: 0.18 +2019-06-04 01:42:29 [Main thread] INFO PatientLevelPrediction evaluatePlp Brier: 0.00 +2019-06-04 01:42:43 [Main thread] INFO PatientLevelPrediction evaluatePlp Calibration gradient: 1.09 intercept: -0.00 +2019-06-04 01:42:44 [Main thread] INFO PatientLevelPrediction evaluatePlp Average Precision: 0.00 +2019-06-04 01:42:44 [Main thread] INFO PatientLevelPrediction Calculating covariate summary @ 2019-06-04 01:42:44 +2019-06-04 01:42:44 [Main thread] INFO PatientLevelPrediction This can take a while... +2019-06-04 01:44:33 [Main thread] INFO PatientLevelPrediction Finished covariate summary @ 2019-06-04 01:44:33 +2019-06-04 01:44:33 [Main thread] INFO PatientLevelPrediction Saving PlpResult +2019-06-04 01:44:36 [Main thread] INFO PatientLevelPrediction plpResult saved to ..\T:/atlasResults/testingDocResults/Analysis_4\plpResult +2019-06-04 01:44:36 [Main thread] INFO PatientLevelPrediction Log saved to T:/atlasResults/testingDocResults/Analysis_4/plplog.txt +2019-06-04 01:44:36 [Main thread] INFO PatientLevelPrediction Run finished successfully. diff --git a/BookOfOhdsiPlp/data/Analysis_4/plpResult.rds b/BookOfOhdsiPlp/data/Analysis_4/plpResult.rds new file mode 100644 index 00000000..45a16b85 Binary files /dev/null and b/BookOfOhdsiPlp/data/Analysis_4/plpResult.rds differ diff --git a/BookOfOhdsiPlp/data/Analysis_5/plpLog.txt b/BookOfOhdsiPlp/data/Analysis_5/plpLog.txt new file mode 100644 index 00000000..1683f3ec --- /dev/null +++ b/BookOfOhdsiPlp/data/Analysis_5/plpLog.txt @@ -0,0 +1,38 @@ +2019-06-03 23:16:38 [Main thread] INFO PatientLevelPrediction Patient-Level Prediction Package version 3.0.5 +2019-06-03 23:16:38 [Main thread] INFO PatientLevelPrediction AnalysisID: Analysis_5 +2019-06-03 23:16:38 [Main thread] INFO PatientLevelPrediction CohortID: 3 +2019-06-03 23:16:38 [Main thread] INFO PatientLevelPrediction OutcomeID: 1 +2019-06-03 23:16:38 [Main thread] INFO PatientLevelPrediction Cohort size: 500000 +2019-06-03 23:16:38 [Main thread] INFO PatientLevelPrediction Covariates: 74348 +2019-06-03 23:16:38 [Main thread] INFO PatientLevelPrediction Population size: 351028 +2019-06-03 23:16:38 [Main thread] INFO PatientLevelPrediction Cases: 2601 +2019-06-03 23:16:38 [Main thread] INFO PatientLevelPrediction personSplitter Creating a 25% test and 75% train (into 3 folds) stratified split by person +2019-06-03 23:16:38 [Main thread] INFO PatientLevelPrediction personSplitter Data split into 87757 test cases and 263271 train cases (87758, 87757, 87756) +2019-06-03 23:16:39 [Main thread] INFO PatientLevelPrediction Training Random forest model +2019-06-03 23:36:49 [Main thread] INFO PatientLevelPrediction fitRandomForest Model with settings: ntrees:500 maxDepth: 4mtry: obtained AUC of 0.707629706784536 +2019-06-03 23:42:28 [Main thread] INFO PatientLevelPrediction fitRandomForest Model with settings: ntrees:500 maxDepth: 10mtry: obtained AUC of 0.718516847753899 +2019-06-03 23:54:40 [Main thread] INFO PatientLevelPrediction fitRandomForest Model with settings: ntrees:500 maxDepth: 17mtry: obtained AUC of 0.724548590513502 +2019-06-04 00:03:54 [Main thread] INFO PatientLevelPrediction predict.pythonReticulate Mapping covariates... +2019-06-04 00:04:12 [Main thread] INFO PatientLevelPrediction predict.pythonReticulate Executing prediction... +2019-06-04 00:04:22 [Main thread] INFO PatientLevelPrediction predict.pythonReticulate Returning results... +2019-06-04 00:04:23 [Main thread] INFO PatientLevelPrediction Train set evaluation +2019-06-04 00:04:26 [Main thread] INFO PatientLevelPrediction evaluatePlp AUC: 90.27 +2019-06-04 00:04:26 [Main thread] INFO PatientLevelPrediction evaluatePlp AUPRC: 66.84 +2019-06-04 00:04:26 [Main thread] INFO PatientLevelPrediction evaluatePlp Brier: 0.01 +2019-06-04 00:04:59 [Main thread] INFO PatientLevelPrediction evaluatePlp Calibration gradient: 2.55 intercept: -0.01 +2019-06-04 00:05:04 [Main thread] INFO PatientLevelPrediction evaluatePlp Average Precision: 0.67 +2019-06-04 00:05:04 [Main thread] INFO PatientLevelPrediction Test set evaluation +2019-06-04 00:05:05 [Main thread] INFO PatientLevelPrediction evaluatePlp AUC: 71.67 +2019-06-04 00:05:05 [Main thread] INFO PatientLevelPrediction evaluatePlp 95% lower AUC: 69.61 +2019-06-04 00:05:05 [Main thread] INFO PatientLevelPrediction evaluatePlp 95% upper AUC: 73.73 +2019-06-04 00:05:05 [Main thread] INFO PatientLevelPrediction evaluatePlp AUPRC: 3.10 +2019-06-04 00:05:05 [Main thread] INFO PatientLevelPrediction evaluatePlp Brier: 0.01 +2019-06-04 00:05:35 [Main thread] INFO PatientLevelPrediction evaluatePlp Calibration gradient: 1.10 intercept: -0.00 +2019-06-04 00:05:37 [Main thread] INFO PatientLevelPrediction evaluatePlp Average Precision: 0.03 +2019-06-04 00:05:37 [Main thread] INFO PatientLevelPrediction Calculating covariate summary @ 2019-06-04 00:05:37 +2019-06-04 00:05:37 [Main thread] INFO PatientLevelPrediction This can take a while... +2019-06-04 00:09:59 [Main thread] INFO PatientLevelPrediction Finished covariate summary @ 2019-06-04 00:09:59 +2019-06-04 00:09:59 [Main thread] INFO PatientLevelPrediction Saving PlpResult +2019-06-04 00:10:04 [Main thread] INFO PatientLevelPrediction plpResult saved to ..\T:/atlasResults/testingDocResults/Analysis_5\plpResult +2019-06-04 00:10:04 [Main thread] INFO PatientLevelPrediction Log saved to T:/atlasResults/testingDocResults/Analysis_5/plplog.txt +2019-06-04 00:10:04 [Main thread] INFO PatientLevelPrediction Run finished successfully. diff --git a/BookOfOhdsiPlp/data/Analysis_5/plpResult.rds b/BookOfOhdsiPlp/data/Analysis_5/plpResult.rds new file mode 100644 index 00000000..ebcf0f36 Binary files /dev/null and b/BookOfOhdsiPlp/data/Analysis_5/plpResult.rds differ diff --git a/BookOfOhdsiPlp/data/Analysis_6/plpLog.txt b/BookOfOhdsiPlp/data/Analysis_6/plpLog.txt new file mode 100644 index 00000000..61f7e186 --- /dev/null +++ b/BookOfOhdsiPlp/data/Analysis_6/plpLog.txt @@ -0,0 +1,40 @@ +2019-06-04 00:28:22 [Main thread] INFO PatientLevelPrediction Patient-Level Prediction Package version 3.0.5 +2019-06-04 00:28:22 [Main thread] INFO PatientLevelPrediction AnalysisID: Analysis_6 +2019-06-04 00:28:22 [Main thread] INFO PatientLevelPrediction CohortID: 4 +2019-06-04 00:28:22 [Main thread] INFO PatientLevelPrediction OutcomeID: 1 +2019-06-04 00:28:22 [Main thread] INFO PatientLevelPrediction Cohort size: 216387 +2019-06-04 00:28:22 [Main thread] INFO PatientLevelPrediction Covariates: 63216 +2019-06-04 00:28:22 [Main thread] INFO PatientLevelPrediction Population size: 152994 +2019-06-04 00:28:22 [Main thread] INFO PatientLevelPrediction Cases: 788 +2019-06-04 00:28:22 [Main thread] INFO PatientLevelPrediction personSplitter Creating a 25% test and 75% train (into 3 folds) stratified split by person +2019-06-04 00:28:22 [Main thread] INFO PatientLevelPrediction personSplitter Data split into 38248 test cases and 114746 train cases (38249, 38249, 38248) +2019-06-04 00:28:22 [Main thread] INFO PatientLevelPrediction Training Random forest model +2019-06-04 00:32:51 [Main thread] INFO PatientLevelPrediction fitRandomForest Model with settings: ntrees:500 maxDepth: 4mtry: obtained AUC of 0.728504539461256 +2019-06-04 00:33:36 [Main thread] INFO PatientLevelPrediction fitRandomForest Model with settings: ntrees:500 maxDepth: 10mtry: obtained AUC of 0.733642157363 +2019-06-04 00:34:48 [Main thread] INFO PatientLevelPrediction fitRandomForest Model with settings: ntrees:500 maxDepth: 17mtry: obtained AUC of 0.732360445000086 +2019-06-04 00:35:59 [Main thread] INFO PatientLevelPrediction predict.pythonReticulate Mapping covariates... +2019-06-04 00:36:06 [Main thread] INFO PatientLevelPrediction predict.pythonReticulate Executing prediction... +2019-06-04 00:36:10 [Main thread] INFO PatientLevelPrediction predict.pythonReticulate Returning results... +2019-06-04 00:36:10 [Main thread] INFO PatientLevelPrediction Train set evaluation +2019-06-04 00:36:11 [Main thread] INFO PatientLevelPrediction evaluatePlp AUC: 86.63 +2019-06-04 00:36:11 [Main thread] INFO PatientLevelPrediction evaluatePlp 95% lower AUC: 84.75 +2019-06-04 00:36:11 [Main thread] INFO PatientLevelPrediction evaluatePlp 95% upper AUC: 88.51 +2019-06-04 00:36:11 [Main thread] INFO PatientLevelPrediction evaluatePlp AUPRC: 49.11 +2019-06-04 00:36:11 [Main thread] INFO PatientLevelPrediction evaluatePlp Brier: 0.00 +2019-06-04 00:36:26 [Main thread] INFO PatientLevelPrediction evaluatePlp Calibration gradient: 2.85 intercept: -0.01 +2019-06-04 00:36:28 [Main thread] INFO PatientLevelPrediction evaluatePlp Average Precision: 0.49 +2019-06-04 00:36:28 [Main thread] INFO PatientLevelPrediction Test set evaluation +2019-06-04 00:36:28 [Main thread] INFO PatientLevelPrediction evaluatePlp AUC: 70.32 +2019-06-04 00:36:28 [Main thread] INFO PatientLevelPrediction evaluatePlp 95% lower AUC: 66.66 +2019-06-04 00:36:28 [Main thread] INFO PatientLevelPrediction evaluatePlp 95% upper AUC: 73.97 +2019-06-04 00:36:28 [Main thread] INFO PatientLevelPrediction evaluatePlp AUPRC: 2.09 +2019-06-04 00:36:28 [Main thread] INFO PatientLevelPrediction evaluatePlp Brier: 0.01 +2019-06-04 00:36:41 [Main thread] INFO PatientLevelPrediction evaluatePlp Calibration gradient: 1.32 intercept: -0.00 +2019-06-04 00:36:42 [Main thread] INFO PatientLevelPrediction evaluatePlp Average Precision: 0.02 +2019-06-04 00:36:42 [Main thread] INFO PatientLevelPrediction Calculating covariate summary @ 2019-06-04 00:36:42 +2019-06-04 00:36:42 [Main thread] INFO PatientLevelPrediction This can take a while... +2019-06-04 00:38:30 [Main thread] INFO PatientLevelPrediction Finished covariate summary @ 2019-06-04 00:38:30 +2019-06-04 00:38:30 [Main thread] INFO PatientLevelPrediction Saving PlpResult +2019-06-04 00:38:33 [Main thread] INFO PatientLevelPrediction plpResult saved to ..\T:/atlasResults/testingDocResults/Analysis_6\plpResult +2019-06-04 00:38:33 [Main thread] INFO PatientLevelPrediction Log saved to T:/atlasResults/testingDocResults/Analysis_6/plplog.txt +2019-06-04 00:38:33 [Main thread] INFO PatientLevelPrediction Run finished successfully. diff --git a/BookOfOhdsiPlp/data/Analysis_6/plpResult.rds b/BookOfOhdsiPlp/data/Analysis_6/plpResult.rds new file mode 100644 index 00000000..dcf9d862 Binary files /dev/null and b/BookOfOhdsiPlp/data/Analysis_6/plpResult.rds differ diff --git a/BookOfOhdsiPlp/data/Analysis_7/plpLog.txt b/BookOfOhdsiPlp/data/Analysis_7/plpLog.txt new file mode 100644 index 00000000..8cbdf8da --- /dev/null +++ b/BookOfOhdsiPlp/data/Analysis_7/plpLog.txt @@ -0,0 +1,40 @@ +2019-06-04 00:56:29 [Main thread] INFO PatientLevelPrediction Patient-Level Prediction Package version 3.0.5 +2019-06-04 00:56:29 [Main thread] INFO PatientLevelPrediction AnalysisID: Analysis_7 +2019-06-04 00:56:29 [Main thread] INFO PatientLevelPrediction CohortID: 3 +2019-06-04 00:56:29 [Main thread] INFO PatientLevelPrediction OutcomeID: 2 +2019-06-04 00:56:29 [Main thread] INFO PatientLevelPrediction Cohort size: 500000 +2019-06-04 00:56:29 [Main thread] INFO PatientLevelPrediction Covariates: 74348 +2019-06-04 00:56:29 [Main thread] INFO PatientLevelPrediction Population size: 350460 +2019-06-04 00:56:29 [Main thread] INFO PatientLevelPrediction Cases: 591 +2019-06-04 00:56:30 [Main thread] INFO PatientLevelPrediction personSplitter Creating a 25% test and 75% train (into 3 folds) stratified split by person +2019-06-04 00:56:30 [Main thread] INFO PatientLevelPrediction personSplitter Data split into 87615 test cases and 262845 train cases (87616, 87615, 87614) +2019-06-04 00:56:31 [Main thread] INFO PatientLevelPrediction Training Random forest model +2019-06-04 01:13:51 [Main thread] INFO PatientLevelPrediction fitRandomForest Model with settings: ntrees:500 maxDepth: 4mtry: obtained AUC of 0.610012224621904 +2019-06-04 01:18:36 [Main thread] INFO PatientLevelPrediction fitRandomForest Model with settings: ntrees:500 maxDepth: 10mtry: obtained AUC of 0.616114737226287 +2019-06-04 01:27:10 [Main thread] INFO PatientLevelPrediction fitRandomForest Model with settings: ntrees:500 maxDepth: 17mtry: obtained AUC of 0.603570641004481 +2019-06-04 01:30:36 [Main thread] INFO PatientLevelPrediction predict.pythonReticulate Mapping covariates... +2019-06-04 01:30:54 [Main thread] INFO PatientLevelPrediction predict.pythonReticulate Executing prediction... +2019-06-04 01:31:05 [Main thread] INFO PatientLevelPrediction predict.pythonReticulate Returning results... +2019-06-04 01:31:06 [Main thread] INFO PatientLevelPrediction Train set evaluation +2019-06-04 01:31:06 [Main thread] INFO PatientLevelPrediction evaluatePlp AUC: 86.14 +2019-06-04 01:31:06 [Main thread] INFO PatientLevelPrediction evaluatePlp 95% lower AUC: 83.50 +2019-06-04 01:31:06 [Main thread] INFO PatientLevelPrediction evaluatePlp 95% upper AUC: 88.78 +2019-06-04 01:31:07 [Main thread] INFO PatientLevelPrediction evaluatePlp AUPRC: 61.18 +2019-06-04 01:31:07 [Main thread] INFO PatientLevelPrediction evaluatePlp Brier: 0.00 +2019-06-04 01:31:40 [Main thread] INFO PatientLevelPrediction evaluatePlp Calibration gradient: 4.87 intercept: -0.01 +2019-06-04 01:31:44 [Main thread] INFO PatientLevelPrediction evaluatePlp Average Precision: 0.61 +2019-06-04 01:31:44 [Main thread] INFO PatientLevelPrediction Test set evaluation +2019-06-04 01:31:45 [Main thread] INFO PatientLevelPrediction evaluatePlp AUC: 64.16 +2019-06-04 01:31:45 [Main thread] INFO PatientLevelPrediction evaluatePlp 95% lower AUC: 59.33 +2019-06-04 01:31:45 [Main thread] INFO PatientLevelPrediction evaluatePlp 95% upper AUC: 69.00 +2019-06-04 01:31:45 [Main thread] INFO PatientLevelPrediction evaluatePlp AUPRC: 2.45 +2019-06-04 01:31:45 [Main thread] INFO PatientLevelPrediction evaluatePlp Brier: 0.00 +2019-06-04 01:32:16 [Main thread] INFO PatientLevelPrediction evaluatePlp Calibration gradient: 1.39 intercept: -0.00 +2019-06-04 01:32:18 [Main thread] INFO PatientLevelPrediction evaluatePlp Average Precision: 0.02 +2019-06-04 01:32:18 [Main thread] INFO PatientLevelPrediction Calculating covariate summary @ 2019-06-04 01:32:18 +2019-06-04 01:32:18 [Main thread] INFO PatientLevelPrediction This can take a while... +2019-06-04 01:36:38 [Main thread] INFO PatientLevelPrediction Finished covariate summary @ 2019-06-04 01:36:38 +2019-06-04 01:36:38 [Main thread] INFO PatientLevelPrediction Saving PlpResult +2019-06-04 01:36:43 [Main thread] INFO PatientLevelPrediction plpResult saved to ..\T:/atlasResults/testingDocResults/Analysis_7\plpResult +2019-06-04 01:36:43 [Main thread] INFO PatientLevelPrediction Log saved to T:/atlasResults/testingDocResults/Analysis_7/plplog.txt +2019-06-04 01:36:43 [Main thread] INFO PatientLevelPrediction Run finished successfully. diff --git a/BookOfOhdsiPlp/data/Analysis_7/plpResult.rds b/BookOfOhdsiPlp/data/Analysis_7/plpResult.rds new file mode 100644 index 00000000..c1e60cc9 Binary files /dev/null and b/BookOfOhdsiPlp/data/Analysis_7/plpResult.rds differ diff --git a/BookOfOhdsiPlp/data/Analysis_8/plpLog.txt b/BookOfOhdsiPlp/data/Analysis_8/plpLog.txt new file mode 100644 index 00000000..8bbf40d4 --- /dev/null +++ b/BookOfOhdsiPlp/data/Analysis_8/plpLog.txt @@ -0,0 +1,40 @@ +2019-06-04 01:44:37 [Main thread] INFO PatientLevelPrediction Patient-Level Prediction Package version 3.0.5 +2019-06-04 01:44:37 [Main thread] INFO PatientLevelPrediction AnalysisID: Analysis_8 +2019-06-04 01:44:37 [Main thread] INFO PatientLevelPrediction CohortID: 4 +2019-06-04 01:44:37 [Main thread] INFO PatientLevelPrediction OutcomeID: 2 +2019-06-04 01:44:37 [Main thread] INFO PatientLevelPrediction Cohort size: 216387 +2019-06-04 01:44:37 [Main thread] INFO PatientLevelPrediction Covariates: 63216 +2019-06-04 01:44:37 [Main thread] INFO PatientLevelPrediction Population size: 152819 +2019-06-04 01:44:37 [Main thread] INFO PatientLevelPrediction Cases: 164 +2019-06-04 01:44:37 [Main thread] INFO PatientLevelPrediction personSplitter Creating a 25% test and 75% train (into 3 folds) stratified split by person +2019-06-04 01:44:37 [Main thread] INFO PatientLevelPrediction personSplitter Data split into 38205 test cases and 114614 train cases (38205, 38205, 38204) +2019-06-04 01:44:37 [Main thread] INFO PatientLevelPrediction Training Random forest model +2019-06-04 01:48:34 [Main thread] INFO PatientLevelPrediction fitRandomForest Model with settings: ntrees:500 maxDepth: 4mtry: obtained AUC of 0.591223451866455 +2019-06-04 01:49:16 [Main thread] INFO PatientLevelPrediction fitRandomForest Model with settings: ntrees:500 maxDepth: 10mtry: obtained AUC of 0.592959094381189 +2019-06-04 01:50:14 [Main thread] INFO PatientLevelPrediction fitRandomForest Model with settings: ntrees:500 maxDepth: 17mtry: obtained AUC of 0.643506611411853 +2019-06-04 01:51:24 [Main thread] INFO PatientLevelPrediction predict.pythonReticulate Mapping covariates... +2019-06-04 01:51:32 [Main thread] INFO PatientLevelPrediction predict.pythonReticulate Executing prediction... +2019-06-04 01:51:35 [Main thread] INFO PatientLevelPrediction predict.pythonReticulate Returning results... +2019-06-04 01:51:35 [Main thread] INFO PatientLevelPrediction Train set evaluation +2019-06-04 01:51:35 [Main thread] INFO PatientLevelPrediction evaluatePlp AUC: 98.17 +2019-06-04 01:51:36 [Main thread] INFO PatientLevelPrediction evaluatePlp 95% lower AUC: 96.43 +2019-06-04 01:51:36 [Main thread] INFO PatientLevelPrediction evaluatePlp 95% upper AUC: 99.91 +2019-06-04 01:51:36 [Main thread] INFO PatientLevelPrediction evaluatePlp AUPRC: 88.80 +2019-06-04 01:51:36 [Main thread] INFO PatientLevelPrediction evaluatePlp Brier: 0.00 +2019-06-04 01:51:51 [Main thread] INFO PatientLevelPrediction evaluatePlp Calibration gradient: 2.35 intercept: -0.00 +2019-06-04 01:51:54 [Main thread] INFO PatientLevelPrediction evaluatePlp Average Precision: 0.89 +2019-06-04 01:51:54 [Main thread] INFO PatientLevelPrediction Test set evaluation +2019-06-04 01:51:54 [Main thread] INFO PatientLevelPrediction evaluatePlp AUC: 67.05 +2019-06-04 01:51:54 [Main thread] INFO PatientLevelPrediction evaluatePlp 95% lower AUC: 59.08 +2019-06-04 01:51:54 [Main thread] INFO PatientLevelPrediction evaluatePlp 95% upper AUC: 75.02 +2019-06-04 01:51:54 [Main thread] INFO PatientLevelPrediction evaluatePlp AUPRC: 0.54 +2019-06-04 01:51:54 [Main thread] INFO PatientLevelPrediction evaluatePlp Brier: 0.00 +2019-06-04 01:52:08 [Main thread] INFO PatientLevelPrediction evaluatePlp Calibration gradient: 0.49 intercept: 0.00 +2019-06-04 01:52:09 [Main thread] INFO PatientLevelPrediction evaluatePlp Average Precision: 0.01 +2019-06-04 01:52:09 [Main thread] INFO PatientLevelPrediction Calculating covariate summary @ 2019-06-04 01:52:09 +2019-06-04 01:52:09 [Main thread] INFO PatientLevelPrediction This can take a while... +2019-06-04 01:53:58 [Main thread] INFO PatientLevelPrediction Finished covariate summary @ 2019-06-04 01:53:58 +2019-06-04 01:53:58 [Main thread] INFO PatientLevelPrediction Saving PlpResult +2019-06-04 01:54:02 [Main thread] INFO PatientLevelPrediction plpResult saved to ..\T:/atlasResults/testingDocResults/Analysis_8\plpResult +2019-06-04 01:54:02 [Main thread] INFO PatientLevelPrediction Log saved to T:/atlasResults/testingDocResults/Analysis_8/plplog.txt +2019-06-04 01:54:02 [Main thread] INFO PatientLevelPrediction Run finished successfully. diff --git a/BookOfOhdsiPlp/data/Analysis_8/plpResult.rds b/BookOfOhdsiPlp/data/Analysis_8/plpResult.rds new file mode 100644 index 00000000..5c2eca54 Binary files /dev/null and b/BookOfOhdsiPlp/data/Analysis_8/plpResult.rds differ diff --git a/BookOfOhdsiPlp/data/settings.csv b/BookOfOhdsiPlp/data/settings.csv new file mode 100644 index 00000000..63359cfc --- /dev/null +++ b/BookOfOhdsiPlp/data/settings.csv @@ -0,0 +1,9 @@ +"outcomeId","cohortId","modelSettingsId","analysisId","devDatabase","populationSettingId","modelSettingId","covariateSettingId","modelSettingName","addExposureDaysToStart","riskWindowStart","addExposureDaysToEnd","riskWindowEnd","plpDataFolder","studyPopFile","plpResultFolder","cohortName","outcomeName" +1,3,1,1,"Optum claims",1,1,1,"Lasso Logistic Regression",0,1,0,365,"T:/atlasResults/testingDocResults/PlpData_L1_T3","T:/atlasResults/testingDocResults/StudyPop_L1_T3_O1.rds","T:/atlasResults/testingDocResults/Analysis_1","New users of ACE inhibitors as first-line monotherapy for hypertension","Acute myocardial infarction events" +1,3,2,5,"Optum claims",1,2,1,"Random forest",0,1,0,365,"T:/atlasResults/testingDocResults/PlpData_L1_T3","T:/atlasResults/testingDocResults/StudyPop_L1_T3_O1.rds","T:/atlasResults/testingDocResults/Analysis_5","New users of ACE inhibitors as first-line monotherapy for hypertension","Acute myocardial infarction events" +1,4,1,2,"Optum claims",1,1,1,"Lasso Logistic Regression",0,1,0,365,"T:/atlasResults/testingDocResults/PlpData_L1_T4","T:/atlasResults/testingDocResults/StudyPop_L1_T4_O1.rds","T:/atlasResults/testingDocResults/Analysis_2","New users of Thiazide-like diuretics as first-line monotherapy for hypertension","Acute myocardial infarction events" +1,4,2,6,"Optum claims",1,2,1,"Random forest",0,1,0,365,"T:/atlasResults/testingDocResults/PlpData_L1_T4","T:/atlasResults/testingDocResults/StudyPop_L1_T4_O1.rds","T:/atlasResults/testingDocResults/Analysis_6","New users of Thiazide-like diuretics as first-line monotherapy for hypertension","Acute myocardial infarction events" +2,3,1,3,"Optum claims",1,1,1,"Lasso Logistic Regression",0,1,0,365,"T:/atlasResults/testingDocResults/PlpData_L1_T3","T:/atlasResults/testingDocResults/StudyPop_L1_T3_O2.rds","T:/atlasResults/testingDocResults/Analysis_3","New users of ACE inhibitors as first-line monotherapy for hypertension","Angioedema events" +2,3,2,7,"Optum claims",1,2,1,"Random forest",0,1,0,365,"T:/atlasResults/testingDocResults/PlpData_L1_T3","T:/atlasResults/testingDocResults/StudyPop_L1_T3_O2.rds","T:/atlasResults/testingDocResults/Analysis_7","New users of ACE inhibitors as first-line monotherapy for hypertension","Angioedema events" +2,4,1,4,"Optum claims",1,1,1,"Lasso Logistic Regression",0,1,0,365,"T:/atlasResults/testingDocResults/PlpData_L1_T4","T:/atlasResults/testingDocResults/StudyPop_L1_T4_O2.rds","T:/atlasResults/testingDocResults/Analysis_4","New users of Thiazide-like diuretics as first-line monotherapy for hypertension","Angioedema events" +2,4,2,8,"Optum claims",1,2,1,"Random forest",0,1,0,365,"T:/atlasResults/testingDocResults/PlpData_L1_T4","T:/atlasResults/testingDocResults/StudyPop_L1_T4_O2.rds","T:/atlasResults/testingDocResults/Analysis_8","New users of Thiazide-like diuretics as first-line monotherapy for hypertension","Angioedema events" diff --git a/BookOfOhdsiPlp/global.R b/BookOfOhdsiPlp/global.R new file mode 100644 index 00000000..d7f2c630 --- /dev/null +++ b/BookOfOhdsiPlp/global.R @@ -0,0 +1,14 @@ +# uncomment if running standalone +##runPlp <- readRDS(file.path("data","results.rds")) +##validatePlp <- readRDS(file.path("data","extValidation.rds")) +source("utils.R") +analysesLocation <- file.path("data") +allPerformance <- summaryPlpAnalyses(analysesLocation) +plpResultLocation <- allPerformance[,c('plpResultLocation', 'plpResultLoad')] +#allPerformance$combinedModelSettingName <- paste0(allPerformance$modelSettingName,'-', allPerformance$modelSettingsId +formatPerformance <- allPerformance[,c('analysisId','devDatabase','valDatabase','cohortName','outcomeName','modelSettingName','riskWindowStart', 'riskWindowEnd', 'AUC','AUPRC', 'populationSize','outcomeCount','incidence', + 'addExposureDaysToStart','addExposureDaysToEnd')] +colnames(formatPerformance) <- c('Analysis','Dev', 'Val', 'T', 'O','Model', 'TAR start', 'TAR end', 'AUC','AUPRC', 'T Size','O Count','O Incidence (%)', 'addExposureDaysToStart','addExposureDaysToEnd') + + + diff --git a/BookOfOhdsiPlp/plots.R b/BookOfOhdsiPlp/plots.R new file mode 100644 index 00000000..4a848eed --- /dev/null +++ b/BookOfOhdsiPlp/plots.R @@ -0,0 +1,480 @@ +#============ DYNAMIC PLOTS ====================== +#++++++++++++++++++++++++++++++++++++++++++++++++++ + +plotShiny <- function(eval, pointOfInterest){ + + data <- eval$thresholdSummary[eval$thresholdSummary$Eval%in%c('test','validation'),] + # pointOfInterest # this is a threshold + pointOfInterest <- data[pointOfInterest,] + rocobject <- plotly::plot_ly(x = 1-c(0,data$specificity,1)) %>% + plotly::add_lines(y = c(1,data$sensitivity,0),name = "hv", + text = paste('Risk Threshold:',c(0,data$predictionThreshold,1)), + line = list(shape = "hv", + color = 'rgb(22, 96, 167)'), + fill = 'tozeroy') %>% + plotly::add_trace(x= c(0,1), y = c(0,1),mode = 'lines', + line = list(dash = "dash"), color = I('black'), + type='scatter') %>% + plotly::add_trace(x= 1-pointOfInterest$specificity, y=pointOfInterest$sensitivity, + mode = 'markers', symbols='x') %>% # change the colour of this! + plotly::add_lines(x=c(1-pointOfInterest$specificity, 1-pointOfInterest$specificity), + y = c(0,1), + line = list(dash ='solid', + color = 'black')) %>% + plotly::layout(title = "ROC Plot", + xaxis = list(title = "1-specificity"), + yaxis = list (title = "Sensitivity"), + showlegend = FALSE) + + popAv <- data$trueCount[1]/(data$trueCount[1] + data$falseCount[1]) + probject <- plotly::plot_ly(x = data$sensitivity) %>% + plotly::add_lines(y = data$positivePredictiveValue, name = "hv", + text = paste('Risk Threshold:',data$predictionThreshold), + line = list(shape = "hv", + color = 'rgb(22, 96, 167)'), + fill = 'tozeroy') %>% + plotly::add_trace(x= c(0,1), y = c(popAv,popAv),mode = 'lines', + line = list(dash = "dash"), color = I('red'), + type='scatter') %>% + plotly::add_trace(x= pointOfInterest$sensitivity, y=pointOfInterest$positivePredictiveValue, + mode = 'markers', symbols='x') %>% + plotly::add_lines(x=c(pointOfInterest$sensitivity, pointOfInterest$sensitivity), + y = c(0,1), + line = list(dash ='solid', + color = 'black')) %>% + plotly::layout(title = "PR Plot", + xaxis = list(title = "Recall"), + yaxis = list (title = "Precision"), + showlegend = FALSE) + + # add F1 score + f1object <- plotly::plot_ly(x = data$predictionThreshold) %>% + plotly::add_lines(y = data$f1Score, name = "hv", + text = paste('Risk Threshold:',data$predictionThreshold), + line = list(shape = "hv", + color = 'rgb(22, 96, 167)'), + fill = 'tozeroy') %>% + plotly::add_trace(x= pointOfInterest$predictionThreshold, y=pointOfInterest$f1Score, + mode = 'markers', symbols='x') %>% + plotly::add_lines(x=c(pointOfInterest$predictionThreshold, pointOfInterest$predictionThreshold), + y = c(0,1), + line = list(dash ='solid', + color = 'black')) %>% + plotly::layout(title = "F1-Score Plot", + xaxis = list(title = "Prediction Threshold"), + yaxis = list (title = "F1-Score"), + showlegend = FALSE) + # create 2x2 table with TP, FP, TN, FN and threshold + threshold <- pointOfInterest$predictionThreshold + TP <- pointOfInterest$truePositiveCount + TN <- pointOfInterest$trueNegativeCount + FP <- pointOfInterest$falsePositiveCount + FN <- pointOfInterest$falseNegativeCount + preferenceThreshold <- pointOfInterest$preferenceThreshold + + return(list(roc = rocobject, + pr = probject, + f1score = f1object, + threshold = threshold, prefthreshold=preferenceThreshold, + TP = TP, TN=TN, + FP= FP, FN=FN)) +} + +plotCovariateSummary <- function(covariateSummary){ + + #writeLines(paste(colnames(covariateSummary))) + #writeLines(paste(covariateSummary[1,])) + # remove na values + covariateSummary$CovariateMeanWithNoOutcome[is.na(covariateSummary$CovariateMeanWithNoOutcome)] <- 0 + covariateSummary$CovariateMeanWithOutcome[is.na(covariateSummary$CovariateMeanWithOutcome)] <- 0 + if(!'covariateValue'%in%colnames(covariateSummary)){ + covariateSummary$covariateValue <- 1 + } + if(sum(is.na(covariateSummary$covariateValue))>0){ + covariateSummary$covariateValue[is.na(covariateSummary$covariateValue)] <- 0 + } + + # SPEED EDIT remove the none model variables + covariateSummary <- covariateSummary[covariateSummary$covariateValue!=0,] + + # save dots based on coef value + covariateSummary$size <- abs(covariateSummary$covariateValue) + covariateSummary$size[is.na(covariateSummary$size)] <- 4 + covariateSummary$size <- 4+4*covariateSummary$size/max(covariateSummary$size) + + # color based on analysis id + covariateSummary$color <- sapply(covariateSummary$covariateName, function(x) ifelse(is.na(x), '', strsplit(as.character(x), ' ')[[1]][1])) + + l <- list(x = 0.01, y = 1, + font = list( + family = "sans-serif", + size = 10, + color = "#000"), + bgcolor = "#E2E2E2", + bordercolor = "#FFFFFF", + borderwidth = 1) + + #covariateSummary$annotation <- sapply(covariateSummary$covariateName, getName) + covariateSummary$annotation <- covariateSummary$covariateName + + + ind <- covariateSummary$CovariateMeanWithNoOutcome <=1 & covariateSummary$CovariateMeanWithOutcome <= 1 + # create two plots -1 or less or g1 + binary <- plotly::plot_ly(x = covariateSummary$CovariateMeanWithNoOutcome[ind], + #size = covariateSummary$size[ind], + showlegend = F) %>% + plotly::add_markers(y = covariateSummary$CovariateMeanWithOutcome[ind], + color=factor(covariateSummary$color[ind]), + text = paste(covariateSummary$annotation[ind]), + showlegend = T + ) %>% + plotly::add_trace(x= c(0,1), y = c(0,1),mode = 'lines', + line = list(dash = "dash"), color = I('black'), + type='scatter', showlegend = FALSE) %>% + plotly::layout(#title = 'Prevalance of baseline predictors in persons with and without outcome', + xaxis = list(title = "Prevalance in persons without outcome", + range = c(0, 1)), + yaxis = list(title = "Prevalance in persons with outcome", + range = c(0, 1)), + legend = l, showlegend = T) + + if(sum(!ind)>0){ + maxValue <- max(c(covariateSummary$CovariateMeanWithNoOutcome[!ind], + covariateSummary$CovariateMeanWithOutcome[!ind]), na.rm = T) + meas <- plotly::plot_ly(x = covariateSummary$CovariateMeanWithNoOutcome[!ind] ) %>% + plotly::add_markers(y = covariateSummary$CovariateMeanWithOutcome[!ind], + text = paste(covariateSummary$annotation[!ind])) %>% + plotly::add_trace(x= c(0,maxValue), y = c(0,maxValue),mode = 'lines', + line = list(dash = "dash"), color = I('black'), + type='scatter', showlegend = FALSE) %>% + plotly::layout(#title = 'Prevalance of baseline predictors in persons with and without outcome', + xaxis = list(title = "Mean in persons without outcome"), + yaxis = list(title = "Mean in persons with outcome"), + showlegend = FALSE) + } else { + meas <- NULL + } + + return(list(binary=binary, + meas = meas)) +} + + + + + + +plotPredictedPDF <- function(evaluation, type='test', fileName=NULL){ + ind <- evaluation$thresholdSummary$Eval==type + + x<- evaluation$thresholdSummary[ind,c('predictionThreshold','truePositiveCount','trueNegativeCount', + 'falsePositiveCount','falseNegativeCount')] + x<- x[order(x$predictionThreshold,-x$truePositiveCount, -x$falsePositiveCount),] + x$out <- c(x$truePositiveCount[-length(x$truePositiveCount)]-x$truePositiveCount[-1], x$truePositiveCount[length(x$truePositiveCount)]) + x$nout <- c(x$falsePositiveCount[-length(x$falsePositiveCount)]-x$falsePositiveCount[-1], x$falsePositiveCount[length(x$falsePositiveCount)]) + + vals <- c() + for(i in 1:length(x$predictionThreshold)){ + if(i!=length(x$predictionThreshold)){ + upper <- x$predictionThreshold[i+1]} else {upper <- min(x$predictionThreshold[i]+0.01,1)} + val <- x$predictionThreshold[i]+runif(x$out[i])*(upper-x$predictionThreshold[i]) + vals <- c(val, vals) + } + vals[!is.na(vals)] + + vals2 <- c() + for(i in 1:length(x$predictionThreshold)){ + if(i!=length(x$predictionThreshold)){ + upper <- x$predictionThreshold[i+1]} else {upper <- min(x$predictionThreshold[i]+0.01,1)} + val2 <- x$predictionThreshold[i]+runif(x$nout[i])*(upper-x$predictionThreshold[i]) + vals2 <- c(val2, vals2) + } + vals2[!is.na(vals2)] + + x <- rbind(data.frame(variable=rep('outcome',length(vals)), value=vals), + data.frame(variable=rep('No outcome',length(vals2)), value=vals2) + ) + + plot <- ggplot2::ggplot(x, ggplot2::aes(x=x$value, + group=x$variable, + fill=x$variable)) + + ggplot2::geom_density(ggplot2::aes(x=x$value, fill=x$variable), alpha=.3) + + ggplot2::scale_x_continuous("Prediction Threshold")+#, limits=c(0,1)) + + ggplot2::scale_y_continuous("Density") + + ggplot2::guides(fill=ggplot2::guide_legend(title="Class")) + + if (!is.null(fileName)) + ggplot2::ggsave(fileName, plot, width = 5, height = 4.5, dpi = 400) + return(plot) +} + + +plotPreferencePDF <- function(evaluation, type='test', fileName=NULL){ + ind <- evaluation$thresholdSummary$Eval==type + + x<- evaluation$thresholdSummary[ind,c('preferenceThreshold','truePositiveCount','trueNegativeCount', + 'falsePositiveCount','falseNegativeCount')] + x<- x[order(x$preferenceThreshold,-x$truePositiveCount),] + x$out <- c(x$truePositiveCount[-length(x$truePositiveCount)]-x$truePositiveCount[-1], x$truePositiveCount[length(x$truePositiveCount)]) + x$nout <- c(x$falsePositiveCount[-length(x$falsePositiveCount)]-x$falsePositiveCount[-1], x$falsePositiveCount[length(x$falsePositiveCount)]) + + vals <- c() + for(i in 1:length(x$preferenceThreshold)){ + if(i!=length(x$preferenceThreshold)){ + upper <- x$preferenceThreshold[i+1]} else {upper <- 1} + val <- x$preferenceThreshold[i]+runif(x$out[i])*(upper-x$preferenceThreshold[i]) + vals <- c(val, vals) + } + vals[!is.na(vals)] + + vals2 <- c() + for(i in 1:length(x$preferenceThreshold)){ + if(i!=length(x$preferenceThreshold)){ + upper <- x$preferenceThreshold[i+1]} else {upper <- 1} + val2 <- x$preferenceThreshold[i]+runif(x$nout[i])*(upper-x$preferenceThreshold[i]) + vals2 <- c(val2, vals2) + } + vals2[!is.na(vals2)] + + x <- rbind(data.frame(variable=rep('outcome',length(vals)), value=vals), + data.frame(variable=rep('No outcome',length(vals2)), value=vals2) + ) + + plot <- ggplot2::ggplot(x, ggplot2::aes(x=x$value, + group=x$variable, + fill=x$variable)) + + ggplot2::geom_density(ggplot2::aes(x=x$value, fill=x$variable), alpha=.3) + + ggplot2::scale_x_continuous("Preference Threshold")+#, limits=c(0,1)) + + ggplot2::scale_y_continuous("Density") + + ggplot2::guides(fill=ggplot2::guide_legend(title="Class")) + + if (!is.null(fileName)) + ggplot2::ggsave(fileName, plot, width = 5, height = 4.5, dpi = 400) + return(plot) +} + + + + +plotDemographicSummary <- function(evaluation, type='test', fileName=NULL){ + if (!all(is.na(evaluation$demographicSummary$averagePredictedProbability))){ + ind <- evaluation$demographicSummary$Eval==type + x<- evaluation$demographicSummary[ind,colnames(evaluation$demographicSummary)%in%c('ageGroup','genGroup','averagePredictedProbability', + 'PersonCountAtRisk', 'PersonCountWithOutcome')] + + # remove -1 values: + x <- x[x$PersonCountWithOutcome != -1,] + if(nrow(x)==0){ + return(NULL) + } + # end remove -1 values + + x$observed <- x$PersonCountWithOutcome/x$PersonCountAtRisk + + + x <- x[,colnames(x)%in%c('ageGroup','genGroup','averagePredictedProbability','observed')] + + # if age or gender missing add + if(sum(colnames(x)=='ageGroup')==1 && sum(colnames(x)=='genGroup')==0 ){ + x$genGroup = rep('Non', nrow(x)) + evaluation$demographicSummary$genGroup = rep('Non', nrow(evaluation$demographicSummary)) + } + if(sum(colnames(x)=='ageGroup')==0 && sum(colnames(x)=='genGroup')==1 ){ + x$ageGroup = rep('-1', nrow(x)) + evaluation$demographicSummary$ageGroup = rep('-1', nrow(evaluation$demographicSummary)) + + } + + x <- reshape2::melt(x, id.vars=c('ageGroup','genGroup')) + + # 1.96*StDevPredictedProbability + ci <- evaluation$demographicSummary[,colnames(evaluation$demographicSummary)%in%c('ageGroup','genGroup','averagePredictedProbability','StDevPredictedProbability')] + ci$StDevPredictedProbability[is.na(ci$StDevPredictedProbability)] <- 1 + ci$lower <- ci$averagePredictedProbability-1.96*ci$StDevPredictedProbability + ci$lower[ci$lower <0] <- 0 + ci$upper <- ci$averagePredictedProbability+1.96*ci$StDevPredictedProbability + ci$upper[ci$upper >1] <- max(ci$upper[ci$upper <1]) + + x$age <- gsub('Age group:','', x$ageGroup) + x$age <- factor(x$age,levels=c(" 0-4"," 5-9"," 10-14", + " 15-19"," 20-24"," 25-29"," 30-34"," 35-39"," 40-44", + " 45-49"," 50-54"," 55-59"," 60-64"," 65-69"," 70-74", + " 75-79"," 80-84"," 85-89"," 90-94"," 95-99","-1"),ordered=TRUE) + + x <- merge(x, ci[,c('ageGroup','genGroup','lower','upper')], by=c('ageGroup','genGroup')) + + plot <- ggplot2::ggplot(data=x, + ggplot2::aes(x=age, group=variable*genGroup)) + + + ggplot2::geom_line(ggplot2::aes(y=value, group=variable, + color=variable, + linetype = variable))+ + ggplot2::geom_ribbon(data=x[x$variable!='observed',], + ggplot2::aes(ymin=lower, ymax=upper + , group=genGroup), + fill="blue", alpha="0.2") + + ggplot2::facet_grid(.~ genGroup, scales = "free") + + ggplot2::theme(axis.text.x = ggplot2::element_text(angle = 90, hjust = 1)) + + #ggplot2::coord_flip() + + ggplot2::scale_y_continuous("Fraction") + + ggplot2::scale_x_discrete("Age") + + ggplot2::scale_color_manual(values = c("royalblue4","red"), + guide = ggplot2::guide_legend(title = NULL), + labels = c("Expected", "Observed")) + + + ggplot2::guides(linetype=FALSE) + + if (!is.null(fileName)) + ggplot2::ggsave(fileName, plot, width = 7, height = 4.5, dpi = 400) + return(plot) + } +} + + +#============= +# CALIBRATIONSUMMARY PLOTS +#============= +#' Plot the calibration +#' +#' @details +#' Create a plot showing the calibration +#' #' +#' @param evaluation A prediction object as generated using the +#' \code{\link{runPlp}} function. +#' @param type options: 'train' or test' +#' @param fileName Name of the file where the plot should be saved, for example +#' 'plot.png'. See the function \code{ggsave} in the ggplot2 package for +#' supported file formats. +#' +#' @return +#' A ggplot object. Use the \code{\link[ggplot2]{ggsave}} function to save to file in a different +#' format. +#' +#' @export +plotSparseCalibration <- function(evaluation, type='test', fileName=NULL){ + ind <- evaluation$calibrationSummary$Eval==type + + x<- evaluation$calibrationSummary[ind,c('averagePredictedProbability','observedIncidence')] + maxVal <- max(x$averagePredictedProbability,x$observedIncidence) + model <- stats::lm(observedIncidence~averagePredictedProbability, data=x) + res <- model$coefficients + names(res) <- c('Intercept','Gradient') + + # confidence int + interceptConf <- stats::confint(model)[1,] + gradientConf <- stats::confint(model)[2,] + + cis <- data.frame(lci = interceptConf[1]+seq(0,1,length.out = nrow(x))*gradientConf[1], + uci = interceptConf[2]+seq(0,1,length.out = nrow(x))*gradientConf[2], + x=seq(0,1,length.out = nrow(x))) + + x <- cbind(x, cis) + # TODO: CHECK INPUT + plot <- ggplot2::ggplot(data=x, + ggplot2::aes(x=averagePredictedProbability, y=observedIncidence + )) + + ggplot2::geom_ribbon(ggplot2::aes(ymin=lci,ymax=uci, x=x), + fill="blue", alpha="0.2") + + ggplot2::geom_point(size=1, color='darkblue') + + ggplot2::coord_cartesian(ylim = c(0, maxVal), xlim =c(0,maxVal)) + + ggplot2::geom_abline(intercept = 0, slope = 1, linetype = 2, size=1, + show.legend = TRUE) + + ggplot2::geom_abline(intercept = res['Intercept'], slope = res['Gradient'], + linetype = 1,show.legend = TRUE, + color='darkblue') + + ggplot2::scale_x_continuous("Average Predicted Probability") + + ggplot2::scale_y_continuous("Observed Fraction With Outcome") + + + + if (!is.null(fileName)) + ggplot2::ggsave(fileName, plot, width = 5, height = 3.5, dpi = 400) + return(plot) +} + +#============= +# CALIBRATIONSUMMARY PLOTS 2 +#============= +#' Plot the conventional calibration +#' +#' @details +#' Create a plot showing the calibration +#' #' +#' @param evaluation A prediction object as generated using the +#' \code{\link{runPlp}} function. +#' @param type options: 'train' or test' +#' @param fileName Name of the file where the plot should be saved, for example +#' 'plot.png'. See the function \code{ggsave} in the ggplot2 package for +#' supported file formats. +#' +#' @return +#' A ggplot object. Use the \code{\link[ggplot2]{ggsave}} function to save to file in a different +#' format. +#' +#' @export +plotSparseCalibration2 <- function(evaluation, type='test', fileName=NULL){ + ind <- evaluation$calibrationSummary$Eval==type + + x<- evaluation$calibrationSummary[ind,c('averagePredictedProbability','observedIncidence', 'PersonCountAtRisk')] + + + cis <- apply(x, 1, function(x) binom.test(x[2]*x[3], x[3], alternative = c("two.sided"), conf.level = 0.95)$conf.int) + x$lci <- cis[1,] + x$uci <- cis[2,] + + maxes <- max(max(x$averagePredictedProbability), max(x$observedIncidence))*1.1 + + # TODO: CHECK INPUT + limits <- ggplot2::aes(ymax = x$uci, ymin= x$lci) + + plot <- ggplot2::ggplot(data=x, + ggplot2::aes(x=averagePredictedProbability, y=observedIncidence + )) + + ggplot2::geom_point(size=2, color='black') + + ggplot2::geom_errorbar(limits) + + #ggplot2::geom_smooth(method=lm, se=F, colour='darkgrey') + + ggplot2::geom_line(colour='darkgrey') + + ggplot2::geom_abline(intercept = 0, slope = 1, linetype = 5, size=0.4, + show.legend = TRUE) + + ggplot2::scale_x_continuous("Average Predicted Probability") + + ggplot2::scale_y_continuous("Observed Fraction With Outcome") + + ggplot2::coord_cartesian(xlim = c(0, maxes), ylim=c(0,maxes)) + + + if (!is.null(fileName)) + ggplot2::ggsave(fileName, plot, width = 5, height = 3.5, dpi = 400) + return(plot) +} + +plotPredictionDistribution <- function(evaluation, type='test', fileName=NULL){ + ind <- evaluation$predictionDistribution$Eval==type + x<- evaluation$predictionDistribution[ind,] + + #(x=Class, y=predictedProbabllity sequence: min->P05->P25->Median->P75->P95->max) + + plot <- ggplot2::ggplot(x, ggplot2::aes(x=as.factor(class), + ymin=MinPredictedProbability, + lower=P25PredictedProbability, + middle=MedianPredictedProbability, + upper=P75PredictedProbability, + ymax=MaxPredictedProbability, + color=as.factor(class))) + + ggplot2::coord_flip() + + ggplot2::geom_boxplot(stat="identity") + + ggplot2::scale_x_discrete("Class") + + ggplot2::scale_y_continuous("Predicted Probability") + + ggplot2::theme(legend.position="none") + + ggplot2::geom_segment(ggplot2::aes(x = 0.9, y = x$P05PredictedProbability[x$class==0], + xend = 1.1, yend = x$P05PredictedProbability[x$class==0]), color='red') + + ggplot2::geom_segment(ggplot2::aes(x = 0.9, y = x$P95PredictedProbability[x$class==0], + xend = 1.1, yend = x$P95PredictedProbability[x$class==0]), color='red') + + ggplot2::geom_segment(ggplot2::aes(x = 1.9, y = x$P05PredictedProbability[x$class==1], + xend = 2.1, yend = x$P05PredictedProbability[x$class==1])) + + ggplot2::geom_segment(ggplot2::aes(x = 1.9, y = x$P95PredictedProbability[x$class==1], + xend = 2.1, yend = x$P95PredictedProbability[x$class==1])) + + + if (!is.null(fileName)) + ggplot2::ggsave(fileName, plot, width = 5, height = 4.5, dpi = 400) + return(plot) +} \ No newline at end of file diff --git a/BookOfOhdsiPlp/server.R b/BookOfOhdsiPlp/server.R new file mode 100644 index 00000000..d637e178 --- /dev/null +++ b/BookOfOhdsiPlp/server.R @@ -0,0 +1,317 @@ +# @file server.R +# +# Copyright 2018 Observational Health Data Sciences and Informatics +# +# This file is part of PatientLevelPrediction +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +library(shiny) +library(plotly) +library(shinycssloaders) + +source("utils.R") +source("plots.R") + +shiny::shinyServer(function(input, output, session) { + session$onSessionEnded(stopApp) + # reactive values - contains the location of the plpResult + ##reactVars <- shiny::reactiveValues(resultLocation=NULL, + ## plpResult= NULL) + #============= + + summaryData <- shiny::reactive({ + ind <- 1:nrow(allPerformance) + if(input$devDatabase!='All'){ + ind <- intersect(ind,which(as.character(allPerformance$devDatabase)==input$devDatabase)) + } + if(input$valDatabase!='All'){ + ind <- intersect(ind,which(as.character(allPerformance$valDatabase)==input$valDatabase)) + } + if(input$T!='All'){ + ind <- intersect(ind,which(allPerformance$cohortName==input$T)) + } + if(input$O!='All'){ + ind <- intersect(ind,which(allPerformance$outcomeName==input$O)) + } + if(input$modelSettingName!='All'){ + ind <- intersect(ind,which(as.character(allPerformance$modelSettingName)==input$modelSettingName)) + } + if(input$riskWindowStart!='All'){ + ind <- intersect(ind,which(allPerformance$riskWindowStart==input$riskWindowStart)) + } + if(input$riskWindowEnd!='All'){ + ind <- intersect(ind,which(allPerformance$riskWindowEnd==input$riskWindowEnd)) + } + + ind + }) + + + + output$summaryTable <- DT::renderDataTable(DT::datatable(formatPerformance[summaryData(),!colnames(formatPerformance)%in%c('addExposureDaysToStart','addExposureDaysToEnd')], + rownames= FALSE, selection = 'single')) + + + dataofint <- shiny::reactive({ + if(is.null(input$summaryTable_rows_selected[1])){ + ind <- 1 + }else{ + ind <- input$summaryTable_rows_selected[1] + } + + loc <- plpResultLocation[summaryData(),][ind,]$plpResultLocation + logLocation <- gsub('validationResult.rds','plpLog.txt',gsub('plpResult.rds','plpLog.txt', as.character(loc))) + if(file.exists(logLocation)){ + txt <- readLines(logLocation) + } else{ + txt <- 'log not available' + } + + covariates <- NULL + population <- NULL + modelset <- NULL + + if(file.exists(as.character(loc))){ + eval <- readRDS(as.character(loc)) + # rounding values to 2dp + for(coln in c('covariateValue','CovariateMeanWithOutcome','CovariateMeanWithNoOutcome')){ + eval$covariateSummary[,coln] <- format(round(eval$covariateSummary[,coln], 4), nsmall = 4) + class(eval$covariateSummary[,coln]) <- "numeric" + } + + } else{ + eval <- NULL + } + if(length(grep('/Validation',loc))>0){ + type <- 'validation' }else{ + type <- 'test' + } + + if(!is.null(eval)){ + covariates <- eval$model$metaData$call$covariateSettings + population <- eval$model$populationSettings + covariates <- data.frame(covariateName = names(covariates), + SettingValue = unlist(lapply(covariates, + function(x) paste0(x, + collapse='-'))) + ) + population$attrition <- NULL # remove the attrition as result and not setting + population <- data.frame(Setting = names(population), + Value = unlist(lapply(population, + function(x) paste0(x, + collapse='-'))) + ) + modelset <- data.frame(Setting = c('Model',names(eval$model$modelSettings[[2]])), + Value = c(eval$model$modelSettings[[1]], unlist(lapply(eval$model$modelSettings[[2]], + function(x) paste0(x, collapse='')))) + ) + + row.names(covariates) <- NULL + row.names(population) <- NULL + row.names(modelset) <- NULL + } + + return(list(eval=eval, type=type, + logtext = txt, + logLocation=logLocation, + covariates = covariates, + population = population, + modelset = modelset)) + }) + + plotters <- shiny::reactive({ + + eval <- dataofint()$eval$performanceEvaluation + if(is.null(eval)){return(NULL)} + + calPlot <- NULL + rocPlot <- NULL + prPlot <- NULL + f1Plot <- NULL + demoPlot <- NULL + boxPlot <- NULL + distPlot <- NULL + txt <- 'Empty' + predictionText <- c() + + if(!is.null(eval)){ + intPlot <- plotShiny(eval, input$slider1) + rocPlot <- intPlot$roc + prPlot <- intPlot$pr + f1Plot <- intPlot$f1score + threshold <- intPlot$threshold + prefthreshold <- intPlot$prefthreshold + TP <- intPlot$TP + FP <- intPlot$FP + TN <- intPlot$TN + FN <- intPlot$FN + prefdistPlot <- plotPreferencePDF(eval, type=dataofint()$type ) + prefdistPlot <- prefdistPlot + ggplot2::geom_vline(xintercept=prefthreshold) + preddistPlot <- plotPredictedPDF(eval, type=dataofint()$type ) + preddistPlot <- preddistPlot + ggplot2::geom_vline(xintercept=threshold) + boxPlot <- plotPredictionDistribution(eval, type=dataofint()$type ) + + calPlot <- plotSparseCalibration2(eval, type=dataofint()$type ) + demoPlot <- tryCatch(plotDemographicSummary(eval, type=dataofint()$type ), + error= function(cond){return(NULL)}) + + if(is.null(input$summaryTable_rows_selected[1])){ + ind <- 1 + }else{ + ind <- input$summaryTable_rows_selected[1] + } + predictionText <- paste0('Within ', formatPerformance[summaryData(),'T'][ind], + ' predict who will develop ', formatPerformance[summaryData(),'O'][ind], + ' during ', formatPerformance[summaryData(),'TAR start'][ind], ' day/s', + ' after ', ifelse(formatPerformance[summaryData(),'addExposureDaysToStart'][ind]==0, ' cohort start ', ' cohort end '), + ' and ', formatPerformance[summaryData(),'TAR end'][ind], ' day/s', + ' after ', ifelse(formatPerformance[summaryData(),'addExposureDaysToEnd'][ind]==0, ' cohort start ', ' cohort end ')) + + } + + twobytwo <- as.data.frame(matrix(c(FP,TP,TN,FN), byrow=T, ncol=2)) + colnames(twobytwo) <- c('Ground Truth Negative','Ground Truth Positive') + rownames(twobytwo) <- c('Predicted Positive','Predicted Negative') + + performance <- data.frame(Incidence = (TP+FN)/(TP+TN+FP+FN), + Threshold = threshold, + Sensitivity = TP/(TP+FN), + Specificity = TN/(TN+FP), + PPV = TP/(TP+FP), + NPV = TN/(TN+FN)) + + list(rocPlot= rocPlot, calPlot=calPlot, + prPlot=prPlot, f1Plot=f1Plot, + demoPlot=demoPlot, boxPlot=boxPlot, + prefdistPlot=prefdistPlot, + preddistPlot=preddistPlot, predictionText=predictionText, + threshold = format(threshold, digits=5), + twobytwo=twobytwo, + performance = performance ) + }) + + output$performance <- shiny::renderTable(plotters()$performance, + rownames = F, digits = 3) + output$twobytwo <- shiny::renderTable(plotters()$twobytwo, + rownames = T, digits = 0) + + output$modelTable <- DT::renderDataTable(dataofint()$modelset) + output$covariateTable <- DT::renderDataTable(dataofint()$covariates) + output$populationTable <- DT::renderDataTable(dataofint()$population) + + output$info <- shiny::renderText(plotters()$predictionText) + output$log <- shiny::renderText( paste(dataofint()$logtext, collapse="\n") ) + output$threshold <- shiny::renderText(plotters()$threshold) + + output$roc <- plotly::renderPlotly({ + plotters()$rocPlot + }) + output$cal <- shiny::renderPlot({ + plotters()$calPlot + }) + output$pr <- plotly::renderPlotly({ + plotters()$prPlot + }) + output$f1 <- plotly::renderPlotly({ + plotters()$f1Plot + }) + output$demo <- shiny::renderPlot({ + plotters()$demoPlot + }) + output$box <- shiny::renderPlot({ + plotters()$boxPlot + }) + output$preddist <- shiny::renderPlot({ + plotters()$preddistPlot + }) + output$prefdist <- shiny::renderPlot({ + plotters()$prefdistPlot + }) + + + covs <- shiny::reactive({ + if(is.null(dataofint()$eval)) + return(NULL) + plotCovariateSummary(dataofint()$eval$covariateSummary) + }) + + output$covariateSummaryBinary <- plotly::renderPlotly({ covs()$binary }) + output$covariateSummaryMeasure <- plotly::renderPlotly({ covs()$meas }) + + + output$modelView <- DT::renderDataTable(dataofint()$eval$covariateSummary[,c('covariateName','covariateValue','CovariateMeanWithOutcome','CovariateMeanWithNoOutcome' )], + colnames = c('Covariate Name', 'Value', 'Outcome Mean', 'Non-outcome Mean')) + + + # dashboard + + output$performanceBoxIncidence <- renderInfoBox({ + infoBox( + "Incidence", paste0(round(plotters()$performance$Incidence*100, digits=3),'%'), icon = icon("ambulance"), + color = "green" + ) + }) + + output$performanceBoxThreshold <- renderInfoBox({ + infoBox( + "Threshold", format((plotters()$performance$Threshold), scientific = F, digits=3), icon = icon("edit"), + color = "yellow" + ) + }) + + output$performanceBoxPPV <- renderInfoBox({ + infoBox( + "PPV", paste0(round(plotters()$performance$PPV*1000)/10, "%"), icon = icon("thumbs-up"), + color = "orange" + ) + }) + + output$performanceBoxSpecificity <- renderInfoBox({ + infoBox( + "Specificity", paste0(round(plotters()$performance$Specificity*1000)/10, "%"), icon = icon("bullseye"), + color = "purple" + ) + }) + + output$performanceBoxSensitivity <- renderInfoBox({ + infoBox( + "Sensitivity", paste0(round(plotters()$performance$Sensitivity*1000)/10, "%"), icon = icon("low-vision"), + color = "blue" + ) + }) + + output$performanceBoxNPV <- renderInfoBox({ + infoBox( + "NPV", paste0(round(plotters()$performance$NPV*1000)/10, "%"), icon = icon("minus-square"), + color = "black" + ) + }) + + + + # Downloadable csv of model ---- + output$downloadData <- downloadHandler( + filename = function(){'model.csv'}, + content = function(file) { + write.csv(dataofint()$eval$covariateSummary[dataofint()$eval$covariateSummary$covariateValue!=0,c('covariateName','covariateValue','CovariateMeanWithOutcome','CovariateMeanWithNoOutcome' )] + , file, row.names = FALSE) + } + ) + + + + + #============= + +}) \ No newline at end of file diff --git a/BookOfOhdsiPlp/ui.R b/BookOfOhdsiPlp/ui.R new file mode 100644 index 00000000..8ed94d16 --- /dev/null +++ b/BookOfOhdsiPlp/ui.R @@ -0,0 +1,238 @@ +# @file Ui.R +# +# Copyright 2018 Observational Health Data Sciences and Informatics +# +# This file is part of PatientLevelPrediction +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +library(shiny) +library(plotly) +library(shinycssloaders) +library(shinydashboard) + +ui <- shinydashboard::dashboardPage(skin = 'black', + + shinydashboard::dashboardHeader(title = "Multiple PLP Viewer", + + tags$li(div(img(src = 'logo.png', + title = "OHDSI PLP", height = "40px", width = "40px"), + style = "padding-top:0px; padding-bottom:0px;"), + class = "dropdown") + + + ), + + shinydashboard::dashboardSidebar( + shinydashboard::sidebarMenu( + shinydashboard::menuItem("Summary", tabName = "Summary", icon = shiny::icon("table")), + shinydashboard::menuItem("Performance", tabName = "Performance", icon = shiny::icon("bar-chart")), + shinydashboard::menuItem("Model", tabName = "Model", icon = shiny::icon("clipboard")), + shinydashboard::menuItem("Log", tabName = "Log", icon = shiny::icon("list")), + shinydashboard::menuItem("Help", tabName = "Help", icon = shiny::icon("info")) + ) + ), + + shinydashboard::dashboardBody( + shinydashboard::tabItems( + + # help tab + shinydashboard::tabItem(tabName = "Help", + shiny::h2("Information"), + shiny::p("Click on a row to explore the results for that model. When you wish to explore a different model, then select the new result row and the tabs will be updated."), + shiny::a("Demo Video", href = 'https://youtu.be/StpV40yl1UE', target='_blank') + ), + + # First tab content + shinydashboard::tabItem(tabName = "Summary", + + shiny::fluidRow( + shiny::column(2, + shiny::h4('Filters'), + shiny::selectInput('devDatabase', 'Development Database', c('All',unique(as.character(allPerformance$devDatabase)))), + shiny::selectInput('valDatabase', 'Validation Database', c('All',unique(as.character(allPerformance$valDatabase)))), + shiny::selectInput('T', 'Target Cohort', c('All',unique(as.character(allPerformance$cohortName)))), + shiny::selectInput('O', 'Outcome Cohort', c('All',unique(as.character(allPerformance$outcomeName)))), + shiny::selectInput('riskWindowStart', 'Time-at-risk start:', c('All',unique(allPerformance$riskWindowStart))), + shiny::selectInput('riskWindowEnd', 'Time-at-risk end:', c('All',unique(as.character(allPerformance$riskWindowEnd)))), + shiny::selectInput('modelSettingName', 'Model:', c('All',unique(as.character(allPerformance$modelSettingName)))) + ), + shiny::column(10, style = "background-color:#F3FAFC;", + + # do this inside tabs: + shiny::tabsetPanel( + + shiny::tabPanel("Results", + shiny::div(DT::dataTableOutput('summaryTable'), + style = "font-size:70%")), + + shiny::tabPanel("Model Settings", + shiny::h3('Model Settings: ', + shiny::a("help", href="https://ohdsi.github.io/PatientLevelPrediction/reference/index.html", target="_blank") + ), + DT::dataTableOutput('modelTable')), + + shiny::tabPanel("Population Settings", + shiny::h3('Population Settings: ', + shiny::a("help", href="https://ohdsi.github.io/PatientLevelPrediction/reference/createStudyPopulation.html", target="_blank") + ), + DT::dataTableOutput('populationTable')), + + shiny::tabPanel("Covariate Settings", + shiny::h3('Covariate Settings: ', + shiny::a("help", href="http://ohdsi.github.io/FeatureExtraction/reference/createCovariateSettings.html", target="_blank") + ), + DT::dataTableOutput('covariateTable')) + ) + + ) + + )), + # second tab + shinydashboard::tabItem(tabName = "Performance", + + shiny::fluidRow( + tabBox( + title = "Performance", + # The id lets us use input$tabset1 on the server to find the current tab + id = "tabset1", height = "100%", width='100%', + tabPanel("Summary", + + shiny::fluidRow( + shiny::column(width = 4, + shinydashboard::box(width = 12, + title = tagList(shiny::icon("question"),"Prediction Question"), status = "info", solidHeader = TRUE, + shiny::textOutput('info') + ), + shinydashboard::box(width = 12, + title = tagList(shiny::icon("gear"), "Input"), + status = "info", solidHeader = TRUE, + shiny::splitLayout( + cellWidths = c('5%', '90%', '5%'), + shiny::h5(' '), + shiny::sliderInput("slider1", + shiny::h4("Threshold value slider: ", strong(shiny::textOutput('threshold'))), + min = 1, max = 100, value = 50, ticks = F), + shiny::h5(' ') + ), + shiny::splitLayout( + cellWidths = c('5%', '90%', '5%'), + shiny::h5(strong('0')), + shiny::h5(' '), + shiny::h5(strong('1')) + ), + shiny::tags$script(shiny::HTML(" + $(document).ready(function() {setTimeout(function() { + supElement = document.getElementById('slider1').parentElement; + $(supElement).find('span.irs-max, span.irs-min, span.irs-single, span.irs-from, span.irs-to').remove(); + }, 50);}) + ")) + ) + + ), + + + shiny::column(width = 8, + shinydashboard::box(width = 12, + title = "Dashboard", + status = "warning", solidHeader = TRUE, + shinydashboard::infoBoxOutput("performanceBoxThreshold"), + shinydashboard::infoBoxOutput("performanceBoxIncidence"), + shinydashboard::infoBoxOutput("performanceBoxPPV"), + shinydashboard::infoBoxOutput("performanceBoxSpecificity"), + shinydashboard::infoBoxOutput("performanceBoxSensitivity"), + shinydashboard::infoBoxOutput("performanceBoxNPV") + + ), + shinydashboard::box(width = 12, + title = "Cutoff Performance", + status = "warning", solidHeader = TRUE, + shiny::tableOutput('twobytwo') + #infoBoxOutput("performanceBox"), + ) + ) + ) + + + ), + tabPanel("Discrimination", + + shiny::fluidRow( + shinydashboard::box( status = 'info', + title = "ROC Plot", solidHeader = TRUE, + shinycssloaders::withSpinner(plotly::plotlyOutput('roc'))), + shinydashboard::box(status = 'info', + title = "Precision recall plot", solidHeader = TRUE, + side = "right", + shinycssloaders::withSpinner(plotly::plotlyOutput('pr')))), + + shiny::fluidRow( + shinydashboard::box(status = 'info', + title = "F1 Score Plot", solidHeader = TRUE, + shinycssloaders::withSpinner(plotly::plotlyOutput('f1'))), + shinydashboard::box(status = 'info', + title = "Box Plot", solidHeader = TRUE, + side = "right", + shinycssloaders::withSpinner(shiny::plotOutput('box')))), + + shiny::fluidRow( + shinydashboard::box(status = 'info', + title = "Prediction Score Distribution", solidHeader = TRUE, + shinycssloaders::withSpinner(shiny::plotOutput('preddist'))), + shinydashboard::box(status = 'info', + title = "Preference Score Distribution", solidHeader = TRUE, + side = "right", + shinycssloaders::withSpinner(shiny::plotOutput('prefdist')))) + + + ), + tabPanel("Calibration", + shiny::fluidRow( + shinydashboard::box(status = 'info', + title = "Calibration Plot", solidHeader = TRUE, + shinycssloaders::withSpinner(shiny::plotOutput('cal'))), + shinydashboard::box(status = 'info', + title = "Demographic Plot", solidHeader = TRUE, + side = "right", + shinycssloaders::withSpinner(shiny::plotOutput('demo'))) + ) + ) + ))), + + # 3rd tab + shinydashboard::tabItem(tabName = "Model", + shiny::fluidRow( + shinydashboard::box( status = 'info', + title = "Binary", solidHeader = TRUE, + shinycssloaders::withSpinner(plotly::plotlyOutput('covariateSummaryBinary'))), + shinydashboard::box(status = 'info', + title = "Measurements", solidHeader = TRUE, + side = "right", + shinycssloaders::withSpinner(plotly::plotlyOutput('covariateSummaryMeasure')))), + + shiny::fluidRow(width=12, + shinydashboard::box(status = 'info', width = 12, + title = "Model Table", solidHeader = TRUE, + shiny::downloadButton("downloadData", "Download Model"), + DT::dataTableOutput('modelView'))) + ), + + # 4th tab + shinydashboard::tabItem(tabName = "Log", + shiny::verbatimTextOutput('log') + ) + + + ) + ) + ) \ No newline at end of file diff --git a/BookOfOhdsiPlp/utils.R b/BookOfOhdsiPlp/utils.R new file mode 100644 index 00000000..8de84575 --- /dev/null +++ b/BookOfOhdsiPlp/utils.R @@ -0,0 +1,121 @@ +summaryPlpAnalyses <- function(analysesLocation){ + # loads the analyses and validations to get summaries + #======================================== + settings <- read.csv(file.path(analysesLocation,'settings.csv')) + settings <- settings[,!colnames(settings)%in%c('plpDataFolder','studyPopFile','plpResultFolder')] + settings$analysisId <- paste0('Analysis_', settings$analysisId) + + analysisIds <- dir(file.path(analysesLocation), recursive = F, full.names = T) + analysisIds <- analysisIds[grep('Analysis_',analysisIds)] + if(is.null(settings$devDatabase)){ + settings$devDatabase <- 'Missing' + } + settings$valDatabase <- settings$devDatabase + devPerformance <- do.call(rbind,lapply(file.path(analysisIds), getPerformance)) + devPerformance <- merge(settings[,c('analysisId','modelSettingsId', 'cohortName', 'outcomeName', + 'populationSettingId','modelSettingName','addExposureDaysToStart', + 'riskWindowStart', 'addExposureDaysToEnd', + 'riskWindowEnd','devDatabase','valDatabase')], + devPerformance, by='analysisId', all.x=T) + + validationLocation <- file.path(analysesLocation,'Validation') + if(length(dir(validationLocation))>0){ + valPerformances <- c() + valDatabases <- dir(validationLocation, recursive = F, full.names = T) + for( valDatabase in valDatabases){ + + valAnalyses <- dir(valDatabase, recursive = F, full.names = T) + valAnalyses <- valAnalyses[grep('Analysis_', valAnalyses)] + valPerformance <- do.call(rbind,lapply(file.path(valAnalyses), function(x) getValidationPerformance(x))) + valSettings <- settings[,c('analysisId','modelSettingsId', 'cohortName', 'outcomeName', + 'populationSettingId','modelSettingName','addExposureDaysToStart', + 'riskWindowStart', 'addExposureDaysToEnd', + 'riskWindowEnd')] + valSettings$devDatabase <- settings$devDatabase[1] + valPerformance <- merge(valSettings, + valPerformance, by='analysisId') + valPerformance <- valPerformance[,colnames(devPerformance)] # make sure same order + valPerformances <- rbind(valPerformances, valPerformance) + } + + if(ncol(valPerformances)==ncol(devPerformance)){ + allPerformance <- rbind(devPerformance,valPerformances) + } else{ + stop('Issue with dev and val performance data.frames') + } + } else { + allPerformance <- devPerformance + } + + allPerformance$AUC <- as.double(allPerformance$AUC) + allPerformance$AUPRC <- as.double(allPerformance$AUPRC) + allPerformance$outcomeCount <- as.double(allPerformance$outcomeCount) + allPerformance$populationSize <- as.double(allPerformance$populationSize) + allPerformance$incidence <- as.double(allPerformance$incidence) + return(allPerformance) +} + +getPerformance <- function(analysisLocation){ + location <- file.path(analysisLocation, 'plpResult.rds') + if(!file.exists(location)){ + analysisId <- strsplit(analysisLocation, '/')[[1]] + return(data.frame(analysisId=analysisId[length(analysisId)], + AUC=0.000, AUPRC=0, outcomeCount=0, + populationSize=0,incidence=0,plpResultLocation=location, + plpResultLoad='loadPlpResult')) + } + # read rds here + res <- readRDS(file.path(analysisLocation,'plpResult.rds')) + res <- as.data.frame(res$performanceEvaluation$evaluationStatistics) + + #if empty do edit? + + res <- tryCatch(reshape2::dcast(res[res$Eval=='test',], analysisId ~ Metric, value.var='Value'), + error = function(cont) return(NULL)) + if(is.null(res)){ + return(NULL) } + res <- res[,!colnames(res)%in%c("BrierScore","BrierScaled")] + res$incidence <- as.double(res$outcomeCount)/as.double(res$populationSize)*100 + res[, !colnames(res)%in%c('analysisId','outcomeCount','populationSize')] <- + format(as.double(res[, !colnames(res)%in%c('analysisId','outcomeCount','populationSize')]), digits = 2, scientific = F) + + if(sum(colnames(res)=='AUC.auc_ub95ci')>0){ + res$AUC <- res$AUC.auc + #res$AUC <- paste0(res$AUC.auc, ' (', res$AUC.auc_lb95ci,'-', res$AUC.auc_ub95ci,')') + } + + res$plpResultLocation <- location + res$plpResultLoad <- 'readRDS'#'loadPlpResult' + return(res[,c('analysisId', 'AUC', 'AUPRC', 'outcomeCount','populationSize','incidence','plpResultLocation', 'plpResultLoad')]) +} + +getValidationPerformance <- function(validationLocation){ + val <- readRDS(file.path(validationLocation,'validationResult.rds')) + if("performanceEvaluation"%in%names(val)){ + valPerformance <- reshape2::dcast(as.data.frame(val$performanceEvaluation$evaluationStatistics), + analysisId ~ Metric, value.var='Value') + } else { + valPerformance <- reshape2::dcast(as.data.frame(val[[1]]$performanceEvaluation$evaluationStatistics), + analysisId ~ Metric, value.var='Value') + } + valPerformance$incidence <- as.double(valPerformance$outcomeCount)/as.double(valPerformance$populationSize)*100 + valPerformance[, !colnames(valPerformance)%in%c('analysisId','outcomeCount','populationSize')] <- + format(as.double(valPerformance[, !colnames(valPerformance)%in%c('analysisId','outcomeCount','populationSize')]), digits = 2, scientific = F) + + if(sum(colnames(valPerformance)=='AUC.auc_ub95ci')>0){ + valPerformance$AUC <- valPerformance$AUC.auc + #valPerformance$AUC <- paste0(valPerformance$AUC.auc, ' (', valPerformance$AUC.auc_lb95ci,'-', valPerformance$AUC.auc_ub95ci,')') + } + valPerformance$analysisId <- strsplit(validationLocation, '/')[[1]][[length(strsplit(validationLocation, '/')[[1]])]] + valPerformance$valDatabase <- strsplit(validationLocation, '/')[[1]][[length(strsplit(validationLocation, '/')[[1]])-1]] + valPerformance <- valPerformance[,c('analysisId','valDatabase', 'AUC', 'AUPRC', 'outcomeCount','populationSize','incidence')] + valPerformance$plpResultLocation <- file.path(validationLocation,'validationResult.rds') + valPerformance$plpResultLoad <- 'readRDS' + #valPerformance$rocplot <- file.path(validationLocation,'plots','sparseROC.pdf') + #valPerformance$calplot <- file.path(validationLocation,'plots','sparseCalibrationConventional.pdf') + return(valPerformance) +} + + + + diff --git a/BookOfOhdsiPlp/www/about.png b/BookOfOhdsiPlp/www/about.png new file mode 100644 index 00000000..c56d5b02 Binary files /dev/null and b/BookOfOhdsiPlp/www/about.png differ diff --git a/BookOfOhdsiPlp/www/custom.css b/BookOfOhdsiPlp/www/custom.css new file mode 100644 index 00000000..26730bbd --- /dev/null +++ b/BookOfOhdsiPlp/www/custom.css @@ -0,0 +1,81 @@ +.plotly.html-widget.html-widget-output.shiny-bound-output.js-plotly-plot { + z-index: 22; + position: relative; +} + +.plotlybars { + padding: 0 10px; + vertical-align: bottom; + width: 100%; + height: 100%; + overflow: hidden; + position: relative; + box-sizing: border-box; +} + +.plotlybars-wrapper { + width: 165px; + height: 100px; + margin: 0 auto; + left: 0; + right: 0; + position: absolute; + z-index: 1; +} + +.plotlybars-text { + color: #447adb; + font-family: 'Open Sans', verdana, arial, sans-serif; + font-size: 80%; + text-align: center; + margin-top: 5px; +} + +.plotlybars-bar { + background-color: #447adb; + height: 100%; + width: 13.3%; + position: absolute; + + -webkit-transform: translateZ(0); + transform: translateZ(0); + + animation-duration: 2s; + animation-iteration-count: infinite; + animation-direction: normal; + animation-timing-function: linear; + + -webkit-animation-duration: 2s; + -webkit-animation-iteration-count: infinite; + -webkit-animation-direction: normal; + -webkit-animation-timing-function: linear; +} + +.b1 { left: 0%; top: 88%; animation-name: b1; -webkit-animation-name: b1; } +.b2 { left: 14.3%; top: 76%; animation-name: b2; -webkit-animation-name: b2; } +.b3 { left: 28.6%; top: 16%; animation-name: b3; -webkit-animation-name: b3; } +.b4 { left: 42.9%; top: 40%; animation-name: b4; -webkit-animation-name: b4; } +.b5 { left: 57.2%; top: 26%; animation-name: b5; -webkit-animation-name: b5; } +.b6 { left: 71.5%; top: 67%; animation-name: b6; -webkit-animation-name: b6; } +.b7 { left: 85.8%; top: 89%; animation-name: b7; -webkit-animation-name: b7; } + +@keyframes b1 { 0% { top: 88%; } 44% { top: 0%; } 94% { top: 100%; } 100% { top: 88%; } } +@-webkit-keyframes b1 { 0% { top: 88%; } 44% { top: 0%; } 94% { top: 100%; } 100% { top: 88%; } } + +@keyframes b2 { 0% { top: 76%; } 38% { top: 0%; } 88% { top: 100%; } 100% { top: 76%; } } +@-webkit-keyframes b2 { 0% { top: 76%; } 38% { top: 0%; } 88% { top: 100%; } 100% { top: 76%; } } + +@keyframes b3 { 0% { top: 16%; } 8% { top: 0%; } 58% { top: 100%; } 100% { top: 16%; } } +@-webkit-keyframes b3 { 0% { top: 16%; } 8% { top: 0%; } 58% { top: 100%; } 100% { top: 16%; } } + +@keyframes b4 { 0% { top: 40%; } 20% { top: 0%; } 70% { top: 100%; } 100% { top: 40%; } } +@-webkit-keyframes b4 { 0% { top: 40%; } 20% { top: 0%; } 70% { top: 100%; } 100% { top: 40%; } } + +@keyframes b5 { 0% { top: 26%; } 13% { top: 0%; } 63% { top: 100%; } 100% { top: 26%; } } +@-webkit-keyframes b5 { 0% { top: 26%; } 13% { top: 0%; } 63% { top: 100%; } 100% { top: 26%; } } + +@keyframes b6 { 0% { top: 67%; } 33.5% { top: 0%; } 83% { top: 100%; } 100% { top: 67%; } } +@-webkit-keyframes b6 { 0% { top: 67%; } 33.5% { top: 0%; } 83% { top: 100%; } 100% { top: 67%; } } + +@keyframes b7 { 0% { top: 89%; } 44.5% { top: 0%; } 94.5% { top: 100%; } 100% { top: 89%; } } +@-webkit-keyframes b7 { 0% { top: 89%; } 44.5% { top: 0%; } 94.5% { top: 100%; } 100% { top: 89%; } } diff --git a/BookOfOhdsiPlp/www/favicon.ico b/BookOfOhdsiPlp/www/favicon.ico new file mode 100644 index 00000000..4421f8ef Binary files /dev/null and b/BookOfOhdsiPlp/www/favicon.ico differ diff --git a/BookOfOhdsiPlp/www/logo.png b/BookOfOhdsiPlp/www/logo.png new file mode 100644 index 00000000..1392aa83 Binary files /dev/null and b/BookOfOhdsiPlp/www/logo.png differ