Skip to content

Commit

Permalink
initial public commit of 'flexible exporter' aka flexporter
Browse files Browse the repository at this point in the history
  • Loading branch information
dehall committed Aug 1, 2023
1 parent dd185eb commit 1c051be
Show file tree
Hide file tree
Showing 14 changed files with 26,381 additions and 8 deletions.
23 changes: 19 additions & 4 deletions build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,11 @@ dependencies {
implementation 'ca.uhn.hapi.fhir:hapi-fhir-structures-dstu2:6.1.0'
implementation 'ca.uhn.hapi.fhir:hapi-fhir-structures-r4:6.1.0'
implementation 'ca.uhn.hapi.fhir:hapi-fhir-client:6.1.0'

implementation 'ca.uhn.hapi.fhir:hapi-fhir-validation:6.1.0'
implementation 'ca.uhn.hapi.fhir:hapi-fhir-validation-resources-r4:6.1.0'
implementation 'ca.uhn.hapi.fhir:hapi-fhir-validation-resources-dstu3:6.1.0'
implementation 'ca.uhn.hapi.fhir:hapi-fhir-validation-resources-dstu2:6.1.0'
// C-CDA export uses Apache FreeMarker templates
implementation 'org.freemarker:freemarker:2.3.31'

Expand Down Expand Up @@ -123,10 +128,6 @@ dependencies {
testImplementation 'org.powermock:powermock-module-junit4:2.0.9'
testImplementation 'org.powermock:powermock-api-mockito2:2.0.9'
testImplementation 'com.github.tomakehurst:wiremock-jre8:2.33.2'
testImplementation 'ca.uhn.hapi.fhir:hapi-fhir-validation:6.1.0'
testImplementation 'ca.uhn.hapi.fhir:hapi-fhir-validation-resources-r4:6.1.0'
testImplementation 'ca.uhn.hapi.fhir:hapi-fhir-validation-resources-dstu3:6.1.0'
testImplementation 'ca.uhn.hapi.fhir:hapi-fhir-validation-resources-dstu2:6.1.0'
testImplementation 'com.helger:ph-schematron:5.6.5'
testImplementation 'com.helger:ph-commons:9.5.5'
testImplementation 'com.squareup.okhttp3:mockwebserver:4.10.0'
Expand Down Expand Up @@ -178,6 +179,7 @@ task graphviz(type: JavaExec) {
mainClass = "Graphviz"
}


task rifMinimize(type: JavaExec) {
group 'Application'
description 'Filter exported RIF files to produce minimal set that covers all claim types'
Expand Down Expand Up @@ -229,6 +231,19 @@ shadowJar {
task uberJar() {
}

task flexporter(type: JavaExec) {
group 'Application'
description 'Apply transformations to FHIR'
classpath sourceSets.main.runtimeClasspath
mainClass = "RunFlexporter"
// args are called "arams" because they are called with -P,
// ex. gradle run -Params="['arg1', 'args2']"
// see https://stackoverflow.com/questions/27604283/gradle-task-pass-arguments-to-java-application
if (project.hasProperty("arams")) {
args Eval.me(arams)
}
}

task concepts(type: JavaExec) {
group 'Application'
description 'Create a list of simulated concepts'
Expand Down
18 changes: 18 additions & 0 deletions run_flexporter
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
#!/usr/bin/env sh

##############################################################################
##
## Flexporter launcher for UN*X
##
##############################################################################

ARGS=

for arg in "$@"
do
ARGS=$ARGS\'$arg\',
# Trailing comma ok, don't need to remove it
done

./gradlew flexporter -Params="[$ARGS]"

147 changes: 147 additions & 0 deletions src/main/java/RunFlexporter.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,147 @@
import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.nio.file.DirectoryStream;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.StandardOpenOption;
import java.util.Arrays;
import java.util.LinkedList;
import java.util.Queue;

import org.hl7.fhir.r4.model.Bundle;
import org.hl7.fhir.r4.model.Bundle.BundleEntryComponent;
import org.hl7.fhir.r4.model.Resource;
import org.mitre.synthea.export.flexporter.Actions;
import org.mitre.synthea.export.flexporter.FhirPathUtils;
import org.mitre.synthea.export.flexporter.Mapping;

import ca.uhn.fhir.parser.IParser;


public class RunFlexporter {
public static void main(String[] args) throws Exception {
Queue<String> argsQ = new LinkedList<String>(Arrays.asList(args));

File igDirectory = null;
File sourceFile = null;
File mappingFile = null;

while (!argsQ.isEmpty()) {
String currArg = argsQ.poll();

if (currArg.equals("-ig")) {
String value = argsQ.poll();

if (value == null) {
throw new FileNotFoundException("No implementation guide directory provided");
}

igDirectory = new File(value);

if (!igDirectory.isDirectory()) {
throw new FileNotFoundException(String.format(
"Specified implementation guide directory (%s) does not exist or is not a directory",
value));
} else if (isDirEmpty(igDirectory.toPath())) {
throw new FileNotFoundException(
String.format("Specified implementation guide directory (%s) is empty", value));
}
}

else if (currArg.equals("-m")) {
String value = argsQ.poll();

if (value == null) {
throw new FileNotFoundException("No mapping file provided");
}

mappingFile = new File(value);

if (!mappingFile.exists()) {
throw new FileNotFoundException(
String.format("Specified mapping file (%s) does not exist", value));
}
}

else if (currArg.equals("-s")) {
String value = argsQ.poll();
sourceFile = new File(value);

if (value == null) {
throw new FileNotFoundException("No Synthea source FHIR provided");
}

if (!sourceFile.exists()) {
throw new FileNotFoundException(
String.format("Specified Synthea source FHIR (%s) does not exist", value));
}
}
}

if (mappingFile == null || sourceFile == null) {
usage();
System.exit(1);
}

convertFhir(mappingFile, igDirectory, sourceFile);
}

public static void convertFhir(File mappingFile, File igDirectory, File sourceFhir)
throws IOException {

Mapping mapping = Mapping.parseMapping(mappingFile);

IParser parser = FhirPathUtils.FHIR_CTX.newJsonParser().setPrettyPrint(true);

if (sourceFhir.isDirectory()) {

// TODO

} else {
String fhirJson = new String(Files.readAllBytes(sourceFhir.toPath()));
Bundle bundle = parser.parseResource(Bundle.class, fhirJson);

for (BundleEntryComponent bec : bundle.getEntry()) {
Resource r = bec.getResource();
if (r.getId().startsWith("urn:uuid:")) {
// HAPI does some weird stuff with IDs
// by default in Synthea they are just plain UUIDs
// and the entry.fullUrl is urn:uuid:(id)
// but somehow when they get parsed back in, the id is urn:uuid:etc
// which then doesn't get written back out at the end
// so this removes the "urn:uuid:" bit if it got added
r.setId(r.getId().substring(9));
}
}

// bundle is modified in-place
convertFhir(bundle, mapping);

String bundleJson = parser.encodeResourceToString(bundle);

File outFile =
new File("./output/" + System.currentTimeMillis() + "_" + sourceFhir.getName());

Files.write(outFile.toPath(), bundleJson.getBytes(), StandardOpenOption.CREATE_NEW);

System.out.println("Wrote " + outFile);
}
}

public static Bundle convertFhir(Bundle bundle, Mapping mapping) {
Actions.applyMapping(bundle, mapping, null);

return bundle;
}

private static boolean isDirEmpty(final Path directory) throws IOException {
try (DirectoryStream<Path> dirStream = Files.newDirectoryStream(directory)) {
return !dirStream.iterator().hasNext();
}
}

private static void usage() {
System.out.println("Usage: run_flexporter -m MAPPING_FILE -s SOURCE_FHIR [-i IG_FOLDER]");
}
}
20 changes: 16 additions & 4 deletions src/main/java/org/mitre/synthea/export/Exporter.java
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
package org.mitre.synthea.export;

import ca.uhn.fhir.parser.IParser;

import java.io.BufferedWriter;
import java.io.File;
import java.io.FileWriter;
Expand All @@ -24,6 +23,8 @@
import org.apache.commons.lang3.tuple.Pair;

import org.mitre.synthea.engine.Generator;
import org.mitre.synthea.export.flexporter.Actions;
import org.mitre.synthea.export.flexporter.Mapping;
import org.mitre.synthea.export.rif.BB2RIFExporter;
import org.mitre.synthea.helpers.Config;
import org.mitre.synthea.helpers.Utilities;
Expand Down Expand Up @@ -68,6 +69,7 @@ public static class ExporterRuntimeOptions {
!Config.get("generate.terminology_service_url", "").isEmpty();
private BlockingQueue<String> recordQueue;
private SupportedFhirVersion fhirVersion;
private List<Mapping> flexporterMappings;

public ExporterRuntimeOptions() {
yearsOfHistory = Integer.parseInt(Config.get("exporter.years_of_history"));
Expand Down Expand Up @@ -227,17 +229,27 @@ private static boolean exportRecord(Person person, String fileTag, long stopTime
}
if (Config.getAsBoolean("exporter.fhir.export")) {
File outDirectory = getOutputFolder("fhir", person);
org.hl7.fhir.r4.model.Bundle bundle = FhirR4.convertToFHIR(person, stopTime);

if (options.flexporterMappings != null) {
for (Mapping mapping : options.flexporterMappings) {
// flexport on the bundle here
Actions.applyMapping(bundle, mapping, person);
}
}

IParser parser = FhirR4.getContext().newJsonParser();
if (Config.getAsBoolean("exporter.fhir.bulk_data")) {
org.hl7.fhir.r4.model.Bundle bundle = FhirR4.convertToFHIR(person, stopTime);
IParser parser = FhirR4.getContext().newJsonParser().setPrettyPrint(false);
parser.setPrettyPrint(false);
for (org.hl7.fhir.r4.model.Bundle.BundleEntryComponent entry : bundle.getEntry()) {
String filename = entry.getResource().getResourceType().toString() + ".ndjson";
Path outFilePath = outDirectory.toPath().resolve(filename);
String entryJson = parser.encodeResourceToString(entry.getResource());
appendToFile(outFilePath, entryJson);
}
} else {
String bundleJson = FhirR4.convertToFHIRJson(person, stopTime);
parser.setPrettyPrint(true);
String bundleJson = parser.encodeResourceToString(bundle);
Path outFilePath = outDirectory.toPath().resolve(filename(person, fileTag, "json"));
writeNewFile(outFilePath, bundleJson);
}
Expand Down
Loading

0 comments on commit 1c051be

Please sign in to comment.