Merge pull request #1096 from synthetichealth/hospital_practitioner_n…

…djson New NDJSON files
synthetichealth · Sep 7, 2022 · 1994bef · 1994bef
2 parents fb5add0 + 43e33f2
commit 1994bef
Show file tree

Hide file tree

Showing 15 changed files with 553 additions and 136 deletions.
diff --git a/build.gradle b/build.gradle
@@ -123,7 +123,7 @@ test {
   }
   maxHeapSize = "6144m"
   jacoco {
-    excludes = ["org.hl7.*", "com.google.*"]
+    includes = ["org.mitre.*"]
   }
 }
 

diff --git a/src/main/java/org/mitre/synthea/export/Exporter.java b/src/main/java/org/mitre/synthea/export/Exporter.java
@@ -427,16 +427,12 @@ public static void runPostCompletionExports(Generator generator, ExporterRuntime
       deferredExports.clear();
     }
 
-    String bulk = Config.get("exporter.fhir.bulk_data");
-
-    // Before we force bulk data to be off...
     try {
       FhirGroupExporterR4.exportAndSave(generator.getRandomizer(), generator.stop);
     } catch (Exception e) {
       e.printStackTrace();
     }
 
-    Config.set("exporter.fhir.bulk_data", "false");
     try {
       HospitalExporterR4.export(generator.getRandomizer(), generator.stop);
     } catch (Exception e) {
@@ -472,7 +468,6 @@ public static void runPostCompletionExports(Generator generator, ExporterRuntime
     } catch (Exception e) {
       e.printStackTrace();
     }
-    Config.set("exporter.fhir.bulk_data", bulk);
 
     if (Config.getAsBoolean("exporter.bfd.export")) {
       try {

diff --git a/src/main/java/org/mitre/synthea/export/FhirGroupExporterR4.java b/src/main/java/org/mitre/synthea/export/FhirGroupExporterR4.java
@@ -1,6 +1,5 @@
 package org.mitre.synthea.export;
 
-import ca.uhn.fhir.context.FhirContext;
 import ca.uhn.fhir.parser.IParser;
 
 import java.io.File;

diff --git a/src/main/java/org/mitre/synthea/export/FhirPractitionerExporterDstu2.java b/src/main/java/org/mitre/synthea/export/FhirPractitionerExporterDstu2.java
@@ -6,18 +6,13 @@
 import ca.uhn.fhir.model.dstu2.resource.Practitioner;
 import ca.uhn.fhir.model.dstu2.valueset.BundleTypeEnum;
 import ca.uhn.fhir.model.primitive.IntegerDt;
+import ca.uhn.fhir.parser.IParser;
 
 import com.google.common.collect.Table;
 
 import java.io.File;
-import java.io.IOException;
-import java.nio.file.Files;
 import java.nio.file.Path;
-import java.nio.file.Paths;
-import java.nio.file.StandardOpenOption;
 import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
 import java.util.Map;
 import java.util.concurrent.atomic.AtomicInteger;
 
@@ -65,21 +60,22 @@ public static void export(long stop) {
         }
       }
 
-      String bundleJson = FhirDstu2.getContext().newJsonParser().setPrettyPrint(true)
-          .encodeResourceToString(bundle);
+      boolean ndjson = Config.getAsBoolean("exporter.fhir.bulk_data", false);
+      File outputFolder = Exporter.getOutputFolder("fhir_dstu2", null);
+      IParser parser = FhirDstu2.getContext().newJsonParser();
 
-      // get output folder
-      List<String> folders = new ArrayList<>();
-      folders.add("fhir_dstu2");
-      String baseDirectory = Config.get("exporter.baseDirectory");
-      File f = Paths.get(baseDirectory, folders.toArray(new String[0])).toFile();
-      f.mkdirs();
-      Path outFilePath = f.toPath().resolve("practitionerInformation" + stop + ".json");
-
-      try {
-        Files.write(outFilePath, Collections.singleton(bundleJson), StandardOpenOption.CREATE_NEW);
-      } catch (IOException e) {
-        e.printStackTrace();
+      if (ndjson) {
+        Path outFilePath = outputFolder.toPath().resolve("Practitioner." + stop + ".ndjson");
+        for (Bundle.Entry entry : bundle.getEntry()) {
+          String entryJson = parser.encodeResourceToString(entry.getResource());
+          Exporter.appendToFile(outFilePath, entryJson);
+        }
+      } else {
+        parser = parser.setPrettyPrint(true);
+        Path outFilePath =
+            outputFolder.toPath().resolve("practitionerInformation" + stop + ".json");
+        String bundleJson = parser.encodeResourceToString(bundle);
+        Exporter.overwriteFile(outFilePath, bundleJson);
       }
     }
   }

diff --git a/src/main/java/org/mitre/synthea/export/FhirPractitionerExporterR4.java b/src/main/java/org/mitre/synthea/export/FhirPractitionerExporterR4.java
@@ -1,17 +1,12 @@
 package org.mitre.synthea.export;
 
-import ca.uhn.fhir.context.FhirContext;
+import ca.uhn.fhir.parser.IParser;
+
 import com.google.common.collect.Table;
 
 import java.io.File;
-import java.io.IOException;
-import java.nio.file.Files;
 import java.nio.file.Path;
-import java.nio.file.Paths;
-import java.nio.file.StandardOpenOption;
 import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
 import java.util.Map;
 import java.util.concurrent.atomic.AtomicInteger;
 
@@ -20,6 +15,7 @@
 import org.hl7.fhir.r4.model.Bundle.BundleType;
 import org.hl7.fhir.r4.model.IntegerType;
 import org.hl7.fhir.r4.model.Practitioner;
+import org.hl7.fhir.r4.model.ResourceType;
 import org.mitre.synthea.helpers.Config;
 import org.mitre.synthea.helpers.RandomNumberGenerator;
 import org.mitre.synthea.world.agents.Clinician;
@@ -65,21 +61,27 @@ public static void export(RandomNumberGenerator rand, long stop) {
         }
       }
 
-      String bundleJson = FhirR4.getContext().newJsonParser().setPrettyPrint(true)
-          .encodeResourceToString(bundle);
-
-      // get output folder
-      List<String> folders = new ArrayList<>();
-      folders.add("fhir");
-      String baseDirectory = Config.get("exporter.baseDirectory");
-      File f = Paths.get(baseDirectory, folders.toArray(new String[0])).toFile();
-      f.mkdirs();
-      Path outFilePath = f.toPath().resolve("practitionerInformation" + stop + ".json");
+      boolean ndjson = Config.getAsBoolean("exporter.fhir.bulk_data", false);
+      File outputFolder = Exporter.getOutputFolder("fhir", null);
+      IParser parser = FhirR4.getContext().newJsonParser();
 
-      try {
-        Files.write(outFilePath, Collections.singleton(bundleJson), StandardOpenOption.CREATE_NEW);
-      } catch (IOException e) {
-        e.printStackTrace();
+      if (ndjson) {
+        Path pracFilePath = outputFolder.toPath().resolve("Practitioner." + stop + ".ndjson");
+        Path roleFilePath = outputFolder.toPath().resolve("PractitionerRole." + stop + ".ndjson");
+        for (BundleEntryComponent entry : bundle.getEntry()) {
+          String entryJson = parser.encodeResourceToString(entry.getResource());
+          if (entry.getResource().getResourceType() == ResourceType.Practitioner) {
+            Exporter.appendToFile(pracFilePath, entryJson);
+          } else {
+            Exporter.appendToFile(roleFilePath, entryJson);
+          }
+        }
+      } else {
+        parser = parser.setPrettyPrint(true);
+        Path outFilePath =
+            outputFolder.toPath().resolve("practitionerInformation" + stop + ".json");
+        String bundleJson = parser.encodeResourceToString(bundle);
+        Exporter.overwriteFile(outFilePath, bundleJson);
       }
     }
   }

diff --git a/src/main/java/org/mitre/synthea/export/FhirPractitionerExporterStu3.java b/src/main/java/org/mitre/synthea/export/FhirPractitionerExporterStu3.java
@@ -1,17 +1,12 @@
 package org.mitre.synthea.export;
 
-import ca.uhn.fhir.context.FhirContext;
+import ca.uhn.fhir.parser.IParser;
+
 import com.google.common.collect.Table;
 
 import java.io.File;
-import java.io.IOException;
-import java.nio.file.Files;
 import java.nio.file.Path;
-import java.nio.file.Paths;
-import java.nio.file.StandardOpenOption;
 import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
 import java.util.Map;
 import java.util.concurrent.atomic.AtomicInteger;
 
@@ -64,21 +59,22 @@ public static void export(long stop) {
         }
       }
 
-      String bundleJson = FhirStu3.getContext().newJsonParser().setPrettyPrint(true)
-          .encodeResourceToString(bundle);
-
-      // get output folder
-      List<String> folders = new ArrayList<>();
-      folders.add("fhir_stu3");
-      String baseDirectory = Config.get("exporter.baseDirectory");
-      File f = Paths.get(baseDirectory, folders.toArray(new String[0])).toFile();
-      f.mkdirs();
-      Path outFilePath = f.toPath().resolve("practitionerInformation" + stop + ".json");
+      boolean ndjson = Config.getAsBoolean("exporter.fhir.bulk_data", false);
+      File outputFolder = Exporter.getOutputFolder("fhir_stu3", null);
+      IParser parser = FhirStu3.getContext().newJsonParser();
 
-      try {
-        Files.write(outFilePath, Collections.singleton(bundleJson), StandardOpenOption.CREATE_NEW);
-      } catch (IOException e) {
-        e.printStackTrace();
+      if (ndjson) {
+        Path outFilePath = outputFolder.toPath().resolve("Practitioner." + stop + ".ndjson");
+        for (BundleEntryComponent entry : bundle.getEntry()) {
+          String entryJson = parser.encodeResourceToString(entry.getResource());
+          Exporter.appendToFile(outFilePath, entryJson);
+        }
+      } else {
+        parser = parser.setPrettyPrint(true);
+        Path outFilePath =
+            outputFolder.toPath().resolve("practitionerInformation" + stop + ".json");
+        String bundleJson = parser.encodeResourceToString(bundle);
+        Exporter.overwriteFile(outFilePath, bundleJson);
       }
     }
   }

diff --git a/src/main/java/org/mitre/synthea/export/HospitalExporterDstu2.java b/src/main/java/org/mitre/synthea/export/HospitalExporterDstu2.java
@@ -6,18 +6,12 @@
 import ca.uhn.fhir.model.dstu2.resource.Organization;
 import ca.uhn.fhir.model.dstu2.valueset.BundleTypeEnum;
 import ca.uhn.fhir.model.primitive.IntegerDt;
+import ca.uhn.fhir.parser.IParser;
 
 import com.google.common.collect.Table;
 
 import java.io.File;
-import java.io.IOException;
-import java.nio.file.Files;
 import java.nio.file.Path;
-import java.nio.file.Paths;
-import java.nio.file.StandardOpenOption;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
 import java.util.concurrent.atomic.AtomicInteger;
 
 import org.mitre.synthea.helpers.Config;
@@ -50,21 +44,21 @@ public static void export(long stop) {
         }
       }
 
-      String bundleJson = FhirDstu2.getContext().newJsonParser().setPrettyPrint(true)
-          .encodeResourceToString(bundle);
+      boolean ndjson = Config.getAsBoolean("exporter.fhir.bulk_data", false);
+      File outputFolder = Exporter.getOutputFolder("fhir_dstu2", null);
+      IParser parser = FhirDstu2.getContext().newJsonParser();
 
-      // get output folder
-      List<String> folders = new ArrayList<>();
-      folders.add("fhir_dstu2");
-      String baseDirectory = Config.get("exporter.baseDirectory");
-      File f = Paths.get(baseDirectory, folders.toArray(new String[0])).toFile();
-      f.mkdirs();
-      Path outFilePath = f.toPath().resolve("hospitalInformation" + stop + ".json");
-
-      try {
-        Files.write(outFilePath, Collections.singleton(bundleJson), StandardOpenOption.CREATE_NEW);
-      } catch (IOException e) {
-        e.printStackTrace();
+      if (ndjson) {
+        Path outFilePath = outputFolder.toPath().resolve("Organization." + stop + ".ndjson");
+        for (Bundle.Entry entry : bundle.getEntry()) {
+          String entryJson = parser.encodeResourceToString(entry.getResource());
+          Exporter.appendToFile(outFilePath, entryJson);
+        }
+      } else {
+        parser = parser.setPrettyPrint(true);
+        Path outFilePath = outputFolder.toPath().resolve("hospitalInformation" + stop + ".json");
+        String bundleJson = parser.encodeResourceToString(bundle);
+        Exporter.overwriteFile(outFilePath, bundleJson);
       }
     }
   }

diff --git a/src/main/java/org/mitre/synthea/export/HospitalExporterR4.java b/src/main/java/org/mitre/synthea/export/HospitalExporterR4.java
@@ -1,16 +1,11 @@
 package org.mitre.synthea.export;
 
+import ca.uhn.fhir.parser.IParser;
+
 import com.google.common.collect.Table;
 
 import java.io.File;
-import java.io.IOException;
-import java.nio.file.Files;
 import java.nio.file.Path;
-import java.nio.file.Paths;
-import java.nio.file.StandardOpenOption;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
 import java.util.concurrent.atomic.AtomicInteger;
 
 import org.hl7.fhir.r4.model.Bundle;
@@ -19,7 +14,7 @@
 import org.hl7.fhir.r4.model.Extension;
 import org.hl7.fhir.r4.model.IntegerType;
 import org.hl7.fhir.r4.model.Organization;
-
+import org.hl7.fhir.r4.model.ResourceType;
 import org.mitre.synthea.helpers.Config;
 import org.mitre.synthea.helpers.RandomNumberGenerator;
 import org.mitre.synthea.world.agents.Provider;
@@ -53,22 +48,26 @@ public static void export(RandomNumberGenerator rand, long stop) {
       // add in the patient's home location
       FhirR4.addPatientHomeLocation(bundle);
 
-
-      String bundleJson = FhirR4.getContext().newJsonParser().setPrettyPrint(true)
-          .encodeResourceToString(bundle);
-
-      // get output folder
-      List<String> folders = new ArrayList<>();
-      folders.add("fhir");
-      String baseDirectory = Config.get("exporter.baseDirectory");
-      File f = Paths.get(baseDirectory, folders.toArray(new String[0])).toFile();
-      f.mkdirs();
-      Path outFilePath = f.toPath().resolve("hospitalInformation" + stop + ".json");
-
-      try {
-        Files.write(outFilePath, Collections.singleton(bundleJson), StandardOpenOption.CREATE_NEW);
-      } catch (IOException e) {
-        e.printStackTrace();
+      boolean ndjson = Config.getAsBoolean("exporter.fhir.bulk_data", false);
+      File outputFolder = Exporter.getOutputFolder("fhir", null);
+      IParser parser = FhirR4.getContext().newJsonParser();
+
+      if (ndjson) {
+        Path orgFilePath = outputFolder.toPath().resolve("Organization." + stop + ".ndjson");
+        Path locFilePath = outputFolder.toPath().resolve("Location." + stop + ".ndjson");
+        for (BundleEntryComponent entry : bundle.getEntry()) {
+          String entryJson = parser.encodeResourceToString(entry.getResource());
+          if (entry.getResource().getResourceType() == ResourceType.Organization) {
+            Exporter.appendToFile(orgFilePath, entryJson);
+          } else {
+            Exporter.appendToFile(locFilePath, entryJson);
+          }
+        }
+      } else {
+        parser = parser.setPrettyPrint(true);
+        Path outFilePath = outputFolder.toPath().resolve("hospitalInformation" + stop + ".json");
+        String bundleJson = parser.encodeResourceToString(bundle);
+        Exporter.overwriteFile(outFilePath, bundleJson);
       }
     }
   }