Plotting in Java for VCell CLI (+ Python live shell "removal" & deprecation)

vcell-cli/pom.xml

@@ -98,6 +98,11 @@
             <artifactId>picocli</artifactId>
             <version>${picocli.version}</version>
         </dependency>
+        <dependency>
+            <groupId>org.jfree</groupId>
+            <artifactId>jfreechart</artifactId>
+            <version>1.5.5</version>
+        </dependency>
         <dependency>
             <groupId>org.junit.jupiter</groupId>
             <artifactId>junit-jupiter</artifactId>

vcell-cli/src/main/java/org/vcell/cli/CLIPythonManager.java

@@ -46,7 +46,10 @@ public class CLIPythonManager {
     /**
      * Retrieve the Python Manager, or create and return if it doesn't exist.
      * @return the manager
+     * @deprecated CLIPythonManager is no longer used in CLI, and has no current use; should this be used again
+     * you need to check whether there are still bugs with Arm Macs, and whether single-instance python shells are not a viable alternative
      */
+    @Deprecated
     public static CLIPythonManager getInstance(){
         lg.trace("Getting Python instance");
         if (instance == null){
@@ -62,7 +65,9 @@ public static CLIPythonManager getInstance(){
      * @param arguments the arguments to provide to the function call, each as their own string, in the correct order
      * @return the return response from Python
      * @throws PythonStreamException if there is any exception encountered in this exchange.
+     * @deprecated See getInstance()
      */
+    @Deprecated
     public String callPython(String functionName, String... arguments) throws PythonStreamException {
         String command = this.formatPythonFunctionCall(functionName, arguments);
         try {
@@ -82,7 +87,9 @@ public String callPython(String functionName, String... arguments) throws Python
      * @param cliCommand the command to run
      * @throws InterruptedException if the python process was interrupted
      * @throws IOException if there was a system IO failure
+     * @deprecated See getInstance()
      */
+    @Deprecated
     private static String callNonSharedPython(String cliCommand)
             throws InterruptedException, IOException, PythonStreamException {
         Path cliWorkingDir = Paths.get(PropertyLoader.getRequiredProperty(PropertyLoader.cliWorkingDir));
@@ -96,7 +103,9 @@ private static String callNonSharedPython(String cliCommand)
      * Shuts down the python session and cleans up.
      * 
      * @throws IOException if there is a system IO issue.
+     * @deprecated See getInstance()
      */
+    @Deprecated
     public void closePythonProcess() throws IOException {
         // Exit the living Python Process
         lg.debug("Closing Python Instance");
@@ -122,7 +131,9 @@ public void closePythonProcess() throws IOException {
      * as means of installation verification.
      * 
      * @throws IOException if there is a problem with System I/O
+     * @deprecated this entire system is unstable on ARM Macs, and still a bit slow.
      */
+    @Deprecated
     public void instantiatePythonProcess() throws IOException, PythonStreamException {
         if (this.pythonProcess != null) return; // prevent override
         lg.info("Initializing Python...");
@@ -338,10 +349,26 @@ private static String runAndPrintProcessStreams(ProcessBuilder pb, String outStr
         return os;
     }
 
+    /**
+     * Checks whether python returned successfully or not
+     * @param returnedString
+     * @throws PythonStreamException
+     * @deprecated See getInstance()
+     */
+    @Deprecated
     public void parsePythonReturn(String returnedString) throws PythonStreamException {
        this.parsePythonReturn(returnedString, null, null);
     }
 
+    /**
+     * Checks whether python returned successfully or not
+     * @param returnedString
+     * @param outString
+     * @param errString
+     * @throws PythonStreamException
+     * @deprecated See getInstance()
+     */
+    @Deprecated
     public void parsePythonReturn(String returnedString, String outString, String errString) throws PythonStreamException {
         boolean DEBUG_NORMAL_OUTPUT = lg.isTraceEnabled(); // Consider getting rid of this, currently redundant
         String ERROR_PHRASE1 = "Traceback", ERROR_PHRASE2 = "File \"<stdin>\"";

vcell-cli/src/main/java/org/vcell/cli/biosimulation/BiosimulationsCommand.java

@@ -42,14 +42,38 @@ public class BiosimulationsCommand implements Callable<Integer> {
     private boolean help;
 
     public Integer call() {
-        CLIRecorder cliRecorder = null;
         int returnCode;
-        
-        if ((returnCode = this.noFurtherActionNeeded(bQuiet, bDebug, bVersion)) != -1)
+
+        if ((returnCode = BiosimulationsCommand.noFurtherActionNeeded(bQuiet, bDebug, bVersion)) != -1)
             return returnCode;
-
+
+        if (this.ARCHIVE == null) {
+            logger.error("ARCHIVE file not specified, try --help for usage");
+            return 1;
+        }
+        if (this.OUT_DIR == null) {
+            logger.error("OUT_DIR not specified, try --help for usage");
+            return 1;
+        }
+        String trace_args =  String.format(
+                "Arguments:\nArchive\t: \"%s\"\nOut Dir\t: \"%s\"\nDebug\t: %b\n" +
+                        "Quiet\t: %b\nVersion\t: %b\nHelp\t: %b",
+                ARCHIVE.getAbsolutePath(), OUT_DIR.getAbsolutePath(), bDebug, bQuiet, bVersion, help
+        );
+
+        logger.trace(trace_args);
+        return BiosimulationsCommand.executeVCellBiosimulationsMode(ARCHIVE, OUT_DIR, bQuiet, bDebug);
+    }
+
+    public static int executeVCellBiosimulationsMode(File inFile, File outDir){
+        return BiosimulationsCommand.executeVCellBiosimulationsMode(inFile, outDir, false, false);
+    }
+
+    public static int executeVCellBiosimulationsMode(File inFile, File outDir, boolean bQuiet, boolean bDebug){
+        CLIRecorder cliRecorder;
+
         try {
-            cliRecorder = new CLIRecorder(OUT_DIR); // CLILogger will throw an execption if our output dir isn't valid.
+            cliRecorder = new CLIRecorder(outDir); // CLILogger will throw an execption if our output dir isn't valid.
             Level logLevel = logger.getLevel();
             if (!bQuiet && bDebug) {
                 logLevel = Level.DEBUG;
@@ -58,59 +82,38 @@ public Integer call() {
             }
             LoggerContext config = (LoggerContext)(LogManager.getContext(false));
             config.getConfiguration().getLoggerConfig(LogManager.getLogger("org.vcell").getName()).setLevel(logLevel);
-            config.getConfiguration().getLoggerConfig(LogManager.getLogger("cbit").getName()).setLevel(logLevel);
+
+            config.getConfiguration().getLoggerConfig(LogManager.getLogger("cbit").getName()).setLevel(bDebug ? logLevel : Level.WARN );
+            config.getConfiguration().getLoggerConfig(LogManager.getLogger("org.jlibsedml").getName()).setLevel(bDebug ? logLevel : Level.WARN);
             config.updateLoggers();
 
             logger.debug("Biosimulations mode requested");
 
-            String trace_args =  String.format(
-                "Arguments:\nArchive\t: \"%s\"\nOut Dir\t: \"%s\"\nDebug\t: %b\n" +
-                    "Quiet\t: %b\nVersion\t: %b\nHelp\t: %b",
-                ARCHIVE.getAbsolutePath(), OUT_DIR.getAbsolutePath(), bDebug, bQuiet, bVersion, help
-            );
-
-            logger.trace(trace_args);
-
             logger.trace("Validating input");
-            if (ARCHIVE == null) {
-                logger.error("ARCHIVE file not specified, try --help for usage");
-                return 1;
-            }
-            if (!ARCHIVE.isFile()) {
-                logger.error("ARCHIVE file " + ARCHIVE.getAbsolutePath() + " not found, try --help for usage");
+            if (!inFile.isFile()) {
+                logger.error("ARCHIVE file " + inFile.getAbsolutePath() + " not found, try --help for usage");
                 return 1;
             }
 
             logger.trace("Validating output");
-            if (OUT_DIR == null) {
-                logger.error("OUT_DIR not specified, try --help for usage");
-                return 1;
-            }
-            if (!OUT_DIR.isDirectory()) {
-                logger.error("OUT_DIR " + OUT_DIR.getAbsolutePath() + " not found or is not a directory, try --help for usage");
+            if (!outDir.isDirectory()) {
+                logger.error("OUT_DIR " + outDir.getAbsolutePath() + " not found or is not a directory, try --help for usage");
                 return 1;
             }
 
             logger.info("Beginning execution");
             File tmpDir = Files.createTempDirectory("VCell_CLI_" + Long.toHexString(new Date().getTime())).toFile();
             try {
-                CLIPythonManager.getInstance().instantiatePythonProcess();
-                ExecuteImpl.singleMode(ARCHIVE, tmpDir, cliRecorder, true);
-                CLIPythonManager.getInstance().closePythonProcess(); // Give the process time to finish
+                ExecuteImpl.singleMode(inFile, tmpDir, cliRecorder, true);
                 if (!Tracer.hasErrors()) return 0;
                 if (!bQuiet) {
                     logger.error("Errors occurred during execution");
                     Tracer.reportErrors(bDebug);
                 }
                 return 1;
             } finally {
-                try {
-                    CLIPythonManager.getInstance().closePythonProcess(); // WARNING: Python will need reinstantiation after this is called
-                } catch (Exception e) {
-                    logger.error(e.getMessage(), e);
-                }
                 logger.debug("Finished all execution.");
-                FileUtils.copyDirectoryContents(tmpDir, OUT_DIR, true, null);
+                FileUtils.copyDirectoryContents(tmpDir, outDir, true, null);
             }
         } catch (Exception e) {
             if (!bQuiet) {
@@ -121,7 +124,7 @@ public Integer call() {
         }
     }
 
-    private int noFurtherActionNeeded(boolean bQuiet, boolean bDebug, boolean bVersion){
+    private static int noFurtherActionNeeded(boolean bQuiet, boolean bDebug, boolean bVersion){
         logger.debug("Validating CLI arguments");
         if (bVersion) {
             String version = PropertyLoader.getRequiredProperty(PropertyLoader.vcellSoftwareVersion);

vcell-cli/src/main/java/org/vcell/cli/exceptions/PreProcessingException.java

@@ -0,0 +1,20 @@
+package org.vcell.cli.exceptions;
+
+public class PreProcessingException extends RuntimeException {
+    public PreProcessingException(String message) {
+        super(message);
+    }
+
+    public PreProcessingException(Throwable cause) {
+        super(cause);
+    }
+
+    public PreProcessingException(String message, Throwable cause){
+        super(message, cause);
+    }
+
+    @Override
+    public int hashCode() {
+        return super.hashCode();
+    }
+}

vcell-cli/src/main/java/org/vcell/cli/run/ExecuteCommand.java

@@ -110,8 +110,6 @@ public Integer call() {
                 System.err.println("cannot specify both debug and quiet, try --help for usage");
                 return 1;
             }
-
-            CLIPythonManager.getInstance().instantiatePythonProcess();
 
 
             Executable.setGlobalTimeoutMS(EXECUTABLE_MAX_WALLCLOCK_MILLIS);
@@ -132,7 +130,7 @@ public Integer call() {
                             bEncapsulateOutput, bSmallMeshOverride);
                 }
             }
-            CLIPythonManager.getInstance().closePythonProcess();
+
             // WARNING: Python needs re-instantiation once the above line is called!
             FileUtils.copyDirectoryContents(tmpDir, outputFilePath, true, null);
             return 0;

vcell-cli/src/main/java/org/vcell/cli/run/ExecuteImpl.java

@@ -5,7 +5,6 @@
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.vcell.cli.CLIRecordable;
-import org.vcell.cli.PythonStreamException;
 import org.vcell.cli.exceptions.ExecutionException;
 import org.vcell.cli.run.hdf5.BiosimulationsHdfWriterException;
 import org.vcell.sedml.log.BiosimulationLog;
@@ -55,9 +54,9 @@ public static void batchMode(File dirOfArchivesToProcess, File outputDir, CLIRec
                 String targetOutputDir = Paths.get(outputBaseDir, bioModelBaseName).toString();
 
                 Span span = null;
-                try {
+                // Initialization call generates Status YAML
+                try (BiosimulationLog bioSimLog = BiosimulationLog.initialize(inputFile.getAbsolutePath(), targetOutputDir)) {
                     span = Tracer.startSpan(Span.ContextType.OMEX_EXECUTE, inputFileName, Map.of("filename", inputFileName));
-                    BiosimulationLog.initialize(inputFile.getAbsolutePath(), targetOutputDir);    // generate Status YAML
 
                     System.out.println("\n\n");
                     logger.info("Processing " + inputFileName + "(" + inputFile + ")");
@@ -79,7 +78,6 @@ public static void batchMode(File dirOfArchivesToProcess, File outputDir, CLIRec
                     if (span != null) {
                         span.close();
                     }
-                    BiosimulationLog.instance().close();
                 }
             }
             if (failedFiles.isEmpty()){
@@ -108,7 +106,8 @@ public static void batchMode(File dirOfArchivesToProcess, File outputDir, CLIRec
 
     private static void runSingleExecOmex(File inputFile, File outputDir, CLIRecordable cliLogger, boolean bKeepTempFiles,
                                           boolean bExactMatchOnly, boolean bSmallMeshOverride)
-            throws IOException, ExecutionException, PythonStreamException, InterruptedException, BiosimulationsHdfWriterException {
+            throws IOException, ExecutionException, BiosimulationsHdfWriterException {
+
         String bioModelBaseName = inputFile.getName().substring(0, inputFile.getName().indexOf(".")); // ".omex"??
         Files.createDirectories(Paths.get(outputDir.getAbsolutePath() + File.separator + bioModelBaseName)); // make output subdir
         final boolean bEncapsulateOutput = true;
@@ -133,19 +132,16 @@ public static void singleMode(File inputFile, File rootOutputDir, CLIRecordable
         String targetOutputDir = bEncapsulateOutput ? Paths.get(outputBaseDir, bioModelBaseName).toString() : outputBaseDir;
         File adjustedOutputDir = new File(targetOutputDir);
 
-        logger.info("Preparing output directory...");
+        if (logger.isDebugEnabled()) logger.info("Preparing output directory...");
         // we don't want to accidentally delete the input...
         // if the output directory is a subset of the input file's housing directory, we shouldn't delete!!
         if (!inputFile.getParentFile().getCanonicalPath().contains(adjustedOutputDir.getCanonicalPath()))
             RunUtils.removeAndMakeDirs(adjustedOutputDir);
 
-        try {
-            BiosimulationLog.initialize(inputFile.getAbsolutePath(), targetOutputDir);    // generate Status YAML
-
+        // Initialization line generates Status YAML
+        try (BiosimulationLog bioSimLog = BiosimulationLog.initialize(inputFile.getAbsolutePath(), targetOutputDir)) {
             ExecuteImpl.singleExecOmex(inputFile, rootOutputDir, cliLogger, bKeepTempFiles, bExactMatchOnly,
                     bEncapsulateOutput, bSmallMeshOverride, bBioSimMode);
-        } finally {
-            BiosimulationLog.instance().close();
         }
     }
 
@@ -205,8 +201,7 @@ public static void singleExecVcml(File vcmlFile, File outputDir, CLIRecordable c
 
             for (String simName : resultsHash.keySet()) {
                 String CSVFilePath = Paths.get(outDirForCurrentVcml.toString(), simName + ".csv").toString();
-                RunUtils.createCSVFromODEResultSet(resultsHash.get(simName), new File(CSVFilePath));
-                PythonCalls.transposeVcmlCsv(CSVFilePath);
+                RunUtils.createCSVFromODEResultSet(resultsHash.get(simName), new File(CSVFilePath), true);
             }
         } catch (IOException e) {
             Tracer.failure(e, "IOException while processing VCML " + vcmlFile.getName());
@@ -237,7 +232,7 @@ public static void singleExecVcml(File vcmlFile, File outputDir, CLIRecordable c
 
     private static void singleExecOmex(File inputFile, File rootOutputDir, CLIRecordable cliRecorder,
             boolean bKeepTempFiles, boolean bExactMatchOnly, boolean bEncapsulateOutput, boolean bSmallMeshOverride, boolean bBioSimMode)
-            throws ExecutionException, PythonStreamException, IOException, BiosimulationsHdfWriterException {
+            throws ExecutionException, IOException, BiosimulationsHdfWriterException {
 
         ExecutionJob requestedExecution = new ExecutionJob(inputFile, rootOutputDir, cliRecorder,
             bKeepTempFiles, bExactMatchOnly, bEncapsulateOutput, bSmallMeshOverride);

vcell-cli/src/main/java/org/vcell/cli/run/ExecuteOmexCommand.java

@@ -93,16 +93,14 @@ public Integer call() {
             config.updateLoggers();
 
 
-            CLIPythonManager.getInstance().instantiatePythonProcess();
-
             Executable.setGlobalTimeoutMS(EXECUTABLE_MAX_WALLCLOCK_MILLIS);
             logger.info("Beginning execution");
             File tmpDir = Files.createTempDirectory("VCell_CLI_" + Long.toHexString(new Date().getTime())).toFile();
 
             Tracer.clearTraceEvents();
             ExecuteImpl.singleMode(inputFilePath, tmpDir, cliTracer, bKeepTempFiles, bExactMatchOnly,
                             bEncapsulateOutput, bSmallMeshOverride);
-            CLIPythonManager.getInstance().closePythonProcess();
+
             // WARNING: Python needs re-instantiation once the above line is called!
             FileUtils.copyDirectoryContents(tmpDir, outputFilePath, true, null);
             final OmexExecSummary omexExecSummary;