Skip to content

Commit

Permalink
Post-python removal patching, bugfixing, and log cleaning
Browse files Browse the repository at this point in the history
  • Loading branch information
@CodeByDrescher
CodeByDrescher committed Jan 21, 2025
1 parent d1de15a commit 15a991e
Show file tree
Hide file tree
Showing 21 changed files with 253 additions and 298 deletions.
20 changes: 20 additions & 0 deletions vcell-cli/src/main/java/org/vcell/cli/exceptions/PreProcessingException.java
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
package org.vcell.cli.exceptions;

public class PreProcessingException extends RuntimeException {
public PreProcessingException(String message) {
super(message);
}

public PreProcessingException(Throwable cause) {
super(cause);
}

public PreProcessingException(String message, Throwable cause){
super(message, cause);
}

@Override
public int hashCode() {
return super.hashCode();
}
}
14 changes: 5 additions & 9 deletions vcell-cli/src/main/java/org/vcell/cli/run/ExecuteImpl.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,9 +55,9 @@ public static void batchMode(File dirOfArchivesToProcess, File outputDir, CLIRec
String targetOutputDir = Paths.get(outputBaseDir, bioModelBaseName).toString();

Span span = null;
try {
// Initialization call generates Status YAML
try (BiosimulationLog bioSimLog = BiosimulationLog.initialize(inputFile.getAbsolutePath(), targetOutputDir)) {
span = Tracer.startSpan(Span.ContextType.OMEX_EXECUTE, inputFileName, Map.of("filename", inputFileName));
BiosimulationLog.initialize(inputFile.getAbsolutePath(), targetOutputDir); // generate Status YAML

System.out.println("\n\n");
logger.info("Processing " + inputFileName + "(" + inputFile + ")");
Expand All @@ -79,7 +79,6 @@ public static void batchMode(File dirOfArchivesToProcess, File outputDir, CLIRec
if (span != null) {
span.close();
}
BiosimulationLog.instance().close();
}
}
if (failedFiles.isEmpty()){
Expand Down Expand Up @@ -134,19 +133,16 @@ public static void singleMode(File inputFile, File rootOutputDir, CLIRecordable
String targetOutputDir = bEncapsulateOutput ? Paths.get(outputBaseDir, bioModelBaseName).toString() : outputBaseDir;
File adjustedOutputDir = new File(targetOutputDir);

logger.info("Preparing output directory...");
if (logger.isDebugEnabled()) logger.info("Preparing output directory...");
// we don't want to accidentally delete the input...
// if the output directory is a subset of the input file's housing directory, we shouldn't delete!!
if (!inputFile.getParentFile().getCanonicalPath().contains(adjustedOutputDir.getCanonicalPath()))
RunUtils.removeAndMakeDirs(adjustedOutputDir);

try {
BiosimulationLog.initialize(inputFile.getAbsolutePath(), targetOutputDir); // generate Status YAML

// Initialization line generates Status YAML
try (BiosimulationLog bioSimLog = BiosimulationLog.initialize(inputFile.getAbsolutePath(), targetOutputDir)) {
ExecuteImpl.singleExecOmex(inputFile, rootOutputDir, cliLogger, bKeepTempFiles, bExactMatchOnly,
bEncapsulateOutput, bSmallMeshOverride, bBioSimMode);
} finally {
BiosimulationLog.instance().close();
}
}

Expand Down
74 changes: 31 additions & 43 deletions vcell-cli/src/main/java/org/vcell/cli/run/ExecutionJob.java
View file
Original file line number Diff line number Diff line change
@@ -1,10 +1,12 @@
package org.vcell.cli.run;

import org.apache.logging.log4j.Level;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.vcell.cli.CLIRecordable;
import org.vcell.cli.PythonStreamException;
import org.vcell.cli.exceptions.ExecutionException;
import org.vcell.cli.exceptions.PreProcessingException;
import org.vcell.cli.run.hdf5.BiosimulationsHdf5Writer;
import org.vcell.cli.run.hdf5.BiosimulationsHdfWriterException;
import org.vcell.cli.run.hdf5.HDF5ExecutionResults;
Expand Down Expand Up @@ -71,8 +73,7 @@ private ExecutionJob(){
/**
* Run the neexed steps to prepare an archive for execution.
* Follow-up call: `executeArchive()`
*
* @throws PythonStreamException if calls to the python-shell instance are not working correctly
*
* @throws IOException if there are system I/O issues.
*/
public void preprocessArchive() throws IOException {
Expand All @@ -81,7 +82,7 @@ public void preprocessArchive() throws IOException {

// Beginning of Execution
logger.info("Executing OMEX archive `{}`", this.inputFile.getName());
logger.info("Archive location: {}", this.inputFile.getAbsolutePath());
if (logger.isDebugEnabled()) logger.info("Archive location: {}", this.inputFile.getAbsolutePath());
RunUtils.drawBreakLine("-", 100);

// Unpack the Omex Archive
Expand Down Expand Up @@ -112,50 +113,43 @@ public void preprocessArchive() throws IOException {

/**
* Run solvers on all the models in the archive.
*
* </br>
* Called after: `preprocessArchive()`
* Called before: `postProcessArchive()`
*
* @throws InterruptedException if there is an issue with accessing data
* @throws PythonStreamException if calls to the python-shell instance are not working correctly
* @throws IOException if there are system I/O issues
* @throws ExecutionException if an execution specfic error occurs
*
* @throws ExecutionException if an execution specific error occurs
*/
public void executeArchive(boolean isBioSimSedml) throws BiosimulationsHdfWriterException, PythonStreamException, ExecutionException {
public void executeArchive(boolean isBioSimSedml) throws BiosimulationsHdfWriterException, ExecutionException {
HDF5ExecutionResults cumulativeHdf5Results = new HDF5ExecutionResults(isBioSimSedml);

try {
HDF5ExecutionResults masterHdf5File = new HDF5ExecutionResults(isBioSimSedml);
this.queueAllSedml();

for (String sedmlLocation : this.sedmlLocations){
SedmlJob job = new SedmlJob(sedmlLocation, this.omexHandler, this.inputFile,
this.outputDir, this.sedmlPath2d3d.toString(), this.cliRecorder,
this.bKeepTempFiles, this.bExactMatchOnly, this.bSmallMeshOverride, this.logOmexMessage);
if (!job.preProcessDoc()){
SedmlStatistics stats = job.getDocStatistics(); // Must process document first
logger.error("Statistics of failed SedML:\n" + stats.toString());
for (String sedmlLocation : this.sedmlLocations) {
try {
this.executeSedmlDocument(sedmlLocation, cumulativeHdf5Results);
} catch (PreProcessingException e) {
this.anySedmlDocumentHasFailed = true;
}
SedmlStatistics stats = job.getDocStatistics();
boolean hasSucceeded = job.simulateSedml(masterHdf5File);
this.anySedmlDocumentHasSucceeded |= hasSucceeded;
this.anySedmlDocumentHasFailed &= hasSucceeded;
if (hasSucceeded){
String formattedStats = stats.toFormattedString();
logger.info("Processing of SedML succeeded.\n" + formattedStats);
}
else logger.error("Processing of SedML has failed.\n" + stats.toString());
}
BiosimulationsHdf5Writer.writeHdf5(masterHdf5File, new File(this.outputDir));

} catch(PythonStreamException e){
logger.error("Python-processing encountered fatal error. Execution is unable to properly continue.", e);
throw e;
} catch(InterruptedException | IOException e){
BiosimulationsHdf5Writer.writeHdf5(cumulativeHdf5Results, new File(this.outputDir));

} catch (IOException e){
logger.error("System IO encountered a fatal error");
throw new ExecutionException(e);
}
}

private void executeSedmlDocument(String sedmlLocation, HDF5ExecutionResults cumulativeHdf5Results) throws IOException, PreProcessingException {
BiosimulationLog.instance().updateSedmlDocStatusYml(sedmlLocation, BiosimulationLog.Status.QUEUED);
SedmlJob job = new SedmlJob(sedmlLocation, this.omexHandler, this.inputFile,
this.outputDir, this.sedmlPath2d3d.toString(), this.cliRecorder,
this.bKeepTempFiles, this.bExactMatchOnly, this.bSmallMeshOverride, this.logOmexMessage);
SedmlStatistics stats = job.preProcessDoc();
boolean hasSucceeded = job.simulateSedml(cumulativeHdf5Results);
this.anySedmlDocumentHasSucceeded |= hasSucceeded;
this.anySedmlDocumentHasFailed &= hasSucceeded;
logger.log(hasSucceeded ? Level.INFO : Level.ERROR, "Processing of SedML ({}) {}", stats.getSedmlName(), hasSucceeded ? "succeeded." : "failed!");
}

/**
* Clean up and analyze the results of the archive's execution
*
Expand All @@ -171,7 +165,7 @@ public void postProcessessArchive() throws IOException {
this.endTime_ms = System.currentTimeMillis();
long elapsedTime_ms = this.endTime_ms - this.startTime_ms;
double duration_s = elapsedTime_ms / 1000.0;
logger.info("Omex " + inputFile.getName() + " processing completed (" + duration_s + "s)");
logger.info("Omex `" + inputFile.getName() + "` processing completed (" + duration_s + "s)");
//
// failure if at least one of the documents in the omex archive fails
//
Expand All @@ -192,13 +186,7 @@ public void postProcessessArchive() throws IOException {
}
BiosimulationLog.instance().setOutputMessage("null", "null", "omex", logOmexMessage.toString());

logger.debug("Finished Execution of Archive: " + bioModelBaseName);
}

private void queueAllSedml() throws PythonStreamException, InterruptedException, IOException {
for (String sedmlLocation: sedmlLocations){
BiosimulationLog.instance().updateSedmlDocStatusYml(sedmlLocation, BiosimulationLog.Status.QUEUED);
}
if (logger.isDebugEnabled()) logger.info("Finished Execution of Archive: {}", this.bioModelBaseName);
}

}
20 changes: 9 additions & 11 deletions vcell-cli/src/main/java/org/vcell/cli/run/RunUtils.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -192,13 +192,13 @@ public static HashMap<String, File> generateReportsAsCSV(SedML sedml, SolverHand
for (Output sedmlOutput : allSedmlOutputs) {
// We only want Reports
if (!(sedmlOutput instanceof Report sedmlReport)) {
logger.info("Ignoring unsupported output `" + sedmlOutput.getId() + "` while CSV generation.");
if (logger.isDebugEnabled()) logger.info("Ignoring unsupported output `" + sedmlOutput.getId() + "` while CSV generation.");
continue;
}

StringBuilder sb = new StringBuilder();

logger.info("Generating report `" + sedmlReport.getId() +"`.");
if (logger.isDebugEnabled()) logger.info("Generating report `" + sedmlReport.getId() +"`.");
/*
* we go through each entry (dataset) in the list of datasets
* for each dataset, we use the data reference to obtain the data generator
Expand Down Expand Up @@ -371,16 +371,16 @@ public static HashMap<String, File> generateReportsAsCSV(SedML sedml, SolverHand

} // end of dataset

if (sb.length() > 0) {
if (!sb.isEmpty()) {
File f = new File(outDirForCurrentSedml, sedmlReport.getId() + ".csv");
PrintWriter out = new PrintWriter(f);
out.print(sb.toString());
out.print(sb);
out.flush();
out.close();
logger.info("created csv file for report " + sedmlReport.getId() + ": " + f.getAbsolutePath());
if (logger.isDebugEnabled()) logger.info("created csv file for report " + sedmlReport.getId() + ": " + f.getAbsolutePath());
reportsHash.put(sedmlReport.getId(), f);
} else {
logger.info("no csv file for report " + sedmlReport.getId());
if (logger.isDebugEnabled()) logger.info("no csv file for report " + sedmlReport.getId());
}
} catch (Exception e) {
throw new RuntimeException("CSV generation failed: " + e.getMessage(), e);
Expand Down Expand Up @@ -442,15 +442,13 @@ public static void zipResFiles(File dirPath) throws IOException {
srcFiles = listFilesForFolder(dirPath, ext);

if (srcFiles.isEmpty()) {
logger.warn("No " + ext.toUpperCase() + " files found, skipping archiving `" + extensionListMap.get(ext) + "` files");
if (logger.isDebugEnabled()) logger.warn("No {} files found, skipping archiving `{}` files", ext.toUpperCase(), extensionListMap.get(ext));
} else {
fileOutputStream = new FileOutputStream(Paths.get(dirPath.toString(), extensionListMap.get(ext)).toFile());
zipOutputStream = new ZipOutputStream(fileOutputStream);
if (!srcFiles.isEmpty()) logger.info("Archiving resultant " + ext.toUpperCase() + " files to `" + extensionListMap.get(ext) + "`.");
if (!srcFiles.isEmpty() && logger.isDebugEnabled()) logger.info("Archiving resultant {} files to `{}`.", ext.toUpperCase(), extensionListMap.get(ext));
for (File srcFile : srcFiles) {

fileInputstream = new FileInputStream(srcFile);

// get relative path
relativePath = dirPath.toURI().relativize(srcFile.toURI()).toString();
zipEntry = new ZipEntry(relativePath);
Expand All @@ -472,7 +470,7 @@ public static void zipResFiles(File dirPath) throws IOException {
public static String getTempDir() throws IOException {
String tempPath = String.valueOf(java.nio.file.Files.createTempDirectory(
RunUtils.VCELL_TEMP_DIR_PREFIX + UUID.randomUUID()).toAbsolutePath());
logger.info("TempPath Created: " + tempPath);
logger.trace("TempPath Created: " + tempPath);
return tempPath;
}

Expand Down
Loading

0 comments on commit 15a991e

Please sign in to comment.