Skip to content

Commit

Permalink
Converted Spatial Unit test to use new reporting framework
Browse files Browse the repository at this point in the history
  • Loading branch information
@CodeByDrescher
CodeByDrescher committed Dec 30, 2024
1 parent f12ecc3 commit 3b47c7e
Show file tree
Hide file tree
Showing 3 changed files with 96 additions and 111 deletions.
2 changes: 2 additions & 0 deletions vcell-cli/src/main/resources/test_cases.ndjson
View file
Original file line number Diff line number Diff line change
Expand Up @@ -106,6 +106,8 @@
{"test_collection":"VCELL_BSTS_SYNTHS","file_path":"sedml/SimulatorSupportsSubstitutingAlgorithms/4.execution-should-succeed.omex","should_fail":false,"known_status":null,"known_failure_type":null,"known_failure_desc":null}
{"test_collection":"VCELL_BSTS_SYNTHS","file_path":"sedml/SimulatorSupportsSubstitutingAlgorithms/1.execute-should-fail.omex","should_fail":true,"known_status":null,"known_failure_type":null,"known_failure_desc":null}
{"test_collection":"VCELL_BSTS_SYNTHS","file_path":"sedml/SimulatorSupportsSubstitutingAlgorithms/2.execution-should-succeed.omex","should_fail":false,"known_status":null,"known_failure_type":null,"known_failure_desc":null}
{"test_collection":"VCELL_SPATIAL","file_path":"TinySpatialProject","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
{"test_collection":"VCELL_SPATIAL","file_path":"SimpleSpatialModel","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000001.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000002.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000003.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
Expand Down
45 changes: 20 additions & 25 deletions vcell-cli/src/test/java/org/vcell/cli/run/SpatialArchiveFiles.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,51 +3,46 @@
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Tag;
import org.junit.jupiter.api.Test;
import org.vcell.sedml.testsupport.OmexTestCase;
import org.vcell.sedml.testsupport.OmexTestingDatabase;

import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
import java.util.Arrays;
import java.util.List;
import java.util.function.Predicate;

import static org.junit.jupiter.api.Assertions.assertNotNull;

@Tag("Fast")
public class SpatialArchiveFiles {
private final static String[] allTestFiles = new String[]{
"TinySpatialProject.omex",
"SimpleSpatialModel.omex"
};

public static String[] getSpatialTestCases() {
Predicate<String> testFilter = t -> true;
public static OmexTestCase[] getSpatialTestCases() throws IOException {
List<OmexTestCase> allTestCases = OmexTestingDatabase.loadOmexTestCases();
Predicate<OmexTestCase> testFilter = t -> t.test_collection == OmexTestingDatabase.TestCollection.VCELL_SPATIAL;

return Arrays.stream(allTestFiles).filter(testFilter).toArray(String[]::new);
return allTestCases.stream().filter(testFilter).toArray(OmexTestCase[]::new);
}

public static InputStream getSpatialTestCase(String testFile) {
if (Arrays.stream(allTestFiles).noneMatch(file -> file.equals(testFile))) {
throw new RuntimeException("file not found for VCell Published Test Suite test "+testFile);
}
try {
return getFileFromResourceAsStream(testFile);
}catch (FileNotFoundException e){
throw new RuntimeException("failed to find test case file '"+testFile+"': " + e.getMessage(), e);
}
public static InputStream getOmex(OmexTestCase testCase) {
String fullPath = testCase.test_collection.pathPrefix + "/" + testCase.file_path;
String path = fullPath.substring(fullPath.indexOf("/spatial"));
return getFileFromResourceAsStream(path + ".omex");
}

private static InputStream getFileFromResourceAsStream(String fileName) throws FileNotFoundException {
Class<org.vcell.cli.run.SpatialArchiveFiles> spatialFilesClass = org.vcell.cli.run.SpatialArchiveFiles.class;
InputStream nextTestFile = spatialFilesClass.getResourceAsStream("/spatial/" + fileName);
if (nextTestFile == null) {
throw new FileNotFoundException("file not found! " + fileName);
private static InputStream getFileFromResourceAsStream(String path) {
InputStream inputStream = SpatialArchiveFiles.class.getResourceAsStream(path);
if (inputStream == null) {
throw new RuntimeException("file not found! " + path);
} else {
return nextTestFile;
return inputStream;
}
}

@Test
public void test_read_Spatial_omex_file() {
InputStream inputStream = getSpatialTestCase(allTestFiles[0]);
assertNotNull(inputStream);
public void test_read_omex_file() throws IOException {
InputStream inputStream = getOmex(getSpatialTestCases()[0]);
Assertions.assertTrue(inputStream != null);
}
}
160 changes: 74 additions & 86 deletions vcell-cli/src/test/java/org/vcell/cli/run/SpatialExecTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,10 @@
import cbit.vcell.mongodb.VCMongoMessage;
import cbit.vcell.resource.PropertyLoader;
import org.apache.commons.io.FileUtils;
import org.apache.logging.log4j.Level;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.core.LoggerContext;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.Assumptions;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Tag;
import org.junit.jupiter.params.ParameterizedTest;
Expand All @@ -13,6 +15,9 @@
import org.vcell.cli.CLIRecordable;
import org.vcell.cli.PythonStreamException;
import org.vcell.sedml.testsupport.FailureType;
import org.vcell.sedml.testsupport.OmexTestCase;
import org.vcell.sedml.testsupport.OmexTestingDatabase;
import org.vcell.trace.TraceEvent;
import org.vcell.trace.Tracer;
import org.vcell.util.VCellUtilityHub;

Expand All @@ -38,6 +43,13 @@ public static void setup() throws PythonStreamException, IOException {
PropertyLoader.setProperty(PropertyLoader.cliWorkingDir, new File("../vcell-cli-utils").getAbsolutePath());
VCMongoMessage.enabled = false;

LoggerContext config = (LoggerContext)(LogManager.getContext(false));
config.getConfiguration().getLoggerConfig(LogManager.getLogger("org.vcell").getName()).setLevel(Level.DEBUG);
config.getConfiguration().getLoggerConfig(LogManager.getLogger("cbit").getName()).setLevel(Level.DEBUG);
config.updateLoggers();
config.getConfiguration().getLoggerConfig(LogManager.getLogger("io.jhdf").getName()).setLevel(Level.WARN);
config.updateLoggers();

CLIPythonManager.getInstance().instantiatePythonProcess();
}

Expand Down Expand Up @@ -81,114 +93,90 @@ public static void teardown() throws Exception {
//
// }

static Set<String> blacklistedModels(){
HashSet<String> blacklistSet = new HashSet<>();
// Hooray! Nothing unsupported yet!
return blacklistSet;
static Map<String, FailureType> knownFailureTypes() {
HashMap<String, FailureType> failureTypes = new HashMap<>();
return failureTypes; // Hooray! no accepted failures yet!
}

static Map<String, FailureType> knownFaults() {
HashMap<String, FailureType> faults = new HashMap<>();
// Hooray! no known faults yet!
return faults;
public static Collection<OmexTestCase> testCases() throws IOException {
Predicate<OmexTestCase> projectFilter;
projectFilter = (t) -> true; // don't skip any for now.

return Arrays.stream(SpatialArchiveFiles.getSpatialTestCases()).filter(projectFilter).collect(Collectors.toList());
}

public static Collection<String> testCases() {
Set<String> modelsToFilter = new HashSet<>(blacklistedModels());
Predicate<String> filter = (t) -> !modelsToFilter.contains(t);
static class TestRecorder implements CLIRecordable {

return Arrays.stream(SpatialArchiveFiles.getSpatialTestCases()).filter(filter).collect(Collectors.toList());
public TestRecorder() {
Tracer.clearTraceEvents();
}

@Override
public void writeDetailedErrorList(Exception e, String message) {
System.err.println("writeDetailedErrorList(): " + message);
Tracer.failure(e, "writeDetailedErrorList(): "+message);
}
@Override
public void writeFullSuccessList(String message) {
System.out.println("writeFullSuccessList(): " + message);
Tracer.success("writeFullSuccessList(): " + message);
}
@Override
public void writeErrorList(Exception e, String message) {
System.err.println("writeErrorList(): " + message);
Tracer.failure(e, "writeErrorList(): " + message);
}
@Override
public void writeDetailedResultList(String message) {
System.out.println("writeDetailedResultList(): " + message);
Tracer.log("writeDetailedResultList(): "+message);
}
@Override
public void writeSpatialList(String message) {
System.out.println("writeSpatialList(): " + message);
Tracer.log("writeSpatialList(): "+message);
}
@Override
public void writeImportErrorList(Exception e, String message) {
System.err.println("writeImportErrorList(): " + message);
Tracer.failure(e, "writeImportErrorList(): " + message);
}
}

@ParameterizedTest
@MethodSource("testCases")
public void testSpatialOmex(String testCaseFilename) throws Exception {
FailureType knownFault = knownFaults().get(testCaseFilename);
public void testSpatialOmex(OmexTestCase testCase) throws Exception {
FailureType knownFailureType = testCase.known_failure_type;
Tracer.clearTraceEvents();
try {
System.out.println("running test " + testCaseFilename);
System.out.println("running testCase " + testCase.test_collection + " " + testCase.file_path);

Path outdirPath = Files.createTempDirectory("Spatial_OmexExecTest");
CLIRecordable cliRecorder = new CLIRecordable() {
@Override
public void writeDetailedErrorList(Exception e, String message) {
System.err.println("writeDetailedErrorList(): " + message);
Tracer.failure(e, "detailedErrorList: " + message);
}
@Override
public void writeFullSuccessList(String message) {
System.out.println("writeFullSuccessList(): " + message);
Tracer.success("fullSuccessList: " + message);
}
@Override
public void writeErrorList(Exception e, String message) {
System.err.println("writeErrorList(): " + message);
Tracer.failure(e, "errorList: " + message);
}
@Override
public void writeDetailedResultList(String message) {
System.out.println("writeDetailedResultList(): " + message);
Tracer.log("detailedResultList: " + message);
}
@Override
public void writeSpatialList(String message) {
System.out.println("writeSpatialList(): " + message);
Tracer.log("spatialList: " + message);
}
@Override
public void writeImportErrorList(Exception e, String message) {
System.err.println("writeImportErrorList(): " + message);
Tracer.failure(e, "importErrorList: " + message);
}
};
InputStream omexInputStream = SpatialArchiveFiles.getSpatialTestCase(testCaseFilename);
InputStream omexInputStream = SpatialArchiveFiles.getOmex(testCase);
Path omexFile = Files.createTempFile("Spatial_OmexFile_", "omex");
FileUtils.copyInputStreamToFile(omexInputStream, omexFile.toFile());

CLIRecordable cliRecorder = new TestRecorder();
ExecuteImpl.singleMode(omexFile.toFile(), outdirPath.toFile(), cliRecorder);

assertFalse(Tracer.hasErrors(), "no exception, but trace errors found: error[0] = "+(Tracer.hasErrors()?(Tracer.getErrors().get(0).message.replace("\n"," | ")):null));
if (knownFault != null){
throw new RuntimeException("test case passed, but expected " + knownFault.name() + ", remove "
+ testCaseFilename + " from known faults");
String errorMessage = (Tracer.hasErrors()) ? "failure: '" + Tracer.getErrors().get(0).message.replace("\n", " | ") : "";
if (Tracer.hasErrors()) {
throw new RuntimeException(errorMessage);
}
if (knownFailureType != null){
throw new RuntimeException("test case passed, but expected " + knownFailureType.name() + ", remove " + testCase.file_path + " from known FailureTypes");
}

} catch (Exception | AssertionError e){
FailureType fault = this.determineFault(e);
if (knownFault == fault) {
} catch (Exception e){
List<TraceEvent> errorEvents = Tracer.getErrors();
FailureType observedFailure = OmexTestingDatabase.determineFault(e, errorEvents);
if (knownFailureType == observedFailure) {
System.err.println("Expected error: " + e.getMessage());
return;
}

System.err.println("add FAULT." + fault.name() + " to " + testCaseFilename);
throw new Exception("Test error: " + testCaseFilename + " failed improperly", e);
}
}

private FailureType determineFault(Throwable caughtException){ // Throwable because Assertion Error
String errorMessage = caughtException.getMessage();
if (errorMessage == null) errorMessage = ""; // Prevent nullptr exception

if (caughtException instanceof Error && caughtException.getCause() != null)
errorMessage = caughtException.getCause().getMessage();

if (errorMessage.contains("refers to either a non-existent model")) { //"refers to either a non-existent model (invalid SED-ML) or to another model with changes (not supported yet)"
return FailureType.SEDML_UNSUPPORTED_MODEL_REFERENCE;
} else if (errorMessage.contains("System IO encountered a fatal error")){
Throwable subException = caughtException.getCause();
//String subMessage = (subException == null) ? "" : subException.getMessage();
if (subException instanceof FileAlreadyExistsException){
return FailureType.HDF5_FILE_ALREADY_EXISTS;
}
} else if (errorMessage.contains("error while processing outputs: null")){
Throwable subException = caughtException.getCause();
if (subException instanceof ArrayIndexOutOfBoundsException){
return FailureType.ARRAY_INDEX_OUT_OF_BOUNDS;
}
} else if (errorMessage.contains("nconsistent unit system in SBML model") ||
errorMessage.contains("ust be of type")){
return FailureType.SEDML_ERRONEOUS_UNIT_SYSTEM;
System.err.println("add FailureType." + observedFailure.name() + " to " + testCase.file_path);
throw new Exception("Test error: " + testCase.file_path + " failed improperly", e);
}

return FailureType.UNCATETORIZED_FAULT;
}
}

0 comments on commit 3b47c7e

Please sign in to comment.