add cache_buster to force rerunning sims when sim caching is implemented

biosim_server/api/main.py

@@ -135,6 +135,7 @@ async def start_verify_omex(
         rel_tol: float = Query(default=0.0001, description="Relative tolerance for proximity comparison."),
         abs_tol_min: float = Query(default=0.001, description="Min absolute tolerance, where atol = max(atol_min, max(arr1,arr2)*atol_scale."),
         abs_tol_scale: float = Query(default=0.00001, description="Scale for absolute tolerance, where atol = max(atol_min, max(arr1,arr2)*atol_scale."),
+        cache_buster: str = Query(default="0", description="Optional unique id for cache busting (unique string to force new simulation runs)."),
         observables: Optional[list[str]] = Query(default=None,
                                                  description="List of observables to include in the return data.")
 ) -> OmexVerifyWorkflowOutput:
@@ -172,7 +173,8 @@ async def start_verify_omex(
         rel_tol=rel_tol,
         abs_tol_min=abs_tol_min,
         abs_tol_scale=abs_tol_scale,
-        observables=observables)
+        observables=observables,
+        cache_buster=cache_buster)
 
     # ---- invoke workflow ---- #
     logger.info(f"starting workflow for {omex_file}")

biosim_server/common/database/data_models.py

@@ -74,7 +74,8 @@ def validate_id(cls, v: str) -> str:
 class BiosimulatorWorkflowRun(BaseModel):
     workflow_id: str
     file_hash_md5: str
-    sim_digest: str
+    image_digest: str
+    cache_buster: str
     omex_file: OmexFile
     simulator_version: BiosimulatorVersion
     biosim_run: BiosimSimulationRun | None = None

biosim_server/common/database/database_service.py

@@ -38,7 +38,8 @@ async def insert_biosimulator_workflow_run(self, input: BiosimulatorWorkflowRun)
         pass
 
     @abstractmethod
-    async def get_biosimulator_workflow_runs(self, file_hash_md5: str, sim_digest: str) -> list[BiosimulatorWorkflowRun]:
+    async def get_biosimulator_workflow_runs(self, file_hash_md5: str, sim_digest: str, cache_buster: str)\
+            -> list[BiosimulatorWorkflowRun]:
         pass
 
     @abstractmethod

biosim_server/common/database/database_service_mongo.py

@@ -91,9 +91,10 @@ async def insert_biosimulator_workflow_run(self, sim_output: BiosimulatorWorkflo
             raise Exception("Insert failed")
 
     @override
-    async def get_biosimulator_workflow_runs(self, file_hash_md5: str, sim_digest: str) -> list[BiosimulatorWorkflowRun]:
-        logger.info(f"Getting OMEX sim workflow output with file hash {file_hash_md5} and sim digest {sim_digest}")
-        query = dict(file_hash_md5=file_hash_md5, sim_digest=sim_digest)
+    async def get_biosimulator_workflow_runs(self, file_hash_md5: str, image_digest: str, cache_buster: str) \
+            -> list[BiosimulatorWorkflowRun]:
+        logger.info(f"Getting OMEX sim workflow output with file hash {file_hash_md5} and sim digest {image_digest} and cache buster {cache_buster}")
+        query = dict(file_hash_md5=file_hash_md5, image_digest=image_digest, cache_buster=cache_buster)
         document = await self._sim_output_col.find(query).to_list(length=100)
         workflow_runs: list[BiosimulatorWorkflowRun] = []
         if document is not None and type(document) is list:

biosim_server/workflows/simulate/biosim_activities.py

@@ -8,7 +8,7 @@
 
 from biosim_server.common.biosim1_client import BiosimService, BiosimServiceRest, HDF5File, Hdf5DataValues
 from biosim_server.common.database.data_models import OmexFile, BiosimSimulationRun, BiosimulatorVersion, \
-     BiosimSimulationRunStatus, BiosimulatorWorkflowRun
+    BiosimSimulationRunStatus, BiosimulatorWorkflowRun
 from biosim_server.common.storage import FileService
 from biosim_server.dependencies import get_file_service, get_biosim_service, get_database_service
 
@@ -75,12 +75,14 @@ async def get_hdf5_metadata(input: GetHdf5MetadataInput) -> HDF5File:
 
 
 @activity.defn
-async def get_biosimulator_workflow_runs_activity(file_hash_md5: str, sim_digest: str) -> list[BiosimulatorWorkflowRun]:
+async def get_biosimulator_workflow_runs_activity(file_hash_md5: str, sim_digest: str,
+                                                  cache_buster: str) -> list[BiosimulatorWorkflowRun]:
     activity.logger.setLevel(logging.INFO)
     activity.logger.info(f"Getting biosimulator workflow runs with file hash {file_hash_md5} and sim digest {sim_digest}")
     database_service = get_database_service()
     assert database_service is not None
-    return await database_service.get_biosimulator_workflow_runs(file_hash_md5=file_hash_md5, sim_digest=sim_digest)
+    return await database_service.get_biosimulator_workflow_runs(file_hash_md5=file_hash_md5, sim_digest=sim_digest,
+                                                                 cache_buster=cache_buster)
 
 
 @activity.defn

biosim_server/workflows/simulate/omex_sim_workflow.py

@@ -17,6 +17,7 @@
 class OmexSimWorkflowInput(BaseModel):
     omex_file: OmexFile
     simulator_version: BiosimulatorVersion
+    cache_buster: str
 
 
 class OmexSimWorkflowStatus(StrEnum):
@@ -100,7 +101,8 @@ async def run(self, sim_input: OmexSimWorkflowInput) -> OmexSimWorkflowOutput:
 
         biosimulator_workflow_run = BiosimulatorWorkflowRun(workflow_id=self.sim_output.workflow_id,
                                                             file_hash_md5=self.sim_input.omex_file.file_hash_md5,
-                                                            sim_digest=self.sim_input.simulator_version.image_digest,
+                                                            image_digest=self.sim_input.simulator_version.image_digest,
+                                                            cache_buster=self.sim_input.cache_buster,
                                                             omex_file=self.sim_input.omex_file,
                                                             simulator_version=self.sim_input.simulator_version,
                                                             biosim_run=biosim_simulation_run,

biosim_server/workflows/simulate/trigger_sim_workflow.py

@@ -18,11 +18,10 @@ async def start_workflow() -> None:
     biosim_service: BiosimService | None = get_biosim_service()
     assert biosim_service is not None
     simulator_version = (await biosim_service.get_simulator_versions())[0]
-    omex_sim_workflow_input = OmexSimWorkflowInput(omex_file=omex_file, simulator_version=simulator_version)
+    omex_sim_workflow_input = OmexSimWorkflowInput(omex_file=omex_file, simulator_version=simulator_version,
+                                                   cache_buster="0")
     handle = await client.start_workflow(
-        OmexSimWorkflow.run,
-        args=[OmexSimWorkflowInput(omex_file=omex_sim_workflow_input.omex_file,
-                                   simulator_version=omex_sim_workflow_input.simulator_version)],
+        OmexSimWorkflow.run, args=[omex_sim_workflow_input],
         task_queue="verification_tasks",
         id=uuid.uuid4().hex,
     )

biosim_server/workflows/verify/omex_verify_workflow.py

@@ -29,6 +29,7 @@ class OmexVerifyWorkflowInput(BaseModel):
     omex_file: OmexFile
     user_description: str
     requested_simulators: list[BiosimulatorVersion]
+    cache_buster: str
     include_outputs: bool
     rel_tol: float
     abs_tol_min: float
@@ -77,7 +78,9 @@ async def run(self, verify_input: OmexVerifyWorkflowInput) -> OmexVerifyWorkflow
         for simulator_spec in verify_input.requested_simulators:
             child_workflows.append(
                 workflow.start_child_workflow(OmexSimWorkflow.run,  # type: ignore
-                                              args=[OmexSimWorkflowInput(omex_file=verify_input.omex_file, simulator_version=simulator_spec)],
+                                              args=[OmexSimWorkflowInput(omex_file=verify_input.omex_file,
+                                                                         simulator_version=simulator_spec,
+                                                                         cache_buster=verify_input.cache_buster)],
                                               result_type=OmexSimWorkflowOutput,
                                               task_queue="verification_tasks", execution_timeout=timedelta(minutes=10), ))
 

biosim_server/workflows/verify/trigger_verify_workflow.py

@@ -27,7 +27,8 @@ async def start_workflow() -> None:
             rel_tol=1e-4,
             abs_tol_min=1e-3,
             abs_tol_scale=1e-5,
-            observables=["time", "concentration"]
+            observables=["time", "concentration"],
+            cache_buster="0"
         )],
         task_queue="verification_tasks",
         id=workflow_id,

tests/common/test_database_service.py

@@ -31,6 +31,7 @@ async def test_omex_file(database_service_mongo: DatabaseServiceMongo) -> None:
 async def test_biosimulator_workflow_run(database_service_mongo: DatabaseServiceMongo,
                                         omex_verify_workflow_output: OmexVerifyWorkflowOutput) -> None:
     workflow_id = "workflow_id"
+    cache_buster = "1234"
     omex_file = OmexFile(file_hash_md5="1234", bucket_name="test_bucket",
                          uploaded_filename="BIOMD0000000010_tellurium_Negative_feedback_and_ultrasen.omex",
                          omex_gcs_path="path/to/omex", file_size=100000)
@@ -40,20 +41,23 @@ async def test_biosimulator_workflow_run(database_service_mongo: DatabaseService
     biosim_run = omex_verify_workflow_output.workflow_results.sims_run_info[0].biosim_sim_run
     hdf5_file = omex_verify_workflow_output.workflow_results.sims_run_info[0].hdf5_file
     sim_output = BiosimulatorWorkflowRun(workflow_id=workflow_id, file_hash_md5=omex_file.file_hash_md5,
-                                         sim_digest=simulator_version.image_digest, omex_file=omex_file,
-                                         simulator_version=simulator_version, biosim_run=biosim_run, hdf5_file=hdf5_file)
+                                         image_digest=simulator_version.image_digest, cache_buster=cache_buster,
+                                         omex_file=omex_file, simulator_version=simulator_version,
+                                         biosim_run=biosim_run, hdf5_file=hdf5_file)
 
     inserted_sim_output1 = await database_service_mongo.insert_biosimulator_workflow_run(sim_output=sim_output)
     assert inserted_sim_output1.database_id is not None
     sim_outputs: list[BiosimulatorWorkflowRun] = await database_service_mongo.get_biosimulator_workflow_runs(
                                                              file_hash_md5=omex_file.file_hash_md5,
-                                                             sim_digest=simulator_version.image_digest)
+                                                             image_digest=simulator_version.image_digest,
+                                                             cache_buster=cache_buster)
     assert sim_outputs == [inserted_sim_output1]
 
     inserted_sim_output2 = await database_service_mongo.insert_biosimulator_workflow_run(sim_output=sim_output)
     assert inserted_sim_output2.database_id is not None
     sim_outputs = await database_service_mongo.get_biosimulator_workflow_runs(file_hash_md5=omex_file.file_hash_md5,
-                                                                              sim_digest=simulator_version.image_digest)
+                                                                              image_digest=simulator_version.image_digest,
+                                                                              cache_buster=cache_buster)
     assert sim_outputs == [inserted_sim_output1, inserted_sim_output2]
 
     await database_service_mongo.delete_biosimulator_workflow_run(database_id=inserted_sim_output1.database_id)

tests/fixtures/local_data/OmexVerifyWorkflowOutput_expected.json

@@ -1,5 +1,5 @@
 {
-  "workflow_id": "d13f4a726d05473ca90d64cf71d6c464",
+  "workflow_id": "3ea78714d33843dbac8e1efc030e71cc",
   "workflow_input": {
     "omex_file": {
       "file_hash_md5": "3b965a2208febec450825569e6089984",
@@ -26,6 +26,7 @@
         "image_digest": "sha256:828b2dc2b983de901c2d68eeb415cb22b46f1db04cdb9e8815d80bf451005216"
       }
     ],
+    "cache_buster": "0",
     "include_outputs": false,
     "rel_tol": 0.0001,
     "abs_tol_min": 0.001,
@@ -36,14 +37,14 @@
     ]
   },
   "workflow_status": "COMPLETED",
-  "timestamp": "2025-01-31 21:21:30.943543+00:00",
+  "timestamp": "2025-01-31 22:26:18.575726+00:00",
   "actual_simulators": null,
-  "workflow_run_id": "deaaf6e6-8d5a-484d-a718-31f4fbbb3766",
+  "workflow_run_id": "bcdb443c-2ecd-4507-a711-1844aa9cefa8",
   "workflow_results": {
     "sims_run_info": [
       {
         "biosim_sim_run": {
-          "id": "679d3edb55ff66742cf1484c",
+          "id": "679d4e0a55ff66742cf14a9e",
           "name": "BIOMD0000000010_tellurium_Negative_feedback_and_ultrasen.omex",
           "simulator_version": {
             "id": "copasi",
@@ -56,8 +57,8 @@
         },
         "hdf5_file": {
           "filename": "reports.h5",
-          "id": "679d3edb55ff66742cf1484c",
-          "uri": "https://storage.googleapis.com/files.biosimulations.org/simulations/679d3edb55ff66742cf1484c/outputs/reports.h5",
+          "id": "679d4e0a55ff66742cf14a9e",
+          "uri": "https://storage.googleapis.com/files.biosimulations.org/simulations/679d4e0a55ff66742cf14a9e/outputs/reports.h5",
           "groups": [
             {
               "name": "BIOMD0000000010_url.sedml",
@@ -489,7 +490,7 @@
       },
       {
         "biosim_sim_run": {
-          "id": "679d3edb902399434ba95632",
+          "id": "679d4e0a55ff66742cf14a9f",
           "name": "BIOMD0000000010_tellurium_Negative_feedback_and_ultrasen.omex",
           "simulator_version": {
             "id": "vcell",
@@ -502,8 +503,8 @@
         },
         "hdf5_file": {
           "filename": "reports.h5",
-          "id": "679d3edb902399434ba95632",
-          "uri": "https://storage.googleapis.com/files.biosimulations.org/simulations/679d3edb902399434ba95632/outputs/reports.h5",
+          "id": "679d4e0a55ff66742cf14a9f",
+          "uri": "https://storage.googleapis.com/files.biosimulations.org/simulations/679d4e0a55ff66742cf14a9f/outputs/reports.h5",
           "groups": [
             {
               "name": "BIOMD0000000010_url.sedml",
@@ -1037,25 +1038,25 @@
             ],
             "score": [
               0.0,
-              0.19520482348612309,
-              0.5733469498777034,
-              0.07456396784090855,
-              0.12396169525663402,
-              0.07828001923749059,
-              0.29933272886715745,
-              0.2004268991480218,
-              0.8911449893341556,
+              0.2109208541519768,
+              0.6160117641611397,
+              0.08073399713312734,
+              0.13396543530030625,
+              0.0844545843285283,
+              0.3207037300083684,
+              0.21449570723816266,
+              0.96109237672241,
               0.0,
-              0.0059296935206195785,
-              0.0011678241047812237,
-              0.01165051415602138,
-              0.012094117814414683,
-              0.010579195522396468,
-              0.009020949943794137,
-              0.08963025772538413,
-              0.09177831504659288,
-              0.0018907352976844827,
-              0.01829085248840209
+              0.006402084786197194,
+              0.0012556976092903502,
+              0.012589597583596512,
+              0.01308331126907003,
+              0.011326124992810022,
+              0.009667698853151248,
+              0.09630682413689742,
+              0.09893510061030918,
+              0.0020428828371769213,
+              0.01972444866994348
             ],
             "is_close": [
               true,
@@ -1111,25 +1112,25 @@
             ],
             "score": [
               0.0,
-              0.19520863405281844,
-              0.5733140790898466,
-              0.07456341186652408,
-              0.12396323192587193,
-              0.07827940646614619,
-              0.2993237691270947,
-              0.20042288213434334,
-              0.8910655824712569,
+              0.21092530300648407,
+              0.6159738194492199,
+              0.08073334534056023,
+              0.1339672299981345,
+              0.08445387107687062,
+              0.3206934452499603,
+              0.21449110649600422,
+              0.9610000157434938,
               0.0,
-              0.005929690004495138,
-              0.0011678239683999257,
-              0.011650527729485206,
-              0.01209413244120094,
-              0.010579184330470517,
-              0.009020941806047688,
-              0.0896294543742746,
-              0.09177747272841226,
-              0.0018907356551725468,
-              0.018290819032934807
+              0.006402080687530857,
+              0.0012556974516127215,
+              0.012589613433413199,
+              0.0130833283863958,
+              0.011326112164713817,
+              0.009667689506720173,
+              0.09630589664539227,
+              0.09893412180457972,
+              0.0020428832545140354,
+              0.01972440976463269
             ],
             "is_close": [
               true,
@@ -1411,8 +1412,8 @@
             ],
             "score": [
               1.2053311887363872e-12,
-              0.19520482348612309,
-              0.5733469498777034
+              0.2109208541519768,
+              0.6160117641611397
             ],
             "is_close": [
               true,
@@ -1434,8 +1435,8 @@
             ],
             "score": [
               1.205331188736387e-12,
-              0.19520863405281844,
-              0.5733140790898466
+              0.21092530300648407,
+              0.6159738194492199
             ],
             "is_close": [
               true,
@@ -1501,8 +1502,8 @@
             ],
             "score": [
               1.1516089720585092e-12,
-              0.1449101929048558,
-              0.1579643595189614
+              0.14491019189531268,
+              0.15796435879920528
             ],
             "is_close": [
               true,
@@ -1524,8 +1525,8 @@
             ],
             "score": [
               1.1516089720585092e-12,
-              0.14491229283168663,
-              0.15796186428448936
+              0.1449122918221143,
+              0.157961863564756
             ],
             "is_close": [
               true,

tests/fixtures/workflow_fixtures.py

@@ -59,7 +59,7 @@ def omex_verify_workflow_input(simulator_version_copasi: BiosimulatorVersion,
     omex_input = OmexVerifyWorkflowInput(omex_file=omex_file, user_description="description",
                                          requested_simulators=simulators, include_outputs=include_outputs,
                                          rel_tol=rel_tol, abs_tol_min=abs_tol_min, abs_tol_scale=abs_tol_scale,
-                                         observables=observables)
+                                         observables=observables, cache_buster="0")
     return omex_input
 
 

tests/workflows/test_sim_workflow.py

@@ -29,7 +29,7 @@ async def test_sim_workflow(temporal_client: Client, temporal_verify_worker: Wor
             simulator_version = sim
     assert simulator_version is not None
 
-    sim_workflow_input = OmexSimWorkflowInput(omex_file=omex_file, simulator_version=simulator_version)
+    sim_workflow_input = OmexSimWorkflowInput(omex_file=omex_file, simulator_version=simulator_version, cache_buster="0")
     workflow_handle = await temporal_client.start_workflow(OmexSimWorkflow.run, args=[sim_workflow_input],
         id=uuid.uuid4().hex, task_queue="verification_tasks", )
     assert isinstance(workflow_handle, WorkflowHandle)