reorganized packages and simplified imports

biosim_server/api/main.py

@@ -6,8 +6,8 @@
 
 from temporalio import workflow
 
-from biosim_server.omex_sim.biosim1.models import BiosimSimulatorSpec, SourceOmex
-from biosim_server.verify.workflows.runs_verify_workflow import RunsVerifyWorkflowOutput, RunsVerifyWorkflowInput, \
+from biosim_server.common.biosim1_client import BiosimSimulatorSpec, SourceOmex
+from biosim_server.workflows.verify.runs_verify_workflow import RunsVerifyWorkflowOutput, RunsVerifyWorkflowInput, \
     RunsVerifyWorkflow, RunsVerifyWorkflowStatus
 
 with workflow.unsafe.imports_passed_through():
@@ -22,7 +22,7 @@
 from biosim_server.dependencies import get_file_service, get_temporal_client, \
     init_standalone, shutdown_standalone
 from biosim_server.log_config import setup_logging
-from biosim_server.verify.workflows.omex_verify_workflow import OmexVerifyWorkflow, OmexVerifyWorkflowInput, \
+from biosim_server.workflows.verify.omex_verify_workflow import OmexVerifyWorkflow, OmexVerifyWorkflowInput, \
     OmexVerifyWorkflowOutput, OmexVerifyWorkflowStatus
 
 logger = logging.getLogger(__name__)

biosim_server/common/biosim1_client/__init__.py

@@ -0,0 +1,16 @@
+from biosim_server.common.biosim1_client.biosim_service import BiosimService
+from biosim_server.common.biosim1_client.biosim_service_rest import BiosimServiceRest
+from biosim_server.common.biosim1_client.models import SourceOmex, BiosimSimulatorSpec, BiosimSimulationRun, \
+    HDF5File, Hdf5DataValues, BiosimSimulationRunStatus, BiosimSimulationRunApiRequest
+
+__all__ = [
+    "BiosimService",
+    "BiosimServiceRest",
+    "SourceOmex",
+    "BiosimSimulatorSpec",
+    "BiosimSimulationRun",
+    "HDF5File",
+    "Hdf5DataValues",
+    "BiosimSimulationRunStatus",
+    "BiosimSimulationRunApiRequest",
+]

biosim_server/omex_sim/biosim1/biosim_service.py → biosim_server/common/biosim1_client/biosim_service.py

@@ -1,7 +1,7 @@
 import logging
 from abc import ABC, abstractmethod
 
-from biosim_server.omex_sim.biosim1.models import BiosimSimulationRun, HDF5File, Hdf5DataValues, BiosimSimulationRunStatus, BiosimSimulatorSpec
+from biosim_server.common.biosim1_client.models import BiosimSimulationRun, HDF5File, Hdf5DataValues, BiosimSimulatorSpec
 
 logger = logging.getLogger(__name__)
 

biosim_server/omex_sim/biosim1/biosim_service_rest.py → biosim_server/common/biosim1_client/biosim_service_rest.py

@@ -8,8 +8,8 @@
 from aiohttp import FormData
 from typing_extensions import override
 
-from biosim_server.omex_sim.biosim1.biosim_service import BiosimService
-from biosim_server.omex_sim.biosim1.models import BiosimSimulationRun, BiosimSimulationRunApiRequest, HDF5File, \
+from biosim_server.common.biosim1_client.biosim_service import BiosimService
+from biosim_server.common.biosim1_client.models import BiosimSimulationRun, BiosimSimulationRunApiRequest, HDF5File, \
     Hdf5DataValues, BiosimSimulationRunStatus, BiosimSimulatorSpec
 
 logger = logging.getLogger(__name__)

biosim_server/common/storage/__init__.py

@@ -0,0 +1,17 @@
+from biosim_server.common.storage.file_service import FileService, ListingItem
+from biosim_server.common.storage.file_service_S3 import FileServiceS3
+from biosim_server.common.storage.file_service_local import FileServiceLocal
+from biosim_server.common.storage.s3_aiobotocore import get_listing_of_s3_path, download_s3_file, upload_bytes_to_s3, \
+    get_s3_modified_date, get_s3_file_contents
+
+__all__ = [
+    "FileService",
+    "ListingItem",
+    "FileServiceS3",
+    "FileServiceLocal",
+    "get_listing_of_s3_path",
+    "download_s3_file",
+    "upload_bytes_to_s3",
+    "get_s3_modified_date",
+    "get_s3_file_contents"
+]

biosim_server/io/file_service_S3.py → biosim_server/common/storage/file_service_S3.py

@@ -8,8 +8,8 @@
 
 from typing_extensions import override
 
-from biosim_server.io.file_service import FileService, ListingItem
-from biosim_server.io.s3_aiobotocore import (
+from biosim_server.common.storage.file_service import FileService, ListingItem
+from biosim_server.common.storage.s3_aiobotocore import (
     download_s3_file,
     get_s3_modified_date,
     get_listing_of_s3_path,

biosim_server/io/file_service_local.py → biosim_server/common/storage/file_service_local.py

@@ -10,7 +10,7 @@
 import aiofiles
 from typing_extensions import override
 
-from biosim_server.io.file_service import FileService, ListingItem
+from biosim_server.common.storage.file_service import FileService, ListingItem
 
 
 def generate_fake_etag(file_path: Path) -> str:

biosim_server/io/s3_aiobotocore.py → biosim_server/common/storage/s3_aiobotocore.py

@@ -10,7 +10,7 @@
 from botocore.exceptions import ClientError
 
 from biosim_server.config import get_settings
-from biosim_server.io.file_service import ListingItem
+from biosim_server.common.storage.file_service import ListingItem
 
 logger = logging.getLogger(__name__)
 

biosim_server/common/temporal/__init__.py

@@ -0,0 +1,5 @@
+from biosim_server.common.temporal.converter import pydantic_data_converter
+
+__all__ = [
+    "pydantic_data_converter"
+]

biosim_server/dependencies.py

@@ -1,9 +1,7 @@
 from temporalio.client import Client as TemporalClient
 
-from biosim_server.io.file_service import FileService
-from biosim_server.io.file_service_S3 import FileServiceS3
-from biosim_server.omex_sim.biosim1.biosim_service import BiosimService
-from biosim_server.omex_sim.biosim1.biosim_service_rest import BiosimServiceRest
+from biosim_server.common.biosim1_client import BiosimService, BiosimServiceRest
+from biosim_server.common.storage import FileService, FileServiceS3
 from biosim_server.config import get_settings
 
 #------ file service (standalone or pytest) ------

biosim_server/worker/worker_main.py

@@ -5,12 +5,12 @@
 from temporalio.worker import Worker, UnsandboxedWorkflowRunner
 
 from biosim_server.dependencies import get_temporal_client, init_standalone
-from biosim_server.omex_sim.workflows.biosim_activities import get_hdf5_metadata, get_hdf5_data
-from biosim_server.omex_sim.workflows.biosim_activities import get_sim_run, submit_biosim_sim
-from biosim_server.omex_sim.workflows.omex_sim_workflow import OmexSimWorkflow
-from biosim_server.verify.workflows.activities import generate_statistics
-from biosim_server.verify.workflows.omex_verify_workflow import OmexVerifyWorkflow
-from biosim_server.verify.workflows.runs_verify_workflow import RunsVerifyWorkflow
+from biosim_server.workflows.simulate.biosim_activities import get_hdf5_metadata, get_hdf5_data
+from biosim_server.workflows.simulate.biosim_activities import get_sim_run, submit_biosim_sim
+from biosim_server.workflows.simulate.omex_sim_workflow import OmexSimWorkflow
+from biosim_server.workflows.verify.activities import generate_statistics
+from biosim_server.workflows.verify.omex_verify_workflow import OmexVerifyWorkflow
+from biosim_server.workflows.verify.runs_verify_workflow import RunsVerifyWorkflow
 
 interrupt_event = asyncio.Event()
 

biosim_server/workflows/simulate/__init__.py

@@ -0,0 +1,20 @@
+from biosim_server.workflows.simulate.biosim_activities import submit_biosim_sim, get_sim_run, get_hdf5_data, \
+    get_hdf5_metadata, SubmitBiosimSimInput, GetSimRunInput, GetHdf5MetadataInput, GetHdf5DataInput
+from biosim_server.workflows.simulate.omex_sim_workflow import OmexSimWorkflow, OmexSimWorkflowInput, \
+    OmexSimWorkflowStatus, OmexSimWorkflowOutput
+
+
+__all__ = [
+    "submit_biosim_sim",
+    "get_sim_run",
+    "get_hdf5_data",
+    "get_hdf5_metadata",
+    "SubmitBiosimSimInput",
+    "GetSimRunInput",
+    "GetHdf5MetadataInput",
+    "GetHdf5DataInput",
+    "OmexSimWorkflow",
+    "OmexSimWorkflowInput",
+    "OmexSimWorkflowStatus",
+    "OmexSimWorkflowOutput",
+]

biosim_server/omex_sim/workflows/biosim_activities.py → biosim_server/workflows/simulate/biosim_activities.py

@@ -4,12 +4,10 @@
 from pydantic import BaseModel
 from temporalio import activity
 
+from biosim_server.common.biosim1_client import BiosimService, BiosimServiceRest, SourceOmex, BiosimSimulatorSpec, \
+    BiosimSimulationRun, HDF5File, Hdf5DataValues
+from biosim_server.common.storage import FileService
 from biosim_server.dependencies import get_file_service, get_biosim_service
-from biosim_server.io.file_service import FileService
-from biosim_server.omex_sim.biosim1.biosim_service import BiosimService
-from biosim_server.omex_sim.biosim1.biosim_service_rest import BiosimServiceRest
-from biosim_server.omex_sim.biosim1.models import SourceOmex, BiosimSimulatorSpec, BiosimSimulationRun, \
-    HDF5File, Hdf5DataValues
 
 
 class GetSimRunInput(BaseModel):

biosim_server/omex_sim/workflows/omex_sim_workflow.py → biosim_server/workflows/simulate/omex_sim_workflow.py

@@ -6,11 +6,10 @@
 from temporalio import workflow
 from temporalio.common import RetryPolicy
 
-from biosim_server.omex_sim.biosim1.models import BiosimSimulationRun, BiosimSimulationRunStatus, HDF5File, \
+from biosim_server.common.biosim1_client import BiosimSimulationRun, BiosimSimulationRunStatus, HDF5File, \
     SourceOmex, BiosimSimulatorSpec
-from biosim_server.omex_sim.workflows.biosim_activities import get_hdf5_metadata
-from biosim_server.omex_sim.workflows.biosim_activities import get_sim_run, submit_biosim_sim, \
-    SubmitBiosimSimInput, GetSimRunInput, GetHdf5MetadataInput
+from biosim_server.workflows.simulate import get_hdf5_metadata, get_sim_run, submit_biosim_sim, SubmitBiosimSimInput, \
+    GetSimRunInput, GetHdf5MetadataInput
 
 
 class OmexSimWorkflowInput(BaseModel):

biosim_server/omex_sim/workflows/trigger_sim_workflow.py → biosim_server/workflows/simulate/trigger_sim_workflow.py

@@ -3,10 +3,9 @@
 
 from temporalio.client import Client
 
-from biosim_server.omex_sim.biosim1.models import SourceOmex, BiosimSimulatorSpec
-from biosim_server.omex_sim.workflows.omex_sim_workflow import OmexSimWorkflow, OmexSimWorkflowInput, \
-    OmexSimWorkflowOutput
-from biosim_server.temporal_utils.converter import pydantic_data_converter
+from biosim_server.common.biosim1_client import SourceOmex, BiosimSimulatorSpec
+from biosim_server.common.temporal import pydantic_data_converter
+from biosim_server.workflows.simulate import OmexSimWorkflow, OmexSimWorkflowInput, OmexSimWorkflowOutput
 
 
 async def start_workflow() -> None:

biosim_server/workflows/verify/__init__.py

@@ -0,0 +1,25 @@
+from biosim_server.workflows.verify.activities import SimulationRunInfo, GenerateStatisticsInput, \
+    GenerateStatisticsOutput, DatasetVar, RunData, ComparisonStatistics, generate_statistics
+from biosim_server.workflows.verify.omex_verify_workflow import OmexVerifyWorkflow, OmexVerifyWorkflowInput, \
+    OmexVerifyWorkflowStatus, OmexVerifyWorkflowOutput
+from biosim_server.workflows.verify.runs_verify_workflow import RunsVerifyWorkflow, RunsVerifyWorkflowInput, \
+    RunsVerifyWorkflowStatus, RunsVerifyWorkflowOutput
+
+
+__all__ = [
+    "SimulationRunInfo",
+    "GenerateStatisticsInput",
+    "GenerateStatisticsOutput",
+    "DatasetVar",
+    "RunData",
+    "ComparisonStatistics",
+    "generate_statistics",
+    "OmexVerifyWorkflow",
+    "OmexVerifyWorkflowInput",
+    "OmexVerifyWorkflowStatus",
+    "OmexVerifyWorkflowOutput",
+    "RunsVerifyWorkflow",
+    "RunsVerifyWorkflowInput",
+    "RunsVerifyWorkflowStatus",
+    "RunsVerifyWorkflowOutput"
+]

biosim_server/verify/workflows/activities.py → biosim_server/workflows/verify/activities.py

@@ -5,9 +5,8 @@
 from pydantic import BaseModel
 from temporalio import activity
 
-from biosim_server.omex_sim.biosim1.models import BiosimSimulationRun
-from biosim_server.omex_sim.biosim1.models import Hdf5DataValues, HDF5File
-from biosim_server.omex_sim.workflows.biosim_activities import get_hdf5_data, GetHdf5DataInput
+from biosim_server.common.biosim1_client import BiosimSimulationRun, Hdf5DataValues, HDF5File
+from biosim_server.workflows.simulate import get_hdf5_data, GetHdf5DataInput
 
 
 class SimulationRunInfo(BaseModel):

biosim_server/verify/workflows/omex_verify_workflow.py → biosim_server/workflows/verify/omex_verify_workflow.py

@@ -9,11 +9,10 @@
 from temporalio.common import RetryPolicy
 from temporalio.workflow import ChildWorkflowHandle
 
-from biosim_server.omex_sim.biosim1.models import BiosimSimulatorSpec, SourceOmex
-from biosim_server.omex_sim.workflows.omex_sim_workflow import OmexSimWorkflow, OmexSimWorkflowInput, \
-    OmexSimWorkflowOutput
-from biosim_server.verify.workflows.activities import generate_statistics, GenerateStatisticsInput, \
-    GenerateStatisticsOutput, SimulationRunInfo
+from biosim_server.common.biosim1_client import BiosimSimulatorSpec, SourceOmex
+from biosim_server.workflows.simulate import OmexSimWorkflow, OmexSimWorkflowInput, OmexSimWorkflowOutput
+from biosim_server.workflows.verify import generate_statistics, GenerateStatisticsInput, GenerateStatisticsOutput, \
+    SimulationRunInfo
 
 
 class OmexVerifyWorkflowStatus(StrEnum):

biosim_server/verify/workflows/runs_verify_workflow.py → biosim_server/workflows/verify/runs_verify_workflow.py

@@ -7,12 +7,10 @@
 from temporalio import workflow
 from temporalio.common import RetryPolicy
 
-from biosim_server.omex_sim.biosim1.models import BiosimSimulationRun
-from biosim_server.omex_sim.biosim1.models import BiosimSimulatorSpec, HDF5File
-from biosim_server.omex_sim.workflows.biosim_activities import GetSimRunInput, get_hdf5_metadata, GetHdf5MetadataInput
-from biosim_server.omex_sim.workflows.biosim_activities import get_sim_run
-from biosim_server.verify.workflows.activities import generate_statistics, GenerateStatisticsOutput, \
-    GenerateStatisticsInput, SimulationRunInfo
+from biosim_server.common.biosim1_client import BiosimSimulationRun, BiosimSimulatorSpec, HDF5File
+from biosim_server.workflows.simulate import GetSimRunInput, get_hdf5_metadata, GetHdf5MetadataInput, get_sim_run
+from biosim_server.workflows.verify import generate_statistics, GenerateStatisticsOutput, GenerateStatisticsInput, \
+    SimulationRunInfo
 
 
 class RunsVerifyWorkflowStatus(StrEnum):

biosim_server/verify/workflows/trigger_verify_workflow.py → biosim_server/workflows/verify/trigger_verify_workflow.py

@@ -3,9 +3,9 @@
 
 from temporalio.client import Client
 
-from biosim_server.omex_sim.biosim1.models import SourceOmex, BiosimSimulatorSpec
-from biosim_server.verify.workflows.omex_verify_workflow import OmexVerifyWorkflow, OmexVerifyWorkflowInput
-from biosim_server.temporal_utils.converter import pydantic_data_converter
+from biosim_server.common.biosim1_client import SourceOmex, BiosimSimulatorSpec
+from biosim_server.common.temporal import pydantic_data_converter
+from biosim_server.workflows.verify import OmexVerifyWorkflow, OmexVerifyWorkflowInput
 
 
 async def start_workflow() -> None:

tests/api/test_main.py

@@ -9,14 +9,11 @@
 from temporalio.worker import Worker
 
 from biosim_server.api.main import app
+from biosim_server.common.biosim1_client import BiosimServiceRest
+from biosim_server.common.storage import FileServiceS3, FileServiceLocal
 from biosim_server.config import get_settings
-from biosim_server.io.file_service_S3 import FileServiceS3
-from biosim_server.io.file_service_local import FileServiceLocal
-from biosim_server.omex_sim.biosim1.biosim_service_rest import BiosimServiceRest
-from biosim_server.verify.workflows.omex_verify_workflow import OmexVerifyWorkflowInput, OmexVerifyWorkflowOutput, \
-    OmexVerifyWorkflowStatus
-from biosim_server.verify.workflows.runs_verify_workflow import RunsVerifyWorkflowInput, RunsVerifyWorkflowOutput, \
-    RunsVerifyWorkflowStatus
+from biosim_server.workflows.verify import OmexVerifyWorkflowInput, OmexVerifyWorkflowOutput, OmexVerifyWorkflowStatus, \
+    RunsVerifyWorkflowInput, RunsVerifyWorkflowOutput, RunsVerifyWorkflowStatus
 from tests.workflows.test_verify_workflows import assert_omex_verify_results, assert_runs_verify_results
 
 
@@ -25,7 +22,7 @@ async def test_root() -> None:
     async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as test_client:
         response = await test_client.get("/")
         assert response.status_code == 200
-        assert response.json() == {'docs': 'https://biochecknet.biosimulations.org/docs'}
+        assert response.json() == {'docs': 'https://biosim.biosimulations.org/docs'}
 
 
 @pytest.mark.asyncio

tests/biosim1/test_hdf5_json.py

@@ -9,7 +9,7 @@
 import os
 from pathlib import Path
 
-from biosim_server.omex_sim.biosim1.models import HDF5File
+from biosim_server.common.biosim1_client import HDF5File
 
 
 def test_hdf5_json() -> None:

tests/database/test_mongo.py

@@ -3,7 +3,7 @@
 from pymongo.results import InsertOneResult
 from testcontainers.mongodb import MongoDbContainer  # type: ignore
 
-from biosim_server.verify.workflows.omex_verify_workflow import OmexVerifyWorkflowOutput
+from biosim_server.workflows.verify import OmexVerifyWorkflowOutput
 from tests.fixtures.database_fixtures import mongo_test_collection
 
 

tests/fixtures/biosim_fixtures.py

@@ -3,7 +3,7 @@
 import pytest_asyncio
 
 from biosim_server.dependencies import get_biosim_service, set_biosim_service
-from biosim_server.omex_sim.biosim1.biosim_service_rest import BiosimServiceRest
+from biosim_server.common.biosim1_client import BiosimServiceRest
 from tests.fixtures.biosim_service_mock import BiosimServiceMock
 
 

tests/fixtures/biosim_service_mock.py

@@ -2,8 +2,7 @@
 
 from typing_extensions import override
 
-from biosim_server.omex_sim.biosim1.biosim_service import BiosimService
-from biosim_server.omex_sim.biosim1.models import Hdf5DataValues, BiosimSimulationRun, HDF5File, \
+from biosim_server.common.biosim1_client import BiosimService, Hdf5DataValues, BiosimSimulationRun, HDF5File, \
     BiosimSimulationRunStatus, BiosimSimulatorSpec
 
 

tests/fixtures/s3_fixtures.py

@@ -4,9 +4,9 @@
 import pytest_asyncio
 from testcontainers.mongodb import MongoDbContainer  # type: ignore
 
+from biosim_server.common.storage import FileServiceLocal
+from biosim_server.common.storage import FileServiceS3
 from biosim_server.dependencies import get_file_service, set_file_service
-from biosim_server.io.file_service_S3 import FileServiceS3
-from biosim_server.io.file_service_local import FileServiceLocal
 
 
 @pytest_asyncio.fixture(scope="function")

tests/fixtures/temporal_fixtures.py

@@ -6,14 +6,11 @@
 from temporalio.testing import WorkflowEnvironment
 from temporalio.worker import Worker, UnsandboxedWorkflowRunner
 
+from biosim_server.common.temporal import pydantic_data_converter
 from biosim_server.dependencies import get_temporal_client, set_temporal_client
-from biosim_server.omex_sim.workflows.biosim_activities import get_sim_run, submit_biosim_sim, get_hdf5_metadata, \
-    get_hdf5_data
-from biosim_server.omex_sim.workflows.omex_sim_workflow import OmexSimWorkflow
-from biosim_server.temporal_utils.converter import pydantic_data_converter
-from biosim_server.verify.workflows.activities import generate_statistics
-from biosim_server.verify.workflows.omex_verify_workflow import OmexVerifyWorkflow
-from biosim_server.verify.workflows.runs_verify_workflow import RunsVerifyWorkflow
+from biosim_server.workflows.simulate import get_sim_run, submit_biosim_sim, get_hdf5_metadata, get_hdf5_data, \
+    OmexSimWorkflow
+from biosim_server.workflows.verify import OmexVerifyWorkflow, RunsVerifyWorkflow, generate_statistics
 
 
 @pytest_asyncio.fixture(scope="session")