Mix fixes, will split

lib/galaxy/managers/jobs.py

@@ -1245,6 +1245,6 @@ def get_jobs_to_check_at_startup(session: galaxy_scoped_session, track_jobs_in_d
     return session.scalars(stmt).all()
 
 
-def get_job(session, *where_clauses):
+def get_job(session: galaxy_scoped_session, *where_clauses):
     stmt = select(Job).where(*where_clauses).limit(1)
     return session.scalars(stmt).first()

lib/galaxy/managers/tools.py

@@ -1,5 +1,7 @@
 import logging
 from typing import (
+    Any,
+    Dict,
     Optional,
     TYPE_CHECKING,
     Union,
@@ -47,7 +49,7 @@ def get_tool_by_id(self, object_id):
         stmt = select(DynamicTool).where(DynamicTool.id == object_id)
         return self.session().scalars(stmt).one_or_none()
 
-    def create_tool(self, trans, tool_payload, allow_load=True):
+    def create_tool(self, trans, tool_payload: Dict[str, Any], allow_load: bool = True):
         if not getattr(self.app.config, "enable_beta_tool_formats", False):
             raise exceptions.ConfigDoesNotAllowException(
                 "Set 'enable_beta_tool_formats' in Galaxy config to create dynamic tools."
@@ -69,18 +71,25 @@ def create_tool(self, trans, tool_payload, allow_load=True):
             src = tool_payload.get("src", "representation")
             is_path = src == "from_path"
             target_object = None
+            proxy = None
 
             if is_path:
                 tool_format, representation, _, target_object = artifact_class(None, tool_payload)
-            else:
-                assert src == "representation"
+            elif src == "representation":
                 representation = tool_payload.get("representation")
                 if not representation:
                     raise exceptions.ObjectAttributeMissingException("A tool 'representation' is required.")
 
                 tool_format = representation.get("class")
                 if not tool_format:
                     raise exceptions.ObjectAttributeMissingException("Current tool representations require 'class'.")
+            elif src == "proxy":
+                proxy = tool_payload.get("proxy")
+                if not proxy:
+                    raise exceptions.ObjectAttributeMissingException("A tool 'proxy' is required if src='proxy'.")
+                tool_format = proxy._class
+            else:
+                raise exceptions.ObjectAttributeInvalidException(f"Invalid 'src': {src}")
 
             # Set tool_path to None so that in ToolBox.create_dynamic_tool()
             # the tool source is by default recovered using
@@ -93,7 +102,9 @@ def create_tool(self, trans, tool_payload, allow_load=True):
                     tool_id = str(uuid)
             elif tool_format in ("CommandLineTool", "ExpressionTool"):
                 # CWL tools
-                if target_object is not None:
+                if proxy:
+                    representation = proxy.to_persistent_representation()
+                elif target_object is not None:
                     representation = {"raw_process_reference": target_object, "uuid": str(uuid), "class": tool_format}
                     proxy = tool_proxy(tool_object=target_object, tool_directory=tool_directory, uuid=uuid)
                 elif is_path:
@@ -121,6 +132,7 @@ def create_tool(self, trans, tool_payload, allow_load=True):
                 active=tool_payload.get("active"),
                 hidden=tool_payload.get("hidden"),
                 value=representation,
+                proxy=proxy,
             )
         self.app.toolbox.load_dynamic_tool(dynamic_tool)
         return dynamic_tool

lib/galaxy/managers/workflows.py

@@ -673,12 +673,12 @@ def normalize_workflow_format(self, trans, as_dict):
             tool_reference_proxies = wf_proxy.tool_reference_proxies()
             for tool_reference_proxy in tool_reference_proxies:
                 # TODO: Namespace IDS in workflows.
-                representation = tool_reference_proxy.to_persistent_representation()
                 self.app.dynamic_tool_manager.create_tool(
                     trans,
                     {
                         "uuid": tool_reference_proxy.uuid,
-                        "representation": representation,
+                        "src": "proxy",
+                        "proxy": tool_reference_proxy,
                     },
                     allow_load=True,
                 )

lib/galaxy/model/__init__.py

@@ -229,6 +229,7 @@
         ObjectStorePopulator,
     )
     from galaxy.schema.invocation import InvocationMessageUnion
+    from galaxy.tool_util.cwl.parser import ToolProxy
 
 log = logging.getLogger(__name__)
 
@@ -1364,17 +1365,24 @@ class DynamicTool(Base, Dictifiable, RepresentById):
     tool_directory: Mapped[Optional[str]] = mapped_column(Unicode(255))
     hidden: Mapped[Optional[bool]] = mapped_column(default=True)
     active: Mapped[Optional[bool]] = mapped_column(default=True)
-    value: Mapped[Optional[bytes]] = mapped_column(MutableJSONType)
+    value: Mapped[Optional[Dict[str, Any]]] = mapped_column(MutableJSONType)
 
     dict_collection_visible_keys = ("id", "tool_id", "tool_format", "tool_version", "uuid", "active", "hidden")
     dict_element_visible_keys = ("id", "tool_id", "tool_format", "tool_version", "uuid", "active", "hidden")
 
-    def __init__(self, active=True, hidden=True, **kwd):
+    def __init__(
+        self,
+        uuid: Optional[Union[UUID, str]] = None,
+        proxy: Optional["ToolProxy"] = None,
+        **kwd,
+    ):
         super().__init__(**kwd)
-        self.active = active
-        self.hidden = hidden
-        _uuid = kwd.get("uuid")
-        self.uuid = get_uuid(_uuid)
+        self.uuid = get_uuid(uuid)
+        self.proxy = proxy
+
+    @reconstructor
+    def init_on_load(self):
+        self.proxy = None
 
 
 class BaseJobMetric(Base):

lib/galaxy/tool_util/cwl/parser.py

@@ -13,6 +13,7 @@
     abstractmethod,
 )
 from typing import (
+    Any,
     Dict,
     List,
     Optional,
@@ -188,7 +189,9 @@ def to_persistent_representation(self):
         }
 
     @staticmethod
-    def from_persistent_representation(as_object, strict_cwl_validation=True, tool_directory=None) -> "ToolProxy":
+    def from_persistent_representation(
+        as_object: Dict[str, Any], strict_cwl_validation: bool = True, tool_directory=None
+    ) -> "ToolProxy":
         """Recover an object serialized with to_persistent_representation."""
         if "class" not in as_object:
             raise Exception("Failed to deserialize tool proxy from JSON object - no class found.")
@@ -300,7 +303,9 @@ class ExpressionToolProxy(CommandLineToolProxy):
 
 
 class JobProxy:
-    def __init__(self, tool_proxy, input_dict, output_dict, job_directory):
+    _is_command_line_job: bool
+
+    def __init__(self, tool_proxy: ToolProxy, input_dict, output_dict, job_directory):
         assert RuntimeContext is not None, "cwltool is not installed, cannot run CWL jobs"
         self._tool_proxy = tool_proxy
         self._input_dict = input_dict
@@ -310,7 +315,6 @@ def __init__(self, tool_proxy, input_dict, output_dict, job_directory):
         self._final_output = None
         self._ok = True
         self._cwl_job = None
-        self._is_command_line_job = None
 
         self._normalize_job()
 
@@ -321,7 +325,6 @@ def cwl_job(self):
     @property
     def is_command_line_job(self):
         self._ensure_cwl_job_initialized()
-        assert self._is_command_line_job is not None
         return self._is_command_line_job
 
     def _ensure_cwl_job_initialized(self):
@@ -480,7 +483,7 @@ def collect_outputs(self, tool_working_directory, rcode):
             return self.cwl_job().collect_outputs(tool_working_directory, rcode)
 
     def save_job(self):
-        job_file = JobProxy._job_file(self._job_directory)
+        job_file = self._job_file(self._job_directory)
         job_objects = {
             # "tool_path": os.path.abspath(self._tool_proxy._tool_path),
             "tool_representation": self._tool_proxy.to_persistent_representation(),
@@ -733,7 +736,7 @@ def cwl_object_to_annotation(self, cwl_obj):
 
 
 def tool_proxy(
-    tool_path=None, tool_object=None, strict_cwl_validation=True, tool_directory=None, uuid=None
+    tool_path=None, tool_object=None, strict_cwl_validation: bool = True, tool_directory=None, uuid=None
 ) -> ToolProxy:
     """Provide a proxy object to cwltool data structures to just
     grab relevant data.
@@ -749,7 +752,7 @@ def tool_proxy(
 
 
 def tool_proxy_from_persistent_representation(
-    persisted_tool, strict_cwl_validation=True, tool_directory=None
+    persisted_tool, strict_cwl_validation: bool = True, tool_directory=None
 ) -> ToolProxy:
     """Load a ToolProxy from a previously persisted representation."""
     ensure_cwltool_available()
@@ -781,8 +784,8 @@ def _to_cwl_tool_object(
     tool_object=None,
     cwl_tool_object=None,
     raw_process_reference=None,
-    strict_cwl_validation=False,
-    tool_directory=None,
+    strict_cwl_validation: bool = False,
+    tool_directory: Optional[str] = None,
     uuid=None,
 ) -> ToolProxy:
     if uuid is None:

lib/galaxy/tool_util/toolbox/base.py

@@ -12,6 +12,7 @@
     List,
     Optional,
     Tuple,
+    TYPE_CHECKING,
     Union,
 )
 from urllib.parse import urlparse
@@ -58,6 +59,9 @@
 )
 from .views.static import StaticToolPanelView
 
+if TYPE_CHECKING:
+    from galaxy.model import DynamicTool
+
 log = logging.getLogger(__name__)
 
 SHED_TOOL_CONF_XML = """<?xml version="1.0"?>
@@ -252,7 +256,7 @@ def _default_panel_view(self, trans):
     def create_tool(self, config_file, tool_cache_data_dir=None, **kwds):
         raise NotImplementedError()
 
-    def create_dynamic_tool(self, dynamic_tool):
+    def create_dynamic_tool(self, dynamic_tool: "DynamicTool"):
         raise NotImplementedError()
 
     def can_load_config_file(self, config_filename):
@@ -392,7 +396,7 @@ def panel_has_tool(self, tool, panel_view_id):
         panel_view_rendered = self._tool_panel_view_rendered[panel_view_id]
         return panel_view_rendered.has_item_recursive(tool)
 
-    def load_dynamic_tool(self, dynamic_tool):
+    def load_dynamic_tool(self, dynamic_tool: "DynamicTool"):
         if not dynamic_tool.active:
             return None
 

lib/galaxy/tools/__init__.py

@@ -81,6 +81,7 @@
     RequiredFiles,
     ToolOutputCollectionPart,
 )
+from galaxy.tool_util.parser.cwl import CwlToolSource
 from galaxy.tool_util.parser.interface import (
     HelpContent,
     InputSource,
@@ -389,6 +390,7 @@ def create_tool_from_source(app, tool_source: ToolSource, config_file: Optional[
         module, cls = tool_module
         mod = __import__(module, globals(), locals(), [cls])
         ToolClass = getattr(mod, cls)
+        assert issubclass(ToolClass, Tool)
     elif tool_type := tool_source.parse_tool_type():
         ToolClass = tool_types.get(tool_type)
         if not ToolClass:
@@ -604,36 +606,41 @@ def get_expanded_tool_source(self, config_file, **kwargs):
             global_tool_errors.add_error(config_file, "Tool XML parsing", e)
             raise e
 
-    def _create_tool_from_source(self, tool_source, **kwds):
+    def _create_tool_from_source(self, tool_source: ToolSource, **kwds):
         return create_tool_from_source(self.app, tool_source, **kwds)
 
-    def create_dynamic_tool(self, dynamic_tool, **kwds):
-        tool_format = dynamic_tool.tool_format
-        tool_representation = dynamic_tool.value
-        if "name" not in tool_representation:
-            tool_representation["name"] = f"dynamic tool {dynamic_tool.uuid}"
+    def create_dynamic_tool(self, dynamic_tool):
         strict_cwl_validation = getattr(self.app.config, "strict_cwl_validation", True)
-        get_source_kwds = dict(
-            tool_format=tool_format,
-            tool_representation=tool_representation,
-            strict_cwl_validation=strict_cwl_validation,
-            uuid=dynamic_tool.uuid,
-        )
-        if dynamic_tool.tool_directory:
-            get_source_kwds["tool_directory"] = dynamic_tool.tool_directory
-        if dynamic_tool.tool_path:
-            config_file = dynamic_tool.tool_path
-            # TODO: uuid probably needed here...
-            tool_source = get_tool_source(
-                config_file,
-                enable_beta_formats=getattr(self.app.config, "enable_beta_tool_formats", True),
-                tool_location_fetcher=self.tool_location_fetcher,
-                strict_cwl_validation=strict_cwl_validation,
+        if dynamic_tool.proxy:
+            # CWL tool
+            tool_source: ToolSource = CwlToolSource(
+                strict_cwl_validation=strict_cwl_validation, uuid=dynamic_tool.uuid, tool_proxy=dynamic_tool.proxy
             )
         else:
-            tool_source = get_tool_source_from_representation(**get_source_kwds)
-        kwds["dynamic"] = True
-        tool = self._create_tool_from_source(tool_source, **kwds)
+            tool_format = dynamic_tool.tool_format
+            tool_representation = dynamic_tool.value
+            if "name" not in tool_representation:
+                tool_representation["name"] = f"dynamic tool {dynamic_tool.uuid}"
+            get_source_kwds = dict(
+                tool_format=tool_format,
+                tool_representation=tool_representation,
+                strict_cwl_validation=strict_cwl_validation,
+                uuid=dynamic_tool.uuid,
+            )
+            if dynamic_tool.tool_directory:
+                get_source_kwds["tool_directory"] = dynamic_tool.tool_directory
+            if dynamic_tool.tool_path:
+                config_file = dynamic_tool.tool_path
+                # TODO: uuid probably needed here...
+                tool_source = get_tool_source(
+                    config_file,
+                    enable_beta_formats=getattr(self.app.config, "enable_beta_tool_formats", True),
+                    tool_location_fetcher=self.tool_location_fetcher,
+                    strict_cwl_validation=strict_cwl_validation,
+                )
+            else:
+                tool_source = get_tool_source_from_representation(**get_source_kwds)
+        tool = self._create_tool_from_source(tool_source, dynamic=True)
         tool.dynamic_tool = dynamic_tool
         tool.uuid = dynamic_tool.uuid
         if not tool.id:

lib/galaxy/workflow/trs_proxy.py

@@ -95,7 +95,7 @@ def match_url(self, url, ip_allowlist: List[IpAllowedListEntryT]):
         if url.lstrip().startswith("file://"):
             # requests doesn't know what to do with file:// anyway, but just in case we swap
             # out the implementation
-            raise RequestParameterInvalidException("Invalid TRS URL %s", url)
+            raise RequestParameterInvalidException(f"Invalid TRS URL {url}")
         validate_non_local(url, ip_allowlist=ip_allowlist or [])
         return self._match_url(url)
 

test/unit/data/test_mutable_json_column.py

@@ -17,7 +17,7 @@ def test_metadata_mutable_column(self):
         session = self.model.session
         session.add(w)
         session.commit()
-        w.value = {"x": "z"}  # type:ignore[assignment]
+        w.value = {"x": "z"}
         persisted = self.persist_and_reload(w)
         assert persisted.value == {"x": "z"}
         persisted.value["x"] = "1"