Parse docstrings for nested argument models

docs/source/conf.py

@@ -29,6 +29,9 @@
     "private-members": True,
     "show-inheritance": True,
     "autosummary-no-nesting": True,
+    "autosummary_no_nesting": True,
+    # valid options: bysource, groupwise, alphabetical
+    "autodoc_member_order": "groupwise",
 }
 autodata_content = "both"
 source_suffix = ".rst"

fractal_tasks_core/__FRACTAL_MANIFEST__.json

@@ -77,7 +77,7 @@
         "definitions": {
           "Window": {
             "title": "Window",
-            "description": "Custom class for Omero-channel window, related to OME-NGFF v0.4\n\nSee https://ngff.openmicroscopy.org/0.4/#omero-md.\nMain difference from the specs:\n\n    1. ``min`` and ``max`` are optional, since we have custom logic to set\n        their values.",
+            "description": "Custom class for Omero-channel window, related to OME-NGFF v0.4\n\nSee https://ngff.openmicroscopy.org/0.4/#omero-md.\nMain difference from the specs:\n\n    1. ``min`` and ``max`` are optional, since we have custom logic to set\n       their values.",
             "type": "object",
             "properties": {
               "min": {
@@ -104,7 +104,7 @@
           },
           "OmeroChannel": {
             "title": "OmeroChannel",
-            "description": "Custom class for Omero channels, related to OME-NGFF v0.4.\n\nDifferences from OME-NGFF v0.4 specs\n(https://ngff.openmicroscopy.org/0.4/#omero-md)\n\n    1. Additional attributes ``wavelength_id`` and ``index``.\n    2. We make ``color`` an optional attribute, since we have custom\n       logic to set its value.\n    3. We make ``window`` an optional attribute, so that we can also\n       process zarr arrays which do not have this attribute.",
+            "description": "Custom class for Omero channels, related to OME-NGFF v0.4.\n\nDifferences from OME-NGFF v0.4 specs\n(https://ngff.openmicroscopy.org/0.4/#omero-md):\n\n    1. Additional attributes ``wavelength_id`` and ``index``.\n    2. We make ``color`` an optional attribute, since we have custom\n       logic to set its value.\n    3. We make ``window`` an optional attribute, so that we can also\n       process zarr arrays which do not have this attribute.",
             "type": "object",
             "properties": {
               "wavelength_id": {
@@ -115,17 +115,17 @@
                 "title": "Index",
                 "type": "integer"
               },
+              "label": {
+                "title": "Label",
+                "type": "string"
+              },
               "window": {
                 "$ref": "#/definitions/Window"
               },
               "color": {
                 "title": "Color",
                 "type": "string"
               },
-              "label": {
-                "title": "Label",
-                "type": "string"
-              },
               "active": {
                 "title": "Active",
                 "default": true,
@@ -804,7 +804,7 @@
         "definitions": {
           "Window": {
             "title": "Window",
-            "description": "Custom class for Omero-channel window, related to OME-NGFF v0.4\n\nSee https://ngff.openmicroscopy.org/0.4/#omero-md.\nMain difference from the specs:\n\n    1. ``min`` and ``max`` are optional, since we have custom logic to set\n        their values.",
+            "description": "Custom class for Omero-channel window, related to OME-NGFF v0.4\n\nSee https://ngff.openmicroscopy.org/0.4/#omero-md.\nMain difference from the specs:\n\n    1. ``min`` and ``max`` are optional, since we have custom logic to set\n       their values.",
             "type": "object",
             "properties": {
               "min": {
@@ -831,7 +831,7 @@
           },
           "OmeroChannel": {
             "title": "OmeroChannel",
-            "description": "Custom class for Omero channels, related to OME-NGFF v0.4.\n\nDifferences from OME-NGFF v0.4 specs\n(https://ngff.openmicroscopy.org/0.4/#omero-md)\n\n    1. Additional attributes ``wavelength_id`` and ``index``.\n    2. We make ``color`` an optional attribute, since we have custom\n       logic to set its value.\n    3. We make ``window`` an optional attribute, so that we can also\n       process zarr arrays which do not have this attribute.",
+            "description": "Custom class for Omero channels, related to OME-NGFF v0.4.\n\nDifferences from OME-NGFF v0.4 specs\n(https://ngff.openmicroscopy.org/0.4/#omero-md):\n\n    1. Additional attributes ``wavelength_id`` and ``index``.\n    2. We make ``color`` an optional attribute, since we have custom\n       logic to set its value.\n    3. We make ``window`` an optional attribute, so that we can also\n       process zarr arrays which do not have this attribute.",
             "type": "object",
             "properties": {
               "wavelength_id": {
@@ -842,17 +842,17 @@
                 "title": "Index",
                 "type": "integer"
               },
+              "label": {
+                "title": "Label",
+                "type": "string"
+              },
               "window": {
                 "$ref": "#/definitions/Window"
               },
               "color": {
                 "title": "Color",
                 "type": "string"
               },
-              "label": {
-                "title": "Label",
-                "type": "string"
-              },
               "active": {
                 "title": "Active",
                 "default": true,

fractal_tasks_core/dev/lib_args_schemas.py

@@ -13,18 +13,18 @@
 
 Helper functions to handle JSON schemas for task arguments.
 """
-import ast
-from pathlib import Path
 from typing import Any
 
-from docstring_parser import parse
 from pydantic.decorator import ALT_V_ARGS
 from pydantic.decorator import ALT_V_KWARGS
 from pydantic.decorator import V_DUPLICATE_KWARGS
 from pydantic.decorator import V_POSITIONAL_ONLY_NAME
 from pydantic.decorator import ValidatedFunction
 
-import fractal_tasks_core
+from fractal_tasks_core.dev.lib_descriptions import _get_args_descriptions
+from fractal_tasks_core.dev.lib_descriptions import (
+    _include_args_descriptions_in_schema,
+)
 from fractal_tasks_core.dev.lib_signature_constraints import _extract_function
 from fractal_tasks_core.dev.lib_signature_constraints import (
     _validate_function_signature,
@@ -78,49 +78,6 @@ def _remove_pydantic_internals(old_schema: _Schema) -> _Schema:
     return new_schema
 
 
-def _get_args_descriptions(executable) -> dict[str, str]:
-    """
-    Extract argument descriptions for a task function
-    """
-    # Read docstring (via ast)
-    module_path = Path(fractal_tasks_core.__file__).parent / executable
-    module_name = module_path.with_suffix("").name
-    tree = ast.parse(module_path.read_text())
-    function = next(
-        f
-        for f in ast.walk(tree)
-        if (isinstance(f, ast.FunctionDef) and f.name == module_name)
-    )
-    docstring = ast.get_docstring(function)
-    # Parse docstring (via docstring_parser) and prepare output
-    parsed_docstring = parse(docstring)
-    descriptions = {
-        param.arg_name: param.description.replace("\n", " ")
-        for param in parsed_docstring.params
-    }
-    return descriptions
-
-
-def _include_args_descriptions_in_schema(*, schema, descriptions):
-    """
-    Merge the descriptions obtained via `_get_args_descriptions` into an
-    existing JSON Schema for task arguments
-    """
-    new_schema = schema.copy()
-    new_properties = schema["properties"].copy()
-    for key, value in schema["properties"].items():
-        if "description" in value:
-            raise ValueError("Property already has description")
-        else:
-            if key in descriptions:
-                value["description"] = descriptions[key]
-            else:
-                value["description"] = "Missing description"
-            new_properties[key] = value
-    new_schema["properties"] = new_properties
-    return new_schema
-
-
 def create_schema_for_single_task(
     executable: str,
     package: str = "fractal_tasks_core.tasks",

fractal_tasks_core/dev/lib_descriptions.py

@@ -0,0 +1,179 @@
+import ast
+from importlib import import_module
+from pathlib import Path
+
+from docstring_parser import parse as docparse
+
+import fractal_tasks_core
+
+
+def _get_function_args_descriptions(
+    package_name: str, module_relative_path: str, function_name: str
+) -> dict[str, str]:
+    """
+    Extract argument descriptions from a function
+    """
+
+    if not module_relative_path.endswith(".py"):
+        raise ValueError(f"Module {module_relative_path} must end with '.py'")
+    else:
+        module_relative_path = module_relative_path.rstrip(".py")
+    module_name = Path(package_name).with_suffix("").name
+    module = import_module(f"{module_name}.{module_relative_path}")
+    function = getattr(module, function_name)
+
+    docstring = function.__doc__
+
+    # Parse docstring (via docstring_parser) and prepare output
+    parsed_docstring = docparse(docstring)
+    descriptions = {
+        param.arg_name: param.description.replace("\n", " ")
+        for param in parsed_docstring.params
+    }
+    return descriptions
+
+
+def _get_class_attrs_descriptions(
+    package_name: str, module_relative_path: str, class_name: str
+) -> dict[str, str]:
+    if not module_relative_path.endswith(".py"):
+        raise ValueError(f"Module {module_relative_path} must end with '.py'")
+
+    # get the class
+    module_path = Path(package_name) / module_relative_path
+    tree = ast.parse(module_path.read_text())
+
+    _class = next(
+        c
+        for c in ast.walk(tree)
+        if (isinstance(c, ast.ClassDef) and c.name == class_name)
+    )
+    attrs_descriptions = {}
+
+    # extract attribute docstrings
+    var_name: str = ""
+    for node in _class.body:
+        if isinstance(node, ast.AnnAssign):
+            attrs_descriptions[node.target.id] = "Missing description"
+            var_name = node.target.id
+        else:
+            if isinstance(node, ast.Expr) and var_name:
+                attrs_descriptions[var_name] = node.value.s
+                var_name = ""
+    return attrs_descriptions
+
+
+def _get_args_descriptions(executable) -> dict[str, str]:
+    """
+    Extract argument descriptions for a task function.
+    """
+    # Read docstring (via ast)
+    module_path = Path(fractal_tasks_core.__file__).parent / executable
+    module_name = module_path.with_suffix("").name
+    tree = ast.parse(module_path.read_text())
+    function = next(
+        f
+        for f in ast.walk(tree)
+        if (isinstance(f, ast.FunctionDef) and f.name == module_name)
+    )
+
+    docstring = ast.get_docstring(function)
+    # Parse docstring (via docstring_parser) and prepare output
+    parsed_docstring = docparse(docstring)
+    descriptions = {
+        param.arg_name: param.description.replace("\n", " ")
+        for param in parsed_docstring.params
+    }
+    return descriptions
+
+
+def _include_args_descriptions_in_schema(*, schema, descriptions):
+    """
+    Merge the descriptions obtained via `_get_args_descriptions` into an
+    existing JSON Schema for task arguments.
+    """
+    new_schema = schema.copy()
+    new_properties = schema["properties"].copy()
+    for key, value in schema["properties"].items():
+        if "description" in value:
+            raise ValueError("Property already has description")
+        else:
+            if key in descriptions:
+                value["description"] = descriptions[key]
+            else:
+                value["description"] = "Missing description"
+            new_properties[key] = value
+    new_schema["properties"] = new_properties
+    return new_schema
+
+
+INNER_PYDANTIC_MODELS = {
+    "OmeroChannel": "lib_channels.py",
+    "Window": "lib_channels.py",
+    "Channel": "lib_input_models.py",
+    "NapariWorkflowsInput": "lib_input_models.py",
+    "NapariWorkflowsOutput": "lib_input_models.py",
+}
+
+
+def _get_attributes_models_descriptions(
+    models: dict[str, str] = INNER_PYDANTIC_MODELS
+) -> dict[str, dict[str, str]]:
+    """
+    Extract attribut descriptions for Pydantic models
+    """
+    descriptions = {}
+
+    for model, module in models.items():
+        # get the class
+        module_path = Path(fractal_tasks_core.__file__).parent / module
+        tree = ast.parse(module_path.read_text())
+        try:
+            _class = next(
+                c
+                for c in ast.walk(tree)
+                if (isinstance(c, ast.ClassDef) and c.name == model)
+            )
+            descriptions[model] = {}
+        except StopIteration:
+            raise ValueError(f"Model {module_path}::{model} not found.")
+
+        # extract attribute docstrings
+        var_name: str = ""
+        for node in _class.body:
+            if isinstance(node, ast.AnnAssign):
+                descriptions[model][node.target.id] = "Missing description"
+                var_name = node.target.id
+            else:
+                if isinstance(node, ast.Expr) and var_name:
+                    descriptions[model][var_name] = node.value.s
+                    var_name = ""
+
+    return descriptions
+
+
+def _include_attributs_descriptions_in_schema(*, schema, descriptions):
+    """
+    Merge the descriptions obtained via `_get_attributes_models_descriptions`
+    into an existing JSON Schema for task arguments.
+    """
+    new_schema = schema.copy()
+
+    if "definitions" not in schema:
+        return new_schema
+    else:
+        new_definitions = schema["definitions"].copy()
+
+    for name, definition in schema["definitions"].items():
+        if name in descriptions.keys():
+            for prop in definition["properties"]:
+                if "description" in new_definitions[name]["properties"][prop]:
+                    raise ValueError(
+                        f"Property {name}.{prop} already has description"
+                    )
+                else:
+                    new_definitions[name]["properties"][prop][
+                        "description"
+                    ] = descriptions[name][prop]
+    new_schema["definitions"] = new_definitions
+    return new_schema