From 6c48134e4b1b87d3e6c4b8ac78429b63ad15bca3 Mon Sep 17 00:00:00 2001 From: mpkocher Date: Thu, 18 Jul 2024 18:33:56 -0700 Subject: [PATCH 01/10] Update deps to Pydantic 2.8.2 --- REQUIREMENTS-TEST.txt | 6 +++--- REQUIREMENTS.txt | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/REQUIREMENTS-TEST.txt b/REQUIREMENTS-TEST.txt index 02f38f8..c74f10b 100644 --- a/REQUIREMENTS-TEST.txt +++ b/REQUIREMENTS-TEST.txt @@ -1,3 +1,3 @@ -pytest >= 6.0.1 -mypy >= 0.930 -black >= 21.7b0 +pytest >= 8.2.2 +mypy >= 1.10.1 +black >= 24.4.2 \ No newline at end of file diff --git a/REQUIREMENTS.txt b/REQUIREMENTS.txt index 06ee8a8..4d2bf1b 100644 --- a/REQUIREMENTS.txt +++ b/REQUIREMENTS.txt @@ -1 +1 @@ -pydantic >= 1.5, < 2 +pydantic >= 2.8.2 From 3e4c6a09f3afc207312f8001e075d4fc4713033d Mon Sep 17 00:00:00 2001 From: mpkocher Date: Thu, 18 Jul 2024 18:37:48 -0700 Subject: [PATCH 02/10] Update core to use Pydantic 2. Remove old bool style --- pydantic_cli/__init__.py | 364 ++++++++++--------------------- pydantic_cli/_version.py | 2 +- pydantic_cli/argparse.py | 1 + pydantic_cli/core.py | 139 ++++-------- pydantic_cli/shell_completion.py | 1 + 5 files changed, 163 insertions(+), 344 deletions(-) diff --git a/pydantic_cli/__init__.py b/pydantic_cli/__init__.py index 25ff983..6027323 100644 --- a/pydantic_cli/__init__.py +++ b/pydantic_cli/__init__.py @@ -1,20 +1,22 @@ +import collections import datetime import sys import traceback import logging +import typing import typing as T -from enum import Enum from typing import Callable as F -import pydantic.fields + +import pydantic +from pydantic.fields import FieldInfo +from pydantic_core import PydanticUndefined from ._version import __version__ -from .core import M, CustomOptsType +from .core import M, Tuple1or2Type, Tuple1Type, Tuple2Type from .core import EpilogueHandlerType, PrologueHandlerType, ExceptionHandlerType from .core import ( - DefaultConfig, CliConfig, - DEFAULT_CLI_CONFIG, _get_cli_config_from_model, ) from .utils import _load_json_file, _resolve_file, _resolve_file_or_none_and_warn @@ -31,7 +33,7 @@ log = logging.getLogger(__name__) NOT_PROVIDED = ... - +NONE_TYPE = type(None) __all__ = [ "to_runner", @@ -42,7 +44,7 @@ "default_minimal_exception_handler", "default_prologue_handler", "default_epilogue_handler", - "DefaultConfig", + "CliConfig", "HAS_AUTOCOMPLETE_SUPPORT", "PrologueHandlerType", "EpilogueHandlerType", @@ -68,20 +70,39 @@ def __repr__(self): return "<{k} func:{f} >".format(**d) -def __try_to_pretty_type(field_type, allow_none: bool) -> str: +def _is_sequence(annotation: T.Any) -> bool: + # FIXME There's probably a better and robust way to do this. + # Lifted from pydantic + LIST_TYPES: list[type] = [list, typing.List, collections.abc.MutableSequence] + SET_TYPES: list[type] = [set, typing.Set, collections.abc.MutableSet] + FROZEN_SET_TYPES: list[type] = [frozenset, typing.FrozenSet, collections.abc.Set] + ALL_SEQ = set(LIST_TYPES + SET_TYPES + FROZEN_SET_TYPES) + + # what is exactly going on here? + return getattr(annotation, "__origin__", "NOTFOUND") in ALL_SEQ + + +def __try_to_pretty_type(field_type) -> str: """ This is a marginal improvement to get the types to be displayed in slightly better format. FIXME. This needs to be display Union types better. """ - prefix = "Optional[" if allow_none else "" - suffix = "]" if allow_none else "" - try: - name = field_type.__name__ - except AttributeError: - name = repr(field_type) - return "".join(["type:", prefix, name, suffix]) + + args = typing.get_args(field_type) + if args: + if len(args) == 1: + name = field_type.__name__ + else: + name = "|".join(map(lambda x: x.__name__, args)) + else: + try: + name = field_type.__name__ + except AttributeError: + name = repr(field_type) + + return f"type:{name}" def __to_type_description( @@ -90,254 +111,121 @@ def __to_type_description( allow_none: bool = False, is_required: bool = False, ): - t = ( - "" - if field_type is NOT_PROVIDED - else __try_to_pretty_type(field_type, allow_none) - ) + t = "" if field_type is NOT_PROVIDED else __try_to_pretty_type(field_type) # FIXME Pydantic has a very odd default of None, which makes often can make the # the "default" is actually None, or is not None # avoid using in with a Set to avoid assumptions that default_value is hashable allowed_defaults: T.List[T.Any] = ( - [NOT_PROVIDED] if allow_none else [NOT_PROVIDED, None] + [NOT_PROVIDED, PydanticUndefined] + if allow_none + else [NOT_PROVIDED, PydanticUndefined, None, type(None)] ) v = ( "" if any((default_value is x) for x in allowed_defaults) else f"default:{default_value}" ) - required = " required=True" if is_required else "" + required = " *required*" if is_required else "" sep = " " if v else "" xs = sep.join([t, v]) + required return xs -def __process_tuple( - tuple_one_or_two: T.Sequence[str], long_arg: str -) -> T.Union[T.Tuple[str], T.Tuple[str, str]]: +@pydantic.validate_call +def __process_tuple(tuple_one_or_two: Tuple1or2Type, long_arg: str) -> Tuple1or2Type: """ If the custom args are provided as only short, then - add the long version. + add the long version. Or just use the """ lx: T.List[str] = list(tuple_one_or_two) - def is_short(xs) -> int: - # xs = '-s' - return len(xs) == 2 - nx = len(lx) if nx == 1: - first = lx[0] - if is_short(first): - return first, long_arg + if len(lx[0]) == 2: # xs = '-s' + return lx[0], long_arg else: # this is the positional only case - return (first,) + return (lx[0],) elif nx == 2: # the explicit form is provided return lx[0], lx[1] else: raise ValueError( - f"Unsupported format for `{tuple_one_or_two}`. Expected 1 or 2 tuple." + f"Unsupported format for `{tuple_one_or_two}` type={type(tuple_one_or_two)}. Expected 1 or 2 tuple." ) -def __add_boolean_arg_negate_to_parser( - parser: CustomArgumentParser, - field_id: str, - cli_custom: CustomOptsType, - default_value: bool, - type_doc: str, - description: T.Optional[str], -) -> CustomArgumentParser: - dx = {True: "store_true", False: "store_false"} - desc = description or "" - help_doc = f"{desc} ({type_doc})" - parser.add_argument( - *cli_custom, - action=dx[not default_value], - dest=field_id, - default=default_value, - help=help_doc, - ) - return parser - - -def __add_boolean_arg_to_parser( +def _add_pydantic_field_to_parser( parser: CustomArgumentParser, field_id: str, - cli_custom: CustomOptsType, - default_value: bool, - is_required: bool, - type_doc: str, - description: T.Optional[str], + field_info: FieldInfo, + override_value: T.Any = ..., + long_prefix: str = "--", ) -> CustomArgumentParser: - # Overall this is a bit messy to add a boolean flag. - - error_msg = ( - f"boolean field ({field_id}) with custom CLI options ({cli_custom}) must be defined " - "as a Tuple2[str, str] of True, False for the field. For example, (--enable-X, --disable-X)." - ) + """ - n = len(cli_custom) - - if n == 2: - # Argparse is really a thorny beast - # if you set the group level required=True, then the add_argument must be - # set to optional (this is encapsulated at the group level). Otherwise - # argparse will raise "mutually exclusive arguments must be optional" - group = parser.add_mutually_exclusive_group(required=is_required) - # log.info(f"Field={field_id}. Creating group {group} required={is_required}") - - # see comments above about Group - if is_required: - is_required = False - - bool_datum = [(True, "store_true"), (False, "store_false")] - - for k, (bool_, store_bool) in zip(cli_custom, bool_datum): - if bool_ != default_value: - desc = description or f"Set {field_id} to {bool_}." - help_doc = f"{desc} ({type_doc})" - group.add_argument( - k, - help=help_doc, - action=store_bool, - default=default_value, - dest=field_id, - required=is_required, - ) - else: - raise ValueError(error_msg) + :param field_id: Global Id used to store + :param field_info: FieldInfo from Pydantic (this is messy from a type standpoint) + :param override_value: override the default value defined in the Field (perhaps define in ENV or JSON file) - return parser + Supported Core cases of primitive types, T (e.g., float, str, int) + alpha: str -> *required* --alpha abc + alpha: Optional[str] -> *required* --alpha None # This is kinda a problem and not well-defined + alpha: Optional[str] = None -> --alpha abc ,or --alpha None (to explicitly set none) -def __get_cli_key_by_alias(d: T.Dict) -> T.Any: - # for backwards compatibility - try: - return d["extras"]["cli"] - except KeyError: - return d["cli"] + alpha: bool -> *required* --alpha "true" (Pydantic will handle the casting) + alpha: Optional[bool] -> *required* --alpha "none" or --alpha true + alpha: Optional[bool] = True -> --alpha "none" or --alpha true -def _add_pydantic_field_to_parser( - parser: CustomArgumentParser, - field_id: str, - field: pydantic.fields.ModelField, - override_value: T.Any = ..., - override_cli: T.Optional[CustomOptsType] = None, - long_prefix: str = "--", - bool_prefix: T.Tuple[str, str] = DefaultConfig.CLI_BOOL_PREFIX, -) -> CustomArgumentParser: - """ + Sequence Types: - :param field_id: Global Id used to store - :param field: Field from Pydantic (this is messy from a type standpoint) - :param override_value: override the default value defined in the Field - :param override_cli: Custom format of the CLI argument + xs: List[str] -> *required* --xs 1 2 3 + xs: Optional[List[str]] -> There's a useful reason to encode this type, however, + it's not well-defined or supported. This should be List[T] """ - # field is Field from Pydantic - description = field.field_info.description - extra: T.Dict[str, T.Any] = field.field_info.extra default_long_arg = "".join([long_prefix, field_id]) + description = field_info.description + # there's mypy type issues here + cli_custom_: Tuple1or2Type = ( + (default_long_arg,) + if field_info.json_schema_extra is None # type: ignore + else field_info.json_schema_extra.get("cli", (default_long_arg,)) # type: ignore + ) + cli_short_long: Tuple1or2Type = __process_tuple(cli_custom_, default_long_arg) - # If a default value is provided, it's not necessarily required? - is_required = field.required is True # required is UndefinedBool + is_required = field_info.is_required() + is_nullable = type(None) in typing.get_args(field_info.annotation) + default_value = field_info.default + is_sequence = _is_sequence(field_info.annotation) - default_value = field.default + # If the value is loaded from JSON, or ENV, this will fundamentally + # change if a field is required. if override_value is not NOT_PROVIDED: default_value = override_value is_required = False - # The bool cases require some attention - # Cases - # 1. x:bool = False|True - # 2. x:bool - # 3 x:Optional[bool] - # 4 x:Optional[bool] = None - # 5 x:Optional[bool] = Field(...) - # 6 x:Optional[bool] = True|False - - # cases 2-6 are handled in the same way with (--enable-X, --disable-X) semantics - # case 5 has limitations because you can't set None from the commandline - # case 1 is a very common cases and the provided CLI custom flags have a different semantic meaning - # to negate the default value. E.g., debug:bool = False, will generate a CLI flag of - # --enable-debug to set the value to True. Very common to set this to (-d, --debug) to True - # avoid using in with a Set {True,False} to avoid assumptions that default_value is hashable - is_bool_with_non_null_default = all( - ( - not is_required, - not field.allow_none, - default_value is True or default_value is False, - ) + type_desc = __to_type_description( + default_value, field_info.annotation, is_nullable, is_required ) - try: - # cli_custom Should be a tuple2[Str, Str] - cli_custom: CustomOptsType = __process_tuple( - __get_cli_key_by_alias(extra), default_long_arg - ) - except KeyError: - if override_cli is None: - if field.type_ == bool: - if is_bool_with_non_null_default: - # flipped to negate - prefix = {True: bool_prefix[1], False: bool_prefix[0]} - cli_custom = (f"{prefix[default_value]}{field_id}",) - else: - cli_custom = ( - f"{bool_prefix[0]}{field_id}", - f"{bool_prefix[1]}{field_id}", - ) - else: - cli_custom = (default_long_arg,) - else: - cli_custom = __process_tuple(override_cli, default_long_arg) - - # log.debug(f"Creating Argument Field={field_id} opts:{cli_custom}, allow_none={field.allow_none} default={default_value} type={field.type_} required={is_required} dest={field_id} desc={description}") + # log.debug(f"Creating Argument Field={field_id} opts:{cli_short_long}, allow_none={field.allow_none} default={default_value} type={field.type_} required={is_required} dest={field_id} desc={description}") - type_desc = __to_type_description( - default_value, field.type_, field.allow_none, is_required + # MK. I don't think there's any point trying to fight with argparse to get + # the types correct here. It's just a mess from a type standpoint. + shape_kw = {"nargs": "+"} if is_sequence else {} + desc = description or "" + parser.add_argument( + *cli_short_long, + help=f"{desc} ({type_desc})", + default=default_value, + dest=field_id, + required=is_required, + **shape_kw, # type: ignore ) - if field.shape in {pydantic.fields.SHAPE_LIST, pydantic.fields.SHAPE_SET}: - shape_kwargs = {"nargs": "+"} - else: - shape_kwargs = {} - - if field.type_ == bool: - # see comments above - # case #1 and has different semantic meaning with how the tuple[str,str] is - # interpreted and added to the parser - if is_bool_with_non_null_default: - __add_boolean_arg_negate_to_parser( - parser, field_id, cli_custom, default_value, type_desc, description - ) - else: - __add_boolean_arg_to_parser( - parser, - field_id, - cli_custom, - default_value, - is_required, - type_desc, - description, - ) - else: - # MK. I don't think there's any point trying to fight with argparse to get - # the types correct here. It's just a mess from a type standpoint. - desc = description or "" - parser.add_argument( - *cli_custom, - help=f"{desc} ({type_desc})", - default=default_value, - dest=field_id, - required=is_required, - **shape_kwargs, # type: ignore - ) - return parser @@ -345,22 +233,9 @@ def _add_pydantic_class_to_parser( p: CustomArgumentParser, cls: T.Type[M], default_overrides: T.Dict[str, T.Any] ) -> CustomArgumentParser: - for ix, field in cls.__fields__.items(): - - cli_config = _get_cli_config_from_model(cls) - default_cli_opts: T.Optional[CustomOptsType] = cli_config.custom_opts.get( - ix, None - ) - + for ix, field in cls.model_fields.items(): override_value = default_overrides.get(ix, ...) - _add_pydantic_field_to_parser( - p, - ix, - field, - override_value=override_value, - override_cli=default_cli_opts, - bool_prefix=cli_config.bool_prefix, - ) + _add_pydantic_field_to_parser(p, ix, field, override_value=override_value) return p @@ -378,17 +253,17 @@ def pydantic_class_to_parser( in the Pydantic data model class. """ - p = CustomArgumentParser(description=description, add_help=False) + p0 = CustomArgumentParser(description=description, add_help=False) - _add_pydantic_class_to_parser(p, cls, default_value_override) + p = _add_pydantic_class_to_parser(p0, cls, default_value_override) cli_config = _get_cli_config_from_model(cls) - if cli_config.json_config_enable: + if cli_config["cli_json_enable"]: _parser_add_arg_json_file(p, cli_config) - if cli_config.shell_completion_enable: - add_shell_completion_arg(p, cli_config.shell_completion_flag) + if cli_config["cli_shell_completion_enable"]: + add_shell_completion_arg(p, cli_config["cli_shell_completion_flag"]) _parser_add_help(p) @@ -411,7 +286,7 @@ def default_exception_handler(ex: BaseException) -> int: Maps/Transforms the Exception type to an integer exit code """ # this might need the opts instance, however - # this isn't really well defined if there's an + # this isn't really well-defined if there's an # error at that level sys.stderr.write(str(ex)) exc_type, exc_value, exc_traceback = sys.exc_info() @@ -492,7 +367,7 @@ def now(): # This is a bit sloppy. There's some fields that are added # to the argparse namespace to get around some of argparse's thorny design - pure_keys = cls.schema()["properties"].keys() + pure_keys = cls.model_json_schema()["properties"].keys() # Remove the items that may have # polluted the namespace (e.g., func, cls, json_config) @@ -503,7 +378,7 @@ def now(): # this validation interface is a bit odd # and the errors aren't particularly pretty in the console - cls.validate(opts) + cls.model_validate(opts) prologue_handler(opts) exit_code = runner_func(opts) except TerminalEagerCommand: @@ -527,14 +402,14 @@ def _parser_add_arg_json_file( validator = ( _resolve_file - if cli_config.json_config_path_validate + if cli_config["cli_json_validate_path"] else _resolve_file_or_none_and_warn ) - field = f"--{cli_config.json_config_key}" + field = f"--{cli_config['cli_json_key']}" - path = cli_config.json_config_path + path = cli_config["cli_json_config_path"] - help = f"Path to configuration JSON file. Can be set using ENV VAR ({cli_config.json_config_env_var}) (default:{path})" + help = f"Path to configuration JSON file. Can be set using ENV VAR ({cli_config['cli_json_config_env_var']}) (default:{path})" p.add_argument( field, @@ -566,7 +441,10 @@ def setup_hook_to_load_json( d = {} - json_config_path = getattr(pjargs, cli_config.json_config_key_sanitized(), None) + # Arg parse will do some munging on this due to it's Namespace attribute style. + json_config_path = getattr( + pjargs, cli_config["cli_json_key"].replace("-", "_"), None + ) if json_config_path is not None: d = _load_json_file(json_config_path) @@ -600,12 +478,12 @@ def to_p(default_override_dict: T.Dict[str, T.Any]) -> CustomArgumentParser: cli_config = _get_cli_config_from_model(cls) - if cli_config.json_config_enable: + if cli_config["cli_json_enable"]: - def __setup(args_): + def __setup(args: list[str]) -> T.Dict[str, T.Any]: c = cli_config.copy() - c.json_config_path_validate = False - return setup_hook_to_load_json(args_, c) + c["cli_json_validate_path"] = False + return setup_hook_to_load_json(args, c) else: __setup = null_setup_hook @@ -711,7 +589,7 @@ def to_subparser( cli_config = _get_cli_config_from_model(sbm.model_class) - if cli_config.json_config_enable: + if cli_config["cli_json_enable"]: _parser_add_arg_json_file(spx, cli_config) _parser_add_help(spx) @@ -736,17 +614,17 @@ def to_runner_sp( # This is a bit messy. The design calling _runner requires a single setup hook. # in principle, there can be different json key names for each subparser # there's not really a clean way to support different key names (which - # you probably don't want for consistency sake. + # you probably don't want for consistency’s sake. for sbm in subparsers.values(): cli_config = _get_cli_config_from_model(sbm.model_class) - if cli_config.json_config_enable: + if cli_config["cli_json_enable"]: def _setup_hook(args: T.List[str]) -> T.Dict[str, T.Any]: # We allow the setup to fail if the JSON config isn't found c = cli_config.copy() - c.json_config_path_validate = False + c["cli_json_validate_path"] = False return setup_hook_to_load_json(args, cli_config) else: diff --git a/pydantic_cli/_version.py b/pydantic_cli/_version.py index 111dc91..ba7be38 100644 --- a/pydantic_cli/_version.py +++ b/pydantic_cli/_version.py @@ -1 +1 @@ -__version__ = "4.3.0" +__version__ = "5.0.0" diff --git a/pydantic_cli/argparse.py b/pydantic_cli/argparse.py index e8020fe..3cd0efc 100644 --- a/pydantic_cli/argparse.py +++ b/pydantic_cli/argparse.py @@ -1,6 +1,7 @@ """ Custom layer/utils for dealing with the every so thorny argparse """ + import sys import typing as T from argparse import ArgumentParser, SUPPRESS, Action diff --git a/pydantic_cli/core.py b/pydantic_cli/core.py index e0997ae..3718c72 100644 --- a/pydantic_cli/core.py +++ b/pydantic_cli/core.py @@ -1,141 +1,80 @@ import os -from pydantic import BaseModel +from pydantic import ConfigDict, BaseModel from typing import Callable as F import typing as T M = T.TypeVar("M", bound=BaseModel) -CustomOptsType = T.Union[T.Tuple[str], T.Tuple[str, str]] +Tuple1Type = T.Tuple[str] +Tuple2Type = T.Tuple[str, str] +Tuple1or2Type = T.Union[Tuple1Type, Tuple2Type] EpilogueHandlerType = F[[int, float], None] PrologueHandlerType = F[[T.Any], None] ExceptionHandlerType = F[[BaseException], int] -# Trying to adhere to Pydantic's style of defining the -# config with a "mixin" or class "husk" that is used as -# configuration mechanism. The internally used model is CliConfig -class DefaultConfig: +class CliConfig(ConfigDict, total=False): """ - Core Default Config "mixin" for CLI configuration. + See `_get_cli_config_from_model` for defaults. + + This is a container to work with pydantic's style of + defining the config. We'll validate this internally + and convert to a BaseModel """ # value used to generate the CLI format --{key} - CLI_JSON_KEY: str = "json-config" + cli_json_key: str # Enable JSON config loading - CLI_JSON_ENABLE: bool = False + cli_json_enable: bool # Set the default ENV var for defining the JSON config path - CLI_JSON_CONFIG_ENV_VAR: str = "PCLI_JSON_CONFIG" + cli_json_config_env_var: str # Set the default Path for JSON config file - CLI_JSON_CONFIG_PATH: T.Optional[str] = None + cli_json_config_path: T.Optional[str] # If a default path is provided or provided from the commandline - CLI_JSON_VALIDATE_PATH: bool = True - - # Can be used to override custom fields - # e.g., {"max_records": ('-m', '--max-records')} - # or {"max_records": ('-m', )} - # ****** THIS SHOULD NO LONGER BE USED **** Use pydantic.Field. - CLI_EXTRA_OPTIONS: T.Dict[str, CustomOptsType] = {} - - # Customize the default prefix that is generated - # if a boolean flag is provided. Boolean custom CLI - # MUST be provided as Tuple[str, str] - CLI_BOOL_PREFIX: T.Tuple[str, str] = ("--enable-", "--disable-") + cli_json_validate_path: bool # Add a flag that will emit the shell completion # this requires 'shtab' # https://github.com/iterative/shtab - CLI_SHELL_COMPLETION_ENABLE: bool = False - CLI_SHELL_COMPLETION_FLAG: str = "--emit-completion" - - -class CliConfig(BaseModel): - """Internal Model for encapsulating the core configuration of the CLI model""" - - class Config: - # allow_mutation: bool = False - validate_all = True - validate_assignment = True - - json_config_key: str = DefaultConfig.CLI_JSON_KEY - json_config_enable: bool = DefaultConfig.CLI_JSON_ENABLE - json_config_env_var: str = DefaultConfig.CLI_JSON_CONFIG_ENV_VAR - json_config_path: T.Optional[str] = DefaultConfig.CLI_JSON_CONFIG_PATH - json_config_path_validate: bool = DefaultConfig.CLI_JSON_VALIDATE_PATH - bool_prefix: T.Tuple[str, str] = DefaultConfig.CLI_BOOL_PREFIX - custom_opts: T.Dict[str, CustomOptsType] = DefaultConfig.CLI_EXTRA_OPTIONS - shell_completion_enable: bool = DefaultConfig.CLI_SHELL_COMPLETION_ENABLE - shell_completion_flag: str = DefaultConfig.CLI_SHELL_COMPLETION_FLAG - - def json_config_key_sanitized(self): - """ - Arg parse will do some munging on this due to - it's Namespace attribute style. - """ - # I don't really understand why argparse - # didn't just use a dict. - return self.json_config_key.replace("-", "_") - - -# This should really use final for 3.8 T.Final[CliConfig] -DEFAULT_CLI_CONFIG = CliConfig() + cli_shell_completion_enable: bool + cli_shell_completion_flag: str def _get_cli_config_from_model(cls: T.Type[M]) -> CliConfig: - enable_json_config: bool = getattr( - cls.Config, "CLI_JSON_ENABLE", DEFAULT_CLI_CONFIG.json_config_enable - ) - json_key_field_name: str = getattr( - cls.Config, "CLI_JSON_KEY", DEFAULT_CLI_CONFIG.json_config_key - ) - json_config_env_var: str = getattr( - cls.Config, "CLI_JSON_CONFIG_ENV_VAR", DEFAULT_CLI_CONFIG.json_config_env_var + cli_json_key = T.cast(str, cls.model_config.get("cli_json_key", "json-config")) + cli_json_enable: bool = T.cast(bool, cls.model_config.get("cli_json_enable", False)) + cli_json_config_env_var: str = T.cast( + str, cls.model_config.get("cli_json_config_env_var", "PCLI_JSON_CONFIG") ) - - json_config_path: T.Optional[str] = getattr( - cls.Config, "CLI_JSON_CONFIG_PATH", DEFAULT_CLI_CONFIG.json_config_path + cli_json_config_path_from_model: T.Optional[str] = T.cast( + T.Optional[str], cls.model_config.get("cli_json_config_path") ) - - json_config_validate_path: bool = getattr( - cls.Config, - "CLI_JSON_VALIDATE_PATH", - DEFAULT_CLI_CONFIG.json_config_path_validate, + cli_json_validate_path: bool = T.cast( + bool, cls.model_config.get("cli_json_validate_path", True) ) # there's an important prioritization to be clear about here. # The env var will override the default set in the Pydantic Model Config - # and the value of othe commandline will override the ENV var - path: T.Optional[str] = os.environ.get(json_config_env_var, json_config_path) - - custom_opts: T.Dict[str, CustomOptsType] = getattr( - cls.Config, "CLI_EXTRA_OPTIONS", DEFAULT_CLI_CONFIG.custom_opts - ) - - custom_bool_prefix: CustomOptsType = getattr( - cls.Config, "CLI_BOOL_PREFIX", DEFAULT_CLI_CONFIG.bool_prefix + # and the value of the commandline will override the ENV var + cli_json_config_path: T.Optional[str] = os.environ.get( + cli_json_config_env_var, cli_json_config_path_from_model ) - shell_compeltion_enable: bool = getattr( - cls.Config, - "CLI_SHELL_COMPLETION_ENABLE", - DEFAULT_CLI_CONFIG.shell_completion_enable, + cli_shell_completion_enable: bool = T.cast( + bool, cls.model_config.get("cli_shell_completion_enable", False) ) - shell_completion_flag: T.Optional[str] = getattr( - cls.Config, - "CLI_SHELL_COMPLETION_FLAG", - DEFAULT_CLI_CONFIG.shell_completion_flag, + cli_shell_completion_flag = T.cast( + str, cls.model_config.get("cli_shell_completion_flag", "--emit-completion") ) - return CliConfig( - json_config_enable=enable_json_config, - json_config_key=json_key_field_name, - json_config_env_var=json_config_env_var, - json_config_path=path, - json_config_path_validate=json_config_validate_path, - bool_prefix=custom_bool_prefix, - custom_opts=custom_opts, - shell_completion_enable=shell_compeltion_enable, - shell_completion_flag=shell_completion_flag, + cli_json_key=cli_json_key, + cli_json_enable=cli_json_enable, + cli_json_config_env_var=cli_json_config_env_var, + cli_json_config_path=cli_json_config_path, + cli_json_validate_path=cli_json_validate_path, + cli_shell_completion_enable=cli_shell_completion_enable, + cli_shell_completion_flag=cli_shell_completion_flag, ) diff --git a/pydantic_cli/shell_completion.py b/pydantic_cli/shell_completion.py index c64c90f..70cf60d 100644 --- a/pydantic_cli/shell_completion.py +++ b/pydantic_cli/shell_completion.py @@ -6,6 +6,7 @@ which can at any point in time call parser.exit() which doesn't work with how pydantic-cli is designed. """ + import sys from argparse import Action, ArgumentParser from .argparse import TerminalEagerCommand From 08e5a2b320efe044d58452eb936a835cd1b4b90c Mon Sep 17 00:00:00 2001 From: mpkocher Date: Thu, 18 Jul 2024 18:38:04 -0700 Subject: [PATCH 03/10] update examples --- pydantic_cli/examples/__init__.py | 8 -- pydantic_cli/examples/simple.py | 1 + pydantic_cli/examples/simple_schema.py | 28 ++++--- pydantic_cli/examples/simple_with_boolean.py | 1 + .../simple_with_boolean_and_config.py | 12 +-- .../examples/simple_with_boolean_custom.py | 19 ++--- pydantic_cli/examples/simple_with_custom.py | 8 +- .../simple_with_custom_and_setup_log.py | 8 +- .../examples/simple_with_enum_by_name.py | 82 +++++++++++++++---- .../examples/simple_with_json_config.py | 9 +- .../simple_with_json_config_not_found.py | 14 ++-- pydantic_cli/examples/simple_with_list.py | 5 +- .../simple_with_shell_autocomplete_support.py | 9 +- pydantic_cli/examples/subparser.py | 15 ++-- 14 files changed, 132 insertions(+), 87 deletions(-) diff --git a/pydantic_cli/examples/__init__.py b/pydantic_cli/examples/__init__.py index 2f187f8..ec674f4 100644 --- a/pydantic_cli/examples/__init__.py +++ b/pydantic_cli/examples/__init__.py @@ -2,17 +2,9 @@ import logging from enum import Enum -from pydantic_cli import DefaultConfig - log = logging.getLogger(__name__) -class ExampleConfigDefaults(DefaultConfig): - # validate_all: bool = True - # validate_assignment: bool = False - allow_mutation: bool = False - - class LogLevel(str, Enum): # wish this was defined in the stdlib's logging as an enum INFO = "INFO" diff --git a/pydantic_cli/examples/simple.py b/pydantic_cli/examples/simple.py index 769c5a4..6c74f3a 100644 --- a/pydantic_cli/examples/simple.py +++ b/pydantic_cli/examples/simple.py @@ -8,6 +8,7 @@ my-tool --input_file file.fasta --max_records 10 ``` """ + from pydantic import BaseModel from pydantic_cli import run_and_exit diff --git a/pydantic_cli/examples/simple_schema.py b/pydantic_cli/examples/simple_schema.py index 5d211df..ace6217 100644 --- a/pydantic_cli/examples/simple_schema.py +++ b/pydantic_cli/examples/simple_schema.py @@ -5,16 +5,17 @@ Note, that this leverages Pydantic's underlying validation mechanism. For example, `max_records` must be > 0. """ + from typing import Optional from pydantic import BaseModel, Field -from pydantic_cli.examples import ExampleConfigDefaults -from pydantic_cli import run_and_exit, HAS_AUTOCOMPLETE_SUPPORT +from pydantic_cli import run_and_exit, HAS_AUTOCOMPLETE_SUPPORT, CliConfig class Options(BaseModel): - class Config(ExampleConfigDefaults): - CLI_SHELL_COMPLETION_ENABLE = HAS_AUTOCOMPLETE_SUPPORT + model_config = CliConfig( + frozen=True, cli_shell_completion_enable=HAS_AUTOCOMPLETE_SUPPORT + ) input_file: str = Field( ..., @@ -36,9 +37,8 @@ class Config(ExampleConfigDefaults): ..., title="Min Score", description="Minimum Score Filter that will be applied to the records", - cli=("-s",), - gt=0 - # or extras={'cli': ('-s', '--min-filter-score', )} + cli=("-s", "--min-filter-score"), + gt=0, ) max_filter_score: Optional[float] = Field( @@ -46,16 +46,20 @@ class Config(ExampleConfigDefaults): title="Max Score", description="Maximum Score Filter that will be applied to the records", gt=0, - cli=("-S",) - # or extras={'cli': ('-S', '--min-filter-score', )} + cli=("-S", "--max-filter-score"), + ) + + name: Optional[str] = Field( + title="Filter Name", + description="Name to Filter on.", + cli=("-n", "--filter-name"), ) def example_runner(opts: Options) -> int: print(f"Mock example running with options {opts}") - print((opts.input_file, type(opts.input_file))) - print(opts.max_records, type(opts.max_records)) - print(opts.min_filter_score, type(opts.min_filter_score)) + for x in (opts.input_file, opts.max_records, opts.min_filter_score, opts.name): + print(f"{x} type={type(x)}") return 0 diff --git a/pydantic_cli/examples/simple_with_boolean.py b/pydantic_cli/examples/simple_with_boolean.py index e93454e..6643529 100644 --- a/pydantic_cli/examples/simple_with_boolean.py +++ b/pydantic_cli/examples/simple_with_boolean.py @@ -5,6 +5,7 @@ Note the optional boolean value must be supplied as `--run_training False` """ + from pydantic import BaseModel from pydantic_cli import run_and_exit diff --git a/pydantic_cli/examples/simple_with_boolean_and_config.py b/pydantic_cli/examples/simple_with_boolean_and_config.py index 7ed2a3f..68fc369 100644 --- a/pydantic_cli/examples/simple_with_boolean_and_config.py +++ b/pydantic_cli/examples/simple_with_boolean_and_config.py @@ -5,18 +5,18 @@ Note the optional boolean value must be supplied as `--run_training False` """ -from pydantic import BaseModel -from pydantic_cli import run_and_exit, DefaultConfig +from pydantic import BaseModel, Field + +from pydantic_cli import run_and_exit, CliConfig class Options(BaseModel): - class Config(DefaultConfig): - CLI_BOOL_PREFIX = ("--yes-", "--no-") + model_config = CliConfig(frozen=True) input_file: str - run_training: bool = True - dry_run: bool = False + run_training: bool = Field(default=False, cli=("-t", "--run-training")) + dry_run: bool = Field(default=False, cli=("-r", "--dry-run")) def example_runner(opts: Options) -> int: diff --git a/pydantic_cli/examples/simple_with_boolean_custom.py b/pydantic_cli/examples/simple_with_boolean_custom.py index 120ef8c..75d8317 100644 --- a/pydantic_cli/examples/simple_with_boolean_custom.py +++ b/pydantic_cli/examples/simple_with_boolean_custom.py @@ -4,7 +4,7 @@ from pydantic import BaseModel from pydantic.fields import Field -from pydantic_cli import run_and_exit, DefaultConfig, default_minimal_exception_handler +from pydantic_cli import run_and_exit, default_minimal_exception_handler, CliConfig from pydantic_cli.examples import setup_logger @@ -17,8 +17,7 @@ class State(str, Enum): class Options(BaseModel): - class Config(DefaultConfig): - pass + model_config = CliConfig(frozen=True) # Simple Arg/Option can be added and a reasonable commandline "long" flag will be created. input_file: str @@ -32,10 +31,8 @@ class Config(DefaultConfig): ..., description="Path to input H5 file", cli=("-f", "--hdf5") ) - # https://pydantic-docs.helpmanual.io/usage/models/#required-optional-fields - # Pydantic has a bit of an odd model on how it treats Optional[T] - # These end up being indistinguishable. - outfile: Optional[str] + ## FIXME This "optional" value has changed semantics in V2 + outfile: Optional[str] = None fasta: Optional[str] = None # This is a "required" value that can be set to None, or str report_json: Optional[str] = Field(...) @@ -52,22 +49,22 @@ class Config(DefaultConfig): ) # Again, note Pydantic will treat these as indistinguishable - gamma: Optional[bool] + gamma: bool delta: Optional[bool] = None # You need to set this to ... to declare it as "Required". The pydantic docs recommend using # Field(...) instead of ... to avoid issues with mypy. # pydantic-cli doesn't have a good mechanism for declaring this 3-state value of None, True, False. # using a boolean commandline flag (e.g., --enable-logging, or --disable-logging) - zeta_mode: Optional[bool] = Field( + zeta_mode: bool = Field( ..., description="Enable/Disable Zeta mode to experimental filtering mode." ) - # this a bit of a contradiction from the commandline perspective. A "optional" value + # this a bit of a contradiction from the commandline perspective. An "optional" value # with a default value. From a pydantic-cli view, the type should just be 'bool' because this 3-state # True, False, None is not well represented (i.e., can't set the value to None from the commandline) # Similar to the other Optional[bool] cases, the custom flag must be provided as a (--enable, --disable) format. - epsilon: Optional[bool] = Field( + epsilon: bool = Field( False, description="Enable epsilon meta-analysis.", cli=("--epsilon", "--disable-epsilon"), diff --git a/pydantic_cli/examples/simple_with_custom.py b/pydantic_cli/examples/simple_with_custom.py index b6ab7ff..ed1dcf0 100644 --- a/pydantic_cli/examples/simple_with_custom.py +++ b/pydantic_cli/examples/simple_with_custom.py @@ -5,21 +5,19 @@ from pydantic import BaseModel, Field from pydantic_cli import __version__ -from pydantic_cli import run_and_exit, DefaultConfig -from pydantic_cli.examples import ExampleConfigDefaults +from pydantic_cli import run_and_exit, CliConfig log = logging.getLogger(__name__) class Options(BaseModel): - class Config(ExampleConfigDefaults, DefaultConfig): - pass + model_config = CliConfig(frozen=True) input_file: str = Field(..., cli=("-i", "--input")) max_records: int = Field(10, cli=("-m", "--max-records")) min_filter_score: float = Field(..., cli=("-f", "--filter-score")) alpha: Union[int, str] = 1 - values: List[str] = ["a", "b", "c"] + # values: List[str] = ["a", "b", "c"] def example_runner(opts: Options) -> int: diff --git a/pydantic_cli/examples/simple_with_custom_and_setup_log.py b/pydantic_cli/examples/simple_with_custom_and_setup_log.py index d22f29d..7517572 100644 --- a/pydantic_cli/examples/simple_with_custom_and_setup_log.py +++ b/pydantic_cli/examples/simple_with_custom_and_setup_log.py @@ -12,21 +12,21 @@ and the `epilogue_handler` will log the runtime and the exit code after the main execution function is called. """ + import sys import logging from pydantic import BaseModel, Field from pydantic_cli import __version__ -from pydantic_cli import run_and_exit -from pydantic_cli.examples import ExampleConfigDefaults, LogLevel +from pydantic_cli import run_and_exit, CliConfig +from pydantic_cli.examples import LogLevel log = logging.getLogger(__name__) class Options(BaseModel): - class Config(ExampleConfigDefaults): - pass + model_config = CliConfig(frozen=True) input_file: str = Field(..., cli=("-i", "--input")) max_records: int = Field(10, cli=("-m", "--max-records")) diff --git a/pydantic_cli/examples/simple_with_enum_by_name.py b/pydantic_cli/examples/simple_with_enum_by_name.py index f4c4d9e..3b774e9 100644 --- a/pydantic_cli/examples/simple_with_enum_by_name.py +++ b/pydantic_cli/examples/simple_with_enum_by_name.py @@ -1,23 +1,59 @@ +import typing +from typing import Annotated from enum import Enum, auto, IntEnum -from typing import Set - -from pydantic import BaseModel, Field +from typing import Set, TypeVar +import logging + +from pydantic import ( + BaseModel, + Field, + BeforeValidator, + AfterValidator, + PlainValidator, + PlainSerializer, + WrapSerializer, + GetCoreSchemaHandler, +) +from pydantic_core import CoreSchema, core_schema from pydantic_cli import run_and_exit +logger = logging.getLogger(__name__) -class CastAbleEnum(Enum): - """Example enum mixin that will cast enum from case-insensitive name""" +EnumType = TypeVar("EnumType", bound=Enum) - @classmethod - def __get_validators__(cls): - yield cls.validate + +class CastAbleEnumMixin: + """Example enum mixin that will cast enum from case-insensitive name. + + This is a bit of non-standard customization of an Enum. + Any customization of coercing, or casting is going to potentially create friction points + at the commandline level. Specifically, in the help description of the Field. + + This is written awkwardly to get around subclassing Enums + """ + + __members__: dict # to make mypy happy + + # FIXME Pydantic is a bit thorny or confusing here. Ideally, the validation + # should be wired into the enum. But it's not. + # Check https://docs.pydantic.dev/latest/migration/#defining-custom-types for more information. + # @classmethod + # def __get_pydantic_core_schema__( + # cls, source_type: typing.Any, handler: GetCoreSchemaHandler + # ) -> CoreSchema: + # return core_schema.no_info_after_validator_function(cls, handler(str)) @classmethod def validate(cls, v): try: lookup = {k.lower(): item.value for k, item in cls.__members__.items()} - return lookup[v.lower()] + logger.debug( + f"*** Got raw state '{v}'. Trying to cast/convert from {lookup}" + ) + value = lookup[v.lower()] + logger.debug(f"*** Successfully got {v}={value}") + return value except KeyError: raise ValueError(f"Invalid value {v}. {cls.cli_help()}") @@ -26,22 +62,38 @@ def cli_help(cls) -> str: return f"Allowed={list(cls.__members__.keys())}" -class Mode(CastAbleEnum, IntEnum): +class Mode(CastAbleEnumMixin, IntEnum): alpha = auto() beta = auto() -class State(CastAbleEnum, str, Enum): +class State(str, CastAbleEnumMixin, Enum): RUNNING = "RUNNING" FAILED = "FAILED" SUCCESSFUL = "SUCCESSFUL" + @classmethod + def defaults(cls): + return {cls.RUNNING, cls.SUCCESSFUL} + + +# I suspect there's a better way to do this. +STATE = Annotated[State, BeforeValidator(State.validate)] + class Options(BaseModel): - states: Set[State] = Field( - ..., description=f"States to filter on. {State.cli_help()}" - ) - mode: Mode = Field(..., description=f"Processing Mode to select. {Mode.cli_help()}") + states: Annotated[ + Set[STATE], + Field( + description=f"States to filter on. {State.cli_help()}", + default=State.defaults(), + ), + ] + mode: Annotated[ + Mode, + BeforeValidator(Mode.validate), + Field(description=f"Processing Mode to select. {Mode.cli_help()}"), + ] max_records: int = 100 diff --git a/pydantic_cli/examples/simple_with_json_config.py b/pydantic_cli/examples/simple_with_json_config.py index 16dab60..3c28e85 100644 --- a/pydantic_cli/examples/simple_with_json_config.py +++ b/pydantic_cli/examples/simple_with_json_config.py @@ -16,18 +16,19 @@ Similarly, `CLI_JSON_ENABLE` """ + import logging from pydantic import BaseModel -from pydantic_cli import run_and_exit, DefaultConfig +from pydantic_cli import run_and_exit, CliConfig from pydantic_cli.examples import epilogue_handler, prologue_handler log = logging.getLogger(__name__) class Opts(BaseModel): - class Config(DefaultConfig): - CLI_JSON_KEY = "json-training" - CLI_JSON_ENABLE = True + model_config = CliConfig( + frozen=True, cli_json_key="json-training", cli_json_enable=True + ) hdf_file: str max_records: int = 10 diff --git a/pydantic_cli/examples/simple_with_json_config_not_found.py b/pydantic_cli/examples/simple_with_json_config_not_found.py index 1bb6d9a..210de8d 100644 --- a/pydantic_cli/examples/simple_with_json_config_not_found.py +++ b/pydantic_cli/examples/simple_with_json_config_not_found.py @@ -3,7 +3,7 @@ from pydantic import BaseModel -from pydantic_cli import run_and_exit, DefaultConfig +from pydantic_cli import run_and_exit, CliConfig log = logging.getLogger(__name__) @@ -11,13 +11,15 @@ class Options(BaseModel): """For cases where you want a global configuration file that is completely ignored if not found, you can set - CLI_JSON_VALIDATE_PATH = False. + cli_json_config_path = False. """ - class Config(DefaultConfig): - CLI_JSON_ENABLE = True - CLI_JSON_CONFIG_PATH = "/path/to/file/that/does/not/exist/simple_schema.json" - CLI_JSON_VALIDATE_PATH = False + model_config = CliConfig( + frozen=True, + cli_json_config_path="/path/to/file/that/does/not/exist/simple_schema.json", + cli_json_enable=True, + cli_json_validate_path=False, + ) input_file: str max_records: int = 10 diff --git a/pydantic_cli/examples/simple_with_list.py b/pydantic_cli/examples/simple_with_list.py index 93c887c..4bf51f0 100644 --- a/pydantic_cli/examples/simple_with_list.py +++ b/pydantic_cli/examples/simple_with_list.py @@ -8,6 +8,7 @@ my-tool --input_file file.fasta file2.fasta --max_records 10 ``` """ + from typing import List, Set from pydantic import BaseModel @@ -15,8 +16,8 @@ class Options(BaseModel): - input_file: List[str] - filters: Set[str] + input_file: list[str] + filters: set[str] max_records: int diff --git a/pydantic_cli/examples/simple_with_shell_autocomplete_support.py b/pydantic_cli/examples/simple_with_shell_autocomplete_support.py index 0ef55df..c64343d 100644 --- a/pydantic_cli/examples/simple_with_shell_autocomplete_support.py +++ b/pydantic_cli/examples/simple_with_shell_autocomplete_support.py @@ -2,22 +2,21 @@ Example that adds an option to emit shell autocomplete for bash/zsh requires `shtab` to be installed. """ + import sys import logging -from pydantic import BaseModel, Field +from pydantic import BaseModel, Field, ConfigDict from pydantic_cli import __version__ -from pydantic_cli import run_and_exit, DefaultConfig -from pydantic_cli.examples import ExampleConfigDefaults +from pydantic_cli import run_and_exit, CliConfig from pydantic_cli.shell_completion import HAS_AUTOCOMPLETE_SUPPORT log = logging.getLogger(__name__) class Options(BaseModel): - class Config(ExampleConfigDefaults, DefaultConfig): - CLI_SHELL_COMPLETION_ENABLE = HAS_AUTOCOMPLETE_SUPPORT + model_config = CliConfig(cli_shell_completion_enable=HAS_AUTOCOMPLETE_SUPPORT) input_file: str = Field(..., cli=("-i", "--input")) min_filter_score: float = Field(..., cli=("-f", "--filter-score")) diff --git a/pydantic_cli/examples/subparser.py b/pydantic_cli/examples/subparser.py index 8c64395..e183047 100644 --- a/pydantic_cli/examples/subparser.py +++ b/pydantic_cli/examples/subparser.py @@ -9,24 +9,22 @@ my-tool alpha --help my-tool beta --help """ + import sys import logging import typing as T from pydantic import BaseModel, AnyUrl, Field -from pydantic_cli.examples import ExampleConfigDefaults, LogLevel, prologue_handler -from pydantic_cli import run_sp_and_exit, SubParser +from pydantic_cli.examples import LogLevel, prologue_handler +from pydantic_cli import run_sp_and_exit, SubParser, CliConfig log = logging.getLogger(__name__) - -class CustomConfig(ExampleConfigDefaults): - CLI_JSON_ENABLE = True +CLI_CONFIG = CliConfig(cli_json_enable=True, frozen=True) class AlphaOptions(BaseModel): - class Config(CustomConfig): - pass + model_config = CLI_CONFIG input_file: str = Field(..., cli=("-i", "--input")) max_records: int = Field(10, cli=("-m", "--max-records")) @@ -34,8 +32,7 @@ class Config(CustomConfig): class BetaOptions(BaseModel): - class Config(CustomConfig): - pass + model_config = CLI_CONFIG url: AnyUrl = Field(..., cli=("-u", "--url")) num_retries: int = Field(3, cli=("-n", "--num-retries")) From b6e46d49adf3eef6518d12bf755cb221850ca9ff Mon Sep 17 00:00:00 2001 From: mpkocher Date: Thu, 18 Jul 2024 18:38:34 -0700 Subject: [PATCH 04/10] Update tests --- pydantic_cli/tests/__init__.py | 1 + .../tests/test_examples_simple_boolean_and_config.py | 6 ++++-- pydantic_cli/tests/test_examples_simple_schema.py | 8 +++----- pydantic_cli/tests/test_examples_simple_with_boolean.py | 8 +++++--- .../tests/test_examples_simple_with_boolean_custom.py | 9 +++++++-- pydantic_cli/tests/test_examples_with_json_config.py | 2 +- 6 files changed, 21 insertions(+), 13 deletions(-) diff --git a/pydantic_cli/tests/__init__.py b/pydantic_cli/tests/__init__.py index 5df2e96..312f1e8 100644 --- a/pydantic_cli/tests/__init__.py +++ b/pydantic_cli/tests/__init__.py @@ -15,6 +15,7 @@ M = TypeVar("M", bound=BaseModel) + # Making this name a bit odd (from TestConfig) # to get around Pytest complaining that # it can't collect the "Test" diff --git a/pydantic_cli/tests/test_examples_simple_boolean_and_config.py b/pydantic_cli/tests/test_examples_simple_boolean_and_config.py index 160dcb1..e56cee8 100644 --- a/pydantic_cli/tests/test_examples_simple_boolean_and_config.py +++ b/pydantic_cli/tests/test_examples_simple_boolean_and_config.py @@ -10,9 +10,11 @@ def test_simple_01(self): self.run_config(["--input_file", "/path/to/file.txt"]) def test_simple_02(self): - self.run_config(["--input_file", "/path/to/file.txt", "--no-run_training"]) + self.run_config( + ["--input_file", "/path/to/file.txt", "--run-training", "false"] + ) def test_simple_03(self): self.run_config( - ["--input_file", "/path/to/file.txt", "--no-run_training", "--yes-dry_run"] + ["--input_file", "/path/to/file.txt", "-r", "false", "--dry-run", "false"] ) diff --git a/pydantic_cli/tests/test_examples_simple_schema.py b/pydantic_cli/tests/test_examples_simple_schema.py index 8b60de0..ce297dc 100644 --- a/pydantic_cli/tests/test_examples_simple_schema.py +++ b/pydantic_cli/tests/test_examples_simple_schema.py @@ -7,14 +7,12 @@ class TestExamples(_TestHarness): CONFIG = HarnessConfig(Options, example_runner) def test_01(self): - args = ( - "-f /path/to/file.txt --max_records 1234 -s 1.234 --max_filter_score 10.234" - ) + args = "-f /path/to/file.txt --max_records 1234 -s 1.234 --max-filter-score 10.234 -n none" self.run_config(args.split()) def test_02(self): - args = "-f /path/to/file.txt -m 1234 -s 1.234 -S 10.234" + args = "-f /path/to/file.txt -m 1234 -s 1.234 -S 10.234 --filter-name alphax" self.run_config(args.split()) def test_03(self): - self.run_config(["-f", "/path/to/file.txt", "-s", "1.234"]) + self.run_config(["-f", "/path/to/file.txt", "-s", "1.234", "-n", "beta.v2"]) diff --git a/pydantic_cli/tests/test_examples_simple_with_boolean.py b/pydantic_cli/tests/test_examples_simple_with_boolean.py index f229e0e..207b927 100644 --- a/pydantic_cli/tests/test_examples_simple_with_boolean.py +++ b/pydantic_cli/tests/test_examples_simple_with_boolean.py @@ -10,14 +10,16 @@ def test_simple_01(self): self.run_config(["--input_file", "/path/to/file.txt"]) def test_simple_02(self): - self.run_config(["--input_file", "/path/to/file.txt", "--disable-run_training"]) + self.run_config(["--input_file", "/path/to/file.txt", "--run_training", "y"]) def test_simple_03(self): self.run_config( [ "--input_file", "/path/to/file.txt", - "--disable-run_training", - "--enable-dry_run", + "--run_training", + "true", + "--dry_run", + "true", ] ) diff --git a/pydantic_cli/tests/test_examples_simple_with_boolean_custom.py b/pydantic_cli/tests/test_examples_simple_with_boolean_custom.py index f77dfa9..37ea236 100644 --- a/pydantic_cli/tests/test_examples_simple_with_boolean_custom.py +++ b/pydantic_cli/tests/test_examples_simple_with_boolean_custom.py @@ -19,9 +19,14 @@ def test_simple_01(self): "output.json", "--fasta", "output.fasta", - "--enable-alpha", - "--enable-zeta_mode", + "--gamma", + "true", + "--alpha", + "true", + "--zeta_mode", + "true", "--epsilon", + "true", "--states", "RUNNING", "FAILED", diff --git a/pydantic_cli/tests/test_examples_with_json_config.py b/pydantic_cli/tests/test_examples_with_json_config.py index fd68db7..ba00b9b 100644 --- a/pydantic_cli/tests/test_examples_with_json_config.py +++ b/pydantic_cli/tests/test_examples_with_json_config.py @@ -25,7 +25,7 @@ def test_simple_json(self): alpha=1.234, beta=9.854, ) - self._util(opt.dict(), []) + self._util(opt.model_dump(), []) def test_simple_partial_json(self): d = dict(max_records=12, min_filter_score=1.024, alpha=1.234, beta=9.854) From cdf05b32a7b63154b87e306f977fda05860aafe6 Mon Sep 17 00:00:00 2001 From: mpkocher Date: Thu, 18 Jul 2024 18:38:52 -0700 Subject: [PATCH 05/10] Update readme to use Pydantic 2 --- README.md | 146 ++++++++++++++---------------------------------------- 1 file changed, 38 insertions(+), 108 deletions(-) diff --git a/README.md b/README.md index a014228..eb90589 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ Turn Pydantic defined Data Models into CLI Tools and enable loading values from JSON files -**Requires Pydantic** `>=1.5.1`. +**Requires Pydantic** `>=2.8.2`. [![Downloads](https://pepy.tech/badge/pydantic-cli)](https://pepy.tech/project/pydantic-cli) @@ -18,18 +18,24 @@ pip install pydantic-cli 1. Thin Schema driven interfaces constructed from [Pydantic](https://github.com/samuelcolvin/pydantic) defined data models 1. Validation is performed in a single location as defined by Pydantic's validation model and defined types -1. CLI parsing is only structurally validating that the args or optional arguments are provided +1. CLI parsing level is only structurally validating that the args or optional arguments are provided 1. Enable loading config defined in JSON to override or set specific values 1. Clear interface between the CLI and your application code 1. Leverage the static analyzing tool [**mypy**](http://mypy.readthedocs.io) to catch type errors in your commandline tool 1. Easy to test (due to reasons defined above) -### Motivating Usecases +### Motivating Use cases - Quick scrapy commandline tools for local development (e.g., webscraper CLI tool, or CLI application that runs a training algo) - Internal tools driven by a Pydantic data model/schema - Configuration heavy tools that are driven by either partial (i.e, "presets") or complete configuration files defined using JSON +Note: Newer version of `Pydantic-settings` has support for commandline functionality. + +https://docs.pydantic.dev/2.8/concepts/pydantic_settings/#settings-management + +`Pydantic-cli` predates the CLI component of `pydantic-settings` and has a few different requirements. + ## Quick Start @@ -107,7 +113,6 @@ if __name__ == '__main__': run_and_exit(MinOptions, example_runner, description="My Tool Description", version='0.1.0') ``` -**WARNING**: Data models that have boolean values and generated CLI flags (e.g., `--enable-filter` or `--disable-filter`) require special attention. See the "Defining Boolean Flags" section for more details. Leveraging `Field` is also useful for validating inputs. @@ -123,7 +128,7 @@ class MinOptions(BaseModel): ## Loading Configuration using JSON -Tools can also load entire models or partially defined Pydantic data models from JSON files. +User created commandline tools using `pydantic-cli` can also load entire models or **partially** defined Pydantic data models from JSON files. For example, given the following Pydantic data model: @@ -157,7 +162,7 @@ Can be run with a JSON file that defines all the (required) values. {"hdf_file": "/path/to/file.hdf5", "max_records": 5, "min_filter_score": 1.5, "alpha": 1.0, "beta": 1.0} ``` -The tool can be executed as shown below. Note, options required at the commandline as defined in the `Opts` model (e.g., 'hdf_file', 'min_filter_score', 'alpha' and 'beta') are NO longer required values supplied to the commandline tool. +The tool can be executed as shown below. Note, options required at the commandline as defined in the `Opts` model (e.g., 'hdf_file', 'min_filter_score', 'alpha' and 'beta') are **NO longer required** values supplied to the commandline tool. ```bash my-tool --json-config /path/to/file.json ``` @@ -270,119 +275,47 @@ Found 1 error in 1 file (checked 1 source file) ``` -## Defining Boolean Flags - -There are a few common cases of boolean values: - -1. `x:bool = True|False` A bool field with a default value -2. `x:bool` A required bool field -3. `x:Optional[bool]` or `x:Optional[bool] = None` An optional boolean with a default value of None -4. `x:Optional[bool] = Field(...)` a required boolean that can be set to `None`, `True` or `False` in Pydantic. - -Case 1 is very common and the semantics of the custom CLI overrides (as a tuple) **are different than the cases 2-4**. -Case 4 has limitations. It isn't possible to set `None` from the commandline when the default is `True` or `False`. - -### Boolean Field with Default - -As demonstrated in a previous example, the common case of defining a type as `bool` with a default value work as expected. - -For example: - - -```python -from pydantic import BaseModel - - -class MinOptions(BaseModel): - debug: bool = False -``` - - -By default, when defining a model with a boolean flag, an "enable" or "disable" prefix will be added to create the commandline flag depending on the default value. - -In this specific case, a commandline flag of `--enable-debug` which will set `debug` in the Pydantic model to `True`. - -If the default was set to `False`, then a `--disable-debug` flag would be created and would set `debug` to `False` in the Pydantic data model. - -The CLI flag can be customized and provided as a `Tuple[str]` or `Tuple[str, str]` as (long, ) or (short, long) flags (respectively) to negate the default value. - -For example, running `-d` or `--debug` will set `debug` to `True` in the Pydantic data model. - -```python -from pydantic import BaseModel, Field - - -class MinOptions(BaseModel): - debug: bool = Field(False, description="Enable debug mode", cli=('-d', '--debug')) -``` - -If the default is `True`, running the example below with `--disable-debug` will set `debug` to `False`. - -```python -from pydantic import BaseModel, Field - - -class MinOptions(BaseModel): - debug: bool = Field(True, description="Disable debug mode", cli=('-d', '--disable-debug')) -``` - -### Boolean Required Field +## Using Boolean Flags -Required boolean fields are handled a bit different than cases where a boolean is provided with a default value. +There's an ergonomic tradeoff to lean on Pydantic to avoid some friction points at CLI level. This yields an explicit model, but added verboseness. -Specifically, the custom flag `Tuple[str, str]` must be provided as a `(--enable, --disable)` format. +Summary: -```python -from pydantic import BaseModel, Field +- `xs:bool` can be set from commandline as `--xs true` or `--xs false`. Or [using Pydantic's casting](https://docs.pydantic.dev/2.8/api/standard_library_types/#booleans), `--xs yes` or `--xs y`. +- `xs:Optional[bool]` can be set from commandline as `--xs true`, `--xs false`, or `--xs none` +For the `None` case, you can configure your Pydantic model to handle the casting/coercing/validation. Similarly, the bool casting should be configured in Pydantic. -class MinOptions(BaseModel): - debug: bool = Field(..., description="Enable/Disable debugging", cli= ('--enable-debug', '--disable-debug')) -``` -**Currently, supplying the short form of each "enable" and "disable" is not supported**. - -### Optional Boolean Fields - -Similar to the required boolean fields case, `Optional[bool]` cases have the same (--enable, --disable) semantics. +Consider a basic model: ```python from typing import Optional from pydantic import BaseModel, Field +from pydantic_cli import run_and_exit +class Options(BaseModel): + input_file: str + dry_run: bool = Field(default=False, description="Enable dry run mode", cli=('-r', '--dry-run')) + filtering: Optional[bool] -class MinOptions(BaseModel): - a: Optional[bool] - b: Optional[bool] = None - c: Optional[bool] = Field(None, cli= ('--yes-c', '--no-c')) - d: Optional[bool] = Field(False, cli=('--enable-d', '--disable-d')) - e: Optional[bool] = Field(..., cli=('--enable-e', '--disable-e')) -``` -Note, that `x:Optional[bool]`, `x:Optional[bool] = None`, `x:Optional[bool] = Field(None)` semantically mean the same thing in Pydantic. - -In each of the above cases, the **custom CLI flags must be provided as (--enable, --disable) format**. - -Also, note it isn't possible to set `None` from the commandline for the `Optional[bool] = False` or `Optional[bool] = Field(...)` case. - -### Customizing default Enable/Disable Bool Prefix -The enable/disable prefix used for all `bool` options can be customized by setting the `Tuple[str, str]` of `CLI_BOOL_PREFIX` on `Config` to the (positive, negative) of prefix flag. +def example_runner(opts: Options) -> int: + print(f"Mock example running with {opts}") + return 0 -The default value of `Config.CLI_BOOL_PREFIX` is `('--enable-', '--disable')`. +if __name__ == "__main__": + run_and_exit(Options, example_runner, description=__doc__, version="0.1.0") +``` -```python -from pydantic import BaseModel +In this case, +- `dry_run` is an optional value with a default and can be set as `--dry-run yes` or `--dry-run no` +- `filtering` is a required value and can be set `--filtering true`, `--filtering False`, and `--filtering None` -class Options(BaseModel): - class Config: - CLI_BOOL_PREFIX = ('--yes-', '--no-') - - debug: bool = False -``` -This will generate an optional `--yes-debug` flag that will set `debug` from the default (`False`) to `True` in the Pydantic data model. +See the Pydantic docs for more details on boolean casting. -In many cases, **it's best to customize the commandline boolean flags** to avoid ambiguities or confusion. +https://docs.pydantic.dev/2.8/api/standard_library_types/#booleans ## Customization and Hooks @@ -525,7 +458,6 @@ Pydantic-cli attempts to stylistically follow Pydantic's approach using a class ```python import typing as T -from pydantic_cli import CustomOptsType class DefaultConfig: """ @@ -728,11 +660,10 @@ For example: ```python from pydantic import BaseModel -from pydantic_cli import DefaultConfig +from pydantic_cli import CliConfig class MinOptions(BaseModel): - class Config(DefaultConfig): - CLI_JSON_ENABLE = True + model_config = CliConfig(cli_json_enable=True) input_file: str input_hdf: str @@ -771,11 +702,10 @@ The simplest fix is to remove the positional arguments in favor of `-i` or simil ```python from pydantic import BaseModel, Field -from pydantic_cli import run_and_exit, to_runner, DefaultConfig +from pydantic_cli import run_and_exit, to_runner, CliConfig class MinOptions(BaseModel): - class Config(DefaultConfig): - CLI_JSON_ENABLE = True + model_config = CliConfig(cli_json_enable=True) input_file: str = Field(..., cli=('-i', )) input_hdf: str = Field(..., cli=('-d', '--hdf')) From dcd90090cb6f5a14de3562eaac6a65fc96314d0f Mon Sep 17 00:00:00 2001 From: mpkocher Date: Thu, 18 Jul 2024 18:39:10 -0700 Subject: [PATCH 06/10] Update changelog with breaking changes --- CHANGELOG.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0e305e3..88a6976 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,14 @@ # CHANGELOG +## Version 5.0.0 (Pydantic 2 support) + +- Support for Pydantic >= 2.8 +- Pydantic 2 has a different "optional" definition +- Use `CliConfig` instead of `DefaultConfig` +- Many backward incompatible changes to how `bool` are used. Use Pydantic bool casting (e.g., `--dry-run y`, or `--dry-run true`). +- There's `mypy` related issues with `Field( ......, cli=('-x', '--filter'))`. I don't think pydantic should remove the current `extra` functionality. + + ## Version 4.3.0 - Leverage Pydantic validation for enum choices, enabling more complex use-cases From 912387d820c86244ca18e72fc215ae3d7d212e4d Mon Sep 17 00:00:00 2001 From: mpkocher Date: Thu, 18 Jul 2024 18:53:35 -0700 Subject: [PATCH 07/10] Update circleci config --- .circleci/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index fe755b5..f3cad01 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -8,7 +8,7 @@ jobs: docker: # specify the version you desire here # use `-browsers` prefix for selenium tests, e.g. `3.6.1-browsers` - - image: circleci/python:3.7.4 + - image: cimg/python:3.12.4 # Specify service dependencies here if necessary # CircleCI maintains a library of pre-built images @@ -56,4 +56,4 @@ jobs: - store_artifacts: path: test-reports - destination: test-reports \ No newline at end of file + destination: test-reports From 6f6abe25c5d56cc24e6a6ec7bfd4d1d4a6e17c44 Mon Sep 17 00:00:00 2001 From: mpkocher Date: Thu, 18 Jul 2024 19:28:07 -0700 Subject: [PATCH 08/10] Migrate to Github CI --- .github/workflows/python-app.yml | 34 ++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 .github/workflows/python-app.yml diff --git a/.github/workflows/python-app.yml b/.github/workflows/python-app.yml new file mode 100644 index 0000000..be9ec4a --- /dev/null +++ b/.github/workflows/python-app.yml @@ -0,0 +1,34 @@ +# This workflow will install Python dependencies, run tests and lint with a single version of Python +# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python + +name: Pydantic-CLI + +on: + push: + branches: [ "master" ] + pull_request: + branches: [ "master" ] + +permissions: + contents: read + +jobs: + build: + + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + - name: Set up Python 3.10 + uses: actions/setup-python@v3 + with: + python-version: "3.10" + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -t REQUIREMENTS.txt + pip install shtab + pip install -r REQUIREMENTS-TEST.txt + - name: Test with pytest + run: | + pytest From 3023efab414cb5fd395fe39ba7074074e967c1a9 Mon Sep 17 00:00:00 2001 From: mpkocher Date: Thu, 18 Jul 2024 19:30:47 -0700 Subject: [PATCH 09/10] Fix typo in CI config --- .github/workflows/python-app.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python-app.yml b/.github/workflows/python-app.yml index be9ec4a..b22fdd7 100644 --- a/.github/workflows/python-app.yml +++ b/.github/workflows/python-app.yml @@ -26,7 +26,7 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - pip install -t REQUIREMENTS.txt + pip install -r REQUIREMENTS.txt pip install shtab pip install -r REQUIREMENTS-TEST.txt - name: Test with pytest From 9fd4148d458c63e1e3f93fe9131837fb5aa8e750 Mon Sep 17 00:00:00 2001 From: "M. Kocher" Date: Thu, 18 Jul 2024 19:39:43 -0700 Subject: [PATCH 10/10] Remove circle ci --- .circleci/config.yml | 59 -------------------------------------------- 1 file changed, 59 deletions(-) delete mode 100644 .circleci/config.yml diff --git a/.circleci/config.yml b/.circleci/config.yml deleted file mode 100644 index f3cad01..0000000 --- a/.circleci/config.yml +++ /dev/null @@ -1,59 +0,0 @@ -# Python CircleCI 2.0 configuration file -# -# Check https://circleci.com/docs/2.0/language-python/ for more details -# -version: 2 -jobs: - build: - docker: - # specify the version you desire here - # use `-browsers` prefix for selenium tests, e.g. `3.6.1-browsers` - - image: cimg/python:3.12.4 - - # Specify service dependencies here if necessary - # CircleCI maintains a library of pre-built images - # documented at https://circleci.com/docs/2.0/circleci-images/ - # - image: circleci/postgres:9.4 - - working_directory: ~/repo - - steps: - - checkout - - # Download and cache dependencies - - restore_cache: - keys: - - v1-dependencies-{{ checksum "REQUIREMENTS.txt" }} - # fallback to using the latest cache if no exact match is found - - v1-dependencies- - - - run: - name: install dependencies - command: | - python3 -m venv venv - . venv/bin/activate - pip install -r REQUIREMENTS.txt - pip install shtab - pip install -r REQUIREMENTS-TEST.txt - - - save_cache: - paths: - - ./venv - key: v1-dependencies-{{ checksum "REQUIREMENTS.txt" }} - - # run tests! - # this example uses Django's built-in test-runner - # other common Python testing frameworks include pytest and nose - # https://pytest.org - # https://nose.readthedocs.io - - run: - name: run tests - command: | - . venv/bin/activate - pytest . - mypy pydantic_cli - black --check pydantic_cli - - - store_artifacts: - path: test-reports - destination: test-reports