From 894be0d4394cd1d8838d127dca10aeab729fb096 Mon Sep 17 00:00:00 2001 From: akhlakm Date: Sat, 24 Feb 2024 22:28:28 -0500 Subject: [PATCH] Add settings module to load yaml configs --- README.md | 15 +-- examples/settings.py | 37 ++++++++ examples/settings.yaml | 4 + make.sh | 7 +- pylogg/settings.py | 209 +++++++++++++++++++++++++++++++++++++++++ pyproject.toml | 6 +- 6 files changed, 261 insertions(+), 17 deletions(-) create mode 100644 examples/settings.py create mode 100644 examples/settings.yaml create mode 100644 pylogg/settings.py diff --git a/README.md b/README.md index ad4f7d7..6579e07 100644 --- a/README.md +++ b/README.md @@ -1,14 +1,5 @@ -# Python-PyLogg -A personally opinionated logging package in Python. - -Features: -- Colors in console. -- Saving log messages into a single file. -- Support for named sub-loggers. -- Override of log levels and settings from the main function. -- Eight levels of verbosity. -- Wrapping and shortening of long messages. -- Automatic logging of elapsed times for long processes. +# PyLogg +Logging and YAML-based configuration modules in Python. ## Installation You can install this package from PyPI with `pip`. @@ -107,4 +98,4 @@ pre-commit install ``` ## About -LICENSE MIT Copyright 2023 Akhlak Mahmood +LICENSE MIT Copyright 2024 Akhlak Mahmood diff --git a/examples/settings.py b/examples/settings.py new file mode 100644 index 0000000..476b744 --- /dev/null +++ b/examples/settings.py @@ -0,0 +1,37 @@ +from typing import NamedTuple + +from pylogg.settings import YAMLSettings + +# Load settings from `settings.yaml` file and environment variables. +# If a file is given as the first argument, load it as the YAML file. +# Environment variables will be searched as `SETT_TEST_ROW1` etc. +# Prefer environment variable definitions over YAML definitions. +yaml = YAMLSettings( + 'sett', first_arg_as_file=True, load_env=True, prefer_env=True) + + +# Define a classmethod to load the settings. +class Test(NamedTuple): + row1: float = 23.6 + row2: str = 'Hello' + row3: str = 'world' + + @classmethod + def settings(c) -> 'Test': return yaml(c) + + +if __name__ == '__main__': + # Use the class method to load the settings. + test = Test.settings() + print(test) + + # Settings can be globally set/updated. + updated = test._replace(row3='Earth') + yaml.set(Test, updated) + + # This now has updated values. + print(yaml) + + if not yaml.is_loaded(): + # Write to the YAML file. + yaml.save() diff --git a/examples/settings.yaml b/examples/settings.yaml new file mode 100644 index 0000000..ecf297e --- /dev/null +++ b/examples/settings.yaml @@ -0,0 +1,4 @@ +Test: + row1: 23.6 + row2: Hello + row3: Earth diff --git a/make.sh b/make.sh index 2cfd15a..d8023d6 100755 --- a/make.sh +++ b/make.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash publish() { - ## Bump the version number + # Bump the version number. grep version pyproject.toml VERSION=$(sed -n 's/version = "\(.*\)"/\1/p' pyproject.toml) VERSION=$(python -c "v='$VERSION'.split('.');print('%s.%s.%d' %(v[0], v[1], int(v[2])+1))") @@ -9,13 +9,14 @@ publish() { sed -i "s/\(version = \"\)[^\"]*\"/\1$VERSION\"/" pyproject.toml sed -i "s/\(__version__ = \"\)[^\"]*\"/\1$VERSION\"/" pylogg/__init__.py grep version pyproject.toml - git add pyproject.toml pylogg/__init__.py + # Add to git and push. + git add pyproject.toml pylogg/__init__.py git commit -m "Bump version" git push # Create a new git tag using the pyproject.toml version - # and push the tag to origin + # and push the tag to origin. version=$(sed -n 's/version = "\(.*\)"/\1/p' pyproject.toml) git tag v$version && git push origin v$version diff --git a/pylogg/settings.py b/pylogg/settings.py new file mode 100644 index 0000000..c4ca157 --- /dev/null +++ b/pylogg/settings.py @@ -0,0 +1,209 @@ +""" + A simple module to load configurations from environment varibles or + a YAML file. + + Subclass the typing.NamedTuple, and define a classmethod to load the + settings. + + yaml = YAMLSettings() + + class Test(NamedTuple): + name: str = 'hello' + + @classmethod + def settings(cls) -> 'Test': return yaml(cls) + + test = Test.settings() + print(test.name) + +""" + +import os +import sys +from typing import NamedTuple + +import yaml + +# For access to the __annotations__ attribute. +assert sys.version_info >= (3, 10), "Minimum Python 3.10 required" + + +class YAMLSettings: + """ + Load settings from environment variables and/or a YAML file. + + name: + Name of the settings. Used as the prefix for environment variables. + + yamlfile: + YAML file to load the settings. + + first_arg_as_file: + Treat the first argument as the settings YAML file if any. + + load_env: + Load settings from environment variables or not. + + prefer_env: + Override YAML vars with environment variables or vice versa. + """ + + def __init__(self, name : str, + yamlfile : str = 'settings.yaml', first_arg_as_file : bool = True, + load_env : bool = True, prefer_env : bool = True): + + # Prefix of the env vars. + self.name = name + + # Environment variables. + self.env = os.environ if load_env else {} + + # Override file vars with env vars. + self.prefer_env = prefer_env + + # Take the first argument as the settings file if any. + self.file = (sys.argv[1] if len(sys.argv) > 1 else yamlfile) \ + if first_arg_as_file else yamlfile + + # YAML file loaded variables. + self.yamlvars : dict[str, dict[str, any]] = {} + + # Cache for the processed sections. + self.cache = {} + + self.load_file() + + + def __repr__(self) -> str: + s = self.__class__.__name__ + ": " + for k, v in self.cache.items(): + s += f"{k} {v._asdict()}" + return s + + def __call__(self, cls : NamedTuple): + """ Populate settings to the current class/section. """ + + assert hasattr(cls, '_fields'), "Settings must be a NamedTuple." + + fields = {} + classname : str = cls.__name__ + + if classname in self.cache: + return self.cache[classname] + + # for each namedtuple fields ... + for field in cls._fields: + data_type = cls.__annotations__[field] + value = cls._field_defaults.get(field, None) + + if value is not None: + try: + value = data_type(value) + except: + raise ValueError( + f"Invalid default for {classname}.{field}: {value}") + + env_var_name = \ + f"{self.name.upper()}_{classname.upper()}_{field.upper()}" + + if not self.prefer_env: + value = self._get_env(env_var_name, value) + + # Override with YAML file vars. + value = self._get_yaml(classname, field, value) + + # Override with env vars. + if self.prefer_env: + value = self._get_env(env_var_name, value) + + # Convert to expected type. + if value is not None: + try: + value = data_type(value) + except: + raise ValueError( + f"Invalid type for {classname}.{field}: {value}") + + # Add to class fields + fields[field] = value + + # Cache for future. + self.cache[classname] = cls(**fields) + + # Return initialized class. + return self.cache[classname] + + + def _get_env(self, var_name, default_value): + return self.env.get(var_name, default_value) + + + def _get_yaml(self, classname, var_name, default_value): + if classname in self.yamlvars: + return self.yamlvars[classname].get(var_name, default_value) + else: + return default_value + + + def set(self, cls : type, instance : NamedTuple): + """ Set the cached instance of a class with new version. + Useful to update the global settings or writing new sections + to YAML file. + """ + assert type(instance) == cls, "Class and instance do not match" + + classname = cls.__name__ + self.cache[classname] = instance + + + def load_file(self): + """ Load the variables of the YAML file. """ + if os.path.isfile(self.file): + yamlfile = yaml.safe_load(open(self.file)) + for section, fields in yamlfile.items(): + for fieldname, value in fields.items(): + if section not in self.yamlvars: + self.yamlvars[section] = {} + self.yamlvars[section][fieldname] = value + + + def is_loaded(self) -> bool: + """ Returns True if at least one section from YAML file was loaded. """ + return len(self.yamlvars) > 0 + + + def save(self, *sections : NamedTuple, yamlfile : str = None): + """ Save the given sections to YAML file. + If no section is specified, all sections are written. + If no yamlfile is given, the initial file is used. + """ + configs = {} + outfile = yamlfile if yamlfile is not None else self.file + + # Use all sections + if len(sections) == 0: + sections = self.cache.values() + + # For each NamedTuple section + for cls in sections: + assert hasattr(cls, '_fields'), "Section must be a NamedTuple" + + try: + # class + section = cls.__name__ + configs[section] = cls.get()._asdict() + + except AttributeError: + # instance + section = cls.__class__.__name__ + configs[section] = cls._asdict() + + yaml.safe_dump(configs, open(outfile, 'w'), indent=4) + print("Save OK:", outfile) + + + def copy_sample(self, sample_file : str): + """ Copy a sample YAML file to the current settings file. """ + import shutil + shutil.copyfile(sample_file, self.file) + print("Save OK:", self.file) diff --git a/pyproject.toml b/pyproject.toml index 67a4e65..11097e6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ version = "0.1.18" authors = [ { name="Akhlak Mahmood", email="akhlakm@gatech.edu" }, ] -description = "Personally opinionated logger in Python." +description = "Logging and YAML-based configuration modules in Python." readme = "README.md" requires-python = ">=3.0" @@ -16,7 +16,9 @@ classifiers = [ "License :: OSI Approved :: MIT License", ] -dependencies = [] +dependencies = [ + "pyyaml", +] [project.optional-dependencies] dev = [