Make sensitive_data_tools.obfuscate_value_if_sensitive() [minor] (#87)

Co-authored-by: github-actions <[email protected]>

.github/workflows/wipac-cicd.yml

@@ -4,18 +4,34 @@ on: [push]
 
 jobs:
 
+  py-versions:
+    runs-on: ubuntu-latest
+    outputs:
+      matrix: ${{ steps.versions.outputs.matrix }}
+    steps:
+      - uses: actions/checkout@v3
+      - id: versions
+        uses: WIPACrepo/[email protected]
+
   flake8:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
-      - uses: actions/setup-python@v4
+      - uses: actions/setup-python@v3
       - uses: WIPACrepo/[email protected]
 
   mypy:
+    needs: [py-versions]
     runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        py3: ${{ fromJSON(needs.py-versions.outputs.matrix) }}
     steps:
       - uses: actions/checkout@v3
-      - uses: actions/setup-python@v4
+      - uses: actions/setup-python@v3
+        with:
+          python-version: ${{ matrix.py3 }}
       - uses: WIPACrepo/[email protected]
 
   py-setup:
@@ -38,18 +54,9 @@ jobs:
           github.ref_type == 'branch' &&
           format('refs/heads/{0}', github.event.repository.default_branch) != github.ref
         name: wipac-dev-py-setup-action (only for non-dependabot non-default branches)
-        uses: WIPACrepo/wipac-dev-py-setup-action@v2.8
+        uses: WIPACrepo/wipac-dev-py-setup-action@v2.9
         with:
-          base-keywords: "WIPAC IceCube"
-
-  py-versions:
-    runs-on: ubuntu-latest
-    outputs:
-      matrix: ${{ steps.versions.outputs.matrix }}
-    steps:
-      - uses: actions/checkout@v3
-      - id: versions
-        uses: WIPACrepo/[email protected]
+          base-keywords: WIPAC IceCube
 
   tests:
     needs: [py-versions]

dependencies-coloredlogs.log

@@ -6,27 +6,27 @@
 ########################################################################
 #  pip freeze
 ########################################################################
-certifi==2023.7.22
-charset-normalizer==3.3.1
+certifi==2023.11.17
+charset-normalizer==3.3.2
 coloredlogs==15.0.1
 humanfriendly==10.0
 idna==3.4
 requests==2.31.0
 typing_extensions==4.8.0
-urllib3==2.0.7
+urllib3==2.1.0
 ########################################################################
 #  pipdeptree
 ########################################################################
 coloredlogs==15.0.1
 └── humanfriendly [required: >=9.1, installed: 10.0]
 pip==23.2.1
-pipdeptree==2.13.0
+pipdeptree==2.13.1
 setuptools==65.5.1
-wheel==0.41.2
+wheel==0.41.3
 wipac-dev-tools
 ├── requests [required: Any, installed: 2.31.0]
-│   ├── certifi [required: >=2017.4.17, installed: 2023.7.22]
-│   ├── charset-normalizer [required: >=2,<4, installed: 3.3.1]
+│   ├── certifi [required: >=2017.4.17, installed: 2023.11.17]
+│   ├── charset-normalizer [required: >=2,<4, installed: 3.3.2]
 │   ├── idna [required: >=2.5,<4, installed: 3.4]
-│   └── urllib3 [required: >=1.21.1,<3, installed: 2.0.7]
+│   └── urllib3 [required: >=1.21.1,<3, installed: 2.1.0]
 └── typing-extensions [required: Any, installed: 4.8.0]

dependencies-dev.log

@@ -7,14 +7,14 @@
 #  pip freeze
 ########################################################################
 attrs==23.1.0
-certifi==2023.7.22
-charset-normalizer==3.3.1
-filelock==3.12.4
+certifi==2023.11.17
+charset-normalizer==3.3.2
+filelock==3.13.1
 flake8==6.1.0
 idna==3.4
 iniconfig==2.0.0
 mccabe==0.7.0
-mypy==1.6.1
+mypy==1.7.0
 mypy-extensions==1.0.0
 packaging==23.2
 pluggy==1.3.0
@@ -25,12 +25,12 @@ pytest-flake8==1.1.1
 pytest-mypy==0.10.3
 requests==2.31.0
 typing_extensions==4.8.0
-urllib3==2.0.7
+urllib3==2.1.0
 ########################################################################
 #  pipdeptree
 ########################################################################
 pip==23.2.1
-pipdeptree==2.13.0
+pipdeptree==2.13.1
 pytest-flake8==1.1.1
 ├── flake8 [required: >=4.0, installed: 6.1.0]
 │   ├── mccabe [required: >=0.7.0,<0.8.0, installed: 0.7.0]
@@ -42,20 +42,20 @@ pytest-flake8==1.1.1
     └── pluggy [required: >=0.12,<2.0, installed: 1.3.0]
 pytest-mypy==0.10.3
 ├── attrs [required: >=19.0, installed: 23.1.0]
-├── filelock [required: >=3.0, installed: 3.12.4]
-├── mypy [required: >=0.900, installed: 1.6.1]
+├── filelock [required: >=3.0, installed: 3.13.1]
+├── mypy [required: >=0.900, installed: 1.7.0]
 │   ├── mypy-extensions [required: >=1.0.0, installed: 1.0.0]
 │   └── typing-extensions [required: >=4.1.0, installed: 4.8.0]
 └── pytest [required: >=6.2, installed: 7.4.3]
     ├── iniconfig [required: Any, installed: 2.0.0]
     ├── packaging [required: Any, installed: 23.2]
     └── pluggy [required: >=0.12,<2.0, installed: 1.3.0]
 setuptools==65.5.1
-wheel==0.41.2
+wheel==0.41.3
 wipac-dev-tools
 ├── requests [required: Any, installed: 2.31.0]
-│   ├── certifi [required: >=2017.4.17, installed: 2023.7.22]
-│   ├── charset-normalizer [required: >=2,<4, installed: 3.3.1]
+│   ├── certifi [required: >=2017.4.17, installed: 2023.11.17]
+│   ├── charset-normalizer [required: >=2,<4, installed: 3.3.2]
 │   ├── idna [required: >=2.5,<4, installed: 3.4]
-│   └── urllib3 [required: >=1.21.1,<3, installed: 2.0.7]
+│   └── urllib3 [required: >=1.21.1,<3, installed: 2.1.0]
 └── typing-extensions [required: Any, installed: 4.8.0]

dependencies-mypy.log

@@ -7,16 +7,16 @@
 #  pip freeze
 ########################################################################
 attrs==23.1.0
-certifi==2023.7.22
-charset-normalizer==3.3.1
+certifi==2023.11.17
+charset-normalizer==3.3.2
 coloredlogs==15.0.1
-filelock==3.12.4
+filelock==3.13.1
 flake8==6.1.0
 humanfriendly==10.0
 idna==3.4
 iniconfig==2.0.0
 mccabe==0.7.0
-mypy==1.6.1
+mypy==1.7.0
 mypy-extensions==1.0.0
 packaging==23.2
 pluggy==1.3.0
@@ -27,14 +27,14 @@ pytest-flake8==1.1.1
 pytest-mypy==0.10.3
 requests==2.31.0
 typing_extensions==4.8.0
-urllib3==2.0.7
+urllib3==2.1.0
 ########################################################################
 #  pipdeptree
 ########################################################################
 coloredlogs==15.0.1
 └── humanfriendly [required: >=9.1, installed: 10.0]
 pip==23.2.1
-pipdeptree==2.13.0
+pipdeptree==2.13.1
 pytest-flake8==1.1.1
 ├── flake8 [required: >=4.0, installed: 6.1.0]
 │   ├── mccabe [required: >=0.7.0,<0.8.0, installed: 0.7.0]
@@ -46,20 +46,20 @@ pytest-flake8==1.1.1
     └── pluggy [required: >=0.12,<2.0, installed: 1.3.0]
 pytest-mypy==0.10.3
 ├── attrs [required: >=19.0, installed: 23.1.0]
-├── filelock [required: >=3.0, installed: 3.12.4]
-├── mypy [required: >=0.900, installed: 1.6.1]
+├── filelock [required: >=3.0, installed: 3.13.1]
+├── mypy [required: >=0.900, installed: 1.7.0]
 │   ├── mypy-extensions [required: >=1.0.0, installed: 1.0.0]
 │   └── typing-extensions [required: >=4.1.0, installed: 4.8.0]
 └── pytest [required: >=6.2, installed: 7.4.3]
     ├── iniconfig [required: Any, installed: 2.0.0]
     ├── packaging [required: Any, installed: 23.2]
     └── pluggy [required: >=0.12,<2.0, installed: 1.3.0]
 setuptools==65.5.1
-wheel==0.41.2
+wheel==0.41.3
 wipac-dev-tools
 ├── requests [required: Any, installed: 2.31.0]
-│   ├── certifi [required: >=2017.4.17, installed: 2023.7.22]
-│   ├── charset-normalizer [required: >=2,<4, installed: 3.3.1]
+│   ├── certifi [required: >=2017.4.17, installed: 2023.11.17]
+│   ├── charset-normalizer [required: >=2,<4, installed: 3.3.2]
 │   ├── idna [required: >=2.5,<4, installed: 3.4]
-│   └── urllib3 [required: >=1.21.1,<3, installed: 2.0.7]
+│   └── urllib3 [required: >=1.21.1,<3, installed: 2.1.0]
 └── typing-extensions [required: Any, installed: 4.8.0]

dependencies.log

@@ -6,23 +6,23 @@
 ########################################################################
 #  pip freeze
 ########################################################################
-certifi==2023.7.22
-charset-normalizer==3.3.1
+certifi==2023.11.17
+charset-normalizer==3.3.2
 idna==3.4
 requests==2.31.0
 typing_extensions==4.8.0
-urllib3==2.0.7
+urllib3==2.1.0
 ########################################################################
 #  pipdeptree
 ########################################################################
 pip==23.2.1
-pipdeptree==2.13.0
+pipdeptree==2.13.1
 setuptools==65.5.1
-wheel==0.41.2
+wheel==0.41.3
 wipac-dev-tools
 ├── requests [required: Any, installed: 2.31.0]
-│   ├── certifi [required: >=2017.4.17, installed: 2023.7.22]
-│   ├── charset-normalizer [required: >=2,<4, installed: 3.3.1]
+│   ├── certifi [required: >=2017.4.17, installed: 2023.11.17]
+│   ├── charset-normalizer [required: >=2,<4, installed: 3.3.2]
 │   ├── idna [required: >=2.5,<4, installed: 3.4]
-│   └── urllib3 [required: >=1.21.1,<3, installed: 2.0.7]
+│   └── urllib3 [required: >=1.21.1,<3, installed: 2.1.0]
 └── typing-extensions [required: Any, installed: 4.8.0]

tests/test_data_safety_tools.py

@@ -0,0 +1,20 @@
+"""Tests for data_safety_tools.py."""
+
+
+from wipac_dev_tools import data_safety_tools
+
+
+def test_00() -> None:
+    """Test with usual values."""
+    senstives = ["my_token", "AUTHOR", "secretive_number", "YouShallNotPass"]
+
+    unimportant_value = "12345"
+
+    for name in ["foo", "bar", "baz"] + senstives:
+        print(name)
+        actual = data_safety_tools.obfuscate_value_if_sensitive(name, unimportant_value)
+        print(actual)
+        if name in senstives:
+            assert actual == "***"
+        else:
+            assert actual == unimportant_value

tests/wipac_dev_tools_test.py

@@ -14,6 +14,7 @@ def test_available() -> None:
         "logging_tools",
         "strtobool",
         "argparse_tools",
+        "data_safety_tools",
     }
     assert set(wipac_dev_tools.__all__) == all_of_em
 

wipac_dev_tools/__init__.py

@@ -1,6 +1,6 @@
 """Init."""
 
-from . import argparse_tools, logging_tools
+from . import argparse_tools, data_safety_tools, logging_tools
 from .enviro_tools import from_environment, from_environment_as_dataclass  # noqa
 from .setup_tools import SetupShop  # noqa
 from .strtobool import strtobool
@@ -12,6 +12,7 @@
     "logging_tools",
     "strtobool",
     "argparse_tools",
+    "data_safety_tools",
 ]
 
 # version is a human-readable version number.

wipac_dev_tools/data_safety_tools.py

@@ -0,0 +1,11 @@
+"""Tools for handling sensitive data."""
+
+_OBFUSCATE_SUBSTRINGS_UPPER = ["TOKEN", "AUTH", "PASS", "SECRET"]
+
+
+def obfuscate_value_if_sensitive(name: str, value: str) -> str:
+    """Return "***" if the included `value` is sensitive (by its `name`)."""
+    if any(s in name.upper() for s in _OBFUSCATE_SUBSTRINGS_UPPER):
+        return "***"
+    else:
+        return value

wipac_dev_tools/logging_tools.py

@@ -6,9 +6,11 @@
 
 from typing_extensions import Literal  # will redirect to Typing for 3.8+
 
+from .data_safety_tools import obfuscate_value_if_sensitive
+
 # fmt: off
 if TYPE_CHECKING:  # _typeshed only exists at runtime
-    from _typeshed import DataclassInstance  # type: ignore[attr-defined]
+    from _typeshed import DataclassInstance
     DataclassT = TypeVar("DataclassT", bound=DataclassInstance)
 else:
     DataclassT = TypeVar("DataclassT")
@@ -20,9 +22,6 @@
 # ---------------------------------------------------------------------------------------
 
 
-OBFUSCATE_SUBSTRINGS_UPPER = ["TOKEN", "AUTH", "PASS", "SECRET"]
-
-
 LoggerLevel = Literal[
     "CRITICAL",
     "ERROR",
@@ -80,10 +79,7 @@ def log_argparse_args(
     logger_fn = get_logger_fn(logger, level)
 
     for arg, val in vars(args).items():
-        if any(s in arg.upper() for s in OBFUSCATE_SUBSTRINGS_UPPER):
-            logger_fn(f"{arg}: ***")
-        else:
-            logger_fn(f"{arg}: {val}")
+        logger_fn(f"{arg}: {obfuscate_value_if_sensitive(arg, val)}")
 
     return args
 
@@ -112,8 +108,7 @@ def log_dataclass(
     for field in dataclasses.fields(dclass):
         val = getattr(dclass, field.name)
         if obfuscate_sensitive_substrings:
-            if any(s in field.name.upper() for s in OBFUSCATE_SUBSTRINGS_UPPER):
-                val = "***"
+            val = obfuscate_value_if_sensitive(field.name, val)
         logger_fn(f"{prefix+' 'if prefix else ''}{field.name}: {val}")
 
     return dclass
@@ -193,7 +188,7 @@ def _set_level(
     # root
     if use_coloredlogs:
         try:
-            import coloredlogs  # type: ignore[import]  # pylint: disable=import-outside-toplevel
+            import coloredlogs  # type: ignore[import-untyped]  # pylint: disable=import-outside-toplevel
 
             coloredlogs.install(level=first_party_level)  # root
         except ImportError: