From f8ba759ad250ddf15b203ab54d74f9b6ef72d446 Mon Sep 17 00:00:00 2001 From: Tobias Ahrens Date: Mon, 15 May 2023 10:54:32 +0200 Subject: [PATCH] Limit Wildcards support to the LabOne wildcards --- CHANGELOG.md | 5 +- docs/source/spelling_wordlist.txt | 4 +- .../toolkit/nodetree/connection_dict.py | 12 ++ src/zhinst/toolkit/nodetree/helper.py | 15 ++ src/zhinst/toolkit/nodetree/node.py | 133 ++++-------------- tests/test_hdawg.py | 20 +-- tests/test_nodetree.py | 114 ++++++++++++--- 7 files changed, 156 insertions(+), 147 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1107113a..f33d2d67 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,9 @@ # zhinst-toolkit Changelog -## Version 0.5.4 +## Version 0.6.0 +* Revert full support of `fnmatch` wildcards and instead use the LabOne wildcard support. + This means only `*` symbols are supported. A `*` in the middle of the path matches + everything instead of a `/`. A `*` at the end of the path matches everything. * `device.factory_reset` now raises an exception if the factory reset was not successful (`#243`). * Fixed issue where calling a `Node` with `dir()` returned duplicate values on some nodes. diff --git a/docs/source/spelling_wordlist.txt b/docs/source/spelling_wordlist.txt index 926d3904..ae563264 100644 --- a/docs/source/spelling_wordlist.txt +++ b/docs/source/spelling_wordlist.txt @@ -132,4 +132,6 @@ subtree misconfiguration pre forwardwave -backwardwave \ No newline at end of file +backwardwave +labone +LabOne \ No newline at end of file diff --git a/src/zhinst/toolkit/nodetree/connection_dict.py b/src/zhinst/toolkit/nodetree/connection_dict.py index 3a80ddab..7482c411 100644 --- a/src/zhinst/toolkit/nodetree/connection_dict.py +++ b/src/zhinst/toolkit/nodetree/connection_dict.py @@ -45,6 +45,11 @@ def _get_value(self, path: str) -> t.Any: return value() return value + def _resolve_wildcards(self, path: str) -> t.List[str]: + path_raw = path.replace("/\\*/", "/[^/]*/") + path_raw_regex = re.compile(path_raw) + return list(filter(path_raw_regex.match, self._values.keys())) + def _set_value(self, path: str, value: t.Any) -> None: """Set the value for a given path. @@ -54,6 +59,13 @@ def _set_value(self, path: str, value: t.Any) -> None: path: Key in the internal values dictionary. value: New value of the path. """ + paths = self._resolve_wildcards(path) + if not paths: + raise KeyError(path) + for path in paths: + self._do_set_value(path, value) + + def _do_set_value(self, path: str, value: t.Any) -> None: if callable(self._values[path]): self._values[path](value) else: diff --git a/src/zhinst/toolkit/nodetree/helper.py b/src/zhinst/toolkit/nodetree/helper.py index b19178ff..51028c50 100644 --- a/src/zhinst/toolkit/nodetree/helper.py +++ b/src/zhinst/toolkit/nodetree/helper.py @@ -3,6 +3,7 @@ from contextlib import contextmanager from functools import lru_cache from collections.abc import Mapping +import re # TypedDict is available in the typing module since 3.8 # Ift we only support 3.8 we should switch to t.TypedDict @@ -73,6 +74,20 @@ def create_or_append_set_transaction(nodetree) -> t.Generator[None, None, None]: yield +def resolve_wildcards_labone(path: str, nodes: t.List[str]) -> t.List[str]: + """Resolves potential wildcards. + + Also will resolve partial nodes to its leaf nodes. + + Returns: + List of matched nodes in the raw path format + """ + node_raw = re.escape(path) + node_raw = node_raw.replace("/\\*/", "/[^/]*/").replace("/\\*", "/*") + "(/.*)?$" + node_raw_regex = re.compile(node_raw) + return list(filter(node_raw_regex.match, nodes)) + + class NodeDict(Mapping): """Mapping of dictionary structure results. diff --git a/src/zhinst/toolkit/nodetree/node.py b/src/zhinst/toolkit/nodetree/node.py index 9b041f10..3bf08e32 100644 --- a/src/zhinst/toolkit/nodetree/node.py +++ b/src/zhinst/toolkit/nodetree/node.py @@ -12,13 +12,11 @@ from functools import lru_cache from zhinst.toolkit.nodetree.helper import ( - create_or_append_set_transaction, - lazy_property, NodeDict, + lazy_property, + resolve_wildcards_labone, ) -from zhinst.core.errors import CoreError - if t.TYPE_CHECKING: # pragma: no cover from zhinst.toolkit.nodetree import NodeTree @@ -563,8 +561,8 @@ def _resolve_wildcards(self) -> t.List[str]: Returns: List of matched nodes in the raw path format """ - return fnmatch.filter( - self._root.raw_dict.keys(), self._root.node_to_raw_path(self) + "*" + return resolve_wildcards_labone( + self._root.node_to_raw_path(self), self._root.raw_dict.keys() ) def _parse_get_value( @@ -711,16 +709,7 @@ def _get_wildcard( result_raw = self._root.connection.get(self.node_info.path, **kwargs) except TypeError: del kwargs["settingsonly"] - try: - result_raw = self._root.connection.get(self.node_info.path, **kwargs) - except (RuntimeError, TypeError): - # resolve wildecard and get the value of the resulting leaf nodes - nodes_raw = self._resolve_wildcards() - result_raw = self._root.connection.get(",".join(nodes_raw), **kwargs) - except RuntimeError: - # resolve wildecard and get the value of the resulting leaf nodes - nodes_raw = self._resolve_wildcards() - result_raw = self._root.connection.get(",".join(nodes_raw), **kwargs) + result_raw = self._root.connection.get(self.node_info.path, **kwargs) if not result_raw: raise KeyError(self.node_info.path) if not kwargs["flat"]: @@ -831,7 +820,7 @@ def _set( TypeError: Connection does not support deep set """ writable = self.node_info.writable - if writable: + if writable or self.node_info.contains_wildcards: if parse: value = self.node_info.set_parser(value) if self._root.transaction.in_progress(): @@ -852,35 +841,12 @@ def _set( else: raise return None - if writable is None and ( - self.node_info.contains_wildcards or self.node_info.is_partial - ): - self._set_wildcard(value, parse=parse, **kwargs) - return None + if self.node_info.is_partial: + return self["*"](value, deep=deep, enum=enum, parse=parse, **kwargs) if writable is False: raise AttributeError(f"{self.node_info.path} is read-only.") raise KeyError(self.node_info.path) - def _set_wildcard(self, value: t.Any, parse: bool = True, **kwargs) -> None: - """Performs a transactional set on all nodes that match the wildcard. - - The kwargs will be forwarded to the mapped zhinst.core function call. - - Args: - value: value - parse: Flag if the SetParser, if present, should be applied or not. - (default=True) - - Raises: - KeyError: if the wildcard does not resolve to a valid node - """ - nodes_raw = self._resolve_wildcards() - if not nodes_raw: - raise KeyError(self._root.node_to_raw_path(self)) - with create_or_append_set_transaction(self._root): - for node_raw in nodes_raw: - self._root.raw_path_to_node(node_raw)(value, parse=parse, **kwargs) - def _set_deep(self, value: t.Any, **kwargs) -> t.Any: """Set the node value from device. @@ -1004,11 +970,7 @@ def subscribe(self) -> None: try: self._root.connection.subscribe(self.node_info.path) except RuntimeError as error: - nodes_raw = self._resolve_wildcards() - if not nodes_raw: - raise KeyError(self.node_info.path) from error - for node_raw in nodes_raw: - self._root.connection.subscribe(node_raw) + raise KeyError(self.node_info.path) from error def unsubscribe(self) -> None: """Unsubscribe this node (its child lead nodes). @@ -1019,11 +981,7 @@ def unsubscribe(self) -> None: try: self._root.connection.unsubscribe(self.node_info.path) except RuntimeError as error: - nodes_raw = self._resolve_wildcards() - if not nodes_raw: - raise KeyError(self.node_info.path) from error - for node_raw in nodes_raw: - self._root.connection.unsubscribe(node_raw) + raise KeyError(self.node_info.path) from error def get_as_event(self) -> None: """Trigger an event for that node (its child lead nodes). @@ -1033,11 +991,7 @@ def get_as_event(self) -> None: try: self._root.connection.getAsEvent(self.node_info.path) except RuntimeError as error: - nodes_raw = self._resolve_wildcards() - if not nodes_raw: - raise KeyError(self.node_info.path) from error - for node_raw in nodes_raw: - self._root.connection.getAsEvent(node_raw) + raise KeyError(self.node_info.path) from error def child_nodes( self, @@ -1050,7 +1004,6 @@ def child_nodes( basechannelonly: bool = False, excludestreaming: bool = False, excludevectors: bool = False, - full_wildcard: bool = False, ) -> t.Generator["Node", None, None]: """Generator for all child nodes that matches the filters. @@ -1086,62 +1039,24 @@ def child_nodes( of multiple channels (default: False). excludestreaming: Exclude streaming nodes (default: False). excludevectors: Exclude vector nodes (default: False). - full_wildcard: Enables full wildcard support. Per default - only the asterisk wildcard is supported. (Automatically sets - recursive and leavesonly) (default = False) Returns: Generator of all child nodes that match the filters """ raw_path = self._root.node_to_raw_path(self) - try: - raw_result = self._root.connection.listNodes( - raw_path, - recursive=recursive, - leavesonly=leavesonly, - settingsonly=settingsonly, - streamingonly=streamingonly, - subscribedonly=subscribedonly, - basechannelonly=basechannelonly, - excludestreaming=excludestreaming, - excludevectors=excludevectors, - ) - for node_raw in raw_result: - yield self._root.raw_path_to_node(node_raw) - except CoreError as error: - if error.code != 32768: # Replace with correct error in 23.02 - raise - if not full_wildcard: - raise RuntimeError( - "The node contains wildcards that the DataServer can not resolve. " - "Use the `full_wildcard` flag to search for child nodes manually." - ) from error - nodes_raw = self._resolve_wildcards() - for node_raw in nodes_raw: - node = self._root.raw_path_to_node(node_raw) - if not ( - (settingsonly and not node.node_info.is_setting) - or (excludevectors and node.node_info.is_vector) - or ( - basechannelonly - and any(number != 0 for number in re.findall(r"\d+", node_raw)) - ) - or ( - (excludestreaming or subscribedonly or streamingonly) - and not self._root.connection.listNodes( - node_raw[:-1] + "*", # TODO remove once listNodes is fixed - recursive=recursive, - leavesonly=leavesonly, - settingsonly=settingsonly, - streamingonly=streamingonly, - subscribedonly=subscribedonly, - basechannelonly=basechannelonly, - excludestreaming=excludestreaming, - excludevectors=excludevectors, - ) - ) - ): - yield node + raw_result = self._root.connection.listNodes( + raw_path, + recursive=recursive, + leavesonly=leavesonly, + settingsonly=settingsonly, + streamingonly=streamingonly, + subscribedonly=subscribedonly, + basechannelonly=basechannelonly, + excludestreaming=excludestreaming, + excludevectors=excludevectors, + ) + for node_raw in raw_result: + yield self._root.raw_path_to_node(node_raw) @lru_cache() def is_valid(self) -> bool: diff --git a/tests/test_hdawg.py b/tests/test_hdawg.py index 522258c7..6cf61ae0 100644 --- a/tests/test_hdawg.py +++ b/tests/test_hdawg.py @@ -36,14 +36,8 @@ def test_enable_qccs_mode(mock_connection, hdawg): ("/dev1234/dios/0/interface", 0), ("/dev1234/dios/0/mode", "qccs"), ("/dev1234/dios/0/drive", 12), - ("/dev1234/awgs/0/dio/strobe/slope", "off"), - ("/dev1234/awgs/1/dio/strobe/slope", "off"), - ("/dev1234/awgs/2/dio/strobe/slope", "off"), - ("/dev1234/awgs/3/dio/strobe/slope", "off"), - ("/dev1234/awgs/0/dio/valid/polarity", "none"), - ("/dev1234/awgs/1/dio/valid/polarity", "none"), - ("/dev1234/awgs/2/dio/valid/polarity", "none"), - ("/dev1234/awgs/3/dio/valid/polarity", "none"), + ("/dev1234/awgs/*/dio/strobe/slope", "off"), + ("/dev1234/awgs/*/dio/valid/polarity", "none"), ] ) mock_connection.reset_mock() @@ -55,14 +49,8 @@ def test_enable_qccs_mode(mock_connection, hdawg): ("/dev1234/dios/0/interface", 0), ("/dev1234/dios/0/mode", "qccs"), ("/dev1234/dios/0/drive", 12), - ("/dev1234/awgs/0/dio/strobe/slope", "off"), - ("/dev1234/awgs/1/dio/strobe/slope", "off"), - ("/dev1234/awgs/2/dio/strobe/slope", "off"), - ("/dev1234/awgs/3/dio/strobe/slope", "off"), - ("/dev1234/awgs/0/dio/valid/polarity", "none"), - ("/dev1234/awgs/1/dio/valid/polarity", "none"), - ("/dev1234/awgs/2/dio/valid/polarity", "none"), - ("/dev1234/awgs/3/dio/valid/polarity", "none"), + ("/dev1234/awgs/*/dio/strobe/slope", "off"), + ("/dev1234/awgs/*/dio/valid/polarity", "none"), ] ) diff --git a/tests/test_nodetree.py b/tests/test_nodetree.py index 98eb6c8f..3c4ca513 100644 --- a/tests/test_nodetree.py +++ b/tests/test_nodetree.py @@ -14,6 +14,7 @@ from zhinst.toolkit.nodetree import Node, NodeTree from zhinst.toolkit.nodetree.connection_dict import ConnectionDict from zhinst.toolkit.nodetree.node import NodeList +from zhinst.toolkit.nodetree.helper import resolve_wildcards_labone from zhinst.core.errors import CoreError @@ -274,6 +275,7 @@ def test_wildcard_node(connection): connection.get.return_value = None with pytest.raises(KeyError) as e_info: tree.hello["*"].rate() + connection.set.side_effect = RuntimeError("test") with pytest.raises(KeyError) as e_info: tree.hello["*"].rate(1) @@ -467,12 +469,6 @@ def test_module_get_wildcard(connection): connection.get.side_effect = [TypeError(), return_value] assert tree.demods()[tree.demods[0].impedance] == 123 - connection.get.side_effect = [TypeError(), RuntimeError(), return_value] - assert tree.demods()[tree.demods[0].impedance] == 123 - - connection.get.side_effect = [RuntimeError(), return_value] - assert tree.demods()[tree.demods[0].impedance] == 123 - def test_set(connection): tree = NodeTree(connection, "DEV1234") @@ -506,6 +502,12 @@ def test_set(connection): tree.demods.test("test") +def test_set_partial(connection): + tree = NodeTree(connection, "DEV1234") + tree.demods[0](22.0) + connection.set.assert_called_with(tree.demods[0]["*"].node_info.path, 22.0) + + def test_runtimerror_set_set_vector(connection): # tree.system.impedance.calib.user.data does work with set, but for # testing purposes call it @@ -715,11 +717,6 @@ def test_get_as_event(connection): tree.demods[0].sample.get_as_event() connection.getAsEvent.assert_called_once_with(tree.demods[0].sample.node_info.path) - connection.getAsEvent.side_effect = [RuntimeError(), None, None] - tree.demods["[1,2]"].sample.get_as_event() - connection.getAsEvent.assert_any_call(tree.demods[1].sample.node_info.path) - connection.getAsEvent.assert_any_call(tree.demods[2].sample.node_info.path) - connection.getAsEvent.side_effect = RuntimeError() with pytest.raises(KeyError): tree.demods["*"].test.get_as_event() @@ -807,13 +804,6 @@ def test_child_nodes(connection): list(tree.demods["*"].child_nodes()) assert e_info.value.args[0] == "Unrelated Error" - connection.listNodes.side_effect = CoreError("Path format invalid", 32768) - with pytest.raises(RuntimeError) as e_info: - list(tree.demods["[0,1]"].child_nodes()) - assert "full_wildcard" in e_info.value.args[0] - - assert list(tree.demods["*"].child_nodes(full_wildcard=True)) - def test_subscribe(connection): tree = NodeTree(connection, "DEV1234") @@ -923,9 +913,9 @@ def test_connection_dict(data_dir): assert tree.car.color() == "1" # subscribe and unsubscribe is not supported - with pytest.raises(RuntimeError) as e_info: + with pytest.raises(KeyError) as e_info: tree.car.seat.subscribe() - with pytest.raises(RuntimeError) as e_info: + with pytest.raises(KeyError) as e_info: tree.car.seat.unsubscribe() tree = NodeTree(connection, list_nodes=["/car/*"]) @@ -948,6 +938,26 @@ def test_connection_dict(data_dir): tree.street.length() +def test_connection_dict_set_list(data_dir): + data = {"/car/seat": 4, "/car/color": "blue", "/street/length": 110.4} + json_path = data_dir / "nodedoc_fake.json" + with json_path.open("r", encoding="UTF-8") as file: + nodes_json = json.loads(file.read()) + connection = ConnectionDict(data, nodes_json) + connection.set([("/car/seat", 1), ("/car/color", "red")]) + assert {"/car/seat": 1, "/car/color": "red", "/street/length": 110.4} == data + + +def test_connection_dict_missing_node(data_dir): + data = {"/car/seat": 4, "/car/color": "blue", "/street/length": 110.4} + json_path = data_dir / "nodedoc_fake.json" + with json_path.open("r", encoding="UTF-8") as file: + nodes_json = json.loads(file.read()) + connection = ConnectionDict(data, nodes_json) + with pytest.raises(KeyError): + connection.set("/car/test", 5) + + def test_connection_dict_callable_nodes(data_dir): global seat seat = 6 @@ -1003,6 +1013,70 @@ def test_nodelist_hash(connection, hdawg): hash(bar) == hash(Node(nt, ("foobar",))) +def test_resolve_wildcards_labone_single_ok(): + paths = ["test/hello/world", "test/hello/there", "test/hi/there"] + assert resolve_wildcards_labone("test/hello/world", paths) == ["test/hello/world"] + + +def test_resolve_wildcards_labone_single_fail(): + paths = ["test/hello/world", "test/hello/there", "test/hi/there"] + assert resolve_wildcards_labone("test/hello/a", paths) == [] + + +def test_resolve_wildcards_labone_wildcard_middle_ok(): + paths = ["test/hello/world", "test/hello/there", "test/hi/there"] + assert resolve_wildcards_labone("test/*/there", paths) == [ + "test/hello/there", + "test/hi/there", + ] + + +def test_resolve_wildcards_labone_wildcard_middle_fail(): + paths = ["test/hello/world", "test/hello/there", "test/hi/there"] + assert resolve_wildcards_labone("test/*/you", paths) == [] + + +def test_resolve_wildcards_labone_wildcard_partial_ok(): + paths = ["test/hello/world", "test/hello/there", "test/hi/there"] + assert resolve_wildcards_labone("test", paths) == [ + "test/hello/world", + "test/hello/there", + "test/hi/there", + ] + + +def test_resolve_wildcards_labone_wildcard_partial_fail(): + paths = ["test/hello/world", "test/hello/there", "test/hi/there"] + assert resolve_wildcards_labone("tes", paths) == [] + + +def test_resolve_wildcards_labone_wildcard_partial_subset(): + paths = [ + "/awgs/0/outputs/0/enables/0", + "/awgs/1/outputs/0/enables/0", + "/awgs/0/outputs/1/enables/0", + "/awgs/0/enable", + "/awgs/1/enable", + ] + assert resolve_wildcards_labone("/awgs/*/enable", paths) == [ + "/awgs/0/enable", + "/awgs/1/enable", + ] + + +def test_resolve_wildcards_labone_wildcard_end_ok(): + paths = ["test/hello/world", "test/hello/there", "test/hi/there"] + assert resolve_wildcards_labone("test/hello/*", paths) == [ + "test/hello/world", + "test/hello/there", + ] + + +def test_resolve_wildcards_labone_wildcard_end_fail(): + paths = ["test/hello/world", "test/hello/there", "test/hi/there"] + assert resolve_wildcards_labone("new/hello/*", paths) == [] + + class TestWildCardResult: @pytest.fixture() def node_tree(self, connection):