diff --git a/docs/user_manual/components.md b/docs/user_manual/components.md index 46cd2f028..1d85da7bc 100644 --- a/docs/user_manual/components.md +++ b/docs/user_manual/components.md @@ -594,7 +594,7 @@ Because of this distribution, at least one appliance is required to be connected | name | data type | unit | description | required | update | valid values | | ------------------------ | ----------------------------------------------------------------------------- | ---------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | :---------------------------------: | :------: | :--------------------------------------------------: | | `measured_terminal_type` | {py:class}`MeasuredTerminalType ` | - | indicate if it measures an `appliance` or a `branch` | ✔ | ❌ | the terminal type should match the `measured_object` | -| `power_sigma` | `double` | volt-ampere (VA) | standard deviation of the measurement error. Usually this is the absolute measurement error range divided by 3. See {hoverxreftooltip}`user_manual/components:Power Sensor Concrete Types`. | ✨ only for state estimation. | ✔ | `> 0` | +| `power_sigma` | `double` | volt-ampere (VA) | standard deviation of the measurement error. Usually this is the absolute measurement error range divided by 3. See {hoverxreftooltip}`user_manual/components:Power Sensor Concrete Types`. | ✨ in certain cases for state estimation. See the explanation for [concrete types](#power-sensor-concrete-types) below. | ✔ | `> 0` | #### Power Sensor Concrete Types @@ -615,6 +615,19 @@ the meaning of `RealValueInput` is different, as shown in the table below. | `p_sigma` | `RealValueInput` | watt (W) | standard deviation of the active power measurement error. Usually this is the absolute measurement error range divided by 3. | ❌ see the explanation below. | ✔ | `> 0` | | `q_sigma` | `RealValueInput` | volt-ampere-reactive (var) | standard deviation of the reactive power measurement error. Usually this is the absolute measurement error range divided by 3. | ❌ see the explanation below. | ✔ | `> 0` | +Valid combinations of `power_sigma`, `p_sigma` and `q_sigma` are: + +| `power_sigma` | `p_sigma` | `q_sigma` | result | +|:-------------:|:---------:|:---------:|:--------:| +| x | x | x | ✔ | +| x | x | | ❌ | +| x | | x | ❌ | +| x | | | ✔ | +| | x | x | ✔ | +| | x | | ❌ | +| | | x | ❌ | +| | | | ❌ | + ```{note} 1. If both `p_sigma` and `q_sigma` are provided, they represent the standard deviation of the active and reactive power, respectively, and the value of `power_sigma` is ignored. Any infinite component invalidates the entire measurement. diff --git a/src/power_grid_model/validation/rules.py b/src/power_grid_model/validation/rules.py index c812e5221..bc21878d3 100644 --- a/src/power_grid_model/validation/rules.py +++ b/src/power_grid_model/validation/rules.py @@ -667,7 +667,9 @@ def all_finite(data: SingleDataset, exceptions: Optional[Dict[str, List[str]]] = return errors -def none_missing(data: SingleDataset, component: str, fields: Union[str, List[str]]) -> List[MissingValueError]: +def none_missing( + data: SingleDataset, component: str, fields: Union[List[Union[str, List[str]]], str, List[str]], index: int = 0 +) -> List[MissingValueError]: """ Check that for all records of a particular type of component, the values in the 'fields' columns are not NaN. Returns an empty list on success, or a list containing a single error object on failure. @@ -685,14 +687,17 @@ def none_missing(data: SingleDataset, component: str, fields: Union[str, List[st if isinstance(fields, str): fields = [fields] for field in fields: + if isinstance(field, list): + field = field[0] nan = nan_type(component, field) if np.isnan(nan): - invalid = np.isnan(data[component][field]) + invalid = np.isnan(data[component][field][index]) else: - invalid = np.equal(data[component][field], nan) + invalid = np.equal(data[component][field][index], nan) + if invalid.any(): - if invalid.ndim > 1: - invalid = invalid.any(axis=1) + if isinstance(invalid, np.ndarray): + invalid = np.any(invalid) ids = data[component]["id"][invalid].flatten().tolist() errors.append(MissingValueError(component, field, ids)) return errors diff --git a/src/power_grid_model/validation/validation.py b/src/power_grid_model/validation/validation.py index fa6f36273..0483fab8e 100644 --- a/src/power_grid_model/validation/validation.py +++ b/src/power_grid_model/validation/validation.py @@ -9,8 +9,9 @@ """ import copy +from collections.abc import Sized as ABCSized from itertools import chain -from typing import Dict, List, Optional +from typing import Dict, List, Optional, Union, cast import numpy as np @@ -221,6 +222,43 @@ def validate_ids_exist(update_data: Dict[str, np.ndarray], input_data: SingleDat return list(chain(*errors)) +def _process_power_sigma_and_p_q_sigma( + data: SingleDataset, sensor: str, required_list: Dict[str, List[Union[str, List[str]]]] +) -> None: + """ + Helper function to process the required list when both `p_sigma` and `q_sigma` exist + and valid but `power_sigma` is missing. The field `power_sigma` is set to the norm of + `p_sigma` and `q_sigma`in this case. Happens only on proxy data (not the original data). + However, note that this value is eventually not used in the calculation. + """ + + def _check_sensor_in_data(_data, _sensor): + return _sensor in _data and isinstance(_data[_sensor], np.ndarray) + + def _contains_p_q_sigma(_sensor_data): + return "p_sigma" in _sensor_data.dtype.names and "q_sigma" in _sensor_data.dtype.names + + def _process_power_sigma_in_list(_sensor_mask, _power_sigma, _p_sigma, _q_sigma): + _mask = np.logical_not(np.logical_or(np.isnan(_p_sigma), np.isnan(_q_sigma))) + if _power_sigma.ndim < _mask.ndim: + _mask = np.any(_mask, axis=tuple(range(_power_sigma.ndim, _mask.ndim))) + + for sublist, should_remove in zip(_sensor_mask, _mask): + if should_remove and "power_sigma" in sublist: + sublist = cast(List[str], sublist) + sublist.remove("power_sigma") + + if _check_sensor_in_data(data, sensor): + sensor_data = data[sensor] + sensor_mask = required_list[sensor] + if _contains_p_q_sigma(sensor_data): + p_sigma = sensor_data["p_sigma"] + q_sigma = sensor_data["q_sigma"] + power_sigma = sensor_data["power_sigma"] + + _process_power_sigma_in_list(sensor_mask, power_sigma, p_sigma, q_sigma) + + def validate_required_values( data: SingleDataset, calculation_type: Optional[CalculationType] = None, symmetric: bool = True ) -> List[MissingValueError]: @@ -236,7 +274,7 @@ def validate_required_values( An empty list if all required data is available, or a list of MissingValueErrors. """ # Base - required = {"base": ["id"]} + required: Dict[str, List[Union[str, List[str]]]] = {"base": ["id"]} # Nodes required["node"] = required["base"] + ["u_rated"] @@ -313,8 +351,14 @@ def validate_required_values( required["power_sensor"] += ["power_sigma", "p_measured", "q_measured"] required["sym_voltage_sensor"] = required["voltage_sensor"].copy() required["asym_voltage_sensor"] = required["voltage_sensor"].copy() - required["sym_power_sensor"] = required["power_sensor"].copy() - required["asym_power_sensor"] = required["power_sensor"].copy() + # Different requirements for individual sensors. Avoid shallow copy. + for sensor_type in ("sym_power_sensor", "asym_power_sensor"): + try: + required[sensor_type] = [ + required["power_sensor"].copy() for _ in range(data[sensor_type].shape[0]) # type: ignore + ] + except KeyError: + pass # Faults required["fault"] = required["base"] + ["fault_object"] @@ -331,7 +375,50 @@ def validate_required_values( required["line"] += ["r0", "x0", "c0", "tan0"] required["shunt"] += ["g0", "b0"] - return list(chain(*(none_missing(data, component, required.get(component, [])) for component in data))) + _process_power_sigma_and_p_q_sigma(data, "sym_power_sensor", required) + _process_power_sigma_and_p_q_sigma(data, "asym_power_sensor", required) + + return _validate_required_in_data(data, required) + + +def _validate_required_in_data(data, required): + """ + Checks if all required data is available. + + Args: + data: A power-grid-model input dataset + required: a list of required fields (a list of str), per component when applicaple (a list of str or str lists) + + Returns: + An empty list if all required data is available, or a list of MissingValueErrors. + """ + + def is_valid_component(data, component): + return ( + not (isinstance(data[component], np.ndarray) and data[component].size == 0) + and data[component] is not None + and isinstance(data[component], ABCSized) + ) + + def is_nested_list(items): + return isinstance(items, list) and all(isinstance(i, list) for i in items) + + def process_nested_items(component, items, data, results): + for index, item in enumerate(sublist for sublist in items): + if index < len(data[component]): + results.append(none_missing(data, component, item, index)) + + results = [] + + for component in data: + if is_valid_component(data, component): + items = required.get(component, []) + if is_nested_list(items): + process_nested_items(component, items, data, results) + else: + results.append(none_missing(data, component, items, 0)) + + return list(chain(*results)) def validate_values(data: SingleDataset, calculation_type: Optional[CalculationType] = None) -> List[ValidationError]: diff --git a/tests/unit/validation/test_validation_functions.py b/tests/unit/validation/test_validation_functions.py index 0db563478..3e07b13d7 100644 --- a/tests/unit/validation/test_validation_functions.py +++ b/tests/unit/validation/test_validation_functions.py @@ -9,8 +9,9 @@ import numpy as np import pytest -from power_grid_model import MeasuredTerminalType, initialize_array, power_grid_meta_data +from power_grid_model import CalculationType, LoadGenType, MeasuredTerminalType, initialize_array, power_grid_meta_data from power_grid_model.enum import CalculationType, FaultPhase, FaultType +from power_grid_model.validation import assert_valid_input_data from power_grid_model.validation.errors import ( IdNotInDatasetError, InfinityError, @@ -646,3 +647,73 @@ def test_validate_generic_power_sensor__terminal_types( all_valid_ids.assert_any_call( ANY, ANY, field=ANY, ref_components=ref_component, measured_terminal_type=measured_terminal_type ) + + +def test_power_sigma_or_p_q_sigma(): + # node + node = initialize_array("input", "node", 2) + node["id"] = np.array([0, 3]) + node["u_rated"] = [10.5e3, 10.5e3] + + # line + line = initialize_array("input", "line", 1) + line["id"] = [2] + line["from_node"] = [0] + line["to_node"] = [3] + line["from_status"] = [1] + line["to_status"] = [1] + line["r1"] = [0.001] + line["x1"] = [0.02] + line["c1"] = [0.0] + line["tan1"] = [0.0] + line["i_n"] = [1000.0] + + # load + sym_load = initialize_array("input", "sym_load", 2) + sym_load["id"] = [4, 9] + sym_load["node"] = [3, 0] + sym_load["status"] = [1, 1] + sym_load["type"] = [LoadGenType.const_power, LoadGenType.const_power] + sym_load["p_specified"] = [1e6, 1e6] + sym_load["q_specified"] = [-1e6, -1e6] + + # source + source = initialize_array("input", "source", 1) + source["id"] = [1] + source["node"] = [0] + source["status"] = [1] + source["u_ref"] = [1.0] + + # voltage sensor + voltage_sensor = initialize_array("input", "sym_voltage_sensor", 1) + voltage_sensor["id"] = 5 + voltage_sensor["measured_object"] = 0 + voltage_sensor["u_sigma"] = [100.0] + voltage_sensor["u_measured"] = [10.5e3] + + # power sensor + sym_power_sensor = initialize_array("input", "sym_power_sensor", 3) + sym_power_sensor["id"] = [6, 7, 8] + sym_power_sensor["measured_object"] = [2, 4, 9] + sym_power_sensor["measured_terminal_type"] = [ + MeasuredTerminalType.branch_from, + MeasuredTerminalType.load, + MeasuredTerminalType.load, + ] + sym_power_sensor["p_measured"] = [1e6, -1e6, -1e6] + sym_power_sensor["q_measured"] = [1e6, -1e6, -1e6] + sym_power_sensor["power_sigma"] = [np.nan, 1e9, 1e9] + sym_power_sensor["p_sigma"] = [1e4, np.nan, 1e4] + sym_power_sensor["q_sigma"] = [1e9, np.nan, 1e9] + + # all + input_data = { + "node": node, + "line": line, + "sym_load": sym_load, + "source": source, + "sym_voltage_sensor": voltage_sensor, + "sym_power_sensor": sym_power_sensor, + } + + assert_valid_input_data(input_data=input_data, calculation_type=CalculationType.state_estimation)