Feature/Updated validation to allow for missing sigma

docs/user_manual/components.md

@@ -594,7 +594,7 @@ Because of this distribution, at least one appliance is required to be connected
 | name                     | data type                                                                     | unit             | description                                                                                                                                                                                 |              required               |  update  |                     valid values                     |
 | ------------------------ | ----------------------------------------------------------------------------- | ---------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | :---------------------------------: | :------: | :--------------------------------------------------: |
 | `measured_terminal_type` | {py:class}`MeasuredTerminalType <power_grid_model.enum.MeasuredTerminalType>` | -                | indicate if it measures an `appliance` or a `branch`                                                                                                                                        |              &#10004;               | &#10060; | the terminal type should match the `measured_object` |
-| `power_sigma`            | `double`                                                                      | volt-ampere (VA) | standard deviation of the measurement error. Usually this is the absolute measurement error range divided by 3. See {hoverxreftooltip}`user_manual/components:Power Sensor Concrete Types`. | &#10024; only for state estimation. | &#10004; |                        `> 0`                         |
+| `power_sigma`            | `double`                                                                      | volt-ampere (VA) | standard deviation of the measurement error. Usually this is the absolute measurement error range divided by 3. See {hoverxreftooltip}`user_manual/components:Power Sensor Concrete Types`. | &#10024; in certain cases for state estimation. See the explanation for [concrete types](#power-sensor-concrete-types) below. | &#10004; |                        `> 0`                         |
 
 #### Power Sensor Concrete Types
 
@@ -615,6 +615,19 @@ the meaning of `RealValueInput` is different, as shown in the table below.
 | `p_sigma`    | `RealValueInput` | watt (W)                   | standard deviation of the active power measurement error. Usually this is the absolute measurement error range divided by 3.   | &#10060; see the explanation below. | &#10004; | `> 0` |
 | `q_sigma`    | `RealValueInput` | volt-ampere-reactive (var) | standard deviation of the reactive power measurement error. Usually this is the absolute measurement error range divided by 3. | &#10060; see the explanation below. | &#10004; | `> 0` |
 
+Valid combinations of `power_sigma`, `p_sigma` and `q_sigma` are:
+
+| `power_sigma` | `p_sigma` | `q_sigma` |  result  |
+|:-------------:|:---------:|:---------:|:--------:|
+|       x       |      x    |     x     | &#10004; |
+|       x       |      x    |           | &#10060; |
+|       x       |           |     x     | &#10060; |
+|       x       |           |           | &#10004; |
+|               |      x    |     x     | &#10004; |
+|               |      x    |           | &#10060; |
+|               |           |     x     | &#10060; |
+|               |           |           | &#10060; |
+
 ```{note}
 1. If both `p_sigma` and `q_sigma` are provided, they represent the standard deviation of the active and reactive power, respectively, and the value of `power_sigma` is ignored. Any infinite component invalidates the entire measurement.
 

src/power_grid_model/validation/rules.py

@@ -667,7 +667,9 @@ def all_finite(data: SingleDataset, exceptions: Optional[Dict[str, List[str]]] =
     return errors
 
 
-def none_missing(data: SingleDataset, component: str, fields: Union[str, List[str]]) -> List[MissingValueError]:
+def none_missing(
+    data: SingleDataset, component: str, fields: Union[List[Union[str, List[str]]], str, List[str]], index: int = 0
+) -> List[MissingValueError]:
     """
     Check that for all records of a particular type of component, the values in the 'fields' columns are not NaN.
     Returns an empty list on success, or a list containing a single error object on failure.
@@ -685,14 +687,17 @@ def none_missing(data: SingleDataset, component: str, fields: Union[str, List[st
     if isinstance(fields, str):
         fields = [fields]
     for field in fields:
+        if isinstance(field, list):
+            field = field[0]
         nan = nan_type(component, field)
         if np.isnan(nan):
-            invalid = np.isnan(data[component][field])
+            invalid = np.isnan(data[component][field][index])
         else:
-            invalid = np.equal(data[component][field], nan)
+            invalid = np.equal(data[component][field][index], nan)
+
         if invalid.any():
-            if invalid.ndim > 1:
-                invalid = invalid.any(axis=1)
+            if isinstance(invalid, np.ndarray):
+                invalid = np.any(invalid)
             ids = data[component]["id"][invalid].flatten().tolist()
             errors.append(MissingValueError(component, field, ids))
     return errors

src/power_grid_model/validation/validation.py

@@ -9,8 +9,9 @@
 
 """
 import copy
+from collections.abc import Sized as ABCSized
 from itertools import chain
-from typing import Dict, List, Optional
+from typing import Dict, List, Optional, Union, cast
 
 import numpy as np
 
@@ -221,6 +222,43 @@ def validate_ids_exist(update_data: Dict[str, np.ndarray], input_data: SingleDat
     return list(chain(*errors))
 
 
+def process_power_sigma_and_p_q_sigma(
+    data: SingleDataset, sensor: str, required_list: Dict[str, List[Union[str, List[str]]]]
+) -> None:
+    """
+    Helper function to process the required list when both `p_sigma` and `q_sigma` exist
+    and valid but `power_sigma` is missing. The field `power_sigma` is set to the norm of
+    `p_sigma` and `q_sigma`in this case. Happens only on proxy data (not the original data).
+    However, note that this value is eventually not used in the calculation.
+    """
+
+    def _check_sensor_in_data(_data, _sensor):
+        return _sensor in _data and isinstance(_data[_sensor], np.ndarray)
+
+    def _contains_p_q_sigma(_sensor_data):
+        return "p_sigma" in _sensor_data.dtype.names and "q_sigma" in _sensor_data.dtype.names
+
+    def _process_power_sigma_in_list(_sensor_mask, _power_sigma, _p_sigma, _q_sigma):
+        _mask = np.logical_not(np.logical_or(np.isnan(_p_sigma), np.isnan(_q_sigma)))
+        if _power_sigma.ndim < _mask.ndim:
+            _mask = np.any(_mask, axis=tuple(range(_power_sigma.ndim, _mask.ndim)))
+
+        for sublist, should_remove in zip(_sensor_mask, _mask):
+            if should_remove and "power_sigma" in sublist:
+                sublist = cast(List[str], sublist)
+                sublist.remove("power_sigma")
+
+    if _check_sensor_in_data(data, sensor):
+        sensor_data = data[sensor]
+        sensor_mask = required_list[sensor]
+        if _contains_p_q_sigma(sensor_data):
+            p_sigma = sensor_data["p_sigma"]
+            q_sigma = sensor_data["q_sigma"]
+            power_sigma = sensor_data["power_sigma"]
+
+            _process_power_sigma_in_list(sensor_mask, power_sigma, p_sigma, q_sigma)
+
+
 def validate_required_values(
     data: SingleDataset, calculation_type: Optional[CalculationType] = None, symmetric: bool = True
 ) -> List[MissingValueError]:
@@ -236,7 +274,7 @@ def validate_required_values(
         An empty list if all required data is available, or a list of MissingValueErrors.
     """
     # Base
-    required = {"base": ["id"]}
+    required: Dict[str, List[Union[str, List[str]]]] = {"base": ["id"]}
 
     # Nodes
     required["node"] = required["base"] + ["u_rated"]
@@ -313,8 +351,19 @@ def validate_required_values(
         required["power_sensor"] += ["power_sigma", "p_measured", "q_measured"]
     required["sym_voltage_sensor"] = required["voltage_sensor"].copy()
     required["asym_voltage_sensor"] = required["voltage_sensor"].copy()
-    required["sym_power_sensor"] = required["power_sensor"].copy()
-    required["asym_power_sensor"] = required["power_sensor"].copy()
+    # Different requirements for individual sensors. Avoid shallow copy.
+    try:
+        required["sym_power_sensor"] = [
+            required["power_sensor"].copy() for _ in range(data["sym_power_sensor"].shape[0])  # type: ignore
+        ]
+    except KeyError:
+        pass
+    try:
+        required["asym_power_sensor"] = [
+            required["power_sensor"].copy() for _ in range(data["asym_power_sensor"].shape[0])  # type: ignore
+        ]
+    except KeyError:
+        pass
 
     # Faults
     required["fault"] = required["base"] + ["fault_object"]
@@ -331,7 +380,50 @@ def validate_required_values(
         required["line"] += ["r0", "x0", "c0", "tan0"]
         required["shunt"] += ["g0", "b0"]
 
-    return list(chain(*(none_missing(data, component, required.get(component, [])) for component in data)))
+    process_power_sigma_and_p_q_sigma(data, "sym_power_sensor", required)
+    process_power_sigma_and_p_q_sigma(data, "asym_power_sensor", required)
+
+    return _validate_required_in_data(data, required)
+
+
+def _validate_required_in_data(data, required):
+    """
+    Checks if all required data is available.
+
+    Args:
+        data: A power-grid-model input dataset
+        required: a list of required fields (a list of str), per component when applicaple (a list of str or str lists)
+
+    Returns:
+        An empty list if all required data is available, or a list of MissingValueErrors.
+    """
+
+    def is_valid_component(data, component):
+        return (
+            not (isinstance(data[component], np.ndarray) and data[component].size == 0)
+            and data[component] is not None
+            and isinstance(data[component], ABCSized)
+        )
+
+    def is_nested_list(items):
+        return isinstance(items, list) and all(isinstance(i, list) for i in items)
+
+    def process_nested_items(component, items, data, results):
+        for index, item in enumerate(sublist for sublist in items):
+            if index < len(data[component]):
+                results.append(none_missing(data, component, item, index))
+
+    results = []
+
+    for component in data:
+        if is_valid_component(data, component):
+            items = required.get(component, [])
+            if is_nested_list(items):
+                process_nested_items(component, items, data, results)
+            else:
+                results.append(none_missing(data, component, items, 0))
+
+    return list(chain(*results))
 
 
 def validate_values(data: SingleDataset, calculation_type: Optional[CalculationType] = None) -> List[ValidationError]:

tests/unit/validation/test_validation_functions.py

@@ -9,8 +9,9 @@
 import numpy as np
 import pytest
 
-from power_grid_model import MeasuredTerminalType, initialize_array, power_grid_meta_data
+from power_grid_model import CalculationType, LoadGenType, MeasuredTerminalType, initialize_array, power_grid_meta_data
 from power_grid_model.enum import CalculationType, FaultPhase, FaultType
+from power_grid_model.validation import assert_valid_input_data
 from power_grid_model.validation.errors import (
     IdNotInDatasetError,
     InfinityError,
@@ -646,3 +647,73 @@ def test_validate_generic_power_sensor__terminal_types(
     all_valid_ids.assert_any_call(
         ANY, ANY, field=ANY, ref_components=ref_component, measured_terminal_type=measured_terminal_type
     )
+
+
+def test_power_sigma_or_p_q_sigma():
+    # node
+    node = initialize_array("input", "node", 2)
+    node["id"] = np.array([0, 3])
+    node["u_rated"] = [10.5e3, 10.5e3]
+
+    # line
+    line = initialize_array("input", "line", 1)
+    line["id"] = [2]
+    line["from_node"] = [0]
+    line["to_node"] = [3]
+    line["from_status"] = [1]
+    line["to_status"] = [1]
+    line["r1"] = [0.001]
+    line["x1"] = [0.02]
+    line["c1"] = [0.0]
+    line["tan1"] = [0.0]
+    line["i_n"] = [1000.0]
+
+    # load
+    sym_load = initialize_array("input", "sym_load", 2)
+    sym_load["id"] = [4, 9]
+    sym_load["node"] = [3, 0]
+    sym_load["status"] = [1, 1]
+    sym_load["type"] = [LoadGenType.const_power, LoadGenType.const_power]
+    sym_load["p_specified"] = [1e6, 1e6]
+    sym_load["q_specified"] = [-1e6, -1e6]
+
+    # source
+    source = initialize_array("input", "source", 1)
+    source["id"] = [1]
+    source["node"] = [0]
+    source["status"] = [1]
+    source["u_ref"] = [1.0]
+
+    # voltage sensor
+    voltage_sensor = initialize_array("input", "sym_voltage_sensor", 1)
+    voltage_sensor["id"] = 5
+    voltage_sensor["measured_object"] = 0
+    voltage_sensor["u_sigma"] = [100.0]
+    voltage_sensor["u_measured"] = [10.5e3]
+
+    # power sensor
+    sym_power_sensor = initialize_array("input", "sym_power_sensor", 3)
+    sym_power_sensor["id"] = [6, 7, 8]
+    sym_power_sensor["measured_object"] = [2, 4, 9]
+    sym_power_sensor["measured_terminal_type"] = [
+        MeasuredTerminalType.branch_from,
+        MeasuredTerminalType.load,
+        MeasuredTerminalType.load,
+    ]
+    sym_power_sensor["p_measured"] = [1e6, -1e6, -1e6]
+    sym_power_sensor["q_measured"] = [1e6, -1e6, -1e6]
+    sym_power_sensor["power_sigma"] = [np.nan, 1e9, 1e9]
+    sym_power_sensor["p_sigma"] = [1e4, np.nan, 1e4]
+    sym_power_sensor["q_sigma"] = [1e9, np.nan, 1e9]
+
+    # all
+    input_data = {
+        "node": node,
+        "line": line,
+        "sym_load": sym_load,
+        "source": source,
+        "sym_voltage_sensor": voltage_sensor,
+        "sym_power_sensor": sym_power_sensor,
+    }
+
+    assert_valid_input_data(input_data=input_data, calculation_type=CalculationType.state_estimation)