From c9fda67b5f4b33a7eff231d59c41f316f2dd0a8b Mon Sep 17 00:00:00 2001 From: Jerry Guo Date: Tue, 20 Feb 2024 17:15:10 +0100 Subject: [PATCH 01/16] Updated validation to allow for missing `power_sigma` when both `p_sigma` and `q_sigma` exist and are valid updated test case updated documentation Signed-off-by: Jerry Guo --- docs/user_manual/components.md | 14 +++- src/power_grid_model/validation/validation.py | 18 +++++ .../validation/test_validation_functions.py | 69 ++++++++++++++++++- 3 files changed, 99 insertions(+), 2 deletions(-) diff --git a/docs/user_manual/components.md b/docs/user_manual/components.md index 46cd2f028..4c895da3d 100644 --- a/docs/user_manual/components.md +++ b/docs/user_manual/components.md @@ -594,7 +594,7 @@ Because of this distribution, at least one appliance is required to be connected | name | data type | unit | description | required | update | valid values | | ------------------------ | ----------------------------------------------------------------------------- | ---------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | :---------------------------------: | :------: | :--------------------------------------------------: | | `measured_terminal_type` | {py:class}`MeasuredTerminalType ` | - | indicate if it measures an `appliance` or a `branch` | ✔ | ❌ | the terminal type should match the `measured_object` | -| `power_sigma` | `double` | volt-ampere (VA) | standard deviation of the measurement error. Usually this is the absolute measurement error range divided by 3. See {hoverxreftooltip}`user_manual/components:Power Sensor Concrete Types`. | ✨ only for state estimation. | ✔ | `> 0` | +| `power_sigma` | `double` | volt-ampere (VA) | standard deviation of the measurement error. Usually this is the absolute measurement error range divided by 3. See {hoverxreftooltip}`user_manual/components:Power Sensor Concrete Types`. | ✨ only for state estimation. See the explanation below. | ✔ | `> 0` | #### Power Sensor Concrete Types @@ -615,6 +615,18 @@ the meaning of `RealValueInput` is different, as shown in the table below. | `p_sigma` | `RealValueInput` | watt (W) | standard deviation of the active power measurement error. Usually this is the absolute measurement error range divided by 3. | ❌ see the explanation below. | ✔ | `> 0` | | `q_sigma` | `RealValueInput` | volt-ampere-reactive (var) | standard deviation of the reactive power measurement error. Usually this is the absolute measurement error range divided by 3. | ❌ see the explanation below. | ✔ | `> 0` | +Valid combinations of `power_sigma`, `p_sigma` and `q_sigma` are: +| `power_sigma` | `p_sigma` | `q_sigma` | result | +|:-------------:|:---------:|:---------:|:--------:| +| ✔ | ✔ | ✔ | valid | +| ✔ | ✔ | ❌ | invalid | +| ✔ | ❌ | ✔ | invalid | +| ✔ | ❌ | ❌ | valid | +| ❌ | ✔ | ✔ | valid | +| ❌ | ✔ | ❌ | invalid | +| ❌ | ❌ | ✔ | invalid | +| ❌ | ❌ | ❌ | invalid | + ```{note} 1. If both `p_sigma` and `q_sigma` are provided, they represent the standard deviation of the active and reactive power, respectively, and the value of `power_sigma` is ignored. Any infinite component invalidates the entire measurement. diff --git a/src/power_grid_model/validation/validation.py b/src/power_grid_model/validation/validation.py index fa6f36273..48ec54307 100644 --- a/src/power_grid_model/validation/validation.py +++ b/src/power_grid_model/validation/validation.py @@ -221,6 +221,20 @@ def validate_ids_exist(update_data: Dict[str, np.ndarray], input_data: SingleDat return list(chain(*errors)) +def process_power_sigma_and_p_q_sigma(data: SingleDataset, sensor: str, required_list: Dict[str, List[str]]) -> None: + """ + Helper function to process the required list when both `p_sigma` and `q_sigma` exist + and valid but `power_sigma` is missing. + """ + if sensor in data and isinstance(data[sensor], np.ndarray): + sensor_data = data[sensor] + if "p_sigma" in sensor_data.dtype.names and "q_sigma" in sensor_data.dtype.names: + p_sigma = sensor_data["p_sigma"] + q_sigma = sensor_data["q_sigma"] + if not (np.isnan(p_sigma).any() or np.isnan(q_sigma).any()): + required_list[sensor].remove("power_sigma") + + def validate_required_values( data: SingleDataset, calculation_type: Optional[CalculationType] = None, symmetric: bool = True ) -> List[MissingValueError]: @@ -331,6 +345,10 @@ def validate_required_values( required["line"] += ["r0", "x0", "c0", "tan0"] required["shunt"] += ["g0", "b0"] + # Allow missing `power_sigma` of both `p_sigma` and `q_sigma` are present + process_power_sigma_and_p_q_sigma(data, "sym_power_sensor", required) + process_power_sigma_and_p_q_sigma(data, "asym_power_sensor", required) + return list(chain(*(none_missing(data, component, required.get(component, [])) for component in data))) diff --git a/tests/unit/validation/test_validation_functions.py b/tests/unit/validation/test_validation_functions.py index 0db563478..8ef85581d 100644 --- a/tests/unit/validation/test_validation_functions.py +++ b/tests/unit/validation/test_validation_functions.py @@ -9,8 +9,10 @@ import numpy as np import pytest -from power_grid_model import MeasuredTerminalType, initialize_array, power_grid_meta_data +# from power_grid_model import PowerGridModel, LoadGenType, MeasuredTerminalType, initialize_array, CalculationType, CalculationMethod +from power_grid_model import CalculationType, LoadGenType, MeasuredTerminalType, initialize_array, power_grid_meta_data from power_grid_model.enum import CalculationType, FaultPhase, FaultType +from power_grid_model.validation import assert_valid_input_data from power_grid_model.validation.errors import ( IdNotInDatasetError, InfinityError, @@ -646,3 +648,68 @@ def test_validate_generic_power_sensor__terminal_types( all_valid_ids.assert_any_call( ANY, ANY, field=ANY, ref_components=ref_component, measured_terminal_type=measured_terminal_type ) + + +def test_power_sigma_or_p_q_sigma(): + # node + node = initialize_array("input", "node", 2) + node["id"] = np.array([0, 3]) + node["u_rated"] = [10.5e3, 10.5e3] + + # line + line = initialize_array("input", "line", 1) + line["id"] = [2] + line["from_node"] = [0] + line["to_node"] = [3] + line["from_status"] = [1] + line["to_status"] = [1] + line["r1"] = [0.001] + line["x1"] = [0.02] + line["c1"] = [0.0] + line["tan1"] = [0.0] + line["i_n"] = [1000.0] + + # load + sym_load = initialize_array("input", "sym_load", 1) + sym_load["id"] = [4] + sym_load["node"] = [3] + sym_load["status"] = [1] + sym_load["type"] = [LoadGenType.const_power] + sym_load["p_specified"] = [1e6] + sym_load["q_specified"] = [-1e6] + + # source + source = initialize_array("input", "source", 1) + source["id"] = [1] + source["node"] = [0] + source["status"] = [1] + source["u_ref"] = [1.0] + + # voltage sensor + voltage_sensor = initialize_array("input", "sym_voltage_sensor", 1) + voltage_sensor["id"] = 5 + voltage_sensor["measured_object"] = 0 + voltage_sensor["u_sigma"] = [100.0] + voltage_sensor["u_measured"] = [10.5e3] + + # power sensor + power_sensor = initialize_array("input", "sym_power_sensor", 2) + power_sensor["id"] = [6, 7] + power_sensor["measured_object"] = [2, 4] + power_sensor["measured_terminal_type"] = [MeasuredTerminalType.branch_from, MeasuredTerminalType.load] + power_sensor["p_measured"] = [1e6, -1e6] + power_sensor["q_measured"] = [1e6, -1e6] + power_sensor["p_sigma"] = [1e4, 1e9] # trust P for sensor 6 + power_sensor["q_sigma"] = [1e9, 1e4] # trust Q for sensor 7 + + # all + input_data = { + "node": node, + "line": line, + "sym_load": sym_load, + "source": source, + "sym_voltage_sensor": voltage_sensor, + "sym_power_sensor": power_sensor, + } + + assert_valid_input_data(input_data=input_data, calculation_type=CalculationType.state_estimation) From dd4bd7c392bd19b7fedfc0f145d9c682f7c79f25 Mon Sep 17 00:00:00 2001 From: Jerry Guo Date: Wed, 21 Feb 2024 17:46:22 +0100 Subject: [PATCH 02/16] Intermediate commit Signed-off-by: Jerry Guo --- src/power_grid_model/validation/validation.py | 3 +++ tests/unit/validation/test_validation_functions.py | 5 +++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/src/power_grid_model/validation/validation.py b/src/power_grid_model/validation/validation.py index 48ec54307..629a2cc10 100644 --- a/src/power_grid_model/validation/validation.py +++ b/src/power_grid_model/validation/validation.py @@ -348,6 +348,9 @@ def validate_required_values( # Allow missing `power_sigma` of both `p_sigma` and `q_sigma` are present process_power_sigma_and_p_q_sigma(data, "sym_power_sensor", required) process_power_sigma_and_p_q_sigma(data, "asym_power_sensor", required) + # ToDo: + # Instead of removing the `power_sigma` key from `required`, we need to make it a list of length data.shape[0] + # Within the list, the `power_sigma` and `p_sigma` and `q_sigma` are marked independently return list(chain(*(none_missing(data, component, required.get(component, [])) for component in data))) diff --git a/tests/unit/validation/test_validation_functions.py b/tests/unit/validation/test_validation_functions.py index 8ef85581d..8536a2f39 100644 --- a/tests/unit/validation/test_validation_functions.py +++ b/tests/unit/validation/test_validation_functions.py @@ -699,8 +699,9 @@ def test_power_sigma_or_p_q_sigma(): power_sensor["measured_terminal_type"] = [MeasuredTerminalType.branch_from, MeasuredTerminalType.load] power_sensor["p_measured"] = [1e6, -1e6] power_sensor["q_measured"] = [1e6, -1e6] - power_sensor["p_sigma"] = [1e4, 1e9] # trust P for sensor 6 - power_sensor["q_sigma"] = [1e9, 1e4] # trust Q for sensor 7 + power_sensor["power_sigma"] = [NaN, 1e9] # trust P for sensor 6 + power_sensor["p_sigma"] = [1e4, NaN] # trust P for sensor 6 + power_sensor["q_sigma"] = [1e9, NaN] # trust Q for sensor 7 # all input_data = { From a4f255b4a74731785d084da7a10ff695e564c20f Mon Sep 17 00:00:00 2001 From: Jerry Guo Date: Thu, 22 Feb 2024 15:09:55 +0100 Subject: [PATCH 03/16] Taking into account that multiple power sensors might have different configurations for power_sigma and p_sigma/q_sigma pair. Signed-off-by: Jerry Guo --- src/power_grid_model/validation/validation.py | 24 +++++++++---------- .../validation/test_validation_functions.py | 6 ++--- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/src/power_grid_model/validation/validation.py b/src/power_grid_model/validation/validation.py index 629a2cc10..eb7fb4c4f 100644 --- a/src/power_grid_model/validation/validation.py +++ b/src/power_grid_model/validation/validation.py @@ -221,18 +221,21 @@ def validate_ids_exist(update_data: Dict[str, np.ndarray], input_data: SingleDat return list(chain(*errors)) -def process_power_sigma_and_p_q_sigma(data: SingleDataset, sensor: str, required_list: Dict[str, List[str]]) -> None: +def process_power_sigma_and_p_q_sigma(data: SingleDataset, sensor: str) -> None: """ Helper function to process the required list when both `p_sigma` and `q_sigma` exist - and valid but `power_sigma` is missing. + and valid but `power_sigma` is missing. The field `power_sigma` is set to 1 in this case. + The argument is that when both `p_sigma` and `q_sigma` are valid, the value pf `power_sigma` + is always ignored. """ if sensor in data and isinstance(data[sensor], np.ndarray): sensor_data = data[sensor] - if "p_sigma" in sensor_data.dtype.names and "q_sigma" in sensor_data.dtype.names: - p_sigma = sensor_data["p_sigma"] - q_sigma = sensor_data["q_sigma"] - if not (np.isnan(p_sigma).any() or np.isnan(q_sigma).any()): - required_list[sensor].remove("power_sigma") + for i in range(len(sensor_data)): + if "p_sigma" in sensor_data.dtype.names and "q_sigma" in sensor_data.dtype.names: + p_sigma = sensor_data["p_sigma"][i] + q_sigma = sensor_data["q_sigma"][i] + if not (np.isnan(p_sigma).any() or np.isnan(q_sigma).any()): + data[sensor]["power_sigma"][i] = 1 # Set to 1 if both p_sigma and q_sigma are valid def validate_required_values( @@ -346,11 +349,8 @@ def validate_required_values( required["shunt"] += ["g0", "b0"] # Allow missing `power_sigma` of both `p_sigma` and `q_sigma` are present - process_power_sigma_and_p_q_sigma(data, "sym_power_sensor", required) - process_power_sigma_and_p_q_sigma(data, "asym_power_sensor", required) - # ToDo: - # Instead of removing the `power_sigma` key from `required`, we need to make it a list of length data.shape[0] - # Within the list, the `power_sigma` and `p_sigma` and `q_sigma` are marked independently + process_power_sigma_and_p_q_sigma(data, "sym_power_sensor") + process_power_sigma_and_p_q_sigma(data, "asym_power_sensor") return list(chain(*(none_missing(data, component, required.get(component, [])) for component in data))) diff --git a/tests/unit/validation/test_validation_functions.py b/tests/unit/validation/test_validation_functions.py index 8536a2f39..e951ea9bb 100644 --- a/tests/unit/validation/test_validation_functions.py +++ b/tests/unit/validation/test_validation_functions.py @@ -699,9 +699,9 @@ def test_power_sigma_or_p_q_sigma(): power_sensor["measured_terminal_type"] = [MeasuredTerminalType.branch_from, MeasuredTerminalType.load] power_sensor["p_measured"] = [1e6, -1e6] power_sensor["q_measured"] = [1e6, -1e6] - power_sensor["power_sigma"] = [NaN, 1e9] # trust P for sensor 6 - power_sensor["p_sigma"] = [1e4, NaN] # trust P for sensor 6 - power_sensor["q_sigma"] = [1e9, NaN] # trust Q for sensor 7 + power_sensor["power_sigma"] = [np.nan, 1e9] + power_sensor["p_sigma"] = [1e4, np.nan] + power_sensor["q_sigma"] = [1e9, np.nan] # all input_data = { From 2b028a6a18520af799247d3c9ffcbb7dfdd7e8d1 Mon Sep 17 00:00:00 2001 From: Jerry Guo Date: Thu, 22 Feb 2024 15:17:22 +0100 Subject: [PATCH 04/16] Doc format. Signed-off-by: Jerry Guo --- docs/user_manual/components.md | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/docs/user_manual/components.md b/docs/user_manual/components.md index 4c895da3d..01778de1b 100644 --- a/docs/user_manual/components.md +++ b/docs/user_manual/components.md @@ -594,7 +594,7 @@ Because of this distribution, at least one appliance is required to be connected | name | data type | unit | description | required | update | valid values | | ------------------------ | ----------------------------------------------------------------------------- | ---------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | :---------------------------------: | :------: | :--------------------------------------------------: | | `measured_terminal_type` | {py:class}`MeasuredTerminalType ` | - | indicate if it measures an `appliance` or a `branch` | ✔ | ❌ | the terminal type should match the `measured_object` | -| `power_sigma` | `double` | volt-ampere (VA) | standard deviation of the measurement error. Usually this is the absolute measurement error range divided by 3. See {hoverxreftooltip}`user_manual/components:Power Sensor Concrete Types`. | ✨ only for state estimation. See the explanation below. | ✔ | `> 0` | +| `power_sigma` | `double` | volt-ampere (VA) | standard deviation of the measurement error. Usually this is the absolute measurement error range divided by 3. See {hoverxreftooltip}`user_manual/components:Power Sensor Concrete Types`. | ✨ in certain cases for state estimation. See the explanation below. | ✔ | `> 0` | #### Power Sensor Concrete Types @@ -618,14 +618,14 @@ the meaning of `RealValueInput` is different, as shown in the table below. Valid combinations of `power_sigma`, `p_sigma` and `q_sigma` are: | `power_sigma` | `p_sigma` | `q_sigma` | result | |:-------------:|:---------:|:---------:|:--------:| -| ✔ | ✔ | ✔ | valid | -| ✔ | ✔ | ❌ | invalid | -| ✔ | ❌ | ✔ | invalid | -| ✔ | ❌ | ❌ | valid | -| ❌ | ✔ | ✔ | valid | -| ❌ | ✔ | ❌ | invalid | -| ❌ | ❌ | ✔ | invalid | -| ❌ | ❌ | ❌ | invalid | +| x | x | x | ✔ | +| x | x | | ❌ | +| x | | x | ❌ | +| x | | | ✔ | +| | x | x | ✔ | +| | x | | ❌ | +| | | x | ❌ | +| | | | ❌ | ```{note} 1. If both `p_sigma` and `q_sigma` are provided, they represent the standard deviation of the active and reactive power, respectively, and the value of `power_sigma` is ignored. Any infinite component invalidates the entire measurement. From ba601b69da5c3108d76b4c8e166b5481c6d3ecb9 Mon Sep 17 00:00:00 2001 From: Jerry Guo Date: Fri, 23 Feb 2024 12:29:05 +0100 Subject: [PATCH 05/16] Updated (away) the for loop; updated the document Signed-off-by: Jerry Guo --- docs/user_manual/components.md | 2 +- src/power_grid_model/validation/validation.py | 17 ++++++--- .../validation/test_validation_functions.py | 38 ++++++++++--------- 3 files changed, 33 insertions(+), 24 deletions(-) diff --git a/docs/user_manual/components.md b/docs/user_manual/components.md index 01778de1b..5003a07eb 100644 --- a/docs/user_manual/components.md +++ b/docs/user_manual/components.md @@ -594,7 +594,7 @@ Because of this distribution, at least one appliance is required to be connected | name | data type | unit | description | required | update | valid values | | ------------------------ | ----------------------------------------------------------------------------- | ---------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | :---------------------------------: | :------: | :--------------------------------------------------: | | `measured_terminal_type` | {py:class}`MeasuredTerminalType ` | - | indicate if it measures an `appliance` or a `branch` | ✔ | ❌ | the terminal type should match the `measured_object` | -| `power_sigma` | `double` | volt-ampere (VA) | standard deviation of the measurement error. Usually this is the absolute measurement error range divided by 3. See {hoverxreftooltip}`user_manual/components:Power Sensor Concrete Types`. | ✨ in certain cases for state estimation. See the explanation below. | ✔ | `> 0` | +| `power_sigma` | `double` | volt-ampere (VA) | standard deviation of the measurement error. Usually this is the absolute measurement error range divided by 3. See {hoverxreftooltip}`user_manual/components:Power Sensor Concrete Types`. | ✨ in certain cases for state estimation. See the explanation for [concrete types](#power-sensor-concrete-types) below. | ✔ | `> 0` | #### Power Sensor Concrete Types diff --git a/src/power_grid_model/validation/validation.py b/src/power_grid_model/validation/validation.py index eb7fb4c4f..743febafd 100644 --- a/src/power_grid_model/validation/validation.py +++ b/src/power_grid_model/validation/validation.py @@ -230,12 +230,17 @@ def process_power_sigma_and_p_q_sigma(data: SingleDataset, sensor: str) -> None: """ if sensor in data and isinstance(data[sensor], np.ndarray): sensor_data = data[sensor] - for i in range(len(sensor_data)): - if "p_sigma" in sensor_data.dtype.names and "q_sigma" in sensor_data.dtype.names: - p_sigma = sensor_data["p_sigma"][i] - q_sigma = sensor_data["q_sigma"][i] - if not (np.isnan(p_sigma).any() or np.isnan(q_sigma).any()): - data[sensor]["power_sigma"][i] = 1 # Set to 1 if both p_sigma and q_sigma are valid + if "p_sigma" in sensor_data.dtype.names and "q_sigma" in sensor_data.dtype.names: + p_sigma = sensor_data["p_sigma"] + q_sigma = sensor_data["q_sigma"] + power_sigma = sensor_data["power_sigma"] + + mask = np.logical_not(np.logical_or(np.isnan(p_sigma), np.isnan(q_sigma))) + if power_sigma.ndim < mask.ndim: + mask = np.any(mask, axis=tuple(range(power_sigma.ndim, mask.ndim))) + power_sigma[mask] = np.sqrt(np.sum(np.square(p_sigma[mask]) + np.square(q_sigma[mask]), axis=-1)) + + data[sensor]["power_sigma"] = power_sigma def validate_required_values( diff --git a/tests/unit/validation/test_validation_functions.py b/tests/unit/validation/test_validation_functions.py index e951ea9bb..9e5f2dfe2 100644 --- a/tests/unit/validation/test_validation_functions.py +++ b/tests/unit/validation/test_validation_functions.py @@ -670,13 +670,13 @@ def test_power_sigma_or_p_q_sigma(): line["i_n"] = [1000.0] # load - sym_load = initialize_array("input", "sym_load", 1) - sym_load["id"] = [4] - sym_load["node"] = [3] - sym_load["status"] = [1] - sym_load["type"] = [LoadGenType.const_power] - sym_load["p_specified"] = [1e6] - sym_load["q_specified"] = [-1e6] + sym_load = initialize_array("input", "sym_load", 2) + sym_load["id"] = [4, 9] + sym_load["node"] = [3, 0] + sym_load["status"] = [1, 1] + sym_load["type"] = [LoadGenType.const_power, LoadGenType.const_power] + sym_load["p_specified"] = [1e6, 1e6] + sym_load["q_specified"] = [-1e6, -1e6] # source source = initialize_array("input", "source", 1) @@ -693,15 +693,19 @@ def test_power_sigma_or_p_q_sigma(): voltage_sensor["u_measured"] = [10.5e3] # power sensor - power_sensor = initialize_array("input", "sym_power_sensor", 2) - power_sensor["id"] = [6, 7] - power_sensor["measured_object"] = [2, 4] - power_sensor["measured_terminal_type"] = [MeasuredTerminalType.branch_from, MeasuredTerminalType.load] - power_sensor["p_measured"] = [1e6, -1e6] - power_sensor["q_measured"] = [1e6, -1e6] - power_sensor["power_sigma"] = [np.nan, 1e9] - power_sensor["p_sigma"] = [1e4, np.nan] - power_sensor["q_sigma"] = [1e9, np.nan] + sym_power_sensor = initialize_array("input", "sym_power_sensor", 3) + sym_power_sensor["id"] = [6, 7, 8] + sym_power_sensor["measured_object"] = [2, 4, 9] + sym_power_sensor["measured_terminal_type"] = [ + MeasuredTerminalType.branch_from, + MeasuredTerminalType.load, + MeasuredTerminalType.load, + ] + sym_power_sensor["p_measured"] = [1e6, -1e6, -1e6] + sym_power_sensor["q_measured"] = [1e6, -1e6, -1e6] + sym_power_sensor["power_sigma"] = [np.nan, 1e9, 1e9] + sym_power_sensor["p_sigma"] = [1e4, np.nan, 1e4] + sym_power_sensor["q_sigma"] = [1e9, np.nan, 1e9] # all input_data = { @@ -710,7 +714,7 @@ def test_power_sigma_or_p_q_sigma(): "sym_load": sym_load, "source": source, "sym_voltage_sensor": voltage_sensor, - "sym_power_sensor": power_sensor, + "sym_power_sensor": sym_power_sensor, } assert_valid_input_data(input_data=input_data, calculation_type=CalculationType.state_estimation) From caa192fcb746c69903f1dc48ccabbe2a6d1e826e Mon Sep 17 00:00:00 2001 From: Jerry Guo Date: Fri, 23 Feb 2024 13:52:25 +0100 Subject: [PATCH 06/16] Removed unused import; function description rewording; Signed-off-by: Jerry Guo --- src/power_grid_model/validation/validation.py | 6 +++--- tests/unit/validation/test_validation_functions.py | 1 - 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/src/power_grid_model/validation/validation.py b/src/power_grid_model/validation/validation.py index 743febafd..e77c94f79 100644 --- a/src/power_grid_model/validation/validation.py +++ b/src/power_grid_model/validation/validation.py @@ -224,9 +224,9 @@ def validate_ids_exist(update_data: Dict[str, np.ndarray], input_data: SingleDat def process_power_sigma_and_p_q_sigma(data: SingleDataset, sensor: str) -> None: """ Helper function to process the required list when both `p_sigma` and `q_sigma` exist - and valid but `power_sigma` is missing. The field `power_sigma` is set to 1 in this case. - The argument is that when both `p_sigma` and `q_sigma` are valid, the value pf `power_sigma` - is always ignored. + and valid but `power_sigma` is missing. The field `power_sigma` is set to the norm of + `p_sigma` and `q_sigma`in this case. + However, note that this value is eventually not used in the calculation. """ if sensor in data and isinstance(data[sensor], np.ndarray): sensor_data = data[sensor] diff --git a/tests/unit/validation/test_validation_functions.py b/tests/unit/validation/test_validation_functions.py index 9e5f2dfe2..3e07b13d7 100644 --- a/tests/unit/validation/test_validation_functions.py +++ b/tests/unit/validation/test_validation_functions.py @@ -9,7 +9,6 @@ import numpy as np import pytest -# from power_grid_model import PowerGridModel, LoadGenType, MeasuredTerminalType, initialize_array, CalculationType, CalculationMethod from power_grid_model import CalculationType, LoadGenType, MeasuredTerminalType, initialize_array, power_grid_meta_data from power_grid_model.enum import CalculationType, FaultPhase, FaultType from power_grid_model.validation import assert_valid_input_data From 5e07457a0d4a856a95f4a5108e3ee8d6cf66ac30 Mon Sep 17 00:00:00 2001 From: Jerry Guo Date: Fri, 23 Feb 2024 14:56:51 +0100 Subject: [PATCH 07/16] Polished documentation; added proxy check, leave input data untouched. Signed-off-by: Jerry Guo --- docs/user_manual/components.md | 1 + src/power_grid_model/validation/validation.py | 13 +++++++------ 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/docs/user_manual/components.md b/docs/user_manual/components.md index 5003a07eb..1d85da7bc 100644 --- a/docs/user_manual/components.md +++ b/docs/user_manual/components.md @@ -616,6 +616,7 @@ the meaning of `RealValueInput` is different, as shown in the table below. | `q_sigma` | `RealValueInput` | volt-ampere-reactive (var) | standard deviation of the reactive power measurement error. Usually this is the absolute measurement error range divided by 3. | ❌ see the explanation below. | ✔ | `> 0` | Valid combinations of `power_sigma`, `p_sigma` and `q_sigma` are: + | `power_sigma` | `p_sigma` | `q_sigma` | result | |:-------------:|:---------:|:---------:|:--------:| | x | x | x | ✔ | diff --git a/src/power_grid_model/validation/validation.py b/src/power_grid_model/validation/validation.py index e77c94f79..c70261cba 100644 --- a/src/power_grid_model/validation/validation.py +++ b/src/power_grid_model/validation/validation.py @@ -225,7 +225,7 @@ def process_power_sigma_and_p_q_sigma(data: SingleDataset, sensor: str) -> None: """ Helper function to process the required list when both `p_sigma` and `q_sigma` exist and valid but `power_sigma` is missing. The field `power_sigma` is set to the norm of - `p_sigma` and `q_sigma`in this case. + `p_sigma` and `q_sigma`in this case. Happens only on proxy data (not the original data). However, note that this value is eventually not used in the calculation. """ if sensor in data and isinstance(data[sensor], np.ndarray): @@ -338,13 +338,14 @@ def validate_required_values( required["sym_power_sensor"] = required["power_sensor"].copy() required["asym_power_sensor"] = required["power_sensor"].copy() + _data = data # proxy # Faults required["fault"] = required["base"] + ["fault_object"] asym_sc = False if calculation_type is None or calculation_type == CalculationType.short_circuit: required["fault"] += ["status", "fault_type"] - if "fault" in data: - for elem in data["fault"]["fault_type"]: + if "fault" in _data: + for elem in _data["fault"]["fault_type"]: if elem not in (FaultType.three_phase, FaultType.nan): asym_sc = True break @@ -354,10 +355,10 @@ def validate_required_values( required["shunt"] += ["g0", "b0"] # Allow missing `power_sigma` of both `p_sigma` and `q_sigma` are present - process_power_sigma_and_p_q_sigma(data, "sym_power_sensor") - process_power_sigma_and_p_q_sigma(data, "asym_power_sensor") + process_power_sigma_and_p_q_sigma(_data, "sym_power_sensor") + process_power_sigma_and_p_q_sigma(_data, "asym_power_sensor") - return list(chain(*(none_missing(data, component, required.get(component, [])) for component in data))) + return list(chain(*(none_missing(_data, component, required.get(component, [])) for component in _data))) def validate_values(data: SingleDataset, calculation_type: Optional[CalculationType] = None) -> List[ValidationError]: From a020fad83571407fc7477995d76b1bcc616d2da4 Mon Sep 17 00:00:00 2001 From: Jerry Guo Date: Fri, 23 Feb 2024 15:01:27 +0100 Subject: [PATCH 08/16] Code formatted Signed-off-by: Jerry Guo --- src/power_grid_model/validation/validation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/power_grid_model/validation/validation.py b/src/power_grid_model/validation/validation.py index c70261cba..d3c08db16 100644 --- a/src/power_grid_model/validation/validation.py +++ b/src/power_grid_model/validation/validation.py @@ -338,7 +338,7 @@ def validate_required_values( required["sym_power_sensor"] = required["power_sensor"].copy() required["asym_power_sensor"] = required["power_sensor"].copy() - _data = data # proxy + _data = data # proxy # Faults required["fault"] = required["base"] + ["fault_object"] asym_sc = False From 1f80854d448d0f9724e61445423676dd23aa2c8f Mon Sep 17 00:00:00 2001 From: Jerry Guo Date: Fri, 23 Feb 2024 20:40:09 +0100 Subject: [PATCH 09/16] Validations on individual `power_sensor` Signed-off-by: Jerry Guo --- src/power_grid_model/validation/rules.py | 24 +++++--- src/power_grid_model/validation/validation.py | 61 ++++++++++++++----- 2 files changed, 62 insertions(+), 23 deletions(-) diff --git a/src/power_grid_model/validation/rules.py b/src/power_grid_model/validation/rules.py index c812e5221..cfba7c0f0 100644 --- a/src/power_grid_model/validation/rules.py +++ b/src/power_grid_model/validation/rules.py @@ -667,7 +667,9 @@ def all_finite(data: SingleDataset, exceptions: Optional[Dict[str, List[str]]] = return errors -def none_missing(data: SingleDataset, component: str, fields: Union[str, List[str]]) -> List[MissingValueError]: +def none_missing( + data: SingleDataset, component: str, fields: Union[str, List[str], List[List[str]]], index: int = 0 +) -> List[MissingValueError]: """ Check that for all records of a particular type of component, the values in the 'fields' columns are not NaN. Returns an empty list on success, or a list containing a single error object on failure. @@ -685,15 +687,23 @@ def none_missing(data: SingleDataset, component: str, fields: Union[str, List[st if isinstance(fields, str): fields = [fields] for field in fields: + if isinstance(field, list): + field = field[0] nan = nan_type(component, field) + invalid = None if np.isnan(nan): - invalid = np.isnan(data[component][field]) + invalid = np.isnan(data[component][field][index]) else: - invalid = np.equal(data[component][field], nan) - if invalid.any(): - if invalid.ndim > 1: - invalid = invalid.any(axis=1) - ids = data[component]["id"][invalid].flatten().tolist() + invalid = np.equal(data[component][field][index], nan) + + if invalid is not None and invalid.any(): + if isinstance(invalid, np.ndarray): + invalid = np.any(invalid) + ids = [] + if isinstance(data[component][field], list) and len(data[component][field]) > index: + ids = data[component]["id"][index][invalid].flatten().tolist() + else: + ids = data[component]["id"][invalid].flatten().tolist() errors.append(MissingValueError(component, field, ids)) return errors diff --git a/src/power_grid_model/validation/validation.py b/src/power_grid_model/validation/validation.py index d3c08db16..676b4f9d4 100644 --- a/src/power_grid_model/validation/validation.py +++ b/src/power_grid_model/validation/validation.py @@ -10,7 +10,7 @@ """ import copy from itertools import chain -from typing import Dict, List, Optional +from typing import Dict, List, Optional, Union, cast import numpy as np @@ -221,7 +221,9 @@ def validate_ids_exist(update_data: Dict[str, np.ndarray], input_data: SingleDat return list(chain(*errors)) -def process_power_sigma_and_p_q_sigma(data: SingleDataset, sensor: str) -> None: +def process_power_sigma_and_p_q_sigma( + data: SingleDataset, sensor: str, required_list: Dict[str, List[Union[str, List[str]]]] +) -> None: """ Helper function to process the required list when both `p_sigma` and `q_sigma` exist and valid but `power_sigma` is missing. The field `power_sigma` is set to the norm of @@ -230,6 +232,7 @@ def process_power_sigma_and_p_q_sigma(data: SingleDataset, sensor: str) -> None: """ if sensor in data and isinstance(data[sensor], np.ndarray): sensor_data = data[sensor] + sensor_mask = required_list[sensor] if "p_sigma" in sensor_data.dtype.names and "q_sigma" in sensor_data.dtype.names: p_sigma = sensor_data["p_sigma"] q_sigma = sensor_data["q_sigma"] @@ -238,9 +241,11 @@ def process_power_sigma_and_p_q_sigma(data: SingleDataset, sensor: str) -> None: mask = np.logical_not(np.logical_or(np.isnan(p_sigma), np.isnan(q_sigma))) if power_sigma.ndim < mask.ndim: mask = np.any(mask, axis=tuple(range(power_sigma.ndim, mask.ndim))) - power_sigma[mask] = np.sqrt(np.sum(np.square(p_sigma[mask]) + np.square(q_sigma[mask]), axis=-1)) - data[sensor]["power_sigma"] = power_sigma + for sublist, should_remove in zip(sensor_mask, mask): + if should_remove and "power_sigma" in sublist: + sublist = cast(List[str], sublist) + sublist.remove("power_sigma") def validate_required_values( @@ -258,8 +263,8 @@ def validate_required_values( An empty list if all required data is available, or a list of MissingValueErrors. """ # Base - required = {"base": ["id"]} - + required: Dict[str, List[Union[str, List[str]]]] = {"base": ["id"]} + # Nodes required["node"] = required["base"] + ["u_rated"] @@ -335,17 +340,27 @@ def validate_required_values( required["power_sensor"] += ["power_sigma", "p_measured", "q_measured"] required["sym_voltage_sensor"] = required["voltage_sensor"].copy() required["asym_voltage_sensor"] = required["voltage_sensor"].copy() - required["sym_power_sensor"] = required["power_sensor"].copy() - required["asym_power_sensor"] = required["power_sensor"].copy() + # Different requirements for individual sensors. Avoid shallow copy. + try: + required["sym_power_sensor"] = [ + required["power_sensor"].copy() for _ in range(data["sym_power_sensor"].shape[0]) + ] + except KeyError: + pass + try: + required["asym_power_sensor"] = [ + required["power_sensor"].copy() for _ in range(data["asym_power_sensor"].shape[0]) + ] + except KeyError: + pass - _data = data # proxy # Faults required["fault"] = required["base"] + ["fault_object"] asym_sc = False if calculation_type is None or calculation_type == CalculationType.short_circuit: required["fault"] += ["status", "fault_type"] - if "fault" in _data: - for elem in _data["fault"]["fault_type"]: + if "fault" in data: + for elem in data["fault"]["fault_type"]: if elem not in (FaultType.three_phase, FaultType.nan): asym_sc = True break @@ -354,11 +369,25 @@ def validate_required_values( required["line"] += ["r0", "x0", "c0", "tan0"] required["shunt"] += ["g0", "b0"] - # Allow missing `power_sigma` of both `p_sigma` and `q_sigma` are present - process_power_sigma_and_p_q_sigma(_data, "sym_power_sensor") - process_power_sigma_and_p_q_sigma(_data, "asym_power_sensor") - - return list(chain(*(none_missing(_data, component, required.get(component, [])) for component in _data))) + # Mark `power_sigma` for each power_sensor based on the state of `p_sigma` and `q_sigma` + process_power_sigma_and_p_q_sigma(data, "sym_power_sensor", required) + process_power_sigma_and_p_q_sigma(data, "asym_power_sensor", required) + + return list( + chain( + *( + none_missing(data, component, item, index) + for component in data + for index, item in enumerate( + (sublist for sublist in required.get(component, [])) + if isinstance(required.get(component, []), list) + and all(isinstance(i, list) for i in required.get(component, [])) + else [required.get(component, [])] + ) + if data.get(component) is not None and index < len(data.get(component)) + ) + ) + ) def validate_values(data: SingleDataset, calculation_type: Optional[CalculationType] = None) -> List[ValidationError]: From 687dbb00d2a390d8e00fca5bdac2c4ceebffa9e8 Mon Sep 17 00:00:00 2001 From: Jerry Guo Date: Fri, 23 Feb 2024 21:30:39 +0100 Subject: [PATCH 10/16] Down to final two mypy complaints Signed-off-by: Jerry Guo --- src/power_grid_model/validation/rules.py | 2 +- src/power_grid_model/validation/validation.py | 13 +++++++------ 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/src/power_grid_model/validation/rules.py b/src/power_grid_model/validation/rules.py index cfba7c0f0..056225ce6 100644 --- a/src/power_grid_model/validation/rules.py +++ b/src/power_grid_model/validation/rules.py @@ -668,7 +668,7 @@ def all_finite(data: SingleDataset, exceptions: Optional[Dict[str, List[str]]] = def none_missing( - data: SingleDataset, component: str, fields: Union[str, List[str], List[List[str]]], index: int = 0 + data: SingleDataset, component: str, fields: Union[List[Union[str, List[str]]], str, List[str]], index: int = 0 ) -> List[MissingValueError]: """ Check that for all records of a particular type of component, the values in the 'fields' columns are not NaN. diff --git a/src/power_grid_model/validation/validation.py b/src/power_grid_model/validation/validation.py index 676b4f9d4..5e1c23cc6 100644 --- a/src/power_grid_model/validation/validation.py +++ b/src/power_grid_model/validation/validation.py @@ -10,7 +10,7 @@ """ import copy from itertools import chain -from typing import Dict, List, Optional, Union, cast +from typing import Dict, List, Optional, Sized, Union, cast import numpy as np @@ -264,7 +264,7 @@ def validate_required_values( """ # Base required: Dict[str, List[Union[str, List[str]]]] = {"base": ["id"]} - + # Nodes required["node"] = required["base"] + ["u_rated"] @@ -368,11 +368,11 @@ def validate_required_values( if not symmetric or asym_sc: required["line"] += ["r0", "x0", "c0", "tan0"] required["shunt"] += ["g0", "b0"] - + # Mark `power_sigma` for each power_sensor based on the state of `p_sigma` and `q_sigma` process_power_sigma_and_p_q_sigma(data, "sym_power_sensor", required) process_power_sigma_and_p_q_sigma(data, "asym_power_sensor", required) - + return list( chain( *( @@ -382,9 +382,10 @@ def validate_required_values( (sublist for sublist in required.get(component, [])) if isinstance(required.get(component, []), list) and all(isinstance(i, list) for i in required.get(component, [])) - else [required.get(component, [])] + else [required[component]] ) - if data.get(component) is not None and index < len(data.get(component)) + if data[component] is not None and isinstance(data[component], Sized) + and index < len(data[component]) ) ) ) From d0e64e43059492d354862fd5c487ae80b714f8db Mon Sep 17 00:00:00 2001 From: Jerry Guo Date: Sat, 24 Feb 2024 12:49:45 +0100 Subject: [PATCH 11/16] mypy type warning ignore Signed-off-by: Jerry Guo --- src/power_grid_model/validation/validation.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/power_grid_model/validation/validation.py b/src/power_grid_model/validation/validation.py index 5e1c23cc6..a4570de60 100644 --- a/src/power_grid_model/validation/validation.py +++ b/src/power_grid_model/validation/validation.py @@ -343,13 +343,13 @@ def validate_required_values( # Different requirements for individual sensors. Avoid shallow copy. try: required["sym_power_sensor"] = [ - required["power_sensor"].copy() for _ in range(data["sym_power_sensor"].shape[0]) + required["power_sensor"].copy() for _ in range(data["sym_power_sensor"].shape[0]) # type: ignore ] except KeyError: pass try: required["asym_power_sensor"] = [ - required["power_sensor"].copy() for _ in range(data["asym_power_sensor"].shape[0]) + required["power_sensor"].copy() for _ in range(data["asym_power_sensor"].shape[0]) # type: ignore ] except KeyError: pass From 10fb4757f286f47cbc3786ae58bfb09813bcc2a5 Mon Sep 17 00:00:00 2001 From: Jerry Guo Date: Sat, 24 Feb 2024 12:54:00 +0100 Subject: [PATCH 12/16] black reformat Signed-off-by: Jerry Guo --- src/power_grid_model/validation/validation.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/power_grid_model/validation/validation.py b/src/power_grid_model/validation/validation.py index a4570de60..56ba6ee2f 100644 --- a/src/power_grid_model/validation/validation.py +++ b/src/power_grid_model/validation/validation.py @@ -368,11 +368,11 @@ def validate_required_values( if not symmetric or asym_sc: required["line"] += ["r0", "x0", "c0", "tan0"] required["shunt"] += ["g0", "b0"] - + # Mark `power_sigma` for each power_sensor based on the state of `p_sigma` and `q_sigma` process_power_sigma_and_p_q_sigma(data, "sym_power_sensor", required) process_power_sigma_and_p_q_sigma(data, "asym_power_sensor", required) - + return list( chain( *( @@ -384,8 +384,7 @@ def validate_required_values( and all(isinstance(i, list) for i in required.get(component, [])) else [required[component]] ) - if data[component] is not None and isinstance(data[component], Sized) - and index < len(data[component]) + if data[component] is not None and isinstance(data[component], Sized) and index < len(data[component]) ) ) ) From 673919be3a20641c2fb8c54a5d67809e9b51bff5 Mon Sep 17 00:00:00 2001 From: Jerry Guo Date: Sat, 24 Feb 2024 13:17:16 +0100 Subject: [PATCH 13/16] Code smells Signed-off-by: Jerry Guo --- src/power_grid_model/validation/rules.py | 9 ++---- src/power_grid_model/validation/validation.py | 30 ++++++++++++------- 2 files changed, 22 insertions(+), 17 deletions(-) diff --git a/src/power_grid_model/validation/rules.py b/src/power_grid_model/validation/rules.py index 056225ce6..bc21878d3 100644 --- a/src/power_grid_model/validation/rules.py +++ b/src/power_grid_model/validation/rules.py @@ -690,20 +690,15 @@ def none_missing( if isinstance(field, list): field = field[0] nan = nan_type(component, field) - invalid = None if np.isnan(nan): invalid = np.isnan(data[component][field][index]) else: invalid = np.equal(data[component][field][index], nan) - if invalid is not None and invalid.any(): + if invalid.any(): if isinstance(invalid, np.ndarray): invalid = np.any(invalid) - ids = [] - if isinstance(data[component][field], list) and len(data[component][field]) > index: - ids = data[component]["id"][index][invalid].flatten().tolist() - else: - ids = data[component]["id"][invalid].flatten().tolist() + ids = data[component]["id"][invalid].flatten().tolist() errors.append(MissingValueError(component, field, ids)) return errors diff --git a/src/power_grid_model/validation/validation.py b/src/power_grid_model/validation/validation.py index 56ba6ee2f..e1fa09a44 100644 --- a/src/power_grid_model/validation/validation.py +++ b/src/power_grid_model/validation/validation.py @@ -230,22 +230,32 @@ def process_power_sigma_and_p_q_sigma( `p_sigma` and `q_sigma`in this case. Happens only on proxy data (not the original data). However, note that this value is eventually not used in the calculation. """ - if sensor in data and isinstance(data[sensor], np.ndarray): + + def _check_sensor_in_data(_data, _sensor): + return _sensor in _data and isinstance(_data[_sensor], np.ndarray) + + def _contains_p_q_sigma(_sensor_data): + return "p_sigma" in _sensor_data.dtype.names and "q_sigma" in _sensor_data.dtype.names + + def _process_power_sigma_in_list(_sensor_mask, _power_sigma, _p_sigma, _q_sigma): + _mask = np.logical_not(np.logical_or(np.isnan(_p_sigma), np.isnan(_q_sigma))) + if _power_sigma.ndim < _mask.ndim: + _mask = np.any(_mask, axis=tuple(range(_power_sigma.ndim, _mask.ndim))) + + for sublist, should_remove in zip(_sensor_mask, _mask): + if should_remove and "power_sigma" in sublist: + sublist = cast(List[str], sublist) + sublist.remove("power_sigma") + + if _check_sensor_in_data(data, sensor): sensor_data = data[sensor] sensor_mask = required_list[sensor] - if "p_sigma" in sensor_data.dtype.names and "q_sigma" in sensor_data.dtype.names: + if _contains_p_q_sigma(sensor_data): p_sigma = sensor_data["p_sigma"] q_sigma = sensor_data["q_sigma"] power_sigma = sensor_data["power_sigma"] - mask = np.logical_not(np.logical_or(np.isnan(p_sigma), np.isnan(q_sigma))) - if power_sigma.ndim < mask.ndim: - mask = np.any(mask, axis=tuple(range(power_sigma.ndim, mask.ndim))) - - for sublist, should_remove in zip(sensor_mask, mask): - if should_remove and "power_sigma" in sublist: - sublist = cast(List[str], sublist) - sublist.remove("power_sigma") + _process_power_sigma_in_list(sensor_mask, power_sigma, p_sigma, q_sigma) def validate_required_values( From 489c7559113f40692ff247bfae11061bcd9b4f13 Mon Sep 17 00:00:00 2001 From: Jerry Guo Date: Fri, 1 Mar 2024 15:09:47 +0100 Subject: [PATCH 14/16] Cleaned up the chained list return statement Signed-off-by: Jerry Guo --- src/power_grid_model/validation/validation.py | 57 +++++++++++++------ 1 file changed, 41 insertions(+), 16 deletions(-) diff --git a/src/power_grid_model/validation/validation.py b/src/power_grid_model/validation/validation.py index e1fa09a44..241b9a12c 100644 --- a/src/power_grid_model/validation/validation.py +++ b/src/power_grid_model/validation/validation.py @@ -9,8 +9,9 @@ """ import copy +from collections.abc import Sized as ABCSized from itertools import chain -from typing import Dict, List, Optional, Sized, Union, cast +from typing import Dict, List, Optional, Union, cast import numpy as np @@ -379,25 +380,49 @@ def validate_required_values( required["line"] += ["r0", "x0", "c0", "tan0"] required["shunt"] += ["g0", "b0"] - # Mark `power_sigma` for each power_sensor based on the state of `p_sigma` and `q_sigma` process_power_sigma_and_p_q_sigma(data, "sym_power_sensor", required) process_power_sigma_and_p_q_sigma(data, "asym_power_sensor", required) - return list( - chain( - *( - none_missing(data, component, item, index) - for component in data - for index, item in enumerate( - (sublist for sublist in required.get(component, [])) - if isinstance(required.get(component, []), list) - and all(isinstance(i, list) for i in required.get(component, [])) - else [required[component]] - ) - if data[component] is not None and isinstance(data[component], Sized) and index < len(data[component]) - ) + return _validate_required_in_data(data, required) + + +def _validate_required_in_data(data, required): + """ + Checks if all required data is available. + + Args: + data: A power-grid-model input dataset + required: a list of required fields (a list of str), per component when applicaple (a list of str or str lists) + + Returns: + An empty list if all required data is available, or a list of MissingValueErrors. + """ + def is_valid_component(data, component): + return ( + not (isinstance(data[component], np.ndarray) and data[component].size == 0) + and data[component] is not None + and isinstance(data[component], ABCSized) ) - ) + + def is_nested_list(items): + return isinstance(items, list) and all(isinstance(i, list) for i in items) + + def process_nested_items(component, items, data, results): + for index, item in enumerate(sublist for sublist in items): + if index < len(data[component]): + results.append(none_missing(data, component, item, index)) + + results = [] + + for component in data: + if is_valid_component(data, component): + items = required.get(component, []) + if is_nested_list(items): + process_nested_items(component, items, data, results) + else: + results.append(none_missing(data, component, items, 0)) + + return list(chain(*results)) def validate_values(data: SingleDataset, calculation_type: Optional[CalculationType] = None) -> List[ValidationError]: From 87c389f78f6ccaffd9346f258ce5a89fe2e16f8a Mon Sep 17 00:00:00 2001 From: Jerry Guo Date: Fri, 1 Mar 2024 15:15:22 +0100 Subject: [PATCH 15/16] Format Signed-off-by: Jerry Guo --- src/power_grid_model/validation/validation.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/power_grid_model/validation/validation.py b/src/power_grid_model/validation/validation.py index 241b9a12c..e6d3c1d14 100644 --- a/src/power_grid_model/validation/validation.py +++ b/src/power_grid_model/validation/validation.py @@ -397,6 +397,7 @@ def _validate_required_in_data(data, required): Returns: An empty list if all required data is available, or a list of MissingValueErrors. """ + def is_valid_component(data, component): return ( not (isinstance(data[component], np.ndarray) and data[component].size == 0) From f1984a5b198e9b59514658d6328b21370b63805d Mon Sep 17 00:00:00 2001 From: Jerry Guo Date: Fri, 1 Mar 2024 15:32:31 +0100 Subject: [PATCH 16/16] Reviews processed. Signed-off-by: Jerry Guo --- src/power_grid_model/validation/validation.py | 25 ++++++++----------- 1 file changed, 10 insertions(+), 15 deletions(-) diff --git a/src/power_grid_model/validation/validation.py b/src/power_grid_model/validation/validation.py index e6d3c1d14..0483fab8e 100644 --- a/src/power_grid_model/validation/validation.py +++ b/src/power_grid_model/validation/validation.py @@ -222,7 +222,7 @@ def validate_ids_exist(update_data: Dict[str, np.ndarray], input_data: SingleDat return list(chain(*errors)) -def process_power_sigma_and_p_q_sigma( +def _process_power_sigma_and_p_q_sigma( data: SingleDataset, sensor: str, required_list: Dict[str, List[Union[str, List[str]]]] ) -> None: """ @@ -352,18 +352,13 @@ def validate_required_values( required["sym_voltage_sensor"] = required["voltage_sensor"].copy() required["asym_voltage_sensor"] = required["voltage_sensor"].copy() # Different requirements for individual sensors. Avoid shallow copy. - try: - required["sym_power_sensor"] = [ - required["power_sensor"].copy() for _ in range(data["sym_power_sensor"].shape[0]) # type: ignore - ] - except KeyError: - pass - try: - required["asym_power_sensor"] = [ - required["power_sensor"].copy() for _ in range(data["asym_power_sensor"].shape[0]) # type: ignore - ] - except KeyError: - pass + for sensor_type in ("sym_power_sensor", "asym_power_sensor"): + try: + required[sensor_type] = [ + required["power_sensor"].copy() for _ in range(data[sensor_type].shape[0]) # type: ignore + ] + except KeyError: + pass # Faults required["fault"] = required["base"] + ["fault_object"] @@ -380,8 +375,8 @@ def validate_required_values( required["line"] += ["r0", "x0", "c0", "tan0"] required["shunt"] += ["g0", "b0"] - process_power_sigma_and_p_q_sigma(data, "sym_power_sensor", required) - process_power_sigma_and_p_q_sigma(data, "asym_power_sensor", required) + _process_power_sigma_and_p_q_sigma(data, "sym_power_sensor", required) + _process_power_sigma_and_p_q_sigma(data, "asym_power_sensor", required) return _validate_required_in_data(data, required)