diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml index a2fb907f..fdf06186 100644 --- a/.github/workflows/integration_tests.yml +++ b/.github/workflows/integration_tests.yml @@ -61,7 +61,22 @@ jobs: else python tests/integration_testing/run_workflow.py --yaml tests/integration_testing/integration_test.yml fi - mv ./results/*.json ./tests/integration_testing/results/ + rm -rf ./tests/integration_testing/results/* + mv ./results/* ./tests/integration_testing/results/ + - name: Compare integration test results + run: | + #FIXME temporarily pull from ci_outputs + git fetch origin master ci_outputs + branch_name="${{ github.ref }}" + if [[ $(git diff --exit-code origin/master ./tests/integration_testing/results/agg_results.json ./tests/integration_testing/results/ecm_results.json) ]]; then + mkdir tests/integration_testing/base_results + git show origin/ci_outputs:tests/integration_testing/results/agg_results.json > tests/integration_testing/base_results/agg_results.json + git show origin/ci_outputs:tests/integration_testing/results/ecm_results.json > tests/integration_testing/base_results/ecm_results.json + git show origin/ci_outputs:tests/integration_testing/results/plots/tech_potential/Summary_Data-TP.xlsx > tests/integration_testing/base_results/Summary_Data-TP.xlsx + git show origin/ci_outputs:tests/integration_testing/results/plots/max_adopt_potential/Summary_Data-MAP.xlsx > tests/integration_testing/base_results/Summary_Data-MAP.xlsx + + python tests/integration_testing/compare_results.py --base-dir tests/integration_testing/base_results --new-dir tests/integration_testing/results + fi - name: Upload artifacts uses: actions/upload-artifact@v3 with: @@ -73,6 +88,7 @@ jobs: git pull origin $branch_name git add ./tests/integration_testing/results/*.json if [[ $(git diff --cached --exit-code) ]]; then + git add ./tests/integration_testing/results/plots git config --system user.email "github-action@users.noreply.github.com" git config --system user.name "GitHub Action" git commit -m "Upload results files from CI build" diff --git a/.gitignore b/.gitignore index 9e661632..18b46e71 100644 --- a/.gitignore +++ b/.gitignore @@ -108,4 +108,10 @@ _build ################### build/ -scout.egg-info/ \ No newline at end of file +scout.egg-info/ + +# Exclude # +########### + +!tests/integration_testing/results/plots/tech_potential/*.xlsx +!tests/integration_testing/results/plots/max_adopt_potential/*.xlsx \ No newline at end of file diff --git a/tests/integration_testing/compare_results.py b/tests/integration_testing/compare_results.py new file mode 100644 index 00000000..283d86cd --- /dev/null +++ b/tests/integration_testing/compare_results.py @@ -0,0 +1,217 @@ +import pandas as pd +import argparse +import json +import re +from pathlib import Path + + +class ScoutCompare(): + """Class to compare results from Scout workflow run. Comparisons are saved as csv files to + summarize differences in results json files (agg_results.json, ecm_results.json) and/or + summary report files (Summary_Data-TP.xlsx, Summary_Data-MAP.xlsx) + """ + + @staticmethod + def load_json(file_path): + with open(file_path, 'r') as file: + return json.load(file) + + @staticmethod + def load_summary_report(file_path): + reports = pd.read_excel(file_path, sheet_name=None, index_col=list(range(5))) + return reports + + def compare_dict_keys(self, dict1, dict2, paths, path='', key_diffs=None): + """Compares nested keys across two dictionaries by recursively searching each level + + Args: + dict1 (dict): baseline dictionary to compare + dict2 (dict): new dictionary to compare + paths (list): paths to the original files from which the dictionaries are imported + path (str, optional): current dictionary path at whcih to compare. Defaults to ''. + key_diffs (pd.DataFrame, optional): existing summary of difference. Defaults to None. + + Returns: + pd.DataFrame: summary of differences specifying the file, the unique keys, and the + path that key is found at. + """ + if key_diffs is None: + key_diffs = pd.DataFrame(columns=["Results file", "Unique key", "Found at"]) + keys1 = set(dict1.keys()) + keys2 = set(dict2.keys()) + only_in_dict1 = keys1 - keys2 + only_in_dict2 = keys2 - keys1 + + if only_in_dict1: + new_row = pd.DataFrame({"Results file": f"{paths[0].parent.name}/{paths[0].name}", + "Unique key": str(only_in_dict1), + "Found at": path[2:]}, index=[0]) + key_diffs = pd.concat([key_diffs, new_row], ignore_index=True) + if only_in_dict2: + new_row = pd.DataFrame({"Results file": f"{paths[1].parent.name}/{paths[1].name}", + "Unique key": str(only_in_dict2), + "Found at": path[2:]}, index=[0]) + key_diffs = pd.concat([key_diffs, new_row], ignore_index=True) + + for key in keys1.intersection(keys2): + if isinstance(dict1[key], dict) and isinstance(dict2[key], dict): + key_diffs = self.compare_dict_keys(dict1[key], + dict2[key], + paths, + path=f"{path}: {key}", + key_diffs=key_diffs) + + return key_diffs + + def compare_dict_values(self, dict1, dict2, percent_threshold=10, abs_threshold=1000): + """Compares values across two dictionary by recursively searching keys until identifying + values at common paths. Both thresholds must be met to report results. + + Args: + dict1 (dict): baseline dictionary to compare + dict2 (dict): new dictionary to compare + percent_threshold (int, optional): the percent difference threshold at which + differences are reported. Defaults to 10. + abs_threshold (int, optional): the abosolute difference threshold at which differences + are reported. Defaults to 10. + + Returns: + pd.DataFrame: summary of percent differences that meet thresholds + """ + diff_report = {} + + def compare_recursive(d1, d2, path=""): + for key in d1.keys(): + current_path = f"{path}['{key}']" + if isinstance(d1[key], dict) and key in d2: + compare_recursive(d1[key], d2[key], current_path) + elif isinstance(d1[key], (int, float)) and key in d2: + if isinstance(d2[key], (int, float)): + val1 = d1[key] + val2 = d2[key] + if val1 != 0: + percent_change = ((val2 - val1) / val1) * 100 + if (abs(percent_change) >= percent_threshold) and \ + (abs(val1) >= abs_threshold or abs(val2) >= abs_threshold): + diff_report[current_path] = percent_change + + compare_recursive(dict1, dict2) + return diff_report + + def split_json_key_path(self, path): + keys = re.findall(r"\['(.*?)'\]", path) + if len(keys) == 5: + keys[4:4] = [None, None, None] + return keys + + def write_dict_key_report(self, diff_report, output_path): + if diff_report.empty: + return + diff_report.to_csv(output_path, index=False) + print(f"Wrote dictionary key report to {output_path}") + + def write_dict_value_report(self, diff_report, output_path): + col_headers = [ + "ECM", + "Markets and Savings Type", + "Adoption Scenario", + "Results Scenario", + "Climate Zone", + "Building Class", + "End Use", + "Year" + ] + df = pd.DataFrame(columns=["Results path"], data=list(diff_report.keys())) + if df.empty: + return + df[col_headers] = df["Results path"].apply(self.split_json_key_path).apply(pd.Series) + df["Percent difference"] = [round(diff, 2) for diff in diff_report.values()] + df = df.dropna(axis=1, how="all") + df.to_csv(output_path, index=False) + print(f"Wrote dictionary value report to {output_path}") + + def compare_jsons(self, json1_path, json2_path, output_dir=True): + """Compare two jsons and report differences in keys and in values + + Args: + json1_path (Path): baseline json file to compare + json2_path (Path): new json file to compare + write_reports (bool, optional): _description_. Defaults to True. + """ + json1 = self.load_json(json1_path) + json2 = self.load_json(json2_path) + + # Compare differences in json keys + key_diffs = self.compare_dict_keys(json1, json2, [json1_path, json2_path]) + if output_dir is None: + output_dir = json2_path.parent + self.write_dict_key_report(key_diffs, output_dir / f"{json2_path.stem}_key_diffs.csv") + + # Compare differences in json values + val_diffs = self.compare_dict_values(json1, json2) + self.write_dict_value_report(val_diffs, output_dir / f"{json2_path.stem}_value_diffs.csv") + + def compare_summary_reports(self, report1_path, report2_path, output_dir=None): + """Compare Summary_Data-TP.xlsx and Summary_Data-MAP.xlsx with baseline files + + Args: + report1_path (Path): baseline summary report to compare + report2_path (Path): new summary report to compare + output_dir (Path, optional): _description_. Defaults to None. + """ + + reports1 = self.load_summary_report(report1_path) + reports2 = self.load_summary_report(report2_path) + if output_dir is None: + output_dir = report2_path.parent + output_path = output_dir / f"{report2_path.stem}_percent_diffs.xlsx" + with pd.ExcelWriter(output_path) as writer: + for (output_type, report1), (_, report2) in zip(reports1.items(), reports2.items()): + diff = (100 * (report2 - report1)/report1).round(2) + diff = diff.reset_index() + diff.to_excel(writer, sheet_name=output_type, index=False) + print(f"Wrote Summary_Data percent difference report to {output_path}") + + +def main(): + parser = argparse.ArgumentParser(description="Compare results files for Scout.") + parser.add_argument("--json-baseline", type=Path, help="Path to the baseline JSON file") + parser.add_argument("--json-new", type=Path, help="Path to the new JSON file") + parser.add_argument("--summary-baseline", type=Path, + help="Path to the baseline summary report (Excel file)") + parser.add_argument("--summary-new", type=Path, + help="Path to the new summary report (Excel file)") + parser.add_argument("--new-dir", type=Path, help="Directory containing files to compare") + parser.add_argument("--base-dir", type=Path, help="Directory containing files to compare") + parser.add_argument("--threshold", type=float, default=10, + help="Threshold for percent difference") + args = parser.parse_args() + + compare = ScoutCompare() + if args.base_dir and args.new_dir: + # Compare all files + base_dir = args.base_dir.resolve() + new_dir = args.new_dir.resolve() + agg_json_base = base_dir / "agg_results.json" + agg_json_new = new_dir / "agg_results.json" + compare.compare_jsons(agg_json_base, agg_json_new, output_dir=new_dir) + ecm_json_base = base_dir / "ecm_results.json" + ecm_json_new = new_dir / "ecm_results.json" + compare.compare_jsons(ecm_json_base, ecm_json_new, output_dir=new_dir) + + summary_tp_base = base_dir / "Summary_Data-TP.xlsx" + summary_tp_new = new_dir / "plots" / "tech_potential" / "Summary_Data-TP.xlsx" + compare.compare_summary_reports(summary_tp_base, summary_tp_new, output_dir=new_dir) + summary_map_base = base_dir / "Summary_Data-MAP.xlsx" + summary_map_new = new_dir / "plots" / "max_adopt_potential" / "Summary_Data-MAP.xlsx" + compare.compare_summary_reports(summary_map_base, summary_map_new, output_dir=new_dir) + else: + # Compare only as specified by the arguments + if args.json_baseline and args.json_new: + compare.compare_jsons(args.json_baseline, args.json_new) + if args.summary_baseline and args.summary_new: + compare.compare_summary_reports(args.summary_baseline, args.summary_new) + + +if __name__ == "__main__": + main() diff --git a/tests/integration_testing/results/plots/max_adopt_potential/Summary_Data-MAP.xlsx b/tests/integration_testing/results/plots/max_adopt_potential/Summary_Data-MAP.xlsx new file mode 100644 index 00000000..5ca135d9 Binary files /dev/null and b/tests/integration_testing/results/plots/max_adopt_potential/Summary_Data-MAP.xlsx differ diff --git a/tests/integration_testing/results/plots/max_adopt_potential/co2/Cost Effective Avoided CO2-MAP.pdf b/tests/integration_testing/results/plots/max_adopt_potential/co2/Cost Effective Avoided CO2-MAP.pdf new file mode 100644 index 00000000..01ce941f Binary files /dev/null and b/tests/integration_testing/results/plots/max_adopt_potential/co2/Cost Effective Avoided CO2-MAP.pdf differ diff --git a/tests/integration_testing/results/plots/max_adopt_potential/co2/Total Avoided CO2-MAP-Aggregate.pdf b/tests/integration_testing/results/plots/max_adopt_potential/co2/Total Avoided CO2-MAP-Aggregate.pdf new file mode 100644 index 00000000..057727af Binary files /dev/null and b/tests/integration_testing/results/plots/max_adopt_potential/co2/Total Avoided CO2-MAP-Aggregate.pdf differ diff --git a/tests/integration_testing/results/plots/max_adopt_potential/co2/Total CO2-MAP-byECM.pdf b/tests/integration_testing/results/plots/max_adopt_potential/co2/Total CO2-MAP-byECM.pdf new file mode 100644 index 00000000..8d4607cb Binary files /dev/null and b/tests/integration_testing/results/plots/max_adopt_potential/co2/Total CO2-MAP-byECM.pdf differ diff --git a/tests/integration_testing/results/plots/max_adopt_potential/cost/Cost Effective Operation Cost Savings-MAP.pdf b/tests/integration_testing/results/plots/max_adopt_potential/cost/Cost Effective Operation Cost Savings-MAP.pdf new file mode 100644 index 00000000..38ed6b6b Binary files /dev/null and b/tests/integration_testing/results/plots/max_adopt_potential/cost/Cost Effective Operation Cost Savings-MAP.pdf differ diff --git a/tests/integration_testing/results/plots/max_adopt_potential/cost/Total Cost Savings-MAP-Aggregate.pdf b/tests/integration_testing/results/plots/max_adopt_potential/cost/Total Cost Savings-MAP-Aggregate.pdf new file mode 100644 index 00000000..b2b3ef44 Binary files /dev/null and b/tests/integration_testing/results/plots/max_adopt_potential/cost/Total Cost Savings-MAP-Aggregate.pdf differ diff --git a/tests/integration_testing/results/plots/max_adopt_potential/cost/Total Cost-MAP-byECM.pdf b/tests/integration_testing/results/plots/max_adopt_potential/cost/Total Cost-MAP-byECM.pdf new file mode 100644 index 00000000..f072b28d Binary files /dev/null and b/tests/integration_testing/results/plots/max_adopt_potential/cost/Total Cost-MAP-byECM.pdf differ diff --git a/tests/integration_testing/results/plots/max_adopt_potential/energy/Cost Effective Energy Savings-MAP.pdf b/tests/integration_testing/results/plots/max_adopt_potential/energy/Cost Effective Energy Savings-MAP.pdf new file mode 100644 index 00000000..3f55da14 Binary files /dev/null and b/tests/integration_testing/results/plots/max_adopt_potential/energy/Cost Effective Energy Savings-MAP.pdf differ diff --git a/tests/integration_testing/results/plots/max_adopt_potential/energy/Total Energy Savings-MAP-Aggregate.pdf b/tests/integration_testing/results/plots/max_adopt_potential/energy/Total Energy Savings-MAP-Aggregate.pdf new file mode 100644 index 00000000..b77d0e25 Binary files /dev/null and b/tests/integration_testing/results/plots/max_adopt_potential/energy/Total Energy Savings-MAP-Aggregate.pdf differ diff --git a/tests/integration_testing/results/plots/max_adopt_potential/energy/Total Energy-MAP-byECM.pdf b/tests/integration_testing/results/plots/max_adopt_potential/energy/Total Energy-MAP-byECM.pdf new file mode 100644 index 00000000..24ed049b Binary files /dev/null and b/tests/integration_testing/results/plots/max_adopt_potential/energy/Total Energy-MAP-byECM.pdf differ diff --git a/tests/integration_testing/results/plots/tech_potential/Summary_Data-TP.xlsx b/tests/integration_testing/results/plots/tech_potential/Summary_Data-TP.xlsx new file mode 100644 index 00000000..dd934d07 Binary files /dev/null and b/tests/integration_testing/results/plots/tech_potential/Summary_Data-TP.xlsx differ diff --git a/tests/integration_testing/results/plots/tech_potential/co2/Cost Effective Avoided CO2-TP.pdf b/tests/integration_testing/results/plots/tech_potential/co2/Cost Effective Avoided CO2-TP.pdf new file mode 100644 index 00000000..e524b4b5 Binary files /dev/null and b/tests/integration_testing/results/plots/tech_potential/co2/Cost Effective Avoided CO2-TP.pdf differ diff --git a/tests/integration_testing/results/plots/tech_potential/co2/Total Avoided CO2-TP-Aggregate.pdf b/tests/integration_testing/results/plots/tech_potential/co2/Total Avoided CO2-TP-Aggregate.pdf new file mode 100644 index 00000000..fe5120d9 Binary files /dev/null and b/tests/integration_testing/results/plots/tech_potential/co2/Total Avoided CO2-TP-Aggregate.pdf differ diff --git a/tests/integration_testing/results/plots/tech_potential/co2/Total CO2-TP-byECM.pdf b/tests/integration_testing/results/plots/tech_potential/co2/Total CO2-TP-byECM.pdf new file mode 100644 index 00000000..99432d21 Binary files /dev/null and b/tests/integration_testing/results/plots/tech_potential/co2/Total CO2-TP-byECM.pdf differ diff --git a/tests/integration_testing/results/plots/tech_potential/cost/Cost Effective Operation Cost Savings-TP.pdf b/tests/integration_testing/results/plots/tech_potential/cost/Cost Effective Operation Cost Savings-TP.pdf new file mode 100644 index 00000000..15241a7b Binary files /dev/null and b/tests/integration_testing/results/plots/tech_potential/cost/Cost Effective Operation Cost Savings-TP.pdf differ diff --git a/tests/integration_testing/results/plots/tech_potential/cost/Total Cost Savings-TP-Aggregate.pdf b/tests/integration_testing/results/plots/tech_potential/cost/Total Cost Savings-TP-Aggregate.pdf new file mode 100644 index 00000000..af5a9e81 Binary files /dev/null and b/tests/integration_testing/results/plots/tech_potential/cost/Total Cost Savings-TP-Aggregate.pdf differ diff --git a/tests/integration_testing/results/plots/tech_potential/cost/Total Cost-TP-byECM.pdf b/tests/integration_testing/results/plots/tech_potential/cost/Total Cost-TP-byECM.pdf new file mode 100644 index 00000000..f8648d33 Binary files /dev/null and b/tests/integration_testing/results/plots/tech_potential/cost/Total Cost-TP-byECM.pdf differ diff --git a/tests/integration_testing/results/plots/tech_potential/energy/Cost Effective Energy Savings-TP.pdf b/tests/integration_testing/results/plots/tech_potential/energy/Cost Effective Energy Savings-TP.pdf new file mode 100644 index 00000000..6af347f9 Binary files /dev/null and b/tests/integration_testing/results/plots/tech_potential/energy/Cost Effective Energy Savings-TP.pdf differ diff --git a/tests/integration_testing/results/plots/tech_potential/energy/Total Energy Savings-TP-Aggregate.pdf b/tests/integration_testing/results/plots/tech_potential/energy/Total Energy Savings-TP-Aggregate.pdf new file mode 100644 index 00000000..fd8753cd Binary files /dev/null and b/tests/integration_testing/results/plots/tech_potential/energy/Total Energy Savings-TP-Aggregate.pdf differ diff --git a/tests/integration_testing/results/plots/tech_potential/energy/Total Energy-TP-byECM.pdf b/tests/integration_testing/results/plots/tech_potential/energy/Total Energy-TP-byECM.pdf new file mode 100644 index 00000000..446aae6a Binary files /dev/null and b/tests/integration_testing/results/plots/tech_potential/energy/Total Energy-TP-byECM.pdf differ