Update calibration factors and regenerate baseline data

scout/ecm_prep.py

@@ -498,7 +498,7 @@ def __init__(self, base_dir, handyfiles, opts):
         # Load metadata including AEO year range
         aeo_yrs = Utils.load_json(handyfiles.metadata)
         # Set minimum modeling year to current year
-        aeo_min = datetime.today().year
+        aeo_min = 2023
         # Set maximum modeling year
         aeo_max = aeo_yrs["max year"]
         # Derive time horizon from min/max years

scout/final_mseg_converter.py

@@ -80,7 +80,7 @@ class UsefulVars(object):
             factors (principally costs) for a range of equipment types.
         aeo_metadata (str): File name for the custom AEO metadata JSON.
         geo_break (str): Indicates the intended geographical data breakdown.
-        res_hvac_adj (str): Data used to calibrate against EIA 861.
+        res_calibrate (str): Data used to calibrate against EIA 861.
 
     Attributes: (if a method is called)
         res_climate_convert (str): File name for the residential buildings
@@ -99,7 +99,7 @@ def __init__(self, geo_break):
         self.conv_factors = fp.CONVERT_DATA / 'ecm_cost_convert.json'
         self.aeo_metadata = fp.METADATA_PATH
         self.geo_break = geo_break
-        self.res_hvac_adj = fp.CONVERT_DATA / 'res_hvac_adj.csv'
+        self.res_calibrate = fp.CONVERT_DATA / 'res_calibrate.csv'
 
     def configure_for_energy_square_footage_stock_data(self):
         """Reconfigure stock and energy data to custom region."""
@@ -1348,7 +1348,7 @@ def walk(cpl_data, conversions, perf_convert, years, json_db,
     return json_db
 
 
-def res_hvac_energy_recalibrate(result, handyvars, flag_map_dat):
+def res_energy_recalibrate(result, handyvars, flag_map_dat):
     """Adjust residential baseline HVAC energy to align with EIA 861 data in calibration year.
 
     Args:
@@ -1361,59 +1361,98 @@ def res_hvac_energy_recalibrate(result, handyvars, flag_map_dat):
     """
 
     # Read in adjustment factors
-    res_hvac_adj = pd.read_csv(handyvars.res_hvac_adj)
-    # Set building types for adjustment
+    res_calibrate = pd.read_csv(handyvars.res_calibrate)
+    # Set building types for adjustment (currently residential-only)
     bts = flag_map_dat["res_bldg_types"]
-    # Set fuel type for adjustment
+    # Set fuel type for adjustment (currently only electricity)
     fuel = "electricity"
-    # Set end use for adjustment
-    eus = ["heating", "cooling"]
-    # Set tech types for adjustment (unique to heating/cooling)
-    ttypes = ["supply", "demand"]
-    # Set calibration year
+    # Set end uses that have explicit calibration factors (heating, cooling, other end uses
+    # lumped together)
+    eus_w_cols = ["heating", "cooling"]
+    # Set calibration year (currently 2018 to match Stock weather data year)
     yr_cal = "2018"
 
-    # Proceed to resiential HVAC microsegments and apply calibration factor
+    # Loop through microsegments and apply calibration factors
 
-    # Loop regions
+    # Loop regions (states)
     for reg in result.keys():
-        # Pull 861 calibration adjustment factor for current region (states only, in rows)
-        res_hvac_adj_reg = res_hvac_adj[res_hvac_adj["state"] == reg]
-        # Loop building types
+        # Pull 861 calibration adjustment factor for current state
+        res_calibrate_reg = res_calibrate[res_calibrate["state"] == reg]
+        # Loop residential building types
         for bt in bts:
             # Loop end uses
-            for eu in eus:
-                # Further constrain adjustment factor to current end use (column)
-                adj_fact_2018 = res_hvac_adj_reg[eu].iloc[0]
-                # Loop tech types
+            for eu in result[reg][bt][fuel].keys():
+                # Determine whether calibration data are available explicitly for the end use;
+                # if so, pull data with direct end use name; otherwise set name to use
+                if eu in eus_w_cols:
+                    eu_col = eu
+                # Secondary heating lumped in with heating in the Scout breakout data that
+                # were used to develop calibration factors
+                elif eu == "secondary heating":
+                    eu_col = "heating"
+                # All non-heating/cooling calibration is lumped together into one set of factors
+                else:
+                    eu_col = "non_hvac"
+                # Further constrain adjustment factor for a given benchmark year to the data
+                # corresponding to the current end use (column)
+                adj_fact_bnch = res_calibrate_reg[eu_col].iloc[0]
+
+                # Set shorthand for results data filtered down to end use level
+                result_eu = result[reg][bt][fuel][eu]
+
+                # Determine whether data are nested by technology type (unique to heating and
+                # cooling segments); if not, set tech. type to None
+                if all([x in result[reg][bt][fuel][eu].keys() for x in ["supply", "demand"]]):
+                    ttypes = ["supply", "demand"]
+                else:
+                    ttypes = [None]
+                # Loop tech types (if applicable)
                 for tech_type in ttypes:
-                    # Shorthand for results to calibrate at the tech type level
-                    result_short = result[reg][bt][fuel][eu][tech_type]
+                    # Set shorthand for results to calibrate at the end use/tech type level
+                    if tech_type is not None:
+                        result_ttype = result_eu[tech_type]
+                    else:
+                        result_ttype = result_eu
+                    # Find technology-level keys if technology is not None
+                    if all([x not in result_ttype.keys() for x in ["stock", "energy"]]):
+                        tech_keys = result_ttype.keys()
+                    else:
+                        tech_keys = [None]
                     # Further loop through all technologies under the current branch
-                    for tech in result_short.keys():
-                        # If terminal energy values by year have been reached, set the
-                        # anchor year value to adjust other years' values against
+                    for tech in tech_keys:
+                        # Terminal energy values (broken out by year) have been reached; set the
+                        # anchor year value to adjust other years' values against; raise Error
+                        # if energy data can't be pulled as expected
                         try:
-                            energy_yr_cal_bnch = result_short[tech]["energy"][yr_cal]
+                            if all([x is not None for x in [tech_type, tech]]):
+                                # Further restrict end use data down to energy level
+                                result_yr = result_eu[tech_type][tech]["energy"]
+                            elif tech is not None:
+                                result_yr = result_eu[tech]["energy"]
+                            else:
+                                result_yr = result_eu["energy"]
+                            # Set benchmark energy value to that in the calibration year
+                            energy_yr_cal_bnch = result_yr[yr_cal]
                         except KeyError:
                             raise KeyError(
-                                "Unexpected keys:val pairs in results dict when attempting to "
-                                "recalibrate residential HVAC energy values to state-level data "
+                                "Unexpected key:val pair(s) in results dict when attempting to "
+                                "recalibrate residential energy values to state-level data "
                                 "for branch " + str((reg, bt, fuel, eu, tech_type, tech)))
-                        # Loop through all years and adjust by factor
-                        for yr in result_short[tech]["energy"].keys():
-                            # Set ratio of current year energy to benchmark year energy;
-                            # do not proceed with calibration if benchmark year is zero
+                        # Loop through all years in the data, compare values against that of the
+                        # benchmark year, and adjust accordingly
+                        for yr in result_yr.keys():
+                            # Set ratio of current year energy (pre-calibration) to benchmark year
+                            # energy (also pre-calibration); do not proceed with calibration if
+                            # benchmark year is zero
                             if energy_yr_cal_bnch != 0:
-                                ratio_2018 = result_short[tech]["energy"][yr] / energy_yr_cal_bnch
+                                ratio_bnch = result_yr[yr] / energy_yr_cal_bnch
                             else:
-                                ratio_2018 = None
-                            # Updated value is original 2018 value times adjustment factor
-                            # for 2018 value times original ratio of current year value to 2018
-                            # value
-                            if ratio_2018:
-                                result_short[tech]["energy"][yr] = \
-                                    energy_yr_cal_bnch * adj_fact_2018 * ratio_2018
+                                ratio_bnch = None
+                            # Updated value is original benchmark year value times adjust. factor
+                            # for benchmark year value times original ratio of current year value
+                            # to benchmark year value
+                            if ratio_bnch:
+                                result_yr[yr] = energy_yr_cal_bnch * adj_fact_bnch * ratio_bnch
 
     return result
 
@@ -1732,10 +1771,9 @@ def main():
                     aia_list, cdiv_list, emm_list)
 
     # When state-level energy data are being updated, add a final step that recalibrates
-    # residential HVAC energy values to EIA 861 data to correct for under-prediction of
-    # heating and over-prediction of cooling in residential AEO estimates.
+    # residential energy values to EIA 861 data for a given historical benchmark year.
     if calibrate and (input_var[0] == '1' and input_var[1] == '3'):
-        result = res_hvac_energy_recalibrate(result, handyvars, flag_map_dat)
+        result = res_energy_recalibrate(result, handyvars, flag_map_dat)
 
     # Write the updated dict of data to a new JSON file
     with open(handyvars.json_out, 'w') as jso:

scout/supporting_data/convert_data/res_calibrate.csv

@@ -0,0 +1,52 @@
+state,heating,cooling,non_hvac
+AL,0.9749004,1.454717,0.940976
+AK,1,1,1
+AZ,2.8012055,1.49509,0.6799689
+AR,0.76565,1.229731,1.0263227
+CA,1.7833909,2.629588,0.6721981
+CO,0.5440454,2.811147,0.8638815
+CT,2.5212576,2.536008,0.8174609
+DE,0.933831,1.514577,0.9698233
+DC,0.6070683,1.311067,0.9479272
+FL,1.1839467,1.328785,0.9807416
+GA,1.1816975,1.645985,0.9139628
+HI,1,1,1
+ID,1.2012739,3.099807,1.0650233
+IL,1.1163314,1.455244,0.9201591
+IN,1.0425325,1.520832,1.0981952
+IA,1.1526145,2.015231,0.8374929
+KS,1.1131118,1.377091,0.8345619
+KY,0.7534368,1.662459,0.9230604
+LA,0.7770065,1.234848,1.132061
+ME,2.7326056,1.74885,0.9254869
+MD,0.8060803,1.399681,0.8224304
+MA,1.4897189,1.868592,0.8407826
+MI,1.0812576,1.464066,0.8629026
+MN,0.8325516,2.479172,0.8927959
+MS,0.9784908,1.327848,1.0238028
+MO,1.3528856,1.376668,0.9674225
+MT,1.0381531,1.942264,1.0986211
+NE,1.0065739,1.881887,0.939345
+NV,2.4438897,1.986621,0.7021737
+NH,3.1905836,1.982672,0.8176229
+NJ,1.4923894,1.904171,0.887267
+NM,1.3867238,1.684285,0.6840088
+NY,0.7817498,1.339381,0.9337395
+NC,0.8902963,1.656045,0.8514143
+ND,1.0536,2.469788,1.2737885
+OH,1.0894048,1.587668,0.9792613
+OK,0.8758596,1.336358,0.8920935
+OR,0.7809908,4.336494,1.0211802
+PA,1.1398991,1.628359,0.9917168
+RI,1.3908362,2.35667,0.8129071
+SC,0.9932098,1.658584,0.8442286
+SD,0.9254695,2.023738,1.2006639
+TN,0.772711,1.506668,1.0416
+TX,0.9442187,1.308654,0.9351565
+UT,1.7508457,2.982859,0.8371403
+VT,4.8581005,1.958301,0.7913297
+VA,0.9291349,1.831784,0.8255146
+WA,0.6414446,2.898317,1.1193712
+WV,1.0181314,1.567535,0.9537265
+WI,0.8459745,2.145644,0.9559308
+WY,1.2687011,2.980608,0.9737056