From e6a40077d2e0008315f798995bb9cde3746c37fd Mon Sep 17 00:00:00 2001 From: Kevin Anderson Date: Wed, 31 May 2023 13:22:16 -0400 Subject: [PATCH 1/8] gallery: rename shifts folder to include time --- docs/examples/data-shifts/README.rst | 4 ---- docs/examples/data-time-shifts/README.rst | 4 ++++ .../examples/{data-shifts => data-time-shifts}/data-shifts.py | 0 3 files changed, 4 insertions(+), 4 deletions(-) delete mode 100644 docs/examples/data-shifts/README.rst create mode 100644 docs/examples/data-time-shifts/README.rst rename docs/examples/{data-shifts => data-time-shifts}/data-shifts.py (100%) diff --git a/docs/examples/data-shifts/README.rst b/docs/examples/data-shifts/README.rst deleted file mode 100644 index 9927c943d..000000000 --- a/docs/examples/data-shifts/README.rst +++ /dev/null @@ -1,4 +0,0 @@ -Data Shifts ------------ - -This includes examples for identifying data/capacity shifts in time series data. \ No newline at end of file diff --git a/docs/examples/data-time-shifts/README.rst b/docs/examples/data-time-shifts/README.rst new file mode 100644 index 000000000..71cb8e9fc --- /dev/null +++ b/docs/examples/data-time-shifts/README.rst @@ -0,0 +1,4 @@ +Data/Time Shifts +---------------- + +This includes examples for identifying data/capacity/time shifts in time series data. \ No newline at end of file diff --git a/docs/examples/data-shifts/data-shifts.py b/docs/examples/data-time-shifts/data-shifts.py similarity index 100% rename from docs/examples/data-shifts/data-shifts.py rename to docs/examples/data-time-shifts/data-shifts.py From b0e3b3e7c50231664f8762827fc5d406eea8d373 Mon Sep 17 00:00:00 2001 From: Kevin Anderson Date: Wed, 31 May 2023 13:40:43 -0400 Subject: [PATCH 2/8] add gallery page --- .../data-time-shifts/shifts-ruptures.py | 84 +++++++++++++++++++ 1 file changed, 84 insertions(+) create mode 100644 docs/examples/data-time-shifts/shifts-ruptures.py diff --git a/docs/examples/data-time-shifts/shifts-ruptures.py b/docs/examples/data-time-shifts/shifts-ruptures.py new file mode 100644 index 000000000..efca313cc --- /dev/null +++ b/docs/examples/data-time-shifts/shifts-ruptures.py @@ -0,0 +1,84 @@ +""" +Identifying and estimating time shifts +====================================== + +Identifying time shifts from clock errors or uncorrected Daylight Saving Time. +""" + +# %% +# Time shifts can occur in measured data due to clock errors and +# time zone issues (for example, assuming a dataset is in local standard time +# when in fact it contains Daylight Saving Time). +# +# This example uses :py:func:`~pvanalytics.quality.time.shifts_ruptures` +# to identify abrupt time shifts in a time series, and estimate the +# corresponding time shift amount. + +import pvlib +import pandas as pd +from pvanalytics.quality.time import shifts_ruptures +from pvanalytics.features.daytime import power_or_irradiance +import matplotlib.pyplot as plt + + +# %% +# Typically this process would be applied to measured data with possibly +# untrustworthy timestamps. However, for instructional purposes here, +# we'll create an artificial example dataset that contains a time shift +# due to DST. + +# use a time zone (US/Eastern) that is affected by DST. +# Etc/GMT+5 is the corresponding local standard time zone. +times = pd.date_range('2019-01-01', '2019-12-31', freq='5T', tz='US/Eastern') +location = pvlib.location.Location(40, -80) +cs = location.get_clearsky(times) +measured_signal = cs['ghi'] + +# %% +# The :py:func:`~pvanalytics.quality.time.shifts_ruptures` function +# is centered around comparing the timing of events observed in the measured +# data with expected timings for those same events. +# In this case, we'll use the timing of solar noon as the event. +# +# First, we'll extract the timing of solar noon from the measured data. +# This could be done in several ways; here we will just take the midpoint +# between sunrise and sunset using times estimated with +# :py:func:`~pvanalytics.features.daytime.power_or_irradiance`. + +is_daytime = power_or_irradiance(measured_signal) +daytime_timestamps = measured_signal.index[is_daytime].to_series() +grouper = daytime_timestamps.resample('d') +sunrise_timestamps = grouper.first() +sunset_timestamps = grouper.last() + +def ts_to_minutes(ts): + # convert timestamps to minutes since midnight + return ts.dt.hour * 60 + ts.dt.minute + ts.dt.second / 60 + +midday_minutes = (ts_to_minutes(sunrise_timestamps) + ts_to_minutes(sunset_timestamps)) / 2 + +# %% +# Now, calculate the expected timing of solar noon at this location for each +# day. Note that we use a time zone without DST for calculating the expected +# timings; this means that if the "measured" data does include DST in its +# timestamps, it will be flagged as a time shift. + +dates = midday_minutes.index.tz_localize(None).tz_localize('Etc/GMT+5') +sp = location.get_sun_rise_set_transit(dates, method='spa') +transit_minutes = ts_to_minutes(sp['transit']) + +# %% +# Finally, ask ruptures if it sees any change points in the difference between +# these two daily event timings, and visualize the result: + +is_shifted, shift_amount = shifts_ruptures(midday_minutes, transit_minutes) + +fig, axes = plt.subplots(2, 1, sharex=True) + +midday_minutes.plot(ax=axes[0], label='"measured" midday') +transit_minutes.plot(ax=axes[0], label='expected midday') +axes[0].set_ylabel('Minutes since midnight') +axes[0].legend() + +shift_amount.plot(ax=axes[1]) +axes[1].set_ylabel('Estimated shift [minutes]') From 45f3e451f9cb79bb4912b1d69566a46c8f8bc574 Mon Sep 17 00:00:00 2001 From: Kevin Anderson Date: Wed, 31 May 2023 13:40:48 -0400 Subject: [PATCH 3/8] whatsnew --- docs/whatsnew/0.2.0.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/whatsnew/0.2.0.rst b/docs/whatsnew/0.2.0.rst index d93f9d83f..8ec7c6b97 100644 --- a/docs/whatsnew/0.2.0.rst +++ b/docs/whatsnew/0.2.0.rst @@ -32,6 +32,8 @@ Documentation ~~~~~~~~~~~~~ * Online docs now use ``pydata-sphinx-theme`` instead of the built-in ``alabaster`` theme. (:issue:`176`, :pull:`178`) +* Added a gallery page for :py:func:`~pvanalytics.quality.time.shifts_ruptures`. + (:pull:`192`) Testing ~~~~~~~ From bbfb4c7f85dbeab24038e4ad6590bb0ea8a39ead Mon Sep 17 00:00:00 2001 From: Kevin Anderson Date: Wed, 31 May 2023 13:44:48 -0400 Subject: [PATCH 4/8] lint --- docs/examples/data-time-shifts/shifts-ruptures.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/docs/examples/data-time-shifts/shifts-ruptures.py b/docs/examples/data-time-shifts/shifts-ruptures.py index efca313cc..a52c6359c 100644 --- a/docs/examples/data-time-shifts/shifts-ruptures.py +++ b/docs/examples/data-time-shifts/shifts-ruptures.py @@ -25,7 +25,7 @@ # Typically this process would be applied to measured data with possibly # untrustworthy timestamps. However, for instructional purposes here, # we'll create an artificial example dataset that contains a time shift -# due to DST. +# due to DST. # use a time zone (US/Eastern) that is affected by DST. # Etc/GMT+5 is the corresponding local standard time zone. @@ -51,11 +51,15 @@ sunrise_timestamps = grouper.first() sunset_timestamps = grouper.last() + def ts_to_minutes(ts): # convert timestamps to minutes since midnight return ts.dt.hour * 60 + ts.dt.minute + ts.dt.second / 60 -midday_minutes = (ts_to_minutes(sunrise_timestamps) + ts_to_minutes(sunset_timestamps)) / 2 + +midday_minutes = ( + ts_to_minutes(sunrise_timestamps) + ts_to_minutes(sunset_timestamps) +) / 2 # %% # Now, calculate the expected timing of solar noon at this location for each @@ -70,7 +74,7 @@ def ts_to_minutes(ts): # %% # Finally, ask ruptures if it sees any change points in the difference between # these two daily event timings, and visualize the result: - + is_shifted, shift_amount = shifts_ruptures(midday_minutes, transit_minutes) fig, axes = plt.subplots(2, 1, sharex=True) From c5f7cdc200b0f05e00744cec20e58ba9a4981c08 Mon Sep 17 00:00:00 2001 From: Kevin Anderson Date: Wed, 31 May 2023 13:22:16 -0400 Subject: [PATCH 5/8] gallery: rename shifts folder to include time --- docs/examples/data-shifts/README.rst | 4 ---- docs/examples/data-time-shifts/README.rst | 4 ++++ .../examples/{data-shifts => data-time-shifts}/data-shifts.py | 0 3 files changed, 4 insertions(+), 4 deletions(-) delete mode 100644 docs/examples/data-shifts/README.rst create mode 100644 docs/examples/data-time-shifts/README.rst rename docs/examples/{data-shifts => data-time-shifts}/data-shifts.py (100%) diff --git a/docs/examples/data-shifts/README.rst b/docs/examples/data-shifts/README.rst deleted file mode 100644 index 9927c943d..000000000 --- a/docs/examples/data-shifts/README.rst +++ /dev/null @@ -1,4 +0,0 @@ -Data Shifts ------------ - -This includes examples for identifying data/capacity shifts in time series data. \ No newline at end of file diff --git a/docs/examples/data-time-shifts/README.rst b/docs/examples/data-time-shifts/README.rst new file mode 100644 index 000000000..71cb8e9fc --- /dev/null +++ b/docs/examples/data-time-shifts/README.rst @@ -0,0 +1,4 @@ +Data/Time Shifts +---------------- + +This includes examples for identifying data/capacity/time shifts in time series data. \ No newline at end of file diff --git a/docs/examples/data-shifts/data-shifts.py b/docs/examples/data-time-shifts/data-shifts.py similarity index 100% rename from docs/examples/data-shifts/data-shifts.py rename to docs/examples/data-time-shifts/data-shifts.py From 8515c3004688dfc1df66f5d28dfe3aaae0fb2241 Mon Sep 17 00:00:00 2001 From: Kevin Anderson Date: Wed, 31 May 2023 13:40:43 -0400 Subject: [PATCH 6/8] add gallery page --- .../data-time-shifts/shifts-ruptures.py | 84 +++++++++++++++++++ 1 file changed, 84 insertions(+) create mode 100644 docs/examples/data-time-shifts/shifts-ruptures.py diff --git a/docs/examples/data-time-shifts/shifts-ruptures.py b/docs/examples/data-time-shifts/shifts-ruptures.py new file mode 100644 index 000000000..efca313cc --- /dev/null +++ b/docs/examples/data-time-shifts/shifts-ruptures.py @@ -0,0 +1,84 @@ +""" +Identifying and estimating time shifts +====================================== + +Identifying time shifts from clock errors or uncorrected Daylight Saving Time. +""" + +# %% +# Time shifts can occur in measured data due to clock errors and +# time zone issues (for example, assuming a dataset is in local standard time +# when in fact it contains Daylight Saving Time). +# +# This example uses :py:func:`~pvanalytics.quality.time.shifts_ruptures` +# to identify abrupt time shifts in a time series, and estimate the +# corresponding time shift amount. + +import pvlib +import pandas as pd +from pvanalytics.quality.time import shifts_ruptures +from pvanalytics.features.daytime import power_or_irradiance +import matplotlib.pyplot as plt + + +# %% +# Typically this process would be applied to measured data with possibly +# untrustworthy timestamps. However, for instructional purposes here, +# we'll create an artificial example dataset that contains a time shift +# due to DST. + +# use a time zone (US/Eastern) that is affected by DST. +# Etc/GMT+5 is the corresponding local standard time zone. +times = pd.date_range('2019-01-01', '2019-12-31', freq='5T', tz='US/Eastern') +location = pvlib.location.Location(40, -80) +cs = location.get_clearsky(times) +measured_signal = cs['ghi'] + +# %% +# The :py:func:`~pvanalytics.quality.time.shifts_ruptures` function +# is centered around comparing the timing of events observed in the measured +# data with expected timings for those same events. +# In this case, we'll use the timing of solar noon as the event. +# +# First, we'll extract the timing of solar noon from the measured data. +# This could be done in several ways; here we will just take the midpoint +# between sunrise and sunset using times estimated with +# :py:func:`~pvanalytics.features.daytime.power_or_irradiance`. + +is_daytime = power_or_irradiance(measured_signal) +daytime_timestamps = measured_signal.index[is_daytime].to_series() +grouper = daytime_timestamps.resample('d') +sunrise_timestamps = grouper.first() +sunset_timestamps = grouper.last() + +def ts_to_minutes(ts): + # convert timestamps to minutes since midnight + return ts.dt.hour * 60 + ts.dt.minute + ts.dt.second / 60 + +midday_minutes = (ts_to_minutes(sunrise_timestamps) + ts_to_minutes(sunset_timestamps)) / 2 + +# %% +# Now, calculate the expected timing of solar noon at this location for each +# day. Note that we use a time zone without DST for calculating the expected +# timings; this means that if the "measured" data does include DST in its +# timestamps, it will be flagged as a time shift. + +dates = midday_minutes.index.tz_localize(None).tz_localize('Etc/GMT+5') +sp = location.get_sun_rise_set_transit(dates, method='spa') +transit_minutes = ts_to_minutes(sp['transit']) + +# %% +# Finally, ask ruptures if it sees any change points in the difference between +# these two daily event timings, and visualize the result: + +is_shifted, shift_amount = shifts_ruptures(midday_minutes, transit_minutes) + +fig, axes = plt.subplots(2, 1, sharex=True) + +midday_minutes.plot(ax=axes[0], label='"measured" midday') +transit_minutes.plot(ax=axes[0], label='expected midday') +axes[0].set_ylabel('Minutes since midnight') +axes[0].legend() + +shift_amount.plot(ax=axes[1]) +axes[1].set_ylabel('Estimated shift [minutes]') From d3d648a6f7ea7a53d1cd6e32cc1f9f839b67a836 Mon Sep 17 00:00:00 2001 From: Kevin Anderson Date: Wed, 31 May 2023 13:40:48 -0400 Subject: [PATCH 7/8] whatsnew --- docs/whatsnew/0.2.0.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/whatsnew/0.2.0.rst b/docs/whatsnew/0.2.0.rst index 82c03fde4..928ea465a 100644 --- a/docs/whatsnew/0.2.0.rst +++ b/docs/whatsnew/0.2.0.rst @@ -33,6 +33,8 @@ Documentation ~~~~~~~~~~~~~ * Online docs now use ``pydata-sphinx-theme`` instead of the built-in ``alabaster`` theme. (:issue:`176`, :pull:`178`) +* Added a gallery page for :py:func:`~pvanalytics.quality.time.shifts_ruptures`. + (:pull:`192`) Testing ~~~~~~~ From 86d0deb0d63ae637818c2eaff3e8e780b735e09e Mon Sep 17 00:00:00 2001 From: Kevin Anderson Date: Wed, 31 May 2023 13:44:48 -0400 Subject: [PATCH 8/8] lint --- docs/examples/data-time-shifts/shifts-ruptures.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/docs/examples/data-time-shifts/shifts-ruptures.py b/docs/examples/data-time-shifts/shifts-ruptures.py index efca313cc..a52c6359c 100644 --- a/docs/examples/data-time-shifts/shifts-ruptures.py +++ b/docs/examples/data-time-shifts/shifts-ruptures.py @@ -25,7 +25,7 @@ # Typically this process would be applied to measured data with possibly # untrustworthy timestamps. However, for instructional purposes here, # we'll create an artificial example dataset that contains a time shift -# due to DST. +# due to DST. # use a time zone (US/Eastern) that is affected by DST. # Etc/GMT+5 is the corresponding local standard time zone. @@ -51,11 +51,15 @@ sunrise_timestamps = grouper.first() sunset_timestamps = grouper.last() + def ts_to_minutes(ts): # convert timestamps to minutes since midnight return ts.dt.hour * 60 + ts.dt.minute + ts.dt.second / 60 -midday_minutes = (ts_to_minutes(sunrise_timestamps) + ts_to_minutes(sunset_timestamps)) / 2 + +midday_minutes = ( + ts_to_minutes(sunrise_timestamps) + ts_to_minutes(sunset_timestamps) +) / 2 # %% # Now, calculate the expected timing of solar noon at this location for each @@ -70,7 +74,7 @@ def ts_to_minutes(ts): # %% # Finally, ask ruptures if it sees any change points in the difference between # these two daily event timings, and visualize the result: - + is_shifted, shift_amount = shifts_ruptures(midday_minutes, transit_minutes) fig, axes = plt.subplots(2, 1, sharex=True)