Skip to content

Commit

Permalink
Formatted everything with black
Browse files Browse the repository at this point in the history
  • Loading branch information
fredmontet committed Aug 21, 2024
1 parent b5c0b85 commit ec442e3
Show file tree
Hide file tree
Showing 32 changed files with 1,681 additions and 1,499 deletions.
2,423 changes: 1,245 additions & 1,178 deletions poetry.lock

Large diffs are not rendered by default.

11 changes: 10 additions & 1 deletion src/ontime/api/modular.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,16 @@
"""

from ..core import detectors, generators, Model, models, Plot, marks, processors, TimeSeries
from ..core import (
detectors,
generators,
Model,
models,
Plot,
marks,
processors,
TimeSeries,
)

from .. import module
from .. import context
Expand Down
4 changes: 1 addition & 3 deletions src/ontime/context/common/anomaly_frequency.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,9 +19,7 @@ def get_number_of_anomaly_in_window(self, window_size: str) -> TimeSeries:
sum_series = self.anomalies_series.rolling(window=window_size).sum()
return TimeSeries.from_series(sum_series)

def get_frequency_of_anomaly_in_window(
self, window_size: str
) -> UnitTimeSeries:
def get_frequency_of_anomaly_in_window(self, window_size: str) -> UnitTimeSeries:
"""
Compute the frequency of anomalies in a time window. The frequency is computed as the number of anomalies in
the window divided by the maximum number of anomalies in a window. So 1 mean that all samples in the window are
Expand Down
19 changes: 10 additions & 9 deletions src/ontime/context/common/data_quality_detector.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,10 +7,12 @@ class DataQualityDetector:
Detects anomalies in a time series given a threshold or quantile.
"""

def __init__(self,
threshold_type: str,
upper_threshold: float = None,
lower_threshold: float = None):
def __init__(
self,
threshold_type: str,
upper_threshold: float = None,
lower_threshold: float = None,
):
"""
Constructor for the DataQualityDetector class.
Expand All @@ -30,20 +32,19 @@ def fit(self, ts: TimeSeries):
:param ts: TimeSeries
"""
match self.threshold_type:
case 'quantile':
case "quantile":
self.detector = Quantile(
low_quantile=self.lower_threshold,
high_quantile=self.upper_threshold
high_quantile=self.upper_threshold,
)
self.detector.fit(ts)
case 'threshold':
case "threshold":
self.detector = Threshold(
low_threshold=self.lower_threshold,
high_threshold=self.upper_threshold
high_threshold=self.upper_threshold,
)
self.is_fitted = True


def detect(self, ts: TimeSeries) -> BinaryTimeSeries:
"""
Detects anomalies in the given time series given a quantile or threshold crossing.
Expand Down
1 change: 0 additions & 1 deletion src/ontime/context/common/missing_data_detector.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,4 +15,3 @@ def detect(self, ts: TimeSeries) -> BinaryTimeSeries:
:return: BinaryTimeSeries with 0 for normal values and 1 for anomalies
"""
return BinaryTimeSeries.from_series(ts.pd_series().isna())

11 changes: 10 additions & 1 deletion src/ontime/core/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,4 +5,13 @@
from .processing import processors, abstract_processor
from .time_series import TimeSeries

__all__ = ["detectors", "generators", "Model", "models", "Plot", "marks", "processors", "TimeSeries"]
__all__ = [
"detectors",
"generators",
"Model",
"models",
"Plot",
"marks",
"processors",
"TimeSeries",
]
12 changes: 6 additions & 6 deletions src/ontime/core/detection/registry/quantile.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,11 +11,11 @@ class Quantile(QuantileDetector, AbstractDetector):
"""

def __init__(
self,
low_quantile: float = None,
high_quantile: float = None,
enable_logging: bool = False,
logger_params: dict = None
self,
low_quantile: float = None,
high_quantile: float = None,
enable_logging: bool = False,
logger_params: dict = None,
):
"""
Constructor for QuantileDetector
Expand All @@ -26,7 +26,7 @@ def __init__(
"""
super().__init__(low_quantile, high_quantile)
self.enable_logging = enable_logging
default_params = {'description': 'QuantileDetector'}
default_params = {"description": "QuantileDetector"}
self.logger_params = default_params if logger_params is None else logger_params
if enable_logging:
self.logger = BinaryAnomalyLogger(**self.logger_params)
Expand Down
12 changes: 6 additions & 6 deletions src/ontime/core/detection/registry/threshold.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,11 +13,11 @@ class Threshold(ThresholdDetector, AbstractDetector):
"""

def __init__(
self,
low_threshold: Union[int, float, None] = None,
high_threshold: Union[int, float, None] = None,
enable_logging: bool = False,
logger_params: dict = None
self,
low_threshold: Union[int, float, None] = None,
high_threshold: Union[int, float, None] = None,
enable_logging: bool = False,
logger_params: dict = None,
):
"""
:param low_threshold: lower bounds
Expand All @@ -28,7 +28,7 @@ def __init__(
"""
super().__init__(low_threshold, high_threshold)
self.enable_logging = enable_logging
default_params = {'description': 'ThresholdDetector'}
default_params = {"description": "ThresholdDetector"}
self.logger_params = default_params if logger_params is None else logger_params
if enable_logging:
self.logger = BinaryAnomalyLogger(**self.logger_params)
Expand Down
9 changes: 5 additions & 4 deletions src/ontime/core/modelling/__init__.py
Original file line number Diff line number Diff line change
@@ -1,12 +1,13 @@
from .model import Model
from .models import Models
from .registry.gru import GRU
#from .registry.autoencoder import Autoencoder
#from .registry.variational_autoencoder import VariationalAutoencoder

# from .registry.autoencoder import Autoencoder
# from .registry.variational_autoencoder import VariationalAutoencoder

models = Models()
models.load("gru", GRU)
#models.load("autoencoder", AutoEncoder)
#models.load("variational_autoencoder", VariationalAutoencoder)
# models.load("autoencoder", AutoEncoder)
# models.load("variational_autoencoder", VariationalAutoencoder)

__all__ = ["Model", "models"]
57 changes: 36 additions & 21 deletions src/ontime/core/modelling/registry/autoencoder.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@
from ontime.module.benchmarking.benchmark import Benchmark


device = 'cpu'
device = "cpu"


class Encoder(nn.Module):
Expand Down Expand Up @@ -64,7 +64,6 @@ def __str__(self):


class Autoencoder(nn.Module):

def __init__(self, entry_size: int, latent_dims: int):
super(Autoencoder, self).__init__()
self.encoder = Encoder(entry_size, latent_dims)
Expand All @@ -76,16 +75,18 @@ def forward(self, x: Tensor) -> Tensor:
return self.decoder(z)

def loss(self, x: Tensor, x_hat: Tensor) -> float:
loss = F.smooth_l1_loss(x, x_hat, reduction='mean')
loss = F.smooth_l1_loss(x, x_hat, reduction="mean")
return loss

def get_reconstructed(self, dataset: TimeSeries, period: int, labels: TimeSeries = None, verbose: bool = False) -> \
list[list]:
def get_reconstructed(
self,
dataset: TimeSeries,
period: int,
labels: TimeSeries = None,
verbose: bool = False,
) -> list[list]:
ds = SlicedDataset(dataset, period, labels)
data = torch.utils.data.DataLoader(
ds,
batch_size=1,
shuffle=False)
data = torch.utils.data.DataLoader(ds, batch_size=1, shuffle=False)
results_x = []
results_xhat = []
results_y = []
Expand All @@ -95,8 +96,8 @@ def get_reconstructed(self, dataset: TimeSeries, period: int, labels: TimeSeries
x = x.to(device) # CPU
x_hat = self(x)
if verbose:
print(f'x:{x.size()} -> x_hat {x_hat.size()}')
x_hat = x_hat.reshape(x.size()).to('cpu').detach()
print(f"x:{x.size()} -> x_hat {x_hat.size()}")
x_hat = x_hat.reshape(x.size()).to("cpu").detach()
loss = self.loss(x, x_hat)
x_hat = x_hat.numpy()
x = x.cpu().numpy()
Expand All @@ -106,15 +107,17 @@ def get_reconstructed(self, dataset: TimeSeries, period: int, labels: TimeSeries
results_loss.append(loss)
if ds.labels is not None:
results_y.append(y)
reconstructed_dataset = Autoencoder._into_timeseries(dataset, results_xhat, results_loss, results_y, period)
reconstructed_dataset = Autoencoder._into_timeseries(
dataset, results_xhat, results_loss, results_y, period
)
return reconstructed_dataset

def train(self, data: SlicedDataset, device: str, epochs: int = 20):
opt = torch.optim.Adam(self.parameters())
for epoch in range(epochs):
i=1
i = 1
for x, y in data:
i+=1
i += 1
x = x.to(device) # CPU
opt.zero_grad()
x_hat = self(x)
Expand All @@ -127,18 +130,26 @@ def train(self, data: SlicedDataset, device: str, epochs: int = 20):
def get_period(dataset: TimeSeries):
periods = []
for col in dataset.columns:
periods.append(pyd.findfrequency(dataset.pd_dataframe()[col].to_numpy(), detrend=True))
periods.append(
pyd.findfrequency(dataset.pd_dataframe()[col].to_numpy(), detrend=True)
)
period = max(periods)
while period < 15:
period += period
return period

@staticmethod
def new_encoder_for_dataset(dataset: TimeSeries, period: int = None) -> 'Autoencoder':
def new_encoder_for_dataset(
dataset: TimeSeries, period: int = None
) -> "Autoencoder":
if period is None:
periods = []
for col in dataset.columns:
periods.append(pyd.findfrequency(dataset.pd_dataframe()[col].to_numpy(), detrend=True))
periods.append(
pyd.findfrequency(
dataset.pd_dataframe()[col].to_numpy(), detrend=True
)
)
period = max(periods)
while period < 15:
period += period
Expand All @@ -157,21 +168,25 @@ def _into_timeseries(input_dataset, xhat, loss, y, period):
sample = xhat[i][0]
for line in sample:
x_hat_flat.append(line)
reconstructed_dataset = pd.DataFrame(x_hat_flat, columns=input_dataset.columns.tolist())
reconstructed_dataset.index = input_dataset.time_index[:len(reconstructed_dataset.index)]
reconstructed_dataset = pd.DataFrame(
x_hat_flat, columns=input_dataset.columns.tolist()
)
reconstructed_dataset.index = input_dataset.time_index[
: len(reconstructed_dataset.index)
]

# loss and y
reconstructed_loss = []
reconstructed_y = []
for l in loss:
for i in range(0, period):
reconstructed_loss.append(l)
reconstructed_dataset['loss'] = reconstructed_loss
reconstructed_dataset["loss"] = reconstructed_loss

if y is not None and len(y) > 0:
for ry in y:
for i in range(0, period):
reconstructed_y.append(ry)
reconstructed_dataset['y'] = reconstructed_y
reconstructed_dataset["y"] = reconstructed_y

return TimeSeries.from_pandas(reconstructed_dataset)
16 changes: 12 additions & 4 deletions src/ontime/core/modelling/registry/gru.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,13 +2,20 @@
from torch import nn
import pytorch_lightning as pl

from ontime.core.modelling.libs.pytorch.abstract_pytorch_model import AbstractPytorchModel
from ontime.core.modelling.libs.pytorch.abstract_pytorch_model import (
AbstractPytorchModel,
)


class GRU(AbstractPytorchModel):
def __init__(self, input_dim, hidden_dim, output_steps, num_layers=1):
super(GRU, self).__init__()
self.gru = nn.GRU(input_size=input_dim, hidden_size=hidden_dim, num_layers=num_layers, batch_first=True)
self.gru = nn.GRU(
input_size=input_dim,
hidden_size=hidden_dim,
num_layers=num_layers,
batch_first=True,
)
self.linear = nn.Linear(hidden_dim, output_steps)

def forward(self, x):
Expand All @@ -20,7 +27,7 @@ def training_step(self, batch, batch_idx):
x, y = batch
y_hat = self(x)
loss = nn.functional.mse_loss(y_hat, y)
self.log('train_loss', loss)
self.log("train_loss", loss)
return loss

def configure_optimizers(self):
Expand All @@ -39,8 +46,9 @@ def predict(self, horizon, *args, **kwargs):
with torch.no_grad():
return self(dummy_input)


# Example usage
# model = TimeSeriesGRU(input_dim=10, hidden_dim=20, output_steps=5)
# ts_loader = YourDataLoader(time_series)
# model.fit(ts_loader)
# predictions = model.predict(horizon=5)
# predictions = model.predict(horizon=5)
Loading

0 comments on commit ec442e3

Please sign in to comment.