diff --git a/darts/ad/__init__.py b/darts/ad/__init__.py new file mode 100644 index 0000000000..3996373070 --- /dev/null +++ b/darts/ad/__init__.py @@ -0,0 +1,51 @@ +""" +Anomaly Detection +----------------- + +A suite of tools for performing anomaly detection and classification +on time series. + +* `Anomaly Scorers `_ + are at the core of the anomaly detection module. They + produce anomaly scores time series, either for single series (``score()``), + or for series accompanied by some predictions (``score_from_prediction()``). + Scorers can be trainable (e.g., ``KMeansScorer``) or not (e.g., ``NormScorer``). + +* `Anomaly Models `_ + offer a convenient way to produce anomaly scores from any of Darts + forecasting models (``ForecastingAnomalyModel``) or filtering models (``FilteringAnomalyModel``), + by comparing models' predictions with actual observations. + These classes take as parameters one Darts model, and one or multiple scorers, and can be readily + used to produce anomaly scores with the ``score()`` method. + +* `Anomaly Detectors `_: + transform raw time series (such as anaomly scores) into binary anomaly time series. + +* `Anomaly Aggregators `_: + combine multiple binary anomaly time series (in the form of multivariate time series) + into a single binary anomaly time series applying boolean logic. +""" + +# anomaly aggregators +from .aggregators import AndAggregator, EnsembleSklearnAggregator, OrAggregator + +# anomaly models +from .anomaly_model import FilteringAnomalyModel, ForecastingAnomalyModel + +# anomaly detectors +from .detectors import QuantileDetector, ThresholdDetector + +# anomaly scorers +from .scorers import ( + CauchyNLLScorer, + DifferenceScorer, + ExponentialNLLScorer, + GammaNLLScorer, + GaussianNLLScorer, + KMeansScorer, + LaplaceNLLScorer, + NormScorer, + PoissonNLLScorer, + PyODScorer, + WassersteinScorer, +) diff --git a/darts/ad/aggregators/__init__.py b/darts/ad/aggregators/__init__.py new file mode 100644 index 0000000000..324b54b24b --- /dev/null +++ b/darts/ad/aggregators/__init__.py @@ -0,0 +1,18 @@ +""" +Anomaly Aggregators +------------------- + +An anomaly aggregator can take multiple detected anomalies +(in the form of binary TimeSeries, as coming from an anomaly detector) +and combine them into one. It can typically be used to combine +the detections of multiple models into one final detection. + +The key method is ``predict()``, which takes as input one (or multiple) +multivariate binary TimeSeries where each component represents the +detection of a single model, and returns one (or multiple) univariate +binary TimeSeries representing the final detection. +""" + +from .and_aggregator import AndAggregator +from .ensemble_sklearn_aggregator import EnsembleSklearnAggregator +from .or_aggregator import OrAggregator diff --git a/darts/ad/aggregators/aggregators.py b/darts/ad/aggregators/aggregators.py new file mode 100644 index 0000000000..b9980922e2 --- /dev/null +++ b/darts/ad/aggregators/aggregators.py @@ -0,0 +1,300 @@ +""" +Anomaly aggregators base classes +""" + +# TODO: +# - add customize aggregators +# - add in trainable aggregators +# - log regression +# - decision tree +# - create show_all_combined (info about correlation, and from what path did +# the anomaly alarm came from) + +from abc import ABC, abstractmethod +from typing import Any, Sequence, Union + +import numpy as np + +from darts import TimeSeries +from darts.ad.utils import _to_list, eval_accuracy_from_binary_prediction +from darts.logging import raise_if_not + + +class Aggregator(ABC): + def __init__(self, *args: Any, **kwargs: Any) -> None: + pass + + @abstractmethod + def __str__(self): + """returns the name of the aggregator""" + pass + + @abstractmethod + def _predict_core(self): + """returns the aggregated results""" + pass + + @abstractmethod + def predict( + self, series: Union[TimeSeries, Sequence[TimeSeries]] + ) -> Union[TimeSeries, Sequence[TimeSeries]]: + """Aggregates the (sequence of) multivariate binary series given as + input into a (sequence of) univariate binary series. + + Parameters + ---------- + series + The (sequence of) multivariate binary series to aggregate + + Returns + ------- + TimeSeries + (Sequence of) aggregated results + """ + pass + + def _check_input(self, series: Union[TimeSeries, Sequence[TimeSeries]]): + """ + Checks for input if: + - it is a (sequence of) multivariate series (width>1) + - (sequence of) series must be: + * a deterministic TimeSeries + * binary (only values equal to 0 or 1) + """ + + list_series = _to_list(series) + + raise_if_not( + all([isinstance(s, TimeSeries) for s in list_series]), + "all series in `series` must be of type TimeSeries.", + ) + + raise_if_not( + all([s.width > 1 for s in list_series]), + "all series in `series` must be multivariate (width>1).", + ) + + raise_if_not( + all([s.is_deterministic for s in list_series]), + "all series in `series` must be deterministic (number of samples=1).", + ) + + raise_if_not( + all( + [ + np.array_equal( + s.values(copy=False), s.values(copy=False).astype(bool) + ) + for s in list_series + ] + ), + "all series in `series` must be binary (only 0 and 1 values).", + ) + + return list_series + + def eval_accuracy( + self, + actual_anomalies: Sequence[TimeSeries], + series: Sequence[TimeSeries], + window: int = 1, + metric: str = "recall", + ) -> Union[float, Sequence[float]]: + """Aggregates the (sequence of) multivariate series given as input into one (sequence of) + series and evaluates the results against true anomalies. + + Parameters + ---------- + actual_anomalies + The (sequence of) ground truth of the anomalies (1 if it is an anomaly and 0 if not) + series + The (sequence of) multivariate binary series to aggregate + window + (Sequence of) integer value indicating the number of past samples each point + represents in the (sequence of) series. The parameter will be used by the + function ``_window_adjustment_anomalies()`` in darts.ad.utils to transform + actual_anomalies. + metric + Metric function to use. Must be one of "recall", "precision", + "f1", and "accuracy". + Default: "recall" + + Returns + ------- + Union[float, Sequence[float]] + (Sequence of) score for the (sequence of) series + """ + + list_actual_anomalies = _to_list(actual_anomalies) + + raise_if_not( + all([isinstance(s, TimeSeries) for s in list_actual_anomalies]), + "all series in `actual_anomalies` must be of type TimeSeries.", + ) + + raise_if_not( + all([s.is_deterministic for s in list_actual_anomalies]), + "all series in `actual_anomalies` must be deterministic (number of samples=1).", + ) + + raise_if_not( + all([s.width == 1 for s in list_actual_anomalies]), + "all series in `actual_anomalies` must be univariate (width=1).", + ) + + raise_if_not( + len(list_actual_anomalies) == len(_to_list(series)), + "`actual_anomalies` and `series` must contain the same number of series.", + ) + + preds = self.predict(series) + + return eval_accuracy_from_binary_prediction( + list_actual_anomalies, preds, window, metric + ) + + +class NonFittableAggregator(Aggregator): + "Base class of Aggregators that do not need training." + + def __init__(self) -> None: + super().__init__() + + # indicates if the Aggregator is trainable or not + self.trainable = False + + def predict( + self, series: Union[TimeSeries, Sequence[TimeSeries]] + ) -> Union[TimeSeries, Sequence[TimeSeries]]: + """Aggregates the (sequence of) multivariate binary series given as + input into a (sequence of) univariate binary series. + + Parameters + ---------- + series + The (sequence of) multivariate binary series to aggregate + + Returns + ------- + TimeSeries + (Sequence of) aggregated results + """ + list_series = self._check_input(series) + + if isinstance(series, TimeSeries): + return self._predict_core(list_series)[0] + else: + return self._predict_core(list_series) + + +class FittableAggregator(Aggregator): + "Base class of Aggregators that do need training." + + def __init__(self) -> None: + super().__init__() + + # indicates if the Aggregator is trainable or not + self.trainable = True + + # indicates if the Aggregator has been trained yet + self._fit_called = False + + def _assert_fit_called(self): + """Checks if the Aggregator has been fitted before calling its `score()` function.""" + + raise_if_not( + self._fit_called, + f"The Aggregator {self.__str__()} has not been fitted yet. Call `fit()` first.", + ) + + def fit( + self, + actual_anomalies: Union[TimeSeries, Sequence[TimeSeries]], + series: Union[TimeSeries, Sequence[TimeSeries]], + ): + """Fit the aggregators on the (sequence of) multivariate binary series. + + If a list of series is given, they must have the same number of components. + + Parameters + ---------- + actual_anomalies + The (sequence of) ground truth of the anomalies (1 if it is an anomaly and 0 if not) + series + The (sequence of) multivariate binary series + """ + list_series = self._check_input(series) + self.width_trained_on = list_series[0].width + + raise_if_not( + all([s.width == self.width_trained_on for s in list_series]), + "all series in `list_series` must have the same number of components.", + ) + + list_actual_anomalies = _to_list(actual_anomalies) + + raise_if_not( + all([isinstance(s, TimeSeries) for s in list_actual_anomalies]), + "all series in `actual_anomalies` must be of type TimeSeries.", + ) + + raise_if_not( + all([s.is_deterministic for s in list_actual_anomalies]), + "all series in `actual_anomalies` must be deterministic (width=1).", + ) + + raise_if_not( + all([s.width == 1 for s in list_actual_anomalies]), + "all series in `actual_anomalies` must be univariate (width=1).", + ) + + raise_if_not( + len(list_actual_anomalies) == len(list_series), + "`actual_anomalies` and `series` must contain the same number of series.", + ) + + same_intersection = list( + zip( + *[ + [anomalies.slice_intersect(series), series.slice_intersect(series)] + for (anomalies, series) in zip(list_actual_anomalies, list_series) + ] + ) + ) + list_actual_anomalies = list(same_intersection[0]) + list_series = list(same_intersection[1]) + + ret = self._fit_core(list_actual_anomalies, list_series) + self._fit_called = True + return ret + + def predict( + self, series: Union[TimeSeries, Sequence[TimeSeries]] + ) -> Union[TimeSeries, Sequence[TimeSeries]]: + """Aggregates the (sequence of) multivariate binary series given as + input into a (sequence of) univariate binary series. + + Parameters + ---------- + series + The (sequence of) multivariate binary series to aggregate + + Returns + ------- + TimeSeries + (Sequence of) aggregated results + """ + self._assert_fit_called() + list_series = self._check_input(series) + + raise_if_not( + all([s.width == self.width_trained_on for s in list_series]), + "all series in `series` must have the same number of components as the data" + + " used for training the detector model, number of components in training:" + + f" {self.width_trained_on}.", + ) + + if isinstance(series, TimeSeries): + return self._predict_core(list_series)[0] + else: + return self._predict_core(list_series) diff --git a/darts/ad/aggregators/and_aggregator.py b/darts/ad/aggregators/and_aggregator.py new file mode 100644 index 0000000000..1e12bc6efc --- /dev/null +++ b/darts/ad/aggregators/and_aggregator.py @@ -0,0 +1,25 @@ +""" +AND Aggregator +-------------- + +Aggregator that identifies a time step as anomalous if all the components +are flagged as anomalous (logical AND). +""" + +from typing import Sequence + +from darts import TimeSeries +from darts.ad.aggregators.aggregators import NonFittableAggregator + + +class AndAggregator(NonFittableAggregator): + def __init__(self) -> None: + super().__init__() + + def __str__(self): + return "AndAggregator" + + def _predict_core(self, series: Sequence[TimeSeries]) -> Sequence[TimeSeries]: + return [ + s.sum(axis=1).map(lambda x: (x >= s.width).astype(s.dtype)) for s in series + ] diff --git a/darts/ad/aggregators/ensemble_sklearn_aggregator.py b/darts/ad/aggregators/ensemble_sklearn_aggregator.py new file mode 100644 index 0000000000..e053819d29 --- /dev/null +++ b/darts/ad/aggregators/ensemble_sklearn_aggregator.py @@ -0,0 +1,63 @@ +""" +Ensemble scikit-learn aggregator +-------------------------------- + +Aggregator wrapped around the Ensemble model of sklearn. +`sklearn https://scikit-learn.org/stable/modules/ensemble.html`_. +""" + +from typing import Sequence + +import numpy as np +from sklearn.ensemble import BaseEnsemble + +from darts import TimeSeries +from darts.ad.aggregators.aggregators import FittableAggregator +from darts.logging import raise_if_not + + +class EnsembleSklearnAggregator(FittableAggregator): + def __init__(self, model) -> None: + + raise_if_not( + isinstance(model, BaseEnsemble), + f"Scorer is expecting a model of type BaseEnsemble (from sklearn ensemble), \ + found type {type(model)}.", + ) + + self.model = model + super().__init__() + + def __str__(self): + return "EnsembleSklearnAggregator: {}".format( + self.model.__str__().split("(")[0] + ) + + def _fit_core( + self, + actual_anomalies: Sequence[TimeSeries], + series: Sequence[TimeSeries], + ): + + X = np.concatenate( + [s.all_values(copy=False).reshape(len(s), -1) for s in series], + axis=0, + ) + + y = np.concatenate( + [s.all_values(copy=False).reshape(len(s)) for s in actual_anomalies], + axis=0, + ) + + self.model.fit(y=y, X=X) + return self + + def _predict_core(self, series: Sequence[TimeSeries]) -> Sequence[TimeSeries]: + + return [ + TimeSeries.from_times_and_values( + s.time_index, + self.model.predict((s).all_values(copy=False).reshape(len(s), -1)), + ) + for s in series + ] diff --git a/darts/ad/aggregators/or_aggregator.py b/darts/ad/aggregators/or_aggregator.py new file mode 100644 index 0000000000..5737839630 --- /dev/null +++ b/darts/ad/aggregators/or_aggregator.py @@ -0,0 +1,24 @@ +""" +OR Aggregator +------------- + +Aggregator that identifies a time step as anomalous if any of the components +is flagged as anomalous (logical OR). +""" + + +from typing import Sequence + +from darts import TimeSeries +from darts.ad.aggregators.aggregators import NonFittableAggregator + + +class OrAggregator(NonFittableAggregator): + def __init__(self) -> None: + super().__init__() + + def __str__(self): + return "OrAggregator" + + def _predict_core(self, series: Sequence[TimeSeries]) -> Sequence[TimeSeries]: + return [s.sum(axis=1).map(lambda x: (x > 0).astype(s.dtype)) for s in series] diff --git a/darts/ad/anomaly_model/__init__.py b/darts/ad/anomaly_model/__init__.py new file mode 100644 index 0000000000..50af79dbc6 --- /dev/null +++ b/darts/ad/anomaly_model/__init__.py @@ -0,0 +1,29 @@ +""" +Anomaly Models +-------------- + +Anomaly models make it possible to use any of Darts' forecasting +or filtering models to detect anomalies in time series. + +The basic idea is to compare the predictions produced by a fitted model (the forecasts +or the filtered series) with the actual observations, and to emit an anomaly score +describing how "different" the observations are from the predictions. + +An anomaly model takes as parameters a model and one or multiple scorer objects. +The key method is ``score()``, which takes as input one (or multiple) +time series and produces one or multiple anomaly scores time series, for each provided series. + +:class:`ForecastingAnomalyModel` works with Darts forecasting models, and :class:`FilteringAnomalyModel` +works with Darts filtering models. +The anomaly models can also be fitted by calling :func:`fit()`, which trains the scorer(s) +(in case some are trainable), and potentially the model as well. + +The function :func:`eval_accuracy()` is the same as :func:`score()`, but outputs the score of an agnostic +threshold metric ("AUC-ROC" or "AUC-PR"), between the predicted anomaly score time series, and some known binary +ground-truth time series indicating the presence of actual anomalies. +Finally, the function :func:`show_anomalies()` can also be used to visualize the predictions +(in-sample predictions and anomaly scores) of the anomaly model. +""" + +from .filtering_am import FilteringAnomalyModel +from .forecasting_am import ForecastingAnomalyModel diff --git a/darts/ad/anomaly_model/anomaly_model.py b/darts/ad/anomaly_model/anomaly_model.py new file mode 100644 index 0000000000..a86d122249 --- /dev/null +++ b/darts/ad/anomaly_model/anomaly_model.py @@ -0,0 +1,148 @@ +""" +Anomaly models base classes +""" + +from abc import ABC, abstractmethod +from typing import Dict, Sequence, Union + +from darts.ad.scorers.scorers import AnomalyScorer +from darts.ad.utils import ( + _to_list, + eval_accuracy_from_scores, + show_anomalies_from_scores, +) +from darts.logging import raise_if_not +from darts.timeseries import TimeSeries + + +class AnomalyModel(ABC): + """Base class for all anomaly models.""" + + def __init__(self, model, scorer): + + self.scorers = _to_list(scorer) + + raise_if_not( + all([isinstance(s, AnomalyScorer) for s in self.scorers]), + "all scorers must be of instance darts.ad.scorers.AnomalyScorer.", + ) + + self.scorers_are_trainable = any(s.trainable for s in self.scorers) + self.univariate_scoring = any(s.univariate_scorer for s in self.scorers) + + self.model = model + + def _check_univariate(self, actual_anomalies): + """Checks if `actual_anomalies` contains only univariate series, which + is required if any of the scorers returns a univariate score. + """ + + if self.univariate_scoring: + raise_if_not( + all([s.width == 1 for s in actual_anomalies]), + "Anomaly model contains scorer {} that will return".format( + [s.__str__() for s in self.scorers if s.univariate_scorer] + ) + + " a univariate anomaly score series (width=1). Found a" + + " multivariate `actual_anomalies`. The evaluation of the" + + " accuracy cannot be computed. If applicable, think about" + + " setting the scorer parameter `componenet_wise` to True.", + ) + + @abstractmethod + def fit( + self, series: Union[TimeSeries, Sequence[TimeSeries]] + ) -> Union[TimeSeries, Sequence[TimeSeries]]: + pass + + @abstractmethod + def score( + self, series: Union[TimeSeries, Sequence[TimeSeries]] + ) -> Union[TimeSeries, Sequence[TimeSeries]]: + pass + + @abstractmethod + def eval_accuracy( + self, + actual_anomalies: Union[TimeSeries, Sequence[TimeSeries]], + series: Union[TimeSeries, Sequence[TimeSeries]], + ) -> Union[TimeSeries, Sequence[TimeSeries]]: + pass + + @abstractmethod + def show_anomalies(self, series: TimeSeries): + pass + + def _show_anomalies( + self, + series: TimeSeries, + model_output: TimeSeries = None, + anomaly_scores: Union[TimeSeries, Sequence[TimeSeries]] = None, + names_of_scorers: Union[str, Sequence[str]] = None, + actual_anomalies: TimeSeries = None, + title: str = None, + metric: str = None, + ): + """Internal function that plots the results of the anomaly model. + Called by the function show_anomalies(). + """ + + if title is None: + title = f"Anomaly results ({self.model.__class__.__name__})" + + if names_of_scorers is None: + names_of_scorers = [s.__str__() for s in self.scorers] + + list_window = [s.window for s in self.scorers] + + return show_anomalies_from_scores( + series, + model_output=model_output, + anomaly_scores=anomaly_scores, + window=list_window, + names_of_scorers=names_of_scorers, + actual_anomalies=actual_anomalies, + title=title, + metric=metric, + ) + + def _eval_accuracy_from_scores( + self, + list_actual_anomalies: Sequence[TimeSeries], + list_anomaly_scores: Sequence[TimeSeries], + metric: str, + ) -> Union[Sequence[Dict[str, float]], Sequence[Dict[str, Sequence[float]]]]: + """Internal function that computes the accuracy of the anomaly scores + computed by the model. Called by the function eval_accuracy(). + """ + windows = [s.window for s in self.scorers] + + # create a list of unique names for each scorer that + # will be used as keys for the dictionary containing + # the accuracy of each scorer. + name_scorers = [] + for scorer in self.scorers: + name = scorer.__str__() + "_w=" + str(scorer.window) + + if name in name_scorers: + i = 1 + new_name = name + "_" + str(i) + while new_name in name_scorers: + i = i + 1 + new_name = name + "_" + str(i) + name = new_name + + name_scorers.append(name) + + acc = [] + for anomalies, scores in zip(list_actual_anomalies, list_anomaly_scores): + acc.append( + eval_accuracy_from_scores( + actual_anomalies=anomalies, + anomaly_score=scores, + window=windows, + metric=metric, + ) + ) + + return [dict(zip(name_scorers, scorer_values)) for scorer_values in acc] diff --git a/darts/ad/anomaly_model/filtering_am.py b/darts/ad/anomaly_model/filtering_am.py new file mode 100644 index 0000000000..9679bc2906 --- /dev/null +++ b/darts/ad/anomaly_model/filtering_am.py @@ -0,0 +1,348 @@ +""" +Filtering Anomaly Model +----------------------- + +A ``FilteringAnomalyModel`` wraps around a Darts filtering model and one or +several anomaly scorer(s) to compute anomaly scores +by comparing how actuals deviate from the model's predictions (filtered series). +""" + +from typing import Dict, Sequence, Union + +from darts.ad.anomaly_model.anomaly_model import AnomalyModel +from darts.ad.scorers.scorers import AnomalyScorer +from darts.ad.utils import _assert_same_length, _to_list +from darts.logging import get_logger, raise_if_not +from darts.models.filtering.filtering_model import FilteringModel +from darts.timeseries import TimeSeries + +logger = get_logger(__name__) + + +class FilteringAnomalyModel(AnomalyModel): + def __init__( + self, + model: FilteringModel, + scorer: Union[AnomalyScorer, Sequence[AnomalyScorer]], + ): + """Filtering-based Anomaly Detection Model + + The filtering model may or may not be already fitted. The underlying assumption is that this model + should be able to adequately filter the series in the absence of anomalies. For this reason, + it is recommended to either provide a model that has already been fitted and evaluated to work + appropriately on a series without anomalies, or to ensure that a simple call to the :func:`fit()` + function of the model will be sufficient to train it to satisfactory performance on series without anomalies. + + Calling :func:`fit()` on the anomaly model will fit the underlying filtering model only + if ``allow_model_training`` is set to ``True`` upon calling ``fit()``. + In addition, calling :func:`fit()` will also fit the fittable scorers, if any. + + Parameters + ---------- + filter + A filtering model from Darts that will be used to filter the actual time series + scorer + One or multiple scorer(s) that will be used to compare the actual and predicted time series in order + to obtain an anomaly score ``TimeSeries``. + If a list of `N` scorer is given, the anomaly model will call each + one of the scorers and output a list of `N` anomaly scores ``TimeSeries``. + """ + + raise_if_not( + isinstance(model, FilteringModel), + f"`model` must be a darts.models.filtering not a {type(model)}.", + ) + self.filter = model + + super().__init__(model=model, scorer=scorer) + + def fit( + self, + series: Union[TimeSeries, Sequence[TimeSeries]], + allow_model_training: bool = False, + **filter_fit_kwargs, + ): + """Fit the underlying filtering model (if applicable) and the fittable scorers, if any. + + Train the filter (if not already fitted and `allow_filter_training` is set to True) + and the scorer(s) on the given time series. + + The filter model will be applied to the given series, and the results will be used + to train the scorer(s). + + Parameters + ---------- + series + The (sequence of) series to be trained on. + allow_model_training + Boolean value that indicates if the filtering model needs to be fitted on the given series. + If set to False, the model needs to be already fitted. + Default: False + filter_fit_kwargs + Parameters to be passed on to the filtering model ``fit()`` method. + + Returns + ------- + self + Fitted model + """ + # TODO: add support for covariates (see eg. Kalman Filter) + + raise_if_not( + type(allow_model_training) is bool, + f"`allow_filter_training` must be Boolean, found type: {type(allow_model_training)}.", + ) + + # checks if model does not need training and all scorer(s) are not fittable + if not allow_model_training and not self.scorers_are_trainable: + logger.warning( + f"The filtering model {self.model.__class__.__name__} is not required to be trained" + + " because the parameter `allow_filter_training` is set to False, and no scorer" + + " fittable. The ``.fit()`` function has no effect." + ) + return + + list_series = _to_list(series) + + raise_if_not( + all([isinstance(s, TimeSeries) for s in list_series]), + "all input `series` must be of type Timeseries.", + ) + + if allow_model_training: + # fit filtering model + if hasattr(self.filter, "fit"): + # TODO: check if filter is already fitted (for now fit it regardless -> only Kalman) + raise_if_not( + len(list_series) == 1, + f"Filter model {self.model.__class__.__name__} can only be fitted on a" + + " single time series, but multiple are provided.", + ) + + self.filter.fit(list_series[0], **filter_fit_kwargs) + else: + raise ValueError( + "`allow_filter_training` was set to True, but the filter" + + f" {self.model.__class__.__name__} has no fit() method." + ) + else: + # TODO: check if Kalman is fitted or not + # if not raise error "fit filter before, or set `allow_filter_training` to TRUE" + pass + + if self.scorers_are_trainable: + list_pred = [self.filter.filter(series) for series in list_series] + + # fit the scorers + for scorer in self.scorers: + if hasattr(scorer, "fit"): + scorer.fit_from_prediction(list_series, list_pred) + + return self + + def show_anomalies( + self, + series: TimeSeries, + actual_anomalies: TimeSeries = None, + names_of_scorers: Union[str, Sequence[str]] = None, + title: str = None, + metric: str = None, + **score_kwargs, + ): + """Plot the results of the anomaly model. + + Computes the score on the given series input and shows the different anomaly scores with respect to time. + + The plot will be composed of the following: + + - the series itself with the output of the filtering model + - the anomaly score of each scorer. The scorer with different windows will be separated. + - the actual anomalies, if given. + + It is possible to: + + - add a title to the figure with the parameter `title` + - give personalized names for the scorers with `names_of_scorers` + - show the results of a metric for each anomaly score (AUC_ROC or AUC_PR), if the actual anomalies are given + + Parameters + ---------- + series + The series to visualize anomalies from. + actual_anomalies + The ground truth of the anomalies (1 if it is an anomaly and 0 if not) + names_of_scorers + Name of the scorers. Must be a list of length equal to the number of scorers in the anomaly_model. + title + Title of the figure + metric + Optionally, Scoring function to use. Must be one of "AUC_ROC" and "AUC_PR". + Default: "AUC_ROC" + score_kwargs + parameters for the `.score()` function + """ + + if isinstance(series, Sequence): + raise_if_not( + len(series) == 1, + f"`show_anomalies` expects one series, found a sequence of length {len(series)} as input.", + ) + + series = series[0] + + anomaly_scores, model_output = self.score( + series, return_model_prediction=True, **score_kwargs + ) + + return self._show_anomalies( + series, + model_output=model_output, + anomaly_scores=anomaly_scores, + names_of_scorers=names_of_scorers, + actual_anomalies=actual_anomalies, + title=title, + metric=metric, + ) + + def score( + self, + series: Union[TimeSeries, Sequence[TimeSeries]], + return_model_prediction: bool = False, + **filter_kwargs, + ): + """Compute the anomaly score(s) for the given series. + + Predicts the given target time series with the filtering model, and applies the scorer(s) + to compare the predicted (filtered) series and the provided series. + + Outputs the anomaly score(s) of the provided time series. + + Parameters + ---------- + series + The (sequence of) series to score. + return_model_prediction + Boolean value indicating if the prediction of the model should be returned along the anomaly score + Default: False + filter_kwargs + parameters of the Darts `.filter()` filtering model + + Returns + ------- + Union[TimeSeries, Sequence[TimeSeries], Sequence[Sequence[TimeSeries]]] + Anomaly scores series generated by the anomaly model scorers + + - ``TimeSeries`` if `series` is a series, and the anomaly model contains one scorer. + - ``Sequence[TimeSeries]`` + + * If `series` is a series, and the anomaly model contains multiple scorers, + returns one series per scorer. + * If `series` is a sequence, and the anomaly model contains one scorer, + returns one series per series in the sequence. + - ``Sequence[Sequence[TimeSeries]]`` if `series` is a sequence, and the anomaly + model contains multiple scorers. + The outer sequence is over the series, and inner sequence is over the scorers. + """ + raise_if_not( + type(return_model_prediction) is bool, + f"`return_model_prediction` must be Boolean, found type: {type(return_model_prediction)}.", + ) + + list_series = _to_list(series) + + # TODO: vectorize this call later on if we have any filtering models allowing this + list_pred = [self.filter.filter(s, **filter_kwargs) for s in list_series] + + scores = list( + zip( + *[ + sc.score_from_prediction(list_series, list_pred) + for sc in self.scorers + ] + ) + ) + + if len(scores) == 1 and not isinstance(series, Sequence): + # there's only one series + scores = scores[0] + if len(scores) == 1: + # there's only one scorer + scores = scores[0] + + if len(list_pred) == 1: + list_pred = list_pred[0] + + if return_model_prediction: + return scores, list_pred + else: + return scores + + def eval_accuracy( + self, + actual_anomalies: Union[TimeSeries, Sequence[TimeSeries]], + series: Union[TimeSeries, Sequence[TimeSeries]], + metric: str = "AUC_ROC", + **filter_kwargs, + ) -> Union[ + Dict[str, float], + Dict[str, Sequence[float]], + Sequence[Dict[str, float]], + Sequence[Dict[str, Sequence[float]]], + ]: + """Compute the accuracy of the anomaly scores computed by the model. + + Predicts the `series` with the filtering model, and applies the + scorer(s) on the filtered time series and the given target time series. Returns the + score(s) of an agnostic threshold metric, based on the anomaly score given by the scorer(s). + + Parameters + ---------- + actual_anomalies + The (sequence of) ground truth of the anomalies (1 if it is an anomaly and 0 if not) + series + The (sequence of) series to predict anomalies on. + metric + Optionally, Scoring function to use. Must be one of "AUC_ROC" and "AUC_PR". + Default: "AUC_ROC" + filter_kwargs + parameters of the Darts `.filter()` filtering model + + Returns + ------- + Union[Dict[str, float], Dict[str, Sequence[float]], Sequence[Dict[str, float]], + Sequence[Dict[str, Sequence[float]]]] + Score for the time series. + A (sequence of) dictionary with the keys being the name of the scorers, and the values being the + metric results on the (sequence of) `series`. If the scorer treats every dimension independently + (by nature of the scorer or if its component_wise is set to True), the values of the dictionary + will be a Sequence containing the score for each dimension. + """ + list_series, list_actual_anomalies = _to_list(series), _to_list( + actual_anomalies + ) + + raise_if_not( + all([isinstance(s, TimeSeries) for s in list_series]), + "all input `series` must be of type Timeseries.", + ) + + raise_if_not( + all([isinstance(s, TimeSeries) for s in list_actual_anomalies]), + "all input `actual_anomalies` must be of type Timeseries.", + ) + + _assert_same_length(list_series, list_actual_anomalies) + self._check_univariate(list_actual_anomalies) + + list_anomaly_scores = self.score(series=list_series, **filter_kwargs) + + acc_anomaly_scores = self._eval_accuracy_from_scores( + list_actual_anomalies=list_actual_anomalies, + list_anomaly_scores=list_anomaly_scores, + metric=metric, + ) + + if len(acc_anomaly_scores) == 1 and not isinstance(series, Sequence): + return acc_anomaly_scores[0] + else: + return acc_anomaly_scores diff --git a/darts/ad/anomaly_model/forecasting_am.py b/darts/ad/anomaly_model/forecasting_am.py new file mode 100644 index 0000000000..b0c074e0ce --- /dev/null +++ b/darts/ad/anomaly_model/forecasting_am.py @@ -0,0 +1,671 @@ +""" +Forecasting Anomaly Model +------------------------- + +A ``ForecastingAnomalyModel`` wraps around a Darts forecasting model and one or several anomaly +scorer(s) to compute anomaly scores by comparing how actuals deviate from the model's forecasts. +""" + +# TODO: +# - put start default value to its minimal value (wait for the release of historical_forecast) + +import inspect +from typing import Dict, Optional, Sequence, Union + +import pandas as pd + +from darts.ad.anomaly_model.anomaly_model import AnomalyModel +from darts.ad.scorers.scorers import AnomalyScorer +from darts.ad.utils import _assert_same_length, _assert_timeseries, _to_list +from darts.logging import get_logger, raise_if_not +from darts.models.forecasting.forecasting_model import ForecastingModel +from darts.timeseries import TimeSeries + +logger = get_logger(__name__) + + +class ForecastingAnomalyModel(AnomalyModel): + def __init__( + self, + model: ForecastingModel, + scorer: Union[AnomalyScorer, Sequence[AnomalyScorer]], + ): + """Forecasting-based Anomaly Detection Model + + The forecasting model may or may not be already fitted. The underlying assumption is that `model` + should be able to accurately forecast the series in the absence of anomalies. For this reason, + it is recommended to either provide a model that has already been fitted and evaluated to work + appropriately on a series without anomalies, or to ensure that a simple call to the :func:`fit()` + method of the model will be sufficient to train it to satisfactory performance on a series without anomalies. + + Calling :func:`fit()` on the anomaly model will fit the underlying forecasting model only + if ``allow_model_training`` is set to ``True`` upon calling ``fit()``. + In addition, calling :func:`fit()` will also fit the fittable scorers, if any. + + Parameters + ---------- + model + An instance of a Darts forecasting model. + scorer + One or multiple scorer(s) that will be used to compare the actual and predicted time series in order + to obtain an anomaly score ``TimeSeries``. + If a list of `N` scorers is given, the anomaly model will call each + one of the scorers and output a list of `N` anomaly scores ``TimeSeries``. + """ + + raise_if_not( + isinstance(model, ForecastingModel), + f"Model must be a darts ForecastingModel not a {type(model)}.", + ) + self.model = model + + super().__init__(model=model, scorer=scorer) + + def fit( + self, + series: Union[TimeSeries, Sequence[TimeSeries]], + past_covariates: Optional[Union[TimeSeries, Sequence[TimeSeries]]] = None, + future_covariates: Optional[Union[TimeSeries, Sequence[TimeSeries]]] = None, + allow_model_training: bool = False, + forecast_horizon: int = 1, + start: Union[pd.Timestamp, float, int] = 0.5, + num_samples: int = 1, + **model_fit_kwargs, + ): + """Fit the underlying forecasting model (if applicable) and the fittable scorers, if any. + + Train the model (if not already fitted and ``allow_model_training`` is set to True) and the + scorer(s) (if fittable) on the given time series. + + Once the model is fitted, the series historical forecasts are computed, + representing what would have been forecasted by this model on the series. + + The prediction and the series are then used to train the scorer(s). + + Parameters + ---------- + series + One or multiple (if the model supports it) target series to be + trained on (generally assumed to be anomaly-free). + past_covariates + Optional past-observed covariate series or sequence of series. This applies only if the model + supports past covariates. + future_covariates + Optional future-known covariate series or sequence of series. This applies only if the model + supports future covariates. + allow_model_training + Boolean value that indicates if the forecasting model needs to be fitted on the given series. + If set to False, the model needs to be already fitted. + Default: False + forecast_horizon + The forecast horizon for the predictions. + start + The first point of time at which a prediction is computed for a future time. + This parameter supports 3 different data types: ``float``, ``int`` and ``pandas.Timestamp``. + In the case of ``float``, the parameter will be treated as the proportion of the time series + that should lie before the first prediction point. + In the case of ``int``, the parameter will be treated as an integer index to the time index of + `series` that will be used as first prediction time. + In case of ``pandas.Timestamp``, this time stamp will be used to determine the first prediction time + directly. + Default: 0.5 + num_samples + Number of times a prediction is sampled from a probabilistic model. Should be left set to 1 for + deterministic models. + model_fit_kwargs + Parameters to be passed on to the forecast model ``fit()`` method. + + Returns + ------- + self + Fitted model + """ + + raise_if_not( + type(allow_model_training) is bool, + f"`allow_model_training` must be Boolean, found type: {type(allow_model_training)}.", + ) + + # checks if model does not need training and all scorer(s) are not fittable + if not allow_model_training and not self.scorers_are_trainable: + logger.warning( + f"The forecasting model {self.model.__class__.__name__} won't be trained" + + " because the parameter `allow_model_training` is set to False, and no scorer" + + " is fittable. ``.fit()`` method has no effect." + ) + return + + list_series = _to_list(series) + + raise_if_not( + all([isinstance(s, TimeSeries) for s in list_series]), + "all input `series` must be of type Timeseries.", + ) + + list_past_covariates = self._prepare_covariates( + past_covariates, list_series, "past" + ) + list_future_covariates = self._prepare_covariates( + future_covariates, list_series, "future" + ) + + model_fit_kwargs["past_covariates"] = list_past_covariates + model_fit_kwargs["future_covariates"] = list_future_covariates + + # fit forecasting model + if allow_model_training: + # the model has not been trained yet + + fit_signature_series = ( + inspect.signature(self.model.fit).parameters["series"].annotation + ) + + # checks if model can be trained on multiple time series or only on a time series + # TODO: check if model can accept multivariate timeseries, raise error if given and model cannot + if "Sequence[darts.timeseries.TimeSeries]" in str(fit_signature_series): + self.model.fit(series=list_series, **model_fit_kwargs) + else: + raise_if_not( + len(list_series) == 1, + f"Forecasting model {self.model.__class__.__name__} only accepts a single time series" + + " for the training phase and not a sequence of multiple of time series.", + ) + self.model.fit(series=list_series[0], **model_fit_kwargs) + else: + raise_if_not( + self.model._fit_called, + f"Model {self.model.__class__.__name__} needs to be trained, consider training " + + "it beforehand or setting " + + "`allow_model_training` to True (default: False). " + + "The model will then be trained on the provided series.", + ) + + # generate the historical_forecast() prediction of the model on the train set + if self.scorers_are_trainable: + # check if the window size of the scorers are lower than the max size allowed + self._check_window_size(list_series, start) + + list_pred = [] + for idx, series in enumerate(list_series): + + if list_past_covariates is not None: + past_covariates = list_past_covariates[idx] + + if list_future_covariates is not None: + future_covariates = list_future_covariates[idx] + + list_pred.append( + self._predict_with_forecasting( + series, + past_covariates=past_covariates, + future_covariates=future_covariates, + forecast_horizon=forecast_horizon, + start=start, + num_samples=num_samples, + ) + ) + + # fit the scorers + for scorer in self.scorers: + if hasattr(scorer, "fit"): + scorer.fit_from_prediction(list_series, list_pred) + + return self + + def _prepare_covariates( + self, + covariates: Union[TimeSeries, Sequence[TimeSeries]], + series: Sequence[TimeSeries], + name_covariates: str, + ) -> Sequence[TimeSeries]: + """Convert `covariates` into Sequence, if not already, and checks if their length is equal to the one of `series`. + + Parameters + ---------- + covariates + Covariate ("future" or "past") of `series`. + series + The series to be trained on. + name_covariates + Internal parameter for error message, a string indicating if it is a "future" or "past" covariates. + + Returns + ------- + Sequence[TimeSeries] + Covariate time series + """ + + if covariates is not None: + list_covariates = _to_list(covariates) + + for covariates in list_covariates: + _assert_timeseries( + covariates, name_covariates + "_covariates input series" + ) + + raise_if_not( + len(list_covariates) == len(series), + f"Number of {name_covariates}_covariates must match the number of given " + + f"series, found length {len(list_covariates)} and expected {len(series)}.", + ) + + return list_covariates if covariates is not None else None + + def show_anomalies( + self, + series: TimeSeries, + past_covariates: Optional[TimeSeries] = None, + future_covariates: Optional[TimeSeries] = None, + forecast_horizon: int = 1, + start: Union[pd.Timestamp, float, int] = 0.5, + num_samples: int = 1, + actual_anomalies: TimeSeries = None, + names_of_scorers: Union[str, Sequence[str]] = None, + title: str = None, + metric: str = None, + ): + """Plot the results of the anomaly model. + + Computes the score on the given series input and shows the different anomaly scores with respect to time. + + The plot will be composed of the following: + + - the series itself with the output of the forecasting model. + - the anomaly score for each scorer. The scorers with different windows will be separated. + - the actual anomalies, if given. + + It is possible to: + + - add a title to the figure with the parameter `title` + - give personalized names for the scorers with `names_of_scorers` + - show the results of a metric for each anomaly score (AUC_ROC or AUC_PR), + if the actual anomalies are provided. + + Parameters + ---------- + series + The series to visualize anomalies from. + past_covariates + An optional past-observed covariate series or sequence of series. This applies only if the model + supports past covariates. + future_covariates + An optional future-known covariate series or sequence of series. This applies only if the model + supports future covariates. + forecast_horizon + The forecast horizon for the predictions. + start + The first point of time at which a prediction is computed for a future time. + This parameter supports 3 different data types: ``float``, ``int`` and ``pandas.Timestamp``. + In the case of ``float``, the parameter will be treated as the proportion of the time series + that should lie before the first prediction point. + In the case of ``int``, the parameter will be treated as an integer index to the time index of + `series` that will be used as first prediction time. + In case of ``pandas.Timestamp``, this time stamp will be used to determine the first prediction time + directly. + num_samples + Number of times a prediction is sampled from a probabilistic model. Should be left set to 1 for + deterministic models. + actual_anomalies + The ground truth of the anomalies (1 if it is an anomaly and 0 if not) + names_of_scorers + Name of the scores. Must be a list of length equal to the number of scorers in the anomaly_model. + title + Title of the figure + metric + Optionally, Scoring function to use. Must be one of "AUC_ROC" and "AUC_PR". + Default: "AUC_ROC" + """ + + if isinstance(series, Sequence): + raise_if_not( + len(series) == 1, + f"`show_anomalies` expects one series, found a list of length {len(series)} as input.", + ) + + series = series[0] + + raise_if_not( + isinstance(series, TimeSeries), + f"`show_anomalies` expects an input of type TimeSeries, found type: {type(series)}.", + ) + + anomaly_scores, model_output = self.score( + series, + past_covariates=past_covariates, + future_covariates=future_covariates, + forecast_horizon=forecast_horizon, + start=start, + num_samples=num_samples, + return_model_prediction=True, + ) + + return self._show_anomalies( + series, + model_output=model_output, + anomaly_scores=anomaly_scores, + names_of_scorers=names_of_scorers, + actual_anomalies=actual_anomalies, + title=title, + metric=metric, + ) + + def score( + self, + series: Union[TimeSeries, Sequence[TimeSeries]], + past_covariates: Optional[Union[TimeSeries, Sequence[TimeSeries]]] = None, + future_covariates: Optional[Union[TimeSeries, Sequence[TimeSeries]]] = None, + forecast_horizon: int = 1, + start: Union[pd.Timestamp, float, int] = 0.5, + num_samples: int = 1, + return_model_prediction: bool = False, + ) -> Union[TimeSeries, Sequence[TimeSeries]]: + """Compute anomaly score(s) for the given series. + + Predicts the given target time series with the forecasting model, and applies the scorer(s) + on the prediction and the target input time series. Outputs the anomaly score of the given + input time series. + + Parameters + ---------- + series + The (sequence of) series to score on. + past_covariates + An optional past-observed covariate series or sequence of series. This applies only if the model + supports past covariates. + future_covariates + An optional future-known covariate series or sequence of series. This applies only if the model + supports future covariates. + forecast_horizon + The forecast horizon for the predictions. + start + The first point of time at which a prediction is computed for a future time. + This parameter supports 3 different data types: ``float``, ``int`` and ``pandas.Timestamp``. + In the case of ``float``, the parameter will be treated as the proportion of the time series + that should lie before the first prediction point. + In the case of ``int``, the parameter will be treated as an integer index to the time index of + `series` that will be used as first prediction time. + In case of ``pandas.Timestamp``, this time stamp will be used to determine the first prediction time + directly. Default: 0.5 + num_samples + Number of times a prediction is sampled from a probabilistic model. Should be left set to 1 for + deterministic models. + return_model_prediction + Boolean value indicating if the prediction of the model should be returned along the anomaly score + Default: False + + Returns + ------- + Union[TimeSeries, Sequence[TimeSeries], Sequence[Sequence[TimeSeries]]] + Anomaly scores series generated by the anomaly model scorers + + - ``TimeSeries`` if `series` is a series, and the anomaly model contains one scorer. + - ``Sequence[TimeSeries]`` + + * if `series` is a series, and the anomaly model contains multiple scorers, + returns one series per scorer. + * if `series` is a sequence, and the anomaly model contains one scorer, + returns one series per series in the sequence. + - ``Sequence[Sequence[TimeSeries]]`` if `series` is a sequence, and the anomaly + model contains multiple scorers. The outer sequence is over the series, + and inner sequence is over the scorers. + """ + raise_if_not( + type(return_model_prediction) is bool, + f"`return_model_prediction` must be Boolean, found type: {type(return_model_prediction)}.", + ) + + raise_if_not( + self.model._fit_called, + f"Model {self.model} has not been trained. Please call ``.fit()``.", + ) + + list_series = _to_list(series) + + list_past_covariates = self._prepare_covariates( + past_covariates, list_series, "past" + ) + list_future_covariates = self._prepare_covariates( + future_covariates, list_series, "future" + ) + + # check if the window size of the scorers are lower than the max size allowed + self._check_window_size(list_series, start) + + list_pred = [] + for idx, s in enumerate(list_series): + + if list_past_covariates is not None: + past_covariates = list_past_covariates[idx] + + if list_future_covariates is not None: + future_covariates = list_future_covariates[idx] + + list_pred.append( + self._predict_with_forecasting( + s, + past_covariates=past_covariates, + future_covariates=future_covariates, + forecast_horizon=forecast_horizon, + start=start, + num_samples=num_samples, + ) + ) + + scores = list( + zip( + *[ + sc.score_from_prediction(list_series, list_pred) + for sc in self.scorers + ] + ) + ) + + if len(scores) == 1 and not isinstance(series, Sequence): + # there's only one series + scores = scores[0] + if len(scores) == 1: + # there's only one scorer + scores = scores[0] + + if len(list_pred) == 1: + list_pred = list_pred[0] + + if return_model_prediction: + return scores, list_pred + else: + return scores + + def _check_window_size( + self, series: Sequence[TimeSeries], start: Union[pd.Timestamp, float, int] + ): + """Checks if the parameters `window` of the scorers are smaller than the maximum window size allowed. + The maximum size allowed is equal to the output length of the .historical_forecast() applied on `series`. + It is defined by the parameter `start` and the series’ length. + + Parameters + ---------- + series + The series given to the .historical_forecast() + start + Parameter of the .historical_forecast(): first point of time at which a prediction is computed + for a future time. + """ + # biggest window of the anomaly_model scorers + max_window = max(scorer.window for scorer in self.scorers) + + for s in series: + max_possible_window = ( + len(s.drop_before(s.get_timestamp_at_point(start))) + 1 + ) + raise_if_not( + max_window <= max_possible_window, + f"Window size {max_window} is greater than the targeted series length {max_possible_window}," + + f" must be lower or equal. Reduce window size, or reduce start value (start: {start}).", + ) + + def _predict_with_forecasting( + self, + series: TimeSeries, + past_covariates: Optional[TimeSeries] = None, + future_covariates: Optional[TimeSeries] = None, + forecast_horizon: int = 1, + start: Union[pd.Timestamp, float, int] = None, + num_samples: int = 1, + ) -> TimeSeries: + + """Compute the historical forecasts that would have been obtained by this model on the `series`. + + `retrain` is set to False if possible (this is not supported by all models). If set to True, it will always + re-train the model on the entire available history, + + Parameters + ---------- + series + The target time series to use to successively train and evaluate the historical forecasts. + past_covariates + An optional past-observed covariate series or sequence of series. This applies only if the model + supports past covariates. + future_covariates + An optional future-known covariate series or sequence of series. This applies only if the model + supports future covariates. + forecast_horizon + The forecast horizon for the predictions + start + The first point of time at which a prediction is computed for a future time. + This parameter supports 3 different data types: ``float``, ``int`` and ``pandas.Timestamp``. + In the case of ``float``, the parameter will be treated as the proportion of the time series + that should lie before the first prediction point. + In the case of ``int``, the parameter will be treated as an integer index to the time index of + `series` that will be used as first prediction time. + In case of ``pandas.Timestamp``, this time stamp will be used to determine the first prediction time + directly. + num_samples + Number of times a prediction is sampled from a probabilistic model. Should be left set to 1 for + deterministic models. + + Returns + ------- + TimeSeries + Single ``TimeSeries`` instance created from the last point of each individual forecast. + """ + + # TODO: raise an exception. We only support models that do not need retrain + # checks if model accepts to not be retrained in the historical_forecasts() + if self.model._supports_non_retrainable_historical_forecasts(): + # default: set to False. Allows a faster computation. + retrain = False + else: + retrain = True + + historical_forecasts_param = { + "past_covariates": past_covariates, + "future_covariates": future_covariates, + "forecast_horizon": forecast_horizon, + "start": start, + "retrain": retrain, + "num_samples": num_samples, + "stride": 1, + "last_points_only": True, + "verbose": False, + } + + return self.model.historical_forecasts(series, **historical_forecasts_param) + + def eval_accuracy( + self, + actual_anomalies: Union[TimeSeries, Sequence[TimeSeries]], + series: Union[TimeSeries, Sequence[TimeSeries]], + past_covariates: Optional[Union[TimeSeries, Sequence[TimeSeries]]] = None, + future_covariates: Optional[Union[TimeSeries, Sequence[TimeSeries]]] = None, + forecast_horizon: int = 1, + start: Union[pd.Timestamp, float, int] = 0.5, + num_samples: int = 1, + metric: str = "AUC_ROC", + ) -> Union[ + Dict[str, float], + Dict[str, Sequence[float]], + Sequence[Dict[str, float]], + Sequence[Dict[str, Sequence[float]]], + ]: + """Compute the accuracy of the anomaly scores computed by the model. + + Predicts the `series` with the forecasting model, and applies the + scorer(s) on the predicted time series and the given target time series. Returns the + score(s) of an agnostic threshold metric, based on the anomaly score given by the scorer(s). + + Parameters + ---------- + actual_anomalies + The (sequence of) ground truth of the anomalies (1 if it is an anomaly and 0 if not) + series + The (sequence of) series to predict anomalies on. + past_covariates + An optional past-observed covariate series or sequence of series. This applies only + if the model supports past covariates. + future_covariates + An optional future-known covariate series or sequence of series. This applies only + if the model supports future covariates. + forecast_horizon + The forecast horizon for the predictions. + start + The first point of time at which a prediction is computed for a future time. + This parameter supports 3 different data types: ``float``, ``int`` and ``pandas.Timestamp``. + In the case of ``float``, the parameter will be treated as the proportion of the time series + that should lie before the first prediction point. + In the case of ``int``, the parameter will be treated as an integer index to the time index of + `series` that will be used as first prediction time. + In case of ``pandas.Timestamp``, this time stamp will be used to determine the first prediction time + directly. + num_samples + Number of times a prediction is sampled from a probabilistic model. Should be left set to 1 for + deterministic models. + metric + Optionally, Scoring function to use. Must be one of "AUC_ROC" and "AUC_PR". + Default: "AUC_ROC" + + Returns + ------- + Union[Dict[str, float], Dict[str, Sequence[float]], Sequence[Dict[str, float]], + Sequence[Dict[str, Sequence[float]]]] + Score for the time series. + A (sequence of) dictionary with the keys being the name of the scorers, and the values being the + metric results on the (sequence of) `series`. If the scorer treats every dimension independently + (by nature of the scorer or if its component_wise is set to True), the values of the dictionary + will be a Sequence containing the score for each dimension. + """ + + list_actual_anomalies = _to_list(actual_anomalies) + list_series = _to_list(series) + + raise_if_not( + all([isinstance(s, TimeSeries) for s in list_series]), + "all input `series` must be of type Timeseries.", + ) + + raise_if_not( + all([isinstance(s, TimeSeries) for s in list_actual_anomalies]), + "all input `actual_anomalies` must be of type Timeseries.", + ) + + _assert_same_length(list_actual_anomalies, list_series) + self._check_univariate(list_actual_anomalies) + + list_anomaly_scores = self.score( + series=list_series, + past_covariates=past_covariates, + future_covariates=future_covariates, + forecast_horizon=forecast_horizon, + start=start, + num_samples=num_samples, + ) + + acc_anomaly_scores = self._eval_accuracy_from_scores( + list_actual_anomalies=list_actual_anomalies, + list_anomaly_scores=list_anomaly_scores, + metric=metric, + ) + + if len(acc_anomaly_scores) == 1 and not isinstance(series, Sequence): + return acc_anomaly_scores[0] + else: + return acc_anomaly_scores diff --git a/darts/ad/detectors/__init__.py b/darts/ad/detectors/__init__.py new file mode 100644 index 0000000000..820b0f71c6 --- /dev/null +++ b/darts/ad/detectors/__init__.py @@ -0,0 +1,23 @@ +""" +Anomaly Detectors +----------------- + +Detectors provide binary anomaly classification on time series. +They can typically be used to transform anomaly scores time series into binary anomaly time series. + +Some detectors are trainable. For instance, ``QuantileDetector`` emits a binary anomaly for +every time step where the observed value(s) are beyond the quantile(s) observed +on the training series. + +The main functions are ``fit()`` (for the trainable detectors), ``detect()`` and ``eval_accuracy()``. + +``fit()`` trains the detector over the history of one or multiple time series. It can +for instance be called on series containing anomaly scores (or even raw values) during normal times. +The function ``detect()`` takes an anomaly score time series as input, and applies the detector +to obtain binary predictions. The function ``eval_accuracy()`` returns the accuracy metric +("accuracy", "precision", "recall" or "f1") between a binary prediction time series and some known +binary ground truth time series indicating the presence of anomalies. +""" + +from .quantile_detector import QuantileDetector +from .threshold_detector import ThresholdDetector diff --git a/darts/ad/detectors/detectors.py b/darts/ad/detectors/detectors.py new file mode 100644 index 0000000000..88f3b3cc7a --- /dev/null +++ b/darts/ad/detectors/detectors.py @@ -0,0 +1,219 @@ +""" +Detector Base Classes +""" + +# TODO: +# - check error message and add name of variable in the message error +# - rethink the positionning of fun _check_param() +# - add possibility to input a list of param rather than only one number +# - add more complex detectors +# - create an ensemble fittable detector + +from abc import ABC, abstractmethod +from typing import Any, Sequence, Union + +from darts import TimeSeries +from darts.ad.utils import eval_accuracy_from_binary_prediction +from darts.logging import raise_if_not + + +class Detector(ABC): + """Base class for all detectors""" + + def __init__(self, *args: Any, **kwargs: Any) -> None: + pass + + def detect( + self, + series: Union[TimeSeries, Sequence[TimeSeries]], + ) -> Union[TimeSeries, Sequence[TimeSeries]]: + """Detect anomalies on given time series. + + Parameters + ---------- + series + series on which to detect anomalies. + + Returns + ------- + Union[TimeSeries, Sequence[TimeSeries]] + binary prediciton (1 if considered as an anomaly, 0 if not) + """ + + list_series = [series] if not isinstance(series, Sequence) else series + + raise_if_not( + all([isinstance(s, TimeSeries) for s in list_series]), + "all series in `series` must be of type TimeSeries.", + ) + + raise_if_not( + all([s.is_deterministic for s in list_series]), + "all series in `series` must be deterministic (number of samples equal to 1).", + ) + + detected_series = [] + for s in list_series: + detected_series.append(self._detect_core(s)) + + if len(detected_series) == 1 and not isinstance(series, Sequence): + return detected_series[0] + else: + return detected_series + + @abstractmethod + def _detect_core(self, input: Any) -> Any: + pass + + def eval_accuracy( + self, + actual_anomalies: Union[TimeSeries, Sequence[TimeSeries]], + anomaly_score: Union[TimeSeries, Sequence[TimeSeries]], + window: int = 1, + metric: str = "recall", + ) -> Union[float, Sequence[float], Sequence[Sequence[float]]]: + """Score the results against true anomalies. + + Parameters + ---------- + actual_anomalies + The ground truth of the anomalies (1 if it is an anomaly and 0 if not). + anomaly_score + Series indicating how anomoulous each window of size w is. + window + Integer value indicating the number of past samples each point represents + in the anomaly_score. + metric + Metric function to use. Must be one of "recall", "precision", + "f1", and "accuracy". + Default: "recall" + + Returns + ------- + Union[float, Sequence[float], Sequence[Sequence[float]]] + Metric results for each anomaly score + """ + + if isinstance(anomaly_score, Sequence): + raise_if_not( + all([isinstance(s, TimeSeries) for s in anomaly_score]), + "all series in `anomaly_score` must be of type TimeSeries.", + ) + + raise_if_not( + all([s.is_deterministic for s in anomaly_score]), + "all series in `anomaly_score` must be deterministic (number of samples equal to 1).", + ) + else: + raise_if_not( + isinstance(anomaly_score, TimeSeries), + f"Input `anomaly_score` must be of type TimeSeries, found {type(anomaly_score)}.", + ) + + raise_if_not( + anomaly_score.is_deterministic, + "Input `anomaly_score` must be deterministic (number of samples equal to 1).", + ) + + return eval_accuracy_from_binary_prediction( + actual_anomalies, self.detect(anomaly_score), window, metric + ) + + +class FittableDetector(Detector): + """Base class of Detectors that need training.""" + + def __init__(self, *args: Any, **kwargs: Any) -> None: + super().__init__(*args, **kwargs) + self._fit_called = False + + def detect( + self, + series: Union[TimeSeries, Sequence[TimeSeries]], + ) -> Union[TimeSeries, Sequence[TimeSeries]]: + """Detect anomalies on given time series. + + Parameters + ---------- + series + series on which to detect anomalies. + + Returns + ------- + Union[TimeSeries, Sequence[TimeSeries]] + binary prediciton (1 if considered as an anomaly, 0 if not) + """ + + list_series = [series] if not isinstance(series, Sequence) else series + + raise_if_not( + self._fit_called, + "The Detector has not been fitted yet. Call `fit()` first.", + ) + + raise_if_not( + all([self.width_trained_on == s.width for s in list_series]), + "all series in `series` must have the same number of components as the data " + + "used for training the detector model, number of components in training: " + + f" {self.width_trained_on}.", + ) + + return super().detect(series) + + @abstractmethod + def _fit_core(self, input: Any) -> Any: + pass + + def fit(self, series: Union[TimeSeries, Sequence[TimeSeries]]) -> None: + """Trains the detector on the given time series. + + Parameters + ---------- + series + Time series to be used to train the detector. + + Returns + ------- + self + Fitted Detector. + """ + + list_series = [series] if not isinstance(series, Sequence) else series + + raise_if_not( + all([isinstance(s, TimeSeries) for s in list_series]), + "all series in `series` must be of type TimeSeries.", + ) + + raise_if_not( + all([s.is_deterministic for s in list_series]), + "all series in `series` must be deterministic (number of samples equal to 1).", + ) + + self.width_trained_on = list_series[0].width + + raise_if_not( + all([s.width == self.width_trained_on for s in list_series]), + "all series in `series` must have the same number of components.", + ) + + self._fit_called = True + return self._fit_core(list_series) + + def fit_detect( + self, series: Union[TimeSeries, Sequence[TimeSeries]] + ) -> Union[TimeSeries, Sequence[TimeSeries]]: + """Trains the detector and detects anomalies on the same series. + + Parameters + ---------- + series + Time series to be used for training and be detected for anomalies. + + Returns + ------- + Union[TimeSeries, Sequence[TimeSeries]] + Binary prediciton (1 if considered as an anomaly, 0 if not) + """ + self.fit(series) + return self.detect(series) diff --git a/darts/ad/detectors/quantile_detector.py b/darts/ad/detectors/quantile_detector.py new file mode 100644 index 0000000000..82a21e50e9 --- /dev/null +++ b/darts/ad/detectors/quantile_detector.py @@ -0,0 +1,172 @@ +""" +Quantile Detector +----------------- + +Flags anomalies that are beyond some quantiles of historical data. +This is similar to a threshold-based detector, where the thresholds are +computed as quantiles of historical data when the detector is fitted. +""" + +from typing import Sequence, Union + +import numpy as np + +from darts.ad.detectors.detectors import FittableDetector +from darts.ad.detectors.threshold_detector import ThresholdDetector +from darts.logging import raise_if, raise_if_not +from darts.timeseries import TimeSeries + + +class QuantileDetector(FittableDetector): + def __init__( + self, + low_quantile: Union[Sequence[float], float, None] = None, + high_quantile: Union[Sequence[float], float, None] = None, + ) -> None: + """ + Flags values that are either + below or above the `low_quantile` and `high_quantile` + quantiles of historical data, respectively. + + If a single value is provided for `low_quantile` or `high_quantile`, this same + value will be used across all components of the series. + + If sequences of values are given for the parameters `low_quantile` and/or `high_quantile`, + they must be of the same length, matching the dimensionality of the series passed + to ``fit()``, or have a length of 1. In the latter case, this single value will be used + across all components of the series. + + If either `low_quantile` or `high_quantile` is None, the corresponding bound will not be used. + However, at least one of the two must be set. + + Parameters + ---------- + low_quantile + (Sequence of) quantile of historical data below which a value is regarded as anomaly. + Must be between 0 and 1. If a sequence, must match the dimensionality of the series + this detector is applied to. + high_quantile + (Sequence of) quantile of historical data above which a value is regarded as anomaly. + Must be between 0 and 1. If a sequence, must match the dimensionality of the series + this detector is applied to. + + Attributes + ---------- + low_threshold + The (sequence of) lower quantile values. + high_threshold + The (sequence of) upper quantile values. + """ + + super().__init__() + + raise_if( + low_quantile is None and high_quantile is None, + "At least one parameter must be not None (`low` and `high` are both None).", + ) + + def _prep_quantile(q): + return ( + q.tolist() + if isinstance(q, np.ndarray) + else [q] + if not isinstance(q, Sequence) + else q + ) + + low = _prep_quantile(low_quantile) + high = _prep_quantile(high_quantile) + + for q in (low, high): + raise_if_not( + all([x is None or 0 <= x <= 1 for x in q]), + "Quantiles must be between 0 and 1, or None.", + ) + + self.low_quantile = low * len(high) if len(low) == 1 else low + self.high_quantile = high * len(low) if len(high) == 1 else high + + # the quantiles parameters are now sequences of the same length, + # possibly containing some None values, but at least one non-None value + + # We'll use an inner Threshold detector once the quantiles are fitted + self.detector = None + + # A few more checks: + raise_if_not( + len(self.low_quantile) == len(self.high_quantile), + "Parameters `low_quantile` and `high_quantile` must be of the same length," + + f" found `low`: {len(self.low_quantile)} and `high`: {len(self.high_quantile)}.", + ) + + raise_if( + all([lo is None for lo in self.low_quantile]) + and all([hi is None for hi in self.high_quantile]), + "All provided quantile values are None.", + ) + + raise_if_not( + all( + [ + l < h + for (l, h) in zip(self.low_quantile, self.high_quantile) + if ((l is not None) and (h is not None)) + ] + ), + "all values in `low_quantile` must be lower than their corresponding value in `high_quantile`.", + ) + + def _fit_core(self, list_series: Sequence[TimeSeries]) -> None: + + # if len(low) > 1 and len(high) > 1, then check it matches input width: + raise_if( + len(self.low_quantile) > 1 + and len(self.low_quantile) != list_series[0].width, + "The number of components of input must be equal to the number" + + " of values given for `high_quantile` or/and `low_quantile`. Found number of " + + f"components equal to {list_series[0].width} and expected {len(self.low_quantile)}.", + ) + + # otherwise, make them the right length + self.low_quantile = ( + self.low_quantile * list_series[0].width + if len(self.low_quantile) == 1 + else self.low_quantile + ) + self.high_quantile = ( + self.high_quantile * list_series[0].width + if len(self.high_quantile) == 1 + else self.high_quantile + ) + + # concatenate everything along time axis + np_series = np.concatenate( + [series.all_values(copy=False) for series in list_series], axis=0 + ) + + # move sample dimension to position 1 + np_series = np.moveaxis(np_series, 2, 1) + + # flatten it in order to obtain an array of shape (time * samples, components) + # where all samples of a given component are concatenated along time + np_series = np_series.reshape(np_series.shape[0] * np_series.shape[1], -1) + + # Compute 2 thresholds (low, high) for each component: + # TODO: we could make this more efficient when low_quantile or high_quantile contain a single value + self.low_threshold = [ + np.quantile(np_series[:, i], q=lo, axis=0) if lo is not None else None + for i, lo in enumerate(self.low_quantile) + ] + self.high_threshold = [ + np.quantile(np_series[:, i], q=hi, axis=0) if hi is not None else None + for i, hi in enumerate(self.high_quantile) + ] + + self.detector = ThresholdDetector( + low_threshold=self.low_threshold, high_threshold=self.high_threshold + ) + + return self + + def _detect_core(self, series: TimeSeries) -> TimeSeries: + return self.detector.detect(series) diff --git a/darts/ad/detectors/threshold_detector.py b/darts/ad/detectors/threshold_detector.py new file mode 100644 index 0000000000..3ab77997bf --- /dev/null +++ b/darts/ad/detectors/threshold_detector.py @@ -0,0 +1,144 @@ +""" +Threshold Detector +------------------ + +Detector that detects anomaly based on user-given threshold. +This detector compares time series values with user-given thresholds, and +identifies time points as anomalous when values are beyond the thresholds. +""" + +from typing import Sequence, Union + +import numpy as np + +from darts.ad.detectors.detectors import Detector +from darts.logging import raise_if, raise_if_not +from darts.timeseries import TimeSeries + + +class ThresholdDetector(Detector): + def __init__( + self, + low_threshold: Union[int, float, Sequence[float], None] = None, + high_threshold: Union[int, float, Sequence[float], None] = None, + ) -> None: + """ + Flags values that are either below or above the `low_threshold` and `high_threshold`, + respectively. + + If a single value is provided for `low_threshold` or `high_threshold`, this same + value will be used across all components of the series. + + If sequences of values are given for the parameters `low_threshold` and/or `high_threshold`, + they must be of the same length, matching the dimensionality of the series passed + to ``detect()``, or have a length of 1. In the latter case, this single value will be used + across all components of the series. + + If either `low_threshold` or `high_threshold` is None, the corresponding bound will not be used. + However, at least one of the two must be set. + + Parameters + ---------- + low_threshold + (Sequence of) lower bounds. + If a sequence, must match the dimensionality of the series + this detector is applied to. + high_threshold + (Sequence of) upper bounds. + If a sequence, must match the dimensionality of the series + this detector is applied to. + """ + + # TODO: could we refactor some code common between ThresholdDetector and QuantileDetector? + + super().__init__() + + raise_if( + low_threshold is None and high_threshold is None, + "At least one parameter must be not None (`low` and `high` are both None).", + ) + + def _prep_thresholds(q): + return ( + q.tolist() + if isinstance(q, np.ndarray) + else [q] + if not isinstance(q, Sequence) + else q + ) + + low = _prep_thresholds(low_threshold) + high = _prep_thresholds(high_threshold) + + self.low_threshold = low * len(high) if len(low) == 1 else low + self.high_threshold = high * len(low) if len(high) == 1 else high + + # the threshold parameters are now sequences of the same length, + # possibly containing some None values, but at least one non-None value + + raise_if_not( + len(self.low_threshold) == len(self.high_threshold), + "Parameters `low_threshold` and `high_threshold` must be of the same length," + + f" found `low`: {len(self.low_threshold)} and `high`: {len(self.high_threshold)}.", + ) + + raise_if( + all([lo is None for lo in self.low_threshold]) + and all([hi is None for hi in self.high_threshold]), + "All provided threshold values are None.", + ) + + raise_if_not( + all( + [ + l < h + for (l, h) in zip(self.low_threshold, self.high_threshold) + if ((l is not None) and (h is not None)) + ] + ), + "all values in `low_threshold` must be lower than their corresponding value in `high_threshold`.", + ) + + def _detect_core(self, series: TimeSeries) -> TimeSeries: + raise_if_not( + series.is_deterministic, "This detector only works on deterministic series." + ) + + raise_if( + len(self.low_threshold) > 1 and len(self.low_threshold) != series.width, + "The number of components of input must be equal to the number" + + " of threshold values. Found number of " + + f"components equal to {series.width} and expected {len(self.low_threshold)}.", + ) + + # if length is 1, tile it to series width: + low_threshold = ( + self.low_threshold * series.width + if len(self.low_threshold) == 1 + else self.low_threshold + ) + high_threshold = ( + self.high_threshold * series.width + if len(self.high_threshold) == 1 + else self.high_threshold + ) + + # (time, components) + np_series = series.all_values(copy=False).squeeze(-1) + + def _detect_fn(x, lo, hi): + # x of shape (time,) for 1 component + return (x < (np.NINF if lo is None else lo)) | ( + x > (np.Inf if hi is None else hi) + ) + + detected = np.zeros_like(np_series, dtype=int) + + for component_idx in range(series.width): + detected[:, component_idx] = _detect_fn( + np_series[:, component_idx], + low_threshold[component_idx], + high_threshold[component_idx], + ) + + return TimeSeries.from_times_and_values(series.time_index, detected) diff --git a/darts/ad/scorers/__init__.py b/darts/ad/scorers/__init__.py new file mode 100644 index 0000000000..b0eec1298d --- /dev/null +++ b/darts/ad/scorers/__init__.py @@ -0,0 +1,79 @@ +""" +Anomaly Scorers +--------------- + +Scorers are at the core of the anomaly detection module. They +produce anomaly scores time series, either for series directly (``score()``), +or for series accompanied by some predictions (``score_from_prediction()``). + +The higher an anomaly score is, the more "anomalous" the corresponding +time period is. Scorers can work over time windows, and the length of the window is related +to the time scale over which anomalies are expected to occur. +The interpretability of the anomaly score is dependent on the scorer. + +The function ``score_from_prediction()`` works by taking some "difference" (or "residual") +between the prediction and the actual series (captured by the ``"diff_fn"`` parameter). +Some scorers are trainable (e.g., ``KMeansScorer``, which learns clusters over historical data), +in which case the ``score()`` function can be used to score new series. +Other scorers are not trainable (e.g., ``NormScorer``, which simply takes the Lp-norm between +predicted values and actual values over windows). In this latter case ``score()`` cannot be +used and scoring is only possible using ``score_from_prediction()``. + +Some scorers can handle probabilistic predictions from models (at the moment all the "NLL" scorers), +while others handle deterministic predictions (e.g., ``KMeansScorer``). + +As an example, the ``KMeansScorer``, which is trainable, can be applied using the functions: + +- ``fit()`` and ``score()``: directly on a series to uncover the relationships between the different + dimensions (over timesteps within windows and/or over dimensions of multivariate series). +- ``fit_from_prediction`` and ``score_from_prediction``: which will compute a difference (residuals) + between the prediction (coming e.g., from a forecasting model) and the series itself. + When scoring, the scorer will attribute a higher score to residuals that are distant + from the clusters found during the training phase. + +Note that `Anomaly Models `_ +can be used to conveniently combine any of Darts forecasting and filtering models with one or multiple scorers. + +Most of the scorers have the following main parameters: + +- `window`: + Integer value indicating the size of the window W used by the scorer to transform the series into + an anomaly score. A scorer will slice the given series into subsequences of size W and returns + a value indicating how anomalous these subset of W values are. The window size should be commensurate + to the expected durations of the anomalies one is looking for. +- `component_wise`: + boolean parameter indicating how the scorer should behave with multivariate series. If set to + True, the model will treat each series dimension independently. If set to False, the model will + consider the dimensions jointly in the considered `window` W to compute the score. + + +Other useful functions are: + +- ``eval_accuracy_from_prediction()`` + Takes as input two (sequence of) series, computes all the anomaly scores, and + returns the value of an agnostic threshold metric (AUC-ROC or AUC-PR) based on some known ground truth + of anomalies. The returned value is between 0 and 1, with 1 indicating that the scorer could perfectly + separate the anomalous point from the normal ones. + +- ``fit_from_prediction()`` + Takes two (sequence of) series as input and fits the scorer. This task is dependent on the scorer, + but as a general case the scorer will calibrate its scoring function based on the training series that is + considered to be anomaly-free. This training phase will allow the scorer to detect anomalies during + the scoring phase, by comparing the series to score with the anomaly-free series seen during training. + + +More details can be found in the API documentation of each scorer. +""" + +from .difference_scorer import DifferenceScorer +from .kmeans_scorer import KMeansScorer +from .nll_cauchy_scorer import CauchyNLLScorer +from .nll_exponential_scorer import ExponentialNLLScorer +from .nll_gamma_scorer import GammaNLLScorer +from .nll_gaussian_scorer import GaussianNLLScorer +from .nll_laplace_scorer import LaplaceNLLScorer +from .nll_poisson_scorer import PoissonNLLScorer +from .norm_scorer import NormScorer +from .pyod_scorer import PyODScorer +from .scorers import FittableAnomalyScorer, NonFittableAnomalyScorer +from .wasserstein_scorer import WassersteinScorer diff --git a/darts/ad/scorers/difference_scorer.py b/darts/ad/scorers/difference_scorer.py new file mode 100644 index 0000000000..191f7254b7 --- /dev/null +++ b/darts/ad/scorers/difference_scorer.py @@ -0,0 +1,28 @@ +""" +Difference Scorer +----------------- + +This scorer simply computes the elementwise difference +between two series. If the two series are multivariate, it +returns a multivariate series. +""" + +from darts.ad.scorers.scorers import NonFittableAnomalyScorer +from darts.timeseries import TimeSeries + + +class DifferenceScorer(NonFittableAnomalyScorer): + def __init__(self) -> None: + super().__init__(univariate_scorer=False, window=1) + + def __str__(self): + return "Difference" + + def _score_core_from_prediction( + self, + actual_series: TimeSeries, + pred_series: TimeSeries, + ) -> TimeSeries: + self._assert_deterministic(actual_series, "actual_series") + self._assert_deterministic(pred_series, "pred_series") + return actual_series - pred_series diff --git a/darts/ad/scorers/kmeans_scorer.py b/darts/ad/scorers/kmeans_scorer.py new file mode 100644 index 0000000000..fb25fc5007 --- /dev/null +++ b/darts/ad/scorers/kmeans_scorer.py @@ -0,0 +1,201 @@ +""" +k-means Scorer +-------------- + +`k`-means Scorer implementing `k`-means clustering [1]_. + +References +---------- +.. [1] https://en.wikipedia.org/wiki/K-means_clustering +""" + +from typing import Sequence + +import numpy as np +from numpy.lib.stride_tricks import sliding_window_view +from sklearn.cluster import KMeans + +from darts.ad.scorers.scorers import FittableAnomalyScorer +from darts.logging import raise_if_not +from darts.timeseries import TimeSeries + + +class KMeansScorer(FittableAnomalyScorer): + def __init__( + self, + window: int = 1, + k: int = 8, + component_wise: bool = False, + diff_fn="abs_diff", + **kwargs, + ) -> None: + """ + When calling ``fit(series)``, a moving window is applied, which results in a set of vectors of size `W`, + where `W` is the window size. The `k`-means model is trained on these vectors. The ``score(series)`` function + applies the same moving window and returns the distance to the closest of the `k` centroids for each + vector of size `W`. + + Alternatively, the scorer has the functions ``fit_from_prediction()`` and ``score_from_prediction()``. + Both require two series (actual and prediction), and compute a "difference" series by applying the + function ``diff_fn`` (default: absolute difference). The resulting series is then passed to the + functions ``fit()`` and ``score()``, respectively. + + `component_wise` is a boolean parameter indicating how the model should behave with multivariate inputs + series. If set to True, the model will treat each component independently by fitting a different + `k`-means model for each dimension. If set to False, the model concatenates the dimensions in + each windows of length `W` and computes the score using only one underlying `k`-means model. + + **Training with** ``fit()``: + + The input can be a series (univariate or multivariate) or multiple series. The series will be sliced + into equal size subsequences. The subsequence will be of size `W` * `D`, with: + + * `W` being the size of the window given as a parameter `window` + * `D` being the dimension of the series (`D` = 1 if univariate or if `component_wise` is set to True) + + For a series of length `N`, (`N` - `W` + 1)/W subsequences will be generated. If a list of series is given + of length L, each series will be partitioned into subsequences, and the results will be concatenated into + an array of length L * number of subsequences of each series. + + The `k`-means model will be fitted on the generated subsequences. The model will find `k` clusters + in the vector space of dimension equal to the length of the subsequences (`D` * `W`). + + If `component_wise` is set to True, the algorithm will be applied to each dimension independently. For each + dimension, a `k`-means model will be trained. + + **Computing score with** ``score()``: + + The input can be a series (univariate or multivariate) or a sequence of series. The given series must have the + same dimension `D` as the data used to train the `k`-means model. + + For each series, if the series is multivariate of dimension `D`: + + * if `component_wise` is set to False: it returns a univariate series (dimension=1). It represents + the anomaly score of the entire series in the considered window at each timestamp. + * if `component_wise` is set to True: it returns a multivariate series of dimension `D`. Each dimension + represents the anomaly score of the corresponding component of the input. + + If the series is univariate, it returns a univariate series regardless of the parameter + `component_wise`. + + A window of size `W` is rolled on the series with a stride equal to 1. It is the same size window `W` used + during the training phase. + Each value in the score series thus represents how anomalous the sample of the `W` previous values is. + + Parameters + ---------- + window + Size of the window used to create the subsequences of the series. + k + The number of clusters to form as well as the number of centroids to generate by the KMeans model. + diff_fn + Optionally, reduction function to use if two series are given. It will transform the two series into one. + This allows the KMeansScorer to apply KMeans on the original series or on its residuals (difference + between the prediction and the original series). + Must be one of "abs_diff" and "diff" (defined in ``_diff_series()``). + Default: "abs_diff" + component_wise + Boolean value indicating if the score needs to be computed for each component independently (True) + or by concatenating the component in the considered window to compute one score (False). + Default: False + kwargs + Additional keyword arguments passed to the internal scikit-learn KMeans model(s). + """ + + raise_if_not( + type(component_wise) is bool, + f"Parameter `component_wise` must be Boolean, found type: {type(component_wise)}.", + ) + self.component_wise = component_wise + + self.kmeans_kwargs = kwargs + self.kmeans_kwargs["n_clusters"] = k + + super().__init__( + univariate_scorer=(not component_wise), window=window, diff_fn=diff_fn + ) + + def __str__(self): + return "k-means Scorer" + + def _fit_core( + self, + list_series: Sequence[TimeSeries], + ): + + list_np_series = [series.all_values(copy=False) for series in list_series] + + if not self.component_wise: + self.model = KMeans(**self.kmeans_kwargs) + self.model.fit( + np.concatenate( + [ + sliding_window_view(ar, window_shape=self.window, axis=0) + .transpose(0, 3, 1, 2) + .reshape(-1, self.window * len(ar[0])) + for ar in list_np_series + ], + axis=0, + ) + ) + else: + models = [] + for component_idx in range(self.width_trained_on): + model = KMeans(**self.kmeans_kwargs) + model.fit( + np.concatenate( + [ + sliding_window_view( + ar[:, component_idx], window_shape=self.window, axis=0 + ) + .transpose(0, 2, 1) + .reshape(-1, self.window) + for ar in list_np_series + ], + axis=0, + ) + ) + models.append(model) + self.models = models + + def _score_core(self, series: TimeSeries) -> TimeSeries: + raise_if_not( + self.width_trained_on == series.width, + "Input must have the same number of components as the data used for" + + " training the KMeans model, found number of components equal to" + + f" {series.width} and expected {self.width_trained_on}.", + ) + + np_series = series.all_values(copy=False) + np_anomaly_score = [] + + if not self.component_wise: + # return distance to the clostest centroid + np_anomaly_score.append( + self.model.transform( + sliding_window_view(np_series, window_shape=self.window, axis=0) + .transpose(0, 3, 1, 2) + .reshape(-1, self.window * series.width) + ).min(axis=1) + ) # only return the closest distance out of the k ones (k centroids) + else: + for component_idx in range(self.width_trained_on): + score = ( + self.models[component_idx] + .transform( + sliding_window_view( + np_series[:, component_idx], + window_shape=self.window, + axis=0, + ) + .transpose(0, 2, 1) + .reshape(-1, self.window) + ) + .min(axis=1) + ) + + np_anomaly_score.append(score) + + return TimeSeries.from_times_and_values( + series.time_index[self.window - 1 :], list(zip(*np_anomaly_score)) + ) diff --git a/darts/ad/scorers/nll_cauchy_scorer.py b/darts/ad/scorers/nll_cauchy_scorer.py new file mode 100644 index 0000000000..6ef9754fe2 --- /dev/null +++ b/darts/ad/scorers/nll_cauchy_scorer.py @@ -0,0 +1,33 @@ +""" +NLL Cauchy Scorer +----------------- + +Cauchy distribution negative log-likelihood Scorer. + +The anomaly score is the negative log likelihood of the actual series values +under a Cauchy distribution estimated from the stochastic prediction. +""" + +import numpy as np +from scipy.stats import cauchy + +from darts.ad.scorers.scorers import NLLScorer + + +class CauchyNLLScorer(NLLScorer): + def __init__(self, window: int = 1) -> None: + super().__init__(window=window) + + def __str__(self): + return "CauchyNLLScorer" + + def _score_core_nllikelihood( + self, + deterministic_values: np.ndarray, + probabilistic_estimations: np.ndarray, + ) -> np.ndarray: + + params = np.apply_along_axis(cauchy.fit, axis=1, arr=probabilistic_estimations) + return -cauchy.logpdf( + deterministic_values, loc=params[:, 0], scale=params[:, 1] + ) diff --git a/darts/ad/scorers/nll_exponential_scorer.py b/darts/ad/scorers/nll_exponential_scorer.py new file mode 100644 index 0000000000..1a16894347 --- /dev/null +++ b/darts/ad/scorers/nll_exponential_scorer.py @@ -0,0 +1,37 @@ +""" +NLL Exponential Scorer +---------------------- + +Exponential distribution negative log-likelihood Scorer. + +The anomaly score is the negative log likelihood of the actual series values +under an Exponential distribution estimated from the stochastic prediction. +""" + +import numpy as np +from scipy.stats import expon + +from darts.ad.scorers.scorers import NLLScorer + + +class ExponentialNLLScorer(NLLScorer): + def __init__(self, window: int = 1) -> None: + super().__init__(window=window) + + def __str__(self): + return "ExponentialNLLScorer" + + def _score_core_nllikelihood( + self, + deterministic_values: np.ndarray, + probabilistic_estimations: np.ndarray, + ) -> np.ndarray: + + # This is the ML estimate for 1/lambda, which is what scipy expects as scale. + mu = np.mean(probabilistic_estimations, axis=1) + + # This is ML estimate for the loc - see: + # https://github.com/scipy/scipy/blob/de80faf9d3480b9dbb9b888568b64499e0e70c19/scipy/stats/_continuous_distns.py#L1705 + loc = np.min(probabilistic_estimations, axis=1) + + return -expon.logpdf(deterministic_values, scale=mu, loc=loc) diff --git a/darts/ad/scorers/nll_gamma_scorer.py b/darts/ad/scorers/nll_gamma_scorer.py new file mode 100644 index 0000000000..40dc113c3c --- /dev/null +++ b/darts/ad/scorers/nll_gamma_scorer.py @@ -0,0 +1,33 @@ +""" +NLL Gamma Scorer +---------------- + +Gamma distribution negative log-likelihood Scorer. + +The anomaly score is the negative log likelihood of the actual series values +under a Gamma distribution estimated from the stochastic prediction. +""" + +import numpy as np +from scipy.stats import gamma + +from darts.ad.scorers.scorers import NLLScorer + + +class GammaNLLScorer(NLLScorer): + def __init__(self, window: int = 1) -> None: + super().__init__(window=window) + + def __str__(self): + return "GammaNLLScorer" + + def _score_core_nllikelihood( + self, + deterministic_values: np.ndarray, + probabilistic_estimations: np.ndarray, + ) -> np.ndarray: + + params = np.apply_along_axis(gamma.fit, axis=1, arr=probabilistic_estimations) + return -gamma.logpdf( + deterministic_values, a=params[:, 0], loc=params[:, 1], scale=params[:, 2] + ) diff --git a/darts/ad/scorers/nll_gaussian_scorer.py b/darts/ad/scorers/nll_gaussian_scorer.py new file mode 100644 index 0000000000..56eb86300b --- /dev/null +++ b/darts/ad/scorers/nll_gaussian_scorer.py @@ -0,0 +1,32 @@ +""" +NLL Gaussian Scorer +------------------- + +Gaussian negative log-likelihood Scorer. + +The anomaly score is the negative log likelihood of the actual series values +under a Gaussian distribution estimated from the stochastic predictions. +""" + +import numpy as np +from scipy.stats import norm + +from darts.ad.scorers.scorers import NLLScorer + + +class GaussianNLLScorer(NLLScorer): + def __init__(self, window: int = 1) -> None: + super().__init__(window=window) + + def __str__(self): + return "GaussianNLLScorer" + + def _score_core_nllikelihood( + self, + deterministic_values: np.ndarray, + probabilistic_estimations: np.ndarray, + ) -> np.ndarray: + + mu = np.mean(probabilistic_estimations, axis=1) + std = np.std(probabilistic_estimations, axis=1) + return -norm.logpdf(deterministic_values, loc=mu, scale=std) diff --git a/darts/ad/scorers/nll_laplace_scorer.py b/darts/ad/scorers/nll_laplace_scorer.py new file mode 100644 index 0000000000..342dab53ef --- /dev/null +++ b/darts/ad/scorers/nll_laplace_scorer.py @@ -0,0 +1,41 @@ +""" +NLL Laplace Scorer +------------------ + +Laplace distribution negative log-likelihood Scorer. + +The anomaly score is the negative log likelihood of the actual series values +under a Laplace distribution estimated from the stochastic prediction. +""" + +import numpy as np +from scipy.stats import laplace + +from darts.ad.scorers.scorers import NLLScorer + + +class LaplaceNLLScorer(NLLScorer): + def __init__(self, window: int = 1) -> None: + super().__init__(window=window) + + def __str__(self): + return "LaplaceNLLScorer" + + def _score_core_nllikelihood( + self, + deterministic_values: np.ndarray, + probabilistic_estimations: np.ndarray, + ) -> np.ndarray: + + # ML estimate for the Laplace loc + loc = np.median(probabilistic_estimations, axis=1) + + # ML estimate for the Laplace scale + # see: https://github.com/scipy/scipy/blob/de80faf9d3480b9dbb9b888568b64499e0e70c19/scipy + # /stats/_continuous_distns.py#L4846 + scale = ( + np.sum(np.abs(probabilistic_estimations.T - loc), axis=0).T + / probabilistic_estimations.shape[1] + ) + + return -laplace.logpdf(deterministic_values, loc=loc, scale=scale) diff --git a/darts/ad/scorers/nll_poisson_scorer.py b/darts/ad/scorers/nll_poisson_scorer.py new file mode 100644 index 0000000000..df5ee411b8 --- /dev/null +++ b/darts/ad/scorers/nll_poisson_scorer.py @@ -0,0 +1,31 @@ +""" +NLL Poisson Scorer +------------------ + +Poisson distribution negative log-likelihood Scorer. + +The anomaly score is the negative log likelihood of the actual series values +under a Poisson distribution estimated from the stochastic prediction. +""" + +import numpy as np +from scipy.stats import poisson + +from darts.ad.scorers.scorers import NLLScorer + + +class PoissonNLLScorer(NLLScorer): + def __init__(self, window: int = 1) -> None: + super().__init__(window=window) + + def __str__(self): + return "PoissonNLLScorer" + + def _score_core_nllikelihood( + self, + deterministic_values: np.ndarray, + probabilistic_estimations: np.ndarray, + ) -> np.ndarray: + + mu = np.mean(probabilistic_estimations, axis=1) + return -poisson.logpmf(deterministic_values, mu=mu) diff --git a/darts/ad/scorers/norm_scorer.py b/darts/ad/scorers/norm_scorer.py new file mode 100644 index 0000000000..6764960994 --- /dev/null +++ b/darts/ad/scorers/norm_scorer.py @@ -0,0 +1,82 @@ +""" +Norm Scorer +----------- + +Norm anomaly score (of given order) [1]_. + +References +---------- +.. [1] https://en.wikipedia.org/wiki/Norm_(mathematics) +""" + +import numpy as np + +from darts.ad.scorers.scorers import NonFittableAnomalyScorer +from darts.logging import raise_if_not +from darts.timeseries import TimeSeries + + +class NormScorer(NonFittableAnomalyScorer): + def __init__(self, ord=None, component_wise: bool = False) -> None: + """ + Returns the elementwise norm of a given order between two series' values. + + If `component_wise` is False, the norm is computed between vectors + made of the series' components (one norm per timestamp). + + If `component_wise` is True, for any `ord` this effectively amounts to computing the absolute + value of the difference. + + The scoring function expects two series. + + If the two series are multivariate of width `w`: + + * if `component_wise` is set to False: it returns a univariate series (width=1). + * if `component_wise` is set to True: it returns a multivariate series of width `w`. + + If the two series are univariate, it returns a univariate series regardless of the parameter + `component_wise`. + + Parameters + ---------- + ord + Order of the norm. Options are listed under 'Notes' at: + . + Default: None + component_wise + Whether to compare components of the two series in isolation (True), or jointly (False). + Default: False + """ + + raise_if_not( + type(component_wise) is bool, + f"`component_wise` must be Boolean, found type: {type(component_wise)}.", + ) + + self.ord = ord + self.component_wise = component_wise + super().__init__(univariate_scorer=(not component_wise), window=1) + + def __str__(self): + return f"Norm (ord={self.ord})" + + def _score_core_from_prediction( + self, + actual_series: TimeSeries, + pred_series: TimeSeries, + ) -> TimeSeries: + + self._assert_deterministic(actual_series, "actual_series") + self._assert_deterministic(pred_series, "pred_series") + + diff = actual_series - pred_series + + if self.component_wise: + return diff.map(lambda x: np.abs(x)) + + else: + diff_np = diff.all_values(copy=False) + + return TimeSeries.from_times_and_values( + diff.time_index, np.linalg.norm(diff_np, ord=self.ord, axis=1) + ) diff --git a/darts/ad/scorers/pyod_scorer.py b/darts/ad/scorers/pyod_scorer.py new file mode 100644 index 0000000000..0a90235bd2 --- /dev/null +++ b/darts/ad/scorers/pyod_scorer.py @@ -0,0 +1,194 @@ +""" +PyODScorer +----- + +This scorer can wrap around detection algorithms of PyOD. +`PyOD https://pyod.readthedocs.io/en/latest/#`_. +""" + +from typing import Sequence + +import numpy as np +from numpy.lib.stride_tricks import sliding_window_view +from pyod.models.base import BaseDetector + +from darts.ad.scorers.scorers import FittableAnomalyScorer +from darts.logging import get_logger, raise_if_not +from darts.timeseries import TimeSeries + +logger = get_logger(__name__) + + +class PyODScorer(FittableAnomalyScorer): + def __init__( + self, + model: BaseDetector, + window: int = 1, + component_wise: bool = False, + diff_fn="abs_diff", + ) -> None: + """ + When calling ``fit(series)``, a moving window is applied, which results in a set of vectors of size `W`, + where `W` is the window size. The PyODScorer model is trained on these vectors. The ``score(series)`` + function will apply the same moving window and return the predicted raw anomaly score of each vector. + + Alternatively, the scorer has the functions ``fit_from_prediction()`` and ``score_from_prediction()``. + Both require two series (actual and prediction), and compute a "difference" series by applying the + function ``diff_fn`` (default: absolute difference). The resulting series is then passed to the + functions ``fit()`` and ``score()``, respectively. + + `component_wise` is a boolean parameter indicating how the model should behave with multivariate inputs + series. If set to True, the model will treat each series dimension independently by fitting a different + PyODScorer model for each dimension. If set to False, the model concatenates the dimensions in + each windows of length `W` and compute the score using only one underlying PyODScorer model. + + **Training with** ``fit()``: + + The input can be a series (univariate or multivariate) or multiple series. The series will be partitioned + into equal size subsequences. The subsequence will be of size `W` * `D`, with: + + * `W` being the size of the window given as a parameter `window` + * `D` being the dimension of the series (`D` = 1 if univariate or if `component_wise` is set to True) + + For a series of length `N`, (`N` - `W` + 1)/W subsequences will be generated. If a list of series is given + of length L, each series will be partitioned into subsequences, and the results will be concatenated into + an array of length L * number of subsequences of each series. + + The PyOD model will be fitted on the generated subsequences. + + If `component_wise` is set to True, the algorithm will be applied to each dimension independently. For each + dimension, a PyOD model will be trained. + + **Computing score with** ``score()``: + + The input can be a series (univariate or multivariate) or a sequence of series. The given series must have the + same dimension `D` as the data used to train the PyOD model. + + For each series, if the series is multivariate of dimension `D`: + + * if `component_wise` is set to False: it returns a univariate series (dimension=1). It represents + the anomaly score of the entire series in the considered window at each timestamp. + * if `component_wise` is set to True: it returns a multivariate series of dimension `D`. Each dimension + represents the anomaly score of the corresponding component of the input. + + If the series is univariate, it returns a univariate series regardless of the parameter + `component_wise`. + + A window of size `W` is rolled on the series with a stride equal to 1. It is the same size window `W` used + during the training phase. + Each value in the score series thus represents how anomalous the sample of the `W` previous values is. + + Parameters + ---------- + model + The (fitted) PyOD BaseDetector model. + window + Size of the window used to create the subsequences of the series. + diff_fn + Optionally, reduced function to use if two series are given. It will transform the two series into one. + This allows the KMeansScorer to apply PyODScorer on the original series or on its residuals (difference + between the prediction and the original series). + Must be one of "abs_diff" and "diff" (defined in ``_diff_series()``). + Default: "abs_diff" + component_wise + Boolean value indicating if the score needs to be computed for each component independently (True) + or by concatenating the component in the considered window to compute one score (False). + Default: False + """ + + raise_if_not( + isinstance(model, BaseDetector), + f"model must be a PyOD BaseDetector, found type: {type(model)}", + ) + self.model = model + + raise_if_not( + type(component_wise) is bool, + f"Parameter `component_wise` must be Boolean, found type: {type(component_wise)}.", + ) + self.component_wise = component_wise + + super().__init__( + univariate_scorer=(not component_wise), window=window, diff_fn=diff_fn + ) + + def __str__(self): + return "PyODScorer (model {})".format(self.model.__str__().split("(")[0]) + + def _fit_core(self, list_series: Sequence[TimeSeries]): + + list_np_series = [series.all_values(copy=False) for series in list_series] + + # TODO: can we factorize code in common bteween PyODScorer and KMeansScorer? + + if not self.component_wise: + self.model.fit( + np.concatenate( + [ + sliding_window_view(ar, window_shape=self.window, axis=0) + .transpose(0, 3, 1, 2) + .reshape(-1, self.window * len(ar[0])) + for ar in list_np_series + ] + ) + ) + else: + models = [] + for component_idx in range(self.width_trained_on): + + model_width = self.model + model_width.fit( + np.concatenate( + [ + sliding_window_view( + ar[:, component_idx], window_shape=self.window, axis=0 + ) + .transpose(0, 2, 1) + .reshape(-1, self.window) + for ar in list_np_series + ] + ) + ) + models.append(model_width) + self.models = models + + def _score_core(self, series: TimeSeries) -> TimeSeries: + + raise_if_not( + self.width_trained_on == series.width, + "Input must have the same number of components as the data used for training" + + " the PyODScorer model {},".format(self.model.__str__().split("(")[0]) + + f" found number of components equal to {series.width} and expected " + + f"{self.width_trained_on}.", + ) + + np_series = series.all_values(copy=False) + np_anomaly_score = [] + + if not self.component_wise: + + np_anomaly_score.append( + self.model.decision_function( + sliding_window_view(np_series, window_shape=self.window, axis=0) + .transpose(0, 3, 1, 2) + .reshape(-1, self.window * series.width) + ) + ) + else: + + for component_idx in range(self.width_trained_on): + score = self.models[component_idx].decision_function( + sliding_window_view( + np_series[:, component_idx], + window_shape=self.window, + axis=0, + ) + .transpose(0, 2, 1) + .reshape(-1, self.window) + ) + + np_anomaly_score.append(score) + + return TimeSeries.from_times_and_values( + series.time_index[self.window - 1 :], list(zip(*np_anomaly_score)) + ) diff --git a/darts/ad/scorers/scorers.py b/darts/ad/scorers/scorers.py new file mode 100644 index 0000000000..de2f878aab --- /dev/null +++ b/darts/ad/scorers/scorers.py @@ -0,0 +1,754 @@ +""" +Scorers Base Classes +""" + +# TODO: +# - add stride for Scorers like Kmeans and Wasserstein +# - add option to normalize the windows for kmeans? capture only the form and not the values. + + +from abc import ABC, abstractmethod +from typing import Any, Sequence, Union + +import numpy as np + +from darts import TimeSeries +from darts.ad.utils import ( + _assert_same_length, + _assert_timeseries, + _intersect, + _sanity_check_two_series, + _to_list, + eval_accuracy_from_scores, + show_anomalies_from_scores, +) +from darts.logging import get_logger, raise_if_not + +logger = get_logger(__name__) + + +class AnomalyScorer(ABC): + """Base class for all anomaly scorers""" + + def __init__(self, univariate_scorer: bool, window: int) -> None: + + raise_if_not( + type(window) is int, + f"Parameter `window` must be an integer, found type {type(window)}.", + ) + + raise_if_not( + window > 0, + f"Parameter `window` must be stricly greater than 0, found size {window}.", + ) + + self.window = window + + self.univariate_scorer = univariate_scorer + + def _check_univariate_scorer(self, actual_anomalies: Sequence[TimeSeries]): + """Checks if `actual_anomalies` contains only univariate series when the scorer has the + parameter 'univariate_scorer' set to True. + + 'univariate_scorer' is: + True -> when the function of the scorer ``score(series)`` (or, if applicable, + ``score_from_prediction(actual_series, pred_series)``) returns a univariate + anomaly score regardless of the input `series` (or, if applicable, `actual_series` + and `pred_series`). + False -> when the scorer will return a series that has the + same number of components as the input (can be univariate or multivariate). + """ + + if self.univariate_scorer: + raise_if_not( + all([isinstance(s, TimeSeries) for s in actual_anomalies]), + "all series in `actual_anomalies` must be of type TimeSeries.", + ) + + raise_if_not( + all([s.width == 1 for s in actual_anomalies]), + f"Scorer {self.__str__()} will return a univariate anomaly score series (width=1)." + + " Found a multivariate `actual_anomalies`." + + " The evaluation of the accuracy cannot be computed between the two series.", + ) + + def _check_window_size(self, series: TimeSeries): + """Checks if the parameter window is less or equal than the length of the given series""" + + raise_if_not( + self.window <= len(series), + f"Window size {self.window} is greater than the targeted series length {len(series)}, " + + "must be lower or equal. Decrease the window size or increase the length series input" + + " to score on.", + ) + + @property + def is_probabilistic(self) -> bool: + """Whether the scorer expects a probabilistic prediction for its first input.""" + return False + + def _assert_stochastic(self, series: TimeSeries, name_series: str): + "Checks if the series is stochastic (number of samples is higher than one)." + + raise_if_not( + series.is_stochastic, + f"Scorer {self.__str__()} is expecting `{name_series}` to be a stochastic timeseries" + + f" (number of samples must be higher than 1, found: {series.n_samples}).", + ) + + def _assert_deterministic(self, series: TimeSeries, name_series: str): + "Checks if the series is deterministic (number of samples is equal to one)." + + if not series.is_deterministic: + logger.warning( + f"Scorer {self.__str__()} is expecting `{name_series}` to be a (sequence of) deterministic" + + f" timeseries (number of samples must be equal to 1, found: {series.n_samples}). The " + + "series will be converted to a deterministic series by taking the median of the samples.", + ) + series = series.quantile_timeseries(quantile=0.5) + + return series + + @abstractmethod + def __str__(self): + """returns the name of the scorer""" + pass + + def eval_accuracy_from_prediction( + self, + actual_anomalies: Union[TimeSeries, Sequence[TimeSeries]], + actual_series: Union[TimeSeries, Sequence[TimeSeries]], + pred_series: Union[TimeSeries, Sequence[TimeSeries]], + metric: str = "AUC_ROC", + ) -> Union[float, Sequence[float], Sequence[Sequence[float]]]: + """Computes the anomaly score between `actual_series` and `pred_series`, and returns the score + of an agnostic threshold metric. + + Parameters + ---------- + actual_anomalies + The (sequence of) ground truth of the anomalies (1 if it is an anomaly and 0 if not) + actual_series + The (sequence of) actual series. + pred_series + The (sequence of) predicted series. + metric + Optionally, metric function to use. Must be one of "AUC_ROC" and "AUC_PR". + Default: "AUC_ROC" + + Returns + ------- + Union[float, Sequence[float], Sequence[Sequence[float]]] + Score of an agnostic threshold metric for the computed anomaly score + - ``float`` if `actual_series` and `actual_series` are univariate series (dimension=1). + - ``Sequence[float]`` + + * if `actual_series` and `actual_series` are multivariate series (dimension>1), + returns one value per dimension, or + * if `actual_series` and `actual_series` are sequences of univariate series, + returns one value per series + - ``Sequence[Sequence[float]]]`` if `actual_series` and `actual_series` are sequences + of multivariate series. Outer Sequence is over the sequence input and the inner + Sequence is over the dimensions of each element in the sequence input. + """ + actual_anomalies = _to_list(actual_anomalies) + self._check_univariate_scorer(actual_anomalies) + + anomaly_score = self.score_from_prediction(actual_series, pred_series) + + return eval_accuracy_from_scores( + actual_anomalies, anomaly_score, self.window, metric + ) + + @abstractmethod + def score_from_prediction(self, actual_series: Any, pred_series: Any) -> Any: + pass + + def show_anomalies_from_prediction( + self, + actual_series: TimeSeries, + pred_series: TimeSeries, + scorer_name: str = None, + actual_anomalies: TimeSeries = None, + title: str = None, + metric: str = None, + ): + """Plot the results of the scorer. + + Computes the anomaly score on the two series. And plots the results. + + The plot will be composed of the following: + - the actual_series and the pred_series. + - the anomaly score of the scorer. + - the actual anomalies, if given. + + It is possible to: + - add a title to the figure with the parameter `title` + - give personalized name to the scorer with `scorer_name` + - show the results of a metric for the anomaly score (AUC_ROC or AUC_PR), + if the actual anomalies is provided. + + Parameters + ---------- + actual_series + The actual series to visualize anomalies from. + pred_series + The predicted series of `actual_series`. + actual_anomalies + The ground truth of the anomalies (1 if it is an anomaly and 0 if not) + scorer_name + Name of the scorer. + title + Title of the figure + metric + Optionally, Scoring function to use. Must be one of "AUC_ROC" and "AUC_PR". + Default: "AUC_ROC" + """ + if isinstance(actual_series, Sequence): + raise_if_not( + len(actual_series) == 1, + "``show_anomalies_from_prediction`` expects only one series for `actual_series`," + + f" found a list of length {len(actual_series)} as input.", + ) + + actual_series = actual_series[0] + + raise_if_not( + isinstance(actual_series, TimeSeries), + "``show_anomalies_from_prediction`` expects an input of type TimeSeries," + + f" found type {type(actual_series)} for `actual_series`.", + ) + + if isinstance(pred_series, Sequence): + raise_if_not( + len(pred_series) == 1, + "``show_anomalies_from_prediction`` expects one series for `pred_series`," + + f" found a list of length {len(pred_series)} as input.", + ) + + pred_series = pred_series[0] + + raise_if_not( + isinstance(pred_series, TimeSeries), + "``show_anomalies_from_prediction`` expects an input of type TimeSeries," + + f" found type: {type(pred_series)} for `pred_series`.", + ) + + anomaly_score = self.score_from_prediction(actual_series, pred_series) + + if title is None: + title = f"Anomaly results by scorer {self.__str__()}" + + if scorer_name is None: + scorer_name = [f"anomaly score by {self.__str__()}"] + + return show_anomalies_from_scores( + actual_series, + model_output=pred_series, + anomaly_scores=anomaly_score, + window=self.window, + names_of_scorers=scorer_name, + actual_anomalies=actual_anomalies, + title=title, + metric=metric, + ) + + +class NonFittableAnomalyScorer(AnomalyScorer): + """Base class of anomaly scorers that do not need training.""" + + def __init__(self, univariate_scorer, window) -> None: + super().__init__(univariate_scorer=univariate_scorer, window=window) + + # indicates if the scorer is trainable or not + self.trainable = False + + @abstractmethod + def _score_core_from_prediction(self, series: Any) -> Any: + pass + + def score_from_prediction( + self, + actual_series: Union[TimeSeries, Sequence[TimeSeries]], + pred_series: Union[TimeSeries, Sequence[TimeSeries]], + ) -> Union[TimeSeries, Sequence[TimeSeries]]: + """Computes the anomaly score on the two (sequence of) series. + + If a pair of sequences is given, they must contain the same number + of series. The scorer will score each pair of series independently + and return an anomaly score for each pair. + + Parameters + ---------- + actual_series: + The (sequence of) actual series. + pred_series + The (sequence of) predicted series. + + Returns + ------- + Union[TimeSeries, Sequence[TimeSeries]] + (Sequence of) anomaly score time series + """ + list_actual_series, list_pred_series = _to_list(actual_series), _to_list( + pred_series + ) + _assert_same_length(list_actual_series, list_pred_series) + + anomaly_scores = [] + + for s1, s2 in zip(list_actual_series, list_pred_series): + _sanity_check_two_series(s1, s2) + s1, s2 = _intersect(s1, s2) + self._check_window_size(s1) + self._check_window_size(s2) + anomaly_scores.append(self._score_core_from_prediction(s1, s2)) + + if ( + len(anomaly_scores) == 1 + and not isinstance(pred_series, Sequence) + and not isinstance(actual_series, Sequence) + ): + return anomaly_scores[0] + else: + return anomaly_scores + + +class FittableAnomalyScorer(AnomalyScorer): + """Base class of scorers that do need training.""" + + def __init__(self, univariate_scorer, window, diff_fn="abs_diff") -> None: + super().__init__(univariate_scorer=univariate_scorer, window=window) + + # indicates if the scorer is trainable or not + self.trainable = True + + # indicates if the scorer has been trained yet + self._fit_called = False + + # function used in ._diff_series() to convert 2 time series into 1 + if diff_fn in {"abs_diff", "diff"}: + self.diff_fn = diff_fn + else: + raise ValueError(f"Metric should be 'diff' or 'abs_diff', found {diff_fn}") + + def check_if_fit_called(self): + """Checks if the scorer has been fitted before calling its `score()` function.""" + + raise_if_not( + self._fit_called, + f"The Scorer {self.__str__()} has not been fitted yet. Call ``fit()`` first.", + ) + + def eval_accuracy( + self, + actual_anomalies: Union[TimeSeries, Sequence[TimeSeries]], + series: Union[TimeSeries, Sequence[TimeSeries]], + metric: str = "AUC_ROC", + ) -> Union[float, Sequence[float], Sequence[Sequence[float]]]: + """Computes the anomaly score of the given time series, and returns the score + of an agnostic threshold metric. + + Parameters + ---------- + actual_anomalies + The ground truth of the anomalies (1 if it is an anomaly and 0 if not) + series + The (sequence of) series to detect anomalies from. + metric + Optionally, metric function to use. Must be one of "AUC_ROC" and "AUC_PR". + Default: "AUC_ROC" + + Returns + ------- + Union[float, Sequence[float], Sequence[Sequence[float]]] + Score of an agnostic threshold metric for the computed anomaly score + - ``float`` if `series` is a univariate series (dimension=1). + - ``Sequence[float]`` + + * if `series` is a multivariate series (dimension>1), returns one + value per dimension, or + * if `series` is a sequence of univariate series, returns one value + per series + - ``Sequence[Sequence[float]]]`` if `series` is a sequence of multivariate + series. Outer Sequence is over the sequence input and the inner Sequence + is over the dimensions of each element in the sequence input. + """ + actual_anomalies = _to_list(actual_anomalies) + self._check_univariate_scorer(actual_anomalies) + anomaly_score = self.score(series) + + return eval_accuracy_from_scores( + actual_anomalies, anomaly_score, self.window, metric + ) + + def score( + self, + series: Union[TimeSeries, Sequence[TimeSeries]], + ) -> Union[TimeSeries, Sequence[TimeSeries]]: + """Computes the anomaly score on the given series. + + If a sequence of series is given, the scorer will score each series independently + and return an anomaly score for each series in the sequence. + + Parameters + ---------- + series + The (sequence of) series to detect anomalies from. + + Returns + ------- + Union[TimeSeries, Sequence[TimeSeries]] + (Sequence of) anomaly score time series + """ + + self.check_if_fit_called() + + list_series = _to_list(series) + + anomaly_scores = [] + for s in list_series: + _assert_timeseries(s) + self._check_window_size(s) + anomaly_scores.append( + self._score_core(self._assert_deterministic(s, "series")) + ) + + if len(anomaly_scores) == 1 and not isinstance(series, Sequence): + return anomaly_scores[0] + else: + return anomaly_scores + + def show_anomalies( + self, + series: TimeSeries, + actual_anomalies: TimeSeries = None, + scorer_name: str = None, + title: str = None, + metric: str = None, + ): + """Plot the results of the scorer. + + Computes the score on the given series input. And plots the results. + + The plot will be composed of the following: + - the series itself. + - the anomaly score of the score. + - the actual anomalies, if given. + + It is possible to: + - add a title to the figure with the parameter `title` + - give personalized name to the scorer with `scorer_name` + - show the results of a metric for the anomaly score (AUC_ROC or AUC_PR), + if the actual anomalies is provided. + + Parameters + ---------- + series + The series to visualize anomalies from. + actual_anomalies + The ground truth of the anomalies (1 if it is an anomaly and 0 if not) + scorer_name + Name of the scorer. + title + Title of the figure + metric + Optionally, Scoring function to use. Must be one of "AUC_ROC" and "AUC_PR". + Default: "AUC_ROC" + """ + + if isinstance(series, Sequence): + raise_if_not( + len(series) == 1, + "``show_anomalies`` expects one series for `series`," + + f" found a list of length {len(series)} as input.", + ) + + series = series[0] + + raise_if_not( + isinstance(series, TimeSeries), + "``show_anomalies`` expects an input of type TimeSeries," + + f" found type {type(series)} for `series`.", + ) + + anomaly_score = self.score(series) + + if title is None: + title = f"Anomaly results by scorer {self.__str__()}" + + if scorer_name is None: + scorer_name = f"anomaly score by {self.__str__()}" + + return show_anomalies_from_scores( + series, + anomaly_scores=anomaly_score, + window=self.window, + names_of_scorers=scorer_name, + actual_anomalies=actual_anomalies, + title=title, + metric=metric, + ) + + def score_from_prediction( + self, + actual_series: Union[TimeSeries, Sequence[TimeSeries]], + pred_series: Union[TimeSeries, Sequence[TimeSeries]], + ) -> Union[TimeSeries, Sequence[TimeSeries]]: + """Computes the anomaly score on the two (sequence of) series. + + The function ``diff_fn`` passed as a parameter to the scorer, will transform `pred_series` and `actual_series` + into one "difference" series. By default, ``diff_fn`` will compute the absolute difference + (Default: "abs_diff"). + If actual_series and pred_series are sequences, ``diff_fn`` will be applied to all pairwise elements + of the sequences. + + The scorer will then transform this series into an anomaly score. If a sequence of series is given, + the scorer will score each series independently and return an anomaly score for each series in the sequence. + + Parameters + ---------- + actual_series + The (sequence of) actual series. + pred_series + The (sequence of) predicted series. + + Returns + ------- + Union[TimeSeries, Sequence[TimeSeries]] + (Sequence of) anomaly score time series + """ + + self.check_if_fit_called() + + list_actual_series, list_pred_series = _to_list(actual_series), _to_list( + pred_series + ) + _assert_same_length(list_actual_series, list_pred_series) + + anomaly_scores = [] + for (s1, s2) in zip(list_actual_series, list_pred_series): + _sanity_check_two_series(s1, s2) + s1 = self._assert_deterministic(s1, "actual_series") + s2 = self._assert_deterministic(s2, "pred_series") + diff = self._diff_series(s1, s2) + self._check_window_size(diff) + anomaly_scores.append(self.score(diff)) + + if ( + len(anomaly_scores) == 1 + and not isinstance(pred_series, Sequence) + and not isinstance(actual_series, Sequence) + ): + return anomaly_scores[0] + else: + return anomaly_scores + + def fit( + self, + series: Union[TimeSeries, Sequence[TimeSeries]], + ): + """Fits the scorer on the given time series input. + + If sequence of series is given, the scorer will be fitted on the concatenation of the sequence. + + The assumption is that the series `series` used for training are generally anomaly-free. + + Parameters + ---------- + series + The (sequence of) series with no anomalies. + + Returns + ------- + self + Fitted Scorer. + """ + list_series = _to_list(series) + + for idx, s in enumerate(list_series): + _assert_timeseries(s) + + if idx == 0: + self.width_trained_on = s.width + else: + raise_if_not( + s.width == self.width_trained_on, + "series in `series` must have the same number of components," + + f" found number of components equal to {self.width_trained_on}" + + f" at index 0 and {s.width} at index {idx}.", + ) + self._check_window_size(s) + + self._assert_deterministic(s, "series") + + self._fit_core(list_series) + self._fit_called = True + + def fit_from_prediction( + self, + actual_series: Union[TimeSeries, Sequence[TimeSeries]], + pred_series: Union[TimeSeries, Sequence[TimeSeries]], + ): + """Fits the scorer on the two (sequence of) series. + + The function ``diff_fn`` passed as a parameter to the scorer, will transform `pred_series` and `actual_series` + into one series. By default, ``diff_fn`` will compute the absolute difference (Default: "abs_diff"). + If `pred_series` and `actual_series` are sequences, ``diff_fn`` will be applied to all pairwise elements + of the sequences. + + The scorer will then be fitted on this (sequence of) series. If a sequence of series is given, + the scorer will be fitted on the concatenation of the sequence. + + The scorer assumes that the (sequence of) actual_series is anomaly-free. + + Parameters + ---------- + actual_series + The (sequence of) actual series. + pred_series + The (sequence of) predicted series. + + Returns + ------- + self + Fitted Scorer. + """ + list_actual_series, list_pred_series = _to_list(actual_series), _to_list( + pred_series + ) + _assert_same_length(list_actual_series, list_pred_series) + + list_fit_series = [] + for s1, s2 in zip(list_actual_series, list_pred_series): + _sanity_check_two_series(s1, s2) + s1 = self._assert_deterministic(s1, "actual_series") + s2 = self._assert_deterministic(s2, "pred_series") + list_fit_series.append(self._diff_series(s1, s2)) + + self.fit(list_fit_series) + self._fit_called = True + + @abstractmethod + def _fit_core(self, series: Any) -> Any: + pass + + @abstractmethod + def _score_core(self, series: Any) -> Any: + pass + + def _diff_series(self, series_1: TimeSeries, series_2: TimeSeries) -> TimeSeries: + """Applies the ``diff_fn`` to the two time series. Converts two time series into 1. + + series_1 and series_2 must: + - have a non empty time intersection + - be of the same width W + + Parameters + ---------- + series_1 + 1st time series + series_2: + 2nd time series + + Returns + ------- + TimeSeries + series of width W + """ + series_1, series_2 = _intersect(series_1, series_2) + + if self.diff_fn == "abs_diff": + return (series_1 - series_2).map(lambda x: np.abs(x)) + elif self.diff_fn == "diff": + return series_1 - series_2 + else: + # found an non-existent diff_fn + raise ValueError( + f"Metric should be 'diff' or 'abs_diff', found {self.diff_fn}" + ) + + +class NLLScorer(NonFittableAnomalyScorer): + """Parent class for all LikelihoodScorer""" + + def __init__(self, window) -> None: + super().__init__(univariate_scorer=False, window=window) + + def _score_core_from_prediction( + self, + actual_series: TimeSeries, + pred_series: TimeSeries, + ) -> TimeSeries: + """For each timestamp of the inputs: + - the parameters of the considered distribution are fitted on the samples of the probabilistic time series + - the negative log-likelihood of the determinisitc time series values are computed + + If the series is multivariate, the score will be computed on each component independently. + + Parameters + ---------- + actual_series: + A determinisict time series (number of samples per timestamp must be equal to 1) + pred_series + A probabilistic time series (number of samples per timestamp must be higher than 1) + + Returns + ------- + TimeSeries + """ + actual_series = self._assert_deterministic(actual_series, "actual_series") + self._assert_stochastic(pred_series, "pred_series") + + np_actual_series = actual_series.all_values(copy=False) + np_pred_series = pred_series.all_values(copy=False) + + np_anomaly_scores = [] + for component_idx in range(pred_series.width): + np_anomaly_scores.append( + self._score_core_nllikelihood( + # shape actual: (time_steps, ) + # shape pred: (time_steps, samples) + np_actual_series[:, component_idx].squeeze(-1), + np_pred_series[:, component_idx], + ) + ) + + anomaly_scores = TimeSeries.from_times_and_values( + pred_series.time_index, list(zip(*np_anomaly_scores)) + ) + + def _window_adjustment_series(series: TimeSeries) -> TimeSeries: + """Slides a window of size self.window along the input series, and replaces the value of + the input time series by the mean of the values contained in the window (past self.window + points, including itself). + A series of length N will be transformed into a series of length N-self.window+1. + """ + + if self.window == 1: + # the process results in replacing every value by itself -> return directly the series + return series + else: + return series.window_transform( + transforms={ + "window": self.window, + "function": "mean", + "mode": "rolling", + "min_periods": self.window, + }, + treat_na="dropna", + ) + + return _window_adjustment_series(anomaly_scores) + + @property + def is_probabilistic(self) -> bool: + return True + + @abstractmethod + def _score_core_nllikelihood(self, input_1: Any, input_2: Any) -> Any: + """For each timestamp, the corresponding distribution is fitted on the probabilistic time-series + input_2, and returns the negative log-likelihood of the deterministic time-series input_1 + given the distribution. + """ + pass diff --git a/darts/ad/scorers/wasserstein_scorer.py b/darts/ad/scorers/wasserstein_scorer.py new file mode 100644 index 0000000000..a332cb4173 --- /dev/null +++ b/darts/ad/scorers/wasserstein_scorer.py @@ -0,0 +1,191 @@ +""" +WassersteinScorer +----- + +Wasserstein Scorer (distance function defined between probability distributions) [1]_. +The implementations is wrapped around `scipy.stats +`_. + +References +---------- +.. [1] https://en.wikipedia.org/wiki/Wasserstein_metric +""" + +from typing import Sequence + +import numpy as np +from numpy.lib.stride_tricks import sliding_window_view +from scipy.stats import wasserstein_distance + +from darts.ad.scorers.scorers import FittableAnomalyScorer +from darts.logging import get_logger, raise_if_not +from darts.timeseries import TimeSeries + +logger = get_logger(__name__) + + +class WassersteinScorer(FittableAnomalyScorer): + def __init__( + self, + window: int = 10, + component_wise: bool = False, + diff_fn="abs_diff", + ) -> None: + """ + When calling ``fit(series)``, a moving window is applied, which results in a set of vectors of size `W`, + where `W` is the window size. These vectors are kept in memory, representing the training + distribution. The ``score(series)`` function will apply the same moving window. + The Wasserstein distance is computed between the training distribution and each vector, + resulting in an anomaly score. + + Alternatively, the scorer has the functions ``fit_from_prediction()`` and ``score_from_prediction()``. + Both require two series (actual and prediction), and compute a "difference" series by applying the + function ``diff_fn`` (default: absolute difference). The resulting series is then passed to the + functions ``fit()`` and ``score()``, respectively. + + `component_wise` is a boolean parameter indicating how the model should behave with multivariate inputs + series. If set to True, the model will treat each series dimension independently. If set to False, the model + concatenates the dimensions in each windows of length `W` and computes a single score for all dimensions. + + **Training with** ``fit()``: + + The input can be a series (univariate or multivariate) or multiple series. The series will be partitioned + into equal size subsequences. The subsequence will be of size `W` * `D`, with: + + * `W` being the size of the window given as a parameter `window` + * `D` being the dimension of the series (`D` = 1 if univariate or if `component_wise` is set to True) + + For a series of length `N`, (`N` - `W` + 1)/W subsequences will be generated. If a list of series is given + of length L, each series will be partitioned into subsequences, and the results will be concatenated into + an array of length L * number of subsequences of each series. + + The arrays will be kept in memory, representing the training data distribution. + In practice, the series or list of series can for instance represent residuals than can be + considered independent and identically distributed (iid). + + If `component_wise` is set to True, the algorithm will be applied to each dimension independently. For each + dimension, a PyOD model will be trained. + + **Computing score with** ``score()``: + + The input can be a series (univariate or multivariate) or a sequence of series. The given series must have the + same dimension `D` as the data used to train the PyOD model. + + For each series, if the series is multivariate of dimension `D`: + + * if `component_wise` is set to False: it returns a univariate series (dimension=1). It represents + the anomaly score of the entire series in the considered window at each timestamp. + * if `component_wise` is set to True: it returns a multivariate series of dimension `D`. Each dimension + represents the anomaly score of the corresponding component of the input. + + If the series is univariate, it returns a univariate series regardless of the parameter + `component_wise`. + + A window of size `W` is rolled on the series with a stride equal to 1. It is the same size window `W` used + during the training phase. + Each value in the score series thus represents how anomalous the sample of the `W` previous values is. + + Parameters + ---------- + window + Size of the sliding window that represents the number of samples in the testing distribution to compare + with the training distribution in the Wasserstein function + diff_fn + Optionally, reduced function to use if two series are given. It will transform the two series into one. + This allows the WassersteinScorer to compute the Wasserstein distance on the original series or on its + residuals (difference between the prediction and the original series). + Must be one of "abs_diff" and "diff" (defined in ``_diff_series()``). + Default: "abs_diff" + component_wise + Boolean value indicating if the score needs to be computed for each component independently (True) + or by concatenating the component in the considered window to compute one score (False). + Default: False + + """ + + # TODO: + # - understand better the math behind the Wasserstein distance when the test distribution contains + # only one sample + # - check if there is an equivalent Wasserstein distance for d-D distributions (currently only accepts 1D) + + if type(window) is int: + if window > 0 and window < 10: + logger.warning( + f"The `window` parameter WassersteinScorer is smaller than 10 (w={window})." + + " The value represents the window length rolled on the series given as" + + " input in the ``score`` function. At each position, the w values will" + + " constitute a subset, and the Wasserstein distance between the subset" + + " and the train distribution will be computed. To better represent the" + + " constituted test distribution, the window parameter should be larger" + + " than 10." + ) + + raise_if_not( + type(component_wise) is bool, + f"Parameter `component_wise` must be Boolean, found type: {type(component_wise)}.", + ) + self.component_wise = component_wise + + super().__init__( + univariate_scorer=(not component_wise), window=window, diff_fn=diff_fn + ) + + def __str__(self): + return "WassersteinScorer" + + def _fit_core( + self, + list_series: Sequence[TimeSeries], + ): + self.training_data = np.concatenate( + [s.all_values(copy=False) for s in list_series] + ).squeeze(-1) + + if not self.component_wise: + self.training_data = self.training_data.flatten() + + def _score_core(self, series: TimeSeries) -> TimeSeries: + raise_if_not( + self.width_trained_on == series.width, + "Input must have the same number of components as the data used for" + + " training the Wasserstein model, found number of components equal" + + f" to {series.width} and expected {self.width_trained_on}.", + ) + + np_series = series.all_values(copy=False) + np_anomaly_score = [] + + if not self.component_wise: + np_anomaly_score = [ + wasserstein_distance(self.training_data, window_samples) + for window_samples in sliding_window_view( + np_series, window_shape=self.window, axis=0 + ) + .transpose(0, 3, 1, 2) + .reshape(-1, self.window * series.width) + ] + + return TimeSeries.from_times_and_values( + series.time_index[self.window - 1 :], np_anomaly_score + ) + + else: + for component_idx in range(self.width_trained_on): + score = [ + wasserstein_distance( + self.training_data[component_idx, :], window_samples + ) + for window_samples in sliding_window_view( + np_series[:, component_idx], + window_shape=self.window, + axis=0, + ) + .transpose(0, 2, 1) + .reshape(-1, self.window) + ] + + np_anomaly_score.append(score) + + return TimeSeries.from_times_and_values( + series.time_index[self.window - 1 :], list(zip(*np_anomaly_score)) + ) diff --git a/darts/ad/utils.py b/darts/ad/utils.py new file mode 100644 index 0000000000..a6f4cddb43 --- /dev/null +++ b/darts/ad/utils.py @@ -0,0 +1,806 @@ +""" +Utils for Anomaly Detection +--------------------------- + +Common functions used by anomaly_model.py, scorers.py, aggregators.py and detectors.py +""" + +# TODO: +# - change structure of eval_accuracy_from_scores and eval_accuracy_from_binary_prediction (a lot of repeated code) +# - migrate metrics function to darts.metric +# - check error message +# - create a zoom option on anomalies for a show function +# - add an option visualize: "by window", "unique", "together" +# - create a normalize option in plot function (norm every anomaly score btw 1 and 0) -> to be seen on the same plot + +from typing import Sequence, Union + +import matplotlib.pyplot as plt +import numpy as np +from sklearn.metrics import ( + accuracy_score, + average_precision_score, + f1_score, + precision_score, + recall_score, + roc_auc_score, +) + +from darts import TimeSeries +from darts.logging import get_logger, raise_if, raise_if_not + +logger = get_logger(__name__) + + +def _assert_binary(series: TimeSeries, name_series: str): + """Checks if series is a binary timeseries (1 and 0)" + + Parameters + ---------- + series + series to check for. + name_series + name str of the series. + """ + + raise_if_not( + np.array_equal( + series.values(copy=False), + series.values(copy=False).astype(bool), + ), + f"Input series {name_series} must be a binary time series.", + ) + + +def eval_accuracy_from_scores( + actual_anomalies: Union[TimeSeries, Sequence[TimeSeries]], + anomaly_score: Union[TimeSeries, Sequence[TimeSeries]], + window: Union[int, Sequence[int]] = 1, + metric: str = "AUC_ROC", +) -> Union[float, Sequence[float], Sequence[Sequence[float]]]: + """Scores the results against true anomalies. + + `actual_anomalies` and `anomaly_score` must have the same shape. + `actual_anomalies` must be binary and have values belonging to the two classes (0 and 1). + + If one series is given for `actual_anomalies` and `anomaly_score` contains more than + one series, the function will consider `actual_anomalies` as the ground truth anomalies for + all scores in `anomaly_score`. + + Parameters + ---------- + actual_anomalies + The ground truth of the anomalies (1 if it is an anomaly and 0 if not). + anomaly_score + Series indicating how anomoulous each window of size w is. + window + Integer value indicating the number of past samples each point represents + in the anomaly_score. The parameter will be used by the function + ``_window_adjustment_anomalies()`` to transform actual_anomalies. + If a list is given. the length must match the number of series in anomaly_score + and actual_anomalies. If only one window is given, the value will be used for every + series in anomaly_score and actual_anomalies. + metric + Optionally, Scoring function to use. Must be one of "AUC_ROC" and "AUC_PR". + Default: "AUC_ROC" + + Returns + ------- + Union[float, Sequence[float], Sequence[Sequence[float]]] + Score of the anomalies score prediction + * ``float`` if `anomaly_score` is a univariate series (dimension=1). + * ``Sequence[float]`` + + * if `anomaly_score` is a multivariate series (dimension>1), + returns one value per dimension. + * if `anomaly_score` is a sequence of univariate series, returns one + value per series + * ``Sequence[Sequence[float]]`` if `anomaly_score` is a sequence of + multivariate series. Outer Sequence is over the sequence input, and the inner + Sequence is over the dimensions of each element in the sequence input. + """ + + raise_if_not( + metric in {"AUC_ROC", "AUC_PR"}, + "Argument `metric` must be one of 'AUC_ROC', 'AUC_PR'", + ) + metric_fn = roc_auc_score if metric == "AUC_ROC" else average_precision_score + + list_actual_anomalies, list_anomaly_scores, list_window = ( + _to_list(actual_anomalies), + _to_list(anomaly_score), + _to_list(window), + ) + + if len(list_actual_anomalies) == 1 and len(list_anomaly_scores) > 1: + list_actual_anomalies = list_actual_anomalies * len(list_anomaly_scores) + + _assert_same_length(list_actual_anomalies, list_anomaly_scores) + + if len(list_window) == 1: + list_window = list_window * len(actual_anomalies) + else: + raise_if_not( + len(list_window) == len(list_actual_anomalies), + "The list of windows must be the same length as the list of `anomaly_score` and" + + " `actual_anomalies`. There must be one window value for each series." + + f" Found length {len(list_window)}, expected {len(list_actual_anomalies)}.", + ) + + sol = [] + for idx, (s_anomalies, s_score) in enumerate( + zip(list_actual_anomalies, list_anomaly_scores) + ): + + _assert_binary(s_anomalies, "actual_anomalies") + + sol.append( + _eval_accuracy_from_data( + s_anomalies, s_score, list_window[idx], metric_fn, metric + ) + ) + + if len(sol) == 1 and not isinstance(anomaly_score, Sequence): + return sol[0] + else: + return sol + + +def eval_accuracy_from_binary_prediction( + actual_anomalies: Union[TimeSeries, Sequence[TimeSeries]], + binary_pred_anomalies: Union[TimeSeries, Sequence[TimeSeries]], + window: Union[int, Sequence[int]] = 1, + metric: str = "recall", +) -> Union[float, Sequence[float], Sequence[Sequence[float]]]: + """Score the results against true anomalies. + + checks that `pred_anomalies` and `actual_anomalies` are the same: + - type, + - length, + - number of components + - binary and has values belonging to the two classes (1 and 0) + + If one series is given for `actual_anomalies` and `pred_anomalies` contains more than + one series, the function will consider `actual_anomalies` as the true anomalies for + all scores in `anomaly_score`. + + Parameters + ---------- + actual_anomalies + The (sequence of) ground truth of the anomalies (1 if it is an anomaly and 0 if not) + binary_pred_anomalies + Anomaly predictions. + window + Integer value indicating the number of past samples each point represents + in the pred_anomalies. The parameter will be used to transform actual_anomalies. + If a list is given. the length must match the number of series in pred_anomalies + and actual_anomalies. If only one window is given, the value will be used for every + series in pred_anomalies and actual_anomalies. + metric + Optionally, Scoring function to use. Must be one of "recall", "precision", + "f1", and "accuracy". + Default: "recall" + + Returns + ------- + Union[float, Sequence[float], Sequence[Sequence[float]]] + Score of the anomalies prediction + + * ``float`` if `binary_pred_anomalies` is a univariate series (dimension=1). + * ``Sequence[float]`` + + * if `binary_pred_anomalies` is a multivariate series (dimension>1), + returns one value per dimension. + * if `binary_pred_anomalies` is a sequence of univariate series, returns one + value per series + * ``Sequence[Sequence[float]]`` if `binary_pred_anomalies` is a sequence of + multivariate series. Outer Sequence is over the sequence input, and the inner + Sequence is over the dimensions of each element in the sequence input. + """ + + raise_if_not( + metric in {"recall", "precision", "f1", "accuracy"}, + "Argument `metric` must be one of 'recall', 'precision', " + "'f1' and 'accuracy'.", + ) + + if metric == "recall": + metric_fn = recall_score + elif metric == "precision": + metric_fn = precision_score + elif metric == "f1": + metric_fn = f1_score + else: + metric_fn = accuracy_score + + list_actual_anomalies, list_binary_pred_anomalies, list_window = ( + _to_list(actual_anomalies), + _to_list(binary_pred_anomalies), + _to_list(window), + ) + + if len(list_actual_anomalies) == 1 and len(list_binary_pred_anomalies) > 1: + list_actual_anomalies = list_actual_anomalies * len(list_binary_pred_anomalies) + + _assert_same_length(list_actual_anomalies, list_binary_pred_anomalies) + + if len(list_window) == 1: + list_window = list_window * len(actual_anomalies) + else: + raise_if_not( + len(list_window) == len(list_actual_anomalies), + "The list of windows must be the same length as the list of `pred_anomalies` and" + + " `actual_anomalies`. There must be one window value for each series." + + f" Found length {len(list_window)}, expected {len(list_actual_anomalies)}.", + ) + + sol = [] + for idx, (s_anomalies, s_pred) in enumerate( + zip(list_actual_anomalies, list_binary_pred_anomalies) + ): + + _assert_binary(s_pred, "pred_anomalies") + _assert_binary(s_anomalies, "actual_anomalies") + + sol.append( + _eval_accuracy_from_data( + s_anomalies, s_pred, list_window[idx], metric_fn, metric + ) + ) + + if len(sol) == 1 and not isinstance(binary_pred_anomalies, Sequence): + return sol[0] + else: + return sol + + +def _eval_accuracy_from_data( + s_anomalies: TimeSeries, + s_data: TimeSeries, + window: int, + metric_fn, + metric_name: str, +) -> Union[float, Sequence[float]]: + """Internal function for: + - ``eval_accuracy_from_binary_prediction()`` + - ``eval_accuracy_from_scores()`` + + Score the results against true anomalies. + + Parameters + ---------- + actual_anomalies + The ground truth of the anomalies (1 if it is an anomaly and 0 if not) + s_data + series prediction + window + Integer value indicating the number of past samples each point represents + in the anomaly_score. The parameter will be used by the function + ``_window_adjustment_anomalies()`` to transform s_anomalies. + metric_fn + Function to use. Can be "average_precision_score", "roc_auc_score", "accuracy_score", + "f1_score", "precision_score" and "recall_score". + metric_name + Name str of the function to use. Can be "AUC_PR", "AUC_ROC", "accuracy", + "f1", "precision" and "recall". + + Returns + ------- + Union[float, Sequence[float]] + Score of the anomalies prediction + - float -> if `s_data` is a univariate series (dimension=1). + - Sequence[float] -> if `s_data` is a multivariate series (dimension>1), + returns one value per dimension. + """ + + _assert_timeseries(s_data, "Prediction series input") + _assert_timeseries(s_anomalies, "actual_anomalies input") + + # if window > 1, the anomalies will be adjusted so that it can be compared timewise with s_data + s_anomalies = _max_pooling(s_anomalies, window) + + _sanity_check_two_series(s_data, s_anomalies) + + s_data, s_anomalies = _intersect(s_data, s_anomalies) + + if metric_name == "AUC_ROC" or metric_name == "AUC_PR": + + nr_anomalies_per_component = ( + s_anomalies.sum(axis=0).values(copy=False).flatten() + ) + + raise_if( + nr_anomalies_per_component.min() == 0, + f"`actual_anomalies` does not contain anomalies. {metric_name} cannot be computed.", + ) + + raise_if( + nr_anomalies_per_component.max() == len(s_anomalies), + f"`actual_anomalies` only contains anomalies. {metric_name} cannot be computed." + + ["", f" Consider decreasing the window size (window={window})"][ + window > 1 + ], + ) + + # TODO: could we vectorize this? + metrics = [] + for component_idx in range(s_data.width): + metrics.append( + metric_fn( + s_anomalies.all_values(copy=False)[:, component_idx], + s_data.all_values(copy=False)[:, component_idx], + ) + ) + + if len(metrics) == 1: + return metrics[0] + else: + return metrics + + +def _intersect( + series_1: TimeSeries, + series_2: TimeSeries, +) -> tuple[TimeSeries, TimeSeries]: + """Returns the sub-series of series_1 and of series_2 that share the same time index. + (Intersection in time of the two time series) + + Parameters + ---------- + series_1 + 1st time series + series_2: + 2nd time series + + Returns + ------- + Tuple[TimeSeries, TimeSeries] + """ + + new_series_1 = series_1.slice_intersect(series_2) + raise_if( + len(new_series_1) == 0, + "Time intersection between the two series must be non empty.", + ) + + return new_series_1, series_2.slice_intersect(series_1) + + +def _assert_timeseries(series: TimeSeries, message: str = None): + """Checks if given input is of type Darts TimeSeries""" + + raise_if_not( + isinstance(series, TimeSeries), + "{} must be type darts.timeseries.TimeSeries and not {}.".format( + message if message is not None else "Series input", type(series) + ), + ) + + +def _sanity_check_two_series( + series_1: TimeSeries, + series_2: TimeSeries, +): + """Performs sanity check on the two given inputs + + Checks if the two inputs: + - type is Darts Timeseries + - have the same number of components + - if their intersection in time is not null + + Parameters + ---------- + series_1 + 1st time series + series_2: + 2nd time series + """ + + _assert_timeseries(series_1) + _assert_timeseries(series_2) + + # check if the two inputs time series have the same number of components + raise_if_not( + series_1.width == series_2.width, + "Series must have the same number of components," + + f" found {series_1.width} and {series_2.width}.", + ) + + # check if the time intersection between the two inputs time series is not empty + raise_if_not( + len(series_1.time_index.intersection(series_2.time_index)) > 0, + "Series must have a non-empty intersection timestamps.", + ) + + +def _max_pooling(series: TimeSeries, window: int) -> TimeSeries: + """Slides a window of size `window` along the input series, and replaces the value of the + input time series by the maximum of the values contained in the window. + + The binary time series output represents if there is an anomaly (=1) or not (=0) in the past + window points. The new series will equal the length of the input series - window. Its first + point will start at the first time index of the input time series + window points. + + Parameters + ---------- + series: + Binary time series. + window: + Integer value indicating the number of past samples each point represents. + + Returns + ------- + Binary TimeSeries + """ + + raise_if_not( + isinstance(window, int), + f"Parameter `window` must be of type int, found {type(window)}.", + ) + + raise_if_not( + window > 0, + f"Parameter `window` must be stricly greater than 0, found size {window}.", + ) + + raise_if_not( + window < len(series), + "Parameter `window` must be smaller than the length of the input series, " + + f" found window size {(window)}, and max size {len(series)}.", + ) + + if window == 1: + # the process results in replacing every value by itself -> return directly the series + return series + else: + return series.window_transform( + transforms={ + "window": window, + "function": "max", + "mode": "rolling", + "min_periods": window, + }, + treat_na="dropna", + ) + + +def _to_list(series: Union[TimeSeries, Sequence[TimeSeries]]) -> Sequence[TimeSeries]: + """If not already, it converts the input into a sequence + + Parameters + ---------- + series + single TimeSeries, or a sequence of TimeSeries + + Returns + ------- + Sequence[TimeSeries] + """ + + return [series] if not isinstance(series, Sequence) else series + + +def _assert_same_length( + list_series_1: Sequence[TimeSeries], + list_series_2: Sequence[TimeSeries], +): + """Checks if the two sequences contain the same number of TimeSeries.""" + + raise_if_not( + len(list_series_1) == len(list_series_2), + "Sequences of series must be of the same length, found length:" + + f" {len(list_series_1)} and {len(list_series_2)}.", + ) + + +def show_anomalies_from_scores( + series: TimeSeries, + model_output: TimeSeries = None, + anomaly_scores: Union[TimeSeries, Sequence[TimeSeries]] = None, + window: Union[int, Sequence[int]] = 1, + names_of_scorers: Union[str, Sequence[str]] = None, + actual_anomalies: TimeSeries = None, + title: str = None, + metric: str = None, +): + """Plot the results generated by an anomaly model. + + The plot will be composed of the following: + - the series itself with the output of the model (if given) + - the anomaly score of each scorer. The scorer with different windows will be separated. + - the actual anomalies, if given. + + If model_output is stochastic (i.e., if it has multiple samples), the function will plot: + - the mean per timestamp + - the quantile 0.95 for an upper bound + - the quantile 0.05 for a lower bound + + Possible to: + - add a title to the figure with the parameter `title` + - give personalized names for the scorers with `names_of_scorers` + - show the results of a metric for each anomaly score (AUC_ROC or AUC_PR), if the actual anomalies is given + + Parameters + ---------- + series + The series to visualize anomalies from. + model_output + Output of the model given as input the series (can be stochastic). + anomaly_scores + Output of the scorers given the output of the model and the series. + window + Window parameter for each anomaly scores. + Default: 1. If a list of anomaly scores is given, the same default window will be used for every score. + names_of_scorers + Name of the scores. Must be a list of length equal to the number of scorers in the anomaly_model. + actual_anomalies + The ground truth of the anomalies (1 if it is an anomaly and 0 if not) + title + Title of the figure + metric + Optionally, Scoring function to use. Must be one of "AUC_ROC" and "AUC_PR". + Default: "AUC_ROC" + """ + + raise_if_not( + isinstance(series, TimeSeries), + f"Input `series` must be of type TimeSeries, found {type(series)}.", + ) + + if title is None: + if anomaly_scores is not None: + title = "Anomaly results" + else: + raise_if_not( + isinstance(title, str), + f"Input `title` must be of type str, found {type(title)}.", + ) + + nbr_plots = 1 + + if model_output is not None: + raise_if_not( + isinstance(model_output, TimeSeries), + f"Input `model_output` must be of type TimeSeries, found {type(model_output)}.", + ) + + if actual_anomalies is not None: + raise_if_not( + isinstance(actual_anomalies, TimeSeries), + f"Input `actual_anomalies` must be of type TimeSeries, found {type(actual_anomalies)}.", + ) + + nbr_plots = nbr_plots + 1 + else: + raise_if_not( + metric is None, + "`actual_anomalies` must be given in order to calculate a metric.", + ) + + if anomaly_scores is not None: + + if isinstance(anomaly_scores, Sequence): + for idx, s in enumerate(anomaly_scores): + raise_if_not( + isinstance(s, TimeSeries), + f"Elements of anomaly_scores must be of type TimeSeries, found {type(s)} at index {idx}.", + ) + else: + raise_if_not( + isinstance(anomaly_scores, TimeSeries), + f"Input `anomaly_scores` must be of type TimeSeries or Sequence, found {type(actual_anomalies)}.", + ) + anomaly_scores = [anomaly_scores] + + if names_of_scorers is not None: + + if isinstance(names_of_scorers, str): + names_of_scorers = [names_of_scorers] + elif isinstance(names_of_scorers, Sequence): + for idx, name in enumerate(names_of_scorers): + raise_if_not( + isinstance(name, str), + f"Elements of names_of_scorers must be of type str, found {type(name)} at index {idx}.", + ) + else: + raise ValueError( + f"Input `names_of_scorers` must be of type str or Sequence, found {type(names_of_scorers)}." + ) + + raise_if_not( + len(names_of_scorers) == len(anomaly_scores), + "The number of names in `names_of_scorers` must match the number of anomaly score " + + f"given as input, found {len(names_of_scorers)} and expected {len(anomaly_scores)}.", + ) + + if isinstance(window, int): + window = [window] + elif isinstance(window, Sequence): + for idx, w in enumerate(window): + raise_if_not( + isinstance(w, int), + f"Every window must be of type int, found {type(w)} at index {idx}.", + ) + else: + raise ValueError( + f"Input `window` must be of type int or Sequence, found {type(window)}." + ) + + raise_if_not( + all([w > 0 for w in window]), + "All windows must be positive integer.", + ) + + if len(window) == 1: + window = window * len(anomaly_scores) + else: + raise_if_not( + len(window) == len(anomaly_scores), + "The number of window in `window` must match the number of anomaly score given as input. One " + + f"window value for each series. Found length {len(window)}, and expected {len(anomaly_scores)}.", + ) + + raise_if_not( + all([w < len(s) for (w, s) in zip(window, anomaly_scores)]), + "All windows must be smaller than the length of their corresponding score.", + ) + + nbr_plots = nbr_plots + len(set(window)) + else: + if window is not None: + logger.warning( + "The parameter `window` is given, but the input `anomaly_scores` is None." + ) + + if names_of_scorers is not None: + logger.warning( + "The parameter `names_of_scorers` is given, but the input `anomaly_scores` is None." + ) + + if metric is not None: + logger.warning( + "The parameter `metric` is given, but the input `anomaly_scores` is None." + ) + + fig, axs = plt.subplots( + nbr_plots, + figsize=(8, 4 + 2 * (nbr_plots - 1)), + sharex=True, + gridspec_kw={"height_ratios": [2] + [1] * (nbr_plots - 1)}, + squeeze=False, + ) + + index_ax = 0 + + _plot_series(series=series, ax_id=axs[index_ax][0], linewidth=0.5, label_name="") + + if model_output is not None: + + _plot_series( + series=model_output, + ax_id=axs[index_ax][0], + linewidth=0.5, + label_name="model output", + ) + + axs[index_ax][0].set_title("") + + if actual_anomalies is not None or anomaly_scores is not None: + axs[index_ax][0].set_xlabel("") + + axs[index_ax][0].legend(loc="upper center", bbox_to_anchor=(0.5, 1.1), ncol=2) + + if anomaly_scores is not None: + + dict_input = {} + + for idx, (score, w) in enumerate(zip(anomaly_scores, window)): + + dict_input[idx] = {"series_score": score, "window": w, "name_id": idx} + + current_window = window[0] + index_ax = index_ax + 1 + + for elem in sorted(dict_input.items(), key=lambda x: x[1]["window"]): + + idx = elem[1]["name_id"] + w = elem[1]["window"] + + if w != current_window: + current_window = w + index_ax = index_ax + 1 + + if metric is not None: + value = round( + eval_accuracy_from_scores( + anomaly_score=anomaly_scores[idx], + actual_anomalies=actual_anomalies, + window=w, + metric=metric, + ), + 3, + ) + else: + value = None + + if names_of_scorers is not None: + label = names_of_scorers[idx] + [f" ({value})", ""][value is None] + else: + label = f"score_{str(idx)}" + [f" ({value})", ""][value is None] + + _plot_series( + series=elem[1]["series_score"], + ax_id=axs[index_ax][0], + linewidth=0.5, + label_name=label, + ) + + axs[index_ax][0].legend( + loc="upper center", bbox_to_anchor=(0.5, 1.19), ncol=2 + ) + axs[index_ax][0].set_title(f"Window: {str(w)}", loc="left") + axs[index_ax][0].set_title("") + axs[index_ax][0].set_xlabel("") + + if actual_anomalies is not None: + + _plot_series( + series=actual_anomalies, + ax_id=axs[index_ax + 1][0], + linewidth=1, + label_name="anomalies", + color="red", + ) + + axs[index_ax + 1][0].set_title("") + axs[index_ax + 1][0].set_ylim([-0.1, 1.1]) + axs[index_ax + 1][0].set_yticks([0, 1]) + axs[index_ax + 1][0].set_yticklabels(["no", "yes"]) + axs[index_ax + 1][0].legend( + loc="upper center", bbox_to_anchor=(0.5, 1.2), ncol=2 + ) + else: + axs[index_ax][0].set_xlabel("timestamp") + + fig.suptitle(title) + + +def _plot_series(series, ax_id, linewidth, label_name, **kwargs): + """Internal function called by ``show_anomalies_from_scores()`` + + Plot the series on the given axes ax_id. + + Parameters + ---------- + series + The series to plot. + ax_id + The axis the series will be ploted on. + linewidth + Thickness of the line. + label_name + Name that will appear in the legend. + """ + + for i, c in enumerate(series._xa.component[:10]): + comp = series._xa.sel(component=c) + + if comp.sample.size > 1: + central_series = comp.mean(dim="sample") + low_series = comp.quantile(q=0.05, dim="sample") + high_series = comp.quantile(q=0.95, dim="sample") + else: + central_series = comp + + label_to_use = ( + (label_name + ("_" + str(i) if len(series.components) > 1 else "")) + if label_name != "" + else "" + str(str(c.values)) + ) + + central_series.plot(ax=ax_id, linewidth=linewidth, label=label_to_use, **kwargs) + + if comp.sample.size > 1: + ax_id.fill_between( + series.time_index, low_series, high_series, alpha=0.25, **kwargs + ) diff --git a/darts/tests/ad/__init__.py b/darts/tests/ad/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/darts/tests/ad/test_aggregators.py b/darts/tests/ad/test_aggregators.py new file mode 100644 index 0000000000..a360568660 --- /dev/null +++ b/darts/tests/ad/test_aggregators.py @@ -0,0 +1,874 @@ +from typing import Sequence + +import numpy as np +from sklearn.ensemble import GradientBoostingClassifier + +from darts import TimeSeries +from darts.ad.aggregators.and_aggregator import AndAggregator +from darts.ad.aggregators.ensemble_sklearn_aggregator import EnsembleSklearnAggregator +from darts.ad.aggregators.or_aggregator import OrAggregator +from darts.models import MovingAverage +from darts.tests.base_test_class import DartsBaseTestClass + +list_NonFittableAggregator = [ + OrAggregator(), + AndAggregator(), +] + +list_FittableAggregator = [ + EnsembleSklearnAggregator(model=GradientBoostingClassifier()) +] + + +class ADAggregatorsTestCase(DartsBaseTestClass): + + np.random.seed(42) + + # univariate series + np_train = np.random.normal(loc=10, scale=0.5, size=100) + train = TimeSeries.from_values(np_train) + + np_real_anomalies = np.random.choice(a=[0, 1], size=100, p=[0.5, 0.5]) + real_anomalies = TimeSeries.from_times_and_values( + train._time_index, np_real_anomalies + ) + + # multivariate series + np_mts_train = np.random.normal(loc=[10, 5], scale=[0.5, 1], size=[100, 2]) + mts_train = TimeSeries.from_values(np_mts_train) + + np_anomalies = np.random.choice(a=[0, 1], size=[100, 2], p=[0.5, 0.5]) + mts_anomalies1 = TimeSeries.from_times_and_values(train._time_index, np_anomalies) + + np_anomalies = np.random.choice(a=[0, 1], size=[100, 2], p=[0.5, 0.5]) + mts_anomalies2 = TimeSeries.from_times_and_values(train._time_index, np_anomalies) + + np_anomalies_w3 = np.random.choice(a=[0, 1], size=[100, 3], p=[0.5, 0.5]) + mts_anomalies3 = TimeSeries.from_times_and_values( + train._time_index, np_anomalies_w3 + ) + + np_probabilistic = np.random.choice(a=[0, 1], p=[0.5, 0.5], size=[100, 2, 5]) + mts_probabilistic = TimeSeries.from_values(np_probabilistic) + + # simple case + np_anomalies_1 = np.random.choice(a=[1], size=100, p=[1]) + onlyones = TimeSeries.from_times_and_values(train._time_index, np_anomalies_1) + + np_anomalies = np.random.choice(a=[1], size=[100, 2], p=[1]) + mts_onlyones = TimeSeries.from_times_and_values(train._time_index, np_anomalies) + + np_anomalies_0 = np.random.choice(a=[0], size=100, p=[1]) + onlyzero = TimeSeries.from_times_and_values(train._time_index, np_anomalies_0) + + series_1_and_0 = TimeSeries.from_values( + np.dstack((np_anomalies_1, np_anomalies_0))[0], + columns=["component 1", "component 2"], + ) + + np_real_anomalies_3w = [ + elem[0] if elem[2] == 1 else elem[1] for elem in np_anomalies_w3 + ] + real_anomalies_3w = TimeSeries.from_times_and_values( + train._time_index, np_real_anomalies_3w + ) + + def test_DetectNonFittableAggregator(self): + + aggregator = OrAggregator() + + # Check return types + self.assertTrue(isinstance(aggregator.predict(self.mts_anomalies1), TimeSeries)) + self.assertTrue( + isinstance( + aggregator.predict([self.mts_anomalies1]), + Sequence, + ) + ) + self.assertTrue( + isinstance( + aggregator.predict([self.mts_anomalies1, self.mts_anomalies2]), + Sequence, + ) + ) + + def test_DetectFittableAggregator(self): + aggregator = EnsembleSklearnAggregator(model=GradientBoostingClassifier()) + + # Check return types + aggregator.fit(self.real_anomalies, self.mts_anomalies1) + + # Check return types + self.assertTrue(isinstance(aggregator.predict(self.mts_anomalies1), TimeSeries)) + self.assertTrue( + isinstance( + aggregator.predict([self.mts_anomalies1]), + Sequence, + ) + ) + self.assertTrue( + isinstance( + aggregator.predict([self.mts_anomalies1, self.mts_anomalies2]), + Sequence, + ) + ) + + def test_eval_accuracy(self): + + aggregator = AndAggregator() + + # Check return types + self.assertTrue( + isinstance( + aggregator.eval_accuracy(self.real_anomalies, self.mts_anomalies1), + float, + ) + ) + self.assertTrue( + isinstance( + aggregator.eval_accuracy([self.real_anomalies], [self.mts_anomalies1]), + Sequence, + ) + ) + self.assertTrue( + isinstance( + aggregator.eval_accuracy(self.real_anomalies, [self.mts_anomalies1]), + Sequence, + ) + ) + self.assertTrue( + isinstance( + aggregator.eval_accuracy( + [self.real_anomalies, self.real_anomalies], + [self.mts_anomalies1, self.mts_anomalies2], + ), + Sequence, + ) + ) + + # intersection between 'actual_anomalies' and the series in the sequence 'list_series' + # must be non empty + with self.assertRaises(ValueError): + aggregator.eval_accuracy(self.real_anomalies[:30], self.mts_anomalies1[40:]) + with self.assertRaises(ValueError): + aggregator.eval_accuracy( + [self.real_anomalies, self.real_anomalies[:30]], + [self.mts_anomalies1, self.mts_anomalies1[40:]], + ) + + # window parameter must be smaller than the length of the input (len = 100) + with self.assertRaises(ValueError): + aggregator.eval_accuracy( + self.real_anomalies, self.mts_anomalies1, window=101 + ) + + def test_NonFittableAggregator(self): + + for aggregator in list_NonFittableAggregator: + + # name must be of type str + self.assertEqual( + type(aggregator.__str__()), + str, + ) + + # Check if trainable is False, being a NonFittableAggregator + self.assertTrue(not aggregator.trainable) + + # predict on (sequence of) univariate series + with self.assertRaises(ValueError): + aggregator.predict([self.real_anomalies]) + with self.assertRaises(ValueError): + aggregator.predict(self.real_anomalies) + with self.assertRaises(ValueError): + aggregator.predict([self.mts_anomalies1, self.real_anomalies]) + + # input a (sequence of) non binary series + with self.assertRaises(ValueError): + aggregator.predict(self.mts_train) + with self.assertRaises(ValueError): + aggregator.predict([self.mts_anomalies1, self.mts_train]) + + # input a (sequence of) probabilistic series + with self.assertRaises(ValueError): + aggregator.predict(self.mts_probabilistic) + with self.assertRaises(ValueError): + aggregator.predict([self.mts_anomalies1, self.mts_probabilistic]) + + # input an element that is not a series + with self.assertRaises(ValueError): + aggregator.predict([self.mts_anomalies1, "random"]) + with self.assertRaises(ValueError): + aggregator.predict([self.mts_anomalies1, 1]) + + # Check width return + # Check if return type is the same number of series in input + self.assertTrue( + len( + aggregator.eval_accuracy( + [self.real_anomalies, self.real_anomalies], + [self.mts_anomalies1, self.mts_anomalies2], + ) + ), + len([self.mts_anomalies1, self.mts_anomalies2]), + ) + + def test_FittableAggregator(self): + + for aggregator in list_FittableAggregator: + + # name must be of type str + self.assertEqual( + type(aggregator.__str__()), + str, + ) + + # Need to call fit() before calling predict() + with self.assertRaises(ValueError): + aggregator.predict([self.mts_anomalies1, self.mts_anomalies1]) + + # Check if trainable is True, being a FittableAggregator + self.assertTrue(aggregator.trainable) + + # Check if _fit_called is False + self.assertTrue(not aggregator._fit_called) + + # fit on sequence with series that have different width + with self.assertRaises(ValueError): + aggregator.fit( + [self.real_anomalies, self.real_anomalies], + [self.mts_anomalies1, self.mts_anomalies3], + ) + + # fit on a (sequence of) univariate series + with self.assertRaises(ValueError): + aggregator.fit(self.real_anomalies, self.real_anomalies) + with self.assertRaises(ValueError): + aggregator.fit(self.real_anomalies, [self.real_anomalies]) + with self.assertRaises(ValueError): + aggregator.fit( + [self.real_anomalies, self.real_anomalies], + [self.mts_anomalies1, self.real_anomalies], + ) + + # fit on a (sequence of) non binary series + with self.assertRaises(ValueError): + aggregator.fit(self.real_anomalies, self.mts_train) + with self.assertRaises(ValueError): + aggregator.fit(self.real_anomalies, [self.mts_train]) + with self.assertRaises(ValueError): + aggregator.fit( + [self.real_anomalies, self.real_anomalies], + [self.mts_anomalies1, self.mts_train], + ) + + # fit on a (sequence of) probabilistic series + with self.assertRaises(ValueError): + aggregator.fit(self.real_anomalies, self.mts_probabilistic) + with self.assertRaises(ValueError): + aggregator.fit(self.real_anomalies, [self.mts_probabilistic]) + with self.assertRaises(ValueError): + aggregator.fit( + [self.real_anomalies, self.real_anomalies], + [self.mts_anomalies1, self.mts_probabilistic], + ) + + # input an element that is not a series + with self.assertRaises(ValueError): + aggregator.fit(self.real_anomalies, "random") + with self.assertRaises(ValueError): + aggregator.fit(self.real_anomalies, [self.mts_anomalies1, "random"]) + with self.assertRaises(ValueError): + aggregator.fit(self.real_anomalies, [self.mts_anomalies1, 1]) + + # fit on a (sequence of) multivariate anomalies + with self.assertRaises(ValueError): + aggregator.fit(self.mts_anomalies1, self.mts_anomalies1) + with self.assertRaises(ValueError): + aggregator.fit([self.mts_anomalies1], [self.mts_anomalies1]) + with self.assertRaises(ValueError): + aggregator.fit( + [self.real_anomalies, self.mts_anomalies1], + [self.mts_anomalies1, self.mts_anomalies1], + ) + + # fit on a (sequence of) non binary anomalies + with self.assertRaises(ValueError): + aggregator.fit(self.train, self.mts_anomalies1) + with self.assertRaises(ValueError): + aggregator.fit([self.train], self.mts_anomalies1) + with self.assertRaises(ValueError): + aggregator.fit( + [self.real_anomalies, self.train], + [self.mts_anomalies1, self.mts_anomalies1], + ) + + # fit on a (sequence of) probabilistic anomalies + with self.assertRaises(ValueError): + aggregator.fit(self.mts_probabilistic, self.mts_anomalies1) + with self.assertRaises(ValueError): + aggregator.fit([self.mts_probabilistic], self.mts_anomalies1) + with self.assertRaises(ValueError): + aggregator.fit( + [self.real_anomalies, self.mts_probabilistic], + [self.mts_anomalies1, self.mts_anomalies1], + ) + + # input an element that is not a anomalies + with self.assertRaises(ValueError): + aggregator.fit("random", self.mts_anomalies1) + with self.assertRaises(ValueError): + aggregator.fit( + [self.real_anomalies, "random"], + [self.mts_anomalies1, self.mts_anomalies1], + ) + with self.assertRaises(ValueError): + aggregator.fit( + [self.real_anomalies, 1], [self.mts_anomalies1, self.mts_anomalies1] + ) + + # nbr of anomalies must match nbr of input series + with self.assertRaises(ValueError): + aggregator.fit( + [self.real_anomalies, self.real_anomalies], self.mts_anomalies1 + ) + with self.assertRaises(ValueError): + aggregator.fit( + [self.real_anomalies, self.real_anomalies], [self.mts_anomalies1] + ) + with self.assertRaises(ValueError): + aggregator.fit( + [self.real_anomalies], [self.mts_anomalies1, self.mts_anomalies1] + ) + + # case1: fit + aggregator.fit(self.real_anomalies, self.mts_anomalies1) + + # Check if _fit_called is True after being fitted + self.assertTrue(aggregator._fit_called) + + # series must be same width as series used for training + with self.assertRaises(ValueError): + aggregator.predict(self.mts_anomalies3) + with self.assertRaises(ValueError): + aggregator.predict([self.mts_anomalies3]) + with self.assertRaises(ValueError): + aggregator.predict([self.mts_anomalies1, self.mts_anomalies3]) + + # predict on (sequence of) univariate series + with self.assertRaises(ValueError): + aggregator.predict([self.real_anomalies]) + with self.assertRaises(ValueError): + aggregator.predict(self.real_anomalies) + with self.assertRaises(ValueError): + aggregator.predict([self.mts_anomalies1, self.real_anomalies]) + + # input a (sequence of) non binary series + with self.assertRaises(ValueError): + aggregator.predict(self.mts_train) + with self.assertRaises(ValueError): + aggregator.predict([self.mts_anomalies1, self.mts_train]) + + # input a (sequence of) probabilistic series + with self.assertRaises(ValueError): + aggregator.predict(self.mts_probabilistic) + with self.assertRaises(ValueError): + aggregator.predict([self.mts_anomalies1, self.mts_probabilistic]) + + # input an element that is not a series + with self.assertRaises(ValueError): + aggregator.predict([self.mts_anomalies1, "random"]) + with self.assertRaises(ValueError): + aggregator.predict([self.mts_anomalies1, 1]) + + # Check width return + # Check if return type is the same number of series in input + self.assertTrue( + len( + aggregator.eval_accuracy( + [self.real_anomalies, self.real_anomalies], + [self.mts_anomalies1, self.mts_anomalies2], + ) + ), + len([self.mts_anomalies1, self.mts_anomalies2]), + ) + + def test_OrAggregator(self): + + aggregator = OrAggregator() + + # simple case + # aggregator must have an accuracy of 0 for input with 2 components + # (only 1 and only 0) and ground truth is only 0 + self.assertAlmostEqual( + aggregator.eval_accuracy( + self.onlyzero, + self.series_1_and_0, + metric="accuracy", + ), + 0, + delta=1e-05, + ) + # aggregator must have an accuracy of 1 for input with 2 components + # (only 1 and only 0) and ground truth is only 1 + self.assertAlmostEqual( + aggregator.eval_accuracy( + self.onlyones, + self.series_1_and_0, + metric="accuracy", + ), + 1, + delta=1e-05, + ) + + # aggregator must have an accuracy of 1 for the input containing only 1 + self.assertAlmostEqual( + aggregator.eval_accuracy( + self.onlyones, + self.mts_onlyones, + metric="accuracy", + ), + 1, + delta=1e-05, + ) + # aggregator must have an accuracy of 1 for the input containing only 1 + self.assertAlmostEqual( + aggregator.eval_accuracy( + self.onlyones, + self.mts_onlyones, + metric="recall", + ), + 1, + delta=1e-05, + ) + # aggregator must have an accuracy of 1 for the input containing only 1 + self.assertAlmostEqual( + aggregator.eval_accuracy( + self.onlyones, + self.mts_onlyones, + metric="precision", + ), + 1, + delta=1e-05, + ) + + # single series case (random example) + # aggregator must found 67 anomalies in the input mts_anomalies1 + self.assertEqual( + aggregator.predict(self.mts_anomalies1) + .sum(axis=0) + .all_values() + .flatten()[0], + 67, + ) + + # aggregator must have an accuracy of 0.56 for the input mts_anomalies1 + self.assertAlmostEqual( + aggregator.eval_accuracy( + self.real_anomalies, + self.mts_anomalies1, + metric="accuracy", + ), + 0.56, + delta=1e-05, + ) + # aggregator must have an recall of 0.72549 for the input mts_anomalies1 + self.assertAlmostEqual( + aggregator.eval_accuracy( + self.real_anomalies, self.mts_anomalies1, metric="recall" + ), + 0.72549, + delta=1e-05, + ) + # aggregator must have an f1 of 0.62711 for the input mts_anomalies1 + self.assertAlmostEqual( + aggregator.eval_accuracy( + self.real_anomalies, self.mts_anomalies1, metric="f1" + ), + 0.62711, + delta=1e-05, + ) + # aggregator must have an precision of 0.55223 for the input mts_anomalies1 + self.assertAlmostEqual( + aggregator.eval_accuracy( + self.real_anomalies, + self.mts_anomalies1, + metric="precision", + ), + 0.55223, + delta=1e-05, + ) + + # multiple series case (random example) + # aggregator must found [67,75] anomalies in the input [mts_anomalies1, mts_anomalies2] + values = aggregator.predict([self.mts_anomalies1, self.mts_anomalies2]) + np.testing.assert_array_almost_equal( + [v.sum(axis=0).all_values().flatten()[0] for v in values], + [67, 75], + decimal=1, + ) + + # aggregator must have an accuracy of [0.56,0.52] for the input [mts_anomalies1, mts_anomalies2] + np.testing.assert_array_almost_equal( + np.array( + aggregator.eval_accuracy( + [self.real_anomalies, self.real_anomalies], + [self.mts_anomalies1, self.mts_anomalies2], + metric="accuracy", + ) + ), + np.array([0.56, 0.52]), + decimal=1, + ) + # aggregator must have an recall of [0.72549,0.764706] for the input [mts_anomalies1, mts_anomalies2] + np.testing.assert_array_almost_equal( + np.array( + aggregator.eval_accuracy( + [self.real_anomalies, self.real_anomalies], + [self.mts_anomalies1, self.mts_anomalies2], + metric="recall", + ) + ), + np.array([0.72549, 0.764706]), + decimal=1, + ) + # aggregator must have an f1 of [0.627119,0.619048] for the input [mts_anomalies1, mts_anomalies2] + np.testing.assert_array_almost_equal( + np.array( + aggregator.eval_accuracy( + [self.real_anomalies, self.real_anomalies], + [self.mts_anomalies1, self.mts_anomalies2], + metric="f1", + ) + ), + np.array([0.627119, 0.619048]), + decimal=1, + ) + # aggregator must have an precision of [0.552239,0.52] for the input [mts_anomalies1, mts_anomalies2] + np.testing.assert_array_almost_equal( + np.array( + aggregator.eval_accuracy( + [self.real_anomalies, self.real_anomalies], + [self.mts_anomalies1, self.mts_anomalies2], + metric="precision", + ) + ), + np.array([0.552239, 0.52]), + decimal=1, + ) + + def test_AndAggregator(self): + + aggregator = AndAggregator() + + # simple case + # aggregator must have an accuracy of 0 for input with 2 components + # (only 1 and only 0) and ground truth is only 1 + self.assertAlmostEqual( + aggregator.eval_accuracy( + self.onlyones, + self.series_1_and_0, + metric="accuracy", + ), + 0, + delta=1e-05, + ) + # aggregator must have an accuracy of 0 for input with 2 components + # (only 1 and only 0) and ground truth is only 0 + self.assertAlmostEqual( + aggregator.eval_accuracy( + self.onlyzero, + self.series_1_and_0, + metric="accuracy", + ), + 1, + delta=1e-05, + ) + + # aggregator must have an accuracy of 1 for the input containing only 1 + self.assertAlmostEqual( + aggregator.eval_accuracy( + self.onlyones, + self.mts_onlyones, + metric="accuracy", + ), + 1, + delta=1e-05, + ) + # aggregator must have an accuracy of 1 for the input containing only 1 + self.assertAlmostEqual( + aggregator.eval_accuracy( + self.onlyones, + self.mts_onlyones, + metric="recall", + ), + 1, + delta=1e-05, + ) + # aggregator must have an accuracy of 1 for the input containing only 1 + self.assertAlmostEqual( + aggregator.eval_accuracy( + self.onlyones, + self.mts_onlyones, + metric="precision", + ), + 1, + delta=1e-05, + ) + + # single series case (random example) + # aggregator must found 27 anomalies in the input mts_anomalies1 + self.assertEqual( + aggregator.predict(self.mts_anomalies1) + .sum(axis=0) + .all_values() + .flatten()[0], + 27, + ) + + # aggregator must have an accuracy of 0.44 for the input mts_anomalies1 + self.assertAlmostEqual( + aggregator.eval_accuracy( + self.real_anomalies, + self.mts_anomalies1, + metric="accuracy", + ), + 0.44, + delta=1e-05, + ) + # aggregator must have an recall of 0.21568 for the input mts_anomalies1 + self.assertAlmostEqual( + aggregator.eval_accuracy( + self.real_anomalies, self.mts_anomalies1, metric="recall" + ), + 0.21568, + delta=1e-05, + ) + # aggregator must have an f1 of 0.28205 for the input mts_anomalies1 + self.assertAlmostEqual( + aggregator.eval_accuracy( + self.real_anomalies, self.mts_anomalies1, metric="f1" + ), + 0.28205, + delta=1e-05, + ) + # aggregator must have an precision of 0.40740 for the input mts_anomalies1 + self.assertAlmostEqual( + aggregator.eval_accuracy( + self.real_anomalies, + self.mts_anomalies1, + metric="precision", + ), + 0.40740, + delta=1e-05, + ) + + # multiple series case (random example) + # aggregator must found [27,24] anomalies in the input [mts_anomalies1, mts_anomalies2] + values = aggregator.predict([self.mts_anomalies1, self.mts_anomalies2]) + np.testing.assert_array_almost_equal( + [v.sum(axis=0).all_values().flatten()[0] for v in values], + [27, 24], + decimal=1, + ) + + # aggregator must have an accuracy of [0.44,0.53] for the input [mts_anomalies1, mts_anomalies2] + np.testing.assert_array_almost_equal( + np.array( + aggregator.eval_accuracy( + [self.real_anomalies, self.real_anomalies], + [self.mts_anomalies1, self.mts_anomalies2], + metric="accuracy", + ) + ), + np.array([0.44, 0.53]), + decimal=1, + ) + # aggregator must have an recall of [0.215686,0.27451] for the input [mts_anomalies1, mts_anomalies2] + np.testing.assert_array_almost_equal( + np.array( + aggregator.eval_accuracy( + [self.real_anomalies, self.real_anomalies], + [self.mts_anomalies1, self.mts_anomalies2], + metric="recall", + ) + ), + np.array([0.215686, 0.27451]), + decimal=1, + ) + # aggregator must have an f1 of [0.282051,0.373333] for the input [mts_anomalies1, mts_anomalies2] + np.testing.assert_array_almost_equal( + np.array( + aggregator.eval_accuracy( + [self.real_anomalies, self.real_anomalies], + [self.mts_anomalies1, self.mts_anomalies2], + metric="f1", + ) + ), + np.array([0.282051, 0.373333]), + decimal=1, + ) + # aggregator must have an precision of [0.407407, 0.583333] for the input [mts_anomalies1, mts_anomalies2] + np.testing.assert_array_almost_equal( + np.array( + aggregator.eval_accuracy( + [self.real_anomalies, self.real_anomalies], + [self.mts_anomalies1, self.mts_anomalies2], + metric="precision", + ) + ), + np.array([0.407407, 0.583333]), + decimal=1, + ) + + def test_EnsembleSklearn(self): + + # Need to input an EnsembleSklearn model + with self.assertRaises(ValueError): + EnsembleSklearnAggregator(model=MovingAverage(window=10)) + + # simple case + # series has 3 components, and real_anomalies_3w is equal to + # - component 1 when component 3 is 1 + # - component 2 when component 3 is 0 + # must have a high accuracy (here 0.92) + aggregator = EnsembleSklearnAggregator( + model=GradientBoostingClassifier( + n_estimators=50, learning_rate=1.0, max_depth=1 + ) + ) + aggregator.fit(self.real_anomalies_3w, self.mts_anomalies3) + + self.assertAlmostEqual( + aggregator.eval_accuracy( + self.real_anomalies_3w, + self.mts_anomalies3, + metric="accuracy", + ), + 0.92, + delta=1e-05, + ) + + np.testing.assert_array_almost_equal( + np.array( + aggregator.eval_accuracy( + [self.real_anomalies_3w, self.real_anomalies_3w], + [self.mts_anomalies3, self.mts_anomalies3], + metric="accuracy", + ) + ), + np.array([0.92, 0.92]), + decimal=1, + ) + + # single series case (random example) + aggregator = EnsembleSklearnAggregator( + model=GradientBoostingClassifier( + n_estimators=50, learning_rate=1.0, max_depth=1 + ) + ) + aggregator.fit(self.real_anomalies, self.mts_anomalies1) + + # aggregator must found 100 anomalies in the input mts_anomalies1 + self.assertEqual( + aggregator.predict(self.mts_anomalies1) + .sum(axis=0) + .all_values() + .flatten()[0], + 100, + ) + + # aggregator must have an accuracy of 0.51 for the input mts_anomalies1 + self.assertAlmostEqual( + aggregator.eval_accuracy( + self.real_anomalies, + self.mts_anomalies1, + metric="accuracy", + ), + 0.51, + delta=1e-05, + ) + # aggregator must have an recall 1.0 for the input mts_anomalies1 + self.assertAlmostEqual( + aggregator.eval_accuracy( + self.real_anomalies, self.mts_anomalies1, metric="recall" + ), + 1.0, + delta=1e-05, + ) + # aggregator must have an f1 of 0.67549 for the input mts_anomalies1 + self.assertAlmostEqual( + aggregator.eval_accuracy( + self.real_anomalies, self.mts_anomalies1, metric="f1" + ), + 0.67549, + delta=1e-05, + ) + # aggregator must have an precision of 0.51 for the input mts_anomalies1 + self.assertAlmostEqual( + aggregator.eval_accuracy( + self.real_anomalies, + self.mts_anomalies1, + metric="precision", + ), + 0.51, + delta=1e-05, + ) + + # multiple series case (random example) + # aggregator must found [100,100] anomalies in the input [mts_anomalies1, mts_anomalies2] + values = aggregator.predict([self.mts_anomalies1, self.mts_anomalies2]) + np.testing.assert_array_almost_equal( + [v.sum(axis=0).all_values().flatten()[0] for v in values], + [100, 100.0], + decimal=1, + ) + + # aggregator must have an accuracy of [0.51, 0.51] for the input [mts_anomalies1, mts_anomalies2] + np.testing.assert_array_almost_equal( + np.array( + aggregator.eval_accuracy( + [self.real_anomalies, self.real_anomalies], + [self.mts_anomalies1, self.mts_anomalies2], + metric="accuracy", + ) + ), + np.array([0.51, 0.51]), + decimal=1, + ) + # aggregator must have an recall of [1,1] for the input [mts_anomalies1, mts_anomalies2] + np.testing.assert_array_almost_equal( + np.array( + aggregator.eval_accuracy( + [self.real_anomalies, self.real_anomalies], + [self.mts_anomalies1, self.mts_anomalies2], + metric="recall", + ) + ), + np.array([1, 1]), + decimal=1, + ) + # aggregator must have an f1 of [0.675497, 0.675497] for the input [mts_anomalies1, mts_anomalies2] + np.testing.assert_array_almost_equal( + np.array( + aggregator.eval_accuracy( + [self.real_anomalies, self.real_anomalies], + [self.mts_anomalies1, self.mts_anomalies2], + metric="f1", + ) + ), + np.array([0.675497, 0.675497]), + decimal=1, + ) + # aggregator must have an precision of [0.51, 0.51] for the input [mts_anomalies1, mts_anomalies2] + np.testing.assert_array_almost_equal( + np.array( + aggregator.eval_accuracy( + [self.real_anomalies, self.real_anomalies], + [self.mts_anomalies1, self.mts_anomalies2], + metric="precision", + ) + ), + np.array([0.51, 0.51]), + decimal=1, + ) diff --git a/darts/tests/ad/test_anomaly_model.py b/darts/tests/ad/test_anomaly_model.py new file mode 100644 index 0000000000..ebc7972130 --- /dev/null +++ b/darts/tests/ad/test_anomaly_model.py @@ -0,0 +1,1571 @@ +from typing import Dict, Sequence, Tuple + +import numpy as np +import pandas as pd +from pyod.models.knn import KNN + +from darts import TimeSeries + +# anomaly aggregators +# import everything in darts.ad (also for testing imports) +from darts.ad import AndAggregator # noqa: F401 +from darts.ad import EnsembleSklearnAggregator # noqa: F401 +from darts.ad import OrAggregator # noqa: F401 +from darts.ad import QuantileDetector # noqa: F401 +from darts.ad import ThresholdDetector # noqa: F401 +from darts.ad import CauchyNLLScorer +from darts.ad import DifferenceScorer as Difference +from darts.ad import ( + ExponentialNLLScorer, + FilteringAnomalyModel, + ForecastingAnomalyModel, + GammaNLLScorer, + GaussianNLLScorer, + KMeansScorer, + LaplaceNLLScorer, + NormScorer, + PoissonNLLScorer, + PyODScorer, + WassersteinScorer, +) +from darts.ad.utils import eval_accuracy_from_scores, show_anomalies_from_scores +from darts.models import MovingAverage, NaiveSeasonal, RegressionModel +from darts.tests.base_test_class import DartsBaseTestClass + + +class ADAnomalyModelTestCase(DartsBaseTestClass): + np.random.seed(42) + + # univariate series + np_train = np.random.normal(loc=10, scale=0.5, size=100) + train = TimeSeries.from_values(np_train) + + np_covariates = np.random.choice(a=[0, 1], size=100, p=[0.5, 0.5]) + covariates = TimeSeries.from_times_and_values(train._time_index, np_covariates) + + np_test = np.random.normal(loc=10, scale=1, size=100) + test = TimeSeries.from_times_and_values(train._time_index, np_test) + + np_anomalies = np.random.choice(a=[0, 1], size=100, p=[0.9, 0.1]) + anomalies = TimeSeries.from_times_and_values(train._time_index, np_anomalies) + + np_only_1_anomalies = np.random.choice(a=[0, 1], size=100, p=[0, 1]) + only_1_anomalies = TimeSeries.from_times_and_values( + train._time_index, np_only_1_anomalies + ) + + np_only_0_anomalies = np.random.choice(a=[0, 1], size=100, p=[1, 0]) + only_0_anomalies = TimeSeries.from_times_and_values( + train._time_index, np_only_0_anomalies + ) + + modified_train = MovingAverage(window=10).filter(train) + modified_test = MovingAverage(window=10).filter(test) + + np_probabilistic = np.random.normal(loc=10, scale=1, size=[100, 1, 20]) + probabilistic = TimeSeries.from_times_and_values( + train._time_index, np_probabilistic + ) + + # multivariate series + np_mts_train = np.random.normal(loc=[10, 5], scale=[0.5, 1], size=[100, 2]) + mts_train = TimeSeries.from_values(np_mts_train) + + np_mts_test = np.random.normal(loc=[10, 5], scale=[1, 1.5], size=[100, 2]) + mts_test = TimeSeries.from_times_and_values(mts_train._time_index, np_mts_test) + + np_mts_anomalies = np.random.choice(a=[0, 1], size=[100, 2], p=[0.9, 0.1]) + mts_anomalies = TimeSeries.from_times_and_values( + mts_train._time_index, np_mts_anomalies + ) + + def test_Scorer(self): + + list_NonFittableAnomalyScorer = [ + NormScorer(), + Difference(), + GaussianNLLScorer(), + ExponentialNLLScorer(), + PoissonNLLScorer(), + LaplaceNLLScorer(), + CauchyNLLScorer(), + GammaNLLScorer(), + ] + + for scorers in list_NonFittableAnomalyScorer: + for anomaly_model in [ + ForecastingAnomalyModel(model=RegressionModel(lags=10), scorer=scorers), + FilteringAnomalyModel(model=MovingAverage(window=20), scorer=scorers), + ]: + + # scorer are trainable + self.assertTrue(anomaly_model.scorers_are_trainable is False) + + list_FittableAnomalyScorer = [ + PyODScorer(model=KNN()), + KMeansScorer(), + WassersteinScorer(), + ] + + for scorers in list_FittableAnomalyScorer: + for anomaly_model in [ + ForecastingAnomalyModel(model=RegressionModel(lags=10), scorer=scorers), + FilteringAnomalyModel(model=MovingAverage(window=20), scorer=scorers), + ]: + + # scorer are not trainable + self.assertTrue(anomaly_model.scorers_are_trainable is True) + + def test_Score(self): + + am1 = ForecastingAnomalyModel( + model=RegressionModel(lags=10), scorer=NormScorer() + ) + am1.fit(self.train, allow_model_training=True) + + am2 = FilteringAnomalyModel(model=MovingAverage(window=20), scorer=NormScorer()) + + for am in [am1, am2]: + # Parameter return_model_prediction + # parameter return_model_prediction must be bool + with self.assertRaises(ValueError): + am.score(self.test, return_model_prediction=1) + with self.assertRaises(ValueError): + am.score(self.test, return_model_prediction="True") + + # if return_model_prediction set to true, output must be tuple + self.assertTrue( + isinstance(am.score(self.test, return_model_prediction=True), Tuple) + ) + + # if return_model_prediction set to false output must be + # Union[TimeSeries, Sequence[TimeSeries], Sequence[Sequence[TimeSeries]]] + self.assertTrue( + not isinstance( + am.score(self.test, return_model_prediction=False), Tuple + ) + ) + + def test_FitFilteringAnomalyModelInput(self): + + for anomaly_model in [ + FilteringAnomalyModel(model=MovingAverage(window=20), scorer=NormScorer()), + FilteringAnomalyModel( + model=MovingAverage(window=20), scorer=[NormScorer(), KMeansScorer()] + ), + FilteringAnomalyModel( + model=MovingAverage(window=20), scorer=KMeansScorer() + ), + ]: + + # filter must be fittable if allow_filter_training is set to True + with self.assertRaises(ValueError): + anomaly_model.fit(self.train, allow_model_training=True) + + # input 'series' must be a series or Sequence of series + with self.assertRaises(ValueError): + anomaly_model.fit([self.train, "str"], allow_model_training=True) + with self.assertRaises(ValueError): + anomaly_model.fit([[self.train, self.train]], allow_model_training=True) + with self.assertRaises(ValueError): + anomaly_model.fit("str", allow_model_training=True) + with self.assertRaises(ValueError): + anomaly_model.fit([1, 2, 3], allow_model_training=True) + + # allow_model_training must be a bool + with self.assertRaises(ValueError): + anomaly_model.fit(self.train, allow_model_training=1) + with self.assertRaises(ValueError): + anomaly_model.fit(self.train, allow_model_training="True") + + def test_FitForecastingAnomalyModelInput(self): + + for anomaly_model in [ + ForecastingAnomalyModel( + model=RegressionModel(lags=10), scorer=NormScorer() + ), + ForecastingAnomalyModel( + model=RegressionModel(lags=10), scorer=[NormScorer(), KMeansScorer()] + ), + ForecastingAnomalyModel( + model=RegressionModel(lags=10), scorer=KMeansScorer() + ), + ]: + + # input 'series' must be a series or Sequence of series + with self.assertRaises(ValueError): + anomaly_model.fit([self.train, "str"], allow_model_training=True) + with self.assertRaises(ValueError): + anomaly_model.fit([[self.train, self.train]], allow_model_training=True) + with self.assertRaises(ValueError): + anomaly_model.fit("str", allow_model_training=True) + with self.assertRaises(ValueError): + anomaly_model.fit([1, 2, 3], allow_model_training=True) + + # allow_model_training must be a bool + with self.assertRaises(ValueError): + anomaly_model.fit(self.train, allow_model_training=1) + with self.assertRaises(ValueError): + anomaly_model.fit(self.train, allow_model_training="True") + + # 'allow_model_training' must be set to True if forecasting model is not fitted + if anomaly_model.scorers_are_trainable: + with self.assertRaises(ValueError): + anomaly_model.fit(self.train, allow_model_training=False) + anomaly_model.score(self.train) + + with self.assertRaises(ValueError): + # number of 'past_covariates' must be the same as the number of Timeseries in 'series' + anomaly_model.fit( + series=[self.train, self.train], + past_covariates=self.covariates, + allow_model_training=True, + ) + + with self.assertRaises(ValueError): + # number of 'past_covariates' must be the same as the number of Timeseries in 'series' + anomaly_model.fit( + series=self.train, + past_covariates=[self.covariates, self.covariates], + allow_model_training=True, + ) + + with self.assertRaises(ValueError): + # number of 'future_covariates' must be the same as the number of Timeseries in 'series' + anomaly_model.fit( + series=[self.train, self.train], + future_covariates=self.covariates, + allow_model_training=True, + ) + + with self.assertRaises(ValueError): + # number of 'future_covariates' must be the same as the number of Timeseries in 'series' + anomaly_model.fit( + series=self.train, + future_covariates=[self.covariates, self.covariates], + allow_model_training=True, + ) + + fitted_model = RegressionModel(lags=10).fit(self.train) + # Fittable scorer must be fitted before calling .score(), even if forecasting model is fitted + with self.assertRaises(ValueError): + ForecastingAnomalyModel(model=fitted_model, scorer=KMeansScorer()).score( + series=self.test + ) + with self.assertRaises(ValueError): + ForecastingAnomalyModel( + model=fitted_model, scorer=[NormScorer(), KMeansScorer()] + ).score(series=self.test) + + # forecasting model that do not accept past/future covariates + # with self.assertRaises(ValueError): + # ForecastingAnomalyModel(model=ExponentialSmoothing(), + # scorer=NormScorer()).fit( + # series=self.train, past_covariates=self.covariates, allow_model_training=True + # ) + # with self.assertRaises(ValueError): + # ForecastingAnomalyModel(model=ExponentialSmoothing(), + # scorer=NormScorer()).fit( + # series=self.train, future_covariates=self.covariates, allow_model_training=True + # ) + + # check window size + # max window size is len(series.drop_before(series.get_timestamp_at_point(start))) + 1 + with self.assertRaises(ValueError): + ForecastingAnomalyModel( + model=RegressionModel(lags=10), scorer=KMeansScorer(window=50) + ).fit(series=self.train, start=0.9) + + # forecasting model that cannot be trained on a list of series + with self.assertRaises(ValueError): + ForecastingAnomalyModel(model=NaiveSeasonal(), scorer=NormScorer()).fit( + series=[self.train, self.train], allow_model_training=True + ) + + def test_ScoreForecastingAnomalyModelInput(self): + + for anomaly_model in [ + ForecastingAnomalyModel( + model=RegressionModel(lags=10), scorer=NormScorer() + ), + ForecastingAnomalyModel( + model=RegressionModel(lags=10), scorer=[NormScorer(), KMeansScorer()] + ), + ForecastingAnomalyModel( + model=RegressionModel(lags=10), scorer=KMeansScorer() + ), + ]: + + anomaly_model.fit(self.train, allow_model_training=True) + + # number of 'past_covariates' must be the same as the number of Timeseries in 'series' + with self.assertRaises(ValueError): + anomaly_model.score( + series=[self.train, self.train], past_covariates=self.covariates + ) + + # number of 'past_covariates' must be the same as the number of Timeseries in 'series' + with self.assertRaises(ValueError): + anomaly_model.score( + series=self.train, + past_covariates=[self.covariates, self.covariates], + ) + + # number of 'future_covariates' must be the same as the number of Timeseries in 'series' + with self.assertRaises(ValueError): + anomaly_model.score( + series=[self.train, self.train], future_covariates=self.covariates + ) + + # number of 'future_covariates' must be the same as the number of Timeseries in 'series' + with self.assertRaises(ValueError): + anomaly_model.score( + series=self.train, + future_covariates=[self.covariates, self.covariates], + ) + + # check window size + # max window size is len(series.drop_before(series.get_timestamp_at_point(start))) + 1 for score() + anomaly_model = ForecastingAnomalyModel( + model=RegressionModel(lags=10), scorer=KMeansScorer(window=30) + ) + anomaly_model.fit(self.train, allow_model_training=True) + with self.assertRaises(ValueError): + anomaly_model.score(series=self.train, start=0.9) + + def test_ScoreFilteringAnomalyModelInput(self): + + for anomaly_model in [ + FilteringAnomalyModel(model=MovingAverage(window=10), scorer=NormScorer()), + FilteringAnomalyModel( + model=MovingAverage(window=10), scorer=[NormScorer(), KMeansScorer()] + ), + FilteringAnomalyModel( + model=MovingAverage(window=10), scorer=KMeansScorer() + ), + ]: + + if anomaly_model.scorers_are_trainable: + anomaly_model.fit(self.train) + + def test_eval_accuracy(self): + + am1 = ForecastingAnomalyModel( + model=RegressionModel(lags=10), scorer=NormScorer() + ) + am1.fit(self.train, allow_model_training=True) + + am2 = FilteringAnomalyModel(model=MovingAverage(window=20), scorer=NormScorer()) + + am3 = ForecastingAnomalyModel( + model=RegressionModel(lags=10), scorer=[NormScorer(), WassersteinScorer()] + ) + am3.fit(self.train, allow_model_training=True) + + am4 = FilteringAnomalyModel( + model=MovingAverage(window=20), scorer=[NormScorer(), WassersteinScorer()] + ) + am4.fit(self.train) + + for am in [am1, am2, am3, am4]: + + # if the anomaly_model have scorers that have the parameter univariate_scorer set to True, + # 'actual_anomalies' must have widths of 1 + if am.univariate_scoring: + with self.assertRaises(ValueError): + am.eval_accuracy( + actual_anomalies=self.mts_anomalies, series=self.test + ) + with self.assertRaises(ValueError): + am.eval_accuracy( + actual_anomalies=self.mts_anomalies, series=self.mts_test + ) + with self.assertRaises(ValueError): + am.eval_accuracy( + actual_anomalies=[self.anomalies, self.mts_anomalies], + series=[self.test, self.mts_test], + ) + + # 'metric' must be str and "AUC_ROC" or "AUC_PR" + with self.assertRaises(ValueError): + am.eval_accuracy( + actual_anomalies=self.anomalies, series=self.test, metric=1 + ) + with self.assertRaises(ValueError): + am.eval_accuracy( + actual_anomalies=self.anomalies, series=self.test, metric="auc_roc" + ) + with self.assertRaises(TypeError): + am.eval_accuracy( + actual_anomalies=self.anomalies, + series=self.test, + metric=["AUC_ROC"], + ) + + # 'actual_anomalies' must be binary + with self.assertRaises(ValueError): + am.eval_accuracy(actual_anomalies=self.test, series=self.test) + + # 'actual_anomalies' must contain anomalies (at least one) + with self.assertRaises(ValueError): + am.eval_accuracy( + actual_anomalies=self.only_0_anomalies, series=self.test + ) + + # 'actual_anomalies' cannot contain only anomalies + with self.assertRaises(ValueError): + am.eval_accuracy( + actual_anomalies=self.only_1_anomalies, series=self.test + ) + + # 'actual_anomalies' must match the number of series + with self.assertRaises(ValueError): + am.eval_accuracy( + actual_anomalies=self.anomalies, series=[self.test, self.test] + ) + with self.assertRaises(ValueError): + am.eval_accuracy( + actual_anomalies=[self.anomalies, self.anomalies], series=self.test + ) + + # 'actual_anomalies' must have non empty intersection with 'series' + with self.assertRaises(ValueError): + am.eval_accuracy( + actual_anomalies=self.anomalies[:20], series=self.test[30:] + ) + with self.assertRaises(ValueError): + am.eval_accuracy( + actual_anomalies=[self.anomalies, self.anomalies[:20]], + series=[self.test, self.test[40:]], + ) + + # Check input type + # 'actual_anomalies' and 'series' must be of same length + with self.assertRaises(ValueError): + am.eval_accuracy([self.anomalies], [self.test, self.test]) + with self.assertRaises(ValueError): + am.eval_accuracy(self.anomalies, [self.test, self.test]) + with self.assertRaises(ValueError): + am.eval_accuracy([self.anomalies, self.anomalies], [self.test]) + with self.assertRaises(ValueError): + am.eval_accuracy([self.anomalies, self.anomalies], self.test) + + # 'actual_anomalies' and 'series' must be of type Timeseries + with self.assertRaises(ValueError): + am.eval_accuracy([self.anomalies], [2, 3, 4]) + with self.assertRaises(ValueError): + am.eval_accuracy([self.anomalies], "str") + with self.assertRaises(ValueError): + am.eval_accuracy([2, 3, 4], self.test) + with self.assertRaises(ValueError): + am.eval_accuracy("str", self.test) + with self.assertRaises(ValueError): + am.eval_accuracy( + [self.anomalies, self.anomalies], [self.test, [3, 2, 1]] + ) + with self.assertRaises(ValueError): + am.eval_accuracy([self.anomalies, [3, 2, 1]], [self.test, self.test]) + + # Check return types + # Check if return type is float when input is a series + self.assertTrue( + isinstance( + am.eval_accuracy(self.anomalies, self.test), + Dict, + ) + ) + + # Check if return type is Sequence when input is a Sequence of series + self.assertTrue( + isinstance( + am.eval_accuracy(self.anomalies, [self.test]), + Sequence, + ) + ) + self.assertTrue( + isinstance( + am.eval_accuracy( + [self.anomalies, self.anomalies], [self.test, self.test] + ), + Sequence, + ) + ) + + def test_ForecastingAnomalyModelInput(self): + + # model input + # model input must be of type ForecastingModel + with self.assertRaises(ValueError): + ForecastingAnomalyModel(model="str", scorer=NormScorer()) + with self.assertRaises(ValueError): + ForecastingAnomalyModel(model=1, scorer=NormScorer()) + with self.assertRaises(ValueError): + ForecastingAnomalyModel(model=MovingAverage(window=10), scorer=NormScorer()) + with self.assertRaises(ValueError): + ForecastingAnomalyModel( + model=[RegressionModel(lags=10), RegressionModel(lags=5)], + scorer=NormScorer(), + ) + + # scorer input + # scorer input must be of type AnomalyScorer + with self.assertRaises(ValueError): + ForecastingAnomalyModel(model=RegressionModel(lags=10), scorer=1) + with self.assertRaises(ValueError): + ForecastingAnomalyModel(model=RegressionModel(lags=10), scorer="str") + with self.assertRaises(ValueError): + ForecastingAnomalyModel( + model=RegressionModel(lags=10), scorer=RegressionModel(lags=10) + ) + with self.assertRaises(ValueError): + ForecastingAnomalyModel( + model=RegressionModel(lags=10), scorer=[NormScorer(), "str"] + ) + + def test_FilteringAnomalyModelInput(self): + + # model input + # model input must be of type FilteringModel + with self.assertRaises(ValueError): + FilteringAnomalyModel(model="str", scorer=NormScorer()) + with self.assertRaises(ValueError): + FilteringAnomalyModel(model=1, scorer=NormScorer()) + with self.assertRaises(ValueError): + FilteringAnomalyModel(model=RegressionModel(lags=10), scorer=NormScorer()) + with self.assertRaises(ValueError): + FilteringAnomalyModel( + model=[MovingAverage(window=10), MovingAverage(window=10)], + scorer=NormScorer(), + ) + + # scorer input + # scorer input must be of type AnomalyScorer + with self.assertRaises(ValueError): + FilteringAnomalyModel(model=MovingAverage(window=10), scorer=1) + with self.assertRaises(ValueError): + FilteringAnomalyModel(model=MovingAverage(window=10), scorer="str") + with self.assertRaises(ValueError): + FilteringAnomalyModel( + model=MovingAverage(window=10), scorer=MovingAverage(window=10) + ) + with self.assertRaises(ValueError): + FilteringAnomalyModel( + model=MovingAverage(window=10), scorer=[NormScorer(), "str"] + ) + + def test_univariate_ForecastingAnomalyModel(self): + + np.random.seed(40) + + np_train_slope = np.array(range(0, 100, 1)) + np_test_slope = np.array(range(0, 100, 1)) + + np_test_slope[30:32] = 29 + np_test_slope[50:65] = np_test_slope[50:65] + 1 + np_test_slope[75:80] = np_test_slope[75:80] * 0.98 + + train_series_slope = TimeSeries.from_values(np_train_slope) + test_series_slope = TimeSeries.from_values(np_test_slope) + + np_anomalies = np.zeros(100) + np_anomalies[30:32] = 1 + np_anomalies[50:55] = 1 + np_anomalies[70:80] = 1 + ts_anomalies = TimeSeries.from_times_and_values( + test_series_slope.time_index, np_anomalies, columns=["is_anomaly"] + ) + + anomaly_model = ForecastingAnomalyModel( + model=RegressionModel(lags=5), + scorer=[ + NormScorer(), + Difference(), + WassersteinScorer(), + KMeansScorer(), + KMeansScorer(window=10), + PyODScorer(model=KNN()), + PyODScorer(model=KNN(), window=10), + WassersteinScorer(window=15), + ], + ) + + anomaly_model.fit(train_series_slope, allow_model_training=True, start=0.1) + score, model_output = anomaly_model.score( + test_series_slope, return_model_prediction=True, start=0.1 + ) + + # check that NormScorer is the abs difference of model_output and test_series_slope + self.assertEqual( + (model_output - test_series_slope.slice_intersect(model_output)).__abs__(), + NormScorer().score_from_prediction(test_series_slope, model_output), + ) + + # check that Difference is the difference of model_output and test_series_slope + self.assertEqual( + test_series_slope.slice_intersect(model_output) - model_output, + Difference().score_from_prediction(test_series_slope, model_output), + ) + + dict_auc_roc = anomaly_model.eval_accuracy( + ts_anomalies, test_series_slope, metric="AUC_ROC", start=0.1 + ) + dict_auc_pr = anomaly_model.eval_accuracy( + ts_anomalies, test_series_slope, metric="AUC_PR", start=0.1 + ) + + auc_roc_from_scores = eval_accuracy_from_scores( + actual_anomalies=[ts_anomalies] * 8, + anomaly_score=score, + window=[1, 1, 10, 1, 10, 1, 10, 15], + metric="AUC_ROC", + ) + + auc_pr_from_scores = eval_accuracy_from_scores( + actual_anomalies=[ts_anomalies] * 8, + anomaly_score=score, + window=[1, 1, 10, 1, 10, 1, 10, 15], + metric="AUC_PR", + ) + + # function eval_accuracy_from_scores and eval_accuracy must return an input of same length + self.assertEqual(len(auc_roc_from_scores), len(dict_auc_roc)) + self.assertEqual(len(auc_pr_from_scores), len(dict_auc_pr)) + + # function eval_accuracy_from_scores and eval_accuracy must return the same values + np.testing.assert_array_almost_equal( + auc_roc_from_scores, list(dict_auc_roc.values()), decimal=1 + ) + np.testing.assert_array_almost_equal( + auc_pr_from_scores, list(dict_auc_pr.values()), decimal=1 + ) + + true_auc_roc = [ + 0.773449920508744, + 0.40659777424483307, + 0.9153708133971291, + 0.7702702702702702, + 0.9135765550239234, + 0.7603338632750397, + 0.9153708133971292, + 0.9006591337099811, + ] + + true_auc_pr = [ + 0.4818991248542174, + 0.20023033665128342, + 0.9144135170539835, + 0.47953161438253644, + 0.9127969832903458, + 0.47039678636225957, + 0.9147124232933175, + 0.9604714100445533, + ] + + # check value of results + np.testing.assert_array_almost_equal( + auc_roc_from_scores, true_auc_roc, decimal=1 + ) + np.testing.assert_array_almost_equal(auc_pr_from_scores, true_auc_pr, decimal=1) + + def test_univariate_FilteringAnomalyModel(self): + + np.random.seed(40) + + np_series_train = np.array(range(0, 100, 1)) + np.random.normal( + loc=0, scale=1, size=100 + ) + np_series_test = np.array(range(0, 100, 1)) + np.random.normal( + loc=0, scale=1, size=100 + ) + + np_series_test[30:35] = np_series_test[30:35] + np.random.normal( + loc=0, scale=10, size=5 + ) + np_series_test[50:60] = np_series_test[50:60] + np.random.normal( + loc=0, scale=4, size=10 + ) + np_series_test[75:80] = np_series_test[75:80] + np.random.normal( + loc=0, scale=3, size=5 + ) + + train_series_noise = TimeSeries.from_values(np_series_train) + test_series_noise = TimeSeries.from_values(np_series_test) + + np_anomalies = np.zeros(100) + np_anomalies[30:35] = 1 + np_anomalies[50:60] = 1 + np_anomalies[75:80] = 1 + ts_anomalies = TimeSeries.from_times_and_values( + test_series_noise.time_index, np_anomalies, columns=["is_anomaly"] + ) + + anomaly_model = FilteringAnomalyModel( + model=MovingAverage(window=5), + scorer=[ + NormScorer(), + Difference(), + WassersteinScorer(), + KMeansScorer(), + KMeansScorer(window=10), + PyODScorer(model=KNN()), + PyODScorer(model=KNN(), window=10), + WassersteinScorer(window=15), + ], + ) + anomaly_model.fit(train_series_noise) + score, model_output = anomaly_model.score( + test_series_noise, return_model_prediction=True + ) + + # check that Difference is the difference of model_output and test_series_noise + self.assertEqual( + test_series_noise.slice_intersect(model_output) - model_output, + Difference().score_from_prediction(test_series_noise, model_output), + ) + + # check that NormScorer is the abs difference of model_output and test_series_noise + self.assertEqual( + (test_series_noise.slice_intersect(model_output) - model_output).__abs__(), + NormScorer().score_from_prediction(test_series_noise, model_output), + ) + + dict_auc_roc = anomaly_model.eval_accuracy( + ts_anomalies, test_series_noise, metric="AUC_ROC" + ) + dict_auc_pr = anomaly_model.eval_accuracy( + ts_anomalies, test_series_noise, metric="AUC_PR" + ) + + auc_roc_from_scores = eval_accuracy_from_scores( + actual_anomalies=[ts_anomalies] * 8, + anomaly_score=score, + window=[1, 1, 10, 1, 10, 1, 10, 15], + metric="AUC_ROC", + ) + + auc_pr_from_scores = eval_accuracy_from_scores( + actual_anomalies=[ts_anomalies] * 8, + anomaly_score=score, + window=[1, 1, 10, 1, 10, 1, 10, 15], + metric="AUC_PR", + ) + + # function eval_accuracy_from_scores and eval_accuracy must return an input of same length + self.assertEqual(len(auc_roc_from_scores), len(dict_auc_roc)) + self.assertEqual(len(auc_pr_from_scores), len(dict_auc_pr)) + + # function eval_accuracy_from_scores and eval_accuracy must return the same values + np.testing.assert_array_almost_equal( + auc_roc_from_scores, list(dict_auc_roc.values()), decimal=1 + ) + np.testing.assert_array_almost_equal( + auc_pr_from_scores, list(dict_auc_pr.values()), decimal=1 + ) + + true_auc_roc = [ + 0.875625, + 0.5850000000000001, + 0.952127659574468, + 0.814375, + 0.9598646034816247, + 0.88125, + 0.9666344294003868, + 0.9731182795698925, + ] + + true_auc_pr = [ + 0.7691407907338141, + 0.5566414178265074, + 0.9720504927710986, + 0.741298584352156, + 0.9744855592642071, + 0.7808056518442923, + 0.9800621192517156, + 0.9911842778990486, + ] + + # check value of results + np.testing.assert_array_almost_equal( + auc_roc_from_scores, true_auc_roc, decimal=1 + ) + np.testing.assert_array_almost_equal(auc_pr_from_scores, true_auc_pr, decimal=1) + + def test_univariate_covariate_ForecastingAnomalyModel(self): + + np.random.seed(40) + + day_week = [0, 1, 2, 3, 4, 5, 6] + np_day_week = np.array(day_week * 10) + + np_train_series = 0.5 * np_day_week + np_test_series = 0.5 * np_day_week + + np_test_series[30:35] = np_test_series[30:35] + np.random.normal( + loc=0, scale=2, size=5 + ) + np_test_series[50:60] = np_test_series[50:60] + np.random.normal( + loc=0, scale=1, size=10 + ) + + covariates = TimeSeries.from_times_and_values( + pd.date_range(start="1949-01-01", end="1949-03-11"), np_day_week + ) + series_train = TimeSeries.from_times_and_values( + pd.date_range(start="1949-01-01", end="1949-03-11"), np_train_series + ) + series_test = TimeSeries.from_times_and_values( + pd.date_range(start="1949-01-01", end="1949-03-11"), np_test_series + ) + + np_anomalies = np.zeros(70) + np_anomalies[30:35] = 1 + np_anomalies[50:60] = 1 + ts_anomalies = TimeSeries.from_times_and_values( + series_test.time_index, np_anomalies, columns=["is_anomaly"] + ) + + anomaly_model = ForecastingAnomalyModel( + model=RegressionModel(lags=2, lags_future_covariates=[0]), + scorer=[ + NormScorer(), + Difference(), + WassersteinScorer(), + KMeansScorer(), + KMeansScorer(window=10), + PyODScorer(model=KNN()), + PyODScorer(model=KNN(), window=10), + WassersteinScorer(window=15), + ], + ) + + anomaly_model.fit( + series_train, + allow_model_training=True, + future_covariates=covariates, + start=0.2, + ) + + score, model_output = anomaly_model.score( + series_test, + return_model_prediction=True, + future_covariates=covariates, + start=0.2, + ) + + # check that NormScorer is the abs difference of model_output and series_test + self.assertEqual( + (series_test.slice_intersect(model_output) - model_output).__abs__(), + NormScorer().score_from_prediction(series_test, model_output), + ) + + # check that Difference is the difference of model_output and series_test + self.assertEqual( + series_test.slice_intersect(model_output) - model_output, + Difference().score_from_prediction(series_test, model_output), + ) + + dict_auc_roc = anomaly_model.eval_accuracy( + ts_anomalies, series_test, metric="AUC_ROC", start=0.2 + ) + dict_auc_pr = anomaly_model.eval_accuracy( + ts_anomalies, series_test, metric="AUC_PR", start=0.2 + ) + + auc_roc_from_scores = eval_accuracy_from_scores( + actual_anomalies=[ts_anomalies] * 8, + anomaly_score=score, + window=[1, 1, 10, 1, 10, 1, 10, 15], + metric="AUC_ROC", + ) + + auc_pr_from_scores = eval_accuracy_from_scores( + actual_anomalies=[ts_anomalies] * 8, + anomaly_score=score, + window=[1, 1, 10, 1, 10, 1, 10, 15], + metric="AUC_PR", + ) + + # function eval_accuracy_from_scores and eval_accuracy must return an input of same length + self.assertEqual(len(auc_roc_from_scores), len(dict_auc_roc)) + self.assertEqual(len(auc_pr_from_scores), len(dict_auc_pr)) + + # function eval_accuracy_from_scores and eval_accuracy must return the same values + np.testing.assert_array_almost_equal( + auc_roc_from_scores, list(dict_auc_roc.values()), decimal=1 + ) + np.testing.assert_array_almost_equal( + auc_pr_from_scores, list(dict_auc_pr.values()), decimal=1 + ) + + true_auc_roc = [1.0, 0.6, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0] + + true_auc_pr = [ + 1.0, + 0.6914399076961142, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 0.9999999999999999, + ] + + # check value of results + np.testing.assert_array_almost_equal( + auc_roc_from_scores, true_auc_roc, decimal=1 + ) + np.testing.assert_array_almost_equal(auc_pr_from_scores, true_auc_pr, decimal=1) + + def test_multivariate__FilteringAnomalyModel(self): + + np.random.seed(40) + + data_1 = np.random.normal(0, 0.1, 100) + data_2 = np.random.normal(0, 0.1, 100) + + mts_series_train = TimeSeries.from_values( + np.dstack((data_1, data_2))[0], columns=["component 1", "component 2"] + ) + + data_1[15:20] = data_1[15:20] + np.random.normal(0, 0.9, 5) + data_1[35:40] = data_1[35:40] + np.random.normal(0, 0.4, 5) + + data_2[50:55] = data_2[50:55] + np.random.normal(0, 0.7, 5) + data_2[65:70] = data_2[65:70] + np.random.normal(0, 0.4, 5) + + data_1[80:85] = data_1[80:85] + np.random.normal(0, 0.6, 5) + data_2[80:85] = data_2[80:85] + np.random.normal(0, 0.6, 5) + + data_1[93:98] = data_1[93:98] + np.random.normal(0, 0.6, 5) + data_2[93:98] = data_2[93:98] + np.random.normal(0, 0.6, 5) + mts_series_test = TimeSeries.from_values( + np.dstack((data_1, data_2))[0], columns=["component 1", "component 2"] + ) + + np1_anomalies = np.zeros(len(data_1)) + np1_anomalies[15:20] = 1 + np1_anomalies[35:40] = 1 + np1_anomalies[80:85] = 1 + np1_anomalies[93:98] = 1 + + np2_anomalies = np.zeros(len(data_2)) + np2_anomalies[50:55] = 1 + np2_anomalies[67:70] = 1 + np2_anomalies[80:85] = 1 + np2_anomalies[93:98] = 1 + + np_anomalies = np.zeros(len(data_2)) + np_anomalies[15:20] = 1 + np_anomalies[35:40] = 1 + np_anomalies[50:55] = 1 + np_anomalies[67:70] = 1 + np_anomalies[80:85] = 1 + np_anomalies[93:98] = 1 + + ts_anomalies = TimeSeries.from_times_and_values( + mts_series_train.time_index, + np.dstack((np1_anomalies, np2_anomalies))[0], + columns=["is_anomaly_1", "is_anomaly_2"], + ) + + mts_anomalies = TimeSeries.from_times_and_values( + mts_series_train.time_index, np_anomalies, columns=["is_anomaly"] + ) + + # first case: scorers that return univariate scores + anomaly_model = FilteringAnomalyModel( + model=MovingAverage(window=10), + scorer=[ + NormScorer(component_wise=False), + WassersteinScorer(), + WassersteinScorer(window=12), + KMeansScorer(), + KMeansScorer(window=5), + PyODScorer(model=KNN()), + PyODScorer(model=KNN(), window=5), + ], + ) + anomaly_model.fit(mts_series_train) + + scores, model_output = anomaly_model.score( + mts_series_test, return_model_prediction=True + ) + + # model_output must be multivariate (same width as input) + self.assertEqual(model_output.width, mts_series_test.width) + + # scores must be of the same length as the number of scorers + self.assertEqual(len(scores), len(anomaly_model.scorers)) + + dict_auc_roc = anomaly_model.eval_accuracy( + mts_anomalies, mts_series_test, metric="AUC_ROC" + ) + dict_auc_pr = anomaly_model.eval_accuracy( + mts_anomalies, mts_series_test, metric="AUC_PR" + ) + + auc_roc_from_scores = eval_accuracy_from_scores( + actual_anomalies=[mts_anomalies] * 7, + anomaly_score=scores, + window=[1, 10, 12, 1, 5, 1, 5], + metric="AUC_ROC", + ) + + auc_pr_from_scores = eval_accuracy_from_scores( + actual_anomalies=[mts_anomalies] * 7, + anomaly_score=scores, + window=[1, 10, 12, 1, 5, 1, 5], + metric="AUC_PR", + ) + + # function eval_accuracy_from_scores and eval_accuracy must return an input of same length + self.assertEqual(len(auc_roc_from_scores), len(dict_auc_roc)) + self.assertEqual(len(auc_pr_from_scores), len(dict_auc_pr)) + + # function eval_accuracy_from_scores and eval_accuracy must return the same values + np.testing.assert_array_almost_equal( + auc_roc_from_scores, list(dict_auc_roc.values()), decimal=1 + ) + np.testing.assert_array_almost_equal( + auc_pr_from_scores, list(dict_auc_pr.values()), decimal=1 + ) + + true_auc_roc = [ + 0.8695436507936507, + 0.9737678855325913, + 0.9930555555555555, + 0.857638888888889, + 0.9639130434782609, + 0.8690476190476191, + 0.9630434782608696, + ] + + true_auc_pr = [ + 0.814256917602188, + 0.9945160041091712, + 0.9992086070916503, + 0.8054288542539664, + 0.9777504211642852, + 0.8164636240285442, + 0.9763049418985656, + ] + + # check value of results + np.testing.assert_array_almost_equal( + auc_roc_from_scores, true_auc_roc, decimal=1 + ) + np.testing.assert_array_almost_equal(auc_pr_from_scores, true_auc_pr, decimal=1) + + # second case: scorers that return scorers that have the same width as the input + anomaly_model = FilteringAnomalyModel( + model=MovingAverage(window=10), + scorer=[ + NormScorer(component_wise=True), + Difference(), + WassersteinScorer(component_wise=True), + WassersteinScorer(window=12, component_wise=True), + KMeansScorer(component_wise=True), + KMeansScorer(window=5, component_wise=True), + PyODScorer(model=KNN(), component_wise=True), + PyODScorer(model=KNN(), window=5, component_wise=True), + ], + ) + anomaly_model.fit(mts_series_train) + + scores, model_output = anomaly_model.score( + mts_series_test, return_model_prediction=True + ) + + # model_output must be multivariate (same width as input) + self.assertEqual(model_output.width, mts_series_test.width) + + # scores must be of the same length as the number of scorers + self.assertEqual(len(scores), len(anomaly_model.scorers)) + + dict_auc_roc = anomaly_model.eval_accuracy( + ts_anomalies, mts_series_test, metric="AUC_ROC" + ) + dict_auc_pr = anomaly_model.eval_accuracy( + ts_anomalies, mts_series_test, metric="AUC_PR" + ) + + auc_roc_from_scores = eval_accuracy_from_scores( + actual_anomalies=[ts_anomalies] * 8, + anomaly_score=scores, + window=[1, 1, 10, 12, 1, 5, 1, 5], + metric="AUC_ROC", + ) + + auc_pr_from_scores = eval_accuracy_from_scores( + actual_anomalies=[ts_anomalies] * 8, + anomaly_score=scores, + window=[1, 1, 10, 12, 1, 5, 1, 5], + metric="AUC_PR", + ) + + # function eval_accuracy_from_scores and eval_accuracy must return an input of same length + self.assertEqual(len(auc_roc_from_scores), len(dict_auc_roc)) + self.assertEqual(len(auc_pr_from_scores), len(dict_auc_pr)) + + # function eval_accuracy_from_scores and eval_accuracy must return the same values + np.testing.assert_array_almost_equal( + auc_roc_from_scores, list(dict_auc_roc.values()), decimal=1 + ) + np.testing.assert_array_almost_equal( + auc_pr_from_scores, list(dict_auc_pr.values()), decimal=1 + ) + + true_auc_roc = [ + [0.859375, 0.9200542005420054], + [0.49875, 0.513550135501355], + [0.997093023255814, 0.9536231884057971], + [0.998960498960499, 0.9739795918367344], + [0.8143750000000001, 0.8218157181571816], + [0.9886148007590132, 0.94677734375], + [0.830625, 0.9369918699186992], + [0.9909867172675522, 0.94580078125], + ] + + true_auc_pr = [ + [0.7213314465376244, 0.8191331553279771], + [0.4172305056124696, 0.49249755343619195], + [0.9975245098039216, 0.9741870252257915], + [0.9992877492877493, 0.9865792868871687], + [0.7095552075210219, 0.7591858780309868], + [0.9827224901431558, 0.9402925739221939], + [0.7095275592303261, 0.8313668186652059], + [0.9858096294704315, 0.9391783485106905], + ] + + # check value of results + np.testing.assert_array_almost_equal( + auc_roc_from_scores, true_auc_roc, decimal=1 + ) + np.testing.assert_array_almost_equal(auc_pr_from_scores, true_auc_pr, decimal=1) + + def test_multivariate__ForecastingAnomalyModel(self): + + np.random.seed(40) + + data_sin = np.array([np.sin(x) for x in np.arange(0, 20 * np.pi, 0.2)]) + data_cos = np.array([np.cos(x) for x in np.arange(0, 20 * np.pi, 0.2)]) + + mts_series_train = TimeSeries.from_values( + np.dstack((data_sin, data_cos))[0], columns=["component 1", "component 2"] + ) + + data_sin[10:20] = 0 + data_cos[60:80] = 0 + + data_sin[100:110] = 1 + data_cos[150:155] = 1 + + data_sin[200:240] = 0.9 * data_cos[200:240] + data_cos[200:240] = 0.9 * data_sin[200:240] + + data_sin[275:295] = data_sin[275:295] + np.random.normal(0, 0.1, 20) + data_cos[275:295] = data_cos[275:295] + np.random.normal(0, 0.1, 20) + + mts_series_test = TimeSeries.from_values( + np.dstack((data_sin, data_cos))[0], columns=["component 1", "component 2"] + ) + + np1_anomalies = np.zeros(len(data_sin)) + np1_anomalies[10:20] = 1 + np1_anomalies[100:110] = 1 + np1_anomalies[200:240] = 1 + np1_anomalies[275:295] = 1 + + np2_anomalies = np.zeros(len(data_cos)) + np2_anomalies[60:80] = 1 + np2_anomalies[150:155] = 1 + np2_anomalies[200:240] = 1 + np2_anomalies[275:295] = 1 + + np_anomalies = np.zeros(len(data_cos)) + np_anomalies[10:20] = 1 + np_anomalies[60:80] = 1 + np_anomalies[100:110] = 1 + np_anomalies[150:155] = 1 + np_anomalies[200:240] = 1 + np_anomalies[275:295] = 1 + + ts_anomalies = TimeSeries.from_times_and_values( + mts_series_train.time_index, + np.dstack((np1_anomalies, np2_anomalies))[0], + columns=["is_anomaly_1", "is_anomaly_2"], + ) + + mts_anomalies = TimeSeries.from_times_and_values( + mts_series_train.time_index, np_anomalies, columns=["is_anomaly"] + ) + + # first case: scorers that return univariate scores + anomaly_model = ForecastingAnomalyModel( + model=RegressionModel(lags=10), + scorer=[ + NormScorer(component_wise=False), + WassersteinScorer(), + WassersteinScorer(window=20), + KMeansScorer(), + KMeansScorer(window=20), + PyODScorer(model=KNN()), + PyODScorer(model=KNN(), window=10), + ], + ) + anomaly_model.fit(mts_series_train, allow_model_training=True, start=0.1) + + scores, model_output = anomaly_model.score( + mts_series_test, return_model_prediction=True, start=0.1 + ) + + # model_output must be multivariate (same width as input) + self.assertEqual(model_output.width, mts_series_test.width) + + # scores must be of the same length as the number of scorers + self.assertEqual(len(scores), len(anomaly_model.scorers)) + + dict_auc_roc = anomaly_model.eval_accuracy( + mts_anomalies, mts_series_test, start=0.1, metric="AUC_ROC" + ) + dict_auc_pr = anomaly_model.eval_accuracy( + mts_anomalies, mts_series_test, start=0.1, metric="AUC_PR" + ) + + auc_roc_from_scores = eval_accuracy_from_scores( + actual_anomalies=[mts_anomalies] * 7, + anomaly_score=scores, + window=[1, 10, 20, 1, 20, 1, 10], + metric="AUC_ROC", + ) + + auc_pr_from_scores = eval_accuracy_from_scores( + actual_anomalies=[mts_anomalies] * 7, + anomaly_score=scores, + window=[1, 10, 20, 1, 20, 1, 10], + metric="AUC_PR", + ) + + # function eval_accuracy_from_scores and eval_accuracy must return an input of same length + self.assertEqual(len(auc_roc_from_scores), len(dict_auc_roc)) + self.assertEqual(len(auc_pr_from_scores), len(dict_auc_pr)) + + # function eval_accuracy_from_scores and eval_accuracy must return the same values + np.testing.assert_array_almost_equal( + auc_roc_from_scores, list(dict_auc_roc.values()), decimal=1 + ) + np.testing.assert_array_almost_equal( + auc_pr_from_scores, list(dict_auc_pr.values()), decimal=1 + ) + + true_auc_roc = [ + 0.9252575884154831, + 0.9130158730158731, + 0.9291228070175439, + 0.9252575884154832, + 0.9211929824561403, + 0.9252575884154831, + 0.915873015873016, + ] + + true_auc_pr = [ + 0.8389462532437767, + 0.9151621069238896, + 0.9685249535885079, + 0.8389462532437765, + 0.9662153835545242, + 0.8389462532437764, + 0.9212725256428517, + ] + + # check value of results + np.testing.assert_array_almost_equal( + auc_roc_from_scores, true_auc_roc, decimal=1 + ) + np.testing.assert_array_almost_equal(auc_pr_from_scores, true_auc_pr, decimal=1) + + # second case: scorers that return scorers that have the same width as the input + anomaly_model = ForecastingAnomalyModel( + model=RegressionModel(lags=10), + scorer=[ + NormScorer(component_wise=True), + Difference(), + WassersteinScorer(component_wise=True), + WassersteinScorer(window=20, component_wise=True), + KMeansScorer(component_wise=True), + KMeansScorer(window=20, component_wise=True), + PyODScorer(model=KNN(), component_wise=True), + PyODScorer(model=KNN(), window=10, component_wise=True), + ], + ) + anomaly_model.fit(mts_series_train, allow_model_training=True, start=0.1) + + scores, model_output = anomaly_model.score( + mts_series_test, return_model_prediction=True, start=0.1 + ) + + # model_output must be multivariate (same width as input) + self.assertEqual(model_output.width, mts_series_test.width) + + # scores must be of the same length as the number of scorers + self.assertEqual(len(scores), len(anomaly_model.scorers)) + + dict_auc_roc = anomaly_model.eval_accuracy( + ts_anomalies, mts_series_test, start=0.1, metric="AUC_ROC" + ) + dict_auc_pr = anomaly_model.eval_accuracy( + ts_anomalies, mts_series_test, start=0.1, metric="AUC_PR" + ) + + auc_roc_from_scores = eval_accuracy_from_scores( + actual_anomalies=[ts_anomalies] * 8, + anomaly_score=scores, + window=[1, 1, 10, 20, 1, 20, 1, 10], + metric="AUC_ROC", + ) + + auc_pr_from_scores = eval_accuracy_from_scores( + actual_anomalies=[ts_anomalies] * 8, + anomaly_score=scores, + window=[1, 1, 10, 20, 1, 20, 1, 10], + metric="AUC_PR", + ) + + # function eval_accuracy_from_scores and eval_accuracy must return an input of same length + self.assertEqual(len(auc_roc_from_scores), len(dict_auc_roc)) + self.assertEqual(len(auc_pr_from_scores), len(dict_auc_pr)) + + # function eval_accuracy_from_scores and eval_accuracy must return the same values + np.testing.assert_array_almost_equal( + auc_roc_from_scores, list(dict_auc_roc.values()), decimal=1 + ) + np.testing.assert_array_almost_equal( + auc_pr_from_scores, list(dict_auc_pr.values()), decimal=1 + ) + + true_auc_roc = [ + [0.8803738317757009, 0.912267218445167], + [0.48898531375166887, 0.5758202778598878], + [0.8375999073323295, 0.9162283996994741], + [0.7798128494807715, 0.8739249880554228], + [0.8803738317757008, 0.912267218445167], + [0.7787287458632889, 0.8633540372670807], + [0.8803738317757009, 0.9122672184451671], + [0.8348777945094406, 0.9137061285821616], + ] + + true_auc_pr = [ + [0.7123114333965317, 0.7579757115620807], + [0.4447973021706103, 0.596776950584551], + [0.744325434474558, 0.8984960888744328], + [0.7653561450296187, 0.9233662817550338], + [0.7123114333965317, 0.7579757115620807], + [0.7852553779986415, 0.9185701347601994], + [0.7123114333965319, 0.7579757115620807], + [0.757208451057927, 0.8967178983419622], + ] + + # check value of results + np.testing.assert_array_almost_equal( + auc_roc_from_scores, true_auc_roc, decimal=1 + ) + np.testing.assert_array_almost_equal(auc_pr_from_scores, true_auc_pr, decimal=1) + + def test_show_anomalies(self): + + forecasting_anomaly_model = ForecastingAnomalyModel( + model=RegressionModel(lags=10), scorer=NormScorer() + ) + forecasting_anomaly_model.fit(self.train, allow_model_training=True) + + filtering_anomaly_model = FilteringAnomalyModel( + model=MovingAverage(window=10), scorer=NormScorer() + ) + + for anomaly_model in [forecasting_anomaly_model, filtering_anomaly_model]: + + # must input only one series + with self.assertRaises(ValueError): + anomaly_model.show_anomalies(series=[self.train, self.train]) + + # input must be a series + with self.assertRaises(ValueError): + anomaly_model.show_anomalies(series=[1, 2, 4]) + + # metric must be "AUC_ROC" or "AUC_PR" + with self.assertRaises(ValueError): + anomaly_model.show_anomalies( + series=self.train, actual_anomalies=self.anomalies, metric="str" + ) + with self.assertRaises(ValueError): + anomaly_model.show_anomalies( + series=self.train, actual_anomalies=self.anomalies, metric="auc_roc" + ) + with self.assertRaises(ValueError): + anomaly_model.show_anomalies( + series=self.train, actual_anomalies=self.anomalies, metric=1 + ) + + # actual_anomalies must be not none if metric is given + with self.assertRaises(ValueError): + anomaly_model.show_anomalies(series=self.train, metric="AUC_ROC") + + # actual_anomalies must be binary + with self.assertRaises(ValueError): + anomaly_model.show_anomalies( + series=self.train, actual_anomalies=self.test, metric="AUC_ROC" + ) + + # actual_anomalies must contain at least 1 anomaly if metric is given + with self.assertRaises(ValueError): + anomaly_model.show_anomalies( + series=self.train, + actual_anomalies=self.only_0_anomalies, + metric="AUC_ROC", + ) + + # actual_anomalies must contain at least 1 non-anomoulous timestamp + # if metric is given + with self.assertRaises(ValueError): + anomaly_model.show_anomalies( + series=self.train, + actual_anomalies=self.only_1_anomalies, + metric="AUC_ROC", + ) + + # names_of_scorers must be str + with self.assertRaises(ValueError): + anomaly_model.show_anomalies(series=self.train, names_of_scorers=2) + # nbr of names_of_scorers must match the nbr of scores (only 1 here) + with self.assertRaises(ValueError): + anomaly_model.show_anomalies( + series=self.train, names_of_scorers=["scorer1", "scorer2"] + ) + + # title must be str + with self.assertRaises(ValueError): + anomaly_model.show_anomalies(series=self.train, title=1) + + def test_show_anomalies_from_scores(self): + + # must input only one series + with self.assertRaises(ValueError): + show_anomalies_from_scores(series=[self.train, self.train]) + + # input must be a series + with self.assertRaises(ValueError): + show_anomalies_from_scores(series=[1, 2, 4]) + + # must input only one model_output + with self.assertRaises(ValueError): + show_anomalies_from_scores( + series=self.train, model_output=[self.test, self.train] + ) + + # metric must be "AUC_ROC" or "AUC_PR" + with self.assertRaises(ValueError): + show_anomalies_from_scores( + series=self.train, + anomaly_scores=self.test, + actual_anomalies=self.anomalies, + metric="str", + ) + with self.assertRaises(ValueError): + show_anomalies_from_scores( + series=self.train, + anomaly_scores=self.test, + actual_anomalies=self.anomalies, + metric="auc_roc", + ) + with self.assertRaises(ValueError): + show_anomalies_from_scores( + series=self.train, + anomaly_scores=self.test, + actual_anomalies=self.anomalies, + metric=1, + ) + + # actual_anomalies must be not none if metric is given + with self.assertRaises(ValueError): + show_anomalies_from_scores( + series=self.train, anomaly_scores=self.test, metric="AUC_ROC" + ) + + # actual_anomalies must be binary + with self.assertRaises(ValueError): + show_anomalies_from_scores( + series=self.train, + anomaly_scores=self.test, + actual_anomalies=self.test, + metric="AUC_ROC", + ) + + # actual_anomalies must contain at least 1 anomaly if metric is given + with self.assertRaises(ValueError): + show_anomalies_from_scores( + series=self.train, + anomaly_scores=self.test, + actual_anomalies=self.only_0_anomalies, + metric="AUC_ROC", + ) + + # actual_anomalies must contain at least 1 non-anomoulous timestamp + # if metric is given + with self.assertRaises(ValueError): + show_anomalies_from_scores( + series=self.train, + anomaly_scores=self.test, + actual_anomalies=self.only_1_anomalies, + metric="AUC_ROC", + ) + + # window must be int + with self.assertRaises(ValueError): + show_anomalies_from_scores( + series=self.train, anomaly_scores=self.test, window="1" + ) + # window must be an int positive + with self.assertRaises(ValueError): + show_anomalies_from_scores( + series=self.train, anomaly_scores=self.test, window=-1 + ) + # window must smaller than the score series + with self.assertRaises(ValueError): + show_anomalies_from_scores( + series=self.train, anomaly_scores=self.test, window=200 + ) + + # must have the same nbr of windows than scores + with self.assertRaises(ValueError): + show_anomalies_from_scores( + series=self.train, anomaly_scores=self.test, window=[1, 2] + ) + with self.assertRaises(ValueError): + show_anomalies_from_scores( + series=self.train, + anomaly_scores=[self.test, self.test], + window=[1, 2, 1], + ) + + # names_of_scorers must be str + with self.assertRaises(ValueError): + show_anomalies_from_scores( + series=self.train, anomaly_scores=self.test, names_of_scorers=2 + ) + # nbr of names_of_scorers must match the nbr of scores + with self.assertRaises(ValueError): + show_anomalies_from_scores( + series=self.train, + anomaly_scores=self.test, + names_of_scorers=["scorer1", "scorer2"], + ) + with self.assertRaises(ValueError): + show_anomalies_from_scores( + series=self.train, + anomaly_scores=[self.test, self.test], + names_of_scorers=["scorer1", "scorer2", "scorer3"], + ) + + # title must be str + with self.assertRaises(ValueError): + show_anomalies_from_scores(series=self.train, title=1) diff --git a/darts/tests/ad/test_detectors.py b/darts/tests/ad/test_detectors.py new file mode 100644 index 0000000000..79ebb5a67e --- /dev/null +++ b/darts/tests/ad/test_detectors.py @@ -0,0 +1,807 @@ +from typing import Sequence + +import numpy as np + +from darts import TimeSeries +from darts.ad.detectors.quantile_detector import QuantileDetector +from darts.ad.detectors.threshold_detector import ThresholdDetector +from darts.tests.base_test_class import DartsBaseTestClass + +list_NonFittableDetector = [ThresholdDetector(low_threshold=0.2)] + +list_FittableDetector = [QuantileDetector(low_quantile=0.2)] + + +class ADDetectorsTestCase(DartsBaseTestClass): + + np.random.seed(42) + + # univariate series + np_train = np.random.normal(loc=10, scale=0.5, size=100) + train = TimeSeries.from_values(np_train) + + np_test = np.random.normal(loc=10, scale=1, size=100) + test = TimeSeries.from_times_and_values(train._time_index, np_test) + + np_anomalies = np.random.choice(a=[0, 1], size=100, p=[0.9, 0.1]) + anomalies = TimeSeries.from_times_and_values(train._time_index, np_anomalies) + + # multivariate series + np_mts_train = np.random.normal(loc=[10, 5], scale=[0.5, 1], size=[100, 2]) + mts_train = TimeSeries.from_values(np_mts_train) + + np_mts_test = np.random.normal(loc=[10, 5], scale=[1, 1.5], size=[100, 2]) + mts_test = TimeSeries.from_times_and_values(mts_train._time_index, np_mts_test) + + np_mts_anomalies = np.random.choice(a=[0, 1], size=[100, 2], p=[0.9, 0.1]) + mts_anomalies = TimeSeries.from_times_and_values( + mts_train._time_index, np_mts_anomalies + ) + + np_probabilistic = np.random.choice(a=[0, 1], p=[0.5, 0.5], size=[100, 1, 5]) + probabilistic = TimeSeries.from_values(np_probabilistic) + + def test_DetectNonFittableDetector(self): + + detector = ThresholdDetector(low_threshold=0.2) + + # Check return types + # Check if return TimeSeries is float when input is a series + self.assertTrue(isinstance(detector.detect(self.test), TimeSeries)) + + # Check if return type is Sequence when input is a Sequence of series + self.assertTrue(isinstance(detector.detect([self.test]), Sequence)) + + # Check if return TimeSeries is Sequence when input is a multivariate series + self.assertTrue(isinstance(detector.detect(self.mts_test), TimeSeries)) + + # Check if return type is Sequence when input is a multivariate series + self.assertTrue(isinstance(detector.detect([self.mts_test]), Sequence)) + + with self.assertRaises(ValueError): + # Input cannot be probabilistic + detector.detect(self.probabilistic) + + def test_DetectFittableDetector(self): + detector = QuantileDetector(low_quantile=0.2) + + # Check return types + + detector.fit(self.train) + # Check if return type is float when input is a series + self.assertTrue(isinstance(detector.detect(self.test), TimeSeries)) + + # Check if return type is Sequence when input is a sequence of series + self.assertTrue(isinstance(detector.detect([self.test]), Sequence)) + + detector.fit(self.mts_train) + # Check if return type is Sequence when input is a multivariate series + self.assertTrue(isinstance(detector.detect(self.mts_test), TimeSeries)) + + # Check if return type is Sequence when input is a sequence of multivariate series + self.assertTrue(isinstance(detector.detect([self.mts_test]), Sequence)) + + with self.assertRaises(ValueError): + # Input cannot be probabilistic + detector.detect(self.probabilistic) + + def test_eval_accuracy(self): + + detector = ThresholdDetector(low_threshold=0.2) + + # Check return types + # Check if return type is float when input is a series + self.assertTrue( + isinstance(detector.eval_accuracy(self.anomalies, self.test), float) + ) + + # Check if return type is Sequence when input is a Sequence of series + self.assertTrue( + isinstance(detector.eval_accuracy(self.anomalies, [self.test]), Sequence) + ) + + # Check if return type is Sequence when input is a multivariate series + self.assertTrue( + isinstance( + detector.eval_accuracy(self.mts_anomalies, self.mts_test), Sequence + ) + ) + + # Check if return type is Sequence when input is a multivariate series + self.assertTrue( + isinstance( + detector.eval_accuracy(self.mts_anomalies, [self.mts_test]), Sequence + ) + ) + + with self.assertRaises(ValueError): + # Input cannot be probabilistic + detector.eval_accuracy(self.anomalies, self.probabilistic) + + def test_FittableDetector(self): + + for detector in list_FittableDetector: + + # Need to call fit() before calling detect() + with self.assertRaises(ValueError): + detector.detect(self.test) + + # Check if _fit_called is False + self.assertTrue(not detector._fit_called) + + with self.assertRaises(ValueError): + # fit on sequence with series that have different width + detector.fit([self.train, self.mts_train]) + + with self.assertRaises(ValueError): + # Input cannot be probabilistic + detector.fit(self.probabilistic) + + # case1: fit on UTS + detector1 = detector + detector1.fit(self.train) + + # Check if _fit_called is True after being fitted + self.assertTrue(detector1._fit_called) + + with self.assertRaises(ValueError): + # series must be same width as series used for training + detector1.detect(self.mts_test) + + # case2: fit on MTS + detector2 = detector + detector2.fit(self.mts_test) + + # Check if _fit_called is True after being fitted + self.assertTrue(detector2._fit_called) + + with self.assertRaises(ValueError): + # series must be same width as series used for training + detector2.detect(self.train) + + def test_QuantileDetector(self): + + # Need to have at least one parameter (low, high) not None + with self.assertRaises(ValueError): + QuantileDetector() + with self.assertRaises(ValueError): + QuantileDetector(low_quantile=None, high_quantile=None) + + # Parameter low must be float or Sequence of float + with self.assertRaises(TypeError): + QuantileDetector(low_quantile="0.5") + with self.assertRaises(TypeError): + QuantileDetector(low_quantile=[0.2, "0.1"]) + + # Parameter high must be float or Sequence of float + with self.assertRaises(TypeError): + QuantileDetector(high_quantile="0.5") + with self.assertRaises(TypeError): + QuantileDetector(high_quantile=[0.2, "0.1"]) + + # if high and low are both sequences of length>1, they must be of the same size + with self.assertRaises(ValueError): + QuantileDetector(low_quantile=[0.2, 0.1], high_quantile=[0.95, 0.8, 0.9]) + with self.assertRaises(ValueError): + QuantileDetector(low_quantile=[0.2, 0.1, 0.7], high_quantile=[0.95, 0.8]) + + # Parameter must be between 0 and 1 + with self.assertRaises(ValueError): + QuantileDetector(high_quantile=1.1) + with self.assertRaises(ValueError): + QuantileDetector(high_quantile=-0.2) + with self.assertRaises(ValueError): + QuantileDetector(high_quantile=[-0.1, 0.9]) + with self.assertRaises(ValueError): + QuantileDetector(low_quantile=1.1) + with self.assertRaises(ValueError): + QuantileDetector(low_quantile=-0.2) + with self.assertRaises(ValueError): + QuantileDetector(low_quantile=[-0.2, 0.3]) + + # Parameter high must be higher than parameter low + with self.assertRaises(ValueError): + QuantileDetector(low_quantile=0.7, high_quantile=0.2) + with self.assertRaises(ValueError): + QuantileDetector(low_quantile=[0.2, 0.9], high_quantile=[0.95, 0.1]) + with self.assertRaises(ValueError): + QuantileDetector(low_quantile=0.2, high_quantile=[0.95, 0.1]) + with self.assertRaises(ValueError): + QuantileDetector(low_quantile=[0.2, 0.9], high_quantile=0.8) + + # Parameter high/low cannot be sequence of only None + with self.assertRaises(ValueError): + QuantileDetector(low_quantile=[None, None, None]) + with self.assertRaises(ValueError): + QuantileDetector(high_quantile=[None, None, None]) + with self.assertRaises(ValueError): + QuantileDetector(low_quantile=[None], high_quantile=[None, None, None]) + + # widths of series used for fitting must match the number of values given for high or/and low, + # if high and low have a length higher than 1 + + detector = QuantileDetector(low_quantile=0.1, high_quantile=[0.8, 0.7]) + with self.assertRaises(ValueError): + detector.fit(self.train) + with self.assertRaises(ValueError): + detector.fit([self.train, self.mts_train]) + + detector = QuantileDetector(low_quantile=[0.1, 0.2], high_quantile=[0.8, 0.9]) + with self.assertRaises(ValueError): + detector.fit(self.train) + with self.assertRaises(ValueError): + detector.fit([self.train, self.mts_train]) + + detector = QuantileDetector(low_quantile=[0.1, 0.2], high_quantile=0.8) + with self.assertRaises(ValueError): + detector.fit(self.train) + with self.assertRaises(ValueError): + detector.fit([self.train, self.mts_train]) + + detector = QuantileDetector(low_quantile=[0.1, 0.2]) + with self.assertRaises(ValueError): + detector.fit(self.train) + with self.assertRaises(ValueError): + detector.fit([self.train, self.mts_train]) + + detector = QuantileDetector(high_quantile=[0.1, 0.2]) + with self.assertRaises(ValueError): + detector.fit(self.train) + with self.assertRaises(ValueError): + detector.fit([self.train, self.mts_train]) + + # widths of series used for scoring must match the number of values given for high or/and low, + # if high and low have a length higher than 1 + + detector = QuantileDetector(low_quantile=0.1, high_quantile=[0.8, 0.7]) + detector.fit(self.mts_train) + with self.assertRaises(ValueError): + detector.detect(self.train) + with self.assertRaises(ValueError): + detector.detect([self.train, self.mts_train]) + + detector = QuantileDetector(low_quantile=[0.1, 0.2], high_quantile=[0.8, 0.9]) + detector.fit(self.mts_train) + with self.assertRaises(ValueError): + detector.detect(self.train) + with self.assertRaises(ValueError): + detector.detect([self.train, self.mts_train]) + + detector = QuantileDetector(low_quantile=[0.1, 0.2], high_quantile=0.8) + detector.fit(self.mts_train) + with self.assertRaises(ValueError): + detector.detect(self.train) + with self.assertRaises(ValueError): + detector.detect([self.train, self.mts_train]) + + detector = QuantileDetector(low_quantile=[0.1, 0.2]) + detector.fit(self.mts_train) + with self.assertRaises(ValueError): + detector.detect(self.train) + with self.assertRaises(ValueError): + detector.detect([self.train, self.mts_train]) + + detector = QuantileDetector(high_quantile=[0.1, 0.2]) + detector.fit(self.mts_train) + with self.assertRaises(ValueError): + detector.detect(self.train) + with self.assertRaises(ValueError): + detector.detect([self.train, self.mts_train]) + + detector = QuantileDetector(low_quantile=0.05, high_quantile=0.95) + detector.fit(self.train) + + binary_detection = detector.detect(self.test) + + # Return of .detect() must be binary + self.assertTrue( + np.array_equal( + binary_detection.values(copy=False), + binary_detection.values(copy=False).astype(bool), + ) + ) + + # Return of .detect() must be same len as input + self.assertTrue(len(binary_detection) == len(self.test)) + + # univariate test + # detector parameter 'abs_low_' must be equal to 9.13658 when trained on the series 'train' + self.assertAlmostEqual(detector.low_threshold[0], 9.13658, delta=1e-05) + + # detector parameter 'abs_high_' must be equal to 10.74007 when trained on the series 'train' + self.assertAlmostEqual(detector.high_threshold[0], 10.74007, delta=1e-05) + + # detector must found 10 anomalies in the series 'train' + self.assertEqual( + detector.detect(self.train).sum(axis=0).all_values().flatten()[0], 10 + ) + + # detector must found 42 anomalies in the series 'test' + self.assertEqual(binary_detection.sum(axis=0).all_values().flatten()[0], 42) + + # detector must have an accuracy of 0.57 for the series 'test' + self.assertAlmostEqual( + detector.eval_accuracy(self.anomalies, self.test, metric="accuracy"), + 0.57, + delta=1e-05, + ) + # detector must have an recall of 0.4 for the series 'test' + self.assertAlmostEqual( + detector.eval_accuracy(self.anomalies, self.test, metric="recall"), + 0.4, + delta=1e-05, + ) + # detector must have an f1 of 0.08510 for the series 'test' + self.assertAlmostEqual( + detector.eval_accuracy(self.anomalies, self.test, metric="f1"), + 0.08510, + delta=1e-05, + ) + # detector must have an precision of 0.04761 for the series 'test' + self.assertAlmostEqual( + detector.eval_accuracy(self.anomalies, self.test, metric="precision"), + 0.04761, + delta=1e-05, + ) + + # multivariate test + detector_1param = QuantileDetector(low_quantile=0.05, high_quantile=0.95) + detector_1param.fit(self.mts_train) + binary_detection = detector_1param.detect(self.mts_test) + + # if two values are given for low and high, and a series of width 2 is given, then the results must + # be the same as a detector that was given only one value for low and high. + # (will duplicate the value for each component) + detector_2param = QuantileDetector( + low_quantile=[0.05, 0.05], high_quantile=[0.95, 0.95] + ) + detector_2param.fit(self.mts_train) + binary_detection_2param = detector_2param.detect(self.mts_test) + self.assertEqual(binary_detection, binary_detection_2param) + + # width of output must be equal to 2 (same dimension as input) + self.assertEqual(binary_detection.width, 2) + self.assertEqual( + len( + detector_1param.eval_accuracy( + self.mts_anomalies, self.mts_test, metric="accuracy" + ) + ), + 2, + ) + self.assertEqual( + len( + detector_1param.eval_accuracy( + self.mts_anomalies, self.mts_test, metric="recall" + ) + ), + 2, + ) + self.assertEqual( + len( + detector_1param.eval_accuracy( + self.mts_anomalies, self.mts_test, metric="f1" + ) + ), + 2, + ) + self.assertEqual( + len( + detector_1param.eval_accuracy( + self.mts_anomalies, self.mts_test, metric="precision" + ) + ), + 2, + ) + + abs_low_ = detector_1param.low_threshold + abs_high_ = detector_1param.high_threshold + + # detector_1param parameter 'abs_high_' must be equal to 10.83047 when trained + # on the series 'train' for the 1st component + self.assertAlmostEqual(abs_high_[0], 10.83047, delta=1e-05) + # detector_1param parameter 'abs_high_' must be equal to 6.47822 when trained + # on the series 'train' for the 2nd component + self.assertAlmostEqual(abs_high_[1], 6.47822, delta=1e-05) + + # detector_1param parameter 'abs_low_' must be equal to 9.20248 when trained + # on the series 'train' for the 1st component + self.assertAlmostEqual(abs_low_[0], 9.20248, delta=1e-05) + # detector_1param parameter 'abs_low_' must be equal to 3.61853 when trained + # on the series 'train' for the 2nd component + self.assertAlmostEqual(abs_low_[1], 3.61853, delta=1e-05) + + # detector_1param must found 37 anomalies on the first component of the series 'test' + self.assertEqual( + binary_detection["0"].sum(axis=0).all_values().flatten()[0], 37 + ) + # detector_1param must found 38 anomalies on the second component of the series 'test' + self.assertEqual( + binary_detection["1"].sum(axis=0).all_values().flatten()[0], 38 + ) + + acc = detector_1param.eval_accuracy( + self.mts_anomalies, self.mts_test, metric="accuracy" + ) + # detector_1param must have an accuracy of 0.58 on the first component of the series 'mts_test' + self.assertAlmostEqual(acc[0], 0.58, delta=1e-05) + # detector_1param must have an accuracy of 0.58 on the second component of the series 'mts_test' + self.assertAlmostEqual(acc[1], 0.58, delta=1e-05) + + precision = detector_1param.eval_accuracy( + self.mts_anomalies, self.mts_test, metric="precision" + ) + # detector_1param must have an precision of 0.08108 on the first component of the series 'mts_test' + self.assertAlmostEqual(precision[0], 0.08108, delta=1e-05) + # detector_1param must have an precision of 0.07894 on the second component of the series 'mts_test' + self.assertAlmostEqual(precision[1], 0.07894, delta=1e-05) + + recall = detector_1param.eval_accuracy( + self.mts_anomalies, self.mts_test, metric="recall" + ) + # detector_1param must have an recall of 0.2727 on the first component of the series 'mts_test' + self.assertAlmostEqual(recall[0], 0.27272, delta=1e-05) + # detector_1param must have an recall of 0.3 on the second component of the series 'mts_test' + self.assertAlmostEqual(recall[1], 0.3, delta=1e-05) + + f1 = detector_1param.eval_accuracy( + self.mts_anomalies, self.mts_test, metric="f1" + ) + # detector_1param must have an f1 of 0.125 on the first component of the series 'mts_test' + self.assertAlmostEqual(f1[0], 0.125, delta=1e-05) + # detector_1param must have an f1 of 0.125 on the second component of the series 'mts_test' + self.assertAlmostEqual(f1[1], 0.125, delta=1e-05) + + # exemple multivariate with Nones + detector = QuantileDetector( + low_quantile=[0.05, None], high_quantile=[None, 0.95] + ) + detector.fit(self.mts_train) + binary_detection = detector.detect(self.mts_test) + + # width of output must be equal to 2 (same dimension as input) + self.assertEqual(binary_detection.width, 2) + self.assertEqual( + len( + detector.eval_accuracy( + self.mts_anomalies, self.mts_test, metric="accuracy" + ) + ), + 2, + ) + self.assertEqual( + len( + detector.eval_accuracy( + self.mts_anomalies, self.mts_test, metric="recall" + ) + ), + 2, + ) + self.assertEqual( + len(detector.eval_accuracy(self.mts_anomalies, self.mts_test, metric="f1")), + 2, + ) + self.assertEqual( + len( + detector.eval_accuracy( + self.mts_anomalies, self.mts_test, metric="precision" + ) + ), + 2, + ) + + # TODO: we should improve these tests to introduce some correlation + # between actual and detected anomalies... + + # detector must found 20 anomalies on the first component of the series 'test' + # Note: there are 200 values (100 time step x 2 components) so this matches + # well a detection rate of 10% (bottom 5% on first component and top 5% on second component) + self.assertEqual( + binary_detection["0"].sum(axis=0).all_values().flatten()[0], 20 + ) + # detector must found 19 anomalies on the second component of the series 'test' + self.assertEqual( + binary_detection["1"].sum(axis=0).all_values().flatten()[0], 19 + ) + + acc = detector.eval_accuracy( + self.mts_anomalies, self.mts_test, metric="accuracy" + ) + self.assertAlmostEqual(acc[0], 0.69, delta=1e-05) + self.assertAlmostEqual(acc[1], 0.75, delta=1e-05) + + precision = detector.eval_accuracy( + self.mts_anomalies, self.mts_test, metric="precision" + ) + self.assertAlmostEqual(precision[0], 0.0, delta=1e-05) + self.assertAlmostEqual(precision[1], 0.10526, delta=1e-05) + + recall = detector.eval_accuracy( + self.mts_anomalies, self.mts_test, metric="recall" + ) + self.assertAlmostEqual(recall[0], 0.0, delta=1e-05) + self.assertAlmostEqual(recall[1], 0.2, delta=1e-05) + + f1 = detector.eval_accuracy(self.mts_anomalies, self.mts_test, metric="f1") + self.assertAlmostEqual(f1[0], 0.0, delta=1e-05) + self.assertAlmostEqual(f1[1], 0.13793, delta=1e-05) + + def test_ThresholdDetector(self): + + # Parameters + # Need to have at least one parameter (low, high) not None + with self.assertRaises(ValueError): + ThresholdDetector() + with self.assertRaises(ValueError): + ThresholdDetector(low_threshold=None, high_threshold=None) + + # if high and low are both sequences of length>1, they must be of the same size + with self.assertRaises(ValueError): + ThresholdDetector(low_threshold=[0.2, 0.1], high_threshold=[0.95, 0.8, 0.9]) + with self.assertRaises(ValueError): + ThresholdDetector(low_threshold=[0.2, 0.1, 0.7], high_threshold=[0.95, 0.8]) + + # Parameter high must be higher than parameter low + with self.assertRaises(ValueError): + ThresholdDetector(low_threshold=0.7, high_threshold=0.2) + with self.assertRaises(ValueError): + ThresholdDetector(low_threshold=[0.2, 0.9], high_threshold=[0.95, 0.1]) + with self.assertRaises(ValueError): + ThresholdDetector(low_threshold=0.2, high_threshold=[0.95, 0.1]) + with self.assertRaises(ValueError): + ThresholdDetector(low_threshold=[0.2, 0.9], high_threshold=0.8) + with self.assertRaises(ValueError): + ThresholdDetector(low_threshold=[0.2, 0.9, None], high_threshold=0.8) + + # Parameter high/low cannot be sequence of only None + with self.assertRaises(ValueError): + ThresholdDetector(low_threshold=[None, None, None]) + with self.assertRaises(ValueError): + ThresholdDetector(high_threshold=[None, None, None]) + with self.assertRaises(ValueError): + ThresholdDetector(low_threshold=[None], high_threshold=[None, None, None]) + + # widths of series used for scoring must match the number of values given for high or/and low, + # if high and low have a length higher than 1 + + detector = ThresholdDetector(low_threshold=0.1, high_threshold=[0.8, 0.7]) + with self.assertRaises(ValueError): + detector.detect(self.train) + with self.assertRaises(ValueError): + detector.detect([self.train, self.mts_train]) + + detector = ThresholdDetector( + low_threshold=[0.1, 0.2], high_threshold=[0.8, 0.9] + ) + with self.assertRaises(ValueError): + detector.detect(self.train) + with self.assertRaises(ValueError): + detector.detect([self.train, self.mts_train]) + + detector = ThresholdDetector(low_threshold=[0.1, 0.2], high_threshold=0.8) + with self.assertRaises(ValueError): + detector.detect(self.train) + with self.assertRaises(ValueError): + detector.detect([self.train, self.mts_train]) + + detector = ThresholdDetector(low_threshold=[0.1, 0.2]) + with self.assertRaises(ValueError): + detector.detect(self.train) + with self.assertRaises(ValueError): + detector.detect([self.train, self.mts_train]) + + detector = ThresholdDetector(high_threshold=[0.1, 0.2]) + with self.assertRaises(ValueError): + detector.detect(self.train) + with self.assertRaises(ValueError): + detector.detect([self.train, self.mts_train]) + + detector = ThresholdDetector(low_threshold=9.5, high_threshold=10.5) + binary_detection = detector.detect(self.test) + + # Return of .detect() must be binary + self.assertTrue( + np.array_equal( + binary_detection.values(copy=False), + binary_detection.values(copy=False).astype(bool), + ) + ) + + # Return of .detect() must be same len as input + self.assertTrue(len(binary_detection) == len(self.test)) + + # univariate test + # detector must found 58 anomalies in the series 'test' + self.assertEqual(binary_detection.sum(axis=0).all_values().flatten()[0], 58) + # detector must have an accuracy of 0.41 for the series 'test' + self.assertAlmostEqual( + detector.eval_accuracy(self.anomalies, self.test, metric="accuracy"), + 0.41, + delta=1e-05, + ) + # detector must have an recall of 0.4 for the series 'test' + self.assertAlmostEqual( + detector.eval_accuracy(self.anomalies, self.test, metric="recall"), + 0.4, + delta=1e-05, + ) + # detector must have an f1 of 0.06349 for the series 'test' + self.assertAlmostEqual( + detector.eval_accuracy(self.anomalies, self.test, metric="f1"), + 0.06349, + delta=1e-05, + ) + # detector must have an precision of 0.03448 for the series 'test' + self.assertAlmostEqual( + detector.eval_accuracy(self.anomalies, self.test, metric="precision"), + 0.03448, + delta=1e-05, + ) + + # multivariate test + detector = ThresholdDetector(low_threshold=4.8, high_threshold=10.5) + binary_detection = detector.detect(self.mts_test) + + # if two values are given for low and high, and a series of width 2 is given, + # then the results must be the same as a detector that was given only one value + # for low and high. (will duplicate the value for each width) + detector_2param = ThresholdDetector( + low_threshold=[4.8, 4.8], high_threshold=[10.5, 10.5] + ) + binary_detection_2param = detector_2param.detect(self.mts_test) + self.assertEqual(binary_detection, binary_detection_2param) + + # width of output must be equal to 2 (same dimension as input) + self.assertEqual(binary_detection.width, 2) + self.assertEqual( + len( + detector.eval_accuracy( + self.mts_anomalies, self.mts_test, metric="accuracy" + ) + ), + 2, + ) + self.assertEqual( + len( + detector.eval_accuracy( + self.mts_anomalies, self.mts_test, metric="recall" + ) + ), + 2, + ) + self.assertEqual( + len(detector.eval_accuracy(self.mts_anomalies, self.mts_test, metric="f1")), + 2, + ) + self.assertEqual( + len( + detector.eval_accuracy( + self.mts_anomalies, self.mts_test, metric="precision" + ) + ), + 2, + ) + + # detector must found 28 anomalies on the first width of the series 'test' + self.assertEqual( + binary_detection["0"].sum(axis=0).all_values().flatten()[0], 28 + ) + # detector must found 52 anomalies on the second width of the series 'test' + self.assertEqual( + binary_detection["1"].sum(axis=0).all_values().flatten()[0], 52 + ) + + acc = detector.eval_accuracy( + self.mts_anomalies, self.mts_test, metric="accuracy" + ) + # detector must have an accuracy of 0.71 on the first width of the series 'mts_test' + self.assertAlmostEqual(acc[0], 0.71, delta=1e-05) + # detector must have an accuracy of 0.48 on the second width of the series 'mts_test' + self.assertAlmostEqual(acc[1], 0.48, delta=1e-05) + + precision = detector.eval_accuracy( + self.mts_anomalies, self.mts_test, metric="precision" + ) + # detector must have an precision of 0.17857 on the first width of the series 'mts_test' + self.assertAlmostEqual(precision[0], 0.17857, delta=1e-05) + # detector must have an precision of 0.09615 on the second width of the series 'mts_test' + self.assertAlmostEqual(precision[1], 0.09615, delta=1e-05) + + recall = detector.eval_accuracy( + self.mts_anomalies, self.mts_test, metric="recall" + ) + # detector must have an recall of 0.45454 on the first width of the series 'mts_test' + self.assertAlmostEqual(recall[0], 0.45454, delta=1e-05) + # detector must have an recall of 0.5 on the second width of the series 'mts_test' + self.assertAlmostEqual(recall[1], 0.5, delta=1e-05) + + f1 = detector.eval_accuracy(self.mts_anomalies, self.mts_test, metric="f1") + # detector must have an f1 of 0.25641 on the first width of the series 'mts_test' + self.assertAlmostEqual(f1[0], 0.25641, delta=1e-05) + # detector must have an f1 of 0.16129 on the second width of the series 'mts_test' + self.assertAlmostEqual(f1[1], 0.16129, delta=1e-05) + + # exemple multivariate with Nones + detector = ThresholdDetector(low_threshold=[10, None], high_threshold=[None, 5]) + binary_detection = detector.detect(self.mts_test) + + # width of output must be equal to 2 (same dimension as input) + self.assertEqual(binary_detection.width, 2) + self.assertEqual( + len( + detector.eval_accuracy( + self.mts_anomalies, self.mts_test, metric="accuracy" + ) + ), + 2, + ) + self.assertEqual( + len( + detector.eval_accuracy( + self.mts_anomalies, self.mts_test, metric="recall" + ) + ), + 2, + ) + self.assertEqual( + len(detector.eval_accuracy(self.mts_anomalies, self.mts_test, metric="f1")), + 2, + ) + self.assertEqual( + len( + detector.eval_accuracy( + self.mts_anomalies, self.mts_test, metric="precision" + ) + ), + 2, + ) + + # detector must found 48 anomalies on the first width of the series 'test' + self.assertEqual( + binary_detection["0"].sum(axis=0).all_values().flatten()[0], 48 + ) + # detector must found 43 anomalies on the second width of the series 'test' + self.assertEqual( + binary_detection["1"].sum(axis=0).all_values().flatten()[0], 43 + ) + + acc = detector.eval_accuracy( + self.mts_anomalies, self.mts_test, metric="accuracy" + ) + # detector must have an accuracy of 0.51 on the first width of the series 'mts_test' + self.assertAlmostEqual(acc[0], 0.51, delta=1e-05) + # detector must have an accuracy of 0.57 on the second width of the series 'mts_test' + self.assertAlmostEqual(acc[1], 0.57, delta=1e-05) + + precision = detector.eval_accuracy( + self.mts_anomalies, self.mts_test, metric="precision" + ) + # detector must have an precision of 0.10416 and on the first width of the series 'mts_test' + self.assertAlmostEqual(precision[0], 0.10416, delta=1e-05) + # detector must have an precision of 0.11627 on the second width of the series 'mts_test' + self.assertAlmostEqual(precision[1], 0.11627, delta=1e-05) + + recall = detector.eval_accuracy( + self.mts_anomalies, self.mts_test, metric="recall" + ) + # detector must have an recall of 0.45454 on the first width of the series 'mts_test' + self.assertAlmostEqual(recall[0], 0.45454, delta=1e-05) + # detector must have an recall of 0.5 on the second width of the series 'mts_test' + self.assertAlmostEqual(recall[1], 0.5, delta=1e-05) + + f1 = detector.eval_accuracy(self.mts_anomalies, self.mts_test, metric="f1") + # detector must have an f1 of 0.16949 on the first width of the series 'mts_test' + self.assertAlmostEqual(f1[0], 0.16949, delta=1e-05) + # detector must have an f1 of 0.18867 on the second width of the series 'mts_test' + self.assertAlmostEqual(f1[1], 0.18867, delta=1e-05) + + def test_fit_detect(self): + + detector1 = QuantileDetector(low_quantile=0.05, high_quantile=0.95) + detector1.fit(self.train) + prediction1 = detector1.detect(self.train) + + detector2 = QuantileDetector(low_quantile=0.05, high_quantile=0.95) + prediction2 = detector2.fit_detect(self.train) + + self.assertEqual(prediction1, prediction2) diff --git a/darts/tests/ad/test_scorers.py b/darts/tests/ad/test_scorers.py new file mode 100644 index 0000000000..10b2805a44 --- /dev/null +++ b/darts/tests/ad/test_scorers.py @@ -0,0 +1,2633 @@ +from typing import Sequence + +import numpy as np +from pyod.models.knn import KNN +from scipy.stats import cauchy, expon, gamma, laplace, norm, poisson + +from darts import TimeSeries +from darts.ad.scorers import CauchyNLLScorer +from darts.ad.scorers import DifferenceScorer as Difference +from darts.ad.scorers import ( + ExponentialNLLScorer, + GammaNLLScorer, + GaussianNLLScorer, + KMeansScorer, + LaplaceNLLScorer, +) +from darts.ad.scorers import NormScorer as Norm +from darts.ad.scorers import PoissonNLLScorer, PyODScorer, WassersteinScorer +from darts.models import MovingAverage +from darts.tests.base_test_class import DartsBaseTestClass + +list_NonFittableAnomalyScorer = [ + Norm(), + Difference(), + GaussianNLLScorer(), + ExponentialNLLScorer(), + PoissonNLLScorer(), + LaplaceNLLScorer(), + CauchyNLLScorer(), + GammaNLLScorer(), +] + +list_FittableAnomalyScorer = [ + PyODScorer(model=KNN()), + KMeansScorer(), + WassersteinScorer(), +] + +list_NLLScorer = [ + GaussianNLLScorer(), + ExponentialNLLScorer(), + PoissonNLLScorer(), + LaplaceNLLScorer(), + CauchyNLLScorer(), + GammaNLLScorer(), +] + + +class ADAnomalyScorerTestCase(DartsBaseTestClass): + + np.random.seed(42) + + # univariate series + np_train = np.random.normal(loc=10, scale=0.5, size=100) + train = TimeSeries.from_values(np_train) + + np_test = np.random.normal(loc=10, scale=2, size=100) + test = TimeSeries.from_times_and_values(train._time_index, np_test) + + np_anomalies = np.random.choice(a=[0, 1], size=100, p=[0.9, 0.1]) + anomalies = TimeSeries.from_times_and_values(train._time_index, np_anomalies) + + np_only_1_anomalies = np.random.choice(a=[0, 1], size=100, p=[0, 1]) + only_1_anomalies = TimeSeries.from_times_and_values( + train._time_index, np_only_1_anomalies + ) + + np_only_0_anomalies = np.random.choice(a=[0, 1], size=100, p=[1, 0]) + only_0_anomalies = TimeSeries.from_times_and_values( + train._time_index, np_only_0_anomalies + ) + + modified_train = MovingAverage(window=10).filter(train) + modified_test = MovingAverage(window=10).filter(test) + + np_probabilistic = np.random.normal(loc=10, scale=2, size=[100, 1, 20]) + probabilistic = TimeSeries.from_times_and_values( + train._time_index, np_probabilistic + ) + + # multivariate series + np_mts_train = np.random.normal(loc=[10, 5], scale=[0.5, 1], size=[100, 2]) + mts_train = TimeSeries.from_values(np_mts_train) + + np_mts_test = np.random.normal(loc=[10, 5], scale=[1, 1.5], size=[100, 2]) + mts_test = TimeSeries.from_times_and_values(mts_train._time_index, np_mts_test) + + np_mts_anomalies = np.random.choice(a=[0, 1], size=[100, 2], p=[0.9, 0.1]) + mts_anomalies = TimeSeries.from_times_and_values( + mts_train._time_index, np_mts_anomalies + ) + + modified_mts_train = MovingAverage(window=10).filter(mts_train) + modified_mts_test = MovingAverage(window=10).filter(mts_test) + + np_mts_probabilistic = np.random.normal( + loc=[[10], [5]], scale=[[1], [1.5]], size=[100, 2, 20] + ) + mts_probabilistic = TimeSeries.from_times_and_values( + mts_train._time_index, np_mts_probabilistic + ) + + def test_ScoreNonFittableAnomalyScorer(self): + scorer = Norm() + + # Check return types for score_from_prediction() + # Check if return type is float when input is a series + self.assertTrue( + isinstance( + scorer.score_from_prediction(self.test, self.modified_test), TimeSeries + ) + ) + + # Check if return type is Sequence when input is a Sequence of series + self.assertTrue( + isinstance( + scorer.score_from_prediction([self.test], [self.modified_test]), + Sequence, + ) + ) + + # Check if return type is Sequence when input is a multivariate series + self.assertTrue( + isinstance( + scorer.score_from_prediction(self.mts_test, self.modified_mts_test), + TimeSeries, + ) + ) + + # Check if return type is Sequence when input is a multivariate series + self.assertTrue( + isinstance( + scorer.score_from_prediction([self.mts_test], [self.modified_mts_test]), + Sequence, + ) + ) + + def test_ScoreFittableAnomalyScorer(self): + scorer = KMeansScorer() + + # Check return types for score() + scorer.fit(self.train) + # Check if return type is float when input is a series + self.assertTrue(isinstance(scorer.score(self.test), TimeSeries)) + + # Check if return type is Sequence when input is a sequence of series + self.assertTrue(isinstance(scorer.score([self.test]), Sequence)) + + scorer.fit(self.mts_train) + # Check if return type is Sequence when input is a multivariate series + self.assertTrue(isinstance(scorer.score(self.mts_test), TimeSeries)) + + # Check if return type is Sequence when input is a sequence of multivariate series + self.assertTrue(isinstance(scorer.score([self.mts_test]), Sequence)) + + # Check return types for score_from_prediction() + scorer.fit_from_prediction(self.train, self.modified_train) + # Check if return type is float when input is a series + self.assertTrue( + isinstance( + scorer.score_from_prediction(self.test, self.modified_test), TimeSeries + ) + ) + + # Check if return type is Sequence when input is a Sequence of series + self.assertTrue( + isinstance( + scorer.score_from_prediction([self.test], [self.modified_test]), + Sequence, + ) + ) + + scorer.fit_from_prediction(self.mts_train, self.modified_mts_train) + # Check if return type is Sequence when input is a multivariate series + self.assertTrue( + isinstance( + scorer.score_from_prediction(self.mts_test, self.modified_mts_test), + TimeSeries, + ) + ) + + # Check if return type is Sequence when input is a multivariate series + self.assertTrue( + isinstance( + scorer.score_from_prediction([self.mts_test], [self.modified_mts_test]), + Sequence, + ) + ) + + def test_eval_accuracy_from_prediction(self): + + scorer = Norm(component_wise=False) + # Check return types + # Check if return type is float when input is a series + self.assertTrue( + isinstance( + scorer.eval_accuracy_from_prediction( + self.anomalies, self.test, self.modified_test + ), + float, + ) + ) + + # Check if return type is Sequence when input is a Sequence of series + self.assertTrue( + isinstance( + scorer.eval_accuracy_from_prediction( + self.anomalies, [self.test], self.modified_test + ), + Sequence, + ) + ) + + # Check if return type is a float when input is a multivariate series and component_wise is set to False + self.assertTrue( + isinstance( + scorer.eval_accuracy_from_prediction( + self.anomalies, self.mts_test, self.modified_mts_test + ), + float, + ) + ) + + # Check if return type is Sequence when input is a multivariate series and component_wise is set to False + self.assertTrue( + isinstance( + scorer.eval_accuracy_from_prediction( + self.anomalies, [self.mts_test], self.modified_mts_test + ), + Sequence, + ) + ) + + scorer = Norm(component_wise=True) + # Check return types + # Check if return type is float when input is a series + self.assertTrue( + isinstance( + scorer.eval_accuracy_from_prediction( + self.anomalies, self.test, self.modified_test + ), + float, + ) + ) + + # Check if return type is Sequence when input is a Sequence of series + self.assertTrue( + isinstance( + scorer.eval_accuracy_from_prediction( + self.anomalies, [self.test], self.modified_test + ), + Sequence, + ) + ) + + # Check if return type is a float when input is a multivariate series and component_wise is set to True + self.assertTrue( + isinstance( + scorer.eval_accuracy_from_prediction( + self.mts_anomalies, self.mts_test, self.modified_mts_test + ), + Sequence, + ) + ) + + # Check if return type is Sequence when input is a multivariate series and component_wise is set to True + self.assertTrue( + isinstance( + scorer.eval_accuracy_from_prediction( + self.mts_anomalies, [self.mts_test], self.modified_mts_test + ), + Sequence, + ) + ) + + non_fittable_scorer = Norm(component_wise=False) + fittable_scorer = KMeansScorer(component_wise=False) + fittable_scorer.fit(self.train) + + # if component_wise set to False, 'actual_anomalies' must have widths of 1 + with self.assertRaises(ValueError): + fittable_scorer.eval_accuracy( + actual_anomalies=self.mts_anomalies, series=self.test + ) + with self.assertRaises(ValueError): + fittable_scorer.eval_accuracy( + actual_anomalies=[self.anomalies, self.mts_anomalies], + series=[self.test, self.test], + ) + + # 'metric' must be str and "AUC_ROC" or "AUC_PR" + with self.assertRaises(ValueError): + fittable_scorer.eval_accuracy( + actual_anomalies=self.anomalies, series=self.test, metric=1 + ) + with self.assertRaises(ValueError): + fittable_scorer.eval_accuracy( + actual_anomalies=self.anomalies, series=self.test, metric="auc_roc" + ) + with self.assertRaises(TypeError): + fittable_scorer.eval_accuracy( + actual_anomalies=self.anomalies, series=self.test, metric=["AUC_ROC"] + ) + + # 'actual_anomalies' must be binary + with self.assertRaises(ValueError): + fittable_scorer.eval_accuracy(actual_anomalies=self.test, series=self.test) + + # 'actual_anomalies' must contain anomalies (at least one) + with self.assertRaises(ValueError): + fittable_scorer.eval_accuracy( + actual_anomalies=self.only_0_anomalies, series=self.test + ) + + # 'actual_anomalies' cannot contain only anomalies + with self.assertRaises(ValueError): + fittable_scorer.eval_accuracy( + actual_anomalies=self.only_1_anomalies, series=self.test + ) + + # 'actual_anomalies' must match the number of series if length higher than 1 + with self.assertRaises(ValueError): + fittable_scorer.eval_accuracy( + actual_anomalies=[self.anomalies, self.anomalies], series=self.test + ) + with self.assertRaises(ValueError): + fittable_scorer.eval_accuracy( + actual_anomalies=[self.anomalies, self.anomalies], + series=[self.test, self.test, self.test], + ) + + # 'actual_anomalies' must have non empty intersection with 'series' + with self.assertRaises(ValueError): + fittable_scorer.eval_accuracy( + actual_anomalies=self.anomalies[:20], series=self.test[30:] + ) + with self.assertRaises(ValueError): + fittable_scorer.eval_accuracy( + actual_anomalies=[self.anomalies, self.anomalies[:20]], + series=[self.test, self.test[40:]], + ) + + for scorer in [non_fittable_scorer, fittable_scorer]: + + # name must be of type str + self.assertEqual( + type(scorer.__str__()), + str, + ) + + # 'metric' must be str and "AUC_ROC" or "AUC_PR" + with self.assertRaises(ValueError): + fittable_scorer.eval_accuracy_from_prediction( + actual_anomalies=self.anomalies, + actual_series=self.test, + pred_series=self.modified_test, + metric=1, + ) + with self.assertRaises(ValueError): + fittable_scorer.eval_accuracy_from_prediction( + actual_anomalies=self.anomalies, + actual_series=self.test, + pred_series=self.modified_test, + metric="auc_roc", + ) + with self.assertRaises(TypeError): + fittable_scorer.eval_accuracy_from_prediction( + actual_anomalies=self.anomalies, + actual_series=self.test, + pred_series=self.modified_test, + metric=["AUC_ROC"], + ) + + # 'actual_anomalies' must be binary + with self.assertRaises(ValueError): + scorer.eval_accuracy_from_prediction( + actual_anomalies=self.test, + actual_series=self.test, + pred_series=self.modified_test, + ) + + # 'actual_anomalies' must contain anomalies (at least one) + with self.assertRaises(ValueError): + scorer.eval_accuracy_from_prediction( + actual_anomalies=self.only_0_anomalies, + actual_series=self.test, + pred_series=self.modified_test, + ) + + # 'actual_anomalies' cannot contain only anomalies + with self.assertRaises(ValueError): + scorer.eval_accuracy_from_prediction( + actual_anomalies=self.only_1_anomalies, + actual_series=self.test, + pred_series=self.modified_test, + ) + + # 'actual_anomalies' must match the number of series if length higher than 1 + with self.assertRaises(ValueError): + scorer.eval_accuracy_from_prediction( + actual_anomalies=[self.anomalies, self.anomalies], + actual_series=[self.test, self.test, self.test], + pred_series=[ + self.modified_test, + self.modified_test, + self.modified_test, + ], + ) + with self.assertRaises(ValueError): + scorer.eval_accuracy_from_prediction( + actual_anomalies=[self.anomalies, self.anomalies], + actual_series=self.test, + pred_series=self.modified_test, + ) + + # 'actual_anomalies' must have non empty intersection with 'actual_series' and 'pred_series' + with self.assertRaises(ValueError): + scorer.eval_accuracy_from_prediction( + actual_anomalies=self.anomalies[:20], + actual_series=self.test[30:], + pred_series=self.modified_test[30:], + ) + with self.assertRaises(ValueError): + scorer.eval_accuracy_from_prediction( + actual_anomalies=[self.anomalies, self.anomalies[:20]], + actual_series=[self.test, self.test[40:]], + pred_series=[self.modified_test, self.modified_test[40:]], + ) + + def test_NonFittableAnomalyScorer(self): + + for scorer in list_NonFittableAnomalyScorer: + # Check if trainable is False, being a NonFittableAnomalyScorer + self.assertTrue(not scorer.trainable) + + # checks for score_from_prediction() + # input must be Timeseries or sequence of Timeseries + with self.assertRaises(ValueError): + scorer.score_from_prediction(self.train, "str") + with self.assertRaises(ValueError): + scorer.score_from_prediction( + [self.train, self.train], [self.modified_train, "str"] + ) + # score on sequence with series that have different width + with self.assertRaises(ValueError): + scorer.score_from_prediction(self.train, self.modified_mts_train) + # input sequences have different length + with self.assertRaises(ValueError): + scorer.score_from_prediction( + [self.train, self.train], [self.modified_train] + ) + # two inputs must have a non zero intersection + with self.assertRaises(ValueError): + scorer.score_from_prediction(self.train[:50], self.train[55:]) + # every pairwise element must have a non zero intersection + with self.assertRaises(ValueError): + scorer.score_from_prediction( + [self.train, self.train[:50]], [self.train, self.train[55:]] + ) + + def test_FittableAnomalyScorer(self): + + for scorer in list_FittableAnomalyScorer: + + # Need to call fit() before calling score() + with self.assertRaises(ValueError): + scorer.score(self.test) + + # Need to call fit() before calling score_from_prediction() + with self.assertRaises(ValueError): + scorer.score_from_prediction(self.test, self.modified_test) + + # Check if trainable is True, being a FittableAnomalyScorer + self.assertTrue(scorer.trainable) + + # Check if _fit_called is False + self.assertTrue(not scorer._fit_called) + + # fit on sequence with series that have different width + with self.assertRaises(ValueError): + scorer.fit([self.train, self.mts_train]) + + # fit on sequence with series that have different width + with self.assertRaises(ValueError): + scorer.fit_from_prediction( + [self.train, self.mts_train], + [self.modified_train, self.modified_mts_train], + ) + + # checks for fit_from_prediction() + # input must be Timeseries or sequence of Timeseries + with self.assertRaises(ValueError): + scorer.score_from_prediction(self.train, "str") + with self.assertRaises(ValueError): + scorer.score_from_prediction( + [self.train, self.train], [self.modified_train, "str"] + ) + # two inputs must have the same length + with self.assertRaises(ValueError): + scorer.fit_from_prediction( + [self.train, self.train], [self.modified_train] + ) + # two inputs must have the same width + with self.assertRaises(ValueError): + scorer.fit_from_prediction([self.train], [self.modified_mts_train]) + # every element must have the same width + with self.assertRaises(ValueError): + scorer.fit_from_prediction( + [self.train, self.mts_train], + [self.modified_train, self.modified_mts_train], + ) + # two inputs must have a non zero intersection + with self.assertRaises(ValueError): + scorer.fit_from_prediction(self.train[:50], self.train[55:]) + # every pairwise element must have a non zero intersection + with self.assertRaises(ValueError): + scorer.fit_from_prediction( + [self.train, self.train[:50]], [self.train, self.train[55:]] + ) + + # checks for fit() + # input must be Timeseries or sequence of Timeseries + with self.assertRaises(ValueError): + scorer.fit("str") + with self.assertRaises(ValueError): + scorer.fit([self.modified_train, "str"]) + + # checks for score_from_prediction() + scorer.fit_from_prediction(self.train, self.modified_train) + # input must be Timeseries or sequence of Timeseries + with self.assertRaises(ValueError): + scorer.score_from_prediction(self.train, "str") + with self.assertRaises(ValueError): + scorer.score_from_prediction( + [self.train, self.train], [self.modified_train, "str"] + ) + # two inputs must have the same length + with self.assertRaises(ValueError): + scorer.score_from_prediction( + [self.train, self.train], [self.modified_train] + ) + # two inputs must have the same width + with self.assertRaises(ValueError): + scorer.score_from_prediction([self.train], [self.modified_mts_train]) + # every element must have the same width + with self.assertRaises(ValueError): + scorer.score_from_prediction( + [self.train, self.mts_train], + [self.modified_train, self.modified_mts_train], + ) + # two inputs must have a non zero intersection + with self.assertRaises(ValueError): + scorer.score_from_prediction(self.train[:50], self.train[55:]) + # every pairwise element must have a non zero intersection + with self.assertRaises(ValueError): + scorer.score_from_prediction( + [self.train, self.train[:50]], [self.train, self.train[55:]] + ) + + # checks for score() + # input must be Timeseries or sequence of Timeseries + with self.assertRaises(ValueError): + scorer.score("str") + with self.assertRaises(ValueError): + scorer.score([self.modified_train, "str"]) + + # caseA: fit with fit() + # case1: fit on UTS + scorerA1 = scorer + scorerA1.fit(self.train) + # Check if _fit_called is True after being fitted + self.assertTrue(scorerA1._fit_called) + with self.assertRaises(ValueError): + # series must be same width as series used for training + scorerA1.score(self.mts_test) + # case2: fit on MTS + scorerA2 = scorer + scorerA2.fit(self.mts_train) + # Check if _fit_called is True after being fitted + self.assertTrue(scorerA2._fit_called) + with self.assertRaises(ValueError): + # series must be same width as series used for training + scorerA2.score(self.test) + + # caseB: fit with fit_from_prediction() + # case1: fit on UTS + scorerB1 = scorer + scorerB1.fit_from_prediction(self.train, self.modified_train) + # Check if _fit_called is True after being fitted + self.assertTrue(scorerB1._fit_called) + with self.assertRaises(ValueError): + # series must be same width as series used for training + scorerB1.score_from_prediction(self.mts_test, self.modified_mts_test) + # case2: fit on MTS + scorerB2 = scorer + scorerB2.fit_from_prediction(self.mts_train, self.modified_mts_train) + # Check if _fit_called is True after being fitted + self.assertTrue(scorerB2._fit_called) + with self.assertRaises(ValueError): + # series must be same width as series used for training + scorerB2.score_from_prediction(self.test, self.modified_test) + + def test_Norm(self): + + # component_wise must be bool + with self.assertRaises(ValueError): + Norm(component_wise=1) + with self.assertRaises(ValueError): + Norm(component_wise="string") + # if component_wise=False must always return a univariate anomaly score + scorer = Norm(component_wise=False) + self.assertTrue( + scorer.score_from_prediction(self.test, self.modified_test).width == 1 + ) + self.assertTrue( + scorer.score_from_prediction(self.mts_test, self.modified_mts_test).width + == 1 + ) + # if component_wise=True must always return the same width as the input + scorer = Norm(component_wise=True) + self.assertTrue( + scorer.score_from_prediction(self.test, self.modified_test).width == 1 + ) + self.assertTrue( + scorer.score_from_prediction(self.mts_test, self.modified_mts_test).width + == self.mts_test.width + ) + + scorer = Norm(component_wise=True) + # always expects a deterministic input + with self.assertRaises(ValueError): + scorer.score_from_prediction(self.train, self.probabilistic) + with self.assertRaises(ValueError): + scorer.score_from_prediction(self.probabilistic, self.train) + + # univariate case (equivalent to abs diff) + self.assertEqual( + scorer.score_from_prediction(self.test, self.test + 1) + .sum(axis=0) + .all_values() + .flatten()[0], + len(self.test), + ) + self.assertEqual( + scorer.score_from_prediction(self.test + 1, self.test) + .sum(axis=0) + .all_values() + .flatten()[0], + len(self.test), + ) + + # multivariate case with component_wise set to True (equivalent to abs diff) + # abs(a - 2a) = a + self.assertEqual( + scorer.score_from_prediction(self.mts_test, self.mts_test * 2)["0"], + self.mts_test["0"], + ) + self.assertEqual( + scorer.score_from_prediction(self.mts_test, self.mts_test * 2)["1"], + self.mts_test["1"], + ) + # abs(2a - a) = a + self.assertEqual( + scorer.score_from_prediction(self.mts_test * 2, self.mts_test)["0"], + self.mts_test["0"], + ) + self.assertEqual( + scorer.score_from_prediction(self.mts_test * 2, self.mts_test)["1"], + self.mts_test["1"], + ) + + scorer = Norm(component_wise=False) + + # always expects a deterministic input + + # univariate case (equivalent to abs diff) + self.assertEqual( + scorer.score_from_prediction(self.test, self.test + 1) + .sum(axis=0) + .all_values() + .flatten()[0], + len(self.test), + ) + self.assertEqual( + scorer.score_from_prediction(self.test + 1, self.test) + .sum(axis=0) + .all_values() + .flatten()[0], + len(self.test), + ) + + # multivariate case with component_wise set to False + # norm(a - a + sqrt(2)) = 2 * len(a) with a being series of dim=2 + self.assertAlmostEqual( + scorer.score_from_prediction(self.mts_test, self.mts_test + np.sqrt(2)) + .sum(axis=0) + .all_values() + .flatten()[0], + 2 * len(self.mts_test), + delta=1e-05, + ) + + self.assertFalse(scorer.is_probabilistic) + + def test_Difference(self): + + scorer = Difference() + + # always expects a deterministic input + with self.assertRaises(ValueError): + scorer.score_from_prediction(self.train, self.probabilistic) + with self.assertRaises(ValueError): + scorer.score_from_prediction(self.probabilistic, self.train) + + # univariate case + self.assertEqual( + scorer.score_from_prediction(self.test, self.test + 1) + .sum(axis=0) + .all_values() + .flatten()[0], + -len(self.test), + ) + self.assertEqual( + scorer.score_from_prediction(self.test + 1, self.test) + .sum(axis=0) + .all_values() + .flatten()[0], + len(self.test), + ) + + # multivariate case + # output of score() must be the same width as the width of the input + self.assertEqual( + scorer.score_from_prediction(self.mts_test, self.mts_test).width, + self.mts_test.width, + ) + + # a - 2a = - a + self.assertEqual( + scorer.score_from_prediction(self.mts_test, self.mts_test * 2)["0"], + -self.mts_test["0"], + ) + self.assertEqual( + scorer.score_from_prediction(self.mts_test, self.mts_test * 2)["1"], + -self.mts_test["1"], + ) + # 2a - a = a + self.assertEqual( + scorer.score_from_prediction(self.mts_test * 2, self.mts_test)["0"], + self.mts_test["0"], + ) + self.assertEqual( + scorer.score_from_prediction(self.mts_test * 2, self.mts_test)["1"], + self.mts_test["1"], + ) + + self.assertFalse(scorer.is_probabilistic) + + def test_WassersteinScorer(self): + + # component_wise parameter + # component_wise must be bool + with self.assertRaises(ValueError): + WassersteinScorer(component_wise=1) + with self.assertRaises(ValueError): + WassersteinScorer(component_wise="string") + # if component_wise=False must always return a univariate anomaly score + scorer = WassersteinScorer(component_wise=False) + scorer.fit(self.train) + self.assertTrue(scorer.score(self.test).width == 1) + scorer.fit(self.mts_train) + self.assertTrue(scorer.score(self.mts_test).width == 1) + # if component_wise=True must always return the same width as the input + scorer = WassersteinScorer(component_wise=True) + scorer.fit(self.train) + self.assertTrue(scorer.score(self.test).width == 1) + scorer.fit(self.mts_train) + self.assertTrue(scorer.score(self.mts_test).width == self.mts_test.width) + + # window parameter + # window must be int + with self.assertRaises(ValueError): + WassersteinScorer(window=True) + with self.assertRaises(ValueError): + WassersteinScorer(window="string") + # window must be non negative + with self.assertRaises(ValueError): + WassersteinScorer(window=-1) + # window must be different from 0 + with self.assertRaises(ValueError): + WassersteinScorer(window=0) + + # diff_fn paramter + # must be None, 'diff' or 'abs_diff' + with self.assertRaises(ValueError): + WassersteinScorer(diff_fn="random") + with self.assertRaises(ValueError): + WassersteinScorer(diff_fn=1) + + # test _diff_series() directly + with self.assertRaises(ValueError): + s_tmp = WassersteinScorer() + s_tmp.diff_fn = "random" + s_tmp._diff_series(self.train, self.test) + WassersteinScorer(diff_fn="diff")._diff_series(self.train, self.test) + WassersteinScorer()._diff_series(self.train, self.test) + + scorer = WassersteinScorer() + + # always expects a deterministic input + with self.assertRaises(ValueError): + scorer.score_from_prediction(self.train, self.probabilistic) + with self.assertRaises(ValueError): + scorer.score_from_prediction(self.probabilistic, self.train) + with self.assertRaises(ValueError): + scorer.score(self.probabilistic) + + # window must be smaller than the input of score() + scorer = WassersteinScorer(window=101) + with self.assertRaises(ValueError): + scorer.fit(self.train) # len(self.train)=100 + + scorer = WassersteinScorer(window=80) + scorer.fit(self.train) + with self.assertRaises(ValueError): + scorer.score(self.test[:50]) # len(self.test)=100 + + # test plotting (just call the functions) + scorer = WassersteinScorer(window=2) + scorer.fit(self.train) + scorer.show_anomalies(self.test, self.anomalies) + with self.assertRaises(ValueError): + # should fail for a sequence of series + scorer.show_anomalies([self.test, self.test], self.anomalies) + scorer.show_anomalies_from_prediction( + actual_series=self.test, + pred_series=self.test + 1, + actual_anomalies=self.anomalies, + ) + with self.assertRaises(ValueError): + # should fail for a sequence of series + scorer.show_anomalies_from_prediction( + actual_series=[self.test, self.test], + pred_series=self.test + 1, + actual_anomalies=self.anomalies, + ) + with self.assertRaises(ValueError): + # should fail for a sequence of series + scorer.show_anomalies_from_prediction( + actual_series=self.test, + pred_series=[self.test + 1, self.test + 2], + actual_anomalies=self.anomalies, + ) + + self.assertFalse(scorer.is_probabilistic) + + def test_univariate_Wasserstein(self): + + # univariate example + np.random.seed(42) + + np_train_wasserstein = np.abs(np.random.normal(loc=0, scale=0.1, size=100)) + train_wasserstein = TimeSeries.from_times_and_values( + self.train._time_index, np_train_wasserstein + ) + + np_test_wasserstein = np.abs(np.random.normal(loc=0, scale=0.1, size=100)) + np_first_anomaly = np.abs(np.random.normal(loc=0, scale=0.25, size=10)) + np_second_anomaly = np.abs(np.random.normal(loc=0.25, scale=0.05, size=5)) + np_third_anomaly = np.abs(np.random.normal(loc=0, scale=0.15, size=15)) + + np_test_wasserstein[10:20] = np_first_anomaly + np_test_wasserstein[40:45] = np_second_anomaly + np_test_wasserstein[70:85] = np_third_anomaly + test_wasserstein = TimeSeries.from_times_and_values( + self.train._time_index, np_test_wasserstein + ) + + # create the anomaly series + np_anomalies = np.zeros(len(test_wasserstein)) + np_anomalies[10:17] = 1 + np_anomalies[40:42] = 1 + np_anomalies[70:85] = 1 + anomalies_wasserstein = TimeSeries.from_times_and_values( + test_wasserstein.time_index, np_anomalies, columns=["is_anomaly"] + ) + + # test model with window of 10 + scorer_10 = WassersteinScorer(window=10) + scorer_10.fit(train_wasserstein) + auc_roc_w10 = scorer_10.eval_accuracy( + anomalies_wasserstein, test_wasserstein, metric="AUC_ROC" + ) + auc_pr_w10 = scorer_10.eval_accuracy( + anomalies_wasserstein, test_wasserstein, metric="AUC_PR" + ) + + # test model with window of 20 + scorer_20 = WassersteinScorer(window=20) + scorer_20.fit(train_wasserstein) + auc_roc_w20 = scorer_20.eval_accuracy( + anomalies_wasserstein, test_wasserstein, metric="AUC_ROC" + ) + auc_pr_w20 = scorer_20.eval_accuracy( + anomalies_wasserstein, test_wasserstein, metric="AUC_PR" + ) + + self.assertAlmostEqual(auc_roc_w10, 0.80637, delta=1e-05) + self.assertAlmostEqual(auc_pr_w10, 0.83390, delta=1e-05) + self.assertAlmostEqual(auc_roc_w20, 0.77828, delta=1e-05) + self.assertAlmostEqual(auc_pr_w20, 0.93934, delta=1e-05) + + def test_multivariate_componentwise_Wasserstein(self): + + # example multivariate WassersteinScorer component wise (True and False) + np.random.seed(3) + np_mts_train_wasserstein = np.abs( + np.random.normal(loc=[0, 0], scale=[0.1, 0.2], size=[100, 2]) + ) + mts_train_wasserstein = TimeSeries.from_times_and_values( + self.train._time_index, np_mts_train_wasserstein + ) + + np_mts_test_wasserstein = np.abs( + np.random.normal(loc=[0, 0], scale=[0.1, 0.2], size=[100, 2]) + ) + np_first_anomaly_width1 = np.abs(np.random.normal(loc=0.5, scale=0.4, size=10)) + np_first_anomaly_width2 = np.abs(np.random.normal(loc=0, scale=0.5, size=10)) + np_first_commmon_anomaly = np.abs( + np.random.normal(loc=0.5, scale=0.5, size=[10, 2]) + ) + + np_mts_test_wasserstein[5:15, 0] = np_first_anomaly_width1 + np_mts_test_wasserstein[35:45, 1] = np_first_anomaly_width2 + np_mts_test_wasserstein[65:75, :] = np_first_commmon_anomaly + + mts_test_wasserstein = TimeSeries.from_times_and_values( + mts_train_wasserstein._time_index, np_mts_test_wasserstein + ) + + # create the anomaly series width 1 + np_anomalies_width1 = np.zeros(len(mts_test_wasserstein)) + np_anomalies_width1[5:15] = 1 + np_anomalies_width1[65:75] = 1 + + # create the anomaly series width 2 + np_anomaly_width2 = np.zeros(len(mts_test_wasserstein)) + np_anomaly_width2[35:45] = 1 + np_anomaly_width2[65:75] = 1 + + anomalies_wasserstein_per_width = TimeSeries.from_times_and_values( + mts_test_wasserstein.time_index, + list(zip(*[np_anomalies_width1, np_anomaly_width2])), + columns=["is_anomaly_0", "is_anomaly_1"], + ) + + # create the anomaly series for the entire series + np_commmon_anomaly = np.zeros(len(mts_test_wasserstein)) + np_commmon_anomaly[5:15] = 1 + np_commmon_anomaly[35:45] = 1 + np_commmon_anomaly[65:75] = 1 + anomalies_common_wasserstein = TimeSeries.from_times_and_values( + mts_test_wasserstein.time_index, np_commmon_anomaly, columns=["is_anomaly"] + ) + + # test scorer with component_wise=False + scorer_w10_cwfalse = WassersteinScorer(window=10, component_wise=False) + scorer_w10_cwfalse.fit(mts_train_wasserstein) + auc_roc_cwfalse = scorer_w10_cwfalse.eval_accuracy( + anomalies_common_wasserstein, mts_test_wasserstein, metric="AUC_ROC" + ) + + # test scorer with component_wise=True + scorer_w10_cwtrue = WassersteinScorer(window=10, component_wise=True) + scorer_w10_cwtrue.fit(mts_train_wasserstein) + auc_roc_cwtrue = scorer_w10_cwtrue.eval_accuracy( + anomalies_wasserstein_per_width, mts_test_wasserstein, metric="AUC_ROC" + ) + + self.assertAlmostEqual(auc_roc_cwfalse, 0.94637, delta=1e-05) + self.assertAlmostEqual(auc_roc_cwtrue[0], 0.98606, delta=1e-05) + self.assertAlmostEqual(auc_roc_cwtrue[1], 0.96722, delta=1e-05) + + def test_kmeansScorer(self): + + # component_wise parameter + # component_wise must be bool + with self.assertRaises(ValueError): + KMeansScorer(component_wise=1) + with self.assertRaises(ValueError): + KMeansScorer(component_wise="string") + # if component_wise=False must always return a univariate anomaly score + scorer = KMeansScorer(component_wise=False) + scorer.fit(self.train) + self.assertTrue(scorer.score(self.test).width == 1) + scorer.fit(self.mts_train) + self.assertTrue(scorer.score(self.mts_test).width == 1) + # if component_wise=True must always return the same width as the input + scorer = KMeansScorer(component_wise=True) + scorer.fit(self.train) + self.assertTrue(scorer.score(self.test).width == 1) + scorer.fit(self.mts_train) + self.assertTrue(scorer.score(self.mts_test).width == self.mts_test.width) + + # window parameter + # window must be int + with self.assertRaises(ValueError): + KMeansScorer(window=True) + with self.assertRaises(ValueError): + KMeansScorer(window="string") + # window must be non negative + with self.assertRaises(ValueError): + KMeansScorer(window=-1) + # window must be different from 0 + with self.assertRaises(ValueError): + KMeansScorer(window=0) + + # diff_fn paramter + # must be None, 'diff' or 'abs_diff' + with self.assertRaises(ValueError): + KMeansScorer(diff_fn="random") + with self.assertRaises(ValueError): + KMeansScorer(diff_fn=1) + + scorer = KMeansScorer() + + # always expects a deterministic input + with self.assertRaises(ValueError): + scorer.score_from_prediction(self.train, self.probabilistic) + with self.assertRaises(ValueError): + scorer.score_from_prediction(self.probabilistic, self.train) + with self.assertRaises(ValueError): + scorer.score(self.probabilistic) + + # window must be smaller than the input of score() + scorer = KMeansScorer(window=101) + with self.assertRaises(ValueError): + scorer.fit(self.train) # len(self.train)=100 + + scorer = KMeansScorer(window=80) + scorer.fit(self.train) + with self.assertRaises(ValueError): + scorer.score(self.test[:50]) # len(self.test)=100 + + self.assertFalse(scorer.is_probabilistic) + + def test_univariate_kmeans(self): + + # univariate example + + np.random.seed(40) + + # create the train set + np_width1 = np.random.choice(a=[0, 1], size=100, p=[0.5, 0.5]) + np_width2 = (np_width1 == 0).astype(float) + KMeans_mts_train = TimeSeries.from_values( + np.dstack((np_width1, np_width2))[0], columns=["component 1", "component 2"] + ) + + # create the test set + # inject anomalies in the test timeseries + np.random.seed(3) + np_width1 = np.random.choice(a=[0, 1], size=100, p=[0.5, 0.5]) + np_width2 = (np_width1 == 0).astype(int) + + # 2 anomalies per type + # type 1: random values for only one width + np_width1[20:21] = 2 + np_width2[30:32] = 2 + + # type 2: shift both widths values (+/- 1 for both widths) + np_width1[45:47] = np_width1[45:47] + 1 + np_width2[45:47] = np_width2[45:47] + 1 + np_width1[60:64] = np_width1[65:69] - 1 + np_width2[60:64] = np_width2[65:69] - 1 + + # type 3: switch one state to another for only one width (1 to 0 for one width) + np_width1[75:82] = (np_width1[75:82] != 1).astype(int) + np_width2[90:96] = (np_width2[90:96] != 1).astype(int) + + KMeans_mts_test = TimeSeries.from_values( + np.dstack((np_width1, np_width2))[0], columns=["component 1", "component 2"] + ) + + # create the anomaly series + anomalies_index = [ + 20, + 30, + 31, + 45, + 46, + 60, + 61, + 62, + 63, + 75, + 76, + 77, + 78, + 79, + 80, + 81, + 90, + 91, + 92, + 93, + 94, + 95, + ] + np_anomalies = np.zeros(len(KMeans_mts_test)) + np_anomalies[anomalies_index] = 1 + KMeans_mts_anomalies = TimeSeries.from_times_and_values( + KMeans_mts_test.time_index, np_anomalies, columns=["is_anomaly"] + ) + + kmeans_scorer = KMeansScorer(k=2, window=1, component_wise=False) + kmeans_scorer.fit(KMeans_mts_train) + + metric_AUC_ROC = kmeans_scorer.eval_accuracy( + KMeans_mts_anomalies, KMeans_mts_test, metric="AUC_ROC" + ) + metric_AUC_PR = kmeans_scorer.eval_accuracy( + KMeans_mts_anomalies, KMeans_mts_test, metric="AUC_PR" + ) + + self.assertEqual(metric_AUC_ROC, 1.0) + self.assertEqual(metric_AUC_PR, 1.0) + + def test_multivariate_window_kmeans(self): + + # multivariate example with different windows + + np.random.seed(1) + + # create the train set + np_series = np.zeros(100) + np_series[0] = 2 + + for i in range(1, len(np_series)): + np_series[i] = np_series[i - 1] + np.random.choice(a=[-1, 1], p=[0.5, 0.5]) + if np_series[i] > 3: + np_series[i] = 3 + if np_series[i] < 0: + np_series[i] = 0 + + ts_train = TimeSeries.from_values(np_series, columns=["series"]) + + # create the test set + np.random.seed(3) + np_series = np.zeros(100) + np_series[0] = 1 + + for i in range(1, len(np_series)): + np_series[i] = np_series[i - 1] + np.random.choice(a=[-1, 1], p=[0.5, 0.5]) + if np_series[i] > 3: + np_series[i] = 3 + if np_series[i] < 0: + np_series[i] = 0 + + # 3 anomalies per type + # type 1: sudden shift between state 0 to state 2 without passing by state 1 + np_series[23] = 3 + np_series[44] = 3 + np_series[91] = 0 + + # type 2: having consecutive timestamps at state 1 or 2 + np_series[3:5] = 2 + np_series[17:19] = 1 + np_series[62:65] = 2 + + ts_test = TimeSeries.from_values(np_series, columns=["series"]) + + anomalies_index = [4, 23, 18, 44, 63, 64, 91] + np_anomalies = np.zeros(100) + np_anomalies[anomalies_index] = 1 + ts_anomalies = TimeSeries.from_times_and_values( + ts_test.time_index, np_anomalies, columns=["is_anomaly"] + ) + + kmeans_scorer_w1 = KMeansScorer(k=4, window=1) + kmeans_scorer_w1.fit(ts_train) + + kmeans_scorer_w2 = KMeansScorer(k=8, window=2) + kmeans_scorer_w2.fit(ts_train) + + auc_roc_w1 = kmeans_scorer_w1.eval_accuracy( + ts_anomalies, ts_test, metric="AUC_ROC" + ) + auc_pr_w1 = kmeans_scorer_w1.eval_accuracy( + ts_anomalies, ts_test, metric="AUC_PR" + ) + + auc_roc_w2 = kmeans_scorer_w2.eval_accuracy( + ts_anomalies, ts_test, metric="AUC_ROC" + ) + auc_pr_w2 = kmeans_scorer_w2.eval_accuracy( + ts_anomalies, ts_test, metric="AUC_PR" + ) + + self.assertAlmostEqual(auc_roc_w1, 0.41551, delta=1e-05) + self.assertAlmostEqual(auc_pr_w1, 0.064761, delta=1e-05) + self.assertAlmostEqual(auc_roc_w2, 0.957513, delta=1e-05) + self.assertAlmostEqual(auc_pr_w2, 0.88584, delta=1e-05) + + def test_multivariate_componentwise_kmeans(self): + + # example multivariate KMeans component wise (True and False) + np.random.seed(1) + + np_mts_train_kmeans = np.abs( + np.random.normal(loc=[0, 0], scale=[0.1, 0.2], size=[100, 2]) + ) + mts_train_kmeans = TimeSeries.from_times_and_values( + self.train._time_index, np_mts_train_kmeans + ) + + np_mts_test_kmeans = np.abs( + np.random.normal(loc=[0, 0], scale=[0.1, 0.2], size=[100, 2]) + ) + np_first_anomaly_width1 = np.abs(np.random.normal(loc=0.5, scale=0.4, size=10)) + np_first_anomaly_width2 = np.abs(np.random.normal(loc=0, scale=0.5, size=10)) + np_first_commmon_anomaly = np.abs( + np.random.normal(loc=0.5, scale=0.5, size=[10, 2]) + ) + + np_mts_test_kmeans[5:15, 0] = np_first_anomaly_width1 + np_mts_test_kmeans[35:45, 1] = np_first_anomaly_width2 + np_mts_test_kmeans[65:75, :] = np_first_commmon_anomaly + + mts_test_kmeans = TimeSeries.from_times_and_values( + mts_train_kmeans._time_index, np_mts_test_kmeans + ) + + # create the anomaly series width 1 + np_anomalies_width1 = np.zeros(len(mts_test_kmeans)) + np_anomalies_width1[5:15] = 1 + np_anomalies_width1[65:75] = 1 + + # create the anomaly series width 2 + np_anomaly_width2 = np.zeros(len(mts_test_kmeans)) + np_anomaly_width2[35:45] = 1 + np_anomaly_width2[65:75] = 1 + + anomalies_kmeans_per_width = TimeSeries.from_times_and_values( + mts_test_kmeans.time_index, + list(zip(*[np_anomalies_width1, np_anomaly_width2])), + columns=["is_anomaly_0", "is_anomaly_1"], + ) + + # create the anomaly series for the entire series + np_commmon_anomaly = np.zeros(len(mts_test_kmeans)) + np_commmon_anomaly[5:15] = 1 + np_commmon_anomaly[35:45] = 1 + np_commmon_anomaly[65:75] = 1 + anomalies_common_kmeans = TimeSeries.from_times_and_values( + mts_test_kmeans.time_index, np_commmon_anomaly, columns=["is_anomaly"] + ) + + # test scorer with component_wise=False + scorer_w10_cwfalse = KMeansScorer(window=10, component_wise=False) + scorer_w10_cwfalse.fit(mts_train_kmeans) + auc_roc_cwfalse = scorer_w10_cwfalse.eval_accuracy( + anomalies_common_kmeans, mts_test_kmeans, metric="AUC_ROC" + ) + + # test scorer with component_wise=True + scorer_w10_cwtrue = KMeansScorer(window=10, component_wise=True) + scorer_w10_cwtrue.fit(mts_train_kmeans) + auc_roc_cwtrue = scorer_w10_cwtrue.eval_accuracy( + anomalies_kmeans_per_width, mts_test_kmeans, metric="AUC_ROC" + ) + + self.assertAlmostEqual(auc_roc_cwfalse, 0.9851, delta=1e-05) + self.assertAlmostEqual(auc_roc_cwtrue[0], 1.0, delta=1e-05) + self.assertAlmostEqual(auc_roc_cwtrue[1], 0.97666, delta=1e-05) + + def test_PyODScorer(self): + + # model parameter must be pyod.models typy BaseDetector + with self.assertRaises(ValueError): + PyODScorer(model=MovingAverage(window=10)) + + # component_wise parameter + # component_wise must be bool + with self.assertRaises(ValueError): + PyODScorer(model=KNN(), component_wise=1) + with self.assertRaises(ValueError): + PyODScorer(model=KNN(), component_wise="string") + # if component_wise=False must always return a univariate anomaly score + scorer = PyODScorer(model=KNN(), component_wise=False) + scorer.fit(self.train) + self.assertTrue(scorer.score(self.test).width == 1) + scorer.fit(self.mts_train) + self.assertTrue(scorer.score(self.mts_test).width == 1) + # if component_wise=True must always return the same width as the input + scorer = PyODScorer(model=KNN(), component_wise=True) + scorer.fit(self.train) + self.assertTrue(scorer.score(self.test).width == 1) + scorer.fit(self.mts_train) + self.assertTrue(scorer.score(self.mts_test).width == self.mts_test.width) + + # window parameter + # window must be int + with self.assertRaises(ValueError): + PyODScorer(model=KNN(), window=True) + with self.assertRaises(ValueError): + PyODScorer(model=KNN(), window="string") + # window must be non negative + with self.assertRaises(ValueError): + PyODScorer(model=KNN(), window=-1) + # window must be different from 0 + with self.assertRaises(ValueError): + PyODScorer(model=KNN(), window=0) + + # diff_fn paramter + # must be None, 'diff' or 'abs_diff' + with self.assertRaises(ValueError): + PyODScorer(model=KNN(), diff_fn="random") + with self.assertRaises(ValueError): + PyODScorer(model=KNN(), diff_fn=1) + + scorer = PyODScorer(model=KNN()) + + # always expects a deterministic input + with self.assertRaises(ValueError): + scorer.score_from_prediction(self.train, self.probabilistic) + with self.assertRaises(ValueError): + scorer.score_from_prediction(self.probabilistic, self.train) + with self.assertRaises(ValueError): + scorer.score(self.probabilistic) + + # window must be smaller than the input of score() + scorer = PyODScorer(model=KNN(), window=101) + with self.assertRaises(ValueError): + scorer.fit(self.train) # len(self.train)=100 + + scorer = PyODScorer(model=KNN(), window=80) + scorer.fit(self.train) + with self.assertRaises(ValueError): + scorer.score(self.test[:50]) # len(self.test)=100 + + self.assertFalse(scorer.is_probabilistic) + + def test_univariate_PyODScorer(self): + + # univariate test + np.random.seed(40) + + # create the train set + np_width1 = np.random.choice(a=[0, 1], size=100, p=[0.5, 0.5]) + np_width2 = (np_width1 == 0).astype(float) + pyod_mts_train = TimeSeries.from_values( + np.dstack((np_width1, np_width2))[0], columns=["component 1", "component 2"] + ) + + # create the test set + # inject anomalies in the test timeseries + np.random.seed(3) + np_width1 = np.random.choice(a=[0, 1], size=100, p=[0.5, 0.5]) + np_width2 = (np_width1 == 0).astype(int) + + # 2 anomalies per type + # type 1: random values for only one width + np_width1[20:21] = 2 + np_width2[30:32] = 2 + + # type 2: shift both widths values (+/- 1 for both widths) + np_width1[45:47] = np_width1[45:47] + 1 + np_width2[45:47] = np_width2[45:47] + 1 + np_width1[60:64] = np_width1[65:69] - 1 + np_width2[60:64] = np_width2[65:69] - 1 + + # type 3: switch one state to another for only one width (1 to 0 for one width) + np_width1[75:82] = (np_width1[75:82] != 1).astype(int) + np_width2[90:96] = (np_width2[90:96] != 1).astype(int) + + pyod_mts_test = TimeSeries.from_values( + np.dstack((np_width1, np_width2))[0], columns=["component 1", "component 2"] + ) + + # create the anomaly series + anomalies_index = [ + 20, + 30, + 31, + 45, + 46, + 60, + 61, + 62, + 63, + 75, + 76, + 77, + 78, + 79, + 80, + 81, + 90, + 91, + 92, + 93, + 94, + 95, + ] + np_anomalies = np.zeros(len(pyod_mts_test)) + np_anomalies[anomalies_index] = 1 + pyod_mts_anomalies = TimeSeries.from_times_and_values( + pyod_mts_test.time_index, np_anomalies, columns=["is_anomaly"] + ) + + pyod_scorer = PyODScorer( + model=KNN(n_neighbors=10), component_wise=False, window=1 + ) + pyod_scorer.fit(pyod_mts_train) + + metric_AUC_ROC = pyod_scorer.eval_accuracy( + pyod_mts_anomalies, pyod_mts_test, metric="AUC_ROC" + ) + metric_AUC_PR = pyod_scorer.eval_accuracy( + pyod_mts_anomalies, pyod_mts_test, metric="AUC_PR" + ) + + self.assertEqual(metric_AUC_ROC, 1.0) + self.assertEqual(metric_AUC_PR, 1.0) + + def test_multivariate_window_PyODScorer(self): + + # multivariate example (with different window) + + np.random.seed(1) + + # create the train set + np_series = np.zeros(100) + np_series[0] = 2 + + for i in range(1, len(np_series)): + np_series[i] = np_series[i - 1] + np.random.choice(a=[-1, 1], p=[0.5, 0.5]) + if np_series[i] > 3: + np_series[i] = 3 + if np_series[i] < 0: + np_series[i] = 0 + + ts_train = TimeSeries.from_values(np_series, columns=["series"]) + + # create the test set + np.random.seed(3) + np_series = np.zeros(100) + np_series[0] = 1 + + for i in range(1, len(np_series)): + np_series[i] = np_series[i - 1] + np.random.choice(a=[-1, 1], p=[0.5, 0.5]) + if np_series[i] > 3: + np_series[i] = 3 + if np_series[i] < 0: + np_series[i] = 0 + + # 3 anomalies per type + # type 1: sudden shift between state 0 to state 2 without passing by state 1 + np_series[23] = 3 + np_series[44] = 3 + np_series[91] = 0 + + # type 2: having consecutive timestamps at state 1 or 2 + np_series[3:5] = 2 + np_series[17:19] = 1 + np_series[62:65] = 2 + + ts_test = TimeSeries.from_values(np_series, columns=["series"]) + + anomalies_index = [4, 23, 18, 44, 63, 64, 91] + np_anomalies = np.zeros(100) + np_anomalies[anomalies_index] = 1 + ts_anomalies = TimeSeries.from_times_and_values( + ts_test.time_index, np_anomalies, columns=["is_anomaly"] + ) + + pyod_scorer_w1 = PyODScorer( + model=KNN(n_neighbors=10), component_wise=False, window=1 + ) + pyod_scorer_w1.fit(ts_train) + + pyod_scorer_w2 = PyODScorer( + model=KNN(n_neighbors=10), component_wise=False, window=2 + ) + pyod_scorer_w2.fit(ts_train) + + auc_roc_w1 = pyod_scorer_w1.eval_accuracy( + ts_anomalies, ts_test, metric="AUC_ROC" + ) + auc_pr_w1 = pyod_scorer_w1.eval_accuracy(ts_anomalies, ts_test, metric="AUC_PR") + + auc_roc_w2 = pyod_scorer_w2.eval_accuracy( + ts_anomalies, ts_test, metric="AUC_ROC" + ) + auc_pr_w2 = pyod_scorer_w2.eval_accuracy(ts_anomalies, ts_test, metric="AUC_PR") + + self.assertAlmostEqual(auc_roc_w1, 0.5, delta=1e-05) + self.assertAlmostEqual(auc_pr_w1, 0.07, delta=1e-05) + self.assertAlmostEqual(auc_roc_w2, 0.957513, delta=1e-05) + self.assertAlmostEqual(auc_pr_w2, 0.88584, delta=1e-05) + + def test_multivariate_componentwise_PyODScorer(self): + + # multivariate example with component wise (True and False) + + np.random.seed(1) + + np_mts_train_PyOD = np.abs( + np.random.normal(loc=[0, 0], scale=[0.1, 0.2], size=[100, 2]) + ) + mts_train_PyOD = TimeSeries.from_times_and_values( + self.train._time_index, np_mts_train_PyOD + ) + + np_mts_test_PyOD = np.abs( + np.random.normal(loc=[0, 0], scale=[0.1, 0.2], size=[100, 2]) + ) + np_first_anomaly_width1 = np.abs(np.random.normal(loc=0.5, scale=0.4, size=10)) + np_first_anomaly_width2 = np.abs(np.random.normal(loc=0, scale=0.5, size=10)) + np_first_commmon_anomaly = np.abs( + np.random.normal(loc=0.5, scale=0.5, size=[10, 2]) + ) + + np_mts_test_PyOD[5:15, 0] = np_first_anomaly_width1 + np_mts_test_PyOD[35:45, 1] = np_first_anomaly_width2 + np_mts_test_PyOD[65:75, :] = np_first_commmon_anomaly + + mts_test_PyOD = TimeSeries.from_times_and_values( + mts_train_PyOD._time_index, np_mts_test_PyOD + ) + + # create the anomaly series width 1 + np_anomalies_width1 = np.zeros(len(mts_test_PyOD)) + np_anomalies_width1[5:15] = 1 + np_anomalies_width1[65:75] = 1 + + # create the anomaly series width 2 + np_anomaly_width2 = np.zeros(len(mts_test_PyOD)) + np_anomaly_width2[35:45] = 1 + np_anomaly_width2[65:75] = 1 + + anomalies_pyod_per_width = TimeSeries.from_times_and_values( + mts_test_PyOD.time_index, + list(zip(*[np_anomalies_width1, np_anomaly_width2])), + columns=["is_anomaly_0", "is_anomaly_1"], + ) + + # create the anomaly series for the entire series + np_commmon_anomaly = np.zeros(len(mts_test_PyOD)) + np_commmon_anomaly[5:15] = 1 + np_commmon_anomaly[35:45] = 1 + np_commmon_anomaly[65:75] = 1 + anomalies_common_PyOD = TimeSeries.from_times_and_values( + mts_test_PyOD.time_index, np_commmon_anomaly, columns=["is_anomaly"] + ) + + # test scorer with component_wise=False + scorer_w10_cwfalse = PyODScorer( + model=KNN(n_neighbors=10), component_wise=False, window=10 + ) + scorer_w10_cwfalse.fit(mts_train_PyOD) + auc_roc_cwfalse = scorer_w10_cwfalse.eval_accuracy( + anomalies_common_PyOD, mts_test_PyOD, metric="AUC_ROC" + ) + + # test scorer with component_wise=True + scorer_w10_cwtrue = PyODScorer( + model=KNN(n_neighbors=10), component_wise=True, window=10 + ) + scorer_w10_cwtrue.fit(mts_train_PyOD) + auc_roc_cwtrue = scorer_w10_cwtrue.eval_accuracy( + anomalies_pyod_per_width, mts_test_PyOD, metric="AUC_ROC" + ) + + self.assertAlmostEqual(auc_roc_cwfalse, 0.990566, delta=1e-05) + self.assertAlmostEqual(auc_roc_cwtrue[0], 1.0, delta=1e-05) + self.assertAlmostEqual(auc_roc_cwtrue[1], 0.98311, delta=1e-05) + + def test_NLLScorer(self): + + for s in list_NLLScorer: + # expects for 'actual_series' a deterministic input and for 'pred_series' a probabilistic input + with self.assertRaises(ValueError): + s.score_from_prediction(actual_series=self.test, pred_series=self.test) + with self.assertRaises(ValueError): + s.score_from_prediction( + actual_series=self.probabilistic, pred_series=self.train + ) + + def test_GaussianNLLScorer(self): + + # window parameter + # window must be int + with self.assertRaises(ValueError): + GaussianNLLScorer(window=True) + with self.assertRaises(ValueError): + GaussianNLLScorer(window="string") + # window must be non negative + with self.assertRaises(ValueError): + GaussianNLLScorer(window=-1) + # window must be different from 0 + with self.assertRaises(ValueError): + GaussianNLLScorer(window=0) + + scorer = GaussianNLLScorer(window=101) + # window must be smaller than the input of score_from_prediction() + with self.assertRaises(ValueError): + scorer.score_from_prediction( + actual_series=self.test, pred_series=self.probabilistic + ) # len(self.test)=100 + + np.random.seed(4) + scorer = GaussianNLLScorer() + + # test 1 univariate (len=1 and window=1) + gaussian_samples_1 = np.random.normal(loc=0, scale=2, size=10000) + distribution_series = TimeSeries.from_values( + gaussian_samples_1.reshape(1, 1, -1) + ) + actual_series = TimeSeries.from_values(np.array([3])) + value_test1 = ( + scorer.score_from_prediction(actual_series, distribution_series) + .all_values() + .flatten()[0] + ) + + # check if value_test1 is the - log likelihood + self.assertAlmostEqual( + value_test1, -np.log(norm.pdf(3, loc=0, scale=2)), delta=1e-01 + ) + + # test 2 univariate (len=1 and window=1) + gaussian_samples_2 = np.random.normal(loc=0, scale=2, size=10000) + distribution_series = TimeSeries.from_values( + gaussian_samples_2.reshape(1, 1, -1) + ) + actual_series = TimeSeries.from_values(np.array([-2])) + value_test2 = ( + scorer.score_from_prediction(actual_series, distribution_series) + .all_values() + .flatten()[0] + ) + + # check if value_test2 is the - log likelihood + self.assertAlmostEqual( + value_test2, -np.log(norm.pdf(-2, loc=0, scale=2)), delta=1e-01 + ) + + # test window univariate (len=2 and window=2) + distribution_series = TimeSeries.from_values( + np.array( + [gaussian_samples_1.reshape(1, -1), gaussian_samples_2.reshape(1, -1)] + ) + ) + actual_series = TimeSeries.from_values(np.array([3, -2])) + value_window = scorer.score_from_prediction(actual_series, distribution_series) + + # check length + self.assertEqual(len(value_window), 2) + # check width + self.assertEqual(value_window.width, 1) + + # check equal value_test1 and value_test2 + self.assertEqual(value_window.all_values().flatten()[0], value_test1) + self.assertEqual(value_window.all_values().flatten()[1], value_test2) + + scorer = GaussianNLLScorer(window=2) + # check avg of two values + self.assertEqual( + scorer.score_from_prediction(actual_series, distribution_series) + .all_values() + .flatten()[0], + (value_test1 + value_test2) / 2, + ) + + # test window multivariate (n_samples=2, len=1, window=1) + scorer = GaussianNLLScorer(window=1) + distribution_series = TimeSeries.from_values( + np.array([gaussian_samples_1, gaussian_samples_2]).reshape(1, 2, -1) + ) + actual_series = TimeSeries.from_values(np.array([3, -2]).reshape(1, -1)) + value_multivariate = scorer.score_from_prediction( + actual_series, distribution_series + ) + + # check length + self.assertEqual(len(value_multivariate), 1) + # check width + self.assertEqual(value_multivariate.width, 2) + + # check equal value_test1 and value_test2 + self.assertEqual(value_multivariate.all_values().flatten()[0], value_test1) + self.assertEqual(value_multivariate.all_values().flatten()[1], value_test2) + + # test window multivariate (n_samples=2, len=2, window=1 and 2) + scorer_w1 = GaussianNLLScorer(window=1) + scorer_w2 = GaussianNLLScorer(window=2) + + gaussian_samples_3 = np.random.normal(loc=0, scale=2, size=10000) + gaussian_samples_4 = np.random.normal(loc=0, scale=2, size=10000) + + distribution_series = TimeSeries.from_values( + np.array( + [ + gaussian_samples_1, + gaussian_samples_2, + gaussian_samples_3, + gaussian_samples_4, + ] + ).reshape(2, 2, -1) + ) + + actual_series = TimeSeries.from_values( + np.array([1.5, 2.1, 0.1, 0.001]).reshape(2, -1) + ) + + score_w1 = scorer_w1.score_from_prediction(actual_series, distribution_series) + score_w2 = scorer_w2.score_from_prediction(actual_series, distribution_series) + + # check length + self.assertEqual(len(score_w1), 2) + self.assertEqual(len(score_w2), 1) + # check width + self.assertEqual(score_w1.width, 2) + self.assertEqual(score_w2.width, 2) + + # check values for window=1 + self.assertAlmostEqual( + score_w1.all_values().flatten()[0], + -np.log(norm.pdf(1.5, loc=0, scale=2)), + delta=1e-01, + ) + self.assertAlmostEqual( + score_w1.all_values().flatten()[1], + -np.log(norm.pdf(2.1, loc=0, scale=2)), + delta=1e-01, + ) + self.assertAlmostEqual( + score_w1.all_values().flatten()[2], + -np.log(norm.pdf(0.1, loc=0, scale=2)), + delta=1e-01, + ) + self.assertAlmostEqual( + score_w1.all_values().flatten()[3], + -np.log(norm.pdf(0.001, loc=0, scale=2)), + delta=1e-01, + ) + + # check values for window=2 (must be equal to the mean of the past 2 values) + self.assertAlmostEqual( + score_w2.all_values().flatten()[0], + ( + -np.log(norm.pdf(1.5, loc=0, scale=2)) + - np.log(norm.pdf(0.1, loc=0, scale=2)) + ) + / 2, + delta=1e-01, + ) + self.assertAlmostEqual( + score_w2.all_values().flatten()[1], + ( + -np.log(norm.pdf(2.1, loc=0, scale=2)) + - np.log(norm.pdf(0.001, loc=0, scale=2)) + ) + / 2, + delta=1e-01, + ) + + self.assertTrue(scorer.is_probabilistic) + + def test_LaplaceNLLScorer(self): + + # window parameter + # window must be int + with self.assertRaises(ValueError): + LaplaceNLLScorer(window=True) + with self.assertRaises(ValueError): + LaplaceNLLScorer(window="string") + # window must be non negative + with self.assertRaises(ValueError): + LaplaceNLLScorer(window=-1) + # window must be different from 0 + with self.assertRaises(ValueError): + LaplaceNLLScorer(window=0) + + scorer = LaplaceNLLScorer(window=101) + # window must be smaller than the input of score_from_prediction() + with self.assertRaises(ValueError): + scorer.score_from_prediction( + actual_series=self.test, pred_series=self.probabilistic + ) # len(self.test)=100 + + np.random.seed(4) + + scorer = LaplaceNLLScorer() + + # test 1 univariate (len=1 and window=1) + laplace_samples_1 = np.random.laplace(loc=0, scale=2, size=1000) + distribution_series = TimeSeries.from_values( + laplace_samples_1.reshape(1, 1, -1) + ) + actual_series = TimeSeries.from_values(np.array([3])) + value_test1 = ( + scorer.score_from_prediction(actual_series, distribution_series) + .all_values() + .flatten()[0] + ) + + # check if value_test1 is the - log likelihood + self.assertAlmostEqual( + # This is approximate because our NLL scorer is fit from samples + value_test1, + -np.log(laplace.pdf(3, loc=0, scale=2)), + delta=1e-01, + ) + + # test 2 univariate (len=1 and window=1) + laplace_samples_2 = np.random.laplace(loc=0, scale=2, size=1000) + distribution_series = TimeSeries.from_values( + laplace_samples_2.reshape(1, 1, -1) + ) + actual_series = TimeSeries.from_values(np.array([-2])) + value_test2 = ( + scorer.score_from_prediction(actual_series, distribution_series) + .all_values() + .flatten()[0] + ) + + # check if value_test2 is the - log likelihood + self.assertAlmostEqual( + # This is approximate because our NLL scorer is fit from samples + value_test2, + -np.log(laplace.pdf(-2, loc=0, scale=2)), + delta=1e-01, + ) + + # test window univariate (len=2 and window=2) + distribution_series = TimeSeries.from_values( + np.array( + [laplace_samples_1.reshape(1, -1), laplace_samples_2.reshape(1, -1)] + ) + ) + actual_series = TimeSeries.from_values(np.array([3, -2])) + value_window = scorer.score_from_prediction(actual_series, distribution_series) + + # check length + self.assertEqual(len(value_window), 2) + # check width + self.assertEqual(value_window.width, 1) + + # check equal value_test1 and value_test2 + self.assertAlmostEqual(value_window.all_values().flatten()[0], value_test1) + self.assertAlmostEqual(value_window.all_values().flatten()[1], value_test2) + + scorer = LaplaceNLLScorer(window=2) + # check avg of two values + self.assertEqual( + scorer.score_from_prediction(actual_series, distribution_series) + .all_values() + .flatten()[0], + (value_test1 + value_test2) / 2, + ) + + # test window multivariate (n_samples=2, len=1, window=1) + scorer = LaplaceNLLScorer(window=1) + distribution_series = TimeSeries.from_values( + np.array([laplace_samples_1, laplace_samples_2]).reshape(1, 2, -1) + ) + actual_series = TimeSeries.from_values(np.array([3, -2]).reshape(1, -1)) + value_multivariate = scorer.score_from_prediction( + actual_series, distribution_series + ) + + # check length + self.assertEqual(len(value_multivariate), 1) + # check width + self.assertEqual(value_multivariate.width, 2) + + # check equal value_test1 and value_test2 + self.assertAlmostEqual( + value_multivariate.all_values().flatten()[0], value_test1 + ) + self.assertAlmostEqual( + value_multivariate.all_values().flatten()[1], value_test2 + ) + + # test window multivariate (n_samples=2, len=2, window=1 and 2) + scorer_w1 = LaplaceNLLScorer(window=1) + scorer_w2 = LaplaceNLLScorer(window=2) + + laplace_samples_3 = np.random.laplace(loc=0, scale=2, size=1000) + laplace_samples_4 = np.random.laplace(loc=0, scale=2, size=1000) + + distribution_series = TimeSeries.from_values( + np.array( + [ + laplace_samples_1, + laplace_samples_2, + laplace_samples_3, + laplace_samples_4, + ] + ).reshape(2, 2, -1) + ) + + actual_series = TimeSeries.from_values( + np.array([1.5, 2, 0.1, 0.001]).reshape(2, -1) + ) + + score_w1 = scorer_w1.score_from_prediction(actual_series, distribution_series) + score_w2 = scorer_w2.score_from_prediction(actual_series, distribution_series) + + # check length + self.assertEqual(len(score_w1), 2) + self.assertEqual(len(score_w2), 1) + # check width + self.assertEqual(score_w1.width, 2) + self.assertEqual(score_w2.width, 2) + + # check values for window=1 + self.assertAlmostEqual( + score_w1.all_values().flatten()[0], + -np.log(laplace.pdf(1.5, loc=0, scale=2)), + delta=1e-01, + ) + self.assertAlmostEqual( + # This is approximate because our NLL scorer is fit from samples + score_w1.all_values().flatten()[1], + -np.log(laplace.pdf(2, loc=0, scale=2)), + delta=0.5, + ) + self.assertAlmostEqual( + score_w1.all_values().flatten()[2], + -np.log(laplace.pdf(0.1, loc=0, scale=2)), + delta=1e-01, + ) + self.assertAlmostEqual( + score_w1.all_values().flatten()[3], + -np.log(laplace.pdf(0.001, loc=0, scale=2)), + delta=1e-01, + ) + + # check values for window=2 (must be equal to the mean of the past 2 values) + self.assertAlmostEqual( + score_w2.all_values().flatten()[0], + ( + -np.log(laplace.pdf(1.5, loc=0, scale=2)) + - np.log(laplace.pdf(0.1, loc=0, scale=2)) + ) + / 2, + delta=1e-01, + ) + self.assertAlmostEqual( + # This is approximate because our NLL scorer is fit from samples + score_w2.all_values().flatten()[1], + ( + -np.log(laplace.pdf(2, loc=0, scale=2)) + - np.log(laplace.pdf(0.001, loc=0, scale=2)) + ) + / 2, + delta=0.5, + ) + + self.assertTrue(scorer.is_probabilistic) + + def test_ExponentialNLLScorer(self): + + # window parameter + # window must be int + with self.assertRaises(ValueError): + ExponentialNLLScorer(window=True) + with self.assertRaises(ValueError): + ExponentialNLLScorer(window="string") + # window must be non negative + with self.assertRaises(ValueError): + ExponentialNLLScorer(window=-1) + # window must be different from 0 + with self.assertRaises(ValueError): + ExponentialNLLScorer(window=0) + + scorer = ExponentialNLLScorer(window=101) + # window must be smaller than the input of score_from_prediction() + with self.assertRaises(ValueError): + scorer.score_from_prediction( + actual_series=self.test, pred_series=self.probabilistic + ) # len(self.test)=100 + + np.random.seed(4) + scorer = ExponentialNLLScorer() + + # test 1 univariate (len=1 and window=1) + exponential_samples_1 = np.random.exponential(scale=2.0, size=1000) + distribution_series = TimeSeries.from_values( + exponential_samples_1.reshape(1, 1, -1) + ) + actual_series = TimeSeries.from_values(np.array([3])) + value_test1 = ( + scorer.score_from_prediction(actual_series, distribution_series) + .all_values() + .flatten()[0] + ) + + # check if value_test1 is the - log likelihood + self.assertAlmostEqual( + # This is approximate because our NLL scorer is fit from samples and also uses loc + value_test1, + -np.log(expon.pdf(3, scale=2.0)), + delta=1e-01, + ) + + # test 2 univariate (len=1 and window=1) + exponential_samples_2 = np.random.exponential(scale=2.0, size=1000) + distribution_series = TimeSeries.from_values( + exponential_samples_2.reshape(1, 1, -1) + ) + actual_series = TimeSeries.from_values(np.array([10])) + value_test2 = ( + scorer.score_from_prediction(actual_series, distribution_series) + .all_values() + .flatten()[0] + ) + + # check if value_test2 is the - log likelihood + self.assertAlmostEqual( + # This is approximate because our NLL scorer is fit from samples and also uses loc + value_test2, + -np.log(expon.pdf(10, scale=2)), + delta=1e-01, + ) + + # test window univariate (len=2 and window=2) + distribution_series = TimeSeries.from_values( + np.array( + [ + exponential_samples_1.reshape(1, -1), + exponential_samples_2.reshape(1, -1), + ] + ) + ) + actual_series = TimeSeries.from_values(np.array([3, 10])) + value_window = scorer.score_from_prediction(actual_series, distribution_series) + + # check length + self.assertEqual(len(value_window), 2) + # check width + self.assertEqual(value_window.width, 1) + + # check equal value_test1 and value_test2 + self.assertEqual(value_window.all_values().flatten()[0], value_test1) + self.assertEqual(value_window.all_values().flatten()[1], value_test2) + + scorer = ExponentialNLLScorer(window=2) + # check avg of two values + self.assertEqual( + scorer.score_from_prediction(actual_series, distribution_series) + .all_values() + .flatten()[0], + (value_test1 + value_test2) / 2, + ) + + # test window multivariate (n_samples=2, len=1, window=1) + scorer = ExponentialNLLScorer(window=1) + distribution_series = TimeSeries.from_values( + np.array([exponential_samples_1, exponential_samples_2]).reshape(1, 2, -1) + ) + actual_series = TimeSeries.from_values(np.array([3, 10]).reshape(1, -1)) + value_multivariate = scorer.score_from_prediction( + actual_series, distribution_series + ) + + # check length + self.assertEqual(len(value_multivariate), 1) + # check width + self.assertEqual(value_multivariate.width, 2) + + # check equal value_test1 and value_test2 + self.assertEqual(value_multivariate.all_values().flatten()[0], value_test1) + self.assertEqual(value_multivariate.all_values().flatten()[1], value_test2) + + # test window multivariate (n_samples=2, len=2, window=1 and 2) + scorer_w1 = ExponentialNLLScorer(window=1) + scorer_w2 = ExponentialNLLScorer(window=2) + + exponential_samples_3 = np.random.exponential(scale=2, size=1000) + exponential_samples_4 = np.random.exponential(scale=2, size=1000) + + distribution_series = TimeSeries.from_values( + np.array( + [ + exponential_samples_1, + exponential_samples_2, + exponential_samples_3, + exponential_samples_4, + ] + ).reshape(2, 2, -1) + ) + + actual_series = TimeSeries.from_values( + np.array([1.5, 2, 0.1, 0.001]).reshape(2, -1) + ) + + score_w1 = scorer_w1.score_from_prediction(actual_series, distribution_series) + score_w2 = scorer_w2.score_from_prediction(actual_series, distribution_series) + + # check length + self.assertEqual(len(score_w1), 2) + self.assertEqual(len(score_w2), 1) + # check width + self.assertEqual(score_w1.width, 2) + self.assertEqual(score_w2.width, 2) + + # check values for window=1 + self.assertAlmostEqual( + score_w1.all_values().flatten()[0], + -np.log(expon.pdf(1.5, scale=2)), + delta=1e-01, + ) + self.assertAlmostEqual( + score_w1.all_values().flatten()[1], + -np.log(expon.pdf(2, scale=2)), + delta=1e-01, + ) + self.assertAlmostEqual( + score_w1.all_values().flatten()[2], + -np.log(expon.pdf(0.1, scale=2)), + delta=1e-01, + ) + self.assertAlmostEqual( + score_w1.all_values().flatten()[3], + -np.log(expon.pdf(0.001, scale=2)), + delta=1e-01, + ) + + # check values for window=2 (must be equal to the mean of the past 2 values) + self.assertAlmostEqual( + score_w2.all_values().flatten()[0], + (-np.log(expon.pdf(1.5, scale=2)) - np.log(expon.pdf(0.1, scale=2))) / 2, + delta=1e-01, + ) + self.assertAlmostEqual( + score_w2.all_values().flatten()[1], + (-np.log(expon.pdf(2, scale=2)) - np.log(expon.pdf(0.001, scale=2))) / 2, + delta=1e-01, + ) + + self.assertTrue(scorer.is_probabilistic) + + def test_GammaNLLScorer(self): + + # window parameter + # window must be int + with self.assertRaises(ValueError): + GammaNLLScorer(window=True) + with self.assertRaises(ValueError): + GammaNLLScorer(window="string") + # window must be non negative + with self.assertRaises(ValueError): + GammaNLLScorer(window=-1) + # window must be different from 0 + with self.assertRaises(ValueError): + GammaNLLScorer(window=0) + + scorer = GammaNLLScorer(window=101) + # window must be smaller than the input of score_from_prediction() + with self.assertRaises(ValueError): + scorer.score_from_prediction( + actual_series=self.test, pred_series=self.probabilistic + ) # len(self.test)=100 + + np.random.seed(4) + scorer = GammaNLLScorer() + + # test 1 univariate (len=1 and window=1) + gamma_samples_1 = np.random.gamma(shape=2, scale=2, size=10000) + distribution_series = TimeSeries.from_values(gamma_samples_1.reshape(1, 1, -1)) + actual_series = TimeSeries.from_values(np.array([3])) + value_test1 = ( + scorer.score_from_prediction(actual_series, distribution_series) + .all_values() + .flatten()[0] + ) + + # check if value_test1 is the - log likelihood + self.assertAlmostEqual( + # This is approximate because our NLL scorer is fit from samples and also uses loc + value_test1, + -np.log(gamma.pdf(3, 2, scale=2)), + delta=1e-01, + ) + + # test 2 univariate (len=1 and window=1) + gamma_samples_2 = np.random.gamma(2, scale=2, size=10000) + distribution_series = TimeSeries.from_values(gamma_samples_2.reshape(1, 1, -1)) + actual_series = TimeSeries.from_values(np.array([10])) + value_test2 = ( + scorer.score_from_prediction(actual_series, distribution_series) + .all_values() + .flatten()[0] + ) + + # check if value_test2 is the - log likelihood + self.assertAlmostEqual( + # This is approximate because our NLL scorer is fit from samples and also uses loc + value_test2, + -np.log(gamma.pdf(10, 2, scale=2)), + delta=1e-01, + ) + + # test window univariate (len=2 and window=2) + distribution_series = TimeSeries.from_values( + np.array([gamma_samples_1.reshape(1, -1), gamma_samples_2.reshape(1, -1)]) + ) + actual_series = TimeSeries.from_values(np.array([3, 10])) + value_window = scorer.score_from_prediction(actual_series, distribution_series) + + # check length + self.assertEqual(len(value_window), 2) + # check width + self.assertEqual(value_window.width, 1) + + # check equal value_test1 and value_test2 + self.assertEqual(value_window.all_values().flatten()[0], value_test1) + self.assertEqual(value_window.all_values().flatten()[1], value_test2) + + scorer = GammaNLLScorer(window=2) + # check avg of two values + self.assertEqual( + scorer.score_from_prediction(actual_series, distribution_series) + .all_values() + .flatten()[0], + (value_test1 + value_test2) / 2, + ) + + # test window multivariate (n_samples=2, len=1, window=1) + scorer = GammaNLLScorer(window=1) + distribution_series = TimeSeries.from_values( + np.array([gamma_samples_1, gamma_samples_2]).reshape(1, 2, -1) + ) + actual_series = TimeSeries.from_values(np.array([3, 10]).reshape(1, -1)) + value_multivariate = scorer.score_from_prediction( + actual_series, distribution_series + ) + + # check length + self.assertEqual(len(value_multivariate), 1) + # check width + self.assertEqual(value_multivariate.width, 2) + + # check equal value_test1 and value_test2 + self.assertEqual(value_multivariate.all_values().flatten()[0], value_test1) + self.assertEqual(value_multivariate.all_values().flatten()[1], value_test2) + + # test window multivariate (n_samples=2, len=2, window=1 and 2) + scorer_w1 = GammaNLLScorer(window=1) + scorer_w2 = GammaNLLScorer(window=2) + + gamma_samples_3 = np.random.gamma(2, scale=2, size=10000) + gamma_samples_4 = np.random.gamma(2, scale=2, size=10000) + + distribution_series = TimeSeries.from_values( + np.array( + [gamma_samples_1, gamma_samples_2, gamma_samples_3, gamma_samples_4] + ).reshape(2, 2, -1) + ) + + actual_series = TimeSeries.from_values( + np.array([1.5, 2, 0.5, 0.9]).reshape(2, -1) + ) + + score_w1 = scorer_w1.score_from_prediction(actual_series, distribution_series) + score_w2 = scorer_w2.score_from_prediction(actual_series, distribution_series) + + # check length + self.assertEqual(len(score_w1), 2) + self.assertEqual(len(score_w2), 1) + # check width + self.assertEqual(score_w1.width, 2) + self.assertEqual(score_w2.width, 2) + + # check values for window=1 + self.assertAlmostEqual( + score_w1.all_values().flatten()[0], + -np.log(gamma.pdf(1.5, 2, scale=2)), + delta=1e-01, + ) + self.assertAlmostEqual( + score_w1.all_values().flatten()[1], + -np.log(gamma.pdf(2, 2, scale=2)), + delta=1e-01, + ) + self.assertAlmostEqual( + score_w1.all_values().flatten()[2], + -np.log(gamma.pdf(0.5, 2, scale=2)), + delta=1e-01, + ) + self.assertAlmostEqual( + score_w1.all_values().flatten()[3], + -np.log(gamma.pdf(0.9, 2, scale=2)), + delta=1e-01, + ) + + # check values for window=2 (must be equal to the mean of the past 2 values) + self.assertAlmostEqual( + score_w2.all_values().flatten()[0], + (-np.log(gamma.pdf(1.5, 2, scale=2)) - np.log(gamma.pdf(0.5, 2, scale=2))) + / 2, + delta=1e-01, + ) + self.assertAlmostEqual( + score_w2.all_values().flatten()[1], + (-np.log(gamma.pdf(2, 2, scale=2)) - np.log(gamma.pdf(0.9, 2, scale=2))) + / 2, + delta=1e-01, + ) + + self.assertTrue(scorer.is_probabilistic) + + def test_CauchyNLLScorer(self): + + # window parameter + # window must be int + with self.assertRaises(ValueError): + CauchyNLLScorer(window=True) + with self.assertRaises(ValueError): + CauchyNLLScorer(window="string") + # window must be non negative + with self.assertRaises(ValueError): + CauchyNLLScorer(window=-1) + # window must be different from 0 + with self.assertRaises(ValueError): + CauchyNLLScorer(window=0) + + scorer = CauchyNLLScorer(window=101) + # window must be smaller than the input of score_from_prediction() + with self.assertRaises(ValueError): + scorer.score_from_prediction( + actual_series=self.test, pred_series=self.probabilistic + ) # len(self.test)=100 + + np.random.seed(4) + scorer = CauchyNLLScorer() + + # test 1 univariate (len=1 and window=1) + cauchy_samples_1 = np.random.standard_cauchy(size=10000) + distribution_series = TimeSeries.from_values(cauchy_samples_1.reshape(1, 1, -1)) + actual_series = TimeSeries.from_values(np.array([3])) + value_test1 = ( + scorer.score_from_prediction(actual_series, distribution_series) + .all_values() + .flatten()[0] + ) + + # check if value_test1 is the - log likelihood + self.assertAlmostEqual(value_test1, -np.log(cauchy.pdf(3)), delta=1e-01) + + # test 2 univariate (len=1 and window=1) + cauchy_samples_2 = np.random.standard_cauchy(size=10000) + distribution_series = TimeSeries.from_values(cauchy_samples_2.reshape(1, 1, -1)) + actual_series = TimeSeries.from_values(np.array([-2])) + value_test2 = ( + scorer.score_from_prediction(actual_series, distribution_series) + .all_values() + .flatten()[0] + ) + + # check if value_test2 is the - log likelihood + self.assertAlmostEqual(value_test2, -np.log(cauchy.pdf(-2)), delta=1e-01) + + # test window univariate (len=2 and window=2) + distribution_series = TimeSeries.from_values( + np.array([cauchy_samples_1.reshape(1, -1), cauchy_samples_2.reshape(1, -1)]) + ) + actual_series = TimeSeries.from_values(np.array([3, -2])) + value_window = scorer.score_from_prediction(actual_series, distribution_series) + + # check length + self.assertEqual(len(value_window), 2) + # check width + self.assertEqual(value_window.width, 1) + + # check equal value_test1 and value_test2 + self.assertEqual(value_window.all_values().flatten()[0], value_test1) + self.assertEqual(value_window.all_values().flatten()[1], value_test2) + + scorer = CauchyNLLScorer(window=2) + # check avg of two values + self.assertEqual( + scorer.score_from_prediction(actual_series, distribution_series) + .all_values() + .flatten()[0], + (value_test1 + value_test2) / 2, + ) + + # test window multivariate (n_samples=2, len=1, window=1) + scorer = CauchyNLLScorer(window=1) + distribution_series = TimeSeries.from_values( + np.array([cauchy_samples_1, cauchy_samples_2]).reshape(1, 2, -1) + ) + actual_series = TimeSeries.from_values(np.array([3, -2]).reshape(1, -1)) + value_multivariate = scorer.score_from_prediction( + actual_series, distribution_series + ) + + # check length + self.assertEqual(len(value_multivariate), 1) + # check width + self.assertEqual(value_multivariate.width, 2) + + # check equal value_test1 and value_test2 + self.assertEqual(value_multivariate.all_values().flatten()[0], value_test1) + self.assertEqual(value_multivariate.all_values().flatten()[1], value_test2) + + # test window multivariate (n_samples=2, len=2, window=1 and 2) + scorer_w1 = CauchyNLLScorer(window=1) + scorer_w2 = CauchyNLLScorer(window=2) + + cauchy_samples_3 = np.random.standard_cauchy(size=10000) + cauchy_samples_4 = np.random.standard_cauchy(size=10000) + + distribution_series = TimeSeries.from_values( + np.array( + [cauchy_samples_1, cauchy_samples_2, cauchy_samples_3, cauchy_samples_4] + ).reshape(2, 2, -1) + ) + + actual_series = TimeSeries.from_values( + np.array([1.5, 2, 0.5, 0.9]).reshape(2, -1) + ) + + score_w1 = scorer_w1.score_from_prediction(actual_series, distribution_series) + score_w2 = scorer_w2.score_from_prediction(actual_series, distribution_series) + + # check length + self.assertEqual(len(score_w1), 2) + self.assertEqual(len(score_w2), 1) + # check width + self.assertEqual(score_w1.width, 2) + self.assertEqual(score_w2.width, 2) + + # check values for window=1 + self.assertAlmostEqual( + score_w1.all_values().flatten()[0], -np.log(cauchy.pdf(1.5)), delta=1e-01 + ) + self.assertAlmostEqual( + score_w1.all_values().flatten()[1], -np.log(cauchy.pdf(2)), delta=1e-01 + ) + self.assertAlmostEqual( + score_w1.all_values().flatten()[2], -np.log(cauchy.pdf(0.5)), delta=1e-01 + ) + self.assertAlmostEqual( + score_w1.all_values().flatten()[3], -np.log(cauchy.pdf(0.9)), delta=1e-01 + ) + + # check values for window=2 (must be equal to the mean of the past 2 values) + self.assertAlmostEqual( + score_w2.all_values().flatten()[0], + (-np.log(cauchy.pdf(1.5)) - np.log(cauchy.pdf(0.5))) / 2, + delta=1e-01, + ) + self.assertAlmostEqual( + score_w2.all_values().flatten()[1], + (-np.log(cauchy.pdf(2)) - np.log(cauchy.pdf(0.9))) / 2, + delta=1e-01, + ) + + self.assertTrue(scorer.is_probabilistic) + + def test_PoissonNLLScorer(self): + + # window parameter + # window must be int + with self.assertRaises(ValueError): + PoissonNLLScorer(window=True) + with self.assertRaises(ValueError): + PoissonNLLScorer(window="string") + # window must be non negative + with self.assertRaises(ValueError): + PoissonNLLScorer(window=-1) + # window must be different from 0 + with self.assertRaises(ValueError): + PoissonNLLScorer(window=0) + + scorer = PoissonNLLScorer(window=101) + # window must be smaller than the input of score_from_prediction() + with self.assertRaises(ValueError): + scorer.score_from_prediction( + actual_series=self.test, pred_series=self.probabilistic + ) # len(self.test)=100 + + np.random.seed(4) + scorer = PoissonNLLScorer() + + # test 1 univariate (len=1 and window=1) + poisson_samples_1 = np.random.poisson(size=10000, lam=1) + distribution_series = TimeSeries.from_values( + poisson_samples_1.reshape(1, 1, -1) + ) + actual_series = TimeSeries.from_values(np.array([3])) + value_test1 = ( + scorer.score_from_prediction(actual_series, distribution_series) + .all_values() + .flatten()[0] + ) + + # check if value_test1 is the - log likelihood + self.assertAlmostEqual(value_test1, -np.log(poisson.pmf(3, mu=1)), delta=1e-02) + + # test 2 univariate (len=1 and window=1) + poisson_samples_2 = np.random.poisson(size=10000, lam=1) + distribution_series = TimeSeries.from_values( + poisson_samples_2.reshape(1, 1, -1) + ) + actual_series = TimeSeries.from_values(np.array([10])) + value_test2 = ( + scorer.score_from_prediction(actual_series, distribution_series) + .all_values() + .flatten()[0] + ) + + # check if value_test2 is the - log likelihood + self.assertAlmostEqual(value_test2, -np.log(poisson.pmf(10, mu=1)), delta=1e-01) + + # test window univariate (len=2 and window=2) + distribution_series = TimeSeries.from_values( + np.array( + [poisson_samples_1.reshape(1, -1), poisson_samples_2.reshape(1, -1)] + ) + ) + actual_series = TimeSeries.from_values(np.array([3, 10])) + value_window = scorer.score_from_prediction(actual_series, distribution_series) + + # check length + self.assertEqual(len(value_window), 2) + # check width + self.assertEqual(value_window.width, 1) + + # check equal value_test1 and value_test2 + self.assertEqual(value_window.all_values().flatten()[0], value_test1) + self.assertEqual(value_window.all_values().flatten()[1], value_test2) + + scorer = PoissonNLLScorer(window=2) + # check avg of two values + self.assertEqual( + scorer.score_from_prediction(actual_series, distribution_series) + .all_values() + .flatten()[0], + (value_test1 + value_test2) / 2, + ) + + # test window multivariate (n_samples=2, len=1, window=1) + scorer = PoissonNLLScorer(window=1) + distribution_series = TimeSeries.from_values( + np.array([poisson_samples_1, poisson_samples_2]).reshape(1, 2, -1) + ) + actual_series = TimeSeries.from_values(np.array([3, 10]).reshape(1, -1)) + value_multivariate = scorer.score_from_prediction( + actual_series, distribution_series + ) + + # check length + self.assertEqual(len(value_multivariate), 1) + # check width + self.assertEqual(value_multivariate.width, 2) + + # check equal value_test1 and value_test2 + self.assertEqual(value_multivariate.all_values().flatten()[0], value_test1) + self.assertEqual(value_multivariate.all_values().flatten()[1], value_test2) + + # test window multivariate (n_samples=2, len=2, window=1 and 2) + scorer_w1 = PoissonNLLScorer(window=1) + scorer_w2 = PoissonNLLScorer(window=2) + + poisson_samples_3 = np.random.poisson(size=10000, lam=1) + poisson_samples_4 = np.random.poisson(size=10000, lam=1) + + distribution_series = TimeSeries.from_values( + np.array( + [ + poisson_samples_1, + poisson_samples_2, + poisson_samples_3, + poisson_samples_4, + ] + ).reshape(2, 2, -1) + ) + + actual_series = TimeSeries.from_values(np.array([1, 2, 3, 4]).reshape(2, -1)) + + score_w1 = scorer_w1.score_from_prediction(actual_series, distribution_series) + score_w2 = scorer_w2.score_from_prediction(actual_series, distribution_series) + + # check length + self.assertEqual(len(score_w1), 2) + self.assertEqual(len(score_w2), 1) + # check width + self.assertEqual(score_w1.width, 2) + self.assertEqual(score_w2.width, 2) + + # check values for window=1 + self.assertAlmostEqual( + score_w1.all_values().flatten()[0], + -np.log(poisson.pmf(1, mu=1)), + delta=1e-01, + ) + self.assertAlmostEqual( + score_w1.all_values().flatten()[1], + -np.log(poisson.pmf(2, mu=1)), + delta=1e-01, + ) + self.assertAlmostEqual( + score_w1.all_values().flatten()[2], + -np.log(poisson.pmf(3, mu=1)), + delta=1e-01, + ) + self.assertAlmostEqual( + score_w1.all_values().flatten()[3], + -np.log(poisson.pmf(4, mu=1)), + delta=1e-01, + ) + + # check values for window=2 (must be equal to the mean of the past 2 values) + self.assertAlmostEqual( + score_w2.all_values().flatten()[0], + (-np.log(poisson.pmf(1, mu=1)) - np.log(poisson.pmf(3, mu=1))) / 2, + delta=1e-01, + ) + self.assertAlmostEqual( + score_w2.all_values().flatten()[1], + (-np.log(poisson.pmf(2, mu=1)) - np.log(poisson.pmf(4, mu=1))) / 2, + delta=1e-01, + ) + + self.assertTrue(scorer.is_probabilistic) diff --git a/docs/source/conf.py b/docs/source/conf.py index 4870d8a2f0..473d2f8f30 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -55,7 +55,7 @@ + "min_train_series_length," + "untrained_model,first_prediction_index,future_covariate_series,past_covariate_series," + "initialize_encoders,register_datapipe_as_function,register_function,functions," - + "SplitTimeSeriesSequence,randint", + + "SplitTimeSeriesSequence,randint,AnomalyModel", } # In order to also have the docstrings of __init__() methods included diff --git a/requirements/core.txt b/requirements/core.txt index 28d72e67ed..ccc0ebe5f4 100644 --- a/requirements/core.txt +++ b/requirements/core.txt @@ -8,6 +8,7 @@ numpy>=1.19.0 pandas>=1.0.5 pmdarima>=1.8.0 prophet>=1.1.1 +pyod>=0.9.5 requests>=2.22.0 scikit-learn>=1.0.1 scipy>=1.3.2