Example for anomaly detection with LSTM autoencoder architectures¶

There is a multitude of successful architecture. In the following we demonstrate the implementation of 3 possible architecture types.

Models¶

In [1]:

from river import compose, preprocessing, metrics, datasets

from deep_river.anomaly import RollingAutoencoder
from torch import nn, manual_seed
import torch
from tqdm import tqdm

from river import compose, preprocessing, metrics, datasets from deep_river.anomaly import RollingAutoencoder from torch import nn, manual_seed import torch from tqdm import tqdm

LSTM Encoder-Decoder architecture by Sutskever et al. 2014 (https://arxiv.org/abs/1409.3215). The decoder only gets access to its own prediction of the previous timestep. Decoding also takes performed backwards.

In [4]:

class LSTMDecoder(nn.Module):
    def __init__(
        self,
        input_size,
        hidden_size,
        sequence_length=None,
        predict_backward=True,
        num_layers=1,
    ):
        super().__init__()

        self.cell = nn.LSTMCell(input_size, hidden_size)
        self.input_size = input_size
        self.hidden_size = hidden_size

        self.predict_backward = predict_backward
        self.sequence_length = sequence_length
        self.num_layers = num_layers
        self.lstm = (
            None
            if num_layers <= 1
            else nn.LSTM(
                input_size=hidden_size,
                hidden_size=hidden_size,
                num_layers=num_layers - 1,
            )
        )
        self.linear = (
            None
            if input_size == hidden_size
            else nn.Linear(hidden_size, input_size)
        )

    def forward(self, h, sequence_length=None):
        """Computes the forward pass.

        Parameters
        ----------
        x:
            Input of shape (batch_size, input_size)

        Returns
        -------
        Tuple[torch.Tensor, Tuple[torch.Tensor, torch.Tensor]]
            Decoder outputs (output, (h, c)) where output has the shape (sequence_length, batch_size, input_size).
        """

        if sequence_length is None:
            sequence_length = self.sequence_length
        x_hat = torch.empty(sequence_length, h.shape[0], self.hidden_size)
        for t in range(sequence_length):
            if t == 0:
                h, c = self.cell(h)
            else:
                input = h if self.linear is None else self.linear(h)
                h, c = self.cell(input, (h, c))
            t_predicted = -t if self.predict_backward else t
            x_hat[t_predicted] = h

        if self.lstm is not None:
            x_hat = self.lstm(x_hat)

        return x_hat, (h, c)


class LSTMAutoencoderSutskever(nn.Module):
    def __init__(self, n_features, hidden_size=30, n_layers=1):
        super().__init__()
        self.n_features = n_features
        self.hidden_size = hidden_size
        self.n_layers = n_layers
        self.encoder = nn.LSTM(
            input_size=n_features, hidden_size=hidden_size, num_layers=n_layers
        )
        self.decoder = LSTMDecoder(
            input_size=hidden_size,
            hidden_size=n_features,
            predict_backward=True,
        )

    def forward(self, x):
        _, (h, _) = self.encoder(x)
        x_hat, _ = self.decoder(h[-1], x.shape[0])
        return x_hat

class LSTMDecoder(nn.Module): def __init__( self, input_size, hidden_size, sequence_length=None, predict_backward=True, num_layers=1, ): super().__init__() self.cell = nn.LSTMCell(input_size, hidden_size) self.input_size = input_size self.hidden_size = hidden_size self.predict_backward = predict_backward self.sequence_length = sequence_length self.num_layers = num_layers self.lstm = ( None if num_layers <= 1 else nn.LSTM( input_size=hidden_size, hidden_size=hidden_size, num_layers=num_layers - 1, ) ) self.linear = ( None if input_size == hidden_size else nn.Linear(hidden_size, input_size) ) def forward(self, h, sequence_length=None): """Computes the forward pass. Parameters ---------- x: Input of shape (batch_size, input_size) Returns ------- Tuple[torch.Tensor, Tuple[torch.Tensor, torch.Tensor]] Decoder outputs (output, (h, c)) where output has the shape (sequence_length, batch_size, input_size). """ if sequence_length is None: sequence_length = self.sequence_length x_hat = torch.empty(sequence_length, h.shape[0], self.hidden_size) for t in range(sequence_length): if t == 0: h, c = self.cell(h) else: input = h if self.linear is None else self.linear(h) h, c = self.cell(input, (h, c)) t_predicted = -t if self.predict_backward else t x_hat[t_predicted] = h if self.lstm is not None: x_hat = self.lstm(x_hat) return x_hat, (h, c) class LSTMAutoencoderSutskever(nn.Module): def __init__(self, n_features, hidden_size=30, n_layers=1): super().__init__() self.n_features = n_features self.hidden_size = hidden_size self.n_layers = n_layers self.encoder = nn.LSTM( input_size=n_features, hidden_size=hidden_size, num_layers=n_layers ) self.decoder = LSTMDecoder( input_size=hidden_size, hidden_size=n_features, predict_backward=True, ) def forward(self, x): _, (h, _) = self.encoder(x) x_hat, _ = self.decoder(h[-1], x.shape[0]) return x_hat

Testing¶

The models can be tested with the code in the following cells. Since River currently does not feature any anomaly detection datasets with temporal dependencies, the results should be expected to be somewhat inaccurate.

In [9]:

_ = manual_seed(42)
dataset = datasets.CreditCard().take(5000)
metric = metrics.ROCAUC(n_thresholds=50)

module = LSTMAutoencoderSutskever  # Set this variable to your architecture of choice
ae = RollingAutoencoder(module=module, lr=0.005)
scaler = preprocessing.StandardScaler()

_ = manual_seed(42) dataset = datasets.CreditCard().take(5000) metric = metrics.ROCAUC(n_thresholds=50) module = LSTMAutoencoderSutskever # Set this variable to your architecture of choice ae = RollingAutoencoder(module=module, lr=0.005) scaler = preprocessing.StandardScaler()

In [10]:

for x, y in tqdm(list(dataset)):
    scaler.learn_one(x)
    x = scaler.transform_one(x)
    score = ae.score_one(x)
    metric.update(y_true=y, y_pred=score)
    ae.learn_one(x=x, y=None)
print(f"ROCAUC: {metric.get():.4f}")

for x, y in tqdm(list(dataset)): scaler.learn_one(x) x = scaler.transform_one(x) score = ae.score_one(x) metric.update(y_true=y, y_pred=score) ae.learn_one(x=x, y=None) print(f"ROCAUC: {metric.get():.4f}")

  0%|          | 0/5000 [00:00<?, ?it/s]

---------------------------------------------------------------------------
RuntimeError                              Traceback (most recent call last)
Input In [10], in <cell line: 1>()
      2 scaler.learn_one(x)
      3 x = scaler.transform_one(x)
----> 4 score = ae.score_one(x)
      5 metric.update(y_true=y, y_pred=score)
      6 ae.learn_one(x=x, y=None)

File ~/Documents/Research/deep-river/deep_river/anomaly/rolling_ae.py:203, in RollingAutoencoder.score_one(self, x)
    201 if not self.module_initialized:
    202     self.kwargs["n_features"] = len(x)
--> 203     self.initialize_module(**self.kwargs)
    205 if len(self._x_window) == self.window_size:
    206     x_win = self._x_window.copy()

File ~/Documents/Research/deep-river/deep_river/base.py:134, in DeepEstimator.initialize_module(self, **kwargs)
    132 self.optimizer = self.optimizer_fn(self.module.parameters(), lr=self.lr)
    133 self.module_initialized = True
--> 134 self._get_input_output_layers(n_features=kwargs["n_features"])

File ~/Documents/Research/deep-river/deep_river/base.py:179, in DeepEstimator._get_input_output_layers(self, n_features)
    176 apply_hooks(module=self.module, hook=tracker, handles=handles)
    178 x_dummy = torch.empty((1, n_features), device=self.device)
--> 179 self.module(x_dummy)
    181 for h in handles:
    182     h.remove()

File ~/miniconda3/envs/deepriver/lib/python3.10/site-packages/torch/nn/modules/module.py:1511, in Module._wrapped_call_impl(self, *args, **kwargs)
   1509     return self._compiled_call_impl(*args, **kwargs)  # type: ignore[misc]
   1510 else:
-> 1511     return self._call_impl(*args, **kwargs)

File ~/miniconda3/envs/deepriver/lib/python3.10/site-packages/torch/nn/modules/module.py:1561, in Module._call_impl(self, *args, **kwargs)
   1558     bw_hook = hooks.BackwardHook(self, full_backward_hooks, backward_pre_hooks)
   1559     args = bw_hook.setup_input_hook(args)
-> 1561 result = forward_call(*args, **kwargs)
   1562 if _global_forward_hooks or self._forward_hooks:
   1563     for hook_id, hook in (
   1564         *_global_forward_hooks.items(),
   1565         *self._forward_hooks.items(),
   1566     ):
   1567         # mark that always called hook is run

Input In [3], in LSTMAutoencoderCho.forward(self, x)
     22 _, (h, _) = self.encoder(x)
     23 target_shape = (
     24     (-1, x.shape[0], -1) if self.batch_first else (x.shape[0], -1, -1)
     25 )
---> 26 h = h[-1].expand(target_shape)
     27 x_hat, _ = self.decoder(h)
     28 return x_hat

RuntimeError: The expanded size of the tensor (-1) isn't allowed in a leading, non-existing dimension 1