Wind Turbine AD problem with TIM Engine API Client for Python

1. Set up Python Libraries

import logging
import pandas as pd
import plotly as plt
import plotly.express as px
import plotly.graph_objects as go
import numpy as np
import json
import copy

import tim_client

2. Credentials and logging

(Do not forget to fill in your credentials in the credentials.json file)

with open('credentials.json') as f:
    credentials_json = json.load(f)

TIM_URL = 'https://timws.tangent.works/v4/api'

SAVE_JSON = False
JSON_SAVING_FOLDER = 'logs/'

LOGGING_LEVEL = 'INFO'

level = logging.getLevelName(LOGGING_LEVEL)
logging.basicConfig(level=level, format='[%(levelname)s] %(asctime)s - %(name)s:%(funcName)s:%(lineno)s - %(message)s')
logger = logging.getLogger(__name__)

credentials = tim_client.Credentials(credentials_json['license_key'], credentials_json['email'], credentials_json['password'], tim_url=TIM_URL)
api_client = tim_client.ApiClient(credentials)

api_client.save_json = SAVE_JSON
api_client.json_saving_folder_path = JSON_SAVING_FOLDER

[INFO] 2020-11-03 15:42:44,972 - tim_client.api_client:save_json:66 - Saving JSONs functionality has been disabled
[INFO] 2020-11-03 15:42:44,975 - tim_client.api_client:json_saving_folder_path:75 - JSON destination folder changed to logs

3. Data preparation

3.1 Load Data

data = tim_client.load_dataset_from_csv_file('WindTurbine.csv', sep=';')
nrow, ncol = data.shape

print("Number of rows: ", nrow)
print("Number of columns: ", ncol)

Number of rows:  52547
Number of columns:  3

3.2 Change row indices to datetimes

(only for visualization purposes, not required for calling the tim_client methods)

data.index = pd.to_datetime(data['Date'])

3.3 Visualize Data

cols = data.columns

fig = plt.subplots.make_subplots(rows=2, cols=1, shared_xaxes=True, vertical_spacing=0.02)
fig.add_trace(go.Scatter(x=data.loc[:, "Date"], y=data.loc[:, "Wind Power"], name="Wind Power"), row=1, col=1)
fig.add_trace(go.Scatter(x=data.loc[:, "Date"], y=data.loc[:, "Wind Speed"], name="Wind Speed"), row=2, col=1)

fig.update_layout(height=700, width=1000, title_text="Data visualization")

fig.show()

4. Model Building

4.1 Configuration/TIM Setup

configuration = {
    "sensitivity": 0,
    "abnormalBehaviorModellingConfiguration": {
      "features": [
        "Residual",
      ]
    },
    "extendedOutputConfiguration": {
      "returnAnomalyIndicator": True,
      "returnSimpleImportances": True,
      "returnExtendedImportances": True
    }
}

4.2 Select data

train_until = "2015-10-02 23:50:00"
data_train = data[:train_until]

4.3 Build a model

model = api_client.detection_build_model(data_train, configuration)
model.status

'FinishedWithWarning'

4. Visualization of Results

4.1 In-sample anomaly detection

normal_behavior = model.normal_behavior
anomaly_indicator = model.anomaly_indicator
anomaly_indices = [i for i, ai in enumerate(anomaly_indicator.loc[:, "Anomaly Indicator"]) if ai > 1]

fig1 = plt.subplots.make_subplots(rows=2, cols=1, shared_xaxes=True, vertical_spacing=0.02)
fig1.add_trace(go.Scatter(x=data.loc[:, "Date"], y=data.loc[:, "Wind Power"], name="Wind Power", 
                          line=dict(color="black")), row=1, col=1)
    
fig1.add_trace(go.Scatter(x=[data.iloc[0, 0], data.iloc[len(data) - 1, 0]], y=[1,1], name="threshold", 
                          line=dict(color="red"), mode="lines"), row=2, col=1)
    
# in-sample anomalies, anomaly indicators and normal behavior

fig1.add_trace(go.Scatter(x=normal_behavior.index, y=normal_behavior["Normal Behavior"], name="Normal Behavior", 
                        line=dict(color="darkgoldenrod")), row=1, col=1)

fig1.add_trace(go.Scatter(x=anomaly_indicator.index, y=anomaly_indicator["Anomaly Indicator"], name="Anomaly Indicator", 
                        line=dict(color="darkgrey")), row=2, col=1)

anomalies = anomaly_indicator[anomaly_indicator["Anomaly Indicator"] > 1]

fig1.add_trace(go.Scatter(x=data.loc[anomalies.index.values, "Date"], y=data.loc[anomalies.index.values, "Wind Power"], mode='markers', 
                        marker=dict(color="red", size=6), name="Anomalies"), row=1, col=1)
    
fig1.update_layout(height=700, width=1000, title_text="In-sample anomaly detection")

fig1.show()

4.2 Retrieve influencers and term importances

si_df = pd.DataFrame(model.predictors_importances['simpleImportances'])
si_df.sort_values(by=['importance'], ascending=False)
ei_df = pd.DataFrame(model.predictors_importances['extendedImportances'])
ei_df = ei_df.sort_values(by=['time', 'importance'], ascending=False)

4.3 Visualize influencers(predictors) importances

fig2 = go.Figure(go.Bar(x=si_df['predictorName'], y=si_df['importance']))
fig2.update_layout(height=400,
                  width=600, 
                  title_text="Importances of predictors",
                  xaxis_title="Predictor name",
                  yaxis_title="Predictor importance (%)")
fig2.show()

4.4 Visualize term(features) importances

fig3 = go.Figure(go.Bar(x=ei_df['termName'], y=ei_df['importance']))
fig3.update_layout(height=700,
                  width=1000,
                  title_text="Importances of features",
                  xaxis_title="Feature name",
                  yaxis_title="Feature importance (%)")
fig3.show()

4.5 Out of sample anomaly detection

check the number of offsets that the model requires for detection

detection_start = "2015-10-05 00:00:00"
model.data_offsets

[{'from': {'offset': -1, 'baseUnit': 'Sample'}, 'uniqueName': 'Wind Speed'},
 {'from': {'offset': 0, 'baseUnit': 'Sample'}, 'uniqueName': 'Wind Power'}]

make sure that you provide enough data, the number of offsets required by the model (see cell output above) should be satisfied for each sample being detected

start = (pd.to_datetime(detection_start) - pd.Timedelta(minutes=(1)*10)).strftime("%Y-%m-%d %H:%M:%S") # get offset of 1 sample

print("start timestamp: {}".format(start))

start timestamp: 2015-10-04 23:50:00

4.5.1 Detection using a model

detection = api_client.detection_detect(data[start:], model)
detection.status

'Finished'

normal_behavior2 = detection.normal_behavior
anomaly_indicator2 = detection.anomaly_indicator
anomaly_indices2 = [i for i, ai in enumerate(anomaly_indicator2.loc[:, "Anomaly Indicator"]) if ai > 1]

4.5.2 Visualization

fig5 = copy.copy(fig1)
fig5.add_trace(go.Scatter(x=normal_behavior2.index, y=normal_behavior2["Normal Behavior"], name="Normal Behavior OutOfS", 
                        line=dict(color="goldenrod"), mode='lines'), row=1, col=1)

fig5.add_trace(go.Scatter(x=anomaly_indicator2.index, y=anomaly_indicator2["Anomaly Indicator"], name="Anomaly Indicator OutOfS", 
                        line=dict(color="grey"), mode='lines'), row=2, col=1)

anomalies2 = anomaly_indicator2[anomaly_indicator2["Anomaly Indicator"] > 1]

fig5.add_trace(go.Scatter(x=data.loc[anomalies2.index.values, "Date"], y=data.loc[anomalies2.index.values, "Wind Power"], mode='markers', 
                        marker=dict(color="red", size=6), name="Anomalies OutOfS"), row=1, col=1)
    
fig5.update_layout(height=700, width=1000, title_text="Anomaly detection")

fig5.show()