Chapter 13
Baseline model for batch monitoring example
NotebookPython 3 (ipykernel)44 cells
Baseline model for batch monitoring example
In [ ]python · cell 2
python
import requests
import datetime
import pandas as pd
from evidently import DataDefinition
from evidently import Dataset
from evidently import Report
from evidently.metrics import ValueDrift, DriftedColumnsCount, MissingValueCount
from joblib import load, dump
from tqdm import tqdm
from sklearn.linear_model import LinearRegression
from sklearn.metrics import mean_absolute_error, mean_absolute_percentage_errorIn [ ]python · cell 3
python
! mkdir dataIn [ ]python · cell 4
python
files = [('green_tripdata_2022-02.parquet', './data'), ('green_tripdata_2022-01.parquet', './data')]
print("Download files:")
for file, path in files:
url=f"https://d37ci6vzurychx.cloudfront.net/trip-data/{file}"
resp=requests.get(url, stream=True)
save_path=f"{path}/{file}"
with open(save_path, "wb") as handle:
for data in tqdm(resp.iter_content(),
desc=f"{file}",
postfix=f"save to {save_path}",
total=int(resp.headers["Content-Length"])):
handle.write(data)In [ ]python · cell 5
python
jan_data = pd.read_parquet('data/green_tripdata_2022-01.parquet')In [ ]python · cell 6
python
jan_data.describe()In [ ]python · cell 7
python
jan_data.shapeIn [ ]python · cell 8
python
# create target
jan_data["duration_min"] = jan_data.lpep_dropoff_datetime - jan_data.lpep_pickup_datetime
jan_data.duration_min = jan_data.duration_min.apply(lambda td : float(td.total_seconds())/60)In [ ]python · cell 9
python
# filter out outliers
jan_data = jan_data[(jan_data.duration_min >= 0) & (jan_data.duration_min <= 60)]
jan_data = jan_data[(jan_data.passenger_count > 0) & (jan_data.passenger_count <= 8)]In [ ]python · cell 10
python
jan_data.duration_min.hist()In [ ]python · cell 11
python
# data labeling
target = "duration_min"
num_features = ["passenger_count", "trip_distance", "fare_amount", "total_amount"]
cat_features = ["PULocationID", "DOLocationID"]In [ ]python · cell 12
python
jan_data.shapeIn [ ]python · cell 13
python
train_data = jan_data[:30000]
val_data = jan_data[30000:]In [ ]python · cell 14
python
model = LinearRegression()In [ ]python · cell 15
python
model.fit(train_data[num_features + cat_features], train_data[target])In [ ]python · cell 16
python
train_preds = model.predict(train_data[num_features + cat_features])
train_data['prediction'] = train_predsIn [ ]python · cell 17
python
val_preds = model.predict(val_data[num_features + cat_features])
val_data['prediction'] = val_predsIn [ ]python · cell 18
python
print(mean_absolute_error(train_data.duration_min, train_data.prediction))
print(mean_absolute_error(val_data.duration_min, val_data.prediction))Dump model and reference data
In [ ]python · cell 20
python
! mkdir modelsIn [ ]python · cell 21
python
with open('models/lin_reg.bin', 'wb') as f_out:
dump(model, f_out)In [ ]python · cell 22
python
val_data.to_parquet('data/reference.parquet')Evidently Report
In [ ]python · cell 24
python
data_definition = DataDefinition(numerical_columns=num_features + ['prediction'], categorical_columns=cat_features)
train_dataset = Dataset.from_pandas(
train_data,
data_definition
)
val_dataset = Dataset.from_pandas(
val_data,
data_definition
)In [ ]python · cell 25
python
report = Report(metrics=[
ValueDrift(column='prediction'),
DriftedColumnsCount(),
MissingValueCount(column='prediction'),
]
)In [ ]python · cell 26
python
snapshot = report.run(reference_data=train_dataset, current_data=val_dataset)In [ ]python · cell 27
python
snapshotIn [ ]python · cell 28
python
result = snapshot.dict()In [ ]python · cell 29
python
resultIn [ ]python · cell 30
python
#prediction drift
result['metrics'][0]['value']In [ ]python · cell 31
python
#number of drifted columns
result['metrics'][1]['value']['count']In [ ]python · cell 32
python
#share of missing values
result['metrics'][2]['value']['count']Evidently Dashboard
In [ ]python · cell 34
python
from evidently.presets import DataDriftPreset, DataSummaryPreset
from evidently.ui.workspace import Workspace
from evidently.sdk.panels import *
from evidently.legacy.renderers.html_widgets import WidgetSizeIn [ ]python · cell 35
python
ws = Workspace("workspace")In [ ]python · cell 36
python
project = ws.create_project("NYC Taxi Data Quality Project")
project.description = "My project description"
project.save()In [ ]python · cell 37
python
regular_report = Report(
metrics=[
DataSummaryPreset()
],
)
data = Dataset.from_pandas(
val_data.loc[val_data.lpep_pickup_datetime.between('2022-01-28', '2022-01-29', inclusive="left")],
data_definition=data_definition,
)
regular_snapshot = regular_report.run(current_data=data, timestamp=datetime.datetime(2022,1,28))
regular_snapshotIn [ ]python · cell 38
python
ws.add_run(project.id, regular_snapshot)note: To view a report please run "evidently ui" command in a separate tab in your terminal.
In [ ]python · cell 40
python
#configure the dashboard
project.dashboard.add_panel(
text_panel(title="NYC taxi data dashboard")
)
project.dashboard.add_panel(
bar_plot_panel(
title="Inference Count",
values=[
PanelMetric(
metric="RowCount",
legend="count",
),
],
size="half",
),
)
project.dashboard.add_panel(
line_plot_panel(
title="Number of Missing Values",
values=[
PanelMetric(
metric="DatasetMissingValueCount",
legend="count"
),
],
size="half",
),
)
project.save()To view a dashboard please run "evidently ui" command in a separate tab in your terminal.
In [ ]python · cell 42
python
regular_report = Report(
metrics=[
DataSummaryPreset()
],
)
data = Dataset.from_pandas(
val_data.loc[val_data.lpep_pickup_datetime.between('2022-01-29', '2022-01-30', inclusive="left")],
data_definition=data_definition,
)
regular_run = regular_report.run(current_data=data, timestamp=datetime.datetime(2022,1,29))
regular_runIn [ ]python · cell 43
python
ws.add_run(project.id, regular_run)In [ ]python · cell 44
python