Chapter 15
homework
NotebookPython 3 (ipykernel)24 cells
In [42]python · cell 1
python
import pandas as pd
import seaborn as sns
from sklearn.feature_extraction import DictVectorizer
from sklearn.linear_model import LinearRegression
from sklearn.metrics import mean_squared_errorIn [35]python · cell 2
python
df = pd.read_parquet('./data/fhv_tripdata_2021-01.parquet')In [20]python · cell 3
python
old_len = len(df)In [36]python · cell 4
python
df['duration'] = df.dropOff_datetime - df.pickup_datetime
df['duration'] = df.duration.dt.total_seconds() / 60In [16]python · cell 5
python
df.duration.mean()Output
19.1672240937939
In [37]python · cell 6
python
df = df[(df.duration >= 1) & (df.duration <= 60)].copy()In [38]python · cell 7
python
categorical = ['PUlocationID', 'DOlocationID']
df[categorical] = df[categorical].fillna(-1).astype('int')In [52]python · cell 8
python
df[categorical] = df[categorical].astype('str')In [55]python · cell 9
python
train_dicts = df[categorical].to_dict(orient='records')In [56]python · cell 10
python
dv = DictVectorizer()
X_train = dv.fit_transform(train_dicts) In [57]python · cell 11
python
X_train.shapeOutput
(1109826, 525)
In [60]python · cell 12
python
y_train = df.duration.valuesIn [59]python · cell 13
python
len(dv.feature_names_)Output
525
In [61]python · cell 14
python
lr = LinearRegression()
lr.fit(X_train, y_train)Output
LinearRegression()
In [62]python · cell 15
python
y_pred = lr.predict(X_train)In [63]python · cell 16
python
mean_squared_error(y_train, y_pred, squared=False)Output
10.528519107212292
In [64]python · cell 17
python
categorical = ['PUlocationID', 'DOlocationID']
def read_data(filename):
df = pd.read_parquet(filename)
df['duration'] = df.dropOff_datetime - df.pickup_datetime
df['duration'] = df.duration.dt.total_seconds() / 60
df = df[(df.duration >= 1) & (df.duration <= 60)].copy()
df[categorical] = df[categorical].fillna(-1).astype('int').astype('str')
return dfIn [65]python · cell 18
python
df_val = read_data('./data/fhv_tripdata_2021-02.parquet')In [68]python · cell 19
python
val_dicts = df_val[categorical].to_dict(orient='records')In [69]python · cell 20
python
X_val = dv.transform(val_dicts) In [70]python · cell 21
python
y_pred = lr.predict(X_val)In [71]python · cell 22
python
y_val = df_val.duration.valuesIn [72]python · cell 23
python
mean_squared_error(y_val, y_pred, squared=False)Output
11.014283211122269
In [ ]python · cell 24
python