Chapter 01
duration prediction
NotebookPython 3 (ipykernel)18 cells
In [1]python · cell 1
python
!python -VOutput
Python 3.9.7
In [2]python · cell 2
python
import pandas as pdIn [3]python · cell 3
python
import pickleIn [4]python · cell 4
python
import seaborn as sns
import matplotlib.pyplot as pltIn [5]python · cell 5
python
from sklearn.feature_extraction import DictVectorizer
from sklearn.linear_model import LinearRegression
from sklearn.linear_model import Lasso
from sklearn.linear_model import Ridge
from sklearn.metrics import root_mean_squared_errorIn [6]python · cell 6
python
df = pd.read_parquet('./data/green_tripdata_2021-01.parquet')
df['duration'] = df.lpep_dropoff_datetime - df.lpep_pickup_datetime
df.duration = df.duration.apply(lambda td: td.total_seconds() / 60)
df = df[(df.duration >= 1) & (df.duration <= 60)]
categorical = ['PULocationID', 'DOLocationID']
numerical = ['trip_distance']
df[categorical] = df[categorical].astype(str)In [7]python · cell 7
python
train_dicts = df[categorical + numerical].to_dict(orient='records')
dv = DictVectorizer()
X_train = dv.fit_transform(train_dicts)
target = 'duration'
y_train = df[target].values
lr = LinearRegression()
lr.fit(X_train, y_train)
y_pred = lr.predict(X_train)
root_mean_squared_error(y_train, y_pred)Output
9.775464208836793
In [8]python · cell 8
python
sns.distplot(y_pred, label='prediction')
sns.distplot(y_train, label='actual')
plt.legend()Output
/home/ubuntu/anaconda3/lib/python3.9/site-packages/seaborn/distributions.py:2619: FutureWarning: `distplot` is a deprecated function and will be removed in a future version. Please adapt your code to use either `displot` (a figure-level function with similar flexibility) or `histplot` (an axes-level function for histograms). warnings.warn(msg, FutureWarning) /home/ubuntu/anaconda3/lib/python3.9/site-packages/seaborn/distributions.py:2619: FutureWarning: `distplot` is a deprecated function and will be removed in a future version. Please adapt your code to use either `displot` (a figure-level function with similar flexibility) or `histplot` (an axes-level function for histograms). warnings.warn(msg, FutureWarning)
<matplotlib.legend.Legend at 0x7fb6397fad60>
<Figure size 432x288 with 1 Axes>
In [9]python · cell 9
python
def read_dataframe(filename):
if filename.endswith('.csv'):
df = pd.read_csv(filename)
df.lpep_dropoff_datetime = pd.to_datetime(df.lpep_dropoff_datetime)
df.lpep_pickup_datetime = pd.to_datetime(df.lpep_pickup_datetime)
elif filename.endswith('.parquet'):
df = pd.read_parquet(filename)
df['duration'] = df.lpep_dropoff_datetime - df.lpep_pickup_datetime
df.duration = df.duration.apply(lambda td: td.total_seconds() / 60)
df = df[(df.duration >= 1) & (df.duration <= 60)]
categorical = ['PULocationID', 'DOLocationID']
df[categorical] = df[categorical].astype(str)
return dfIn [10]python · cell 10
python
df_train = read_dataframe('./data/green_tripdata_2021-01.parquet')
df_val = read_dataframe('./data/green_tripdata_2021-02.parquet')In [11]python · cell 11
python
len(df_train), len(df_val)Output
(73908, 61921)
In [12]python · cell 12
python
df_train['PU_DO'] = df_train['PULocationID'] + '_' + df_train['DOLocationID']
df_val['PU_DO'] = df_val['PULocationID'] + '_' + df_val['DOLocationID']In [13]python · cell 13
python
categorical = ['PU_DO'] #'PULocationID', 'DOLocationID']
numerical = ['trip_distance']
dv = DictVectorizer()
train_dicts = df_train[categorical + numerical].to_dict(orient='records')
X_train = dv.fit_transform(train_dicts)
val_dicts = df_val[categorical + numerical].to_dict(orient='records')
X_val = dv.transform(val_dicts)In [14]python · cell 14
python
target = 'duration'
y_train = df_train[target].values
y_val = df_val[target].valuesIn [15]python · cell 15
python
lr = LinearRegression()
lr.fit(X_train, y_train)
y_pred = lr.predict(X_val)
root_mean_squared_error(y_val, y_pred)Output
7.479513631630414
In [16]python · cell 16
python
with open('models/lin_reg.bin', 'wb') as f_out:
pickle.dump((dv, lr), f_out)In [17]python · cell 17
python
lr = Lasso(0.01)
lr.fit(X_train, y_train)
y_pred = lr.predict(X_val)
root_mean_squared_error(y_val, y_pred)Output
11.167275941179728
In [ ]python · cell 18
python