Chapter 16
starter
NotebookPython 3 (ipykernel)12 cells
In [1]python · cell 1
python
!pip freeze | grep scikit-learnOutput
scikit-learn==1.0.2 scikit-learn-intelex==2021.20210714.120553
In [1]python · cell 2
python
import pickle
import pandas as pdIn [15]python · cell 3
python
year = 2021
month = 2
input_file = f'https://nyc-tlc.s3.amazonaws.com/trip+data/fhv_tripdata_{year:04d}-{month:02d}.parquet'
output_file = f'output/fhv_tripdata_{year:04d}-{month:02d}.parquet'In [2]python · cell 4
python
with open('model.bin', 'rb') as f_in:
dv, lr = pickle.load(f_in)In [3]python · cell 5
python
categorical = ['PUlocationID', 'DOlocationID']
def read_data(filename):
df = pd.read_parquet(filename)
df['duration'] = df.dropOff_datetime - df.pickup_datetime
df['duration'] = df.duration.dt.total_seconds() / 60
df = df[(df.duration >= 1) & (df.duration <= 60)].copy()
df[categorical] = df[categorical].fillna(-1).astype('int').astype('str')
return dfIn [10]python · cell 6
python
df = read_data(input_file)
df['ride_id'] = f'{year:04d}/{month:02d}_' + df.index.astype('str')In [6]python · cell 7
python
dicts = df[categorical].to_dict(orient='records')
X_val = dv.transform(dicts)
y_pred = lr.predict(X_val)In [7]python · cell 8
python
y_pred.mean()Output
16.191691679979066
In [13]python · cell 9
python
df_result = pd.DataFrame()
df_result['ride_id'] = df['ride_id']
df_result['predicted_duration'] = y_predIn [17]python · cell 10
python
df_result.to_parquet(
output_file,
engine='pyarrow',
compression=None,
index=False
)In [18]python · cell 11
python
!ls -lh output/Output
total 19M -rw-rw-r-- 1 ubuntu ubuntu 19M Jun 30 08:43 fhv_tripdata_2021-02.parquet
In [ ]python · cell 12
python