Chapter 55
Best practices for the real world
NotebookPython 335 cells
This is a companion notebook for the book Deep Learning with Python, Second Edition. For readability, it only contains runnable code blocks and section titles, and omits everything else in the book: text paragraphs, figures, and pseudocode.
If you want to be able to follow what's going on, I recommend reading the notebook side by side with your copy of the book.
This notebook was generated for TensorFlow 2.6.
Best practices for the real world
Getting the most out of your models
Hyperparameter optimization
Using KerasTuner
In [0]python · cell 6
python
!pip install keras-tuner -qA KerasTuner model-building function
In [0]python · cell 8
python
from tensorflow import keras
from tensorflow.keras import layers
def build_model(hp):
units = hp.Int(name="units", min_value=16, max_value=64, step=16)
model = keras.Sequential([
layers.Dense(units, activation="relu"),
layers.Dense(10, activation="softmax")
])
optimizer = hp.Choice(name="optimizer", values=["rmsprop", "adam"])
model.compile(
optimizer=optimizer,
loss="sparse_categorical_crossentropy",
metrics=["accuracy"])
return modelA KerasTuner HyperModel
In [0]python · cell 10
python
import kerastuner as kt
class SimpleMLP(kt.HyperModel):
def __init__(self, num_classes):
self.num_classes = num_classes
def build(self, hp):
units = hp.Int(name="units", min_value=16, max_value=64, step=16)
model = keras.Sequential([
layers.Dense(units, activation="relu"),
layers.Dense(self.num_classes, activation="softmax")
])
optimizer = hp.Choice(name="optimizer", values=["rmsprop", "adam"])
model.compile(
optimizer=optimizer,
loss="sparse_categorical_crossentropy",
metrics=["accuracy"])
return model
hypermodel = SimpleMLP(num_classes=10)In [0]python · cell 11
python
tuner = kt.BayesianOptimization(
build_model,
objective="val_accuracy",
max_trials=100,
executions_per_trial=2,
directory="mnist_kt_test",
overwrite=True,
)In [0]python · cell 12
python
tuner.search_space_summary()In [0]python · cell 13
python
(x_train, y_train), (x_test, y_test) = keras.datasets.mnist.load_data()
x_train = x_train.reshape((-1, 28 * 28)).astype("float32") / 255
x_test = x_test.reshape((-1, 28 * 28)).astype("float32") / 255
x_train_full = x_train[:]
y_train_full = y_train[:]
num_val_samples = 10000
x_train, x_val = x_train[:-num_val_samples], x_train[-num_val_samples:]
y_train, y_val = y_train[:-num_val_samples], y_train[-num_val_samples:]
callbacks = [
keras.callbacks.EarlyStopping(monitor="val_loss", patience=5),
]
tuner.search(
x_train, y_train,
batch_size=128,
epochs=100,
validation_data=(x_val, y_val),
callbacks=callbacks,
verbose=2,
)Querying the best hyperparameter configurations
In [0]python · cell 15
python
top_n = 4
best_hps = tuner.get_best_hyperparameters(top_n)In [0]python · cell 16
python
def get_best_epoch(hp):
model = build_model(hp)
callbacks=[
keras.callbacks.EarlyStopping(
monitor="val_loss", mode="min", patience=10)
]
history = model.fit(
x_train, y_train,
validation_data=(x_val, y_val),
epochs=100,
batch_size=128,
callbacks=callbacks)
val_loss_per_epoch = history.history["val_loss"]
best_epoch = val_loss_per_epoch.index(min(val_loss_per_epoch)) + 1
print(f"Best epoch: {best_epoch}")
return best_epochIn [0]python · cell 17
python
def get_best_trained_model(hp):
best_epoch = get_best_epoch(hp)
model = build_model(hp)
model.fit(
x_train_full, y_train_full,
batch_size=128, epochs=int(best_epoch * 1.2))
return model
best_models = []
for hp in best_hps:
model = get_best_trained_model(hp)
model.evaluate(x_test, y_test)
best_models.append(model)In [0]python · cell 18
python
best_models = tuner.get_best_models(top_n)The art of crafting the right search space
The future of hyperparameter tuning: automated machine learning
Model ensembling
Scaling-up model training
Speeding up training on GPU with mixed precision
Understanding floating-point precision
In [0]python · cell 25
python
import tensorflow as tf
import numpy as np
np_array = np.zeros((2, 2))
tf_tensor = tf.convert_to_tensor(np_array)
tf_tensor.dtypeIn [0]python · cell 26
python
np_array = np.zeros((2, 2))
tf_tensor = tf.convert_to_tensor(np_array, dtype="float32")
tf_tensor.dtypeMixed-precision training in practice
In [0]python · cell 28
python
from tensorflow import keras
keras.mixed_precision.set_global_policy("mixed_float16")