Chapter 03

Classification and regression

NotebookPython 393 cells

This is a companion notebook for the book Deep Learning with Python, Third Edition. For readability, it only contains runnable code blocks and section titles, and omits everything else in the book: text paragraphs, figures, and pseudocode.

If you want to be able to follow what's going on, I recommend reading the notebook side by side with your copy of the book.

The book's contents are available online at deeplearningwithpython.io.

In [0]python · cell 2

python

!pip install keras keras-hub --upgrade -q

In [0]python · cell 3

python

import os
os.environ["KERAS_BACKEND"] = "jax"

In [0]python · cell 4

python

# @title
import os
from IPython.core.magic import register_cell_magic

@register_cell_magic
def backend(line, cell):
    current, required = os.environ.get("KERAS_BACKEND", ""), line.split()[-1]
    if current == required:
        get_ipython().run_cell(cell)
    else:
        print(
            f"This cell requires the {required} backend. To run it, change KERAS_BACKEND to "
            f"\"{required}\" at the top of the notebook, restart the runtime, and rerun the notebook."
        )

Classification and regression

Classifying movie reviews: A binary classification example

The IMDb dataset

In [0]python · cell 8

python

from keras.datasets import imdb

(train_data, train_labels), (test_data, test_labels) = imdb.load_data(
    num_words=10000
)

In [0]python · cell 9

python

train_data[0]

In [0]python · cell 10

python

train_labels[0]

In [0]python · cell 11

python

max([max(sequence) for sequence in train_data])

In [0]python · cell 12

python

word_index = imdb.get_word_index()
reverse_word_index = dict([(value, key) for (key, value) in word_index.items()])
decoded_review = " ".join(
    [reverse_word_index.get(i - 3, "?") for i in train_data[0]]
)

In [0]python · cell 13

python

decoded_review[:100]

Preparing the data

In [0]python · cell 15

python

import numpy as np

def multi_hot_encode(sequences, num_classes):
    results = np.zeros((len(sequences), num_classes))
    for i, sequence in enumerate(sequences):
        results[i][sequence] = 1.0
    return results

x_train = multi_hot_encode(train_data, num_classes=10000)
x_test = multi_hot_encode(test_data, num_classes=10000)

In [0]python · cell 16

python

x_train[0]

In [0]python · cell 17

python

y_train = train_labels.astype("float32")
y_test = test_labels.astype("float32")

Building your model

In [0]python · cell 19

python

import keras
from keras import layers

model = keras.Sequential(
    [
        layers.Dense(16, activation="relu"),
        layers.Dense(16, activation="relu"),
        layers.Dense(1, activation="sigmoid"),
    ]
)

In [0]python · cell 20

python

model.compile(
    optimizer="adam",
    loss="binary_crossentropy",
    metrics=["accuracy"],
)

Validating your approach

In [0]python · cell 22

python

x_val = x_train[:10000]
partial_x_train = x_train[10000:]
y_val = y_train[:10000]
partial_y_train = y_train[10000:]

In [0]python · cell 23

python

history = model.fit(
    partial_x_train,
    partial_y_train,
    epochs=20,
    batch_size=512,
    validation_data=(x_val, y_val),
)

In [0]python · cell 24

python

history = model.fit(
    x_train,
    y_train,
    epochs=20,
    batch_size=512,
    validation_split=0.2,
)

In [0]python · cell 25

python

history_dict = history.history
history_dict.keys()

In [0]python · cell 26

python

import matplotlib.pyplot as plt

history_dict = history.history
loss_values = history_dict["loss"]
val_loss_values = history_dict["val_loss"]
epochs = range(1, len(loss_values) + 1)
plt.plot(epochs, loss_values, "r--", label="Training loss")
plt.plot(epochs, val_loss_values, "b", label="Validation loss")
plt.title("[IMDB] Training and validation loss")
plt.xlabel("Epochs")
plt.xticks(epochs)
plt.ylabel("Loss")
plt.legend()
plt.show()

In [0]python · cell 27

python

plt.clf()
acc = history_dict["accuracy"]
val_acc = history_dict["val_accuracy"]
plt.plot(epochs, acc, "r--", label="Training acc")
plt.plot(epochs, val_acc, "b", label="Validation acc")
plt.title("[IMDB] Training and validation accuracy")
plt.xlabel("Epochs")
plt.xticks(epochs)
plt.ylabel("Accuracy")
plt.legend()
plt.show()

In [0]python · cell 28

python

model = keras.Sequential(
    [
        layers.Dense(16, activation="relu"),
        layers.Dense(16, activation="relu"),
        layers.Dense(1, activation="sigmoid"),
    ]
)
model.compile(
    optimizer="adam",
    loss="binary_crossentropy",
    metrics=["accuracy"],
)
model.fit(x_train, y_train, epochs=4, batch_size=512)
results = model.evaluate(x_test, y_test)

In [0]python · cell 29

python

results

Using a trained model to generate predictions on new data

In [0]python · cell 31

python

model.predict(x_test)

Further experiments

Wrapping up

Classifying newswires: A multiclass classification example

The Reuters dataset

In [0]python · cell 36

python

from keras.datasets import reuters

(train_data, train_labels), (test_data, test_labels) = reuters.load_data(
    num_words=10000
)

In [0]python · cell 37

python

len(train_data)

In [0]python · cell 38

python

len(test_data)

In [0]python · cell 39

python

train_data[10]

In [0]python · cell 40

python

word_index = reuters.get_word_index()
reverse_word_index = dict([(value, key) for (key, value) in word_index.items()])
decoded_newswire = " ".join(
    [reverse_word_index.get(i - 3, "?") for i in train_data[10]]
)

In [0]python · cell 41

python

train_labels[10]

Preparing the data

In [0]python · cell 43

python

x_train = multi_hot_encode(train_data, num_classes=10000)
x_test = multi_hot_encode(test_data, num_classes=10000)

In [0]python · cell 44

python

def one_hot_encode(labels, num_classes=46):
    results = np.zeros((len(labels), num_classes))
    for i, label in enumerate(labels):
        results[i, label] = 1.0
    return results

y_train = one_hot_encode(train_labels)
y_test = one_hot_encode(test_labels)

In [0]python · cell 45

python

from keras.utils import to_categorical

y_train = to_categorical(train_labels)
y_test = to_categorical(test_labels)

Building your model

In [0]python · cell 47

python

model = keras.Sequential(
    [
        layers.Dense(64, activation="relu"),
        layers.Dense(64, activation="relu"),
        layers.Dense(46, activation="softmax"),
    ]
)

In [0]python · cell 48

python

top_3_accuracy = keras.metrics.TopKCategoricalAccuracy(
    k=3, name="top_3_accuracy"
)
model.compile(
    optimizer="adam",
    loss="categorical_crossentropy",
    metrics=["accuracy", top_3_accuracy],
)

Validating your approach

In [0]python · cell 50

python

x_val = x_train[:1000]
partial_x_train = x_train[1000:]
y_val = y_train[:1000]
partial_y_train = y_train[1000:]

In [0]python · cell 51

python

history = model.fit(
    partial_x_train,
    partial_y_train,
    epochs=20,
    batch_size=512,
    validation_data=(x_val, y_val),
)

In [0]python · cell 52

python

loss = history.history["loss"]
val_loss = history.history["val_loss"]
epochs = range(1, len(loss) + 1)
plt.plot(epochs, loss, "r--", label="Training loss")
plt.plot(epochs, val_loss, "b", label="Validation loss")
plt.title("Training and validation loss")
plt.xlabel("Epochs")
plt.xticks(epochs)
plt.ylabel("Loss")
plt.legend()
plt.show()

In [0]python · cell 53

python

plt.clf()
acc = history.history["accuracy"]
val_acc = history.history["val_accuracy"]
plt.plot(epochs, acc, "r--", label="Training accuracy")
plt.plot(epochs, val_acc, "b", label="Validation accuracy")
plt.title("Training and validation accuracy")
plt.xlabel("Epochs")
plt.xticks(epochs)
plt.ylabel("Accuracy")
plt.legend()
plt.show()

In [0]python · cell 54

python

plt.clf()
acc = history.history["top_3_accuracy"]
val_acc = history.history["val_top_3_accuracy"]
plt.plot(epochs, acc, "r--", label="Training top-3 accuracy")
plt.plot(epochs, val_acc, "b", label="Validation top-3 accuracy")
plt.title("Training and validation top-3 accuracy")
plt.xlabel("Epochs")
plt.xticks(epochs)
plt.ylabel("Top-3 accuracy")
plt.legend()
plt.show()

In [0]python · cell 55

python

model = keras.Sequential(
    [
        layers.Dense(64, activation="relu"),
        layers.Dense(64, activation="relu"),
        layers.Dense(46, activation="softmax"),
    ]
)
model.compile(
    optimizer="adam",
    loss="categorical_crossentropy",
    metrics=["accuracy"],
)
model.fit(
    x_train,
    y_train,
    epochs=9,
    batch_size=512,
)
results = model.evaluate(x_test, y_test)

In [0]python · cell 56

python

results

In [0]python · cell 57

python

import copy
test_labels_copy = copy.copy(test_labels)
np.random.shuffle(test_labels_copy)
hits_array = np.array(test_labels == test_labels_copy)
hits_array.mean()

Generating predictions on new data

In [0]python · cell 59

python

predictions = model.predict(x_test)

In [0]python · cell 60

python

predictions[0].shape

In [0]python · cell 61

python

np.sum(predictions[0])

In [0]python · cell 62

python

np.argmax(predictions[0])

A different way to handle the labels and the loss

In [0]python · cell 64

python

y_train = train_labels
y_test = test_labels

In [0]python · cell 65

python

model.compile(
    optimizer="adam",
    loss="sparse_categorical_crossentropy",
    metrics=["accuracy"],
)

The importance of having sufficiently large intermediate layers

In [0]python · cell 67

python

model = keras.Sequential(
    [
        layers.Dense(64, activation="relu"),
        layers.Dense(4, activation="relu"),
        layers.Dense(46, activation="softmax"),
    ]
)
model.compile(
    optimizer="adam",
    loss="categorical_crossentropy",
    metrics=["accuracy"],
)
model.fit(
    partial_x_train,
    partial_y_train,
    epochs=20,
    batch_size=128,
    validation_data=(x_val, y_val),
)

Further experiments

Wrapping up

Predicting house prices: A regression example

The California Housing Price dataset

In [0]python · cell 72

python

from keras.datasets import california_housing

(train_data, train_targets), (test_data, test_targets) = (
    california_housing.load_data(version="small")
)

In [0]python · cell 73

python

train_data.shape

In [0]python · cell 74

python

test_data.shape

In [0]python · cell 75

python

train_targets

Preparing the data

In [0]python · cell 77

python

mean = train_data.mean(axis=0)
std = train_data.std(axis=0)
x_train = (train_data - mean) / std
x_test = (test_data - mean) / std

In [0]python · cell 78

python

y_train = train_targets / 100000
y_test = test_targets / 100000

Building your model

In [0]python · cell 80

python

def get_model():
    model = keras.Sequential(
        [
            layers.Dense(64, activation="relu"),
            layers.Dense(64, activation="relu"),
            layers.Dense(1),
        ]
    )
    model.compile(
        optimizer="adam",
        loss="mean_squared_error",
        metrics=["mean_absolute_error"],
    )
    return model

Validating your approach using K-fold validation

In [0]python · cell 82

python

k = 4
num_val_samples = len(x_train) // k
num_epochs = 50
all_scores = []
for i in range(k):
    print(f"Processing fold #{i + 1}")
    fold_x_val = x_train[i * num_val_samples : (i + 1) * num_val_samples]
    fold_y_val = y_train[i * num_val_samples : (i + 1) * num_val_samples]
    fold_x_train = np.concatenate(
        [x_train[: i * num_val_samples], x_train[(i + 1) * num_val_samples :]],
        axis=0,
    )
    fold_y_train = np.concatenate(
        [y_train[: i * num_val_samples], y_train[(i + 1) * num_val_samples :]],
        axis=0,
    )
    model = get_model()
    model.fit(
        fold_x_train,
        fold_y_train,
        epochs=num_epochs,
        batch_size=16,
        verbose=0,
    )
    scores = model.evaluate(fold_x_val, fold_y_val, verbose=0)
    val_loss, val_mae = scores
    all_scores.append(val_mae)

In [0]python · cell 83

python

[round(value, 3) for value in all_scores]

In [0]python · cell 84

python

round(np.mean(all_scores), 3)

In [0]python · cell 85

python

k = 4
num_val_samples = len(x_train) // k
num_epochs = 200
all_mae_histories = []
for i in range(k):
    print(f"Processing fold #{i + 1}")
    fold_x_val = x_train[i * num_val_samples : (i + 1) * num_val_samples]
    fold_y_val = y_train[i * num_val_samples : (i + 1) * num_val_samples]
    fold_x_train = np.concatenate(
        [x_train[: i * num_val_samples], x_train[(i + 1) * num_val_samples :]],
        axis=0,
    )
    fold_y_train = np.concatenate(
        [y_train[: i * num_val_samples], y_train[(i + 1) * num_val_samples :]],
        axis=0,
    )
    model = get_model()
    history = model.fit(
        fold_x_train,
        fold_y_train,
        validation_data=(fold_x_val, fold_y_val),
        epochs=num_epochs,
        batch_size=16,
        verbose=0,
    )
    mae_history = history.history["val_mean_absolute_error"]
    all_mae_histories.append(mae_history)

In [0]python · cell 86

python

average_mae_history = [
    np.mean([x[i] for x in all_mae_histories]) for i in range(num_epochs)
]

In [0]python · cell 87

python

epochs = range(1, len(average_mae_history) + 1)
plt.plot(epochs, average_mae_history)
plt.xlabel("Epochs")
plt.ylabel("Validation MAE")
plt.show()

In [0]python · cell 88

python

truncated_mae_history = average_mae_history[10:]
epochs = range(10, len(truncated_mae_history) + 10)
plt.plot(epochs, truncated_mae_history)
plt.xlabel("Epochs")
plt.ylabel("Validation MAE")
plt.show()

In [0]python · cell 89

python

model = get_model()
model.fit(x_train, y_train, epochs=130, batch_size=16, verbose=0)
test_mean_squared_error, test_mean_absolute_error = model.evaluate(
    x_test, y_test
)

In [0]python · cell 90

python

round(test_mean_absolute_error, 3)

Generating predictions on new data

In [0]python · cell 92

python

predictions = model.predict(x_test)
predictions[0]

Wrapping up

# @title import os from IPython.core.magic import register_cell_magic @register_cell_magic def backend(line, cell): current, required = os.environ.get("KERAS_BACKEND", ""), line.split()[-1] if current == required: get_ipython().run_cell(cell) else: print( f"This cell requires the {required} backend. To run it, change KERAS_BACKEND to " f"\"{required}\" at the top of the notebook, restart the runtime, and rerun the notebook." )

word_index = imdb.get_word_index() reverse_word_index = dict([(value, key) for (key, value) in word_index.items()]) decoded_review = " ".join( [reverse_word_index.get(i - 3, "?") for i in train_data[0]] )

import numpy as np def multi_hot_encode(sequences, num_classes): results = np.zeros((len(sequences), num_classes)) for i, sequence in enumerate(sequences): results[i][sequence] = 1.0 return results x_train = multi_hot_encode(train_data, num_classes=10000) x_test = multi_hot_encode(test_data, num_classes=10000)

import keras from keras import layers model = keras.Sequential( [ layers.Dense(16, activation="relu"), layers.Dense(16, activation="relu"), layers.Dense(1, activation="sigmoid"), ] )

import matplotlib.pyplot as plt history_dict = history.history loss_values = history_dict["loss"] val_loss_values = history_dict["val_loss"] epochs = range(1, len(loss_values) + 1) plt.plot(epochs, loss_values, "r--", label="Training loss") plt.plot(epochs, val_loss_values, "b", label="Validation loss") plt.title("[IMDB] Training and validation loss") plt.xlabel("Epochs") plt.xticks(epochs) plt.ylabel("Loss") plt.legend() plt.show()

plt.clf() acc = history_dict["accuracy"] val_acc = history_dict["val_accuracy"] plt.plot(epochs, acc, "r--", label="Training acc") plt.plot(epochs, val_acc, "b", label="Validation acc") plt.title("[IMDB] Training and validation accuracy") plt.xlabel("Epochs") plt.xticks(epochs) plt.ylabel("Accuracy") plt.legend() plt.show()

model = keras.Sequential( [ layers.Dense(16, activation="relu"), layers.Dense(16, activation="relu"), layers.Dense(1, activation="sigmoid"), ] ) model.compile( optimizer="adam", loss="binary_crossentropy", metrics=["accuracy"], ) model.fit(x_train, y_train, epochs=4, batch_size=512) results = model.evaluate(x_test, y_test)

word_index = reuters.get_word_index() reverse_word_index = dict([(value, key) for (key, value) in word_index.items()]) decoded_newswire = " ".join( [reverse_word_index.get(i - 3, "?") for i in train_data[10]] )

def one_hot_encode(labels, num_classes=46): results = np.zeros((len(labels), num_classes)) for i, label in enumerate(labels): results[i, label] = 1.0 return results y_train = one_hot_encode(train_labels) y_test = one_hot_encode(test_labels)

top_3_accuracy = keras.metrics.TopKCategoricalAccuracy( k=3, name="top_3_accuracy" ) model.compile( optimizer="adam", loss="categorical_crossentropy", metrics=["accuracy", top_3_accuracy], )

loss = history.history["loss"] val_loss = history.history["val_loss"] epochs = range(1, len(loss) + 1) plt.plot(epochs, loss, "r--", label="Training loss") plt.plot(epochs, val_loss, "b", label="Validation loss") plt.title("Training and validation loss") plt.xlabel("Epochs") plt.xticks(epochs) plt.ylabel("Loss") plt.legend() plt.show()

plt.clf() acc = history.history["accuracy"] val_acc = history.history["val_accuracy"] plt.plot(epochs, acc, "r--", label="Training accuracy") plt.plot(epochs, val_acc, "b", label="Validation accuracy") plt.title("Training and validation accuracy") plt.xlabel("Epochs") plt.xticks(epochs) plt.ylabel("Accuracy") plt.legend() plt.show()

plt.clf() acc = history.history["top_3_accuracy"] val_acc = history.history["val_top_3_accuracy"] plt.plot(epochs, acc, "r--", label="Training top-3 accuracy") plt.plot(epochs, val_acc, "b", label="Validation top-3 accuracy") plt.title("Training and validation top-3 accuracy") plt.xlabel("Epochs") plt.xticks(epochs) plt.ylabel("Top-3 accuracy") plt.legend() plt.show()

model = keras.Sequential( [ layers.Dense(64, activation="relu"), layers.Dense(64, activation="relu"), layers.Dense(46, activation="softmax"), ] ) model.compile( optimizer="adam", loss="categorical_crossentropy", metrics=["accuracy"], ) model.fit( x_train, y_train, epochs=9, batch_size=512, ) results = model.evaluate(x_test, y_test)

model = keras.Sequential( [ layers.Dense(64, activation="relu"), layers.Dense(4, activation="relu"), layers.Dense(46, activation="softmax"), ] ) model.compile( optimizer="adam", loss="categorical_crossentropy", metrics=["accuracy"], ) model.fit( partial_x_train, partial_y_train, epochs=20, batch_size=128, validation_data=(x_val, y_val), )

def get_model(): model = keras.Sequential( [ layers.Dense(64, activation="relu"), layers.Dense(64, activation="relu"), layers.Dense(1), ] ) model.compile( optimizer="adam", loss="mean_squared_error", metrics=["mean_absolute_error"], ) return model

k = 4 num_val_samples = len(x_train) // k num_epochs = 50 all_scores = [] for i in range(k): print(f"Processing fold #{i + 1}") fold_x_val = x_train[i * num_val_samples : (i + 1) * num_val_samples] fold_y_val = y_train[i * num_val_samples : (i + 1) * num_val_samples] fold_x_train = np.concatenate( [x_train[: i * num_val_samples], x_train[(i + 1) * num_val_samples :]], axis=0, ) fold_y_train = np.concatenate( [y_train[: i * num_val_samples], y_train[(i + 1) * num_val_samples :]], axis=0, ) model = get_model() model.fit( fold_x_train, fold_y_train, epochs=num_epochs, batch_size=16, verbose=0, ) scores = model.evaluate(fold_x_val, fold_y_val, verbose=0) val_loss, val_mae = scores all_scores.append(val_mae)

k = 4 num_val_samples = len(x_train) // k num_epochs = 200 all_mae_histories = [] for i in range(k): print(f"Processing fold #{i + 1}") fold_x_val = x_train[i * num_val_samples : (i + 1) * num_val_samples] fold_y_val = y_train[i * num_val_samples : (i + 1) * num_val_samples] fold_x_train = np.concatenate( [x_train[: i * num_val_samples], x_train[(i + 1) * num_val_samples :]], axis=0, ) fold_y_train = np.concatenate( [y_train[: i * num_val_samples], y_train[(i + 1) * num_val_samples :]], axis=0, ) model = get_model() history = model.fit( fold_x_train, fold_y_train, validation_data=(fold_x_val, fold_y_val), epochs=num_epochs, batch_size=16, verbose=0, ) mae_history = history.history["val_mean_absolute_error"] all_mae_histories.append(mae_history)

truncated_mae_history = average_mae_history[10:] epochs = range(10, len(truncated_mae_history) + 10) plt.plot(epochs, truncated_mae_history) plt.xlabel("Epochs") plt.ylabel("Validation MAE") plt.show()