Chapter 01

The mathematical building blocks of neural networks

NotebookPython 3113 cells

This is a companion notebook for the book Deep Learning with Python, Third Edition. For readability, it only contains runnable code blocks and section titles, and omits everything else in the book: text paragraphs, figures, and pseudocode.

If you want to be able to follow what's going on, I recommend reading the notebook side by side with your copy of the book.

The book's contents are available online at deeplearningwithpython.io.

In [0]python · cell 2

python

!pip install keras keras-hub --upgrade -q

In [0]python · cell 3

python

import os
os.environ["KERAS_BACKEND"] = "tensorflow"

In [0]python · cell 4

python

# @title
import os
from IPython.core.magic import register_cell_magic

@register_cell_magic
def backend(line, cell):
    current, required = os.environ.get("KERAS_BACKEND", ""), line.split()[-1]
    if current == required:
        get_ipython().run_cell(cell)
    else:
        print(
            f"This cell requires the {required} backend. To run it, change KERAS_BACKEND to "
            f"\"{required}\" at the top of the notebook, restart the runtime, and rerun the notebook."
        )

The mathematical building blocks of neural networks

A first look at a neural network

In [0]python · cell 7

python

from keras.datasets import mnist

(train_images, train_labels), (test_images, test_labels) = mnist.load_data()

In [0]python · cell 8

python

train_images.shape

In [0]python · cell 9

python

len(train_labels)

In [0]python · cell 10

python

train_labels

In [0]python · cell 11

python

test_images.shape

In [0]python · cell 12

python

len(test_labels)

In [0]python · cell 13

python

test_labels

In [0]python · cell 14

python

import keras
from keras import layers

model = keras.Sequential(
    [
        layers.Dense(512, activation="relu"),
        layers.Dense(10, activation="softmax"),
    ]
)

In [0]python · cell 15

python

model.compile(
    optimizer="adam",
    loss="sparse_categorical_crossentropy",
    metrics=["accuracy"],
)

In [0]python · cell 16

python

train_images = train_images.reshape((60000, 28 * 28))
train_images = train_images.astype("float32") / 255
test_images = test_images.reshape((10000, 28 * 28))
test_images = test_images.astype("float32") / 255

In [0]python · cell 17

python

model.fit(train_images, train_labels, epochs=5, batch_size=128)

In [0]python · cell 18

python

test_digits = test_images[0:10]
predictions = model.predict(test_digits)
predictions[0]

In [0]python · cell 19

python

predictions[0].argmax()

In [0]python · cell 20

python

predictions[0][7]

In [0]python · cell 21

python

test_labels[0]

In [0]python · cell 22

python

test_loss, test_acc = model.evaluate(test_images, test_labels)
print(f"test_acc: {test_acc}")

Data representations for neural networks

Scalars (rank-0 tensors)

In [0]python · cell 25

python

import numpy as np
x = np.array(12)
x

In [0]python · cell 26

python

x.ndim

Vectors (rank-1 tensors)

In [0]python · cell 28

python

x = np.array([12, 3, 6, 14, 7])
x

In [0]python · cell 29

python

x.ndim

Matrices (rank-2 tensors)

In [0]python · cell 31

python

x = np.array([[5, 78, 2, 34, 0],
              [6, 79, 3, 35, 1],
              [7, 80, 4, 36, 2]])
x.ndim

Rank-3 tensors and higher-rank tensors

In [0]python · cell 33

python

x = np.array([[[5, 78, 2, 34, 0],
               [6, 79, 3, 35, 1],
               [7, 80, 4, 36, 2]],
              [[5, 78, 2, 34, 0],
               [6, 79, 3, 35, 1],
               [7, 80, 4, 36, 2]],
              [[5, 78, 2, 34, 0],
               [6, 79, 3, 35, 1],
               [7, 80, 4, 36, 2]]])
x.ndim

Key attributes

In [0]python · cell 35

python

from keras.datasets import mnist

(train_images, train_labels), (test_images, test_labels) = mnist.load_data()

In [0]python · cell 36

python

train_images.ndim

In [0]python · cell 37

python

train_images.shape

In [0]python · cell 38

python

train_images.dtype

In [0]python · cell 39

python

import matplotlib.pyplot as plt

digit = train_images[4]
plt.imshow(digit, cmap=plt.cm.binary)
plt.show()

In [0]python · cell 40

python

train_labels[4]

Manipulating tensors in NumPy

In [0]python · cell 42

python

my_slice = train_images[10:100]
my_slice.shape

In [0]python · cell 43

python

my_slice = train_images[10:100, :, :]
my_slice.shape

In [0]python · cell 44

python

my_slice = train_images[10:100, 0:28, 0:28]
my_slice.shape

In [0]python · cell 45

python

my_slice = train_images[:, 14:, 14:]

In [0]python · cell 46

python

my_slice = train_images[:, 7:-7, 7:-7]

The notion of data batches

In [0]python · cell 48

python

batch = train_images[:128]

In [0]python · cell 49

python

batch = train_images[128:256]

In [0]python · cell 50

python

n = 3
batch = train_images[128 * n : 128 * (n + 1)]

Real-world examples of data tensors

Vector data

Timeseries data or sequence data

Image data

Video data

The gears of neural networks: Tensor operations

Element-wise operations

In [0]python · cell 58

python

def naive_relu(x):
    assert len(x.shape) == 2
    x = x.copy()
    for i in range(x.shape[0]):
        for j in range(x.shape[1]):
            x[i, j] = max(x[i, j], 0)
    return x

In [0]python · cell 59

python

def naive_add(x, y):
    assert len(x.shape) == 2
    assert x.shape == y.shape
    x = x.copy()
    for i in range(x.shape[0]):
        for j in range(x.shape[1]):
            x[i, j] += y[i, j]
    return x

In [0]python · cell 60

python

import time

x = np.random.random((20, 100))
y = np.random.random((20, 100))

t0 = time.time()
for _ in range(1000):
    z = x + y
    z = np.maximum(z, 0.0)
print("Took: {0:.2f} s".format(time.time() - t0))

In [0]python · cell 61

python

t0 = time.time()
for _ in range(1000):
    z = naive_add(x, y)
    z = naive_relu(z)
print("Took: {0:.2f} s".format(time.time() - t0))

Broadcasting

In [0]python · cell 63

python

import numpy as np

X = np.random.random((32, 10))
y = np.random.random((10,))

In [0]python · cell 64

python

y = np.expand_dims(y, axis=0)

In [0]python · cell 65

python

Y = np.tile(y, (32, 1))

In [0]python · cell 66

python

def naive_add_matrix_and_vector(x, y):
    assert len(x.shape) == 2
    assert len(y.shape) == 1
    assert x.shape[1] == y.shape[0]
    x = x.copy()
    for i in range(x.shape[0]):
        for j in range(x.shape[1]):
            x[i, j] += y[j]
    return x

In [0]python · cell 67

python

import numpy as np

x = np.random.random((64, 3, 32, 10))
y = np.random.random((32, 10))
z = np.maximum(x, y)

Tensor product

In [0]python · cell 69

python

x = np.random.random((32,))
y = np.random.random((32,))

z = np.matmul(x, y)
z = x @ y

In [0]python · cell 70

python

def naive_vector_product(x, y):
    assert len(x.shape) == 1
    assert len(y.shape) == 1
    assert x.shape[0] == y.shape[0]
    z = 0.0
    for i in range(x.shape[0]):
        z += x[i] * y[i]
    return z

In [0]python · cell 71

python

def naive_matrix_vector_product(x, y):
    assert len(x.shape) == 2
    assert len(y.shape) == 1
    assert x.shape[1] == y.shape[0]
    z = np.zeros(x.shape[0])
    for i in range(x.shape[0]):
        for j in range(x.shape[1]):
            z[i] += x[i, j] * y[j]
    return z

In [0]python · cell 72

python

def naive_matrix_vector_product(x, y):
    z = np.zeros(x.shape[0])
    for i in range(x.shape[0]):
        z[i] = naive_vector_product(x[i, :], y)
    return z

In [0]python · cell 73

python

def naive_matrix_product(x, y):
    assert len(x.shape) == 2
    assert len(y.shape) == 2
    assert x.shape[1] == y.shape[0]
    z = np.zeros((x.shape[0], y.shape[1]))
    for i in range(x.shape[0]):
        for j in range(y.shape[1]):
            row_x = x[i, :]
            column_y = y[:, j]
            z[i, j] = naive_vector_product(row_x, column_y)
    return z

Tensor reshaping

In [0]python · cell 75

python

train_images = train_images.reshape((60000, 28 * 28))

In [0]python · cell 76

python

x = np.array([[0., 1.],
              [2., 3.],
              [4., 5.]])
x.shape

In [0]python · cell 77

python

x = x.reshape((6, 1))
x

In [0]python · cell 78

python

x = x.reshape((2, 3))
x

In [0]python · cell 79

python

x = np.zeros((300, 20))
x = np.transpose(x)
x.shape

Geometric interpretation of tensor operations

A geometric interpretation of deep learning

The engine of neural networks: Gradient-based optimization

What's a derivative?

Derivative of a tensor operation: The gradient

Stochastic gradient descent

Chaining derivatives: The Backpropagation algorithm

The chain rule

Automatic differentiation with computation graphs

Looking back at our first example

In [0]python · cell 90

python

(train_images, train_labels), (test_images, test_labels) = mnist.load_data()
train_images = train_images.reshape((60000, 28 * 28))
train_images = train_images.astype("float32") / 255
test_images = test_images.reshape((10000, 28 * 28))
test_images = test_images.astype("float32") / 255

In [0]python · cell 91

python

model = keras.Sequential(
    [
        layers.Dense(512, activation="relu"),
        layers.Dense(10, activation="softmax"),
    ]
)

In [0]python · cell 92

python

model.compile(
    optimizer="adam",
    loss="sparse_categorical_crossentropy",
    metrics=["accuracy"],
)

In [0]python · cell 93

python

model.fit(
    train_images,
    train_labels,
    epochs=5,
    batch_size=128,
)

Reimplementing our first example from scratch

A simple Dense class

In [0]python · cell 96

python

import keras
from keras import ops

class NaiveDense:
    def __init__(self, input_size, output_size, activation=None):
        self.activation = activation
        self.W = keras.Variable(
            shape=(input_size, output_size), initializer="uniform"
        )
        self.b = keras.Variable(shape=(output_size,), initializer="zeros")

    def __call__(self, inputs):
        x = ops.matmul(inputs, self.W)
        x = x + self.b
        if self.activation is not None:
            x = self.activation(x)
        return x

    @property
    def weights(self):
        return [self.W, self.b]

A simple Sequential class

In [0]python · cell 98

python

class NaiveSequential:
    def __init__(self, layers):
        self.layers = layers

    def __call__(self, inputs):
        x = inputs
        for layer in self.layers:
            x = layer(x)
        return x

    @property
    def weights(self):
        weights = []
        for layer in self.layers:
            weights += layer.weights
        return weights

In [0]python · cell 99

python

model = NaiveSequential(
    [
        NaiveDense(input_size=28 * 28, output_size=512, activation=ops.relu),
        NaiveDense(input_size=512, output_size=10, activation=ops.softmax),
    ]
)
assert len(model.weights) == 4

A batch generator

In [0]python · cell 101

python

import math

class BatchGenerator:
    def __init__(self, images, labels, batch_size=128):
        assert len(images) == len(labels)
        self.index = 0
        self.images = images
        self.labels = labels
        self.batch_size = batch_size
        self.num_batches = math.ceil(len(images) / batch_size)

    def next(self):
        images = self.images[self.index : self.index + self.batch_size]
        labels = self.labels[self.index : self.index + self.batch_size]
        self.index += self.batch_size
        return images, labels

Running one training step

The weight update step

In [0]python · cell 104

python

learning_rate = 1e-3

def update_weights(gradients, weights):
    for g, w in zip(gradients, weights):
        w.assign(w - g * learning_rate)

In [0]python · cell 105

python

from keras import optimizers

optimizer = optimizers.SGD(learning_rate=1e-3)

def update_weights(gradients, weights):
    optimizer.apply_gradients(zip(gradients, weights))

Gradient computation

In [0]python · cell 107

python

%%backend tensorflow
import tensorflow as tf

x = tf.zeros(shape=())
with tf.GradientTape() as tape:
    y = 2 * x + 3
grad_of_y_wrt_x = tape.gradient(y, x)

In [0]python · cell 108

python

%%backend tensorflow
def one_training_step(model, images_batch, labels_batch):
    with tf.GradientTape() as tape:
        predictions = model(images_batch)
        loss = ops.sparse_categorical_crossentropy(labels_batch, predictions)
        average_loss = ops.mean(loss)
    gradients = tape.gradient(average_loss, model.weights)
    update_weights(gradients, model.weights)
    return average_loss

The full training loop

In [0]python · cell 110

python

%%backend tensorflow
def fit(model, images, labels, epochs, batch_size=128):
    for epoch_counter in range(epochs):
        print(f"Epoch {epoch_counter}")
        batch_generator = BatchGenerator(images, labels)
        for batch_counter in range(batch_generator.num_batches):
            images_batch, labels_batch = batch_generator.next()
            loss = one_training_step(model, images_batch, labels_batch)
            if batch_counter % 100 == 0:
                print(f"loss at batch {batch_counter}: {loss:.2f}")

In [0]python · cell 111

python

%%backend tensorflow
from keras.datasets import mnist

(train_images, train_labels), (test_images, test_labels) = mnist.load_data()

train_images = train_images.reshape((60000, 28 * 28))
train_images = train_images.astype("float32") / 255
test_images = test_images.reshape((10000, 28 * 28))
test_images = test_images.astype("float32") / 255

fit(model, train_images, train_labels, epochs=10, batch_size=128)

Evaluating the model

In [0]python · cell 113

python

%%backend tensorflow
predictions = model(test_images)
predicted_labels = ops.argmax(predictions, axis=1)
matches = predicted_labels == test_labels
f"accuracy: {ops.mean(matches):.2f}"

# @title import os from IPython.core.magic import register_cell_magic @register_cell_magic def backend(line, cell): current, required = os.environ.get("KERAS_BACKEND", ""), line.split()[-1] if current == required: get_ipython().run_cell(cell) else: print( f"This cell requires the {required} backend. To run it, change KERAS_BACKEND to " f"\"{required}\" at the top of the notebook, restart the runtime, and rerun the notebook." )

train_images = train_images.reshape((60000, 28 * 28)) train_images = train_images.astype("float32") / 255 test_images = test_images.reshape((10000, 28 * 28)) test_images = test_images.astype("float32") / 255

x = np.array([[[5, 78, 2, 34, 0], [6, 79, 3, 35, 1], [7, 80, 4, 36, 2]], [[5, 78, 2, 34, 0], [6, 79, 3, 35, 1], [7, 80, 4, 36, 2]], [[5, 78, 2, 34, 0], [6, 79, 3, 35, 1], [7, 80, 4, 36, 2]]]) x.ndim

def naive_add(x, y): assert len(x.shape) == 2 assert x.shape == y.shape x = x.copy() for i in range(x.shape[0]): for j in range(x.shape[1]): x[i, j] += y[i, j] return x

import time x = np.random.random((20, 100)) y = np.random.random((20, 100)) t0 = time.time() for _ in range(1000): z = x + y z = np.maximum(z, 0.0) print("Took: {0:.2f} s".format(time.time() - t0))

def naive_add_matrix_and_vector(x, y): assert len(x.shape) == 2 assert len(y.shape) == 1 assert x.shape[1] == y.shape[0] x = x.copy() for i in range(x.shape[0]): for j in range(x.shape[1]): x[i, j] += y[j] return x

def naive_vector_product(x, y): assert len(x.shape) == 1 assert len(y.shape) == 1 assert x.shape[0] == y.shape[0] z = 0.0 for i in range(x.shape[0]): z += x[i] * y[i] return z

def naive_matrix_vector_product(x, y): assert len(x.shape) == 2 assert len(y.shape) == 1 assert x.shape[1] == y.shape[0] z = np.zeros(x.shape[0]) for i in range(x.shape[0]): for j in range(x.shape[1]): z[i] += x[i, j] * y[j] return z

def naive_matrix_product(x, y): assert len(x.shape) == 2 assert len(y.shape) == 2 assert x.shape[1] == y.shape[0] z = np.zeros((x.shape[0], y.shape[1])) for i in range(x.shape[0]): for j in range(y.shape[1]): row_x = x[i, :] column_y = y[:, j] z[i, j] = naive_vector_product(row_x, column_y) return z

(train_images, train_labels), (test_images, test_labels) = mnist.load_data() train_images = train_images.reshape((60000, 28 * 28)) train_images = train_images.astype("float32") / 255 test_images = test_images.reshape((10000, 28 * 28)) test_images = test_images.astype("float32") / 255

import keras from keras import ops class NaiveDense: def __init__(self, input_size, output_size, activation=None): self.activation = activation self.W = keras.Variable( shape=(input_size, output_size), initializer="uniform" ) self.b = keras.Variable(shape=(output_size,), initializer="zeros") def __call__(self, inputs): x = ops.matmul(inputs, self.W) x = x + self.b if self.activation is not None: x = self.activation(x) return x @property def weights(self): return [self.W, self.b]

class NaiveSequential: def __init__(self, layers): self.layers = layers def __call__(self, inputs): x = inputs for layer in self.layers: x = layer(x) return x @property def weights(self): weights = [] for layer in self.layers: weights += layer.weights return weights

model = NaiveSequential( [ NaiveDense(input_size=28 * 28, output_size=512, activation=ops.relu), NaiveDense(input_size=512, output_size=10, activation=ops.softmax), ] ) assert len(model.weights) == 4

import math class BatchGenerator: def __init__(self, images, labels, batch_size=128): assert len(images) == len(labels) self.index = 0 self.images = images self.labels = labels self.batch_size = batch_size self.num_batches = math.ceil(len(images) / batch_size) def next(self): images = self.images[self.index : self.index + self.batch_size] labels = self.labels[self.index : self.index + self.batch_size] self.index += self.batch_size return images, labels

%%backend tensorflow def one_training_step(model, images_batch, labels_batch): with tf.GradientTape() as tape: predictions = model(images_batch) loss = ops.sparse_categorical_crossentropy(labels_batch, predictions) average_loss = ops.mean(loss) gradients = tape.gradient(average_loss, model.weights) update_weights(gradients, model.weights) return average_loss

%%backend tensorflow def fit(model, images, labels, epochs, batch_size=128): for epoch_counter in range(epochs): print(f"Epoch {epoch_counter}") batch_generator = BatchGenerator(images, labels) for batch_counter in range(batch_generator.num_batches): images_batch, labels_batch = batch_generator.next() loss = one_training_step(model, images_batch, labels_batch) if batch_counter % 100 == 0: print(f"loss at batch {batch_counter}: {loss:.2f}")

%%backend tensorflow from keras.datasets import mnist (train_images, train_labels), (test_images, test_labels) = mnist.load_data() train_images = train_images.reshape((60000, 28 * 28)) train_images = train_images.astype("float32") / 255 test_images = test_images.reshape((10000, 28 * 28)) test_images = test_images.astype("float32") / 255 fit(model, train_images, train_labels, epochs=10, batch_size=128)