00. PyTorch Fundamentals

import torch
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
print(torch.__version__)

# scalar
scalar = torch.tensor(7)
scalar

scalar.ndim

# Get tensor back as Python int
scalar.item()

# Vector
vector = torch.tensor([7, 7])
vector

vector.ndim

vector.shape

# MATRIX
MATRIX = torch.tensor([[7, 8],
                       [9, 10]])
MATRIX

MATRIX.ndim

MATRIX[1]

MATRIX.shape

# TENSOR
TENSOR = torch.tensor([[[1, 2, 3],
                        [3, 6, 9],
                        [2, 4, 5]]])

TENSOR

TENSOR.ndim

TENSOR.shape

TENSOR[0]

# Create a random tensor of size (3, 4)
random_tensor = torch.rand(3, 4)
random_tensor

# Create a random tensor with similar shape to an image tensor
random_image_size_tensor = torch.rand(size=(3, 224, 224)) # height, width, colour channels (R, G, B)
random_image_size_tensor.shape, random_image_size_tensor.ndim

# Create a tensor of all zeros
zeros = torch.zeros(size=(3, 4))
zeros

# Create a tensor of all ones
ones = torch.ones(size=(3, 4))
ones

ones.dtype

random_tensor.dtype

# Use torch.range() and get deprecated message, use torch.arange()
one_to_ten = torch.arange(start=1, end=11, step=1)
one_to_ten

# Creating tensors like
ten_zeros = torch.zeros_like(input=one_to_ten)
ten_zeros

# Float 32 tensor
float_32_tensor = torch.tensor([3.0, 6.0, 9.0],
                               dtype=None, # what datatype is the tensor (e.g. float32 or float16)
                               device=None, # What device is your tensor on
                               requires_grad=False) # whether or not to track gradients with this tensors operations
float_32_tensor

float_32_tensor.dtype

float_16_tensor = float_32_tensor.type(torch.float16)
float_16_tensor

float_16_tensor * float_32_tensor

int_32_tensor = torch.tensor([3, 6, 9], dtype=torch.long)
int_32_tensor

float_32_tensor * int_32_tensor

# Create a tensor
some_tensor = torch.rand(3, 4)
some_tensor

# Find out details about some tensor
print(some_tensor)
print(f"Datatype of tensor: {some_tensor.dtype}")
print(f"Shape of tensor: {some_tensor.shape}")
print(f"Device tensor is on: {some_tensor.device}")

# Create a tensor and add 10 to it
tensor = torch.tensor([1, 2, 3])
tensor + 10

# Multiply tensor by 10
tensor * 10

tensor

# Substract 10
tensor - 10

# Try out PyTorch in-built functions
torch.mul(tensor, 10)

torch.add(tensor, 10)

# Element wise multiplication
print(tensor, "*", tensor)
print(f"Equals: {tensor * tensor}")

# Matrix multiplication
torch.matmul(tensor, tensor)

tensor

# Matrix multiplication by hand
1*1 + 2*2 + 3*3

%%time
value = 0
for i in range(len(tensor)):
  value += tensor[i] * tensor[i]
print(value)

%%time
torch.matmul(tensor, tensor)

# Shapes for matrix multiplication 
tensor_A = torch.tensor([[1, 2],
                         [3, 4],
                         [5, 6]])

tensor_B = torch.tensor([[7, 10],
                         [8, 11],
                         [9, 12]])

# torch.mm(tensor_A, tensor_B) # torch.mm is the same as torch.matmul (it's an alias for writing less code)
torch.matmul(tensor_A, tensor_B)

tensor_B.T

tensor_A.shape, tensor_B.shape

tensor_B, tensor_B.shape

tensor_B.T, tensor_B.T.shape

# The matrix multiplication operation works when tensor_B is transposed
print(f"Original shapes: tensor_A = {tensor_A.shape}, tensor_B = {tensor_B.shape}")
print(f"New shapes: tensor_A = {tensor_A.shape} (same shape as above), tensor_B.T = {tensor_B.T.shape}")
print(f"Multiplying: {tensor_A.shape} @ {tensor_B.T.shape} <- inner dimensions must match")
print("Output:\n")
output = torch.matmul(tensor_A, tensor_B.T)
print(output) 
print(f"\nOutput shape: {output.shape}")

# Create a tensor
x = torch.arange(1, 100, 10)
x, x.dtype

# Find the min
torch.min(x), x.min()

# Find the max
torch.max(x), x.max()

# Find the mean - note: the torch.mean() function requires a tensor of float32 datatype to work
torch.mean(x.type(torch.float32)), x.type(torch.float32).mean()

# Find the sum
torch.sum(x), x.sum()

x

# Find the position in tensor that has the minimum value with argmin() -> returns index position of targt tensor where the minimum value occurs 
x.argmin()

x[0]

# Find the position in tensor that has the maximum value with argmax()
x.argmax()

x[9]

# Let's create a tensor
import torch
x = torch.arange(1., 10.)
x, x.shape

# Add an extra dimension
x_reshaped = x.reshape(1, 9)
x_reshaped, x_reshaped.shape

# Change the view 
z = x.view(1, 9)
z, z.shape

# Changing z changes x (because a view of a tensor shares the same memory as the original input)
z[:, 0] = 5
z, x

# Stack tensors on top of each other
x_stacked = torch.stack([x, x, x, x], dim=0)
x_stacked

# torch.squeeze() - removes all single dimensions from a target tensor
print(f"Previous tensor: {x_reshaped}")
print(f"Previous shape: {x_reshaped.shape}")

# Remove extra dimensions from x_reshaped
x_squeezed = x_reshaped.squeeze()
print(f"\nNew tensor: {x_squeezed}")
print(f"New shape: {x_squeezed.shape}")

# torch.unsqueeze() - adds a single dimension to a target tensor at a specific dim (dimension)
print(f"Previous target: {x_squeezed}")
print(f"Previous shape: {x_squeezed.shape}")

# Add an extra dimension with unsqueeze
x_unsqueezed = x_squeezed.unsqueeze(dim=0)
print(f"\nNew tensor: {x_unsqueezed}")
print(f"New shape: {x_unsqueezed.shape}")

# torch.permute - rearranges the dimensions of a target tensor in a specified order
x_original = torch.rand(size=(224, 224, 3)) # [height, width, colour_channels]

# Permute the original tensor to rearrange the axis (or dim) order
x_permuted = x_original.permute(2, 0, 1) # shifts axis 0->1, 1->2, 2->0

print(f"Previous shape: {x_original.shape}") 
print(f"New shape: {x_permuted.shape}") # [colour_channels, height, width]

x_original[0, 0, 0] = 728218
x_original[0, 0, 0], x_permuted[0, 0, 0]

# Create a tensor
import torch
x = torch.arange(1, 10).reshape(1, 3, 3)
x, x.shape

# Let's index on our new tensor
x[0]

# Let's index on the middle bracket (dim=1)
x[0][0]

# Let's index on the most inner bracket (last dimension)
x[0][1][1]

# You can also use ":" to select "all" of a target dimension
x[:, 0]

# Get all values of 0th and 1st dimensions but only index 1 of 2nd dimension
x[:, :, 1]

# Get all values of the 0 dimension but only the 1 index value of 1st and 2nd dimension
x[:, 1, 1]

# Get index 0 of 0th and 1st dimension and all values of 2nd dimension
x[0, 0, :]

# Index on x to return 9
print(x[0][2][2])

# Index on x to return 3, 6, 9
print(x[:, :, 2])

# NumPy array to tensor
import torch
import numpy as np

array = np.arange(1.0, 8.0)
tensor = torch.from_numpy(array) # warning: when converting from numpy -> pytorch, pytorch reflects numpy's default datatype of float64 unless specified otherwise
array, tensor

# Change the value of array, what will this do to `tensor`?
array = array + 1
array, tensor

# Tensor to NumPy array
tensor = torch.ones(7)
numpy_tensor = tensor.numpy()
tensor, numpy_tensor

# Change the tesnor, what happens to `numpy_tensor`?
tensor = tensor + 1
tensor, numpy_tensor 

import torch

# Create two random tensors
random_tensor_A = torch.rand(3, 4)
random_tensor_B = torch.rand(3, 4)

print(random_tensor_A)
print(random_tensor_B)
print(random_tensor_A == random_tensor_B)

# Let's make some random but reproducible tensors
import torch

# Set the random seed
RANDOM_SEED = 42
torch.manual_seed(RANDOM_SEED)
random_tensor_C = torch.rand(3, 4)

torch.manual_seed(RANDOM_SEED)
random_tensor_D = torch.rand(3, 4)

print(random_tensor_C)
print(random_tensor_D)
print(random_tensor_C == random_tensor_D)

!nvidia-smi

# Check for GPU access with PyTorch
import torch
torch.cuda.is_available()

# Setup device agnostic code 
device = "cuda" if torch.cuda.is_available() else "cpu"
device

# Count number of devices
torch.cuda.device_count()

# Create a tensor (default on the CPU)
tensor = torch.tensor([1, 2, 3])

# Tensor not on GPU
print(tensor, tensor.device)

# Move tensor to GPU (if available)
tensor_on_gpu = tensor.to(device)
tensor_on_gpu

# If tensor is on GPU, can't transform it to NumPy
tensor_on_gpu.numpy()

# To fix the GPU tensor with NumPy issue, we can first set it to the CPU
tensor_back_on_cpu = tensor_on_gpu.cpu().numpy()
tensor_back_on_cpu

tensor_on_gpu