View file src/colab/scratchpad_lenet.py - Download
# -*- coding: utf-8 -*-
"""scratchpad_lenet.ipynb
Automatically generated by Colab.
Original file is located at
https://colab.research.google.com/drive/1D99JxMvPlB9SsOuyZmrFn1A0VGUzlQA_
Source : https://colab.research.google.com/drive/1d2AM2f7o4BmA5ikKfIwtvATmug-uStEW
"""
import torch
import torch.nn as nn
import torch.optim as optim
from torchvision import datasets, transforms
from torch.utils.data import DataLoader, random_split
transform = transforms.Compose([transforms.ToTensor()])
# this is done using 0.1307 as mean intensity of the gray channel
# and 0.3081 as the standard deviation of the intensity of the gray channel
#transforms.Normalize((0.1307), (0.3081))])
# download dataset from the internet (PyTorch repo)
dataset = datasets.MNIST(root='./data', train=True, download=True, transform=transform)
len(dataset)
dummy_dataloader = DataLoader(dataset, batch_size = 60000, shuffle=False)
images, labels = next(iter(dummy_dataloader))
mean_intensity = images.mean()
mean_intensity
standard_deviation_intensity = images.std()
standard_deviation_intensity
images.shape
labels.shape
images.squeeze().shape
import matplotlib.pyplot as plt
# @title Dataset Visualization {run:'auto'}
slider_value = 49368 # @param {type: "slider", min: 0, max: 59999}
print(f'Label = {labels[slider_value]}')
plt.imshow(images.squeeze()[slider_value, :, :]);
"""## Train Set, Validation Set, and Test Set
The validation set is also called a 'dev' set (as in development set), because it is the one that we use to select the best settings of the model
"""
# device configuration
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
device
torch.cuda.is_available()
!nvidia-smi
# the purpose of the normalization is to keep values in a range close to -1 to 1
# this makes gradient descent easier
transform = transforms.Compose([
transforms.ToTensor(),
transforms.Pad(padding=2, fill=0, padding_mode='constant'),
transforms.Normalize((mean_intensity),
(standard_deviation_intensity),)
])
data = datasets.MNIST(root='./data', train=True, download=False,transform=transform )
data
!ls './data'
!ls './data/MNIST/raw/t10k-images-idx3-ubyte'
# this dataset is for us to geet a final benchmark
# this is for us to know how would the model perform 'in the real world'
test_data =datasets.MNIST(root='./data', train=False, transform=transform)
test_data
train_size = int(len(data) * 0.7)
train_size
val_size = len(data) - train_size
val_size
generator1 = torch.Generator().manual_seed(42)
# this makes the split reproducible, it will always be the
# same training data and the same validation data
train_data, val_data = random_split(data, [train_size, val_size], generator1)
train_data[0][0].squeeze()[:,14]
batch_size=256
# shuffle = True is important on the training set, not on the validation
# or test sets
train_loader = DataLoader(train_data, batch_size=batch_size, shuffle=True)
images, labels = next(iter(train_loader))
images.mean(), images.max(), images.min()
len(train_loader)
len(train_loader) * batch_size
val_loader = DataLoader(val_data, batch_size=batch_size, shuffle=False)
test_loader = DataLoader(test_data, batch_size=batch_size, shuffle=False)
import torch.nn as nn
import torch.nn.functional as f
class Model(nn.Module):
def __init__(self):
super().__init__()
self.conv1 = nn.Conv2d(1, 20, kernel_size=(5,5), stride=(1, 1))
#self.relu1 = nn.ReLU() # this is the simplest non-linear activation function
# that we can use to learn something 'useful'
self.conv2 = nn.Conv2d(20, 20, 5)
#self.relu2 = nn.ReLU()
def forward(self, x):
x = f.relu(self.conv1(x))
return f.relu(self.conv2(x))
Model()
f.relu(torch.tensor(-1.))
?nn.Conv2d
""""""
# this is inheritance in Python
# the class LeNet inherits all methods
# and all attributes that are defined in nn.Module
import torch.nn.functional as f
class LeNet5(nn.Module):
def __init__(self):
super(LeNet5, self).__init__()
self.C1 = nn.Conv2d(1, 6, kernel_size=(5,5), padding=(0, 0), stride=(1, 1))
self.S2 = nn.Conv2d(6, 6, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1))
self.C3 = nn.Conv2d(6, 16,kernel_size=(5,5),stride=(1,1), padding=(0, 0))
self.S4 = nn.Conv2d(16, 16,kernel_size=(3,3),stride=(2,2), padding=(1,1))
self.C5 = nn.Linear(400, 120, bias=True)
self.F6 = nn.Linear(120, 84, bias=True)
self.OUTPUT = nn.Linear(84, 10, bias=True)
def forward(self, x):
# this defines a breakpoint for pdb
# import pdb; pdb.set_trace()
x = f.relu(self.C1(x))
x = f.relu(self.S2(x))
x = f.relu(self.C3(x))
x = f.relu(self.S4(x))
x = f.relu(self.C5(x.flatten()))
x = f.relu(self.F6(x))
x = f.softmax(self.OUTPUT(x))
return x
an_instance_of_lenet5 = LeNet5()
an_instance_of_lenet5
output = an_instance_of_lenet5(images[0, :, :, :])
print(output.shape)
print(output)
print(torch.sum(output))
dir(an_instance_of_lenet5.C1)
an_instance_of_lenet5.C1._parameters
# an_instance_of_lenet5.forward(images)
"""---------------------------------------------------------------------------
RuntimeError Traceback (most recent call last)
in | ()
----> 1 an_instance_of_lenet5.forward(images)
3 frames
/usr/local/lib/python3.10/dist-packages/torch/nn/modules/linear.py in forward(self, input)
114
115 def forward(self, input: Tensor) -> Tensor:
--> 116 return F.linear(input, self.weight, self.bias)
117
118 def extra_repr(self) -> str:
RuntimeError: mat1 and mat2 shapes cannot be multiplied (1x102400 and 400x120)
"""
images.shape
an_instance_of_lenet5.OUTPUT = nn.Linear(84, 26)
an_instance_of_lenet5
another_instance_of_lenet5 = LeNet5()
another_instance_of_lenet5
# Only the convolutions, for us to understand the diagram
C1 = nn.Conv2d(1, 6, (5, 5), stride=(1, 1), padding=(0, 0))
output_C1 = C1(images[0, : , :, :])
output_C1.shape
S2 = nn.Conv2d(6, 6, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1))
output_S2 = S2(output_C1)
output_S2.shape
C3 = nn.Conv2d(6, 16, kernel_size=(5, 5),padding=(0, 0), stride=(1, 1))
output_C3 = C3(output_S2)
output_C3.shape
S4 = nn.Conv2d(16, 16, kernel_size=(3, 3), stride=(2,2), padding=(1, 1))
output_S4 = S4(output_C3)
output_S4.shape
output_S4.flatten().size()
#print(output_S4.view(output_S4.size(0), -1).shape)
C5 = nn.Linear(400, 120)
output_C5 = C5(output_S4.flatten())
output_C5.shape
F6 = nn.Linear(120, 84)
output_F6 = F6(output_C5)
output_F6.shape
OUTPUT = nn.Linear(84, 10)
final_output = OUTPUT(output_F6)
final_output.shape
C5
output_S4.view(1, 16*5*5).shape
output_S4.size(0)
output_S4.size()
# 1 in the first position specifies the batch size
output_S4.view(1, -1).shape # the -1 is a wildcard
C5(output_S4.view(1, -1)).shape
images.shape
plt.imshow(output_C1.detach()[0, :, :], cmap='gray');
plt.imshow(output_C1.detach()[1, :, :], cmap='gray');
plt.imshow(output_C1.detach()[2, :, :], cmap='gray');
plt.imshow(output_C1.detach()[3, :, :], cmap='gray');
# @title First Convolution Visualization {run:'auto'}
slider_value = 0 # @param {type: "slider", min: 0, max: 5}
plt.imshow(output_C1.detach().squeeze()[slider_value, :, :], cmap='gray');
"""# Homework
Rewrite the LeNet5 so that it runs using CrossEntropyLoss (we need to change the final activation) and put it into a training loop that monitors train and validation loss. Use ChatGPT as help here.
https://chat.openai.com/share/309c3e50-53dc-4020-a69c-8c99d5da0592
"""
# this is inheritance in Python
# the class LeNet inherits all methods
# and all attributes that are defined in nn.Module
import torch.nn.functional as f
class LeNet5(nn.Module):
def __init__(self):
super(LeNet5, self).__init__()
self.C1 = nn.Conv2d(1, 6, kernel_size=(5,5), padding=(0, 0), stride=(1, 1))
self.S2 = nn.Conv2d(6, 6, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1))
self.C3 = nn.Conv2d(6, 16,kernel_size=(5,5),stride=(1,1), padding=(0, 0))
self.S4 = nn.Conv2d(16, 16,kernel_size=(3,3),stride=(2,2), padding=(1,1))
self.C5 = nn.Linear(400, 120, bias=True)
self.F6 = nn.Linear(120, 84, bias=True)
self.OUTPUT = nn.Linear(84, 10, bias=True)
def forward(self, x):
# this defines a breakpoint for pdb
# import pdb; pdb.set_trace()
x = f.relu(self.C1(x))
x = f.relu(self.S2(x))
x = f.relu(self.C3(x))
x = f.relu(self.S4(x))
x = x.view(-1, 400) # Flatten the tensor for the fully connected layer
x = f.relu(self.C5(x))
x = f.relu(self.F6(x))
# x = f.softmax(self.OUTPUT(x))
x = self.OUTPUT(x)
return x
# Initialize model, loss, and optimizer
model = LeNet5()
criterion = nn.CrossEntropyLoss() # Cross-entropy loss
optimizer = optim.Adam(model.parameters(), lr=0.001) # Adam optimizer
# optimizer = torch.optim.SGD(model.parameters(), lr=0.001)
print(f"images:{images.shape}")
input = images[0, :, :, :]
print(f"input:{input.shape}")
output = model(images[0, :, :, :])
print(f"output:{output.shape}")
print(output)
print(torch.sum(output))
output = model(images)
print(output.shape)
# Assuming you have `train_loader` which loads your data
def train(model, train_loader, criterion, optimizer, epochs=10):
# model.train() # Set the model to training mode
for epoch in range(epochs):
print(f"Epoch {epoch}")
print("Training :")
model.train()
total_loss = 0
for batch_idx, (data, target) in enumerate(train_loader):
optimizer.zero_grad() # Clear gradients
# print(f"data:{data.shape}")
output = model(data) # Forward pass
loss = criterion(output, target) # Compute loss
total_loss = total_loss + loss
loss.backward() # Backpropagation
optimizer.step() # Update weights
if batch_idx % 10 == 0:
print(f' Batch {batch_idx:3}, Loss: {loss.item()}')
print(f"Total loss: {total_loss}")
print("Validation :")
model.eval()
with torch.no_grad():
total_loss = 0
correct = 0
incorrect = 0
for batch_idx, (data, target) in enumerate(val_loader):
output = model(data) # Forward pass
loss = criterion(output, target) # Compute loss
total_loss = total_loss + loss
correct += (output.argmax(1) == target).type(torch.float32).sum().item()
incorrect += (output.argmax(1) != target).type(torch.float32).sum().item()
if batch_idx % 10 == 0:
print(f' Batch {batch_idx:3}, Loss: {loss.item()}')
print(f"Total loss: {total_loss}")
accuracy = correct / (correct + incorrect)
print(f"correct: {correct}\nincorrect: {incorrect}\naccuracy: {accuracy*100:6.3}%")
# print(f"Accuracy: {(100*correct):>0.1f}%")
print("")
# Example usage
# train(model, train_loader, criterion, optimizer)
train(model, train_loader, criterion, optimizer, epochs=5) |