import torch
import torch.nn as nn
import numpy as np
import matplotlib.pyplot as plt

from torch.utils.data import Dataset, DataLoader, SubsetRandomSampler

class TwoMoonsDataset(Dataset):
    def __init__(self, sigma, N):
        self.sigma = sigma
        self.N = N
        self.radius = 1
        self.angle_offset = -np.pi/8
        self.positive_center = torch.tensor([0, 0])
        self.negative_center = torch.tensor([1, 0])
        # generate angles of each moon
        positive_angles = torch.rand(N)*(np.pi-2*self.angle_offset)+self.angle_offset
        negative_angles = torch.rand(N)*(np.pi-2*self.angle_offset)+self.angle_offset
        # generate each moon
        self.positive_data = self.positive_center + torch.stack((self.radius*torch.cos(positive_angles),
                                                                 self.radius*torch.sin(positive_angles)), dim=-1)
        self.positive_data = self.positive_data + torch.randn(N, 2)*sigma
        self.negative_data = self.negative_center + torch.stack((self.radius*torch.cos(-negative_angles),
                                                                 self.radius*torch.sin(-negative_angles)), dim=-1)
        self.negative_data = self.negative_data + torch.randn(N, 2)*sigma
        # wrap up all data and labels
        self.data = torch.cat((self.positive_data, self.negative_data), dim=0)
        self.labels = torch.cat((torch.ones(N), torch.zeros(N))).long()

    def __len__(self):
        return len(self.labels)

    def __getitem__(self, idx):
        return self.data[idx], self.labels[idx]

class MulticlassLogisticRegression(nn.Module):
    def __init__(self, N, M):
        super().__init__()
        self.N = N # input dimension
        self.M = M # number of classes
        self.weight_matrix = nn.Linear(N, M, bias=True) # N input dimensions, M output dimensions

    def forward(self, x):
        return self.weight_matrix(x)

def plot_model_probs(model, plus_class, negative_class):
    x = torch.linspace(-2, 3, 100)
    y = torch.linspace(-2, 2, 100)
    X, Y = torch.meshgrid(x, y, indexing='ij')
    meshgrid_inputs = torch.stack((X.flatten(), Y.flatten()), dim=1)
    with torch.no_grad():
        meshgrid_outputs = torch.softmax(model(meshgrid_inputs), dim=1)[:, 1]

    plt.figure(figsize=(8, 6))
    plt.contourf(X.numpy(), Y.numpy(), meshgrid_outputs.reshape(100, 100).numpy(), cmap='RdBu_r', levels=100)
    plt.colorbar()
    plt.title('Probability of positive class')
    plt.scatter(plus_class[:, 0].numpy(), plus_class[:, 1].numpy(), color='tomato', s=50, edgecolor='black')
    plt.scatter(negative_class[:, 0].numpy(), negative_class[:, 1].numpy(), color='cornflowerblue', s=50, edgecolor='black')
    plt.axis(False)
    plt.tight_layout()

def plot_model_probs_with_component_lines(model, plus_class, negative_class, W, b, layers):
    x = torch.linspace(-2, 3, 100)
    y = torch.linspace(-2, 2, 100)
    X, Y = torch.meshgrid(x, y, indexing='ij')
    meshgrid_inputs = torch.stack((X.flatten(), Y.flatten()), dim=1)
    with torch.no_grad():
        meshgrid_outputs = torch.softmax(model(meshgrid_inputs), dim=1)[:, 1]

    plt.figure(figsize=(8, 6))
    plt.contourf(X.numpy(), Y.numpy(), meshgrid_outputs.reshape(100, 100).numpy(), cmap='RdBu_r', levels=100)
    plt.colorbar()
    plt.title('Probability of positive class')
    plt.scatter(plus_class[:, 0].numpy(), plus_class[:, 1].numpy(), color='tomato', s=50, edgecolor='black')
    plt.scatter(negative_class[:, 0].numpy(), negative_class[:, 1].numpy(), color='cornflowerblue', s=50, edgecolor='black')
    plt.axis(False)
    plt.tight_layout()
    
    # Plot the hidden layer linear decision boundaries
    #W = model.weight_matrix1.weight.detach().numpy()  # Shape: (L, 2)
    #b = model.weight_matrix1.bias.detach().numpy()    # Shape: (L,)

    x_vals = np.linspace(-2, 3, 200)
    for i in range(layers):
        w = W[i]
        bias = b[i]
        # Line equation: w0 * x + w1 * y + b = 0 ⟹ y = (-w0 * x - b) / w1
        if np.abs(w[1]) > 1e-6:  # avoid division by zero
            y_vals = (-w[0] * x_vals - bias) / w[1]
            plt.plot(x_vals, y_vals, linestyle='--', color='black', alpha=0.5)
    
    plt.xlim(-2, 3)
    plt.ylim(-2, 2)    
    
def multiclass_model_accuracy(model, input_data, labels):
    predictions = model(input_data) # no need to squeeze/unsqueeze dimensions now!
    predicted_classes = torch.argmax(predictions, dim=1) # find highest scoring class along the columns
    n_correct = torch.sum(torch.eq(predicted_classes, labels))
    return n_correct

def print_model_params(model):
    for name, param in model.named_parameters():
        if param.requires_grad:
            print(name, param.data)


# visualize example of two moons dataset
N = 100
sigma = 0.1
dataset = TwoMoonsDataset(sigma, N)
plus_data = dataset.positive_data
negative_data = dataset.negative_data
plt.figure(figsize=(8, 8))
plt.scatter(plus_data[:, 0].numpy(), plus_data[:, 1].numpy(), color='tomato', s=50, edgecolor='black', label='Positive Class')
plt.scatter(negative_data[:, 0].numpy(), negative_data[:, 1].numpy(), color='cornflowerblue', s=50, edgecolor='black', label='Negative Class')
plt.title('Two Moons Dataset')
plt.legend()
plt.tight_layout()

# loss function, model, and optimizer
criterion = nn.CrossEntropyLoss(reduction='mean') # cross-entropy loss, use mean of loss
lr = 1e-2 # learning rate
M = 2 # two classes
N = 2 # data is two-dimensional
model = MulticlassLogisticRegression(M, N)
optimizer = torch.optim.SGD(model.parameters(), lr=lr, momentum=0.99, weight_decay=1e-3) 

# create training dataloader
batch_size = 16
train_loader = DataLoader(dataset, batch_size=batch_size, shuffle=True)

# training loop
n_epoch = 200
loss_values, accuracies = [], []
for n in range(n_epoch):
    epoch_loss, epoch_acc = 0, 0
    for x_batch, y_batch in train_loader:
        # zero out gradients
        optimizer.zero_grad()
        # pass batch to model
        predictions = model(x_batch)
        # calculate loss
        loss = criterion(predictions, y_batch)
        # backpropagate and update
        loss.backward() # backprop
        optimizer.step()
        # logging to update epoch_loss (add loss value) and epoch_acc (add current batch accuracy)
        epoch_loss += loss.item()
        epoch_acc += multiclass_model_accuracy(model, x_batch, y_batch)
    loss_values.append(epoch_loss/len(train_loader))
    accuracies.append(epoch_acc/len(dataset))
    
# plot model probabilities
plot_model_probs(model, dataset.positive_data, dataset.negative_data)
# plot loss values
plt.figure(figsize=(12,6))
plt.subplot(121)
plt.semilogy(loss_values)
plt.grid(True)
plt.title('Training Loss')
plt.xlabel('Epoch')
plt.ylabel('Loss')
plt.subplot(122)
plt.plot(accuracies)
plt.grid(True)
plt.title('Classification Accuracy')
plt.xlabel('Epoch')
plt.ylabel('Accuracy')

Text(0, 0.5, 'Accuracy')

plot_model_probs_with_component_lines(model, dataset.positive_data, dataset.negative_data, model.weight_matrix.weight.detach().numpy(), model.weight_matrix.bias.detach().numpy(), 2)
print_model_params(model)

weight_matrix.weight tensor([[ 0.8102, -1.9734],
        [-0.8240,  2.0188]])
weight_matrix.bias tensor([-0.5790,  0.5570])

class MulticlassLogisticRegression_Test1(nn.Module):
    def __init__(self):
        super().__init__()
        self.N = 2  # input dimension
        self.M = 2  # number of classes
        self.weight_matrix = nn.Linear(N, M, bias=True)

        # Set custom initial weights and biases
        custom_weights_1 = torch.tensor([[-0.6106, -2.2457],
                                       [ 0.6192,  2.2808]])
        custom_biases_1 = torch.tensor([0.2708,  0.2180])

        custom_weights_2 = torch.tensor([[0.6106, -2.2457],
                                       [ -0.6192,  2.2808]])
        custom_biases_2 = torch.tensor([-0.2708,  0.2180])

        with torch.no_grad():
            self.weight_matrix.weight.copy_(custom_weights_1)
            self.weight_matrix.bias.copy_(custom_biases_1)

    def forward(self, x):
        return self.weight_matrix(x)

model_test = MulticlassLogisticRegression_Test1()
plot_model_probs_with_component_lines(model_test, dataset.positive_data, dataset.negative_data, model_test.weight_matrix.weight.detach().numpy(), model_test.weight_matrix.bias.detach().numpy(), 2)
print_model_params(model_test)

weight_matrix.weight tensor([[-0.6106, -2.2457],
        [ 0.6192,  2.2808]])
weight_matrix.bias tensor([0.2708, 0.2180])

print(model_test(torch.tensor([1.0,1.0])))

tensor([-2.5855,  3.1180], grad_fn=<ViewBackward0>)

class MulticlassLogisticRegression_Test2(nn.Module):
    def __init__(self):
        super().__init__()
        self.N = 2  # input dimension
        self.M = 2  # number of classes
        self.weight_matrix = nn.Linear(N, 2, bias=True)
        self.weight_matrix_2 = nn.Linear(2, M, bias=True)
        
        # Set custom initial weights and biases
        custom_weights_1 = torch.tensor([[-0.6106, -2.2457],
                                         [ 0.6106, -2.2457]])
        custom_biases_1 = torch.tensor([0.2708,  -0.2708])

        # Set custom initial weights and biases
        custom_weights_2 = torch.tensor([[1, 1],
                                         [ -1, -1]])
        custom_biases_2 = torch.tensor([0.0,  0.0])

        
        with torch.no_grad():
            self.weight_matrix.weight.copy_(custom_weights_1)
            self.weight_matrix.bias.copy_(custom_biases_1)
            self.weight_matrix_2.weight.copy_(custom_weights_2)
            self.weight_matrix_2.bias.copy_(custom_biases_2)

    def forward(self, x):
        x = self.weight_matrix(x)
        x = torch.nn.functional.relu(x)
        z = self.weight_matrix_2(x)
        return z
        #return self.weight_matrix_1(x)

model_test = MulticlassLogisticRegression_Test2()
plot_model_probs_with_component_lines(model_test, dataset.positive_data, dataset.negative_data, model_test.weight_matrix.weight.detach().numpy(), model_test.weight_matrix.bias.detach().numpy(), 2)
print_model_params(model_test)

weight_matrix.weight tensor([[-0.6106, -2.2457],
        [ 0.6106, -2.2457]])
weight_matrix.bias tensor([ 0.2708, -0.2708])
weight_matrix_2.weight tensor([[ 1.,  1.],
        [-1., -1.]])
weight_matrix_2.bias tensor([0., 0.])

class MulticlassLogisticRegression_Test3(nn.Module):
    def __init__(self):
        super().__init__()
        self.N = 2  # input dimension
        self.M = 2  # number of classes
        self.weight_matrix = nn.Linear(N, 2, bias=True)
        self.weight_matrix_2 = nn.Linear(2, M, bias=True)
        
        # Set custom initial weights and biases
        custom_weights_1 = torch.tensor([[-0.6106, -2.2457],
                                         [ 0.6106, -2.2457]])
        custom_biases_1 = torch.tensor([0.2708,  -0.2708])

        # Set custom initial weights and biases
        custom_weights_2 = torch.tensor([[1, 1],
                                         [ -1, -1]])
        custom_biases_2 = torch.tensor([0.0,  0.0])

        
        with torch.no_grad():
            self.weight_matrix.weight.copy_(custom_weights_1)
            self.weight_matrix.bias.copy_(custom_biases_1)
            self.weight_matrix_2.weight.copy_(custom_weights_2)
            self.weight_matrix_2.bias.copy_(custom_biases_2)

    def forward(self, x):
        x = self.weight_matrix(x)
        #x = torch.nn.functional.relu(x)
        z = self.weight_matrix_2(x)
        return z
        #return self.weight_matrix_1(x)

model_test = MulticlassLogisticRegression_Test3()
plot_model_probs_with_component_lines(model_test, dataset.positive_data, dataset.negative_data, model_test.weight_matrix.weight.detach().numpy(), model_test.weight_matrix.bias.detach().numpy(), 2)
print_model_params(model_test)

weight_matrix.weight tensor([[-0.6106, -2.2457],
        [ 0.6106, -2.2457]])
weight_matrix.bias tensor([ 0.2708, -0.2708])
weight_matrix_2.weight tensor([[ 1.,  1.],
        [-1., -1.]])
weight_matrix_2.bias tensor([0., 0.])

class TwoLayerModel(nn.Module):
    def __init__(self, N, L, M):
        super().__init__()
        self.N = N # input dimension
        self.M = M # number of classes
        self.weight_matrix1 = nn.Linear(N, L, bias=True) # N input dimensions, L hidden dimensions
        self.weight_matrix2 = nn.Linear(L, M)

    def forward(self, x):
        x = self.weight_matrix1(x)
        x = torch.nn.functional.relu(x)
        z = self.weight_matrix2(x)
        return z

# loss function, model, and optimizer
criterion = nn.CrossEntropyLoss(reduction='mean') # cross-entropy loss, use mean of loss
lr = 1e-2 # learning rate
M = 2 # two classes
N = 2 # data is two-dimensional
L = 8 # number of hidden features
fancy_new_model = TwoLayerModel(N, L, M)
optimizer = torch.optim.SGD(fancy_new_model.parameters(), lr=lr, momentum=0.99, weight_decay=1e-3) 

# create training dataloader
batch_size = 16
train_loader = DataLoader(dataset, batch_size=batch_size, shuffle=True)

# training loop
n_epoch = 200
loss_values, accuracies = [], []
for n in range(n_epoch):
    epoch_loss, epoch_acc = 0, 0
    for x_batch, y_batch in train_loader:
        # zero out gradients
        optimizer.zero_grad()
        # pass batch to model
        predictions = fancy_new_model(x_batch)
        # calculate loss
        loss = criterion(predictions, y_batch)
        # backpropagate and update
        loss.backward() # backprop
        optimizer.step()
        # logging to update epoch_loss (add loss value) and epoch_acc (add current batch accuracy)
        epoch_loss += loss.item()
        epoch_acc += multiclass_model_accuracy(fancy_new_model, x_batch, y_batch)
    loss_values.append(epoch_loss/len(train_loader))
    accuracies.append(epoch_acc/len(dataset))
    
# plot model probabilities
plot_model_probs(fancy_new_model, dataset.positive_data, dataset.negative_data)
# plot loss values
plt.figure(figsize=(12,6))
plt.subplot(121)
plt.semilogy(loss_values)
plt.grid(True)
plt.title('Training Loss')
plt.xlabel('Epoch')
plt.ylabel('Loss')
plt.subplot(122)
plt.plot(accuracies)
plt.grid(True)
plt.title('Classification Accuracy')
plt.xlabel('Epoch')
plt.ylabel('Accuracy')

Text(0, 0.5, 'Accuracy')

plot_model_probs_with_component_lines(fancy_new_model, dataset.positive_data, dataset.negative_data, fancy_new_model.weight_matrix1.weight.detach().numpy(), fancy_new_model.weight_matrix1.bias.detach().numpy(), L)

for name, param in fancy_new_model.named_parameters():
    if param.requires_grad:
        print(name, param.data)

weight_matrix1.weight tensor([[-0.0984, -0.0491],
        [ 1.0634, -0.3411],
        [ 0.1274, -1.2722],
        [-0.0078,  0.0162],
        [ 1.9571, -0.5915],
        [-0.0313, -0.0305],
        [-3.8406,  0.6589],
        [-2.2012, -1.3838]])
weight_matrix1.bias tensor([-0.1895, -1.0570, -0.4503, -0.0288, -1.9345, -0.1282,  0.1808,  1.8989])
weight_matrix2.weight tensor([[-0.0573,  1.0909,  0.9631,  0.0124,  1.9715,  0.0703, -2.7555,  2.2609],
        [ 0.0421, -1.0758, -0.9455, -0.0158, -1.9905, -0.0833,  2.7518, -2.2542]])
weight_matrix2.bias tensor([-1.9892,  1.9645])

import torch
from torchvision import datasets, transforms
import matplotlib.pyplot as plt

# Define a transform to convert the data to tensor and normalize it
transform = transforms.ToTensor()

# Download the FashionMNIST training dataset
fashion_mnist = datasets.FashionMNIST(root='./data', train=True, download=True, transform=transform)

# Create a DataLoader to iterate through the dataset
data_loader = torch.utils.data.DataLoader(fashion_mnist, batch_size=8, shuffle=True)

# Define class labels for FashionMNIST
class_labels = [
    "T-shirt/top", "Trouser", "Pullover", "Dress", "Coat",
    "Sandal", "Shirt", "Sneaker", "Bag", "Ankle boot"
]

# Get a batch of data
images, labels = next(iter(data_loader))

# Plot the images
plt.figure(figsize=(10, 4))
for i in range(len(images)):
    plt.subplot(2, 4, i+1)
    plt.imshow(images[i].squeeze(), cmap='gray')
    plt.title(class_labels[labels[i]])
    plt.axis('off')
plt.tight_layout()
plt.show()

import torch
import torch.nn as nn

class MyMLPModel(nn.Module):
    # add hidden dimension sizes to constructor as you see fit!
    def __init__(self, input_dim, h1, output_dim, activation_fn):
        super().__init__()
        # create layers using nn.Linear(input_dimension, output_dimension) objects
        self.fc1 = nn.Linear(input_dim, h1)
        self.fc2 = nn.Linear(h1, output_dim)
        # assign non-linear activation function to class
        self.activation = activation_fn
    
    def forward(self, x):
        # implement forward pass
        x = self.fc1(x)
        x = self.activation(x)
        z = self.fc2(x)
        return z

import torchvision
import torchvision.transforms as transforms

from tqdm import tqdm

# set up data
class FashionMNIST(Dataset):
    def __init__(self):
        self.data = torchvision.datasets.FashionMNIST(root='./',
                                            download=True,
                                            train=True,
                                            transform=transforms.Compose([transforms.ToTensor()]))
        
    def __len__(self):
        return len(self.data)
    
    def __getitem__(self, idx):
        image, label = self.data[idx]
        return image.reshape(-1), label

N = 1000
N_train = 600
N_val = 200
N_test = 200
dataset = FashionMNIST()
indices = np.random.choice(np.arange(len(dataset)), size=N, replace=False)
np.random.shuffle(indices)
train_indices = indices[:N_train]
val_indices = indices[N_train:N_train+N_val]
test_indices = indices[N_train+N_val]

batch_size = 8
train_loader = DataLoader(dataset, batch_size=batch_size, sampler=SubsetRandomSampler(train_indices))
val_loader = DataLoader(dataset, batch_size=batch_size, sampler=SubsetRandomSampler(val_indices))

# initialize model
input_dim = 784 # dimension of images after being vectorized
h1 = 128
output_dim = 10 # number of classes
activation_fn = nn.ReLU()
model = MyMLPModel(input_dim, h1, output_dim, activation_fn) # fill this is based on your implementation!

# initialize loss function and optimizer
criterion = nn.CrossEntropyLoss()
lr = 1e-3
optimizer = torch.optim.SGD(model.parameters(), lr=lr, momentum=0.99, weight_decay=1e-4)

# logging info
loss_values, train_accuracies, val_accuracies = [], [], []
n_epoch = 300 # set this value
for n in tqdm(range(n_epoch)):
    epoch_loss, epoch_acc = 0, 0
    for x_batch, y_batch in train_loader:
        # zero out gradients
        optimizer.zero_grad()
        # pass batch to model, no need to worry about using squeeze/unsqueeze now
        predictions = model(x_batch)
        # calculate loss
        loss = criterion(predictions, y_batch)
        # backpropagate and update
        loss.backward() # backprop
        optimizer.step()
        # logging to update epoch_loss (add loss value) and epoch_acc (add current batch accuracy)
        epoch_loss += loss.item()
        epoch_acc += multiclass_model_accuracy(model, x_batch, y_batch)

    loss_values.append(epoch_loss/len(train_loader))
    train_accuracies.append(epoch_acc/N_train)
    # validation performance
    val_acc = 0
    for x_batch, y_batch in val_loader:
        # don't compute gradients since we are only evaluating the model
        with torch.no_grad():
            # validation batch accuracy
            val_acc += multiclass_model_accuracy(model, x_batch, y_batch)
    val_accuracies.append(val_acc/N_val)

plt.figure(figsize=(12,6))
plt.subplot(131)
plt.semilogy(loss_values)
plt.grid(True)
plt.title('Loss values')
plt.xlabel('Epoch')
plt.subplot(132)
plt.plot(train_accuracies)
plt.grid(True)
plt.title('Training Accuracy')
plt.xlabel('Epoch')
plt.subplot(133)
plt.plot(val_accuracies)
plt.grid(True)
plt.title('Validation Accuracy')
plt.xlabel('Epoch')

  0%|                                                                                                                                                                             | 0/300 [00:00<?, ?it/s]

  0%|▌                                                                                                                                                                    | 1/300 [00:00<01:20,  3.71it/s]

  1%|█                                                                                                                                                                    | 2/300 [00:00<01:15,  3.93it/s]

  1%|█▋                                                                                                                                                                   | 3/300 [00:00<01:23,  3.54it/s]

  1%|██▏                                                                                                                                                                  | 4/300 [00:01<01:58,  2.50it/s]

  2%|██▊                                                                                                                                                                  | 5/300 [00:01<02:10,  2.25it/s]

  2%|███▎                                                                                                                                                                 | 6/300 [00:02<01:55,  2.54it/s]

  2%|███▊                                                                                                                                                                 | 7/300 [00:02<01:44,  2.79it/s]

  3%|████▍                                                                                                                                                                | 8/300 [00:02<01:38,  2.97it/s]

  3%|████▉                                                                                                                                                                | 9/300 [00:03<01:35,  3.06it/s]

Lecture 14 - Multi-layer perceptrons¶

ECE364 - Programming Methods for Machine Learning¶

Nickvash Kani¶

Slides based off prior lectures by Alex Schwing, Aigou Han, Farzad Kamalabadi, Corey Snyder. All mistakes are my own!¶

Limits of Logistic Regression¶

How can we represent more complex functions?¶

What's with the ReLU?¶

Activation Functions¶

Multi-layer Perceptron¶

Larger Deepnet: Image Classification with MLP¶

There's a problem¶

That's all for today¶