Discriminative vs Generative Models

from sklearn.svm import SVC
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler
from sklearn.metrics import classification_report
import numpy as np

# Generate sample data
np.random.seed(42)
X = np.random.randn(1000, 20)  # 1000 samples, 20 features
y = (X[:, 0] + X[:, 1] > 0).astype(int)  # Simple decision boundary

# Split and scale data
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)
scaler = StandardScaler()
X_train_scaled = scaler.fit_transform(X_train)
X_test_scaled = scaler.transform(X_test)

# Train discriminative model (SVM)
svm = SVC(kernel='rbf', probability=True, random_state=42)
svm.fit(X_train_scaled, y_train)

# Predictions focus only on decision boundary
y_pred = svm.predict(X_test_scaled)
y_proba = svm.predict_proba(X_test_scaled)

print("Discriminative Model (SVM) Results:")
print(classification_report(y_test, y_pred))
print(f"Decision function range: {svm.decision_function(X_test_scaled).min():.2f} to {svm.decision_function(X_test_scaled).max():.2f}")

# Key insight: SVM learns decision boundary directly
# Does not model how data is generated, only how to separate classes

from sklearn.naive_bayes import GaussianNB
from sklearn.metrics import classification_report
import numpy as np
import matplotlib.pyplot as plt

# Same data as discriminative example
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)

# Train generative model (Naive Bayes)
nb = GaussianNB()
nb.fit(X_train, y_train)

# Predictions based on learned distributions
y_pred = nb.predict(X_test)
y_proba = nb.predict_proba(X_test)

print("Generative Model (Naive Bayes) Results:")
print(classification_report(y_test, y_pred))

# Key insight: Model learns P(X|y) for each class
print(f"\nLearned parameters for class 0:")
print(f"Mean features: {nb.theta_[0][:5]}")  # First 5 features
print(f"Variance: {nb.var_[0][:5]}")

print(f"\nLearned parameters for class 1:")
print(f"Mean features: {nb.theta_[1][:5]}")
print(f"Variance: {nb.var_[1][:5]}")

# Generate new samples from learned distribution (generative capability)
def generate_samples(nb_model, class_label, n_samples=100):
    """Generate new samples from learned Gaussian distributions"""
    mean = nb_model.theta_[class_label]
    var = nb_model.var_[class_label]
    return np.random.normal(mean, np.sqrt(var), (n_samples, len(mean)))

# Generate synthetic data for each class
synthetic_class_0 = generate_samples(nb, 0, 50)
synthetic_class_1 = generate_samples(nb, 1, 50)

print(f"\nGenerated {len(synthetic_class_0)} samples for class 0")
print(f"Generated {len(synthetic_class_1)} samples for class 1")
print("Generative models can create new data - discriminative models cannot!")

import torch
import torch.nn as nn
import torch.nn.functional as F
from torch.utils.data import DataLoader, TensorDataset

class VAE(nn.Module):
    def __init__(self, input_dim=784, hidden_dim=400, latent_dim=20):
        super(VAE, self).__init__()
        
        # Encoder network - learns P(z|x)
        self.fc1 = nn.Linear(input_dim, hidden_dim)
        self.fc21 = nn.Linear(hidden_dim, latent_dim)  # Mean
        self.fc22 = nn.Linear(hidden_dim, latent_dim)  # Log variance
        
        # Decoder network - learns P(x|z)
        self.fc3 = nn.Linear(latent_dim, hidden_dim)
        self.fc4 = nn.Linear(hidden_dim, input_dim)
        
    def encode(self, x):
        h1 = F.relu(self.fc1(x))
        return self.fc21(h1), self.fc22(h1)  # mu, logvar
    
    def reparameterize(self, mu, logvar):
        std = torch.exp(0.5 * logvar)
        eps = torch.randn_like(std)
        return mu + eps * std
    
    def decode(self, z):
        h3 = F.relu(self.fc3(z))
        return torch.sigmoid(self.fc4(h3))
    
    def forward(self, x):
        mu, logvar = self.encode(x.view(-1, 784))
        z = self.reparameterize(mu, logvar)
        return self.decode(z), mu, logvar

def loss_function(recon_x, x, mu, logvar):
    # Reconstruction loss + KL divergence
    BCE = F.binary_cross_entropy(recon_x, x.view(-1, 784), reduction='sum')
    KLD = -0.5 * torch.sum(1 + logvar - mu.pow(2) - logvar.exp())
    return BCE + KLD

# Training loop
model = VAE()
optimizer = torch.optim.Adam(model.parameters(), lr=1e-3)

def train_vae(model, dataloader, epochs=10):
    model.train()
    for epoch in range(epochs):
        train_loss = 0
        for batch_idx, (data, _) in enumerate(dataloader):
            optimizer.zero_grad()
            recon_batch, mu, logvar = model(data)
            loss = loss_function(recon_batch, data, mu, logvar)
            loss.backward()
            optimizer.step()
            train_loss += loss.item()
        
        print(f'Epoch {epoch}: Average loss: {train_loss / len(dataloader.dataset):.4f}')

# Key generative capabilities:
def generate_new_samples(model, num_samples=64):
    """Generate completely new samples from the learned distribution"""
    model.eval()
    with torch.no_grad():
        # Sample from standard normal distribution
        z = torch.randn(num_samples, 20)  # latent_dim = 20
        samples = model.decode(z)
        return samples

# Generate new data
new_samples = generate_new_samples(model, 10)
print(f"Generated {len(new_samples)} new samples from learned distribution")
print("This is the key power of generative models!")