Training AlexNet faster: mixed precision, gradient accumulation and checkpointing

Illustrative image: A convolutional neural network: an input image passes through convolution and pooling layers to a classification output; illustrative of the deep learning approach AlexNet popularised. Image: Irisbox, CC BY 4.0, via Wikimedia Commons.
Here is the upgraded training script combining Automatic Mixed Precision (AMP / FP16), Gradient Accumulation (to simulate larger batch sizes on smaller GPUs), and Smart Model Checkpointing that saves only your best validation model.
1. Hardware Tuning & Speedup Guide
Choose your settings based on the hardware you are running:
| Hardware Environment | Recommended batch_size | accum_steps | Effective Batch Size | Mixed Precision (use_amp) |
|---|---|---|---|---|
| Colab Free / Entry GPU (T4, RTX 3060 — 8–16 GB VRAM) | 128 | 1 | 128 | True (FP16) |
| Low-VRAM GPU (GTX 1650, RTX 2060 — 4–6 GB VRAM) | 32 | 4 | 128 | True (FP16) |
| Workstation / Cloud GPU (A100, RTX 4090 — 24+ GB VRAM) | 256 | 1 | 256 | True (BF16/FP16) |
| CPU Only | 32 | 2 | 64 | False (Disable AMP) |
- How Mixed Precision (FP16) helps: PyTorch's torch.amp runs convolutions and matrix multiplications in 16-bit floating point while keeping critical accumulations in 32-bit. This cuts GPU memory usage nearly in half and speeds up training by 1.5× to 2.5× on NVIDIA Tensor Core GPUs.
- How Gradient Accumulation helps: If batch_size=128 triggers an OutOfMemoryError, set batch_size=32 and accum_steps=4. The model accumulates gradients across four mini-batches before updating weights, giving you the exact training stability of a 128-image batch using a quarter of the VRAM.
2. Complete Script with AMP, Accumulation & Checkpointing
import os
import torch
import torch.nn as nn
import torch.optim as optim
from torch.utils.data import DataLoader
from torchvision import datasets, transforms
# ==========================================
# 1. Modernized AlexNet with BatchNorm
# ==========================================
class ModernAlexNet(nn.Module):
def __init__(self, num_classes: int = 10, dropout: float = 0.5):
super().__init__()
self.features = nn.Sequential(
nn.Conv2d(3, 64, kernel_size=11, stride=4, padding=2, bias=False),
nn.BatchNorm2d(64),
nn.ReLU(inplace=True),
nn.MaxPool2d(kernel_size=3, stride=2),
nn.Conv2d(64, 192, kernel_size=5, padding=2, bias=False),
nn.BatchNorm2d(192),
nn.ReLU(inplace=True),
nn.MaxPool2d(kernel_size=3, stride=2),
nn.Conv2d(192, 384, kernel_size=3, padding=1, bias=False),
nn.BatchNorm2d(384),
nn.ReLU(inplace=True),
nn.Conv2d(384, 256, kernel_size=3, padding=1, bias=False),
nn.BatchNorm2d(256),
nn.ReLU(inplace=True),
nn.Conv2d(256, 256, kernel_size=3, padding=1, bias=False),
nn.BatchNorm2d(256),
nn.ReLU(inplace=True),
nn.MaxPool2d(kernel_size=3, stride=2),
)
self.avgpool = nn.AdaptiveAvgPool2d((6, 6))
self.classifier = nn.Sequential(
nn.Dropout(p=dropout),
nn.Linear(256 * 6 * 6, 4096, bias=False),
nn.BatchNorm1d(4096),
nn.ReLU(inplace=True),
nn.Dropout(p=dropout),
nn.Linear(4096, 4096, bias=False),
nn.BatchNorm1d(4096),
nn.ReLU(inplace=True),
nn.Linear(4096, num_classes),
)
def forward(self, x: torch.Tensor) -> torch.Tensor:
x = self.features(x)
x = self.avgpool(x)
x = torch.flatten(x, 1)
return self.classifier(x)
# ==========================================
# 2. Data Pipeline (Safe for BatchNorm1d)
# ==========================================
def get_dataloaders(dataset_type: str = "cifar10", data_dir: str = "./data", batch_size: int = 128):
norm_mean = [0.485, 0.456, 0.406]
norm_std = [0.229, 0.224, 0.225]
train_transforms = transforms.Compose([
transforms.Resize((256, 256)),
transforms.RandomCrop((224, 224)),
transforms.RandomHorizontalFlip(p=0.5),
transforms.ColorJitter(brightness=0.2, contrast=0.2, saturation=0.2),
transforms.ToTensor(),
transforms.Normalize(mean=norm_mean, std=norm_std),
])
val_transforms = transforms.Compose([
transforms.Resize((256, 256)),
transforms.CenterCrop((224, 224)),
transforms.ToTensor(),
transforms.Normalize(mean=norm_mean, std=norm_std),
])
if dataset_type == "cifar10":
train_dataset = datasets.CIFAR10(root=data_dir, train=True, download=True, transform=train_transforms)
val_dataset = datasets.CIFAR10(root=data_dir, train=False, download=True, transform=val_transforms)
num_classes = 10
elif dataset_type == "folder":
train_dataset = datasets.ImageFolder(root=os.path.join(data_dir, "train"), transform=train_transforms)
val_dataset = datasets.ImageFolder(root=os.path.join(data_dir, "val"), transform=val_transforms)
num_classes = len(train_dataset.classes)
else:
raise ValueError("dataset_type must be 'cifar10' or 'folder'")
# drop_last=True prevents a final batch of size 1 from crashing BatchNorm1d
num_workers = min(4, os.cpu_count() or 1)
train_loader = DataLoader(
train_dataset, batch_size=batch_size, shuffle=True,
num_workers=num_workers, pin_memory=True, drop_last=True
)
val_loader = DataLoader(
val_dataset, batch_size=batch_size, shuffle=False,
num_workers=num_workers, pin_memory=True
)
return train_loader, val_loader, num_classes
# ==========================================
# 3. AMP Training & Checkpointing Utilities
# ==========================================
def train_one_epoch(model, loader, criterion, optimizer, scaler, device, use_amp: bool, accum_steps: int):
model.train()
running_loss, correct, total = 0.0, 0, 0
optimizer.zero_grad(set_to_none=True)
for step, (images, labels) in enumerate(loader):
images = images.to(device, non_blocking=True)
labels = labels.to(device, non_blocking=True)
# Forward pass in mixed precision (FP16)
with torch.amp.autocast(device_type=device.type, enabled=use_amp):
outputs = model(images)
loss = criterion(outputs, labels) / accum_steps
# Scaled backward pass to prevent FP16 gradient underflow
scaler.scale(loss).backward()
# Update weights every `accum_steps` batches
if (step + 1) % accum_steps == 0 or (step + 1) == len(loader):
scaler.step(optimizer)
scaler.update()
optimizer.zero_grad(set_to_none=True)
running_loss += (loss.item() * accum_steps) * images.size(0)
_, predicted = outputs.max(1)
total += labels.size(0)
correct += predicted.eq(labels).sum().item()
return running_loss / total, 100.0 * correct / total
@torch.no_grad()
def evaluate(model, loader, criterion, device, use_amp: bool):
model.eval()
running_loss, correct, total = 0.0, 0, 0
for images, labels in loader:
images = images.to(device, non_blocking=True)
labels = labels.to(device, non_blocking=True)
with torch.amp.autocast(device_type=device.type, enabled=use_amp):
outputs = model(images)
loss = criterion(outputs, labels)
running_loss += loss.item() * images.size(0)
_, predicted = outputs.max(1)
total += labels.size(0)
correct += predicted.eq(labels).sum().item()
return running_loss / total, 100.0 * correct / total
def save_checkpoint(filepath, epoch, model, optimizer, scheduler, scaler, best_acc):
os.makedirs(os.path.dirname(filepath), exist_ok=True)
torch.save({
"epoch": epoch,
"model_state_dict": model.state_dict(),
"optimizer_state_dict": optimizer.state_dict(),
"scheduler_state_dict": scheduler.state_dict(),
"scaler_state_dict": scaler.state_dict(),
"best_acc": best_acc,
}, filepath)
def load_checkpoint(filepath, model, optimizer=None, scheduler=None, scaler=None, device="cpu"):
checkpoint = torch.load(filepath, map_location=device, weights_only=True)
model.load_state_dict(checkpoint["model_state_dict"])
if optimizer and "optimizer_state_dict" in checkpoint:
optimizer.load_state_dict(checkpoint["optimizer_state_dict"])
if scheduler and "scheduler_state_dict" in checkpoint:
scheduler.load_state_dict(checkpoint["scheduler_state_dict"])
if scaler and "scaler_state_dict" in checkpoint:
scaler.load_state_dict(checkpoint["scaler_state_dict"])
return checkpoint.get("epoch", 0), checkpoint.get("best_acc", 0.0)
# ==========================================
# 4. Main Execution Loop
# ==========================================
if __name__ == "__main__":
# --- Configuration ---
BATCH_SIZE = 128 # Lower to 32 if running on <6GB VRAM
ACCUM_STEPS = 1 # Increase to 4 if BATCH_SIZE=32 (keeps effective batch=128)
EPOCHS = 15
LEARNING_RATE = 1e-3
CHECKPOINT_DIR = "./checkpoints"
RESUME_PATH = None # Set to "./checkpoints/last_model.pt" to resume interrupted runs
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
use_amp = (device.type == "cuda")
# Enables cuDNN auto-tuner for fixed 224x224 input sizes (extra speed boost)
if device.type == "cuda":
torch.backends.cudnn.benchmark = True
train_loader, val_loader, num_classes = get_dataloaders(
dataset_type="cifar10", data_dir="./data", batch_size=BATCH_SIZE
)
model = ModernAlexNet(num_classes=num_classes).to(device)
criterion = nn.CrossEntropyLoss(label_smoothing=0.1)
optimizer = optim.AdamW(model.parameters(), lr=LEARNING_RATE, weight_decay=1e-2)
scheduler = optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=EPOCHS)
scaler = torch.amp.GradScaler(device=device.type, enabled=use_amp)
start_epoch = 1
best_val_acc = 0.0
# Resume from checkpoint if specified
if RESUME_PATH and os.path.exists(RESUME_PATH):
last_epoch, best_val_acc = load_checkpoint(
RESUME_PATH, model, optimizer, scheduler, scaler, device
)
start_epoch = last_epoch + 1
print(f"Resumed from epoch {last_epoch} (Best Val Acc: {best_val_acc:.2f}%)")
print(f"Training on {device} | AMP Enabled: {use_amp} | Effective Batch Size: {BATCH_SIZE * ACCUM_STEPS}")
for epoch in range(start_epoch, EPOCHS + 1):
train_loss, train_acc = train_one_epoch(
model, train_loader, criterion, optimizer, scaler, device, use_amp, ACCUM_STEPS
)
val_loss, val_acc = evaluate(model, val_loader, criterion, device, use_amp)
scheduler.step()
# Always save latest state (protects against Colab timeouts or crashes)
save_checkpoint(
os.path.join(CHECKPOINT_DIR, "last_model.pt"),
epoch, model, optimizer, scheduler, scaler, best_val_acc
)
# Save separate file whenever validation accuracy improves
saved_flag = ""
if val_acc > best_val_acc:
best_val_acc = val_acc
save_checkpoint(
os.path.join(CHECKPOINT_DIR, "best_model.pt"),
epoch, model, optimizer, scheduler, scaler, best_val_acc
)
saved_flag = " [Saved Best]"
print(f"Epoch [{epoch:02d}/{EPOCHS}] "
f"Train Loss: {train_loss:.4f} | Train Acc: {train_acc:.2f}% | "
f"Val Loss: {val_loss:.4f} | Val Acc: {val_acc:.2f}%{saved_flag}")3. Key Optimisations Included
- Dual Checkpointing (best_model.pt & last_model.pt). last_model.pt saves your exact optimizer, scheduler, and FP16 scaler states every epoch so you can resume seamlessly after a crash or Google Colab disconnect. best_model.pt preserves the weights from whichever epoch achieved the highest validation accuracy.
- Label Smoothing (label_smoothing=0.1). Prevents AlexNet's large fully connected layers from becoming overconfident on noisy training labels, improving generalisation.
- Memory & Speed Tweaks. optimizer.zero_grad(set_to_none=True) reduces memory writes compared to zeroing tensors, and torch.backends.cudnn.benchmark = True selects the fastest convolution algorithms for your fixed 224×224 input shape.