YouTip

PyTorch Model Evaluation and Debugging \\n\\nThe training and optimization of deep learning models require systematic evaluation and debugging methods. This section introduces how to analyze Loss curves, identify overfitting and underfitting, use confusion matrices to evaluate classification models, and common debugging techniques.\\n\\n* * *\\n\\n## 1. Loss Function Analysis\\n\\nThe loss function is the core metric during the training process, reflecting the gap between the model's predictions and the true values. Correctly analyzing the Loss curve is key to debugging a model.\\n\\n### 1.1 Training vs. Validation Loss Comparison\\n\\nBy comparing the training Loss and validation Loss, you can determine the state of the model:\\n\\n* Training Loss decreases, Validation Loss decreases: Normal, indicating the model is learning\\n* Training Loss decreases, Validation Loss stabilizes or increases: Overfitting, requires regularization\\n* Training Loss does not decrease: Learning rate issue or model architecture issue\\n* Training Loss oscillates: Learning rate is too large or batch size is too small\\n\\n### 1.2 Loss Curve Visualization\\n\\n## Instance\\n\\nimport matplotlib.pyplot as plt\\n\\nimport numpy as np\\n\\ndef plot_training_history(history):\\n\\n"""\\n\\n Plot Training History\\n\\n history: Include 'train_loss', 'val_loss', 'train_acc', 'val_acc' dictionary of\\n\\n """\\n\\n fig, axes = plt.subplots(1,2, figsize=(14,5))\\n\\n# Loss Curve\\n\\n axes.plot(history['train_loss'], label='Train Loss', alpha=0.8)\\n\\n axes.plot(history['val_loss'], label='Val Loss', alpha=0.8)\\n\\n axes.set_xlabel('Epoch')\\n\\n axes.set_ylabel('Loss')\\n\\n axes.set_title('Training and Validation Loss')\\n\\n axes.legend()\\n\\n axes.grid(True, alpha=0.3)\\n\\n# Accuracy Curve\\n\\n axes.plot(history['train_acc'], label='Train Acc', alpha=0.8)\\n\\n axes.plot(history['val_acc'], label='Val Acc', alpha=0.8)\\n\\n axes.set_xlabel('Epoch')\\n\\n axes.set_ylabel('Accuracy')\\n\\n axes.set_title('Training and Validation Accuracy')\\n\\n axes.legend()\\n\\n axes.grid(True, alpha=0.3)\\n\\nplt.tight_layout()\\n\\n plt.show()\\n\\n# Simulate Training Data\\n\\n history ={\\n\\n'train_loss': np.linspace(2.0,0.2,50) + np.random.normal(0,0.05,50),\\n\\n'val_loss': np.linspace(2.0,0.3,50) + np.random.normal(0,0.08,50),\\n\\n'train_acc': np.linspace(0.3,0.95,50),\\n\\n'val_acc': np.linspace(0.3,0.88,50),\\n\\n}\\n\\nplot_training_history(history)\\n\\n### 1.3 Identifying Overfitting\\n\\n> When the validation Loss starts to increase while the training Loss continues to decrease, that is the beginning of overfitting. At this point, the model starts "memorizing" the training data rather than learning general patterns.\\n\\n## Instance\\n\\nimport matplotlib.pyplot as plt\\n\\nimport numpy as np\\n\\ndef detect_overfitting(history, patience=5):\\n\\n"""\\n\\n Detect Overfitting\\n\\n patience: Stop training after how many consecutive epochs of increasing validation loss\\n\\n """\\n\\n val_loss = history['val_loss']\\n\\n best_loss =float('inf')\\n\\n best_epoch =0\\n\\n early_stop_epoch =None\\n\\nfor epoch in range(patience,len(val_loss)):\\n\\n# Check the recent patience epochs\\n\\n recent_losses = val_loss[epoch - patience:epoch]\\n\\n current_loss = val_loss\\n\\n# If the current loss is higher than the recent minimum loss\\n\\nif min(recent_losses)< current_loss:\\n\\n early_stop_epoch = epoch\\n\\nbreak\\n\\nif current_loss < best_loss:\\n\\n best_loss = current_loss\\n\\n best_epoch = epoch\\n\\nif early_stop_epoch:\\n\\nprint(f"Overfitting detected! Best epoch: {best_epoch}, Should be at epoch {early_stop_epoch} Stop")\\n\\nelse:\\n\\nprint("No overfitting detected")\\n\\nreturn early_stop_epoch, best_epoch\\n\\n# Overfitting Example\\n\\n overfitting_history ={\\n\\n'train_loss': np.linspace(1.5,0.1,30),\\n\\n'val_loss': np.concatenate([np.linspace(1.5,0.3,15), np.linspace(0.3,0.8,15)]),\\n\\n}\\n\\ndetect_overfitting(overfitting_history)\\n\\n* * *\\n\\n## 2. Overfitting and Underfitting Handling\\n\\n### 2.1 Overfitting Solutions\\n\\n| Method | Description | Code Implementation |\\n| --- | --- | --- |\\n| Increase data volume | Collect more training data | Data augmentation |\\n| Dropout | Randomly drop neurons | `nn.Dropout(0.5)` |\\n| Weight decay | L2 regularization | `weight_decay=1e-4` |\\n| Early Stopping | Stop training when validation loss stops decreasing | patience parameter |\\n| Reduce model complexity | Reduce the number of layers or neurons | Decrease hidden_size |\\n| Data augmentation | Random transformations to increase data diversity | Rotation, flipping, cropping |\\n\\n### 2.2 Underfitting Solutions\\n\\n| Method | Description |\\n| --- | --- |\\n| Increase model complexity | Increase the number of layers or neurons |\\n| Train longer | Increase the number of training epochs |\\n| Reduce regularization | Lower Dropout or weight_decay |\\n| Adjust learning rate | Try a larger learning rate |\\n\\n### 2.3 Early Stopping Implementation\\n\\n## Instance\\n\\nimport torch\\n\\nimport torch.nn as nn\\n\\nimport torch.optim as optim\\n\\nclass EarlyStopping:\\n\\n"""\\n\\n Early Stopping Mechanism: Stop training when validation loss consecutively fails to decrease\\n\\n """\\n\\ndef __init__ (self, patience=7, min_delta=0, mode='min'):\\n\\n"""\\n\\n patience: Stop after how many consecutive epochs without improvement\\n\\n min_delta: Minimum change considered as improvement\\n\\n mode: 'min' or 'max'，Whether lower or higher metric is better\\n\\n """\\n\\nself.patience= patience\\n\\nself.min_delta= min_delta\\n\\nself.mode= mode\\n\\nself.counter=0\\n\\nself.best_score=None\\n\\nself.early_stop=False\\n\\ndef __call__ (self, val_loss):\\n\\n score = -val_loss if self.mode=='min'else val_loss\\n\\nif self.best_score is None:\\n\\nself.best_score= score\\n\\nelif score =self.patience:\\n\\nself.early_stop=True\\n\\nelse:\\n\\nself.best_score= score\\n\\nself.counter=0\\n\\nreturn self.early_stop\\n\\n# Use Early Stopping\\n\\ndef train_with_early_stopping(model, train_loader, val_loader, patience=7):\\n\\n device = torch.device("cuda"if torch.cuda.is_available()else"cpu")\\n\\n model = model.to(device)\\n\\ncriterion = nn.CrossEntropyLoss()\\n\\n optimizer = optim.Adam(model.parameters(), lr=1e-3)\\n\\n early_stopping = EarlyStopping(patience=patience, mode='min')\\n\\nbest_val_loss =float('inf')\\n\\n best_model_state =None\\n\\nfor epoch in range(100):\\n\\n# Training\\n\\n model.train()\\n\\n train_loss =0\\n\\nfor inputs, labels in train_loader:\\n\\n inputs, labels = inputs.to(device), labels.to(device)\\n\\n optimizer.zero_grad()\\n\\n outputs = model(inputs)\\n\\n loss = criterion(outputs, labels)\\n\\n loss.backward()\\n\\n optimizer.step()\\n\\n train_loss += loss.item()\\n\\n# validation\\n\\n model.eval()\\n\\n val_loss =0\\n\\nwith torch.no_grad():\\n\\nfor inputs, labels in val_loader:\\n\\n inputs, labels = inputs.to(device), labels.to(device)\\n\\n outputs = model(inputs)\\n\\n loss = criterion(outputs, labels)\\n\\n val_loss += loss.item()\\n\\nval_loss /=len(val_loader)\\n\\nprint(f"Epoch {epoch+1}: Train Loss={train_loss:.4f}, Val Loss={val_loss:.4f}")\\n\\n# Early Stopping Check\\n\\nif early_stopping(val_loss):\\n\\nprint(f"Early stopping triggered! Consecutive {patience} items epoch validation loss Not decreased")\\n\\n# Save Best Model\\n\\nif val_loss < best_val_loss:\\n\\n best_val_loss = val_loss\\n\\n best_model_state = model.state_dict().copy()\\n\\n# Restore Best Model\\n\\n model.load_state_dict(best_model_state)\\n\\nreturn model\\n\\n* * *\\n\\n## 3. Confusion Matrix\\n\\nThe confusion matrix is an important tool for evaluating classification models; it shows the relationship between the model's predictions and the true labels.\\n\\n### 3.1 Confusion Matrix Calculation\\n\\n## Instance\\n\\nimport torch\\n\\nimport numpy as np\\n\\nfrom sklearn.metrics import confusion_matrix\\n\\nimport matplotlib.pyplot as plt\\n\\nimport seaborn as sns\\n\\ndef calculate_confusion_matrix(model, dataloader, device, num_classes):\\n\\n"""\\n\\n Compute Confusion Matrix\\n\\n """\\n\\n model.eval()\\n\\n all_preds =[]\\n\\n all_labels =[]\\n\\nwith torch.no_grad():\\n\\nfor inputs, labels in dataloader:\\n\\n inputs = inputs.to(device)\\n\\n outputs = model(inputs)\\n\\n _, preds = torch.max(outputs,1)\\n\\nall_preds.extend(preds.cpu().numpy())\\n\\n all_labels.extend(labels.numpy())\\n\\ncm = confusion_matrix(all_labels, all_preds)\\n\\nreturn cm\\n\\ndef plot_confusion_matrix(cm, class_names):\\n\\n"""\\n\\n Plot Confusion Matrix\\n\\n """\\n\\n plt.figure(figsize=(10,8))\\n\\n sns.heatmap(cm, annot=True, fmt='d', cmap='Blues',\\n\\n xticklabels=class_names, yticklabels=class_names)\\n\\n plt.xlabel('Predicted Label')\\n\\n plt.ylabel('True Label')\\n\\n plt.title('Confusion Matrix')\\n\\n plt.tight_layout()\\n\\n plt.show()\\n\\n# Usage example\\n\\n# Assume there are 10 classes\\n\\n class_names =[f'Class {i}'for i in range(10)]\\n\\n cm = np.array([\\n\\n[45,1,0,0,0,0,2,1,1,0],\\n\\n[0,42,