Implementation Hints

• For model calibration: calibrated_clf = CalibratedClassifierCV(clf, method='sigmoid', cv='prefit')
• For PR curves: precision, recall, _ = precision_recall_curve(y_true, y_scores)
• Learning curves: train_sizes, train_scores, test_scores = learning_curve(model, X, y, cv=5)
• Residual analysis: residuals = y_true - y_pred
• Statistical test: from scipy.stats import ttest_rel for paired t-tests

Complete Solution:

# COMPLETE SOLUTION FOR MODEL EVALUATION
# (Partial solution showing advanced techniques)

# 1.7 Model calibration implementation
print("\n\n=== Model Calibration Analysis ===")
from sklearn.calibration import CalibratedClassifierCV

# Select a model to calibrate (e.g., SVM which often needs calibration)
svm_model = SVC(probability=False, random_state=42)
svm_model.fit(X_train_clf_scaled, y_train_clf)

# Calibrate using Platt scaling
calibrated_svm = CalibratedClassifierCV(svm_model, method='sigmoid', cv=3)
calibrated_svm.fit(X_train_clf_scaled, y_train_clf)

# Get predicted probabilities
y_pred_proba_svm = calibrated_svm.predict_proba(X_test_clf_scaled)[:, 1]

# Calculate calibration metrics
prob_true_svm, prob_pred_svm = calibration_curve(y_test_clf, y_pred_proba_svm, n_bins=10)
brier_svm = brier_score_loss(y_test_clf, y_pred_proba_svm)

print(f"SVM Brier Score (calibrated): {brier_svm:.4f}")

# 2.5 Learning curves implementation
print("\n\n=== Learning Curves Analysis ===")
from sklearn.model_selection import learning_curve

# Use the best regression model (e.g., Gradient Boosting)
best_reg = GradientBoostingRegressor(n_estimators=100, random_state=42)

train_sizes, train_scores, test_scores = learning_curve(
    best_reg, X_reg, y_reg, cv=5, scoring='neg_mean_squared_error',
    train_sizes=np.linspace(0.1, 1.0, 10), n_jobs=-1
)

# Convert to positive MSE
train_scores_mean = -train_scores.mean(axis=1)
test_scores_mean = -test_scores.mean(axis=1)

print(f"Training MSE at 100% data: {train_scores_mean[-1]:.2f}")
print(f"Validation MSE at 100% data: {test_scores_mean[-1]:.2f}")
print(f"Generalization gap: {test_scores_mean[-1] - train_scores_mean[-1]:.2f}")

# 2.6 Residual analysis
print("\n\n=== Residual Analysis ===")
best_reg.fit(X_train_reg_scaled, y_train_reg)
y_pred_reg = best_reg.predict(X_test_reg_scaled)
residuals = y_test_reg - y_pred_reg

# Calculate residual statistics
residual_mean = residuals.mean()
residual_std = residuals.std()
residual_skew = pd.Series(residuals).skew()

print(f"Residual mean: {residual_mean:.4f} (should be close to 0)")
print(f"Residual std: {residual_std:.4f}")
print(f"Residual skewness: {residual_skew:.4f} (should be close to 0 for normality)")

# Check for homoscedasticity (constant variance)
# Calculate correlation between absolute residuals and predictions
abs_residuals = np.abs(residuals)
correlation = np.corrcoef(abs_residuals, y_pred_reg)[0, 1]
print(f"Correlation between |residuals| and predictions: {correlation:.4f}")
print("(Should be close to 0 for homoscedasticity)")

# 3.1 Precision-Recall curves
print("\n=== Precision-Recall Curves ===")
from sklearn.metrics import precision_recall_curve, average_precision_score

# Get probabilities from best classifier
best_clf = RandomForestClassifier(**grid_search.best_params_, random_state=42)
best_clf.fit(X_train_clf_scaled, y_train_clf)
y_pred_proba_best = best_clf.predict_proba(X_test_clf_scaled)[:, 1]

precision, recall, thresholds = precision_recall_curve(y_test_clf, y_pred_proba_best)
average_precision = average_precision_score(y_test_clf, y_pred_proba_best)

print(f"Average Precision Score: {average_precision:.4f}")
print(f"Precision at recall=0.5: {precision[np.argmin(np.abs(recall - 0.5))]:.4f}")

# 3.2 Feature importance
print("\n=== Feature Importance Analysis ===")
importances = best_clf.feature_importances_
indices = np.argsort(importances)[::-1]

print("Top 5 most important features:")
for i in range(5):
    print(f"  Feature {indices[i]}: {importances[indices[i]]:.4f}")

# 3.3 Statistical model comparison
print("\n=== Statistical Model Comparison ===")
from scipy.stats import ttest_rel

# Compare Random Forest vs Gradient Boosting using cross-validation
cv_scores_rf = cross_val_score(best_clf, X_clf, y_clf, cv=5, scoring='roc_auc')
cv_scores_gb = cross_val_score(GradientBoostingRegressor(), X_reg, y_reg, cv=5, scoring='r2')

# Paired t-test for classification
if len(cv_scores_rf) == len(cv_scores_gb):
    t_stat, p_value = ttest_rel(cv_scores_rf, cv_scores_gb[:len(cv_scores_rf)])
    print(f"Paired t-test p-value: {p_value:.4f}")
    if p_value < 0.05:
        print("Statistically significant difference between models")
    else:
        print("No statistically significant difference")

print("\nAdvanced evaluation techniques completed!")

Your Score: 0/10