💰 Business Value: Save $50,000+ with Cost-Sensitive Optimization

ROI: Turn model into profit center with cost-aware thresholds
Time: 10 minutes to understand real business impact
Next: See 03_multiclass.ipynb for advanced scenarios

This example shows how to optimize for business metrics (dollars) rather than statistical metrics (F1, accuracy). Real applications have different costs for different types of errors.

from sklearn.datasets import make_classification
from sklearn.ensemble import GradientBoostingClassifier
from sklearn.metrics import confusion_matrix, f1_score
from sklearn.model_selection import train_test_split

from optimal_cutoffs import optimize_thresholds

print("💰 BUSINESS VALUE: COST-SENSITIVE OPTIMIZATION")
print("=" * 55)

🏦 SCENARIO: Credit Card Fraud Detection

  • Average transaction: $150

  • Fraud investigation cost: $25 per case

  • Customer friction cost: $5 per false alarm

  • Fraud loss: 100% of transaction value

# Business costs
AVERAGE_TRANSACTION = 150
INVESTIGATION_COST = 25
FALSE_ALARM_COST = 5
FRAUD_LOSS = AVERAGE_TRANSACTION  # 100% loss

print("🏦 SCENARIO: Credit Card Fraud Detection")
print("-" * 45)
print(f"• Average transaction: ${AVERAGE_TRANSACTION}")
print(f"• Fraud investigation cost: ${INVESTIGATION_COST} per case")
print(f"• Customer friction cost: ${FALSE_ALARM_COST} per false alarm")
print(f"• Fraud loss: ${FRAUD_LOSS} (100% of transaction)")
print()

Generate fraud detection dataset

# Generate realistic fraud detection dataset
X, y = make_classification(
    n_samples=5000,
    n_features=15,
    n_classes=2,
    weights=[0.995, 0.005],  # 99.5% legitimate, 0.5% fraud (very imbalanced)
    flip_y=0.01,  # Small amount of noise
    random_state=123,
)

# Split data
X_train, X_test, y_train, y_test = train_test_split(
    X, y, test_size=0.3, random_state=123, stratify=y
)

# Train fraud detection model
model = GradientBoostingClassifier(
    n_estimators=100, learning_rate=0.1, random_state=123
)
model.fit(X_train, y_train)
y_prob = model.predict_proba(X_test)[:, 1]

n_fraud = y_test.sum()
n_legit = len(y_test) - n_fraud
fraud_rate = y_test.mean()

print(f"📊 Test set: {len(y_test)} transactions")
print(f"   • {n_fraud} fraudulent ({fraud_rate:.1%})")
print(f"   • {n_legit} legitimate ({(1-fraud_rate):.1%})")
print(f"   • Total value at risk: ${n_fraud * FRAUD_LOSS:,}")
print()

Define cost function

def calculate_business_cost(y_true, y_pred):
    """
    Calculate total business cost based on confusion matrix.
    
    Cost structure:
    - True Positive (TP): Investigation cost (catch fraud)
    - False Positive (FP): Investigation cost + false alarm cost
    - True Negative (TN): No cost (normal operation)
    - False Negative (FN): Fraud loss (missed fraud)
    """
    tn, fp, fn, tp = confusion_matrix(y_true, y_pred).ravel()
    
    # Calculate costs
    tp_cost = tp * INVESTIGATION_COST  # Investigate caught fraud
    fp_cost = fp * (INVESTIGATION_COST + FALSE_ALARM_COST)  # Investigate + customer friction
    tn_cost = 0  # Normal transactions, no cost
    fn_cost = fn * FRAUD_LOSS  # Missed fraud = full loss
    
    total_cost = tp_cost + fp_cost + tn_cost + fn_cost
    
    return {
        'total_cost': total_cost,
        'tp_cost': tp_cost,
        'fp_cost': fp_cost,
        'tn_cost': tn_cost,
        'fn_cost': fn_cost,
        'tp': tp,
        'fp': fp,
        'tn': tn,
        'fn': fn,
    }

print("💰 Business Cost Function:")
print(f"   • True Positive (caught fraud): ${INVESTIGATION_COST} investigation")
print(f"   • False Positive (false alarm): ${INVESTIGATION_COST + FALSE_ALARM_COST} (investigation + friction)")
print(f"   • True Negative (normal): $0")
print(f"   • False Negative (missed fraud): ${FRAUD_LOSS} (full loss)")
print()

❌ BEFORE: Statistical optimization (F1)

# Find threshold that maximizes F1 score
result_f1 = optimize_thresholds(y_test, y_prob, metric='f1')
y_pred_f1 = result_f1.predict(y_prob)

# Calculate F1 performance
f1_score_value = f1_score(y_test, y_pred_f1)
f1_costs = calculate_business_cost(y_test, y_pred_f1)

print("❌ BEFORE: F1-Optimized Threshold")
print(f"   Threshold: {result_f1.thresholds[0]:.3f}")
print(f"   F1 Score: {f1_score_value:.3f}")
print(f"   Confusion Matrix: TP={f1_costs['tp']}, FP={f1_costs['fp']}, TN={f1_costs['tn']}, FN={f1_costs['fn']}")
print(f"   💸 Total Cost: ${f1_costs['total_cost']:,.0f}")
print(f"      • Investigation costs: ${f1_costs['tp_cost'] + f1_costs['fp_cost']:,.0f}")
print(f"      • Fraud losses: ${f1_costs['fn_cost']:,.0f}")
print()

✅ AFTER: Business-optimized threshold

# Find threshold that minimizes business cost
# We'll test different thresholds and find the one with minimum cost
import numpy as np

thresholds = np.linspace(0.01, 0.99, 99)
costs = []

for threshold in thresholds:
    y_pred_temp = (y_prob >= threshold).astype(int)
    cost_info = calculate_business_cost(y_test, y_pred_temp)
    costs.append(cost_info['total_cost'])

# Find threshold with minimum cost
min_cost_idx = np.argmin(costs)
business_threshold = thresholds[min_cost_idx]
min_cost = costs[min_cost_idx]

# Make predictions with business-optimal threshold
y_pred_business = (y_prob >= business_threshold).astype(int)
business_costs = calculate_business_cost(y_test, y_pred_business)
business_f1 = f1_score(y_test, y_pred_business)

print("✅ AFTER: Business-Optimized Threshold")
print(f"   Threshold: {business_threshold:.3f}")
print(f"   F1 Score: {business_f1:.3f}")
print(f"   Confusion Matrix: TP={business_costs['tp']}, FP={business_costs['fp']}, TN={business_costs['tn']}, FN={business_costs['fn']}")
print(f"   💰 Total Cost: ${business_costs['total_cost']:,.0f}")
print(f"      • Investigation costs: ${business_costs['tp_cost'] + business_costs['fp_cost']:,.0f}")
print(f"      • Fraud losses: ${business_costs['fn_cost']:,.0f}")
print()

🎯 THE BUSINESS IMPACT

# Calculate cost savings
cost_savings = f1_costs['total_cost'] - business_costs['total_cost']
cost_reduction_pct = (cost_savings / f1_costs['total_cost']) * 100

# Annualized savings (assuming this test set represents daily transactions)
annual_savings = cost_savings * 365

print("🎯 THE BUSINESS IMPACT")
print("-" * 35)
print(f"💰 Cost Reduction: ${cost_savings:,.0f} ({cost_reduction_pct:.1f}% decrease)")
print(f"📈 Annual Savings: ${annual_savings:,.0f}")
print()

print("📊 Trade-off Analysis:")
tp_diff = business_costs['tp'] - f1_costs['tp']
fp_diff = business_costs['fp'] - f1_costs['fp']
fn_diff = business_costs['fn'] - f1_costs['fn']

print(f"   • Caught fraud: {tp_diff:+d} ({business_costs['tp']} vs {f1_costs['tp']})")
print(f"   • False alarms: {fp_diff:+d} ({business_costs['fp']} vs {f1_costs['fp']})")
print(f"   • Missed fraud: {fn_diff:+d} ({business_costs['fn']} vs {f1_costs['fn']})")
print()

print("🔑 KEY INSIGHT:")
print("   Business-optimal threshold balances investigation costs vs fraud losses")
if business_costs['fn'] < f1_costs['fn']:
    print("   → Catches MORE fraud by accepting more false positives")
else:
    print("   → Reduces false positives even if some fraud is missed")
print(f"   → Net result: ${cost_savings:,.0f} savings!")

📊 Threshold sensitivity analysis

import matplotlib.pyplot as plt

# Calculate F1 scores for visualization
f1_scores = []
for threshold in thresholds:
    y_pred_temp = (y_prob >= threshold).astype(int)
    f1_scores.append(f1_score(y_test, y_pred_temp))

# Create figure with two subplots
fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(15, 5))

# Plot 1: Business cost vs threshold
ax1.plot(thresholds, costs, 'g-', linewidth=2, label='Business Cost')
ax1.axvline(business_threshold, color='green', linestyle='--', 
           label=f'Business Optimal ({business_threshold:.3f})')
ax1.axvline(result_f1.thresholds[0], color='red', linestyle='--', 
           label=f'F1 Optimal ({result_f1.thresholds[0]:.3f})')
ax1.set_xlabel('Decision Threshold')
ax1.set_ylabel('Total Business Cost ($)')
ax1.set_title('Business Cost vs Decision Threshold')
ax1.legend()
ax1.grid(True, alpha=0.3)

# Plot 2: F1 score vs threshold  
ax2.plot(thresholds, f1_scores, 'r-', linewidth=2, label='F1 Score')
ax2.axvline(business_threshold, color='green', linestyle='--',
           label=f'Business Optimal ({business_threshold:.3f})')
ax2.axvline(result_f1.thresholds[0], color='red', linestyle='--',
           label=f'F1 Optimal ({result_f1.thresholds[0]:.3f})')
ax2.set_xlabel('Decision Threshold')
ax2.set_ylabel('F1 Score')
ax2.set_title('F1 Score vs Decision Threshold')
ax2.legend()
ax2.grid(True, alpha=0.3)

plt.tight_layout()
plt.show()

print(f"\nBusiness-optimal threshold ({business_threshold:.3f}) differs from F1-optimal ({result_f1.thresholds[0]:.3f})")
print(f"This difference saves ${cost_savings:,.0f} in business costs!")

🚀 What’s Next?

  • 03_multiclass.ipynb: Handle complex multi-class business scenarios

  • 04_interactive_demo.ipynb: Understand the mathematical foundations

💡 Business Optimization Tips

  1. Quantify your costs: Get real numbers for each type of error

  2. Consider indirect costs: Customer churn, reputation, etc.

  3. Regular recalibration: Business costs and data distributions change

  4. A/B testing: Validate your cost assumptions in production

  5. Risk appetite: Some organizations prefer conservative vs aggressive strategies

🎯 Real-World Applications

  • Medical diagnosis: Balance false positives (unnecessary treatment) vs false negatives (missed disease)

  • Credit approval: Balance default risk vs lost customers

  • Marketing: Balance campaign costs vs missed opportunities

  • Quality control: Balance inspection costs vs defect costs