Lab - Fraud Detection with Tree Ensembles¶

%matplotlib inline
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt

from sklearn.model_selection import train_test_split
from sklearn.tree import DecisionTreeClassifier
from sklearn.ensemble import RandomForestClassifier, GradientBoostingClassifier
from sklearn.metrics import (
    classification_report,
    roc_auc_score,
    average_precision_score,
    precision_recall_curve,
)

rng = np.random.default_rng(42)
n = 12000

# Synthetic transaction features
amount = rng.lognormal(mean=3.1, sigma=0.9, size=n)
hour = rng.integers(0, 24, size=n)
country_risk = rng.beta(2.0, 5.0, size=n)
velocity_1h = rng.poisson(1.3, size=n)
merchant_risk = rng.beta(2.5, 3.5, size=n)
account_age_days = rng.integers(1, 2500, size=n)
device_trust = rng.beta(3.0, 2.5, size=n)

# Fraud propensity: nonlinear + interaction effects
score = (
    0.0028 * amount
    + 0.9 * country_risk
    + 0.45 * merchant_risk
    + 0.34 * velocity_1h
    - 0.00045 * account_age_days
    - 0.8 * device_trust
    + 0.22 * ((hour <= 5) | (hour >= 23)).astype(float)
)

prob = 1 / (1 + np.exp(-(score - 3.8)))
y = rng.binomial(1, np.clip(prob, 0, 1))

# Force stronger imbalance close to real fraud settings
mask_keep = (y == 0) | (rng.random(n) < 0.32)

X = pd.DataFrame(
    {
        "amount": amount,
        "hour": hour,
        "country_risk": country_risk,
        "velocity_1h": velocity_1h,
        "merchant_risk": merchant_risk,
        "account_age_days": account_age_days,
        "device_trust": device_trust,
    }
).loc[mask_keep].reset_index(drop=True)
y = y[mask_keep]

print(f"Samples: {len(X):,}")
print(f"Fraud rate: {y.mean()*100:.2f}%")

X_train, X_test, y_train, y_test = train_test_split(
    X, y, test_size=0.3, random_state=42, stratify=y
)

models = {
    "Decision Tree": DecisionTreeClassifier(max_depth=6, min_samples_leaf=20, random_state=42),
    "Random Forest": RandomForestClassifier(
        n_estimators=320,
        max_depth=8,
        min_samples_leaf=8,
        random_state=42,
        n_jobs=-1,
    ),
    "Gradient Boosting": GradientBoostingClassifier(
        n_estimators=260,
        learning_rate=0.05,
        max_depth=3,
        random_state=42,
    ),
}

rows = []
probas = {}
for name, model in models.items():
    model.fit(X_train, y_train)
    p = model.predict_proba(X_test)[:, 1]
    y_hat = (p >= 0.5).astype(int)
    probas[name] = p
    rows.append(
        {
            "model": name,
            "roc_auc": roc_auc_score(y_test, p),
            "pr_auc": average_precision_score(y_test, p),
            "f1_at_0.5": classification_report(y_test, y_hat, output_dict=True)["1"]["f1-score"],
        }
    )

results = pd.DataFrame(rows).sort_values("pr_auc", ascending=False)
print("\nModel comparison:")
print(results.round(4).to_string(index=False))

%matplotlib inline
import numpy as np
import matplotlib.pyplot as plt
from sklearn.metrics import confusion_matrix

best_model_name = results.iloc[0]["model"]
p_best = probas[best_model_name]

thresholds = np.linspace(0.05, 0.95, 91)
value_tp, cost_fp, cost_fn = 250.0, 20.0, 400.0

evs = []
for t in thresholds:
    y_hat = (p_best >= t).astype(int)
    tn, fp, fn, tp = confusion_matrix(y_test, y_hat).ravel()
    ev = tp * value_tp - fp * cost_fp - fn * cost_fn
    evs.append(ev)

best_idx = int(np.argmax(evs))
best_t = thresholds[best_idx]
print(f"Best model by PR-AUC: {best_model_name}")
print(f"Best threshold by EV: {best_t:.2f}")
print(f"Max expected value: {evs[best_idx]:,.0f}")

plt.figure(figsize=(8, 4))
plt.plot(thresholds, evs, color="#d62828", lw=2)
plt.axvline(best_t, linestyle="--", color="black", label=f"best t={best_t:.2f}")
plt.title("Expected Value vs Decision Threshold")
plt.xlabel("Threshold")
plt.ylabel("Expected value")
plt.grid(alpha=0.25)
plt.legend()
plt.tight_layout()
plt.show()

%matplotlib inline
import pandas as pd
import matplotlib.pyplot as plt

best_model = models[best_model_name]
imp = pd.Series(best_model.feature_importances_, index=X.columns).sort_values(ascending=False)
print("Top feature importances:")
print(imp.head(7).round(4).to_string())

plt.figure(figsize=(7, 4.5))
imp.sort_values().plot.barh(color="#457b9d")
plt.title(f"Feature Importance - {best_model_name}")
plt.xlabel("Importance")
plt.grid(axis="x", alpha=0.25)
plt.tight_layout()
plt.show()