Chapter 1: What Actually Wins on Kaggle

import numpy as np
from sklearn.ensemble import HistGradientBoostingClassifier
from sklearn.metrics import roc_auc_score
from sklearn.model_selection import GroupKFold, KFold

from kaggle_companion.activities._common import clean

SEED = 1
REPLICATES = 8                  # independent constructed studies; every estimate is their mean
N_SUBJECTS, ROWS_EACH = 36, 40  # development data: 36 subjects with 40 sensor windows each
NEW_SUBJECTS, NEW_ROWS = 150, 8 # the real test: 150 subjects never seen in development
ACTIVITY_WEIGHTS = np.array([0.9, -0.7, 0.5])
BASELINE_LEAVES = 2             # the model before the "upgrade": one split per tree


def generate(n_subjects, rows_each, rng):
    """Sensor windows. Three noisy activity channels carry a shared signal. Four more channels are a
    subject fingerprint (constant within a subject), and each subject also has its own label bias."""
    fingerprint = rng.normal(0, 1, (n_subjects, 4))
    bias = rng.normal(0, 1.5, n_subjects)
    subject = np.repeat(np.arange(n_subjects), rows_each)
    activity = rng.normal(size=(len(subject), 3))
    logit = activity @ ACTIVITY_WEIGHTS + bias[subject]
    y = (rng.random(len(subject)) < 1 / (1 + np.exp(-logit))).astype(int)
    X = np.column_stack([activity + rng.normal(0, 0.8, activity.shape),
                         fingerprint[subject] + rng.normal(0, 0.15, (len(subject), 4))])
    return X, y, subject


def make_model(leaves):
    return HistGradientBoostingClassifier(max_leaf_nodes=leaves, max_iter=40, learning_rate=0.2,
                                          min_samples_leaf=5, random_state=0)


def pooled_auc(leaves, X, y, splits):
    """Predict every row from a model that did not see it, then compute one AUC over all rows."""
    pred = np.zeros(len(y))
    for fit, out in splits:
        pred[out] = make_model(leaves).fit(X[fit], y[fit]).predict_proba(X[out])[:, 1]
    return roc_auc_score(y, pred)


def one_study(leaves, seed):
    rng = np.random.default_rng(seed)
    X, y, subject = generate(N_SUBJECTS, ROWS_EACH, rng)
    X_new, y_new, _ = generate(NEW_SUBJECTS, NEW_ROWS, rng)
    out = {}
    for name, size in (("model", leaves), ("baseline", BASELINE_LEAVES)):
        out[name] = {
            "random_folds": pooled_auc(size, X, y, KFold(5, shuffle=True, random_state=0).split(X)),
            "subject_folds": pooled_auc(size, X, y, GroupKFold(5).split(X, y, subject)),
            "new_subjects": roc_auc_score(y_new, make_model(size).fit(X, y).predict_proba(X_new)[:, 1]),
        }
    return out


def run(leaves):
    studies = [one_study(leaves, SEED * 1000 + i) for i in range(REPLICATES)]
    keys = ["random_folds", "subject_folds", "new_subjects"]
    mean = lambda arm, k: float(np.mean([s[arm][k] for s in studies]))
    return clean({
        "leaves": leaves,
        "model": {k: mean("model", k) for k in keys},
        "baseline": {k: mean("baseline", k) for k in keys},
        "per_study": {k: [s["model"][k] for s in studies] for k in keys},
        "replicates": REPLICATES, "subjects": N_SUBJECTS, "rows_each": ROWS_EACH,
        "new_subjects_count": NEW_SUBJECTS, "baseline_leaves": BASELINE_LEAVES,
    })

import pandas as pd
from kaggle_companion.activities.ch01 import explain

VALUES = [2, 8, 31]
results = {value: run(value) for value in VALUES}
table = pd.DataFrame({str(v): explain(results[v], v)['metrics'] for v in VALUES})
print(table.to_string())

import matplotlib.pyplot as plt
from kaggle_companion.activities.ch01 import draw, NCOLS, HEIGHT

DEFAULT = 31
fig, axes = plt.subplots(1, NCOLS, figsize=(6.6 if NCOLS == 1 else 10.4, HEIGHT), layout='constrained')
draw(axes, results[DEFAULT], DEFAULT)
plt.show()