Chapter 12: The GBM Trio

import numpy as np
from sklearn.ensemble import HistGradientBoostingClassifier
from sklearn.metrics import roc_auc_score

from kaggle_companion.activities._common import clean

SEED = 12
REPLICATES = 5                 # independent constructed datasets; every estimate is their mean
N_TRAIN, N_NEW = 3000, 6000
NUMERIC = 4                    # numeric feature columns before the category column


def generate(levels, seed):
    """Four numeric features plus one category with `levels` levels. Each level has a random effect on the log odds,
    and the integer code given to a level is arbitrary, as it is for a label-encoded column."""
    rng = np.random.default_rng(seed)
    effect = rng.normal(0.0, 1.0, levels)
    code = rng.permutation(levels)

    def rows(n):
        level = rng.integers(0, levels, n)
        x = rng.normal(size=(n, NUMERIC))
        logit = 0.9 * x[:, 0] - 0.6 * x[:, 1] + 0.5 * x[:, 2] * x[:, 3] + effect[level]
        y = (rng.random(n) < 1 / (1 + np.exp(-logit))).astype(int)
        return x, level, y

    return rows(N_TRAIN), rows(N_NEW), code


def learner(**extra):
    """One fixed configuration for every representation: the comparison changes the input, not the budget."""
    return HistGradientBoostingClassifier(max_iter=100, learning_rate=0.1, max_leaf_nodes=15, random_state=0, **extra)


def run(levels):
    scores = {"none": [], "ordinal": [], "native": [], "onehot": []}
    for rep in range(REPLICATES):
        (x, lv, y), (x_new, lv_new, y_new), code = generate(levels, SEED * 100 + rep)
        ordinal, ordinal_new = np.column_stack([x, code[lv]]), np.column_stack([x_new, code[lv_new]])
        onehot, onehot_new = np.column_stack([x, np.eye(levels)[lv]]), np.column_stack([x_new, np.eye(levels)[lv_new]])
        fits = {"none": (learner(), x, x_new),                                   # no category column at all
                "ordinal": (learner(), ordinal, ordinal_new),                    # codes treated as ordered numbers
                "native": (learner(categorical_features=[NUMERIC]), ordinal, ordinal_new),   # native categorical splits
                "onehot": (learner(), onehot, onehot_new)}                       # one 0/1 column per level
        for name, (model, a, b) in fits.items():
            scores[name].append(roc_auc_score(y_new, model.fit(a, y).predict_proba(b)[:, 1]))
    return clean({
        "levels": levels, "columns": {"none": NUMERIC, "ordinal": NUMERIC + 1, "native": NUMERIC + 1, "onehot": NUMERIC + levels},
        "auc": {k: float(np.mean(v)) for k, v in scores.items()}, "per_replicate": scores,
        "rows_per_level": N_TRAIN / levels, "replicates": REPLICATES, "train_rows": N_TRAIN, "new_rows": N_NEW,
    })

import pandas as pd
from kaggle_companion.activities.ch12 import explain

VALUES = [8, 32, 64, 128]
results = {value: run(value) for value in VALUES}
table = pd.DataFrame({str(v): explain(results[v], v)['metrics'] for v in VALUES})
print(table.to_string())

import matplotlib.pyplot as plt
from kaggle_companion.activities.ch12 import draw, NCOLS, HEIGHT

DEFAULT = 128
fig, axes = plt.subplots(1, NCOLS, figsize=(6.6 if NCOLS == 1 else 10.4, HEIGHT), layout='constrained')
draw(axes, results[DEFAULT], DEFAULT)
plt.show()