Chapter 2: The Process Mindset

import numpy as np
from sklearn.linear_model import LogisticRegression

from kaggle_companion.activities._common import clean

SEED = 2
REPLICATES = 16                      # independent constructed competitions; every estimate is their mean
SPLITS = 10                          # random public/private splits of each competition's test rows
N_TRAIN, N_TEST, N_FRESH = 800, 2000, 20000
PUBLIC_SHARE = 0.20                  # public leaderboard rows: 400 of the 2,000 test rows
N_BASE, N_CANDIDATES, N_USEFUL, USEFUL_WEIGHT = 6, 100, 2, 0.25
LOGGED = [3, 10, 30, 100]            # numbers of logged versions the control can take


def generate(n, rng, w_base, w_cand):
    """Six base columns and 100 candidate columns. Only two candidates truly help, and only a little."""
    base = rng.normal(size=(n, N_BASE))
    cand = rng.normal(size=(n, N_CANDIDATES))
    p = 1 / (1 + np.exp(-(base @ w_base + cand @ w_cand)))
    return np.column_stack([base, cand]), (rng.random(n) < p).astype(int)


def row_loss(p, y):
    p = np.clip(p, 1e-6, 1 - 1e-6)
    return -(y * np.log(p) + (1 - y) * np.log(1 - p))


def one_competition(seed):
    """Version 0 is the baseline. Version k adds candidate column k and refits. Returns every version's
    per-row test loss and its mean loss on a large fresh sample (the truth)."""
    rng = np.random.default_rng(seed)
    w_base = rng.normal(0, 0.6, N_BASE)
    w_cand = np.zeros(N_CANDIDATES)
    w_cand[rng.choice(N_CANDIDATES, N_USEFUL, replace=False)] = USEFUL_WEIGHT
    X, y = generate(N_TRAIN, rng, w_base, w_cand)
    X_test, y_test = generate(N_TEST, rng, w_base, w_cand)
    X_fresh, y_fresh = generate(N_FRESH, rng, w_base, w_cand)
    test_loss = np.zeros((max(LOGGED), N_TEST))
    fresh_loss = np.zeros(max(LOGGED))
    for k in range(max(LOGGED)):
        cols = list(range(N_BASE)) + ([] if k == 0 else [N_BASE + k - 1])
        model = LogisticRegression(max_iter=200).fit(X[:, cols], y)
        test_loss[k] = row_loss(model.predict_proba(X_test[:, cols])[:, 1], y_test)
        fresh_loss[k] = row_loss(model.predict_proba(X_fresh[:, cols])[:, 1], y_fresh).mean()
    # Each split draws a public sample; the winner is the version with the lowest public loss among the first N.
    gains = {n: [] for n in LOGGED}
    for _ in range(SPLITS):
        public = np.zeros(N_TEST, bool)
        public[rng.choice(N_TEST, int(PUBLIC_SHARE * N_TEST), replace=False)] = True
        pub_loss, priv_loss = test_loss[:, public].mean(1), test_loss[:, ~public].mean(1)
        for n in LOGGED:
            k = int(np.argmin(pub_loss[:n]))
            # gain = baseline loss minus the winner's loss, so positive means the winner looks better
            gains[n].append([pub_loss[0] - pub_loss[k], priv_loss[0] - priv_loss[k], fresh_loss[0] - fresh_loss[k]])
    return {n: np.mean(g, axis=0) for n, g in gains.items()}, {n: np.mean(np.array(g)[:, 2] > 0) for n, g in gains.items()}


def run(versions):
    studies = [one_competition(SEED * 1000 + i) for i in range(REPLICATES)]
    curve = {}
    for n in LOGGED:
        g = np.array([s[0][n] for s in studies])           # replicates x (public, private, fresh)
        curve[n] = {"public": g[:, 0].mean(), "private": g[:, 1].mean(), "fresh": g[:, 2].mean(),
                    "public_sd": g[:, 0].std(), "private_sd": g[:, 1].std(),
                    "truly_better": float(np.mean([s[1][n] for s in studies])),
                    "per_study_public": g[:, 0], "per_study_private": g[:, 1]}
    return clean({"versions": versions, "selected": curve[versions],
                  "curve": [{"versions": n, **curve[n]} for n in LOGGED],
                  "replicates": REPLICATES, "splits": SPLITS, "public_rows": int(PUBLIC_SHARE * N_TEST),
                  "private_rows": N_TEST - int(PUBLIC_SHARE * N_TEST), "fresh_rows": N_FRESH})

import pandas as pd
from kaggle_companion.activities.ch02 import explain

VALUES = [3, 10, 30, 100]
results = {value: run(value) for value in VALUES}
table = pd.DataFrame({str(v): explain(results[v], v)['metrics'] for v in VALUES})
print(table.to_string())

import matplotlib.pyplot as plt
from kaggle_companion.activities.ch02 import draw, NCOLS, HEIGHT

DEFAULT = 30
fig, axes = plt.subplots(1, NCOLS, figsize=(6.6 if NCOLS == 1 else 10.4, HEIGHT), layout='constrained')
draw(axes, results[DEFAULT], DEFAULT)
plt.show()