Chapter 61: Playground Series Strategy

import numpy as np

from kaggle_companion.activities._common import clean

SEED = 61
DATASETS = 12                        # independent constructed competitions; every estimate is their mean
SPLITS = 300                         # random public/private splits of the test rows per competition
F, N_TRAIN, N_TEST, N_FRESH = 30, 1500, 2000, 10000
PUBLIC_SHARE = 0.2                   # 400 public rows, 1,600 private rows
NOISE = 3.0


def make_data(rng, beta, n):
    X = rng.normal(size=(n, F))
    return X, X @ beta + rng.normal(0, NOISE, n)


def ridge(X, y, alpha):
    """Closed-form ridge without an intercept (the constructed target has mean zero)."""
    return np.linalg.solve(X.T @ X + alpha * np.eye(X.shape[1]), X.T @ y)


def rmse(y, pred):
    return float(np.sqrt(np.mean((y - pred) ** 2)))


def one_competition(n_candidates, seed):
    """Candidates are near-duplicate ridge recipes: each drops two features, trains on a random 75% of the rows and
    picks its own penalty. Returns each candidate's squared errors on the test rows, its 5-fold CV RMSE on the
    training rows and its RMSE on 10,000 fresh rows (the 'truth' that nobody gets to see in a real competition)."""
    rng = np.random.default_rng(seed)
    beta = rng.normal(0, 0.35, F)
    Xtr, ytr = make_data(rng, beta, N_TRAIN)
    Xte, yte = make_data(rng, beta, N_TEST)
    Xfr, yfr = make_data(rng, beta, N_FRESH)
    folds = rng.permutation(N_TRAIN) % 5
    err_test, cv, truth = [], [], []
    for _ in range(n_candidates):
        cols = np.sort(rng.choice(F, F - 2, replace=False))
        alpha = 10 ** rng.uniform(-1, 2)
        rows = rng.random(N_TRAIN) < 0.75
        w = ridge(Xtr[rows][:, cols], ytr[rows], alpha)
        err_test.append((yte - Xte[:, cols] @ w) ** 2)
        truth.append(rmse(yfr, Xfr[:, cols] @ w))
        oof = np.zeros(N_TRAIN)
        for k in range(5):
            fit = rows & (folds != k)
            oof[folds == k] = Xtr[folds == k][:, cols] @ ridge(Xtr[fit][:, cols], ytr[fit], alpha)
        cv.append(rmse(ytr, oof))
    return np.array(err_test), np.array(cv), np.array(truth), rng


def run(n_candidates):
    n_pub = int(PUBLIC_SHARE * N_TEST)
    per = {k: [] for k in ("public_shown", "private_earned", "truth_earned", "regret_public", "regret_cv", "regret_random",
                           "leader_stays", "leader_is_best")}
    for d in range(DATASETS):
        err, cv, truth, rng = one_competition(n_candidates, SEED * 100 + d)
        mask = np.zeros((SPLITS, N_TEST))
        for s in range(SPLITS):
            mask[s, rng.permutation(N_TEST)[:n_pub]] = 1
        sp = np.sqrt(mask @ err.T / n_pub)                              # public RMSE of every candidate, per split
        sv = np.sqrt((1 - mask) @ err.T / (N_TEST - n_pub))             # private RMSE of every candidate, per split
        pick = sp.argmin(1)                                             # the public leader in each split
        rows = np.arange(SPLITS)
        per["public_shown"].append(float(np.mean(np.median(sp, 1) - sp[rows, pick])))
        per["private_earned"].append(float(np.mean(np.median(sv, 1) - sv[rows, pick])))
        per["truth_earned"].append(float(np.mean(np.median(truth) - truth[pick])))
        per["regret_public"].append(float(np.mean(truth[pick] - truth.min())))
        per["regret_cv"].append(float(truth[cv.argmin()] - truth.min()))
        per["regret_random"].append(float(truth.mean() - truth.min()))
        per["leader_stays"].append(float(np.mean(sv.argmin(1) == pick)))
        per["leader_is_best"].append(float(np.mean(truth.argmin() == pick)))
    mean = {k: float(np.mean(v)) for k, v in per.items()}
    return clean({"candidates": n_candidates, **mean, "per_dataset": per, "datasets": DATASETS, "splits": SPLITS,
                  "public_rows": n_pub, "private_rows": N_TEST - n_pub, "train_rows": N_TRAIN})

import pandas as pd
from kaggle_companion.activities.ch61 import explain

VALUES = [3, 10, 30, 100]
results = {value: run(value) for value in VALUES}
table = pd.DataFrame({str(v): explain(results[v], v)['metrics'] for v in VALUES})
print(table.to_string())

import matplotlib.pyplot as plt
from kaggle_companion.activities.ch61 import draw, NCOLS, HEIGHT

DEFAULT = 30
fig, axes = plt.subplots(1, NCOLS, figsize=(6.6 if NCOLS == 1 else 10.4, HEIGHT), layout='constrained')
draw(axes, results[DEFAULT], DEFAULT)
plt.show()