Chapter 7: Domain Features as Testable Hypotheses

import numpy as np
from sklearn.ensemble import HistGradientBoostingClassifier
from sklearn.metrics import roc_auc_score
from sklearn.model_selection import StratifiedKFold

from kaggle_companion.activities._common import clean

SEED = 7
REPLICATES = 8        # independent constructed transaction logs; every score is their mean
N_CUSTOMERS = 400
CUTOFF = 0.7          # transactions before this time are development rows, the rest are the forward holdout
FRAUD_RATE, FRAUD_MULTIPLE = 0.06, 3.0


def generate(per_key, seed):
    """One transaction log sorted by time. Customers differ a lot in spend level; fraud is 3x the customer's own level.
    The pipeline never sees the customer, only a pseudo-ID key shared by `per_key` customers chosen at random."""
    rng = np.random.default_rng(seed)
    level = np.exp(rng.normal(3.5, 1.2, N_CUSTOMERS))           # each customer's typical amount
    customer = np.repeat(np.arange(N_CUSTOMERS), rng.integers(6, 15, N_CUSTOMERS))
    time = rng.random(len(customer))
    fraud = (rng.random(len(customer)) < FRAUD_RATE).astype(int)
    amount = level[customer] * np.exp(rng.normal(0, 0.5, len(customer)) + fraud * np.log(FRAUD_MULTIPLE))
    key = (rng.permutation(N_CUSTOMERS) // per_key)[customer]   # per_key = 1 is a perfect entity ID
    order = np.argsort(time, kind="stable")
    return time[order], key[order], amount[order], fraud[order]


def history_means(time, key, amount):
    """Two entity means for every row. Past-only: earlier rows of the same key. Whole-period: every development
    row of the key (past, present and future) for development rows; rows seen up to now for holdout rows."""
    n = len(amount)
    dev = time < CUTOFF
    past, to_date = np.full(n, np.nan), np.empty(n)
    total, count = {}, {}
    for i in range(n):                                          # rows are in time order
        s, c = total.get(key[i], 0.0), count.get(key[i], 0)
        past[i] = s / c if c else np.nan                        # the first row of a key has no history
        total[key[i]], count[key[i]] = s + amount[i], c + 1
        to_date[i] = total[key[i]] / count[key[i]]              # includes the current row, no future
    dev_total, dev_count = {}, {}
    for i in np.where(dev)[0]:
        dev_total[key[i]] = dev_total.get(key[i], 0.0) + amount[i]
        dev_count[key[i]] = dev_count.get(key[i], 0) + 1
    whole = np.array([dev_total[key[i]] / dev_count[key[i]] if dev[i] else to_date[i] for i in range(n)])
    return dev, past, whole


def columns(amount, mean):
    """log(amount / mean) plus a flag for rows with no history (ratio set to 0 there)."""
    missing = np.isnan(mean)
    ratio = np.where(missing, 0.0, np.log(amount) - np.log(np.where(missing, 1.0, mean)))
    return [ratio, missing.astype(float)]


def model():
    return HistGradientBoostingClassifier(max_iter=80, learning_rate=0.1, max_depth=3, random_state=0)


def one_log(per_key, seed):
    time, key, amount, y = generate(per_key, seed)
    dev, past, whole = history_means(time, key, amount)
    sets = {"raw": [np.log(amount)],
            "past_only": [np.log(amount)] + columns(amount, past),
            "whole_period": [np.log(amount)] + columns(amount, whole)}
    out = {}
    for name, cols in sets.items():
        X = np.column_stack(cols)
        Xd, yd, Xh, yh = X[dev], y[dev], X[~dev], y[~dev]
        oof = np.zeros(len(yd))
        for a, b in StratifiedKFold(5, shuffle=True, random_state=1).split(Xd, yd):
            oof[b] = model().fit(Xd[a], yd[a]).predict_proba(Xd[b])[:, 1]
        out[name] = {"cv": roc_auc_score(yd, oof),               # development estimate: 5-fold on development rows
                     "holdout": roc_auc_score(yh, model().fit(Xd, yd).predict_proba(Xh)[:, 1])}
    out["rows"] = [int(dev.sum()), int((~dev).sum())]
    return out


def run(customers_per_key):
    logs = [one_log(customers_per_key, SEED * 100 + i) for i in range(REPLICATES)]
    names = ["raw", "past_only", "whole_period"]
    result = {"customers_per_key": customers_per_key, "replicates": REPLICATES, "customers": N_CUSTOMERS,
              "development_rows": float(np.mean([g["rows"][0] for g in logs])),
              "holdout_rows": float(np.mean([g["rows"][1] for g in logs]))}
    for name in names:
        result[name] = {m: float(np.mean([g[name][m] for g in logs])) for m in ("cv", "holdout")}
        result[name]["holdout_per_log"] = [g[name]["holdout"] for g in logs]
    return clean(result)

import pandas as pd
from kaggle_companion.activities.ch07 import explain

VALUES = [1, 2, 5, 20]
results = {value: run(value) for value in VALUES}
table = pd.DataFrame({str(v): explain(results[v], v)['metrics'] for v in VALUES})
print(table.to_string())

import matplotlib.pyplot as plt
from kaggle_companion.activities.ch07 import draw, NCOLS, HEIGHT

DEFAULT = 1
fig, axes = plt.subplots(1, NCOLS, figsize=(6.6 if NCOLS == 1 else 10.4, HEIGHT), layout='constrained')
draw(axes, results[DEFAULT], DEFAULT)
plt.show()