Chapter 47: Medical Imaging Patterns

import numpy as np
from sklearn.ensemble import ExtraTreesClassifier
from sklearn.metrics import roc_auc_score
from sklearn.model_selection import GroupKFold, KFold

from kaggle_companion.activities._common import clean

SEED = 47
DRAWS = 8            # independent cohorts; every estimate is a mean over draws
PATIENTS = 150       # development patients; 600 further patients form the fresh test cohort
FRESH = 600
DIM = 12             # image features


def generate(n_patients, images_each, rng):
    """Patient-level label. Each image = the patient's own anatomy (shared by all their images) + a weak disease signal + image noise."""
    label = (rng.random(n_patients) < 0.5).astype(int)
    anatomy = 1.5 * rng.normal(size=(n_patients, DIM))
    patient = np.repeat(np.arange(n_patients), images_each)
    y = label[patient]
    X = anatomy[patient] + 0.8 * rng.normal(size=(len(patient), DIM))
    X[:, :3] += 0.5 * (2 * y[:, None] - 1)
    return X, y, patient


def model(seed=0):
    return ExtraTreesClassifier(n_estimators=40, max_features=0.5, random_state=seed, n_jobs=1)


def pooled_auc(X, y, splits):
    """Out-of-fold probabilities pooled over all folds, then one AUC."""
    oof = np.zeros(len(y))
    for fit, hold in splits:
        oof[hold] = model().fit(X[fit], y[fit]).predict_proba(X[hold])[:, 1]
    return roc_auc_score(y, oof)


def run(images_per_patient):
    rng = np.random.default_rng(SEED)
    X_new, y_new, _ = generate(FRESH, images_per_patient, rng)       # fresh patients: the truth the CV should predict
    rows = {k: [] for k in ("kfold", "grouped", "fresh", "overlap_kfold", "overlap_grouped")}
    for draw in range(DRAWS):
        X, y, patient = generate(PATIENTS, images_per_patient, rng)
        kfold = list(KFold(5, shuffle=True, random_state=draw).split(X))              # each image on its own
        grouped = list(GroupKFold(5).split(X, y, patient))                              # whole patients together
        rows["kfold"].append(pooled_auc(X, y, kfold))
        rows["grouped"].append(pooled_auc(X, y, grouped))
        rows["fresh"].append(roc_auc_score(y_new, model().fit(X, y).predict_proba(X_new)[:, 1]))
        # The chapter's check: do validation patients also appear in training? (share of validation images)
        rows["overlap_kfold"].append(np.mean([np.isin(patient[b], patient[a]).mean() for a, b in kfold]))
        rows["overlap_grouped"].append(np.mean([np.isin(patient[b], patient[a]).mean() for a, b in grouped]))
    mean = {k: float(np.mean(v)) for k, v in rows.items()}
    return clean({
        "images_per_patient": images_per_patient, **mean,
        "kfold_error": mean["kfold"] - mean["fresh"], "grouped_error": mean["grouped"] - mean["fresh"],
        "per_draw": {k: rows[k] for k in ("kfold", "grouped", "fresh")},
        "patients": PATIENTS, "images": PATIENTS * images_per_patient, "fresh_patients": FRESH, "draws": DRAWS,
    })

import pandas as pd
from kaggle_companion.activities.ch47 import explain

VALUES = [1, 2, 4, 8]
results = {value: run(value) for value in VALUES}
table = pd.DataFrame({str(v): explain(results[v], v)['metrics'] for v in VALUES})
print(table.to_string())

import matplotlib.pyplot as plt
from kaggle_companion.activities.ch47 import draw, NCOLS, HEIGHT

DEFAULT = 4
fig, axes = plt.subplots(1, NCOLS, figsize=(6.6 if NCOLS == 1 else 10.4, HEIGHT), layout='constrained')
draw(axes, results[DEFAULT], DEFAULT)
plt.show()