Chapter 49: Text Feature Engineering

import numpy as np
from scipy.sparse import hstack
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.linear_model import LogisticRegression
from sklearn.metrics import log_loss

from kaggle_companion.activities._common import clean

SEED = 49
DRAWS = 8                  # independent corpora; every estimate is a mean over draws
N_DOCS, N_CLASSES, TITLE_WORDS = 800, 4, 7
LETTERS = np.array(list("abcdefghijklmnopqrstuvwxyz"))


def invent_words(rng, n):
    """Pronounceable-enough made-up words of 5 to 9 letters (no real language, so nothing is memorised from outside)."""
    return ["".join(rng.choice(LETTERS, rng.integers(5, 10))) for _ in range(n)]


def misspell(word, rng):
    """One typo: substitute a letter, drop a letter or swap two neighbours."""
    i, kind = rng.integers(len(word)), rng.integers(3)
    if kind == 0:
        return word[:i] + str(rng.choice(LETTERS)) + word[i + 1:]
    if kind == 1 and len(word) > 3:
        return word[:i] + word[i + 1:]
    return word[:i] + word[i + 1] + word[i] + word[i + 2:] if i < len(word) - 1 else word


def titles(n, vocabulary, rng, typo_rate=0.0):
    """Product-style titles: 30% of words come from the class's own list, the rest from a shared Zipf-weighted background."""
    own, background = vocabulary
    weights = 1 / np.arange(1, len(background) + 1)
    weights /= weights.sum()
    labels = rng.integers(0, N_CLASSES, n)
    docs = []
    for c in labels:
        words = [own[c][rng.integers(len(own[c]))] if rng.random() < 0.3 else background[rng.choice(len(background), p=weights)]
                 for _ in range(TITLE_WORDS)]
        docs.append(" ".join(misspell(w, rng) if rng.random() < typo_rate else w for w in words))
    return docs, labels


def coverage(train_docs, test_docs):
    """Step Zero: the share of assessment tokens (counted with repeats, so weighted by frequency) seen in training."""
    seen = set(" ".join(train_docs).split())
    tokens = " ".join(test_docs).split()
    return float(np.mean([t in seen for t in tokens]))


def run(typo_rate):
    rng = np.random.default_rng(SEED)
    names = ("word", "char", "both")
    rows = {k: {"log_loss": [], "accuracy": []} for k in names}
    cover = []
    for _ in range(DRAWS):
        vocabulary = ([invent_words(rng, 40) for _ in range(N_CLASSES)], invent_words(rng, 200))
        train, y_train = titles(N_DOCS, vocabulary, rng)                        # clean training text
        test, y_test = titles(N_DOCS, vocabulary, rng, typo_rate)               # typos appear only here
        cover.append(coverage(train, test))
        word = TfidfVectorizer(analyzer="word", sublinear_tf=True)             # vocabularies are fitted on training text only
        char = TfidfVectorizer(analyzer="char", ngram_range=(3, 5), sublinear_tf=True)
        Wtr, Ctr, Wte, Cte = word.fit_transform(train), char.fit_transform(train), word.transform(test), char.transform(test)
        for name, A, B in (("word", Wtr, Wte), ("char", Ctr, Cte), ("both", hstack([Wtr, Ctr]).tocsr(), hstack([Wte, Cte]).tocsr())):
            p = LogisticRegression(C=10, max_iter=500).fit(A, y_train).predict_proba(B)
            rows[name]["log_loss"].append(log_loss(y_test, p, labels=range(N_CLASSES)))
            rows[name]["accuracy"].append((p.argmax(1) == y_test).mean())
    advantage = np.array(rows["word"]["log_loss"]) - np.array(rows["char"]["log_loss"])      # positive: characters are better
    return clean({
        "typo_rate": typo_rate, "coverage": np.mean(cover),
        **{k: {"log_loss": np.mean(v["log_loss"]), "accuracy": np.mean(v["accuracy"])} for k, v in rows.items()},
        "char_advantage": advantage.mean(), "char_advantage_se": advantage.std(ddof=1) / np.sqrt(DRAWS),
        "char_win_rate": float((advantage > 0).mean()),
        "per_draw": {k: v["log_loss"] for k, v in rows.items()},
        "documents": N_DOCS, "draws": DRAWS,
    })

import pandas as pd
from kaggle_companion.activities.ch49 import explain

VALUES = [0, 0.1, 0.2, 0.4]
results = {value: run(value) for value in VALUES}
table = pd.DataFrame({str(v): explain(results[v], v)['metrics'] for v in VALUES})
print(table.to_string())

import matplotlib.pyplot as plt
from kaggle_companion.activities.ch49 import draw, NCOLS, HEIGHT

DEFAULT = 0.2
fig, axes = plt.subplots(1, NCOLS, figsize=(6.6 if NCOLS == 1 else 10.4, HEIGHT), layout='constrained')
draw(axes, results[DEFAULT], DEFAULT)
plt.show()