Worked exampleOne request, three answers

Show me the common words in this file. Nothing in that sentence says what a word is, or what common means. Three faithful readings, three different answers.

Run it in Colab

The notebook carries the source, the install step and every run below, so nothing has to be on your machine. Open it, then choose Runtime › Run all.

Colab opens it read only. Click Copy to Drive to keep your changes. You can also download bridging_the_gap.ipynb, or run it locally: python3 bridging_the_gap.py.

What to try

1. python3 bridging_the_gap.py

   All three readings of the same request, and the counts each one returns.

The source

Download bridging_the_gap.py · served verbatim at https://nlp.jcrlabz.com/code/bridging_the_gap.py

"""Bridging the gap: one English sentence, three defensible programs.

Backs the "Bridging the Gap" frames in Latex/FrameLibrary/L1L2Review.tex.
The request is "Show me the common words in this file". Nothing in that
sentence fixes what a word is, or what common means. Each reading below is a
faithful implementation of the same sentence, and each returns a different
answer.

Run: python3 bridging_the_gap.py

Install:  nothing, the Python standard library is enough
"""

import re
from collections import Counter

TEXT = "The cat sat on the mat. The Cat did not sit on the dog."

STOPWORDS = {"the", "on", "did", "not", "a", "an", "and", "of", "to", "is"}


def reading_a(text):
    """Word means whatever whitespace separates. Common means most frequent."""
    return Counter(text.split())


def reading_b(text):
    """Word means letters only, case folded. Common means most frequent."""
    return Counter(re.findall(r"[a-z]+", text.lower()))


def reading_c(text):
    """Reading B, minus the words that carry no topic."""
    counts = reading_b(text)
    return Counter({w: c for w, c in counts.items() if w not in STOPWORDS})


def top(counts, k=3):
    """Rank by count, break ties alphabetically so the answer is repeatable."""
    return sorted(counts.items(), key=lambda kv: (-kv[1], kv[0]))[:k]


def rank_of(counts, word):
    order = sorted(counts.items(), key=lambda kv: (-kv[1], kv[0]))
    for i, (w, _) in enumerate(order, start=1):
        if w == word:
            return i
    return None


def show(name, counts):
    print(f"\n{name}")
    print(f"  token count : {sum(counts.values())}")
    print(f"  vocabulary  : {len(counts)}")
    print(f"  top 3       : {top(counts)}")
    print(f"  full        : {top(counts, k=len(counts))}")


if __name__ == "__main__":
    print(f"input: {TEXT}")

    a = reading_a(TEXT)
    b = reading_b(TEXT)
    c = reading_c(TEXT)

    show("Reading A  split on spaces, keep case, keep punctuation", a)
    show("Reading B  lower case, letters only", b)
    show("Reading C  reading B without stop words", c)

    print("\nWhere does 'cat' land?")
    for name, counts in (("A", a), ("B", b), ("C", c)):
        hits = counts.get("cat", 0)
        print(f"  reading {name}: count {hits}, rank {rank_of(counts, 'cat')}")

    print("\nTokens produced by reading A:")
    print(" ", TEXT.split())
    print("Tokens produced by reading B:")
    print(" ", re.findall(r"[a-z]+", TEXT.lower()))