← All files for this project · the case study
build_page.py
"""Build the case study page (page/index.html) from results.json. Every number on the page comes from results.json.Run: python3 build_page.py (after python3 analysis.py)The page is self-contained (no scripts), in the tabrizianalytics.com look."""import htmlimport jsonfrom pathlib import Path R = json.loads(Path("results.json").read_text())NAMES = R["names"]TYPES = ["leaks_lid", "temperature", "damage", "wrong_item", "size_design", "cleaning_taste", "price"]BRANDS = [b["brand"] for b in R["by_brand"]]E = html.escape def pct(x, d=0): return "n/a" if x is None else f"{x * 100:.{d}f}%" def reliability_svg(before, after): W, H, L, B, T, Rm = 460, 300, 46, 40, 14, 14 X = lambda v: L + (v - 0.5) / 0.5 * (W - L - Rm) Y = lambda v: H - B - (v - 0.5) / 0.5 * (H - B - T) s = [f'<svg viewBox="0 0 {W} {H}" role="img" aria-label="Calibration: what the model said against how often it was right">'] for v in [0.5, 0.6, 0.7, 0.8, 0.9, 1.0]: s.append(f'<line x1="{L}" x2="{W-Rm}" y1="{Y(v):.0f}" y2="{Y(v):.0f}" class="grid"/>' f'<text x="{L-6}" y="{Y(v)+4:.0f}" text-anchor="end">{round(v*100)}%</text>' f'<text x="{X(v):.0f}" y="{H-B+16}" text-anchor="middle">{round(v*100)}%</text>') s.append(f'<line x1="{X(0.5):.0f}" y1="{Y(0.5):.0f}" x2="{X(1):.0f}" y2="{Y(1):.0f}" class="diag"/>') for pts, cls in [(before, "b"), (after, "a")]: pts = [p for p in pts if p["n"] >= 20] # bands with under 20 reviews are too few to judge d = " ".join(f"{X(p['said']):.0f},{Y(max(p['right'], 0.5)):.0f}" for p in pts) s.append(f'<polyline points="{d}" class="ln {cls}"/>') for p in pts: s.append(f'<circle cx="{X(p["said"]):.0f}" cy="{Y(max(p["right"], 0.5)):.0f}" r="{3 + min(p["n"], 400) ** 0.5 / 4:.1f}" class="pt {cls}"/>') s.append(f'<text x="{(L + W - Rm) / 2:.0f}" y="{H-6}" text-anchor="middle" class="ax">How sure the model said it was</text>' f'<text x="12" y="{(T + H - B) / 2:.0f}" text-anchor="middle" transform="rotate(-90 12 {(T + H - B) / 2:.0f})" class="ax">How often it was right</text>') s.append("</svg>") return "".join(s) def heat_table(): rows = [] head = "".join(f"<th>{E(NAMES[t])}</th>" for t in TYPES + ["other"]) for b in R["by_brand"]: cells = [] for t in TYPES + ["other"]: v = b[t] alpha = min(1, v / 0.6) cells.append(f'<td style="background:color-mix(in srgb, var(--accent) {alpha * 70:.0f}%, transparent)">{pct(v)}</td>') rows.append(f'<tr><th class="b">{E(b["brand"])}<span>{b["unhappy"]} of {b["reviews"]} reviews ({pct(b["unhappy_share"])}) are 1 or 2 stars</span></th>{"".join(cells)}</tr>') return f'<div class="scroll"><table class="heat"><thead><tr><th></th>{head}</tr></thead><tbody>{"".join(rows)}</tbody></table></div>' def top_complaints(b, k=2): pairs = sorted(((b[t], t) for t in TYPES), reverse=True)[:k] # "other" is not a complaint type return " and ".join(f"{NAMES[t].lower()} ({pct(v)})" for v, t in pairs) mood, cal, routes, human = R["mood"], R["calibration"], R["routing"], R["human_check"]chk = R["checks"]pick = next((r for r in routes if r["cut"] == 0.9), routes[-1])yeti = next(b for b in R["by_brand"] if b["brand"] == "YETI")others = [b for b in R["by_brand"] if b["brand"] != "YETI"]lid = R["yeti_missing_lid_part"]top3 = sorted(((yeti[t], t) for t in TYPES), reverse=True)[:3]# The YETI lines below are written for these three complaint types leading YETI's list; stop if new data changes that.assert {t for _, t in top3} == {"wrong_item", "leaks_lid", "temperature"}, top3stars_line = ", ".join(f"{b['brand']} {pct(b['unhappy_share'], 1)}" for b in others)few, many = min(b["unhappy"] for b in R["by_brand"]), max(b["unhappy"] for b in R["by_brand"]) if human: # filled in once data/my_labels.csv exists (analysis.py Step 7) human_card = f"""<section class="card"> <p class="kicker">Check 3</p> <h2>Does the LLM agree with a person on what the complaint is?</h2> <p>I labeled {human['n']} one- and two-star reviews myself, without seeing either AI's answers, and compared both with my labels.</p> <div class="facts"><div class="fact"><b>{pct(human['llm_agree'])}</b><span>of {human['n']} reviews: the LLM's main complaint matched mine (kappa {human['llm_kappa']:.2f})</span></div> <div class="fact"><b>{pct(human['dm_agree'])}</b><span>the decision model's match on the same reviews (kappa {human['dm_kappa']:.2f})</span></div></div> <table class="plain"><thead><tr><th>Main complaint (my label)</th><th>Reviews</th><th>LLM agreed</th><th>Decision model agreed</th></tr></thead><tbody> {''.join(f"<tr><td>{E(p['name'])}</td><td>{p['mine']}</td><td>{p['llm_matched']}</td><td>{p['dm_matched']}</td></tr>" for p in human['per_type'])} </tbody></table></section>"""else: human_card = ""page = f"""<!doctype html><html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1"><title>What makes drinkware buyers unhappy · Tabrizi Analytics</title><link rel="icon" href="/favicon.svg" type="image/svg+xml"><link rel="preconnect" href="https://fonts.googleapis.com"><link rel="preconnect" href="https://fonts.gstatic.com" crossorigin><link rel="stylesheet" href="https://fonts.googleapis.com/css2?family=Inter:wght@400;500;600;700&family=JetBrains+Mono:wght@400;600&display=swap"><style>:root{{--bg:#fbfbfc;--surface:#fff;--panel:#f2f4f7;--ink:#111827;--ink2:#2b3341;--muted:#5d6675;--line:#e2e6ec;--grid:#edf0f4;--accent:#3346b8;--soft:#eef0fb;--s2:#0f8a7e;--bar:#161b2e}}@media (prefers-color-scheme: dark){{:root:not([data-theme="light"]){{--bg:#0d1018;--surface:#141925;--panel:#1a2030;--ink:#eef1f7;--ink2:#cdd3de;--muted:#98a1b2;--line:#283043;--grid:#1e2534;--accent:#8fa0ff;--soft:#1f2547;--s2:#43c6b8;--bar:#090b12;color-scheme:dark}}}}:root[data-theme="dark"]{{--bg:#0d1018;--surface:#141925;--panel:#1a2030;--ink:#eef1f7;--ink2:#cdd3de;--muted:#98a1b2;--line:#283043;--grid:#1e2534;--accent:#8fa0ff;--soft:#1f2547;--s2:#43c6b8;--bar:#090b12;color-scheme:dark}}*{{box-sizing:border-box}} html,body{{margin:0}}body{{background:var(--bg);color:var(--ink2);font:16.5px/1.6 Inter,"Helvetica Neue",Arial,sans-serif;-webkit-font-smoothing:antialiased}}a{{color:var(--accent)}}.top{{background:var(--bar);color:#fff;padding:11px 16px;font-size:13.5px}} .top a{{color:#fff;text-decoration:none}} .top a:hover{{text-decoration:underline}}.wrap{{max-width:920px;margin:0 auto;padding:0 16px 56px}}.card{{background:var(--surface);border:1px solid var(--line);border-radius:14px;padding:26px clamp(18px,4vw,40px);margin-top:16px}}.kicker{{font-size:13px;font-weight:700;color:var(--accent);margin:0 0 8px;letter-spacing:.02em}}h1{{font-size:clamp(28px,4.6vw,42px);line-height:1.12;letter-spacing:-.02em;color:var(--ink);margin:0 0 14px;text-wrap:balance}}h2{{font-size:clamp(22px,3vw,28px);line-height:1.2;letter-spacing:-.015em;color:var(--ink);margin:0 0 10px;text-wrap:balance}}h3{{font-size:17px;color:var(--ink);margin:18px 0 6px}}p{{margin:0 0 12px;max-width:70ch}} .lede{{font-size:18px}} .note{{color:var(--muted);font-size:14.5px}}.facts{{display:grid;grid-template-columns:repeat(auto-fit,minmax(170px,1fr));gap:10px;margin:16px 0}}.fact{{background:var(--panel);border-radius:10px;padding:12px 14px}} .fact b{{display:block;font-size:24px;color:var(--ink);letter-spacing:-.01em}}.fact span{{display:block;font-size:13px;line-height:1.35;color:var(--muted);margin-top:2px}}ul.tight,ol.tight{{margin:6px 0 12px;padding-left:20px}} .tight li{{margin:5px 0}}.so{{border-left:4px solid var(--accent);background:var(--soft);border-radius:0 10px 10px 0;padding:10px 14px;margin:14px 0 4px}}table.plain{{border-collapse:collapse;width:100%;font-size:15px;margin:10px 0}} table.plain th,table.plain td{{text-align:left;padding:8px 10px;border-bottom:1px solid var(--line)}}table.plain th{{font-size:12.5px;text-transform:uppercase;letter-spacing:.04em;color:var(--muted)}}.scroll{{overflow-x:auto}} table.heat{{border-collapse:separate;border-spacing:3px;font-size:14px;min-width:720px}}table.heat th{{font-size:12px;font-weight:600;color:var(--muted);text-align:center;padding:4px;vertical-align:bottom}}table.heat th.b{{text-align:left;color:var(--ink);font-size:14.5px;min-width:150px}} table.heat th.b span{{display:block;font-weight:400;font-size:12px;color:var(--muted)}}table.heat td{{text-align:center;padding:10px 6px;border-radius:6px;color:var(--ink);font-weight:600}}figure{{margin:12px 0}} figure svg{{width:100%;max-width:520px;height:auto;display:block}}figure text{{font:12px Inter,Arial,sans-serif;fill:var(--muted)}} figure .ax{{font-size:12.5px;fill:var(--ink2)}}.grid{{stroke:var(--grid)}} .diag{{stroke:var(--muted);stroke-dasharray:5 5}}.ln{{fill:none;stroke-width:2.5}} .ln.b{{stroke:var(--muted)}} .ln.a{{stroke:var(--accent)}}.pt.b{{fill:var(--muted)}} .pt.a{{fill:var(--accent)}}.key i{{display:inline-block;width:10px;height:10px;border-radius:50%;margin:0 6px 0 12px;vertical-align:0}}details.how{{border:1px solid var(--line);border-radius:10px;margin-top:14px}} details.how summary{{cursor:pointer;padding:10px 14px;font-weight:600}}details.how .hb{{padding:2px 16px 12px;border-top:1px solid var(--line);font-size:15px}}pre{{background:var(--panel);border-radius:8px;padding:10px 12px;overflow:auto;font:13px/1.5 "JetBrains Mono",ui-monospace,Menlo,monospace}}.q{{font-family:"JetBrains Mono",ui-monospace,monospace;font-size:13.5px;background:var(--panel);border-radius:8px;padding:10px 12px}}footer{{color:var(--muted);font-size:13.5px;padding:24px 16px 40px;text-align:center}}</style></head><body><div class="top"><a href="/">← Tabrizi Analytics</a> · Drinkware reviews · <a href="code/">Code</a></div><main class="wrap"> <section class="card"> <p class="kicker">Case study · consumer insights</p> <h1>What makes drinkware buyers unhappy, and can an AI sort the reviews?</h1> <p class="lede">I took {chk['rows']:,} Amazon reviews of insulated drinkware from four brands ({', '.join(BRANDS)}) and asked two questions: what do unhappy buyers write about, and can an AI read reviews well enough to watch them every month? An open AI model read every review, an LLM (Claude) named the main complaint in the {R['unhappy_total']} unhappy ones, and I checked the AI against the reviewers' own stars before using any of it.</p> <div class="facts"> <div class="fact"><b>{lid['n']} of {yeti['unhappy']}</b><span>YETI one- and two-star reviews say a lid piece was missing, {lid['slider']} of them the MagSlider</span></div> <div class="fact"><b>{pct(yeti['unhappy_share'], 1)}</b><span>of YETI's reviews are 1 or 2 stars ({stars_line})</span></div> <div class="fact"><b>{pct(pick['auto_share'])}</b><span>of reviews the AI sorts on its own at {pct(pick['auto_right'], 1)} accuracy; a person reads the rest</span></div> </div> <h3>What I found</h3> <ul class="tight"> <li><b>YETI's unhappy reviews split three ways, and one is easy to fix.</b> {NAMES[top3[0][1]]} ({pct(top3[0][0])}), {NAMES[top3[1][1]].lower()} ({pct(top3[1][0])}) and {NAMES[top3[2][1]].lower()} ({pct(top3[2][0])}). The missing parts stand out: {lid['n']} of those {lid['of']} reviews say a lid piece was missing, {lid['slider']} of them the MagSlider.</li> <li><b>Each brand has its own weak spot.</b> {'; '.join(f"{b['brand']}: {top_complaints(b)}" for b in others)}.</li> <li><b>The AI is good enough to read every review.</b> It read the mood the way the stars say {pct(mood['accuracy'], 1)} of the time (kappa {mood['kappa']:.2f}), and after a one-number fix its confidence means what it says, so a cut-off can decide which reviews a person reads.</li> </ul> <h3>What YETI could do with it</h3> <ol class="tight"> <li><b>Find out where the lid pieces go missing.</b> Packing, returned units sold again as new, or third-party sellers: each has a different fix, and the review dates and listings help tell them apart.</li> <li><b>Watch the temperature complaints.</b> Keeping drinks cold or hot is the core promise, and it is {pct(yeti['temperature'])} of YETI's unhappy reviews. A monthly view shows whether that share moves after a product or lid change.</li> <li><b>Let the AI read every review, every month.</b> At a 90% cut-off it sorts {pct(pick['auto_share'])} on its own and is right {pct(pick['auto_right'], 1)} of the time; a person reads the other {pct(1 - pick['auto_share'])}, the hard cases.</li> </ol></section> <section class="card"> <p class="kicker">The main finding</p> <h2>What unhappy buyers write about, by brand</h2> <p>Each one- and two-star review got one main complaint, named by the LLM with a short codebook. Each row adds up to 100%.</p> {heat_table()} <div class="so"><b>So what.</b> {'; '.join(f"{b['brand']}: {top_complaints(b)}" for b in R['by_brand'])}. With {few} to {many} unhappy reviews per brand, these are directions to check, not final numbers; a monthly tracker would confirm them.</div> <h3>Reading behind YETI's missing parts</h3> <p>I read the {lid['of']} YETI reviews about a wrong or missing part. {lid['n']} say a lid piece was missing, and {lid['slider']} of those name the slider (the MagSlider). The other {lid['of'] - lid['n']}: a straw lid that didn't fit, a handle that didn't match the listing, and items that seemed used or fake. So the fix sits with packing, returns and sellers more than with product design.</p></section> <section class="card"> <p class="kicker">The data</p> <h2>{chk['rows']:,} real reviews, checked before use</h2> <p>Source: <a href="https://huggingface.co/datasets/McAuley-Lab/Amazon-Reviews-2023" target="_blank" rel="noopener">Amazon Reviews 2023</a> (McAuley Lab, UC San Diego), the newest public release of Amazon reviews. I kept insulated tumblers, mugs and bottles from the four brands, the newest 21 months in the release ({chk['first_day']} to {chk['last_day']}), and drew 500 reviews per brand at random. A title filter dropped coolers and most accessories; a few lids and accessories sold on their own (mostly Hydro Flask's press-in lid) got through and are kept.</p> <div class="facts"> <div class="fact"><b>{chk['rows']:,}</b><span>reviews, 500 per brand</span></div> <div class="fact"><b>{chk['duplicate_ids']} / {chk['empty_text']}</b><span>duplicate IDs / empty reviews</span></div> <div class="fact"><b>{chk['spanish']}</b><span>Spanish reviews set aside (this model reads English)</span></div> <div class="fact"><b>{pct(chk['stars'][4] / chk['rows'])}</b><span>are 5 stars; {chk['stars'][0] + chk['stars'][1]} are 1 or 2 stars</span></div> </div> <p class="note">Limits: Amazon reviewers are not all buyers, and the release ends in September 2023. The method is the point: the same steps run on a brand's own reviews or survey comments.</p></section> <section class="card"> <p class="kicker">How the AI read the reviews</p> <h2>A fast model for every review, an LLM for the complaints</h2> <p><b>Step 1.</b> <a href="https://huggingface.co/convaiinnovations/laya" target="_blank" rel="noopener">Laya</a> is an open "decision model" (Apache 2.0, 421M parameters, runs on a laptop). Instead of writing text, it answers typed questions with a probability for each answer, so there is nothing to parse and nothing it can make up. Each review got these questions:</p> <div class="q">Is the reviewer happy with the product? A: yes · B: no<br> For 1-2 star reviews, seven yes/no questions: does the reviewer complain that it leaks or the lid is a problem · it doesn't keep drinks cold or hot · it arrived damaged or broke · a part was wrong or missing · the size, weight or design · cleaning, smell or taste · the price?</div> <p style="margin-top:12px"><b>Step 2.</b> Used as is, the decision model is very good at mood but mixes up complaint types: it agreed with the LLM on only {pct(R['coders_agree'])} of the unhappy reviews. So the {R['unhappy_total']} unhappy reviews also went to <b>Claude</b> with a one-paragraph codebook: the same complaint types and the rule "pick the one the reviewer stresses most". The complaint numbers on this page come from Claude.</p> <h3>Check 1 · Does it read the mood the way the reviewer rated it?</h3> <p>The stars are the reviewer's own verdict: 4-5 stars = happy, 1-2 = unhappy (3 stars are mixed and left out).</p> <table class="plain"><thead><tr><th></th><th>Model: happy</th><th>Model: unhappy</th></tr></thead><tbody> <tr><td>Stars: happy (4-5)</td><td>{mood['table']['stars_happy_ai_happy']:,}</td><td>{mood['table']['stars_happy_ai_unhappy']:,}</td></tr> <tr><td>Stars: unhappy (1-2)</td><td>{mood['table']['stars_unhappy_ai_happy']:,}</td><td>{mood['table']['stars_unhappy_ai_unhappy']:,}</td></tr></tbody></table> <div class="so"><b>So what.</b> It agreed {pct(mood['accuracy'], 1)} of the time (kappa {mood['kappa']:.2f}) and missed only {mood['unhappy_missed']} of {mood['unhappy_n']} unhappy reviews. Reading those misses: most are calm reports of a missing piece or a dent from buyers who still like the product, and one reads like a five-star review.</div> <h3>Check 2 · When it says "90% sure", is it right 90% of the time?</h3> <p>I grouped the reviews by how sure the model said it was and checked how often it was right. Then I fixed it with one number (a "temperature", T = {cal['T']}), learned on one half of the reviews and tested on the other half.</p> <figure>{reliability_svg(cal['bands_before'], cal['bands_after'])} <figcaption class="note"><span class="key"><i style="background:var(--muted)"></i>Before the fix<i style="background:var(--accent)"></i>After the fix</span> · dashed line = perfectly calibrated · dot size = number of reviews · bands with fewer than 20 reviews are left off the chart (too few to judge)</figcaption></figure> <div class="so"><b>So what.</b> Calibration error on the test half went from {cal['ece_test_before']:.3f} to {cal['ece_test_after']:.3f}. {"Before the fix the model was right more often than it claimed (under-confident), so the fix sharpened its probabilities." if cal['T'] < 1 else "Before the fix the model claimed more than it delivered (over-confident), so the fix softened its probabilities."} Now a cut-off on its confidence means something.</div> <h3>The decision · Which reviews can it handle alone?</h3> <table class="plain"><thead><tr><th>Cut-off (model at least this sure)</th><th>Handled by the model</th><th>Right when it handled them</th><th>Sent to a person</th></tr></thead><tbody> {''.join(f"<tr><td>{pct(r['cut'])}</td><td>{pct(r['auto_share'])}</td><td>{pct(r['auto_right'], 1)}</td><td>{r['to_person']} of {cal['test_n']}</td></tr>" for r in routes)} </tbody></table> <div class="so"><b>So what.</b> At a 90% cut-off the model handles {pct(pick['auto_share'])} of reviews and is right {pct(pick['auto_right'], 1)} of the time; a person reads the rest. That is the human-in-the-loop setup: the model does the volume, people do the hard cases.</div></section>{human_card}<section class="card"> <p class="kicker">Next</p> <h2>What I'd set up to run every month</h2> <ol class="tight"> <li><b>A review tracker.</b> Every new review scored the same way: mood, complaint type, model confidence. One table by product and month.</li> <li><b>A check set.</b> 100 reviews labeled by a person, re-run whenever the model or the questions change, so a drop in agreement shows up before the numbers move. It also measures how far to trust the complaint types.</li> <li><b>A routing rule with a budget.</b> The cut-off set by how many reviews a person can read each week.</li> <li><b>Alerts.</b> When one complaint type jumps for a product (for example leaks after a lid change), flag it to the product team with five example reviews.</li> </ol> <details class="how"><summary>How it's built (code, files, limits)</summary><div class="hb"> <p><b>fetch_data.py</b> streams the public review files (41 GB) and draws the sample · <b>run_model.py</b> asks the decision model the questions and saves its answers (ai_answers.csv) · <b>llm_code.py</b> has Claude name the main complaint of each unhappy review (llm_complaints.csv) · <b>analysis.py</b> runs the checks and the brand comparison and writes results.json · <b>build_page.py</b> builds this page from results.json. All files: <a href="code/">code</a>.</p> <pre>m["ai_happy"] = (m["p_happy"] >= 0.5).astype(int)accuracy = (m["ai_happy"] == m["happy_by_stars"]).mean()p_fixed = p**(1/T) / (p**(1/T) + (1-p)**(1/T)) # the one-number fix (temperature)</pre> <p class="note">Limits: one marketplace and one period; Amazon reviews skew to strong opinions; the complaint types are one LLM's reading and still need a person's check on a labeled sample (the check set above); the decision model was used as is, with no training on this data.</p> </div></details></section></main><footer><a href="/">Tabrizi Analytics</a> · a case study on public data. Not affiliated with any brand in the data.Data: Amazon Reviews 2023 (McAuley Lab, UC San Diego). Models: Laya (Convai Innovations, Apache 2.0) and Claude (Anthropic).</footer></body></html>"""Path("page").mkdir(exist_ok=True)Path("page/index.html").write_text(page)print("Wrote page/index.html")