← All files for this project · the case study

llm_code.py

"""Second AI step: an LLM (Claude) reads each one- and two-star review and names its main complaint.Why a second model: the decision model (Laya) is fast and excellent at mood, but zero-shot it is weak at tellingcomplaint types apart (its own model card says so, and it agreed with this LLM on only about 6 in 10 reviews).So the pipeline is a cascade: the fast model reads every review, the LLM reads only the unhappy 15%, and a personcan check a sample of both (label.html).data/llm_complaints.csv holds the answers used on the page: coded on 2026-10-11 by Claude (claude-opus-5-5) withexactly the PROMPT below, one main complaint per review, before any hand labels existed.Re-run (optional; uses your Claude Code login, about 12 calls of 25 reviews):    python3 llm_code.py            -> writes data/llm_complaints_rerun.csv (keeps the original file as it is)"""import jsonimport subprocess import pandas as pd CODES = ["leaks_lid", "temperature", "damage", "wrong_item", "size_design", "cleaning_taste", "price", "other"]PROMPT = """You are coding Amazon reviews of insulated drinkware for a consumer insights team.For each review below, pick its MAIN complaint from this list (exactly one code):- leaks_lid: it leaks or spills, or the lid, straw, cap or seal is hard to use or doesn't work- temperature: it doesn't keep drinks cold or hot- damage: it arrived damaged (dented, scratched), or it dented, chipped, rusted or a part broke during use- wrong_item: a wrong or missing part (e.g. a missing lid or slider), the wrong size or color, a used item, or a fake- size_design: size, weight, shape or handle; doesn't fit a cup holder- cleaning_taste: cleaning, stains, smell, taste or mold- price: too expensive for what it is (only when price is the main point)- other: anything else (delivery never arrived, made-in-country complaints, unclear)If a review has two complaints, pick the one the reviewer stresses most (often the title).Answer with JSON only: {"R0001": "code", ...}Reviews:"""  def ask_claude(batch):    text = PROMPT + "\n".join(f'{r.review_id}: {r.title}. {r.text}' for r in batch.itertuples())    out = subprocess.run(["claude", "-p", text], capture_output=True, text=True, timeout=300).stdout    return json.loads(out[out.index("{"): out.rindex("}") + 1])  if __name__ == "__main__":    reviews = pd.read_csv("data/reviews.csv").fillna("")    unhappy = reviews[(reviews["rating"] <= 2) & (reviews["language"] == "en")].sort_values("review_id")    answers = {}    for start in range(0, len(unhappy), 25):        answers.update(ask_claude(unhappy.iloc[start:start + 25]))        print(len(answers), "reviews coded")    bad = {k: v for k, v in answers.items() if v not in CODES}    if bad:        print("Answers outside the code list (check by hand):", bad)    pd.DataFrame({"review_id": list(answers), "llm_complaint": list(answers.values())}).to_csv("data/llm_complaints_rerun.csv", index=False)    print("Saved data/llm_complaints_rerun.csv")

"""Second AI step: an LLM (Claude) reads each one- and two-star review and names its main complaint.

Why a second model: the decision model (Laya) is fast and excellent at mood, but zero-shot it is weak at telling
complaint types apart (its own model card says so, and it agreed with this LLM on only about 6 in 10 reviews).
So the pipeline is a cascade: the fast model reads every review, the LLM reads only the unhappy 15%, and a person
can check a sample of both (label.html).

data/llm_complaints.csv holds the answers used on the page: coded on 2026-10-11 by Claude (claude-opus-5-5) with
exactly the PROMPT below, one main complaint per review, before any hand labels existed.

Re-run (optional; uses your Claude Code login, about 12 calls of 25 reviews):
    python3 llm_code.py            -> writes data/llm_complaints_rerun.csv (keeps the original file as it is)
"""
import json
import subprocess

import pandas as pd

CODES = ["leaks_lid", "temperature", "damage", "wrong_item", "size_design", "cleaning_taste", "price", "other"]
PROMPT = """You are coding Amazon reviews of insulated drinkware for a consumer insights team.
For each review below, pick its MAIN complaint from this list (exactly one code):
- leaks_lid: it leaks or spills, or the lid, straw, cap or seal is hard to use or doesn't work
- temperature: it doesn't keep drinks cold or hot
- damage: it arrived damaged (dented, scratched), or it dented, chipped, rusted or a part broke during use
- wrong_item: a wrong or missing part (e.g. a missing lid or slider), the wrong size or color, a used item, or a fake
- size_design: size, weight, shape or handle; doesn't fit a cup holder
- cleaning_taste: cleaning, stains, smell, taste or mold
- price: too expensive for what it is (only when price is the main point)
- other: anything else (delivery never arrived, made-in-country complaints, unclear)
If a review has two complaints, pick the one the reviewer stresses most (often the title).
Answer with JSON only: {"R0001": "code", ...}

Reviews:
"""

def ask_claude(batch):
    text = PROMPT + "\n".join(f'{r.review_id}: {r.title}. {r.text}' for r in batch.itertuples())
    out = subprocess.run(["claude", "-p", text], capture_output=True, text=True, timeout=300).stdout
    return json.loads(out[out.index("{"): out.rindex("}") + 1])

if __name__ == "__main__":
    reviews = pd.read_csv("data/reviews.csv").fillna("")
    unhappy = reviews[(reviews["rating"] <= 2) & (reviews["language"] == "en")].sort_values("review_id")
    answers = {}
    for start in range(0, len(unhappy), 25):
        answers.update(ask_claude(unhappy.iloc[start:start + 25]))
        print(len(answers), "reviews coded")
    bad = {k: v for k, v in answers.items() if v not in CODES}
    if bad:
        print("Answers outside the code list (check by hand):", bad)
    pd.DataFrame({"review_id": list(answers), "llm_complaint": list(answers.values())}).to_csv("data/llm_complaints_rerun.csv", index=False)
    print("Saved data/llm_complaints_rerun.csv")