← All files for this project · the case study

fetch_data.py

"""Build the review sample used in this case study (already done; you only run this to rebuild it from scratch).Source: Amazon Reviews 2023 (McAuley Lab, UC San Diego), the newest public release of Amazon product reviews:https://huggingface.co/datasets/McAuley-Lab/Amazon-Reviews-2023  (reviews up to September 2023).Steps:  1. Read the product files for Home & Kitchen and Sports & Outdoors and keep the insulated drinkware sold by four     brands: YETI, Stanley, Hydro Flask and Owala (tumblers, mugs, bottles; the title filter drops coolers and most     lids and accessories sold alone; a few get through, mostly Hydro Flask's press-in lid).  2. Read the review files and keep the reviews of those products.  3. Keep January 2022 to September 2023 (the newest 21 months), drop exact duplicates (the same review shown on     several color versions), and draw 500 reviews per brand at random (seed 2026), 2,000 in all.The big files are 41 GB together, so they are streamed in pieces and filtered on the fly (grep); nothing big isstored. Needs curl and grep. Takes about 5 minutes on a fast connection."""import csv, datetime as dt, html, json, random, re, subprocess BASE = "https://huggingface.co/datasets/McAuley-Lab/Amazon-Reviews-2023/resolve/main/raw"BRANDS = {"yeti": "YETI", "stanley": "Stanley", "owala": "Owala", "hydro flask": "Hydro Flask"}DRINK = re.compile(r"tumbler|bottle|mug|\bcup\b|quencher|rambler|freesip|jug|colster|lowball|growler|flask|iceflow|pint|stein|wine", re.I)NOT = re.compile(r"cooler|hopper|tundra|roadie|camino|\btote\b|\bbag\b|blanket|bucket|chair|bowl|brush|cleaner|opener|\bboot\b|sleeve|"                 r"backpack|lantern|food jar|lunch|bento|knife|cutlery|stove|grill|replacement|holder|pouch|carrier|sling|keychain|"                 r"sticker|gift card|cook ?set|cookware|\bpot\b|camp set|french press|flatware|utensil|spork|shaker", re.I)ACCESSORY = re.compile(r"^\W*(yeti|stanley|owala|hydro flask)?\s*(rambler\s+)?(bottle\s+)?(wide mouth\s+|standard mouth\s+)?"                       r"(magslider|straw|chug|flex|stronghold|flip|twist|press-in|sip|hotshot|tumbler|bottle|mug)?\s*(lid|cap|straw|spout)s?\b"                       r"|\b(lid|cap)s?\b\s*(for|,?\s*fits)\b", re.I)  def stream(url, pattern_args):    """Lines of a big remote file that match grep, without saving the file."""    p = subprocess.run(f"curl -sSL {url} | grep {pattern_args}", shell=True, capture_output=True, text=True)    return p.stdout.splitlines()  def brand_of(store):    s = (store or "").lower()    return next((v for k, v in BRANDS.items() if k in s), None)  SPANISH = re.compile(r"[ñ¿¡]|\b(muy|es|el|la|que|lo|para|excelente|producto|vaso|calidad|llegó|llego|compra|bueno|buena|pero|con|sin|una|un|mi|me|no es)\b", re.I)ENGLISH = re.compile(r"\b(the|it|and|is|i|my|this|to|a|for|of|was|in|love|great|good|cup|bottle|mug|not|but|so|very|keeps|cold|hot|ice|lid|best|yet|nice|perfect|works|size)\b", re.I)  def language(text):    """'es' when a review has more common Spanish words than common English ones (the English model can't read those)."""    return "es" if len(SPANISH.findall(text)) > len(ENGLISH.findall(text)) else "en"  def clean(text):    text = re.sub(r"\[\[VIDEOID:[^\]]*\]\]", " ", text or "")    text = re.sub(r"<br\s*/?>", " ", text)    return re.sub(r"\s+", " ", html.unescape(text)).strip()  products = {}for cat in ["Home_and_Kitchen", "Sports_and_Outdoors"]:    for line in stream(f"{BASE}/meta_categories/meta_{cat}.jsonl", "-i -E '\"store\": \"(yeti|stanley|stanley 1913|owala|hydro flask)\"'"):        p = json.loads(line)        b, title = brand_of(p.get("store")), p.get("title") or ""        if b and DRINK.search(title) and not NOT.search(title) and not ACCESSORY.search(title):            products[p["parent_asin"]] = (b, title)open("data/asin_patterns.txt", "w").write("".join(f'"parent_asin": "{a}"\n' for a in products)) reviews, seen = [], set()for cat in ["Home_and_Kitchen", "Sports_and_Outdoors"]:    for line in stream(f"{BASE}/review_categories/{cat}.jsonl", "-F -f data/asin_patterns.txt"):        r = json.loads(line)        key = (r.get("user_id"), r.get("timestamp"), (r.get("text") or "")[:60])        if r.get("parent_asin") not in products or key in seen:            continue        seen.add(key)        day = dt.datetime.fromtimestamp(r["timestamp"] / 1000, dt.timezone.utc).date().isoformat()        if "2022-01-01" <= day <= "2023-09-30":            b, title = products[r["parent_asin"]]            reviews.append({"brand": b, "product": title[:90], "rating": int(r["rating"]), "date": day,                            "verified": r.get("verified_purchase"), "helpful_votes": r.get("helpful_vote"),                            "title": clean(r.get("title")), "text": clean(r.get("text"))}) random.seed(2026)sample = []for b in ["YETI", "Stanley", "Hydro Flask", "Owala"]:    mine = [r for r in reviews if r["brand"] == b]    random.shuffle(mine)    sample += mine[:500]for i, r in enumerate(sample, 1):    r["review_id"] = f"R{i:04d}"    r["language"] = language(r["title"].rstrip(".") + ". " + r["text"])with open("data/reviews.csv", "w", newline="") as f:    w = csv.DictWriter(f, fieldnames=["review_id", "brand", "product", "rating", "date", "verified", "helpful_votes", "language", "title", "text"])    w.writeheader()    w.writerows(sample)print(len(products), "products,", len(reviews), "reviews in the window,", len(sample), "in the sample")

"""Build the review sample used in this case study (already done; you only run this to rebuild it from scratch).

Source: Amazon Reviews 2023 (McAuley Lab, UC San Diego), the newest public release of Amazon product reviews:
https://huggingface.co/datasets/McAuley-Lab/Amazon-Reviews-2023  (reviews up to September 2023).

Steps:
  1. Read the product files for Home & Kitchen and Sports & Outdoors and keep the insulated drinkware sold by four
     brands: YETI, Stanley, Hydro Flask and Owala (tumblers, mugs, bottles; the title filter drops coolers and most
     lids and accessories sold alone; a few get through, mostly Hydro Flask's press-in lid).
  2. Read the review files and keep the reviews of those products.
  3. Keep January 2022 to September 2023 (the newest 21 months), drop exact duplicates (the same review shown on
     several color versions), and draw 500 reviews per brand at random (seed 2026), 2,000 in all.
The big files are 41 GB together, so they are streamed in pieces and filtered on the fly (grep); nothing big is
stored. Needs curl and grep. Takes about 5 minutes on a fast connection.
"""
import csv, datetime as dt, html, json, random, re, subprocess

BASE = "https://huggingface.co/datasets/McAuley-Lab/Amazon-Reviews-2023/resolve/main/raw"
BRANDS = {"yeti": "YETI", "stanley": "Stanley", "owala": "Owala", "hydro flask": "Hydro Flask"}
DRINK = re.compile(r"tumbler|bottle|mug|\bcup\b|quencher|rambler|freesip|jug|colster|lowball|growler|flask|iceflow|pint|stein|wine", re.I)
NOT = re.compile(r"cooler|hopper|tundra|roadie|camino|\btote\b|\bbag\b|blanket|bucket|chair|bowl|brush|cleaner|opener|\bboot\b|sleeve|"
                 r"backpack|lantern|food jar|lunch|bento|knife|cutlery|stove|grill|replacement|holder|pouch|carrier|sling|keychain|"
                 r"sticker|gift card|cook ?set|cookware|\bpot\b|camp set|french press|flatware|utensil|spork|shaker", re.I)
ACCESSORY = re.compile(r"^\W*(yeti|stanley|owala|hydro flask)?\s*(rambler\s+)?(bottle\s+)?(wide mouth\s+|standard mouth\s+)?"
                       r"(magslider|straw|chug|flex|stronghold|flip|twist|press-in|sip|hotshot|tumbler|bottle|mug)?\s*(lid|cap|straw|spout)s?\b"
                       r"|\b(lid|cap)s?\b\s*(for|,?\s*fits)\b", re.I)

def stream(url, pattern_args):
    """Lines of a big remote file that match grep, without saving the file."""
    p = subprocess.run(f"curl -sSL {url} | grep {pattern_args}", shell=True, capture_output=True, text=True)
    return p.stdout.splitlines()

def brand_of(store):
    s = (store or "").lower()
    return next((v for k, v in BRANDS.items() if k in s), None)

SPANISH = re.compile(r"[ñ¿¡]|\b(muy|es|el|la|que|lo|para|excelente|producto|vaso|calidad|llegó|llego|compra|bueno|buena|pero|con|sin|una|un|mi|me|no es)\b", re.I)
ENGLISH = re.compile(r"\b(the|it|and|is|i|my|this|to|a|for|of|was|in|love|great|good|cup|bottle|mug|not|but|so|very|keeps|cold|hot|ice|lid|best|yet|nice|perfect|works|size)\b", re.I)

def language(text):
    """'es' when a review has more common Spanish words than common English ones (the English model can't read those)."""
    return "es" if len(SPANISH.findall(text)) > len(ENGLISH.findall(text)) else "en"

def clean(text):
    text = re.sub(r"\[\[VIDEOID:[^\]]*\]\]", " ", text or "")
    text = re.sub(r"<br\s*/?>", " ", text)
    return re.sub(r"\s+", " ", html.unescape(text)).strip()

products = {}
for cat in ["Home_and_Kitchen", "Sports_and_Outdoors"]:
    for line in stream(f"{BASE}/meta_categories/meta_{cat}.jsonl", "-i -E '\"store\": \"(yeti|stanley|stanley 1913|owala|hydro flask)\"'"):
        p = json.loads(line)
        b, title = brand_of(p.get("store")), p.get("title") or ""
        if b and DRINK.search(title) and not NOT.search(title) and not ACCESSORY.search(title):
            products[p["parent_asin"]] = (b, title)
open("data/asin_patterns.txt", "w").write("".join(f'"parent_asin": "{a}"\n' for a in products))

reviews, seen = [], set()
for cat in ["Home_and_Kitchen", "Sports_and_Outdoors"]:
    for line in stream(f"{BASE}/review_categories/{cat}.jsonl", "-F -f data/asin_patterns.txt"):
        r = json.loads(line)
        key = (r.get("user_id"), r.get("timestamp"), (r.get("text") or "")[:60])
        if r.get("parent_asin") not in products or key in seen:
            continue
        seen.add(key)
        day = dt.datetime.fromtimestamp(r["timestamp"] / 1000, dt.timezone.utc).date().isoformat()
        if "2022-01-01" <= day <= "2023-09-30":
            b, title = products[r["parent_asin"]]
            reviews.append({"brand": b, "product": title[:90], "rating": int(r["rating"]), "date": day,
                            "verified": r.get("verified_purchase"), "helpful_votes": r.get("helpful_vote"),
                            "title": clean(r.get("title")), "text": clean(r.get("text"))})

random.seed(2026)
sample = []
for b in ["YETI", "Stanley", "Hydro Flask", "Owala"]:
    mine = [r for r in reviews if r["brand"] == b]
    random.shuffle(mine)
    sample += mine[:500]
for i, r in enumerate(sample, 1):
    r["review_id"] = f"R{i:04d}"
    r["language"] = language(r["title"].rstrip(".") + ". " + r["text"])
with open("data/reviews.csv", "w", newline="") as f:
    w = csv.DictWriter(f, fieldnames=["review_id", "brand", "product", "rating", "date", "verified", "helpful_votes", "language", "title", "text"])
    w.writeheader()
    w.writerows(sample)
print(len(products), "products,", len(reviews), "reviews in the window,", len(sample), "in the sample")