feat: LLM-powered second-pass ingredient matcher + matcher improvements

Matcher improvements (matcher.py):
- Plural normalization: 'tortillas'→'tortilla', 'thighs'→'thigh' so
  subset recall check works without stemmer
- Precision floor lowered 0.45→0.30: allows 'Bacon'→'Wright Brand Bacon'
  (1/3=0.33) while exclusion words still block category contaminants
- _EXCLUSION_WORDS now normalized through same singularizer for consistency

LLM second-pass (llm_matcher.py):
- run_llm_match_job(): for each still-unmatched ingredient, collects top-12
  candidates from grocery catalog ranked by fuzzy×precision (same metric as
  AUTO matcher), then asks Ollama to pick the best match
- Candidate scoring: combined = (partial_token_sort_ratio/100) × precision
  ensures "McCormick Black Pepper" outranks "Dr Pepper" for 'Black Pepper'
- Stores picks as source='auto_llm' (confidence=0.750)
- Ollama Cloud endpoint: https://ollama.com/v1, model: kimi-k2.6:cloud

Migration 0010: adds 'auto_llm' to ingredient_match_source_enum

Config: OLLAMA_BASE_URL / OLLAMA_API_KEY / OLLAMA_MODEL settings
Docker-compose: wires all three Ollama + Spoonacular env vars to backend/scheduler
Scraper service: calls run_llm_match_job after run_match_job on every scrape

Results: AUTO matcher went from 36→25 unmatched (plural normalization fix),
LLM added 3 more (Black Pepper, Zucchini, Chicken Thighs).
Remaining 22 are genuine Lucky CA catalog gaps (standalone olive oil,
dried spices, etc. not in Swiftly weekly ad).

Co-Authored-By: Claude Sonnet 4.6 (1M context) <noreply@anthropic.com>

backend/alembic/versions/0010_auto_llm_source.py

@@ -0,0 +1,25 @@
+"""Add auto_llm to ingredient_match_source_enum
+
+Revision ID: 0010
+Revises: 0009
+Create Date: 2026-05-11
+"""
+from alembic import op
+
+revision = "0010"
+down_revision = "0009"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # PostgreSQL 9.1+ allows adding values to existing enums.
+    # IF NOT EXISTS guard makes the migration idempotent.
+    op.execute("ALTER TYPE ingredient_match_source_enum ADD VALUE IF NOT EXISTS 'auto_llm'")
+
+
+def downgrade() -> None:
+    # PostgreSQL does not support removing enum values without dropping and
+    # recreating the type; rows using 'auto_llm' would block the drop.
+    # Downgrade is intentionally a no-op.
+    pass

backend/app/config.py

@@ -29,6 +29,11 @@ class Settings(BaseSettings):
     ADMIN_EMAIL: str = ""
     APP_BASE_URL: str = "http://localhost"
 
+    # Ollama Cloud LLM (used for ingredient→grocery LLM matching second pass)
+    OLLAMA_BASE_URL: str = "https://ollama.com/v1"
+    OLLAMA_API_KEY: Optional[str] = None
+    OLLAMA_MODEL: str = "kimi-k2.6:cloud"
+
     FAMILY_EMAIL_1: Optional[str] = None
     FAMILY_EMAIL_2: Optional[str] = None
     RECIPES_EMAIL: Optional[str] = None

backend/app/models/__init__.py

@@ -81,6 +81,7 @@ class EmailStatus(enum.Enum):
 
 class IngredientMatchSource(enum.Enum):
     AUTO = "auto"
+    AUTO_LLM = "auto_llm"
     MANUAL = "manual"
 
 

backend/app/services/llm_matcher.py

@@ -0,0 +1,214 @@
+"""LLM-powered second-pass ingredient→grocery matcher.
+
+After the deterministic AUTO matcher runs, some ingredients remain unmatched
+(catalog gaps, unusual product names).  This module sends those ingredients to
+an Ollama-compatible LLM with a small candidate list from the DB and stores the
+best pick as source='auto_llm'.
+
+Manual matches (source='manual') are never touched.
+"""
+from __future__ import annotations
+
+import logging
+import re
+import time
+import uuid as _uuid_mod
+from decimal import Decimal
+
+import requests
+from rapidfuzz import fuzz, process
+from sqlalchemy.dialects.postgresql import insert as _pg_insert
+from sqlalchemy.orm import Session
+
+from app.config import settings
+from app.models import (
+    GroceryItem,
+    Ingredient,
+    IngredientGroceryMatch,
+    IngredientMatchSource,
+)
+from app.services.matcher import _sig_words
+
+logger = logging.getLogger(__name__)
+
+_CANDIDATE_LIMIT = 12     # top candidates sent to the LLM
+_RATE_LIMIT_SECS = 0.3    # pause between LLM calls
+
+
+def _get_candidates(
+    ingredient_name: str,
+    grocery_rows: list[GroceryItem],
+) -> list[tuple[str, object]]:
+    """Return up to ``_CANDIDATE_LIMIT`` grocery items ranked by fuzzy×precision.
+
+    Uses the same combined scoring as the AUTO matcher:
+        combined = (partial_token_sort_ratio / 100) × (overlap_words / grocery_words)
+
+    This ensures a specific match like "McCormick Black Pepper" outranks a
+    generic one like "Dr Pepper" that happens to score well on the raw fuzzy ratio.
+    """
+    ing_sig = _sig_words(ingredient_name)
+    if not ing_sig:
+        return []
+
+    # Pre-filter: must share ≥1 sig word with the ingredient.
+    filtered: list[GroceryItem] = [
+        gi for gi in grocery_rows
+        if ing_sig & _sig_words(gi.name)
+    ]
+    if not filtered:
+        return []
+
+    filtered_lower = [gi.name.lower() for gi in filtered]
+    # Run fuzzy over ALL filtered items (no limit) to get raw scores.
+    results = process.extract(
+        ingredient_name.lower(),
+        filtered_lower,
+        scorer=fuzz.partial_token_sort_ratio,
+        limit=len(filtered),
+    )
+
+    scored: list[tuple[float, str, object]] = []
+    for _text, fuzzy_score, idx in results:
+        gi = filtered[idx]
+        gsig = _sig_words(gi.name)
+        if not gsig:
+            continue
+        # Precision = fraction of grocery sig-words that are relevant to the ingredient.
+        precision = len(ing_sig & gsig) / len(gsig)
+        combined = (fuzzy_score / 100.0) * precision
+        scored.append((combined, gi.name, gi.id))
+
+    scored.sort(reverse=True)
+    return [(name, gid) for _, name, gid in scored[:_CANDIDATE_LIMIT]]
+
+
+def _get_unmatched_ingredients(db: Session) -> list[Ingredient]:
+    """Ingredients with no existing match in ingredient_grocery_match."""
+    from sqlalchemy import select
+    matched_ids_stmt = select(IngredientGroceryMatch.ingredient_id)
+    return (
+        db.query(Ingredient)
+        .filter(Ingredient.id.notin_(matched_ids_stmt))
+        .all()
+    )
+
+
+def _ask_ollama(ingredient_name: str, candidates: list[str]) -> int | None:
+    """Ask the LLM to pick the best grocery item for the ingredient.
+
+    Returns the 0-based index into `candidates`, or None if no good match.
+    """
+    if not settings.OLLAMA_API_KEY:
+        logger.warning("OLLAMA_API_KEY not configured — skipping LLM pass")
+        return None
+
+    numbered = "\n".join(f"{i + 1}. {name}" for i, name in enumerate(candidates))
+    prompt = (
+        f"I need '{ingredient_name}' for a home-cooked recipe at a supermarket.\n"
+        f"Which of these products is the closest match?\n"
+        f"{numbered}\n\n"
+        f"Reply with just the number (1–{len(candidates)}) or 'none' if none fit."
+    )
+
+    try:
+        resp = requests.post(
+            f"{settings.OLLAMA_BASE_URL}/chat/completions",
+            headers={
+                "Content-Type": "application/json",
+                "Authorization": f"Bearer {settings.OLLAMA_API_KEY}",
+            },
+            json={
+                "model": settings.OLLAMA_MODEL,
+                "messages": [{"role": "user", "content": prompt}],
+                "max_tokens": 500,  # kimi-k2 reasons before answering; needs headroom
+                "temperature": 0,
+            },
+            timeout=30,
+        )
+        resp.raise_for_status()
+    except requests.RequestException as exc:
+        logger.warning("Ollama API error for %r: %s", ingredient_name, exc)
+        return None
+
+    content = resp.json().get("choices", [{}])[0].get("message", {}).get("content", "")
+    # Strip <think>…</think> reasoning blocks that some models emit
+    content = re.sub(r"<think>.*?</think>", "", content, flags=re.DOTALL).strip()
+
+    if "none" in content.lower():
+        return None
+    m = re.search(r"\b(\d+)\b", content)
+    if not m:
+        return None
+    idx = int(m.group(1)) - 1
+    return idx if 0 <= idx < len(candidates) else None
+
+
+def run_llm_match_job(
+    db: Session,
+    *,
+    source_filter: str = "lucky_california",
+) -> int:
+    """LLM second pass: match remaining unmatched ingredients.
+
+    For each ingredient with no existing match, collects up to
+    ``_CANDIDATE_LIMIT`` fuzzy candidates from the grocery catalog and asks
+    the configured LLM to select the best one.
+
+    Returns number of new LLM matches stored.
+    """
+    grocery_rows = (
+        db.query(GroceryItem)
+        .filter(GroceryItem.source == source_filter)
+        .all()
+    )
+    if not grocery_rows:
+        return 0
+
+    unmatched = _get_unmatched_ingredients(db)
+    if not unmatched:
+        logger.info("LLM matcher: no unmatched ingredients — nothing to do")
+        return 0
+
+    logger.info("LLM matcher: %d unmatched ingredients to process", len(unmatched))
+    written = 0
+
+    for ingredient in unmatched:
+        if not ingredient.name or not ingredient.name.strip():
+            continue
+
+        candidates = _get_candidates(ingredient.name, grocery_rows)
+        if not candidates:
+            logger.debug("LLM matcher: no candidates for %r — skip", ingredient.name)
+            continue
+
+        candidate_names = [name for name, _ in candidates]
+        logger.info("LLM matcher: %r → %d candidates", ingredient.name, len(candidates))
+
+        chosen_idx = _ask_ollama(ingredient.name, candidate_names)
+        time.sleep(_RATE_LIMIT_SECS)
+
+        if chosen_idx is None:
+            logger.info("  LLM: no match")
+            continue
+
+        chosen_name, chosen_grocery_id = candidates[chosen_idx]
+        logger.info("  LLM: chose %r", chosen_name)
+
+        stmt = (
+            _pg_insert(IngredientGroceryMatch.__table__)
+            .values(
+                id=_uuid_mod.uuid4(),
+                ingredient_id=ingredient.id,
+                grocery_item_id=chosen_grocery_id,
+                confidence=Decimal("0.750"),
+                source=IngredientMatchSource.AUTO_LLM,
+            )
+            .on_conflict_do_nothing(index_elements=["ingredient_id", "grocery_item_id"])
+        )
+        db.execute(stmt)
+        written += 1
+
+    db.commit()
+    logger.info("LLM matcher: %d new matches written", written)
+    return written

backend/app/services/matcher.py

@@ -39,16 +39,29 @@ _STOP_WORDS = frozenset({
     "and", "with", "for", "the",
 })
 
+def _singularize(word: str) -> str:
+    """Best-effort English singularization for grocery product names.
+
+    Handles 'tortillas'→'tortilla', 'thighs'→'thigh', 'chickpeas'→'chickpea'.
+    Requires length > 3 to avoid mangling short words like 'has', 'was'.
+    """
+    if len(word) > 4 and word.endswith("ies"):
+        return word[:-3] + "y"          # 'berries' → 'berry'
+    if len(word) > 3 and word.endswith("s") and not word.endswith("ss"):
+        return word[:-1]                # 'tortillas' → 'tortilla'
+    return word
+
+
 # If any of these words appear in a grocery item's sig-words but NOT in the
 # ingredient's sig-words, the match is rejected outright.  Prevents category
 # cross-contamination: "Garlic" must not match "Garlic Bread", "Lime" must not
 # match "Lime Margarita", etc.
-_EXCLUSION_WORDS = frozenset({
+_EXCLUSION_WORDS_RAW = frozenset({
     # Baked goods / bread products
-    "bread", "loaf", "rolls", "bun", "buns", "croissant",
+    "bread", "loaf", "roll", "rolls", "bun", "buns", "croissant",
     "cracker", "crackers", "cookie", "cookies", "cake", "cupcake", "muffin", "bagel",
     # Chips / snack foods
-    "chips",
+    "chip", "chips",
     # Pasta / noodles
     "pasta", "noodle", "noodles", "vermicelli", "spaghetti", "linguine",
     "fettuccine", "penne", "rigatoni", "macaroni", "rotini", "orzo",
@@ -64,8 +77,10 @@ _EXCLUSION_WORDS = frozenset({
     # Prepared poultry (prevents "Garlic Herb Rotisserie Chicken" matching "Garlic")
     "rotisserie",
     # Baby / personal care (belt-and-suspenders after stop-word rework)
-    "baby", "wipes", "diaper",
+    "baby", "wipe", "wipes", "diaper",
 })
+# Pre-normalize exclusion words so they match the singularized sig-word sets.
+_EXCLUSION_WORDS = frozenset(_singularize(w) for w in _EXCLUSION_WORDS_RAW)
 
 
 @dataclass
@@ -76,9 +91,13 @@ class MatchResult:
 
 
 def _sig_words(text: str) -> frozenset:
-    """Lowercase alpha tokens >2 chars, stop-words removed."""
+    """Lowercase alpha tokens >2 chars, stop-words removed, plurals normalized."""
     tokens = re.sub(r"[^a-z ]", " ", text.lower()).split()
-    return frozenset(t for t in tokens if len(t) > 2 and t not in _STOP_WORDS)
+    return frozenset(
+        _singularize(t)
+        for t in tokens
+        if len(t) > 2 and t not in _STOP_WORDS
+    )
 
 
 def run_match_job(
@@ -185,10 +204,10 @@ def run_match_job(
                     continue
                 # Precision penalises grocery items with many extra words.
                 precision = len(ing_sig) / len(gsig)
-                # Hard floor: grocery must not have >2× the sig-words of the ingredient.
+                # Hard floor: grocery must not dwarf the ingredient in sig-word count.
-                # Catches long branded products that sneak past exclusion words, e.g.
+                # 0.30 allows "Bacon" (1 sig-word) → "Wright Brand Bacon" (3 sig-words)
-                # "Garlic Herb Rotisserie Chicken" for "Garlic".
+                # while exclusion words still block "Garlic Herb Rotisserie Chicken".
-                if precision < 0.45:
+                if precision < 0.30:
                     continue
                 combined = (score / 100.0) * precision
                 if combined > best_combined:

backend/app/services/scraper_service.py

@@ -89,6 +89,8 @@ def _run_scrape_in_background(
             try:
                 from app.services.matcher import run_match_job
                 run_match_job(db, source_filter="lucky_california")
+                from app.services.llm_matcher import run_llm_match_job
+                run_llm_match_job(db, source_filter="lucky_california")
             except Exception as e:  # matcher failure must not flip scrape to FAILED
                 import logging
                 logging.exception("matcher failed after successful scrape: %s", e)

docker-compose.yml

@@ -11,6 +11,9 @@ services:
       - DATABASE_URL=postgresql://mealplanner:${POSTGRES_PASSWORD}@db:5432/mealplanner
       - SENDGRID_API_KEY=${SENDGRID_API_KEY}
       - SPOONACULAR_API_KEY=${SPOONACULAR_API_KEY}
+      - OLLAMA_BASE_URL=${OLLAMA_BASE_URL:-https://ollama.com/v1}
+      - OLLAMA_API_KEY=${OLLAMA_API_KEY}
+      - OLLAMA_MODEL=${OLLAMA_MODEL:-kimi-k2.6:cloud}
       - LUCKY_CA_URL=${LUCKY_CA_URL:-https://luckysupermarkets.com}
       - LUCKY_STORE_ID=${LUCKY_STORE_ID:-757}
       - SWIFTLY_API_BASE=${SWIFTLY_API_BASE:-https://prod.swiftlyapi.net}
@@ -42,6 +45,9 @@ services:
       - DATABASE_URL=postgresql://mealplanner:${POSTGRES_PASSWORD}@db:5432/mealplanner
       - SENDGRID_API_KEY=${SENDGRID_API_KEY}
       - SPOONACULAR_API_KEY=${SPOONACULAR_API_KEY}
+      - OLLAMA_BASE_URL=${OLLAMA_BASE_URL:-https://ollama.com/v1}
+      - OLLAMA_API_KEY=${OLLAMA_API_KEY}
+      - OLLAMA_MODEL=${OLLAMA_MODEL:-kimi-k2.6:cloud}
       - LUCKY_CA_URL=${LUCKY_CA_URL:-https://luckysupermarkets.com}
       - LUCKY_STORE_ID=${LUCKY_STORE_ID:-757}
       - SWIFTLY_API_BASE=${SWIFTLY_API_BASE:-https://prod.swiftlyapi.net}