Public Access
feat: LLM-powered second-pass ingredient matcher + matcher improvements
Matcher improvements (matcher.py): - Plural normalization: 'tortillas'→'tortilla', 'thighs'→'thigh' so subset recall check works without stemmer - Precision floor lowered 0.45→0.30: allows 'Bacon'→'Wright Brand Bacon' (1/3=0.33) while exclusion words still block category contaminants - _EXCLUSION_WORDS now normalized through same singularizer for consistency LLM second-pass (llm_matcher.py): - run_llm_match_job(): for each still-unmatched ingredient, collects top-12 candidates from grocery catalog ranked by fuzzy×precision (same metric as AUTO matcher), then asks Ollama to pick the best match - Candidate scoring: combined = (partial_token_sort_ratio/100) × precision ensures "McCormick Black Pepper" outranks "Dr Pepper" for 'Black Pepper' - Stores picks as source='auto_llm' (confidence=0.750) - Ollama Cloud endpoint: https://ollama.com/v1, model: kimi-k2.6:cloud Migration 0010: adds 'auto_llm' to ingredient_match_source_enum Config: OLLAMA_BASE_URL / OLLAMA_API_KEY / OLLAMA_MODEL settings Docker-compose: wires all three Ollama + Spoonacular env vars to backend/scheduler Scraper service: calls run_llm_match_job after run_match_job on every scrape Results: AUTO matcher went from 36→25 unmatched (plural normalization fix), LLM added 3 more (Black Pepper, Zucchini, Chicken Thighs). Remaining 22 are genuine Lucky CA catalog gaps (standalone olive oil, dried spices, etc. not in Swiftly weekly ad). Co-Authored-By: Claude Sonnet 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
Claude Sonnet 4.6
@@ -0,0 +1,214 @@
|
||||
"""LLM-powered second-pass ingredient→grocery matcher.
|
||||
|
||||
After the deterministic AUTO matcher runs, some ingredients remain unmatched
|
||||
(catalog gaps, unusual product names). This module sends those ingredients to
|
||||
an Ollama-compatible LLM with a small candidate list from the DB and stores the
|
||||
best pick as source='auto_llm'.
|
||||
|
||||
Manual matches (source='manual') are never touched.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import re
|
||||
import time
|
||||
import uuid as _uuid_mod
|
||||
from decimal import Decimal
|
||||
|
||||
import requests
|
||||
from rapidfuzz import fuzz, process
|
||||
from sqlalchemy.dialects.postgresql import insert as _pg_insert
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.config import settings
|
||||
from app.models import (
|
||||
GroceryItem,
|
||||
Ingredient,
|
||||
IngredientGroceryMatch,
|
||||
IngredientMatchSource,
|
||||
)
|
||||
from app.services.matcher import _sig_words
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_CANDIDATE_LIMIT = 12 # top candidates sent to the LLM
|
||||
_RATE_LIMIT_SECS = 0.3 # pause between LLM calls
|
||||
|
||||
|
||||
def _get_candidates(
|
||||
ingredient_name: str,
|
||||
grocery_rows: list[GroceryItem],
|
||||
) -> list[tuple[str, object]]:
|
||||
"""Return up to ``_CANDIDATE_LIMIT`` grocery items ranked by fuzzy×precision.
|
||||
|
||||
Uses the same combined scoring as the AUTO matcher:
|
||||
combined = (partial_token_sort_ratio / 100) × (overlap_words / grocery_words)
|
||||
|
||||
This ensures a specific match like "McCormick Black Pepper" outranks a
|
||||
generic one like "Dr Pepper" that happens to score well on the raw fuzzy ratio.
|
||||
"""
|
||||
ing_sig = _sig_words(ingredient_name)
|
||||
if not ing_sig:
|
||||
return []
|
||||
|
||||
# Pre-filter: must share ≥1 sig word with the ingredient.
|
||||
filtered: list[GroceryItem] = [
|
||||
gi for gi in grocery_rows
|
||||
if ing_sig & _sig_words(gi.name)
|
||||
]
|
||||
if not filtered:
|
||||
return []
|
||||
|
||||
filtered_lower = [gi.name.lower() for gi in filtered]
|
||||
# Run fuzzy over ALL filtered items (no limit) to get raw scores.
|
||||
results = process.extract(
|
||||
ingredient_name.lower(),
|
||||
filtered_lower,
|
||||
scorer=fuzz.partial_token_sort_ratio,
|
||||
limit=len(filtered),
|
||||
)
|
||||
|
||||
scored: list[tuple[float, str, object]] = []
|
||||
for _text, fuzzy_score, idx in results:
|
||||
gi = filtered[idx]
|
||||
gsig = _sig_words(gi.name)
|
||||
if not gsig:
|
||||
continue
|
||||
# Precision = fraction of grocery sig-words that are relevant to the ingredient.
|
||||
precision = len(ing_sig & gsig) / len(gsig)
|
||||
combined = (fuzzy_score / 100.0) * precision
|
||||
scored.append((combined, gi.name, gi.id))
|
||||
|
||||
scored.sort(reverse=True)
|
||||
return [(name, gid) for _, name, gid in scored[:_CANDIDATE_LIMIT]]
|
||||
|
||||
|
||||
def _get_unmatched_ingredients(db: Session) -> list[Ingredient]:
|
||||
"""Ingredients with no existing match in ingredient_grocery_match."""
|
||||
from sqlalchemy import select
|
||||
matched_ids_stmt = select(IngredientGroceryMatch.ingredient_id)
|
||||
return (
|
||||
db.query(Ingredient)
|
||||
.filter(Ingredient.id.notin_(matched_ids_stmt))
|
||||
.all()
|
||||
)
|
||||
|
||||
|
||||
def _ask_ollama(ingredient_name: str, candidates: list[str]) -> int | None:
|
||||
"""Ask the LLM to pick the best grocery item for the ingredient.
|
||||
|
||||
Returns the 0-based index into `candidates`, or None if no good match.
|
||||
"""
|
||||
if not settings.OLLAMA_API_KEY:
|
||||
logger.warning("OLLAMA_API_KEY not configured — skipping LLM pass")
|
||||
return None
|
||||
|
||||
numbered = "\n".join(f"{i + 1}. {name}" for i, name in enumerate(candidates))
|
||||
prompt = (
|
||||
f"I need '{ingredient_name}' for a home-cooked recipe at a supermarket.\n"
|
||||
f"Which of these products is the closest match?\n"
|
||||
f"{numbered}\n\n"
|
||||
f"Reply with just the number (1–{len(candidates)}) or 'none' if none fit."
|
||||
)
|
||||
|
||||
try:
|
||||
resp = requests.post(
|
||||
f"{settings.OLLAMA_BASE_URL}/chat/completions",
|
||||
headers={
|
||||
"Content-Type": "application/json",
|
||||
"Authorization": f"Bearer {settings.OLLAMA_API_KEY}",
|
||||
},
|
||||
json={
|
||||
"model": settings.OLLAMA_MODEL,
|
||||
"messages": [{"role": "user", "content": prompt}],
|
||||
"max_tokens": 500, # kimi-k2 reasons before answering; needs headroom
|
||||
"temperature": 0,
|
||||
},
|
||||
timeout=30,
|
||||
)
|
||||
resp.raise_for_status()
|
||||
except requests.RequestException as exc:
|
||||
logger.warning("Ollama API error for %r: %s", ingredient_name, exc)
|
||||
return None
|
||||
|
||||
content = resp.json().get("choices", [{}])[0].get("message", {}).get("content", "")
|
||||
# Strip <think>…</think> reasoning blocks that some models emit
|
||||
content = re.sub(r"<think>.*?</think>", "", content, flags=re.DOTALL).strip()
|
||||
|
||||
if "none" in content.lower():
|
||||
return None
|
||||
m = re.search(r"\b(\d+)\b", content)
|
||||
if not m:
|
||||
return None
|
||||
idx = int(m.group(1)) - 1
|
||||
return idx if 0 <= idx < len(candidates) else None
|
||||
|
||||
|
||||
def run_llm_match_job(
|
||||
db: Session,
|
||||
*,
|
||||
source_filter: str = "lucky_california",
|
||||
) -> int:
|
||||
"""LLM second pass: match remaining unmatched ingredients.
|
||||
|
||||
For each ingredient with no existing match, collects up to
|
||||
``_CANDIDATE_LIMIT`` fuzzy candidates from the grocery catalog and asks
|
||||
the configured LLM to select the best one.
|
||||
|
||||
Returns number of new LLM matches stored.
|
||||
"""
|
||||
grocery_rows = (
|
||||
db.query(GroceryItem)
|
||||
.filter(GroceryItem.source == source_filter)
|
||||
.all()
|
||||
)
|
||||
if not grocery_rows:
|
||||
return 0
|
||||
|
||||
unmatched = _get_unmatched_ingredients(db)
|
||||
if not unmatched:
|
||||
logger.info("LLM matcher: no unmatched ingredients — nothing to do")
|
||||
return 0
|
||||
|
||||
logger.info("LLM matcher: %d unmatched ingredients to process", len(unmatched))
|
||||
written = 0
|
||||
|
||||
for ingredient in unmatched:
|
||||
if not ingredient.name or not ingredient.name.strip():
|
||||
continue
|
||||
|
||||
candidates = _get_candidates(ingredient.name, grocery_rows)
|
||||
if not candidates:
|
||||
logger.debug("LLM matcher: no candidates for %r — skip", ingredient.name)
|
||||
continue
|
||||
|
||||
candidate_names = [name for name, _ in candidates]
|
||||
logger.info("LLM matcher: %r → %d candidates", ingredient.name, len(candidates))
|
||||
|
||||
chosen_idx = _ask_ollama(ingredient.name, candidate_names)
|
||||
time.sleep(_RATE_LIMIT_SECS)
|
||||
|
||||
if chosen_idx is None:
|
||||
logger.info(" LLM: no match")
|
||||
continue
|
||||
|
||||
chosen_name, chosen_grocery_id = candidates[chosen_idx]
|
||||
logger.info(" LLM: chose %r", chosen_name)
|
||||
|
||||
stmt = (
|
||||
_pg_insert(IngredientGroceryMatch.__table__)
|
||||
.values(
|
||||
id=_uuid_mod.uuid4(),
|
||||
ingredient_id=ingredient.id,
|
||||
grocery_item_id=chosen_grocery_id,
|
||||
confidence=Decimal("0.750"),
|
||||
source=IngredientMatchSource.AUTO_LLM,
|
||||
)
|
||||
.on_conflict_do_nothing(index_elements=["ingredient_id", "grocery_item_id"])
|
||||
)
|
||||
db.execute(stmt)
|
||||
written += 1
|
||||
|
||||
db.commit()
|
||||
logger.info("LLM matcher: %d new matches written", written)
|
||||
return written
|
||||
Reference in New Issue
Block a user