feat(meals): enforce three lower-calorie dinners with media
CI / backend (pytest + alembic) (push) Canceled after 0s
CI / frontend (build) (push) Canceled after 0s

This commit is contained in:
2026-09-01 10:11:36 -07:00
parent 7838c49721
commit 2695ba0232
85 changed files with 8103 additions and 795 deletions
+140 -23
View File
@@ -1,12 +1,32 @@
from fastapi import APIRouter, BackgroundTasks, Depends, HTTPException
from fastapi import APIRouter, BackgroundTasks, Depends, HTTPException, Query, Response, status
from sqlalchemy.exc import IntegrityError
from sqlalchemy.orm import Session
from app.database import get_db
from app.models import ScrapeLog, EmailLog, MealPlan, Recipe
from app.models import (
FamilyProfile,
ScrapeLog,
EmailLog,
MealPlan,
Recipe,
RecipeImportStaging,
RecipeImportStatus,
)
from app.schemas.recipe_import import (
RecipeImportDetail,
RecipeImportListResponse,
RecipeImportRejectRequest,
)
from app.security import require_admin
from app.services.scraper_service import ScraperService, enqueue_scrape
from app.services.feedback_analyzer import FeedbackAnalyzer
from app.services.recipe_discovery import RecipeDiscoveryService
from app.services.recipe_ingestion import RecipeIngestionService
from app.services.recipe_import_review import (
RecipeImportConflictError,
RecipeImportNotFoundError,
approve_recipe_import,
reject_recipe_import,
)
from app.services.wikibooks_import import WikibooksImportError, WikibooksImportService
from app.config import settings
from typing import List, Optional
from datetime import datetime, timedelta
from uuid import UUID
@@ -17,6 +37,99 @@ logger = logging.getLogger(__name__)
router = APIRouter(dependencies=[Depends(require_admin)])
def _import_list_item(row: RecipeImportStaging) -> dict:
normalized = row.normalized_recipe if isinstance(row.normalized_recipe, dict) else {}
return {
"id": row.id,
"source": row.source,
"external_id": row.external_id,
"source_url": row.source_url,
"source_revision": row.source_revision,
"name": normalized.get("name"),
"status": row.status.value,
"acquired_at": row.acquired_at,
"reviewed_at": row.reviewed_at,
"reviewed_by": row.reviewed_by,
"recipe_id": row.recipe_id,
}
@router.get("/recipe-imports", response_model=RecipeImportListResponse)
def list_recipe_imports(
import_status: RecipeImportStatus | None = Query(default=None, alias="status"),
source: str | None = Query(default=None, min_length=1, max_length=100),
limit: int = Query(default=50, ge=1, le=100),
offset: int = Query(default=0, ge=0),
db: Session = Depends(get_db),
):
query = db.query(RecipeImportStaging)
if import_status is not None:
query = query.filter(RecipeImportStaging.status == import_status)
if source is not None:
query = query.filter(RecipeImportStaging.source == source)
total = query.count()
rows = (
query.order_by(RecipeImportStaging.acquired_at.desc(), RecipeImportStaging.id)
.offset(offset)
.limit(limit)
.all()
)
return {
"items": [_import_list_item(row) for row in rows],
"total": total,
"limit": limit,
"offset": offset,
"has_next": offset + len(rows) < total,
}
@router.get("/recipe-imports/{import_id}", response_model=RecipeImportDetail)
def get_recipe_import(import_id: UUID, db: Session = Depends(get_db)):
staged = db.get(RecipeImportStaging, import_id)
if staged is None:
raise HTTPException(status_code=404, detail="Recipe import not found")
return staged
@router.post("/recipe-imports/{import_id}/approve", response_model=RecipeImportDetail)
def approve_staged_recipe(
import_id: UUID,
response: Response,
db: Session = Depends(get_db),
reviewer: str = Depends(require_admin),
):
try:
staged, created = approve_recipe_import(db, import_id, reviewer)
except RecipeImportNotFoundError as exc:
db.rollback()
raise HTTPException(status_code=404, detail=str(exc)) from exc
except RecipeImportConflictError as exc:
db.rollback()
raise HTTPException(status_code=409, detail=str(exc)) from exc
except IntegrityError as exc:
db.rollback()
raise HTTPException(status_code=409, detail="Recipe publication conflicted") from exc
response.status_code = status.HTTP_201_CREATED if created else status.HTTP_200_OK
return staged
@router.post("/recipe-imports/{import_id}/reject", response_model=RecipeImportDetail)
def reject_staged_recipe(
import_id: UUID,
payload: RecipeImportRejectRequest,
db: Session = Depends(get_db),
reviewer: str = Depends(require_admin),
):
try:
return reject_recipe_import(db, import_id, payload.reason, reviewer)
except RecipeImportNotFoundError as exc:
db.rollback()
raise HTTPException(status_code=404, detail=str(exc)) from exc
except RecipeImportConflictError as exc:
db.rollback()
raise HTTPException(status_code=409, detail=str(exc)) from exc
@router.post("/scrape", status_code=202)
def trigger_scrape(
background_tasks: BackgroundTasks,
@@ -214,10 +327,9 @@ def trigger_images(
def trigger_discovery(
family_profile_id: UUID,
force: bool = False,
limit: int = Query(default=5, ge=1, le=25),
db: Session = Depends(get_db),
):
from uuid import UUID as UUIDType
profile = (
db.query(FamilyProfile)
.filter(FamilyProfile.id == family_profile_id)
@@ -232,25 +344,30 @@ def trigger_discovery(
if not analysis.discovery_queries and not force:
return {"status": "skipped", "reason": "No discovery queries and force=false"}
discovery = RecipeDiscoveryService()
if not analysis.discovery_queries and force:
queries = ["popular recipes"]
else:
queries = analysis.discovery_queries
candidates = discovery.discover(queries)
if not candidates:
return {"status": "no_candidates"}
try:
staged = WikibooksImportService(
settings.WIKIBOOKS_USER_AGENT
).stage_discovered_pages(
db,
limit=limit,
queries=analysis.discovery_queries or None,
)
db.commit()
except WikibooksImportError as exc:
db.rollback()
raise HTTPException(status_code=502, detail="Wikibooks staging failed") from exc
ingestion = RecipeIngestionService()
added = ingestion.ingest(
db, profile.id, candidates, analysis.to_dict()
pending = sum(row.status == RecipeImportStatus.PENDING for row in staged)
quarantined = sum(row.status == RecipeImportStatus.QUARANTINED for row in staged)
logger.info(
"Manual discovery: staged %d Wikibooks candidates for review", len(staged)
)
logger.info("Manual discovery: added %d recipes for family %s", added, profile.id)
return {
"status": "success",
"candidates_found": len(candidates),
"recipes_added": added,
"discovery_queries": queries,
"candidates_staged": len(staged),
"pending": pending,
"quarantined": quarantined,
"recipes_added": 0,
"discovery_queries": analysis.discovery_queries,
"confidence": analysis.confidence,
}
}
+83 -97
View File
@@ -1,21 +1,16 @@
"""Sprint 13 — F9-lite: free-text meal-plan synthesis via Ollama Cloud.
"""Free-text preferences constrained by the canonical three-dinner planner.
This endpoint lets the user describe what they want for the week
("Italian-inspired, vegetarian", "easy weeknight dinners, no fish")
and asks the configured LLM (kimi-k2.6:cloud on ollama.com) to pick
meals from the local recipe library. The LLM's picks are inserted
into a fresh plan; the remaining slots are filled by the existing
Sprint 6+ `fillEmptySlots` partial-success pattern.
meals from the local recipe library. The canonical planner validates
and persists all picks so LLM output cannot bypass planner constraints.
Reuses:
- The LLM call pattern from `app.services.llm_matcher._ask_ollama`
(POST ${OLLAMA_BASE_URL}/chat/completions, same headers, same
max_tokens=500, temperature=0, strip <think> blocks).
- The Sprint 6+ `meals.create` + `meals.fillEmptySlots` semantics
for the plan create + fill flow.
The LLM call follows `app.services.llm_matcher._ask_ollama`; plan
selection and persistence are delegated to `generate_meal_plan`.
The endpoint never crashes on LLM failure: parse errors, timeout,
or empty picks all fall through to the `fillEmptySlots` path.
Parse errors, timeout, invalid picks, and infeasible picks fall back to
the canonical planner.
"""
from __future__ import annotations
@@ -29,13 +24,20 @@ from typing import List, Optional
import requests
from fastapi import APIRouter, Depends, HTTPException
from pydantic import BaseModel, Field
from sqlalchemy import or_
from sqlalchemy.orm import Session
from app.config import settings
from app.database import get_db
from app.models import FamilyProfile, MealPlan, MealPlanItem, MealType, Recipe
from app.models import (
FamilyProfile,
MealPlan,
MealType,
Recipe,
RecipeCatalogStatus,
)
from app.security import require_session
from app.services.meal_pairings import components_with_suggested_sides
from app.services.planner.generate import generate_meal_plan
logger = logging.getLogger(__name__)
@@ -49,7 +51,7 @@ class LLMPlanRequest(BaseModel):
class LLMPickedItem(BaseModel):
day_of_week: int
meal_type: str # "breakfast" | "lunch" | "dinner"
meal_type: str
recipe_id: str
@@ -82,7 +84,13 @@ def _serialize_library(db: Session, profile_id: uuid.UUID) -> List[dict]:
Sorted alphabetically by name for deterministic LLM input."""
rows = (
db.query(Recipe)
.filter(Recipe.family_profile_id == profile_id)
.filter(
Recipe.catalog_status == RecipeCatalogStatus.ACTIVE,
or_(
Recipe.family_profile_id == profile_id,
Recipe.family_profile_id.is_(None),
),
)
.order_by(Recipe.name.asc())
.limit(_MAX_LIBRARY_PROMPTED)
.all()
@@ -95,6 +103,7 @@ def _serialize_library(db: Session, profile_id: uuid.UUID) -> List[dict]:
"cuisine_tags": r.cuisine_tags or [],
"dietary_tags": r.dietary_tags or [],
"protein_type": r.protein_type,
"calories_per_serving": r.calories_per_serving,
"total_time_minutes": (r.prep_time_minutes or 0) + (r.cook_time_minutes or 0),
})
return out
@@ -115,7 +124,7 @@ def _ask_llm(prompt: str) -> Optional[str]:
json={
"model": settings.OLLAMA_MODEL,
"messages": [{"role": "user", "content": prompt}],
"max_tokens": 4000, # 47-recipe library: 21 picks × ~100 chars + reasoning + boilerplate ≈ 2100+ chars; 4000 gives 2x headroom
"max_tokens": 4000, # Reasoning models may consume most of the budget before returning JSON.
"temperature": 0,
},
timeout=_LLM_TIMEOUT_SECS,
@@ -171,7 +180,10 @@ def _parse_picks(raw: str) -> List[dict]:
def _validate_picks(picks: List[dict], valid_recipe_ids: set[str]) -> List[LLMPickedItem]:
"""Drop invalid entries: missing fields, out-of-range day_of_week,
unknown meal_type, unknown recipe_id."""
valid_meal_types = {m.value for m in MealType}
valid_days = {1, 3, 5}
valid_meal_types = {MealType.DINNER.value}
used_days: set[int] = set()
used_recipes: set[str] = set()
out: List[LLMPickedItem] = []
for p in picks:
try:
@@ -180,13 +192,17 @@ def _validate_picks(picks: List[dict], valid_recipe_ids: set[str]) -> List[LLMPi
rid = str(p.get("recipe_id", ""))
except (TypeError, ValueError):
continue
if day < 1 or day > 7:
if day not in valid_days or day in used_days:
continue
if meal not in valid_meal_types:
continue
if rid not in valid_recipe_ids:
if rid not in valid_recipe_ids or rid in used_recipes:
continue
out.append(LLMPickedItem(day_of_week=day, meal_type=meal, recipe_id=rid))
used_days.add(day)
used_recipes.add(rid)
if len(out) == 3:
break
return out
@@ -194,14 +210,17 @@ def _validate_picks(picks: List[dict], valid_recipe_ids: set[str]) -> List[LLMPi
def synthesize_plan(
payload: LLMPlanRequest,
db: Session = Depends(get_db),
_user: str = Depends(require_session),
session_family_id: str = Depends(require_session),
) -> LLMPlanResponse:
"""Synthesize a 7-day meal plan from a free-text prompt + the
local recipe library via the configured LLM. Falls through to
the Sprint 6+ library-fill for any slots the LLM didn't pick."""
"""Use LLM preferences only when the canonical planner accepts all three."""
_ensure_ollama_configured()
profile = db.query(FamilyProfile).first()
try:
profile_id = uuid.UUID(session_family_id)
except ValueError:
raise HTTPException(status_code=404, detail="family profile not found")
profile = db.query(FamilyProfile).filter(FamilyProfile.id == profile_id).first()
if not profile:
raise HTTPException(status_code=404, detail="family profile not found")
@@ -234,21 +253,19 @@ def synthesize_plan(
f"cuisine={','.join(r['cuisine_tags']) or 'n/a'} | "
f"diet={','.join(r['dietary_tags']) or 'n/a'} | "
f"protein={r['protein_type'] or 'n/a'} | "
f"calories={r['calories_per_serving'] or 'unknown'} | "
f"time={r['total_time_minutes']}min"
for r in library
)
prompt = (
"You are planning a 7-day meal plan (Monday through Sunday) "
"for a family. Each day has 3 meals: breakfast, lunch, dinner. "
"Pick up to 21 meals total from the recipe library below. "
"If a slot has no good match for the user's request, OMIT it "
"(do not invent a recipe). Use only recipe_ids from the list.\n\n"
"You are choosing exactly 3 dinners for Monday, Wednesday, and Friday "
"(day_of_week 1, 3, and 5). Pick one different recipe for each day. "
"Do not invent recipes and use only recipe_ids from the list.\n\n"
f"USER REQUEST: {payload.prompt}\n\n"
f"RECIPE LIBRARY ({len(library)} recipes):\n{compact}\n\n"
'RETURN FORMAT — valid JSON only, no markdown, no commentary:\n'
'[\n'
' {"day_of_week": 1, "meal_type": "breakfast", "recipe_id": "<uuid>"},\n'
' {"day_of_week": 1, "meal_type": "lunch", "recipe_id": "<uuid>"},\n'
' {"day_of_week": 1, "meal_type": "dinner", "recipe_id": "<uuid>"},\n'
' ...\n'
"]"
)
@@ -258,80 +275,49 @@ def synthesize_plan(
raw_picks = _parse_picks(raw or "")
valid_recipe_ids = {r["id"] for r in library}
picks = _validate_picks(raw_picks, valid_recipe_ids)
recipe_by_id = {
str(r.id): r
for r in db.query(Recipe).filter(Recipe.id.in_([uuid.UUID(rid) for rid in valid_recipe_ids])).all()
}
logger.info(
"LLM plan: prompt=%d chars, raw_picks=%d, valid_picks=%d",
len(payload.prompt), len(raw_picks), len(picks),
)
# 4) Create the empty plan.
plan = MealPlan(
family_profile_id=profile.id,
week_start_date=payload.week_start,
status="draft",
notes=f"LLM-synthesized from prompt: {payload.prompt[:200]}",
)
db.add(plan)
db.flush()
picked_count = 0
result = None
if len(picks) == 3:
preferred_ids = {uuid.UUID(pick.recipe_id) for pick in picks}
active_ids = {
recipe_id
for (recipe_id,) in db.query(Recipe.id).filter(
Recipe.catalog_status == RecipeCatalogStatus.ACTIVE
).all()
}
result = generate_meal_plan(
db,
family_id=profile.id,
week_start_date=payload.week_start,
exclude_recipe_ids=active_ids - preferred_ids,
)
if len(result.selected) == 3:
picked_count = 3
else:
generated_plan = db.query(MealPlan).filter(MealPlan.id == result.meal_plan_id).first()
if generated_plan is not None:
db.delete(generated_plan)
db.commit()
result = None
# 5) Insert the LLM-picked items.
for pick in picks:
db.add(MealPlanItem(
meal_plan_id=plan.id,
recipe_id=uuid.UUID(pick.recipe_id),
day_of_week=pick.day_of_week,
meal_type=MealType(pick.meal_type),
components=components_with_suggested_sides(
recipe_by_id.get(pick.recipe_id),
pick.meal_type,
),
))
db.flush()
if result is None:
result = generate_meal_plan(
db,
family_id=profile.id,
week_start_date=payload.week_start,
)
picked_count = len(picks)
picked_slots = {(p.day_of_week, p.meal_type) for p in picks}
# 6) Fill the rest from the library (Sprint 6+ pattern).
# Re-implemented inline (not via HTTP) to avoid a self-call.
requested_meal_types = ["breakfast", "lunch", "dinner"]
used_recipe_ids = {uuid.UUID(p.recipe_id) for p in picks}
filled_count = 0
failed_count = 0
for day in range(1, 8):
for mt in requested_meal_types:
if (day, mt) in picked_slots:
continue
# Pick the first library recipe not already used in the plan
# and with a matching meal_type preference (best-effort). The
# LLM call above was the smart pick; the library fill is the
# fallback for any slots the LLM didn't cover.
candidates = [r for r in library if uuid.UUID(r["id"]) not in used_recipe_ids]
if not candidates:
failed_count += 1
continue
chosen = candidates[0]
db.add(MealPlanItem(
meal_plan_id=plan.id,
recipe_id=uuid.UUID(chosen["id"]),
day_of_week=day,
meal_type=MealType(mt),
components=components_with_suggested_sides(
recipe_by_id.get(chosen["id"]),
mt,
),
))
used_recipe_ids.add(uuid.UUID(chosen["id"]))
filled_count += 1
db.commit()
db.refresh(plan)
selected_count = len(result.selected)
filled_count = selected_count - picked_count
failed_count = max(0, 3 - selected_count)
return LLMPlanResponse(
plan_id=str(plan.id),
plan_id=str(result.meal_plan_id),
picked_count=picked_count,
filled_count=filled_count,
failed_count=failed_count,
+51 -1
View File
@@ -14,9 +14,10 @@ from app.schemas.meal_plan_generation import (
GenerationDebug,
GenerationItem,
GenerationResponse,
PublicGenerateRequest,
RegenerateRequest,
)
from app.security import require_admin
from app.security import require_admin, require_session
from app.services.planner.config import DEFAULT
from app.services.planner.generate import generate_meal_plan
from app.services.planner.types import GenerationResult
@@ -82,6 +83,55 @@ def generate(payload: GenerateRequest, db: Session = Depends(get_db)) -> Generat
return _to_response(payload.week_start_date, result.meal_plan_id, items, result)
@public_router.post("/generate", response_model=GenerationResponse, status_code=status.HTTP_201_CREATED)
def generate_for_family(
payload: PublicGenerateRequest,
db: Session = Depends(get_db),
session_family_id: str = Depends(require_session),
) -> GenerationResponse:
try:
family_id = UUID(session_family_id)
except ValueError:
raise HTTPException(status_code=404, detail="family_profile not found")
family = db.query(FamilyProfile).filter(FamilyProfile.id == family_id).first()
if family is None:
raise HTTPException(status_code=404, detail="family_profile not found")
existing = (
db.query(MealPlan)
.filter(
MealPlan.family_profile_id == family_id,
MealPlan.week_start_date == payload.week_start_date,
)
.first()
)
if existing is not None:
has_items = (
db.query(MealPlanItem.id)
.filter(MealPlanItem.meal_plan_id == existing.id)
.first()
is not None
)
if has_items:
raise HTTPException(status_code=409, detail="meal plan already has items")
db.delete(existing)
db.commit()
result = generate_meal_plan(
db,
family_id=family_id,
week_start_date=payload.week_start_date,
)
items = (
db.query(MealPlanItem)
.filter(MealPlanItem.meal_plan_id == result.meal_plan_id)
.order_by(MealPlanItem.day_of_week)
.all()
)
return _to_response(payload.week_start_date, result.meal_plan_id, items, result)
@admin_router.post("/regenerate", response_model=GenerationResponse, status_code=status.HTTP_201_CREATED)
def regenerate(payload: RegenerateRequest, db: Session = Depends(get_db)) -> GenerationResponse:
family = db.query(FamilyProfile).filter(FamilyProfile.id == payload.family_profile_id).first()
+73 -4
View File
@@ -9,6 +9,7 @@ from app.models import (
MealPlan, MealPlanItem, MealPlanVote, Recipe, Ingredient,
FamilyProfile, FamilyMember, ApprovalToken, NeverSuggest, NeverSuggestReason,
MealPlanStatus, MealPlanItemStatus, MealType, ApprovalTokenStatus,
RecipeCatalogStatus,
)
from app.schemas import (
MealPlanResponse, MealPlanCreate,
@@ -154,6 +155,7 @@ def get_planned_meals(
if not meal_plan:
return None
_enrich_recipe_ingredient_names(db, meal_plan)
return meal_plan
@@ -170,6 +172,23 @@ def create_meal_plan(meal_plan_data: MealPlanCreate, db: Session = Depends(get_d
if existing:
raise HTTPException(status_code=400, detail="Meal plan for this week already exists")
recipe_ids = {item.recipe_id for item in meal_plan_data.items}
if recipe_ids:
active_recipe_ids = {
recipe_id
for (recipe_id,) in db.query(Recipe.id)
.filter(
Recipe.id.in_(recipe_ids),
Recipe.catalog_status == RecipeCatalogStatus.ACTIVE,
)
.all()
}
if active_recipe_ids != recipe_ids:
raise HTTPException(
status_code=400,
detail="Meal plan contains unavailable recipes",
)
db_meal_plan = MealPlan(
family_profile_id=profile.id,
week_start_date=meal_plan_data.week_start_date,
@@ -201,6 +220,7 @@ def get_meal_plan(meal_plan_id: UUID, db: Session = Depends(get_db)):
meal_plan = db.query(MealPlan).filter(MealPlan.id == meal_plan_id).first()
if not meal_plan:
raise HTTPException(status_code=404, detail="Meal plan not found")
_enrich_recipe_ingredient_names(db, meal_plan)
return meal_plan
@@ -221,6 +241,39 @@ _DAY_NAMES = {
}
def _enrich_recipe_ingredient_names(db: Session, meal_plan: MealPlan) -> None:
"""Mutate each embedded Recipe's `ingredients` JSONB in-place to add
a resolved `name` from the canonical Ingredient table. Mirrors the
enrichment the public /api/recipes endpoints do; the meal-plan path
uses pydantic from_attributes which reads the raw JSONB and would
otherwise leave `name=None`. Idempotent: only sets name when
missing.
"""
recipes: list[Recipe] = []
for item in meal_plan.items:
if item.recipe is not None and item.recipe not in recipes:
recipes.append(item.recipe)
if not recipes:
return
all_ids: set = set()
for r in recipes:
for ref in (r.ingredients or []):
iid = ref.get("ingredient_id") if isinstance(ref, dict) else None
if iid:
all_ids.add(iid)
if not all_ids:
return
rows = db.query(Ingredient.id, Ingredient.name).filter(Ingredient.id.in_(all_ids)).all()
name_map = {str(rid): rname for rid, rname in rows}
for r in recipes:
for ref in (r.ingredients or []):
if not isinstance(ref, dict):
continue
iid = str(ref.get("ingredient_id") or "")
if iid and not ref.get("name"):
ref["name"] = name_map.get(iid)
class VoteSubmission(BaseModel):
"""Body for POST /vote/{item_id}?token=...
@@ -556,7 +609,10 @@ def swap_meal_item(item_id: UUID, new_recipe_id: UUID, db: Session = Depends(get
if not item:
raise HTTPException(status_code=404, detail="Meal plan item not found")
new_recipe = db.query(Recipe).filter(Recipe.id == new_recipe_id).first()
new_recipe = db.query(Recipe).filter(
Recipe.id == new_recipe_id,
Recipe.catalog_status == RecipeCatalogStatus.ACTIVE,
).first()
if not new_recipe:
raise HTTPException(status_code=404, detail="New recipe not found")
@@ -668,7 +724,17 @@ def generate_single_item(
raise HTTPException(status_code=400, detail="Slot already occupied")
# Get all recipes, score them, pick the best not already in this plan
all_recipes = db.query(Recipe).all()
all_recipes = db.query(Recipe).filter(
Recipe.catalog_status == RecipeCatalogStatus.ACTIVE
).all()
if not all_recipes:
raise HTTPException(
status_code=409,
detail=(
"No active recipes are available. "
"Approve a staged recipe before generating meals."
),
)
used_ids = {i.recipe_id for i in plan.items if i.recipe_id is not None}
# Simple heuristic: pick a random un-used recipe (we can improve scoring later)
@@ -728,7 +794,9 @@ def fill_empty_slots(
if not requested:
return FillEmptySlotsResult(filled=[], failed=[])
all_recipes = db.query(Recipe).all()
all_recipes = db.query(Recipe).filter(
Recipe.catalog_status == RecipeCatalogStatus.ACTIVE
).all()
if not all_recipes:
# No recipes at all — every requested slot fails.
return FillEmptySlotsResult(
@@ -877,6 +945,7 @@ def discovered_count(db: Session = Depends(get_db)):
db.query(Recipe)
.filter(
Recipe.family_profile_id == profile.id,
Recipe.catalog_status == RecipeCatalogStatus.ACTIVE,
Recipe.external_source.isnot(None),
Recipe.created_at >= since,
)
@@ -908,4 +977,4 @@ def discovery_insights(db: Session = Depends(get_db)):
"positive_signals": analysis.get("positive_signals", []),
"negative_signals": analysis.get("negative_signals", []),
"top_rated_recipes": analysis.get("top_rated_recipes", []),
}
}
+15 -292
View File
@@ -1,308 +1,31 @@
"""Sprint 12 — external recipe search (Spoonacular) + import.
"""Retired direct third-party recipe search and publication endpoints."""
from fastapi import APIRouter, Depends, HTTPException, Query, status
This is a thin HTTP layer on top of the pre-existing Spoonacular
free-tier API. The service-class `RecipeDiscoveryService` in
`app.services.recipe_discovery` is the bulk-orchestrator used by
the offline FeedbackAnalyzer; we re-implement the call shape here
because the webui wants:
1. A search that returns *summary* data (no info endpoint call) so
10 results cost 1.1 points, not 11.1.
2. An import that fetches the full info for ONE recipe and writes
a local Recipe row.
Quota: Spoonacular free tier = 150 points/day. complexSearch = 1 +
0.01 per result. /information = 1 point. We gate at 140 to leave
a safety margin and return 503 once exhausted.
"""
from __future__ import annotations
import logging
import threading
from typing import List, Optional
import requests
from fastapi import APIRouter, Depends, HTTPException, Query
from sqlalchemy.orm import Session
from app.config import settings
from app.database import get_db
from app.models import FamilyProfile, Ingredient, Recipe
from app.schemas import (
RecipeImportRequest,
RecipeSearchHit,
)
from app.schemas import RecipeImportRequest
from app.security import require_session
logger = logging.getLogger(__name__)
router = APIRouter()
# Quota counter: a per-process point budget. Survives across
# requests in the same uvicorn worker. Process restart resets to 0
# (so the operator can recover by bouncing the backend). Thread-safe
# with a single lock — the counter is touched on every request and
# we don't want a race between two parallel searches.
_quota_lock = threading.Lock()
_points_used: float = 0.0
_DAILY_LIMIT: float = 45.0 # 50 free, leave 5pt safety margin (corrected from 140; Sprint 15 + Sprint 16)
_INFO_URL = "https://api.spoonacular.com/recipes/{id}/information"
_SEARCH_URL = "https://api.spoonacular.com/recipes/complexSearch"
_RETIRED_DETAIL = (
"Direct external recipe ingestion is retired; recipes must be staged and "
"approved through the admin import queue."
)
def _points_available() -> float:
with _quota_lock:
return _DAILY_LIMIT - _points_used
def _charge_points(pts: float) -> None:
global _points_used
with _quota_lock:
_points_used += pts
def _ensure_spoonacular_configured() -> str:
"""Return the API key, or 503 if unconfigured."""
key = settings.SPOONACULAR_API_KEY
if not key:
logger.error("SPOONACULAR_API_KEY not configured")
raise HTTPException(
status_code=503,
detail="SPOONACULAR_API_KEY not configured; set it in the backend env",
)
return key
def _normalize_spoonacular_summary(item: dict) -> Optional[RecipeSearchHit]:
"""Map a complexSearch result item into RecipeSearchHit."""
ext_id = str(item.get("id") or "")
title = item.get("title")
if not ext_id or not title:
return None
cuisines = [str(c).lower() for c in (item.get("cuisines") or []) if c]
diets = [str(d).lower() for d in (item.get("diets") or []) if d]
return RecipeSearchHit(
external_id=ext_id,
external_source="spoonacular",
name=title,
image_url=item.get("image"),
source_url=item.get("sourceUrl") or item.get("spoonacularSourceUrl"),
prep_time_minutes=int(item["preparationMinutes"]) if item.get("preparationMinutes") else None,
cook_time_minutes=int(item["cookingMinutes"]) if item.get("cookingMinutes") else None,
servings=int(item["servings"]) if item.get("servings") else None,
cuisine_tags=cuisines,
dietary_tags=diets,
protein_type=None, # would need to call /information or _infer_protein
)
@router.get("/search", response_model=List[RecipeSearchHit])
@router.get("/search")
def search_recipes(
q: str = Query(..., min_length=1, max_length=200),
limit: int = Query(10, ge=1, le=25),
_user: str = Depends(require_session),
) -> List[RecipeSearchHit]:
"""Search Spoonacular's complexSearch index and return a
summary list (no per-recipe /information call). The frontend
uses this for the "Search the web" panel."""
if _points_available() < 1.05: # 1 base + 0.01 * 5 average
raise HTTPException(
status_code=503,
detail="spoonacular daily quota reached; try again tomorrow",
)
api_key = _ensure_spoonacular_configured()
try:
resp = requests.get(
_SEARCH_URL,
params={
"apiKey": api_key,
"query": q,
"number": limit,
"addRecipeInformation": "true",
"fillIngredients": "true",
"instructionsRequired": "true",
},
timeout=15,
)
resp.raise_for_status()
except requests.RequestException as exc:
logger.warning("Spoonacular search failed for %r: %s", q, exc)
raise HTTPException(
status_code=502,
detail=f"spoonacular search failed: {exc}",
) from exc
data = resp.json()
results = data.get("results", [])
# 1 base + 0.01 per result
cost = 1.0 + len(results) * 0.01
_charge_points(cost)
logger.info("Spoonacular search %r → %d hits (%.2f pts, total %.1f/%.0f)",
q, len(results), cost, _points_used, _DAILY_LIMIT)
out: List[RecipeSearchHit] = []
for item in results:
hit = _normalize_spoonacular_summary(item)
if hit:
out.append(hit)
return out
):
"""Reject the legacy Spoonacular search flow without contacting a vendor."""
raise HTTPException(status_code=status.HTTP_410_GONE, detail=_RETIRED_DETAIL)
def _infer_protein_simple(title: str, ingredient_names: List[str]) -> Optional[str]:
"""Lightweight protein inference; mirrors recipe_discovery._infer_protein
but doesn't import the service (to avoid pulling in the rest of the
offline path)."""
text = (title + " " + " ".join(ingredient_names)).lower()
proteins = {
"chicken": ["chicken"],
"beef": ["beef", "steak", "ground beef"],
"pork": ["pork", "bacon", "ham"],
"fish": ["salmon", "tilapia", "cod", "fish fillet"],
"shrimp": ["shrimp", "prawn"],
"turkey": ["turkey"],
"lamb": ["lamb"],
"vegetarian": ["tofu", "tempeh", "vegetarian"],
}
for ptype, kws in proteins.items():
for kw in kws:
if kw in text:
return ptype
return None
def _upsert_ingredient(db: Session, name: str) -> Ingredient:
"""Idempotent insert by name_lower. Mirrors the public
POST /api/ingredients logic without the HTTP roundtrip."""
name_lower = name.lower()
existing = db.query(Ingredient).filter(Ingredient.name_lower == name_lower).first()
if existing:
return existing
row = Ingredient(name=name, name_lower=name_lower, aliases=[])
db.add(row)
try:
db.commit()
db.refresh(row)
except Exception:
db.rollback()
existing = db.query(Ingredient).filter(Ingredient.name_lower == name_lower).first()
if existing:
return existing
raise
return row
@router.post("/import", response_model=dict, status_code=201)
@router.post("/import")
def import_recipe(
payload: RecipeImportRequest,
db: Session = Depends(get_db),
_user: str = Depends(require_session),
) -> dict:
"""Import a Spoonacular recipe into the local library. One
/information call (1 point) + ingredient upserts + Recipe insert.
Returns the new recipe id."""
if payload.external_source != "spoonacular":
raise HTTPException(status_code=400, detail=f"unknown external_source: {payload.external_source}")
if _points_available() < 1.05:
raise HTTPException(status_code=503, detail="spoonacular daily quota reached; try again tomorrow")
api_key = _ensure_spoonacular_configured()
# 1) Reject duplicates
existing = (
db.query(Recipe)
.filter(Recipe.external_source == "spoonacular", Recipe.external_id == payload.external_id)
.first()
)
if existing:
raise HTTPException(status_code=409, detail=f"recipe already imported: {existing.id}")
# 2) Fetch full info
try:
resp = requests.get(
_INFO_URL.format(id=payload.external_id),
params={"apiKey": api_key, "includeNutrition": "false"},
timeout=15,
)
resp.raise_for_status()
except requests.RequestException as exc:
logger.warning("Spoonacular /information failed for %s: %s", payload.external_id, exc)
raise HTTPException(status_code=502, detail=f"spoonacular info failed: {exc}") from exc
full = resp.json()
_charge_points(1.0)
logger.info("Spoonacular import %s (1 pt, total %.1f/%.0f)",
payload.external_id, _points_used, _DAILY_LIMIT)
# 3) Resolve ingredients (upsert)
ingredient_names: List[str] = []
ingredients_json: List[dict] = []
for ing in full.get("extendedIngredients", []):
name = (ing.get("name") or ing.get("originalName") or "").strip()
if not name:
continue
ingredient_names.append(name)
local = _upsert_ingredient(db, name)
qty = ing.get("amount")
ingredients_json.append({
"ingredient_id": str(local.id),
"name": local.name,
"qty": float(qty) if qty is not None else 1.0,
"unit": ing.get("unit", "") or None,
"notes": None,
})
# 4) Extract instructions
instructions: List[str] = []
analyzed = full.get("analyzedInstructions", [])
if analyzed:
for step in analyzed[0].get("steps", []):
txt = step.get("step", "")
if txt:
instructions.append(txt)
if not instructions:
raw = full.get("instructions", "")
if raw:
instructions = [raw]
# 5) Build the Recipe
title = full.get("title") or "Untitled"
cuisines = [str(c).lower() for c in (full.get("cuisines") or []) if c]
diets = [str(d).lower() for d in (full.get("diets") or []) if d]
# 6) Resolve family_profile_id (require_session auto-fills first profile)
profile = db.query(FamilyProfile).first()
if not profile:
raise HTTPException(status_code=404, detail="family profile not found")
row = Recipe(
family_profile_id=profile.id,
name=title,
description=full.get("summary"),
image_url=full.get("image"),
image_source="spoonacular",
prep_time_minutes=int(full["preparationMinutes"]) if full.get("preparationMinutes") else None,
cook_time_minutes=int(full["cookingMinutes"]) if full.get("cookingMinutes") else None,
servings=int(full.get("servings", 4)),
cuisine_tags=cuisines,
dietary_tags=diets,
protein_type=_infer_protein_simple(title, ingredient_names),
ingredients=ingredients_json,
side_dishes=[],
instructions=instructions or ["See source for instructions."],
source_url=full.get("sourceUrl") or full.get("spoonacularSourceUrl"),
scraped_at=None,
is_manually_added=True,
external_source="spoonacular",
external_id=payload.external_id,
discovery_reason="user imported via webui",
)
db.add(row)
db.commit()
db.refresh(row)
return {
"id": str(row.id),
"name": row.name,
"external_id": row.external_id,
"ingredients_imported": len(ingredients_json),
}
):
"""Reject direct publication; reviewed staging is the sole import path."""
raise HTTPException(status_code=status.HTTP_410_GONE, detail=_RETIRED_DETAIL)
+37 -11
View File
@@ -10,7 +10,13 @@ from sqlalchemy import or_
from sqlalchemy.orm import Session
from app.database import get_db
from app.models import FamilyProfile, Ingredient, NeverSuggest, Recipe
from app.models import (
FamilyProfile,
Ingredient,
NeverSuggest,
Recipe,
RecipeCatalogStatus,
)
from app.schemas.recipe import (
RecipeCreate,
RecipeRead,
@@ -45,7 +51,19 @@ def _validate_ingredient_ids(db: Session, ingredient_refs) -> None:
)
def _serialize(row: Recipe) -> dict:
def _serialize(row: Recipe, db: Session) -> dict:
raw_ingredients = row.ingredients or []
# Resolve ingredient names from the canonical Ingredient table
ingredient_ids = [ref["ingredient_id"] for ref in raw_ingredients if ref.get("ingredient_id")]
name_map = {}
if ingredient_ids:
name_rows = db.query(Ingredient.id, Ingredient.name).filter(Ingredient.id.in_(ingredient_ids)).all()
name_map = {str(r[0]): r[1] for r in name_rows}
enriched_ingredients = []
for ref in raw_ingredients:
enriched = dict(ref)
enriched["name"] = name_map.get(str(ref.get("ingredient_id")), "unknown ingredient")
enriched_ingredients.append(enriched)
return {
"id": row.id,
"name": row.name,
@@ -59,7 +77,7 @@ def _serialize(row: Recipe) -> dict:
"protein_type": row.protein_type,
"spice_level": row.spice_level,
"calories_per_serving": row.calories_per_serving,
"ingredients": row.ingredients or [],
"ingredients": enriched_ingredients,
"side_dishes": row.side_dishes or [],
"instructions": list(row.instructions or []),
"source_url": row.source_url,
@@ -91,7 +109,9 @@ def list_recommended_recipes(
}
# Start from all recipes
query = db.query(Recipe).filter(~Recipe.id.in_(blocked)) if blocked else db.query(Recipe)
query = db.query(Recipe).filter(Recipe.catalog_status == RecipeCatalogStatus.ACTIVE)
if blocked:
query = query.filter(~Recipe.id.in_(blocked))
# Apply positive signal filters
pos_cuisines = {s.value for s in result.positive_signals if s.type == "prefer_cuisine"}
@@ -119,6 +139,7 @@ def list_recommended_recipes(
top_rated = result.top_rated_recipe_names or []
if top_rated:
top_rows = db.query(Recipe).filter(
Recipe.catalog_status == RecipeCatalogStatus.ACTIVE,
Recipe.name.in_(top_rated),
(~Recipe.id.in_(blocked) if blocked else True),
).limit(limit).all()
@@ -133,7 +154,7 @@ def list_recommended_recipes(
ids_seen.add(r.id)
ordered.append(r)
return [_serialize(r) for r in ordered]
return [_serialize(r, db) for r in ordered]
@public_router.get("", response_model=List[RecipeRead])
@@ -150,7 +171,9 @@ def list_recipes(
limit: int = Query(default=100, le=500),
db: Session = Depends(get_db),
):
query = db.query(Recipe)
query = db.query(Recipe).filter(
Recipe.catalog_status == RecipeCatalogStatus.ACTIVE
)
if q:
like = f"%{q.lower()}%"
query = query.filter(Recipe.name.ilike(like))
@@ -194,15 +217,18 @@ def list_recipes(
if blocked:
query = query.filter(~Recipe.id.in_(blocked))
rows = query.order_by(Recipe.name).limit(limit).all()
return [_serialize(r) for r in rows]
return [_serialize(r, db) for r in rows]
@public_router.get("/{recipe_id}", response_model=RecipeRead)
def get_recipe(recipe_id: UUID, db: Session = Depends(get_db)):
row = db.query(Recipe).filter(Recipe.id == recipe_id).first()
row = db.query(Recipe).filter(
Recipe.id == recipe_id,
Recipe.catalog_status == RecipeCatalogStatus.ACTIVE,
).first()
if row is None:
raise HTTPException(status_code=404, detail="recipe not found")
return _serialize(row)
return _serialize(row, db)
@admin_router.post("", response_model=RecipeRead, status_code=status.HTTP_201_CREATED)
@@ -228,7 +254,7 @@ def create_recipe(payload: RecipeCreate, db: Session = Depends(get_db)):
db.add(row)
db.commit()
db.refresh(row)
return _serialize(row)
return _serialize(row, db)
@admin_router.patch("/{recipe_id}", response_model=RecipeRead)
@@ -245,7 +271,7 @@ def update_recipe(recipe_id: UUID, payload: RecipeUpdate, db: Session = Depends(
setattr(row, field, value)
db.commit()
db.refresh(row)
return _serialize(row)
return _serialize(row, db)
@admin_router.delete("/{recipe_id}", status_code=status.HTTP_204_NO_CONTENT, response_class=Response)
+1 -6
View File
@@ -39,12 +39,7 @@ class Settings(BaseSettings):
OLLAMA_API_KEY: Optional[str] = None
OLLAMA_MODEL: str = "gpt-oss:20b"
# Sprint 12: Spoonacular external recipe search. Free tier is
# 150 points/day. ComplexSearch = 1 point + 0.01 per result. The
# /information endpoint = 1 point per call. The recipe_search
# router gates calls to stay under 140 points/day to leave a
# safety margin.
SPOONACULAR_API_KEY: Optional[str] = None
WIKIBOOKS_USER_AGENT: str = "MealPlanner/1.0 (recipe catalog staging)"
FAMILY_EMAIL_1: Optional[str] = None
FAMILY_EMAIL_2: Optional[str] = None
File diff suppressed because it is too large Load Diff
+27 -4
View File
@@ -1,3 +1,4 @@
from contextlib import asynccontextmanager
from fastapi import FastAPI, Depends, HTTPException, Request
from fastapi.responses import JSONResponse
from fastapi.staticfiles import StaticFiles
@@ -11,10 +12,34 @@ import logging
logging.basicConfig(level=settings.LOG_LEVEL)
logger = logging.getLogger(__name__)
@asynccontextmanager
async def lifespan(app: FastAPI):
# Auto-migrate on boot. Idempotent: alembic upgrade head is a no-op
# when the schema is already at head. Catches the recurring failure
# where the DB is at revision N but the image expects N+1 (e.g. a
# new column like denial_expires_at), which would otherwise 500 the
# first request that touches the new column. Skip when SKIP_AUTO_MIGRATE
# is set (tests / read-only replicas).
import os
if not os.getenv("SKIP_AUTO_MIGRATE"):
try:
from alembic.config import Config
from alembic import command
cfg = Config("alembic.ini")
command.upgrade(cfg, "head")
logger.info("auto-migrate: alembic upgrade head OK")
except Exception as e:
# Don't crash boot — log loud so it's visible in `docker logs`.
logger.error(f"auto-migrate FAILED: {e!r}")
yield
app = FastAPI(
title="MealPlanner",
description="Self-hosted meal planning system",
version="0.1.0",
lifespan=lifespan,
)
# Serve generated recipe images from local filesystem.
@@ -77,10 +102,8 @@ app.include_router(shopping_list.router, prefix="/api/shopping-list", tags=["sho
app.include_router(pantry.router, prefix="/api/pantry", tags=["pantry"])
app.include_router(admin.router, prefix="/api/admin", tags=["admin"])
app.include_router(orchestrate_api.router, prefix="/api/orchestrate", tags=["orchestrate"])
# Sprint 12: external recipe search (Spoonacular) + import.
# Mounted BEFORE recipes_api.public_router so /search and /import are
# not shadowed by the WIP's GET /{recipe_id} (which would 422 on a
# non-UUID "search" path segment). See sprint15-verification.md.
# Retired external search/import routes stay mounted before /{recipe_id}
# so legacy clients receive 410 rather than UUID validation errors.
app.include_router(recipe_search_api.router, prefix="/api/recipes", tags=["recipes"])
app.include_router(ingredients_api.public_router)
app.include_router(ingredients_api.admin_router)
+83 -1
View File
@@ -1,6 +1,6 @@
from sqlalchemy import (
Column, String, Integer, Numeric, Float, Text, Boolean, Date, DateTime,
ForeignKey, CheckConstraint, UniqueConstraint, Enum as SQLEnum
ForeignKey, CheckConstraint, UniqueConstraint, Index, Enum as SQLEnum
)
from sqlalchemy.dialects.postgresql import UUID, JSONB, ARRAY
from sqlalchemy.orm import relationship
@@ -85,6 +85,18 @@ class IngredientMatchSource(enum.Enum):
MANUAL = "manual"
class RecipeImportStatus(str, enum.Enum):
PENDING = "pending"
QUARANTINED = "quarantined"
APPROVED = "approved"
REJECTED = "rejected"
class RecipeCatalogStatus(str, enum.Enum):
ACTIVE = "active"
QUARANTINED = "quarantined"
class FamilyProfile(Base):
__tablename__ = "family_profile"
@@ -186,6 +198,19 @@ class Recipe(Base):
external_source = Column(String(50))
external_id = Column(String(100))
discovery_reason = Column(Text)
catalog_status = Column(
SQLEnum(
RecipeCatalogStatus,
name="recipe_catalog_status_enum",
create_type=False,
values_callable=lambda obj: [e.value for e in obj],
),
nullable=False,
default=RecipeCatalogStatus.ACTIVE,
server_default="active",
)
catalog_findings = Column(JSONB, nullable=False, default=list, server_default="[]")
catalog_reviewed_at = Column(DateTime(timezone=True))
created_at = Column(DateTime(timezone=True), server_default=func.now())
updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now())
@@ -197,6 +222,63 @@ class Recipe(Base):
return (self.prep_time_minutes or 0) + (self.cook_time_minutes or 0)
class RecipeImportStaging(Base):
"""Auditable, globally scoped recipe import candidate awaiting review."""
__tablename__ = "recipe_import_staging"
id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
source = Column(String(100), nullable=False)
external_id = Column(String(255), nullable=False)
source_url = Column(Text, nullable=False)
source_revision = Column(String(255), nullable=False)
license_name = Column(String(255), nullable=False)
license_url = Column(Text, nullable=False)
attribution = Column(Text, nullable=False)
raw_payload = Column(JSONB, nullable=False)
payload_sha256 = Column(String(64), nullable=False)
normalized_recipe = Column(JSONB, nullable=False)
validation_report = Column(JSONB, nullable=False)
status = Column(
SQLEnum(
RecipeImportStatus,
name="recipe_import_status_enum",
create_type=False,
values_callable=lambda obj: [e.value for e in obj],
),
nullable=False,
default=RecipeImportStatus.PENDING,
server_default="pending",
)
rejection_reason = Column(Text)
recipe_id = Column(UUID(as_uuid=True), ForeignKey("recipe.id", ondelete="SET NULL"))
acquired_at = Column(
DateTime(timezone=True), nullable=False, server_default=func.now()
)
reviewed_at = Column(DateTime(timezone=True))
reviewed_by = Column(String(300))
__table_args__ = (
UniqueConstraint(
"source",
"external_id",
"source_revision",
name="uq_recipe_import_staging_source_revision",
),
CheckConstraint(
"payload_sha256 ~ '^[0-9a-f]{64}$'",
name="ck_recipe_import_staging_payload_sha256",
),
CheckConstraint(
"(reviewed_at IS NULL) = (reviewed_by IS NULL)",
name="ck_recipe_import_staging_review_metadata",
),
Index("ix_recipe_import_staging_status_acquired_at", "status", "acquired_at"),
)
recipe = relationship("Recipe")
class MealPlan(Base):
__tablename__ = "meal_plan"
+1 -19
View File
@@ -147,7 +147,7 @@ class PlannerConfigOverride(BaseModel):
time_full_minutes: Optional[int] = Field(default=None, ge=0)
recency_full_weeks: Optional[int] = Field(default=None, ge=0)
top_k: Optional[int] = Field(default=None, ge=1)
set_size: Optional[int] = Field(default=None, ge=1)
set_size: Optional[int] = Field(default=None, ge=3, le=3)
p_protein: Optional[float] = Field(default=None, ge=0)
p_cuisine: Optional[float] = Field(default=None, ge=0)
@@ -392,24 +392,6 @@ class ShoppingListResponse(BaseModel):
by_aisle: dict[str, List[ShoppingListItem]]
# Sprint 12: Spoonacular external recipe search. The hit shape is
# what the webui shows in the "Search the web" panel; the import
# request is what the import button POSTs.
class RecipeSearchHit(BaseModel):
external_id: str
external_source: str = "spoonacular"
name: str
image_url: Optional[str] = None
source_url: Optional[str] = None
prep_time_minutes: Optional[int] = None
cook_time_minutes: Optional[int] = None
servings: Optional[int] = None
cuisine_tags: List[str] = Field(default_factory=list)
dietary_tags: List[str] = Field(default_factory=list)
protein_type: Optional[str] = None
calories_per_serving: Optional[int] = None
class RecipeImportRequest(BaseModel):
external_id: str
external_source: str = "spoonacular"
@@ -13,6 +13,10 @@ class GenerateRequest(BaseModel):
week_start_date: date
class PublicGenerateRequest(BaseModel):
week_start_date: date
class RegenerateRequest(BaseModel):
family_profile_id: UUID
week_start_date: date
+1 -1
View File
@@ -19,7 +19,7 @@ class PlannerConfigOverride(BaseModel):
time_full_minutes: Optional[int] = Field(default=None, ge=0)
recency_full_weeks: Optional[int] = Field(default=None, ge=0)
top_k: Optional[int] = Field(default=None, ge=1)
set_size: Optional[int] = Field(default=None, ge=1)
set_size: Optional[int] = Field(default=None, ge=3, le=3)
p_protein: Optional[float] = Field(default=None, ge=0)
p_cuisine: Optional[float] = Field(default=None, ge=0)
+5 -1
View File
@@ -9,6 +9,7 @@ from pydantic import BaseModel, Field, field_validator
class RecipeIngredientRef(BaseModel):
ingredient_id: UUID
name: Optional[str] = Field(default=None, description="resolved name from Ingredient table")
qty: float = Field(gt=0)
unit: Optional[str] = Field(default=None, max_length=50)
notes: Optional[str] = None
@@ -66,10 +67,13 @@ class RecipeUpdate(BaseModel):
class RecipeRead(RecipeBase):
id: UUID
prep_time_minutes: Optional[int] = Field(default=None, ge=0)
cook_time_minutes: Optional[int] = Field(default=None, ge=0)
protein_type: Optional[str] = None
external_source: Optional[str] = None
external_id: Optional[str] = None
discovery_reason: Optional[str] = None
side_dishes: List[SideDish] = Field(default_factory=list)
side_dishes: Optional[List[SideDish]] = None
model_config = {"from_attributes": True}
+64
View File
@@ -0,0 +1,64 @@
from __future__ import annotations
from datetime import datetime
from typing import Any
from uuid import UUID
from pydantic import BaseModel, Field, field_validator
class RecipeImportListItem(BaseModel):
id: UUID
source: str
external_id: str
source_url: str
source_revision: str
name: str | None
status: str
acquired_at: datetime
reviewed_at: datetime | None
reviewed_by: str | None
recipe_id: UUID | None
class RecipeImportListResponse(BaseModel):
items: list[RecipeImportListItem]
total: int
limit: int
offset: int
has_next: bool
class RecipeImportDetail(BaseModel):
id: UUID
source: str
external_id: str
source_url: str
source_revision: str
license_name: str
license_url: str
attribution: str
raw_payload: dict[str, Any]
payload_sha256: str
normalized_recipe: dict[str, Any]
validation_report: dict[str, Any]
status: str
rejection_reason: str | None
recipe_id: UUID | None
acquired_at: datetime
reviewed_at: datetime | None
reviewed_by: str | None
model_config = {"from_attributes": True}
class RecipeImportRejectRequest(BaseModel):
reason: str = Field(min_length=1, max_length=1000)
@field_validator("reason")
@classmethod
def normalize_reason(cls, value: str) -> str:
cleaned = value.strip()
if not cleaned:
raise ValueError("reason must not be blank")
return cleaned
+3 -2
View File
@@ -15,8 +15,8 @@ SESSION_COOKIE = "mp_session"
SESSION_MAX_AGE = 60 * 60 * 24 * 30 # 30 days
def require_admin(request: Request) -> None:
"""Enforce a shared bearer token. 401 on bad/missing token, 503 if unset."""
def require_admin(request: Request) -> str:
"""Enforce the shared bearer token and return its configured principal."""
expected = settings.ADMIN_TOKEN
if not expected:
raise HTTPException(
@@ -29,6 +29,7 @@ def require_admin(request: Request) -> None:
status_code=status.HTTP_401_UNAUTHORIZED,
detail="Invalid admin token",
)
return (settings.ADMIN_EMAIL or "").strip() or "admin"
def _signer() -> TimestampSigner:
+5 -2
View File
@@ -17,7 +17,7 @@ import httpx
from sqlalchemy.orm import Session
from app.config import settings
from app.models import Recipe
from app.models import Recipe, RecipeCatalogStatus
logger = logging.getLogger(__name__)
@@ -156,6 +156,7 @@ def generate_image_for_recipe(
# Persist on recipe row if a DB session was provided.
if db is not None:
recipe.image_url = public_path
recipe.image_source = f"openai:{model}"
db.add(recipe)
db.commit()
logger.info("Image saved for '%s' -> %s", recipe.name, public_path)
@@ -183,7 +184,9 @@ def generate_images_batch(
"""
from sqlalchemy import or_
query = db.query(Recipe)
query = db.query(Recipe).filter(
Recipe.catalog_status == RecipeCatalogStatus.ACTIVE
)
if recipe_ids:
query = query.filter(Recipe.id.in_(recipe_ids))
+43 -33
View File
@@ -17,7 +17,7 @@ from datetime import datetime, timezone
from typing import TYPE_CHECKING
from app.config import settings
from app.models import FamilyMember, FamilyProfile, Ingredient, IngredientGroceryMatch, MealPlan, MealPlanItemStatus, MealPlanVote
from app.models import FamilyMember, FamilyProfile, Ingredient, IngredientGroceryMatch, MealPlan, MealPlanItemStatus, MealPlanVote, RecipeCatalogStatus
from app.services.approval import issue_token
from app.services.email import get_email_backend
from app.services.orchestrator.alerts import send_admin_alert
@@ -26,8 +26,7 @@ from app.services.scraper_service import ScraperService
from app.utils.units import convert_qty
from app.services.feedback_analyzer import FeedbackAnalyzer
from app.services.recipe_discovery import RecipeDiscoveryService
from app.services.recipe_ingestion import RecipeIngestionService
from app.services.wikibooks_import import WikibooksImportService
if TYPE_CHECKING:
from sqlalchemy.orm import Session
@@ -56,6 +55,15 @@ def _suggested_sides_html(components: dict | None) -> str:
)
def _active_recipe_items(items: list) -> list:
return [
item
for item in items
if item.recipe is not None
and item.recipe.catalog_status == RecipeCatalogStatus.ACTIVE
]
def step_scrape(run: "WeeklyRun", db: "Session") -> None:
if run.scraped_at is not None:
logger.info("step_scrape: already done for %s, skipping", run.week_start_date)
@@ -132,6 +140,12 @@ def step_email(run: "WeeklyRun", db: "Session") -> None:
f"No MealPlan for family {run.family_id} week {run.week_start_date}"
)
pending_items = [
item
for item in _active_recipe_items(plan.items)
if item.approval_status == MealPlanItemStatus.pending
]
members = (
db.query(FamilyMember)
.filter(
@@ -150,11 +164,10 @@ def step_email(run: "WeeklyRun", db: "Session") -> None:
# Preload ingredient names for all pending items
ing_ids: set = set()
for _item in plan.items:
if _item.approval_status == MealPlanItemStatus.pending and _item.recipe:
for _ing in (_item.recipe.ingredients or []):
if "ingredient_id" in _ing:
ing_ids.add(_ing["ingredient_id"])
for _item in pending_items:
for _ing in (_item.recipe.ingredients or []):
if "ingredient_id" in _ing:
ing_ids.add(_ing["ingredient_id"])
ingredient_names: dict = {}
if ing_ids:
@@ -165,10 +178,8 @@ def step_email(run: "WeeklyRun", db: "Session") -> None:
# Build shopping list preview once (shared across all member emails)
all_ingredients: dict[str, tuple[str, str, str, str]] = {}
for item in plan.items:
if item.approval_status != MealPlanItemStatus.pending:
continue
for ing in (item.recipe.ingredients or [] if item.recipe else []):
for item in pending_items:
for ing in (item.recipe.ingredients or []):
ing_name = ingredient_names.get(str(ing.get("ingredient_id", "")), ing.get("name", "unknown")).strip()
if not ing_name or ing_name in all_ingredients:
continue
@@ -217,9 +228,7 @@ def step_email(run: "WeeklyRun", db: "Session") -> None:
backend = get_email_backend()
for member in members:
item_html_parts = []
for item in plan.items:
if item.approval_status != MealPlanItemStatus.pending:
continue
for item in pending_items:
token = issue_token(item.id, member.id)
vote_url = (
f"{settings.APP_BASE_URL}/api/meals/vote/{item.id}?token={token}"
@@ -400,7 +409,7 @@ def step_finalize(run: "WeeklyRun", db: "Session") -> None:
approved_items = [
item
for item in (plan.items if plan else [])
for item in _active_recipe_items(plan.items if plan else [])
if item.approval_status == MealPlanItemStatus.approved
]
@@ -524,18 +533,19 @@ def step_finalize(run: "WeeklyRun", db: "Session") -> None:
analysis.discovery_queries and
analysis.confidence >= 0.5
):
discovery = RecipeDiscoveryService()
candidates = discovery.discover(analysis.discovery_queries)
if candidates:
ingestion = RecipeIngestionService()
added = ingestion.ingest(db, run.family_id, candidates, analysis.to_dict())
logger.info(
"step_finalize: recipe discovery added %d new recipes for family %s",
added, run.family_id,
# Roll back a partial discovery batch without losing finalization.
with db.begin_nested():
staged = WikibooksImportService(
settings.WIKIBOOKS_USER_AGENT
).stage_discovered_pages(
db,
limit=min(5, len(analysis.discovery_queries)),
queries=analysis.discovery_queries,
)
# ingestion deliberately does not commit so orchestrator
# can keep everything in one transaction
db.commit()
logger.info(
"step_finalize: staged %d Wikibooks candidates for review",
len(staged),
)
except Exception as exc:
# Discovery is best-effort; never block finalization
logger.warning("step_finalize: recipe discovery failed: %s", exc)
@@ -565,11 +575,12 @@ def step_reminder(run: "WeeklyRun", db: "Session") -> None:
db.commit()
return
pending_item_ids = [
item.id
for item in plan.items
pending_items = [
item
for item in _active_recipe_items(plan.items)
if item.approval_status == MealPlanItemStatus.pending
]
pending_item_ids = [item.id for item in pending_items]
if not pending_item_ids:
run.reminded_at = datetime.now(timezone.utc)
db.commit()
@@ -597,9 +608,8 @@ def step_reminder(run: "WeeklyRun", db: "Session") -> None:
}
unvoted = [
item
for item in plan.items
if item.approval_status == MealPlanItemStatus.pending
and item.id not in voted_ids
for item in pending_items
if item.id not in voted_ids
]
if not unvoted:
continue
+2
View File
@@ -37,6 +37,8 @@ class PlannerConfig:
total = self.w_savings + self.w_coverage + self.w_pantry + self.w_time + self.w_recency
if abs(total - 1.0) > 1e-6:
raise ValueError(f"weights must sum to 1.0, got {total}")
if self.set_size != 3:
raise ValueError("set_size must be 3 dinners per week")
def merge(self, overrides: "dict[str, Any]") -> "PlannerConfig":
"""Return a new PlannerConfig with overrides applied."""
+8 -1
View File
@@ -3,6 +3,7 @@ from __future__ import annotations
from datetime import date
from decimal import Decimal
import logging
from typing import Dict, List, Optional, Set
from uuid import UUID
@@ -22,6 +23,7 @@ from app.models import (
MealType,
NeverSuggest,
Recipe,
RecipeCatalogStatus,
)
from app.services.planner.config import DEFAULT, PlannerConfig
from app.services.planner.cost import compute_recipe_cost
@@ -32,6 +34,9 @@ from app.services.planner.types import GenerationResult
from app.services.meal_pairings import components_with_suggested_sides
logger = logging.getLogger(__name__)
def _load_match_index(db: Session) -> Dict[UUID, List[dict]]:
"""ingredient_id → list of match dicts ordered by confidence DESC."""
rows = (
@@ -148,7 +153,9 @@ def generate_meal_plan(
except (ValueError, TypeError) as exc:
logger.warning("Invalid planner_config for family %s: %s", family_id, exc)
recipes = db.query(Recipe).all()
recipes = db.query(Recipe).filter(
Recipe.catalog_status == RecipeCatalogStatus.ACTIVE
).all()
exclude_set = exclude_recipe_ids or set()
recipe_dicts = [
{
@@ -0,0 +1,221 @@
"""Deterministic provenance and quality audit for the published recipe catalog."""
from __future__ import annotations
from datetime import datetime, timezone
from decimal import Decimal
from typing import Any
from sqlalchemy import and_
from sqlalchemy.orm import Session
from app.models import (
Ingredient,
Recipe,
RecipeCatalogStatus,
RecipeImportStaging,
RecipeImportStatus,
)
from app.services.wikibooks_import import WIKIBOOKS_SOURCE, WikibooksImportService
OPERATOR_QUARANTINE_FINDING_CODE = "operator_quarantine"
def audit_recipe_catalog(
db: Session,
*,
apply: bool = False,
reviewed_at: datetime | None = None,
) -> dict[str, Any]:
"""Audit every recipe and optionally apply active/quarantined state changes."""
timestamp = reviewed_at or datetime.now(timezone.utc)
ingredient_names = {
str(ingredient_id): name
for ingredient_id, name in db.query(Ingredient.id, Ingredient.name).all()
}
catalog_query = (
db.query(
Recipe,
RecipeImportStaging.source,
RecipeImportStaging.external_id,
RecipeImportStaging.source_url,
)
.outerjoin(
RecipeImportStaging,
and_(
RecipeImportStaging.recipe_id == Recipe.id,
RecipeImportStaging.status == RecipeImportStatus.APPROVED,
),
)
.order_by(Recipe.id, RecipeImportStaging.id)
)
if apply:
catalog_query = catalog_query.with_for_update(of=Recipe)
recipes: dict[str, Recipe] = {}
approved_links: dict[str, set[tuple[str, str, str]]] = {}
for recipe, source, external_id, source_url in catalog_query.all():
recipe_id = str(recipe.id)
recipes[recipe_id] = recipe
if source is not None and external_id is not None and source_url is not None:
approved_links.setdefault(recipe_id, set()).add(
(source, external_id, source_url)
)
validator = WikibooksImportService("MealPlanner catalog audit")
records: list[dict[str, Any]] = []
for recipe_id, recipe in recipes.items():
prior_findings = (
recipe.catalog_findings
if isinstance(recipe.catalog_findings, list)
else []
)
operator_quarantine_findings = [
finding
for finding in prior_findings
if isinstance(finding, dict)
and finding.get("code") == OPERATOR_QUARANTINE_FINDING_CODE
]
provenance, findings = _provenance_findings(
recipe, approved_links.get(recipe_id, set())
)
candidate, reference_findings = _validation_candidate(recipe, ingredient_names)
findings.extend(reference_findings)
validation = validator.validate_recipe(candidate)
findings.extend(
{**issue, "severity": severity}
for severity, key in (("error", "errors"), ("warning", "warnings"))
for issue in validation[key]
)
findings.extend(operator_quarantine_findings)
target_status = (
RecipeCatalogStatus.QUARANTINED
if operator_quarantine_findings
or any(item["severity"] == "error" for item in findings)
else RecipeCatalogStatus.ACTIVE
)
if apply:
recipe.catalog_status = target_status
recipe.catalog_findings = findings
recipe.catalog_reviewed_at = timestamp
records.append(
{
"id": str(recipe.id),
"name": recipe.name,
"provenance": provenance,
"catalog_status": target_status.value,
"findings": findings,
}
)
if apply:
db.flush()
quarantined = sum(
record["catalog_status"] == RecipeCatalogStatus.QUARANTINED.value
for record in records
)
return {
"version": 1,
"generated_at": timestamp.isoformat(),
"applied": apply,
"total": len(records),
"summary": {
"active": len(records) - quarantined,
"quarantined": quarantined,
},
"recipes": records,
}
def _provenance_findings(
recipe: Recipe, approved_links: set[tuple[str, str, str]]
) -> tuple[str, list[dict[str, str]]]:
if recipe.is_manually_added and not recipe.external_source:
return "manual", []
if recipe.external_source == WIKIBOOKS_SOURCE:
findings = []
expected_link = (recipe.external_source, recipe.external_id, recipe.source_url)
if expected_link not in approved_links:
findings.append(
_finding(
"missing_import_provenance",
"external_source",
"Published Wikibooks recipe is not linked to an approved staged import.",
)
)
if not recipe.source_url:
findings.append(
_finding("missing_source_url", "source_url", "External recipe has no source URL.")
)
return "staged_wikibooks", findings
if recipe.external_source == "spoonacular":
return "spoonacular", [
_finding(
"unlicensed_external_source",
"external_source",
"Spoonacular data was persisted without durable catalog rights.",
)
]
if recipe.external_source:
return "external_unverified", [
_finding(
"unverified_external_source",
"external_source",
"External recipe has no approved staged license record.",
)
]
return "missing", [
_finding(
"missing_provenance",
"external_source",
"Non-manual recipe has no auditable source or license record.",
)
]
def _validation_candidate(
recipe: Recipe, ingredient_names: dict[str, str]
) -> tuple[dict[str, Any], list[dict[str, str]]]:
ingredients: list[dict[str, Any]] = []
findings: list[dict[str, str]] = []
for index, item in enumerate(recipe.ingredients or []):
if not isinstance(item, dict):
findings.append(
_finding(
"invalid_ingredient_reference",
f"ingredients[{index}]",
"Ingredient reference must be an object.",
)
)
continue
ingredient_id = str(item.get("ingredient_id") or "")
name = ingredient_names.get(ingredient_id)
if not name:
findings.append(
_finding(
"orphan_ingredient_reference",
f"ingredients[{index}].ingredient_id",
"Ingredient reference does not resolve to the canonical ingredient table.",
)
)
name = str(item.get("name") or "")
quantity = item.get("qty")
if isinstance(quantity, Decimal):
quantity = float(quantity)
ingredients.append({"name": name, "quantity": quantity})
return (
{
"name": recipe.name,
"description": recipe.description,
"servings": recipe.servings,
"prep_time_minutes": recipe.prep_time_minutes,
"cook_time_minutes": recipe.cook_time_minutes,
"baking_time_minutes": None,
"ingredients": ingredients,
"instructions": list(recipe.instructions or []),
},
findings,
)
def _finding(code: str, field: str, message: str) -> dict[str, str]:
return {"code": code, "severity": "error", "field": field, "message": message}
@@ -0,0 +1,196 @@
from __future__ import annotations
from datetime import datetime, timezone
from typing import Any, Mapping
from uuid import UUID
from sqlalchemy import select
from sqlalchemy.dialects.postgresql import insert as pg_insert
from sqlalchemy.orm import Session
from app.models import (
Ingredient,
Recipe,
RecipeCatalogStatus,
RecipeImportStaging,
RecipeImportStatus,
)
from app.services.wikibooks_import import (
WIKIBOOKS_SOURCE,
WikibooksImportService,
canonical_json_sha256,
)
class RecipeImportNotFoundError(LookupError):
"""Raised when a staged recipe does not exist."""
class RecipeImportConflictError(ValueError):
"""Raised when a review transition or publication is unsafe."""
def approve_recipe_import(
db: Session, import_id: UUID, reviewed_by: str
) -> tuple[RecipeImportStaging, bool]:
"""Atomically publish one pending Wikibooks candidate."""
staged = _locked_import(db, import_id)
if staged.status == RecipeImportStatus.APPROVED:
if staged.recipe_id and db.get(Recipe, staged.recipe_id):
return staged, False
elif staged.status != RecipeImportStatus.PENDING:
raise RecipeImportConflictError(
f"Only pending imports can be approved; current status is {staged.status.value}"
)
if staged.source != WIKIBOOKS_SOURCE:
raise RecipeImportConflictError(f"Unsupported recipe source: {staged.source}")
raw_payload = _mapping(staged.raw_payload, "raw_payload")
if canonical_json_sha256(raw_payload) != staged.payload_sha256:
raise RecipeImportConflictError("Raw payload checksum does not match provenance")
normalizer = WikibooksImportService("MealPlanner recipe review")
normalized = normalizer.normalize_page(raw_payload)
validation_report = normalizer.validate_recipe(normalized)
if not validation_report["valid"]:
raise RecipeImportConflictError("Candidate no longer passes publication validation")
existing = (
db.query(Recipe)
.filter(
Recipe.external_source == staged.source,
Recipe.external_id == staged.external_id,
)
.first()
)
if existing:
raise RecipeImportConflictError(
"A recipe from this source page is already published"
)
ingredient_refs = _ingredient_refs(db, normalized["ingredients"])
cook_time = (normalized.get("cook_time_minutes") or 0) + (
normalized.get("baking_time_minutes") or 0
)
recipe = Recipe(
family_profile_id=None,
name=normalized["name"],
description=normalized.get("description"),
image_url=None,
image_source=None,
prep_time_minutes=normalized.get("prep_time_minutes"),
cook_time_minutes=cook_time or None,
servings=normalized["servings"],
cuisine_tags=[],
dietary_tags=[],
protein_type=None,
spice_level=None,
calories_per_serving=None,
ingredients=ingredient_refs,
side_dishes=[],
instructions=list(normalized["instructions"]),
source_url=staged.source_url,
scraped_at=staged.acquired_at,
is_manually_added=False,
external_source=staged.source,
external_id=staged.external_id,
discovery_reason=f"Approved import from revision {staged.source_revision}",
catalog_status=RecipeCatalogStatus.ACTIVE,
catalog_findings=[],
)
db.add(recipe)
db.flush()
staged.normalized_recipe = normalized
staged.validation_report = validation_report
staged.status = RecipeImportStatus.APPROVED
staged.rejection_reason = None
staged.recipe_id = recipe.id
staged.reviewed_at = datetime.now(timezone.utc)
staged.reviewed_by = reviewed_by[:300]
db.commit()
db.refresh(staged)
return staged, True
def reject_recipe_import(
db: Session, import_id: UUID, reason: str, reviewed_by: str
) -> RecipeImportStaging:
"""Atomically reject a pending or quarantined candidate."""
staged = _locked_import(db, import_id)
if staged.status == RecipeImportStatus.APPROVED:
raise RecipeImportConflictError("Approved imports cannot be rejected")
if staged.status == RecipeImportStatus.REJECTED:
if staged.rejection_reason == reason:
return staged
raise RecipeImportConflictError("Import was already rejected with another reason")
staged.status = RecipeImportStatus.REJECTED
staged.rejection_reason = reason
staged.reviewed_at = datetime.now(timezone.utc)
staged.reviewed_by = reviewed_by[:300]
db.commit()
db.refresh(staged)
return staged
def _locked_import(db: Session, import_id: UUID) -> RecipeImportStaging:
staged = db.execute(
select(RecipeImportStaging)
.where(RecipeImportStaging.id == import_id)
.with_for_update()
).scalar_one_or_none()
if staged is None:
raise RecipeImportNotFoundError("Recipe import not found")
return staged
def _ingredient_refs(
db: Session, ingredients: list[Mapping[str, Any]]
) -> list[dict[str, Any]]:
prepared: list[tuple[str, str, float, str | None, str | None]] = []
for item in ingredients:
name = " ".join(str(item.get("name", "")).split())
name_lower = name.casefold()
quantity = item.get("quantity")
unit = item.get("unit")
source_text = item.get("source_text")
if not name or len(name) > 200:
raise RecipeImportConflictError("Ingredient name is invalid")
if not isinstance(quantity, (int, float)) or isinstance(quantity, bool) or quantity <= 0:
raise RecipeImportConflictError("Ingredient quantity is invalid")
if unit is not None and (not isinstance(unit, str) or len(unit) > 50):
raise RecipeImportConflictError("Ingredient unit is invalid")
prepared.append((name, name_lower, float(quantity), unit, source_text))
for name, name_lower, _, unit, _ in prepared:
db.execute(
pg_insert(Ingredient.__table__)
.values(name=name, name_lower=name_lower, unit=unit, aliases=[])
.on_conflict_do_nothing(index_elements=[Ingredient.name_lower])
)
canonical = {
row.name_lower: row
for row in db.query(Ingredient)
.filter(Ingredient.name_lower.in_([item[1] for item in prepared]))
.all()
}
if len(canonical) != len({item[1] for item in prepared}):
raise RecipeImportConflictError("Canonical ingredient creation failed")
return [
{
"ingredient_id": str(canonical[name_lower].id),
"qty": quantity,
"unit": unit,
"notes": source_text,
}
for _, name_lower, quantity, unit, source_text in prepared
]
def _mapping(value: Any, field: str) -> Mapping[str, Any]:
if not isinstance(value, Mapping):
raise RecipeImportConflictError(f"{field} must be an object")
return value
@@ -0,0 +1,540 @@
"""Validate and atomically load the bundled starter recipe catalog."""
from __future__ import annotations
import json
import math
import re
from pathlib import Path
from typing import Any, Mapping
from uuid import UUID
from sqlalchemy import func, select
from sqlalchemy.dialects.postgresql import insert as pg_insert
from sqlalchemy.orm import Session
from app.models import Ingredient, Recipe, RecipeCatalogStatus
STARTER_CATALOG_VERSION = 2
STARTER_RECIPE_COUNT = 50
STARTER_DISCOVERY_REASON = "bundled_starter_catalog_v1"
# Fixed PostgreSQL bigint advisory-lock key for this loader (ASCII: MPLSTART).
# Changing it would allow concurrent catalog loads to bypass each other.
STARTER_CATALOG_ADVISORY_LOCK_ID = 0x4D504C5354415254
DEFAULT_CATALOG_PATH = Path(__file__).resolve().parents[1] / "data" / "starter_recipes.json"
MAX_SERVINGS = 50
MAX_TIME_MINUTES = 1_440
MAX_INGREDIENT_QUANTITY = 10_000
MAX_CALORIES_PER_SERVING = 500
MAX_SPICE_LEVEL = 5
_PLACEHOLDER_TEXT = frozenset(
{"coming soon", "directions", "instructions", "method", "n a", "tbd", "todo"}
)
_PLACEHOLDER_PHRASES = ("lorem ipsum", "placeholder")
_RECIPE_KEYS = frozenset(
{
"name",
"description",
"servings",
"prep_time_minutes",
"cook_time_minutes",
"cuisine_tags",
"dietary_tags",
"protein_type",
"spice_level",
"calories_per_serving",
"ingredients",
"instructions",
}
)
_INGREDIENT_KEYS = frozenset({"name", "quantity", "unit"})
_CATALOG_SCALAR_FIELDS = (
"description",
"servings",
"prep_time_minutes",
"cook_time_minutes",
"cuisine_tags",
"dietary_tags",
"protein_type",
"spice_level",
"calories_per_serving",
)
class StarterRecipeCatalogError(ValueError):
"""Raised when the bundled catalog is incomplete or unsafe to load."""
def read_starter_catalog(path: Path = DEFAULT_CATALOG_PATH) -> Mapping[str, Any]:
"""Read the JSON catalog without accepting non-standard numeric constants."""
try:
with path.open(encoding="utf-8") as catalog_file:
catalog = json.load(catalog_file, parse_constant=_reject_json_constant)
except OSError as exc:
raise StarterRecipeCatalogError(f"Catalog cannot be read: {path}") from exc
except ValueError as exc:
raise StarterRecipeCatalogError("Catalog is not valid JSON") from exc
if not isinstance(catalog, Mapping):
raise StarterRecipeCatalogError("Catalog root must be an object")
return catalog
def validate_starter_catalog(catalog: Mapping[str, Any]) -> list[dict[str, Any]]:
"""Return normalized recipes after validating the complete, fixed-size catalog."""
_require_exact_keys(catalog, {"version", "recipes"}, "catalog")
if catalog["version"] != STARTER_CATALOG_VERSION:
raise StarterRecipeCatalogError(
f"Catalog version must be {STARTER_CATALOG_VERSION}"
)
recipes = catalog["recipes"]
if not isinstance(recipes, list) or len(recipes) != STARTER_RECIPE_COUNT:
raise StarterRecipeCatalogError(
f"Catalog must contain exactly {STARTER_RECIPE_COUNT} recipes"
)
normalized: list[dict[str, Any]] = []
names: set[str] = set()
for index, recipe in enumerate(recipes, 1):
if not isinstance(recipe, Mapping):
raise StarterRecipeCatalogError(f"Recipe {index} must be an object")
_require_exact_keys(recipe, _RECIPE_KEYS, f"Recipe {index}")
name = _text(recipe["name"], f"Recipe {index} name", maximum=300)
name_key = _canonical_name(name)
if name_key in names:
raise StarterRecipeCatalogError(f"Recipe {index} name is not unique")
names.add(name_key)
ingredients = _validate_ingredients(recipe["ingredients"], index)
instructions = _validate_instructions(recipe["instructions"], index)
normalized.append(
{
"name": name,
"description": _text(
recipe["description"], f"Recipe {index} description", maximum=10_000
),
"servings": _positive_int(
recipe["servings"], f"Recipe {index} servings", MAX_SERVINGS
),
"prep_time_minutes": _positive_int(
recipe["prep_time_minutes"],
f"Recipe {index} prep_time_minutes",
MAX_TIME_MINUTES,
),
"cook_time_minutes": _positive_int(
recipe["cook_time_minutes"],
f"Recipe {index} cook_time_minutes",
MAX_TIME_MINUTES,
),
"cuisine_tags": _tags(recipe["cuisine_tags"], f"Recipe {index} cuisine_tags"),
"dietary_tags": _tags(recipe["dietary_tags"], f"Recipe {index} dietary_tags"),
"protein_type": _text(
recipe["protein_type"], f"Recipe {index} protein_type", maximum=50
),
"spice_level": _bounded_int(
recipe["spice_level"], f"Recipe {index} spice_level", 0, MAX_SPICE_LEVEL
),
"calories_per_serving": _positive_int(
recipe["calories_per_serving"],
f"Recipe {index} calories_per_serving",
MAX_CALORIES_PER_SERVING,
),
"ingredients": ingredients,
"instructions": instructions,
}
)
return normalized
def load_bundled_starter_catalog(
db: Session,
*,
path: Path = DEFAULT_CATALOG_PATH,
dry_run: bool = False,
update_existing: bool = False,
) -> dict[str, Any]:
"""Load the bundled JSON catalog after validating all recipes before writes."""
return load_starter_catalog(
db,
read_starter_catalog(path),
dry_run=dry_run,
update_existing=update_existing,
)
def load_starter_catalog(
db: Session,
catalog: Mapping[str, Any],
*,
dry_run: bool = False,
update_existing: bool = False,
) -> dict[str, Any]:
"""Load or explicitly synchronize bundled recipes in one transaction."""
recipes = validate_starter_catalog(catalog)
catalog_names = {recipe["name"] for recipe in recipes}
catalog_by_name = {recipe["name"]: recipe for recipe in recipes}
ingredient_names = {
_canonical_name(ingredient["name"]): ingredient
for recipe in recipes
for ingredient in recipe["ingredients"]
}
with db.begin():
# This transaction-scoped lock must precede every catalog Recipe read or write.
db.execute(select(func.pg_advisory_xact_lock(STARTER_CATALOG_ADVISORY_LOCK_ID)))
marked_recipes = list(
db.execute(
select(Recipe)
.where(Recipe.discovery_reason == STARTER_DISCOVERY_REASON)
.with_for_update()
).scalars()
)
marked_by_name = {recipe.name: recipe for recipe in marked_recipes}
if len(marked_by_name) != len(marked_recipes):
raise StarterRecipeCatalogError("Bundled catalog has duplicate persisted recipe names")
unknown_marked_names = set(marked_by_name).difference(catalog_names)
if unknown_marked_names:
raise StarterRecipeCatalogError(
"Bundled catalog marker is attached to an unknown recipe"
)
resolved_recipe_ingredients = _resolve_recipe_ingredients(db, marked_recipes)
missing_marked_names = catalog_names.difference(marked_by_name)
if update_existing and missing_marked_names:
raise StarterRecipeCatalogError(
"Update requires exactly all "
f"{STARTER_RECIPE_COUNT} existing marker-owned recipe names"
)
recipes_to_update: list[Recipe] = []
for recipe in marked_recipes:
expected = catalog_by_name[recipe.name]
if update_existing:
_assert_bundled_recipe_ownership(recipe)
if not _bundled_recipe_content_matches(
recipe, expected, resolved_recipe_ingredients
):
recipes_to_update.append(recipe)
else:
_assert_bundled_recipe(
recipe,
expected,
resolved_recipe_ingredients,
)
name_collisions = list(
db.execute(
select(Recipe).where(Recipe.name.in_(catalog_names)).with_for_update()
).scalars()
)
if any(recipe.discovery_reason != STARTER_DISCOVERY_REASON for recipe in name_collisions):
raise StarterRecipeCatalogError(
"Catalog recipe name conflicts with an existing non-bundled recipe"
)
canonical_ingredients = {
ingredient.name_lower: ingredient
for ingredient in db.execute(
select(Ingredient).where(Ingredient.name_lower.in_(ingredient_names))
).scalars()
}
new_recipes = (
[]
if update_existing
else [recipe for recipe in recipes if recipe["name"] not in marked_by_name]
)
summary = {
"catalog_version": STARTER_CATALOG_VERSION,
"dry_run": dry_run,
"validated_recipes": len(recipes),
"existing_recipes": len(marked_recipes),
"created_recipes": len(new_recipes),
"updated_recipes": len(recipes_to_update),
"existing_ingredients": len(canonical_ingredients),
"created_ingredients": len(ingredient_names) - len(canonical_ingredients),
}
if dry_run:
return summary
for name_lower, ingredient in ingredient_names.items():
db.execute(
pg_insert(Ingredient.__table__)
.values(
name=ingredient["name"],
name_lower=name_lower,
unit=ingredient["unit"],
aliases=[],
)
.on_conflict_do_nothing(index_elements=[Ingredient.name_lower])
)
canonical_ingredients = {
ingredient.name_lower: ingredient
for ingredient in db.execute(
select(Ingredient).where(Ingredient.name_lower.in_(ingredient_names))
).scalars()
}
if set(canonical_ingredients) != set(ingredient_names):
raise StarterRecipeCatalogError("Canonical ingredient creation failed")
for recipe in recipes_to_update:
_synchronize_bundled_recipe(
recipe,
catalog_by_name[recipe.name],
canonical_ingredients,
)
for recipe in new_recipes:
db.add(
Recipe(
family_profile_id=None,
name=recipe["name"],
description=recipe["description"],
image_url=None,
image_source=None,
prep_time_minutes=recipe["prep_time_minutes"],
cook_time_minutes=recipe["cook_time_minutes"],
servings=recipe["servings"],
cuisine_tags=recipe["cuisine_tags"],
dietary_tags=recipe["dietary_tags"],
protein_type=recipe["protein_type"],
spice_level=recipe["spice_level"],
calories_per_serving=recipe["calories_per_serving"],
ingredients=_catalog_recipe_ingredients(recipe, canonical_ingredients),
side_dishes=[],
instructions=recipe["instructions"],
source_url=None,
scraped_at=None,
is_manually_added=True,
external_source=None,
external_id=None,
discovery_reason=STARTER_DISCOVERY_REASON,
catalog_status=RecipeCatalogStatus.ACTIVE,
catalog_findings=[],
)
)
db.flush()
return summary
def _validate_ingredients(value: Any, recipe_index: int) -> list[dict[str, Any]]:
if not isinstance(value, list) or len(value) < 2:
raise StarterRecipeCatalogError(f"Recipe {recipe_index} needs at least two ingredients")
ingredients: list[dict[str, Any]] = []
for index, ingredient in enumerate(value, 1):
if not isinstance(ingredient, Mapping):
raise StarterRecipeCatalogError(
f"Recipe {recipe_index} ingredient {index} must be an object"
)
_require_exact_keys(
ingredient,
_INGREDIENT_KEYS,
f"Recipe {recipe_index} ingredient {index}",
)
ingredients.append(
{
"name": _text(
ingredient["name"],
f"Recipe {recipe_index} ingredient {index} name",
maximum=200,
),
"quantity": _positive_number(
ingredient["quantity"],
f"Recipe {recipe_index} ingredient {index} quantity",
MAX_INGREDIENT_QUANTITY,
),
"unit": _text(
ingredient["unit"], f"Recipe {recipe_index} ingredient {index} unit", maximum=50
),
}
)
return ingredients
def _validate_instructions(value: Any, recipe_index: int) -> list[str]:
if not isinstance(value, list) or len(value) < 2:
raise StarterRecipeCatalogError(f"Recipe {recipe_index} needs at least two instructions")
instructions = [
_text(instruction, f"Recipe {recipe_index} instruction {index}", maximum=5_000)
for index, instruction in enumerate(value, 1)
]
for instruction in instructions:
if len(instruction) < 20 or len(re.findall(r"[A-Za-z0-9]+", instruction)) < 4:
raise StarterRecipeCatalogError(
f"Recipe {recipe_index} instructions must be substantive"
)
return instructions
def _tags(value: Any, field: str) -> list[str]:
if not isinstance(value, list):
raise StarterRecipeCatalogError(f"{field} must be an array")
return [_text(tag, field, maximum=50) for tag in value]
def _text(value: Any, field: str, *, maximum: int) -> str:
if not isinstance(value, str):
raise StarterRecipeCatalogError(f"{field} must be text")
text = " ".join(value.split())
if not text or len(text) > maximum or _is_placeholder(text):
raise StarterRecipeCatalogError(f"{field} is missing, too long, or placeholder text")
return text
def _positive_int(value: Any, field: str, maximum: int) -> int:
return _bounded_int(value, field, 1, maximum)
def _bounded_int(value: Any, field: str, minimum: int, maximum: int) -> int:
if isinstance(value, bool) or not isinstance(value, int) or not minimum <= value <= maximum:
raise StarterRecipeCatalogError(f"{field} must be an integer between {minimum} and {maximum}")
return value
def _positive_number(value: Any, field: str, maximum: int) -> int | float:
if isinstance(value, bool) or not isinstance(value, (int, float)):
raise StarterRecipeCatalogError(f"{field} must be a number")
if not math.isfinite(value) or not 0 < value <= maximum:
raise StarterRecipeCatalogError(f"{field} must be greater than zero and at most {maximum}")
return value
def _require_exact_keys(
value: Mapping[str, Any], expected: set[str] | frozenset[str], field: str
) -> None:
if set(value) != set(expected):
raise StarterRecipeCatalogError(f"{field} has unexpected or missing fields")
def _canonical_name(value: str) -> str:
return " ".join(value.split()).casefold()
def _is_placeholder(value: str) -> bool:
normalized = re.sub(r"[^a-z0-9]+", " ", value.casefold()).strip()
return normalized in _PLACEHOLDER_TEXT or any(
phrase in normalized for phrase in _PLACEHOLDER_PHRASES
)
def _resolve_recipe_ingredients(
db: Session, recipes: list[Recipe]
) -> dict[str, Ingredient]:
ingredient_ids: set[UUID] = set()
for recipe in recipes:
if not isinstance(recipe.ingredients, list):
continue
for item in recipe.ingredients:
if not isinstance(item, Mapping):
continue
try:
ingredient_ids.add(UUID(str(item.get("ingredient_id"))))
except (TypeError, ValueError, AttributeError):
continue
if not ingredient_ids:
return {}
return {
str(ingredient.id): ingredient
for ingredient in db.execute(
select(Ingredient).where(Ingredient.id.in_(ingredient_ids))
).scalars()
}
def _assert_bundled_recipe(
recipe: Recipe,
expected: Mapping[str, Any],
resolved_ingredients: Mapping[str, Ingredient],
) -> None:
_assert_bundled_recipe_ownership(recipe)
if not _bundled_recipe_content_matches(recipe, expected, resolved_ingredients):
raise StarterRecipeCatalogError("Existing bundled recipe content has drifted")
def _assert_bundled_recipe_ownership(recipe: Recipe) -> None:
if (
recipe.family_profile_id is not None
or not recipe.is_manually_added
or recipe.external_source is not None
or recipe.external_id is not None
or recipe.source_url is not None
or recipe.scraped_at is not None
or recipe.image_url is not None
or recipe.image_source is not None
or recipe.servings_scaled is not None
or recipe.side_dishes != []
or recipe.discovery_reason != STARTER_DISCOVERY_REASON
or recipe.catalog_status != RecipeCatalogStatus.ACTIVE
or recipe.catalog_findings != []
or recipe.catalog_reviewed_at is not None
):
raise StarterRecipeCatalogError(
"Existing bundled recipe does not match the required catalog ownership"
)
def _bundled_recipe_content_matches(
recipe: Recipe,
expected: Mapping[str, Any],
resolved_ingredients: Mapping[str, Ingredient],
) -> bool:
if any(getattr(recipe, field) != expected[field] for field in _CATALOG_SCALAR_FIELDS):
return False
if list(recipe.instructions or []) != expected["instructions"]:
return False
if not isinstance(recipe.ingredients, list) or len(recipe.ingredients) != len(
expected["ingredients"]
):
return False
for actual, expected_ingredient in zip(recipe.ingredients, expected["ingredients"]):
if not isinstance(actual, Mapping) or set(actual) != {
"ingredient_id",
"qty",
"unit",
"notes",
}:
return False
resolved = resolved_ingredients.get(str(actual["ingredient_id"]))
quantity = actual["qty"]
if (
resolved is None
or resolved.name_lower != _canonical_name(expected_ingredient["name"])
or isinstance(quantity, bool)
or not isinstance(quantity, (int, float))
or not math.isfinite(quantity)
or quantity != expected_ingredient["quantity"]
or actual["unit"] != expected_ingredient["unit"]
or actual["notes"] is not None
):
return False
return True
def _synchronize_bundled_recipe(
recipe: Recipe,
expected: Mapping[str, Any],
canonical_ingredients: Mapping[str, Ingredient],
) -> None:
for field in _CATALOG_SCALAR_FIELDS:
value = expected[field]
setattr(recipe, field, list(value) if field.endswith("_tags") else value)
recipe.ingredients = _catalog_recipe_ingredients(expected, canonical_ingredients)
recipe.instructions = list(expected["instructions"])
def _catalog_recipe_ingredients(
recipe: Mapping[str, Any], canonical_ingredients: Mapping[str, Ingredient]
) -> list[dict[str, Any]]:
return [
{
"ingredient_id": str(
canonical_ingredients[_canonical_name(item["name"])].id
),
"qty": item["quantity"],
"unit": item["unit"],
"notes": None,
}
for item in recipe["ingredients"]
]
def _reject_json_constant(value: str) -> None:
raise ValueError(f"Unsupported JSON numeric constant: {value}")
+973
View File
@@ -0,0 +1,973 @@
"""Bounded synchronous acquisition and staging for Wikibooks cookbook pages."""
from __future__ import annotations
import copy
import hashlib
import json
import math
import re
import time
from datetime import datetime, timezone
from email.utils import parsedate_to_datetime
from typing import Any, Callable, Mapping
from urllib.parse import quote
import requests
from bs4 import BeautifulSoup, Tag
from sqlalchemy import select
from sqlalchemy.dialects.postgresql import insert as pg_insert
from sqlalchemy.orm import Session
from app.models import RecipeImportStaging, RecipeImportStatus
WIKIBOOKS_API_URL = "https://en.wikibooks.org/w/api.php"
WIKIBOOKS_SOURCE = "wikibooks"
CC_BY_SA_4_LICENSE_NAME = "CC BY-SA 4.0"
CC_BY_SA_4_LICENSE_URL = "https://creativecommons.org/licenses/by-sa/4.0/"
MAX_DISCOVERY_LIMIT = 100
MAX_DISCOVERY_REQUESTS = 10
MAX_TITLE_LENGTH = 200
MAX_PAYLOAD_BYTES = 1_000_000
MAX_HTTP_ATTEMPTS = 3
MAX_POSTGRES_INTEGER = 2_147_483_647
RETRYABLE_HTTP_STATUSES = frozenset({429, 500, 502, 503, 504})
RETRYABLE_MEDIAWIKI_ERRORS = frozenset({"maxlag", "ratelimited"})
_UNICODE_FRACTIONS = {
"¼": 0.25,
"½": 0.5,
"¾": 0.75,
"⅓": 1 / 3,
"⅔": 2 / 3,
"⅛": 0.125,
"⅜": 0.375,
"⅝": 0.625,
"⅞": 0.875,
}
_UNICODE_FRACTION_CHARS = "".join(_UNICODE_FRACTIONS)
_UNIT_PATTERN = re.compile(
r"^(?P<unit>(?:cups?|tablespoons?|tbsp\.?|teaspoons?|tsp\.?|ounces?|oz\.?|"
r"pounds?|lbs?\.?|grams?|g|kilograms?|kgs?|kg|millilit(?:er|re)s?|ml|"
r"lit(?:er|re)s?|l|cloves?|cans?|slices?|packages?|packs?|pinches?))\b\.?",
re.IGNORECASE,
)
_ALTERNATE_METRIC_QUANTITY = re.compile(
r"\s*\(\s*(?:\d+(?:\.\d+)?(?:\s*/\s*\d+)?|[" + _UNICODE_FRACTION_CHARS + r"]|"
r"\d+[" + _UNICODE_FRACTION_CHARS + r"])\s*(?:ml|millilit(?:er|re)s?|l|lit(?:er|re)s?|"
r"g|grams?|kg|kilograms?|oz|ounces?|lb|lbs?|pounds?)\b[^)]*\)",
re.IGNORECASE,
)
_TITLE_GENERIC_WORDS = frozenset(
{
"a",
"an",
"and",
"american",
"baked",
"baking",
"bread",
"boiled",
"braised",
"classic",
"cookbook",
"cooked",
"cookie",
"dessert",
"dish",
"dressing",
"for",
"fried",
"glaze",
"glazed",
"grilled",
"in",
"meal",
"minute",
"of",
"on",
"or",
"out",
"pan",
"pie",
"recipe",
"roasted",
"salad",
"sauce",
"seared",
"steamed",
"stroganoff",
"style",
"the",
"with",
}
)
_TITLE_VARIANT_TOKENS = frozenset({"I", "II", "III", "IV", "V", "VI", "VII", "VIII", "IX", "X"})
_PLACEHOLDER_INSTRUCTIONS = frozenset(
{
"coming soon",
"directions",
"instructions",
"method",
"n a",
"tbd",
"todo",
}
)
class WikibooksImportError(RuntimeError):
"""Raised when a Wikibooks response cannot safely be used."""
class WikibooksResponseError(WikibooksImportError):
"""Raised for malformed or unexpected API response data."""
def canonical_json_sha256(value: Mapping[str, Any]) -> str:
"""Return the stable SHA-256 checksum for a JSON-compatible mapping."""
canonical = json.dumps(value, ensure_ascii=False, separators=(",", ":"), sort_keys=True)
return hashlib.sha256(canonical.encode("utf-8")).hexdigest()
class WikibooksImportService:
"""Acquire and stage text-only recipe candidates from the fixed Wikibooks API."""
def __init__(
self,
user_agent: str,
*,
session: requests.Session | Any | None = None,
timeout: tuple[float, float] = (3.05, 20.0),
min_request_interval: float = 1.0,
max_payload_bytes: int = MAX_PAYLOAD_BYTES,
max_attempts: int = MAX_HTTP_ATTEMPTS,
backoff_seconds: float = 1.0,
max_retry_delay: float = 60.0,
sleep: Callable[[float], None] = time.sleep,
monotonic: Callable[[], float] = time.monotonic,
) -> None:
cleaned_user_agent = user_agent.strip() if isinstance(user_agent, str) else ""
if len(cleaned_user_agent) < 12 or not re.search(r"[A-Za-z]", cleaned_user_agent):
raise ValueError("user_agent must be a meaningful caller-supplied identifier")
if isinstance(max_payload_bytes, bool) or not isinstance(max_payload_bytes, int) or max_payload_bytes <= 0:
raise ValueError("max_payload_bytes must be a positive integer")
if isinstance(max_attempts, bool) or not isinstance(max_attempts, int) or max_attempts < 1:
raise ValueError("max_attempts must be a positive integer")
self._user_agent = cleaned_user_agent
self._session = session or requests.Session()
self._timeout = _timeout_pair(timeout)
self._min_request_interval = _finite_nonnegative(
min_request_interval, "min_request_interval"
)
self._max_payload_bytes = max_payload_bytes
self._max_attempts = max_attempts
self._backoff_seconds = _finite_positive(backoff_seconds, "backoff_seconds")
self._max_retry_delay = _finite_positive(max_retry_delay, "max_retry_delay")
self._sleep = sleep
self._monotonic = monotonic
self._last_request_at: float | None = None
def discover_recipe_titles(self, limit: int = MAX_DISCOVERY_LIMIT) -> list[str]:
"""Return bounded Cookbook page titles from ``Category:Recipes``."""
if isinstance(limit, bool) or not isinstance(limit, int) or limit < 1:
raise ValueError("limit must be a positive integer")
return self._discover_category_recipe_titles(
requested=min(limit, MAX_DISCOVERY_LIMIT),
max_requests=MAX_DISCOVERY_REQUESTS,
)
def _discover_category_recipe_titles(
self, *, requested: int, max_requests: int
) -> list[str]:
titles: list[str] = []
seen_titles: set[str] = set()
continuation: str | None = None
for _ in range(max_requests):
params: dict[str, str | int] = {
"action": "query",
"list": "categorymembers",
"cmtitle": "Category:Recipes",
"cmtype": "page",
"cmlimit": min(requested - len(titles), MAX_DISCOVERY_LIMIT),
"format": "json",
"formatversion": 2,
"maxlag": 5,
}
if continuation:
params["cmcontinue"] = continuation
payload = self._request(params)
query = _mapping(payload.get("query"), "query")
members = query.get("categorymembers")
if not isinstance(members, list):
raise WikibooksResponseError("query.categorymembers must be a list")
for member in members:
member_data = _mapping(member, "query.categorymembers item")
title = member_data.get("title")
if not isinstance(title, str):
raise WikibooksResponseError("query.categorymembers item title must be a string")
if title.startswith("Cookbook:") and title not in seen_titles:
_validate_title(title)
seen_titles.add(title)
titles.append(title)
if len(titles) == requested:
return titles
next_data = payload.get("continue")
if next_data is None:
return titles
next_mapping = _mapping(next_data, "continue")
next_continuation = next_mapping.get("cmcontinue")
if not isinstance(next_continuation, str) or not next_continuation:
raise WikibooksResponseError("continue.cmcontinue must be a non-empty string")
if next_continuation == continuation:
raise WikibooksResponseError("repeated category continuation token")
continuation = next_continuation
return titles
def search_recipe_titles(
self, queries: list[str], limit: int = MAX_DISCOVERY_LIMIT
) -> list[str]:
"""Return bounded Cookbook titles relevant to feedback search queries."""
if isinstance(limit, bool) or not isinstance(limit, int) or limit < 1:
raise ValueError("limit must be a positive integer")
requested = min(limit, MAX_DISCOVERY_LIMIT)
cleaned_queries: list[str] = []
for query in queries:
cleaned = " ".join(query.split()) if isinstance(query, str) else ""
if cleaned and cleaned not in cleaned_queries:
cleaned_queries.append(cleaned[:MAX_TITLE_LENGTH])
if len(cleaned_queries) == MAX_DISCOVERY_REQUESTS:
break
titles: list[str] = []
seen_titles: set[str] = set()
for query in cleaned_queries:
payload = self._request(
{
"action": "query",
"list": "search",
"srsearch": query,
"srnamespace": 102,
"srlimit": requested - len(titles),
"format": "json",
"formatversion": 2,
"maxlag": 5,
}
)
search = _mapping(payload.get("query"), "query").get("search")
if not isinstance(search, list):
raise WikibooksResponseError("query.search must be a list")
for result in search:
title = _mapping(result, "query.search item").get("title")
if not isinstance(title, str):
raise WikibooksResponseError(
"query.search item title must be a string"
)
if title.startswith("Cookbook:") and title not in seen_titles:
_validate_title(title)
seen_titles.add(title)
titles.append(title)
if len(titles) == requested:
return titles
return titles
def fetch_page(self, title: str) -> dict[str, Any]:
"""Fetch one validated ``Cookbook:`` parse object without augmentation."""
_validate_title(title)
payload = self._request(
{
"action": "parse",
"page": title,
"redirects": 1,
"prop": "text|wikitext|sections|revid|displaytitle",
"format": "json",
"formatversion": 2,
"maxlag": 5,
}
)
parse_payload = _mapping(payload.get("parse"), "parse")
_validate_parse_payload(parse_payload)
return parse_payload
def normalize_page(self, parse_payload: Mapping[str, Any]) -> dict[str, Any]:
"""Convert a validated rendered cookbook page into text-only recipe data."""
_validate_parse_payload(parse_payload)
title = str(parse_payload["title"])
soup = BeautifulSoup(str(parse_payload["text"]), "html.parser")
root = soup.select_one(".mw-parser-output") or soup
elements = root.find_all(["h2", "h3", "h4", "h5", "h6", "p", "ul", "ol"])
ingredients_heading = _find_heading(elements, {"ingredients"})
instruction_heading = _find_heading(
elements, {"procedure", "instructions", "method", "directions"}
)
description = _description_before_heading(elements, ingredients_heading)
infobox = _infobox_values(root)
ingredients = [
_normalize_ingredient(item)
for item in _section_list_items(ingredients_heading, {"ul", "ol"})
]
instructions = _section_list_items(instruction_heading, {"ol"})
return {
"name": title.removeprefix("Cookbook:").replace("_", " ").strip(),
"description": description,
"servings": _servings_from_infobox(infobox),
"prep_time_minutes": _time_from_infobox(infobox, ("prep", "preparation")),
"cook_time_minutes": _time_from_infobox(infobox, ("cook", "cooking")),
"baking_time_minutes": _time_from_infobox(infobox, ("bake", "baking")),
"ingredients": ingredients,
"instructions": instructions,
}
def validate_recipe(self, candidate: Mapping[str, Any]) -> dict[str, Any]:
"""Return deterministic structural and cross-field validation results."""
errors: list[dict[str, Any]] = []
warnings: list[dict[str, Any]] = []
servings = candidate.get("servings")
if (
isinstance(servings, bool)
or not isinstance(servings, int)
or servings <= 0
):
errors.append(_issue("invalid_servings", "servings", "Servings must be a positive integer."))
elif servings > MAX_POSTGRES_INTEGER:
errors.append(
_issue(
"servings_out_of_range",
"servings",
"Servings exceed the supported database range.",
)
)
for field in ("prep_time_minutes", "cook_time_minutes", "baking_time_minutes"):
duration = candidate.get(field)
if isinstance(duration, int) and not isinstance(duration, bool) and duration > MAX_POSTGRES_INTEGER:
errors.append(
_issue(
"time_out_of_range",
field,
"Duration exceeds the supported database range.",
)
)
combined_cook_time = (candidate.get("cook_time_minutes") or 0) + (
candidate.get("baking_time_minutes") or 0
)
if (
isinstance(combined_cook_time, int)
and not isinstance(combined_cook_time, bool)
and combined_cook_time > MAX_POSTGRES_INTEGER
):
errors.append(
_issue(
"combined_cook_time_out_of_range",
"times",
"Combined cook and baking time exceed the supported database range.",
)
)
ingredients = candidate.get("ingredients")
if not isinstance(ingredients, list) or len(ingredients) < 2:
errors.append(
_issue("too_few_ingredients", "ingredients", "At least two ingredients are required.")
)
ingredient_items: list[Any] = []
else:
ingredient_items = ingredients
instructions = candidate.get("instructions")
if not isinstance(instructions, list) or len(instructions) < 2:
errors.append(
_issue("too_few_instructions", "instructions", "At least two instructions are required.")
)
instruction_items: list[Any] = []
else:
instruction_items = instructions
for index, ingredient in enumerate(ingredient_items):
quantity = ingredient.get("quantity") if isinstance(ingredient, Mapping) else None
if (
isinstance(quantity, bool)
or not isinstance(quantity, (int, float))
or not math.isfinite(quantity)
or quantity <= 0
):
errors.append(
_issue(
"invalid_ingredient_quantity",
f"ingredients[{index}].quantity",
"Ingredient quantity must be positive.",
)
)
for index, instruction in enumerate(instruction_items):
if not isinstance(instruction, str) or _is_placeholder_instruction(instruction):
errors.append(
_issue(
"placeholder_instruction",
f"instructions[{index}]",
"Instruction is missing or a placeholder.",
)
)
title_name = candidate.get("name")
if isinstance(title_name, str) and ingredient_items:
missing_tokens = _missing_title_ingredient_tokens(title_name, ingredient_items)
if missing_tokens:
errors.append(
_issue(
"title_ingredient_mismatch",
"name",
"Distinctive title food tokens are absent from ingredients.",
missing_tokens=missing_tokens,
)
)
if not isinstance(candidate.get("description"), str) or not candidate["description"].strip():
warnings.append(_issue("missing_description", "description", "Intro description is unavailable."))
if not any(
isinstance(candidate.get(field), int) and candidate[field] > 0
for field in ("prep_time_minutes", "cook_time_minutes", "baking_time_minutes")
):
warnings.append(_issue("missing_time", "times", "No preparation, cook, or baking time is available."))
return {"valid": not errors, "errors": errors, "warnings": warnings}
def stage_page(self, db: Session, title: str) -> RecipeImportStaging:
"""Fetch, normalize, validate, and flush one candidate without committing."""
return self.stage_parse_payload(db, self.fetch_page(title))
def stage_discovered_pages(
self,
db: Session,
limit: int = 5,
queries: list[str] | None = None,
) -> list[RecipeImportStaging]:
"""Discover and stage a bounded relevant or category batch without committing."""
titles = (
self.search_recipe_titles(queries, limit)
if queries
else self.discover_recipe_titles(limit)
)
return [self.stage_page(db, title) for title in titles]
def stage_parse_payload(
self, db: Session, parse_payload: Mapping[str, Any]
) -> RecipeImportStaging:
"""Stage a validated parse object, reusing an existing page revision."""
snapshot = copy.deepcopy(dict(parse_payload))
_validate_parse_payload(snapshot)
external_id = str(snapshot["pageid"])
source_revision = str(snapshot["revid"])
normalized = self.normalize_page(snapshot)
validation_report = self.validate_recipe(normalized)
status = (
RecipeImportStatus.PENDING
if validation_report["valid"]
else RecipeImportStatus.QUARANTINED
)
title = str(snapshot["title"])
values = {
"source": WIKIBOOKS_SOURCE,
"external_id": external_id,
"source_url": _page_url(title),
"source_revision": source_revision,
"license_name": CC_BY_SA_4_LICENSE_NAME,
"license_url": CC_BY_SA_4_LICENSE_URL,
"attribution": f"Wikibooks contributors; revision history: {_history_url(title)}",
"raw_payload": snapshot,
"payload_sha256": canonical_json_sha256(snapshot),
"normalized_recipe": normalized,
"validation_report": validation_report,
"status": status,
"rejection_reason": (
None
if validation_report["valid"]
else "; ".join(error["code"] for error in validation_report["errors"])
),
}
identity = {
"source": WIKIBOOKS_SOURCE,
"external_id": external_id,
"source_revision": source_revision,
}
with db.no_autoflush:
db.execute(
pg_insert(RecipeImportStaging.__table__)
.values(**values)
.on_conflict_do_nothing(
constraint="uq_recipe_import_staging_source_revision"
)
.returning(RecipeImportStaging.id)
)
return db.execute(
select(RecipeImportStaging).filter_by(**identity)
).scalar_one()
def revalidate_staged_imports(
self, db: Session, *, limit: int = 100
) -> dict[str, int]:
"""Recompute unreviewed Wikibooks candidates without committing."""
if isinstance(limit, bool) or not isinstance(limit, int) or not 1 <= limit <= 1000:
raise ValueError("limit must be between 1 and 1000")
candidates = (
db.execute(
select(RecipeImportStaging)
.where(
RecipeImportStaging.source == WIKIBOOKS_SOURCE,
RecipeImportStaging.status.in_(
[RecipeImportStatus.PENDING, RecipeImportStatus.QUARANTINED]
),
RecipeImportStaging.reviewed_at.is_(None),
RecipeImportStaging.reviewed_by.is_(None),
)
.order_by(RecipeImportStaging.acquired_at, RecipeImportStaging.id)
.limit(limit)
.with_for_update(skip_locked=True)
)
.scalars()
.all()
)
counts = {"processed": len(candidates), "pending": 0, "quarantined": 0}
for staged in candidates:
snapshot = copy.deepcopy(staged.raw_payload)
if canonical_json_sha256(snapshot) != staged.payload_sha256:
staged.validation_report = {
"valid": False,
"errors": [
_issue(
"payload_checksum_mismatch",
"raw_payload",
"Raw payload checksum does not match immutable provenance.",
)
],
"warnings": [],
}
staged.status = RecipeImportStatus.QUARANTINED
staged.rejection_reason = "payload_checksum_mismatch"
counts[RecipeImportStatus.QUARANTINED.value] += 1
continue
_validate_parse_payload(snapshot)
normalized = self.normalize_page(snapshot)
validation_report = self.validate_recipe(normalized)
status = (
RecipeImportStatus.PENDING
if validation_report["valid"]
else RecipeImportStatus.QUARANTINED
)
staged.normalized_recipe = normalized
staged.validation_report = validation_report
staged.status = status
staged.rejection_reason = (
None
if validation_report["valid"]
else "; ".join(error["code"] for error in validation_report["errors"])
)
counts[status.value] += 1
db.flush()
return counts
def _request(self, params: Mapping[str, str | int]) -> dict[str, Any]:
for attempt in range(self._max_attempts):
response: Any | None = None
retry_error: WikibooksImportError | None = None
retry_delay: float | None = None
try:
self._wait_for_request_slot()
response = self._session.get(
WIKIBOOKS_API_URL,
params=dict(params),
headers={"Accept": "application/json", "User-Agent": self._user_agent},
timeout=self._timeout,
stream=True,
allow_redirects=False,
)
status_code = getattr(response, "status_code", None)
if isinstance(status_code, bool) or not isinstance(status_code, int):
raise WikibooksResponseError("HTTP response status code is invalid")
if status_code in RETRYABLE_HTTP_STATUSES:
retry_error = WikibooksImportError(f"Wikibooks returned retryable HTTP {status_code}")
retry_delay = _retry_after_seconds(getattr(response, "headers", {}))
elif 300 <= status_code < 400:
raise WikibooksImportError(f"Wikibooks redirect HTTP {status_code} rejected")
elif status_code < 200 or status_code >= 300:
raise WikibooksImportError(f"Wikibooks returned HTTP {status_code}")
else:
payload = self._read_json_response(response)
api_error = payload.get("error")
if api_error is None:
return payload
error_data = _mapping(api_error, "error")
error_code = error_data.get("code")
if not isinstance(error_code, str) or not error_code:
raise WikibooksResponseError("error.code must be a non-empty string")
if error_code.lower() in RETRYABLE_MEDIAWIKI_ERRORS:
retry_error = WikibooksImportError(
f"Wikibooks returned retryable MediaWiki error {error_code}"
)
else:
raise WikibooksImportError(
f"Wikibooks returned MediaWiki error {error_code}"
)
except requests.RequestException as exc:
retry_error = WikibooksImportError(f"Wikibooks request failed: {exc}")
finally:
if response is not None:
response.close()
if retry_error is None:
raise AssertionError("request did not return or produce an error")
if attempt == self._max_attempts - 1:
raise retry_error
delay = retry_delay if retry_delay is not None else self._backoff_seconds * (2**attempt)
self._sleep(min(delay, self._max_retry_delay))
raise AssertionError("unreachable")
def _wait_for_request_slot(self) -> None:
if self._last_request_at is not None:
wait_seconds = self._min_request_interval - (
self._monotonic() - self._last_request_at
)
if wait_seconds > 0:
self._sleep(wait_seconds)
self._last_request_at = self._monotonic()
def _read_json_response(self, response: Any) -> dict[str, Any]:
headers = getattr(response, "headers", {})
if not isinstance(headers, Mapping):
raise WikibooksResponseError("HTTP response headers are invalid")
content_length = headers.get("Content-Length")
if content_length is not None:
try:
declared_size = int(content_length)
except (TypeError, ValueError) as exc:
raise WikibooksResponseError("HTTP Content-Length is invalid") from exc
if declared_size < 0 or declared_size > self._max_payload_bytes:
raise WikibooksResponseError("Wikibooks payload exceeds configured size limit")
body = bytearray()
for chunk in response.iter_content(chunk_size=64 * 1024):
if not isinstance(chunk, (bytes, bytearray)):
raise WikibooksResponseError("HTTP response body chunk is invalid")
body.extend(chunk)
if len(body) > self._max_payload_bytes:
raise WikibooksResponseError("Wikibooks payload exceeds configured size limit")
try:
decoded = json.loads(bytes(body).decode("utf-8"))
except (UnicodeDecodeError, json.JSONDecodeError) as exc:
raise WikibooksResponseError("Wikibooks response is not valid UTF-8 JSON") from exc
return _mapping(decoded, "response")
def _mapping(value: Any, name: str) -> dict[str, Any]:
if not isinstance(value, dict):
raise WikibooksResponseError(f"{name} must be an object")
return value
def _validate_title(title: str) -> None:
if (
not isinstance(title, str)
or not title.startswith("Cookbook:")
or len(title) <= len("Cookbook:")
or len(title) > MAX_TITLE_LENGTH
or any(character in title for character in "\r\n\x00")
):
raise WikibooksResponseError("title must be a sane Cookbook: page title")
def _validate_parse_payload(parse_payload: Mapping[str, Any]) -> None:
if not isinstance(parse_payload, Mapping):
raise WikibooksResponseError("parse must be an object")
_positive_integer(parse_payload.get("pageid"), "parse.pageid")
_positive_integer(parse_payload.get("revid"), "parse.revid")
title = parse_payload.get("title")
_validate_title(title)
for field in ("text", "wikitext", "displaytitle"):
if not isinstance(parse_payload.get(field), str):
raise WikibooksResponseError(f"parse.{field} must be a string")
sections = parse_payload.get("sections")
if not isinstance(sections, list):
raise WikibooksResponseError("parse.sections must be a list")
for section in sections:
section_data = _mapping(section, "parse.sections item")
if not isinstance(section_data.get("line"), str):
raise WikibooksResponseError("parse.sections item line must be a string")
def _positive_integer(value: Any, name: str) -> int:
if isinstance(value, bool) or not isinstance(value, int) or value <= 0:
raise WikibooksResponseError(f"{name} must be a positive integer")
return value
def _finite_positive(value: Any, name: str) -> float:
if (
isinstance(value, bool)
or not isinstance(value, (int, float))
or not math.isfinite(value)
or value <= 0
):
raise ValueError(f"{name} must be finite and positive")
return float(value)
def _finite_nonnegative(value: Any, name: str) -> float:
if (
isinstance(value, bool)
or not isinstance(value, (int, float))
or not math.isfinite(value)
or value < 0
):
raise ValueError(f"{name} must be finite and non-negative")
return float(value)
def _timeout_pair(timeout: Any) -> tuple[float, float]:
if not isinstance(timeout, (tuple, list)) or len(timeout) != 2:
raise ValueError("timeout must be a (connect, read) pair")
return (
_finite_positive(timeout[0], "timeout connect value"),
_finite_positive(timeout[1], "timeout read value"),
)
def _retry_after_seconds(headers: Mapping[str, Any]) -> float | None:
value = headers.get("Retry-After")
if value is None:
return None
try:
seconds = float(value)
return max(0.0, seconds) if math.isfinite(seconds) else None
except (TypeError, ValueError):
try:
retry_at = parsedate_to_datetime(str(value))
except (TypeError, ValueError, IndexError):
return None
if retry_at.tzinfo is None:
retry_at = retry_at.replace(tzinfo=timezone.utc)
seconds = (retry_at - datetime.now(timezone.utc)).total_seconds()
return max(0.0, seconds) if math.isfinite(seconds) else None
def _page_url(title: str) -> str:
return "https://en.wikibooks.org/wiki/" + quote(title.replace(" ", "_"), safe=":")
def _history_url(title: str) -> str:
return "https://en.wikibooks.org/w/index.php?title=" + quote(title, safe="") + "&action=history"
def _text(tag: Tag) -> str:
return re.sub(r"\s+", " ", " ".join(tag.stripped_strings)).strip()
def _heading_text(tag: Tag) -> str:
return re.sub(r"\s*\[edit\]\s*$", "", _text(tag), flags=re.IGNORECASE).strip().lower()
def _find_heading(elements: list[Tag], names: set[str]) -> Tag | None:
for element in elements:
if element.name in {"h2", "h3", "h4", "h5", "h6"} and _heading_text(element) in names:
return element
return None
def _description_before_heading(elements: list[Tag], heading: Tag | None) -> str | None:
if heading is None:
return None
descriptions: list[str] = []
for element in elements:
if element is heading:
break
if element.name == "p" and element.find_parent(["li", "td", "th"]) is None:
text = _text(element)
if text:
descriptions.append(text)
return " ".join(descriptions) or None
def _infobox_values(root: Tag | BeautifulSoup) -> dict[str, str]:
values: dict[str, str] = {}
for table in root.find_all("table"):
classes = " ".join(table.get("class", []))
if "infobox" not in classes.lower():
continue
for row in table.find_all("tr"):
cells = row.find_all(["th", "td"], recursive=False)
if len(cells) < 2:
continue
label = _text(cells[0]).lower()
value = _text(cells[1])
if label and value and label not in values:
values[label] = value
return values
def _servings_from_infobox(values: Mapping[str, str]) -> int | None:
for label, value in values.items():
if any(term in label for term in ("servings", "serves", "yield", "makes")):
match = re.search(r"\b(\d+)\b", value)
if match:
return int(match.group(1))
return None
def _time_from_infobox(values: Mapping[str, str], labels: tuple[str, ...]) -> int | None:
for label, value in values.items():
if any(expected in label for expected in labels):
return _parse_duration(value)
return None
def _parse_duration(value: str) -> int | None:
clock = re.search(r"\b(\d{1,2}):(\d{2})\b", value)
if clock:
hours, minutes = (int(part) for part in clock.groups())
total = hours * 60 + minutes
return total if total > 0 else None
hours = re.search(r"\b(\d+)\s*(?:hours?|hrs?|hr|h)\b", value, re.IGNORECASE)
minutes = re.search(r"\b(\d+)\s*(?:minutes?|mins?|min|m)\b", value, re.IGNORECASE)
total = (int(hours.group(1)) * 60 if hours else 0) + (int(minutes.group(1)) if minutes else 0)
return total if total > 0 else None
def _section_list_items(heading: Tag | None, list_names: set[str]) -> list[str]:
if heading is None:
return []
heading_level = int(heading.name[1])
items: list[str] = []
for element in heading.find_all_next(["h2", "h3", "h4", "h5", "h6", "ul", "ol"]):
if element.name and element.name.startswith("h") and int(element.name[1]) <= heading_level:
break
if element.name not in list_names:
continue
for item in element.find_all("li", recursive=False):
text = _list_item_text(item)
if text:
items.append(text)
return items
def _list_item_text(item: Tag) -> str:
clone = copy.copy(item)
for nested_list in clone.find_all(["ul", "ol"]):
nested_list.decompose()
return _text(clone)
def _normalize_ingredient(source_text: str) -> dict[str, Any]:
quantity, quantity_text, remainder = _parse_leading_quantity(source_text)
unit: str | None = None
name = remainder if quantity_text is not None else source_text
if quantity_text is not None:
unit_match = _UNIT_PATTERN.match(name.lstrip())
if unit_match:
unit = unit_match.group("unit").lower().rstrip(".")
name = name.lstrip()[unit_match.end() :]
name = _ALTERNATE_METRIC_QUANTITY.sub("", name)
name = name.strip(" \t,;:-") or source_text
return {
"quantity": quantity,
"quantity_text": quantity_text,
"unit": unit,
"name": name,
"source_text": source_text,
}
def _parse_leading_quantity(value: str) -> tuple[float | None, str | None, str]:
text = value.strip()
if re.match(r"^\d+(?:\.\d+)?\s*(?:-|–|—)\s*\d", text):
return None, None, text
fraction_prefix = re.match(r"^\d+(?:\.\d+)?(?:\s+\d+)?\s*/\s*(\d*)", text)
if fraction_prefix and (not fraction_prefix.group(1) or int(fraction_prefix.group(1)) == 0):
return None, None, text
mixed = re.match(r"^(\d+(?:\.\d+)?)\s+(\d+\s*/\s*\d+|[" + _UNICODE_FRACTION_CHARS + r"])", text)
if mixed:
fraction = _fraction_value(mixed.group(2))
if fraction is not None:
return float(mixed.group(1)) + fraction, mixed.group(0), text[mixed.end() :]
compact = re.match(r"^(\d+)([" + _UNICODE_FRACTION_CHARS + r"])", text)
if compact:
return int(compact.group(1)) + _UNICODE_FRACTIONS[compact.group(2)], compact.group(0), text[compact.end() :]
ascii_fraction = re.match(r"^(\d+)\s*/\s*(\d+)", text)
if ascii_fraction:
denominator = int(ascii_fraction.group(2))
if denominator:
return (
int(ascii_fraction.group(1)) / denominator,
ascii_fraction.group(0),
text[ascii_fraction.end() :],
)
number = re.match(r"^\d+(?:\.\d+)?", text)
if number:
return float(number.group(0)), number.group(0), text[number.end() :]
unicode_fraction = re.match(r"^(?:[" + _UNICODE_FRACTION_CHARS + r"])", text)
if unicode_fraction:
token = unicode_fraction.group(0)
return _UNICODE_FRACTIONS[token], token, text[unicode_fraction.end() :]
return None, None, text
def _fraction_value(value: str) -> float | None:
if value in _UNICODE_FRACTIONS:
return _UNICODE_FRACTIONS[value]
numerator, denominator = (int(part.strip()) for part in value.split("/", 1))
return numerator / denominator if denominator else None
def _issue(code: str, field: str, message: str, **extra: Any) -> dict[str, Any]:
return {"code": code, "field": field, "message": message, **extra}
def _is_placeholder_instruction(instruction: str) -> bool:
normalized = re.sub(r"[^a-z0-9]+", " ", instruction.lower()).strip()
return normalized in _PLACEHOLDER_INSTRUCTIONS or bool(
re.fullmatch(r"(?:step )?\d+", normalized)
)
def _missing_title_ingredient_tokens(title: str, ingredients: list[Any]) -> list[str]:
title_tokens: list[str] = []
for raw_token in re.findall(r"[A-Za-z]+", title):
if raw_token in _TITLE_VARIANT_TOKENS:
continue
token = _singular_token(raw_token.lower())
if len(token) > 1 and token not in _TITLE_GENERIC_WORDS:
title_tokens.append(token)
ingredient_tokens: set[str] = set()
for ingredient in ingredients:
if isinstance(ingredient, Mapping) and isinstance(ingredient.get("name"), str):
ingredient_tokens.update(
_singular_token(token)
for token in re.findall(r"[a-z]+", ingredient["name"].lower())
)
return [token for token in dict.fromkeys(title_tokens) if token not in ingredient_tokens]
def _singular_token(token: str) -> str:
if token == "pies":
return "pie"
if token == "cookies":
return "cookie"
if token.endswith("ies") and len(token) > 3:
return token[:-3] + "y"
if token.endswith(("ches", "shes", "sses", "xes", "zes")) and len(token) > 3:
return token[:-2]
if token.endswith("s") and not token.endswith("ss") and len(token) > 2:
return token[:-1]
return token