admin/Meal-Planner
Public Access
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(meals): enforce three lower-calorie dinners with media
CI / backend (pytest + alembic) (push) Canceled after 0s
Details
CI / frontend (build) (push) Canceled after 0s
Details

Browse Source
This commit is contained in:
admin
2026-09-01 10:11:36 -07:00
parent 7838c49721
commit 2695ba0232
85 changed files with 8103 additions and 795 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/alembic/versions/0017_recipe_import_staging.py
+120
View File
@@ -0,0 +1,120 @@
"""Add global recipe import staging and provenance storage.
Revision ID: 0017
Revises: 0016
Create Date: 2026-08-14
"""
from typing import Sequence, Union
from alembic import op
import sqlalchemy as sa
from sqlalchemy.dialects import postgresql
# revision identifiers, used by Alembic.
revision: str = "0017"
down_revision: Union[str, None] = "0016"
branch_labels: Union[str, Sequence[str], None] = None
depends_on: Union[str, Sequence[str], None] = None
recipe_import_status_enum = postgresql.ENUM(
"pending",
"quarantined",
"approved",
"rejected",
name="recipe_import_status_enum",
create_type=False,
)
def upgrade() -> None:
recipe_import_status_enum.create(op.get_bind(), checkfirst=True)
op.create_table(
"recipe_import_staging",
sa.Column("id", postgresql.UUID(as_uuid=True), nullable=False),
sa.Column("source", sa.String(length=100), nullable=False),
sa.Column("external_id", sa.String(length=255), nullable=False),
sa.Column("source_url", sa.Text(), nullable=False),
sa.Column("source_revision", sa.String(length=255), nullable=False),
sa.Column("license_name", sa.String(length=255), nullable=False),
sa.Column("license_url", sa.Text(), nullable=False),
sa.Column("attribution", sa.Text(), nullable=False),
sa.Column("raw_payload", postgresql.JSONB(), nullable=False),
sa.Column("payload_sha256", sa.String(length=64), nullable=False),
sa.Column("normalized_recipe", postgresql.JSONB(), nullable=False),
sa.Column("validation_report", postgresql.JSONB(), nullable=False),
sa.Column(
"status",
recipe_import_status_enum,
nullable=False,
server_default="pending",
),
sa.Column("rejection_reason", sa.Text(), nullable=True),
sa.Column("recipe_id", postgresql.UUID(as_uuid=True), nullable=True),
sa.Column(
"acquired_at",
sa.DateTime(timezone=True),
nullable=False,
server_default=sa.text("now()"),
),
sa.Column("reviewed_at", sa.DateTime(timezone=True), nullable=True),
sa.Column("reviewed_by", sa.String(length=300), nullable=True),
sa.CheckConstraint(
"payload_sha256 ~ '^[0-9a-f]{64}$'",
name="ck_recipe_import_staging_payload_sha256",
),
sa.CheckConstraint(
"(reviewed_at IS NULL) = (reviewed_by IS NULL)",
name="ck_recipe_import_staging_review_metadata",
),
sa.ForeignKeyConstraint(["recipe_id"], ["recipe.id"], ondelete="SET NULL"),
sa.PrimaryKeyConstraint("id"),
sa.UniqueConstraint(
"source",
"external_id",
"source_revision",
name="uq_recipe_import_staging_source_revision",
),
)
op.create_index(
"ix_recipe_import_staging_status_acquired_at",
"recipe_import_staging",
["status", "acquired_at"],
unique=False,
)
op.execute(
"""
CREATE FUNCTION prevent_recipe_import_staging_raw_payload_update()
RETURNS trigger AS $$
BEGIN
IF NEW.raw_payload IS DISTINCT FROM OLD.raw_payload THEN
RAISE EXCEPTION 'recipe_import_staging.raw_payload is immutable';
END IF;
RETURN NEW;
END;
$$ LANGUAGE plpgsql
"""
)
op.execute(
"""
CREATE TRIGGER trg_recipe_import_staging_raw_payload_immutable
BEFORE UPDATE OF raw_payload ON recipe_import_staging
FOR EACH ROW
EXECUTE FUNCTION prevent_recipe_import_staging_raw_payload_update()
"""
)
def downgrade() -> None:
op.execute(
"DROP TRIGGER trg_recipe_import_staging_raw_payload_immutable "
"ON recipe_import_staging"
)
op.execute("DROP FUNCTION prevent_recipe_import_staging_raw_payload_update()")
op.drop_index(
"ix_recipe_import_staging_status_acquired_at",
table_name="recipe_import_staging",
)
op.drop_table("recipe_import_staging")
recipe_import_status_enum.drop(op.get_bind(), checkfirst=True)
backend/alembic/versions/0018_recipe_catalog_quarantine.py
+65
View File
@@ -0,0 +1,65 @@
"""Add recipe catalog quarantine state.
Revision ID: 0018
Revises: 0017
Create Date: 2026-08-14
"""
from typing import Sequence, Union
from alembic import op
import sqlalchemy as sa
from sqlalchemy.dialects import postgresql
revision: str = "0018"
down_revision: Union[str, None] = "0017"
branch_labels: Union[str, Sequence[str], None] = None
depends_on: Union[str, Sequence[str], None] = None
recipe_catalog_status_enum = postgresql.ENUM(
"active",
"quarantined",
name="recipe_catalog_status_enum",
create_type=False,
)
def upgrade() -> None:
recipe_catalog_status_enum.create(op.get_bind(), checkfirst=True)
op.add_column(
"recipe",
sa.Column(
"catalog_status",
recipe_catalog_status_enum,
nullable=False,
server_default="active",
),
)
op.add_column(
"recipe",
sa.Column(
"catalog_findings",
postgresql.JSONB(),
nullable=False,
server_default=sa.text("'[]'::jsonb"),
),
)
op.add_column(
"recipe",
sa.Column("catalog_reviewed_at", sa.DateTime(timezone=True), nullable=True),
)
op.create_index(
"ix_recipe_catalog_status",
"recipe",
["catalog_status"],
unique=False,
)
def downgrade() -> None:
op.drop_index("ix_recipe_catalog_status", table_name="recipe")
op.drop_column("recipe", "catalog_reviewed_at")
op.drop_column("recipe", "catalog_findings")
op.drop_column("recipe", "catalog_status")
recipe_catalog_status_enum.drop(op.get_bind(), checkfirst=True)
backend/app/api/admin.py
+140 -23
View File
@@ -1,12 +1,32 @@
from fastapi import APIRouter, BackgroundTasks, Depends, HTTPException
from fastapi import APIRouter, BackgroundTasks, Depends, HTTPException, Query, Response, status
from sqlalchemy.exc import IntegrityError
from sqlalchemy.orm import Session
from app.database import get_db
from app.models import ScrapeLog, EmailLog, MealPlan, Recipe
from app.models import (
FamilyProfile,
ScrapeLog,
EmailLog,
MealPlan,
Recipe,
RecipeImportStaging,
RecipeImportStatus,
)
from app.schemas.recipe_import import (
RecipeImportDetail,
RecipeImportListResponse,
RecipeImportRejectRequest,
)
from app.security import require_admin
from app.services.scraper_service import ScraperService, enqueue_scrape
from app.services.feedback_analyzer import FeedbackAnalyzer
from app.services.recipe_discovery import RecipeDiscoveryService
from app.services.recipe_ingestion import RecipeIngestionService
from app.services.recipe_import_review import (
RecipeImportConflictError,
RecipeImportNotFoundError,
approve_recipe_import,
reject_recipe_import,
)
from app.services.wikibooks_import import WikibooksImportError, WikibooksImportService
from app.config import settings
from typing import List, Optional
from datetime import datetime, timedelta
from uuid import UUID
@@ -17,6 +37,99 @@ logger = logging.getLogger(__name__)
router = APIRouter(dependencies=[Depends(require_admin)])
def _import_list_item(row: RecipeImportStaging) -> dict:
normalized = row.normalized_recipe if isinstance(row.normalized_recipe, dict) else {}
return {
"id": row.id,
"source": row.source,
"external_id": row.external_id,
"source_url": row.source_url,
"source_revision": row.source_revision,
"name": normalized.get("name"),
"status": row.status.value,
"acquired_at": row.acquired_at,
"reviewed_at": row.reviewed_at,
"reviewed_by": row.reviewed_by,
"recipe_id": row.recipe_id,
}
@router.get("/recipe-imports", response_model=RecipeImportListResponse)
def list_recipe_imports(
import_status: RecipeImportStatus | None = Query(default=None, alias="status"),
source: str | None = Query(default=None, min_length=1, max_length=100),
limit: int = Query(default=50, ge=1, le=100),
offset: int = Query(default=0, ge=0),
db: Session = Depends(get_db),
):
query = db.query(RecipeImportStaging)
if import_status is not None:
query = query.filter(RecipeImportStaging.status == import_status)
if source is not None:
query = query.filter(RecipeImportStaging.source == source)
total = query.count()
rows = (
query.order_by(RecipeImportStaging.acquired_at.desc(), RecipeImportStaging.id)
.offset(offset)
.limit(limit)
.all()
)
return {
"items": [_import_list_item(row) for row in rows],
"total": total,
"limit": limit,
"offset": offset,
"has_next": offset + len(rows) < total,
}
@router.get("/recipe-imports/{import_id}", response_model=RecipeImportDetail)
def get_recipe_import(import_id: UUID, db: Session = Depends(get_db)):
staged = db.get(RecipeImportStaging, import_id)
if staged is None:
raise HTTPException(status_code=404, detail="Recipe import not found")
return staged
@router.post("/recipe-imports/{import_id}/approve", response_model=RecipeImportDetail)
def approve_staged_recipe(
import_id: UUID,
response: Response,
db: Session = Depends(get_db),
reviewer: str = Depends(require_admin),
):
try:
staged, created = approve_recipe_import(db, import_id, reviewer)
except RecipeImportNotFoundError as exc:
db.rollback()
raise HTTPException(status_code=404, detail=str(exc)) from exc
except RecipeImportConflictError as exc:
db.rollback()
raise HTTPException(status_code=409, detail=str(exc)) from exc
except IntegrityError as exc:
db.rollback()
raise HTTPException(status_code=409, detail="Recipe publication conflicted") from exc
response.status_code = status.HTTP_201_CREATED if created else status.HTTP_200_OK
return staged
@router.post("/recipe-imports/{import_id}/reject", response_model=RecipeImportDetail)
def reject_staged_recipe(
import_id: UUID,
payload: RecipeImportRejectRequest,
db: Session = Depends(get_db),
reviewer: str = Depends(require_admin),
):
try:
return reject_recipe_import(db, import_id, payload.reason, reviewer)
except RecipeImportNotFoundError as exc:
db.rollback()
raise HTTPException(status_code=404, detail=str(exc)) from exc
except RecipeImportConflictError as exc:
db.rollback()
raise HTTPException(status_code=409, detail=str(exc)) from exc
@router.post("/scrape", status_code=202)
def trigger_scrape(
background_tasks: BackgroundTasks,
@@ -214,10 +327,9 @@ def trigger_images(
def trigger_discovery(
family_profile_id: UUID,
force: bool = False,
limit: int = Query(default=5, ge=1, le=25),
db: Session = Depends(get_db),
):
from uuid import UUID as UUIDType
profile = (
db.query(FamilyProfile)
.filter(FamilyProfile.id == family_profile_id)
@@ -232,25 +344,30 @@ def trigger_discovery(
if not analysis.discovery_queries and not force:
return {"status": "skipped", "reason": "No discovery queries and force=false"}
discovery = RecipeDiscoveryService()
if not analysis.discovery_queries and force:
queries = ["popular recipes"]
else:
queries = analysis.discovery_queries
candidates = discovery.discover(queries)
if not candidates:
return {"status": "no_candidates"}
try:
staged = WikibooksImportService(
settings.WIKIBOOKS_USER_AGENT
).stage_discovered_pages(
db,
limit=limit,
queries=analysis.discovery_queries or None,
)
db.commit()
except WikibooksImportError as exc:
db.rollback()
raise HTTPException(status_code=502, detail="Wikibooks staging failed") from exc
ingestion = RecipeIngestionService()
added = ingestion.ingest(
db, profile.id, candidates, analysis.to_dict()
pending = sum(row.status == RecipeImportStatus.PENDING for row in staged)
quarantined = sum(row.status == RecipeImportStatus.QUARANTINED for row in staged)
logger.info(
"Manual discovery: staged %d Wikibooks candidates for review", len(staged)
)
logger.info("Manual discovery: added %d recipes for family %s", added, profile.id)
return {
"status": "success",
"candidates_found": len(candidates),
"recipes_added": added,
"discovery_queries": queries,
"candidates_staged": len(staged),
"pending": pending,
"quarantined": quarantined,
"recipes_added": 0,
"discovery_queries": analysis.discovery_queries,
"confidence": analysis.confidence,
}
}
backend/app/api/llm_plan.py
+83 -97
View File
@@ -1,21 +1,16 @@
"""Sprint 13 — F9-lite: free-text meal-plan synthesis via Ollama Cloud.
"""Free-text preferences constrained by the canonical three-dinner planner.
This endpoint lets the user describe what they want for the week
("Italian-inspired, vegetarian", "easy weeknight dinners, no fish")
and asks the configured LLM (kimi-k2.6:cloud on ollama.com) to pick
meals from the local recipe library. The LLM's picks are inserted
into a fresh plan; the remaining slots are filled by the existing
Sprint 6+ `fillEmptySlots` partial-success pattern.
meals from the local recipe library. The canonical planner validates
and persists all picks so LLM output cannot bypass planner constraints.
Reuses:
- The LLM call pattern from `app.services.llm_matcher._ask_ollama`
(POST ${OLLAMA_BASE_URL}/chat/completions, same headers, same
max_tokens=500, temperature=0, strip <think> blocks).
- The Sprint 6+ `meals.create` + `meals.fillEmptySlots` semantics
for the plan create + fill flow.
The LLM call follows `app.services.llm_matcher._ask_ollama`; plan
selection and persistence are delegated to `generate_meal_plan`.
The endpoint never crashes on LLM failure: parse errors, timeout,
or empty picks all fall through to the `fillEmptySlots` path.
Parse errors, timeout, invalid picks, and infeasible picks fall back to
the canonical planner.
"""
from __future__ import annotations
@@ -29,13 +24,20 @@ from typing import List, Optional
import requests
from fastapi import APIRouter, Depends, HTTPException
from pydantic import BaseModel, Field
from sqlalchemy import or_
from sqlalchemy.orm import Session
from app.config import settings
from app.database import get_db
from app.models import FamilyProfile, MealPlan, MealPlanItem, MealType, Recipe
from app.models import (
FamilyProfile,
MealPlan,
MealType,
Recipe,
RecipeCatalogStatus,
)
from app.security import require_session
from app.services.meal_pairings import components_with_suggested_sides
from app.services.planner.generate import generate_meal_plan
logger = logging.getLogger(__name__)
@@ -49,7 +51,7 @@ class LLMPlanRequest(BaseModel):
class LLMPickedItem(BaseModel):
day_of_week: int
meal_type: str # "breakfast" | "lunch" | "dinner"
meal_type: str
recipe_id: str
@@ -82,7 +84,13 @@ def _serialize_library(db: Session, profile_id: uuid.UUID) -> List[dict]:
Sorted alphabetically by name for deterministic LLM input."""
rows = (
db.query(Recipe)
.filter(Recipe.family_profile_id == profile_id)
.filter(
Recipe.catalog_status == RecipeCatalogStatus.ACTIVE,
or_(
Recipe.family_profile_id == profile_id,
Recipe.family_profile_id.is_(None),
),
)
.order_by(Recipe.name.asc())
.limit(_MAX_LIBRARY_PROMPTED)
.all()
@@ -95,6 +103,7 @@ def _serialize_library(db: Session, profile_id: uuid.UUID) -> List[dict]:
"cuisine_tags": r.cuisine_tags or [],
"dietary_tags": r.dietary_tags or [],
"protein_type": r.protein_type,
"calories_per_serving": r.calories_per_serving,
"total_time_minutes": (r.prep_time_minutes or 0) + (r.cook_time_minutes or 0),
})
return out
@@ -115,7 +124,7 @@ def _ask_llm(prompt: str) -> Optional[str]:
json={
"model": settings.OLLAMA_MODEL,
"messages": [{"role": "user", "content": prompt}],
"max_tokens": 4000, # 47-recipe library: 21 picks × ~100 chars + reasoning + boilerplate ≈ 2100+ chars; 4000 gives 2x headroom
"max_tokens": 4000, # Reasoning models may consume most of the budget before returning JSON.
"temperature": 0,
},
timeout=_LLM_TIMEOUT_SECS,
@@ -171,7 +180,10 @@ def _parse_picks(raw: str) -> List[dict]:
def _validate_picks(picks: List[dict], valid_recipe_ids: set[str]) -> List[LLMPickedItem]:
"""Drop invalid entries: missing fields, out-of-range day_of_week,
unknown meal_type, unknown recipe_id."""
valid_meal_types = {m.value for m in MealType}
valid_days = {1, 3, 5}
valid_meal_types = {MealType.DINNER.value}
used_days: set[int] = set()
used_recipes: set[str] = set()
out: List[LLMPickedItem] = []
for p in picks:
try:
@@ -180,13 +192,17 @@ def _validate_picks(picks: List[dict], valid_recipe_ids: set[str]) -> List[LLMPi
rid = str(p.get("recipe_id", ""))
except (TypeError, ValueError):
continue
if day < 1 or day > 7:
if day not in valid_days or day in used_days:
continue
if meal not in valid_meal_types:
continue
if rid not in valid_recipe_ids:
if rid not in valid_recipe_ids or rid in used_recipes:
continue
out.append(LLMPickedItem(day_of_week=day, meal_type=meal, recipe_id=rid))
used_days.add(day)
used_recipes.add(rid)
if len(out) == 3:
break
return out
@@ -194,14 +210,17 @@ def _validate_picks(picks: List[dict], valid_recipe_ids: set[str]) -> List[LLMPi
def synthesize_plan(
payload: LLMPlanRequest,
db: Session = Depends(get_db),
_user: str = Depends(require_session),
session_family_id: str = Depends(require_session),
) -> LLMPlanResponse:
"""Synthesize a 7-day meal plan from a free-text prompt + the
local recipe library via the configured LLM. Falls through to
the Sprint 6+ library-fill for any slots the LLM didn't pick."""
"""Use LLM preferences only when the canonical planner accepts all three."""
_ensure_ollama_configured()
profile = db.query(FamilyProfile).first()
try:
profile_id = uuid.UUID(session_family_id)
except ValueError:
raise HTTPException(status_code=404, detail="family profile not found")
profile = db.query(FamilyProfile).filter(FamilyProfile.id == profile_id).first()
if not profile:
raise HTTPException(status_code=404, detail="family profile not found")
@@ -234,21 +253,19 @@ def synthesize_plan(
f"cuisine={','.join(r['cuisine_tags']) or 'n/a'} | "
f"diet={','.join(r['dietary_tags']) or 'n/a'} | "
f"protein={r['protein_type'] or 'n/a'} | "
f"calories={r['calories_per_serving'] or 'unknown'} | "
f"time={r['total_time_minutes']}min"
for r in library
)
prompt = (
"You are planning a 7-day meal plan (Monday through Sunday) "
"for a family. Each day has 3 meals: breakfast, lunch, dinner. "
"Pick up to 21 meals total from the recipe library below. "
"If a slot has no good match for the user's request, OMIT it "
"(do not invent a recipe). Use only recipe_ids from the list.\n\n"
"You are choosing exactly 3 dinners for Monday, Wednesday, and Friday "
"(day_of_week 1, 3, and 5). Pick one different recipe for each day. "
"Do not invent recipes and use only recipe_ids from the list.\n\n"
f"USER REQUEST: {payload.prompt}\n\n"
f"RECIPE LIBRARY ({len(library)} recipes):\n{compact}\n\n"
'RETURN FORMAT — valid JSON only, no markdown, no commentary:\n'
'[\n'
' {"day_of_week": 1, "meal_type": "breakfast", "recipe_id": "<uuid>"},\n'
' {"day_of_week": 1, "meal_type": "lunch", "recipe_id": "<uuid>"},\n'
' {"day_of_week": 1, "meal_type": "dinner", "recipe_id": "<uuid>"},\n'
' ...\n'
"]"
)
@@ -258,80 +275,49 @@ def synthesize_plan(
raw_picks = _parse_picks(raw or "")
valid_recipe_ids = {r["id"] for r in library}
picks = _validate_picks(raw_picks, valid_recipe_ids)
recipe_by_id = {
str(r.id): r
for r in db.query(Recipe).filter(Recipe.id.in_([uuid.UUID(rid) for rid in valid_recipe_ids])).all()
}
logger.info(
"LLM plan: prompt=%d chars, raw_picks=%d, valid_picks=%d",
len(payload.prompt), len(raw_picks), len(picks),
)
# 4) Create the empty plan.
plan = MealPlan(
family_profile_id=profile.id,
week_start_date=payload.week_start,
status="draft",
notes=f"LLM-synthesized from prompt: {payload.prompt[:200]}",
)
db.add(plan)
db.flush()
picked_count = 0
result = None
if len(picks) == 3:
preferred_ids = {uuid.UUID(pick.recipe_id) for pick in picks}
active_ids = {
recipe_id
for (recipe_id,) in db.query(Recipe.id).filter(
Recipe.catalog_status == RecipeCatalogStatus.ACTIVE
).all()
}
result = generate_meal_plan(
db,
family_id=profile.id,
week_start_date=payload.week_start,
exclude_recipe_ids=active_ids - preferred_ids,
)
if len(result.selected) == 3:
picked_count = 3
else:
generated_plan = db.query(MealPlan).filter(MealPlan.id == result.meal_plan_id).first()
if generated_plan is not None:
db.delete(generated_plan)
db.commit()
result = None
# 5) Insert the LLM-picked items.
for pick in picks:
db.add(MealPlanItem(
meal_plan_id=plan.id,
recipe_id=uuid.UUID(pick.recipe_id),
day_of_week=pick.day_of_week,
meal_type=MealType(pick.meal_type),
components=components_with_suggested_sides(
recipe_by_id.get(pick.recipe_id),
pick.meal_type,
),
))
db.flush()
if result is None:
result = generate_meal_plan(
db,
family_id=profile.id,
week_start_date=payload.week_start,
)
picked_count = len(picks)
picked_slots = {(p.day_of_week, p.meal_type) for p in picks}
# 6) Fill the rest from the library (Sprint 6+ pattern).
# Re-implemented inline (not via HTTP) to avoid a self-call.
requested_meal_types = ["breakfast", "lunch", "dinner"]
used_recipe_ids = {uuid.UUID(p.recipe_id) for p in picks}
filled_count = 0
failed_count = 0
for day in range(1, 8):
for mt in requested_meal_types:
if (day, mt) in picked_slots:
continue
# Pick the first library recipe not already used in the plan
# and with a matching meal_type preference (best-effort). The
# LLM call above was the smart pick; the library fill is the
# fallback for any slots the LLM didn't cover.
candidates = [r for r in library if uuid.UUID(r["id"]) not in used_recipe_ids]
if not candidates:
failed_count += 1
continue
chosen = candidates[0]
db.add(MealPlanItem(
meal_plan_id=plan.id,
recipe_id=uuid.UUID(chosen["id"]),
day_of_week=day,
meal_type=MealType(mt),
components=components_with_suggested_sides(
recipe_by_id.get(chosen["id"]),
mt,
),
))
used_recipe_ids.add(uuid.UUID(chosen["id"]))
filled_count += 1
db.commit()
db.refresh(plan)
selected_count = len(result.selected)
filled_count = selected_count - picked_count
failed_count = max(0, 3 - selected_count)
return LLMPlanResponse(
plan_id=str(plan.id),
plan_id=str(result.meal_plan_id),
picked_count=picked_count,
filled_count=filled_count,
failed_count=failed_count,
backend/app/api/meal_plans.py
+51 -1
View File
@@ -14,9 +14,10 @@ from app.schemas.meal_plan_generation import (
GenerationDebug,
GenerationItem,
GenerationResponse,
PublicGenerateRequest,
RegenerateRequest,
)
from app.security import require_admin
from app.security import require_admin, require_session
from app.services.planner.config import DEFAULT
from app.services.planner.generate import generate_meal_plan
from app.services.planner.types import GenerationResult
@@ -82,6 +83,55 @@ def generate(payload: GenerateRequest, db: Session = Depends(get_db)) -> Generat
return _to_response(payload.week_start_date, result.meal_plan_id, items, result)
@public_router.post("/generate", response_model=GenerationResponse, status_code=status.HTTP_201_CREATED)
def generate_for_family(
payload: PublicGenerateRequest,
db: Session = Depends(get_db),
session_family_id: str = Depends(require_session),
) -> GenerationResponse:
try:
family_id = UUID(session_family_id)
except ValueError:
raise HTTPException(status_code=404, detail="family_profile not found")
family = db.query(FamilyProfile).filter(FamilyProfile.id == family_id).first()
if family is None:
raise HTTPException(status_code=404, detail="family_profile not found")
existing = (
db.query(MealPlan)
.filter(
MealPlan.family_profile_id == family_id,
MealPlan.week_start_date == payload.week_start_date,
)
.first()
)
if existing is not None:
has_items = (
db.query(MealPlanItem.id)
.filter(MealPlanItem.meal_plan_id == existing.id)
.first()
is not None
)
if has_items:
raise HTTPException(status_code=409, detail="meal plan already has items")
db.delete(existing)
db.commit()
result = generate_meal_plan(
db,
family_id=family_id,
week_start_date=payload.week_start_date,
)
items = (
db.query(MealPlanItem)
.filter(MealPlanItem.meal_plan_id == result.meal_plan_id)
.order_by(MealPlanItem.day_of_week)
.all()
)
return _to_response(payload.week_start_date, result.meal_plan_id, items, result)
@admin_router.post("/regenerate", response_model=GenerationResponse, status_code=status.HTTP_201_CREATED)
def regenerate(payload: RegenerateRequest, db: Session = Depends(get_db)) -> GenerationResponse:
family = db.query(FamilyProfile).filter(FamilyProfile.id == payload.family_profile_id).first()
backend/app/api/meals.py
+73 -4
View File
@@ -9,6 +9,7 @@ from app.models import (
MealPlan, MealPlanItem, MealPlanVote, Recipe, Ingredient,
FamilyProfile, FamilyMember, ApprovalToken, NeverSuggest, NeverSuggestReason,
MealPlanStatus, MealPlanItemStatus, MealType, ApprovalTokenStatus,
RecipeCatalogStatus,
)
from app.schemas import (
MealPlanResponse, MealPlanCreate,
@@ -154,6 +155,7 @@ def get_planned_meals(
if not meal_plan:
return None
_enrich_recipe_ingredient_names(db, meal_plan)
return meal_plan
@@ -170,6 +172,23 @@ def create_meal_plan(meal_plan_data: MealPlanCreate, db: Session = Depends(get_d
if existing:
raise HTTPException(status_code=400, detail="Meal plan for this week already exists")
recipe_ids = {item.recipe_id for item in meal_plan_data.items}
if recipe_ids:
active_recipe_ids = {
recipe_id
for (recipe_id,) in db.query(Recipe.id)
.filter(
Recipe.id.in_(recipe_ids),
Recipe.catalog_status == RecipeCatalogStatus.ACTIVE,
)
.all()
}
if active_recipe_ids != recipe_ids:
raise HTTPException(
status_code=400,
detail="Meal plan contains unavailable recipes",
)
db_meal_plan = MealPlan(
family_profile_id=profile.id,
week_start_date=meal_plan_data.week_start_date,
@@ -201,6 +220,7 @@ def get_meal_plan(meal_plan_id: UUID, db: Session = Depends(get_db)):
meal_plan = db.query(MealPlan).filter(MealPlan.id == meal_plan_id).first()
if not meal_plan:
raise HTTPException(status_code=404, detail="Meal plan not found")
_enrich_recipe_ingredient_names(db, meal_plan)
return meal_plan
@@ -221,6 +241,39 @@ _DAY_NAMES = {
}
def _enrich_recipe_ingredient_names(db: Session, meal_plan: MealPlan) -> None:
"""Mutate each embedded Recipe's `ingredients` JSONB in-place to add
a resolved `name` from the canonical Ingredient table. Mirrors the
enrichment the public /api/recipes endpoints do; the meal-plan path
uses pydantic from_attributes which reads the raw JSONB and would
otherwise leave `name=None`. Idempotent: only sets name when
missing.
"""
recipes: list[Recipe] = []
for item in meal_plan.items:
if item.recipe is not None and item.recipe not in recipes:
recipes.append(item.recipe)
if not recipes:
return
all_ids: set = set()
for r in recipes:
for ref in (r.ingredients or []):
iid = ref.get("ingredient_id") if isinstance(ref, dict) else None
if iid:
all_ids.add(iid)
if not all_ids:
return
rows = db.query(Ingredient.id, Ingredient.name).filter(Ingredient.id.in_(all_ids)).all()
name_map = {str(rid): rname for rid, rname in rows}
for r in recipes:
for ref in (r.ingredients or []):
if not isinstance(ref, dict):
continue
iid = str(ref.get("ingredient_id") or "")
if iid and not ref.get("name"):
ref["name"] = name_map.get(iid)
class VoteSubmission(BaseModel):
"""Body for POST /vote/{item_id}?token=...
@@ -556,7 +609,10 @@ def swap_meal_item(item_id: UUID, new_recipe_id: UUID, db: Session = Depends(get
if not item:
raise HTTPException(status_code=404, detail="Meal plan item not found")
new_recipe = db.query(Recipe).filter(Recipe.id == new_recipe_id).first()
new_recipe = db.query(Recipe).filter(
Recipe.id == new_recipe_id,
Recipe.catalog_status == RecipeCatalogStatus.ACTIVE,
).first()
if not new_recipe:
raise HTTPException(status_code=404, detail="New recipe not found")
@@ -668,7 +724,17 @@ def generate_single_item(
raise HTTPException(status_code=400, detail="Slot already occupied")
# Get all recipes, score them, pick the best not already in this plan
all_recipes = db.query(Recipe).all()
all_recipes = db.query(Recipe).filter(
Recipe.catalog_status == RecipeCatalogStatus.ACTIVE
).all()
if not all_recipes:
raise HTTPException(
status_code=409,
detail=(
"No active recipes are available. "
"Approve a staged recipe before generating meals."
),
)
used_ids = {i.recipe_id for i in plan.items if i.recipe_id is not None}
# Simple heuristic: pick a random un-used recipe (we can improve scoring later)
@@ -728,7 +794,9 @@ def fill_empty_slots(
if not requested:
return FillEmptySlotsResult(filled=[], failed=[])
all_recipes = db.query(Recipe).all()
all_recipes = db.query(Recipe).filter(
Recipe.catalog_status == RecipeCatalogStatus.ACTIVE
).all()
if not all_recipes:
# No recipes at all — every requested slot fails.
return FillEmptySlotsResult(
@@ -877,6 +945,7 @@ def discovered_count(db: Session = Depends(get_db)):
db.query(Recipe)
.filter(
Recipe.family_profile_id == profile.id,
Recipe.catalog_status == RecipeCatalogStatus.ACTIVE,
Recipe.external_source.isnot(None),
Recipe.created_at >= since,
)
@@ -908,4 +977,4 @@ def discovery_insights(db: Session = Depends(get_db)):
"positive_signals": analysis.get("positive_signals", []),
"negative_signals": analysis.get("negative_signals", []),
"top_rated_recipes": analysis.get("top_rated_recipes", []),
}
}
backend/app/api/recipe_search.py
+15 -292
View File
@@ -1,308 +1,31 @@
"""Sprint 12 — external recipe search (Spoonacular) + import.
"""Retired direct third-party recipe search and publication endpoints."""
from fastapi import APIRouter, Depends, HTTPException, Query, status
This is a thin HTTP layer on top of the pre-existing Spoonacular
free-tier API. The service-class `RecipeDiscoveryService` in
`app.services.recipe_discovery` is the bulk-orchestrator used by
the offline FeedbackAnalyzer; we re-implement the call shape here
because the webui wants:
1. A search that returns *summary* data (no info endpoint call) so
10 results cost 1.1 points, not 11.1.
2. An import that fetches the full info for ONE recipe and writes
a local Recipe row.
Quota: Spoonacular free tier = 150 points/day. complexSearch = 1 +
0.01 per result. /information = 1 point. We gate at 140 to leave
a safety margin and return 503 once exhausted.
"""
from __future__ import annotations
import logging
import threading
from typing import List, Optional
import requests
from fastapi import APIRouter, Depends, HTTPException, Query
from sqlalchemy.orm import Session
from app.config import settings
from app.database import get_db
from app.models import FamilyProfile, Ingredient, Recipe
from app.schemas import (
RecipeImportRequest,
RecipeSearchHit,
)
from app.schemas import RecipeImportRequest
from app.security import require_session
logger = logging.getLogger(__name__)
router = APIRouter()
# Quota counter: a per-process point budget. Survives across
# requests in the same uvicorn worker. Process restart resets to 0
# (so the operator can recover by bouncing the backend). Thread-safe
# with a single lock — the counter is touched on every request and
# we don't want a race between two parallel searches.
_quota_lock = threading.Lock()
_points_used: float = 0.0
_DAILY_LIMIT: float = 45.0 # 50 free, leave 5pt safety margin (corrected from 140; Sprint 15 + Sprint 16)
_INFO_URL = "https://api.spoonacular.com/recipes/{id}/information"
_SEARCH_URL = "https://api.spoonacular.com/recipes/complexSearch"
_RETIRED_DETAIL = (
"Direct external recipe ingestion is retired; recipes must be staged and "
"approved through the admin import queue."
)
def _points_available() -> float:
with _quota_lock:
return _DAILY_LIMIT - _points_used
def _charge_points(pts: float) -> None:
global _points_used
with _quota_lock:
_points_used += pts
def _ensure_spoonacular_configured() -> str:
"""Return the API key, or 503 if unconfigured."""
key = settings.SPOONACULAR_API_KEY
if not key:
logger.error("SPOONACULAR_API_KEY not configured")
raise HTTPException(
status_code=503,
detail="SPOONACULAR_API_KEY not configured; set it in the backend env",
)
return key
def _normalize_spoonacular_summary(item: dict) -> Optional[RecipeSearchHit]:
"""Map a complexSearch result item into RecipeSearchHit."""
ext_id = str(item.get("id") or "")
title = item.get("title")
if not ext_id or not title:
return None
cuisines = [str(c).lower() for c in (item.get("cuisines") or []) if c]
diets = [str(d).lower() for d in (item.get("diets") or []) if d]
return RecipeSearchHit(
external_id=ext_id,
external_source="spoonacular",
name=title,
image_url=item.get("image"),
source_url=item.get("sourceUrl") or item.get("spoonacularSourceUrl"),
prep_time_minutes=int(item["preparationMinutes"]) if item.get("preparationMinutes") else None,
cook_time_minutes=int(item["cookingMinutes"]) if item.get("cookingMinutes") else None,
servings=int(item["servings"]) if item.get("servings") else None,
cuisine_tags=cuisines,
dietary_tags=diets,
protein_type=None, # would need to call /information or _infer_protein
)
@router.get("/search", response_model=List[RecipeSearchHit])
@router.get("/search")
def search_recipes(
q: str = Query(..., min_length=1, max_length=200),
limit: int = Query(10, ge=1, le=25),
_user: str = Depends(require_session),
) -> List[RecipeSearchHit]:
"""Search Spoonacular's complexSearch index and return a
summary list (no per-recipe /information call). The frontend
uses this for the "Search the web" panel."""
if _points_available() < 1.05: # 1 base + 0.01 * 5 average
raise HTTPException(
status_code=503,
detail="spoonacular daily quota reached; try again tomorrow",
)
api_key = _ensure_spoonacular_configured()
try:
resp = requests.get(
_SEARCH_URL,
params={
"apiKey": api_key,
"query": q,
"number": limit,
"addRecipeInformation": "true",
"fillIngredients": "true",
"instructionsRequired": "true",
},
timeout=15,
)
resp.raise_for_status()
except requests.RequestException as exc:
logger.warning("Spoonacular search failed for %r: %s", q, exc)
raise HTTPException(
status_code=502,
detail=f"spoonacular search failed: {exc}",
) from exc
data = resp.json()
results = data.get("results", [])
# 1 base + 0.01 per result
cost = 1.0 + len(results) * 0.01
_charge_points(cost)
logger.info("Spoonacular search %r → %d hits (%.2f pts, total %.1f/%.0f)",
q, len(results), cost, _points_used, _DAILY_LIMIT)
out: List[RecipeSearchHit] = []
for item in results:
hit = _normalize_spoonacular_summary(item)
if hit:
out.append(hit)
return out
):
"""Reject the legacy Spoonacular search flow without contacting a vendor."""
raise HTTPException(status_code=status.HTTP_410_GONE, detail=_RETIRED_DETAIL)
def _infer_protein_simple(title: str, ingredient_names: List[str]) -> Optional[str]:
"""Lightweight protein inference; mirrors recipe_discovery._infer_protein
but doesn't import the service (to avoid pulling in the rest of the
offline path)."""
text = (title + " " + " ".join(ingredient_names)).lower()
proteins = {
"chicken": ["chicken"],
"beef": ["beef", "steak", "ground beef"],
"pork": ["pork", "bacon", "ham"],
"fish": ["salmon", "tilapia", "cod", "fish fillet"],
"shrimp": ["shrimp", "prawn"],
"turkey": ["turkey"],
"lamb": ["lamb"],
"vegetarian": ["tofu", "tempeh", "vegetarian"],
}
for ptype, kws in proteins.items():
for kw in kws:
if kw in text:
return ptype
return None
def _upsert_ingredient(db: Session, name: str) -> Ingredient:
"""Idempotent insert by name_lower. Mirrors the public
POST /api/ingredients logic without the HTTP roundtrip."""
name_lower = name.lower()
existing = db.query(Ingredient).filter(Ingredient.name_lower == name_lower).first()
if existing:
return existing
row = Ingredient(name=name, name_lower=name_lower, aliases=[])
db.add(row)
try:
db.commit()
db.refresh(row)
except Exception:
db.rollback()
existing = db.query(Ingredient).filter(Ingredient.name_lower == name_lower).first()
if existing:
return existing
raise
return row
@router.post("/import", response_model=dict, status_code=201)
@router.post("/import")
def import_recipe(
payload: RecipeImportRequest,
db: Session = Depends(get_db),
_user: str = Depends(require_session),
) -> dict:
"""Import a Spoonacular recipe into the local library. One
/information call (1 point) + ingredient upserts + Recipe insert.
Returns the new recipe id."""
if payload.external_source != "spoonacular":
raise HTTPException(status_code=400, detail=f"unknown external_source: {payload.external_source}")
if _points_available() < 1.05:
raise HTTPException(status_code=503, detail="spoonacular daily quota reached; try again tomorrow")
api_key = _ensure_spoonacular_configured()
# 1) Reject duplicates
existing = (
db.query(Recipe)
.filter(Recipe.external_source == "spoonacular", Recipe.external_id == payload.external_id)
.first()
)
if existing:
raise HTTPException(status_code=409, detail=f"recipe already imported: {existing.id}")
# 2) Fetch full info
try:
resp = requests.get(
_INFO_URL.format(id=payload.external_id),
params={"apiKey": api_key, "includeNutrition": "false"},
timeout=15,
)
resp.raise_for_status()
except requests.RequestException as exc:
logger.warning("Spoonacular /information failed for %s: %s", payload.external_id, exc)
raise HTTPException(status_code=502, detail=f"spoonacular info failed: {exc}") from exc
full = resp.json()
_charge_points(1.0)
logger.info("Spoonacular import %s (1 pt, total %.1f/%.0f)",
payload.external_id, _points_used, _DAILY_LIMIT)
# 3) Resolve ingredients (upsert)
ingredient_names: List[str] = []
ingredients_json: List[dict] = []
for ing in full.get("extendedIngredients", []):
name = (ing.get("name") or ing.get("originalName") or "").strip()
if not name:
continue
ingredient_names.append(name)
local = _upsert_ingredient(db, name)
qty = ing.get("amount")
ingredients_json.append({
"ingredient_id": str(local.id),
"name": local.name,
"qty": float(qty) if qty is not None else 1.0,
"unit": ing.get("unit", "") or None,
"notes": None,
})
# 4) Extract instructions
instructions: List[str] = []
analyzed = full.get("analyzedInstructions", [])
if analyzed:
for step in analyzed[0].get("steps", []):
txt = step.get("step", "")
if txt:
instructions.append(txt)
if not instructions:
raw = full.get("instructions", "")
if raw:
instructions = [raw]
# 5) Build the Recipe
title = full.get("title") or "Untitled"
cuisines = [str(c).lower() for c in (full.get("cuisines") or []) if c]
diets = [str(d).lower() for d in (full.get("diets") or []) if d]
# 6) Resolve family_profile_id (require_session auto-fills first profile)
profile = db.query(FamilyProfile).first()
if not profile:
raise HTTPException(status_code=404, detail="family profile not found")
row = Recipe(
family_profile_id=profile.id,
name=title,
description=full.get("summary"),
image_url=full.get("image"),
image_source="spoonacular",
prep_time_minutes=int(full["preparationMinutes"]) if full.get("preparationMinutes") else None,
cook_time_minutes=int(full["cookingMinutes"]) if full.get("cookingMinutes") else None,
servings=int(full.get("servings", 4)),
cuisine_tags=cuisines,
dietary_tags=diets,
protein_type=_infer_protein_simple(title, ingredient_names),
ingredients=ingredients_json,
side_dishes=[],
instructions=instructions or ["See source for instructions."],
source_url=full.get("sourceUrl") or full.get("spoonacularSourceUrl"),
scraped_at=None,
is_manually_added=True,
external_source="spoonacular",
external_id=payload.external_id,
discovery_reason="user imported via webui",
)
db.add(row)
db.commit()
db.refresh(row)
return {
"id": str(row.id),
"name": row.name,
"external_id": row.external_id,
"ingredients_imported": len(ingredients_json),
}
):
"""Reject direct publication; reviewed staging is the sole import path."""
raise HTTPException(status_code=status.HTTP_410_GONE, detail=_RETIRED_DETAIL)
backend/app/api/recipes.py
+37 -11
View File
@@ -10,7 +10,13 @@ from sqlalchemy import or_
from sqlalchemy.orm import Session
from app.database import get_db
from app.models import FamilyProfile, Ingredient, NeverSuggest, Recipe
from app.models import (
FamilyProfile,
Ingredient,
NeverSuggest,
Recipe,
RecipeCatalogStatus,
)
from app.schemas.recipe import (
RecipeCreate,
RecipeRead,
@@ -45,7 +51,19 @@ def _validate_ingredient_ids(db: Session, ingredient_refs) -> None:
)
def _serialize(row: Recipe) -> dict:
def _serialize(row: Recipe, db: Session) -> dict:
raw_ingredients = row.ingredients or []
# Resolve ingredient names from the canonical Ingredient table
ingredient_ids = [ref["ingredient_id"] for ref in raw_ingredients if ref.get("ingredient_id")]
name_map = {}
if ingredient_ids:
name_rows = db.query(Ingredient.id, Ingredient.name).filter(Ingredient.id.in_(ingredient_ids)).all()
name_map = {str(r[0]): r[1] for r in name_rows}
enriched_ingredients = []
for ref in raw_ingredients:
enriched = dict(ref)
enriched["name"] = name_map.get(str(ref.get("ingredient_id")), "unknown ingredient")
enriched_ingredients.append(enriched)
return {
"id": row.id,
"name": row.name,
@@ -59,7 +77,7 @@ def _serialize(row: Recipe) -> dict:
"protein_type": row.protein_type,
"spice_level": row.spice_level,
"calories_per_serving": row.calories_per_serving,
"ingredients": row.ingredients or [],
"ingredients": enriched_ingredients,
"side_dishes": row.side_dishes or [],
"instructions": list(row.instructions or []),
"source_url": row.source_url,
@@ -91,7 +109,9 @@ def list_recommended_recipes(
}
# Start from all recipes
query = db.query(Recipe).filter(~Recipe.id.in_(blocked)) if blocked else db.query(Recipe)
query = db.query(Recipe).filter(Recipe.catalog_status == RecipeCatalogStatus.ACTIVE)
if blocked:
query = query.filter(~Recipe.id.in_(blocked))
# Apply positive signal filters
pos_cuisines = {s.value for s in result.positive_signals if s.type == "prefer_cuisine"}
@@ -119,6 +139,7 @@ def list_recommended_recipes(
top_rated = result.top_rated_recipe_names or []
if top_rated:
top_rows = db.query(Recipe).filter(
Recipe.catalog_status == RecipeCatalogStatus.ACTIVE,
Recipe.name.in_(top_rated),
(~Recipe.id.in_(blocked) if blocked else True),
).limit(limit).all()
@@ -133,7 +154,7 @@ def list_recommended_recipes(
ids_seen.add(r.id)
ordered.append(r)
return [_serialize(r) for r in ordered]
return [_serialize(r, db) for r in ordered]
@public_router.get("", response_model=List[RecipeRead])
@@ -150,7 +171,9 @@ def list_recipes(
limit: int = Query(default=100, le=500),
db: Session = Depends(get_db),
):
query = db.query(Recipe)
query = db.query(Recipe).filter(
Recipe.catalog_status == RecipeCatalogStatus.ACTIVE
)
if q:
like = f"%{q.lower()}%"
query = query.filter(Recipe.name.ilike(like))
@@ -194,15 +217,18 @@ def list_recipes(
if blocked:
query = query.filter(~Recipe.id.in_(blocked))
rows = query.order_by(Recipe.name).limit(limit).all()
return [_serialize(r) for r in rows]
return [_serialize(r, db) for r in rows]
@public_router.get("/{recipe_id}", response_model=RecipeRead)
def get_recipe(recipe_id: UUID, db: Session = Depends(get_db)):
row = db.query(Recipe).filter(Recipe.id == recipe_id).first()
row = db.query(Recipe).filter(
Recipe.id == recipe_id,
Recipe.catalog_status == RecipeCatalogStatus.ACTIVE,
).first()
if row is None:
raise HTTPException(status_code=404, detail="recipe not found")
return _serialize(row)
return _serialize(row, db)
@admin_router.post("", response_model=RecipeRead, status_code=status.HTTP_201_CREATED)
@@ -228,7 +254,7 @@ def create_recipe(payload: RecipeCreate, db: Session = Depends(get_db)):
db.add(row)
db.commit()
db.refresh(row)
return _serialize(row)
return _serialize(row, db)
@admin_router.patch("/{recipe_id}", response_model=RecipeRead)
@@ -245,7 +271,7 @@ def update_recipe(recipe_id: UUID, payload: RecipeUpdate, db: Session = Depends(
setattr(row, field, value)
db.commit()
db.refresh(row)
return _serialize(row)
return _serialize(row, db)
@admin_router.delete("/{recipe_id}", status_code=status.HTTP_204_NO_CONTENT, response_class=Response)
backend/app/config.py
+1 -6
View File
@@ -39,12 +39,7 @@ class Settings(BaseSettings):
OLLAMA_API_KEY: Optional[str] = None
OLLAMA_MODEL: str = "gpt-oss:20b"
# Sprint 12: Spoonacular external recipe search. Free tier is
# 150 points/day. ComplexSearch = 1 point + 0.01 per result. The
# /information endpoint = 1 point per call. The recipe_search
# router gates calls to stay under 140 points/day to leave a
# safety margin.
SPOONACULAR_API_KEY: Optional[str] = None
WIKIBOOKS_USER_AGENT: str = "MealPlanner/1.0 (recipe catalog staging)"
FAMILY_EMAIL_1: Optional[str] = None
FAMILY_EMAIL_2: Optional[str] = None
backend/app/data/starter_recipes.json
+1357
View File
File diff suppressed because it is too large Load Diff
backend/app/main.py
+27 -4
View File
@@ -1,3 +1,4 @@
from contextlib import asynccontextmanager
from fastapi import FastAPI, Depends, HTTPException, Request
from fastapi.responses import JSONResponse
from fastapi.staticfiles import StaticFiles
@@ -11,10 +12,34 @@ import logging
logging.basicConfig(level=settings.LOG_LEVEL)
logger = logging.getLogger(__name__)
@asynccontextmanager
async def lifespan(app: FastAPI):
# Auto-migrate on boot. Idempotent: alembic upgrade head is a no-op
# when the schema is already at head. Catches the recurring failure
# where the DB is at revision N but the image expects N+1 (e.g. a
# new column like denial_expires_at), which would otherwise 500 the
# first request that touches the new column. Skip when SKIP_AUTO_MIGRATE
# is set (tests / read-only replicas).
import os
if not os.getenv("SKIP_AUTO_MIGRATE"):
try:
from alembic.config import Config
from alembic import command
cfg = Config("alembic.ini")
command.upgrade(cfg, "head")
logger.info("auto-migrate: alembic upgrade head OK")
except Exception as e:
# Don't crash boot — log loud so it's visible in `docker logs`.
logger.error(f"auto-migrate FAILED: {e!r}")
yield
app = FastAPI(
title="MealPlanner",
description="Self-hosted meal planning system",
version="0.1.0",
lifespan=lifespan,
)
# Serve generated recipe images from local filesystem.
@@ -77,10 +102,8 @@ app.include_router(shopping_list.router, prefix="/api/shopping-list", tags=["sho
app.include_router(pantry.router, prefix="/api/pantry", tags=["pantry"])
app.include_router(admin.router, prefix="/api/admin", tags=["admin"])
app.include_router(orchestrate_api.router, prefix="/api/orchestrate", tags=["orchestrate"])
# Sprint 12: external recipe search (Spoonacular) + import.
# Mounted BEFORE recipes_api.public_router so /search and /import are
# not shadowed by the WIP's GET /{recipe_id} (which would 422 on a
# non-UUID "search" path segment). See sprint15-verification.md.
# Retired external search/import routes stay mounted before /{recipe_id}
# so legacy clients receive 410 rather than UUID validation errors.
app.include_router(recipe_search_api.router, prefix="/api/recipes", tags=["recipes"])
app.include_router(ingredients_api.public_router)
app.include_router(ingredients_api.admin_router)
backend/app/models/__init__.py
+83 -1
View File
@@ -1,6 +1,6 @@
from sqlalchemy import (
Column, String, Integer, Numeric, Float, Text, Boolean, Date, DateTime,
ForeignKey, CheckConstraint, UniqueConstraint, Enum as SQLEnum
ForeignKey, CheckConstraint, UniqueConstraint, Index, Enum as SQLEnum
)
from sqlalchemy.dialects.postgresql import UUID, JSONB, ARRAY
from sqlalchemy.orm import relationship
@@ -85,6 +85,18 @@ class IngredientMatchSource(enum.Enum):
MANUAL = "manual"
class RecipeImportStatus(str, enum.Enum):
PENDING = "pending"
QUARANTINED = "quarantined"
APPROVED = "approved"
REJECTED = "rejected"
class RecipeCatalogStatus(str, enum.Enum):
ACTIVE = "active"
QUARANTINED = "quarantined"
class FamilyProfile(Base):
__tablename__ = "family_profile"
@@ -186,6 +198,19 @@ class Recipe(Base):
external_source = Column(String(50))
external_id = Column(String(100))
discovery_reason = Column(Text)
catalog_status = Column(
SQLEnum(
RecipeCatalogStatus,
name="recipe_catalog_status_enum",
create_type=False,
values_callable=lambda obj: [e.value for e in obj],
),
nullable=False,
default=RecipeCatalogStatus.ACTIVE,
server_default="active",
)
catalog_findings = Column(JSONB, nullable=False, default=list, server_default="[]")
catalog_reviewed_at = Column(DateTime(timezone=True))
created_at = Column(DateTime(timezone=True), server_default=func.now())
updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now())
@@ -197,6 +222,63 @@ class Recipe(Base):
return (self.prep_time_minutes or 0) + (self.cook_time_minutes or 0)
class RecipeImportStaging(Base):
"""Auditable, globally scoped recipe import candidate awaiting review."""
__tablename__ = "recipe_import_staging"
id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
source = Column(String(100), nullable=False)
external_id = Column(String(255), nullable=False)
source_url = Column(Text, nullable=False)
source_revision = Column(String(255), nullable=False)
license_name = Column(String(255), nullable=False)
license_url = Column(Text, nullable=False)
attribution = Column(Text, nullable=False)
raw_payload = Column(JSONB, nullable=False)
payload_sha256 = Column(String(64), nullable=False)
normalized_recipe = Column(JSONB, nullable=False)
validation_report = Column(JSONB, nullable=False)
status = Column(
SQLEnum(
RecipeImportStatus,
name="recipe_import_status_enum",
create_type=False,
values_callable=lambda obj: [e.value for e in obj],
),
nullable=False,
default=RecipeImportStatus.PENDING,
server_default="pending",
)
rejection_reason = Column(Text)
recipe_id = Column(UUID(as_uuid=True), ForeignKey("recipe.id", ondelete="SET NULL"))
acquired_at = Column(
DateTime(timezone=True), nullable=False, server_default=func.now()
)
reviewed_at = Column(DateTime(timezone=True))
reviewed_by = Column(String(300))
__table_args__ = (
UniqueConstraint(
"source",
"external_id",
"source_revision",
name="uq_recipe_import_staging_source_revision",
),
CheckConstraint(
"payload_sha256 ~ '^[0-9a-f]{64}$'",
name="ck_recipe_import_staging_payload_sha256",
),
CheckConstraint(
"(reviewed_at IS NULL) = (reviewed_by IS NULL)",
name="ck_recipe_import_staging_review_metadata",
),
Index("ix_recipe_import_staging_status_acquired_at", "status", "acquired_at"),
)
recipe = relationship("Recipe")
class MealPlan(Base):
__tablename__ = "meal_plan"
backend/app/schemas/__init__.py
+1 -19
View File
@@ -147,7 +147,7 @@ class PlannerConfigOverride(BaseModel):
time_full_minutes: Optional[int] = Field(default=None, ge=0)
recency_full_weeks: Optional[int] = Field(default=None, ge=0)
top_k: Optional[int] = Field(default=None, ge=1)
set_size: Optional[int] = Field(default=None, ge=1)
set_size: Optional[int] = Field(default=None, ge=3, le=3)
p_protein: Optional[float] = Field(default=None, ge=0)
p_cuisine: Optional[float] = Field(default=None, ge=0)
@@ -392,24 +392,6 @@ class ShoppingListResponse(BaseModel):
by_aisle: dict[str, List[ShoppingListItem]]
# Sprint 12: Spoonacular external recipe search. The hit shape is
# what the webui shows in the "Search the web" panel; the import
# request is what the import button POSTs.
class RecipeSearchHit(BaseModel):
external_id: str
external_source: str = "spoonacular"
name: str
image_url: Optional[str] = None
source_url: Optional[str] = None
prep_time_minutes: Optional[int] = None
cook_time_minutes: Optional[int] = None
servings: Optional[int] = None
cuisine_tags: List[str] = Field(default_factory=list)
dietary_tags: List[str] = Field(default_factory=list)
protein_type: Optional[str] = None
calories_per_serving: Optional[int] = None
class RecipeImportRequest(BaseModel):
external_id: str
external_source: str = "spoonacular"
backend/app/schemas/meal_plan_generation.py
+4
View File
@@ -13,6 +13,10 @@ class GenerateRequest(BaseModel):
week_start_date: date
class PublicGenerateRequest(BaseModel):
week_start_date: date
class RegenerateRequest(BaseModel):
family_profile_id: UUID
week_start_date: date
backend/app/schemas/planner_config.py
+1 -1
View File
@@ -19,7 +19,7 @@ class PlannerConfigOverride(BaseModel):
time_full_minutes: Optional[int] = Field(default=None, ge=0)
recency_full_weeks: Optional[int] = Field(default=None, ge=0)
top_k: Optional[int] = Field(default=None, ge=1)
set_size: Optional[int] = Field(default=None, ge=1)
set_size: Optional[int] = Field(default=None, ge=3, le=3)
p_protein: Optional[float] = Field(default=None, ge=0)
p_cuisine: Optional[float] = Field(default=None, ge=0)
backend/app/schemas/recipe.py
+5 -1
View File
@@ -9,6 +9,7 @@ from pydantic import BaseModel, Field, field_validator
class RecipeIngredientRef(BaseModel):
ingredient_id: UUID
name: Optional[str] = Field(default=None, description="resolved name from Ingredient table")
qty: float = Field(gt=0)
unit: Optional[str] = Field(default=None, max_length=50)
notes: Optional[str] = None
@@ -66,10 +67,13 @@ class RecipeUpdate(BaseModel):
class RecipeRead(RecipeBase):
id: UUID
prep_time_minutes: Optional[int] = Field(default=None, ge=0)
cook_time_minutes: Optional[int] = Field(default=None, ge=0)
protein_type: Optional[str] = None
external_source: Optional[str] = None
external_id: Optional[str] = None
discovery_reason: Optional[str] = None
side_dishes: List[SideDish] = Field(default_factory=list)
side_dishes: Optional[List[SideDish]] = None
model_config = {"from_attributes": True}
backend/app/schemas/recipe_import.py
+64
View File
@@ -0,0 +1,64 @@
from __future__ import annotations
from datetime import datetime
from typing import Any
from uuid import UUID
from pydantic import BaseModel, Field, field_validator
class RecipeImportListItem(BaseModel):
id: UUID
source: str
external_id: str
source_url: str
source_revision: str
name: str | None
status: str
acquired_at: datetime
reviewed_at: datetime | None
reviewed_by: str | None
recipe_id: UUID | None
class RecipeImportListResponse(BaseModel):
items: list[RecipeImportListItem]
total: int
limit: int
offset: int
has_next: bool
class RecipeImportDetail(BaseModel):
id: UUID
source: str
external_id: str
source_url: str
source_revision: str
license_name: str
license_url: str
attribution: str
raw_payload: dict[str, Any]
payload_sha256: str
normalized_recipe: dict[str, Any]
validation_report: dict[str, Any]
status: str
rejection_reason: str | None
recipe_id: UUID | None
acquired_at: datetime
reviewed_at: datetime | None
reviewed_by: str | None
model_config = {"from_attributes": True}
class RecipeImportRejectRequest(BaseModel):
reason: str = Field(min_length=1, max_length=1000)
@field_validator("reason")
@classmethod
def normalize_reason(cls, value: str) -> str:
cleaned = value.strip()
if not cleaned:
raise ValueError("reason must not be blank")
return cleaned
backend/app/security.py
+3 -2
View File
@@ -15,8 +15,8 @@ SESSION_COOKIE = "mp_session"
SESSION_MAX_AGE = 60 * 60 * 24 * 30 # 30 days
def require_admin(request: Request) -> None:
"""Enforce a shared bearer token. 401 on bad/missing token, 503 if unset."""
def require_admin(request: Request) -> str:
"""Enforce the shared bearer token and return its configured principal."""
expected = settings.ADMIN_TOKEN
if not expected:
raise HTTPException(
@@ -29,6 +29,7 @@ def require_admin(request: Request) -> None:
status_code=status.HTTP_401_UNAUTHORIZED,
detail="Invalid admin token",
)
return (settings.ADMIN_EMAIL or "").strip() or "admin"
def _signer() -> TimestampSigner:
backend/app/services/image_generation.py
+5 -2
View File
@@ -17,7 +17,7 @@ import httpx
from sqlalchemy.orm import Session
from app.config import settings
from app.models import Recipe
from app.models import Recipe, RecipeCatalogStatus
logger = logging.getLogger(__name__)
@@ -156,6 +156,7 @@ def generate_image_for_recipe(
# Persist on recipe row if a DB session was provided.
if db is not None:
recipe.image_url = public_path
recipe.image_source = f"openai:{model}"
db.add(recipe)
db.commit()
logger.info("Image saved for '%s' -> %s", recipe.name, public_path)
@@ -183,7 +184,9 @@ def generate_images_batch(
"""
from sqlalchemy import or_
query = db.query(Recipe)
query = db.query(Recipe).filter(
Recipe.catalog_status == RecipeCatalogStatus.ACTIVE
)
if recipe_ids:
query = query.filter(Recipe.id.in_(recipe_ids))
backend/app/services/orchestrator/steps.py
+43 -33
View File
@@ -17,7 +17,7 @@ from datetime import datetime, timezone
from typing import TYPE_CHECKING
from app.config import settings
from app.models import FamilyMember, FamilyProfile, Ingredient, IngredientGroceryMatch, MealPlan, MealPlanItemStatus, MealPlanVote
from app.models import FamilyMember, FamilyProfile, Ingredient, IngredientGroceryMatch, MealPlan, MealPlanItemStatus, MealPlanVote, RecipeCatalogStatus
from app.services.approval import issue_token
from app.services.email import get_email_backend
from app.services.orchestrator.alerts import send_admin_alert
@@ -26,8 +26,7 @@ from app.services.scraper_service import ScraperService
from app.utils.units import convert_qty
from app.services.feedback_analyzer import FeedbackAnalyzer
from app.services.recipe_discovery import RecipeDiscoveryService
from app.services.recipe_ingestion import RecipeIngestionService
from app.services.wikibooks_import import WikibooksImportService
if TYPE_CHECKING:
from sqlalchemy.orm import Session
@@ -56,6 +55,15 @@ def _suggested_sides_html(components: dict | None) -> str:
)
def _active_recipe_items(items: list) -> list:
return [
item
for item in items
if item.recipe is not None
and item.recipe.catalog_status == RecipeCatalogStatus.ACTIVE
]
def step_scrape(run: "WeeklyRun", db: "Session") -> None:
if run.scraped_at is not None:
logger.info("step_scrape: already done for %s, skipping", run.week_start_date)
@@ -132,6 +140,12 @@ def step_email(run: "WeeklyRun", db: "Session") -> None:
f"No MealPlan for family {run.family_id} week {run.week_start_date}"
)
pending_items = [
item
for item in _active_recipe_items(plan.items)
if item.approval_status == MealPlanItemStatus.pending
]
members = (
db.query(FamilyMember)
.filter(
@@ -150,11 +164,10 @@ def step_email(run: "WeeklyRun", db: "Session") -> None:
# Preload ingredient names for all pending items
ing_ids: set = set()
for _item in plan.items:
if _item.approval_status == MealPlanItemStatus.pending and _item.recipe:
for _ing in (_item.recipe.ingredients or []):
if "ingredient_id" in _ing:
ing_ids.add(_ing["ingredient_id"])
for _item in pending_items:
for _ing in (_item.recipe.ingredients or []):
if "ingredient_id" in _ing:
ing_ids.add(_ing["ingredient_id"])
ingredient_names: dict = {}
if ing_ids:
@@ -165,10 +178,8 @@ def step_email(run: "WeeklyRun", db: "Session") -> None:
# Build shopping list preview once (shared across all member emails)
all_ingredients: dict[str, tuple[str, str, str, str]] = {}
for item in plan.items:
if item.approval_status != MealPlanItemStatus.pending:
continue
for ing in (item.recipe.ingredients or [] if item.recipe else []):
for item in pending_items:
for ing in (item.recipe.ingredients or []):
ing_name = ingredient_names.get(str(ing.get("ingredient_id", "")), ing.get("name", "unknown")).strip()
if not ing_name or ing_name in all_ingredients:
continue
@@ -217,9 +228,7 @@ def step_email(run: "WeeklyRun", db: "Session") -> None:
backend = get_email_backend()
for member in members:
item_html_parts = []
for item in plan.items:
if item.approval_status != MealPlanItemStatus.pending:
continue
for item in pending_items:
token = issue_token(item.id, member.id)
vote_url = (
f"{settings.APP_BASE_URL}/api/meals/vote/{item.id}?token={token}"
@@ -400,7 +409,7 @@ def step_finalize(run: "WeeklyRun", db: "Session") -> None:
approved_items = [
item
for item in (plan.items if plan else [])
for item in _active_recipe_items(plan.items if plan else [])
if item.approval_status == MealPlanItemStatus.approved
]
@@ -524,18 +533,19 @@ def step_finalize(run: "WeeklyRun", db: "Session") -> None:
analysis.discovery_queries and
analysis.confidence >= 0.5
):
discovery = RecipeDiscoveryService()
candidates = discovery.discover(analysis.discovery_queries)
if candidates:
ingestion = RecipeIngestionService()
added = ingestion.ingest(db, run.family_id, candidates, analysis.to_dict())
logger.info(
"step_finalize: recipe discovery added %d new recipes for family %s",
added, run.family_id,
# Roll back a partial discovery batch without losing finalization.
with db.begin_nested():
staged = WikibooksImportService(
settings.WIKIBOOKS_USER_AGENT
).stage_discovered_pages(
db,
limit=min(5, len(analysis.discovery_queries)),
queries=analysis.discovery_queries,
)
# ingestion deliberately does not commit so orchestrator
# can keep everything in one transaction
db.commit()
logger.info(
"step_finalize: staged %d Wikibooks candidates for review",
len(staged),
)
except Exception as exc:
# Discovery is best-effort; never block finalization
logger.warning("step_finalize: recipe discovery failed: %s", exc)
@@ -565,11 +575,12 @@ def step_reminder(run: "WeeklyRun", db: "Session") -> None:
db.commit()
return
pending_item_ids = [
item.id
for item in plan.items
pending_items = [
item
for item in _active_recipe_items(plan.items)
if item.approval_status == MealPlanItemStatus.pending
]
pending_item_ids = [item.id for item in pending_items]
if not pending_item_ids:
run.reminded_at = datetime.now(timezone.utc)
db.commit()
@@ -597,9 +608,8 @@ def step_reminder(run: "WeeklyRun", db: "Session") -> None:
}
unvoted = [
item
for item in plan.items
if item.approval_status == MealPlanItemStatus.pending
and item.id not in voted_ids
for item in pending_items
if item.id not in voted_ids
]
if not unvoted:
continue
backend/app/services/planner/config.py
+2
View File
@@ -37,6 +37,8 @@ class PlannerConfig:
total = self.w_savings + self.w_coverage + self.w_pantry + self.w_time + self.w_recency
if abs(total - 1.0) > 1e-6:
raise ValueError(f"weights must sum to 1.0, got {total}")
if self.set_size != 3:
raise ValueError("set_size must be 3 dinners per week")
def merge(self, overrides: "dict[str, Any]") -> "PlannerConfig":
"""Return a new PlannerConfig with overrides applied."""
backend/app/services/planner/generate.py
+8 -1
View File
@@ -3,6 +3,7 @@ from __future__ import annotations
from datetime import date
from decimal import Decimal
import logging
from typing import Dict, List, Optional, Set
from uuid import UUID
@@ -22,6 +23,7 @@ from app.models import (
MealType,
NeverSuggest,
Recipe,
RecipeCatalogStatus,
)
from app.services.planner.config import DEFAULT, PlannerConfig
from app.services.planner.cost import compute_recipe_cost
@@ -32,6 +34,9 @@ from app.services.planner.types import GenerationResult
from app.services.meal_pairings import components_with_suggested_sides
logger = logging.getLogger(__name__)
def _load_match_index(db: Session) -> Dict[UUID, List[dict]]:
"""ingredient_id → list of match dicts ordered by confidence DESC."""
rows = (
@@ -148,7 +153,9 @@ def generate_meal_plan(
except (ValueError, TypeError) as exc:
logger.warning("Invalid planner_config for family %s: %s", family_id, exc)
recipes = db.query(Recipe).all()
recipes = db.query(Recipe).filter(
Recipe.catalog_status == RecipeCatalogStatus.ACTIVE
).all()
exclude_set = exclude_recipe_ids or set()
recipe_dicts = [
{
backend/app/services/recipe_catalog_audit.py
+221
View File
@@ -0,0 +1,221 @@
"""Deterministic provenance and quality audit for the published recipe catalog."""
from __future__ import annotations
from datetime import datetime, timezone
from decimal import Decimal
from typing import Any
from sqlalchemy import and_
from sqlalchemy.orm import Session
from app.models import (
Ingredient,
Recipe,
RecipeCatalogStatus,
RecipeImportStaging,
RecipeImportStatus,
)
from app.services.wikibooks_import import WIKIBOOKS_SOURCE, WikibooksImportService
OPERATOR_QUARANTINE_FINDING_CODE = "operator_quarantine"
def audit_recipe_catalog(
db: Session,
*,
apply: bool = False,
reviewed_at: datetime | None = None,
) -> dict[str, Any]:
"""Audit every recipe and optionally apply active/quarantined state changes."""
timestamp = reviewed_at or datetime.now(timezone.utc)
ingredient_names = {
str(ingredient_id): name
for ingredient_id, name in db.query(Ingredient.id, Ingredient.name).all()
}
catalog_query = (
db.query(
Recipe,
RecipeImportStaging.source,
RecipeImportStaging.external_id,
RecipeImportStaging.source_url,
)
.outerjoin(
RecipeImportStaging,
and_(
RecipeImportStaging.recipe_id == Recipe.id,
RecipeImportStaging.status == RecipeImportStatus.APPROVED,
),
)
.order_by(Recipe.id, RecipeImportStaging.id)
)
if apply:
catalog_query = catalog_query.with_for_update(of=Recipe)
recipes: dict[str, Recipe] = {}
approved_links: dict[str, set[tuple[str, str, str]]] = {}
for recipe, source, external_id, source_url in catalog_query.all():
recipe_id = str(recipe.id)
recipes[recipe_id] = recipe
if source is not None and external_id is not None and source_url is not None:
approved_links.setdefault(recipe_id, set()).add(
(source, external_id, source_url)
)
validator = WikibooksImportService("MealPlanner catalog audit")
records: list[dict[str, Any]] = []
for recipe_id, recipe in recipes.items():
prior_findings = (
recipe.catalog_findings
if isinstance(recipe.catalog_findings, list)
else []
)
operator_quarantine_findings = [
finding
for finding in prior_findings
if isinstance(finding, dict)
and finding.get("code") == OPERATOR_QUARANTINE_FINDING_CODE
]
provenance, findings = _provenance_findings(
recipe, approved_links.get(recipe_id, set())
)
candidate, reference_findings = _validation_candidate(recipe, ingredient_names)
findings.extend(reference_findings)
validation = validator.validate_recipe(candidate)
findings.extend(
{**issue, "severity": severity}
for severity, key in (("error", "errors"), ("warning", "warnings"))
for issue in validation[key]
)
findings.extend(operator_quarantine_findings)
target_status = (
RecipeCatalogStatus.QUARANTINED
if operator_quarantine_findings
or any(item["severity"] == "error" for item in findings)
else RecipeCatalogStatus.ACTIVE
)
if apply:
recipe.catalog_status = target_status
recipe.catalog_findings = findings
recipe.catalog_reviewed_at = timestamp
records.append(
{
"id": str(recipe.id),
"name": recipe.name,
"provenance": provenance,
"catalog_status": target_status.value,
"findings": findings,
}
)
if apply:
db.flush()
quarantined = sum(
record["catalog_status"] == RecipeCatalogStatus.QUARANTINED.value
for record in records
)
return {
"version": 1,
"generated_at": timestamp.isoformat(),
"applied": apply,
"total": len(records),
"summary": {
"active": len(records) - quarantined,
"quarantined": quarantined,
},
"recipes": records,
}
def _provenance_findings(
recipe: Recipe, approved_links: set[tuple[str, str, str]]
) -> tuple[str, list[dict[str, str]]]:
if recipe.is_manually_added and not recipe.external_source:
return "manual", []
if recipe.external_source == WIKIBOOKS_SOURCE:
findings = []
expected_link = (recipe.external_source, recipe.external_id, recipe.source_url)
if expected_link not in approved_links:
findings.append(
_finding(
"missing_import_provenance",
"external_source",
"Published Wikibooks recipe is not linked to an approved staged import.",
)
)
if not recipe.source_url:
findings.append(
_finding("missing_source_url", "source_url", "External recipe has no source URL.")
)
return "staged_wikibooks", findings
if recipe.external_source == "spoonacular":
return "spoonacular", [
_finding(
"unlicensed_external_source",
"external_source",
"Spoonacular data was persisted without durable catalog rights.",
)
]
if recipe.external_source:
return "external_unverified", [
_finding(
"unverified_external_source",
"external_source",
"External recipe has no approved staged license record.",
)
]
return "missing", [
_finding(
"missing_provenance",
"external_source",
"Non-manual recipe has no auditable source or license record.",
)
]
def _validation_candidate(
recipe: Recipe, ingredient_names: dict[str, str]
) -> tuple[dict[str, Any], list[dict[str, str]]]:
ingredients: list[dict[str, Any]] = []
findings: list[dict[str, str]] = []
for index, item in enumerate(recipe.ingredients or []):
if not isinstance(item, dict):
findings.append(
_finding(
"invalid_ingredient_reference",
f"ingredients[{index}]",
"Ingredient reference must be an object.",
)
)
continue
ingredient_id = str(item.get("ingredient_id") or "")
name = ingredient_names.get(ingredient_id)
if not name:
findings.append(
_finding(
"orphan_ingredient_reference",
f"ingredients[{index}].ingredient_id",
"Ingredient reference does not resolve to the canonical ingredient table.",
)
)
name = str(item.get("name") or "")
quantity = item.get("qty")
if isinstance(quantity, Decimal):
quantity = float(quantity)
ingredients.append({"name": name, "quantity": quantity})
return (
{
"name": recipe.name,
"description": recipe.description,
"servings": recipe.servings,
"prep_time_minutes": recipe.prep_time_minutes,
"cook_time_minutes": recipe.cook_time_minutes,
"baking_time_minutes": None,
"ingredients": ingredients,
"instructions": list(recipe.instructions or []),
},
findings,
)
def _finding(code: str, field: str, message: str) -> dict[str, str]:
return {"code": code, "severity": "error", "field": field, "message": message}
backend/app/services/recipe_import_review.py
+196
View File
@@ -0,0 +1,196 @@
from __future__ import annotations
from datetime import datetime, timezone
from typing import Any, Mapping
from uuid import UUID
from sqlalchemy import select
from sqlalchemy.dialects.postgresql import insert as pg_insert
from sqlalchemy.orm import Session
from app.models import (
Ingredient,
Recipe,
RecipeCatalogStatus,
RecipeImportStaging,
RecipeImportStatus,
)
from app.services.wikibooks_import import (
WIKIBOOKS_SOURCE,
WikibooksImportService,
canonical_json_sha256,
)
class RecipeImportNotFoundError(LookupError):
"""Raised when a staged recipe does not exist."""
class RecipeImportConflictError(ValueError):
"""Raised when a review transition or publication is unsafe."""
def approve_recipe_import(
db: Session, import_id: UUID, reviewed_by: str
) -> tuple[RecipeImportStaging, bool]:
"""Atomically publish one pending Wikibooks candidate."""
staged = _locked_import(db, import_id)
if staged.status == RecipeImportStatus.APPROVED:
if staged.recipe_id and db.get(Recipe, staged.recipe_id):
return staged, False
elif staged.status != RecipeImportStatus.PENDING:
raise RecipeImportConflictError(
f"Only pending imports can be approved; current status is {staged.status.value}"
)
if staged.source != WIKIBOOKS_SOURCE:
raise RecipeImportConflictError(f"Unsupported recipe source: {staged.source}")
raw_payload = _mapping(staged.raw_payload, "raw_payload")
if canonical_json_sha256(raw_payload) != staged.payload_sha256:
raise RecipeImportConflictError("Raw payload checksum does not match provenance")
normalizer = WikibooksImportService("MealPlanner recipe review")
normalized = normalizer.normalize_page(raw_payload)
validation_report = normalizer.validate_recipe(normalized)
if not validation_report["valid"]:
raise RecipeImportConflictError("Candidate no longer passes publication validation")
existing = (
db.query(Recipe)
.filter(
Recipe.external_source == staged.source,
Recipe.external_id == staged.external_id,
)
.first()
)
if existing:
raise RecipeImportConflictError(
"A recipe from this source page is already published"
)
ingredient_refs = _ingredient_refs(db, normalized["ingredients"])
cook_time = (normalized.get("cook_time_minutes") or 0) + (
normalized.get("baking_time_minutes") or 0
)
recipe = Recipe(
family_profile_id=None,
name=normalized["name"],
description=normalized.get("description"),
image_url=None,
image_source=None,
prep_time_minutes=normalized.get("prep_time_minutes"),
cook_time_minutes=cook_time or None,
servings=normalized["servings"],
cuisine_tags=[],
dietary_tags=[],
protein_type=None,
spice_level=None,
calories_per_serving=None,
ingredients=ingredient_refs,
side_dishes=[],
instructions=list(normalized["instructions"]),
source_url=staged.source_url,
scraped_at=staged.acquired_at,
is_manually_added=False,
external_source=staged.source,
external_id=staged.external_id,
discovery_reason=f"Approved import from revision {staged.source_revision}",
catalog_status=RecipeCatalogStatus.ACTIVE,
catalog_findings=[],
)
db.add(recipe)
db.flush()
staged.normalized_recipe = normalized
staged.validation_report = validation_report
staged.status = RecipeImportStatus.APPROVED
staged.rejection_reason = None
staged.recipe_id = recipe.id
staged.reviewed_at = datetime.now(timezone.utc)
staged.reviewed_by = reviewed_by[:300]
db.commit()
db.refresh(staged)
return staged, True
def reject_recipe_import(
db: Session, import_id: UUID, reason: str, reviewed_by: str
) -> RecipeImportStaging:
"""Atomically reject a pending or quarantined candidate."""
staged = _locked_import(db, import_id)
if staged.status == RecipeImportStatus.APPROVED:
raise RecipeImportConflictError("Approved imports cannot be rejected")
if staged.status == RecipeImportStatus.REJECTED:
if staged.rejection_reason == reason:
return staged
raise RecipeImportConflictError("Import was already rejected with another reason")
staged.status = RecipeImportStatus.REJECTED
staged.rejection_reason = reason
staged.reviewed_at = datetime.now(timezone.utc)
staged.reviewed_by = reviewed_by[:300]
db.commit()
db.refresh(staged)
return staged
def _locked_import(db: Session, import_id: UUID) -> RecipeImportStaging:
staged = db.execute(
select(RecipeImportStaging)
.where(RecipeImportStaging.id == import_id)
.with_for_update()
).scalar_one_or_none()
if staged is None:
raise RecipeImportNotFoundError("Recipe import not found")
return staged
def _ingredient_refs(
db: Session, ingredients: list[Mapping[str, Any]]
) -> list[dict[str, Any]]:
prepared: list[tuple[str, str, float, str | None, str | None]] = []
for item in ingredients:
name = " ".join(str(item.get("name", "")).split())
name_lower = name.casefold()
quantity = item.get("quantity")
unit = item.get("unit")
source_text = item.get("source_text")
if not name or len(name) > 200:
raise RecipeImportConflictError("Ingredient name is invalid")
if not isinstance(quantity, (int, float)) or isinstance(quantity, bool) or quantity <= 0:
raise RecipeImportConflictError("Ingredient quantity is invalid")
if unit is not None and (not isinstance(unit, str) or len(unit) > 50):
raise RecipeImportConflictError("Ingredient unit is invalid")
prepared.append((name, name_lower, float(quantity), unit, source_text))
for name, name_lower, _, unit, _ in prepared:
db.execute(
pg_insert(Ingredient.__table__)
.values(name=name, name_lower=name_lower, unit=unit, aliases=[])
.on_conflict_do_nothing(index_elements=[Ingredient.name_lower])
)
canonical = {
row.name_lower: row
for row in db.query(Ingredient)
.filter(Ingredient.name_lower.in_([item[1] for item in prepared]))
.all()
}
if len(canonical) != len({item[1] for item in prepared}):
raise RecipeImportConflictError("Canonical ingredient creation failed")
return [
{
"ingredient_id": str(canonical[name_lower].id),
"qty": quantity,
"unit": unit,
"notes": source_text,
}
for _, name_lower, quantity, unit, source_text in prepared
]
def _mapping(value: Any, field: str) -> Mapping[str, Any]:
if not isinstance(value, Mapping):
raise RecipeImportConflictError(f"{field} must be an object")
return value
backend/app/services/starter_recipe_catalog.py
+540
View File
@@ -0,0 +1,540 @@
"""Validate and atomically load the bundled starter recipe catalog."""
from __future__ import annotations
import json
import math
import re
from pathlib import Path
from typing import Any, Mapping
from uuid import UUID
from sqlalchemy import func, select
from sqlalchemy.dialects.postgresql import insert as pg_insert
from sqlalchemy.orm import Session
from app.models import Ingredient, Recipe, RecipeCatalogStatus
STARTER_CATALOG_VERSION = 2
STARTER_RECIPE_COUNT = 50
STARTER_DISCOVERY_REASON = "bundled_starter_catalog_v1"
# Fixed PostgreSQL bigint advisory-lock key for this loader (ASCII: MPLSTART).
# Changing it would allow concurrent catalog loads to bypass each other.
STARTER_CATALOG_ADVISORY_LOCK_ID = 0x4D504C5354415254
DEFAULT_CATALOG_PATH = Path(__file__).resolve().parents[1] / "data" / "starter_recipes.json"
MAX_SERVINGS = 50
MAX_TIME_MINUTES = 1_440
MAX_INGREDIENT_QUANTITY = 10_000
MAX_CALORIES_PER_SERVING = 500
MAX_SPICE_LEVEL = 5
_PLACEHOLDER_TEXT = frozenset(
{"coming soon", "directions", "instructions", "method", "n a", "tbd", "todo"}
)
_PLACEHOLDER_PHRASES = ("lorem ipsum", "placeholder")
_RECIPE_KEYS = frozenset(
{
"name",
"description",
"servings",
"prep_time_minutes",
"cook_time_minutes",
"cuisine_tags",
"dietary_tags",
"protein_type",
"spice_level",
"calories_per_serving",
"ingredients",
"instructions",
}
)
_INGREDIENT_KEYS = frozenset({"name", "quantity", "unit"})
_CATALOG_SCALAR_FIELDS = (
"description",
"servings",
"prep_time_minutes",
"cook_time_minutes",
"cuisine_tags",
"dietary_tags",
"protein_type",
"spice_level",
"calories_per_serving",
)
class StarterRecipeCatalogError(ValueError):
"""Raised when the bundled catalog is incomplete or unsafe to load."""
def read_starter_catalog(path: Path = DEFAULT_CATALOG_PATH) -> Mapping[str, Any]:
"""Read the JSON catalog without accepting non-standard numeric constants."""
try:
with path.open(encoding="utf-8") as catalog_file:
catalog = json.load(catalog_file, parse_constant=_reject_json_constant)
except OSError as exc:
raise StarterRecipeCatalogError(f"Catalog cannot be read: {path}") from exc
except ValueError as exc:
raise StarterRecipeCatalogError("Catalog is not valid JSON") from exc
if not isinstance(catalog, Mapping):
raise StarterRecipeCatalogError("Catalog root must be an object")
return catalog
def validate_starter_catalog(catalog: Mapping[str, Any]) -> list[dict[str, Any]]:
"""Return normalized recipes after validating the complete, fixed-size catalog."""
_require_exact_keys(catalog, {"version", "recipes"}, "catalog")
if catalog["version"] != STARTER_CATALOG_VERSION:
raise StarterRecipeCatalogError(
f"Catalog version must be {STARTER_CATALOG_VERSION}"
)
recipes = catalog["recipes"]
if not isinstance(recipes, list) or len(recipes) != STARTER_RECIPE_COUNT:
raise StarterRecipeCatalogError(
f"Catalog must contain exactly {STARTER_RECIPE_COUNT} recipes"
)
normalized: list[dict[str, Any]] = []
names: set[str] = set()
for index, recipe in enumerate(recipes, 1):
if not isinstance(recipe, Mapping):
raise StarterRecipeCatalogError(f"Recipe {index} must be an object")
_require_exact_keys(recipe, _RECIPE_KEYS, f"Recipe {index}")
name = _text(recipe["name"], f"Recipe {index} name", maximum=300)
name_key = _canonical_name(name)
if name_key in names:
raise StarterRecipeCatalogError(f"Recipe {index} name is not unique")
names.add(name_key)
ingredients = _validate_ingredients(recipe["ingredients"], index)
instructions = _validate_instructions(recipe["instructions"], index)
normalized.append(
{
"name": name,
"description": _text(
recipe["description"], f"Recipe {index} description", maximum=10_000
),
"servings": _positive_int(
recipe["servings"], f"Recipe {index} servings", MAX_SERVINGS
),
"prep_time_minutes": _positive_int(
recipe["prep_time_minutes"],
f"Recipe {index} prep_time_minutes",
MAX_TIME_MINUTES,
),
"cook_time_minutes": _positive_int(
recipe["cook_time_minutes"],
f"Recipe {index} cook_time_minutes",
MAX_TIME_MINUTES,
),
"cuisine_tags": _tags(recipe["cuisine_tags"], f"Recipe {index} cuisine_tags"),
"dietary_tags": _tags(recipe["dietary_tags"], f"Recipe {index} dietary_tags"),
"protein_type": _text(
recipe["protein_type"], f"Recipe {index} protein_type", maximum=50
),
"spice_level": _bounded_int(
recipe["spice_level"], f"Recipe {index} spice_level", 0, MAX_SPICE_LEVEL
),
"calories_per_serving": _positive_int(
recipe["calories_per_serving"],
f"Recipe {index} calories_per_serving",
MAX_CALORIES_PER_SERVING,
),
"ingredients": ingredients,
"instructions": instructions,
}
)
return normalized
def load_bundled_starter_catalog(
db: Session,
*,
path: Path = DEFAULT_CATALOG_PATH,
dry_run: bool = False,
update_existing: bool = False,
) -> dict[str, Any]:
"""Load the bundled JSON catalog after validating all recipes before writes."""
return load_starter_catalog(
db,
read_starter_catalog(path),
dry_run=dry_run,
update_existing=update_existing,
)
def load_starter_catalog(
db: Session,
catalog: Mapping[str, Any],
*,
dry_run: bool = False,
update_existing: bool = False,
) -> dict[str, Any]:
"""Load or explicitly synchronize bundled recipes in one transaction."""
recipes = validate_starter_catalog(catalog)
catalog_names = {recipe["name"] for recipe in recipes}
catalog_by_name = {recipe["name"]: recipe for recipe in recipes}
ingredient_names = {
_canonical_name(ingredient["name"]): ingredient
for recipe in recipes
for ingredient in recipe["ingredients"]
}
with db.begin():
# This transaction-scoped lock must precede every catalog Recipe read or write.
db.execute(select(func.pg_advisory_xact_lock(STARTER_CATALOG_ADVISORY_LOCK_ID)))
marked_recipes = list(
db.execute(
select(Recipe)
.where(Recipe.discovery_reason == STARTER_DISCOVERY_REASON)
.with_for_update()
).scalars()
)
marked_by_name = {recipe.name: recipe for recipe in marked_recipes}
if len(marked_by_name) != len(marked_recipes):
raise StarterRecipeCatalogError("Bundled catalog has duplicate persisted recipe names")
unknown_marked_names = set(marked_by_name).difference(catalog_names)
if unknown_marked_names:
raise StarterRecipeCatalogError(
"Bundled catalog marker is attached to an unknown recipe"
)
resolved_recipe_ingredients = _resolve_recipe_ingredients(db, marked_recipes)
missing_marked_names = catalog_names.difference(marked_by_name)
if update_existing and missing_marked_names:
raise StarterRecipeCatalogError(
"Update requires exactly all "
f"{STARTER_RECIPE_COUNT} existing marker-owned recipe names"
)
recipes_to_update: list[Recipe] = []
for recipe in marked_recipes:
expected = catalog_by_name[recipe.name]
if update_existing:
_assert_bundled_recipe_ownership(recipe)
if not _bundled_recipe_content_matches(
recipe, expected, resolved_recipe_ingredients
):
recipes_to_update.append(recipe)
else:
_assert_bundled_recipe(
recipe,
expected,
resolved_recipe_ingredients,
)
name_collisions = list(
db.execute(
select(Recipe).where(Recipe.name.in_(catalog_names)).with_for_update()
).scalars()
)
if any(recipe.discovery_reason != STARTER_DISCOVERY_REASON for recipe in name_collisions):
raise StarterRecipeCatalogError(
"Catalog recipe name conflicts with an existing non-bundled recipe"
)
canonical_ingredients = {
ingredient.name_lower: ingredient
for ingredient in db.execute(
select(Ingredient).where(Ingredient.name_lower.in_(ingredient_names))
).scalars()
}
new_recipes = (
[]
if update_existing
else [recipe for recipe in recipes if recipe["name"] not in marked_by_name]
)
summary = {
"catalog_version": STARTER_CATALOG_VERSION,
"dry_run": dry_run,
"validated_recipes": len(recipes),
"existing_recipes": len(marked_recipes),
"created_recipes": len(new_recipes),
"updated_recipes": len(recipes_to_update),
"existing_ingredients": len(canonical_ingredients),
"created_ingredients": len(ingredient_names) - len(canonical_ingredients),
}
if dry_run:
return summary
for name_lower, ingredient in ingredient_names.items():
db.execute(
pg_insert(Ingredient.__table__)
.values(
name=ingredient["name"],
name_lower=name_lower,
unit=ingredient["unit"],
aliases=[],
)
.on_conflict_do_nothing(index_elements=[Ingredient.name_lower])
)
canonical_ingredients = {
ingredient.name_lower: ingredient
for ingredient in db.execute(
select(Ingredient).where(Ingredient.name_lower.in_(ingredient_names))
).scalars()
}
if set(canonical_ingredients) != set(ingredient_names):
raise StarterRecipeCatalogError("Canonical ingredient creation failed")
for recipe in recipes_to_update:
_synchronize_bundled_recipe(
recipe,
catalog_by_name[recipe.name],
canonical_ingredients,
)
for recipe in new_recipes:
db.add(
Recipe(
family_profile_id=None,
name=recipe["name"],
description=recipe["description"],
image_url=None,
image_source=None,
prep_time_minutes=recipe["prep_time_minutes"],
cook_time_minutes=recipe["cook_time_minutes"],
servings=recipe["servings"],
cuisine_tags=recipe["cuisine_tags"],
dietary_tags=recipe["dietary_tags"],
protein_type=recipe["protein_type"],
spice_level=recipe["spice_level"],
calories_per_serving=recipe["calories_per_serving"],
ingredients=_catalog_recipe_ingredients(recipe, canonical_ingredients),
side_dishes=[],
instructions=recipe["instructions"],
source_url=None,
scraped_at=None,
is_manually_added=True,
external_source=None,
external_id=None,
discovery_reason=STARTER_DISCOVERY_REASON,
catalog_status=RecipeCatalogStatus.ACTIVE,
catalog_findings=[],
)
)
db.flush()
return summary
def _validate_ingredients(value: Any, recipe_index: int) -> list[dict[str, Any]]:
if not isinstance(value, list) or len(value) < 2:
raise StarterRecipeCatalogError(f"Recipe {recipe_index} needs at least two ingredients")
ingredients: list[dict[str, Any]] = []
for index, ingredient in enumerate(value, 1):
if not isinstance(ingredient, Mapping):
raise StarterRecipeCatalogError(
f"Recipe {recipe_index} ingredient {index} must be an object"
)
_require_exact_keys(
ingredient,
_INGREDIENT_KEYS,
f"Recipe {recipe_index} ingredient {index}",
)
ingredients.append(
{
"name": _text(
ingredient["name"],
f"Recipe {recipe_index} ingredient {index} name",
maximum=200,
),
"quantity": _positive_number(
ingredient["quantity"],
f"Recipe {recipe_index} ingredient {index} quantity",
MAX_INGREDIENT_QUANTITY,
),
"unit": _text(
ingredient["unit"], f"Recipe {recipe_index} ingredient {index} unit", maximum=50
),
}
)
return ingredients
def _validate_instructions(value: Any, recipe_index: int) -> list[str]:
if not isinstance(value, list) or len(value) < 2:
raise StarterRecipeCatalogError(f"Recipe {recipe_index} needs at least two instructions")
instructions = [
_text(instruction, f"Recipe {recipe_index} instruction {index}", maximum=5_000)
for index, instruction in enumerate(value, 1)
]
for instruction in instructions:
if len(instruction) < 20 or len(re.findall(r"[A-Za-z0-9]+", instruction)) < 4:
raise StarterRecipeCatalogError(
f"Recipe {recipe_index} instructions must be substantive"
)
return instructions
def _tags(value: Any, field: str) -> list[str]:
if not isinstance(value, list):
raise StarterRecipeCatalogError(f"{field} must be an array")
return [_text(tag, field, maximum=50) for tag in value]
def _text(value: Any, field: str, *, maximum: int) -> str:
if not isinstance(value, str):
raise StarterRecipeCatalogError(f"{field} must be text")
text = " ".join(value.split())
if not text or len(text) > maximum or _is_placeholder(text):
raise StarterRecipeCatalogError(f"{field} is missing, too long, or placeholder text")
return text
def _positive_int(value: Any, field: str, maximum: int) -> int:
return _bounded_int(value, field, 1, maximum)
def _bounded_int(value: Any, field: str, minimum: int, maximum: int) -> int:
if isinstance(value, bool) or not isinstance(value, int) or not minimum <= value <= maximum:
raise StarterRecipeCatalogError(f"{field} must be an integer between {minimum} and {maximum}")
return value
def _positive_number(value: Any, field: str, maximum: int) -> int | float:
if isinstance(value, bool) or not isinstance(value, (int, float)):
raise StarterRecipeCatalogError(f"{field} must be a number")
if not math.isfinite(value) or not 0 < value <= maximum:
raise StarterRecipeCatalogError(f"{field} must be greater than zero and at most {maximum}")
return value
def _require_exact_keys(
value: Mapping[str, Any], expected: set[str] | frozenset[str], field: str
) -> None:
if set(value) != set(expected):
raise StarterRecipeCatalogError(f"{field} has unexpected or missing fields")
def _canonical_name(value: str) -> str:
return " ".join(value.split()).casefold()
def _is_placeholder(value: str) -> bool:
normalized = re.sub(r"[^a-z0-9]+", " ", value.casefold()).strip()
return normalized in _PLACEHOLDER_TEXT or any(
phrase in normalized for phrase in _PLACEHOLDER_PHRASES
)
def _resolve_recipe_ingredients(
db: Session, recipes: list[Recipe]
) -> dict[str, Ingredient]:
ingredient_ids: set[UUID] = set()
for recipe in recipes:
if not isinstance(recipe.ingredients, list):
continue
for item in recipe.ingredients:
if not isinstance(item, Mapping):
continue
try:
ingredient_ids.add(UUID(str(item.get("ingredient_id"))))
except (TypeError, ValueError, AttributeError):
continue
if not ingredient_ids:
return {}
return {
str(ingredient.id): ingredient
for ingredient in db.execute(
select(Ingredient).where(Ingredient.id.in_(ingredient_ids))
).scalars()
}
def _assert_bundled_recipe(
recipe: Recipe,
expected: Mapping[str, Any],
resolved_ingredients: Mapping[str, Ingredient],
) -> None:
_assert_bundled_recipe_ownership(recipe)
if not _bundled_recipe_content_matches(recipe, expected, resolved_ingredients):
raise StarterRecipeCatalogError("Existing bundled recipe content has drifted")
def _assert_bundled_recipe_ownership(recipe: Recipe) -> None:
if (
recipe.family_profile_id is not None
or not recipe.is_manually_added
or recipe.external_source is not None
or recipe.external_id is not None
or recipe.source_url is not None
or recipe.scraped_at is not None
or recipe.image_url is not None
or recipe.image_source is not None
or recipe.servings_scaled is not None
or recipe.side_dishes != []
or recipe.discovery_reason != STARTER_DISCOVERY_REASON
or recipe.catalog_status != RecipeCatalogStatus.ACTIVE
or recipe.catalog_findings != []
or recipe.catalog_reviewed_at is not None
):
raise StarterRecipeCatalogError(
"Existing bundled recipe does not match the required catalog ownership"
)
def _bundled_recipe_content_matches(
recipe: Recipe,
expected: Mapping[str, Any],
resolved_ingredients: Mapping[str, Ingredient],
) -> bool:
if any(getattr(recipe, field) != expected[field] for field in _CATALOG_SCALAR_FIELDS):
return False
if list(recipe.instructions or []) != expected["instructions"]:
return False
if not isinstance(recipe.ingredients, list) or len(recipe.ingredients) != len(
expected["ingredients"]
):
return False
for actual, expected_ingredient in zip(recipe.ingredients, expected["ingredients"]):
if not isinstance(actual, Mapping) or set(actual) != {
"ingredient_id",
"qty",
"unit",
"notes",
}:
return False
resolved = resolved_ingredients.get(str(actual["ingredient_id"]))
quantity = actual["qty"]
if (
resolved is None
or resolved.name_lower != _canonical_name(expected_ingredient["name"])
or isinstance(quantity, bool)
or not isinstance(quantity, (int, float))
or not math.isfinite(quantity)
or quantity != expected_ingredient["quantity"]
or actual["unit"] != expected_ingredient["unit"]
or actual["notes"] is not None
):
return False
return True
def _synchronize_bundled_recipe(
recipe: Recipe,
expected: Mapping[str, Any],
canonical_ingredients: Mapping[str, Ingredient],
) -> None:
for field in _CATALOG_SCALAR_FIELDS:
value = expected[field]
setattr(recipe, field, list(value) if field.endswith("_tags") else value)
recipe.ingredients = _catalog_recipe_ingredients(expected, canonical_ingredients)
recipe.instructions = list(expected["instructions"])
def _catalog_recipe_ingredients(
recipe: Mapping[str, Any], canonical_ingredients: Mapping[str, Ingredient]
) -> list[dict[str, Any]]:
return [
{
"ingredient_id": str(
canonical_ingredients[_canonical_name(item["name"])].id
),
"qty": item["quantity"],
"unit": item["unit"],
"notes": None,
}
for item in recipe["ingredients"]
]
def _reject_json_constant(value: str) -> None:
raise ValueError(f"Unsupported JSON numeric constant: {value}")
backend/app/services/wikibooks_import.py
+973
View File
@@ -0,0 +1,973 @@
"""Bounded synchronous acquisition and staging for Wikibooks cookbook pages."""
from __future__ import annotations
import copy
import hashlib
import json
import math
import re
import time
from datetime import datetime, timezone
from email.utils import parsedate_to_datetime
from typing import Any, Callable, Mapping
from urllib.parse import quote
import requests
from bs4 import BeautifulSoup, Tag
from sqlalchemy import select
from sqlalchemy.dialects.postgresql import insert as pg_insert
from sqlalchemy.orm import Session
from app.models import RecipeImportStaging, RecipeImportStatus
WIKIBOOKS_API_URL = "https://en.wikibooks.org/w/api.php"
WIKIBOOKS_SOURCE = "wikibooks"
CC_BY_SA_4_LICENSE_NAME = "CC BY-SA 4.0"
CC_BY_SA_4_LICENSE_URL = "https://creativecommons.org/licenses/by-sa/4.0/"
MAX_DISCOVERY_LIMIT = 100
MAX_DISCOVERY_REQUESTS = 10
MAX_TITLE_LENGTH = 200
MAX_PAYLOAD_BYTES = 1_000_000
MAX_HTTP_ATTEMPTS = 3
MAX_POSTGRES_INTEGER = 2_147_483_647
RETRYABLE_HTTP_STATUSES = frozenset({429, 500, 502, 503, 504})
RETRYABLE_MEDIAWIKI_ERRORS = frozenset({"maxlag", "ratelimited"})
_UNICODE_FRACTIONS = {
"¼": 0.25,
"½": 0.5,
"¾": 0.75,
"⅓": 1 / 3,
"⅔": 2 / 3,
"⅛": 0.125,
"⅜": 0.375,
"⅝": 0.625,
"⅞": 0.875,
}
_UNICODE_FRACTION_CHARS = "".join(_UNICODE_FRACTIONS)
_UNIT_PATTERN = re.compile(
r"^(?P<unit>(?:cups?|tablespoons?|tbsp\.?|teaspoons?|tsp\.?|ounces?|oz\.?|"
r"pounds?|lbs?\.?|grams?|g|kilograms?|kgs?|kg|millilit(?:er|re)s?|ml|"
r"lit(?:er|re)s?|l|cloves?|cans?|slices?|packages?|packs?|pinches?))\b\.?",
re.IGNORECASE,
)
_ALTERNATE_METRIC_QUANTITY = re.compile(
r"\s*\(\s*(?:\d+(?:\.\d+)?(?:\s*/\s*\d+)?|[" + _UNICODE_FRACTION_CHARS + r"]|"
r"\d+[" + _UNICODE_FRACTION_CHARS + r"])\s*(?:ml|millilit(?:er|re)s?|l|lit(?:er|re)s?|"
r"g|grams?|kg|kilograms?|oz|ounces?|lb|lbs?|pounds?)\b[^)]*\)",
re.IGNORECASE,
)
_TITLE_GENERIC_WORDS = frozenset(
{
"a",
"an",
"and",
"american",
"baked",
"baking",
"bread",
"boiled",
"braised",
"classic",
"cookbook",
"cooked",
"cookie",
"dessert",
"dish",
"dressing",
"for",
"fried",
"glaze",
"glazed",
"grilled",
"in",
"meal",
"minute",
"of",
"on",
"or",
"out",
"pan",
"pie",
"recipe",
"roasted",
"salad",
"sauce",
"seared",
"steamed",
"stroganoff",
"style",
"the",
"with",
}
)
_TITLE_VARIANT_TOKENS = frozenset({"I", "II", "III", "IV", "V", "VI", "VII", "VIII", "IX", "X"})
_PLACEHOLDER_INSTRUCTIONS = frozenset(
{
"coming soon",
"directions",
"instructions",
"method",
"n a",
"tbd",
"todo",
}
)
class WikibooksImportError(RuntimeError):
"""Raised when a Wikibooks response cannot safely be used."""
class WikibooksResponseError(WikibooksImportError):
"""Raised for malformed or unexpected API response data."""
def canonical_json_sha256(value: Mapping[str, Any]) -> str:
"""Return the stable SHA-256 checksum for a JSON-compatible mapping."""
canonical = json.dumps(value, ensure_ascii=False, separators=(",", ":"), sort_keys=True)
return hashlib.sha256(canonical.encode("utf-8")).hexdigest()
class WikibooksImportService:
"""Acquire and stage text-only recipe candidates from the fixed Wikibooks API."""
def __init__(
self,
user_agent: str,
*,
session: requests.Session | Any | None = None,
timeout: tuple[float, float] = (3.05, 20.0),
min_request_interval: float = 1.0,
max_payload_bytes: int = MAX_PAYLOAD_BYTES,
max_attempts: int = MAX_HTTP_ATTEMPTS,
backoff_seconds: float = 1.0,
max_retry_delay: float = 60.0,
sleep: Callable[[float], None] = time.sleep,
monotonic: Callable[[], float] = time.monotonic,
) -> None:
cleaned_user_agent = user_agent.strip() if isinstance(user_agent, str) else ""
if len(cleaned_user_agent) < 12 or not re.search(r"[A-Za-z]", cleaned_user_agent):
raise ValueError("user_agent must be a meaningful caller-supplied identifier")
if isinstance(max_payload_bytes, bool) or not isinstance(max_payload_bytes, int) or max_payload_bytes <= 0:
raise ValueError("max_payload_bytes must be a positive integer")
if isinstance(max_attempts, bool) or not isinstance(max_attempts, int) or max_attempts < 1:
raise ValueError("max_attempts must be a positive integer")
self._user_agent = cleaned_user_agent
self._session = session or requests.Session()
self._timeout = _timeout_pair(timeout)
self._min_request_interval = _finite_nonnegative(
min_request_interval, "min_request_interval"
)
self._max_payload_bytes = max_payload_bytes
self._max_attempts = max_attempts
self._backoff_seconds = _finite_positive(backoff_seconds, "backoff_seconds")
self._max_retry_delay = _finite_positive(max_retry_delay, "max_retry_delay")
self._sleep = sleep
self._monotonic = monotonic
self._last_request_at: float | None = None
def discover_recipe_titles(self, limit: int = MAX_DISCOVERY_LIMIT) -> list[str]:
"""Return bounded Cookbook page titles from ``Category:Recipes``."""
if isinstance(limit, bool) or not isinstance(limit, int) or limit < 1:
raise ValueError("limit must be a positive integer")
return self._discover_category_recipe_titles(
requested=min(limit, MAX_DISCOVERY_LIMIT),
max_requests=MAX_DISCOVERY_REQUESTS,
)
def _discover_category_recipe_titles(
self, *, requested: int, max_requests: int
) -> list[str]:
titles: list[str] = []
seen_titles: set[str] = set()
continuation: str | None = None
for _ in range(max_requests):
params: dict[str, str | int] = {
"action": "query",
"list": "categorymembers",
"cmtitle": "Category:Recipes",
"cmtype": "page",
"cmlimit": min(requested - len(titles), MAX_DISCOVERY_LIMIT),
"format": "json",
"formatversion": 2,
"maxlag": 5,
}
if continuation:
params["cmcontinue"] = continuation
payload = self._request(params)
query = _mapping(payload.get("query"), "query")
members = query.get("categorymembers")
if not isinstance(members, list):
raise WikibooksResponseError("query.categorymembers must be a list")
for member in members:
member_data = _mapping(member, "query.categorymembers item")
title = member_data.get("title")
if not isinstance(title, str):
raise WikibooksResponseError("query.categorymembers item title must be a string")
if title.startswith("Cookbook:") and title not in seen_titles:
_validate_title(title)
seen_titles.add(title)
titles.append(title)
if len(titles) == requested:
return titles
next_data = payload.get("continue")
if next_data is None:
return titles
next_mapping = _mapping(next_data, "continue")
next_continuation = next_mapping.get("cmcontinue")
if not isinstance(next_continuation, str) or not next_continuation:
raise WikibooksResponseError("continue.cmcontinue must be a non-empty string")
if next_continuation == continuation:
raise WikibooksResponseError("repeated category continuation token")
continuation = next_continuation
return titles
def search_recipe_titles(
self, queries: list[str], limit: int = MAX_DISCOVERY_LIMIT
) -> list[str]:
"""Return bounded Cookbook titles relevant to feedback search queries."""
if isinstance(limit, bool) or not isinstance(limit, int) or limit < 1:
raise ValueError("limit must be a positive integer")
requested = min(limit, MAX_DISCOVERY_LIMIT)
cleaned_queries: list[str] = []
for query in queries:
cleaned = " ".join(query.split()) if isinstance(query, str) else ""
if cleaned and cleaned not in cleaned_queries:
cleaned_queries.append(cleaned[:MAX_TITLE_LENGTH])
if len(cleaned_queries) == MAX_DISCOVERY_REQUESTS:
break
titles: list[str] = []
seen_titles: set[str] = set()
for query in cleaned_queries:
payload = self._request(
{
"action": "query",
"list": "search",
"srsearch": query,
"srnamespace": 102,
"srlimit": requested - len(titles),
"format": "json",
"formatversion": 2,
"maxlag": 5,
}
)
search = _mapping(payload.get("query"), "query").get("search")
if not isinstance(search, list):
raise WikibooksResponseError("query.search must be a list")
for result in search:
title = _mapping(result, "query.search item").get("title")
if not isinstance(title, str):
raise WikibooksResponseError(
"query.search item title must be a string"
)
if title.startswith("Cookbook:") and title not in seen_titles:
_validate_title(title)
seen_titles.add(title)
titles.append(title)
if len(titles) == requested:
return titles
return titles
def fetch_page(self, title: str) -> dict[str, Any]:
"""Fetch one validated ``Cookbook:`` parse object without augmentation."""
_validate_title(title)
payload = self._request(
{
"action": "parse",
"page": title,
"redirects": 1,
"prop": "text|wikitext|sections|revid|displaytitle",
"format": "json",
"formatversion": 2,
"maxlag": 5,
}
)
parse_payload = _mapping(payload.get("parse"), "parse")
_validate_parse_payload(parse_payload)
return parse_payload
def normalize_page(self, parse_payload: Mapping[str, Any]) -> dict[str, Any]:
"""Convert a validated rendered cookbook page into text-only recipe data."""
_validate_parse_payload(parse_payload)
title = str(parse_payload["title"])
soup = BeautifulSoup(str(parse_payload["text"]), "html.parser")
root = soup.select_one(".mw-parser-output") or soup
elements = root.find_all(["h2", "h3", "h4", "h5", "h6", "p", "ul", "ol"])
ingredients_heading = _find_heading(elements, {"ingredients"})
instruction_heading = _find_heading(
elements, {"procedure", "instructions", "method", "directions"}
)
description = _description_before_heading(elements, ingredients_heading)
infobox = _infobox_values(root)
ingredients = [
_normalize_ingredient(item)
for item in _section_list_items(ingredients_heading, {"ul", "ol"})
]
instructions = _section_list_items(instruction_heading, {"ol"})
return {
"name": title.removeprefix("Cookbook:").replace("_", " ").strip(),
"description": description,
"servings": _servings_from_infobox(infobox),
"prep_time_minutes": _time_from_infobox(infobox, ("prep", "preparation")),
"cook_time_minutes": _time_from_infobox(infobox, ("cook", "cooking")),
"baking_time_minutes": _time_from_infobox(infobox, ("bake", "baking")),
"ingredients": ingredients,
"instructions": instructions,
}
def validate_recipe(self, candidate: Mapping[str, Any]) -> dict[str, Any]:
"""Return deterministic structural and cross-field validation results."""
errors: list[dict[str, Any]] = []
warnings: list[dict[str, Any]] = []
servings = candidate.get("servings")
if (
isinstance(servings, bool)
or not isinstance(servings, int)
or servings <= 0
):
errors.append(_issue("invalid_servings", "servings", "Servings must be a positive integer."))
elif servings > MAX_POSTGRES_INTEGER:
errors.append(
_issue(
"servings_out_of_range",
"servings",
"Servings exceed the supported database range.",
)
)
for field in ("prep_time_minutes", "cook_time_minutes", "baking_time_minutes"):
duration = candidate.get(field)
if isinstance(duration, int) and not isinstance(duration, bool) and duration > MAX_POSTGRES_INTEGER:
errors.append(
_issue(
"time_out_of_range",
field,
"Duration exceeds the supported database range.",
)
)
combined_cook_time = (candidate.get("cook_time_minutes") or 0) + (
candidate.get("baking_time_minutes") or 0
)
if (
isinstance(combined_cook_time, int)
and not isinstance(combined_cook_time, bool)
and combined_cook_time > MAX_POSTGRES_INTEGER
):
errors.append(
_issue(
"combined_cook_time_out_of_range",
"times",
"Combined cook and baking time exceed the supported database range.",
)
)
ingredients = candidate.get("ingredients")
if not isinstance(ingredients, list) or len(ingredients) < 2:
errors.append(
_issue("too_few_ingredients", "ingredients", "At least two ingredients are required.")
)
ingredient_items: list[Any] = []
else:
ingredient_items = ingredients
instructions = candidate.get("instructions")
if not isinstance(instructions, list) or len(instructions) < 2:
errors.append(
_issue("too_few_instructions", "instructions", "At least two instructions are required.")
)
instruction_items: list[Any] = []
else:
instruction_items = instructions
for index, ingredient in enumerate(ingredient_items):
quantity = ingredient.get("quantity") if isinstance(ingredient, Mapping) else None
if (
isinstance(quantity, bool)
or not isinstance(quantity, (int, float))
or not math.isfinite(quantity)
or quantity <= 0
):
errors.append(
_issue(
"invalid_ingredient_quantity",
f"ingredients[{index}].quantity",
"Ingredient quantity must be positive.",
)
)
for index, instruction in enumerate(instruction_items):
if not isinstance(instruction, str) or _is_placeholder_instruction(instruction):
errors.append(
_issue(
"placeholder_instruction",
f"instructions[{index}]",
"Instruction is missing or a placeholder.",
)
)
title_name = candidate.get("name")
if isinstance(title_name, str) and ingredient_items:
missing_tokens = _missing_title_ingredient_tokens(title_name, ingredient_items)
if missing_tokens:
errors.append(
_issue(
"title_ingredient_mismatch",
"name",
"Distinctive title food tokens are absent from ingredients.",
missing_tokens=missing_tokens,
)
)
if not isinstance(candidate.get("description"), str) or not candidate["description"].strip():
warnings.append(_issue("missing_description", "description", "Intro description is unavailable."))
if not any(
isinstance(candidate.get(field), int) and candidate[field] > 0
for field in ("prep_time_minutes", "cook_time_minutes", "baking_time_minutes")
):
warnings.append(_issue("missing_time", "times", "No preparation, cook, or baking time is available."))
return {"valid": not errors, "errors": errors, "warnings": warnings}
def stage_page(self, db: Session, title: str) -> RecipeImportStaging:
"""Fetch, normalize, validate, and flush one candidate without committing."""
return self.stage_parse_payload(db, self.fetch_page(title))
def stage_discovered_pages(
self,
db: Session,
limit: int = 5,
queries: list[str] | None = None,
) -> list[RecipeImportStaging]:
"""Discover and stage a bounded relevant or category batch without committing."""
titles = (
self.search_recipe_titles(queries, limit)
if queries
else self.discover_recipe_titles(limit)
)
return [self.stage_page(db, title) for title in titles]
def stage_parse_payload(
self, db: Session, parse_payload: Mapping[str, Any]
) -> RecipeImportStaging:
"""Stage a validated parse object, reusing an existing page revision."""
snapshot = copy.deepcopy(dict(parse_payload))
_validate_parse_payload(snapshot)
external_id = str(snapshot["pageid"])
source_revision = str(snapshot["revid"])
normalized = self.normalize_page(snapshot)
validation_report = self.validate_recipe(normalized)
status = (
RecipeImportStatus.PENDING
if validation_report["valid"]
else RecipeImportStatus.QUARANTINED
)
title = str(snapshot["title"])
values = {
"source": WIKIBOOKS_SOURCE,
"external_id": external_id,
"source_url": _page_url(title),
"source_revision": source_revision,
"license_name": CC_BY_SA_4_LICENSE_NAME,
"license_url": CC_BY_SA_4_LICENSE_URL,
"attribution": f"Wikibooks contributors; revision history: {_history_url(title)}",
"raw_payload": snapshot,
"payload_sha256": canonical_json_sha256(snapshot),
"normalized_recipe": normalized,
"validation_report": validation_report,
"status": status,
"rejection_reason": (
None
if validation_report["valid"]
else "; ".join(error["code"] for error in validation_report["errors"])
),
}
identity = {
"source": WIKIBOOKS_SOURCE,
"external_id": external_id,
"source_revision": source_revision,
}
with db.no_autoflush:
db.execute(
pg_insert(RecipeImportStaging.__table__)
.values(**values)
.on_conflict_do_nothing(
constraint="uq_recipe_import_staging_source_revision"
)
.returning(RecipeImportStaging.id)
)
return db.execute(
select(RecipeImportStaging).filter_by(**identity)
).scalar_one()
def revalidate_staged_imports(
self, db: Session, *, limit: int = 100
) -> dict[str, int]:
"""Recompute unreviewed Wikibooks candidates without committing."""
if isinstance(limit, bool) or not isinstance(limit, int) or not 1 <= limit <= 1000:
raise ValueError("limit must be between 1 and 1000")
candidates = (
db.execute(
select(RecipeImportStaging)
.where(
RecipeImportStaging.source == WIKIBOOKS_SOURCE,
RecipeImportStaging.status.in_(
[RecipeImportStatus.PENDING, RecipeImportStatus.QUARANTINED]
),
RecipeImportStaging.reviewed_at.is_(None),
RecipeImportStaging.reviewed_by.is_(None),
)
.order_by(RecipeImportStaging.acquired_at, RecipeImportStaging.id)
.limit(limit)
.with_for_update(skip_locked=True)
)
.scalars()
.all()
)
counts = {"processed": len(candidates), "pending": 0, "quarantined": 0}
for staged in candidates:
snapshot = copy.deepcopy(staged.raw_payload)
if canonical_json_sha256(snapshot) != staged.payload_sha256:
staged.validation_report = {
"valid": False,
"errors": [
_issue(
"payload_checksum_mismatch",
"raw_payload",
"Raw payload checksum does not match immutable provenance.",
)
],
"warnings": [],
}
staged.status = RecipeImportStatus.QUARANTINED
staged.rejection_reason = "payload_checksum_mismatch"
counts[RecipeImportStatus.QUARANTINED.value] += 1
continue
_validate_parse_payload(snapshot)
normalized = self.normalize_page(snapshot)
validation_report = self.validate_recipe(normalized)
status = (
RecipeImportStatus.PENDING
if validation_report["valid"]
else RecipeImportStatus.QUARANTINED
)
staged.normalized_recipe = normalized
staged.validation_report = validation_report
staged.status = status
staged.rejection_reason = (
None
if validation_report["valid"]
else "; ".join(error["code"] for error in validation_report["errors"])
)
counts[status.value] += 1
db.flush()
return counts
def _request(self, params: Mapping[str, str | int]) -> dict[str, Any]:
for attempt in range(self._max_attempts):
response: Any | None = None
retry_error: WikibooksImportError | None = None
retry_delay: float | None = None
try:
self._wait_for_request_slot()
response = self._session.get(
WIKIBOOKS_API_URL,
params=dict(params),
headers={"Accept": "application/json", "User-Agent": self._user_agent},
timeout=self._timeout,
stream=True,
allow_redirects=False,
)
status_code = getattr(response, "status_code", None)
if isinstance(status_code, bool) or not isinstance(status_code, int):
raise WikibooksResponseError("HTTP response status code is invalid")
if status_code in RETRYABLE_HTTP_STATUSES:
retry_error = WikibooksImportError(f"Wikibooks returned retryable HTTP {status_code}")
retry_delay = _retry_after_seconds(getattr(response, "headers", {}))
elif 300 <= status_code < 400:
raise WikibooksImportError(f"Wikibooks redirect HTTP {status_code} rejected")
elif status_code < 200 or status_code >= 300:
raise WikibooksImportError(f"Wikibooks returned HTTP {status_code}")
else:
payload = self._read_json_response(response)
api_error = payload.get("error")
if api_error is None:
return payload
error_data = _mapping(api_error, "error")
error_code = error_data.get("code")
if not isinstance(error_code, str) or not error_code:
raise WikibooksResponseError("error.code must be a non-empty string")
if error_code.lower() in RETRYABLE_MEDIAWIKI_ERRORS:
retry_error = WikibooksImportError(
f"Wikibooks returned retryable MediaWiki error {error_code}"
)
else:
raise WikibooksImportError(
f"Wikibooks returned MediaWiki error {error_code}"
)
except requests.RequestException as exc:
retry_error = WikibooksImportError(f"Wikibooks request failed: {exc}")
finally:
if response is not None:
response.close()
if retry_error is None:
raise AssertionError("request did not return or produce an error")
if attempt == self._max_attempts - 1:
raise retry_error
delay = retry_delay if retry_delay is not None else self._backoff_seconds * (2**attempt)
self._sleep(min(delay, self._max_retry_delay))
raise AssertionError("unreachable")
def _wait_for_request_slot(self) -> None:
if self._last_request_at is not None:
wait_seconds = self._min_request_interval - (
self._monotonic() - self._last_request_at
)
if wait_seconds > 0:
self._sleep(wait_seconds)
self._last_request_at = self._monotonic()
def _read_json_response(self, response: Any) -> dict[str, Any]:
headers = getattr(response, "headers", {})
if not isinstance(headers, Mapping):
raise WikibooksResponseError("HTTP response headers are invalid")
content_length = headers.get("Content-Length")
if content_length is not None:
try:
declared_size = int(content_length)
except (TypeError, ValueError) as exc:
raise WikibooksResponseError("HTTP Content-Length is invalid") from exc
if declared_size < 0 or declared_size > self._max_payload_bytes:
raise WikibooksResponseError("Wikibooks payload exceeds configured size limit")
body = bytearray()
for chunk in response.iter_content(chunk_size=64 * 1024):
if not isinstance(chunk, (bytes, bytearray)):
raise WikibooksResponseError("HTTP response body chunk is invalid")
body.extend(chunk)
if len(body) > self._max_payload_bytes:
raise WikibooksResponseError("Wikibooks payload exceeds configured size limit")
try:
decoded = json.loads(bytes(body).decode("utf-8"))
except (UnicodeDecodeError, json.JSONDecodeError) as exc:
raise WikibooksResponseError("Wikibooks response is not valid UTF-8 JSON") from exc
return _mapping(decoded, "response")
def _mapping(value: Any, name: str) -> dict[str, Any]:
if not isinstance(value, dict):
raise WikibooksResponseError(f"{name} must be an object")
return value
def _validate_title(title: str) -> None:
if (
not isinstance(title, str)
or not title.startswith("Cookbook:")
or len(title) <= len("Cookbook:")
or len(title) > MAX_TITLE_LENGTH
or any(character in title for character in "\r\n\x00")
):
raise WikibooksResponseError("title must be a sane Cookbook: page title")
def _validate_parse_payload(parse_payload: Mapping[str, Any]) -> None:
if not isinstance(parse_payload, Mapping):
raise WikibooksResponseError("parse must be an object")
_positive_integer(parse_payload.get("pageid"), "parse.pageid")
_positive_integer(parse_payload.get("revid"), "parse.revid")
title = parse_payload.get("title")
_validate_title(title)
for field in ("text", "wikitext", "displaytitle"):
if not isinstance(parse_payload.get(field), str):
raise WikibooksResponseError(f"parse.{field} must be a string")
sections = parse_payload.get("sections")
if not isinstance(sections, list):
raise WikibooksResponseError("parse.sections must be a list")
for section in sections:
section_data = _mapping(section, "parse.sections item")
if not isinstance(section_data.get("line"), str):
raise WikibooksResponseError("parse.sections item line must be a string")
def _positive_integer(value: Any, name: str) -> int:
if isinstance(value, bool) or not isinstance(value, int) or value <= 0:
raise WikibooksResponseError(f"{name} must be a positive integer")
return value
def _finite_positive(value: Any, name: str) -> float:
if (
isinstance(value, bool)
or not isinstance(value, (int, float))
or not math.isfinite(value)
or value <= 0
):
raise ValueError(f"{name} must be finite and positive")
return float(value)
def _finite_nonnegative(value: Any, name: str) -> float:
if (
isinstance(value, bool)
or not isinstance(value, (int, float))
or not math.isfinite(value)
or value < 0
):
raise ValueError(f"{name} must be finite and non-negative")
return float(value)
def _timeout_pair(timeout: Any) -> tuple[float, float]:
if not isinstance(timeout, (tuple, list)) or len(timeout) != 2:
raise ValueError("timeout must be a (connect, read) pair")
return (
_finite_positive(timeout[0], "timeout connect value"),
_finite_positive(timeout[1], "timeout read value"),
)
def _retry_after_seconds(headers: Mapping[str, Any]) -> float | None:
value = headers.get("Retry-After")
if value is None:
return None
try:
seconds = float(value)
return max(0.0, seconds) if math.isfinite(seconds) else None
except (TypeError, ValueError):
try:
retry_at = parsedate_to_datetime(str(value))
except (TypeError, ValueError, IndexError):
return None
if retry_at.tzinfo is None:
retry_at = retry_at.replace(tzinfo=timezone.utc)
seconds = (retry_at - datetime.now(timezone.utc)).total_seconds()
return max(0.0, seconds) if math.isfinite(seconds) else None
def _page_url(title: str) -> str:
return "https://en.wikibooks.org/wiki/" + quote(title.replace(" ", "_"), safe=":")
def _history_url(title: str) -> str:
return "https://en.wikibooks.org/w/index.php?title=" + quote(title, safe="") + "&action=history"
def _text(tag: Tag) -> str:
return re.sub(r"\s+", " ", " ".join(tag.stripped_strings)).strip()
def _heading_text(tag: Tag) -> str:
return re.sub(r"\s*\[edit\]\s*$", "", _text(tag), flags=re.IGNORECASE).strip().lower()
def _find_heading(elements: list[Tag], names: set[str]) -> Tag | None:
for element in elements:
if element.name in {"h2", "h3", "h4", "h5", "h6"} and _heading_text(element) in names:
return element
return None
def _description_before_heading(elements: list[Tag], heading: Tag | None) -> str | None:
if heading is None:
return None
descriptions: list[str] = []
for element in elements:
if element is heading:
break
if element.name == "p" and element.find_parent(["li", "td", "th"]) is None:
text = _text(element)
if text:
descriptions.append(text)
return " ".join(descriptions) or None
def _infobox_values(root: Tag | BeautifulSoup) -> dict[str, str]:
values: dict[str, str] = {}
for table in root.find_all("table"):
classes = " ".join(table.get("class", []))
if "infobox" not in classes.lower():
continue
for row in table.find_all("tr"):
cells = row.find_all(["th", "td"], recursive=False)
if len(cells) < 2:
continue
label = _text(cells[0]).lower()
value = _text(cells[1])
if label and value and label not in values:
values[label] = value
return values
def _servings_from_infobox(values: Mapping[str, str]) -> int | None:
for label, value in values.items():
if any(term in label for term in ("servings", "serves", "yield", "makes")):
match = re.search(r"\b(\d+)\b", value)
if match:
return int(match.group(1))
return None
def _time_from_infobox(values: Mapping[str, str], labels: tuple[str, ...]) -> int | None:
for label, value in values.items():
if any(expected in label for expected in labels):
return _parse_duration(value)
return None
def _parse_duration(value: str) -> int | None:
clock = re.search(r"\b(\d{1,2}):(\d{2})\b", value)
if clock:
hours, minutes = (int(part) for part in clock.groups())
total = hours * 60 + minutes
return total if total > 0 else None
hours = re.search(r"\b(\d+)\s*(?:hours?|hrs?|hr|h)\b", value, re.IGNORECASE)
minutes = re.search(r"\b(\d+)\s*(?:minutes?|mins?|min|m)\b", value, re.IGNORECASE)
total = (int(hours.group(1)) * 60 if hours else 0) + (int(minutes.group(1)) if minutes else 0)
return total if total > 0 else None
def _section_list_items(heading: Tag | None, list_names: set[str]) -> list[str]:
if heading is None:
return []
heading_level = int(heading.name[1])
items: list[str] = []
for element in heading.find_all_next(["h2", "h3", "h4", "h5", "h6", "ul", "ol"]):
if element.name and element.name.startswith("h") and int(element.name[1]) <= heading_level:
break
if element.name not in list_names:
continue
for item in element.find_all("li", recursive=False):
text = _list_item_text(item)
if text:
items.append(text)
return items
def _list_item_text(item: Tag) -> str:
clone = copy.copy(item)
for nested_list in clone.find_all(["ul", "ol"]):
nested_list.decompose()
return _text(clone)
def _normalize_ingredient(source_text: str) -> dict[str, Any]:
quantity, quantity_text, remainder = _parse_leading_quantity(source_text)
unit: str | None = None
name = remainder if quantity_text is not None else source_text
if quantity_text is not None:
unit_match = _UNIT_PATTERN.match(name.lstrip())
if unit_match:
unit = unit_match.group("unit").lower().rstrip(".")
name = name.lstrip()[unit_match.end() :]
name = _ALTERNATE_METRIC_QUANTITY.sub("", name)
name = name.strip(" \t,;:-") or source_text
return {
"quantity": quantity,
"quantity_text": quantity_text,
"unit": unit,
"name": name,
"source_text": source_text,
}
def _parse_leading_quantity(value: str) -> tuple[float | None, str | None, str]:
text = value.strip()
if re.match(r"^\d+(?:\.\d+)?\s*(?:-|–|—)\s*\d", text):
return None, None, text
fraction_prefix = re.match(r"^\d+(?:\.\d+)?(?:\s+\d+)?\s*/\s*(\d*)", text)
if fraction_prefix and (not fraction_prefix.group(1) or int(fraction_prefix.group(1)) == 0):
return None, None, text
mixed = re.match(r"^(\d+(?:\.\d+)?)\s+(\d+\s*/\s*\d+|[" + _UNICODE_FRACTION_CHARS + r"])", text)
if mixed:
fraction = _fraction_value(mixed.group(2))
if fraction is not None:
return float(mixed.group(1)) + fraction, mixed.group(0), text[mixed.end() :]
compact = re.match(r"^(\d+)([" + _UNICODE_FRACTION_CHARS + r"])", text)
if compact:
return int(compact.group(1)) + _UNICODE_FRACTIONS[compact.group(2)], compact.group(0), text[compact.end() :]
ascii_fraction = re.match(r"^(\d+)\s*/\s*(\d+)", text)
if ascii_fraction:
denominator = int(ascii_fraction.group(2))
if denominator:
return (
int(ascii_fraction.group(1)) / denominator,
ascii_fraction.group(0),
text[ascii_fraction.end() :],
)
number = re.match(r"^\d+(?:\.\d+)?", text)
if number:
return float(number.group(0)), number.group(0), text[number.end() :]
unicode_fraction = re.match(r"^(?:[" + _UNICODE_FRACTION_CHARS + r"])", text)
if unicode_fraction:
token = unicode_fraction.group(0)
return _UNICODE_FRACTIONS[token], token, text[unicode_fraction.end() :]
return None, None, text
def _fraction_value(value: str) -> float | None:
if value in _UNICODE_FRACTIONS:
return _UNICODE_FRACTIONS[value]
numerator, denominator = (int(part.strip()) for part in value.split("/", 1))
return numerator / denominator if denominator else None
def _issue(code: str, field: str, message: str, **extra: Any) -> dict[str, Any]:
return {"code": code, "field": field, "message": message, **extra}
def _is_placeholder_instruction(instruction: str) -> bool:
normalized = re.sub(r"[^a-z0-9]+", " ", instruction.lower()).strip()
return normalized in _PLACEHOLDER_INSTRUCTIONS or bool(
re.fullmatch(r"(?:step )?\d+", normalized)
)
def _missing_title_ingredient_tokens(title: str, ingredients: list[Any]) -> list[str]:
title_tokens: list[str] = []
for raw_token in re.findall(r"[A-Za-z]+", title):
if raw_token in _TITLE_VARIANT_TOKENS:
continue
token = _singular_token(raw_token.lower())
if len(token) > 1 and token not in _TITLE_GENERIC_WORDS:
title_tokens.append(token)
ingredient_tokens: set[str] = set()
for ingredient in ingredients:
if isinstance(ingredient, Mapping) and isinstance(ingredient.get("name"), str):
ingredient_tokens.update(
_singular_token(token)
for token in re.findall(r"[a-z]+", ingredient["name"].lower())
)
return [token for token in dict.fromkeys(title_tokens) if token not in ingredient_tokens]
def _singular_token(token: str) -> str:
if token == "pies":
return "pie"
if token == "cookies":
return "cookie"
if token.endswith("ies") and len(token) > 3:
return token[:-3] + "y"
if token.endswith(("ches", "shes", "sses", "xes", "zes")) and len(token) > 3:
return token[:-2]
if token.endswith("s") and not token.endswith("ss") and len(token) > 2:
return token[:-1]
return token
backend/scripts/audit_recipe_catalog.py
+42
View File
@@ -0,0 +1,42 @@
#!/usr/bin/env python3
"""Report or apply deterministic recipe catalog quarantine decisions."""
from __future__ import annotations
import argparse
import json
import os
import sys
_ = sys.path.insert(0, os.path.join(os.path.dirname(__file__), ".."))
from app.database import SessionLocal
from app.services.recipe_catalog_audit import audit_recipe_catalog
def main() -> int:
parser = argparse.ArgumentParser(
description="Audit recipe provenance and semantic quality"
)
parser.add_argument(
"--apply",
action="store_true",
help="Persist active/quarantined status and findings atomically",
)
args = parser.parse_args()
db = SessionLocal()
try:
report = audit_recipe_catalog(db, apply=args.apply)
if args.apply:
db.commit()
print(json.dumps(report, indent=2, sort_keys=True))
return 0
except Exception:
db.rollback()
raise
finally:
db.close()
if __name__ == "__main__":
raise SystemExit(main())
backend/scripts/load_starter_recipes.py
+55
View File
@@ -0,0 +1,55 @@
#!/usr/bin/env python3
"""Load the bundled starter recipe catalog as one atomic operation."""
from __future__ import annotations
import argparse
import json
import os
import sys
from typing import Sequence
_ = sys.path.insert(0, os.path.join(os.path.dirname(__file__), ".."))
from app.database import SessionLocal
from app.services.starter_recipe_catalog import (
StarterRecipeCatalogError,
load_bundled_starter_catalog,
)
def main(argv: Sequence[str] | None = None) -> int:
parser = argparse.ArgumentParser(description="Load the bundled starter recipe catalog")
parser.add_argument(
"--dry-run",
action="store_true",
help="Validate the catalog and report planned inserts or updates without writing",
)
parser.add_argument(
"--update-existing",
action="store_true",
help="Atomically synchronize all existing bundled recipes in place",
)
args = parser.parse_args(argv)
db = SessionLocal()
try:
summary = load_bundled_starter_catalog(
db,
dry_run=args.dry_run,
update_existing=args.update_existing,
)
print(json.dumps(summary, sort_keys=True))
return 0
except StarterRecipeCatalogError as exc:
db.rollback()
print(json.dumps({"dry_run": args.dry_run, "error": str(exc)}, sort_keys=True))
return 1
except Exception:
db.rollback()
raise
finally:
db.close()
if __name__ == "__main__":
raise SystemExit(main())
backend/static/images/baked_ziti_with_spinach_a4eda6ba.png
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 1.5 MiB

backend/static/images/beefy_stuffed_bell_peppers_4e83898d.png
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 1.4 MiB

backend/static/images/red_bean_rice_pot_ef59586e.png
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 1.5 MiB

backend/tests/fixtures/wikibooks/apple_glaze_missing_apple_parse.json
Vendored
+14
View File
@@ -0,0 +1,14 @@
{
"parse": {
"title": "Cookbook:Pan-Seared Pork Chops with Apple Glaze",
"pageid": 902,
"revid": 5002,
"displaytitle": "Pan-Seared Pork Chops with Apple Glaze",
"wikitext": "Recipe text",
"sections": [
{"line": "Ingredients"},
{"line": "Directions"}
],
"text": "<div class=\"mw-parser-output\"><table class=\"infobox\"><tr><th>Servings</th><td>4</td></tr><tr><th>Cook time</th><td>20 minutes</td></tr></table><p>A deliberately incomplete apple glaze recipe.</p><h2><span id=\"Ingredients\">Ingredients</span></h2><ul><li>2 pork chops</li><li>1 tbsp butter</li><li>2 cloves garlic</li><li>1 cup broth</li></ul><h2><span id=\"Directions\">Directions</span></h2><ol><li>Brown the pork chops.</li><li>Simmer with butter, garlic, and broth.</li></ol></div>"
}
}
backend/tests/fixtures/wikibooks/apple_glazed_pork_chops_parse.json
Vendored
+14
View File
@@ -0,0 +1,14 @@
{
"parse": {
"title": "Cookbook:Apple Glazed Pork Chops",
"pageid": 901,
"revid": 5001,
"displaytitle": "Apple Glazed Pork Chops",
"wikitext": "Recipe text",
"sections": [
{"line": "Ingredients"},
{"line": "Procedure"}
],
"text": "<div class=\"mw-parser-output\"><table class=\"infobox\"><tr><th>Servings</th><td>4</td></tr><tr><th>Prep time</th><td>15 minutes</td></tr><tr><th>Cook time</th><td>20 minutes</td></tr><tr><th>Baking time</th><td>10 minutes</td></tr></table><p>Sweet apples balance savory pork chops.</p><h2><span id=\"Ingredients\">Ingredients</span></h2><ul><li>1 1/2 cups (360 ml) apple cider</li><li>½ tbsp butter</li></ul><h3>Seasoning</h3><ul><li>1¼ tsp salt</li><li>2 pork chops</li></ul><h2><span id=\"Procedure\">Procedure</span></h2><ol><li>Brown the pork chops in butter.</li></ol><h3>Finish</h3><ol><li>Add cider and cook until glazed.</li></ol><h2><span id=\"Notes\">Notes</span></h2><p>Do not import this note.</p></div>"
}
}
backend/tests/fixtures/wikibooks/apple_pie_i_live_parse.json
Vendored
+122
View File
File diff suppressed because one or more lines are too long
backend/tests/test_auth.py
+16
View File
@@ -12,6 +12,7 @@ from __future__ import annotations
import os
import pytest
from starlette.requests import Request
# Configure auth secrets BEFORE app import. conftest.py runs first and sets
@@ -39,6 +40,21 @@ def _reload_settings(monkeypatch):
# ---------------------------------------------------------------------------
# Admin bearer token
# ---------------------------------------------------------------------------
def test_admin_principal_defaults_when_email_is_unset(monkeypatch):
from app import security
monkeypatch.setattr(security.settings, "ADMIN_TOKEN", "test-admin-token")
monkeypatch.setattr(security.settings, "ADMIN_EMAIL", None)
request = Request(
{
"type": "http",
"headers": [(b"authorization", b"Bearer test-admin-token")],
}
)
assert security.require_admin(request) == "admin"
@pytest.mark.requires_postgres
def test_admin_requires_token(client):
"""No bearer → 401. Valid bearer → not 401 (handler runs)."""
backend/tests/test_image_generation.py
+42
View File
@@ -0,0 +1,42 @@
import base64
from types import SimpleNamespace
from unittest.mock import Mock
from uuid import UUID
from app.services import image_generation
def test_generate_image_persists_local_url_and_openai_provenance(tmp_path, monkeypatch):
recipe = SimpleNamespace(
id=UUID("12345678-1234-5678-1234-567812345678"),
name="Spiced Chickpea Bowl",
ingredients=[{"name": "chickpeas"}],
cuisine_tags=["Mediterranean"],
image_url=None,
image_source=None,
)
db = Mock()
image_bytes = b"mock-png-bytes"
response = Mock(status_code=200)
response.json.return_value = {
"data": [{"b64_json": base64.b64encode(image_bytes).decode()}]
}
post = Mock(return_value=response)
monkeypatch.setattr(image_generation.settings, "AI_IMAGE_ENABLED", True)
monkeypatch.setattr(image_generation, "_openai_api_key", lambda: "test-key")
monkeypatch.setattr(image_generation, "STATIC_IMAGES_DIR", tmp_path)
monkeypatch.setattr(image_generation.httpx, "post", post)
monkeypatch.setenv("AI_IMAGE_MODEL", "gpt-image-1")
image_url = image_generation.generate_image_for_recipe(recipe, db=db)
assert image_url is not None
assert image_url.startswith("/static/images/")
assert (tmp_path / image_url.removeprefix("/static/images/")).read_bytes() == image_bytes
assert recipe.image_url == image_url
db.add.assert_called_once_with(recipe)
db.commit.assert_called_once_with()
assert recipe.image_source
assert "openai" in recipe.image_source.lower()
assert "gpt-image-1" in recipe.image_source.lower()
backend/tests/test_llm_plan.py
+39
View File
@@ -0,0 +1,39 @@
import uuid
from app.api.llm_plan import _validate_picks
def test_validate_picks_accepts_only_dinners():
recipe_id = str(uuid.uuid4())
picks = _validate_picks(
[
{"day_of_week": 1, "meal_type": "breakfast", "recipe_id": recipe_id},
{"day_of_week": 1, "meal_type": "lunch", "recipe_id": recipe_id},
{"day_of_week": 1, "meal_type": "dinner", "recipe_id": recipe_id},
],
{recipe_id},
)
assert [(pick.day_of_week, pick.meal_type) for pick in picks] == [(1, "dinner")]
def test_validate_picks_limits_schedule_to_three_unique_dinners():
recipe_ids = [str(uuid.uuid4()) for _ in range(5)]
picks = _validate_picks(
[
{"day_of_week": 1, "meal_type": "dinner", "recipe_id": recipe_ids[0]},
{"day_of_week": 2, "meal_type": "dinner", "recipe_id": recipe_ids[1]},
{"day_of_week": 3, "meal_type": "dinner", "recipe_id": recipe_ids[2]},
{"day_of_week": 3, "meal_type": "dinner", "recipe_id": recipe_ids[3]},
{"day_of_week": 5, "meal_type": "dinner", "recipe_id": recipe_ids[4]},
],
set(recipe_ids),
)
assert [(pick.day_of_week, pick.recipe_id) for pick in picks] == [
(1, recipe_ids[0]),
(3, recipe_ids[2]),
(5, recipe_ids[4]),
]
backend/tests/test_load_starter_recipes.py
+90
View File
@@ -0,0 +1,90 @@
"""Direct CLI coverage for the bundled starter recipe loader."""
from __future__ import annotations
import importlib.util
import json
from pathlib import Path
import pytest
from app.services.starter_recipe_catalog import StarterRecipeCatalogError
SCRIPT_PATH = Path(__file__).resolve().parent.parent / "scripts" / "load_starter_recipes.py"
class FakeSession:
def __init__(self):
self.rollback_count = 0
self.close_count = 0
def rollback(self):
self.rollback_count += 1
def close(self):
self.close_count += 1
@pytest.fixture()
def loader_cli():
spec = importlib.util.spec_from_file_location("load_starter_recipes", SCRIPT_PATH)
assert spec and spec.loader
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
return module
def test_cli_forwards_update_existing_and_writes_json(loader_cli, monkeypatch, capsys):
db = FakeSession()
calls = []
summary = {"catalog_version": 2, "updated_recipes": 50}
monkeypatch.setattr(loader_cli, "SessionLocal", lambda: db)
monkeypatch.setattr(
loader_cli,
"load_bundled_starter_catalog",
lambda session, **kwargs: calls.append((session, kwargs)) or summary,
)
assert loader_cli.main(["--update-existing"]) == 0
assert calls == [(db, {"dry_run": False, "update_existing": True})]
assert json.loads(capsys.readouterr().out) == summary
assert db.rollback_count == 0
assert db.close_count == 1
def test_cli_reports_catalog_errors_as_json_and_closes_rolled_back_session(
loader_cli, monkeypatch, capsys
):
db = FakeSession()
monkeypatch.setattr(loader_cli, "SessionLocal", lambda: db)
def fail_catalog_load(session, **kwargs):
raise StarterRecipeCatalogError("catalog validation failed")
monkeypatch.setattr(loader_cli, "load_bundled_starter_catalog", fail_catalog_load)
assert loader_cli.main(["--dry-run"]) == 1
assert json.loads(capsys.readouterr().out) == {
"dry_run": True,
"error": "catalog validation failed",
}
assert db.rollback_count == 1
assert db.close_count == 1
def test_cli_rolls_back_and_closes_session_for_database_failures(loader_cli, monkeypatch):
db = FakeSession()
monkeypatch.setattr(loader_cli, "SessionLocal", lambda: db)
def fail_catalog_load(session, **kwargs):
raise RuntimeError("database write failed")
monkeypatch.setattr(loader_cli, "load_bundled_starter_catalog", fail_catalog_load)
with pytest.raises(RuntimeError, match="database write failed"):
loader_cli.main([])
assert db.rollback_count == 1
assert db.close_count == 1
backend/tests/test_meal_plan_generate_api.py
+87 -1
View File
@@ -1,4 +1,4 @@
from datetime import datetime, timezone
from datetime import date, datetime, timezone
from decimal import Decimal
from uuid import uuid4
@@ -70,6 +70,92 @@ def test_generate_endpoint_requires_admin_token(client, family_with_groceries):
assert r.status_code == 401
def test_public_generate_uses_session_family_and_three_dinner_schedule(
client, family_with_groceries
):
from app.security import SESSION_COOKIE, issue_session
client.cookies.set(
SESSION_COOKIE,
issue_session(str(family_with_groceries.id)),
)
r = client.post(
"/api/meal-plans/generate",
json={"week_start_date": "2026-06-01"},
)
assert r.status_code == 201, r.text
data = r.json()
assert [item["day_of_week"] for item in data["items"]] == [1, 3, 5]
def test_public_generate_requires_session(client):
r = client.post(
"/api/meal-plans/generate",
json={"week_start_date": "2026-06-08"},
)
assert r.status_code == 401
def test_public_generate_preserves_populated_plan(client, db, family_with_groceries):
from app.models import MealPlan, MealPlanItem
from app.security import SESSION_COOKIE, issue_session
client.cookies.set(
SESSION_COOKIE,
issue_session(str(family_with_groceries.id)),
)
payload = {"week_start_date": "2026-06-15"}
first = client.post("/api/meal-plans/generate", json=payload)
assert first.status_code == 201, first.text
plan_id = first.json()["meal_plan_id"]
item_ids = {
item_id
for (item_id,) in db.query(MealPlanItem.id).filter(
MealPlanItem.meal_plan_id == plan_id
).all()
}
second = client.post("/api/meal-plans/generate", json=payload)
assert second.status_code == 409
assert db.query(MealPlan).filter(MealPlan.id == plan_id).first() is not None
assert {
item_id
for (item_id,) in db.query(MealPlanItem.id).filter(
MealPlanItem.meal_plan_id == plan_id
).all()
} == item_ids
def test_public_generate_replaces_only_empty_plan(client, db, family_with_groceries):
from app.models import MealPlan
from app.security import SESSION_COOKIE, issue_session
empty_plan = MealPlan(
family_profile_id=family_with_groceries.id,
week_start_date=date(2026, 6, 22),
status="draft",
)
db.add(empty_plan)
db.flush()
empty_plan_id = empty_plan.id
client.cookies.set(
SESSION_COOKIE,
issue_session(str(family_with_groceries.id)),
)
response = client.post(
"/api/meal-plans/generate",
json={"week_start_date": "2026-06-22"},
)
assert response.status_code == 201, response.text
assert response.json()["meal_plan_id"] != str(empty_plan_id)
assert db.query(MealPlan).filter(MealPlan.id == empty_plan_id).first() is None
def test_generate_endpoint_returns_404_for_unknown_family(client):
body = {
"family_profile_id": str(uuid4()),
backend/tests/test_orchestrator.py
+246
View File
@@ -11,6 +11,37 @@ pytestmark = pytest.mark.requires_postgres
WEEK = date(2026, 5, 8) # a Friday
def _add_quarantined_item(db, meal_plan, *, name, approval_status, day_of_week):
from app.models import (
MealPlanItem,
MealType,
Recipe,
RecipeCatalogStatus,
)
recipe = Recipe(
id=uuid4(),
name=name,
servings=4,
ingredients=[{"name": "legacy ingredient", "qty": "1", "unit": "cup"}],
instructions=["Do not include this legacy recipe."],
catalog_status=RecipeCatalogStatus.QUARANTINED,
)
db.add(recipe)
db.flush()
item = MealPlanItem(
id=uuid4(),
meal_plan_id=meal_plan.id,
recipe_id=recipe.id,
day_of_week=day_of_week,
meal_type=MealType.DINNER,
approval_status=approval_status,
)
db.add(item)
db.flush()
return item
@pytest.fixture()
def family(db):
from app.models import FamilyProfile
@@ -339,6 +370,38 @@ def test_step_email_escapes_recipe_name(
assert "&lt;script&gt;" in sent[0]["html"]
def test_step_email_suppresses_quarantined_recipes(
db, weekly_run_generated, meal_plan, pending_item, member, monkeypatch
):
from app.models import MealPlanItemStatus
_add_quarantined_item(
db,
meal_plan,
name="Legacy Mango Pork",
approval_status=MealPlanItemStatus.pending,
day_of_week=6,
)
sent = []
class FakeBackend:
def send(self, **kwargs):
sent.append(kwargs)
monkeypatch.setattr(
"app.services.orchestrator.steps.get_email_backend",
lambda: FakeBackend(),
)
from app.services.orchestrator.steps import step_email
step_email(weekly_run_generated, db)
assert len(sent) == 1
assert "Test Pasta" in sent[0]["html"]
assert "Legacy Mango Pork" not in sent[0]["html"]
assert "legacy ingredient" not in sent[0]["html"]
# ── step_reminder ──────────────────────────────────────────────────────────
@@ -447,6 +510,37 @@ def test_step_reminder_no_sends_when_all_voted(
assert weekly_run_emailed.reminded_at is not None
def test_step_reminder_suppresses_quarantined_recipes(
db, weekly_run_emailed, meal_plan, pending_item, member, monkeypatch
):
from app.models import MealPlanItemStatus
_add_quarantined_item(
db,
meal_plan,
name="Legacy Mango Pork",
approval_status=MealPlanItemStatus.pending,
day_of_week=6,
)
sent = []
class FakeBackend:
def send(self, **kwargs):
sent.append(kwargs)
monkeypatch.setattr(
"app.services.orchestrator.steps.get_email_backend",
lambda: FakeBackend(),
)
from app.services.orchestrator.steps import step_reminder
step_reminder(weekly_run_emailed, db)
assert len(sent) == 1
assert "Test Pasta" in sent[0]["html"]
assert "Legacy Mango Pork" not in sent[0]["html"]
# ---------------------------------------------------------------------------
# step_deadline tests
# ---------------------------------------------------------------------------
@@ -573,6 +667,158 @@ def test_step_finalize_no_approved_sends_empty_message(db, weekly_run_deadline_p
assert "No meals were approved" in sent[0]["html"]
def test_step_finalize_suppresses_quarantined_recipes(
db, weekly_run_deadline_passed, meal_plan, approved_item, member, monkeypatch
):
from app.models import MealPlanItemStatus
_add_quarantined_item(
db,
meal_plan,
name="Legacy Mango Pork",
approval_status=MealPlanItemStatus.approved,
day_of_week=7,
)
sent = []
class FakeBackend:
def send(self, **kwargs):
sent.append(kwargs)
monkeypatch.setattr(
"app.services.orchestrator.steps.get_email_backend",
lambda: FakeBackend(),
)
from app.services.orchestrator.steps import step_finalize
step_finalize(weekly_run_deadline_passed, db)
assert len(sent) == 1
assert "Approved Chicken" in sent[0]["html"]
assert "Legacy Mango Pork" not in sent[0]["html"]
assert "legacy ingredient" not in sent[0]["html"]
def test_step_finalize_stages_candidates_without_publishing(
db, weekly_run_deadline_passed, meal_plan, member, monkeypatch
):
from types import SimpleNamespace
from app.models import Recipe, RecipeImportStaging, RecipeImportStatus
class FakeBackend:
def send(self, **kwargs):
pass
analysis = SimpleNamespace(discovery_queries=["apple dinner"], confidence=0.9)
analysis.to_dict = lambda: {
"discovery_queries": analysis.discovery_queries,
"confidence": analysis.confidence,
}
monkeypatch.setattr(
"app.services.orchestrator.steps.get_email_backend", lambda: FakeBackend()
)
monkeypatch.setattr(
"app.services.orchestrator.steps.FeedbackAnalyzer.analyze",
lambda self, database, family_id: analysis,
)
external_id = str(uuid4())
def stage(service, database, limit, queries):
assert queries == ["apple dinner"]
row = RecipeImportStaging(
source="wikibooks",
external_id=external_id,
source_url="https://en.wikibooks.org/wiki/Cookbook:Finalize_Test",
source_revision=str(uuid4()),
license_name="CC BY-SA 4.0",
license_url="https://creativecommons.org/licenses/by-sa/4.0/",
attribution="Wikibooks contributors",
raw_payload={"title": "Cookbook:Finalize Test"},
payload_sha256="a" * 64,
normalized_recipe={"name": "Finalize Test"},
validation_report={"valid": True, "errors": [], "warnings": []},
status=RecipeImportStatus.PENDING,
)
database.add(row)
database.flush()
return [row]
monkeypatch.setattr(
"app.services.orchestrator.steps.WikibooksImportService.stage_discovered_pages",
stage,
)
before = db.query(Recipe).count()
from app.services.orchestrator.steps import step_finalize
step_finalize(weekly_run_deadline_passed, db)
assert weekly_run_deadline_passed.status == "completed"
assert (
db.query(RecipeImportStaging).filter_by(external_id=external_id).count() == 1
)
assert db.query(Recipe).count() == before
def test_step_finalize_rolls_back_partial_staging_failure(
db, weekly_run_deadline_passed, meal_plan, member, monkeypatch
):
from types import SimpleNamespace
from app.models import RecipeImportStaging, RecipeImportStatus
class FakeBackend:
def send(self, **kwargs):
pass
analysis = SimpleNamespace(discovery_queries=["apple dinner"], confidence=0.9)
analysis.to_dict = lambda: {
"discovery_queries": analysis.discovery_queries,
"confidence": analysis.confidence,
}
monkeypatch.setattr(
"app.services.orchestrator.steps.get_email_backend", lambda: FakeBackend()
)
monkeypatch.setattr(
"app.services.orchestrator.steps.FeedbackAnalyzer.analyze",
lambda self, database, family_id: analysis,
)
external_id = str(uuid4())
def stage_then_fail(service, database, limit, queries):
assert queries == ["apple dinner"]
database.add(
RecipeImportStaging(
source="wikibooks",
external_id=external_id,
source_url="https://en.wikibooks.org/wiki/Cookbook:Partial_Test",
source_revision=str(uuid4()),
license_name="CC BY-SA 4.0",
license_url="https://creativecommons.org/licenses/by-sa/4.0/",
attribution="Wikibooks contributors",
raw_payload={"title": "Cookbook:Partial Test"},
payload_sha256="b" * 64,
normalized_recipe={"name": "Partial Test"},
validation_report={"valid": True, "errors": [], "warnings": []},
status=RecipeImportStatus.PENDING,
)
)
database.flush()
raise RuntimeError("upstream failed after first page")
monkeypatch.setattr(
"app.services.orchestrator.steps.WikibooksImportService.stage_discovered_pages",
stage_then_fail,
)
from app.services.orchestrator.steps import step_finalize
step_finalize(weekly_run_deadline_passed, db)
assert weekly_run_deadline_passed.status == "completed"
assert (
db.query(RecipeImportStaging).filter_by(external_id=external_id).count() == 0
)
# ---------------------------------------------------------------------------
# runner tests
# ---------------------------------------------------------------------------
backend/tests/test_planner_generate.py
+90
View File
@@ -87,3 +87,93 @@ def test_generate_meal_plan_against_seeded_data():
finally:
cleanup.close()
work.close()
def test_generate_meal_plan_uses_three_active_calorie_eligible_recipes(db):
from app.models import (
FamilyProfile,
MealPlanItem,
MealType,
Recipe,
RecipeCatalogStatus,
)
from app.services.planner.generate import generate_meal_plan
db.query(Recipe).update(
{Recipe.catalog_status: RecipeCatalogStatus.QUARANTINED},
synchronize_session=False,
)
family = FamilyProfile(
id=uuid4(),
name="Three Dinner Family",
household_size=2,
adult_count=2,
child_count=0,
calorie_target=400,
planner_config={"set_size": 7},
)
db.add(family)
eligible = [
Recipe(
id=uuid4(),
name=f"Lower Calorie Dinner {index}",
servings=4,
calories_per_serving=calories,
prep_time_minutes=10,
cook_time_minutes=20,
ingredients=[],
instructions=["Cook."],
catalog_status=RecipeCatalogStatus.ACTIVE,
)
for index, calories in enumerate((360, 400, 440), start=1)
]
high_calorie = Recipe(
id=uuid4(),
name="Old High Calorie Dinner",
servings=4,
calories_per_serving=700,
prep_time_minutes=10,
cook_time_minutes=20,
ingredients=[],
instructions=["Cook."],
catalog_status=RecipeCatalogStatus.ACTIVE,
)
quarantined = Recipe(
id=uuid4(),
name="Quarantined Legacy Dinner",
servings=4,
calories_per_serving=400,
prep_time_minutes=10,
cook_time_minutes=20,
ingredients=[],
instructions=["Cook."],
catalog_status=RecipeCatalogStatus.QUARANTINED,
)
db.add_all([*eligible, high_calorie, quarantined])
db.flush()
result = generate_meal_plan(
db,
family_id=family.id,
week_start_date=date(2026, 8, 31),
today=date(2026, 8, 31),
)
items = (
db.query(MealPlanItem)
.filter(MealPlanItem.meal_plan_id == result.meal_plan_id)
.order_by(MealPlanItem.day_of_week)
.all()
)
assert {selected.recipe_id for selected in result.selected} == {
recipe.id for recipe in eligible
}
assert [(item.day_of_week, item.meal_type) for item in items] == [
(1, MealType.DINNER),
(3, MealType.DINNER),
(5, MealType.DINNER),
]
assert high_calorie.id not in {item.recipe_id for item in items}
assert quarantined.id not in {item.recipe_id for item in items}
assert result.rejected_summary["calories"] == 1
backend/tests/test_recipe_catalog_audit.py
+400
View File
@@ -0,0 +1,400 @@
"""Catalog quarantine migration, audit, and visibility tests."""
from __future__ import annotations
import importlib.util
import os
import subprocess
from datetime import date, datetime, timezone
from pathlib import Path
from urllib.parse import urlparse
from uuid import uuid4
import pytest
from app.models import (
Ingredient,
MealPlan,
Recipe,
RecipeCatalogStatus,
RecipeImportStaging,
RecipeImportStatus,
)
from app.services.recipe_catalog_audit import audit_recipe_catalog
pytestmark = pytest.mark.requires_postgres
BACKEND_ROOT = Path(__file__).resolve().parent.parent
MIGRATION_PATH = BACKEND_ROOT / "alembic" / "versions" / "0018_recipe_catalog_quarantine.py"
def _ingredient(db, name: str) -> Ingredient:
row = Ingredient(name=name, name_lower=name.casefold(), aliases=[])
db.add(row)
db.flush()
return row
def _recipe(name: str, ingredients: list[Ingredient], **overrides) -> Recipe:
values = {
"name": name,
"description": "A complete test recipe.",
"prep_time_minutes": 10,
"cook_time_minutes": 20,
"servings": 4,
"ingredients": [
{"ingredient_id": str(item.id), "qty": 1.0, "unit": "cup"}
for item in ingredients
],
"instructions": ["Prepare the ingredients.", "Cook until done."],
"is_manually_added": True,
}
values.update(overrides)
return Recipe(**values)
def _approved_link(recipe: Recipe, **overrides) -> RecipeImportStaging:
values = {
"source": "wikibooks",
"external_id": recipe.external_id,
"source_url": recipe.source_url,
"source_revision": str(uuid4()),
"license_name": "CC BY-SA 4.0",
"license_url": "https://creativecommons.org/licenses/by-sa/4.0/",
"attribution": "Wikibooks contributors",
"raw_payload": {"title": recipe.name},
"payload_sha256": "a" * 64,
"normalized_recipe": {"name": recipe.name},
"validation_report": {"valid": True, "errors": [], "warnings": []},
"status": RecipeImportStatus.APPROVED,
"recipe_id": recipe.id,
"reviewed_at": datetime(2026, 8, 14, tzinfo=timezone.utc),
"reviewed_by": "test-admin",
}
values.update(overrides)
return RecipeImportStaging(**values)
def test_recipe_catalog_model_and_migration_chain():
table = Recipe.__table__
assert table.c.catalog_status.type.enums == ["active", "quarantined"]
assert table.c.catalog_status.default.arg is RecipeCatalogStatus.ACTIVE
assert table.c.catalog_status.server_default.arg == "active"
assert table.c.catalog_findings.nullable is False
spec = importlib.util.spec_from_file_location("catalog_quarantine", MIGRATION_PATH)
assert spec and spec.loader
migration = importlib.util.module_from_spec(spec)
spec.loader.exec_module(migration)
assert migration.revision == "0018"
assert migration.down_revision == "0017"
def test_recipe_catalog_alembic_round_trip_from_0017():
dsn = os.environ.get("TEST_DATABASE_URL")
if not dsn:
pytest.skip("TEST_DATABASE_URL is required to avoid modifying a non-test database")
database_name = urlparse(dsn).path.rsplit("/", 1)[-1]
if not database_name.endswith("_test"):
pytest.skip("TEST_DATABASE_URL must target a database ending in '_test'")
env = os.environ.copy()
env["DATABASE_URL"] = dsn
def alembic(*args: str) -> subprocess.CompletedProcess[str]:
return subprocess.run(
["alembic", *args],
cwd=str(BACKEND_ROOT),
env=env,
capture_output=True,
text=True,
)
try:
at_head = alembic("upgrade", "head")
assert at_head.returncode == 0, at_head.stderr
at_0017 = alembic("downgrade", "0017")
assert at_0017.returncode == 0, at_0017.stderr
at_0018 = alembic("upgrade", "0018")
assert at_0018.returncode == 0, at_0018.stderr
back_to_0017 = alembic("downgrade", "0017")
assert back_to_0017.returncode == 0, back_to_0017.stderr
back_to_0018 = alembic("upgrade", "0018")
assert back_to_0018.returncode == 0, back_to_0018.stderr
finally:
restore_head = alembic("upgrade", "head")
assert restore_head.returncode == 0, restore_head.stderr
def test_audit_reports_without_mutation_and_applies_quarantine(db):
pork = _ingredient(db, f"Audit Pork {uuid4()}")
salt = _ingredient(db, f"Audit Salt {uuid4()}")
manual = _recipe("Audit Pork Salt", [pork, salt])
unsafe = _recipe(
"Audit Apple Pork Glaze",
[pork, salt],
is_manually_added=False,
external_source="spoonacular",
external_id=str(uuid4()),
source_url="https://example.invalid/source",
)
db.add_all([manual, unsafe])
db.flush()
report = audit_recipe_catalog(
db, apply=False, reviewed_at=datetime(2026, 8, 14, tzinfo=timezone.utc)
)
records = {row["id"]: row for row in report["recipes"]}
assert records[str(manual.id)]["catalog_status"] == "active"
assert records[str(unsafe.id)]["catalog_status"] == "quarantined"
unsafe_codes = {item["code"] for item in records[str(unsafe.id)]["findings"]}
assert "unlicensed_external_source" in unsafe_codes
assert "title_ingredient_mismatch" in unsafe_codes
assert unsafe.catalog_status is RecipeCatalogStatus.ACTIVE
audit_recipe_catalog(
db, apply=True, reviewed_at=datetime(2026, 8, 14, tzinfo=timezone.utc)
)
assert manual.catalog_status is RecipeCatalogStatus.ACTIVE
assert unsafe.catalog_status is RecipeCatalogStatus.QUARANTINED
assert unsafe.catalog_reviewed_at is not None
assert unsafe.catalog_findings
def test_audit_requires_exact_approved_wikibooks_provenance(db):
first = _ingredient(db, f"Provenance First {uuid4()}")
second = _ingredient(db, f"Provenance Second {uuid4()}")
recipe = _recipe(
"Provenance First Second",
[first, second],
is_manually_added=False,
external_source="wikibooks",
external_id=str(uuid4()),
source_url="https://en.wikibooks.org/wiki/Cookbook:Provenance_Test",
)
db.add(recipe)
db.flush()
db.add(_approved_link(recipe, external_id="unrelated-page"))
db.flush()
report = audit_recipe_catalog(
db, reviewed_at=datetime(2026, 8, 14, tzinfo=timezone.utc)
)
record = next(row for row in report["recipes"] if row["id"] == str(recipe.id))
assert "missing_import_provenance" in {
finding["code"] for finding in record["findings"]
}
db.add(_approved_link(recipe))
db.flush()
report = audit_recipe_catalog(
db, reviewed_at=datetime(2026, 8, 14, tzinfo=timezone.utc)
)
record = next(row for row in report["recipes"] if row["id"] == str(recipe.id))
assert "missing_import_provenance" not in {
finding["code"] for finding in record["findings"]
}
def test_audit_preserves_operator_quarantine_for_valid_wikibooks_recipe(db):
first = _ingredient(db, f"Operator First {uuid4()}")
second = _ingredient(db, f"Operator Second {uuid4()}")
operator_finding = {
"code": "operator_quarantine",
"severity": "error",
"field": "catalog_status",
"message": "Quarantined by an operator.",
}
recipe = _recipe(
"Operator First Second",
[first, second],
is_manually_added=False,
external_source="wikibooks",
external_id=str(uuid4()),
source_url="https://en.wikibooks.org/wiki/Cookbook:Operator_Test",
catalog_findings=[operator_finding],
)
db.add(recipe)
db.flush()
db.add(_approved_link(recipe))
db.flush()
report = audit_recipe_catalog(
db, apply=False, reviewed_at=datetime(2026, 8, 14, tzinfo=timezone.utc)
)
record = next(row for row in report["recipes"] if row["id"] == str(recipe.id))
assert record["catalog_status"] == "quarantined"
assert operator_finding in record["findings"]
applied_report = audit_recipe_catalog(
db, apply=True, reviewed_at=datetime(2026, 8, 14, tzinfo=timezone.utc)
)
applied_record = next(
row for row in applied_report["recipes"] if row["id"] == str(recipe.id)
)
assert applied_record["catalog_status"] == "quarantined"
assert operator_finding in applied_record["findings"]
assert recipe.catalog_status is RecipeCatalogStatus.QUARANTINED
assert operator_finding in recipe.catalog_findings
@pytest.mark.parametrize(
"prior_findings",
[
1,
{"code": "operator_quarantine", "severity": "error"},
None,
],
ids=["truthy-scalar", "truthy-object", "null"],
)
def test_audit_discards_malformed_prior_catalog_findings(db, prior_findings):
first = _ingredient(db, f"Malformed First {uuid4()}")
second = _ingredient(db, f"Malformed Second {uuid4()}")
recipe = _recipe(
"Malformed First Second",
[first, second],
catalog_findings=prior_findings,
)
db.add(recipe)
db.flush()
report = audit_recipe_catalog(
db, apply=True, reviewed_at=datetime(2026, 8, 14, tzinfo=timezone.utc)
)
record = next(row for row in report["recipes"] if row["id"] == str(recipe.id))
assert record["catalog_status"] == "active"
assert record["findings"] == []
assert recipe.catalog_findings == []
def test_audit_discards_stale_normal_findings(db):
first = _ingredient(db, f"Stale First {uuid4()}")
second = _ingredient(db, f"Stale Second {uuid4()}")
stale_finding = {
"code": "obsolete_validation_error",
"severity": "error",
"field": "name",
"message": "No longer applies.",
}
recipe = _recipe(
"Stale First Second",
[first, second],
catalog_findings=[stale_finding],
)
db.add(recipe)
db.flush()
report = audit_recipe_catalog(
db, apply=False, reviewed_at=datetime(2026, 8, 14, tzinfo=timezone.utc)
)
record = next(row for row in report["recipes"] if row["id"] == str(recipe.id))
assert record["catalog_status"] == "active"
assert record["findings"] == []
assert recipe.catalog_findings == [stale_finding]
audit_recipe_catalog(
db, apply=True, reviewed_at=datetime(2026, 8, 14, tzinfo=timezone.utc)
)
assert recipe.catalog_findings == []
def test_public_catalog_hides_quarantined_recipes(client, db):
first = _ingredient(db, f"Visible First {uuid4()}")
second = _ingredient(db, f"Visible Second {uuid4()}")
active = _recipe("Visible First Second", [first, second])
quarantined = _recipe(
"Hidden First Second",
[first, second],
catalog_status=RecipeCatalogStatus.QUARANTINED,
catalog_findings=[{"code": "test_quarantine", "severity": "error"}],
)
db.add_all([active, quarantined])
db.flush()
response = client.get("/api/recipes")
assert response.status_code == 200
ids = {row["id"] for row in response.json()}
assert str(active.id) in ids
assert str(quarantined.id) not in ids
assert client.get(f"/api/recipes/{quarantined.id}").status_code == 404
def test_meal_plan_creation_rejects_quarantined_recipe(client, db):
first = _ingredient(db, f"Plan First {uuid4()}")
second = _ingredient(db, f"Plan Second {uuid4()}")
quarantined = _recipe(
"Plan First Second",
[first, second],
catalog_status=RecipeCatalogStatus.QUARANTINED,
)
db.add(quarantined)
db.flush()
login = client.post("/api/auth/login", json={"password": "test-family-password"})
assert login.status_code == 204, login.text
response = client.post(
"/api/meals",
json={
"week_start_date": "2031-01-03",
"items": [
{
"recipe_id": str(quarantined.id),
"day_of_week": 1,
"meal_type": "dinner",
}
],
},
)
assert response.status_code == 400
assert response.json()["detail"] == "Meal plan contains unavailable recipes"
def test_generate_single_item_reports_empty_active_catalog(client, db):
db.query(Recipe).update(
{Recipe.catalog_status: RecipeCatalogStatus.QUARANTINED},
synchronize_session=False,
)
plan = MealPlan(week_start_date=date(2031, 1, 13))
db.add(plan)
db.flush()
login = client.post("/api/auth/login", json={"password": "test-family-password"})
assert login.status_code == 204, login.text
response = client.post(
f"/api/meals/{plan.id}/generate-item",
params={"day_of_week": 4, "meal_type": "dinner"},
)
assert response.status_code == 409
assert response.json()["detail"] == (
"No active recipes are available. Approve a staged recipe before generating meals."
)
def test_llm_library_includes_active_global_and_excludes_quarantined(db):
from app.models import FamilyProfile
from app.api.llm_plan import _serialize_library
family = FamilyProfile(
name=f"Audit Family {uuid4()}",
household_size=2,
adult_count=2,
child_count=0,
)
first = _ingredient(db, f"Library First {uuid4()}")
second = _ingredient(db, f"Library Second {uuid4()}")
active = _recipe("Library First Second", [first, second], family_profile_id=None)
quarantined = _recipe(
"Quarantined First Second",
[first, second],
family_profile_id=None,
catalog_status=RecipeCatalogStatus.QUARANTINED,
)
db.add_all([family, active, quarantined])
db.flush()
ids = {row["id"] for row in _serialize_library(db, family.id)}
assert str(active.id) in ids
assert str(quarantined.id) not in ids
backend/tests/test_recipe_import_admin.py
+384
View File
@@ -0,0 +1,384 @@
from __future__ import annotations
import copy
import itertools
import json
import os
import threading
from pathlib import Path
import pytest
from sqlalchemy.exc import IntegrityError
from sqlalchemy.orm import sessionmaker
os.environ.setdefault("ADMIN_TOKEN", "test-admin-token")
os.environ.setdefault("ADMIN_EMAIL", "reviewer@example.com")
from app.models import Ingredient, Recipe, RecipeImportStaging, RecipeImportStatus
from app.services.wikibooks_import import WikibooksImportService
from app.services.recipe_import_review import (
RecipeImportConflictError,
approve_recipe_import,
)
pytestmark = pytest.mark.requires_postgres
FIXTURES = Path(__file__).parent / "fixtures" / "wikibooks"
ADMIN_HEADERS = {"Authorization": "Bearer test-admin-token"}
_IDENTITIES = itertools.count(1)
def _stage(db, fixture: str = "apple_glazed_pork_chops_parse.json"):
payload = json.loads((FIXTURES / fixture).read_text(encoding="utf-8"))["parse"]
identity = next(_IDENTITIES) * 1_000_000
payload["pageid"] += identity
payload["revid"] += identity
staged = WikibooksImportService("MealPlanner admin API tests").stage_parse_payload(
db, payload
)
db.commit()
return staged, payload
@pytest.fixture
def api_client():
from fastapi.testclient import TestClient
from app.main import app
with TestClient(app) as test_client:
yield test_client
def test_recipe_import_routes_require_admin(api_client):
response = api_client.get("/api/admin/recipe-imports")
assert response.status_code == 401
def test_list_filters_and_detail_exposes_provenance(api_client, db_session):
pending, _ = _stage(db_session)
pending.source = "wikibooks-list-test"
db_session.commit()
quarantined, _ = _stage(db_session, "apple_glaze_missing_apple_parse.json")
response = api_client.get(
"/api/admin/recipe-imports?status=pending&source=wikibooks-list-test&limit=1",
headers=ADMIN_HEADERS,
)
assert response.status_code == 200, response.text
body = response.json()
assert body["total"] == 1
assert body["has_next"] is False
assert body["items"][0]["id"] == str(pending.id)
assert body["items"][0]["name"] == "Apple Glazed Pork Chops"
response = api_client.get(
f"/api/admin/recipe-imports/{quarantined.id}", headers=ADMIN_HEADERS
)
assert response.status_code == 200, response.text
detail = response.json()
assert detail["status"] == "quarantined"
assert detail["license_name"] == "CC BY-SA 4.0"
assert detail["raw_payload"]["revid"] == quarantined.raw_payload["revid"]
assert detail["validation_report"]["valid"] is False
def test_approve_publishes_atomically_and_is_idempotent(api_client, db_session):
staged, _ = _stage(db_session)
staged_id = staged.id
external_id = staged.external_id
response = api_client.post(
f"/api/admin/recipe-imports/{staged_id}/approve", headers=ADMIN_HEADERS
)
assert response.status_code == 201, response.text
approved = response.json()
assert approved["status"] == "approved"
assert approved["reviewed_by"] == "admin"
assert approved["recipe_id"] is not None
db_session.expire_all()
recipe = db_session.get(Recipe, approved["recipe_id"])
assert recipe is not None
assert recipe.family_profile_id is None
assert recipe.image_url is None
assert recipe.external_source == "wikibooks"
assert recipe.external_id == external_id
assert recipe.cook_time_minutes == 30
assert len(recipe.ingredients) == 4
ingredient_ids = [item["ingredient_id"] for item in recipe.ingredients]
assert (
db_session.query(Ingredient)
.filter(Ingredient.id.in_(ingredient_ids))
.count()
== 4
)
repeated = api_client.post(
f"/api/admin/recipe-imports/{staged_id}/approve", headers=ADMIN_HEADERS
)
assert repeated.status_code == 200, repeated.text
assert repeated.json()["recipe_id"] == approved["recipe_id"]
assert db_session.query(Recipe).filter(Recipe.id == recipe.id).count() == 1
def test_approved_import_can_republish_after_recipe_deletion(api_client, db_session):
staged, _ = _stage(db_session)
first = api_client.post(
f"/api/admin/recipe-imports/{staged.id}/approve", headers=ADMIN_HEADERS
)
assert first.status_code == 201, first.text
first_recipe_id = first.json()["recipe_id"]
recipe = db_session.get(Recipe, first_recipe_id)
db_session.delete(recipe)
db_session.commit()
repeated = api_client.post(
f"/api/admin/recipe-imports/{staged.id}/approve", headers=ADMIN_HEADERS
)
assert repeated.status_code == 201, repeated.text
assert repeated.json()["recipe_id"] != first_recipe_id
assert repeated.json()["status"] == "approved"
def test_approve_rejects_quarantined_candidate(api_client, db_session):
staged, _ = _stage(db_session, "apple_glaze_missing_apple_parse.json")
staged_id = staged.id
external_id = staged.external_id
response = api_client.post(
f"/api/admin/recipe-imports/{staged_id}/approve", headers=ADMIN_HEADERS
)
assert response.status_code == 409, response.text
assert (
db_session.query(Recipe)
.filter(Recipe.external_source == "wikibooks", Recipe.external_id == external_id)
.count()
== 0
)
db_session.expire_all()
unchanged = db_session.get(RecipeImportStaging, staged_id)
assert unchanged.status == RecipeImportStatus.QUARANTINED
assert unchanged.reviewed_at is None
def test_reject_is_idempotent_and_blocks_later_approval(api_client, db_session):
staged, _ = _stage(db_session)
staged_id = staged.id
external_id = staged.external_id
request = {"reason": "Instructions need editorial review."}
response = api_client.post(
f"/api/admin/recipe-imports/{staged_id}/reject",
headers=ADMIN_HEADERS,
json=request,
)
assert response.status_code == 200, response.text
rejected = response.json()
assert rejected["status"] == "rejected"
assert rejected["rejection_reason"] == request["reason"]
assert rejected["reviewed_by"] == "admin"
repeated = api_client.post(
f"/api/admin/recipe-imports/{staged_id}/reject",
headers=ADMIN_HEADERS,
json=request,
)
assert repeated.status_code == 200, repeated.text
approve = api_client.post(
f"/api/admin/recipe-imports/{staged_id}/approve", headers=ADMIN_HEADERS
)
assert approve.status_code == 409, approve.text
assert (
db_session.query(Recipe)
.filter(Recipe.external_source == "wikibooks", Recipe.external_id == external_id)
.count()
== 0
)
def test_reject_accepts_quarantined_candidate(api_client, db_session):
staged, _ = _stage(db_session, "apple_glaze_missing_apple_parse.json")
response = api_client.post(
f"/api/admin/recipe-imports/{staged.id}/reject",
headers=ADMIN_HEADERS,
json={"reason": "Title does not match the ingredient list."},
)
assert response.status_code == 200, response.text
assert response.json()["status"] == "rejected"
def test_new_source_revision_conflicts_with_published_page(api_client, db_session):
first, payload = _stage(db_session)
first_id = first.id
second_payload = copy.deepcopy(payload)
second_payload["revid"] += 1
second = WikibooksImportService(
"MealPlanner admin API tests"
).stage_parse_payload(db_session, second_payload)
db_session.commit()
second_id = second.id
first_response = api_client.post(
f"/api/admin/recipe-imports/{first_id}/approve", headers=ADMIN_HEADERS
)
assert first_response.status_code == 201, first_response.text
second_response = api_client.post(
f"/api/admin/recipe-imports/{second_id}/approve", headers=ADMIN_HEADERS
)
assert second_response.status_code == 409, second_response.text
db_session.expire_all()
assert (
db_session.get(RecipeImportStaging, second_id).status
== RecipeImportStatus.PENDING
)
assert (
db_session.query(Recipe)
.filter(Recipe.external_source == "wikibooks", Recipe.external_id == first.external_id)
.count()
== 1
)
def test_checksum_conflict_rolls_back_review_state(api_client, db_session):
staged, _ = _stage(db_session)
staged_id = staged.id
external_id = staged.external_id
staged.payload_sha256 = "0" * 64
db_session.commit()
response = api_client.post(
f"/api/admin/recipe-imports/{staged_id}/approve", headers=ADMIN_HEADERS
)
assert response.status_code == 409, response.text
assert (
db_session.query(Recipe)
.filter(Recipe.external_source == "wikibooks", Recipe.external_id == external_id)
.count()
== 0
)
db_session.expire_all()
unchanged = db_session.get(RecipeImportStaging, staged_id)
assert unchanged.status == RecipeImportStatus.PENDING
assert unchanged.reviewed_at is None
assert unchanged.recipe_id is None
def test_oversized_numeric_fields_are_quarantined_before_approval(
api_client, db_session
):
payload = json.loads(
(FIXTURES / "apple_glazed_pork_chops_parse.json").read_text(
encoding="utf-8"
)
)["parse"]
identity = next(_IDENTITIES) * 1_000_000
payload["pageid"] += identity
payload["revid"] += identity
payload["text"] = payload["text"].replace(
"<th>Servings</th><td>4</td>",
"<th>Servings</th><td>2147483648</td>",
)
staged = WikibooksImportService(
"MealPlanner admin API tests"
).stage_parse_payload(db_session, payload)
db_session.commit()
assert staged.status == RecipeImportStatus.QUARANTINED
assert any(
error["code"] == "servings_out_of_range"
for error in staged.validation_report["errors"]
)
response = api_client.post(
f"/api/admin/recipe-imports/{staged.id}/approve", headers=ADMIN_HEADERS
)
assert response.status_code == 409, response.text
def test_concurrent_revision_approval_publishes_only_one_recipe(db_session):
first, payload = _stage(db_session)
second_payload = copy.deepcopy(payload)
second_payload["revid"] += 1
second = WikibooksImportService(
"MealPlanner admin API tests"
).stage_parse_payload(db_session, second_payload)
db_session.commit()
engine = db_session.get_bind()
Session = sessionmaker(bind=engine, autocommit=False, autoflush=False)
barrier = threading.Barrier(2)
outcomes: list[str] = []
def approve(import_id):
session = Session()
try:
barrier.wait(timeout=5)
approve_recipe_import(session, import_id, "concurrent-reviewer")
outcomes.append("approved")
except (IntegrityError, RecipeImportConflictError):
session.rollback()
outcomes.append("conflict")
finally:
session.close()
threads = [
threading.Thread(target=approve, args=(first.id,)),
threading.Thread(target=approve, args=(second.id,)),
]
for thread in threads:
thread.start()
for thread in threads:
thread.join(timeout=10)
assert not thread.is_alive()
assert sorted(outcomes) == ["approved", "conflict"]
db_session.expire_all()
assert (
db_session.query(Recipe)
.filter(
Recipe.external_source == "wikibooks",
Recipe.external_id == first.external_id,
)
.count()
== 1
)
def test_invalid_token_cannot_mutate_import(api_client, db_session):
staged, _ = _stage(db_session)
response = api_client.post(
f"/api/admin/recipe-imports/{staged.id}/approve",
headers={"Authorization": "Bearer wrong-token"},
)
assert response.status_code == 401
db_session.expire_all()
assert (
db_session.get(RecipeImportStaging, staged.id).status
== RecipeImportStatus.PENDING
)
@pytest.mark.parametrize(
"url",
[
"/api/admin/recipe-imports?status=unknown",
"/api/admin/recipe-imports?limit=0",
"/api/admin/recipe-imports?offset=-1",
],
)
def test_list_rejects_invalid_filters(api_client, url):
response = api_client.get(url, headers=ADMIN_HEADERS)
assert response.status_code == 422
def test_reject_requires_nonblank_reason(api_client, db_session):
staged, _ = _stage(db_session)
response = api_client.post(
f"/api/admin/recipe-imports/{staged.id}/reject",
headers=ADMIN_HEADERS,
json={"reason": " "},
)
assert response.status_code == 422
backend/tests/test_recipe_import_staging.py
+184
View File
@@ -0,0 +1,184 @@
"""Focused persistence tests for global recipe import staging."""
from __future__ import annotations
import importlib.util
import os
import pathlib
import subprocess
from urllib.parse import urlparse
import pytest
from sqlalchemy import CheckConstraint, UniqueConstraint
from sqlalchemy.dialects.postgresql import JSONB
from sqlalchemy.exc import DBAPIError, IntegrityError
from app.models import RecipeImportStaging, RecipeImportStatus
BACKEND_ROOT = pathlib.Path(__file__).resolve().parent.parent
MIGRATION_PATH = BACKEND_ROOT / "alembic" / "versions" / "0017_recipe_import_staging.py"
def _staged_recipe(**overrides):
payload = {
"source": "wikibooks",
"revision": "12345",
"title": "Tomato Soup",
}
values = {
"source": "wikibooks",
"external_id": "Cookbook:Tomato_Soup",
"source_url": "https://en.wikibooks.org/wiki/Cookbook:Tomato_Soup",
"source_revision": "12345",
"license_name": "CC BY-SA 4.0",
"license_url": "https://creativecommons.org/licenses/by-sa/4.0/",
"attribution": "Wikibooks contributors",
"raw_payload": payload,
"payload_sha256": "a" * 64,
"normalized_recipe": {"name": "Tomato Soup"},
"validation_report": {"valid": True, "errors": []},
}
values.update(overrides)
return RecipeImportStaging(**values)
def test_recipe_import_staging_model_metadata():
table = RecipeImportStaging.__table__
assert "family_profile_id" not in table.c
assert set(table.c.keys()) == {
"id",
"source",
"external_id",
"source_url",
"source_revision",
"license_name",
"license_url",
"attribution",
"raw_payload",
"payload_sha256",
"normalized_recipe",
"validation_report",
"status",
"rejection_reason",
"recipe_id",
"acquired_at",
"reviewed_at",
"reviewed_by",
}
assert isinstance(table.c.raw_payload.type, JSONB)
assert isinstance(table.c.normalized_recipe.type, JSONB)
assert isinstance(table.c.validation_report.type, JSONB)
assert table.c.status.type.enums == [
"pending",
"quarantined",
"approved",
"rejected",
]
assert table.c.status.default.arg is RecipeImportStatus.PENDING
assert table.c.status.server_default.arg == "pending"
assert table.c.acquired_at.nullable is False
assert table.c.recipe_id.nullable is True
foreign_key = next(iter(table.c.recipe_id.foreign_keys))
assert foreign_key.target_fullname == "recipe.id"
assert foreign_key.ondelete == "SET NULL"
unique_constraints = {
tuple(column.name for column in constraint.columns)
for constraint in table.constraints
if isinstance(constraint, UniqueConstraint)
}
assert ("source", "external_id", "source_revision") in unique_constraints
check_constraints = {
constraint.name: str(constraint.sqltext)
for constraint in table.constraints
if isinstance(constraint, CheckConstraint)
}
assert "payload_sha256" in check_constraints[
"ck_recipe_import_staging_payload_sha256"
]
assert "reviewed_at" in check_constraints[
"ck_recipe_import_staging_review_metadata"
]
index = next(
index
for index in table.indexes
if index.name == "ix_recipe_import_staging_status_acquired_at"
)
assert [column.name for column in index.columns] == ["status", "acquired_at"]
def test_recipe_import_staging_migration_follows_0016():
spec = importlib.util.spec_from_file_location("recipe_import_staging", MIGRATION_PATH)
assert spec and spec.loader
migration = importlib.util.module_from_spec(spec)
spec.loader.exec_module(migration)
assert migration.revision == "0017"
assert migration.down_revision == "0016"
@pytest.mark.requires_postgres
def test_recipe_import_staging_alembic_round_trip_from_0016():
dsn = os.environ.get("TEST_DATABASE_URL")
if not dsn:
pytest.skip("TEST_DATABASE_URL is required to avoid modifying a non-test database")
database_name = urlparse(dsn).path.rsplit("/", 1)[-1]
if not database_name.endswith("_test"):
pytest.skip("TEST_DATABASE_URL must target a database ending in '_test'")
env = os.environ.copy()
env["DATABASE_URL"] = dsn
def alembic(*args: str) -> subprocess.CompletedProcess[str]:
return subprocess.run(
["alembic", *args],
cwd=str(BACKEND_ROOT),
env=env,
capture_output=True,
text=True,
)
try:
at_0016 = alembic("upgrade", "0016")
assert at_0016.returncode == 0, at_0016.stderr
at_0017 = alembic("upgrade", "0017")
assert at_0017.returncode == 0, at_0017.stderr
back_to_0016 = alembic("downgrade", "0016")
assert back_to_0016.returncode == 0, back_to_0016.stderr
back_to_0017 = alembic("upgrade", "0017")
assert back_to_0017.returncode == 0, back_to_0017.stderr
finally:
# Never cross irreversible 0015; leave the dedicated test database at head.
restore_head = alembic("upgrade", "head")
assert restore_head.returncode == 0, restore_head.stderr
@pytest.mark.requires_postgres
def test_recipe_import_staging_versions_revisions_and_preserves_raw_payload(db):
staged = _staged_recipe()
revised = _staged_recipe(source_revision="12346")
db.add_all([staged, revised])
db.flush()
assert staged.status is RecipeImportStatus.PENDING
assert staged.acquired_at is not None
with db.begin_nested():
db.add(_staged_recipe())
with pytest.raises(IntegrityError):
db.flush()
with db.begin_nested():
staged.raw_payload = {"source": "wikibooks", "revision": "tampered"}
with pytest.raises(DBAPIError, match="raw_payload is immutable"):
db.flush()
db.refresh(staged)
assert staged.raw_payload["revision"] == "12345"
backend/tests/test_recipe_schema.py
+23 -1
View File
@@ -3,7 +3,7 @@ from uuid import uuid4
import pytest
from pydantic import ValidationError
from app.schemas.recipe import RecipeCreate, RecipeIngredientRef, ResolveIngredientRequest
from app.schemas.recipe import RecipeCreate, RecipeIngredientRef, RecipeRead, ResolveIngredientRequest
def test_recipe_create_requires_canonical_ingredient_ids() -> None:
@@ -36,6 +36,28 @@ def test_recipe_create_rejects_empty_ingredients() -> None:
)
def test_recipe_read_allows_nullable_discovered_recipe_fields() -> None:
iid = uuid4()
payload = RecipeRead(
id=uuid4(),
name="Imported Recipe With Sparse Metadata",
servings=4,
prep_time_minutes=None,
cook_time_minutes=None,
cuisine_tags=[],
dietary_tags=[],
protein_type=None,
ingredients=[RecipeIngredientRef(ingredient_id=iid, qty=1.0, unit="ea")],
instructions=["Cook until done"],
side_dishes=None,
)
assert payload.prep_time_minutes is None
assert payload.cook_time_minutes is None
assert payload.protein_type is None
def test_resolve_ingredient_request_round_trip() -> None:
req = ResolveIngredientRequest(text="1 lb chicken thighs")
assert req.text == "1 lb chicken thighs"
backend/tests/test_recipe_search_retired.py
+75
View File
@@ -0,0 +1,75 @@
"""Regression tests for retired direct third-party acquisition endpoints."""
from types import SimpleNamespace
from uuid import uuid4
import pytest
pytestmark = pytest.mark.requires_postgres
def _login(client):
response = client.post("/api/auth/login", json={"password": "test-family-password"})
assert response.status_code == 204, response.text
def test_direct_recipe_search_and_import_are_gone(client):
_login(client)
search = client.get("/api/recipes/search", params={"q": "apple", "limit": 5})
assert search.status_code == 410
assert "staged and approved" in search.json()["detail"]
imported = client.post(
"/api/recipes/import",
json={"external_id": "123", "external_source": "spoonacular"},
)
assert imported.status_code == 410
assert "staged and approved" in imported.json()["detail"]
def test_admin_discovery_stages_without_publishing(client, db, monkeypatch):
from app.models import FamilyProfile, Recipe, RecipeImportStatus
family = FamilyProfile(
name=f"Staging Family {uuid4()}",
household_size=2,
adult_count=2,
child_count=0,
)
db.add(family)
db.flush()
analysis = SimpleNamespace(
discovery_queries=["apple dinner"],
confidence=0.9,
)
analysis.to_dict = lambda: {
"discovery_queries": analysis.discovery_queries,
"confidence": analysis.confidence,
}
monkeypatch.setattr(
"app.api.admin.FeedbackAnalyzer.analyze",
lambda self, database, family_id: analysis,
)
staged = [
SimpleNamespace(status=RecipeImportStatus.PENDING),
SimpleNamespace(status=RecipeImportStatus.QUARANTINED),
]
monkeypatch.setattr(
"app.api.admin.WikibooksImportService.stage_discovered_pages",
lambda self, database, limit, queries: staged,
)
before = db.query(Recipe).count()
response = client.post(
"/api/admin/trigger-discovery",
params={"family_profile_id": str(family.id), "limit": 2},
headers={"Authorization": "Bearer test-admin-token"},
)
assert response.status_code == 200, response.text
assert response.json()["candidates_staged"] == 2
assert response.json()["pending"] == 1
assert response.json()["quarantined"] == 1
assert response.json()["recipes_added"] == 0
assert db.query(Recipe).count() == before
backend/tests/test_starter_recipe_catalog.py
+606
View File
@@ -0,0 +1,606 @@
"""Offline and PostgreSQL coverage for the bundled starter recipe loader."""
from __future__ import annotations
import copy
import threading
from datetime import date
from queue import Empty, Queue
import pytest
from sqlalchemy import event
from sqlalchemy.orm import sessionmaker
from app.models import (
Ingredient,
MealPlan,
MealPlanItem,
MealType,
Recipe,
RecipeCatalogStatus,
)
from app.services.starter_recipe_catalog import (
DEFAULT_CATALOG_PATH,
MAX_CALORIES_PER_SERVING,
STARTER_CATALOG_VERSION,
STARTER_DISCOVERY_REASON,
STARTER_RECIPE_COUNT,
StarterRecipeCatalogError,
load_bundled_starter_catalog,
load_starter_catalog,
read_starter_catalog,
validate_starter_catalog,
)
@pytest.fixture(name="db")
def starter_catalog_db(_engine):
"""Use a commit-aware session and remove only rows created by each loader test."""
if _engine is None:
pytest.skip("Postgres not reachable")
Session = sessionmaker(
bind=_engine,
autocommit=False,
autoflush=False,
expire_on_commit=False,
)
baseline = Session()
try:
recipe_ids = {recipe.id for recipe in baseline.query(Recipe).all()}
ingredient_ids = {ingredient.id for ingredient in baseline.query(Ingredient).all()}
meal_plan_ids = {meal_plan.id for meal_plan in baseline.query(MealPlan).all()}
meal_plan_item_ids = {
meal_plan_item.id for meal_plan_item in baseline.query(MealPlanItem).all()
}
finally:
baseline.rollback()
baseline.close()
session = Session()
try:
yield session
finally:
session.rollback()
session.close()
cleanup = Session()
try:
for meal_plan_item in cleanup.query(MealPlanItem).all():
if meal_plan_item.id not in meal_plan_item_ids:
cleanup.delete(meal_plan_item)
cleanup.flush()
for meal_plan in cleanup.query(MealPlan).all():
if meal_plan.id not in meal_plan_ids:
cleanup.delete(meal_plan)
cleanup.flush()
for recipe in cleanup.query(Recipe).all():
if recipe.id not in recipe_ids:
cleanup.delete(recipe)
cleanup.flush()
for ingredient in cleanup.query(Ingredient).all():
if ingredient.id not in ingredient_ids:
cleanup.delete(ingredient)
cleanup.commit()
finally:
cleanup.rollback()
cleanup.close()
def _catalog() -> dict[str, object]:
return dict(read_starter_catalog(DEFAULT_CATALOG_PATH))
def _catalog_recipe(catalog: dict[str, object], name: str) -> dict[str, object]:
return next(
recipe for recipe in catalog["recipes"] if recipe["name"] == name # type: ignore[index]
)
def _prior_catalog() -> dict[str, object]:
catalog = copy.deepcopy(_catalog())
for recipe in catalog["recipes"]: # type: ignore[index]
recipe["description"] = "A prior bundled formulation before the version two update."
recipe["calories_per_serving"] = (
recipe["calories_per_serving"] - 1
if recipe["calories_per_serving"] == MAX_CALORIES_PER_SERVING
else recipe["calories_per_serving"] + 1
)
recipe["ingredients"][0]["quantity"] += 1
recipe["instructions"][0] = (
"Prepare the prior bundled formulation with measured ingredients and careful cooking."
)
return catalog
def _ingredient_values(db, recipe: Recipe) -> list[tuple[str, object, object]]:
ingredients = {
str(ingredient.id): ingredient.name_lower
for ingredient in db.query(Ingredient)
.filter(Ingredient.id.in_([item["ingredient_id"] for item in recipe.ingredients]))
.all()
}
return [
(ingredients[item["ingredient_id"]], item["qty"], item["unit"])
for item in recipe.ingredients
]
def _unrelated_recipes() -> tuple[Recipe, Recipe]:
approved = Recipe(
name="Rigatoni Pasta with Ricotta in Tomato Cream Sauce",
description="An already approved Wikibooks provenance row.",
servings=4,
ingredients=[],
instructions=[
"Keep the approved provenance recipe unchanged during catalog loading."
],
is_manually_added=False,
external_source="wikibooks",
external_id="approved-rigatoni-provenance-row",
source_url="https://en.wikibooks.org/wiki/Cookbook:Rigatoni",
discovery_reason="Approved import from revision 1",
catalog_status=RecipeCatalogStatus.ACTIVE,
catalog_findings=[],
)
legacy = Recipe(
name="Legacy quarantined recipe remains unchanged",
description="A historical row outside the bundled catalog.",
servings=2,
ingredients=[],
instructions=[
"Keep historical links to this quarantined recipe available without changes."
],
is_manually_added=False,
catalog_status=RecipeCatalogStatus.QUARANTINED,
catalog_findings=[{"code": "legacy", "severity": "error"}],
)
return approved, legacy
def test_default_catalog_path_is_validated_as_the_complete_catalog():
assert DEFAULT_CATALOG_PATH.is_file()
catalog = _catalog()
recipes = validate_starter_catalog(catalog)
assert catalog["version"] == STARTER_CATALOG_VERSION
assert len(recipes) == STARTER_RECIPE_COUNT
assert len({recipe["name"] for recipe in recipes}) == STARTER_RECIPE_COUNT
assert max(recipe["calories_per_serving"] for recipe in recipes) <= MAX_CALORIES_PER_SERVING
@pytest.mark.parametrize("version", [1, 3])
def test_validation_rejects_catalog_versions_other_than_two(version):
catalog = copy.deepcopy(_catalog())
catalog["version"] = version
with pytest.raises(StarterRecipeCatalogError, match="Catalog version must be 2"):
validate_starter_catalog(catalog)
def test_validation_rejects_calories_per_serving_over_limit():
catalog = copy.deepcopy(_catalog())
catalog["recipes"][0]["calories_per_serving"] = MAX_CALORIES_PER_SERVING + 1 # type: ignore[index]
with pytest.raises(
StarterRecipeCatalogError,
match="calories_per_serving must be an integer between 1 and 500",
):
validate_starter_catalog(catalog)
def test_validation_rejects_partial_catalog_before_database_access():
catalog = copy.deepcopy(_catalog())
catalog["recipes"] = catalog["recipes"][:-1] # type: ignore[index]
class NoDatabase:
def __getattr__(self, name):
raise AssertionError(f"Database access attempted through {name}")
with pytest.raises(StarterRecipeCatalogError, match="exactly 50"):
load_starter_catalog(NoDatabase(), catalog) # type: ignore[arg-type]
def test_nonstandard_json_number_is_a_catalog_error(tmp_path):
invalid_catalog = tmp_path / "invalid_catalog.json"
invalid_catalog.write_text('{"version": NaN, "recipes": []}', encoding="utf-8")
with pytest.raises(StarterRecipeCatalogError, match="not valid JSON"):
read_starter_catalog(invalid_catalog)
@pytest.mark.requires_postgres
def test_dry_run_validates_actual_catalog_without_writes(db):
before = (
db.query(Recipe).count(),
db.query(Ingredient).count(),
db.query(Recipe).filter(Recipe.discovery_reason == STARTER_DISCOVERY_REASON).count(),
)
db.rollback()
summary = load_bundled_starter_catalog(db, dry_run=True)
assert summary["dry_run"] is True
assert summary["validated_recipes"] == STARTER_RECIPE_COUNT
assert summary["created_recipes"] == STARTER_RECIPE_COUNT
after = (
db.query(Recipe).count(),
db.query(Ingredient).count(),
db.query(Recipe).filter(Recipe.discovery_reason == STARTER_DISCOVERY_REASON).count(),
)
assert after == before
@pytest.mark.requires_postgres
def test_loader_uses_actual_catalog_and_preserves_unrelated_rows(db):
catalog = _catalog()
approved, legacy = _unrelated_recipes()
db.add_all([approved, legacy])
db.commit()
approved_before = {
"name": approved.name,
"external_source": approved.external_source,
"external_id": approved.external_id,
"source_url": approved.source_url,
"discovery_reason": approved.discovery_reason,
"catalog_status": approved.catalog_status,
}
legacy_before = {
"name": legacy.name,
"catalog_status": legacy.catalog_status,
"catalog_findings": legacy.catalog_findings,
}
db.rollback()
first = load_bundled_starter_catalog(db)
assert first["created_recipes"] == STARTER_RECIPE_COUNT
assert (
db.query(Recipe)
.filter(Recipe.discovery_reason == STARTER_DISCOVERY_REASON)
.count()
== STARTER_RECIPE_COUNT
)
loaded = (
db.query(Recipe)
.filter(Recipe.name == catalog["recipes"][0]["name"]) # type: ignore[index]
.one()
)
expected = _catalog_recipe(catalog, loaded.name)
assert _ingredient_values(db, loaded) == [
(" ".join(item["name"].split()).casefold(), item["quantity"], item["unit"])
for item in expected["ingredients"]
]
approved_after = db.get(Recipe, approved.id)
legacy_after = db.get(Recipe, legacy.id)
assert {
"name": approved_after.name,
"external_source": approved_after.external_source,
"external_id": approved_after.external_id,
"source_url": approved_after.source_url,
"discovery_reason": approved_after.discovery_reason,
"catalog_status": approved_after.catalog_status,
} == approved_before
assert {
"name": legacy_after.name,
"catalog_status": legacy_after.catalog_status,
"catalog_findings": legacy_after.catalog_findings,
} == legacy_before
db.rollback()
second = load_bundled_starter_catalog(db)
assert second["created_recipes"] == 0
assert second["existing_recipes"] == STARTER_RECIPE_COUNT
@pytest.mark.requires_postgres
@pytest.mark.parametrize("field", ["description", "instructions", "ingredients"])
def test_loader_rejects_drifted_bundled_content(db, field):
catalog = _catalog()
load_bundled_starter_catalog(db)
drifted = (
db.query(Recipe)
.filter(Recipe.name == catalog["recipes"][0]["name"]) # type: ignore[index]
.one()
)
if field == "description":
drifted.description = "This description was changed after the initial catalog load."
elif field == "instructions":
drifted.instructions = ["This instruction was changed after the initial catalog load."]
else:
ingredients = copy.deepcopy(drifted.ingredients)
ingredients[0]["qty"] += 1
drifted.ingredients = ingredients
db.commit()
with pytest.raises(StarterRecipeCatalogError, match="content has drifted"):
load_bundled_starter_catalog(db)
@pytest.mark.requires_postgres
def test_update_existing_preserves_recipe_and_meal_plan_item_ids(db):
catalog = _catalog()
load_starter_catalog(db, _prior_catalog())
target = (
db.query(Recipe)
.filter(Recipe.name == catalog["recipes"][0]["name"]) # type: ignore[index]
.one()
)
recipe_id = target.id
prior_ingredients = copy.deepcopy(target.ingredients)
meal_plan = MealPlan(week_start_date=date(2026, 1, 5))
db.add(meal_plan)
db.flush()
meal_plan_item = MealPlanItem(
meal_plan_id=meal_plan.id,
recipe_id=recipe_id,
day_of_week=1,
meal_type=MealType.DINNER,
)
db.add(meal_plan_item)
db.commit()
with pytest.raises(StarterRecipeCatalogError, match="content has drifted"):
load_bundled_starter_catalog(db)
summary = load_bundled_starter_catalog(db, update_existing=True)
assert summary["created_recipes"] == 0
assert summary["updated_recipes"] == STARTER_RECIPE_COUNT
db.expire_all()
updated = db.get(Recipe, recipe_id)
expected = _catalog_recipe(catalog, updated.name)
assert updated.id == recipe_id
assert db.get(MealPlanItem, meal_plan_item.id).recipe_id == recipe_id
assert updated.description == expected["description"]
assert updated.calories_per_serving == expected["calories_per_serving"]
assert updated.instructions == expected["instructions"]
assert updated.ingredients != prior_ingredients
assert _ingredient_values(db, updated) == [
(" ".join(item["name"].split()).casefold(), item["quantity"], item["unit"])
for item in expected["ingredients"]
]
db.rollback()
assert load_bundled_starter_catalog(db, update_existing=True)["updated_recipes"] == 0
@pytest.mark.requires_postgres
def test_update_existing_dry_run_reports_updates_without_writes(db):
catalog = _catalog()
load_starter_catalog(db, _prior_catalog())
target = (
db.query(Recipe)
.filter(Recipe.name == catalog["recipes"][0]["name"]) # type: ignore[index]
.one()
)
recipe_id = target.id
before = (
target.description,
target.calories_per_serving,
copy.deepcopy(target.ingredients),
list(target.instructions),
)
db.rollback()
summary = load_bundled_starter_catalog(db, dry_run=True, update_existing=True)
assert summary["dry_run"] is True
assert summary["created_recipes"] == 0
assert summary["updated_recipes"] == STARTER_RECIPE_COUNT
db.expire_all()
unchanged = db.get(Recipe, recipe_id)
assert (
unchanged.description,
unchanged.calories_per_serving,
unchanged.ingredients,
unchanged.instructions,
) == before
@pytest.mark.requires_postgres
def test_update_existing_rejects_an_incomplete_marker_owned_catalog(db):
catalog = _catalog()
load_starter_catalog(db, _prior_catalog())
missing = (
db.query(Recipe)
.filter(Recipe.name == catalog["recipes"][0]["name"]) # type: ignore[index]
.one()
)
missing.discovery_reason = None
db.commit()
with pytest.raises(StarterRecipeCatalogError, match="exactly all 50"):
load_bundled_starter_catalog(db, update_existing=True)
@pytest.mark.requires_postgres
def test_update_existing_rolls_back_all_content_changes_on_flush_failure(db):
catalog = _catalog()
load_starter_catalog(db, _prior_catalog())
target = (
db.query(Recipe)
.filter(Recipe.name == catalog["recipes"][0]["name"]) # type: ignore[index]
.one()
)
recipe_id = target.id
before = (
target.description,
target.calories_per_serving,
copy.deepcopy(target.ingredients),
list(target.instructions),
)
db.rollback()
def fail_recipe_update(mapper, connection, target):
raise RuntimeError("forced recipe update failure")
event.listen(Recipe, "before_update", fail_recipe_update)
try:
with pytest.raises(RuntimeError, match="forced recipe update failure"):
load_bundled_starter_catalog(db, update_existing=True)
finally:
event.remove(Recipe, "before_update", fail_recipe_update)
db.expire_all()
unchanged = db.get(Recipe, recipe_id)
assert (
unchanged.description,
unchanged.calories_per_serving,
unchanged.ingredients,
unchanged.instructions,
) == before
@pytest.mark.requires_postgres
def test_loader_rolls_back_all_catalog_writes_if_recipe_flush_fails(db):
before = (
db.query(Recipe).count(),
db.query(Ingredient).count(),
db.query(Recipe).filter(Recipe.discovery_reason == STARTER_DISCOVERY_REASON).count(),
)
db.rollback()
def fail_recipe_flush(mapper, connection, target):
raise RuntimeError("forced recipe flush failure")
event.listen(Recipe, "before_insert", fail_recipe_flush)
try:
with pytest.raises(RuntimeError, match="forced recipe flush failure"):
load_bundled_starter_catalog(db)
finally:
event.remove(Recipe, "before_insert", fail_recipe_flush)
after = (
db.query(Recipe).count(),
db.query(Ingredient).count(),
db.query(Recipe).filter(Recipe.discovery_reason == STARTER_DISCOVERY_REASON).count(),
)
assert after == before
@pytest.mark.requires_postgres
def test_concurrent_sessions_create_one_bundled_catalog(db):
engine = db.get_bind().engine
Session = sessionmaker(bind=engine, autocommit=False, autoflush=False)
baseline = Session()
try:
initial_ingredient_ids = {ingredient.id for ingredient in baseline.query(Ingredient).all()}
assert (
baseline.query(Recipe)
.filter(Recipe.discovery_reason == STARTER_DISCOVERY_REASON)
.count()
== 0
)
finally:
baseline.rollback()
baseline.close()
barrier = threading.Barrier(2)
results: Queue[dict[str, object]] = Queue()
errors: Queue[BaseException] = Queue()
def load_in_session() -> None:
session = Session()
try:
barrier.wait(timeout=5)
results.put(load_bundled_starter_catalog(session))
except BaseException as exc:
session.rollback()
errors.put(exc)
finally:
session.close()
threads = [threading.Thread(target=load_in_session) for _ in range(2)]
for thread in threads:
thread.start()
for thread in threads:
thread.join(timeout=20)
try:
assert all(not thread.is_alive() for thread in threads)
if not errors.empty():
raise errors.get_nowait()
try:
summaries = [results.get_nowait(), results.get_nowait()]
except Empty as exc:
raise AssertionError("concurrent catalog load did not return two summaries") from exc
assert sorted(summary["created_recipes"] for summary in summaries) == [0, 50]
verifier = Session()
try:
bundled = (
verifier.query(Recipe)
.filter(Recipe.discovery_reason == STARTER_DISCOVERY_REASON)
.all()
)
assert len(bundled) == STARTER_RECIPE_COUNT
assert len({recipe.name for recipe in bundled}) == STARTER_RECIPE_COUNT
finally:
verifier.rollback()
verifier.close()
finally:
cleanup = Session()
try:
bundled = (
cleanup.query(Recipe)
.filter(Recipe.discovery_reason == STARTER_DISCOVERY_REASON)
.all()
)
for recipe in bundled:
cleanup.delete(recipe)
cleanup.flush()
for ingredient in cleanup.query(Ingredient).all():
if ingredient.id not in initial_ingredient_ids:
cleanup.delete(ingredient)
cleanup.commit()
finally:
cleanup.rollback()
cleanup.close()
@pytest.mark.requires_postgres
def test_concurrent_update_existing_calls_serialize(db):
catalog = _catalog()
load_starter_catalog(db, _prior_catalog())
engine = db.get_bind().engine
Session = sessionmaker(bind=engine, autocommit=False, autoflush=False)
barrier = threading.Barrier(2)
results: Queue[dict[str, object]] = Queue()
errors: Queue[BaseException] = Queue()
def update_in_session() -> None:
session = Session()
try:
barrier.wait(timeout=5)
results.put(load_bundled_starter_catalog(session, update_existing=True))
except BaseException as exc:
session.rollback()
errors.put(exc)
finally:
session.close()
threads = [threading.Thread(target=update_in_session) for _ in range(2)]
for thread in threads:
thread.start()
for thread in threads:
thread.join(timeout=20)
assert all(not thread.is_alive() for thread in threads)
if not errors.empty():
raise errors.get_nowait()
try:
summaries = [results.get_nowait(), results.get_nowait()]
except Empty as exc:
raise AssertionError("concurrent catalog update did not return two summaries") from exc
assert sorted(summary["updated_recipes"] for summary in summaries) == [0, 50]
verifier = Session()
try:
target = (
verifier.query(Recipe)
.filter(Recipe.name == catalog["recipes"][0]["name"]) # type: ignore[index]
.one()
)
expected = _catalog_recipe(catalog, target.name)
assert target.description == expected["description"]
assert target.calories_per_serving == expected["calories_per_serving"]
finally:
verifier.rollback()
verifier.close()
backend/tests/test_wikibooks_import.py
+729
View File
@@ -0,0 +1,729 @@
"""Offline tests for the bounded Wikibooks recipe staging service."""
from __future__ import annotations
import copy
import json
import math
import threading
import uuid
from contextlib import nullcontext
from datetime import datetime, timezone
from pathlib import Path
from queue import Empty, Queue
import pytest
from sqlalchemy import text
from sqlalchemy.dialects import postgresql
from sqlalchemy.orm import sessionmaker
from app.models import RecipeImportStaging, RecipeImportStatus
from app.services.wikibooks_import import (
CC_BY_SA_4_LICENSE_URL,
WIKIBOOKS_API_URL,
WikibooksImportError,
WikibooksImportService,
WikibooksResponseError,
_normalize_ingredient,
canonical_json_sha256,
)
FIXTURE_DIR = Path(__file__).parent / "fixtures" / "wikibooks"
class FakeResponse:
def __init__(self, payload, *, status_code=200, headers=None):
self.status_code = status_code
self.headers = {"Content-Length": str(len(json.dumps(payload).encode("utf-8")))}
self.headers.update(headers or {})
self._body = json.dumps(payload).encode("utf-8")
self.closed = False
def iter_content(self, chunk_size):
for offset in range(0, len(self._body), chunk_size):
yield self._body[offset : offset + chunk_size]
def close(self):
self.closed = True
class FakeSession:
def __init__(self, responses):
self.responses = list(responses)
self.calls = []
def get(self, url, **kwargs):
self.calls.append((url, kwargs))
return self.responses.pop(0)
class FakeStageResult:
def __init__(self, value):
self.value = value
def scalar_one(self):
return self.value
def scalars(self):
return self
def all(self):
return [] if self.value is None else [self.value]
class FakeStageDb:
def __init__(self):
self.winner = object()
self.statements = []
self.no_autoflush = nullcontext()
def execute(self, statement):
self.statements.append(statement)
return FakeStageResult(self.winner)
def flush(self):
pass
def _parse_fixture(name: str):
return json.loads((FIXTURE_DIR / name).read_text())["parse"]
def test_stage_discovered_pages_stages_each_title(monkeypatch):
service = WikibooksImportService("MealPlanner recipe test")
titles = ["Cookbook:Apple Pie I", "Cookbook:Banana Bread"]
staged = [object(), object()]
monkeypatch.setattr(service, "discover_recipe_titles", lambda limit: titles[:limit])
monkeypatch.setattr(
service,
"stage_page",
lambda db, title: staged[titles.index(title)],
)
assert service.stage_discovered_pages(object(), limit=2) == staged
def test_stage_discovered_pages_uses_feedback_queries(monkeypatch):
service = WikibooksImportService("MealPlanner recipe test")
titles = ["Cookbook:Apple Pie I"]
staged = [object()]
captured = {}
monkeypatch.setattr(
service,
"search_recipe_titles",
lambda queries, limit: captured.update(queries=queries, limit=limit) or titles,
)
monkeypatch.setattr(service, "stage_page", lambda db, title: staged[0])
assert service.stage_discovered_pages(
object(), limit=1, queries=["apple dinner"]
) == staged
assert captured == {"queries": ["apple dinner"], "limit": 1}
def _service(session=None, **kwargs):
return WikibooksImportService(
"MealPlanner-Wikibooks-Import/1.0 (tests@example.invalid)",
session=session,
min_request_interval=0,
**kwargs,
)
def test_discovery_uses_fixed_params_and_bounded_continuation():
session = FakeSession(
[
FakeResponse(
{
"continue": {"cmcontinue": "page|2", "continue": "-||"},
"query": {
"categorymembers": [
{"pageid": 1, "title": "Cookbook:First"}
]
},
}
),
FakeResponse(
{
"query": {
"categorymembers": [
{"pageid": 2, "title": "Cookbook:Second"}
]
}
}
),
]
)
assert _service(session).discover_recipe_titles(2) == ["Cookbook:First", "Cookbook:Second"]
first_url, first_kwargs = session.calls[0]
assert first_url == WIKIBOOKS_API_URL
assert first_kwargs["params"] == {
"action": "query",
"list": "categorymembers",
"cmtitle": "Category:Recipes",
"cmtype": "page",
"cmlimit": 2,
"format": "json",
"formatversion": 2,
"maxlag": 5,
}
assert first_kwargs["headers"]["User-Agent"].startswith("MealPlanner-Wikibooks-Import/")
assert first_kwargs["stream"] is True
assert first_kwargs["allow_redirects"] is False
assert session.calls[1][1]["params"]["cmcontinue"] == "page|2"
assert session.calls[1][1]["params"]["cmlimit"] == 1
def test_feedback_search_uses_cookbook_namespace_and_deduplicates():
session = FakeSession(
[
FakeResponse(
{
"query": {
"search": [
{"pageid": 1, "title": "Cookbook:Apple Pie I"},
{"pageid": 2, "title": "Cookbook:Apple Sauce"},
]
}
}
),
FakeResponse(
{
"query": {
"search": [
{"pageid": 1, "title": "Cookbook:Apple Pie I"},
{"pageid": 3, "title": "Cookbook:Pork Chops"},
]
}
}
),
]
)
assert _service(session).search_recipe_titles(
[" apple dinner ", "pork dinner"], 3
) == [
"Cookbook:Apple Pie I",
"Cookbook:Apple Sauce",
"Cookbook:Pork Chops",
]
assert session.calls[0][1]["params"] == {
"action": "query",
"list": "search",
"srsearch": "apple dinner",
"srnamespace": 102,
"srlimit": 3,
"format": "json",
"formatversion": 2,
"maxlag": 5,
}
assert session.calls[1][1]["params"]["srsearch"] == "pork dinner"
assert session.calls[1][1]["params"]["srlimit"] == 1
def test_fetch_retries_retry_after_and_mediawiki_maxlag():
parse_payload = _parse_fixture("apple_glazed_pork_chops_parse.json")
session = FakeSession(
[
FakeResponse({"error": "busy"}, status_code=429, headers={"Retry-After": "3"}),
FakeResponse({"error": {"code": "maxlag"}}),
FakeResponse({"parse": parse_payload}),
]
)
sleeps = []
assert _service(session, backoff_seconds=0.25, sleep=sleeps.append).fetch_page(
"Cookbook:Apple Glazed Pork Chops"
) == parse_payload
assert sleeps == [3.0, 0.5]
assert len(session.calls) == 3
assert session.calls[-1][1]["params"] == {
"action": "parse",
"page": "Cookbook:Apple Glazed Pork Chops",
"redirects": 1,
"prop": "text|wikitext|sections|revid|displaytitle",
"format": "json",
"formatversion": 2,
"maxlag": 5,
}
def test_fetch_rejects_redirect_responses():
response = FakeResponse({}, status_code=302)
session = FakeSession([response])
with pytest.raises(WikibooksImportError, match="redirect HTTP 302 rejected"):
_service(session).fetch_page("Cookbook:Apple Glazed Pork Chops")
assert session.calls[0][1]["allow_redirects"] is False
assert response.closed is True
def test_retry_delays_are_capped():
parse_payload = _parse_fixture("apple_glazed_pork_chops_parse.json")
retry_after_sleeps = []
retry_after_session = FakeSession(
[
FakeResponse({"error": "busy"}, status_code=429, headers={"Retry-After": "300"}),
FakeResponse({"parse": parse_payload}),
]
)
_service(
retry_after_session,
max_attempts=2,
max_retry_delay=2,
sleep=retry_after_sleeps.append,
).fetch_page("Cookbook:Apple Glazed Pork Chops")
assert retry_after_sleeps == [2.0]
backoff_sleeps = []
backoff_session = FakeSession(
[
FakeResponse({"error": {"code": "maxlag"}}),
FakeResponse({"parse": parse_payload}),
]
)
_service(
backoff_session,
backoff_seconds=10,
max_retry_delay=2,
sleep=backoff_sleeps.append,
).fetch_page("Cookbook:Apple Glazed Pork Chops")
assert backoff_sleeps == [2.0]
@pytest.mark.parametrize(
"kwargs",
[
{"timeout": None},
{"timeout": (0, 1)},
{"timeout": (1, math.inf)},
{"max_attempts": 0},
{"max_attempts": 1.5},
{"backoff_seconds": math.nan},
{"max_retry_delay": 0},
],
)
def test_constructor_rejects_invalid_timeout_and_retry_configuration(kwargs):
with pytest.raises(ValueError):
_service(**kwargs)
def test_payload_limit_boundary_and_malformed_parse_shape():
parse_payload = _parse_fixture("apple_glazed_pork_chops_parse.json")
response_payload = {"parse": parse_payload}
size = len(json.dumps(response_payload).encode("utf-8"))
assert _service(
FakeSession([FakeResponse(response_payload)]), max_payload_bytes=size
).fetch_page("Cookbook:Apple Glazed Pork Chops") == parse_payload
with pytest.raises(WikibooksResponseError, match="payload exceeds"):
_service(
FakeSession([FakeResponse(response_payload)]), max_payload_bytes=size - 1
).fetch_page("Cookbook:Apple Glazed Pork Chops")
malformed = copy.deepcopy(parse_payload)
malformed["sections"] = [{}]
with pytest.raises(WikibooksResponseError, match="sections item line"):
_service(FakeSession([FakeResponse({"parse": malformed})])).fetch_page(
"Cookbook:Apple Glazed Pork Chops"
)
def test_normalization_extracts_text_sections_times_and_fractions():
candidate = _service().normalize_page(_parse_fixture("apple_glazed_pork_chops_parse.json"))
assert candidate["name"] == "Apple Glazed Pork Chops"
assert candidate["description"] == "Sweet apples balance savory pork chops."
assert candidate["servings"] == 4
assert candidate["prep_time_minutes"] == 15
assert candidate["cook_time_minutes"] == 20
assert candidate["baking_time_minutes"] == 10
assert [item["quantity"] for item in candidate["ingredients"]] == [1.5, 0.5, 1.25, 2.0]
assert [item["quantity_text"] for item in candidate["ingredients"]] == ["1 1/2", "½", "1¼", "2"]
assert candidate["ingredients"][0] == {
"quantity": 1.5,
"quantity_text": "1 1/2",
"unit": "cups",
"name": "apple cider",
"source_text": "1 1/2 cups (360 ml) apple cider",
}
assert candidate["instructions"] == [
"Brown the pork chops in butter.",
"Add cider and cook until glazed.",
]
assert _service().validate_recipe(candidate) == {"valid": True, "errors": [], "warnings": []}
def test_validation_rejects_invalid_servings_and_ingredient_quantities():
valid_candidate = _service().normalize_page(_parse_fixture("apple_glazed_pork_chops_parse.json"))
for servings in (None, 0, -1):
invalid_servings = copy.deepcopy(valid_candidate)
invalid_servings["servings"] = servings
assert any(
error["code"] == "invalid_servings"
for error in _service().validate_recipe(invalid_servings)["errors"]
)
for quantity in (None, 0, -1):
invalid_quantity = copy.deepcopy(valid_candidate)
invalid_quantity["ingredients"][0]["quantity"] = quantity
assert any(
error["code"] == "invalid_ingredient_quantity"
for error in _service().validate_recipe(invalid_quantity)["errors"]
)
def test_validation_reports_apple_title_mismatch():
bad_candidate = _service().normalize_page(
_parse_fixture("apple_glaze_missing_apple_parse.json")
)
report = _service().validate_recipe(bad_candidate)
mismatch = next(error for error in report["errors"] if error["code"] == "title_ingredient_mismatch")
assert mismatch["missing_tokens"] == ["apple"]
assert report["valid"] is False
def test_validation_quarantines_database_integer_overflow():
candidate = _service().normalize_page(
_parse_fixture("apple_glazed_pork_chops_parse.json")
)
candidate["servings"] = 2_147_483_648
candidate["prep_time_minutes"] = 2_147_483_648
candidate["cook_time_minutes"] = 2_147_483_640
candidate["baking_time_minutes"] = 10
report = _service().validate_recipe(candidate)
assert report["valid"] is False
assert {error["code"] for error in report["errors"]} >= {
"servings_out_of_range",
"time_out_of_range",
"combined_cook_time_out_of_range",
}
def test_validation_accepts_generic_pie_form_while_rejecting_missing_apple():
candidate = _service().normalize_page(_parse_fixture("apple_glazed_pork_chops_parse.json"))
apple_pie = copy.deepcopy(candidate)
apple_pie["name"] = "Apple Pie"
apple_pie["ingredients"][0]["name"] = "apples"
assert not any(
error["code"] == "title_ingredient_mismatch"
for error in _service().validate_recipe(apple_pie)["errors"]
)
missing_apple = _service().normalize_page(
_parse_fixture("apple_glaze_missing_apple_parse.json")
)
assert any(
error["code"] == "title_ingredient_mismatch"
for error in _service().validate_recipe(missing_apple)["errors"]
)
@pytest.mark.parametrize(
("title", "ingredient_name"),
[
("Fried Apple Pies", "apples"),
("Classic American Apple Pie", "apples"),
("20-Minute Beef Stroganoff", "beef"),
("Apple Pie II", "apples"),
("8 Desserts in 1 Pan", "apples"),
("Classic American Salad Dressing Out", "apples"),
("Chocolate Chip Cookies I", "chocolate chips"),
],
)
def test_validation_ignores_generic_title_descriptors_and_variants(title, ingredient_name):
candidate = _service().normalize_page(_parse_fixture("apple_glazed_pork_chops_parse.json"))
candidate["name"] = title
candidate["ingredients"][0]["name"] = ingredient_name
report = _service().validate_recipe(candidate)
assert report["valid"] is True
assert not any(error["code"] == "title_ingredient_mismatch" for error in report["errors"])
def test_live_apple_pie_fixture_preserves_missing_quantity_failures():
payload = json.loads((FIXTURE_DIR / "apple_pie_i_live_parse.json").read_text())
candidate = _service().normalize_page(payload)
report = _service().validate_recipe(candidate)
assert candidate["name"] == "Apple Pie I"
assert sum(
error["code"] == "invalid_ingredient_quantity" for error in report["errors"]
) == 4
assert not any(
error["code"] == "title_ingredient_mismatch" for error in report["errors"]
)
assert report["valid"] is False
@pytest.mark.parametrize("source_text", ["1-2 cups apples", "1–2 cups apples", "1 — 2 cups apples"])
def test_validation_accepts_plural_title_matches_and_quarantines_quantity_ranges(source_text):
candidate = _service().normalize_page(_parse_fixture("apple_glazed_pork_chops_parse.json"))
banana_bread = copy.deepcopy(candidate)
banana_bread["name"] = "Banana Bread"
banana_bread["ingredients"][0]["name"] = "bananas"
assert not any(
error["code"] == "title_ingredient_mismatch"
for error in _service().validate_recipe(banana_bread)["errors"]
)
ranged_ingredient = _normalize_ingredient(source_text)
assert ranged_ingredient["quantity"] is None
assert ranged_ingredient["quantity_text"] is None
ranged_candidate = copy.deepcopy(candidate)
ranged_candidate["ingredients"][0] = ranged_ingredient
assert any(
error["code"] == "invalid_ingredient_quantity"
for error in _service().validate_recipe(ranged_candidate)["errors"]
)
@pytest.mark.parametrize("source_text", ["1/0 cup apples", "1/ cups apples", "1 1/0 cups apples"])
def test_validation_quarantines_invalid_fraction_prefixes_and_nonfinite_quantities(source_text):
candidate = _service().normalize_page(_parse_fixture("apple_glazed_pork_chops_parse.json"))
invalid_fraction = _normalize_ingredient(source_text)
assert invalid_fraction["quantity"] is None
assert invalid_fraction["quantity_text"] is None
invalid_candidate = copy.deepcopy(candidate)
invalid_candidate["ingredients"][0] = invalid_fraction
assert any(
error["code"] == "invalid_ingredient_quantity"
for error in _service().validate_recipe(invalid_candidate)["errors"]
)
for value in (math.nan, math.inf, -math.inf):
nonfinite_candidate = copy.deepcopy(candidate)
nonfinite_candidate["ingredients"][0]["quantity"] = value
assert any(
error["code"] == "invalid_ingredient_quantity"
for error in _service().validate_recipe(nonfinite_candidate)["errors"]
)
def test_staging_uses_atomic_insert_and_owns_payload_snapshot():
parse_payload = _parse_fixture("apple_glazed_pork_chops_parse.json")
original_text = parse_payload["text"]
db = FakeStageDb()
assert _service().stage_parse_payload(db, parse_payload) is db.winner
parse_payload["text"] = "mutated after staging"
parse_payload["sections"][0]["line"] = "mutated"
compiled = db.statements[0].compile(dialect=postgresql.dialect())
assert compiled.params["raw_payload"]["text"] == original_text
assert compiled.params["raw_payload"]["sections"][0]["line"] == "Ingredients"
assert "ON CONFLICT ON CONSTRAINT uq_recipe_import_staging_source_revision DO NOTHING" in str(
compiled
)
assert "RETURNING recipe_import_staging.id" in str(compiled)
assert len(db.statements) == 2
@pytest.mark.requires_postgres
def test_staging_checksums_deduplicates_and_quarantines_invalid_revisions(db, monkeypatch):
valid_parse = _parse_fixture("apple_glazed_pork_chops_parse.json")
invalid_parse = _parse_fixture("apple_glaze_missing_apple_parse.json")
service = _service()
monkeypatch.setattr(service, "fetch_page", lambda title: copy.deepcopy(valid_parse))
staged = service.stage_page(db, "Cookbook:Apple Glazed Pork Chops")
same_revision = service.stage_page(db, "Cookbook:Apple Glazed Pork Chops")
assert staged is same_revision
assert staged.status is RecipeImportStatus.PENDING
assert staged.raw_payload == valid_parse
assert staged.payload_sha256 == canonical_json_sha256(valid_parse)
assert staged.source_url == "https://en.wikibooks.org/wiki/Cookbook:Apple_Glazed_Pork_Chops"
assert staged.license_url == CC_BY_SA_4_LICENSE_URL
assert "Wikibooks contributors" in staged.attribution
assert "action=history" in staged.attribution
assert "normalized_recipe" not in staged.raw_payload
monkeypatch.setattr(service, "fetch_page", lambda title: copy.deepcopy(invalid_parse))
quarantined = service.stage_page(db, "Cookbook:Pan-Seared Pork Chops with Apple Glaze")
assert quarantined.status is RecipeImportStatus.QUARANTINED
assert any(
error["code"] == "title_ingredient_mismatch"
for error in quarantined.validation_report["errors"]
)
assert db.query(RecipeImportStaging).count() == 2
@pytest.mark.requires_postgres
def test_revalidation_updates_only_unreviewed_candidates_without_changing_raw_payload(db):
service = _service()
parse_payload = _parse_fixture("apple_glazed_pork_chops_parse.json")
parse_payload["pageid"] = 2_000_000_001
parse_payload["revid"] = 7_001
staged = service.stage_parse_payload(db, parse_payload)
raw_payload = copy.deepcopy(staged.raw_payload)
payload_sha256 = staged.payload_sha256
staged.status = RecipeImportStatus.QUARANTINED
staged.validation_report = {
"valid": False,
"errors": [{"code": "obsolete_validator_error"}],
"warnings": [],
}
staged.rejection_reason = "obsolete_validator_error"
tampered_checksum_payload = copy.deepcopy(parse_payload)
tampered_checksum_payload["pageid"] = 2_000_000_002
tampered_checksum_payload["revid"] = 7_002
tampered_checksum = service.stage_parse_payload(db, tampered_checksum_payload)
tampered_checksum.payload_sha256 = "0" * 64
reviewed_payload = copy.deepcopy(parse_payload)
reviewed_payload["pageid"] = 2_000_000_003
reviewed_payload["revid"] = 7_003
reviewed = service.stage_parse_payload(db, reviewed_payload)
reviewed.status = RecipeImportStatus.REJECTED
reviewed.reviewed_at = datetime.now(timezone.utc)
reviewed.reviewed_by = "reviewer@example.com"
reviewed.rejection_reason = "manual rejection"
approved_payload = copy.deepcopy(parse_payload)
approved_payload["pageid"] = 2_000_000_004
approved_payload["revid"] = 7_004
approved = service.stage_parse_payload(db, approved_payload)
approved.status = RecipeImportStatus.APPROVED
approved.reviewed_at = datetime.now(timezone.utc)
approved.reviewed_by = "reviewer@example.com"
reviewed_quarantine_payload = copy.deepcopy(parse_payload)
reviewed_quarantine_payload["pageid"] = 2_000_000_005
reviewed_quarantine_payload["revid"] = 7_005
reviewed_quarantine = service.stage_parse_payload(db, reviewed_quarantine_payload)
reviewed_quarantine.status = RecipeImportStatus.QUARANTINED
reviewed_quarantine.reviewed_at = datetime.now(timezone.utc)
reviewed_quarantine.reviewed_by = "reviewer@example.com"
reviewed_quarantine.rejection_reason = "manual quarantine"
db.flush()
result = service.revalidate_staged_imports(db, limit=10)
assert result == {"processed": 2, "pending": 1, "quarantined": 1}
assert staged.status is RecipeImportStatus.PENDING
assert staged.validation_report["valid"] is True
assert staged.rejection_reason is None
assert staged.raw_payload == raw_payload
assert staged.payload_sha256 == payload_sha256
assert tampered_checksum.status is RecipeImportStatus.QUARANTINED
assert tampered_checksum.validation_report["errors"] == [
{
"code": "payload_checksum_mismatch",
"field": "raw_payload",
"message": "Raw payload checksum does not match immutable provenance.",
}
]
assert reviewed.status is RecipeImportStatus.REJECTED
assert reviewed.rejection_reason == "manual rejection"
assert approved.status is RecipeImportStatus.APPROVED
assert reviewed_quarantine.status is RecipeImportStatus.QUARANTINED
assert reviewed_quarantine.rejection_reason == "manual quarantine"
def test_revalidation_uses_skip_locked():
db = FakeStageDb()
db.winner = None
assert _service().revalidate_staged_imports(db, limit=10) == {
"processed": 0,
"pending": 0,
"quarantined": 0,
}
compiled = db.statements[0].compile(dialect=postgresql.dialect())
assert "FOR UPDATE SKIP LOCKED" in str(compiled)
@pytest.mark.requires_postgres
def test_staging_deduplicates_two_concurrent_sessions(db):
engine = db.get_bind().engine
Session = sessionmaker(bind=engine, autoflush=False)
parse_payload = _parse_fixture("apple_glazed_pork_chops_parse.json")
external_id = str(uuid.uuid4().int % 2_000_000_000 + 1)
parse_payload["pageid"] = int(external_id)
parse_payload["revid"] = 6001
identity = {
"source": "wikibooks",
"external_id": external_id,
"source_revision": "6001",
}
start = threading.Barrier(2)
release_winner = threading.Event()
results = Queue()
errors = Queue()
service = _service()
def stage_in_session():
session = Session()
try:
session.execute(text("SET LOCAL lock_timeout = '3s'"))
session.execute(text("SET LOCAL statement_timeout = '10s'"))
start.wait(timeout=5)
staged = service.stage_parse_payload(session, copy.deepcopy(parse_payload))
results.put(staged.id)
if not release_winner.wait(timeout=5):
raise TimeoutError("winner transaction was not released")
session.commit()
except Exception as exc: # Propagate worker failures to the test thread.
session.rollback()
errors.put(exc)
finally:
session.rollback()
session.close()
threads = [threading.Thread(target=stage_in_session) for _ in range(2)]
for thread in threads:
thread.start()
def next_result(timeout: float):
try:
return results.get(timeout=timeout)
except Empty:
if not errors.empty():
raise errors.get()
pytest.fail("concurrent staging did not complete before timeout")
try:
first_id = next_result(5)
release_winner.set()
second_id = next_result(10)
for thread in threads:
thread.join(timeout=5)
assert not thread.is_alive(), "concurrent staging thread did not exit"
if not errors.empty():
raise errors.get()
verifier = Session()
try:
persisted = verifier.query(RecipeImportStaging).filter_by(**identity).all()
assert first_id == second_id == persisted[0].id
assert len(persisted) == 1
finally:
verifier.rollback()
verifier.close()
finally:
release_winner.set()
for thread in threads:
thread.join(timeout=5)
cleanup = Session()
try:
cleanup.query(RecipeImportStaging).filter_by(**identity).delete(
synchronize_session=False
)
cleanup.commit()
finally:
cleanup.rollback()
cleanup.close()