admin/Meal-Planner
Public Access
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(meals): enforce three lower-calorie dinners with media
CI / backend (pytest + alembic) (push) Canceled after 0s
Details
CI / frontend (build) (push) Canceled after 0s
Details

Browse Source
This commit is contained in:
admin
2026-09-01 10:11:36 -07:00
parent 7838c49721
commit 2695ba0232
85 changed files with 8103 additions and 795 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/app/services/image_generation.py
+5 -2
View File
@@ -17,7 +17,7 @@ import httpx
from sqlalchemy.orm import Session
from app.config import settings
from app.models import Recipe
from app.models import Recipe, RecipeCatalogStatus
logger = logging.getLogger(__name__)
@@ -156,6 +156,7 @@ def generate_image_for_recipe(
# Persist on recipe row if a DB session was provided.
if db is not None:
recipe.image_url = public_path
recipe.image_source = f"openai:{model}"
db.add(recipe)
db.commit()
logger.info("Image saved for '%s' -> %s", recipe.name, public_path)
@@ -183,7 +184,9 @@ def generate_images_batch(
"""
from sqlalchemy import or_
query = db.query(Recipe)
query = db.query(Recipe).filter(
Recipe.catalog_status == RecipeCatalogStatus.ACTIVE
)
if recipe_ids:
query = query.filter(Recipe.id.in_(recipe_ids))
backend/app/services/orchestrator/steps.py
+43 -33
View File
@@ -17,7 +17,7 @@ from datetime import datetime, timezone
from typing import TYPE_CHECKING
from app.config import settings
from app.models import FamilyMember, FamilyProfile, Ingredient, IngredientGroceryMatch, MealPlan, MealPlanItemStatus, MealPlanVote
from app.models import FamilyMember, FamilyProfile, Ingredient, IngredientGroceryMatch, MealPlan, MealPlanItemStatus, MealPlanVote, RecipeCatalogStatus
from app.services.approval import issue_token
from app.services.email import get_email_backend
from app.services.orchestrator.alerts import send_admin_alert
@@ -26,8 +26,7 @@ from app.services.scraper_service import ScraperService
from app.utils.units import convert_qty
from app.services.feedback_analyzer import FeedbackAnalyzer
from app.services.recipe_discovery import RecipeDiscoveryService
from app.services.recipe_ingestion import RecipeIngestionService
from app.services.wikibooks_import import WikibooksImportService
if TYPE_CHECKING:
from sqlalchemy.orm import Session
@@ -56,6 +55,15 @@ def _suggested_sides_html(components: dict | None) -> str:
)
def _active_recipe_items(items: list) -> list:
return [
item
for item in items
if item.recipe is not None
and item.recipe.catalog_status == RecipeCatalogStatus.ACTIVE
]
def step_scrape(run: "WeeklyRun", db: "Session") -> None:
if run.scraped_at is not None:
logger.info("step_scrape: already done for %s, skipping", run.week_start_date)
@@ -132,6 +140,12 @@ def step_email(run: "WeeklyRun", db: "Session") -> None:
f"No MealPlan for family {run.family_id} week {run.week_start_date}"
)
pending_items = [
item
for item in _active_recipe_items(plan.items)
if item.approval_status == MealPlanItemStatus.pending
]
members = (
db.query(FamilyMember)
.filter(
@@ -150,11 +164,10 @@ def step_email(run: "WeeklyRun", db: "Session") -> None:
# Preload ingredient names for all pending items
ing_ids: set = set()
for _item in plan.items:
if _item.approval_status == MealPlanItemStatus.pending and _item.recipe:
for _ing in (_item.recipe.ingredients or []):
if "ingredient_id" in _ing:
ing_ids.add(_ing["ingredient_id"])
for _item in pending_items:
for _ing in (_item.recipe.ingredients or []):
if "ingredient_id" in _ing:
ing_ids.add(_ing["ingredient_id"])
ingredient_names: dict = {}
if ing_ids:
@@ -165,10 +178,8 @@ def step_email(run: "WeeklyRun", db: "Session") -> None:
# Build shopping list preview once (shared across all member emails)
all_ingredients: dict[str, tuple[str, str, str, str]] = {}
for item in plan.items:
if item.approval_status != MealPlanItemStatus.pending:
continue
for ing in (item.recipe.ingredients or [] if item.recipe else []):
for item in pending_items:
for ing in (item.recipe.ingredients or []):
ing_name = ingredient_names.get(str(ing.get("ingredient_id", "")), ing.get("name", "unknown")).strip()
if not ing_name or ing_name in all_ingredients:
continue
@@ -217,9 +228,7 @@ def step_email(run: "WeeklyRun", db: "Session") -> None:
backend = get_email_backend()
for member in members:
item_html_parts = []
for item in plan.items:
if item.approval_status != MealPlanItemStatus.pending:
continue
for item in pending_items:
token = issue_token(item.id, member.id)
vote_url = (
f"{settings.APP_BASE_URL}/api/meals/vote/{item.id}?token={token}"
@@ -400,7 +409,7 @@ def step_finalize(run: "WeeklyRun", db: "Session") -> None:
approved_items = [
item
for item in (plan.items if plan else [])
for item in _active_recipe_items(plan.items if plan else [])
if item.approval_status == MealPlanItemStatus.approved
]
@@ -524,18 +533,19 @@ def step_finalize(run: "WeeklyRun", db: "Session") -> None:
analysis.discovery_queries and
analysis.confidence >= 0.5
):
discovery = RecipeDiscoveryService()
candidates = discovery.discover(analysis.discovery_queries)
if candidates:
ingestion = RecipeIngestionService()
added = ingestion.ingest(db, run.family_id, candidates, analysis.to_dict())
logger.info(
"step_finalize: recipe discovery added %d new recipes for family %s",
added, run.family_id,
# Roll back a partial discovery batch without losing finalization.
with db.begin_nested():
staged = WikibooksImportService(
settings.WIKIBOOKS_USER_AGENT
).stage_discovered_pages(
db,
limit=min(5, len(analysis.discovery_queries)),
queries=analysis.discovery_queries,
)
# ingestion deliberately does not commit so orchestrator
# can keep everything in one transaction
db.commit()
logger.info(
"step_finalize: staged %d Wikibooks candidates for review",
len(staged),
)
except Exception as exc:
# Discovery is best-effort; never block finalization
logger.warning("step_finalize: recipe discovery failed: %s", exc)
@@ -565,11 +575,12 @@ def step_reminder(run: "WeeklyRun", db: "Session") -> None:
db.commit()
return
pending_item_ids = [
item.id
for item in plan.items
pending_items = [
item
for item in _active_recipe_items(plan.items)
if item.approval_status == MealPlanItemStatus.pending
]
pending_item_ids = [item.id for item in pending_items]
if not pending_item_ids:
run.reminded_at = datetime.now(timezone.utc)
db.commit()
@@ -597,9 +608,8 @@ def step_reminder(run: "WeeklyRun", db: "Session") -> None:
}
unvoted = [
item
for item in plan.items
if item.approval_status == MealPlanItemStatus.pending
and item.id not in voted_ids
for item in pending_items
if item.id not in voted_ids
]
if not unvoted:
continue
backend/app/services/planner/config.py
+2
View File
@@ -37,6 +37,8 @@ class PlannerConfig:
total = self.w_savings + self.w_coverage + self.w_pantry + self.w_time + self.w_recency
if abs(total - 1.0) > 1e-6:
raise ValueError(f"weights must sum to 1.0, got {total}")
if self.set_size != 3:
raise ValueError("set_size must be 3 dinners per week")
def merge(self, overrides: "dict[str, Any]") -> "PlannerConfig":
"""Return a new PlannerConfig with overrides applied."""
backend/app/services/planner/generate.py
+8 -1
View File
@@ -3,6 +3,7 @@ from __future__ import annotations
from datetime import date
from decimal import Decimal
import logging
from typing import Dict, List, Optional, Set
from uuid import UUID
@@ -22,6 +23,7 @@ from app.models import (
MealType,
NeverSuggest,
Recipe,
RecipeCatalogStatus,
)
from app.services.planner.config import DEFAULT, PlannerConfig
from app.services.planner.cost import compute_recipe_cost
@@ -32,6 +34,9 @@ from app.services.planner.types import GenerationResult
from app.services.meal_pairings import components_with_suggested_sides
logger = logging.getLogger(__name__)
def _load_match_index(db: Session) -> Dict[UUID, List[dict]]:
"""ingredient_id → list of match dicts ordered by confidence DESC."""
rows = (
@@ -148,7 +153,9 @@ def generate_meal_plan(
except (ValueError, TypeError) as exc:
logger.warning("Invalid planner_config for family %s: %s", family_id, exc)
recipes = db.query(Recipe).all()
recipes = db.query(Recipe).filter(
Recipe.catalog_status == RecipeCatalogStatus.ACTIVE
).all()
exclude_set = exclude_recipe_ids or set()
recipe_dicts = [
{
backend/app/services/recipe_catalog_audit.py
+221
View File
@@ -0,0 +1,221 @@
"""Deterministic provenance and quality audit for the published recipe catalog."""
from __future__ import annotations
from datetime import datetime, timezone
from decimal import Decimal
from typing import Any
from sqlalchemy import and_
from sqlalchemy.orm import Session
from app.models import (
Ingredient,
Recipe,
RecipeCatalogStatus,
RecipeImportStaging,
RecipeImportStatus,
)
from app.services.wikibooks_import import WIKIBOOKS_SOURCE, WikibooksImportService
OPERATOR_QUARANTINE_FINDING_CODE = "operator_quarantine"
def audit_recipe_catalog(
db: Session,
*,
apply: bool = False,
reviewed_at: datetime | None = None,
) -> dict[str, Any]:
"""Audit every recipe and optionally apply active/quarantined state changes."""
timestamp = reviewed_at or datetime.now(timezone.utc)
ingredient_names = {
str(ingredient_id): name
for ingredient_id, name in db.query(Ingredient.id, Ingredient.name).all()
}
catalog_query = (
db.query(
Recipe,
RecipeImportStaging.source,
RecipeImportStaging.external_id,
RecipeImportStaging.source_url,
)
.outerjoin(
RecipeImportStaging,
and_(
RecipeImportStaging.recipe_id == Recipe.id,
RecipeImportStaging.status == RecipeImportStatus.APPROVED,
),
)
.order_by(Recipe.id, RecipeImportStaging.id)
)
if apply:
catalog_query = catalog_query.with_for_update(of=Recipe)
recipes: dict[str, Recipe] = {}
approved_links: dict[str, set[tuple[str, str, str]]] = {}
for recipe, source, external_id, source_url in catalog_query.all():
recipe_id = str(recipe.id)
recipes[recipe_id] = recipe
if source is not None and external_id is not None and source_url is not None:
approved_links.setdefault(recipe_id, set()).add(
(source, external_id, source_url)
)
validator = WikibooksImportService("MealPlanner catalog audit")
records: list[dict[str, Any]] = []
for recipe_id, recipe in recipes.items():
prior_findings = (
recipe.catalog_findings
if isinstance(recipe.catalog_findings, list)
else []
)
operator_quarantine_findings = [
finding
for finding in prior_findings
if isinstance(finding, dict)
and finding.get("code") == OPERATOR_QUARANTINE_FINDING_CODE
]
provenance, findings = _provenance_findings(
recipe, approved_links.get(recipe_id, set())
)
candidate, reference_findings = _validation_candidate(recipe, ingredient_names)
findings.extend(reference_findings)
validation = validator.validate_recipe(candidate)
findings.extend(
{**issue, "severity": severity}
for severity, key in (("error", "errors"), ("warning", "warnings"))
for issue in validation[key]
)
findings.extend(operator_quarantine_findings)
target_status = (
RecipeCatalogStatus.QUARANTINED
if operator_quarantine_findings
or any(item["severity"] == "error" for item in findings)
else RecipeCatalogStatus.ACTIVE
)
if apply:
recipe.catalog_status = target_status
recipe.catalog_findings = findings
recipe.catalog_reviewed_at = timestamp
records.append(
{
"id": str(recipe.id),
"name": recipe.name,
"provenance": provenance,
"catalog_status": target_status.value,
"findings": findings,
}
)
if apply:
db.flush()
quarantined = sum(
record["catalog_status"] == RecipeCatalogStatus.QUARANTINED.value
for record in records
)
return {
"version": 1,
"generated_at": timestamp.isoformat(),
"applied": apply,
"total": len(records),
"summary": {
"active": len(records) - quarantined,
"quarantined": quarantined,
},
"recipes": records,
}
def _provenance_findings(
recipe: Recipe, approved_links: set[tuple[str, str, str]]
) -> tuple[str, list[dict[str, str]]]:
if recipe.is_manually_added and not recipe.external_source:
return "manual", []
if recipe.external_source == WIKIBOOKS_SOURCE:
findings = []
expected_link = (recipe.external_source, recipe.external_id, recipe.source_url)
if expected_link not in approved_links:
findings.append(
_finding(
"missing_import_provenance",
"external_source",
"Published Wikibooks recipe is not linked to an approved staged import.",
)
)
if not recipe.source_url:
findings.append(
_finding("missing_source_url", "source_url", "External recipe has no source URL.")
)
return "staged_wikibooks", findings
if recipe.external_source == "spoonacular":
return "spoonacular", [
_finding(
"unlicensed_external_source",
"external_source",
"Spoonacular data was persisted without durable catalog rights.",
)
]
if recipe.external_source:
return "external_unverified", [
_finding(
"unverified_external_source",
"external_source",
"External recipe has no approved staged license record.",
)
]
return "missing", [
_finding(
"missing_provenance",
"external_source",
"Non-manual recipe has no auditable source or license record.",
)
]
def _validation_candidate(
recipe: Recipe, ingredient_names: dict[str, str]
) -> tuple[dict[str, Any], list[dict[str, str]]]:
ingredients: list[dict[str, Any]] = []
findings: list[dict[str, str]] = []
for index, item in enumerate(recipe.ingredients or []):
if not isinstance(item, dict):
findings.append(
_finding(
"invalid_ingredient_reference",
f"ingredients[{index}]",
"Ingredient reference must be an object.",
)
)
continue
ingredient_id = str(item.get("ingredient_id") or "")
name = ingredient_names.get(ingredient_id)
if not name:
findings.append(
_finding(
"orphan_ingredient_reference",
f"ingredients[{index}].ingredient_id",
"Ingredient reference does not resolve to the canonical ingredient table.",
)
)
name = str(item.get("name") or "")
quantity = item.get("qty")
if isinstance(quantity, Decimal):
quantity = float(quantity)
ingredients.append({"name": name, "quantity": quantity})
return (
{
"name": recipe.name,
"description": recipe.description,
"servings": recipe.servings,
"prep_time_minutes": recipe.prep_time_minutes,
"cook_time_minutes": recipe.cook_time_minutes,
"baking_time_minutes": None,
"ingredients": ingredients,
"instructions": list(recipe.instructions or []),
},
findings,
)
def _finding(code: str, field: str, message: str) -> dict[str, str]:
return {"code": code, "severity": "error", "field": field, "message": message}
backend/app/services/recipe_import_review.py
+196
View File
@@ -0,0 +1,196 @@
from __future__ import annotations
from datetime import datetime, timezone
from typing import Any, Mapping
from uuid import UUID
from sqlalchemy import select
from sqlalchemy.dialects.postgresql import insert as pg_insert
from sqlalchemy.orm import Session
from app.models import (
Ingredient,
Recipe,
RecipeCatalogStatus,
RecipeImportStaging,
RecipeImportStatus,
)
from app.services.wikibooks_import import (
WIKIBOOKS_SOURCE,
WikibooksImportService,
canonical_json_sha256,
)
class RecipeImportNotFoundError(LookupError):
"""Raised when a staged recipe does not exist."""
class RecipeImportConflictError(ValueError):
"""Raised when a review transition or publication is unsafe."""
def approve_recipe_import(
db: Session, import_id: UUID, reviewed_by: str
) -> tuple[RecipeImportStaging, bool]:
"""Atomically publish one pending Wikibooks candidate."""
staged = _locked_import(db, import_id)
if staged.status == RecipeImportStatus.APPROVED:
if staged.recipe_id and db.get(Recipe, staged.recipe_id):
return staged, False
elif staged.status != RecipeImportStatus.PENDING:
raise RecipeImportConflictError(
f"Only pending imports can be approved; current status is {staged.status.value}"
)
if staged.source != WIKIBOOKS_SOURCE:
raise RecipeImportConflictError(f"Unsupported recipe source: {staged.source}")
raw_payload = _mapping(staged.raw_payload, "raw_payload")
if canonical_json_sha256(raw_payload) != staged.payload_sha256:
raise RecipeImportConflictError("Raw payload checksum does not match provenance")
normalizer = WikibooksImportService("MealPlanner recipe review")
normalized = normalizer.normalize_page(raw_payload)
validation_report = normalizer.validate_recipe(normalized)
if not validation_report["valid"]:
raise RecipeImportConflictError("Candidate no longer passes publication validation")
existing = (
db.query(Recipe)
.filter(
Recipe.external_source == staged.source,
Recipe.external_id == staged.external_id,
)
.first()
)
if existing:
raise RecipeImportConflictError(
"A recipe from this source page is already published"
)
ingredient_refs = _ingredient_refs(db, normalized["ingredients"])
cook_time = (normalized.get("cook_time_minutes") or 0) + (
normalized.get("baking_time_minutes") or 0
)
recipe = Recipe(
family_profile_id=None,
name=normalized["name"],
description=normalized.get("description"),
image_url=None,
image_source=None,
prep_time_minutes=normalized.get("prep_time_minutes"),
cook_time_minutes=cook_time or None,
servings=normalized["servings"],
cuisine_tags=[],
dietary_tags=[],
protein_type=None,
spice_level=None,
calories_per_serving=None,
ingredients=ingredient_refs,
side_dishes=[],
instructions=list(normalized["instructions"]),
source_url=staged.source_url,
scraped_at=staged.acquired_at,
is_manually_added=False,
external_source=staged.source,
external_id=staged.external_id,
discovery_reason=f"Approved import from revision {staged.source_revision}",
catalog_status=RecipeCatalogStatus.ACTIVE,
catalog_findings=[],
)
db.add(recipe)
db.flush()
staged.normalized_recipe = normalized
staged.validation_report = validation_report
staged.status = RecipeImportStatus.APPROVED
staged.rejection_reason = None
staged.recipe_id = recipe.id
staged.reviewed_at = datetime.now(timezone.utc)
staged.reviewed_by = reviewed_by[:300]
db.commit()
db.refresh(staged)
return staged, True
def reject_recipe_import(
db: Session, import_id: UUID, reason: str, reviewed_by: str
) -> RecipeImportStaging:
"""Atomically reject a pending or quarantined candidate."""
staged = _locked_import(db, import_id)
if staged.status == RecipeImportStatus.APPROVED:
raise RecipeImportConflictError("Approved imports cannot be rejected")
if staged.status == RecipeImportStatus.REJECTED:
if staged.rejection_reason == reason:
return staged
raise RecipeImportConflictError("Import was already rejected with another reason")
staged.status = RecipeImportStatus.REJECTED
staged.rejection_reason = reason
staged.reviewed_at = datetime.now(timezone.utc)
staged.reviewed_by = reviewed_by[:300]
db.commit()
db.refresh(staged)
return staged
def _locked_import(db: Session, import_id: UUID) -> RecipeImportStaging:
staged = db.execute(
select(RecipeImportStaging)
.where(RecipeImportStaging.id == import_id)
.with_for_update()
).scalar_one_or_none()
if staged is None:
raise RecipeImportNotFoundError("Recipe import not found")
return staged
def _ingredient_refs(
db: Session, ingredients: list[Mapping[str, Any]]
) -> list[dict[str, Any]]:
prepared: list[tuple[str, str, float, str | None, str | None]] = []
for item in ingredients:
name = " ".join(str(item.get("name", "")).split())
name_lower = name.casefold()
quantity = item.get("quantity")
unit = item.get("unit")
source_text = item.get("source_text")
if not name or len(name) > 200:
raise RecipeImportConflictError("Ingredient name is invalid")
if not isinstance(quantity, (int, float)) or isinstance(quantity, bool) or quantity <= 0:
raise RecipeImportConflictError("Ingredient quantity is invalid")
if unit is not None and (not isinstance(unit, str) or len(unit) > 50):
raise RecipeImportConflictError("Ingredient unit is invalid")
prepared.append((name, name_lower, float(quantity), unit, source_text))
for name, name_lower, _, unit, _ in prepared:
db.execute(
pg_insert(Ingredient.__table__)
.values(name=name, name_lower=name_lower, unit=unit, aliases=[])
.on_conflict_do_nothing(index_elements=[Ingredient.name_lower])
)
canonical = {
row.name_lower: row
for row in db.query(Ingredient)
.filter(Ingredient.name_lower.in_([item[1] for item in prepared]))
.all()
}
if len(canonical) != len({item[1] for item in prepared}):
raise RecipeImportConflictError("Canonical ingredient creation failed")
return [
{
"ingredient_id": str(canonical[name_lower].id),
"qty": quantity,
"unit": unit,
"notes": source_text,
}
for _, name_lower, quantity, unit, source_text in prepared
]
def _mapping(value: Any, field: str) -> Mapping[str, Any]:
if not isinstance(value, Mapping):
raise RecipeImportConflictError(f"{field} must be an object")
return value
backend/app/services/starter_recipe_catalog.py
+540
View File
@@ -0,0 +1,540 @@
"""Validate and atomically load the bundled starter recipe catalog."""
from __future__ import annotations
import json
import math
import re
from pathlib import Path
from typing import Any, Mapping
from uuid import UUID
from sqlalchemy import func, select
from sqlalchemy.dialects.postgresql import insert as pg_insert
from sqlalchemy.orm import Session
from app.models import Ingredient, Recipe, RecipeCatalogStatus
STARTER_CATALOG_VERSION = 2
STARTER_RECIPE_COUNT = 50
STARTER_DISCOVERY_REASON = "bundled_starter_catalog_v1"
# Fixed PostgreSQL bigint advisory-lock key for this loader (ASCII: MPLSTART).
# Changing it would allow concurrent catalog loads to bypass each other.
STARTER_CATALOG_ADVISORY_LOCK_ID = 0x4D504C5354415254
DEFAULT_CATALOG_PATH = Path(__file__).resolve().parents[1] / "data" / "starter_recipes.json"
MAX_SERVINGS = 50
MAX_TIME_MINUTES = 1_440
MAX_INGREDIENT_QUANTITY = 10_000
MAX_CALORIES_PER_SERVING = 500
MAX_SPICE_LEVEL = 5
_PLACEHOLDER_TEXT = frozenset(
{"coming soon", "directions", "instructions", "method", "n a", "tbd", "todo"}
)
_PLACEHOLDER_PHRASES = ("lorem ipsum", "placeholder")
_RECIPE_KEYS = frozenset(
{
"name",
"description",
"servings",
"prep_time_minutes",
"cook_time_minutes",
"cuisine_tags",
"dietary_tags",
"protein_type",
"spice_level",
"calories_per_serving",
"ingredients",
"instructions",
}
)
_INGREDIENT_KEYS = frozenset({"name", "quantity", "unit"})
_CATALOG_SCALAR_FIELDS = (
"description",
"servings",
"prep_time_minutes",
"cook_time_minutes",
"cuisine_tags",
"dietary_tags",
"protein_type",
"spice_level",
"calories_per_serving",
)
class StarterRecipeCatalogError(ValueError):
"""Raised when the bundled catalog is incomplete or unsafe to load."""
def read_starter_catalog(path: Path = DEFAULT_CATALOG_PATH) -> Mapping[str, Any]:
"""Read the JSON catalog without accepting non-standard numeric constants."""
try:
with path.open(encoding="utf-8") as catalog_file:
catalog = json.load(catalog_file, parse_constant=_reject_json_constant)
except OSError as exc:
raise StarterRecipeCatalogError(f"Catalog cannot be read: {path}") from exc
except ValueError as exc:
raise StarterRecipeCatalogError("Catalog is not valid JSON") from exc
if not isinstance(catalog, Mapping):
raise StarterRecipeCatalogError("Catalog root must be an object")
return catalog
def validate_starter_catalog(catalog: Mapping[str, Any]) -> list[dict[str, Any]]:
"""Return normalized recipes after validating the complete, fixed-size catalog."""
_require_exact_keys(catalog, {"version", "recipes"}, "catalog")
if catalog["version"] != STARTER_CATALOG_VERSION:
raise StarterRecipeCatalogError(
f"Catalog version must be {STARTER_CATALOG_VERSION}"
)
recipes = catalog["recipes"]
if not isinstance(recipes, list) or len(recipes) != STARTER_RECIPE_COUNT:
raise StarterRecipeCatalogError(
f"Catalog must contain exactly {STARTER_RECIPE_COUNT} recipes"
)
normalized: list[dict[str, Any]] = []
names: set[str] = set()
for index, recipe in enumerate(recipes, 1):
if not isinstance(recipe, Mapping):
raise StarterRecipeCatalogError(f"Recipe {index} must be an object")
_require_exact_keys(recipe, _RECIPE_KEYS, f"Recipe {index}")
name = _text(recipe["name"], f"Recipe {index} name", maximum=300)
name_key = _canonical_name(name)
if name_key in names:
raise StarterRecipeCatalogError(f"Recipe {index} name is not unique")
names.add(name_key)
ingredients = _validate_ingredients(recipe["ingredients"], index)
instructions = _validate_instructions(recipe["instructions"], index)
normalized.append(
{
"name": name,
"description": _text(
recipe["description"], f"Recipe {index} description", maximum=10_000
),
"servings": _positive_int(
recipe["servings"], f"Recipe {index} servings", MAX_SERVINGS
),
"prep_time_minutes": _positive_int(
recipe["prep_time_minutes"],
f"Recipe {index} prep_time_minutes",
MAX_TIME_MINUTES,
),
"cook_time_minutes": _positive_int(
recipe["cook_time_minutes"],
f"Recipe {index} cook_time_minutes",
MAX_TIME_MINUTES,
),
"cuisine_tags": _tags(recipe["cuisine_tags"], f"Recipe {index} cuisine_tags"),
"dietary_tags": _tags(recipe["dietary_tags"], f"Recipe {index} dietary_tags"),
"protein_type": _text(
recipe["protein_type"], f"Recipe {index} protein_type", maximum=50
),
"spice_level": _bounded_int(
recipe["spice_level"], f"Recipe {index} spice_level", 0, MAX_SPICE_LEVEL
),
"calories_per_serving": _positive_int(
recipe["calories_per_serving"],
f"Recipe {index} calories_per_serving",
MAX_CALORIES_PER_SERVING,
),
"ingredients": ingredients,
"instructions": instructions,
}
)
return normalized
def load_bundled_starter_catalog(
db: Session,
*,
path: Path = DEFAULT_CATALOG_PATH,
dry_run: bool = False,
update_existing: bool = False,
) -> dict[str, Any]:
"""Load the bundled JSON catalog after validating all recipes before writes."""
return load_starter_catalog(
db,
read_starter_catalog(path),
dry_run=dry_run,
update_existing=update_existing,
)
def load_starter_catalog(
db: Session,
catalog: Mapping[str, Any],
*,
dry_run: bool = False,
update_existing: bool = False,
) -> dict[str, Any]:
"""Load or explicitly synchronize bundled recipes in one transaction."""
recipes = validate_starter_catalog(catalog)
catalog_names = {recipe["name"] for recipe in recipes}
catalog_by_name = {recipe["name"]: recipe for recipe in recipes}
ingredient_names = {
_canonical_name(ingredient["name"]): ingredient
for recipe in recipes
for ingredient in recipe["ingredients"]
}
with db.begin():
# This transaction-scoped lock must precede every catalog Recipe read or write.
db.execute(select(func.pg_advisory_xact_lock(STARTER_CATALOG_ADVISORY_LOCK_ID)))
marked_recipes = list(
db.execute(
select(Recipe)
.where(Recipe.discovery_reason == STARTER_DISCOVERY_REASON)
.with_for_update()
).scalars()
)
marked_by_name = {recipe.name: recipe for recipe in marked_recipes}
if len(marked_by_name) != len(marked_recipes):
raise StarterRecipeCatalogError("Bundled catalog has duplicate persisted recipe names")
unknown_marked_names = set(marked_by_name).difference(catalog_names)
if unknown_marked_names:
raise StarterRecipeCatalogError(
"Bundled catalog marker is attached to an unknown recipe"
)
resolved_recipe_ingredients = _resolve_recipe_ingredients(db, marked_recipes)
missing_marked_names = catalog_names.difference(marked_by_name)
if update_existing and missing_marked_names:
raise StarterRecipeCatalogError(
"Update requires exactly all "
f"{STARTER_RECIPE_COUNT} existing marker-owned recipe names"
)
recipes_to_update: list[Recipe] = []
for recipe in marked_recipes:
expected = catalog_by_name[recipe.name]
if update_existing:
_assert_bundled_recipe_ownership(recipe)
if not _bundled_recipe_content_matches(
recipe, expected, resolved_recipe_ingredients
):
recipes_to_update.append(recipe)
else:
_assert_bundled_recipe(
recipe,
expected,
resolved_recipe_ingredients,
)
name_collisions = list(
db.execute(
select(Recipe).where(Recipe.name.in_(catalog_names)).with_for_update()
).scalars()
)
if any(recipe.discovery_reason != STARTER_DISCOVERY_REASON for recipe in name_collisions):
raise StarterRecipeCatalogError(
"Catalog recipe name conflicts with an existing non-bundled recipe"
)
canonical_ingredients = {
ingredient.name_lower: ingredient
for ingredient in db.execute(
select(Ingredient).where(Ingredient.name_lower.in_(ingredient_names))
).scalars()
}
new_recipes = (
[]
if update_existing
else [recipe for recipe in recipes if recipe["name"] not in marked_by_name]
)
summary = {
"catalog_version": STARTER_CATALOG_VERSION,
"dry_run": dry_run,
"validated_recipes": len(recipes),
"existing_recipes": len(marked_recipes),
"created_recipes": len(new_recipes),
"updated_recipes": len(recipes_to_update),
"existing_ingredients": len(canonical_ingredients),
"created_ingredients": len(ingredient_names) - len(canonical_ingredients),
}
if dry_run:
return summary
for name_lower, ingredient in ingredient_names.items():
db.execute(
pg_insert(Ingredient.__table__)
.values(
name=ingredient["name"],
name_lower=name_lower,
unit=ingredient["unit"],
aliases=[],
)
.on_conflict_do_nothing(index_elements=[Ingredient.name_lower])
)
canonical_ingredients = {
ingredient.name_lower: ingredient
for ingredient in db.execute(
select(Ingredient).where(Ingredient.name_lower.in_(ingredient_names))
).scalars()
}
if set(canonical_ingredients) != set(ingredient_names):
raise StarterRecipeCatalogError("Canonical ingredient creation failed")
for recipe in recipes_to_update:
_synchronize_bundled_recipe(
recipe,
catalog_by_name[recipe.name],
canonical_ingredients,
)
for recipe in new_recipes:
db.add(
Recipe(
family_profile_id=None,
name=recipe["name"],
description=recipe["description"],
image_url=None,
image_source=None,
prep_time_minutes=recipe["prep_time_minutes"],
cook_time_minutes=recipe["cook_time_minutes"],
servings=recipe["servings"],
cuisine_tags=recipe["cuisine_tags"],
dietary_tags=recipe["dietary_tags"],
protein_type=recipe["protein_type"],
spice_level=recipe["spice_level"],
calories_per_serving=recipe["calories_per_serving"],
ingredients=_catalog_recipe_ingredients(recipe, canonical_ingredients),
side_dishes=[],
instructions=recipe["instructions"],
source_url=None,
scraped_at=None,
is_manually_added=True,
external_source=None,
external_id=None,
discovery_reason=STARTER_DISCOVERY_REASON,
catalog_status=RecipeCatalogStatus.ACTIVE,
catalog_findings=[],
)
)
db.flush()
return summary
def _validate_ingredients(value: Any, recipe_index: int) -> list[dict[str, Any]]:
if not isinstance(value, list) or len(value) < 2:
raise StarterRecipeCatalogError(f"Recipe {recipe_index} needs at least two ingredients")
ingredients: list[dict[str, Any]] = []
for index, ingredient in enumerate(value, 1):
if not isinstance(ingredient, Mapping):
raise StarterRecipeCatalogError(
f"Recipe {recipe_index} ingredient {index} must be an object"
)
_require_exact_keys(
ingredient,
_INGREDIENT_KEYS,
f"Recipe {recipe_index} ingredient {index}",
)
ingredients.append(
{
"name": _text(
ingredient["name"],
f"Recipe {recipe_index} ingredient {index} name",
maximum=200,
),
"quantity": _positive_number(
ingredient["quantity"],
f"Recipe {recipe_index} ingredient {index} quantity",
MAX_INGREDIENT_QUANTITY,
),
"unit": _text(
ingredient["unit"], f"Recipe {recipe_index} ingredient {index} unit", maximum=50
),
}
)
return ingredients
def _validate_instructions(value: Any, recipe_index: int) -> list[str]:
if not isinstance(value, list) or len(value) < 2:
raise StarterRecipeCatalogError(f"Recipe {recipe_index} needs at least two instructions")
instructions = [
_text(instruction, f"Recipe {recipe_index} instruction {index}", maximum=5_000)
for index, instruction in enumerate(value, 1)
]
for instruction in instructions:
if len(instruction) < 20 or len(re.findall(r"[A-Za-z0-9]+", instruction)) < 4:
raise StarterRecipeCatalogError(
f"Recipe {recipe_index} instructions must be substantive"
)
return instructions
def _tags(value: Any, field: str) -> list[str]:
if not isinstance(value, list):
raise StarterRecipeCatalogError(f"{field} must be an array")
return [_text(tag, field, maximum=50) for tag in value]
def _text(value: Any, field: str, *, maximum: int) -> str:
if not isinstance(value, str):
raise StarterRecipeCatalogError(f"{field} must be text")
text = " ".join(value.split())
if not text or len(text) > maximum or _is_placeholder(text):
raise StarterRecipeCatalogError(f"{field} is missing, too long, or placeholder text")
return text
def _positive_int(value: Any, field: str, maximum: int) -> int:
return _bounded_int(value, field, 1, maximum)
def _bounded_int(value: Any, field: str, minimum: int, maximum: int) -> int:
if isinstance(value, bool) or not isinstance(value, int) or not minimum <= value <= maximum:
raise StarterRecipeCatalogError(f"{field} must be an integer between {minimum} and {maximum}")
return value
def _positive_number(value: Any, field: str, maximum: int) -> int | float:
if isinstance(value, bool) or not isinstance(value, (int, float)):
raise StarterRecipeCatalogError(f"{field} must be a number")
if not math.isfinite(value) or not 0 < value <= maximum:
raise StarterRecipeCatalogError(f"{field} must be greater than zero and at most {maximum}")
return value
def _require_exact_keys(
value: Mapping[str, Any], expected: set[str] | frozenset[str], field: str
) -> None:
if set(value) != set(expected):
raise StarterRecipeCatalogError(f"{field} has unexpected or missing fields")
def _canonical_name(value: str) -> str:
return " ".join(value.split()).casefold()
def _is_placeholder(value: str) -> bool:
normalized = re.sub(r"[^a-z0-9]+", " ", value.casefold()).strip()
return normalized in _PLACEHOLDER_TEXT or any(
phrase in normalized for phrase in _PLACEHOLDER_PHRASES
)
def _resolve_recipe_ingredients(
db: Session, recipes: list[Recipe]
) -> dict[str, Ingredient]:
ingredient_ids: set[UUID] = set()
for recipe in recipes:
if not isinstance(recipe.ingredients, list):
continue
for item in recipe.ingredients:
if not isinstance(item, Mapping):
continue
try:
ingredient_ids.add(UUID(str(item.get("ingredient_id"))))
except (TypeError, ValueError, AttributeError):
continue
if not ingredient_ids:
return {}
return {
str(ingredient.id): ingredient
for ingredient in db.execute(
select(Ingredient).where(Ingredient.id.in_(ingredient_ids))
).scalars()
}
def _assert_bundled_recipe(
recipe: Recipe,
expected: Mapping[str, Any],
resolved_ingredients: Mapping[str, Ingredient],
) -> None:
_assert_bundled_recipe_ownership(recipe)
if not _bundled_recipe_content_matches(recipe, expected, resolved_ingredients):
raise StarterRecipeCatalogError("Existing bundled recipe content has drifted")
def _assert_bundled_recipe_ownership(recipe: Recipe) -> None:
if (
recipe.family_profile_id is not None
or not recipe.is_manually_added
or recipe.external_source is not None
or recipe.external_id is not None
or recipe.source_url is not None
or recipe.scraped_at is not None
or recipe.image_url is not None
or recipe.image_source is not None
or recipe.servings_scaled is not None
or recipe.side_dishes != []
or recipe.discovery_reason != STARTER_DISCOVERY_REASON
or recipe.catalog_status != RecipeCatalogStatus.ACTIVE
or recipe.catalog_findings != []
or recipe.catalog_reviewed_at is not None
):
raise StarterRecipeCatalogError(
"Existing bundled recipe does not match the required catalog ownership"
)
def _bundled_recipe_content_matches(
recipe: Recipe,
expected: Mapping[str, Any],
resolved_ingredients: Mapping[str, Ingredient],
) -> bool:
if any(getattr(recipe, field) != expected[field] for field in _CATALOG_SCALAR_FIELDS):
return False
if list(recipe.instructions or []) != expected["instructions"]:
return False
if not isinstance(recipe.ingredients, list) or len(recipe.ingredients) != len(
expected["ingredients"]
):
return False
for actual, expected_ingredient in zip(recipe.ingredients, expected["ingredients"]):
if not isinstance(actual, Mapping) or set(actual) != {
"ingredient_id",
"qty",
"unit",
"notes",
}:
return False
resolved = resolved_ingredients.get(str(actual["ingredient_id"]))
quantity = actual["qty"]
if (
resolved is None
or resolved.name_lower != _canonical_name(expected_ingredient["name"])
or isinstance(quantity, bool)
or not isinstance(quantity, (int, float))
or not math.isfinite(quantity)
or quantity != expected_ingredient["quantity"]
or actual["unit"] != expected_ingredient["unit"]
or actual["notes"] is not None
):
return False
return True
def _synchronize_bundled_recipe(
recipe: Recipe,
expected: Mapping[str, Any],
canonical_ingredients: Mapping[str, Ingredient],
) -> None:
for field in _CATALOG_SCALAR_FIELDS:
value = expected[field]
setattr(recipe, field, list(value) if field.endswith("_tags") else value)
recipe.ingredients = _catalog_recipe_ingredients(expected, canonical_ingredients)
recipe.instructions = list(expected["instructions"])
def _catalog_recipe_ingredients(
recipe: Mapping[str, Any], canonical_ingredients: Mapping[str, Ingredient]
) -> list[dict[str, Any]]:
return [
{
"ingredient_id": str(
canonical_ingredients[_canonical_name(item["name"])].id
),
"qty": item["quantity"],
"unit": item["unit"],
"notes": None,
}
for item in recipe["ingredients"]
]
def _reject_json_constant(value: str) -> None:
raise ValueError(f"Unsupported JSON numeric constant: {value}")
backend/app/services/wikibooks_import.py
+973
View File
@@ -0,0 +1,973 @@
"""Bounded synchronous acquisition and staging for Wikibooks cookbook pages."""
from __future__ import annotations
import copy
import hashlib
import json
import math
import re
import time
from datetime import datetime, timezone
from email.utils import parsedate_to_datetime
from typing import Any, Callable, Mapping
from urllib.parse import quote
import requests
from bs4 import BeautifulSoup, Tag
from sqlalchemy import select
from sqlalchemy.dialects.postgresql import insert as pg_insert
from sqlalchemy.orm import Session
from app.models import RecipeImportStaging, RecipeImportStatus
WIKIBOOKS_API_URL = "https://en.wikibooks.org/w/api.php"
WIKIBOOKS_SOURCE = "wikibooks"
CC_BY_SA_4_LICENSE_NAME = "CC BY-SA 4.0"
CC_BY_SA_4_LICENSE_URL = "https://creativecommons.org/licenses/by-sa/4.0/"
MAX_DISCOVERY_LIMIT = 100
MAX_DISCOVERY_REQUESTS = 10
MAX_TITLE_LENGTH = 200
MAX_PAYLOAD_BYTES = 1_000_000
MAX_HTTP_ATTEMPTS = 3
MAX_POSTGRES_INTEGER = 2_147_483_647
RETRYABLE_HTTP_STATUSES = frozenset({429, 500, 502, 503, 504})
RETRYABLE_MEDIAWIKI_ERRORS = frozenset({"maxlag", "ratelimited"})
_UNICODE_FRACTIONS = {
"¼": 0.25,
"½": 0.5,
"¾": 0.75,
"⅓": 1 / 3,
"⅔": 2 / 3,
"⅛": 0.125,
"⅜": 0.375,
"⅝": 0.625,
"⅞": 0.875,
}
_UNICODE_FRACTION_CHARS = "".join(_UNICODE_FRACTIONS)
_UNIT_PATTERN = re.compile(
r"^(?P<unit>(?:cups?|tablespoons?|tbsp\.?|teaspoons?|tsp\.?|ounces?|oz\.?|"
r"pounds?|lbs?\.?|grams?|g|kilograms?|kgs?|kg|millilit(?:er|re)s?|ml|"
r"lit(?:er|re)s?|l|cloves?|cans?|slices?|packages?|packs?|pinches?))\b\.?",
re.IGNORECASE,
)
_ALTERNATE_METRIC_QUANTITY = re.compile(
r"\s*\(\s*(?:\d+(?:\.\d+)?(?:\s*/\s*\d+)?|[" + _UNICODE_FRACTION_CHARS + r"]|"
r"\d+[" + _UNICODE_FRACTION_CHARS + r"])\s*(?:ml|millilit(?:er|re)s?|l|lit(?:er|re)s?|"
r"g|grams?|kg|kilograms?|oz|ounces?|lb|lbs?|pounds?)\b[^)]*\)",
re.IGNORECASE,
)
_TITLE_GENERIC_WORDS = frozenset(
{
"a",
"an",
"and",
"american",
"baked",
"baking",
"bread",
"boiled",
"braised",
"classic",
"cookbook",
"cooked",
"cookie",
"dessert",
"dish",
"dressing",
"for",
"fried",
"glaze",
"glazed",
"grilled",
"in",
"meal",
"minute",
"of",
"on",
"or",
"out",
"pan",
"pie",
"recipe",
"roasted",
"salad",
"sauce",
"seared",
"steamed",
"stroganoff",
"style",
"the",
"with",
}
)
_TITLE_VARIANT_TOKENS = frozenset({"I", "II", "III", "IV", "V", "VI", "VII", "VIII", "IX", "X"})
_PLACEHOLDER_INSTRUCTIONS = frozenset(
{
"coming soon",
"directions",
"instructions",
"method",
"n a",
"tbd",
"todo",
}
)
class WikibooksImportError(RuntimeError):
"""Raised when a Wikibooks response cannot safely be used."""
class WikibooksResponseError(WikibooksImportError):
"""Raised for malformed or unexpected API response data."""
def canonical_json_sha256(value: Mapping[str, Any]) -> str:
"""Return the stable SHA-256 checksum for a JSON-compatible mapping."""
canonical = json.dumps(value, ensure_ascii=False, separators=(",", ":"), sort_keys=True)
return hashlib.sha256(canonical.encode("utf-8")).hexdigest()
class WikibooksImportService:
"""Acquire and stage text-only recipe candidates from the fixed Wikibooks API."""
def __init__(
self,
user_agent: str,
*,
session: requests.Session | Any | None = None,
timeout: tuple[float, float] = (3.05, 20.0),
min_request_interval: float = 1.0,
max_payload_bytes: int = MAX_PAYLOAD_BYTES,
max_attempts: int = MAX_HTTP_ATTEMPTS,
backoff_seconds: float = 1.0,
max_retry_delay: float = 60.0,
sleep: Callable[[float], None] = time.sleep,
monotonic: Callable[[], float] = time.monotonic,
) -> None:
cleaned_user_agent = user_agent.strip() if isinstance(user_agent, str) else ""
if len(cleaned_user_agent) < 12 or not re.search(r"[A-Za-z]", cleaned_user_agent):
raise ValueError("user_agent must be a meaningful caller-supplied identifier")
if isinstance(max_payload_bytes, bool) or not isinstance(max_payload_bytes, int) or max_payload_bytes <= 0:
raise ValueError("max_payload_bytes must be a positive integer")
if isinstance(max_attempts, bool) or not isinstance(max_attempts, int) or max_attempts < 1:
raise ValueError("max_attempts must be a positive integer")
self._user_agent = cleaned_user_agent
self._session = session or requests.Session()
self._timeout = _timeout_pair(timeout)
self._min_request_interval = _finite_nonnegative(
min_request_interval, "min_request_interval"
)
self._max_payload_bytes = max_payload_bytes
self._max_attempts = max_attempts
self._backoff_seconds = _finite_positive(backoff_seconds, "backoff_seconds")
self._max_retry_delay = _finite_positive(max_retry_delay, "max_retry_delay")
self._sleep = sleep
self._monotonic = monotonic
self._last_request_at: float | None = None
def discover_recipe_titles(self, limit: int = MAX_DISCOVERY_LIMIT) -> list[str]:
"""Return bounded Cookbook page titles from ``Category:Recipes``."""
if isinstance(limit, bool) or not isinstance(limit, int) or limit < 1:
raise ValueError("limit must be a positive integer")
return self._discover_category_recipe_titles(
requested=min(limit, MAX_DISCOVERY_LIMIT),
max_requests=MAX_DISCOVERY_REQUESTS,
)
def _discover_category_recipe_titles(
self, *, requested: int, max_requests: int
) -> list[str]:
titles: list[str] = []
seen_titles: set[str] = set()
continuation: str | None = None
for _ in range(max_requests):
params: dict[str, str | int] = {
"action": "query",
"list": "categorymembers",
"cmtitle": "Category:Recipes",
"cmtype": "page",
"cmlimit": min(requested - len(titles), MAX_DISCOVERY_LIMIT),
"format": "json",
"formatversion": 2,
"maxlag": 5,
}
if continuation:
params["cmcontinue"] = continuation
payload = self._request(params)
query = _mapping(payload.get("query"), "query")
members = query.get("categorymembers")
if not isinstance(members, list):
raise WikibooksResponseError("query.categorymembers must be a list")
for member in members:
member_data = _mapping(member, "query.categorymembers item")
title = member_data.get("title")
if not isinstance(title, str):
raise WikibooksResponseError("query.categorymembers item title must be a string")
if title.startswith("Cookbook:") and title not in seen_titles:
_validate_title(title)
seen_titles.add(title)
titles.append(title)
if len(titles) == requested:
return titles
next_data = payload.get("continue")
if next_data is None:
return titles
next_mapping = _mapping(next_data, "continue")
next_continuation = next_mapping.get("cmcontinue")
if not isinstance(next_continuation, str) or not next_continuation:
raise WikibooksResponseError("continue.cmcontinue must be a non-empty string")
if next_continuation == continuation:
raise WikibooksResponseError("repeated category continuation token")
continuation = next_continuation
return titles
def search_recipe_titles(
self, queries: list[str], limit: int = MAX_DISCOVERY_LIMIT
) -> list[str]:
"""Return bounded Cookbook titles relevant to feedback search queries."""
if isinstance(limit, bool) or not isinstance(limit, int) or limit < 1:
raise ValueError("limit must be a positive integer")
requested = min(limit, MAX_DISCOVERY_LIMIT)
cleaned_queries: list[str] = []
for query in queries:
cleaned = " ".join(query.split()) if isinstance(query, str) else ""
if cleaned and cleaned not in cleaned_queries:
cleaned_queries.append(cleaned[:MAX_TITLE_LENGTH])
if len(cleaned_queries) == MAX_DISCOVERY_REQUESTS:
break
titles: list[str] = []
seen_titles: set[str] = set()
for query in cleaned_queries:
payload = self._request(
{
"action": "query",
"list": "search",
"srsearch": query,
"srnamespace": 102,
"srlimit": requested - len(titles),
"format": "json",
"formatversion": 2,
"maxlag": 5,
}
)
search = _mapping(payload.get("query"), "query").get("search")
if not isinstance(search, list):
raise WikibooksResponseError("query.search must be a list")
for result in search:
title = _mapping(result, "query.search item").get("title")
if not isinstance(title, str):
raise WikibooksResponseError(
"query.search item title must be a string"
)
if title.startswith("Cookbook:") and title not in seen_titles:
_validate_title(title)
seen_titles.add(title)
titles.append(title)
if len(titles) == requested:
return titles
return titles
def fetch_page(self, title: str) -> dict[str, Any]:
"""Fetch one validated ``Cookbook:`` parse object without augmentation."""
_validate_title(title)
payload = self._request(
{
"action": "parse",
"page": title,
"redirects": 1,
"prop": "text|wikitext|sections|revid|displaytitle",
"format": "json",
"formatversion": 2,
"maxlag": 5,
}
)
parse_payload = _mapping(payload.get("parse"), "parse")
_validate_parse_payload(parse_payload)
return parse_payload
def normalize_page(self, parse_payload: Mapping[str, Any]) -> dict[str, Any]:
"""Convert a validated rendered cookbook page into text-only recipe data."""
_validate_parse_payload(parse_payload)
title = str(parse_payload["title"])
soup = BeautifulSoup(str(parse_payload["text"]), "html.parser")
root = soup.select_one(".mw-parser-output") or soup
elements = root.find_all(["h2", "h3", "h4", "h5", "h6", "p", "ul", "ol"])
ingredients_heading = _find_heading(elements, {"ingredients"})
instruction_heading = _find_heading(
elements, {"procedure", "instructions", "method", "directions"}
)
description = _description_before_heading(elements, ingredients_heading)
infobox = _infobox_values(root)
ingredients = [
_normalize_ingredient(item)
for item in _section_list_items(ingredients_heading, {"ul", "ol"})
]
instructions = _section_list_items(instruction_heading, {"ol"})
return {
"name": title.removeprefix("Cookbook:").replace("_", " ").strip(),
"description": description,
"servings": _servings_from_infobox(infobox),
"prep_time_minutes": _time_from_infobox(infobox, ("prep", "preparation")),
"cook_time_minutes": _time_from_infobox(infobox, ("cook", "cooking")),
"baking_time_minutes": _time_from_infobox(infobox, ("bake", "baking")),
"ingredients": ingredients,
"instructions": instructions,
}
def validate_recipe(self, candidate: Mapping[str, Any]) -> dict[str, Any]:
"""Return deterministic structural and cross-field validation results."""
errors: list[dict[str, Any]] = []
warnings: list[dict[str, Any]] = []
servings = candidate.get("servings")
if (
isinstance(servings, bool)
or not isinstance(servings, int)
or servings <= 0
):
errors.append(_issue("invalid_servings", "servings", "Servings must be a positive integer."))
elif servings > MAX_POSTGRES_INTEGER:
errors.append(
_issue(
"servings_out_of_range",
"servings",
"Servings exceed the supported database range.",
)
)
for field in ("prep_time_minutes", "cook_time_minutes", "baking_time_minutes"):
duration = candidate.get(field)
if isinstance(duration, int) and not isinstance(duration, bool) and duration > MAX_POSTGRES_INTEGER:
errors.append(
_issue(
"time_out_of_range",
field,
"Duration exceeds the supported database range.",
)
)
combined_cook_time = (candidate.get("cook_time_minutes") or 0) + (
candidate.get("baking_time_minutes") or 0
)
if (
isinstance(combined_cook_time, int)
and not isinstance(combined_cook_time, bool)
and combined_cook_time > MAX_POSTGRES_INTEGER
):
errors.append(
_issue(
"combined_cook_time_out_of_range",
"times",
"Combined cook and baking time exceed the supported database range.",
)
)
ingredients = candidate.get("ingredients")
if not isinstance(ingredients, list) or len(ingredients) < 2:
errors.append(
_issue("too_few_ingredients", "ingredients", "At least two ingredients are required.")
)
ingredient_items: list[Any] = []
else:
ingredient_items = ingredients
instructions = candidate.get("instructions")
if not isinstance(instructions, list) or len(instructions) < 2:
errors.append(
_issue("too_few_instructions", "instructions", "At least two instructions are required.")
)
instruction_items: list[Any] = []
else:
instruction_items = instructions
for index, ingredient in enumerate(ingredient_items):
quantity = ingredient.get("quantity") if isinstance(ingredient, Mapping) else None
if (
isinstance(quantity, bool)
or not isinstance(quantity, (int, float))
or not math.isfinite(quantity)
or quantity <= 0
):
errors.append(
_issue(
"invalid_ingredient_quantity",
f"ingredients[{index}].quantity",
"Ingredient quantity must be positive.",
)
)
for index, instruction in enumerate(instruction_items):
if not isinstance(instruction, str) or _is_placeholder_instruction(instruction):
errors.append(
_issue(
"placeholder_instruction",
f"instructions[{index}]",
"Instruction is missing or a placeholder.",
)
)
title_name = candidate.get("name")
if isinstance(title_name, str) and ingredient_items:
missing_tokens = _missing_title_ingredient_tokens(title_name, ingredient_items)
if missing_tokens:
errors.append(
_issue(
"title_ingredient_mismatch",
"name",
"Distinctive title food tokens are absent from ingredients.",
missing_tokens=missing_tokens,
)
)
if not isinstance(candidate.get("description"), str) or not candidate["description"].strip():
warnings.append(_issue("missing_description", "description", "Intro description is unavailable."))
if not any(
isinstance(candidate.get(field), int) and candidate[field] > 0
for field in ("prep_time_minutes", "cook_time_minutes", "baking_time_minutes")
):
warnings.append(_issue("missing_time", "times", "No preparation, cook, or baking time is available."))
return {"valid": not errors, "errors": errors, "warnings": warnings}
def stage_page(self, db: Session, title: str) -> RecipeImportStaging:
"""Fetch, normalize, validate, and flush one candidate without committing."""
return self.stage_parse_payload(db, self.fetch_page(title))
def stage_discovered_pages(
self,
db: Session,
limit: int = 5,
queries: list[str] | None = None,
) -> list[RecipeImportStaging]:
"""Discover and stage a bounded relevant or category batch without committing."""
titles = (
self.search_recipe_titles(queries, limit)
if queries
else self.discover_recipe_titles(limit)
)
return [self.stage_page(db, title) for title in titles]
def stage_parse_payload(
self, db: Session, parse_payload: Mapping[str, Any]
) -> RecipeImportStaging:
"""Stage a validated parse object, reusing an existing page revision."""
snapshot = copy.deepcopy(dict(parse_payload))
_validate_parse_payload(snapshot)
external_id = str(snapshot["pageid"])
source_revision = str(snapshot["revid"])
normalized = self.normalize_page(snapshot)
validation_report = self.validate_recipe(normalized)
status = (
RecipeImportStatus.PENDING
if validation_report["valid"]
else RecipeImportStatus.QUARANTINED
)
title = str(snapshot["title"])
values = {
"source": WIKIBOOKS_SOURCE,
"external_id": external_id,
"source_url": _page_url(title),
"source_revision": source_revision,
"license_name": CC_BY_SA_4_LICENSE_NAME,
"license_url": CC_BY_SA_4_LICENSE_URL,
"attribution": f"Wikibooks contributors; revision history: {_history_url(title)}",
"raw_payload": snapshot,
"payload_sha256": canonical_json_sha256(snapshot),
"normalized_recipe": normalized,
"validation_report": validation_report,
"status": status,
"rejection_reason": (
None
if validation_report["valid"]
else "; ".join(error["code"] for error in validation_report["errors"])
),
}
identity = {
"source": WIKIBOOKS_SOURCE,
"external_id": external_id,
"source_revision": source_revision,
}
with db.no_autoflush:
db.execute(
pg_insert(RecipeImportStaging.__table__)
.values(**values)
.on_conflict_do_nothing(
constraint="uq_recipe_import_staging_source_revision"
)
.returning(RecipeImportStaging.id)
)
return db.execute(
select(RecipeImportStaging).filter_by(**identity)
).scalar_one()
def revalidate_staged_imports(
self, db: Session, *, limit: int = 100
) -> dict[str, int]:
"""Recompute unreviewed Wikibooks candidates without committing."""
if isinstance(limit, bool) or not isinstance(limit, int) or not 1 <= limit <= 1000:
raise ValueError("limit must be between 1 and 1000")
candidates = (
db.execute(
select(RecipeImportStaging)
.where(
RecipeImportStaging.source == WIKIBOOKS_SOURCE,
RecipeImportStaging.status.in_(
[RecipeImportStatus.PENDING, RecipeImportStatus.QUARANTINED]
),
RecipeImportStaging.reviewed_at.is_(None),
RecipeImportStaging.reviewed_by.is_(None),
)
.order_by(RecipeImportStaging.acquired_at, RecipeImportStaging.id)
.limit(limit)
.with_for_update(skip_locked=True)
)
.scalars()
.all()
)
counts = {"processed": len(candidates), "pending": 0, "quarantined": 0}
for staged in candidates:
snapshot = copy.deepcopy(staged.raw_payload)
if canonical_json_sha256(snapshot) != staged.payload_sha256:
staged.validation_report = {
"valid": False,
"errors": [
_issue(
"payload_checksum_mismatch",
"raw_payload",
"Raw payload checksum does not match immutable provenance.",
)
],
"warnings": [],
}
staged.status = RecipeImportStatus.QUARANTINED
staged.rejection_reason = "payload_checksum_mismatch"
counts[RecipeImportStatus.QUARANTINED.value] += 1
continue
_validate_parse_payload(snapshot)
normalized = self.normalize_page(snapshot)
validation_report = self.validate_recipe(normalized)
status = (
RecipeImportStatus.PENDING
if validation_report["valid"]
else RecipeImportStatus.QUARANTINED
)
staged.normalized_recipe = normalized
staged.validation_report = validation_report
staged.status = status
staged.rejection_reason = (
None
if validation_report["valid"]
else "; ".join(error["code"] for error in validation_report["errors"])
)
counts[status.value] += 1
db.flush()
return counts
def _request(self, params: Mapping[str, str | int]) -> dict[str, Any]:
for attempt in range(self._max_attempts):
response: Any | None = None
retry_error: WikibooksImportError | None = None
retry_delay: float | None = None
try:
self._wait_for_request_slot()
response = self._session.get(
WIKIBOOKS_API_URL,
params=dict(params),
headers={"Accept": "application/json", "User-Agent": self._user_agent},
timeout=self._timeout,
stream=True,
allow_redirects=False,
)
status_code = getattr(response, "status_code", None)
if isinstance(status_code, bool) or not isinstance(status_code, int):
raise WikibooksResponseError("HTTP response status code is invalid")
if status_code in RETRYABLE_HTTP_STATUSES:
retry_error = WikibooksImportError(f"Wikibooks returned retryable HTTP {status_code}")
retry_delay = _retry_after_seconds(getattr(response, "headers", {}))
elif 300 <= status_code < 400:
raise WikibooksImportError(f"Wikibooks redirect HTTP {status_code} rejected")
elif status_code < 200 or status_code >= 300:
raise WikibooksImportError(f"Wikibooks returned HTTP {status_code}")
else:
payload = self._read_json_response(response)
api_error = payload.get("error")
if api_error is None:
return payload
error_data = _mapping(api_error, "error")
error_code = error_data.get("code")
if not isinstance(error_code, str) or not error_code:
raise WikibooksResponseError("error.code must be a non-empty string")
if error_code.lower() in RETRYABLE_MEDIAWIKI_ERRORS:
retry_error = WikibooksImportError(
f"Wikibooks returned retryable MediaWiki error {error_code}"
)
else:
raise WikibooksImportError(
f"Wikibooks returned MediaWiki error {error_code}"
)
except requests.RequestException as exc:
retry_error = WikibooksImportError(f"Wikibooks request failed: {exc}")
finally:
if response is not None:
response.close()
if retry_error is None:
raise AssertionError("request did not return or produce an error")
if attempt == self._max_attempts - 1:
raise retry_error
delay = retry_delay if retry_delay is not None else self._backoff_seconds * (2**attempt)
self._sleep(min(delay, self._max_retry_delay))
raise AssertionError("unreachable")
def _wait_for_request_slot(self) -> None:
if self._last_request_at is not None:
wait_seconds = self._min_request_interval - (
self._monotonic() - self._last_request_at
)
if wait_seconds > 0:
self._sleep(wait_seconds)
self._last_request_at = self._monotonic()
def _read_json_response(self, response: Any) -> dict[str, Any]:
headers = getattr(response, "headers", {})
if not isinstance(headers, Mapping):
raise WikibooksResponseError("HTTP response headers are invalid")
content_length = headers.get("Content-Length")
if content_length is not None:
try:
declared_size = int(content_length)
except (TypeError, ValueError) as exc:
raise WikibooksResponseError("HTTP Content-Length is invalid") from exc
if declared_size < 0 or declared_size > self._max_payload_bytes:
raise WikibooksResponseError("Wikibooks payload exceeds configured size limit")
body = bytearray()
for chunk in response.iter_content(chunk_size=64 * 1024):
if not isinstance(chunk, (bytes, bytearray)):
raise WikibooksResponseError("HTTP response body chunk is invalid")
body.extend(chunk)
if len(body) > self._max_payload_bytes:
raise WikibooksResponseError("Wikibooks payload exceeds configured size limit")
try:
decoded = json.loads(bytes(body).decode("utf-8"))
except (UnicodeDecodeError, json.JSONDecodeError) as exc:
raise WikibooksResponseError("Wikibooks response is not valid UTF-8 JSON") from exc
return _mapping(decoded, "response")
def _mapping(value: Any, name: str) -> dict[str, Any]:
if not isinstance(value, dict):
raise WikibooksResponseError(f"{name} must be an object")
return value
def _validate_title(title: str) -> None:
if (
not isinstance(title, str)
or not title.startswith("Cookbook:")
or len(title) <= len("Cookbook:")
or len(title) > MAX_TITLE_LENGTH
or any(character in title for character in "\r\n\x00")
):
raise WikibooksResponseError("title must be a sane Cookbook: page title")
def _validate_parse_payload(parse_payload: Mapping[str, Any]) -> None:
if not isinstance(parse_payload, Mapping):
raise WikibooksResponseError("parse must be an object")
_positive_integer(parse_payload.get("pageid"), "parse.pageid")
_positive_integer(parse_payload.get("revid"), "parse.revid")
title = parse_payload.get("title")
_validate_title(title)
for field in ("text", "wikitext", "displaytitle"):
if not isinstance(parse_payload.get(field), str):
raise WikibooksResponseError(f"parse.{field} must be a string")
sections = parse_payload.get("sections")
if not isinstance(sections, list):
raise WikibooksResponseError("parse.sections must be a list")
for section in sections:
section_data = _mapping(section, "parse.sections item")
if not isinstance(section_data.get("line"), str):
raise WikibooksResponseError("parse.sections item line must be a string")
def _positive_integer(value: Any, name: str) -> int:
if isinstance(value, bool) or not isinstance(value, int) or value <= 0:
raise WikibooksResponseError(f"{name} must be a positive integer")
return value
def _finite_positive(value: Any, name: str) -> float:
if (
isinstance(value, bool)
or not isinstance(value, (int, float))
or not math.isfinite(value)
or value <= 0
):
raise ValueError(f"{name} must be finite and positive")
return float(value)
def _finite_nonnegative(value: Any, name: str) -> float:
if (
isinstance(value, bool)
or not isinstance(value, (int, float))
or not math.isfinite(value)
or value < 0
):
raise ValueError(f"{name} must be finite and non-negative")
return float(value)
def _timeout_pair(timeout: Any) -> tuple[float, float]:
if not isinstance(timeout, (tuple, list)) or len(timeout) != 2:
raise ValueError("timeout must be a (connect, read) pair")
return (
_finite_positive(timeout[0], "timeout connect value"),
_finite_positive(timeout[1], "timeout read value"),
)
def _retry_after_seconds(headers: Mapping[str, Any]) -> float | None:
value = headers.get("Retry-After")
if value is None:
return None
try:
seconds = float(value)
return max(0.0, seconds) if math.isfinite(seconds) else None
except (TypeError, ValueError):
try:
retry_at = parsedate_to_datetime(str(value))
except (TypeError, ValueError, IndexError):
return None
if retry_at.tzinfo is None:
retry_at = retry_at.replace(tzinfo=timezone.utc)
seconds = (retry_at - datetime.now(timezone.utc)).total_seconds()
return max(0.0, seconds) if math.isfinite(seconds) else None
def _page_url(title: str) -> str:
return "https://en.wikibooks.org/wiki/" + quote(title.replace(" ", "_"), safe=":")
def _history_url(title: str) -> str:
return "https://en.wikibooks.org/w/index.php?title=" + quote(title, safe="") + "&action=history"
def _text(tag: Tag) -> str:
return re.sub(r"\s+", " ", " ".join(tag.stripped_strings)).strip()
def _heading_text(tag: Tag) -> str:
return re.sub(r"\s*\[edit\]\s*$", "", _text(tag), flags=re.IGNORECASE).strip().lower()
def _find_heading(elements: list[Tag], names: set[str]) -> Tag | None:
for element in elements:
if element.name in {"h2", "h3", "h4", "h5", "h6"} and _heading_text(element) in names:
return element
return None
def _description_before_heading(elements: list[Tag], heading: Tag | None) -> str | None:
if heading is None:
return None
descriptions: list[str] = []
for element in elements:
if element is heading:
break
if element.name == "p" and element.find_parent(["li", "td", "th"]) is None:
text = _text(element)
if text:
descriptions.append(text)
return " ".join(descriptions) or None
def _infobox_values(root: Tag | BeautifulSoup) -> dict[str, str]:
values: dict[str, str] = {}
for table in root.find_all("table"):
classes = " ".join(table.get("class", []))
if "infobox" not in classes.lower():
continue
for row in table.find_all("tr"):
cells = row.find_all(["th", "td"], recursive=False)
if len(cells) < 2:
continue
label = _text(cells[0]).lower()
value = _text(cells[1])
if label and value and label not in values:
values[label] = value
return values
def _servings_from_infobox(values: Mapping[str, str]) -> int | None:
for label, value in values.items():
if any(term in label for term in ("servings", "serves", "yield", "makes")):
match = re.search(r"\b(\d+)\b", value)
if match:
return int(match.group(1))
return None
def _time_from_infobox(values: Mapping[str, str], labels: tuple[str, ...]) -> int | None:
for label, value in values.items():
if any(expected in label for expected in labels):
return _parse_duration(value)
return None
def _parse_duration(value: str) -> int | None:
clock = re.search(r"\b(\d{1,2}):(\d{2})\b", value)
if clock:
hours, minutes = (int(part) for part in clock.groups())
total = hours * 60 + minutes
return total if total > 0 else None
hours = re.search(r"\b(\d+)\s*(?:hours?|hrs?|hr|h)\b", value, re.IGNORECASE)
minutes = re.search(r"\b(\d+)\s*(?:minutes?|mins?|min|m)\b", value, re.IGNORECASE)
total = (int(hours.group(1)) * 60 if hours else 0) + (int(minutes.group(1)) if minutes else 0)
return total if total > 0 else None
def _section_list_items(heading: Tag | None, list_names: set[str]) -> list[str]:
if heading is None:
return []
heading_level = int(heading.name[1])
items: list[str] = []
for element in heading.find_all_next(["h2", "h3", "h4", "h5", "h6", "ul", "ol"]):
if element.name and element.name.startswith("h") and int(element.name[1]) <= heading_level:
break
if element.name not in list_names:
continue
for item in element.find_all("li", recursive=False):
text = _list_item_text(item)
if text:
items.append(text)
return items
def _list_item_text(item: Tag) -> str:
clone = copy.copy(item)
for nested_list in clone.find_all(["ul", "ol"]):
nested_list.decompose()
return _text(clone)
def _normalize_ingredient(source_text: str) -> dict[str, Any]:
quantity, quantity_text, remainder = _parse_leading_quantity(source_text)
unit: str | None = None
name = remainder if quantity_text is not None else source_text
if quantity_text is not None:
unit_match = _UNIT_PATTERN.match(name.lstrip())
if unit_match:
unit = unit_match.group("unit").lower().rstrip(".")
name = name.lstrip()[unit_match.end() :]
name = _ALTERNATE_METRIC_QUANTITY.sub("", name)
name = name.strip(" \t,;:-") or source_text
return {
"quantity": quantity,
"quantity_text": quantity_text,
"unit": unit,
"name": name,
"source_text": source_text,
}
def _parse_leading_quantity(value: str) -> tuple[float | None, str | None, str]:
text = value.strip()
if re.match(r"^\d+(?:\.\d+)?\s*(?:-|–|—)\s*\d", text):
return None, None, text
fraction_prefix = re.match(r"^\d+(?:\.\d+)?(?:\s+\d+)?\s*/\s*(\d*)", text)
if fraction_prefix and (not fraction_prefix.group(1) or int(fraction_prefix.group(1)) == 0):
return None, None, text
mixed = re.match(r"^(\d+(?:\.\d+)?)\s+(\d+\s*/\s*\d+|[" + _UNICODE_FRACTION_CHARS + r"])", text)
if mixed:
fraction = _fraction_value(mixed.group(2))
if fraction is not None:
return float(mixed.group(1)) + fraction, mixed.group(0), text[mixed.end() :]
compact = re.match(r"^(\d+)([" + _UNICODE_FRACTION_CHARS + r"])", text)
if compact:
return int(compact.group(1)) + _UNICODE_FRACTIONS[compact.group(2)], compact.group(0), text[compact.end() :]
ascii_fraction = re.match(r"^(\d+)\s*/\s*(\d+)", text)
if ascii_fraction:
denominator = int(ascii_fraction.group(2))
if denominator:
return (
int(ascii_fraction.group(1)) / denominator,
ascii_fraction.group(0),
text[ascii_fraction.end() :],
)
number = re.match(r"^\d+(?:\.\d+)?", text)
if number:
return float(number.group(0)), number.group(0), text[number.end() :]
unicode_fraction = re.match(r"^(?:[" + _UNICODE_FRACTION_CHARS + r"])", text)
if unicode_fraction:
token = unicode_fraction.group(0)
return _UNICODE_FRACTIONS[token], token, text[unicode_fraction.end() :]
return None, None, text
def _fraction_value(value: str) -> float | None:
if value in _UNICODE_FRACTIONS:
return _UNICODE_FRACTIONS[value]
numerator, denominator = (int(part.strip()) for part in value.split("/", 1))
return numerator / denominator if denominator else None
def _issue(code: str, field: str, message: str, **extra: Any) -> dict[str, Any]:
return {"code": code, "field": field, "message": message, **extra}
def _is_placeholder_instruction(instruction: str) -> bool:
normalized = re.sub(r"[^a-z0-9]+", " ", instruction.lower()).strip()
return normalized in _PLACEHOLDER_INSTRUCTIONS or bool(
re.fullmatch(r"(?:step )?\d+", normalized)
)
def _missing_title_ingredient_tokens(title: str, ingredients: list[Any]) -> list[str]:
title_tokens: list[str] = []
for raw_token in re.findall(r"[A-Za-z]+", title):
if raw_token in _TITLE_VARIANT_TOKENS:
continue
token = _singular_token(raw_token.lower())
if len(token) > 1 and token not in _TITLE_GENERIC_WORDS:
title_tokens.append(token)
ingredient_tokens: set[str] = set()
for ingredient in ingredients:
if isinstance(ingredient, Mapping) and isinstance(ingredient.get("name"), str):
ingredient_tokens.update(
_singular_token(token)
for token in re.findall(r"[a-z]+", ingredient["name"].lower())
)
return [token for token in dict.fromkeys(title_tokens) if token not in ingredient_tokens]
def _singular_token(token: str) -> str:
if token == "pies":
return "pie"
if token == "cookies":
return "cookie"
if token.endswith("ies") and len(token) > 3:
return token[:-3] + "y"
if token.endswith(("ches", "shes", "sses", "xes", "zes")) and len(token) > 3:
return token[:-2]
if token.endswith("s") and not token.endswith("ss") and len(token) > 2:
return token[:-1]
return token