feat: phase r1+r2 recovery + r3-0 swiftly api ingestion

R1 stabilization: pytest harness with transactional db fixture, smoke
+ alembic + auth + scrape + approval + swiftly tests, github actions
ci yaml. Bearer-token admin auth + signed-cookie session for family
ui mutations. Async POST /api/admin/scrape (BackgroundTasks, returns
202). Path canonicalization (no /list, /planned suffixes). DATABASE_URL
fail-fast on empty.

R2 deferred-risk spikes: live lucky california fetch (R2-A), full
email+per-voter approval click round trip with single-use enforcement
(R2-B, console email backend, sendgrid stub).

R3-0 phase 3 redesign: replaced playwright html scraper with requests
based swiftly json api client. 17 categories, ~10k products per scrape,
upsert by (source, external_id). 401 surfaces actionable token-refresh
message via ScrapeLog.error_message.

Pre-existing defects fixed: shopping_list.py syntax error blocking app
import, MealPlan.votes orphan relationship, JSONB(astext=True) invalid
kwarg, missing requests dep, calorie_target schema drift, every SQLEnum
needed values_callable, 0001 had empty downgrade(), seed had duplicate
ingredient rows.

Migrations added: 0003 grocery_item.description, 0004 family_profile.
calorie_target, 0005 grocery_item.external_id + source + composite index.

Verified: 31/31 pytest green, alembic upgrade->downgrade->upgrade clean,
frontend npm run build clean, live scrape 9,960 grocery_item rows in 36s.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

scripts/send_test_approval.py

@@ -0,0 +1,191 @@
+#!/usr/bin/env python
+"""
+R2-B end-to-end approval round-trip spike.
+
+Usage:
+  # Print URL only (you can click it in a browser running the server):
+  python scripts/send_test_approval.py
+
+  # Bypass the email and exercise the full POST against a TestClient:
+  python scripts/send_test_approval.py --simulate-click approve
+  python scripts/send_test_approval.py --simulate-click deny
+
+This is the actual proof that the email + per-voter approval click
+round-trip works against the real schema (`family_member`,
+`meal_plan_item`, `meal_plan_vote`). If the script exits 0 with
+`item_status=approved` (or `denied`) the spike is green.
+"""
+
+from __future__ import annotations
+
+import argparse
+import os
+import sys
+import uuid
+from datetime import date, timedelta
+from pathlib import Path
+
+# Make backend/ importable when invoked from repo root.
+_REPO_ROOT = Path(__file__).resolve().parent.parent
+_BACKEND_ROOT = _REPO_ROOT / "backend"
+if str(_BACKEND_ROOT) not in sys.path:
+    sys.path.insert(0, str(_BACKEND_ROOT))
+
+# DATABASE_URL must be set before importing app.config.
+os.environ.setdefault(
+    "DATABASE_URL",
+    os.environ.get(
+        "TEST_DATABASE_URL",
+        "postgresql://mealplanner:password@localhost:5432/mealplanner_test",
+    ),
+)
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument(
+        "--simulate-click",
+        choices=["approve", "deny"],
+        help="Bypass email; POST the vote via TestClient and assert success.",
+    )
+    parser.add_argument(
+        "--base-url",
+        default="http://localhost:8000",
+        help="Base URL for the printed link (informational only).",
+    )
+    args = parser.parse_args()
+
+    from sqlalchemy import create_engine
+    from sqlalchemy.orm import sessionmaker
+
+    from app.config import settings
+    from app.models import (
+        FamilyMember,
+        FamilyMemberRole,
+        FamilyProfile,
+        MealPlan,
+        MealPlanItem,
+        MealPlanStatus,
+        MealType,
+        Recipe,
+    )
+    from app.services import approval as approval_service
+    from app.services.email import get_email_backend
+
+    engine = create_engine(
+        settings.DATABASE_URL.replace("postgresql://", "postgresql+psycopg2://")
+    )
+    Session = sessionmaker(bind=engine, autoflush=False, autocommit=False)
+    db = Session()
+
+    # ------------------------------------------------------------------
+    # Bootstrap a minimal scenario. Suffix everything with a UUID4 so the
+    # script is idempotent and can run repeatedly without unique-constraint
+    # collisions.
+    # ------------------------------------------------------------------
+    suffix = uuid.uuid4().hex[:8]
+
+    profile = FamilyProfile(
+        name=f"Spike Family {suffix}",
+        household_size=1,
+        adult_count=1,
+        child_count=0,
+    )
+    db.add(profile)
+    db.flush()
+
+    voter = FamilyMember(
+        family_profile_id=profile.id,
+        name="Spike Voter",
+        email=f"spike+{suffix}@example.com",
+        role=FamilyMemberRole.ADULT,
+    )
+    db.add(voter)
+    db.flush()
+
+    recipe = Recipe(
+        family_profile_id=profile.id,
+        name=f"Spike Pasta {suffix}",
+        servings=2,
+        ingredients=[{"name": "pasta", "qty": "200g"}],
+        instructions=["Boil water", "Cook pasta"],
+        is_manually_added=True,
+    )
+    db.add(recipe)
+    db.flush()
+
+    plan = MealPlan(
+        family_profile_id=profile.id,
+        week_start_date=date.today() + timedelta(days=(7 - date.today().weekday())),
+        status=MealPlanStatus.DRAFT,
+    )
+    db.add(plan)
+    db.flush()
+
+    item = MealPlanItem(
+        meal_plan_id=plan.id,
+        recipe_id=recipe.id,
+        day_of_week=1,
+        meal_type=MealType.DINNER,
+    )
+    db.add(item)
+    db.commit()
+
+    token = approval_service.issue_token(item.id, voter.id)
+    url = f"{args.base_url}/api/meals/vote/{item.id}?token={token}"
+
+    # Send via configured email backend (Console writes to stdout + outbox).
+    backend = get_email_backend()
+    subject = "Action required: please vote on this week's meal"
+    html = (
+        f"<p>Hi {voter.name}, please review this meal:</p>"
+        f"<p><strong>{recipe.name}</strong></p>"
+        f'<p><a href="{url}">Open the approval page</a></p>'
+    )
+    text = f"Hi {voter.name}, please review {recipe.name}: {url}"
+    backend.send(to=voter.email, subject=subject, html=html, text=text)
+
+    print(f"item_id={item.id}")
+    print(f"voter_id={voter.id}")
+    print(f"approval_url={url}")
+
+    if args.simulate_click is None:
+        return 0
+
+    # ------------------------------------------------------------------
+    # End-to-end proof: drive the POST through a TestClient.
+    # ------------------------------------------------------------------
+    from fastapi.testclient import TestClient
+
+    from app.main import app
+
+    with TestClient(app) as client:
+        # Render the GET page first to mirror a real click.
+        r_get = client.get(f"/api/meals/vote/{item.id}", params={"token": token})
+        assert r_get.status_code == 200, (r_get.status_code, r_get.text)
+        assert "text/html" in r_get.headers.get("content-type", "")
+
+        r_post = client.post(
+            f"/api/meals/vote/{item.id}",
+            params={"token": token},
+            json={"vote": args.simulate_click},
+        )
+        assert r_post.status_code == 200, (r_post.status_code, r_post.text)
+        body = r_post.json()
+        print(f"item_status={body['item_status']}")
+        assert body["status"] == "recorded"
+
+        # Single-use: a second POST must 409.
+        r_post2 = client.post(
+            f"/api/meals/vote/{item.id}",
+            params={"token": token},
+            json={"vote": args.simulate_click},
+        )
+        assert r_post2.status_code == 409, (r_post2.status_code, r_post2.text)
+        print("single_use=enforced")
+
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

scripts/spike_lucky_scrape.py

@@ -0,0 +1,169 @@
+"""
+R2-A live-scrape spike for Lucky California weekly ad.
+
+Performs ONE live fetch of https://luckysupermarkets.com weekly-ad page,
+saves the rendered HTML, a full-page screenshot, and a META.md describing
+the fetch result. This is the deferred-risk spike demanded by review §2.4.
+
+Run once. Do not loop. Polite citizen: identifying user-agent, single
+request, no auth bypass attempts.
+
+Usage:
+    python scripts/spike_lucky_scrape.py
+"""
+
+from __future__ import annotations
+
+import sys
+from datetime import datetime, timezone
+from pathlib import Path
+
+from playwright.sync_api import sync_playwright
+
+# Path to backend/app on import path so we can reuse the parser.
+REPO_ROOT = Path(__file__).resolve().parent.parent
+sys.path.insert(0, str(REPO_ROOT / "backend"))
+
+from app.scraper.lucky_ca_scraper import LuckyCaliforniaScraper  # noqa: E402
+from bs4 import BeautifulSoup  # noqa: E402
+
+FIXTURE_DIR = REPO_ROOT / "backend" / "tests" / "fixtures" / "lucky_ca"
+TARGET_URL = "https://luckysupermarkets.com/coupons/Coupon%2Flu-featured-in-ad"
+USER_AGENT = (
+    "Mozilla/5.0 (compatible; MealPlannerSpike/0.1; "
+    "+https://github.com/MealPlanner; spike=R2-A)"
+)
+
+
+def main() -> int:
+    FIXTURE_DIR.mkdir(parents=True, exist_ok=True)
+    html_path = FIXTURE_DIR / "weekly_ad.html"
+    png_path = FIXTURE_DIR / "weekly_ad.png"
+    meta_path = FIXTURE_DIR / "META.md"
+
+    started = datetime.now(timezone.utc)
+    print(f"[spike] Fetching {TARGET_URL}")
+
+    status_code: int | None = None
+    final_url: str = TARGET_URL
+    error: str | None = None
+    captcha_or_block = False
+    captcha_signal = ""
+
+    with sync_playwright() as p:
+        browser = p.chromium.launch(headless=True)
+        try:
+            ctx = browser.new_context(user_agent=USER_AGENT)
+            page = ctx.new_page()
+
+            response = None
+
+            def _capture(resp):
+                nonlocal response
+                # Capture only the main document response.
+                if response is None and resp.url.rstrip("/") == TARGET_URL.rstrip("/"):
+                    response = resp
+
+            page.on("response", _capture)
+
+            try:
+                nav_resp = page.goto(
+                    TARGET_URL,
+                    wait_until="networkidle",
+                    timeout=45_000,
+                )
+                if nav_resp is not None:
+                    status_code = nav_resp.status
+                    final_url = nav_resp.url
+                elif response is not None:
+                    status_code = response.status
+                    final_url = response.url
+            except Exception as exc:  # noqa: BLE001 -- spike, capture and report
+                error = f"{type(exc).__name__}: {exc}"
+
+            try:
+                html = page.content()
+            except Exception as exc:  # noqa: BLE001
+                html = ""
+                error = (error or "") + f" content_err={exc}"
+
+            html_path.write_text(html, encoding="utf-8")
+
+            try:
+                page.screenshot(path=str(png_path), full_page=True)
+            except Exception as exc:  # noqa: BLE001
+                error = (error or "") + f" screenshot_err={exc}"
+
+            # Heuristic captcha / block detection.
+            lower = html.lower()
+            for needle in (
+                "captcha",
+                "are you a robot",
+                "access denied",
+                "akamai",
+                "cloudflare",
+                "px-captcha",
+                "perimeterx",
+                "incapsula",
+            ):
+                if needle in lower:
+                    captcha_or_block = True
+                    captcha_signal = needle
+                    break
+
+        finally:
+            browser.close()
+
+    # Run parser portion against the captured HTML (no network).
+    scraper = LuckyCaliforniaScraper()
+    items: list[dict] = scraper.parse_featured_coupons_html(html)
+    _ = BeautifulSoup  # keep import for type-stability if reused later
+
+    completed = datetime.now(timezone.utc)
+
+    meta = f"""# Lucky California weekly-ad spike (R2-A)
+
+| Field | Value |
+| --- | --- |
+| URL | {TARGET_URL} |
+| Final URL | {final_url} |
+| Started (UTC) | {started.isoformat()} |
+| Completed (UTC) | {completed.isoformat()} |
+| HTTP status | {status_code} |
+| HTML bytes | {len(html)} |
+| Items parsed | {len(items)} |
+| Captcha/block signal | {"YES (" + captcha_signal + ")" if captcha_or_block else "no"} |
+| Error | {error or "none"} |
+| User-Agent | `{USER_AGENT}` |
+
+## First parsed item (sample)
+
+```json
+{__import__("json").dumps(items[0], indent=2) if items else "null"}
+```
+
+## Notes
+
+- Single live fetch performed. Do not rerun without reason.
+- HTML and PNG saved alongside this file.
+- Parser used: `LuckyCaliforniaScraper._parse_coupon_item` against
+  elements matching `h2/h3/a` with `$N.NN` text.
+"""
+    meta_path.write_text(meta, encoding="utf-8")
+
+    print(f"[spike] status={status_code} html_bytes={len(html)} items={len(items)} "
+          f"block={captcha_or_block}")
+    print(f"[spike] wrote: {html_path}")
+    print(f"[spike] wrote: {png_path}")
+    print(f"[spike] wrote: {meta_path}")
+
+    if captcha_or_block:
+        print("[spike] WARNING: captcha/anti-bot signal detected; review META.md")
+    if not items:
+        print("[spike] WARNING: zero items parsed; selectors may be stale")
+
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

scripts/spike_swiftly_ingest.py

@@ -0,0 +1,101 @@
+"""Live spike for the Swiftly product API (R3-0, ad-hoc admin tool).
+
+Usage:
+    python scripts/spike_swiftly_ingest.py --confirm-live
+
+Fetches the categories page + ONE category (default ``Product/meat_seafood``)
+against the live Swiftly API, prints the item count, and pretty-prints
+two sample mapped products. Does NOT persist anything to Postgres — the
+purpose is to verify the token + parser round-trip on demand without
+running the full scrape.
+"""
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import sys
+from pathlib import Path
+
+# Make `app.*` importable when invoked from the repo root.
+REPO_ROOT = Path(__file__).resolve().parent.parent
+BACKEND_DIR = REPO_ROOT / "backend"
+if str(BACKEND_DIR) not in sys.path:
+    sys.path.insert(0, str(BACKEND_DIR))
+
+# Allow running outside docker without a real Postgres URL just to exercise
+# the scraper. Settings still requires DATABASE_URL to be non-empty.
+os.environ.setdefault("DATABASE_URL", "postgresql://placeholder@localhost:5432/placeholder")
+
+from app.scraper.lucky_ca_scraper import LuckyCaliforniaScraper, SwiftlyAuthError  # noqa: E402
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument(
+        "--confirm-live",
+        action="store_true",
+        help="Required: hit the live Swiftly API. Without this flag the script no-ops.",
+    )
+    parser.add_argument(
+        "--category",
+        default="Product/meat_seafood",
+        help="API slug to fetch (default: Product/meat_seafood).",
+    )
+    parser.add_argument(
+        "--limit",
+        type=int,
+        default=2,
+        help="How many sample mapped products to print (default 2).",
+    )
+    args = parser.parse_args()
+
+    if not args.confirm_live:
+        print(
+            "Refusing to make live HTTP calls without --confirm-live.\n"
+            "Pass --confirm-live to opt in.",
+            file=sys.stderr,
+        )
+        return 2
+
+    scraper = LuckyCaliforniaScraper()
+    try:
+        slugs = scraper.discover_categories()
+        print(f"discovered {len(slugs)} categories")
+        for s in slugs[:5]:
+            print(f"  - {s}")
+        if len(slugs) > 5:
+            print(f"  ... ({len(slugs) - 5} more)")
+
+        try:
+            raw_items = scraper.fetch_category(args.category)
+        except SwiftlyAuthError as exc:
+            print(f"\nAUTH FAILURE: {exc}", file=sys.stderr)
+            return 1
+
+        print(f"\ncategory={args.category!r}: {len(raw_items)} raw items")
+
+        aisle = LuckyCaliforniaScraper._aisle_from_slug(args.category)
+        mapped = []
+        for raw in raw_items:
+            m = LuckyCaliforniaScraper.map_product(
+                raw, aisle=aisle, source_slug=args.category
+            )
+            if m is not None:
+                mapped.append(m)
+        print(f"{len(mapped)} mapped products (after dropping unparseable rows)")
+
+        print("\n=== sample mapped products ===")
+        for m in mapped[: args.limit]:
+            # Decimal isn't JSON-serializable; coerce for display.
+            display = {k: (str(v) if k in {"current_price", "regular_price", "sale_price"} else v) for k, v in m.items()}
+            print(json.dumps(display, indent=2, default=str))
+            print()
+
+        return 0
+    finally:
+        scraper.cleanup()
+
+
+if __name__ == "__main__":
+    sys.exit(main())