feat: phase r1+r2 recovery + r3-0 swiftly api ingestion

R1 stabilization: pytest harness with transactional db fixture, smoke + alembic + auth + scrape + approval + swiftly tests, github actions ci yaml. Bearer-token admin auth + signed-cookie session for family ui mutations. Async POST /api/admin/scrape (BackgroundTasks, returns 202). Path canonicalization (no /list, /planned suffixes). DATABASE_URL fail-fast on empty. R2 deferred-risk spikes: live lucky california fetch (R2-A), full email+per-voter approval click round trip with single-use enforcement (R2-B, console email backend, sendgrid stub). R3-0 phase 3 redesign: replaced playwright html scraper with requests based swiftly json api client. 17 categories, ~10k products per scrape, upsert by (source, external_id). 401 surfaces actionable token-refresh message via ScrapeLog.error_message. Pre-existing defects fixed: shopping_list.py syntax error blocking app import, MealPlan.votes orphan relationship, JSONB(astext=True) invalid kwarg, missing requests dep, calorie_target schema drift, every SQLEnum needed values_callable, 0001 had empty downgrade(), seed had duplicate ingredient rows. Migrations added: 0003 grocery_item.description, 0004 family_profile. calorie_target, 0005 grocery_item.external_id + source + composite index. Verified: 31/31 pytest green, alembic upgrade->downgrade->upgrade clean, frontend npm run build clean, live scrape 9,960 grocery_item rows in 36s. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
2026-05-05 14:08:19 -07:00
parent b9434967ed
commit 8e89f793d5
58 changed files with 3594 additions and 348 deletions
@@ -0,0 +1,101 @@
+"""Live spike for the Swiftly product API (R3-0, ad-hoc admin tool).
+
+Usage:
+    python scripts/spike_swiftly_ingest.py --confirm-live
+
+Fetches the categories page + ONE category (default ``Product/meat_seafood``)
+against the live Swiftly API, prints the item count, and pretty-prints
+two sample mapped products. Does NOT persist anything to Postgres — the
+purpose is to verify the token + parser round-trip on demand without
+running the full scrape.
+"""
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import sys
+from pathlib import Path
+
+# Make `app.*` importable when invoked from the repo root.
+REPO_ROOT = Path(__file__).resolve().parent.parent
+BACKEND_DIR = REPO_ROOT / "backend"
+if str(BACKEND_DIR) not in sys.path:
+    sys.path.insert(0, str(BACKEND_DIR))
+
+# Allow running outside docker without a real Postgres URL just to exercise
+# the scraper. Settings still requires DATABASE_URL to be non-empty.
+os.environ.setdefault("DATABASE_URL", "postgresql://placeholder@localhost:5432/placeholder")
+
+from app.scraper.lucky_ca_scraper import LuckyCaliforniaScraper, SwiftlyAuthError  # noqa: E402
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument(
+        "--confirm-live",
+        action="store_true",
+        help="Required: hit the live Swiftly API. Without this flag the script no-ops.",
+    )
+    parser.add_argument(
+        "--category",
+        default="Product/meat_seafood",
+        help="API slug to fetch (default: Product/meat_seafood).",
+    )
+    parser.add_argument(
+        "--limit",
+        type=int,
+        default=2,
+        help="How many sample mapped products to print (default 2).",
+    )
+    args = parser.parse_args()
+
+    if not args.confirm_live:
+        print(
+            "Refusing to make live HTTP calls without --confirm-live.\n"
+            "Pass --confirm-live to opt in.",
+            file=sys.stderr,
+        )
+        return 2
+
+    scraper = LuckyCaliforniaScraper()
+    try:
+        slugs = scraper.discover_categories()
+        print(f"discovered {len(slugs)} categories")
+        for s in slugs[:5]:
+            print(f"  - {s}")
+        if len(slugs) > 5:
+            print(f"  ... ({len(slugs) - 5} more)")
+
+        try:
+            raw_items = scraper.fetch_category(args.category)
+        except SwiftlyAuthError as exc:
+            print(f"\nAUTH FAILURE: {exc}", file=sys.stderr)
+            return 1
+
+        print(f"\ncategory={args.category!r}: {len(raw_items)} raw items")
+
+        aisle = LuckyCaliforniaScraper._aisle_from_slug(args.category)
+        mapped = []
+        for raw in raw_items:
+            m = LuckyCaliforniaScraper.map_product(
+                raw, aisle=aisle, source_slug=args.category
+            )
+            if m is not None:
+                mapped.append(m)
+        print(f"{len(mapped)} mapped products (after dropping unparseable rows)")
+
+        print("\n=== sample mapped products ===")
+        for m in mapped[: args.limit]:
+            # Decimal isn't JSON-serializable; coerce for display.
+            display = {k: (str(v) if k in {"current_price", "regular_price", "sale_price"} else v) for k, v in m.items()}
+            print(json.dumps(display, indent=2, default=str))
+            print()
+
+        return 0
+    finally:
+        scraper.cleanup()
+
+
+if __name__ == "__main__":
+    sys.exit(main())