Public Access
adminandClaude Opus 4.7
8e89f793d5
R1 stabilization: pytest harness with transactional db fixture, smoke + alembic + auth + scrape + approval + swiftly tests, github actions ci yaml. Bearer-token admin auth + signed-cookie session for family ui mutations. Async POST /api/admin/scrape (BackgroundTasks, returns 202). Path canonicalization (no /list, /planned suffixes). DATABASE_URL fail-fast on empty. R2 deferred-risk spikes: live lucky california fetch (R2-A), full email+per-voter approval click round trip with single-use enforcement (R2-B, console email backend, sendgrid stub). R3-0 phase 3 redesign: replaced playwright html scraper with requests based swiftly json api client. 17 categories, ~10k products per scrape, upsert by (source, external_id). 401 surfaces actionable token-refresh message via ScrapeLog.error_message. Pre-existing defects fixed: shopping_list.py syntax error blocking app import, MealPlan.votes orphan relationship, JSONB(astext=True) invalid kwarg, missing requests dep, calorie_target schema drift, every SQLEnum needed values_callable, 0001 had empty downgrade(), seed had duplicate ingredient rows. Migrations added: 0003 grocery_item.description, 0004 family_profile. calorie_target, 0005 grocery_item.external_id + source + composite index. Verified: 31/31 pytest green, alembic upgrade->downgrade->upgrade clean, frontend npm run build clean, live scrape 9,960 grocery_item rows in 36s. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
102 lines
3.3 KiB
Python
102 lines
3.3 KiB
Python
"""Live spike for the Swiftly product API (R3-0, ad-hoc admin tool).
|
|
|
|
Usage:
|
|
python scripts/spike_swiftly_ingest.py --confirm-live
|
|
|
|
Fetches the categories page + ONE category (default ``Product/meat_seafood``)
|
|
against the live Swiftly API, prints the item count, and pretty-prints
|
|
two sample mapped products. Does NOT persist anything to Postgres — the
|
|
purpose is to verify the token + parser round-trip on demand without
|
|
running the full scrape.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import json
|
|
import os
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
# Make `app.*` importable when invoked from the repo root.
|
|
REPO_ROOT = Path(__file__).resolve().parent.parent
|
|
BACKEND_DIR = REPO_ROOT / "backend"
|
|
if str(BACKEND_DIR) not in sys.path:
|
|
sys.path.insert(0, str(BACKEND_DIR))
|
|
|
|
# Allow running outside docker without a real Postgres URL just to exercise
|
|
# the scraper. Settings still requires DATABASE_URL to be non-empty.
|
|
os.environ.setdefault("DATABASE_URL", "postgresql://placeholder@localhost:5432/placeholder")
|
|
|
|
from app.scraper.lucky_ca_scraper import LuckyCaliforniaScraper, SwiftlyAuthError # noqa: E402
|
|
|
|
|
|
def main() -> int:
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument(
|
|
"--confirm-live",
|
|
action="store_true",
|
|
help="Required: hit the live Swiftly API. Without this flag the script no-ops.",
|
|
)
|
|
parser.add_argument(
|
|
"--category",
|
|
default="Product/meat_seafood",
|
|
help="API slug to fetch (default: Product/meat_seafood).",
|
|
)
|
|
parser.add_argument(
|
|
"--limit",
|
|
type=int,
|
|
default=2,
|
|
help="How many sample mapped products to print (default 2).",
|
|
)
|
|
args = parser.parse_args()
|
|
|
|
if not args.confirm_live:
|
|
print(
|
|
"Refusing to make live HTTP calls without --confirm-live.\n"
|
|
"Pass --confirm-live to opt in.",
|
|
file=sys.stderr,
|
|
)
|
|
return 2
|
|
|
|
scraper = LuckyCaliforniaScraper()
|
|
try:
|
|
slugs = scraper.discover_categories()
|
|
print(f"discovered {len(slugs)} categories")
|
|
for s in slugs[:5]:
|
|
print(f" - {s}")
|
|
if len(slugs) > 5:
|
|
print(f" ... ({len(slugs) - 5} more)")
|
|
|
|
try:
|
|
raw_items = scraper.fetch_category(args.category)
|
|
except SwiftlyAuthError as exc:
|
|
print(f"\nAUTH FAILURE: {exc}", file=sys.stderr)
|
|
return 1
|
|
|
|
print(f"\ncategory={args.category!r}: {len(raw_items)} raw items")
|
|
|
|
aisle = LuckyCaliforniaScraper._aisle_from_slug(args.category)
|
|
mapped = []
|
|
for raw in raw_items:
|
|
m = LuckyCaliforniaScraper.map_product(
|
|
raw, aisle=aisle, source_slug=args.category
|
|
)
|
|
if m is not None:
|
|
mapped.append(m)
|
|
print(f"{len(mapped)} mapped products (after dropping unparseable rows)")
|
|
|
|
print("\n=== sample mapped products ===")
|
|
for m in mapped[: args.limit]:
|
|
# Decimal isn't JSON-serializable; coerce for display.
|
|
display = {k: (str(v) if k in {"current_price", "regular_price", "sale_price"} else v) for k, v in m.items()}
|
|
print(json.dumps(display, indent=2, default=str))
|
|
print()
|
|
|
|
return 0
|
|
finally:
|
|
scraper.cleanup()
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|