admin/Meal-Planner
Public Access
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: phase r1+r2 recovery + r3-0 swiftly api ingestion

Browse Source 
R1 stabilization: pytest harness with transactional db fixture, smoke
+ alembic + auth + scrape + approval + swiftly tests, github actions
ci yaml. Bearer-token admin auth + signed-cookie session for family
ui mutations. Async POST /api/admin/scrape (BackgroundTasks, returns
202). Path canonicalization (no /list, /planned suffixes). DATABASE_URL
fail-fast on empty.

R2 deferred-risk spikes: live lucky california fetch (R2-A), full
email+per-voter approval click round trip with single-use enforcement
(R2-B, console email backend, sendgrid stub).

R3-0 phase 3 redesign: replaced playwright html scraper with requests
based swiftly json api client. 17 categories, ~10k products per scrape,
upsert by (source, external_id). 401 surfaces actionable token-refresh
message via ScrapeLog.error_message.

Pre-existing defects fixed: shopping_list.py syntax error blocking app
import, MealPlan.votes orphan relationship, JSONB(astext=True) invalid
kwarg, missing requests dep, calorie_target schema drift, every SQLEnum
needed values_callable, 0001 had empty downgrade(), seed had duplicate
ingredient rows.

Migrations added: 0003 grocery_item.description, 0004 family_profile.
calorie_target, 0005 grocery_item.external_id + source + composite index.

Verified: 31/31 pytest green, alembic upgrade->downgrade->upgrade clean,
frontend npm run build clean, live scrape 9,960 grocery_item rows in 36s.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
admin
2026-05-05 14:08:19 -07:00
co-authored by Claude Opus 4.7
parent b9434967ed
commit 8e89f793d5
58 changed files with 3594 additions and 348 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/app/services/__init__.py
View File
backend/app/services/approval.py
+89
View File
@@ -0,0 +1,89 @@
"""
Per-voter approval tokens.
Stateless signed tokens (itsdangerous) keyed on settings.SECRET_KEY with a
versioned salt. Single-use is enforced by the presence of a MealPlanVote
row for (item, voter) — the table already has UniqueConstraint on that
pair, so the DB is the source of truth, not a token-status column.
"""
from __future__ import annotations
from typing import TYPE_CHECKING
from uuid import UUID
from fastapi import HTTPException
from itsdangerous import (
BadSignature,
SignatureExpired,
URLSafeTimedSerializer,
)
from app.config import settings
from app.models import FamilyMember, MealPlanVote
if TYPE_CHECKING: # pragma: no cover
from sqlalchemy.orm import Session
SALT = "meal-approval-v1"
DEFAULT_MAX_AGE_SECONDS = 7 * 24 * 3600
def _serializer() -> URLSafeTimedSerializer:
return URLSafeTimedSerializer(secret_key=settings.SECRET_KEY, salt=SALT)
def issue_token(meal_plan_item_id: UUID, family_member_id: UUID) -> str:
payload = {
"item": str(meal_plan_item_id),
"voter": str(family_member_id),
}
return _serializer().dumps(payload)
def verify_token(token: str, max_age_seconds: int = DEFAULT_MAX_AGE_SECONDS) -> dict:
try:
payload = _serializer().loads(token, max_age=max_age_seconds)
except SignatureExpired:
raise HTTPException(status_code=401, detail="Token expired")
except BadSignature:
raise HTTPException(status_code=401, detail="Invalid token")
if not isinstance(payload, dict) or "item" not in payload or "voter" not in payload:
raise HTTPException(status_code=401, detail="Invalid token payload")
return payload
def consume_token(
db: "Session",
token: str,
meal_plan_item_id: UUID,
max_age_seconds: int = DEFAULT_MAX_AGE_SECONDS,
) -> FamilyMember:
"""Verify + match URL + enforce single-use. Returns the voter on success.
Single-use is checked by looking for an existing MealPlanVote row for
(item, voter). If one exists, raise 409.
"""
payload = verify_token(token, max_age_seconds=max_age_seconds)
if str(payload["item"]) != str(meal_plan_item_id):
raise HTTPException(status_code=400, detail="Token not valid for this meal")
voter_id = UUID(str(payload["voter"]))
voter = db.query(FamilyMember).filter(FamilyMember.id == voter_id).first()
if not voter:
raise HTTPException(status_code=404, detail="Voter not found")
existing = (
db.query(MealPlanVote)
.filter(
MealPlanVote.meal_plan_item_id == meal_plan_item_id,
MealPlanVote.family_member_id == voter_id,
)
.first()
)
if existing:
raise HTTPException(status_code=409, detail="Already voted")
return voter
backend/app/services/email.py
+89
View File
@@ -0,0 +1,89 @@
"""
Email backends.
R2-B spike: only ConsoleEmailBackend is functional. SendGrid is a stub
intentionally left to be wired in R3-C.
"""
from __future__ import annotations
import json
import os
import sys
from datetime import datetime, timezone
from pathlib import Path
from typing import Optional, Protocol
from app.config import settings
# Repository convention: backend/var/email_outbox.jsonl
# This module lives at backend/app/services/email.py — go up two levels to
# reach backend/, then var/.
_BACKEND_ROOT = Path(__file__).resolve().parent.parent.parent
OUTBOX_PATH = _BACKEND_ROOT / "var" / "email_outbox.jsonl"
class EmailBackend(Protocol):
def send(
self,
to: str,
subject: str,
html: str,
text: Optional[str] = None,
) -> None:
...
class ConsoleEmailBackend:
"""Dev/spike backend. Prints to stdout AND appends a JSON line to
backend/var/email_outbox.jsonl so the round-trip can be inspected
after the fact.
"""
def send(
self,
to: str,
subject: str,
html: str,
text: Optional[str] = None,
) -> None:
record = {
"ts": datetime.now(timezone.utc).isoformat(),
"to": to,
"subject": subject,
"html": html,
"text": text,
}
print(
f"[ConsoleEmailBackend] -> {to} | {subject}",
file=sys.stdout,
flush=True,
)
OUTBOX_PATH.parent.mkdir(parents=True, exist_ok=True)
with OUTBOX_PATH.open("a", encoding="utf-8") as f:
f.write(json.dumps(record) + "\n")
class SendGridEmailBackend:
"""Stub. Wire in R3-C (real SendGrid client + sandbox mode + retries).
Deliberately raises so a misconfigured prod env fails loudly instead of
silently dropping mail.
"""
def send(
self,
to: str,
subject: str,
html: str,
text: Optional[str] = None,
) -> None: # pragma: no cover - stub
raise NotImplementedError("Wire SendGrid in R3-C")
def get_email_backend() -> EmailBackend:
backend = (settings.EMAIL_BACKEND or "console").lower()
if backend == "sendgrid":
return SendGridEmailBackend()
return ConsoleEmailBackend()
backend/app/services/scraper_service.py
+204 -67
View File
@@ -1,87 +1,204 @@
"""Scraper service.
Two entry points:
- ``enqueue_scrape(db, source, scrape_type, background_tasks)``: synchronously
inserts a ``ScrapeLog`` row in status ``STARTED`` (the existing enum has no
``pending`` member; ``STARTED`` is reused for the queued state) and registers
``_run_scrape_in_background`` to fire after the response is sent.
- ``_run_scrape_in_background(log_id, source, scrape_type)``: runs in a
FastAPI background task with its OWN ``SessionLocal()`` (the request-scoped
``db`` is closed by the time this fires). Writes terminal status
(``SUCCESS``/``FAILED``) and ``error_message``.
``ScraperService.run_scrape`` is preserved for direct/test invocation; the
``/api/admin/scrape`` endpoint now goes through ``enqueue_scrape``.
"""
from __future__ import annotations
import logging
from typing import Dict, Any, Optional
from datetime import datetime
from uuid import uuid4
from typing import Any, Dict, Optional
from datetime import datetime, timezone
from uuid import UUID, uuid4
from fastapi import BackgroundTasks
from sqlalchemy.orm import Session
from app.database import SessionLocal
from app.models import GroceryItem, Ingredient, ScrapeLog, ScrapeStatus
logger = logging.getLogger(__name__)
# ---------------------------------------------------------------------------
# Public API: enqueue + background runner
# ---------------------------------------------------------------------------
def enqueue_scrape(
db: Session,
*,
source: str,
scrape_type: str,
background_tasks: BackgroundTasks,
) -> ScrapeLog:
"""Create the ScrapeLog row, commit, and schedule the background scrape.
Returns the persisted ``ScrapeLog`` instance (refreshed). The actual
scraping work runs after FastAPI sends the 202 response.
"""
log = ScrapeLog(
id=uuid4(),
source=source,
scrape_type=scrape_type,
status=ScrapeStatus.STARTED,
started_at=datetime.now(timezone.utc),
)
db.add(log)
db.commit()
db.refresh(log)
background_tasks.add_task(
_run_scrape_in_background, log.id, source, scrape_type
)
return log
def _run_scrape_in_background(
log_id: UUID, source: str, scrape_type: str
) -> None:
"""Background entry point. Opens a fresh DB session — the request-scoped
session is gone by the time this runs.
"""
db: Session = SessionLocal()
try:
log = db.query(ScrapeLog).filter(ScrapeLog.id == log_id).first()
if log is None:
logger.error("ScrapeLog %s vanished before background run", log_id)
return
# No "running" state in the enum; STARTED already covers in-flight.
# Mark started_at fresh in case there was lag between enqueue and run.
try:
service = ScraperService(db)
saved_count, items_found = service._do_scrape(source=source)
log.status = ScrapeStatus.SUCCESS
log.items_scraped = saved_count
log.completed_at = datetime.now(timezone.utc)
log.duration_seconds = int(
(log.completed_at - log.started_at).total_seconds()
)
db.commit()
logger.info(
"Background scrape %s complete: %s/%s items saved",
log_id, saved_count, items_found,
)
except Exception as exc: # noqa: BLE001 — must catch all to mark failed
logger.exception("Background scrape %s failed", log_id)
db.rollback()
# Re-fetch in case the rollback detached the instance.
log = db.query(ScrapeLog).filter(ScrapeLog.id == log_id).first()
if log is not None:
log.status = ScrapeStatus.FAILED
log.error_message = str(exc)
log.completed_at = datetime.now(timezone.utc)
if log.started_at:
log.duration_seconds = int(
(log.completed_at - log.started_at).total_seconds()
)
db.commit()
finally:
db.close()
# ---------------------------------------------------------------------------
# Service class (kept for direct/test use)
# ---------------------------------------------------------------------------
class ScraperService:
def __init__(self, db: Session):
self.db = db
def run_scrape(self, source: str = "lucky_california", scrape_type: str = "weekly_ad") -> Dict[str, Any]:
from app.scraper import LuckyCaliforniaScraper
from app.models import ScrapeLog, GroceryItem, Ingredient
scrape_log = ScrapeLog(
def run_scrape(
self, source: str = "lucky_california", scrape_type: str = "weekly_ad"
) -> Dict[str, Any]:
"""Synchronous end-to-end scrape (legacy path). Creates the log row,
runs the scrape, commits terminal status. Used by direct callers and
tests; the API endpoint goes through ``enqueue_scrape``.
"""
log = ScrapeLog(
id=uuid4(),
source=source,
scrape_type=scrape_type,
status="started",
started_at=datetime.now()
status=ScrapeStatus.STARTED,
started_at=datetime.now(timezone.utc),
)
self.db.add(scrape_log)
self.db.add(log)
self.db.commit()
logger.info(f"Starting {source} {scrape_type} scrape")
logger.info("Starting %s %s scrape", source, scrape_type)
try:
scraper = LuckyCaliforniaScraper()
result = scraper.scrape()
scraper.cleanup()
items = result.get("items", [])
saved_count = 0
for item_data in items:
saved_item = self._save_grocery_item(item_data)
if saved_item:
saved_count += 1
scrape_log.status = "success"
scrape_log.items_scraped = saved_count
scrape_log.completed_at = datetime.now()
scrape_log.duration_seconds = int(
(scrape_log.completed_at - scrape_log.started_at).total_seconds()
saved_count, items_found = self._do_scrape(source=source)
log.status = ScrapeStatus.SUCCESS
log.items_scraped = saved_count
log.completed_at = datetime.now(timezone.utc)
log.duration_seconds = int(
(log.completed_at - log.started_at).total_seconds()
)
self.db.commit()
logger.info(f"Scrape complete: {saved_count} items saved")
logger.info("Scrape complete: %s items saved", saved_count)
return {
"scrape_id": str(scrape_log.id),
"scrape_id": str(log.id),
"status": "success",
"items_scraped": saved_count,
"items_found": len(items)
"items_found": items_found,
}
except Exception as e:
logger.error(f"Scrape failed: {e}")
scrape_log.status = "failed"
scrape_log.error_message = str(e)
scrape_log.completed_at = datetime.now()
scrape_log.duration_seconds = int(
(scrape_log.completed_at - scrape_log.started_at).total_seconds()
except Exception as e: # noqa: BLE001
logger.error("Scrape failed: %s", e)
log.status = ScrapeStatus.FAILED
log.error_message = str(e)
log.completed_at = datetime.now(timezone.utc)
log.duration_seconds = int(
(log.completed_at - log.started_at).total_seconds()
)
self.db.commit()
return {
"scrape_id": str(scrape_log.id),
"scrape_id": str(log.id),
"status": "failed",
"error": str(e)
"error": str(e),
}
def _save_grocery_item(self, item_data: Dict[str, Any]) -> Optional[GroceryItem]:
from app.models import GroceryItem, Ingredient
# ------------------------------------------------------------------
# Internals
# ------------------------------------------------------------------
def _do_scrape(self, *, source: str) -> tuple[int, int]:
"""Run the scraper and persist items. Returns (saved, found).
name = item_data.get("name", "").strip()
Raises any scraper exception (including ``SwiftlyAuthError``) to
the caller for status mapping.
"""
from app.scraper import LuckyCaliforniaScraper
scraper = LuckyCaliforniaScraper()
saved_count = 0
found_count = 0
try:
for item_data in scraper.fetch_all():
found_count += 1
if self._save_grocery_item(item_data) is not None:
saved_count += 1
finally:
scraper.cleanup()
return saved_count, found_count
def _save_grocery_item(
self, item_data: Dict[str, Any]
) -> Optional[GroceryItem]:
name = (item_data.get("name") or "").strip()
if not name:
return None
name_lower = name.lower()
source = item_data.get("source") or "lucky_california"
external_id = item_data.get("external_id")
ingredient = self.db.query(Ingredient).filter(
Ingredient.name_lower == name_lower
@@ -93,22 +210,41 @@ class ScraperService:
name=name,
name_lower=name_lower,
aisle=item_data.get("aisle"),
typical_price=item_data.get("current_price")
typical_price=item_data.get("current_price"),
)
self.db.add(ingredient)
self.db.flush()
existing = self.db.query(GroceryItem).filter(
GroceryItem.name == name,
GroceryItem.scraped_url == item_data.get("scraped_url")
).first()
# Idempotency:
# 1) (source, external_id) when both present (Swiftly path);
# 2) fall back to (name, scraped_url) for legacy rows from R2-A.
existing: Optional[GroceryItem] = None
if external_id:
existing = self.db.query(GroceryItem).filter(
GroceryItem.source == source,
GroceryItem.external_id == external_id,
).first()
if existing is None:
existing = self.db.query(GroceryItem).filter(
GroceryItem.name == name,
GroceryItem.scraped_url == item_data.get("scraped_url"),
).first()
if existing:
existing.ingredient_id = ingredient.id
existing.brand = item_data.get("brand")
existing.current_price = item_data.get("current_price")
existing.is_on_sale = item_data.get("is_on_sale", True)
existing.regular_price = item_data.get("regular_price")
existing.unit = item_data.get("unit")
existing.aisle = item_data.get("aisle")
existing.image_url = item_data.get("image_url")
existing.product_url = item_data.get("product_url")
existing.scraped_at = datetime.now()
existing.description = item_data.get("description")
existing.is_on_sale = bool(item_data.get("is_on_sale", False))
existing.scraped_at = datetime.now(timezone.utc)
existing.scraped_url = item_data.get("scraped_url")
existing.external_id = external_id
existing.source = source
self.db.flush()
return existing
@@ -116,31 +252,31 @@ class ScraperService:
id=uuid4(),
ingredient_id=ingredient.id,
name=name,
brand=item_data.get("brand"),
current_price=item_data.get("current_price"),
regular_price=item_data.get("regular_price"),
unit=item_data.get("unit"),
aisle=item_data.get("aisle"),
image_url=item_data.get("image_url"),
product_url=item_data.get("product_url"),
is_on_sale=item_data.get("is_on_sale", True),
description=item_data.get("description"),
is_on_sale=bool(item_data.get("is_on_sale", False)),
sale_start_date=item_data.get("sale_start_date"),
sale_end_date=item_data.get("sale_end_date"),
in_season=item_data.get("in_season", False),
scraped_at=datetime.now(),
scraped_url=item_data.get("scraped_url")
scraped_at=datetime.now(timezone.utc),
scraped_url=item_data.get("scraped_url"),
external_id=external_id,
source=source,
)
self.db.add(grocery_item)
self.db.commit()
self.db.refresh(grocery_item)
self.db.flush()
return grocery_item
def get_sale_items(self, limit: int = 50) -> list:
from app.models import GroceryItem
items = self.db.query(GroceryItem).filter(
GroceryItem.is_on_sale == True
GroceryItem.is_on_sale == True # noqa: E712 — SQLAlchemy idiom
).order_by(GroceryItem.scraped_at.desc()).limit(limit).all()
return [
@@ -152,7 +288,8 @@ class ScraperService:
"aisle": item.aisle,
"image_url": item.image_url,
"product_url": item.product_url,
"scraped_at": item.scraped_at.isoformat() if item.scraped_at else None
"description": item.description,
"scraped_at": item.scraped_at.isoformat() if item.scraped_at else None,
}
for item in items
]
]