admin/Meal-Planner
Public Access
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: phase r1+r2 recovery + r3-0 swiftly api ingestion

Browse Source 
R1 stabilization: pytest harness with transactional db fixture, smoke
+ alembic + auth + scrape + approval + swiftly tests, github actions
ci yaml. Bearer-token admin auth + signed-cookie session for family
ui mutations. Async POST /api/admin/scrape (BackgroundTasks, returns
202). Path canonicalization (no /list, /planned suffixes). DATABASE_URL
fail-fast on empty.

R2 deferred-risk spikes: live lucky california fetch (R2-A), full
email+per-voter approval click round trip with single-use enforcement
(R2-B, console email backend, sendgrid stub).

R3-0 phase 3 redesign: replaced playwright html scraper with requests
based swiftly json api client. 17 categories, ~10k products per scrape,
upsert by (source, external_id). 401 surfaces actionable token-refresh
message via ScrapeLog.error_message.

Pre-existing defects fixed: shopping_list.py syntax error blocking app
import, MealPlan.votes orphan relationship, JSONB(astext=True) invalid
kwarg, missing requests dep, calorie_target schema drift, every SQLEnum
needed values_callable, 0001 had empty downgrade(), seed had duplicate
ingredient rows.

Migrations added: 0003 grocery_item.description, 0004 family_profile.
calorie_target, 0005 grocery_item.external_id + source + composite index.

Verified: 31/31 pytest green, alembic upgrade->downgrade->upgrade clean,
frontend npm run build clean, live scrape 9,960 grocery_item rows in 36s.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
admin
2026-05-05 14:08:19 -07:00
co-authored by Claude Opus 4.7
parent b9434967ed
commit 8e89f793d5
58 changed files with 3594 additions and 348 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/app/services/scraper_service.py
+204 -67
View File
@@ -1,87 +1,204 @@
"""Scraper service.
Two entry points:
- ``enqueue_scrape(db, source, scrape_type, background_tasks)``: synchronously
inserts a ``ScrapeLog`` row in status ``STARTED`` (the existing enum has no
``pending`` member; ``STARTED`` is reused for the queued state) and registers
``_run_scrape_in_background`` to fire after the response is sent.
- ``_run_scrape_in_background(log_id, source, scrape_type)``: runs in a
FastAPI background task with its OWN ``SessionLocal()`` (the request-scoped
``db`` is closed by the time this fires). Writes terminal status
(``SUCCESS``/``FAILED``) and ``error_message``.
``ScraperService.run_scrape`` is preserved for direct/test invocation; the
``/api/admin/scrape`` endpoint now goes through ``enqueue_scrape``.
"""
from __future__ import annotations
import logging
from typing import Dict, Any, Optional
from datetime import datetime
from uuid import uuid4
from typing import Any, Dict, Optional
from datetime import datetime, timezone
from uuid import UUID, uuid4
from fastapi import BackgroundTasks
from sqlalchemy.orm import Session
from app.database import SessionLocal
from app.models import GroceryItem, Ingredient, ScrapeLog, ScrapeStatus
logger = logging.getLogger(__name__)
# ---------------------------------------------------------------------------
# Public API: enqueue + background runner
# ---------------------------------------------------------------------------
def enqueue_scrape(
db: Session,
*,
source: str,
scrape_type: str,
background_tasks: BackgroundTasks,
) -> ScrapeLog:
"""Create the ScrapeLog row, commit, and schedule the background scrape.
Returns the persisted ``ScrapeLog`` instance (refreshed). The actual
scraping work runs after FastAPI sends the 202 response.
"""
log = ScrapeLog(
id=uuid4(),
source=source,
scrape_type=scrape_type,
status=ScrapeStatus.STARTED,
started_at=datetime.now(timezone.utc),
)
db.add(log)
db.commit()
db.refresh(log)
background_tasks.add_task(
_run_scrape_in_background, log.id, source, scrape_type
)
return log
def _run_scrape_in_background(
log_id: UUID, source: str, scrape_type: str
) -> None:
"""Background entry point. Opens a fresh DB session — the request-scoped
session is gone by the time this runs.
"""
db: Session = SessionLocal()
try:
log = db.query(ScrapeLog).filter(ScrapeLog.id == log_id).first()
if log is None:
logger.error("ScrapeLog %s vanished before background run", log_id)
return
# No "running" state in the enum; STARTED already covers in-flight.
# Mark started_at fresh in case there was lag between enqueue and run.
try:
service = ScraperService(db)
saved_count, items_found = service._do_scrape(source=source)
log.status = ScrapeStatus.SUCCESS
log.items_scraped = saved_count
log.completed_at = datetime.now(timezone.utc)
log.duration_seconds = int(
(log.completed_at - log.started_at).total_seconds()
)
db.commit()
logger.info(
"Background scrape %s complete: %s/%s items saved",
log_id, saved_count, items_found,
)
except Exception as exc: # noqa: BLE001 — must catch all to mark failed
logger.exception("Background scrape %s failed", log_id)
db.rollback()
# Re-fetch in case the rollback detached the instance.
log = db.query(ScrapeLog).filter(ScrapeLog.id == log_id).first()
if log is not None:
log.status = ScrapeStatus.FAILED
log.error_message = str(exc)
log.completed_at = datetime.now(timezone.utc)
if log.started_at:
log.duration_seconds = int(
(log.completed_at - log.started_at).total_seconds()
)
db.commit()
finally:
db.close()
# ---------------------------------------------------------------------------
# Service class (kept for direct/test use)
# ---------------------------------------------------------------------------
class ScraperService:
def __init__(self, db: Session):
self.db = db
def run_scrape(self, source: str = "lucky_california", scrape_type: str = "weekly_ad") -> Dict[str, Any]:
from app.scraper import LuckyCaliforniaScraper
from app.models import ScrapeLog, GroceryItem, Ingredient
scrape_log = ScrapeLog(
def run_scrape(
self, source: str = "lucky_california", scrape_type: str = "weekly_ad"
) -> Dict[str, Any]:
"""Synchronous end-to-end scrape (legacy path). Creates the log row,
runs the scrape, commits terminal status. Used by direct callers and
tests; the API endpoint goes through ``enqueue_scrape``.
"""
log = ScrapeLog(
id=uuid4(),
source=source,
scrape_type=scrape_type,
status="started",
started_at=datetime.now()
status=ScrapeStatus.STARTED,
started_at=datetime.now(timezone.utc),
)
self.db.add(scrape_log)
self.db.add(log)
self.db.commit()
logger.info(f"Starting {source} {scrape_type} scrape")
logger.info("Starting %s %s scrape", source, scrape_type)
try:
scraper = LuckyCaliforniaScraper()
result = scraper.scrape()
scraper.cleanup()
items = result.get("items", [])
saved_count = 0
for item_data in items:
saved_item = self._save_grocery_item(item_data)
if saved_item:
saved_count += 1
scrape_log.status = "success"
scrape_log.items_scraped = saved_count
scrape_log.completed_at = datetime.now()
scrape_log.duration_seconds = int(
(scrape_log.completed_at - scrape_log.started_at).total_seconds()
saved_count, items_found = self._do_scrape(source=source)
log.status = ScrapeStatus.SUCCESS
log.items_scraped = saved_count
log.completed_at = datetime.now(timezone.utc)
log.duration_seconds = int(
(log.completed_at - log.started_at).total_seconds()
)
self.db.commit()
logger.info(f"Scrape complete: {saved_count} items saved")
logger.info("Scrape complete: %s items saved", saved_count)
return {
"scrape_id": str(scrape_log.id),
"scrape_id": str(log.id),
"status": "success",
"items_scraped": saved_count,
"items_found": len(items)
"items_found": items_found,
}
except Exception as e:
logger.error(f"Scrape failed: {e}")
scrape_log.status = "failed"
scrape_log.error_message = str(e)
scrape_log.completed_at = datetime.now()
scrape_log.duration_seconds = int(
(scrape_log.completed_at - scrape_log.started_at).total_seconds()
except Exception as e: # noqa: BLE001
logger.error("Scrape failed: %s", e)
log.status = ScrapeStatus.FAILED
log.error_message = str(e)
log.completed_at = datetime.now(timezone.utc)
log.duration_seconds = int(
(log.completed_at - log.started_at).total_seconds()
)
self.db.commit()
return {
"scrape_id": str(scrape_log.id),
"scrape_id": str(log.id),
"status": "failed",
"error": str(e)
"error": str(e),
}
def _save_grocery_item(self, item_data: Dict[str, Any]) -> Optional[GroceryItem]:
from app.models import GroceryItem, Ingredient
# ------------------------------------------------------------------
# Internals
# ------------------------------------------------------------------
def _do_scrape(self, *, source: str) -> tuple[int, int]:
"""Run the scraper and persist items. Returns (saved, found).
name = item_data.get("name", "").strip()
Raises any scraper exception (including ``SwiftlyAuthError``) to
the caller for status mapping.
"""
from app.scraper import LuckyCaliforniaScraper
scraper = LuckyCaliforniaScraper()
saved_count = 0
found_count = 0
try:
for item_data in scraper.fetch_all():
found_count += 1
if self._save_grocery_item(item_data) is not None:
saved_count += 1
finally:
scraper.cleanup()
return saved_count, found_count
def _save_grocery_item(
self, item_data: Dict[str, Any]
) -> Optional[GroceryItem]:
name = (item_data.get("name") or "").strip()
if not name:
return None
name_lower = name.lower()
source = item_data.get("source") or "lucky_california"
external_id = item_data.get("external_id")
ingredient = self.db.query(Ingredient).filter(
Ingredient.name_lower == name_lower
@@ -93,22 +210,41 @@ class ScraperService:
name=name,
name_lower=name_lower,
aisle=item_data.get("aisle"),
typical_price=item_data.get("current_price")
typical_price=item_data.get("current_price"),
)
self.db.add(ingredient)
self.db.flush()
existing = self.db.query(GroceryItem).filter(
GroceryItem.name == name,
GroceryItem.scraped_url == item_data.get("scraped_url")
).first()
# Idempotency:
# 1) (source, external_id) when both present (Swiftly path);
# 2) fall back to (name, scraped_url) for legacy rows from R2-A.
existing: Optional[GroceryItem] = None
if external_id:
existing = self.db.query(GroceryItem).filter(
GroceryItem.source == source,
GroceryItem.external_id == external_id,
).first()
if existing is None:
existing = self.db.query(GroceryItem).filter(
GroceryItem.name == name,
GroceryItem.scraped_url == item_data.get("scraped_url"),
).first()
if existing:
existing.ingredient_id = ingredient.id
existing.brand = item_data.get("brand")
existing.current_price = item_data.get("current_price")
existing.is_on_sale = item_data.get("is_on_sale", True)
existing.regular_price = item_data.get("regular_price")
existing.unit = item_data.get("unit")
existing.aisle = item_data.get("aisle")
existing.image_url = item_data.get("image_url")
existing.product_url = item_data.get("product_url")
existing.scraped_at = datetime.now()
existing.description = item_data.get("description")
existing.is_on_sale = bool(item_data.get("is_on_sale", False))
existing.scraped_at = datetime.now(timezone.utc)
existing.scraped_url = item_data.get("scraped_url")
existing.external_id = external_id
existing.source = source
self.db.flush()
return existing
@@ -116,31 +252,31 @@ class ScraperService:
id=uuid4(),
ingredient_id=ingredient.id,
name=name,
brand=item_data.get("brand"),
current_price=item_data.get("current_price"),
regular_price=item_data.get("regular_price"),
unit=item_data.get("unit"),
aisle=item_data.get("aisle"),
image_url=item_data.get("image_url"),
product_url=item_data.get("product_url"),
is_on_sale=item_data.get("is_on_sale", True),
description=item_data.get("description"),
is_on_sale=bool(item_data.get("is_on_sale", False)),
sale_start_date=item_data.get("sale_start_date"),
sale_end_date=item_data.get("sale_end_date"),
in_season=item_data.get("in_season", False),
scraped_at=datetime.now(),
scraped_url=item_data.get("scraped_url")
scraped_at=datetime.now(timezone.utc),
scraped_url=item_data.get("scraped_url"),
external_id=external_id,
source=source,
)
self.db.add(grocery_item)
self.db.commit()
self.db.refresh(grocery_item)
self.db.flush()
return grocery_item
def get_sale_items(self, limit: int = 50) -> list:
from app.models import GroceryItem
items = self.db.query(GroceryItem).filter(
GroceryItem.is_on_sale == True
GroceryItem.is_on_sale == True # noqa: E712 — SQLAlchemy idiom
).order_by(GroceryItem.scraped_at.desc()).limit(limit).all()
return [
@@ -152,7 +288,8 @@ class ScraperService:
"aisle": item.aisle,
"image_url": item.image_url,
"product_url": item.product_url,
"scraped_at": item.scraped_at.isoformat() if item.scraped_at else None
"description": item.description,
"scraped_at": item.scraped_at.isoformat() if item.scraped_at else None,
}
for item in items
]
]