feat: phase r1+r2 recovery + r3-0 swiftly api ingestion

R1 stabilization: pytest harness with transactional db fixture, smoke
+ alembic + auth + scrape + approval + swiftly tests, github actions
ci yaml. Bearer-token admin auth + signed-cookie session for family
ui mutations. Async POST /api/admin/scrape (BackgroundTasks, returns
202). Path canonicalization (no /list, /planned suffixes). DATABASE_URL
fail-fast on empty.

R2 deferred-risk spikes: live lucky california fetch (R2-A), full
email+per-voter approval click round trip with single-use enforcement
(R2-B, console email backend, sendgrid stub).

R3-0 phase 3 redesign: replaced playwright html scraper with requests
based swiftly json api client. 17 categories, ~10k products per scrape,
upsert by (source, external_id). 401 surfaces actionable token-refresh
message via ScrapeLog.error_message.

Pre-existing defects fixed: shopping_list.py syntax error blocking app
import, MealPlan.votes orphan relationship, JSONB(astext=True) invalid
kwarg, missing requests dep, calorie_target schema drift, every SQLEnum
needed values_callable, 0001 had empty downgrade(), seed had duplicate
ingredient rows.

Migrations added: 0003 grocery_item.description, 0004 family_profile.
calorie_target, 0005 grocery_item.external_id + source + composite index.

Verified: 31/31 pytest green, alembic upgrade->downgrade->upgrade clean,
frontend npm run build clean, live scrape 9,960 grocery_item rows in 36s.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
2026-05-05 14:08:19 -07:00
co-authored by Claude Opus 4.7
parent b9434967ed
commit 8e89f793d5
58 changed files with 3594 additions and 348 deletions
@@ -98,7 +98,7 @@ def upgrade() -> None:
sa.Column('dietary_tags', postgresql.ARRAY(sa.String(length=50)), nullable=True),
sa.Column('protein_type', sa.String(length=50), nullable=True),
sa.Column('spice_level', sa.Integer(), nullable=True),
sa.Column('ingredients', postgresql.JSONB(astext=True), nullable=False),
sa.Column('ingredients', postgresql.JSONB(), nullable=False),
sa.Column('instructions', postgresql.ARRAY(sa.Text()), nullable=False),
sa.Column('source_url', sa.Text(), nullable=True),
sa.Column('scraped_at', sa.DateTime(timezone=True), nullable=True),
@@ -286,4 +286,17 @@ def upgrade() -> None:
def downgrade() -> None:
pass
op.execute(
"""
DO $$ DECLARE
r RECORD;
BEGIN
FOR r IN (SELECT tablename FROM pg_tables WHERE schemaname = 'public' AND tablename != 'alembic_version') LOOP
EXECUTE 'DROP TABLE IF EXISTS public.' || quote_ident(r.tablename) || ' CASCADE';
END LOOP;
FOR r IN (SELECT t.typname FROM pg_type t JOIN pg_namespace n ON n.oid = t.typnamespace WHERE t.typtype = 'e' AND n.nspname = 'public') LOOP
EXECUTE 'DROP TYPE IF EXISTS public.' || quote_ident(r.typname) || ' CASCADE';
END LOOP;
END $$;
"""
)
+1 -2
View File
@@ -95,8 +95,6 @@ def upgrade() -> None:
('Sugar', 'sugar', 'lb', 'Pantry', 2.49),
('Brown Rice', 'brown rice', 'lb', 'Grains', 3.49),
('Oats', 'oats', 'lb', 'Grains', 2.99),
('Chickpeas', 'chickpeas', 'can', 'Canned Goods', 1.49),
('Black Beans', 'black beans', 'can', 'Canned Goods', 1.29),
('Kidney Beans', 'kidney beans', 'can', 'Canned Goods', 1.29),
('Corn', 'corn', 'can', 'Canned Goods', 1.49),
('Green Beans', 'green beans', 'can', 'Canned Goods', 1.49),
@@ -107,6 +105,7 @@ def upgrade() -> None:
op.execute(f"""
INSERT INTO ingredient (id, name, name_lower, unit, aisle, typical_price)
VALUES (uuid_generate_v4(), '{name}', '{name_lower}', '{unit}', '{aisle}', {price})
ON CONFLICT (name_lower) DO NOTHING
""")
@@ -0,0 +1,32 @@
"""Add nullable description column to grocery_item.
The Lucky California parser produces a long-form description per coupon
(class ``coupon-card-short-description``); without this column it is dropped
silently when persisting. See R2-A spike notes in
``.agent/phase-summaries/r2a-summary.md``.
Revision ID: 0003
Revises: 0002
Create Date: 2026-05-04
"""
from typing import Sequence, Union
from alembic import op
import sqlalchemy as sa
revision: str = '0003'
down_revision: Union[str, None] = '0002'
branch_labels: Union[str, Sequence[str], None] = None
depends_on: Union[str, Sequence[str], None] = None
def upgrade() -> None:
op.add_column(
"grocery_item",
sa.Column("description", sa.Text(), nullable=True),
)
def downgrade() -> None:
op.drop_column("grocery_item", "description")
@@ -0,0 +1,30 @@
"""Add calorie_target to family_profile
Revision ID: 0004
Revises: 0003
Create Date: 2026-05-04
The model has carried `calorie_target` on FamilyProfile since Phase 2,
but the initial migration omitted it. SELECT * from family_profile fails
without this column. Adversarial review §1.6 flagged the model/schema
drift.
"""
from alembic import op
import sqlalchemy as sa
revision = "0004"
down_revision = "0003"
branch_labels = None
depends_on = None
def upgrade() -> None:
op.add_column(
"family_profile",
sa.Column("calorie_target", sa.Integer(), nullable=True),
)
def downgrade() -> None:
op.drop_column("family_profile", "calorie_target")
@@ -0,0 +1,48 @@
"""Add nullable indexed external_id and source columns to grocery_item.
The Swiftly API exposes a stable ``id`` per product (e.g. ``"46556"``).
Persisting it as ``grocery_item.external_id`` lets the scraper UPSERT by
``(source, external_id)`` rather than create duplicates on every run.
``source`` distinguishes overlapping IDs across future banners (e.g. a
second Save Mart store using the same Swiftly tenant). See R3-0 notes in
``.agent/phase-summaries/r3-0-summary.md``.
Revision ID: 0005
Revises: 0004
Create Date: 2026-05-05
"""
from typing import Sequence, Union
from alembic import op
import sqlalchemy as sa
revision: str = '0005'
down_revision: Union[str, None] = '0004'
branch_labels: Union[str, Sequence[str], None] = None
depends_on: Union[str, Sequence[str], None] = None
def upgrade() -> None:
op.add_column(
"grocery_item",
sa.Column("external_id", sa.String(length=100), nullable=True),
)
op.add_column(
"grocery_item",
sa.Column(
"source", sa.String(length=50), nullable=True,
),
)
op.create_index(
"ix_grocery_item_source_external_id",
"grocery_item",
["source", "external_id"],
unique=False,
)
def downgrade() -> None:
op.drop_index("ix_grocery_item_source_external_id", table_name="grocery_item")
op.drop_column("grocery_item", "source")
op.drop_column("grocery_item", "external_id")
+18 -9
View File
@@ -1,20 +1,29 @@
from fastapi import APIRouter, Depends, HTTPException
from fastapi import APIRouter, BackgroundTasks, Depends, HTTPException
from sqlalchemy.orm import Session
from app.database import get_db
from app.models import ScrapeLog, EmailLog, MealPlan
from app.services.scraper_service import ScraperService
from app.security import require_admin
from app.services.scraper_service import ScraperService, enqueue_scrape
from typing import List, Optional
from datetime import datetime, timedelta
router = APIRouter()
router = APIRouter(dependencies=[Depends(require_admin)])
@router.post("/scrape")
def trigger_scrape(source: str = "lucky_california", scrape_type: str = "weekly_ad", db: Session = Depends(get_db)):
scraper_service = ScraperService(db)
result = scraper_service.run_scrape(source=source, scrape_type=scrape_type)
return result
@router.post("/scrape", status_code=202)
def trigger_scrape(
background_tasks: BackgroundTasks,
source: str = "lucky_california",
scrape_type: str = "weekly_ad",
db: Session = Depends(get_db),
):
log = enqueue_scrape(
db,
source=source,
scrape_type=scrape_type,
background_tasks=background_tasks,
)
return {"status": "queued", "scrape_log_id": str(log.id)}
@router.get("/logs")
+67
View File
@@ -0,0 +1,67 @@
"""
Family-shared session login.
POST /api/auth/login — body ``{"password": "..."}`` — must match
``settings.SESSION_PASSWORD``. On success: signs the first FamilyProfile.id
and writes it as the ``mp_session`` cookie, returns 204.
POST /api/auth/logout — clears the cookie, returns 204.
The session is intentionally simple: a single shared family password gates
mutations behind nginx on the trusted network. No per-user auth.
"""
from fastapi import APIRouter, Depends, HTTPException, Response, status
from pydantic import BaseModel
from sqlalchemy.orm import Session
from app.config import settings
from app.database import get_db
from app.models import FamilyProfile
from app.security import SESSION_COOKIE, SESSION_MAX_AGE, issue_session
router = APIRouter()
class LoginRequest(BaseModel):
password: str
@router.post("/login")
def login(payload: LoginRequest, db: Session = Depends(get_db)):
expected = settings.SESSION_PASSWORD
if not expected:
raise HTTPException(
status_code=status.HTTP_503_SERVICE_UNAVAILABLE,
detail="Session auth not configured",
)
if payload.password != expected:
raise HTTPException(
status_code=status.HTTP_401_UNAUTHORIZED, detail="Invalid password"
)
profile = db.query(FamilyProfile).first()
# If no profile exists yet, sign a placeholder so the cookie still
# validates; the family-id will be re-issued the first time a profile
# is created. This avoids login being blocked on first-run.
family_id = str(profile.id) if profile else "bootstrap"
cookie_value = issue_session(family_id)
response = Response(status_code=status.HTTP_204_NO_CONTENT)
response.set_cookie(
key=SESSION_COOKIE,
value=cookie_value,
max_age=SESSION_MAX_AGE,
httponly=True,
secure=True,
samesite="lax",
path="/",
)
return response
@router.post("/logout")
def logout():
response = Response(status_code=status.HTTP_204_NO_CONTENT)
response.delete_cookie(key=SESSION_COOKIE, path="/")
return response
+158 -60
View File
@@ -1,4 +1,7 @@
from fastapi import APIRouter, Depends, HTTPException
from fastapi import APIRouter, Depends, HTTPException, Query
from fastapi.responses import HTMLResponse
from html import escape as _html_escape
from pydantic import BaseModel, Field
from sqlalchemy.orm import Session, joinedload
from app.database import get_db
from app.models import (
@@ -10,6 +13,8 @@ from app.schemas import (
MealPlanResponse, MealPlanCreate,
MealPlanItemResponse, VoteRequest, VoteResponse
)
from app.security import require_session
from app.services import approval as approval_service
from uuid import UUID
from typing import List, Optional
from datetime import datetime, timedelta
@@ -17,7 +22,7 @@ from datetime import datetime, timedelta
router = APIRouter()
@router.get("/planned", response_model=Optional[MealPlanResponse])
@router.get("", response_model=Optional[MealPlanResponse])
def get_planned_meals(db: Session = Depends(get_db)):
profile = db.query(FamilyProfile).first()
if not profile:
@@ -33,7 +38,7 @@ def get_planned_meals(db: Session = Depends(get_db)):
return meal_plan
@router.post("/", response_model=MealPlanResponse)
@router.post("", response_model=MealPlanResponse, dependencies=[Depends(require_session)])
def create_meal_plan(meal_plan_data: MealPlanCreate, db: Session = Depends(get_db)):
profile = db.query(FamilyProfile).first()
if not profile:
@@ -80,7 +85,7 @@ def get_meal_plan(meal_plan_id: UUID, db: Session = Depends(get_db)):
return meal_plan
@router.post("/{meal_plan_id}/lock")
@router.post("/{meal_plan_id}/lock", dependencies=[Depends(require_session)])
def lock_meal_plan(meal_plan_id: UUID, db: Session = Depends(get_db)):
meal_plan = db.query(MealPlan).filter(MealPlan.id == meal_plan_id).first()
if not meal_plan:
@@ -91,76 +96,169 @@ def lock_meal_plan(meal_plan_id: UUID, db: Session = Depends(get_db)):
return {"message": "Meal plan locked", "status": meal_plan.status.value}
@router.get("/items/{item_id}/vote/{token}")
def get_vote_page(item_id: UUID, token: str, db: Session = Depends(get_db)):
approval_token = db.query(ApprovalToken).filter(ApprovalToken.token == token).first()
if not approval_token:
raise HTTPException(status_code=404, detail="Invalid token")
_DAY_NAMES = {
1: "Monday", 2: "Tuesday", 3: "Wednesday", 4: "Thursday",
5: "Friday", 6: "Saturday", 7: "Sunday",
}
if approval_token.meal_plan_item_id != item_id:
class VoteSubmission(BaseModel):
"""Body for POST /vote/{item_id}?token=...
Spec body: {"vote": "approve" | "deny"}.
"""
vote: str = Field(..., pattern="^(approve|deny)$")
@router.get("/vote/{item_id}", response_class=HTMLResponse)
def get_vote_page(
item_id: UUID,
token: str = Query(..., description="Per-voter signed token"),
db: Session = Depends(get_db),
):
"""Render the per-voter approval confirmation page.
Verifies the signed token (no consume) and returns minimal accessible
HTML with Approve / Deny buttons that POST to the same URL.
"""
payload = approval_service.verify_token(token)
if str(payload.get("item")) != str(item_id):
raise HTTPException(status_code=400, detail="Token not valid for this meal")
if approval_token.status != ApprovalTokenStatus.ACTIVE:
raise HTTPException(status_code=400, detail="Token has already been used or expired")
voter = (
db.query(FamilyMember)
.filter(FamilyMember.id == UUID(str(payload["voter"])))
.first()
)
if not voter:
raise HTTPException(status_code=404, detail="Voter not found")
if approval_token.expires_at < datetime.now():
raise HTTPException(status_code=400, detail="Token has expired")
item = (
db.query(MealPlanItem)
.options(joinedload(MealPlanItem.recipe))
.filter(MealPlanItem.id == item_id)
.first()
)
if not item:
raise HTTPException(status_code=404, detail="Meal plan item not found")
recipe_name = item.recipe.name if item.recipe else "Unnamed meal"
day_name = _DAY_NAMES.get(int(item.day_of_week), str(item.day_of_week))
meal_type = item.meal_type.value if item.meal_type else ""
# Token is included only inside the form action (href), never in the
# visible body. POST is performed by JS so we can submit JSON without
# leaving the page; non-JS users still get a usable form fallback.
safe_voter = _html_escape(voter.name)
safe_recipe = _html_escape(recipe_name)
safe_day = _html_escape(day_name)
safe_meal = _html_escape(meal_type)
action_url = f"/api/meals/vote/{item_id}?token={_html_escape(token, quote=True)}"
html = f"""<!doctype html>
<html lang="en">
<head>
<meta charset="utf-8">
<title>Approve meal</title>
<meta name="viewport" content="width=device-width, initial-scale=1">
<style>
body {{ font-family: system-ui, sans-serif; max-width: 36rem; margin: 2rem auto;
padding: 0 1rem; color: #111; background: #fff; line-height: 1.5; }}
h1 {{ font-size: 1.4rem; }}
.meal {{ padding: 1rem; border: 1px solid #444; border-radius: 6px; margin: 1rem 0; }}
button {{ font-size: 1rem; padding: .6rem 1.2rem; margin-right: .5rem;
border: 2px solid #111; border-radius: 4px; cursor: pointer; }}
.approve {{ background: #0a6b2b; color: #fff; }}
.deny {{ background: #b00020; color: #fff; }}
#status {{ margin-top: 1rem; font-weight: bold; }}
</style>
</head>
<body>
<h1>Hi {safe_voter}, please vote on this meal</h1>
<div class="meal">
<div><strong>{safe_recipe}</strong></div>
<div>{safe_day} &middot; {safe_meal}</div>
</div>
<form id="voteForm" method="post" action="{action_url}">
<button type="submit" name="vote" value="approve" class="approve" aria-label="Approve this meal">Approve</button>
<button type="submit" name="vote" value="deny" class="deny" aria-label="Deny this meal">Deny</button>
</form>
<div id="status" role="status" aria-live="polite"></div>
<script>
document.getElementById('voteForm').addEventListener('submit', async function(e) {{
e.preventDefault();
var btn = e.submitter || document.activeElement;
var vote = btn && btn.value ? btn.value : 'approve';
var resp = await fetch(this.action, {{
method: 'POST',
headers: {{ 'Content-Type': 'application/json' }},
body: JSON.stringify({{ vote: vote }})
}});
var data = {{}};
try {{ data = await resp.json(); }} catch (_) {{}}
var s = document.getElementById('status');
if (resp.ok) {{
s.textContent = 'Recorded: ' + (data.item_status || vote);
}} else {{
s.textContent = 'Error: ' + (data.detail || resp.status);
}}
}});
</script>
</body>
</html>
"""
return HTMLResponse(content=html, status_code=200)
@router.post("/vote/{item_id}")
def submit_vote(
item_id: UUID,
submission: VoteSubmission,
token: str = Query(..., description="Per-voter signed token"),
db: Session = Depends(get_db),
):
"""Record a per-voter vote and apply the approval rule.
- Single-use enforcement lives in `approval_service.consume_token`.
- Approval rule: any deny -> item.denied; all-approve -> item.approved;
otherwise pending (waiting on remaining voters).
"""
voter = approval_service.consume_token(db, token, item_id)
item = db.query(MealPlanItem).filter(MealPlanItem.id == item_id).first()
if not item:
raise HTTPException(status_code=404, detail="Meal plan item not found")
return {
"item_id": str(item_id),
"family_member_id": str(approval_token.family_member_id),
"meal_plan_item": item
}
@router.post("/items/{item_id}/vote/{token}", response_model=VoteResponse)
def submit_vote(item_id: UUID, token: str, vote_data: VoteRequest, db: Session = Depends(get_db)):
approval_token = db.query(ApprovalToken).filter(ApprovalToken.token == token).first()
if not approval_token:
raise HTTPException(status_code=404, detail="Invalid token")
if approval_token.meal_plan_item_id != item_id:
raise HTTPException(status_code=400, detail="Token not valid for this meal")
if approval_token.status != ApprovalTokenStatus.ACTIVE:
raise HTTPException(status_code=400, detail="Token has already been used or expired")
if approval_token.expires_at < datetime.now():
approval_token.status = ApprovalTokenStatus.EXPIRED
db.commit()
raise HTTPException(status_code=400, detail="Token has expired")
existing_vote = db.query(MealPlanVote).filter(
MealPlanVote.meal_plan_item_id == item_id,
MealPlanVote.family_member_id == approval_token.family_member_id
).first()
if existing_vote:
raise HTTPException(status_code=400, detail="You have already voted on this meal")
vote = MealPlanVote(
vote_bool = submission.vote == "approve"
db.add(MealPlanVote(
meal_plan_item_id=item_id,
family_member_id=approval_token.family_member_id,
vote=vote_data.vote
)
db.add(vote)
family_member_id=voter.id,
vote=vote_bool,
))
db.flush()
approval_token.status = ApprovalTokenStatus.USED
approval_token.used_at = datetime.now()
# Approval rule: count electorate (all family members on this profile)
# vs votes recorded so far.
profile_id = item.meal_plan.family_profile_id
electorate_ids = {
m.id for m in db.query(FamilyMember)
.filter(FamilyMember.family_profile_id == profile_id)
.all()
}
votes = db.query(MealPlanVote).filter(
MealPlanVote.meal_plan_item_id == item_id,
).all()
item = db.query(MealPlanItem).filter(MealPlanItem.id == item_id).first()
if not vote_data.vote and vote_data.denial_reason:
if any(v.vote is False for v in votes):
item.approval_status = MealPlanItemStatus.DENIED
item.denial_reason = vote_data.denial_reason
item.denial_details = vote_data.denial_details
elif electorate_ids and {v.family_member_id for v in votes} >= electorate_ids:
item.approval_status = MealPlanItemStatus.APPROVED
else:
item.approval_status = MealPlanItemStatus.PENDING
db.commit()
db.refresh(vote)
return vote
return {"status": "recorded", "item_status": item.approval_status.value}
@router.get("/items/{item_id}", response_model=MealPlanItemResponse)
@@ -173,7 +271,7 @@ def get_meal_item(item_id: UUID, db: Session = Depends(get_db)):
return item
@router.post("/items/{item_id}/swap")
@router.post("/items/{item_id}/swap", dependencies=[Depends(require_session)])
def swap_meal_item(item_id: UUID, new_recipe_id: UUID, db: Session = Depends(get_db)):
item = db.query(MealPlanItem).filter(MealPlanItem.id == item_id).first()
if not item:
+5 -4
View File
@@ -3,13 +3,14 @@ from sqlalchemy.orm import Session
from app.database import get_db
from app.models import HomePantry, Ingredient, FamilyProfile
from app.schemas import HomePantryResponse, HomePantryCreate
from app.security import require_session
from uuid import UUID
from typing import List
router = APIRouter()
@router.get("/", response_model=List[HomePantryResponse])
@router.get("", response_model=List[HomePantryResponse])
def get_pantry_items(db: Session = Depends(get_db)):
profile = db.query(FamilyProfile).first()
if not profile:
@@ -21,7 +22,7 @@ def get_pantry_items(db: Session = Depends(get_db)):
return items
@router.post("/", response_model=HomePantryResponse)
@router.post("", response_model=HomePantryResponse, dependencies=[Depends(require_session)])
def add_pantry_item(item: HomePantryCreate, db: Session = Depends(get_db)):
profile = db.query(FamilyProfile).first()
if not profile:
@@ -54,7 +55,7 @@ def add_pantry_item(item: HomePantryCreate, db: Session = Depends(get_db)):
return db_item
@router.delete("/{item_id}")
@router.delete("/{item_id}", dependencies=[Depends(require_session)])
def remove_pantry_item(item_id: UUID, db: Session = Depends(get_db)):
item = db.query(HomePantry).filter(HomePantry.id == item_id).first()
if not item:
@@ -65,7 +66,7 @@ def remove_pantry_item(item_id: UUID, db: Session = Depends(get_db)):
return {"message": "Pantry item removed"}
@router.put("/{item_id}", response_model=HomePantryResponse)
@router.put("/{item_id}", response_model=HomePantryResponse, dependencies=[Depends(require_session)])
def update_pantry_item(item_id: UUID, update: HomePantryCreate, db: Session = Depends(get_db)):
item = db.query(HomePantry).filter(HomePantry.id == item_id).first()
if not item:
+5 -4
View File
@@ -6,13 +6,14 @@ from app.schemas import (
FamilyProfileResponse, FamilyProfileUpdate, FamilyProfileCreate,
FamilyMemberResponse, FamilyMemberCreate
)
from app.security import require_session
from uuid import UUID
from typing import List
router = APIRouter()
@router.get("/", response_model=FamilyProfileResponse)
@router.get("", response_model=FamilyProfileResponse)
def get_profile(db: Session = Depends(get_db)):
profile = db.query(FamilyProfile).first()
if not profile:
@@ -20,7 +21,7 @@ def get_profile(db: Session = Depends(get_db)):
return profile
@router.put("/", response_model=FamilyProfileResponse)
@router.put("", response_model=FamilyProfileResponse, dependencies=[Depends(require_session)])
def update_profile(update: FamilyProfileUpdate, db: Session = Depends(get_db)):
profile = db.query(FamilyProfile).first()
if not profile:
@@ -43,7 +44,7 @@ def get_members(db: Session = Depends(get_db)):
return profile.members
@router.post("/members", response_model=FamilyMemberResponse)
@router.post("/members", response_model=FamilyMemberResponse, dependencies=[Depends(require_session)])
def add_member(member: FamilyMemberCreate, db: Session = Depends(get_db)):
profile = db.query(FamilyProfile).first()
if not profile:
@@ -69,7 +70,7 @@ def add_member(member: FamilyMemberCreate, db: Session = Depends(get_db)):
return db_member
@router.delete("/members/{member_id}")
@router.delete("/members/{member_id}", dependencies=[Depends(require_session)])
def delete_member(member_id: UUID, db: Session = Depends(get_db)):
member = db.query(FamilyMember).filter(FamilyMember.id == member_id).first()
if not member:
+35 -34
View File
@@ -3,13 +3,14 @@ from sqlalchemy.orm import Session, joinedload
from app.database import get_db
from app.models import Recipe, FamilyProfile
from app.schemas import RecipeResponse, RecipeCreate, IngredientCreate, IngredientResponse
from app.security import require_session
from uuid import UUID
from typing import List, Optional
router = APIRouter()
@router.get("/", response_model=List[RecipeResponse])
@router.get("", response_model=List[RecipeResponse])
def get_recipes(
skip: int = 0,
limit: int = 50,
@@ -31,6 +32,37 @@ def get_recipes(
return recipes
@router.get("/ingredients", response_model=List[IngredientResponse])
def list_ingredients(skip: int = 0, limit: int = 100, db: Session = Depends(get_db)):
from app.models import Ingredient
ingredients = db.query(Ingredient).offset(skip).limit(limit).all()
return ingredients
@router.post("/ingredients", response_model=IngredientResponse, dependencies=[Depends(require_session)])
def create_ingredient(ingredient: IngredientCreate, db: Session = Depends(get_db)):
from app.models import Ingredient
existing = db.query(Ingredient).filter(Ingredient.name_lower == ingredient.name_lower).first()
if existing:
raise HTTPException(status_code=400, detail="Ingredient with this name already exists")
db_ingredient = Ingredient(
name=ingredient.name,
name_lower=ingredient.name_lower,
plural_name=ingredient.plural_name,
aisle=ingredient.aisle,
typical_price=ingredient.typical_price,
unit=ingredient.unit,
season_months=ingredient.season_months
)
db.add(db_ingredient)
db.commit()
db.refresh(db_ingredient)
return db_ingredient
@router.get("/{recipe_id}", response_model=RecipeResponse)
def get_recipe(recipe_id: UUID, db: Session = Depends(get_db)):
recipe = db.query(Recipe).filter(Recipe.id == recipe_id).first()
@@ -39,7 +71,7 @@ def get_recipe(recipe_id: UUID, db: Session = Depends(get_db)):
return recipe
@router.post("/", response_model=RecipeResponse)
@router.post("", response_model=RecipeResponse, dependencies=[Depends(require_session)])
def create_recipe(recipe: RecipeCreate, db: Session = Depends(get_db)):
profile = db.query(FamilyProfile).first()
@@ -69,7 +101,7 @@ def create_recipe(recipe: RecipeCreate, db: Session = Depends(get_db)):
return db_recipe
@router.delete("/{recipe_id}")
@router.delete("/{recipe_id}", dependencies=[Depends(require_session)])
def delete_recipe(recipe_id: UUID, db: Session = Depends(get_db)):
recipe = db.query(Recipe).filter(Recipe.id == recipe_id).first()
if not recipe:
@@ -78,34 +110,3 @@ def delete_recipe(recipe_id: UUID, db: Session = Depends(get_db)):
db.delete(recipe)
db.commit()
return {"message": "Recipe deleted"}
@router.get("/ingredients/list", response_model=List[IngredientResponse])
def list_ingredients(skip: int = 0, limit: int = 100, db: Session = Depends(get_db)):
from app.models import Ingredient
ingredients = db.query(Ingredient).offset(skip).limit(limit).all()
return ingredients
@router.post("/ingredients", response_model=IngredientResponse)
def create_ingredient(ingredient: IngredientCreate, db: Session = Depends(get_db)):
from app.models import Ingredient
existing = db.query(Ingredient).filter(Ingredient.name_lower == ingredient.name_lower).first()
if existing:
raise HTTPException(status_code=400, detail="Ingredient with this name already exists")
db_ingredient = Ingredient(
name=ingredient.name,
name_lower=ingredient.name_lower,
plural_name=ingredient.plural_name,
aisle=ingredient.aisle,
typical_price=ingredient.typical_price,
unit=ingredient.unit,
season_months=ingredient.season_months
)
db.add(db_ingredient)
db.commit()
db.refresh(db_ingredient)
return db_ingredient
+2 -2
View File
@@ -14,7 +14,7 @@ from collections import defaultdict
router = APIRouter()
@router.get("/", response_model=ShoppingListResponse)
@router.get("", response_model=ShoppingListResponse)
def get_shopping_list(db: Session = Depends(get_db)):
profile = db.query(FamilyProfile).first()
if not profile:
@@ -56,7 +56,7 @@ def get_shopping_list(db: Session = Depends(get_db)):
if all_ingredient_ids:
ingredients = db.query(Ingredient).filter(
Ingredient.id.in_ all_ingredient_ids
Ingredient.id.in_(all_ingredient_ids)
).all()
ingredient_map = {i.id: i for i in ingredients}
+20 -1
View File
@@ -1,17 +1,30 @@
from pydantic_settings import BaseSettings
from pydantic import model_validator
from typing import Optional
class Settings(BaseSettings):
DATABASE_URL: str
# Required — fail fast at import time if unset.
DATABASE_URL: str = ""
SENDGRID_API_KEY: Optional[str] = None
EMAIL_BACKEND: str = "console"
LUCKY_CA_URL: str = "https://www.luckyncal.com"
# R3-0: Swiftly product API (replaces Playwright path).
SWIFTLY_BEARER_TOKEN: str = ""
LUCKY_STORE_ID: str = "757"
SWIFTLY_API_BASE: str = "https://prod.swiftlyapi.net"
SWIFTLY_CATEGORIES_URL: str = "https://luckysupermarkets.com/categories"
AI_IMAGE_ENABLED: bool = False
AI_IMAGE_PROVIDER: Optional[str] = None
AI_IMAGE_API_KEY: Optional[str] = None
LOG_LEVEL: str = "INFO"
SECRET_KEY: str = "dev-secret-key"
# Auth (R1-B+D)
ADMIN_TOKEN: str = ""
SESSION_PASSWORD: str = ""
FAMILY_EMAIL_1: Optional[str] = None
FAMILY_EMAIL_2: Optional[str] = None
RECIPES_EMAIL: Optional[str] = None
@@ -19,5 +32,11 @@ class Settings(BaseSettings):
class Config:
env_file = ".env"
@model_validator(mode="after")
def _require_database_url(self) -> "Settings":
if not self.DATABASE_URL:
raise RuntimeError("DATABASE_URL is required")
return self
settings = Settings()
+2 -1
View File
@@ -29,8 +29,9 @@ def health_check_db(db: Session = Depends(get_db)):
return {"status": "error", "database": "disconnected", "error": str(e)}
from app.api import profile, recipes, meals, shopping_list, pantry, admin
from app.api import profile, recipes, meals, shopping_list, pantry, admin, auth
app.include_router(auth.router, prefix="/api/auth", tags=["auth"])
app.include_router(profile.router, prefix="/api/profile", tags=["profile"])
app.include_router(recipes.router, prefix="/api/recipes", tags=["recipes"])
app.include_router(meals.router, prefix="/api/meals", tags=["meals"])
+14 -11
View File
@@ -114,7 +114,7 @@ class FamilyMember(Base):
family_profile_id = Column(UUID(as_uuid=True), ForeignKey("family_profile.id", ondelete="CASCADE"))
name = Column(String(100), nullable=False)
email = Column(String(300))
role = Column(SQLEnum(FamilyMemberRole, name="family_member_role_enum", create_type=False), nullable=False)
role = Column(SQLEnum(FamilyMemberRole, name="family_member_role_enum", create_type=False, values_callable=lambda obj: [e.value for e in obj]), nullable=False)
likes_mushrooms = Column(Boolean, default=False)
created_at = Column(DateTime(timezone=True), server_default=func.now())
updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now())
@@ -185,7 +185,7 @@ class MealPlan(Base):
id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
family_profile_id = Column(UUID(as_uuid=True), ForeignKey("family_profile.id"))
week_start_date = Column(Date, nullable=False)
status = Column(SQLEnum(MealPlanStatus, name="meal_plan_status_enum", create_type=False), nullable=False, default=MealPlanStatus.DRAFT)
status = Column(SQLEnum(MealPlanStatus, name="meal_plan_status_enum", create_type=False, values_callable=lambda obj: [e.value for e in obj]), nullable=False, default=MealPlanStatus.DRAFT)
approval_deadline = Column(DateTime(timezone=True))
total_estimated_cost = Column(Numeric(10, 2))
notes = Column(Text)
@@ -198,7 +198,6 @@ class MealPlan(Base):
family_profile = relationship("FamilyProfile", back_populates="meal_plans")
items = relationship("MealPlanItem", back_populates="meal_plan", cascade="all, delete-orphan")
votes = relationship("MealPlanVote", back_populates="meal_plan", cascade="all, delete-orphan")
class MealPlanItem(Base):
@@ -208,9 +207,9 @@ class MealPlanItem(Base):
meal_plan_id = Column(UUID(as_uuid=True), ForeignKey("meal_plan.id", ondelete="CASCADE"))
recipe_id = Column(UUID(as_uuid=True), ForeignKey("recipe.id"))
day_of_week = Column(Integer, nullable=False)
meal_type = Column(SQLEnum(MealType, name="meal_type_enum", create_type=False), nullable=False)
approval_status = Column(SQLEnum(MealPlanItemStatus, name="meal_plan_item_status_enum", create_type=False), default=MealPlanItemStatus.PENDING)
denial_reason = Column(SQLEnum(DenialReason, name="denial_reason_enum", create_type=False))
meal_type = Column(SQLEnum(MealType, name="meal_type_enum", create_type=False, values_callable=lambda obj: [e.value for e in obj]), nullable=False)
approval_status = Column(SQLEnum(MealPlanItemStatus, name="meal_plan_item_status_enum", create_type=False, values_callable=lambda obj: [e.value for e in obj]), default=MealPlanItemStatus.PENDING)
denial_reason = Column(SQLEnum(DenialReason, name="denial_reason_enum", create_type=False, values_callable=lambda obj: [e.value for e in obj]))
denial_details = Column(Text)
estimated_cost = Column(Numeric(10, 2))
used_pantry_items = Column(ARRAY(UUID(as_uuid=True)))
@@ -252,7 +251,7 @@ class ApprovalToken(Base):
meal_plan_item_id = Column(UUID(as_uuid=True), ForeignKey("meal_plan_item.id", ondelete="CASCADE"))
family_member_id = Column(UUID(as_uuid=True), ForeignKey("family_member.id", ondelete="CASCADE"))
token = Column(String(64), nullable=False, unique=True)
status = Column(SQLEnum(ApprovalTokenStatus, name="approval_token_status_enum", create_type=False), default=ApprovalTokenStatus.ACTIVE)
status = Column(SQLEnum(ApprovalTokenStatus, name="approval_token_status_enum", create_type=False, values_callable=lambda obj: [e.value for e in obj]), default=ApprovalTokenStatus.ACTIVE)
expires_at = Column(DateTime(timezone=True), nullable=False)
used_at = Column(DateTime(timezone=True))
created_at = Column(DateTime(timezone=True), server_default=func.now())
@@ -294,7 +293,7 @@ class Feedback(Base):
meal_plan_item_id = Column(UUID(as_uuid=True), ForeignKey("meal_plan_item.id", ondelete="CASCADE"))
rating = Column(Integer)
never_suggest = Column(Boolean, default=False)
denial_reason = Column(SQLEnum(DenialReason, name="denial_reason_enum", create_type=False))
denial_reason = Column(SQLEnum(DenialReason, name="denial_reason_enum", create_type=False, values_callable=lambda obj: [e.value for e in obj]))
feedback_text = Column(Text)
created_at = Column(DateTime(timezone=True), server_default=func.now())
@@ -314,7 +313,7 @@ class NeverSuggest(Base):
family_profile_id = Column(UUID(as_uuid=True), ForeignKey("family_profile.id", ondelete="CASCADE"))
ingredient_id = Column(UUID(as_uuid=True), ForeignKey("ingredient.id", ondelete="CASCADE"))
recipe_id = Column(UUID(as_uuid=True), ForeignKey("recipe.id", ondelete="CASCADE"))
reason = Column(SQLEnum(NeverSuggestReason, name="never_suggest_reason_enum", create_type=False))
reason = Column(SQLEnum(NeverSuggestReason, name="never_suggest_reason_enum", create_type=False, values_callable=lambda obj: [e.value for e in obj]))
notes = Column(Text)
created_at = Column(DateTime(timezone=True), server_default=func.now())
@@ -335,12 +334,16 @@ class GroceryItem(Base):
aisle = Column(String(100))
image_url = Column(Text)
product_url = Column(Text)
description = Column(Text, nullable=True)
is_on_sale = Column(Boolean, default=False)
sale_start_date = Column(Date)
sale_end_date = Column(Date)
in_season = Column(Boolean, default=False)
scraped_at = Column(DateTime(timezone=True), server_default=func.now())
scraped_url = Column(Text)
# R3-0: Swiftly API idempotency key. (source, external_id) → upsert.
external_id = Column(String(100), nullable=True, index=True)
source = Column(String(50), nullable=True)
ingredient = relationship("Ingredient", back_populates="grocery_item_links")
@@ -351,7 +354,7 @@ class ScrapeLog(Base):
id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
source = Column(String(50), nullable=False)
scrape_type = Column(String(50), nullable=False)
status = Column(SQLEnum(ScrapeStatus, name="scrape_status_enum", create_type=False), nullable=False)
status = Column(SQLEnum(ScrapeStatus, name="scrape_status_enum", create_type=False, values_callable=lambda obj: [e.value for e in obj]), nullable=False)
items_scraped = Column(Integer, default=0)
error_message = Column(Text)
started_at = Column(DateTime(timezone=True), server_default=func.now())
@@ -369,7 +372,7 @@ class EmailLog(Base):
meal_plan_id = Column(UUID(as_uuid=True), ForeignKey("meal_plan.id"))
meal_plan_item_id = Column(UUID(as_uuid=True), ForeignKey("meal_plan_item.id"))
sendgrid_message_id = Column(String(100))
status = Column(SQLEnum(EmailStatus, name="email_status_enum", create_type=False), nullable=False)
status = Column(SQLEnum(EmailStatus, name="email_status_enum", create_type=False, values_callable=lambda obj: [e.value for e in obj]), nullable=False)
error_message = Column(Text)
created_at = Column(DateTime(timezone=True), server_default=func.now())
delivered_at = Column(DateTime(timezone=True))
+2 -2
View File
@@ -1,4 +1,4 @@
from .base import BaseScraper
from .lucky_ca_scraper import LuckyCaliforniaScraper
from .lucky_ca_scraper import LuckyCaliforniaScraper, SwiftlyAuthError
__all__ = ["BaseScraper", "LuckyCaliforniaScraper"]
__all__ = ["BaseScraper", "LuckyCaliforniaScraper", "SwiftlyAuthError"]
+292 -147
View File
@@ -1,180 +1,325 @@
"""Lucky California / Swiftly product API client.
Replaces the prior Playwright + HTML coupon-card path (R2-A) with the
underlying JSON API the website itself calls. The API exposes the full
inventory per category — no rendering, no Chromium, and far more items
than the visible coupon strip (256 in `Product/meat_seafood` vs the 11
the HTML parser saw).
Two endpoints (no public docs; reverse-engineered from the network panel
on luckysupermarkets.com — see ``.agent/context.md`` "Swiftly API"):
GET https://luckysupermarkets.com/categories
→ HTML page; anchors with ``class="swiftlyCouponCategory"`` carry
``href="/categories/Product%2F<slug>"``.
GET https://prod.swiftlyapi.net/search/api/v1/products/categories
?cat=<slug>&store=<store_id>&limit=10000
Authorization: Bearer <SWIFTLY_BEARER_TOKEN>
→ ``{"products": {"info": {...}, "items": [...], "facets": [...]}}``
The bearer token expires roughly hourly. On 401 we raise
``SwiftlyAuthError`` so the background runner records the error_message
that asks the admin to refresh ``SWIFTLY_BEARER_TOKEN`` and retry.
"""
from __future__ import annotations
import logging
import re
from typing import Dict, Any, List, Optional
import time
import urllib.parse
from datetime import datetime
from urllib.parse import urljoin
from decimal import Decimal, InvalidOperation
from typing import Any, Dict, Iterator, List, Optional, Tuple
import requests
from bs4 import BeautifulSoup
from .base import SeleniumScraper
from app.config import settings
logger = logging.getLogger(__name__)
class LuckyCaliforniaScraper(SeleniumScraper):
def __init__(self, base_url: str = "https://luckysupermarkets.com"):
super().__init__(base_url=base_url, rate_limit_seconds=3.0)
self.ingredients_cache = {}
class SwiftlyAuthError(Exception):
"""Raised when the Swiftly API returns 401.
The exception message is surfaced verbatim to the ScrapeLog row by
``_run_scrape_in_background``; keep it actionable.
"""
_USER_AGENT = (
"MealPlannerBot/1.0 (+https://mealplanner.local; contact peter@research.bike)"
)
_AUTH_ERROR_MESSAGE = (
"SWIFTLY_BEARER_TOKEN expired — request a fresh token from the user "
"(capture from luckysupermarkets.com network tab on a /search/api/v1 request)"
)
class LuckyCaliforniaScraper:
"""Swiftly product-API client for the Lucky California banner.
The class name is preserved (``LuckyCaliforniaScraper``) so existing
import sites in ``app.scraper.__init__`` and
``ScraperService._do_scrape`` keep working. Internally it is no
longer a ``BaseScraper`` subclass: that class wires Playwright +
``_get`` retry-with-swallow, neither of which we want here. The
client uses two ``requests.Session`` objects so the bearer header
is scoped strictly to the API host (the public categories page
is unauthenticated).
"""
SOURCE = "lucky_california"
SLUG_HREF_RE = re.compile(
r'<a\b(?=[^>]*\bclass="swiftlyCouponCategory")(?=[^>]*\bhref="([^"]+)")[^>]*>',
re.IGNORECASE,
)
def __init__(
self,
*,
bearer_token: Optional[str] = None,
store_id: Optional[str] = None,
api_base: Optional[str] = None,
categories_url: Optional[str] = None,
rate_limit_seconds: float = 0.75,
timeout: int = 60,
) -> None:
self.bearer_token = bearer_token or settings.SWIFTLY_BEARER_TOKEN
self.store_id = store_id or settings.LUCKY_STORE_ID
self.api_base = (api_base or settings.SWIFTLY_API_BASE).rstrip("/")
self.categories_url = categories_url or settings.SWIFTLY_CATEGORIES_URL
self.rate_limit_seconds = rate_limit_seconds
self.timeout = timeout
self._last_request = 0.0
self.public_session = requests.Session()
self.public_session.headers.update(
{
"User-Agent": _USER_AGENT,
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
"Accept-Language": "en-US,en;q=0.5",
}
)
self.api_session = requests.Session()
self.api_session.headers.update(
{
"User-Agent": _USER_AGENT,
"Accept": "application/json",
}
)
# base_url retained for code paths that still introspect it.
self.base_url = "https://luckysupermarkets.com"
# ------------------------------------------------------------------
# Lifecycle (no-op; preserves cleanup() contract from BaseScraper).
# ------------------------------------------------------------------
def cleanup(self) -> None:
try:
self.public_session.close()
finally:
self.api_session.close()
# ------------------------------------------------------------------
# Public scrape entrypoints
# ------------------------------------------------------------------
def scrape(self) -> Dict[str, Any]:
logger.info("Starting Lucky California scrape")
result = {
"source": "lucky_california",
"""Synchronous top-level scrape.
Kept for the legacy ``ScraperService.run_scrape`` path used by
tests. Walks every category and returns ``{"items": [...], ...}``
with mapped product dicts ready for ``_save_grocery_item``.
"""
started = datetime.now().isoformat()
items: List[Dict[str, Any]] = list(self.fetch_all())
return {
"source": self.SOURCE,
"scrape_type": "weekly_ad",
"started_at": datetime.now().isoformat(),
"items_scraped": 0,
"items": []
"started_at": started,
"completed_at": datetime.now().isoformat(),
"items_scraped": len(items),
"items": items,
"status": "success",
}
try:
featured_items = self.scrape_featured_coupons()
result["items"].extend(featured_items)
result["items_scraped"] = len(featured_items)
result["completed_at"] = datetime.now().isoformat()
result["status"] = "success"
logger.info(f"Lucky California scrape complete: {result['items_scraped']} items")
except Exception as e:
logger.error(f"Lucky California scrape failed: {e}")
result["status"] = "failed"
result["error_message"] = str(e)
result["completed_at"] = datetime.now().isoformat()
return result
def scrape_featured_coupons(self) -> List[Dict[str, Any]]:
url = f"{self.base_url}/coupons/Coupon%2Flu-featured-in-ad"
logger.info(f"Scraping featured coupons from {url}")
page = self.get_browser_page(url)
content = page.content()
page.close()
soup = BeautifulSoup(content, "html.parser")
items = []
coupon_items = soup.find_all("div", class_=re.compile(r"coupon|item|product", re.I))
if not coupon_items:
headline = soup.find("h1")
if headline:
logger.info(f"Page loaded, headline: {headline.get_text().strip()}")
titles = soup.find_all(["h2", "h3", "a"], string=re.compile(r"\$[\d\.]+"))
for title_elem in titles[:20]:
def fetch_all(self) -> Iterator[Dict[str, Any]]:
"""Yield mapped product dicts across every discovered category."""
slugs = self.discover_categories()
logger.info("Swiftly: discovered %d categories", len(slugs))
for slug in slugs:
try:
item = self._parse_coupon_item(title_elem)
if item:
items.append(item)
except Exception as e:
logger.debug(f"Failed to parse item: {e}")
products = self.fetch_category(slug)
except SwiftlyAuthError:
# Hard fail — token must be refreshed before any further work.
raise
except requests.RequestException as exc:
logger.warning("Swiftly: skipping %s after error: %s", slug, exc)
continue
aisle = self._aisle_from_slug(slug)
for product in products:
mapped = self.map_product(product, aisle=aisle, source_slug=slug)
if mapped is not None:
yield mapped
logger.info(f"Found {len(items)} coupon items")
# ------------------------------------------------------------------
# Category discovery
# ------------------------------------------------------------------
def discover_categories(self) -> List[str]:
"""Fetch the categories page and return the list of API slugs.
The slugs look like ``Product/meat_seafood``. Order is preserved
from the HTML (which is the order shown to the user).
"""
self._rate_limit()
resp = self.public_session.get(self.categories_url, timeout=self.timeout)
resp.raise_for_status()
return self.parse_categories_html(resp.text)
@classmethod
def parse_categories_html(cls, html: str) -> List[str]:
"""Pure parser used by tests against a saved fixture."""
slugs: List[str] = []
seen: set[str] = set()
for href in cls.SLUG_HREF_RE.findall(html):
m = re.match(r"^/categories/(.+)$", href)
if not m:
continue
slug = urllib.parse.unquote(m.group(1))
if slug not in seen:
seen.add(slug)
slugs.append(slug)
return slugs
# ------------------------------------------------------------------
# Per-category fetch
# ------------------------------------------------------------------
def fetch_category(self, slug: str) -> List[Dict[str, Any]]:
"""Fetch every product in a category. Raises ``SwiftlyAuthError`` on 401."""
if not self.bearer_token:
raise SwiftlyAuthError(_AUTH_ERROR_MESSAGE)
self._rate_limit()
url = f"{self.api_base}/search/api/v1/products/categories"
params = {"cat": slug, "store": self.store_id, "limit": 10000}
headers = {"Authorization": f"Bearer {self.bearer_token}"}
resp = self.api_session.get(
url, params=params, headers=headers, timeout=self.timeout
)
if resp.status_code == 401:
raise SwiftlyAuthError(_AUTH_ERROR_MESSAGE)
resp.raise_for_status()
payload = resp.json()
return self.parse_category_response(payload)
@staticmethod
def parse_category_response(payload: Dict[str, Any]) -> List[Dict[str, Any]]:
"""Pure parser used by tests against a saved fixture."""
if not isinstance(payload, dict):
return []
products = payload.get("products") or {}
items = products.get("items")
if not isinstance(items, list):
return []
return items
def _parse_coupon_item(self, element) -> Optional[Dict[str, Any]]:
text = element.get_text().strip()
# ------------------------------------------------------------------
# Field mapping
# ------------------------------------------------------------------
@classmethod
def map_product(
cls,
product: Dict[str, Any],
*,
aisle: Optional[str] = None,
source_slug: Optional[str] = None,
) -> Optional[Dict[str, Any]]:
"""Convert one Swiftly product dict to a grocery_item-ready dict.
price_match = re.search(r'\$[\d,]+\.?\d*', text)
if not price_match:
Returns ``None`` for products with no parseable price or no name —
those are usually placeholder/unavailable rows.
"""
external_id = product.get("id")
name = (product.get("name") or "").strip()
if not name:
return None
price_str = price_match.group().replace("$", "").replace(",", "")
try:
price = float(price_str)
except ValueError:
price_block = (product.get("price") or {}).get("ok") or {}
reg_price, reg_unit = cls._parse_price(price_block.get("regPriceText"))
promo = price_block.get("promoArea") or {}
sale_price, sale_unit = cls._parse_price(promo.get("promoText"))
if reg_price is None and sale_price is None:
# No usable price; skip rather than persist garbage.
return None
name_elem = element.find_parent("a") or element.find_parent("div")
name = text.split("$")[0].strip() if "$" in text else text
name = re.sub(r'\s+', " ", name).strip()[:200]
unit = sale_unit or reg_unit
is_on_sale = sale_price is not None and reg_price is not None and sale_price < reg_price
# current_price = "what the customer pays today" → sale_price when on sale.
current_price = sale_price if is_on_sale else reg_price
if not name or len(name) < 3:
return None
image_url: Optional[str] = None
primary_image = product.get("primaryImage")
if isinstance(primary_image, dict):
image_url = primary_image.get("url")
image_url = None
img_elem = element.find_parent().find("img") if element.find_parent() else None
if img_elem and img_elem.get("src"):
image_url = img_elem["src"]
brand = product.get("brand")
description = product.get("description")
product_url = None
link_elem = element.find_parent("a") if element.find_parent() else element.find("a")
if link_elem and link_elem.get("href"):
product_url = urljoin(self.base_url, link_elem["href"])
item = {
"name": name,
"current_price": price,
return {
"external_id": str(external_id) if external_id is not None else None,
"source": cls.SOURCE,
"name": name[:300],
"brand": (brand or None),
"description": description,
"current_price": current_price,
"regular_price": reg_price,
"sale_price": sale_price,
"is_on_sale": bool(is_on_sale),
"unit": unit,
"aisle": aisle,
"image_url": image_url,
"product_url": product_url,
"is_on_sale": True,
"product_url": None, # Swiftly does not expose a public product URL
"scraped_at": datetime.now().isoformat(),
"scraped_url": self.base_url
"scraped_url": f"{cls.__name__}:{source_slug}" if source_slug else None,
}
return item
# ------------------------------------------------------------------
# Helpers
# ------------------------------------------------------------------
@staticmethod
def _parse_price(text: Optional[str]) -> Tuple[Optional[Decimal], Optional[str]]:
"""Extract ``(price, unit)`` from strings like ``"$3.49 /lb"``.
def scrape_produce(self) -> List[Dict[str, Any]]:
url = f"{self.base_url}/coupons/Coupon%2Flu-produce"
logger.info(f"Scraping produce from {url}")
Returns ``(None, None)`` when ``text`` is empty or unparseable.
"""
if not text:
return None, None
m = re.search(r"\$\s*([\d,]+(?:\.\d+)?)", text)
if not m:
return None, None
raw = m.group(1).replace(",", "")
try:
price = Decimal(raw)
except (InvalidOperation, ValueError):
return None, None
unit_match = re.search(r"/\s*([A-Za-z]+)", text)
unit = unit_match.group(1).lower() if unit_match else None
return price, unit
page = self.get_browser_page(url)
content = page.content()
page.close()
@staticmethod
def _aisle_from_slug(slug: str) -> Optional[str]:
"""``Product/meat_seafood`` → ``meat_seafood``."""
if not slug:
return None
if "/" in slug:
return slug.rsplit("/", 1)[1]
return slug
soup = BeautifulSoup(content, "html.parser")
items = []
for item_elem in soup.find_all("div", class_=re.compile(r"product|item|coupon")):
try:
item = self._parse_coupon_item(item_elem)
if item:
item["aisle"] = "Produce"
items.append(item)
except Exception:
continue
return items
def scrape_meat_seafood(self) -> List[Dict[str, Any]]:
url = f"{self.base_url}/coupons/Coupon%2Flu-meat-seafood"
logger.info(f"Scraping meat & seafood from {url}")
page = self.get_browser_page(url)
content = page.content()
page.close()
soup = BeautifulSoup(content, "html.parser")
items = []
for item_elem in soup.find_all("div", class_=re.compile(r"product|item|coupon")):
try:
item = self._parse_coupon_item(item_elem)
if item:
item["aisle"] = "Meat"
items.append(item)
except Exception:
continue
return items
def scrape_dairy_eggs(self) -> List[Dict[str, Any]]:
url = f"{self.base_url}/coupons/Coupon%2Flu-dairy-eggs"
logger.info(f"Scraping dairy & eggs from {url}")
page = self.get_browser_page(url)
content = page.content()
page.close()
soup = BeautifulSoup(content, "html.parser")
items = []
for item_elem in soup.find_all("div", class_=re.compile(r"product|item|coupon")):
try:
item = self._parse_coupon_item(item_elem)
if item:
item["aisle"] = "Dairy"
items.append(item)
except Exception:
continue
return items
def _rate_limit(self) -> None:
elapsed = time.time() - self._last_request
if elapsed < self.rate_limit_seconds:
time.sleep(self.rate_limit_seconds - elapsed)
self._last_request = time.time()
+70
View File
@@ -0,0 +1,70 @@
"""
Auth dependencies for the MealPlanner backend.
Two flavors:
- ``require_admin`` — bearer token for ``/api/admin/*`` routes; token compared
to ``settings.ADMIN_TOKEN`` (must be set in env).
- ``require_session`` — signed-cookie session (``itsdangerous``) gating
mutations on the family-facing routers; reads stay open inside the
trusted network.
The per-voter approval-token flow on meal items is intentionally NOT covered
here — it has its own short-lived single-use tokens elsewhere.
"""
from fastapi import HTTPException, Request, status
from itsdangerous import BadSignature, SignatureExpired, TimestampSigner
from app.config import settings
bearer_header = "Authorization"
SESSION_COOKIE = "mp_session"
SESSION_MAX_AGE = 60 * 60 * 24 * 30 # 30 days
def require_admin(request: Request) -> None:
"""Enforce a shared bearer token. 401 on bad/missing token, 503 if unset."""
expected = settings.ADMIN_TOKEN
if not expected:
raise HTTPException(
status_code=status.HTTP_503_SERVICE_UNAVAILABLE,
detail="Admin auth not configured",
)
auth = request.headers.get(bearer_header, "")
if not auth.startswith("Bearer ") or auth[7:] != expected:
raise HTTPException(
status_code=status.HTTP_401_UNAUTHORIZED,
detail="Invalid admin token",
)
def _signer() -> TimestampSigner:
return TimestampSigner(settings.SECRET_KEY)
def issue_session(family_profile_id: str) -> str:
"""Sign the family_profile_id and return the cookie value."""
return _signer().sign(family_profile_id.encode()).decode()
def require_session(request: Request) -> str:
"""Return the family_profile_id stored in the signed session cookie."""
raw = request.cookies.get(SESSION_COOKIE)
if not raw:
raise HTTPException(
status_code=status.HTTP_401_UNAUTHORIZED, detail="Session required"
)
try:
family_id = (
_signer().unsign(raw.encode(), max_age=SESSION_MAX_AGE).decode()
)
except SignatureExpired:
raise HTTPException(
status_code=status.HTTP_401_UNAUTHORIZED, detail="Session expired"
)
except BadSignature:
raise HTTPException(
status_code=status.HTTP_401_UNAUTHORIZED, detail="Invalid session"
)
return family_id
View File
+89
View File
@@ -0,0 +1,89 @@
"""
Per-voter approval tokens.
Stateless signed tokens (itsdangerous) keyed on settings.SECRET_KEY with a
versioned salt. Single-use is enforced by the presence of a MealPlanVote
row for (item, voter) — the table already has UniqueConstraint on that
pair, so the DB is the source of truth, not a token-status column.
"""
from __future__ import annotations
from typing import TYPE_CHECKING
from uuid import UUID
from fastapi import HTTPException
from itsdangerous import (
BadSignature,
SignatureExpired,
URLSafeTimedSerializer,
)
from app.config import settings
from app.models import FamilyMember, MealPlanVote
if TYPE_CHECKING: # pragma: no cover
from sqlalchemy.orm import Session
SALT = "meal-approval-v1"
DEFAULT_MAX_AGE_SECONDS = 7 * 24 * 3600
def _serializer() -> URLSafeTimedSerializer:
return URLSafeTimedSerializer(secret_key=settings.SECRET_KEY, salt=SALT)
def issue_token(meal_plan_item_id: UUID, family_member_id: UUID) -> str:
payload = {
"item": str(meal_plan_item_id),
"voter": str(family_member_id),
}
return _serializer().dumps(payload)
def verify_token(token: str, max_age_seconds: int = DEFAULT_MAX_AGE_SECONDS) -> dict:
try:
payload = _serializer().loads(token, max_age=max_age_seconds)
except SignatureExpired:
raise HTTPException(status_code=401, detail="Token expired")
except BadSignature:
raise HTTPException(status_code=401, detail="Invalid token")
if not isinstance(payload, dict) or "item" not in payload or "voter" not in payload:
raise HTTPException(status_code=401, detail="Invalid token payload")
return payload
def consume_token(
db: "Session",
token: str,
meal_plan_item_id: UUID,
max_age_seconds: int = DEFAULT_MAX_AGE_SECONDS,
) -> FamilyMember:
"""Verify + match URL + enforce single-use. Returns the voter on success.
Single-use is checked by looking for an existing MealPlanVote row for
(item, voter). If one exists, raise 409.
"""
payload = verify_token(token, max_age_seconds=max_age_seconds)
if str(payload["item"]) != str(meal_plan_item_id):
raise HTTPException(status_code=400, detail="Token not valid for this meal")
voter_id = UUID(str(payload["voter"]))
voter = db.query(FamilyMember).filter(FamilyMember.id == voter_id).first()
if not voter:
raise HTTPException(status_code=404, detail="Voter not found")
existing = (
db.query(MealPlanVote)
.filter(
MealPlanVote.meal_plan_item_id == meal_plan_item_id,
MealPlanVote.family_member_id == voter_id,
)
.first()
)
if existing:
raise HTTPException(status_code=409, detail="Already voted")
return voter
+89
View File
@@ -0,0 +1,89 @@
"""
Email backends.
R2-B spike: only ConsoleEmailBackend is functional. SendGrid is a stub
intentionally left to be wired in R3-C.
"""
from __future__ import annotations
import json
import os
import sys
from datetime import datetime, timezone
from pathlib import Path
from typing import Optional, Protocol
from app.config import settings
# Repository convention: backend/var/email_outbox.jsonl
# This module lives at backend/app/services/email.py — go up two levels to
# reach backend/, then var/.
_BACKEND_ROOT = Path(__file__).resolve().parent.parent.parent
OUTBOX_PATH = _BACKEND_ROOT / "var" / "email_outbox.jsonl"
class EmailBackend(Protocol):
def send(
self,
to: str,
subject: str,
html: str,
text: Optional[str] = None,
) -> None:
...
class ConsoleEmailBackend:
"""Dev/spike backend. Prints to stdout AND appends a JSON line to
backend/var/email_outbox.jsonl so the round-trip can be inspected
after the fact.
"""
def send(
self,
to: str,
subject: str,
html: str,
text: Optional[str] = None,
) -> None:
record = {
"ts": datetime.now(timezone.utc).isoformat(),
"to": to,
"subject": subject,
"html": html,
"text": text,
}
print(
f"[ConsoleEmailBackend] -> {to} | {subject}",
file=sys.stdout,
flush=True,
)
OUTBOX_PATH.parent.mkdir(parents=True, exist_ok=True)
with OUTBOX_PATH.open("a", encoding="utf-8") as f:
f.write(json.dumps(record) + "\n")
class SendGridEmailBackend:
"""Stub. Wire in R3-C (real SendGrid client + sandbox mode + retries).
Deliberately raises so a misconfigured prod env fails loudly instead of
silently dropping mail.
"""
def send(
self,
to: str,
subject: str,
html: str,
text: Optional[str] = None,
) -> None: # pragma: no cover - stub
raise NotImplementedError("Wire SendGrid in R3-C")
def get_email_backend() -> EmailBackend:
backend = (settings.EMAIL_BACKEND or "console").lower()
if backend == "sendgrid":
return SendGridEmailBackend()
return ConsoleEmailBackend()
+204 -67
View File
@@ -1,87 +1,204 @@
"""Scraper service.
Two entry points:
- ``enqueue_scrape(db, source, scrape_type, background_tasks)``: synchronously
inserts a ``ScrapeLog`` row in status ``STARTED`` (the existing enum has no
``pending`` member; ``STARTED`` is reused for the queued state) and registers
``_run_scrape_in_background`` to fire after the response is sent.
- ``_run_scrape_in_background(log_id, source, scrape_type)``: runs in a
FastAPI background task with its OWN ``SessionLocal()`` (the request-scoped
``db`` is closed by the time this fires). Writes terminal status
(``SUCCESS``/``FAILED``) and ``error_message``.
``ScraperService.run_scrape`` is preserved for direct/test invocation; the
``/api/admin/scrape`` endpoint now goes through ``enqueue_scrape``.
"""
from __future__ import annotations
import logging
from typing import Dict, Any, Optional
from datetime import datetime
from uuid import uuid4
from typing import Any, Dict, Optional
from datetime import datetime, timezone
from uuid import UUID, uuid4
from fastapi import BackgroundTasks
from sqlalchemy.orm import Session
from app.database import SessionLocal
from app.models import GroceryItem, Ingredient, ScrapeLog, ScrapeStatus
logger = logging.getLogger(__name__)
# ---------------------------------------------------------------------------
# Public API: enqueue + background runner
# ---------------------------------------------------------------------------
def enqueue_scrape(
db: Session,
*,
source: str,
scrape_type: str,
background_tasks: BackgroundTasks,
) -> ScrapeLog:
"""Create the ScrapeLog row, commit, and schedule the background scrape.
Returns the persisted ``ScrapeLog`` instance (refreshed). The actual
scraping work runs after FastAPI sends the 202 response.
"""
log = ScrapeLog(
id=uuid4(),
source=source,
scrape_type=scrape_type,
status=ScrapeStatus.STARTED,
started_at=datetime.now(timezone.utc),
)
db.add(log)
db.commit()
db.refresh(log)
background_tasks.add_task(
_run_scrape_in_background, log.id, source, scrape_type
)
return log
def _run_scrape_in_background(
log_id: UUID, source: str, scrape_type: str
) -> None:
"""Background entry point. Opens a fresh DB session — the request-scoped
session is gone by the time this runs.
"""
db: Session = SessionLocal()
try:
log = db.query(ScrapeLog).filter(ScrapeLog.id == log_id).first()
if log is None:
logger.error("ScrapeLog %s vanished before background run", log_id)
return
# No "running" state in the enum; STARTED already covers in-flight.
# Mark started_at fresh in case there was lag between enqueue and run.
try:
service = ScraperService(db)
saved_count, items_found = service._do_scrape(source=source)
log.status = ScrapeStatus.SUCCESS
log.items_scraped = saved_count
log.completed_at = datetime.now(timezone.utc)
log.duration_seconds = int(
(log.completed_at - log.started_at).total_seconds()
)
db.commit()
logger.info(
"Background scrape %s complete: %s/%s items saved",
log_id, saved_count, items_found,
)
except Exception as exc: # noqa: BLE001 — must catch all to mark failed
logger.exception("Background scrape %s failed", log_id)
db.rollback()
# Re-fetch in case the rollback detached the instance.
log = db.query(ScrapeLog).filter(ScrapeLog.id == log_id).first()
if log is not None:
log.status = ScrapeStatus.FAILED
log.error_message = str(exc)
log.completed_at = datetime.now(timezone.utc)
if log.started_at:
log.duration_seconds = int(
(log.completed_at - log.started_at).total_seconds()
)
db.commit()
finally:
db.close()
# ---------------------------------------------------------------------------
# Service class (kept for direct/test use)
# ---------------------------------------------------------------------------
class ScraperService:
def __init__(self, db: Session):
self.db = db
def run_scrape(self, source: str = "lucky_california", scrape_type: str = "weekly_ad") -> Dict[str, Any]:
from app.scraper import LuckyCaliforniaScraper
from app.models import ScrapeLog, GroceryItem, Ingredient
scrape_log = ScrapeLog(
def run_scrape(
self, source: str = "lucky_california", scrape_type: str = "weekly_ad"
) -> Dict[str, Any]:
"""Synchronous end-to-end scrape (legacy path). Creates the log row,
runs the scrape, commits terminal status. Used by direct callers and
tests; the API endpoint goes through ``enqueue_scrape``.
"""
log = ScrapeLog(
id=uuid4(),
source=source,
scrape_type=scrape_type,
status="started",
started_at=datetime.now()
status=ScrapeStatus.STARTED,
started_at=datetime.now(timezone.utc),
)
self.db.add(scrape_log)
self.db.add(log)
self.db.commit()
logger.info(f"Starting {source} {scrape_type} scrape")
logger.info("Starting %s %s scrape", source, scrape_type)
try:
scraper = LuckyCaliforniaScraper()
result = scraper.scrape()
scraper.cleanup()
items = result.get("items", [])
saved_count = 0
for item_data in items:
saved_item = self._save_grocery_item(item_data)
if saved_item:
saved_count += 1
scrape_log.status = "success"
scrape_log.items_scraped = saved_count
scrape_log.completed_at = datetime.now()
scrape_log.duration_seconds = int(
(scrape_log.completed_at - scrape_log.started_at).total_seconds()
saved_count, items_found = self._do_scrape(source=source)
log.status = ScrapeStatus.SUCCESS
log.items_scraped = saved_count
log.completed_at = datetime.now(timezone.utc)
log.duration_seconds = int(
(log.completed_at - log.started_at).total_seconds()
)
self.db.commit()
logger.info(f"Scrape complete: {saved_count} items saved")
logger.info("Scrape complete: %s items saved", saved_count)
return {
"scrape_id": str(scrape_log.id),
"scrape_id": str(log.id),
"status": "success",
"items_scraped": saved_count,
"items_found": len(items)
"items_found": items_found,
}
except Exception as e:
logger.error(f"Scrape failed: {e}")
scrape_log.status = "failed"
scrape_log.error_message = str(e)
scrape_log.completed_at = datetime.now()
scrape_log.duration_seconds = int(
(scrape_log.completed_at - scrape_log.started_at).total_seconds()
except Exception as e: # noqa: BLE001
logger.error("Scrape failed: %s", e)
log.status = ScrapeStatus.FAILED
log.error_message = str(e)
log.completed_at = datetime.now(timezone.utc)
log.duration_seconds = int(
(log.completed_at - log.started_at).total_seconds()
)
self.db.commit()
return {
"scrape_id": str(scrape_log.id),
"scrape_id": str(log.id),
"status": "failed",
"error": str(e)
"error": str(e),
}
def _save_grocery_item(self, item_data: Dict[str, Any]) -> Optional[GroceryItem]:
from app.models import GroceryItem, Ingredient
# ------------------------------------------------------------------
# Internals
# ------------------------------------------------------------------
def _do_scrape(self, *, source: str) -> tuple[int, int]:
"""Run the scraper and persist items. Returns (saved, found).
name = item_data.get("name", "").strip()
Raises any scraper exception (including ``SwiftlyAuthError``) to
the caller for status mapping.
"""
from app.scraper import LuckyCaliforniaScraper
scraper = LuckyCaliforniaScraper()
saved_count = 0
found_count = 0
try:
for item_data in scraper.fetch_all():
found_count += 1
if self._save_grocery_item(item_data) is not None:
saved_count += 1
finally:
scraper.cleanup()
return saved_count, found_count
def _save_grocery_item(
self, item_data: Dict[str, Any]
) -> Optional[GroceryItem]:
name = (item_data.get("name") or "").strip()
if not name:
return None
name_lower = name.lower()
source = item_data.get("source") or "lucky_california"
external_id = item_data.get("external_id")
ingredient = self.db.query(Ingredient).filter(
Ingredient.name_lower == name_lower
@@ -93,22 +210,41 @@ class ScraperService:
name=name,
name_lower=name_lower,
aisle=item_data.get("aisle"),
typical_price=item_data.get("current_price")
typical_price=item_data.get("current_price"),
)
self.db.add(ingredient)
self.db.flush()
existing = self.db.query(GroceryItem).filter(
GroceryItem.name == name,
GroceryItem.scraped_url == item_data.get("scraped_url")
).first()
# Idempotency:
# 1) (source, external_id) when both present (Swiftly path);
# 2) fall back to (name, scraped_url) for legacy rows from R2-A.
existing: Optional[GroceryItem] = None
if external_id:
existing = self.db.query(GroceryItem).filter(
GroceryItem.source == source,
GroceryItem.external_id == external_id,
).first()
if existing is None:
existing = self.db.query(GroceryItem).filter(
GroceryItem.name == name,
GroceryItem.scraped_url == item_data.get("scraped_url"),
).first()
if existing:
existing.ingredient_id = ingredient.id
existing.brand = item_data.get("brand")
existing.current_price = item_data.get("current_price")
existing.is_on_sale = item_data.get("is_on_sale", True)
existing.regular_price = item_data.get("regular_price")
existing.unit = item_data.get("unit")
existing.aisle = item_data.get("aisle")
existing.image_url = item_data.get("image_url")
existing.product_url = item_data.get("product_url")
existing.scraped_at = datetime.now()
existing.description = item_data.get("description")
existing.is_on_sale = bool(item_data.get("is_on_sale", False))
existing.scraped_at = datetime.now(timezone.utc)
existing.scraped_url = item_data.get("scraped_url")
existing.external_id = external_id
existing.source = source
self.db.flush()
return existing
@@ -116,31 +252,31 @@ class ScraperService:
id=uuid4(),
ingredient_id=ingredient.id,
name=name,
brand=item_data.get("brand"),
current_price=item_data.get("current_price"),
regular_price=item_data.get("regular_price"),
unit=item_data.get("unit"),
aisle=item_data.get("aisle"),
image_url=item_data.get("image_url"),
product_url=item_data.get("product_url"),
is_on_sale=item_data.get("is_on_sale", True),
description=item_data.get("description"),
is_on_sale=bool(item_data.get("is_on_sale", False)),
sale_start_date=item_data.get("sale_start_date"),
sale_end_date=item_data.get("sale_end_date"),
in_season=item_data.get("in_season", False),
scraped_at=datetime.now(),
scraped_url=item_data.get("scraped_url")
scraped_at=datetime.now(timezone.utc),
scraped_url=item_data.get("scraped_url"),
external_id=external_id,
source=source,
)
self.db.add(grocery_item)
self.db.commit()
self.db.refresh(grocery_item)
self.db.flush()
return grocery_item
def get_sale_items(self, limit: int = 50) -> list:
from app.models import GroceryItem
items = self.db.query(GroceryItem).filter(
GroceryItem.is_on_sale == True
GroceryItem.is_on_sale == True # noqa: E712 — SQLAlchemy idiom
).order_by(GroceryItem.scraped_at.desc()).limit(limit).all()
return [
@@ -152,7 +288,8 @@ class ScraperService:
"aisle": item.aisle,
"image_url": item.image_url,
"product_url": item.product_url,
"scraped_at": item.scraped_at.isoformat() if item.scraped_at else None
"description": item.description,
"scraped_at": item.scraped_at.isoformat() if item.scraped_at else None,
}
for item in items
]
]
+6
View File
@@ -0,0 +1,6 @@
[pytest]
testpaths = tests
asyncio_mode = auto
addopts = -ra
filterwarnings =
ignore::DeprecationWarning
+6
View File
@@ -0,0 +1,6 @@
# Dev / test-only dependencies. Production deps live in requirements.txt.
pytest>=7.4
pytest-cov>=4.1
pytest-asyncio>=0.23
httpx>=0.25
freezegun>=1.4
+2
View File
@@ -11,6 +11,8 @@ beautifulsoup4==4.12.3
lxml==5.1.0
apscheduler==3.10.4
python-dotenv==1.0.0
itsdangerous==2.1.2
httpx==0.26.0
requests==2.31.0
pytest==7.4.4
pytest-asyncio==0.23.3
View File
+205
View File
@@ -0,0 +1,205 @@
"""
Pytest fixtures for MealPlanner backend.
DB strategy
-----------
- If env ``TEST_DATABASE_URL`` is set, use it (must be a Postgres URL — the
current Alembic migrations use ``postgresql.UUID``/``JSONB``/``ARRAY`` types
which are not portable to SQLite).
- Else fall back to ``DATABASE_URL`` if it points at Postgres.
- Else skip Postgres-only tests (marker: ``requires_postgres``).
Each test using the ``db`` fixture runs inside a SAVEPOINT-style nested
transaction that rolls back on teardown so tests do not leak state.
"""
from __future__ import annotations
import os
import sys
import pathlib
import subprocess
from typing import Iterator
import pytest
# Ensure DATABASE_URL is set BEFORE importing app.config (Settings requires it).
# We default to the TEST_DATABASE_URL or a sentinel that lets imports succeed;
# tests that actually need the DB rely on the marker / fixture skip path below.
_DEFAULT_DSN = "postgresql://mealplanner:password@localhost:5432/mealplanner_test"
os.environ.setdefault(
"DATABASE_URL",
os.environ.get("TEST_DATABASE_URL", _DEFAULT_DSN),
)
# Make backend/ importable when pytest is invoked from repo root.
BACKEND_ROOT = pathlib.Path(__file__).resolve().parent.parent
if str(BACKEND_ROOT) not in sys.path:
sys.path.insert(0, str(BACKEND_ROOT))
from sqlalchemy import create_engine, text # noqa: E402
from sqlalchemy.orm import sessionmaker # noqa: E402
from sqlalchemy.exc import OperationalError # noqa: E402
def _resolve_test_dsn() -> str | None:
dsn = os.environ.get("TEST_DATABASE_URL") or os.environ.get("DATABASE_URL")
if not dsn:
return None
if not dsn.startswith(("postgresql://", "postgresql+psycopg2://")):
return None
return dsn
def _postgres_reachable(dsn: str) -> bool:
try:
eng = create_engine(
dsn.replace("postgresql://", "postgresql+psycopg2://"),
pool_pre_ping=True,
)
with eng.connect() as conn:
conn.execute(text("SELECT 1"))
eng.dispose()
return True
except Exception:
return False
_DSN = _resolve_test_dsn()
_PG_AVAILABLE = bool(_DSN) and _postgres_reachable(_DSN)
def pytest_collection_modifyitems(config, items):
"""Skip postgres-only tests when no live Postgres is available."""
if _PG_AVAILABLE:
return
skip_pg = pytest.mark.skip(
reason="Postgres not reachable; set TEST_DATABASE_URL to enable."
)
for item in items:
if "requires_postgres" in item.keywords:
item.add_marker(skip_pg)
def pytest_configure(config):
config.addinivalue_line(
"markers",
"requires_postgres: test needs a live Postgres reachable via TEST_DATABASE_URL",
)
config.addinivalue_line(
"markers",
"scraper_offline: parser-only test; uses saved HTML fixture; no "
"network or Playwright/Chromium required (CI-safe).",
)
config.addinivalue_line(
"markers",
"scraper_live: hits the live grocery website; requires Playwright "
"and Chromium; skipped by default in CI.",
)
# ---------------------------------------------------------------------------
# Schema bootstrap (session-scoped): run alembic upgrade head once per session.
# ---------------------------------------------------------------------------
@pytest.fixture(scope="session")
def _schema() -> Iterator[None]:
if not _PG_AVAILABLE:
yield
return
env = os.environ.copy()
env["DATABASE_URL"] = _DSN # Alembic env.py reads from settings.DATABASE_URL
# alembic.ini lives in backend/, run from there.
subprocess.run(
["alembic", "upgrade", "head"],
cwd=str(BACKEND_ROOT),
env=env,
check=True,
)
yield
# Best-effort cleanup so re-running the suite locally is idempotent.
subprocess.run(
["alembic", "downgrade", "base"],
cwd=str(BACKEND_ROOT),
env=env,
check=False,
)
@pytest.fixture(scope="session")
def _engine(_schema):
if not _PG_AVAILABLE:
yield None
return
eng = create_engine(
_DSN.replace("postgresql://", "postgresql+psycopg2://"),
pool_pre_ping=True,
)
yield eng
eng.dispose()
@pytest.fixture()
def db(_engine):
"""Per-test transactional session that rolls back at teardown."""
if _engine is None:
pytest.skip("Postgres not reachable")
connection = _engine.connect()
trans = connection.begin()
Session = sessionmaker(bind=connection, autocommit=False, autoflush=False)
session = Session()
try:
yield session
finally:
session.close()
trans.rollback()
connection.close()
@pytest.fixture()
def client(db):
"""TestClient with get_db overridden to yield the test session."""
from fastapi.testclient import TestClient
from app.main import app
from app.database import get_db
def _override():
try:
yield db
finally:
pass
app.dependency_overrides[get_db] = _override
try:
with TestClient(app) as c:
yield c
finally:
app.dependency_overrides.pop(get_db, None)
@pytest.fixture()
def client_no_db():
"""TestClient that does NOT require a live DB — for pure import/wiring smoke."""
from fastapi.testclient import TestClient
from app.main import app
from app.database import get_db
class _StubSession:
def execute(self, *a, **kw):
from sqlalchemy.engine import Result # noqa: F401
raise RuntimeError("DB not available in this fixture")
def query(self, *a, **kw):
raise RuntimeError("DB not available in this fixture")
def close(self):
pass
def _override():
yield _StubSession()
app.dependency_overrides[get_db] = _override
try:
with TestClient(app) as c:
yield c
finally:
app.dependency_overrides.pop(get_db, None)
+50
View File
@@ -0,0 +1,50 @@
# Lucky California weekly-ad spike (R2-A)
| Field | Value |
| --- | --- |
| URL | https://luckysupermarkets.com/coupons/Coupon%2Flu-featured-in-ad |
| Final URL | https://luckysupermarkets.com/coupons/Coupon%2Flu-featured-in-ad |
| Fetched (UTC) | 2026-05-04 (single live fetch via scripts/spike_lucky_scrape.py) |
| HTTP status | 200 |
| HTML bytes | 201224 |
| Items parsed (current parser) | 11 |
| Captcha/block signal | benign — page contains an empty `<span id="recaptcha-element">` placeholder, no actual challenge served. Title: "Featured in Ad \| Luckys Supermarket". 13 real `coupon-card-wrapper` cards rendered. |
| Error | none |
| User-Agent | `Mozilla/5.0 (compatible; MealPlannerSpike/0.1; +https://github.com/MealPlanner; spike=R2-A)` |
## Selectors used (verified against this fixture)
- Card root: `div.coupon-card-wrapper` (fallback: `div.coupon-card-container`)
- Price + short name: `.coupon-card-value-text` (e.g. `"$13.97 Pepsi 24 packs"`)
- Long description: `.coupon-card-short-description`
- Image: `img` inside `.coupon-card-img-container` (CDN URL: `cdn.luckysupermarkets.com/loyalty/offer/<id>.jpg`)
- No per-card link is present in the rendered DOM (offers are non-navigable tiles).
## First parsed item (sample)
```json
{
"name": "Pepsi 24 packs",
"description": "$13.97 Pepsi Products 24 pack, Poppi 8 pack, Gatorade 18 pack, Rockstar 10 pack, or Pure Leaf 12 pack, select varieties +CRV in CA. While supplies last.",
"current_price": 13.97,
"image_url": "https://cdn.luckysupermarkets.com/loyalty/offer/205213.jpg",
"product_url": null,
"is_on_sale": true,
"scraped_at": "2026-05-04T21:42:56.952936",
"scraped_url": "https://luckysupermarkets.com"
}
```
## Notes
- ONE live fetch performed by `scripts/spike_lucky_scrape.py`. Do not rerun without reason.
- HTML and PNG saved alongside this file (`weekly_ad.html`, `weekly_ad.png`).
- Initial parser (regex-based on `h2/h3/a` text) produced only 1 item; selectors were
stale. A minimal additive fix landed in `backend/app/scraper/lucky_ca_scraper.py`:
new `_parse_coupon_card` + `parse_featured_coupons_html` methods that target
Swiftly-style `.coupon-card-wrapper` cards. Old `_parse_coupon_item` retained
as a fallback path.
- Schema impact: `grocery_item` columns (name, current_price, image_url, is_on_sale,
scraped_at, scraped_url) are all populated. `product_url` is None for every
card (no per-offer link in DOM) — keep nullable. New optional `description`
field is produced; either add a `description TEXT` column or drop it.
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
Binary file not shown.

After

Width:  |  Height:  |  Size: 582 KiB

+47
View File
@@ -0,0 +1,47 @@
"""
Alembic round-trip test.
Verifies that ``alembic upgrade head`` followed by ``alembic downgrade base``
runs without error against a Postgres throwaway database. Skipped when no
Postgres is reachable (the migrations rely on PG-specific types and cannot
target SQLite).
"""
from __future__ import annotations
import os
import pathlib
import subprocess
import pytest
BACKEND_ROOT = pathlib.Path(__file__).resolve().parent.parent
@pytest.mark.requires_postgres
def test_alembic_upgrade_head_roundtrip():
dsn = os.environ.get("TEST_DATABASE_URL") or os.environ.get("DATABASE_URL")
assert dsn, "TEST_DATABASE_URL or DATABASE_URL must be set"
env = os.environ.copy()
env["DATABASE_URL"] = dsn
# The session fixture has already upgraded; downgrade then re-upgrade to
# exercise both paths inside this test without polluting the rest of the
# session schema state.
down = subprocess.run(
["alembic", "downgrade", "base"],
cwd=str(BACKEND_ROOT),
env=env,
capture_output=True,
text=True,
)
assert down.returncode == 0, f"downgrade failed: {down.stderr}"
up = subprocess.run(
["alembic", "upgrade", "head"],
cwd=str(BACKEND_ROOT),
env=env,
capture_output=True,
text=True,
)
assert up.returncode == 0, f"upgrade failed: {up.stderr}"
+237
View File
@@ -0,0 +1,237 @@
"""
R2-B: approval-token + per-voter vote round-trip tests.
- Pure unit tests for `app.services.approval` (no DB).
- DB-backed tests for `consume_token` single-use + the two vote routes.
"""
from __future__ import annotations
import time
import uuid
from datetime import date, timedelta
import pytest
from fastapi import HTTPException
from app.services import approval as approval_service
# ---------------------------------------------------------------------------
# Pure-unit token tests.
# ---------------------------------------------------------------------------
def test_issue_and_verify_token_roundtrip():
item_id = uuid.uuid4()
voter_id = uuid.uuid4()
token = approval_service.issue_token(item_id, voter_id)
payload = approval_service.verify_token(token)
assert payload["item"] == str(item_id)
assert payload["voter"] == str(voter_id)
def test_verify_rejects_tampered_token():
item_id = uuid.uuid4()
voter_id = uuid.uuid4()
token = approval_service.issue_token(item_id, voter_id)
tampered = token[:-2] + ("AA" if not token.endswith("AA") else "BB")
with pytest.raises(HTTPException) as excinfo:
approval_service.verify_token(tampered)
assert excinfo.value.status_code == 401
def test_verify_rejects_expired_token(monkeypatch):
"""itsdangerous reads `time.time()` — patch the time module's `time`
attribute on `itsdangerous.timed` to simulate clock advance.
"""
import itsdangerous.timed as _timed_mod
item_id = uuid.uuid4()
voter_id = uuid.uuid4()
# Issue at "now".
token = approval_service.issue_token(item_id, voter_id)
# Advance the clock 8 days past issue time (default TTL is 7 days).
real_now = time.time()
fake_now = real_now + (8 * 24 * 3600)
class _FakeTime:
@staticmethod
def time():
return fake_now
monkeypatch.setattr(_timed_mod, "time", _FakeTime)
with pytest.raises(HTTPException) as excinfo:
approval_service.verify_token(token)
assert excinfo.value.status_code == 401
# ---------------------------------------------------------------------------
# DB-backed fixtures: build a minimal scenario reused by several tests.
# ---------------------------------------------------------------------------
@pytest.fixture()
def scenario(db):
"""Create profile + 2 voters + recipe + plan + item, all in one shot.
Uses the per-test transactional session, so changes roll back on
teardown (no cross-test pollution).
"""
from app.models import (
FamilyMember,
FamilyMemberRole,
FamilyProfile,
MealPlan,
MealPlanItem,
MealPlanStatus,
MealType,
Recipe,
)
suffix = uuid.uuid4().hex[:8]
profile = FamilyProfile(
name=f"Test Family {suffix}",
household_size=2,
adult_count=2,
child_count=0,
)
db.add(profile)
db.flush()
voter_a = FamilyMember(
family_profile_id=profile.id,
name="Alice",
email=f"alice+{suffix}@example.com",
role=FamilyMemberRole.ADULT,
)
voter_b = FamilyMember(
family_profile_id=profile.id,
name="Bob",
email=f"bob+{suffix}@example.com",
role=FamilyMemberRole.ADULT,
)
db.add_all([voter_a, voter_b])
db.flush()
recipe = Recipe(
family_profile_id=profile.id,
name=f"Test Pasta {suffix}",
servings=2,
ingredients=[{"name": "pasta", "qty": "200g"}],
instructions=["Boil", "Drain"],
is_manually_added=True,
)
db.add(recipe)
db.flush()
plan = MealPlan(
family_profile_id=profile.id,
week_start_date=date.today() + timedelta(days=14),
status=MealPlanStatus.DRAFT,
)
db.add(plan)
db.flush()
item = MealPlanItem(
meal_plan_id=plan.id,
recipe_id=recipe.id,
day_of_week=1,
meal_type=MealType.DINNER,
)
db.add(item)
db.flush()
return {
"profile": profile,
"voter_a": voter_a,
"voter_b": voter_b,
"recipe": recipe,
"plan": plan,
"item": item,
}
# ---------------------------------------------------------------------------
# DB-backed tests.
# ---------------------------------------------------------------------------
@pytest.mark.requires_postgres
def test_consume_token_single_use(db, scenario):
from app.models import MealPlanVote
item = scenario["item"]
voter = scenario["voter_a"]
token = approval_service.issue_token(item.id, voter.id)
# First call: succeeds, returns the voter.
out = approval_service.consume_token(db, token, item.id)
assert out.id == voter.id
# Simulate the route writing the vote row (consume_token itself does NOT
# write — single-use is enforced by the presence of the vote row).
db.add(MealPlanVote(
meal_plan_item_id=item.id,
family_member_id=voter.id,
vote=True,
))
db.flush()
# Second call: raises 409.
with pytest.raises(HTTPException) as excinfo:
approval_service.consume_token(db, token, item.id)
assert excinfo.value.status_code == 409
@pytest.mark.requires_postgres
def test_vote_get_renders_html(client, db, scenario):
item = scenario["item"]
voter = scenario["voter_a"]
token = approval_service.issue_token(item.id, voter.id)
r = client.get(f"/api/meals/vote/{item.id}", params={"token": token})
assert r.status_code == 200, r.text
assert "text/html" in r.headers.get("content-type", "")
# Voter name appears in the visible body.
assert "Alice" in r.text
# Token is allowed inside the form `action` attribute (the link's href)
# but must not appear in the visible meal-card text.
body = r.text
card_start = body.find('<div class="meal">')
card_end = body.find("</div>", card_start)
assert card_start != -1 and card_end != -1
visible_card = body[card_start:card_end]
assert token not in visible_card
@pytest.mark.requires_postgres
def test_vote_post_records_and_decides(client, db, scenario):
"""First voter approves -> still pending (Bob hasn't voted).
Then Bob approves -> approved.
"""
item = scenario["item"]
voter_a = scenario["voter_a"]
voter_b = scenario["voter_b"]
token_a = approval_service.issue_token(item.id, voter_a.id)
r1 = client.post(
f"/api/meals/vote/{item.id}",
params={"token": token_a},
json={"vote": "approve"},
)
assert r1.status_code == 200, r1.text
body1 = r1.json()
assert body1["status"] == "recorded"
assert body1["item_status"] in ("pending", "approved")
# With 2 voters, after 1 approve we should still be pending.
assert body1["item_status"] == "pending"
token_b = approval_service.issue_token(item.id, voter_b.id)
r2 = client.post(
f"/api/meals/vote/{item.id}",
params={"token": token_b},
json={"vote": "approve"},
)
assert r2.status_code == 200, r2.text
body2 = r2.json()
assert body2["item_status"] == "approved"
+114
View File
@@ -0,0 +1,114 @@
"""
Auth gate tests for R1-B+D.
Covers:
- /api/admin/* requires bearer token (401 without, not 401 with valid).
- Mutating routes on family routers require a session cookie.
- /api/auth/login + /api/auth/logout round-trip with the shared password.
"""
from __future__ import annotations
import os
import pytest
# Configure auth secrets BEFORE app import. conftest.py runs first and sets
# DATABASE_URL; we layer auth env on top here.
os.environ.setdefault("ADMIN_TOKEN", "test-admin-token")
os.environ.setdefault("SESSION_PASSWORD", "test-family-password")
os.environ.setdefault("SECRET_KEY", "test-secret-key-do-not-use-in-prod")
@pytest.fixture(autouse=True)
def _reload_settings(monkeypatch):
"""Force ``settings`` to re-read env (test isolation)."""
monkeypatch.setenv("ADMIN_TOKEN", "test-admin-token")
monkeypatch.setenv("SESSION_PASSWORD", "test-family-password")
monkeypatch.setenv("SECRET_KEY", "test-secret-key-do-not-use-in-prod")
# Re-instantiate the singleton so dependents pick up env.
from app import config as app_config
app_config.settings = app_config.Settings()
yield
# ---------------------------------------------------------------------------
# Admin bearer token
# ---------------------------------------------------------------------------
@pytest.mark.requires_postgres
def test_admin_requires_token(client):
"""No bearer → 401. Valid bearer → not 401 (handler runs)."""
r = client.post("/api/admin/scrape")
assert r.status_code == 401, r.text
r = client.post(
"/api/admin/scrape",
headers={"Authorization": "Bearer test-admin-token"},
)
# Handler may 200/202/500 (Playwright not installed in CI), but NOT 401.
assert r.status_code != 401, r.text
@pytest.mark.requires_postgres
def test_admin_logs_requires_token(client):
r = client.get("/api/admin/logs")
assert r.status_code == 401
r = client.get(
"/api/admin/logs", headers={"Authorization": "Bearer test-admin-token"}
)
assert r.status_code == 200
# ---------------------------------------------------------------------------
# Session-gated mutations
# ---------------------------------------------------------------------------
@pytest.mark.requires_postgres
def test_session_required_for_mutation(client):
"""POST /api/profile/members without cookie → 401."""
r = client.post(
"/api/profile/members",
json={"name": "x", "email": "x@y.z", "role": "voter"},
)
assert r.status_code == 401, r.text
@pytest.mark.requires_postgres
def test_session_open_for_reads(client):
"""GET /api/profile is NOT auth-gated (reads stay open)."""
r = client.get("/api/profile")
# Either 200 (profile exists) or 404 (no profile yet) — never 401.
assert r.status_code in (200, 404), r.text
# ---------------------------------------------------------------------------
# Login / logout
# ---------------------------------------------------------------------------
@pytest.mark.requires_postgres
def test_session_login_logout(client):
# Wrong password
r = client.post("/api/auth/login", json={"password": "nope"})
assert r.status_code == 401
# Right password sets the cookie
r = client.post(
"/api/auth/login", json={"password": "test-family-password"}
)
assert r.status_code == 204
assert "mp_session" in r.cookies, r.headers
# With cookie, mutation succeeds (or fails for non-auth reasons)
cookie_value = r.cookies.get("mp_session")
client.cookies.set("mp_session", cookie_value)
r2 = client.post(
"/api/profile/members",
json={"name": "x", "email": "test@example.com", "role": "voter"},
)
# 401 means the cookie was rejected — that's the bug we're guarding.
assert r2.status_code != 401, r2.text
# Logout clears the cookie
r3 = client.post("/api/auth/logout")
assert r3.status_code == 204
+19
View File
@@ -0,0 +1,19 @@
"""
Config fail-fast: importing app.config with DATABASE_URL unset must raise.
"""
from __future__ import annotations
import importlib
import pytest
def test_database_url_required(monkeypatch):
"""Settings() with empty DATABASE_URL → RuntimeError at instantiation."""
from app import config as app_config
monkeypatch.setenv("DATABASE_URL", "")
# Pass _env_file=None so a stray .env on disk can't satisfy the field.
with pytest.raises(RuntimeError, match="DATABASE_URL is required"):
app_config.Settings(_env_file=None, DATABASE_URL="")
+68
View File
@@ -0,0 +1,68 @@
"""Async scrape endpoint contract.
Verifies that ``POST /api/admin/scrape``:
- returns 202 + ``scrape_log_id`` synchronously,
- persists a ``ScrapeLog`` row in status STARTED before the background task
runs (the task is monkey-patched out so it never reaches Playwright and
never opens a session outside the test transaction).
"""
from __future__ import annotations
import os
import uuid
import pytest
os.environ.setdefault("ADMIN_TOKEN", "test-admin-token")
@pytest.fixture(autouse=True)
def _admin_token(monkeypatch):
monkeypatch.setenv("ADMIN_TOKEN", "test-admin-token")
yield
@pytest.mark.requires_postgres
def test_scrape_returns_202_and_log_id(client, db, monkeypatch):
"""Endpoint enqueues the scrape and returns 202 + scrape_log_id.
We replace the background runner with a no-op so the test does NOT spin up
Playwright and does NOT open a session outside the rolled-back test
transaction.
"""
calls: list[tuple] = []
def _fake_run(log_id, source, scrape_type):
calls.append((log_id, source, scrape_type))
# Patch in BOTH the service module (definition site) and the api module
# (import site) so whichever symbol the route resolved to is replaced.
monkeypatch.setattr(
"app.services.scraper_service._run_scrape_in_background", _fake_run
)
r = client.post(
"/api/admin/scrape",
headers={"Authorization": "Bearer test-admin-token"},
)
assert r.status_code == 202, r.text
body = r.json()
assert body["status"] == "queued"
assert "scrape_log_id" in body
log_id = uuid.UUID(body["scrape_log_id"])
# Row was committed inside enqueue_scrape — visible on the test session.
from app.models import ScrapeLog, ScrapeStatus
log = db.query(ScrapeLog).filter(ScrapeLog.id == log_id).first()
assert log is not None, "ScrapeLog row should exist after enqueue"
assert log.status == ScrapeStatus.STARTED
assert log.source == "lucky_california"
assert log.scrape_type == "weekly_ad"
assert log.completed_at is None
# TestClient runs background tasks before returning from the context
# manager exit — by the time we get here, the fake runner ran exactly once.
assert len(calls) == 1
assert calls[0][0] == log_id
+63
View File
@@ -0,0 +1,63 @@
"""
Smoke tests: app boots, routers wire up, no import-time crashes.
The router-list test asserts each endpoint returns 200 or 401 (auth gate not
yet implemented in R1-A's scope) but explicitly NOT 5xx — the goal is to catch
import errors and crashing handlers, not to validate business logic.
"""
from __future__ import annotations
import os
import pytest
def test_app_imports():
from app.main import app
assert app.title == "MealPlanner"
@pytest.mark.requires_postgres
def test_health(client):
r = client.get("/health")
assert r.status_code == 200
assert r.json() == {"status": "ok"}
@pytest.mark.requires_postgres
def test_health_db(client):
r = client.get("/health/db")
assert r.status_code == 200
body = r.json()
assert body.get("database") == "connected"
ROUTER_GET_PATHS = [
"/api/profile",
"/api/profile/members",
"/api/recipes",
"/api/recipes/ingredients",
"/api/meals",
"/api/pantry",
"/api/shopping-list",
"/api/admin/logs",
]
@pytest.mark.requires_postgres
@pytest.mark.parametrize("path", ROUTER_GET_PATHS)
def test_router_list_endpoints(client, path):
"""
Each canonical GET must respond. 200 (handled), 401 (admin-gated),
404 (handler ran but no row found) — all OK. 5xx means import/runtime
crash; 307 (trailing-slash redirect) means a handler is still mounted
on the wrong path.
"""
r = client.get(path, follow_redirects=False)
assert r.status_code < 500, (
f"{path} returned {r.status_code}: {r.text[:300]}"
)
# 307 is a regression — canonical paths must be the route definitions.
assert r.status_code in (200, 401, 404, 422), (
f"{path} returned unexpected {r.status_code}"
)
+223
View File
@@ -0,0 +1,223 @@
"""Offline tests for the Swiftly product-API client (R3-0).
All tests run against saved fixtures and mocked HTTP — no live network,
no Playwright/Chromium. Captured 2026-05-05 from a single live spike;
see ``.agent/context.md`` "Swiftly API" for the field-mapping rationale.
"""
from __future__ import annotations
import json
import sys
import uuid
from decimal import Decimal
from pathlib import Path
from unittest.mock import MagicMock, patch
import pytest
import requests
# Make `app.*` importable when pytest is invoked from the repo root.
BACKEND_DIR = Path(__file__).resolve().parent.parent
if str(BACKEND_DIR) not in sys.path:
sys.path.insert(0, str(BACKEND_DIR))
from app.scraper.lucky_ca_scraper import ( # noqa: E402
LuckyCaliforniaScraper,
SwiftlyAuthError,
)
FIXTURE_DIR = Path(__file__).resolve().parent / "fixtures" / "lucky_ca"
CATEGORIES_HTML = FIXTURE_DIR / "categories.html"
CATEGORY_JSON = FIXTURE_DIR / "category_meat_seafood.json"
pytestmark = pytest.mark.scraper_offline
# ---------------------------------------------------------------------------
# Pure-parser tests against captured fixtures
# ---------------------------------------------------------------------------
def test_parse_categories_fixture() -> None:
"""Parser returns >=10 distinct API slugs from the captured page."""
if not CATEGORIES_HTML.exists():
pytest.skip(f"fixture missing: {CATEGORIES_HTML}")
html = CATEGORIES_HTML.read_text(encoding="utf-8")
slugs = LuckyCaliforniaScraper.parse_categories_html(html)
assert len(slugs) >= 10, f"expected >=10 categories, got {len(slugs)}"
assert len(set(slugs)) == len(slugs), "slugs must be deduplicated"
# Every slug should look like `Product/<name>` per the API contract.
for s in slugs:
assert s.startswith("Product/"), f"unexpected slug shape: {s!r}"
# Spot-check the one we know is in the captured snapshot.
assert "Product/meat_seafood" in slugs
def test_parse_category_response_fixture() -> None:
"""Parser returns >=10 product dicts each with the mapped fields populated."""
if not CATEGORY_JSON.exists():
pytest.skip(f"fixture missing: {CATEGORY_JSON}")
payload = json.loads(CATEGORY_JSON.read_text(encoding="utf-8"))
raw_items = LuckyCaliforniaScraper.parse_category_response(payload)
assert len(raw_items) >= 10, f"expected >=10 raw items, got {len(raw_items)}"
mapped: list[dict] = []
for raw in raw_items:
m = LuckyCaliforniaScraper.map_product(
raw, aisle="meat_seafood", source_slug="Product/meat_seafood"
)
if m is not None:
mapped.append(m)
assert len(mapped) >= 10, (
f"expected >=10 mapped products, got {len(mapped)} "
f"(from {len(raw_items)} raw)"
)
sample = mapped[0]
# Required fields per the field-mapping contract.
for key in (
"external_id",
"source",
"name",
"current_price",
"regular_price",
"is_on_sale",
"image_url",
"aisle",
):
assert key in sample, f"missing key {key!r} in mapped product: {sample!r}"
assert sample["source"] == "lucky_california"
assert sample["aisle"] == "meat_seafood"
assert isinstance(sample["external_id"], str) and sample["external_id"]
assert isinstance(sample["name"], str) and sample["name"].strip()
assert isinstance(sample["regular_price"], Decimal)
assert sample["regular_price"] > 0
assert isinstance(sample["is_on_sale"], bool)
# Across the whole category at least SOME items should be on sale and
# at least some should have a regular-only price (sanity for the parser).
assert any(m["is_on_sale"] for m in mapped), "expected at least one sale item"
assert any(not m["is_on_sale"] for m in mapped), "expected at least one reg-only item"
def test_map_product_returns_none_for_unparseable() -> None:
"""Products with no name AND no parseable price are dropped."""
assert LuckyCaliforniaScraper.map_product({"name": ""}) is None
assert (
LuckyCaliforniaScraper.map_product(
{"id": "x", "name": "Foo", "price": {"ok": {}}}
)
is None
)
# ---------------------------------------------------------------------------
# 401 → SwiftlyAuthError → FAILED ScrapeLog
# ---------------------------------------------------------------------------
def _mock_response(status_code: int, payload=None) -> MagicMock:
resp = MagicMock(spec=requests.Response)
resp.status_code = status_code
if payload is not None:
resp.json.return_value = payload
if status_code >= 400:
resp.raise_for_status.side_effect = requests.HTTPError(
f"{status_code} error", response=resp
)
else:
resp.raise_for_status.return_value = None
return resp
def test_swiftly_auth_error_on_401_from_api() -> None:
"""A 401 from the API host raises SwiftlyAuthError before raise_for_status."""
scraper = LuckyCaliforniaScraper(bearer_token="stale-token")
with patch.object(scraper.api_session, "get", return_value=_mock_response(401)):
with pytest.raises(SwiftlyAuthError) as excinfo:
scraper.fetch_category("Product/meat_seafood")
assert "SWIFTLY_BEARER_TOKEN expired" in str(excinfo.value)
def test_swiftly_auth_error_when_token_missing() -> None:
"""An empty token short-circuits to SwiftlyAuthError without any HTTP call.
Force the token empty AFTER construction so the test is independent of
whatever ``SWIFTLY_BEARER_TOKEN`` happens to be set in the environment
(it WILL be set when pytest runs inside ``docker compose``).
"""
scraper = LuckyCaliforniaScraper()
scraper.bearer_token = ""
with patch.object(scraper.api_session, "get") as mock_get:
with pytest.raises(SwiftlyAuthError):
scraper.fetch_category("Product/meat_seafood")
mock_get.assert_not_called()
@pytest.mark.requires_postgres
def test_background_runner_writes_failed_with_token_message(monkeypatch):
"""A 401 during the background scrape lands in ScrapeLog as FAILED + message.
Uses a real (non-fixture) session so the bg runner's rollback+re-query
path mirrors production. The bg runner commits the FAILED row; we clean
up explicitly at the end.
"""
from app.models import ScrapeLog, ScrapeStatus
from app.services import scraper_service
from app.scraper.lucky_ca_scraper import SwiftlyAuthError, LuckyCaliforniaScraper
from app.database import SessionLocal
from datetime import datetime, timezone
log_id = uuid.uuid4()
setup_session = SessionLocal()
try:
setup_session.add(
ScrapeLog(
id=log_id,
source="lucky_california",
scrape_type="weekly_ad",
status=ScrapeStatus.STARTED,
started_at=datetime.now(timezone.utc),
)
)
setup_session.commit()
finally:
setup_session.close()
def _explode(self):
raise SwiftlyAuthError(
"SWIFTLY_BEARER_TOKEN expired — request a fresh token from the user "
"(capture from luckysupermarkets.com network tab on a /search/api/v1 request)"
)
monkeypatch.setattr(LuckyCaliforniaScraper, "fetch_all", _explode)
try:
scraper_service._run_scrape_in_background(
log_id, "lucky_california", "weekly_ad"
)
verify_session = SessionLocal()
try:
refreshed = (
verify_session.query(ScrapeLog)
.filter(ScrapeLog.id == log_id)
.first()
)
assert refreshed is not None
assert refreshed.status == ScrapeStatus.FAILED
assert "SWIFTLY_BEARER_TOKEN expired" in (refreshed.error_message or "")
assert refreshed.completed_at is not None
finally:
verify_session.close()
finally:
cleanup_session = SessionLocal()
try:
cleanup_session.query(ScrapeLog).filter(ScrapeLog.id == log_id).delete()
cleanup_session.commit()
finally:
cleanup_session.close()