feat: phase r1+r2 recovery + r3-0 swiftly api ingestion

R1 stabilization: pytest harness with transactional db fixture, smoke
+ alembic + auth + scrape + approval + swiftly tests, github actions
ci yaml. Bearer-token admin auth + signed-cookie session for family
ui mutations. Async POST /api/admin/scrape (BackgroundTasks, returns
202). Path canonicalization (no /list, /planned suffixes). DATABASE_URL
fail-fast on empty.

R2 deferred-risk spikes: live lucky california fetch (R2-A), full
email+per-voter approval click round trip with single-use enforcement
(R2-B, console email backend, sendgrid stub).

R3-0 phase 3 redesign: replaced playwright html scraper with requests
based swiftly json api client. 17 categories, ~10k products per scrape,
upsert by (source, external_id). 401 surfaces actionable token-refresh
message via ScrapeLog.error_message.

Pre-existing defects fixed: shopping_list.py syntax error blocking app
import, MealPlan.votes orphan relationship, JSONB(astext=True) invalid
kwarg, missing requests dep, calorie_target schema drift, every SQLEnum
needed values_callable, 0001 had empty downgrade(), seed had duplicate
ingredient rows.

Migrations added: 0003 grocery_item.description, 0004 family_profile.
calorie_target, 0005 grocery_item.external_id + source + composite index.

Verified: 31/31 pytest green, alembic upgrade->downgrade->upgrade clean,
frontend npm run build clean, live scrape 9,960 grocery_item rows in 36s.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
2026-05-05 14:08:19 -07:00
co-authored by Claude Opus 4.7
parent b9434967ed
commit 8e89f793d5
58 changed files with 3594 additions and 348 deletions
+18 -9
View File
@@ -1,20 +1,29 @@
from fastapi import APIRouter, Depends, HTTPException
from fastapi import APIRouter, BackgroundTasks, Depends, HTTPException
from sqlalchemy.orm import Session
from app.database import get_db
from app.models import ScrapeLog, EmailLog, MealPlan
from app.services.scraper_service import ScraperService
from app.security import require_admin
from app.services.scraper_service import ScraperService, enqueue_scrape
from typing import List, Optional
from datetime import datetime, timedelta
router = APIRouter()
router = APIRouter(dependencies=[Depends(require_admin)])
@router.post("/scrape")
def trigger_scrape(source: str = "lucky_california", scrape_type: str = "weekly_ad", db: Session = Depends(get_db)):
scraper_service = ScraperService(db)
result = scraper_service.run_scrape(source=source, scrape_type=scrape_type)
return result
@router.post("/scrape", status_code=202)
def trigger_scrape(
background_tasks: BackgroundTasks,
source: str = "lucky_california",
scrape_type: str = "weekly_ad",
db: Session = Depends(get_db),
):
log = enqueue_scrape(
db,
source=source,
scrape_type=scrape_type,
background_tasks=background_tasks,
)
return {"status": "queued", "scrape_log_id": str(log.id)}
@router.get("/logs")
+67
View File
@@ -0,0 +1,67 @@
"""
Family-shared session login.
POST /api/auth/login — body ``{"password": "..."}`` — must match
``settings.SESSION_PASSWORD``. On success: signs the first FamilyProfile.id
and writes it as the ``mp_session`` cookie, returns 204.
POST /api/auth/logout — clears the cookie, returns 204.
The session is intentionally simple: a single shared family password gates
mutations behind nginx on the trusted network. No per-user auth.
"""
from fastapi import APIRouter, Depends, HTTPException, Response, status
from pydantic import BaseModel
from sqlalchemy.orm import Session
from app.config import settings
from app.database import get_db
from app.models import FamilyProfile
from app.security import SESSION_COOKIE, SESSION_MAX_AGE, issue_session
router = APIRouter()
class LoginRequest(BaseModel):
password: str
@router.post("/login")
def login(payload: LoginRequest, db: Session = Depends(get_db)):
expected = settings.SESSION_PASSWORD
if not expected:
raise HTTPException(
status_code=status.HTTP_503_SERVICE_UNAVAILABLE,
detail="Session auth not configured",
)
if payload.password != expected:
raise HTTPException(
status_code=status.HTTP_401_UNAUTHORIZED, detail="Invalid password"
)
profile = db.query(FamilyProfile).first()
# If no profile exists yet, sign a placeholder so the cookie still
# validates; the family-id will be re-issued the first time a profile
# is created. This avoids login being blocked on first-run.
family_id = str(profile.id) if profile else "bootstrap"
cookie_value = issue_session(family_id)
response = Response(status_code=status.HTTP_204_NO_CONTENT)
response.set_cookie(
key=SESSION_COOKIE,
value=cookie_value,
max_age=SESSION_MAX_AGE,
httponly=True,
secure=True,
samesite="lax",
path="/",
)
return response
@router.post("/logout")
def logout():
response = Response(status_code=status.HTTP_204_NO_CONTENT)
response.delete_cookie(key=SESSION_COOKIE, path="/")
return response
+158 -60
View File
@@ -1,4 +1,7 @@
from fastapi import APIRouter, Depends, HTTPException
from fastapi import APIRouter, Depends, HTTPException, Query
from fastapi.responses import HTMLResponse
from html import escape as _html_escape
from pydantic import BaseModel, Field
from sqlalchemy.orm import Session, joinedload
from app.database import get_db
from app.models import (
@@ -10,6 +13,8 @@ from app.schemas import (
MealPlanResponse, MealPlanCreate,
MealPlanItemResponse, VoteRequest, VoteResponse
)
from app.security import require_session
from app.services import approval as approval_service
from uuid import UUID
from typing import List, Optional
from datetime import datetime, timedelta
@@ -17,7 +22,7 @@ from datetime import datetime, timedelta
router = APIRouter()
@router.get("/planned", response_model=Optional[MealPlanResponse])
@router.get("", response_model=Optional[MealPlanResponse])
def get_planned_meals(db: Session = Depends(get_db)):
profile = db.query(FamilyProfile).first()
if not profile:
@@ -33,7 +38,7 @@ def get_planned_meals(db: Session = Depends(get_db)):
return meal_plan
@router.post("/", response_model=MealPlanResponse)
@router.post("", response_model=MealPlanResponse, dependencies=[Depends(require_session)])
def create_meal_plan(meal_plan_data: MealPlanCreate, db: Session = Depends(get_db)):
profile = db.query(FamilyProfile).first()
if not profile:
@@ -80,7 +85,7 @@ def get_meal_plan(meal_plan_id: UUID, db: Session = Depends(get_db)):
return meal_plan
@router.post("/{meal_plan_id}/lock")
@router.post("/{meal_plan_id}/lock", dependencies=[Depends(require_session)])
def lock_meal_plan(meal_plan_id: UUID, db: Session = Depends(get_db)):
meal_plan = db.query(MealPlan).filter(MealPlan.id == meal_plan_id).first()
if not meal_plan:
@@ -91,76 +96,169 @@ def lock_meal_plan(meal_plan_id: UUID, db: Session = Depends(get_db)):
return {"message": "Meal plan locked", "status": meal_plan.status.value}
@router.get("/items/{item_id}/vote/{token}")
def get_vote_page(item_id: UUID, token: str, db: Session = Depends(get_db)):
approval_token = db.query(ApprovalToken).filter(ApprovalToken.token == token).first()
if not approval_token:
raise HTTPException(status_code=404, detail="Invalid token")
_DAY_NAMES = {
1: "Monday", 2: "Tuesday", 3: "Wednesday", 4: "Thursday",
5: "Friday", 6: "Saturday", 7: "Sunday",
}
if approval_token.meal_plan_item_id != item_id:
class VoteSubmission(BaseModel):
"""Body for POST /vote/{item_id}?token=...
Spec body: {"vote": "approve" | "deny"}.
"""
vote: str = Field(..., pattern="^(approve|deny)$")
@router.get("/vote/{item_id}", response_class=HTMLResponse)
def get_vote_page(
item_id: UUID,
token: str = Query(..., description="Per-voter signed token"),
db: Session = Depends(get_db),
):
"""Render the per-voter approval confirmation page.
Verifies the signed token (no consume) and returns minimal accessible
HTML with Approve / Deny buttons that POST to the same URL.
"""
payload = approval_service.verify_token(token)
if str(payload.get("item")) != str(item_id):
raise HTTPException(status_code=400, detail="Token not valid for this meal")
if approval_token.status != ApprovalTokenStatus.ACTIVE:
raise HTTPException(status_code=400, detail="Token has already been used or expired")
voter = (
db.query(FamilyMember)
.filter(FamilyMember.id == UUID(str(payload["voter"])))
.first()
)
if not voter:
raise HTTPException(status_code=404, detail="Voter not found")
if approval_token.expires_at < datetime.now():
raise HTTPException(status_code=400, detail="Token has expired")
item = (
db.query(MealPlanItem)
.options(joinedload(MealPlanItem.recipe))
.filter(MealPlanItem.id == item_id)
.first()
)
if not item:
raise HTTPException(status_code=404, detail="Meal plan item not found")
recipe_name = item.recipe.name if item.recipe else "Unnamed meal"
day_name = _DAY_NAMES.get(int(item.day_of_week), str(item.day_of_week))
meal_type = item.meal_type.value if item.meal_type else ""
# Token is included only inside the form action (href), never in the
# visible body. POST is performed by JS so we can submit JSON without
# leaving the page; non-JS users still get a usable form fallback.
safe_voter = _html_escape(voter.name)
safe_recipe = _html_escape(recipe_name)
safe_day = _html_escape(day_name)
safe_meal = _html_escape(meal_type)
action_url = f"/api/meals/vote/{item_id}?token={_html_escape(token, quote=True)}"
html = f"""<!doctype html>
<html lang="en">
<head>
<meta charset="utf-8">
<title>Approve meal</title>
<meta name="viewport" content="width=device-width, initial-scale=1">
<style>
body {{ font-family: system-ui, sans-serif; max-width: 36rem; margin: 2rem auto;
padding: 0 1rem; color: #111; background: #fff; line-height: 1.5; }}
h1 {{ font-size: 1.4rem; }}
.meal {{ padding: 1rem; border: 1px solid #444; border-radius: 6px; margin: 1rem 0; }}
button {{ font-size: 1rem; padding: .6rem 1.2rem; margin-right: .5rem;
border: 2px solid #111; border-radius: 4px; cursor: pointer; }}
.approve {{ background: #0a6b2b; color: #fff; }}
.deny {{ background: #b00020; color: #fff; }}
#status {{ margin-top: 1rem; font-weight: bold; }}
</style>
</head>
<body>
<h1>Hi {safe_voter}, please vote on this meal</h1>
<div class="meal">
<div><strong>{safe_recipe}</strong></div>
<div>{safe_day} &middot; {safe_meal}</div>
</div>
<form id="voteForm" method="post" action="{action_url}">
<button type="submit" name="vote" value="approve" class="approve" aria-label="Approve this meal">Approve</button>
<button type="submit" name="vote" value="deny" class="deny" aria-label="Deny this meal">Deny</button>
</form>
<div id="status" role="status" aria-live="polite"></div>
<script>
document.getElementById('voteForm').addEventListener('submit', async function(e) {{
e.preventDefault();
var btn = e.submitter || document.activeElement;
var vote = btn && btn.value ? btn.value : 'approve';
var resp = await fetch(this.action, {{
method: 'POST',
headers: {{ 'Content-Type': 'application/json' }},
body: JSON.stringify({{ vote: vote }})
}});
var data = {{}};
try {{ data = await resp.json(); }} catch (_) {{}}
var s = document.getElementById('status');
if (resp.ok) {{
s.textContent = 'Recorded: ' + (data.item_status || vote);
}} else {{
s.textContent = 'Error: ' + (data.detail || resp.status);
}}
}});
</script>
</body>
</html>
"""
return HTMLResponse(content=html, status_code=200)
@router.post("/vote/{item_id}")
def submit_vote(
item_id: UUID,
submission: VoteSubmission,
token: str = Query(..., description="Per-voter signed token"),
db: Session = Depends(get_db),
):
"""Record a per-voter vote and apply the approval rule.
- Single-use enforcement lives in `approval_service.consume_token`.
- Approval rule: any deny -> item.denied; all-approve -> item.approved;
otherwise pending (waiting on remaining voters).
"""
voter = approval_service.consume_token(db, token, item_id)
item = db.query(MealPlanItem).filter(MealPlanItem.id == item_id).first()
if not item:
raise HTTPException(status_code=404, detail="Meal plan item not found")
return {
"item_id": str(item_id),
"family_member_id": str(approval_token.family_member_id),
"meal_plan_item": item
}
@router.post("/items/{item_id}/vote/{token}", response_model=VoteResponse)
def submit_vote(item_id: UUID, token: str, vote_data: VoteRequest, db: Session = Depends(get_db)):
approval_token = db.query(ApprovalToken).filter(ApprovalToken.token == token).first()
if not approval_token:
raise HTTPException(status_code=404, detail="Invalid token")
if approval_token.meal_plan_item_id != item_id:
raise HTTPException(status_code=400, detail="Token not valid for this meal")
if approval_token.status != ApprovalTokenStatus.ACTIVE:
raise HTTPException(status_code=400, detail="Token has already been used or expired")
if approval_token.expires_at < datetime.now():
approval_token.status = ApprovalTokenStatus.EXPIRED
db.commit()
raise HTTPException(status_code=400, detail="Token has expired")
existing_vote = db.query(MealPlanVote).filter(
MealPlanVote.meal_plan_item_id == item_id,
MealPlanVote.family_member_id == approval_token.family_member_id
).first()
if existing_vote:
raise HTTPException(status_code=400, detail="You have already voted on this meal")
vote = MealPlanVote(
vote_bool = submission.vote == "approve"
db.add(MealPlanVote(
meal_plan_item_id=item_id,
family_member_id=approval_token.family_member_id,
vote=vote_data.vote
)
db.add(vote)
family_member_id=voter.id,
vote=vote_bool,
))
db.flush()
approval_token.status = ApprovalTokenStatus.USED
approval_token.used_at = datetime.now()
# Approval rule: count electorate (all family members on this profile)
# vs votes recorded so far.
profile_id = item.meal_plan.family_profile_id
electorate_ids = {
m.id for m in db.query(FamilyMember)
.filter(FamilyMember.family_profile_id == profile_id)
.all()
}
votes = db.query(MealPlanVote).filter(
MealPlanVote.meal_plan_item_id == item_id,
).all()
item = db.query(MealPlanItem).filter(MealPlanItem.id == item_id).first()
if not vote_data.vote and vote_data.denial_reason:
if any(v.vote is False for v in votes):
item.approval_status = MealPlanItemStatus.DENIED
item.denial_reason = vote_data.denial_reason
item.denial_details = vote_data.denial_details
elif electorate_ids and {v.family_member_id for v in votes} >= electorate_ids:
item.approval_status = MealPlanItemStatus.APPROVED
else:
item.approval_status = MealPlanItemStatus.PENDING
db.commit()
db.refresh(vote)
return vote
return {"status": "recorded", "item_status": item.approval_status.value}
@router.get("/items/{item_id}", response_model=MealPlanItemResponse)
@@ -173,7 +271,7 @@ def get_meal_item(item_id: UUID, db: Session = Depends(get_db)):
return item
@router.post("/items/{item_id}/swap")
@router.post("/items/{item_id}/swap", dependencies=[Depends(require_session)])
def swap_meal_item(item_id: UUID, new_recipe_id: UUID, db: Session = Depends(get_db)):
item = db.query(MealPlanItem).filter(MealPlanItem.id == item_id).first()
if not item:
+5 -4
View File
@@ -3,13 +3,14 @@ from sqlalchemy.orm import Session
from app.database import get_db
from app.models import HomePantry, Ingredient, FamilyProfile
from app.schemas import HomePantryResponse, HomePantryCreate
from app.security import require_session
from uuid import UUID
from typing import List
router = APIRouter()
@router.get("/", response_model=List[HomePantryResponse])
@router.get("", response_model=List[HomePantryResponse])
def get_pantry_items(db: Session = Depends(get_db)):
profile = db.query(FamilyProfile).first()
if not profile:
@@ -21,7 +22,7 @@ def get_pantry_items(db: Session = Depends(get_db)):
return items
@router.post("/", response_model=HomePantryResponse)
@router.post("", response_model=HomePantryResponse, dependencies=[Depends(require_session)])
def add_pantry_item(item: HomePantryCreate, db: Session = Depends(get_db)):
profile = db.query(FamilyProfile).first()
if not profile:
@@ -54,7 +55,7 @@ def add_pantry_item(item: HomePantryCreate, db: Session = Depends(get_db)):
return db_item
@router.delete("/{item_id}")
@router.delete("/{item_id}", dependencies=[Depends(require_session)])
def remove_pantry_item(item_id: UUID, db: Session = Depends(get_db)):
item = db.query(HomePantry).filter(HomePantry.id == item_id).first()
if not item:
@@ -65,7 +66,7 @@ def remove_pantry_item(item_id: UUID, db: Session = Depends(get_db)):
return {"message": "Pantry item removed"}
@router.put("/{item_id}", response_model=HomePantryResponse)
@router.put("/{item_id}", response_model=HomePantryResponse, dependencies=[Depends(require_session)])
def update_pantry_item(item_id: UUID, update: HomePantryCreate, db: Session = Depends(get_db)):
item = db.query(HomePantry).filter(HomePantry.id == item_id).first()
if not item:
+5 -4
View File
@@ -6,13 +6,14 @@ from app.schemas import (
FamilyProfileResponse, FamilyProfileUpdate, FamilyProfileCreate,
FamilyMemberResponse, FamilyMemberCreate
)
from app.security import require_session
from uuid import UUID
from typing import List
router = APIRouter()
@router.get("/", response_model=FamilyProfileResponse)
@router.get("", response_model=FamilyProfileResponse)
def get_profile(db: Session = Depends(get_db)):
profile = db.query(FamilyProfile).first()
if not profile:
@@ -20,7 +21,7 @@ def get_profile(db: Session = Depends(get_db)):
return profile
@router.put("/", response_model=FamilyProfileResponse)
@router.put("", response_model=FamilyProfileResponse, dependencies=[Depends(require_session)])
def update_profile(update: FamilyProfileUpdate, db: Session = Depends(get_db)):
profile = db.query(FamilyProfile).first()
if not profile:
@@ -43,7 +44,7 @@ def get_members(db: Session = Depends(get_db)):
return profile.members
@router.post("/members", response_model=FamilyMemberResponse)
@router.post("/members", response_model=FamilyMemberResponse, dependencies=[Depends(require_session)])
def add_member(member: FamilyMemberCreate, db: Session = Depends(get_db)):
profile = db.query(FamilyProfile).first()
if not profile:
@@ -69,7 +70,7 @@ def add_member(member: FamilyMemberCreate, db: Session = Depends(get_db)):
return db_member
@router.delete("/members/{member_id}")
@router.delete("/members/{member_id}", dependencies=[Depends(require_session)])
def delete_member(member_id: UUID, db: Session = Depends(get_db)):
member = db.query(FamilyMember).filter(FamilyMember.id == member_id).first()
if not member:
+35 -34
View File
@@ -3,13 +3,14 @@ from sqlalchemy.orm import Session, joinedload
from app.database import get_db
from app.models import Recipe, FamilyProfile
from app.schemas import RecipeResponse, RecipeCreate, IngredientCreate, IngredientResponse
from app.security import require_session
from uuid import UUID
from typing import List, Optional
router = APIRouter()
@router.get("/", response_model=List[RecipeResponse])
@router.get("", response_model=List[RecipeResponse])
def get_recipes(
skip: int = 0,
limit: int = 50,
@@ -31,6 +32,37 @@ def get_recipes(
return recipes
@router.get("/ingredients", response_model=List[IngredientResponse])
def list_ingredients(skip: int = 0, limit: int = 100, db: Session = Depends(get_db)):
from app.models import Ingredient
ingredients = db.query(Ingredient).offset(skip).limit(limit).all()
return ingredients
@router.post("/ingredients", response_model=IngredientResponse, dependencies=[Depends(require_session)])
def create_ingredient(ingredient: IngredientCreate, db: Session = Depends(get_db)):
from app.models import Ingredient
existing = db.query(Ingredient).filter(Ingredient.name_lower == ingredient.name_lower).first()
if existing:
raise HTTPException(status_code=400, detail="Ingredient with this name already exists")
db_ingredient = Ingredient(
name=ingredient.name,
name_lower=ingredient.name_lower,
plural_name=ingredient.plural_name,
aisle=ingredient.aisle,
typical_price=ingredient.typical_price,
unit=ingredient.unit,
season_months=ingredient.season_months
)
db.add(db_ingredient)
db.commit()
db.refresh(db_ingredient)
return db_ingredient
@router.get("/{recipe_id}", response_model=RecipeResponse)
def get_recipe(recipe_id: UUID, db: Session = Depends(get_db)):
recipe = db.query(Recipe).filter(Recipe.id == recipe_id).first()
@@ -39,7 +71,7 @@ def get_recipe(recipe_id: UUID, db: Session = Depends(get_db)):
return recipe
@router.post("/", response_model=RecipeResponse)
@router.post("", response_model=RecipeResponse, dependencies=[Depends(require_session)])
def create_recipe(recipe: RecipeCreate, db: Session = Depends(get_db)):
profile = db.query(FamilyProfile).first()
@@ -69,7 +101,7 @@ def create_recipe(recipe: RecipeCreate, db: Session = Depends(get_db)):
return db_recipe
@router.delete("/{recipe_id}")
@router.delete("/{recipe_id}", dependencies=[Depends(require_session)])
def delete_recipe(recipe_id: UUID, db: Session = Depends(get_db)):
recipe = db.query(Recipe).filter(Recipe.id == recipe_id).first()
if not recipe:
@@ -78,34 +110,3 @@ def delete_recipe(recipe_id: UUID, db: Session = Depends(get_db)):
db.delete(recipe)
db.commit()
return {"message": "Recipe deleted"}
@router.get("/ingredients/list", response_model=List[IngredientResponse])
def list_ingredients(skip: int = 0, limit: int = 100, db: Session = Depends(get_db)):
from app.models import Ingredient
ingredients = db.query(Ingredient).offset(skip).limit(limit).all()
return ingredients
@router.post("/ingredients", response_model=IngredientResponse)
def create_ingredient(ingredient: IngredientCreate, db: Session = Depends(get_db)):
from app.models import Ingredient
existing = db.query(Ingredient).filter(Ingredient.name_lower == ingredient.name_lower).first()
if existing:
raise HTTPException(status_code=400, detail="Ingredient with this name already exists")
db_ingredient = Ingredient(
name=ingredient.name,
name_lower=ingredient.name_lower,
plural_name=ingredient.plural_name,
aisle=ingredient.aisle,
typical_price=ingredient.typical_price,
unit=ingredient.unit,
season_months=ingredient.season_months
)
db.add(db_ingredient)
db.commit()
db.refresh(db_ingredient)
return db_ingredient
+2 -2
View File
@@ -14,7 +14,7 @@ from collections import defaultdict
router = APIRouter()
@router.get("/", response_model=ShoppingListResponse)
@router.get("", response_model=ShoppingListResponse)
def get_shopping_list(db: Session = Depends(get_db)):
profile = db.query(FamilyProfile).first()
if not profile:
@@ -56,7 +56,7 @@ def get_shopping_list(db: Session = Depends(get_db)):
if all_ingredient_ids:
ingredients = db.query(Ingredient).filter(
Ingredient.id.in_ all_ingredient_ids
Ingredient.id.in_(all_ingredient_ids)
).all()
ingredient_map = {i.id: i for i in ingredients}