Public Access
feat: phase r1+r2 recovery + r3-0 swiftly api ingestion
R1 stabilization: pytest harness with transactional db fixture, smoke + alembic + auth + scrape + approval + swiftly tests, github actions ci yaml. Bearer-token admin auth + signed-cookie session for family ui mutations. Async POST /api/admin/scrape (BackgroundTasks, returns 202). Path canonicalization (no /list, /planned suffixes). DATABASE_URL fail-fast on empty. R2 deferred-risk spikes: live lucky california fetch (R2-A), full email+per-voter approval click round trip with single-use enforcement (R2-B, console email backend, sendgrid stub). R3-0 phase 3 redesign: replaced playwright html scraper with requests based swiftly json api client. 17 categories, ~10k products per scrape, upsert by (source, external_id). 401 surfaces actionable token-refresh message via ScrapeLog.error_message. Pre-existing defects fixed: shopping_list.py syntax error blocking app import, MealPlan.votes orphan relationship, JSONB(astext=True) invalid kwarg, missing requests dep, calorie_target schema drift, every SQLEnum needed values_callable, 0001 had empty downgrade(), seed had duplicate ingredient rows. Migrations added: 0003 grocery_item.description, 0004 family_profile. calorie_target, 0005 grocery_item.external_id + source + composite index. Verified: 31/31 pytest green, alembic upgrade->downgrade->upgrade clean, frontend npm run build clean, live scrape 9,960 grocery_item rows in 36s. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
Claude Opus 4.7
@@ -1,20 +1,29 @@
|
||||
from fastapi import APIRouter, Depends, HTTPException
|
||||
from fastapi import APIRouter, BackgroundTasks, Depends, HTTPException
|
||||
from sqlalchemy.orm import Session
|
||||
from app.database import get_db
|
||||
from app.models import ScrapeLog, EmailLog, MealPlan
|
||||
from app.services.scraper_service import ScraperService
|
||||
from app.security import require_admin
|
||||
from app.services.scraper_service import ScraperService, enqueue_scrape
|
||||
from typing import List, Optional
|
||||
from datetime import datetime, timedelta
|
||||
|
||||
router = APIRouter()
|
||||
router = APIRouter(dependencies=[Depends(require_admin)])
|
||||
|
||||
|
||||
@router.post("/scrape")
|
||||
def trigger_scrape(source: str = "lucky_california", scrape_type: str = "weekly_ad", db: Session = Depends(get_db)):
|
||||
scraper_service = ScraperService(db)
|
||||
result = scraper_service.run_scrape(source=source, scrape_type=scrape_type)
|
||||
|
||||
return result
|
||||
@router.post("/scrape", status_code=202)
|
||||
def trigger_scrape(
|
||||
background_tasks: BackgroundTasks,
|
||||
source: str = "lucky_california",
|
||||
scrape_type: str = "weekly_ad",
|
||||
db: Session = Depends(get_db),
|
||||
):
|
||||
log = enqueue_scrape(
|
||||
db,
|
||||
source=source,
|
||||
scrape_type=scrape_type,
|
||||
background_tasks=background_tasks,
|
||||
)
|
||||
return {"status": "queued", "scrape_log_id": str(log.id)}
|
||||
|
||||
|
||||
@router.get("/logs")
|
||||
|
||||
@@ -0,0 +1,67 @@
|
||||
"""
|
||||
Family-shared session login.
|
||||
|
||||
POST /api/auth/login — body ``{"password": "..."}`` — must match
|
||||
``settings.SESSION_PASSWORD``. On success: signs the first FamilyProfile.id
|
||||
and writes it as the ``mp_session`` cookie, returns 204.
|
||||
|
||||
POST /api/auth/logout — clears the cookie, returns 204.
|
||||
|
||||
The session is intentionally simple: a single shared family password gates
|
||||
mutations behind nginx on the trusted network. No per-user auth.
|
||||
"""
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, Response, status
|
||||
from pydantic import BaseModel
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.config import settings
|
||||
from app.database import get_db
|
||||
from app.models import FamilyProfile
|
||||
from app.security import SESSION_COOKIE, SESSION_MAX_AGE, issue_session
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
|
||||
class LoginRequest(BaseModel):
|
||||
password: str
|
||||
|
||||
|
||||
@router.post("/login")
|
||||
def login(payload: LoginRequest, db: Session = Depends(get_db)):
|
||||
expected = settings.SESSION_PASSWORD
|
||||
if not expected:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_503_SERVICE_UNAVAILABLE,
|
||||
detail="Session auth not configured",
|
||||
)
|
||||
if payload.password != expected:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_401_UNAUTHORIZED, detail="Invalid password"
|
||||
)
|
||||
|
||||
profile = db.query(FamilyProfile).first()
|
||||
# If no profile exists yet, sign a placeholder so the cookie still
|
||||
# validates; the family-id will be re-issued the first time a profile
|
||||
# is created. This avoids login being blocked on first-run.
|
||||
family_id = str(profile.id) if profile else "bootstrap"
|
||||
cookie_value = issue_session(family_id)
|
||||
|
||||
response = Response(status_code=status.HTTP_204_NO_CONTENT)
|
||||
response.set_cookie(
|
||||
key=SESSION_COOKIE,
|
||||
value=cookie_value,
|
||||
max_age=SESSION_MAX_AGE,
|
||||
httponly=True,
|
||||
secure=True,
|
||||
samesite="lax",
|
||||
path="/",
|
||||
)
|
||||
return response
|
||||
|
||||
|
||||
@router.post("/logout")
|
||||
def logout():
|
||||
response = Response(status_code=status.HTTP_204_NO_CONTENT)
|
||||
response.delete_cookie(key=SESSION_COOKIE, path="/")
|
||||
return response
|
||||
+158
-60
@@ -1,4 +1,7 @@
|
||||
from fastapi import APIRouter, Depends, HTTPException
|
||||
from fastapi import APIRouter, Depends, HTTPException, Query
|
||||
from fastapi.responses import HTMLResponse
|
||||
from html import escape as _html_escape
|
||||
from pydantic import BaseModel, Field
|
||||
from sqlalchemy.orm import Session, joinedload
|
||||
from app.database import get_db
|
||||
from app.models import (
|
||||
@@ -10,6 +13,8 @@ from app.schemas import (
|
||||
MealPlanResponse, MealPlanCreate,
|
||||
MealPlanItemResponse, VoteRequest, VoteResponse
|
||||
)
|
||||
from app.security import require_session
|
||||
from app.services import approval as approval_service
|
||||
from uuid import UUID
|
||||
from typing import List, Optional
|
||||
from datetime import datetime, timedelta
|
||||
@@ -17,7 +22,7 @@ from datetime import datetime, timedelta
|
||||
router = APIRouter()
|
||||
|
||||
|
||||
@router.get("/planned", response_model=Optional[MealPlanResponse])
|
||||
@router.get("", response_model=Optional[MealPlanResponse])
|
||||
def get_planned_meals(db: Session = Depends(get_db)):
|
||||
profile = db.query(FamilyProfile).first()
|
||||
if not profile:
|
||||
@@ -33,7 +38,7 @@ def get_planned_meals(db: Session = Depends(get_db)):
|
||||
return meal_plan
|
||||
|
||||
|
||||
@router.post("/", response_model=MealPlanResponse)
|
||||
@router.post("", response_model=MealPlanResponse, dependencies=[Depends(require_session)])
|
||||
def create_meal_plan(meal_plan_data: MealPlanCreate, db: Session = Depends(get_db)):
|
||||
profile = db.query(FamilyProfile).first()
|
||||
if not profile:
|
||||
@@ -80,7 +85,7 @@ def get_meal_plan(meal_plan_id: UUID, db: Session = Depends(get_db)):
|
||||
return meal_plan
|
||||
|
||||
|
||||
@router.post("/{meal_plan_id}/lock")
|
||||
@router.post("/{meal_plan_id}/lock", dependencies=[Depends(require_session)])
|
||||
def lock_meal_plan(meal_plan_id: UUID, db: Session = Depends(get_db)):
|
||||
meal_plan = db.query(MealPlan).filter(MealPlan.id == meal_plan_id).first()
|
||||
if not meal_plan:
|
||||
@@ -91,76 +96,169 @@ def lock_meal_plan(meal_plan_id: UUID, db: Session = Depends(get_db)):
|
||||
return {"message": "Meal plan locked", "status": meal_plan.status.value}
|
||||
|
||||
|
||||
@router.get("/items/{item_id}/vote/{token}")
|
||||
def get_vote_page(item_id: UUID, token: str, db: Session = Depends(get_db)):
|
||||
approval_token = db.query(ApprovalToken).filter(ApprovalToken.token == token).first()
|
||||
if not approval_token:
|
||||
raise HTTPException(status_code=404, detail="Invalid token")
|
||||
_DAY_NAMES = {
|
||||
1: "Monday", 2: "Tuesday", 3: "Wednesday", 4: "Thursday",
|
||||
5: "Friday", 6: "Saturday", 7: "Sunday",
|
||||
}
|
||||
|
||||
if approval_token.meal_plan_item_id != item_id:
|
||||
|
||||
class VoteSubmission(BaseModel):
|
||||
"""Body for POST /vote/{item_id}?token=...
|
||||
|
||||
Spec body: {"vote": "approve" | "deny"}.
|
||||
"""
|
||||
|
||||
vote: str = Field(..., pattern="^(approve|deny)$")
|
||||
|
||||
|
||||
@router.get("/vote/{item_id}", response_class=HTMLResponse)
|
||||
def get_vote_page(
|
||||
item_id: UUID,
|
||||
token: str = Query(..., description="Per-voter signed token"),
|
||||
db: Session = Depends(get_db),
|
||||
):
|
||||
"""Render the per-voter approval confirmation page.
|
||||
|
||||
Verifies the signed token (no consume) and returns minimal accessible
|
||||
HTML with Approve / Deny buttons that POST to the same URL.
|
||||
"""
|
||||
payload = approval_service.verify_token(token)
|
||||
if str(payload.get("item")) != str(item_id):
|
||||
raise HTTPException(status_code=400, detail="Token not valid for this meal")
|
||||
|
||||
if approval_token.status != ApprovalTokenStatus.ACTIVE:
|
||||
raise HTTPException(status_code=400, detail="Token has already been used or expired")
|
||||
voter = (
|
||||
db.query(FamilyMember)
|
||||
.filter(FamilyMember.id == UUID(str(payload["voter"])))
|
||||
.first()
|
||||
)
|
||||
if not voter:
|
||||
raise HTTPException(status_code=404, detail="Voter not found")
|
||||
|
||||
if approval_token.expires_at < datetime.now():
|
||||
raise HTTPException(status_code=400, detail="Token has expired")
|
||||
item = (
|
||||
db.query(MealPlanItem)
|
||||
.options(joinedload(MealPlanItem.recipe))
|
||||
.filter(MealPlanItem.id == item_id)
|
||||
.first()
|
||||
)
|
||||
if not item:
|
||||
raise HTTPException(status_code=404, detail="Meal plan item not found")
|
||||
|
||||
recipe_name = item.recipe.name if item.recipe else "Unnamed meal"
|
||||
day_name = _DAY_NAMES.get(int(item.day_of_week), str(item.day_of_week))
|
||||
meal_type = item.meal_type.value if item.meal_type else ""
|
||||
|
||||
# Token is included only inside the form action (href), never in the
|
||||
# visible body. POST is performed by JS so we can submit JSON without
|
||||
# leaving the page; non-JS users still get a usable form fallback.
|
||||
safe_voter = _html_escape(voter.name)
|
||||
safe_recipe = _html_escape(recipe_name)
|
||||
safe_day = _html_escape(day_name)
|
||||
safe_meal = _html_escape(meal_type)
|
||||
action_url = f"/api/meals/vote/{item_id}?token={_html_escape(token, quote=True)}"
|
||||
|
||||
html = f"""<!doctype html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<title>Approve meal</title>
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||
<style>
|
||||
body {{ font-family: system-ui, sans-serif; max-width: 36rem; margin: 2rem auto;
|
||||
padding: 0 1rem; color: #111; background: #fff; line-height: 1.5; }}
|
||||
h1 {{ font-size: 1.4rem; }}
|
||||
.meal {{ padding: 1rem; border: 1px solid #444; border-radius: 6px; margin: 1rem 0; }}
|
||||
button {{ font-size: 1rem; padding: .6rem 1.2rem; margin-right: .5rem;
|
||||
border: 2px solid #111; border-radius: 4px; cursor: pointer; }}
|
||||
.approve {{ background: #0a6b2b; color: #fff; }}
|
||||
.deny {{ background: #b00020; color: #fff; }}
|
||||
#status {{ margin-top: 1rem; font-weight: bold; }}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<h1>Hi {safe_voter}, please vote on this meal</h1>
|
||||
<div class="meal">
|
||||
<div><strong>{safe_recipe}</strong></div>
|
||||
<div>{safe_day} · {safe_meal}</div>
|
||||
</div>
|
||||
<form id="voteForm" method="post" action="{action_url}">
|
||||
<button type="submit" name="vote" value="approve" class="approve" aria-label="Approve this meal">Approve</button>
|
||||
<button type="submit" name="vote" value="deny" class="deny" aria-label="Deny this meal">Deny</button>
|
||||
</form>
|
||||
<div id="status" role="status" aria-live="polite"></div>
|
||||
<script>
|
||||
document.getElementById('voteForm').addEventListener('submit', async function(e) {{
|
||||
e.preventDefault();
|
||||
var btn = e.submitter || document.activeElement;
|
||||
var vote = btn && btn.value ? btn.value : 'approve';
|
||||
var resp = await fetch(this.action, {{
|
||||
method: 'POST',
|
||||
headers: {{ 'Content-Type': 'application/json' }},
|
||||
body: JSON.stringify({{ vote: vote }})
|
||||
}});
|
||||
var data = {{}};
|
||||
try {{ data = await resp.json(); }} catch (_) {{}}
|
||||
var s = document.getElementById('status');
|
||||
if (resp.ok) {{
|
||||
s.textContent = 'Recorded: ' + (data.item_status || vote);
|
||||
}} else {{
|
||||
s.textContent = 'Error: ' + (data.detail || resp.status);
|
||||
}}
|
||||
}});
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
"""
|
||||
return HTMLResponse(content=html, status_code=200)
|
||||
|
||||
|
||||
@router.post("/vote/{item_id}")
|
||||
def submit_vote(
|
||||
item_id: UUID,
|
||||
submission: VoteSubmission,
|
||||
token: str = Query(..., description="Per-voter signed token"),
|
||||
db: Session = Depends(get_db),
|
||||
):
|
||||
"""Record a per-voter vote and apply the approval rule.
|
||||
|
||||
- Single-use enforcement lives in `approval_service.consume_token`.
|
||||
- Approval rule: any deny -> item.denied; all-approve -> item.approved;
|
||||
otherwise pending (waiting on remaining voters).
|
||||
"""
|
||||
voter = approval_service.consume_token(db, token, item_id)
|
||||
|
||||
item = db.query(MealPlanItem).filter(MealPlanItem.id == item_id).first()
|
||||
if not item:
|
||||
raise HTTPException(status_code=404, detail="Meal plan item not found")
|
||||
|
||||
return {
|
||||
"item_id": str(item_id),
|
||||
"family_member_id": str(approval_token.family_member_id),
|
||||
"meal_plan_item": item
|
||||
}
|
||||
|
||||
|
||||
@router.post("/items/{item_id}/vote/{token}", response_model=VoteResponse)
|
||||
def submit_vote(item_id: UUID, token: str, vote_data: VoteRequest, db: Session = Depends(get_db)):
|
||||
approval_token = db.query(ApprovalToken).filter(ApprovalToken.token == token).first()
|
||||
if not approval_token:
|
||||
raise HTTPException(status_code=404, detail="Invalid token")
|
||||
|
||||
if approval_token.meal_plan_item_id != item_id:
|
||||
raise HTTPException(status_code=400, detail="Token not valid for this meal")
|
||||
|
||||
if approval_token.status != ApprovalTokenStatus.ACTIVE:
|
||||
raise HTTPException(status_code=400, detail="Token has already been used or expired")
|
||||
|
||||
if approval_token.expires_at < datetime.now():
|
||||
approval_token.status = ApprovalTokenStatus.EXPIRED
|
||||
db.commit()
|
||||
raise HTTPException(status_code=400, detail="Token has expired")
|
||||
|
||||
existing_vote = db.query(MealPlanVote).filter(
|
||||
MealPlanVote.meal_plan_item_id == item_id,
|
||||
MealPlanVote.family_member_id == approval_token.family_member_id
|
||||
).first()
|
||||
|
||||
if existing_vote:
|
||||
raise HTTPException(status_code=400, detail="You have already voted on this meal")
|
||||
|
||||
vote = MealPlanVote(
|
||||
vote_bool = submission.vote == "approve"
|
||||
db.add(MealPlanVote(
|
||||
meal_plan_item_id=item_id,
|
||||
family_member_id=approval_token.family_member_id,
|
||||
vote=vote_data.vote
|
||||
)
|
||||
db.add(vote)
|
||||
family_member_id=voter.id,
|
||||
vote=vote_bool,
|
||||
))
|
||||
db.flush()
|
||||
|
||||
approval_token.status = ApprovalTokenStatus.USED
|
||||
approval_token.used_at = datetime.now()
|
||||
# Approval rule: count electorate (all family members on this profile)
|
||||
# vs votes recorded so far.
|
||||
profile_id = item.meal_plan.family_profile_id
|
||||
electorate_ids = {
|
||||
m.id for m in db.query(FamilyMember)
|
||||
.filter(FamilyMember.family_profile_id == profile_id)
|
||||
.all()
|
||||
}
|
||||
votes = db.query(MealPlanVote).filter(
|
||||
MealPlanVote.meal_plan_item_id == item_id,
|
||||
).all()
|
||||
|
||||
item = db.query(MealPlanItem).filter(MealPlanItem.id == item_id).first()
|
||||
if not vote_data.vote and vote_data.denial_reason:
|
||||
if any(v.vote is False for v in votes):
|
||||
item.approval_status = MealPlanItemStatus.DENIED
|
||||
item.denial_reason = vote_data.denial_reason
|
||||
item.denial_details = vote_data.denial_details
|
||||
elif electorate_ids and {v.family_member_id for v in votes} >= electorate_ids:
|
||||
item.approval_status = MealPlanItemStatus.APPROVED
|
||||
else:
|
||||
item.approval_status = MealPlanItemStatus.PENDING
|
||||
|
||||
db.commit()
|
||||
db.refresh(vote)
|
||||
return vote
|
||||
return {"status": "recorded", "item_status": item.approval_status.value}
|
||||
|
||||
|
||||
@router.get("/items/{item_id}", response_model=MealPlanItemResponse)
|
||||
@@ -173,7 +271,7 @@ def get_meal_item(item_id: UUID, db: Session = Depends(get_db)):
|
||||
return item
|
||||
|
||||
|
||||
@router.post("/items/{item_id}/swap")
|
||||
@router.post("/items/{item_id}/swap", dependencies=[Depends(require_session)])
|
||||
def swap_meal_item(item_id: UUID, new_recipe_id: UUID, db: Session = Depends(get_db)):
|
||||
item = db.query(MealPlanItem).filter(MealPlanItem.id == item_id).first()
|
||||
if not item:
|
||||
|
||||
@@ -3,13 +3,14 @@ from sqlalchemy.orm import Session
|
||||
from app.database import get_db
|
||||
from app.models import HomePantry, Ingredient, FamilyProfile
|
||||
from app.schemas import HomePantryResponse, HomePantryCreate
|
||||
from app.security import require_session
|
||||
from uuid import UUID
|
||||
from typing import List
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
|
||||
@router.get("/", response_model=List[HomePantryResponse])
|
||||
@router.get("", response_model=List[HomePantryResponse])
|
||||
def get_pantry_items(db: Session = Depends(get_db)):
|
||||
profile = db.query(FamilyProfile).first()
|
||||
if not profile:
|
||||
@@ -21,7 +22,7 @@ def get_pantry_items(db: Session = Depends(get_db)):
|
||||
return items
|
||||
|
||||
|
||||
@router.post("/", response_model=HomePantryResponse)
|
||||
@router.post("", response_model=HomePantryResponse, dependencies=[Depends(require_session)])
|
||||
def add_pantry_item(item: HomePantryCreate, db: Session = Depends(get_db)):
|
||||
profile = db.query(FamilyProfile).first()
|
||||
if not profile:
|
||||
@@ -54,7 +55,7 @@ def add_pantry_item(item: HomePantryCreate, db: Session = Depends(get_db)):
|
||||
return db_item
|
||||
|
||||
|
||||
@router.delete("/{item_id}")
|
||||
@router.delete("/{item_id}", dependencies=[Depends(require_session)])
|
||||
def remove_pantry_item(item_id: UUID, db: Session = Depends(get_db)):
|
||||
item = db.query(HomePantry).filter(HomePantry.id == item_id).first()
|
||||
if not item:
|
||||
@@ -65,7 +66,7 @@ def remove_pantry_item(item_id: UUID, db: Session = Depends(get_db)):
|
||||
return {"message": "Pantry item removed"}
|
||||
|
||||
|
||||
@router.put("/{item_id}", response_model=HomePantryResponse)
|
||||
@router.put("/{item_id}", response_model=HomePantryResponse, dependencies=[Depends(require_session)])
|
||||
def update_pantry_item(item_id: UUID, update: HomePantryCreate, db: Session = Depends(get_db)):
|
||||
item = db.query(HomePantry).filter(HomePantry.id == item_id).first()
|
||||
if not item:
|
||||
|
||||
@@ -6,13 +6,14 @@ from app.schemas import (
|
||||
FamilyProfileResponse, FamilyProfileUpdate, FamilyProfileCreate,
|
||||
FamilyMemberResponse, FamilyMemberCreate
|
||||
)
|
||||
from app.security import require_session
|
||||
from uuid import UUID
|
||||
from typing import List
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
|
||||
@router.get("/", response_model=FamilyProfileResponse)
|
||||
@router.get("", response_model=FamilyProfileResponse)
|
||||
def get_profile(db: Session = Depends(get_db)):
|
||||
profile = db.query(FamilyProfile).first()
|
||||
if not profile:
|
||||
@@ -20,7 +21,7 @@ def get_profile(db: Session = Depends(get_db)):
|
||||
return profile
|
||||
|
||||
|
||||
@router.put("/", response_model=FamilyProfileResponse)
|
||||
@router.put("", response_model=FamilyProfileResponse, dependencies=[Depends(require_session)])
|
||||
def update_profile(update: FamilyProfileUpdate, db: Session = Depends(get_db)):
|
||||
profile = db.query(FamilyProfile).first()
|
||||
if not profile:
|
||||
@@ -43,7 +44,7 @@ def get_members(db: Session = Depends(get_db)):
|
||||
return profile.members
|
||||
|
||||
|
||||
@router.post("/members", response_model=FamilyMemberResponse)
|
||||
@router.post("/members", response_model=FamilyMemberResponse, dependencies=[Depends(require_session)])
|
||||
def add_member(member: FamilyMemberCreate, db: Session = Depends(get_db)):
|
||||
profile = db.query(FamilyProfile).first()
|
||||
if not profile:
|
||||
@@ -69,7 +70,7 @@ def add_member(member: FamilyMemberCreate, db: Session = Depends(get_db)):
|
||||
return db_member
|
||||
|
||||
|
||||
@router.delete("/members/{member_id}")
|
||||
@router.delete("/members/{member_id}", dependencies=[Depends(require_session)])
|
||||
def delete_member(member_id: UUID, db: Session = Depends(get_db)):
|
||||
member = db.query(FamilyMember).filter(FamilyMember.id == member_id).first()
|
||||
if not member:
|
||||
|
||||
+35
-34
@@ -3,13 +3,14 @@ from sqlalchemy.orm import Session, joinedload
|
||||
from app.database import get_db
|
||||
from app.models import Recipe, FamilyProfile
|
||||
from app.schemas import RecipeResponse, RecipeCreate, IngredientCreate, IngredientResponse
|
||||
from app.security import require_session
|
||||
from uuid import UUID
|
||||
from typing import List, Optional
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
|
||||
@router.get("/", response_model=List[RecipeResponse])
|
||||
@router.get("", response_model=List[RecipeResponse])
|
||||
def get_recipes(
|
||||
skip: int = 0,
|
||||
limit: int = 50,
|
||||
@@ -31,6 +32,37 @@ def get_recipes(
|
||||
return recipes
|
||||
|
||||
|
||||
@router.get("/ingredients", response_model=List[IngredientResponse])
|
||||
def list_ingredients(skip: int = 0, limit: int = 100, db: Session = Depends(get_db)):
|
||||
from app.models import Ingredient
|
||||
ingredients = db.query(Ingredient).offset(skip).limit(limit).all()
|
||||
return ingredients
|
||||
|
||||
|
||||
@router.post("/ingredients", response_model=IngredientResponse, dependencies=[Depends(require_session)])
|
||||
def create_ingredient(ingredient: IngredientCreate, db: Session = Depends(get_db)):
|
||||
from app.models import Ingredient
|
||||
|
||||
existing = db.query(Ingredient).filter(Ingredient.name_lower == ingredient.name_lower).first()
|
||||
if existing:
|
||||
raise HTTPException(status_code=400, detail="Ingredient with this name already exists")
|
||||
|
||||
db_ingredient = Ingredient(
|
||||
name=ingredient.name,
|
||||
name_lower=ingredient.name_lower,
|
||||
plural_name=ingredient.plural_name,
|
||||
aisle=ingredient.aisle,
|
||||
typical_price=ingredient.typical_price,
|
||||
unit=ingredient.unit,
|
||||
season_months=ingredient.season_months
|
||||
)
|
||||
|
||||
db.add(db_ingredient)
|
||||
db.commit()
|
||||
db.refresh(db_ingredient)
|
||||
return db_ingredient
|
||||
|
||||
|
||||
@router.get("/{recipe_id}", response_model=RecipeResponse)
|
||||
def get_recipe(recipe_id: UUID, db: Session = Depends(get_db)):
|
||||
recipe = db.query(Recipe).filter(Recipe.id == recipe_id).first()
|
||||
@@ -39,7 +71,7 @@ def get_recipe(recipe_id: UUID, db: Session = Depends(get_db)):
|
||||
return recipe
|
||||
|
||||
|
||||
@router.post("/", response_model=RecipeResponse)
|
||||
@router.post("", response_model=RecipeResponse, dependencies=[Depends(require_session)])
|
||||
def create_recipe(recipe: RecipeCreate, db: Session = Depends(get_db)):
|
||||
profile = db.query(FamilyProfile).first()
|
||||
|
||||
@@ -69,7 +101,7 @@ def create_recipe(recipe: RecipeCreate, db: Session = Depends(get_db)):
|
||||
return db_recipe
|
||||
|
||||
|
||||
@router.delete("/{recipe_id}")
|
||||
@router.delete("/{recipe_id}", dependencies=[Depends(require_session)])
|
||||
def delete_recipe(recipe_id: UUID, db: Session = Depends(get_db)):
|
||||
recipe = db.query(Recipe).filter(Recipe.id == recipe_id).first()
|
||||
if not recipe:
|
||||
@@ -78,34 +110,3 @@ def delete_recipe(recipe_id: UUID, db: Session = Depends(get_db)):
|
||||
db.delete(recipe)
|
||||
db.commit()
|
||||
return {"message": "Recipe deleted"}
|
||||
|
||||
|
||||
@router.get("/ingredients/list", response_model=List[IngredientResponse])
|
||||
def list_ingredients(skip: int = 0, limit: int = 100, db: Session = Depends(get_db)):
|
||||
from app.models import Ingredient
|
||||
ingredients = db.query(Ingredient).offset(skip).limit(limit).all()
|
||||
return ingredients
|
||||
|
||||
|
||||
@router.post("/ingredients", response_model=IngredientResponse)
|
||||
def create_ingredient(ingredient: IngredientCreate, db: Session = Depends(get_db)):
|
||||
from app.models import Ingredient
|
||||
|
||||
existing = db.query(Ingredient).filter(Ingredient.name_lower == ingredient.name_lower).first()
|
||||
if existing:
|
||||
raise HTTPException(status_code=400, detail="Ingredient with this name already exists")
|
||||
|
||||
db_ingredient = Ingredient(
|
||||
name=ingredient.name,
|
||||
name_lower=ingredient.name_lower,
|
||||
plural_name=ingredient.plural_name,
|
||||
aisle=ingredient.aisle,
|
||||
typical_price=ingredient.typical_price,
|
||||
unit=ingredient.unit,
|
||||
season_months=ingredient.season_months
|
||||
)
|
||||
|
||||
db.add(db_ingredient)
|
||||
db.commit()
|
||||
db.refresh(db_ingredient)
|
||||
return db_ingredient
|
||||
@@ -14,7 +14,7 @@ from collections import defaultdict
|
||||
router = APIRouter()
|
||||
|
||||
|
||||
@router.get("/", response_model=ShoppingListResponse)
|
||||
@router.get("", response_model=ShoppingListResponse)
|
||||
def get_shopping_list(db: Session = Depends(get_db)):
|
||||
profile = db.query(FamilyProfile).first()
|
||||
if not profile:
|
||||
@@ -56,7 +56,7 @@ def get_shopping_list(db: Session = Depends(get_db)):
|
||||
|
||||
if all_ingredient_ids:
|
||||
ingredients = db.query(Ingredient).filter(
|
||||
Ingredient.id.in_ all_ingredient_ids
|
||||
Ingredient.id.in_(all_ingredient_ids)
|
||||
).all()
|
||||
ingredient_map = {i.id: i for i in ingredients}
|
||||
|
||||
|
||||
+20
-1
@@ -1,17 +1,30 @@
|
||||
from pydantic_settings import BaseSettings
|
||||
from pydantic import model_validator
|
||||
from typing import Optional
|
||||
|
||||
|
||||
class Settings(BaseSettings):
|
||||
DATABASE_URL: str
|
||||
# Required — fail fast at import time if unset.
|
||||
DATABASE_URL: str = ""
|
||||
|
||||
SENDGRID_API_KEY: Optional[str] = None
|
||||
EMAIL_BACKEND: str = "console"
|
||||
LUCKY_CA_URL: str = "https://www.luckyncal.com"
|
||||
# R3-0: Swiftly product API (replaces Playwright path).
|
||||
SWIFTLY_BEARER_TOKEN: str = ""
|
||||
LUCKY_STORE_ID: str = "757"
|
||||
SWIFTLY_API_BASE: str = "https://prod.swiftlyapi.net"
|
||||
SWIFTLY_CATEGORIES_URL: str = "https://luckysupermarkets.com/categories"
|
||||
AI_IMAGE_ENABLED: bool = False
|
||||
AI_IMAGE_PROVIDER: Optional[str] = None
|
||||
AI_IMAGE_API_KEY: Optional[str] = None
|
||||
LOG_LEVEL: str = "INFO"
|
||||
SECRET_KEY: str = "dev-secret-key"
|
||||
|
||||
# Auth (R1-B+D)
|
||||
ADMIN_TOKEN: str = ""
|
||||
SESSION_PASSWORD: str = ""
|
||||
|
||||
FAMILY_EMAIL_1: Optional[str] = None
|
||||
FAMILY_EMAIL_2: Optional[str] = None
|
||||
RECIPES_EMAIL: Optional[str] = None
|
||||
@@ -19,5 +32,11 @@ class Settings(BaseSettings):
|
||||
class Config:
|
||||
env_file = ".env"
|
||||
|
||||
@model_validator(mode="after")
|
||||
def _require_database_url(self) -> "Settings":
|
||||
if not self.DATABASE_URL:
|
||||
raise RuntimeError("DATABASE_URL is required")
|
||||
return self
|
||||
|
||||
|
||||
settings = Settings()
|
||||
|
||||
+2
-1
@@ -29,8 +29,9 @@ def health_check_db(db: Session = Depends(get_db)):
|
||||
return {"status": "error", "database": "disconnected", "error": str(e)}
|
||||
|
||||
|
||||
from app.api import profile, recipes, meals, shopping_list, pantry, admin
|
||||
from app.api import profile, recipes, meals, shopping_list, pantry, admin, auth
|
||||
|
||||
app.include_router(auth.router, prefix="/api/auth", tags=["auth"])
|
||||
app.include_router(profile.router, prefix="/api/profile", tags=["profile"])
|
||||
app.include_router(recipes.router, prefix="/api/recipes", tags=["recipes"])
|
||||
app.include_router(meals.router, prefix="/api/meals", tags=["meals"])
|
||||
|
||||
@@ -114,7 +114,7 @@ class FamilyMember(Base):
|
||||
family_profile_id = Column(UUID(as_uuid=True), ForeignKey("family_profile.id", ondelete="CASCADE"))
|
||||
name = Column(String(100), nullable=False)
|
||||
email = Column(String(300))
|
||||
role = Column(SQLEnum(FamilyMemberRole, name="family_member_role_enum", create_type=False), nullable=False)
|
||||
role = Column(SQLEnum(FamilyMemberRole, name="family_member_role_enum", create_type=False, values_callable=lambda obj: [e.value for e in obj]), nullable=False)
|
||||
likes_mushrooms = Column(Boolean, default=False)
|
||||
created_at = Column(DateTime(timezone=True), server_default=func.now())
|
||||
updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now())
|
||||
@@ -185,7 +185,7 @@ class MealPlan(Base):
|
||||
id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
|
||||
family_profile_id = Column(UUID(as_uuid=True), ForeignKey("family_profile.id"))
|
||||
week_start_date = Column(Date, nullable=False)
|
||||
status = Column(SQLEnum(MealPlanStatus, name="meal_plan_status_enum", create_type=False), nullable=False, default=MealPlanStatus.DRAFT)
|
||||
status = Column(SQLEnum(MealPlanStatus, name="meal_plan_status_enum", create_type=False, values_callable=lambda obj: [e.value for e in obj]), nullable=False, default=MealPlanStatus.DRAFT)
|
||||
approval_deadline = Column(DateTime(timezone=True))
|
||||
total_estimated_cost = Column(Numeric(10, 2))
|
||||
notes = Column(Text)
|
||||
@@ -198,7 +198,6 @@ class MealPlan(Base):
|
||||
|
||||
family_profile = relationship("FamilyProfile", back_populates="meal_plans")
|
||||
items = relationship("MealPlanItem", back_populates="meal_plan", cascade="all, delete-orphan")
|
||||
votes = relationship("MealPlanVote", back_populates="meal_plan", cascade="all, delete-orphan")
|
||||
|
||||
|
||||
class MealPlanItem(Base):
|
||||
@@ -208,9 +207,9 @@ class MealPlanItem(Base):
|
||||
meal_plan_id = Column(UUID(as_uuid=True), ForeignKey("meal_plan.id", ondelete="CASCADE"))
|
||||
recipe_id = Column(UUID(as_uuid=True), ForeignKey("recipe.id"))
|
||||
day_of_week = Column(Integer, nullable=False)
|
||||
meal_type = Column(SQLEnum(MealType, name="meal_type_enum", create_type=False), nullable=False)
|
||||
approval_status = Column(SQLEnum(MealPlanItemStatus, name="meal_plan_item_status_enum", create_type=False), default=MealPlanItemStatus.PENDING)
|
||||
denial_reason = Column(SQLEnum(DenialReason, name="denial_reason_enum", create_type=False))
|
||||
meal_type = Column(SQLEnum(MealType, name="meal_type_enum", create_type=False, values_callable=lambda obj: [e.value for e in obj]), nullable=False)
|
||||
approval_status = Column(SQLEnum(MealPlanItemStatus, name="meal_plan_item_status_enum", create_type=False, values_callable=lambda obj: [e.value for e in obj]), default=MealPlanItemStatus.PENDING)
|
||||
denial_reason = Column(SQLEnum(DenialReason, name="denial_reason_enum", create_type=False, values_callable=lambda obj: [e.value for e in obj]))
|
||||
denial_details = Column(Text)
|
||||
estimated_cost = Column(Numeric(10, 2))
|
||||
used_pantry_items = Column(ARRAY(UUID(as_uuid=True)))
|
||||
@@ -252,7 +251,7 @@ class ApprovalToken(Base):
|
||||
meal_plan_item_id = Column(UUID(as_uuid=True), ForeignKey("meal_plan_item.id", ondelete="CASCADE"))
|
||||
family_member_id = Column(UUID(as_uuid=True), ForeignKey("family_member.id", ondelete="CASCADE"))
|
||||
token = Column(String(64), nullable=False, unique=True)
|
||||
status = Column(SQLEnum(ApprovalTokenStatus, name="approval_token_status_enum", create_type=False), default=ApprovalTokenStatus.ACTIVE)
|
||||
status = Column(SQLEnum(ApprovalTokenStatus, name="approval_token_status_enum", create_type=False, values_callable=lambda obj: [e.value for e in obj]), default=ApprovalTokenStatus.ACTIVE)
|
||||
expires_at = Column(DateTime(timezone=True), nullable=False)
|
||||
used_at = Column(DateTime(timezone=True))
|
||||
created_at = Column(DateTime(timezone=True), server_default=func.now())
|
||||
@@ -294,7 +293,7 @@ class Feedback(Base):
|
||||
meal_plan_item_id = Column(UUID(as_uuid=True), ForeignKey("meal_plan_item.id", ondelete="CASCADE"))
|
||||
rating = Column(Integer)
|
||||
never_suggest = Column(Boolean, default=False)
|
||||
denial_reason = Column(SQLEnum(DenialReason, name="denial_reason_enum", create_type=False))
|
||||
denial_reason = Column(SQLEnum(DenialReason, name="denial_reason_enum", create_type=False, values_callable=lambda obj: [e.value for e in obj]))
|
||||
feedback_text = Column(Text)
|
||||
created_at = Column(DateTime(timezone=True), server_default=func.now())
|
||||
|
||||
@@ -314,7 +313,7 @@ class NeverSuggest(Base):
|
||||
family_profile_id = Column(UUID(as_uuid=True), ForeignKey("family_profile.id", ondelete="CASCADE"))
|
||||
ingredient_id = Column(UUID(as_uuid=True), ForeignKey("ingredient.id", ondelete="CASCADE"))
|
||||
recipe_id = Column(UUID(as_uuid=True), ForeignKey("recipe.id", ondelete="CASCADE"))
|
||||
reason = Column(SQLEnum(NeverSuggestReason, name="never_suggest_reason_enum", create_type=False))
|
||||
reason = Column(SQLEnum(NeverSuggestReason, name="never_suggest_reason_enum", create_type=False, values_callable=lambda obj: [e.value for e in obj]))
|
||||
notes = Column(Text)
|
||||
created_at = Column(DateTime(timezone=True), server_default=func.now())
|
||||
|
||||
@@ -335,12 +334,16 @@ class GroceryItem(Base):
|
||||
aisle = Column(String(100))
|
||||
image_url = Column(Text)
|
||||
product_url = Column(Text)
|
||||
description = Column(Text, nullable=True)
|
||||
is_on_sale = Column(Boolean, default=False)
|
||||
sale_start_date = Column(Date)
|
||||
sale_end_date = Column(Date)
|
||||
in_season = Column(Boolean, default=False)
|
||||
scraped_at = Column(DateTime(timezone=True), server_default=func.now())
|
||||
scraped_url = Column(Text)
|
||||
# R3-0: Swiftly API idempotency key. (source, external_id) → upsert.
|
||||
external_id = Column(String(100), nullable=True, index=True)
|
||||
source = Column(String(50), nullable=True)
|
||||
|
||||
ingredient = relationship("Ingredient", back_populates="grocery_item_links")
|
||||
|
||||
@@ -351,7 +354,7 @@ class ScrapeLog(Base):
|
||||
id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
|
||||
source = Column(String(50), nullable=False)
|
||||
scrape_type = Column(String(50), nullable=False)
|
||||
status = Column(SQLEnum(ScrapeStatus, name="scrape_status_enum", create_type=False), nullable=False)
|
||||
status = Column(SQLEnum(ScrapeStatus, name="scrape_status_enum", create_type=False, values_callable=lambda obj: [e.value for e in obj]), nullable=False)
|
||||
items_scraped = Column(Integer, default=0)
|
||||
error_message = Column(Text)
|
||||
started_at = Column(DateTime(timezone=True), server_default=func.now())
|
||||
@@ -369,7 +372,7 @@ class EmailLog(Base):
|
||||
meal_plan_id = Column(UUID(as_uuid=True), ForeignKey("meal_plan.id"))
|
||||
meal_plan_item_id = Column(UUID(as_uuid=True), ForeignKey("meal_plan_item.id"))
|
||||
sendgrid_message_id = Column(String(100))
|
||||
status = Column(SQLEnum(EmailStatus, name="email_status_enum", create_type=False), nullable=False)
|
||||
status = Column(SQLEnum(EmailStatus, name="email_status_enum", create_type=False, values_callable=lambda obj: [e.value for e in obj]), nullable=False)
|
||||
error_message = Column(Text)
|
||||
created_at = Column(DateTime(timezone=True), server_default=func.now())
|
||||
delivered_at = Column(DateTime(timezone=True))
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
from .base import BaseScraper
|
||||
from .lucky_ca_scraper import LuckyCaliforniaScraper
|
||||
from .lucky_ca_scraper import LuckyCaliforniaScraper, SwiftlyAuthError
|
||||
|
||||
__all__ = ["BaseScraper", "LuckyCaliforniaScraper"]
|
||||
__all__ = ["BaseScraper", "LuckyCaliforniaScraper", "SwiftlyAuthError"]
|
||||
@@ -1,180 +1,325 @@
|
||||
"""Lucky California / Swiftly product API client.
|
||||
|
||||
Replaces the prior Playwright + HTML coupon-card path (R2-A) with the
|
||||
underlying JSON API the website itself calls. The API exposes the full
|
||||
inventory per category — no rendering, no Chromium, and far more items
|
||||
than the visible coupon strip (256 in `Product/meat_seafood` vs the 11
|
||||
the HTML parser saw).
|
||||
|
||||
Two endpoints (no public docs; reverse-engineered from the network panel
|
||||
on luckysupermarkets.com — see ``.agent/context.md`` "Swiftly API"):
|
||||
|
||||
GET https://luckysupermarkets.com/categories
|
||||
→ HTML page; anchors with ``class="swiftlyCouponCategory"`` carry
|
||||
``href="/categories/Product%2F<slug>"``.
|
||||
|
||||
GET https://prod.swiftlyapi.net/search/api/v1/products/categories
|
||||
?cat=<slug>&store=<store_id>&limit=10000
|
||||
Authorization: Bearer <SWIFTLY_BEARER_TOKEN>
|
||||
→ ``{"products": {"info": {...}, "items": [...], "facets": [...]}}``
|
||||
|
||||
The bearer token expires roughly hourly. On 401 we raise
|
||||
``SwiftlyAuthError`` so the background runner records the error_message
|
||||
that asks the admin to refresh ``SWIFTLY_BEARER_TOKEN`` and retry.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import re
|
||||
from typing import Dict, Any, List, Optional
|
||||
import time
|
||||
import urllib.parse
|
||||
from datetime import datetime
|
||||
from urllib.parse import urljoin
|
||||
from decimal import Decimal, InvalidOperation
|
||||
from typing import Any, Dict, Iterator, List, Optional, Tuple
|
||||
|
||||
import requests
|
||||
from bs4 import BeautifulSoup
|
||||
from .base import SeleniumScraper
|
||||
|
||||
from app.config import settings
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class LuckyCaliforniaScraper(SeleniumScraper):
|
||||
def __init__(self, base_url: str = "https://luckysupermarkets.com"):
|
||||
super().__init__(base_url=base_url, rate_limit_seconds=3.0)
|
||||
self.ingredients_cache = {}
|
||||
class SwiftlyAuthError(Exception):
|
||||
"""Raised when the Swiftly API returns 401.
|
||||
|
||||
The exception message is surfaced verbatim to the ScrapeLog row by
|
||||
``_run_scrape_in_background``; keep it actionable.
|
||||
"""
|
||||
|
||||
|
||||
_USER_AGENT = (
|
||||
"MealPlannerBot/1.0 (+https://mealplanner.local; contact peter@research.bike)"
|
||||
)
|
||||
|
||||
_AUTH_ERROR_MESSAGE = (
|
||||
"SWIFTLY_BEARER_TOKEN expired — request a fresh token from the user "
|
||||
"(capture from luckysupermarkets.com network tab on a /search/api/v1 request)"
|
||||
)
|
||||
|
||||
|
||||
class LuckyCaliforniaScraper:
|
||||
"""Swiftly product-API client for the Lucky California banner.
|
||||
|
||||
The class name is preserved (``LuckyCaliforniaScraper``) so existing
|
||||
import sites in ``app.scraper.__init__`` and
|
||||
``ScraperService._do_scrape`` keep working. Internally it is no
|
||||
longer a ``BaseScraper`` subclass: that class wires Playwright +
|
||||
``_get`` retry-with-swallow, neither of which we want here. The
|
||||
client uses two ``requests.Session`` objects so the bearer header
|
||||
is scoped strictly to the API host (the public categories page
|
||||
is unauthenticated).
|
||||
"""
|
||||
|
||||
SOURCE = "lucky_california"
|
||||
SLUG_HREF_RE = re.compile(
|
||||
r'<a\b(?=[^>]*\bclass="swiftlyCouponCategory")(?=[^>]*\bhref="([^"]+)")[^>]*>',
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
bearer_token: Optional[str] = None,
|
||||
store_id: Optional[str] = None,
|
||||
api_base: Optional[str] = None,
|
||||
categories_url: Optional[str] = None,
|
||||
rate_limit_seconds: float = 0.75,
|
||||
timeout: int = 60,
|
||||
) -> None:
|
||||
self.bearer_token = bearer_token or settings.SWIFTLY_BEARER_TOKEN
|
||||
self.store_id = store_id or settings.LUCKY_STORE_ID
|
||||
self.api_base = (api_base or settings.SWIFTLY_API_BASE).rstrip("/")
|
||||
self.categories_url = categories_url or settings.SWIFTLY_CATEGORIES_URL
|
||||
self.rate_limit_seconds = rate_limit_seconds
|
||||
self.timeout = timeout
|
||||
self._last_request = 0.0
|
||||
|
||||
self.public_session = requests.Session()
|
||||
self.public_session.headers.update(
|
||||
{
|
||||
"User-Agent": _USER_AGENT,
|
||||
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
|
||||
"Accept-Language": "en-US,en;q=0.5",
|
||||
}
|
||||
)
|
||||
|
||||
self.api_session = requests.Session()
|
||||
self.api_session.headers.update(
|
||||
{
|
||||
"User-Agent": _USER_AGENT,
|
||||
"Accept": "application/json",
|
||||
}
|
||||
)
|
||||
# base_url retained for code paths that still introspect it.
|
||||
self.base_url = "https://luckysupermarkets.com"
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Lifecycle (no-op; preserves cleanup() contract from BaseScraper).
|
||||
# ------------------------------------------------------------------
|
||||
def cleanup(self) -> None:
|
||||
try:
|
||||
self.public_session.close()
|
||||
finally:
|
||||
self.api_session.close()
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Public scrape entrypoints
|
||||
# ------------------------------------------------------------------
|
||||
def scrape(self) -> Dict[str, Any]:
|
||||
logger.info("Starting Lucky California scrape")
|
||||
result = {
|
||||
"source": "lucky_california",
|
||||
"""Synchronous top-level scrape.
|
||||
|
||||
Kept for the legacy ``ScraperService.run_scrape`` path used by
|
||||
tests. Walks every category and returns ``{"items": [...], ...}``
|
||||
with mapped product dicts ready for ``_save_grocery_item``.
|
||||
"""
|
||||
started = datetime.now().isoformat()
|
||||
items: List[Dict[str, Any]] = list(self.fetch_all())
|
||||
return {
|
||||
"source": self.SOURCE,
|
||||
"scrape_type": "weekly_ad",
|
||||
"started_at": datetime.now().isoformat(),
|
||||
"items_scraped": 0,
|
||||
"items": []
|
||||
"started_at": started,
|
||||
"completed_at": datetime.now().isoformat(),
|
||||
"items_scraped": len(items),
|
||||
"items": items,
|
||||
"status": "success",
|
||||
}
|
||||
|
||||
try:
|
||||
featured_items = self.scrape_featured_coupons()
|
||||
result["items"].extend(featured_items)
|
||||
result["items_scraped"] = len(featured_items)
|
||||
|
||||
result["completed_at"] = datetime.now().isoformat()
|
||||
result["status"] = "success"
|
||||
logger.info(f"Lucky California scrape complete: {result['items_scraped']} items")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Lucky California scrape failed: {e}")
|
||||
result["status"] = "failed"
|
||||
result["error_message"] = str(e)
|
||||
result["completed_at"] = datetime.now().isoformat()
|
||||
|
||||
return result
|
||||
|
||||
def scrape_featured_coupons(self) -> List[Dict[str, Any]]:
|
||||
url = f"{self.base_url}/coupons/Coupon%2Flu-featured-in-ad"
|
||||
logger.info(f"Scraping featured coupons from {url}")
|
||||
|
||||
page = self.get_browser_page(url)
|
||||
content = page.content()
|
||||
page.close()
|
||||
|
||||
soup = BeautifulSoup(content, "html.parser")
|
||||
items = []
|
||||
|
||||
coupon_items = soup.find_all("div", class_=re.compile(r"coupon|item|product", re.I))
|
||||
if not coupon_items:
|
||||
headline = soup.find("h1")
|
||||
if headline:
|
||||
logger.info(f"Page loaded, headline: {headline.get_text().strip()}")
|
||||
|
||||
titles = soup.find_all(["h2", "h3", "a"], string=re.compile(r"\$[\d\.]+"))
|
||||
for title_elem in titles[:20]:
|
||||
def fetch_all(self) -> Iterator[Dict[str, Any]]:
|
||||
"""Yield mapped product dicts across every discovered category."""
|
||||
slugs = self.discover_categories()
|
||||
logger.info("Swiftly: discovered %d categories", len(slugs))
|
||||
for slug in slugs:
|
||||
try:
|
||||
item = self._parse_coupon_item(title_elem)
|
||||
if item:
|
||||
items.append(item)
|
||||
except Exception as e:
|
||||
logger.debug(f"Failed to parse item: {e}")
|
||||
products = self.fetch_category(slug)
|
||||
except SwiftlyAuthError:
|
||||
# Hard fail — token must be refreshed before any further work.
|
||||
raise
|
||||
except requests.RequestException as exc:
|
||||
logger.warning("Swiftly: skipping %s after error: %s", slug, exc)
|
||||
continue
|
||||
aisle = self._aisle_from_slug(slug)
|
||||
for product in products:
|
||||
mapped = self.map_product(product, aisle=aisle, source_slug=slug)
|
||||
if mapped is not None:
|
||||
yield mapped
|
||||
|
||||
logger.info(f"Found {len(items)} coupon items")
|
||||
# ------------------------------------------------------------------
|
||||
# Category discovery
|
||||
# ------------------------------------------------------------------
|
||||
def discover_categories(self) -> List[str]:
|
||||
"""Fetch the categories page and return the list of API slugs.
|
||||
|
||||
The slugs look like ``Product/meat_seafood``. Order is preserved
|
||||
from the HTML (which is the order shown to the user).
|
||||
"""
|
||||
self._rate_limit()
|
||||
resp = self.public_session.get(self.categories_url, timeout=self.timeout)
|
||||
resp.raise_for_status()
|
||||
return self.parse_categories_html(resp.text)
|
||||
|
||||
@classmethod
|
||||
def parse_categories_html(cls, html: str) -> List[str]:
|
||||
"""Pure parser used by tests against a saved fixture."""
|
||||
slugs: List[str] = []
|
||||
seen: set[str] = set()
|
||||
for href in cls.SLUG_HREF_RE.findall(html):
|
||||
m = re.match(r"^/categories/(.+)$", href)
|
||||
if not m:
|
||||
continue
|
||||
slug = urllib.parse.unquote(m.group(1))
|
||||
if slug not in seen:
|
||||
seen.add(slug)
|
||||
slugs.append(slug)
|
||||
return slugs
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Per-category fetch
|
||||
# ------------------------------------------------------------------
|
||||
def fetch_category(self, slug: str) -> List[Dict[str, Any]]:
|
||||
"""Fetch every product in a category. Raises ``SwiftlyAuthError`` on 401."""
|
||||
if not self.bearer_token:
|
||||
raise SwiftlyAuthError(_AUTH_ERROR_MESSAGE)
|
||||
|
||||
self._rate_limit()
|
||||
url = f"{self.api_base}/search/api/v1/products/categories"
|
||||
params = {"cat": slug, "store": self.store_id, "limit": 10000}
|
||||
headers = {"Authorization": f"Bearer {self.bearer_token}"}
|
||||
resp = self.api_session.get(
|
||||
url, params=params, headers=headers, timeout=self.timeout
|
||||
)
|
||||
if resp.status_code == 401:
|
||||
raise SwiftlyAuthError(_AUTH_ERROR_MESSAGE)
|
||||
resp.raise_for_status()
|
||||
payload = resp.json()
|
||||
return self.parse_category_response(payload)
|
||||
|
||||
@staticmethod
|
||||
def parse_category_response(payload: Dict[str, Any]) -> List[Dict[str, Any]]:
|
||||
"""Pure parser used by tests against a saved fixture."""
|
||||
if not isinstance(payload, dict):
|
||||
return []
|
||||
products = payload.get("products") or {}
|
||||
items = products.get("items")
|
||||
if not isinstance(items, list):
|
||||
return []
|
||||
return items
|
||||
|
||||
def _parse_coupon_item(self, element) -> Optional[Dict[str, Any]]:
|
||||
text = element.get_text().strip()
|
||||
# ------------------------------------------------------------------
|
||||
# Field mapping
|
||||
# ------------------------------------------------------------------
|
||||
@classmethod
|
||||
def map_product(
|
||||
cls,
|
||||
product: Dict[str, Any],
|
||||
*,
|
||||
aisle: Optional[str] = None,
|
||||
source_slug: Optional[str] = None,
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
"""Convert one Swiftly product dict to a grocery_item-ready dict.
|
||||
|
||||
price_match = re.search(r'\$[\d,]+\.?\d*', text)
|
||||
if not price_match:
|
||||
Returns ``None`` for products with no parseable price or no name —
|
||||
those are usually placeholder/unavailable rows.
|
||||
"""
|
||||
external_id = product.get("id")
|
||||
name = (product.get("name") or "").strip()
|
||||
if not name:
|
||||
return None
|
||||
|
||||
price_str = price_match.group().replace("$", "").replace(",", "")
|
||||
try:
|
||||
price = float(price_str)
|
||||
except ValueError:
|
||||
price_block = (product.get("price") or {}).get("ok") or {}
|
||||
reg_price, reg_unit = cls._parse_price(price_block.get("regPriceText"))
|
||||
promo = price_block.get("promoArea") or {}
|
||||
sale_price, sale_unit = cls._parse_price(promo.get("promoText"))
|
||||
|
||||
if reg_price is None and sale_price is None:
|
||||
# No usable price; skip rather than persist garbage.
|
||||
return None
|
||||
|
||||
name_elem = element.find_parent("a") or element.find_parent("div")
|
||||
name = text.split("$")[0].strip() if "$" in text else text
|
||||
name = re.sub(r'\s+', " ", name).strip()[:200]
|
||||
unit = sale_unit or reg_unit
|
||||
is_on_sale = sale_price is not None and reg_price is not None and sale_price < reg_price
|
||||
# current_price = "what the customer pays today" → sale_price when on sale.
|
||||
current_price = sale_price if is_on_sale else reg_price
|
||||
|
||||
if not name or len(name) < 3:
|
||||
return None
|
||||
image_url: Optional[str] = None
|
||||
primary_image = product.get("primaryImage")
|
||||
if isinstance(primary_image, dict):
|
||||
image_url = primary_image.get("url")
|
||||
|
||||
image_url = None
|
||||
img_elem = element.find_parent().find("img") if element.find_parent() else None
|
||||
if img_elem and img_elem.get("src"):
|
||||
image_url = img_elem["src"]
|
||||
brand = product.get("brand")
|
||||
description = product.get("description")
|
||||
|
||||
product_url = None
|
||||
link_elem = element.find_parent("a") if element.find_parent() else element.find("a")
|
||||
if link_elem and link_elem.get("href"):
|
||||
product_url = urljoin(self.base_url, link_elem["href"])
|
||||
|
||||
item = {
|
||||
"name": name,
|
||||
"current_price": price,
|
||||
return {
|
||||
"external_id": str(external_id) if external_id is not None else None,
|
||||
"source": cls.SOURCE,
|
||||
"name": name[:300],
|
||||
"brand": (brand or None),
|
||||
"description": description,
|
||||
"current_price": current_price,
|
||||
"regular_price": reg_price,
|
||||
"sale_price": sale_price,
|
||||
"is_on_sale": bool(is_on_sale),
|
||||
"unit": unit,
|
||||
"aisle": aisle,
|
||||
"image_url": image_url,
|
||||
"product_url": product_url,
|
||||
"is_on_sale": True,
|
||||
"product_url": None, # Swiftly does not expose a public product URL
|
||||
"scraped_at": datetime.now().isoformat(),
|
||||
"scraped_url": self.base_url
|
||||
"scraped_url": f"{cls.__name__}:{source_slug}" if source_slug else None,
|
||||
}
|
||||
|
||||
return item
|
||||
# ------------------------------------------------------------------
|
||||
# Helpers
|
||||
# ------------------------------------------------------------------
|
||||
@staticmethod
|
||||
def _parse_price(text: Optional[str]) -> Tuple[Optional[Decimal], Optional[str]]:
|
||||
"""Extract ``(price, unit)`` from strings like ``"$3.49 /lb"``.
|
||||
|
||||
def scrape_produce(self) -> List[Dict[str, Any]]:
|
||||
url = f"{self.base_url}/coupons/Coupon%2Flu-produce"
|
||||
logger.info(f"Scraping produce from {url}")
|
||||
Returns ``(None, None)`` when ``text`` is empty or unparseable.
|
||||
"""
|
||||
if not text:
|
||||
return None, None
|
||||
m = re.search(r"\$\s*([\d,]+(?:\.\d+)?)", text)
|
||||
if not m:
|
||||
return None, None
|
||||
raw = m.group(1).replace(",", "")
|
||||
try:
|
||||
price = Decimal(raw)
|
||||
except (InvalidOperation, ValueError):
|
||||
return None, None
|
||||
unit_match = re.search(r"/\s*([A-Za-z]+)", text)
|
||||
unit = unit_match.group(1).lower() if unit_match else None
|
||||
return price, unit
|
||||
|
||||
page = self.get_browser_page(url)
|
||||
content = page.content()
|
||||
page.close()
|
||||
@staticmethod
|
||||
def _aisle_from_slug(slug: str) -> Optional[str]:
|
||||
"""``Product/meat_seafood`` → ``meat_seafood``."""
|
||||
if not slug:
|
||||
return None
|
||||
if "/" in slug:
|
||||
return slug.rsplit("/", 1)[1]
|
||||
return slug
|
||||
|
||||
soup = BeautifulSoup(content, "html.parser")
|
||||
items = []
|
||||
|
||||
for item_elem in soup.find_all("div", class_=re.compile(r"product|item|coupon")):
|
||||
try:
|
||||
item = self._parse_coupon_item(item_elem)
|
||||
if item:
|
||||
item["aisle"] = "Produce"
|
||||
items.append(item)
|
||||
except Exception:
|
||||
continue
|
||||
|
||||
return items
|
||||
|
||||
def scrape_meat_seafood(self) -> List[Dict[str, Any]]:
|
||||
url = f"{self.base_url}/coupons/Coupon%2Flu-meat-seafood"
|
||||
logger.info(f"Scraping meat & seafood from {url}")
|
||||
|
||||
page = self.get_browser_page(url)
|
||||
content = page.content()
|
||||
page.close()
|
||||
|
||||
soup = BeautifulSoup(content, "html.parser")
|
||||
items = []
|
||||
|
||||
for item_elem in soup.find_all("div", class_=re.compile(r"product|item|coupon")):
|
||||
try:
|
||||
item = self._parse_coupon_item(item_elem)
|
||||
if item:
|
||||
item["aisle"] = "Meat"
|
||||
items.append(item)
|
||||
except Exception:
|
||||
continue
|
||||
|
||||
return items
|
||||
|
||||
def scrape_dairy_eggs(self) -> List[Dict[str, Any]]:
|
||||
url = f"{self.base_url}/coupons/Coupon%2Flu-dairy-eggs"
|
||||
logger.info(f"Scraping dairy & eggs from {url}")
|
||||
|
||||
page = self.get_browser_page(url)
|
||||
content = page.content()
|
||||
page.close()
|
||||
|
||||
soup = BeautifulSoup(content, "html.parser")
|
||||
items = []
|
||||
|
||||
for item_elem in soup.find_all("div", class_=re.compile(r"product|item|coupon")):
|
||||
try:
|
||||
item = self._parse_coupon_item(item_elem)
|
||||
if item:
|
||||
item["aisle"] = "Dairy"
|
||||
items.append(item)
|
||||
except Exception:
|
||||
continue
|
||||
|
||||
return items
|
||||
def _rate_limit(self) -> None:
|
||||
elapsed = time.time() - self._last_request
|
||||
if elapsed < self.rate_limit_seconds:
|
||||
time.sleep(self.rate_limit_seconds - elapsed)
|
||||
self._last_request = time.time()
|
||||
|
||||
@@ -0,0 +1,70 @@
|
||||
"""
|
||||
Auth dependencies for the MealPlanner backend.
|
||||
|
||||
Two flavors:
|
||||
- ``require_admin`` — bearer token for ``/api/admin/*`` routes; token compared
|
||||
to ``settings.ADMIN_TOKEN`` (must be set in env).
|
||||
- ``require_session`` — signed-cookie session (``itsdangerous``) gating
|
||||
mutations on the family-facing routers; reads stay open inside the
|
||||
trusted network.
|
||||
|
||||
The per-voter approval-token flow on meal items is intentionally NOT covered
|
||||
here — it has its own short-lived single-use tokens elsewhere.
|
||||
"""
|
||||
|
||||
from fastapi import HTTPException, Request, status
|
||||
from itsdangerous import BadSignature, SignatureExpired, TimestampSigner
|
||||
|
||||
from app.config import settings
|
||||
|
||||
bearer_header = "Authorization"
|
||||
|
||||
SESSION_COOKIE = "mp_session"
|
||||
SESSION_MAX_AGE = 60 * 60 * 24 * 30 # 30 days
|
||||
|
||||
|
||||
def require_admin(request: Request) -> None:
|
||||
"""Enforce a shared bearer token. 401 on bad/missing token, 503 if unset."""
|
||||
expected = settings.ADMIN_TOKEN
|
||||
if not expected:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_503_SERVICE_UNAVAILABLE,
|
||||
detail="Admin auth not configured",
|
||||
)
|
||||
auth = request.headers.get(bearer_header, "")
|
||||
if not auth.startswith("Bearer ") or auth[7:] != expected:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_401_UNAUTHORIZED,
|
||||
detail="Invalid admin token",
|
||||
)
|
||||
|
||||
|
||||
def _signer() -> TimestampSigner:
|
||||
return TimestampSigner(settings.SECRET_KEY)
|
||||
|
||||
|
||||
def issue_session(family_profile_id: str) -> str:
|
||||
"""Sign the family_profile_id and return the cookie value."""
|
||||
return _signer().sign(family_profile_id.encode()).decode()
|
||||
|
||||
|
||||
def require_session(request: Request) -> str:
|
||||
"""Return the family_profile_id stored in the signed session cookie."""
|
||||
raw = request.cookies.get(SESSION_COOKIE)
|
||||
if not raw:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_401_UNAUTHORIZED, detail="Session required"
|
||||
)
|
||||
try:
|
||||
family_id = (
|
||||
_signer().unsign(raw.encode(), max_age=SESSION_MAX_AGE).decode()
|
||||
)
|
||||
except SignatureExpired:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_401_UNAUTHORIZED, detail="Session expired"
|
||||
)
|
||||
except BadSignature:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_401_UNAUTHORIZED, detail="Invalid session"
|
||||
)
|
||||
return family_id
|
||||
@@ -0,0 +1,89 @@
|
||||
"""
|
||||
Per-voter approval tokens.
|
||||
|
||||
Stateless signed tokens (itsdangerous) keyed on settings.SECRET_KEY with a
|
||||
versioned salt. Single-use is enforced by the presence of a MealPlanVote
|
||||
row for (item, voter) — the table already has UniqueConstraint on that
|
||||
pair, so the DB is the source of truth, not a token-status column.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
from uuid import UUID
|
||||
|
||||
from fastapi import HTTPException
|
||||
from itsdangerous import (
|
||||
BadSignature,
|
||||
SignatureExpired,
|
||||
URLSafeTimedSerializer,
|
||||
)
|
||||
|
||||
from app.config import settings
|
||||
from app.models import FamilyMember, MealPlanVote
|
||||
|
||||
if TYPE_CHECKING: # pragma: no cover
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
|
||||
SALT = "meal-approval-v1"
|
||||
DEFAULT_MAX_AGE_SECONDS = 7 * 24 * 3600
|
||||
|
||||
|
||||
def _serializer() -> URLSafeTimedSerializer:
|
||||
return URLSafeTimedSerializer(secret_key=settings.SECRET_KEY, salt=SALT)
|
||||
|
||||
|
||||
def issue_token(meal_plan_item_id: UUID, family_member_id: UUID) -> str:
|
||||
payload = {
|
||||
"item": str(meal_plan_item_id),
|
||||
"voter": str(family_member_id),
|
||||
}
|
||||
return _serializer().dumps(payload)
|
||||
|
||||
|
||||
def verify_token(token: str, max_age_seconds: int = DEFAULT_MAX_AGE_SECONDS) -> dict:
|
||||
try:
|
||||
payload = _serializer().loads(token, max_age=max_age_seconds)
|
||||
except SignatureExpired:
|
||||
raise HTTPException(status_code=401, detail="Token expired")
|
||||
except BadSignature:
|
||||
raise HTTPException(status_code=401, detail="Invalid token")
|
||||
if not isinstance(payload, dict) or "item" not in payload or "voter" not in payload:
|
||||
raise HTTPException(status_code=401, detail="Invalid token payload")
|
||||
return payload
|
||||
|
||||
|
||||
def consume_token(
|
||||
db: "Session",
|
||||
token: str,
|
||||
meal_plan_item_id: UUID,
|
||||
max_age_seconds: int = DEFAULT_MAX_AGE_SECONDS,
|
||||
) -> FamilyMember:
|
||||
"""Verify + match URL + enforce single-use. Returns the voter on success.
|
||||
|
||||
Single-use is checked by looking for an existing MealPlanVote row for
|
||||
(item, voter). If one exists, raise 409.
|
||||
"""
|
||||
payload = verify_token(token, max_age_seconds=max_age_seconds)
|
||||
|
||||
if str(payload["item"]) != str(meal_plan_item_id):
|
||||
raise HTTPException(status_code=400, detail="Token not valid for this meal")
|
||||
|
||||
voter_id = UUID(str(payload["voter"]))
|
||||
voter = db.query(FamilyMember).filter(FamilyMember.id == voter_id).first()
|
||||
if not voter:
|
||||
raise HTTPException(status_code=404, detail="Voter not found")
|
||||
|
||||
existing = (
|
||||
db.query(MealPlanVote)
|
||||
.filter(
|
||||
MealPlanVote.meal_plan_item_id == meal_plan_item_id,
|
||||
MealPlanVote.family_member_id == voter_id,
|
||||
)
|
||||
.first()
|
||||
)
|
||||
if existing:
|
||||
raise HTTPException(status_code=409, detail="Already voted")
|
||||
|
||||
return voter
|
||||
@@ -0,0 +1,89 @@
|
||||
"""
|
||||
Email backends.
|
||||
|
||||
R2-B spike: only ConsoleEmailBackend is functional. SendGrid is a stub
|
||||
intentionally left to be wired in R3-C.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Optional, Protocol
|
||||
|
||||
from app.config import settings
|
||||
|
||||
|
||||
# Repository convention: backend/var/email_outbox.jsonl
|
||||
# This module lives at backend/app/services/email.py — go up two levels to
|
||||
# reach backend/, then var/.
|
||||
_BACKEND_ROOT = Path(__file__).resolve().parent.parent.parent
|
||||
OUTBOX_PATH = _BACKEND_ROOT / "var" / "email_outbox.jsonl"
|
||||
|
||||
|
||||
class EmailBackend(Protocol):
|
||||
def send(
|
||||
self,
|
||||
to: str,
|
||||
subject: str,
|
||||
html: str,
|
||||
text: Optional[str] = None,
|
||||
) -> None:
|
||||
...
|
||||
|
||||
|
||||
class ConsoleEmailBackend:
|
||||
"""Dev/spike backend. Prints to stdout AND appends a JSON line to
|
||||
backend/var/email_outbox.jsonl so the round-trip can be inspected
|
||||
after the fact.
|
||||
"""
|
||||
|
||||
def send(
|
||||
self,
|
||||
to: str,
|
||||
subject: str,
|
||||
html: str,
|
||||
text: Optional[str] = None,
|
||||
) -> None:
|
||||
record = {
|
||||
"ts": datetime.now(timezone.utc).isoformat(),
|
||||
"to": to,
|
||||
"subject": subject,
|
||||
"html": html,
|
||||
"text": text,
|
||||
}
|
||||
print(
|
||||
f"[ConsoleEmailBackend] -> {to} | {subject}",
|
||||
file=sys.stdout,
|
||||
flush=True,
|
||||
)
|
||||
OUTBOX_PATH.parent.mkdir(parents=True, exist_ok=True)
|
||||
with OUTBOX_PATH.open("a", encoding="utf-8") as f:
|
||||
f.write(json.dumps(record) + "\n")
|
||||
|
||||
|
||||
class SendGridEmailBackend:
|
||||
"""Stub. Wire in R3-C (real SendGrid client + sandbox mode + retries).
|
||||
|
||||
Deliberately raises so a misconfigured prod env fails loudly instead of
|
||||
silently dropping mail.
|
||||
"""
|
||||
|
||||
def send(
|
||||
self,
|
||||
to: str,
|
||||
subject: str,
|
||||
html: str,
|
||||
text: Optional[str] = None,
|
||||
) -> None: # pragma: no cover - stub
|
||||
raise NotImplementedError("Wire SendGrid in R3-C")
|
||||
|
||||
|
||||
def get_email_backend() -> EmailBackend:
|
||||
backend = (settings.EMAIL_BACKEND or "console").lower()
|
||||
if backend == "sendgrid":
|
||||
return SendGridEmailBackend()
|
||||
return ConsoleEmailBackend()
|
||||
@@ -1,87 +1,204 @@
|
||||
"""Scraper service.
|
||||
|
||||
Two entry points:
|
||||
|
||||
- ``enqueue_scrape(db, source, scrape_type, background_tasks)``: synchronously
|
||||
inserts a ``ScrapeLog`` row in status ``STARTED`` (the existing enum has no
|
||||
``pending`` member; ``STARTED`` is reused for the queued state) and registers
|
||||
``_run_scrape_in_background`` to fire after the response is sent.
|
||||
- ``_run_scrape_in_background(log_id, source, scrape_type)``: runs in a
|
||||
FastAPI background task with its OWN ``SessionLocal()`` (the request-scoped
|
||||
``db`` is closed by the time this fires). Writes terminal status
|
||||
(``SUCCESS``/``FAILED``) and ``error_message``.
|
||||
|
||||
``ScraperService.run_scrape`` is preserved for direct/test invocation; the
|
||||
``/api/admin/scrape`` endpoint now goes through ``enqueue_scrape``.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from typing import Dict, Any, Optional
|
||||
from datetime import datetime
|
||||
from uuid import uuid4
|
||||
from typing import Any, Dict, Optional
|
||||
from datetime import datetime, timezone
|
||||
from uuid import UUID, uuid4
|
||||
|
||||
from fastapi import BackgroundTasks
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.database import SessionLocal
|
||||
from app.models import GroceryItem, Ingredient, ScrapeLog, ScrapeStatus
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Public API: enqueue + background runner
|
||||
# ---------------------------------------------------------------------------
|
||||
def enqueue_scrape(
|
||||
db: Session,
|
||||
*,
|
||||
source: str,
|
||||
scrape_type: str,
|
||||
background_tasks: BackgroundTasks,
|
||||
) -> ScrapeLog:
|
||||
"""Create the ScrapeLog row, commit, and schedule the background scrape.
|
||||
|
||||
Returns the persisted ``ScrapeLog`` instance (refreshed). The actual
|
||||
scraping work runs after FastAPI sends the 202 response.
|
||||
"""
|
||||
log = ScrapeLog(
|
||||
id=uuid4(),
|
||||
source=source,
|
||||
scrape_type=scrape_type,
|
||||
status=ScrapeStatus.STARTED,
|
||||
started_at=datetime.now(timezone.utc),
|
||||
)
|
||||
db.add(log)
|
||||
db.commit()
|
||||
db.refresh(log)
|
||||
|
||||
background_tasks.add_task(
|
||||
_run_scrape_in_background, log.id, source, scrape_type
|
||||
)
|
||||
return log
|
||||
|
||||
|
||||
def _run_scrape_in_background(
|
||||
log_id: UUID, source: str, scrape_type: str
|
||||
) -> None:
|
||||
"""Background entry point. Opens a fresh DB session — the request-scoped
|
||||
session is gone by the time this runs.
|
||||
"""
|
||||
db: Session = SessionLocal()
|
||||
try:
|
||||
log = db.query(ScrapeLog).filter(ScrapeLog.id == log_id).first()
|
||||
if log is None:
|
||||
logger.error("ScrapeLog %s vanished before background run", log_id)
|
||||
return
|
||||
|
||||
# No "running" state in the enum; STARTED already covers in-flight.
|
||||
# Mark started_at fresh in case there was lag between enqueue and run.
|
||||
try:
|
||||
service = ScraperService(db)
|
||||
saved_count, items_found = service._do_scrape(source=source)
|
||||
log.status = ScrapeStatus.SUCCESS
|
||||
log.items_scraped = saved_count
|
||||
log.completed_at = datetime.now(timezone.utc)
|
||||
log.duration_seconds = int(
|
||||
(log.completed_at - log.started_at).total_seconds()
|
||||
)
|
||||
db.commit()
|
||||
logger.info(
|
||||
"Background scrape %s complete: %s/%s items saved",
|
||||
log_id, saved_count, items_found,
|
||||
)
|
||||
except Exception as exc: # noqa: BLE001 — must catch all to mark failed
|
||||
logger.exception("Background scrape %s failed", log_id)
|
||||
db.rollback()
|
||||
# Re-fetch in case the rollback detached the instance.
|
||||
log = db.query(ScrapeLog).filter(ScrapeLog.id == log_id).first()
|
||||
if log is not None:
|
||||
log.status = ScrapeStatus.FAILED
|
||||
log.error_message = str(exc)
|
||||
log.completed_at = datetime.now(timezone.utc)
|
||||
if log.started_at:
|
||||
log.duration_seconds = int(
|
||||
(log.completed_at - log.started_at).total_seconds()
|
||||
)
|
||||
db.commit()
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Service class (kept for direct/test use)
|
||||
# ---------------------------------------------------------------------------
|
||||
class ScraperService:
|
||||
def __init__(self, db: Session):
|
||||
self.db = db
|
||||
|
||||
def run_scrape(self, source: str = "lucky_california", scrape_type: str = "weekly_ad") -> Dict[str, Any]:
|
||||
from app.scraper import LuckyCaliforniaScraper
|
||||
from app.models import ScrapeLog, GroceryItem, Ingredient
|
||||
|
||||
scrape_log = ScrapeLog(
|
||||
def run_scrape(
|
||||
self, source: str = "lucky_california", scrape_type: str = "weekly_ad"
|
||||
) -> Dict[str, Any]:
|
||||
"""Synchronous end-to-end scrape (legacy path). Creates the log row,
|
||||
runs the scrape, commits terminal status. Used by direct callers and
|
||||
tests; the API endpoint goes through ``enqueue_scrape``.
|
||||
"""
|
||||
log = ScrapeLog(
|
||||
id=uuid4(),
|
||||
source=source,
|
||||
scrape_type=scrape_type,
|
||||
status="started",
|
||||
started_at=datetime.now()
|
||||
status=ScrapeStatus.STARTED,
|
||||
started_at=datetime.now(timezone.utc),
|
||||
)
|
||||
self.db.add(scrape_log)
|
||||
self.db.add(log)
|
||||
self.db.commit()
|
||||
|
||||
logger.info(f"Starting {source} {scrape_type} scrape")
|
||||
logger.info("Starting %s %s scrape", source, scrape_type)
|
||||
|
||||
try:
|
||||
scraper = LuckyCaliforniaScraper()
|
||||
result = scraper.scrape()
|
||||
scraper.cleanup()
|
||||
|
||||
items = result.get("items", [])
|
||||
saved_count = 0
|
||||
|
||||
for item_data in items:
|
||||
saved_item = self._save_grocery_item(item_data)
|
||||
if saved_item:
|
||||
saved_count += 1
|
||||
|
||||
scrape_log.status = "success"
|
||||
scrape_log.items_scraped = saved_count
|
||||
scrape_log.completed_at = datetime.now()
|
||||
scrape_log.duration_seconds = int(
|
||||
(scrape_log.completed_at - scrape_log.started_at).total_seconds()
|
||||
saved_count, items_found = self._do_scrape(source=source)
|
||||
log.status = ScrapeStatus.SUCCESS
|
||||
log.items_scraped = saved_count
|
||||
log.completed_at = datetime.now(timezone.utc)
|
||||
log.duration_seconds = int(
|
||||
(log.completed_at - log.started_at).total_seconds()
|
||||
)
|
||||
|
||||
self.db.commit()
|
||||
|
||||
logger.info(f"Scrape complete: {saved_count} items saved")
|
||||
|
||||
logger.info("Scrape complete: %s items saved", saved_count)
|
||||
return {
|
||||
"scrape_id": str(scrape_log.id),
|
||||
"scrape_id": str(log.id),
|
||||
"status": "success",
|
||||
"items_scraped": saved_count,
|
||||
"items_found": len(items)
|
||||
"items_found": items_found,
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Scrape failed: {e}")
|
||||
scrape_log.status = "failed"
|
||||
scrape_log.error_message = str(e)
|
||||
scrape_log.completed_at = datetime.now()
|
||||
scrape_log.duration_seconds = int(
|
||||
(scrape_log.completed_at - scrape_log.started_at).total_seconds()
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.error("Scrape failed: %s", e)
|
||||
log.status = ScrapeStatus.FAILED
|
||||
log.error_message = str(e)
|
||||
log.completed_at = datetime.now(timezone.utc)
|
||||
log.duration_seconds = int(
|
||||
(log.completed_at - log.started_at).total_seconds()
|
||||
)
|
||||
self.db.commit()
|
||||
|
||||
return {
|
||||
"scrape_id": str(scrape_log.id),
|
||||
"scrape_id": str(log.id),
|
||||
"status": "failed",
|
||||
"error": str(e)
|
||||
"error": str(e),
|
||||
}
|
||||
|
||||
def _save_grocery_item(self, item_data: Dict[str, Any]) -> Optional[GroceryItem]:
|
||||
from app.models import GroceryItem, Ingredient
|
||||
# ------------------------------------------------------------------
|
||||
# Internals
|
||||
# ------------------------------------------------------------------
|
||||
def _do_scrape(self, *, source: str) -> tuple[int, int]:
|
||||
"""Run the scraper and persist items. Returns (saved, found).
|
||||
|
||||
name = item_data.get("name", "").strip()
|
||||
Raises any scraper exception (including ``SwiftlyAuthError``) to
|
||||
the caller for status mapping.
|
||||
"""
|
||||
from app.scraper import LuckyCaliforniaScraper
|
||||
|
||||
scraper = LuckyCaliforniaScraper()
|
||||
saved_count = 0
|
||||
found_count = 0
|
||||
try:
|
||||
for item_data in scraper.fetch_all():
|
||||
found_count += 1
|
||||
if self._save_grocery_item(item_data) is not None:
|
||||
saved_count += 1
|
||||
finally:
|
||||
scraper.cleanup()
|
||||
return saved_count, found_count
|
||||
|
||||
def _save_grocery_item(
|
||||
self, item_data: Dict[str, Any]
|
||||
) -> Optional[GroceryItem]:
|
||||
name = (item_data.get("name") or "").strip()
|
||||
if not name:
|
||||
return None
|
||||
|
||||
name_lower = name.lower()
|
||||
source = item_data.get("source") or "lucky_california"
|
||||
external_id = item_data.get("external_id")
|
||||
|
||||
ingredient = self.db.query(Ingredient).filter(
|
||||
Ingredient.name_lower == name_lower
|
||||
@@ -93,22 +210,41 @@ class ScraperService:
|
||||
name=name,
|
||||
name_lower=name_lower,
|
||||
aisle=item_data.get("aisle"),
|
||||
typical_price=item_data.get("current_price")
|
||||
typical_price=item_data.get("current_price"),
|
||||
)
|
||||
self.db.add(ingredient)
|
||||
self.db.flush()
|
||||
|
||||
existing = self.db.query(GroceryItem).filter(
|
||||
GroceryItem.name == name,
|
||||
GroceryItem.scraped_url == item_data.get("scraped_url")
|
||||
).first()
|
||||
# Idempotency:
|
||||
# 1) (source, external_id) when both present (Swiftly path);
|
||||
# 2) fall back to (name, scraped_url) for legacy rows from R2-A.
|
||||
existing: Optional[GroceryItem] = None
|
||||
if external_id:
|
||||
existing = self.db.query(GroceryItem).filter(
|
||||
GroceryItem.source == source,
|
||||
GroceryItem.external_id == external_id,
|
||||
).first()
|
||||
if existing is None:
|
||||
existing = self.db.query(GroceryItem).filter(
|
||||
GroceryItem.name == name,
|
||||
GroceryItem.scraped_url == item_data.get("scraped_url"),
|
||||
).first()
|
||||
|
||||
if existing:
|
||||
existing.ingredient_id = ingredient.id
|
||||
existing.brand = item_data.get("brand")
|
||||
existing.current_price = item_data.get("current_price")
|
||||
existing.is_on_sale = item_data.get("is_on_sale", True)
|
||||
existing.regular_price = item_data.get("regular_price")
|
||||
existing.unit = item_data.get("unit")
|
||||
existing.aisle = item_data.get("aisle")
|
||||
existing.image_url = item_data.get("image_url")
|
||||
existing.product_url = item_data.get("product_url")
|
||||
existing.scraped_at = datetime.now()
|
||||
existing.description = item_data.get("description")
|
||||
existing.is_on_sale = bool(item_data.get("is_on_sale", False))
|
||||
existing.scraped_at = datetime.now(timezone.utc)
|
||||
existing.scraped_url = item_data.get("scraped_url")
|
||||
existing.external_id = external_id
|
||||
existing.source = source
|
||||
self.db.flush()
|
||||
return existing
|
||||
|
||||
@@ -116,31 +252,31 @@ class ScraperService:
|
||||
id=uuid4(),
|
||||
ingredient_id=ingredient.id,
|
||||
name=name,
|
||||
brand=item_data.get("brand"),
|
||||
current_price=item_data.get("current_price"),
|
||||
regular_price=item_data.get("regular_price"),
|
||||
unit=item_data.get("unit"),
|
||||
aisle=item_data.get("aisle"),
|
||||
image_url=item_data.get("image_url"),
|
||||
product_url=item_data.get("product_url"),
|
||||
is_on_sale=item_data.get("is_on_sale", True),
|
||||
description=item_data.get("description"),
|
||||
is_on_sale=bool(item_data.get("is_on_sale", False)),
|
||||
sale_start_date=item_data.get("sale_start_date"),
|
||||
sale_end_date=item_data.get("sale_end_date"),
|
||||
in_season=item_data.get("in_season", False),
|
||||
scraped_at=datetime.now(),
|
||||
scraped_url=item_data.get("scraped_url")
|
||||
scraped_at=datetime.now(timezone.utc),
|
||||
scraped_url=item_data.get("scraped_url"),
|
||||
external_id=external_id,
|
||||
source=source,
|
||||
)
|
||||
|
||||
self.db.add(grocery_item)
|
||||
self.db.commit()
|
||||
self.db.refresh(grocery_item)
|
||||
|
||||
self.db.flush()
|
||||
return grocery_item
|
||||
|
||||
def get_sale_items(self, limit: int = 50) -> list:
|
||||
from app.models import GroceryItem
|
||||
|
||||
items = self.db.query(GroceryItem).filter(
|
||||
GroceryItem.is_on_sale == True
|
||||
GroceryItem.is_on_sale == True # noqa: E712 — SQLAlchemy idiom
|
||||
).order_by(GroceryItem.scraped_at.desc()).limit(limit).all()
|
||||
|
||||
return [
|
||||
@@ -152,7 +288,8 @@ class ScraperService:
|
||||
"aisle": item.aisle,
|
||||
"image_url": item.image_url,
|
||||
"product_url": item.product_url,
|
||||
"scraped_at": item.scraped_at.isoformat() if item.scraped_at else None
|
||||
"description": item.description,
|
||||
"scraped_at": item.scraped_at.isoformat() if item.scraped_at else None,
|
||||
}
|
||||
for item in items
|
||||
]
|
||||
]
|
||||
|
||||
Reference in New Issue
Block a user