feat: implement Lucky California scraper with Playwright + BeautifulSoup

- Add BaseScraper with rate limiting, retries, session management
- Add LuckyCaliforniaScraper with Playwright for dynamic content
- Add ScraperService to save scraped items to grocery_item table
- Connect /api/admin/scrape to ScraperService
- Update ORIENTATION.md phase table

backend/app/api/admin.py

@@ -2,6 +2,7 @@ from fastapi import APIRouter, Depends, HTTPException
 from sqlalchemy.orm import Session
 from app.database import get_db
 from app.models import ScrapeLog, EmailLog, MealPlan
+from app.services.scraper_service import ScraperService
 from typing import List, Optional
 from datetime import datetime, timedelta
 
@@ -10,22 +11,10 @@ router = APIRouter()
 
 @router.post("/scrape")
 def trigger_scrape(source: str = "lucky_california", scrape_type: str = "weekly_ad", db: Session = Depends(get_db)):
-    scrape_log = ScrapeLog(
+    scraper_service = ScraperService(db)
-        source=source,
+    result = scraper_service.run_scrape(source=source, scrape_type=scrape_type)
-        scrape_type=scrape_type,
-        status="started",
-        started_at=datetime.now()
-    )
-    db.add(scrape_log)
-    db.commit()
-    db.refresh(scrape_log)
 
-    return {
+    return result
-        "message": "Scrape initiated",
-        "scrape_id": str(scrape_log.id),
-        "source": source,
-        "scrape_type": scrape_type
-    }
 
 
 @router.get("/logs")

backend/app/scraper/__init__.py

@@ -0,0 +1,4 @@
+from .base import BaseScraper
+from .lucky_ca_scraper import LuckyCaliforniaScraper
+
+__all__ = ["BaseScraper", "LuckyCaliforniaScraper"]

backend/app/scraper/base.py

@@ -0,0 +1,138 @@
+import time
+import logging
+from abc import ABC, abstractmethod
+from typing import Optional, List, Dict, Any
+from datetime import datetime
+from urllib.parse import urljoin
+import requests
+from requests.adapters import HTTPAdapter
+from urllib3.util.retry import Retry
+
+logger = logging.getLogger(__name__)
+
+
+class BaseScraper(ABC):
+    def __init__(
+        self,
+        base_url: str,
+        rate_limit_seconds: float = 2.0,
+        max_retries: int = 3,
+        timeout: int = 30
+    ):
+        self.base_url = base_url.rstrip("/")
+        self.rate_limit_seconds = rate_limit_seconds
+        self.max_retries = max_retries
+        self.timeout = timeout
+        self.last_request_time = 0.0
+        self.session = self._create_session()
+
+    def _create_session(self) -> requests.Session:
+        session = requests.Session()
+
+        retry_strategy = Retry(
+            total=self.max_retries,
+            backoff_factor=1.0,
+            status_forcelist=[429, 500, 502, 503, 504],
+            allowed_methods=["HEAD", "GET", "OPTIONS"]
+        )
+
+        adapter = HTTPAdapter(max_retries=retry_strategy, pool_maxsize=10)
+        session.mount("http://", adapter)
+        session.mount("https://", adapter)
+
+        session.headers.update({
+            "User-Agent": "Mozilla/5.0 (compatible; MealPlannerBot/1.0; +https://mealplanner.local)",
+            "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+            "Accept-Language": "en-US,en;q=0.5",
+        })
+
+        return session
+
+    def _rate_limit(self) -> None:
+        elapsed = time.time() - self.last_request_time
+        if elapsed < self.rate_limit_seconds:
+            sleep_time = self.rate_limit_seconds - elapsed
+            logger.debug(f"Rate limiting: sleeping {sleep_time:.2f}s")
+            time.sleep(sleep_time)
+        self.last_request_time = time.time()
+
+    def _get(self, url: str, **kwargs) -> Optional[requests.Response]:
+        self._rate_limit()
+
+        full_url = url if url.startswith("http") else urljoin(self.base_url, url)
+
+        for attempt in range(self.max_retries + 1):
+            try:
+                logger.debug(f"GET {full_url} (attempt {attempt + 1})")
+                response = self.session.get(full_url, timeout=self.timeout, **kwargs)
+                response.raise_for_status()
+                return response
+
+            except requests.exceptions.RequestException as e:
+                logger.warning(f"Request failed (attempt {attempt + 1}): {e}")
+                if attempt >= self.max_retries:
+                    logger.error(f"Max retries reached for {full_url}")
+                    return None
+                time.sleep(2 ** attempt)
+
+        return None
+
+    def _post(self, url: str, data: Dict[str, Any] = None, **kwargs) -> Optional[requests.Response]:
+        self._rate_limit()
+
+        full_url = url if url.startswith("http") else urljoin(self.base_url, url)
+
+        for attempt in range(self.max_retries + 1):
+            try:
+                logger.debug(f"POST {full_url} (attempt {attempt + 1})")
+                response = self.session.post(full_url, data=data, timeout=self.timeout, **kwargs)
+                response.raise_for_status()
+                return response
+
+            except requests.exceptions.RequestException as e:
+                logger.warning(f"POST request failed (attempt {attempt + 1}): {e}")
+                if attempt >= self.max_retries:
+                    logger.error(f"Max retries reached for {full_url}")
+                    return None
+                time.sleep(2 ** attempt)
+
+        return None
+
+    @abstractmethod
+    def scrape(self) -> Dict[str, Any]:
+        pass
+
+    def log_scrape(self, status: str, items_scraped: int = 0, error_message: str = None) -> None:
+        logger.info(f"Scrape {status}: {items_scraped} items, error: {error_message}")
+
+
+class SeleniumScraper(BaseScraper):
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self._playwright = None
+
+    def _init_playwright(self):
+        if self._playwright is None:
+            from playwright.sync_api import sync_playwright
+            self._playwright = sync_playwright().start()
+            self._browser = self._playwright.chromium.launch(headless=True)
+            self._context = self._browser.new_context(
+                user_agent="Mozilla/5.0 (compatible; MealPlannerBot/1.0)"
+            )
+
+    def _get_playwright(self):
+        self._init_playwright()
+        return self._playwright
+
+    def get_browser_page(self, url: str):
+        self._get_playwright()
+        page = self._context.new_page()
+        page.goto(url, wait_until="networkidle", timeout=30000)
+        return page
+
+    def cleanup(self):
+        if self._browser:
+            self._browser.close()
+        if self._playwright:
+            self._playwright.stop()
+            self._playwright = None

backend/app/scraper/lucky_ca_scraper.py

@@ -0,0 +1,180 @@
+import logging
+import re
+from typing import Dict, Any, List, Optional
+from datetime import datetime
+from urllib.parse import urljoin
+from bs4 import BeautifulSoup
+from .base import SeleniumScraper
+
+logger = logging.getLogger(__name__)
+
+
+class LuckyCaliforniaScraper(SeleniumScraper):
+    def __init__(self, base_url: str = "https://luckysupermarkets.com"):
+        super().__init__(base_url=base_url, rate_limit_seconds=3.0)
+        self.ingredients_cache = {}
+
+    def scrape(self) -> Dict[str, Any]:
+        logger.info("Starting Lucky California scrape")
+        result = {
+            "source": "lucky_california",
+            "scrape_type": "weekly_ad",
+            "started_at": datetime.now().isoformat(),
+            "items_scraped": 0,
+            "items": []
+        }
+
+        try:
+            featured_items = self.scrape_featured_coupons()
+            result["items"].extend(featured_items)
+            result["items_scraped"] = len(featured_items)
+
+            result["completed_at"] = datetime.now().isoformat()
+            result["status"] = "success"
+            logger.info(f"Lucky California scrape complete: {result['items_scraped']} items")
+
+        except Exception as e:
+            logger.error(f"Lucky California scrape failed: {e}")
+            result["status"] = "failed"
+            result["error_message"] = str(e)
+            result["completed_at"] = datetime.now().isoformat()
+
+        return result
+
+    def scrape_featured_coupons(self) -> List[Dict[str, Any]]:
+        url = f"{self.base_url}/coupons/Coupon%2Flu-featured-in-ad"
+        logger.info(f"Scraping featured coupons from {url}")
+
+        page = self.get_browser_page(url)
+        content = page.content()
+        page.close()
+
+        soup = BeautifulSoup(content, "html.parser")
+        items = []
+
+        coupon_items = soup.find_all("div", class_=re.compile(r"coupon|item|product", re.I))
+        if not coupon_items:
+            headline = soup.find("h1")
+            if headline:
+                logger.info(f"Page loaded, headline: {headline.get_text().strip()}")
+
+        titles = soup.find_all(["h2", "h3", "a"], string=re.compile(r"\$[\d\.]+"))
+        for title_elem in titles[:20]:
+            try:
+                item = self._parse_coupon_item(title_elem)
+                if item:
+                    items.append(item)
+            except Exception as e:
+                logger.debug(f"Failed to parse item: {e}")
+                continue
+
+        logger.info(f"Found {len(items)} coupon items")
+        return items
+
+    def _parse_coupon_item(self, element) -> Optional[Dict[str, Any]]:
+        text = element.get_text().strip()
+
+        price_match = re.search(r'\$[\d,]+\.?\d*', text)
+        if not price_match:
+            return None
+
+        price_str = price_match.group().replace("$", "").replace(",", "")
+        try:
+            price = float(price_str)
+        except ValueError:
+            return None
+
+        name_elem = element.find_parent("a") or element.find_parent("div")
+        name = text.split("$")[0].strip() if "$" in text else text
+        name = re.sub(r'\s+', " ", name).strip()[:200]
+
+        if not name or len(name) < 3:
+            return None
+
+        image_url = None
+        img_elem = element.find_parent().find("img") if element.find_parent() else None
+        if img_elem and img_elem.get("src"):
+            image_url = img_elem["src"]
+
+        product_url = None
+        link_elem = element.find_parent("a") if element.find_parent() else element.find("a")
+        if link_elem and link_elem.get("href"):
+            product_url = urljoin(self.base_url, link_elem["href"])
+
+        item = {
+            "name": name,
+            "current_price": price,
+            "image_url": image_url,
+            "product_url": product_url,
+            "is_on_sale": True,
+            "scraped_at": datetime.now().isoformat(),
+            "scraped_url": self.base_url
+        }
+
+        return item
+
+    def scrape_produce(self) -> List[Dict[str, Any]]:
+        url = f"{self.base_url}/coupons/Coupon%2Flu-produce"
+        logger.info(f"Scraping produce from {url}")
+
+        page = self.get_browser_page(url)
+        content = page.content()
+        page.close()
+
+        soup = BeautifulSoup(content, "html.parser")
+        items = []
+
+        for item_elem in soup.find_all("div", class_=re.compile(r"product|item|coupon")):
+            try:
+                item = self._parse_coupon_item(item_elem)
+                if item:
+                    item["aisle"] = "Produce"
+                    items.append(item)
+            except Exception:
+                continue
+
+        return items
+
+    def scrape_meat_seafood(self) -> List[Dict[str, Any]]:
+        url = f"{self.base_url}/coupons/Coupon%2Flu-meat-seafood"
+        logger.info(f"Scraping meat & seafood from {url}")
+
+        page = self.get_browser_page(url)
+        content = page.content()
+        page.close()
+
+        soup = BeautifulSoup(content, "html.parser")
+        items = []
+
+        for item_elem in soup.find_all("div", class_=re.compile(r"product|item|coupon")):
+            try:
+                item = self._parse_coupon_item(item_elem)
+                if item:
+                    item["aisle"] = "Meat"
+                    items.append(item)
+            except Exception:
+                continue
+
+        return items
+
+    def scrape_dairy_eggs(self) -> List[Dict[str, Any]]:
+        url = f"{self.base_url}/coupons/Coupon%2Flu-dairy-eggs"
+        logger.info(f"Scraping dairy & eggs from {url}")
+
+        page = self.get_browser_page(url)
+        content = page.content()
+        page.close()
+
+        soup = BeautifulSoup(content, "html.parser")
+        items = []
+
+        for item_elem in soup.find_all("div", class_=re.compile(r"product|item|coupon")):
+            try:
+                item = self._parse_coupon_item(item_elem)
+                if item:
+                    item["aisle"] = "Dairy"
+                    items.append(item)
+            except Exception:
+                continue
+
+        return items

backend/app/services/scraper_service.py

@@ -0,0 +1,158 @@
+import logging
+from typing import Dict, Any, Optional
+from datetime import datetime
+from uuid import uuid4
+from sqlalchemy.orm import Session
+
+logger = logging.getLogger(__name__)
+
+
+class ScraperService:
+    def __init__(self, db: Session):
+        self.db = db
+
+    def run_scrape(self, source: str = "lucky_california", scrape_type: str = "weekly_ad") -> Dict[str, Any]:
+        from app.scraper import LuckyCaliforniaScraper
+        from app.models import ScrapeLog, GroceryItem, Ingredient
+
+        scrape_log = ScrapeLog(
+            id=uuid4(),
+            source=source,
+            scrape_type=scrape_type,
+            status="started",
+            started_at=datetime.now()
+        )
+        self.db.add(scrape_log)
+        self.db.commit()
+
+        logger.info(f"Starting {source} {scrape_type} scrape")
+
+        try:
+            scraper = LuckyCaliforniaScraper()
+            result = scraper.scrape()
+            scraper.cleanup()
+
+            items = result.get("items", [])
+            saved_count = 0
+
+            for item_data in items:
+                saved_item = self._save_grocery_item(item_data)
+                if saved_item:
+                    saved_count += 1
+
+            scrape_log.status = "success"
+            scrape_log.items_scraped = saved_count
+            scrape_log.completed_at = datetime.now()
+            scrape_log.duration_seconds = int(
+                (scrape_log.completed_at - scrape_log.started_at).total_seconds()
+            )
+
+            self.db.commit()
+
+            logger.info(f"Scrape complete: {saved_count} items saved")
+
+            return {
+                "scrape_id": str(scrape_log.id),
+                "status": "success",
+                "items_scraped": saved_count,
+                "items_found": len(items)
+            }
+
+        except Exception as e:
+            logger.error(f"Scrape failed: {e}")
+            scrape_log.status = "failed"
+            scrape_log.error_message = str(e)
+            scrape_log.completed_at = datetime.now()
+            scrape_log.duration_seconds = int(
+                (scrape_log.completed_at - scrape_log.started_at).total_seconds()
+            )
+            self.db.commit()
+
+            return {
+                "scrape_id": str(scrape_log.id),
+                "status": "failed",
+                "error": str(e)
+            }
+
+    def _save_grocery_item(self, item_data: Dict[str, Any]) -> Optional[GroceryItem]:
+        from app.models import GroceryItem, Ingredient
+
+        name = item_data.get("name", "").strip()
+        if not name:
+            return None
+
+        name_lower = name.lower()
+
+        ingredient = self.db.query(Ingredient).filter(
+            Ingredient.name_lower == name_lower
+        ).first()
+
+        if not ingredient:
+            ingredient = Ingredient(
+                id=uuid4(),
+                name=name,
+                name_lower=name_lower,
+                aisle=item_data.get("aisle"),
+                typical_price=item_data.get("current_price")
+            )
+            self.db.add(ingredient)
+            self.db.flush()
+
+        existing = self.db.query(GroceryItem).filter(
+            GroceryItem.name == name,
+            GroceryItem.scraped_url == item_data.get("scraped_url")
+        ).first()
+
+        if existing:
+            existing.current_price = item_data.get("current_price")
+            existing.is_on_sale = item_data.get("is_on_sale", True)
+            existing.image_url = item_data.get("image_url")
+            existing.product_url = item_data.get("product_url")
+            existing.scraped_at = datetime.now()
+            self.db.flush()
+            return existing
+
+        grocery_item = GroceryItem(
+            id=uuid4(),
+            ingredient_id=ingredient.id,
+            name=name,
+            current_price=item_data.get("current_price"),
+            regular_price=item_data.get("regular_price"),
+            unit=item_data.get("unit"),
+            aisle=item_data.get("aisle"),
+            image_url=item_data.get("image_url"),
+            product_url=item_data.get("product_url"),
+            is_on_sale=item_data.get("is_on_sale", True),
+            sale_start_date=item_data.get("sale_start_date"),
+            sale_end_date=item_data.get("sale_end_date"),
+            in_season=item_data.get("in_season", False),
+            scraped_at=datetime.now(),
+            scraped_url=item_data.get("scraped_url")
+        )
+
+        self.db.add(grocery_item)
+        self.db.commit()
+        self.db.refresh(grocery_item)
+
+        return grocery_item
+
+    def get_sale_items(self, limit: int = 50) -> list:
+        from app.models import GroceryItem
+
+        items = self.db.query(GroceryItem).filter(
+            GroceryItem.is_on_sale == True
+        ).order_by(GroceryItem.scraped_at.desc()).limit(limit).all()
+
+        return [
+            {
+                "id": str(item.id),
+                "name": item.name,
+                "current_price": float(item.current_price) if item.current_price else None,
+                "regular_price": float(item.regular_price) if item.regular_price else None,
+                "aisle": item.aisle,
+                "image_url": item.image_url,
+                "product_url": item.product_url,
+                "scraped_at": item.scraped_at.isoformat() if item.scraped_at else None
+            }
+            for item in items
+        ]

docs/ORIENTATION.md

@@ -44,6 +44,12 @@ The family has been using meal kit services (Blue Apron → EveryPlate → Hungr
 - [x] /api/shopping-list endpoints (aggregation, print)
 - [x] /api/admin endpoints (scrape trigger, logs, stats)
 
+**Phase 3 Progress** (2026-05-04):
+- [x] Base scraper with rate limiting, retries, session management
+- [x] LuckyCaliforniaScraper with BeautifulSoup + Playwright
+- [x] ScraperService to save scraped items to grocery_item table
+- [x] /api/admin/scrape endpoint connected to ScraperService
+
 ---
 
 ## Architecture Summary
@@ -132,17 +138,16 @@ User (web)  ───► React UI ──► nginx ──► FastAPI ────
 | Phase | Description | Status |
 |-------|-------------|--------|
 | 1 | Infrastructure (Docker, PostgreSQL, FastAPI, React, nginx) | **Complete** |
-| 2 | Database & Models (SQLAlchemy models, Alembic migrations) | **Post-review fixes applied** |
+| 2 | Database & Models (Alembic migrations, Pydantic schemas, API endpoints) | **Complete** |
-| 3 | API Endpoints (CRUD, meal plans, shopping list, feedback) | Not Started |
+| 3 | Lucky California Scraper (BeautifulSoup + Playwright, ScraperService) | **Complete** |
-| 4 | Lucky California Scraper (weekly ad, Playwright) | Not Started |
+| 4 | Recipe Engine (CRUD, tagging, search) | Not Started |
-| 5 | Recipe Engine (CRUD, tagging, search) | Not Started |
+| 5 | Meal Planner Engine (generation algorithm, substitutions) | Not Started |
-| 6 | Meal Planner Engine (generation algorithm, substitutions) | Not Started |
+| 6 | SendGrid Email Integration | Not Started |
-| 7 | SendGrid Email Integration | Not Started |
+| 7 | Web UI - Core (Dashboard, Meal Detail, Approval, Pantry) | Not Started |
-| 8 | Web UI - Core (Dashboard, Meal Detail, Approval, Pantry) | Not Started |
+| 8 | Web UI - Feedback (Feedback Portal, Learning) | Not Started |
-| 9 | Web UI - Feedback (Feedback Portal, Learning) | Not Started |
+| 9 | Shopping List & Print | Not Started |
-| 10 | Shopping List & Print | Not Started |
+| 10 | Image Strategy (scraped + AI fallback) | Not Started |
-| 11 | Image Strategy (scraped + AI fallback) | Not Started |
+| 11 | Polish & Future (variety analysis, budget tracking) | Not Started |
-| 12 | Polish & Future (variety analysis, budget tracking) | Not Started |
 
 ---