From 933a0cc9dbf40bb67f44a4cb01db6d30247aad25 Mon Sep 17 00:00:00 2001 From: Peter Woolery Date: Mon, 4 May 2026 20:50:27 -0700 Subject: [PATCH] feat: implement Lucky California scraper with Playwright + BeautifulSoup - Add BaseScraper with rate limiting, retries, session management - Add LuckyCaliforniaScraper with Playwright for dynamic content - Add ScraperService to save scraped items to grocery_item table - Connect /api/admin/scrape to ScraperService - Update ORIENTATION.md phase table --- backend/app/api/admin.py | 19 +-- backend/app/scraper/__init__.py | 4 + backend/app/scraper/base.py | 138 ++++++++++++++++++ backend/app/scraper/lucky_ca_scraper.py | 180 ++++++++++++++++++++++++ backend/app/services/scraper_service.py | 158 +++++++++++++++++++++ docs/ORIENTATION.md | 27 ++-- 6 files changed, 500 insertions(+), 26 deletions(-) create mode 100644 backend/app/scraper/__init__.py create mode 100644 backend/app/scraper/base.py create mode 100644 backend/app/scraper/lucky_ca_scraper.py create mode 100644 backend/app/services/scraper_service.py diff --git a/backend/app/api/admin.py b/backend/app/api/admin.py index f15ad60..f62fe11 100644 --- a/backend/app/api/admin.py +++ b/backend/app/api/admin.py @@ -2,6 +2,7 @@ from fastapi import APIRouter, Depends, HTTPException from sqlalchemy.orm import Session from app.database import get_db from app.models import ScrapeLog, EmailLog, MealPlan +from app.services.scraper_service import ScraperService from typing import List, Optional from datetime import datetime, timedelta @@ -10,22 +11,10 @@ router = APIRouter() @router.post("/scrape") def trigger_scrape(source: str = "lucky_california", scrape_type: str = "weekly_ad", db: Session = Depends(get_db)): - scrape_log = ScrapeLog( - source=source, - scrape_type=scrape_type, - status="started", - started_at=datetime.now() - ) - db.add(scrape_log) - db.commit() - db.refresh(scrape_log) + scraper_service = ScraperService(db) + result = scraper_service.run_scrape(source=source, scrape_type=scrape_type) - return { - "message": "Scrape initiated", - "scrape_id": str(scrape_log.id), - "source": source, - "scrape_type": scrape_type - } + return result @router.get("/logs") diff --git a/backend/app/scraper/__init__.py b/backend/app/scraper/__init__.py new file mode 100644 index 0000000..8f9829f --- /dev/null +++ b/backend/app/scraper/__init__.py @@ -0,0 +1,4 @@ +from .base import BaseScraper +from .lucky_ca_scraper import LuckyCaliforniaScraper + +__all__ = ["BaseScraper", "LuckyCaliforniaScraper"] \ No newline at end of file diff --git a/backend/app/scraper/base.py b/backend/app/scraper/base.py new file mode 100644 index 0000000..1c3dc5e --- /dev/null +++ b/backend/app/scraper/base.py @@ -0,0 +1,138 @@ +import time +import logging +from abc import ABC, abstractmethod +from typing import Optional, List, Dict, Any +from datetime import datetime +from urllib.parse import urljoin +import requests +from requests.adapters import HTTPAdapter +from urllib3.util.retry import Retry + +logger = logging.getLogger(__name__) + + +class BaseScraper(ABC): + def __init__( + self, + base_url: str, + rate_limit_seconds: float = 2.0, + max_retries: int = 3, + timeout: int = 30 + ): + self.base_url = base_url.rstrip("/") + self.rate_limit_seconds = rate_limit_seconds + self.max_retries = max_retries + self.timeout = timeout + self.last_request_time = 0.0 + self.session = self._create_session() + + def _create_session(self) -> requests.Session: + session = requests.Session() + + retry_strategy = Retry( + total=self.max_retries, + backoff_factor=1.0, + status_forcelist=[429, 500, 502, 503, 504], + allowed_methods=["HEAD", "GET", "OPTIONS"] + ) + + adapter = HTTPAdapter(max_retries=retry_strategy, pool_maxsize=10) + session.mount("http://", adapter) + session.mount("https://", adapter) + + session.headers.update({ + "User-Agent": "Mozilla/5.0 (compatible; MealPlannerBot/1.0; +https://mealplanner.local)", + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", + "Accept-Language": "en-US,en;q=0.5", + }) + + return session + + def _rate_limit(self) -> None: + elapsed = time.time() - self.last_request_time + if elapsed < self.rate_limit_seconds: + sleep_time = self.rate_limit_seconds - elapsed + logger.debug(f"Rate limiting: sleeping {sleep_time:.2f}s") + time.sleep(sleep_time) + self.last_request_time = time.time() + + def _get(self, url: str, **kwargs) -> Optional[requests.Response]: + self._rate_limit() + + full_url = url if url.startswith("http") else urljoin(self.base_url, url) + + for attempt in range(self.max_retries + 1): + try: + logger.debug(f"GET {full_url} (attempt {attempt + 1})") + response = self.session.get(full_url, timeout=self.timeout, **kwargs) + response.raise_for_status() + return response + + except requests.exceptions.RequestException as e: + logger.warning(f"Request failed (attempt {attempt + 1}): {e}") + if attempt >= self.max_retries: + logger.error(f"Max retries reached for {full_url}") + return None + time.sleep(2 ** attempt) + + return None + + def _post(self, url: str, data: Dict[str, Any] = None, **kwargs) -> Optional[requests.Response]: + self._rate_limit() + + full_url = url if url.startswith("http") else urljoin(self.base_url, url) + + for attempt in range(self.max_retries + 1): + try: + logger.debug(f"POST {full_url} (attempt {attempt + 1})") + response = self.session.post(full_url, data=data, timeout=self.timeout, **kwargs) + response.raise_for_status() + return response + + except requests.exceptions.RequestException as e: + logger.warning(f"POST request failed (attempt {attempt + 1}): {e}") + if attempt >= self.max_retries: + logger.error(f"Max retries reached for {full_url}") + return None + time.sleep(2 ** attempt) + + return None + + @abstractmethod + def scrape(self) -> Dict[str, Any]: + pass + + def log_scrape(self, status: str, items_scraped: int = 0, error_message: str = None) -> None: + logger.info(f"Scrape {status}: {items_scraped} items, error: {error_message}") + + +class SeleniumScraper(BaseScraper): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self._playwright = None + + def _init_playwright(self): + if self._playwright is None: + from playwright.sync_api import sync_playwright + self._playwright = sync_playwright().start() + self._browser = self._playwright.chromium.launch(headless=True) + self._context = self._browser.new_context( + user_agent="Mozilla/5.0 (compatible; MealPlannerBot/1.0)" + ) + + def _get_playwright(self): + self._init_playwright() + return self._playwright + + def get_browser_page(self, url: str): + self._get_playwright() + page = self._context.new_page() + page.goto(url, wait_until="networkidle", timeout=30000) + return page + + def cleanup(self): + if self._browser: + self._browser.close() + if self._playwright: + self._playwright.stop() + self._playwright = None \ No newline at end of file diff --git a/backend/app/scraper/lucky_ca_scraper.py b/backend/app/scraper/lucky_ca_scraper.py new file mode 100644 index 0000000..f231443 --- /dev/null +++ b/backend/app/scraper/lucky_ca_scraper.py @@ -0,0 +1,180 @@ +import logging +import re +from typing import Dict, Any, List, Optional +from datetime import datetime +from urllib.parse import urljoin +from bs4 import BeautifulSoup +from .base import SeleniumScraper + +logger = logging.getLogger(__name__) + + +class LuckyCaliforniaScraper(SeleniumScraper): + def __init__(self, base_url: str = "https://luckysupermarkets.com"): + super().__init__(base_url=base_url, rate_limit_seconds=3.0) + self.ingredients_cache = {} + + def scrape(self) -> Dict[str, Any]: + logger.info("Starting Lucky California scrape") + result = { + "source": "lucky_california", + "scrape_type": "weekly_ad", + "started_at": datetime.now().isoformat(), + "items_scraped": 0, + "items": [] + } + + try: + featured_items = self.scrape_featured_coupons() + result["items"].extend(featured_items) + result["items_scraped"] = len(featured_items) + + result["completed_at"] = datetime.now().isoformat() + result["status"] = "success" + logger.info(f"Lucky California scrape complete: {result['items_scraped']} items") + + except Exception as e: + logger.error(f"Lucky California scrape failed: {e}") + result["status"] = "failed" + result["error_message"] = str(e) + result["completed_at"] = datetime.now().isoformat() + + return result + + def scrape_featured_coupons(self) -> List[Dict[str, Any]]: + url = f"{self.base_url}/coupons/Coupon%2Flu-featured-in-ad" + logger.info(f"Scraping featured coupons from {url}") + + page = self.get_browser_page(url) + content = page.content() + page.close() + + soup = BeautifulSoup(content, "html.parser") + items = [] + + coupon_items = soup.find_all("div", class_=re.compile(r"coupon|item|product", re.I)) + if not coupon_items: + headline = soup.find("h1") + if headline: + logger.info(f"Page loaded, headline: {headline.get_text().strip()}") + + titles = soup.find_all(["h2", "h3", "a"], string=re.compile(r"\$[\d\.]+")) + for title_elem in titles[:20]: + try: + item = self._parse_coupon_item(title_elem) + if item: + items.append(item) + except Exception as e: + logger.debug(f"Failed to parse item: {e}") + continue + + logger.info(f"Found {len(items)} coupon items") + return items + + def _parse_coupon_item(self, element) -> Optional[Dict[str, Any]]: + text = element.get_text().strip() + + price_match = re.search(r'\$[\d,]+\.?\d*', text) + if not price_match: + return None + + price_str = price_match.group().replace("$", "").replace(",", "") + try: + price = float(price_str) + except ValueError: + return None + + name_elem = element.find_parent("a") or element.find_parent("div") + name = text.split("$")[0].strip() if "$" in text else text + name = re.sub(r'\s+', " ", name).strip()[:200] + + if not name or len(name) < 3: + return None + + image_url = None + img_elem = element.find_parent().find("img") if element.find_parent() else None + if img_elem and img_elem.get("src"): + image_url = img_elem["src"] + + product_url = None + link_elem = element.find_parent("a") if element.find_parent() else element.find("a") + if link_elem and link_elem.get("href"): + product_url = urljoin(self.base_url, link_elem["href"]) + + item = { + "name": name, + "current_price": price, + "image_url": image_url, + "product_url": product_url, + "is_on_sale": True, + "scraped_at": datetime.now().isoformat(), + "scraped_url": self.base_url + } + + return item + + def scrape_produce(self) -> List[Dict[str, Any]]: + url = f"{self.base_url}/coupons/Coupon%2Flu-produce" + logger.info(f"Scraping produce from {url}") + + page = self.get_browser_page(url) + content = page.content() + page.close() + + soup = BeautifulSoup(content, "html.parser") + items = [] + + for item_elem in soup.find_all("div", class_=re.compile(r"product|item|coupon")): + try: + item = self._parse_coupon_item(item_elem) + if item: + item["aisle"] = "Produce" + items.append(item) + except Exception: + continue + + return items + + def scrape_meat_seafood(self) -> List[Dict[str, Any]]: + url = f"{self.base_url}/coupons/Coupon%2Flu-meat-seafood" + logger.info(f"Scraping meat & seafood from {url}") + + page = self.get_browser_page(url) + content = page.content() + page.close() + + soup = BeautifulSoup(content, "html.parser") + items = [] + + for item_elem in soup.find_all("div", class_=re.compile(r"product|item|coupon")): + try: + item = self._parse_coupon_item(item_elem) + if item: + item["aisle"] = "Meat" + items.append(item) + except Exception: + continue + + return items + + def scrape_dairy_eggs(self) -> List[Dict[str, Any]]: + url = f"{self.base_url}/coupons/Coupon%2Flu-dairy-eggs" + logger.info(f"Scraping dairy & eggs from {url}") + + page = self.get_browser_page(url) + content = page.content() + page.close() + + soup = BeautifulSoup(content, "html.parser") + items = [] + + for item_elem in soup.find_all("div", class_=re.compile(r"product|item|coupon")): + try: + item = self._parse_coupon_item(item_elem) + if item: + item["aisle"] = "Dairy" + items.append(item) + except Exception: + continue + + return items \ No newline at end of file diff --git a/backend/app/services/scraper_service.py b/backend/app/services/scraper_service.py new file mode 100644 index 0000000..ed6f3e4 --- /dev/null +++ b/backend/app/services/scraper_service.py @@ -0,0 +1,158 @@ +import logging +from typing import Dict, Any, Optional +from datetime import datetime +from uuid import uuid4 +from sqlalchemy.orm import Session + +logger = logging.getLogger(__name__) + + +class ScraperService: + def __init__(self, db: Session): + self.db = db + + def run_scrape(self, source: str = "lucky_california", scrape_type: str = "weekly_ad") -> Dict[str, Any]: + from app.scraper import LuckyCaliforniaScraper + from app.models import ScrapeLog, GroceryItem, Ingredient + + scrape_log = ScrapeLog( + id=uuid4(), + source=source, + scrape_type=scrape_type, + status="started", + started_at=datetime.now() + ) + self.db.add(scrape_log) + self.db.commit() + + logger.info(f"Starting {source} {scrape_type} scrape") + + try: + scraper = LuckyCaliforniaScraper() + result = scraper.scrape() + scraper.cleanup() + + items = result.get("items", []) + saved_count = 0 + + for item_data in items: + saved_item = self._save_grocery_item(item_data) + if saved_item: + saved_count += 1 + + scrape_log.status = "success" + scrape_log.items_scraped = saved_count + scrape_log.completed_at = datetime.now() + scrape_log.duration_seconds = int( + (scrape_log.completed_at - scrape_log.started_at).total_seconds() + ) + + self.db.commit() + + logger.info(f"Scrape complete: {saved_count} items saved") + + return { + "scrape_id": str(scrape_log.id), + "status": "success", + "items_scraped": saved_count, + "items_found": len(items) + } + + except Exception as e: + logger.error(f"Scrape failed: {e}") + scrape_log.status = "failed" + scrape_log.error_message = str(e) + scrape_log.completed_at = datetime.now() + scrape_log.duration_seconds = int( + (scrape_log.completed_at - scrape_log.started_at).total_seconds() + ) + self.db.commit() + + return { + "scrape_id": str(scrape_log.id), + "status": "failed", + "error": str(e) + } + + def _save_grocery_item(self, item_data: Dict[str, Any]) -> Optional[GroceryItem]: + from app.models import GroceryItem, Ingredient + + name = item_data.get("name", "").strip() + if not name: + return None + + name_lower = name.lower() + + ingredient = self.db.query(Ingredient).filter( + Ingredient.name_lower == name_lower + ).first() + + if not ingredient: + ingredient = Ingredient( + id=uuid4(), + name=name, + name_lower=name_lower, + aisle=item_data.get("aisle"), + typical_price=item_data.get("current_price") + ) + self.db.add(ingredient) + self.db.flush() + + existing = self.db.query(GroceryItem).filter( + GroceryItem.name == name, + GroceryItem.scraped_url == item_data.get("scraped_url") + ).first() + + if existing: + existing.current_price = item_data.get("current_price") + existing.is_on_sale = item_data.get("is_on_sale", True) + existing.image_url = item_data.get("image_url") + existing.product_url = item_data.get("product_url") + existing.scraped_at = datetime.now() + self.db.flush() + return existing + + grocery_item = GroceryItem( + id=uuid4(), + ingredient_id=ingredient.id, + name=name, + current_price=item_data.get("current_price"), + regular_price=item_data.get("regular_price"), + unit=item_data.get("unit"), + aisle=item_data.get("aisle"), + image_url=item_data.get("image_url"), + product_url=item_data.get("product_url"), + is_on_sale=item_data.get("is_on_sale", True), + sale_start_date=item_data.get("sale_start_date"), + sale_end_date=item_data.get("sale_end_date"), + in_season=item_data.get("in_season", False), + scraped_at=datetime.now(), + scraped_url=item_data.get("scraped_url") + ) + + self.db.add(grocery_item) + self.db.commit() + self.db.refresh(grocery_item) + + return grocery_item + + def get_sale_items(self, limit: int = 50) -> list: + from app.models import GroceryItem + + items = self.db.query(GroceryItem).filter( + GroceryItem.is_on_sale == True + ).order_by(GroceryItem.scraped_at.desc()).limit(limit).all() + + return [ + { + "id": str(item.id), + "name": item.name, + "current_price": float(item.current_price) if item.current_price else None, + "regular_price": float(item.regular_price) if item.regular_price else None, + "aisle": item.aisle, + "image_url": item.image_url, + "product_url": item.product_url, + "scraped_at": item.scraped_at.isoformat() if item.scraped_at else None + } + for item in items + ] \ No newline at end of file diff --git a/docs/ORIENTATION.md b/docs/ORIENTATION.md index 728a2f9..fb4d13b 100644 --- a/docs/ORIENTATION.md +++ b/docs/ORIENTATION.md @@ -44,6 +44,12 @@ The family has been using meal kit services (Blue Apron → EveryPlate → Hungr - [x] /api/shopping-list endpoints (aggregation, print) - [x] /api/admin endpoints (scrape trigger, logs, stats) +**Phase 3 Progress** (2026-05-04): +- [x] Base scraper with rate limiting, retries, session management +- [x] LuckyCaliforniaScraper with BeautifulSoup + Playwright +- [x] ScraperService to save scraped items to grocery_item table +- [x] /api/admin/scrape endpoint connected to ScraperService + --- ## Architecture Summary @@ -132,17 +138,16 @@ User (web) ───► React UI ──► nginx ──► FastAPI ──── | Phase | Description | Status | |-------|-------------|--------| | 1 | Infrastructure (Docker, PostgreSQL, FastAPI, React, nginx) | **Complete** | -| 2 | Database & Models (SQLAlchemy models, Alembic migrations) | **Post-review fixes applied** | -| 3 | API Endpoints (CRUD, meal plans, shopping list, feedback) | Not Started | -| 4 | Lucky California Scraper (weekly ad, Playwright) | Not Started | -| 5 | Recipe Engine (CRUD, tagging, search) | Not Started | -| 6 | Meal Planner Engine (generation algorithm, substitutions) | Not Started | -| 7 | SendGrid Email Integration | Not Started | -| 8 | Web UI - Core (Dashboard, Meal Detail, Approval, Pantry) | Not Started | -| 9 | Web UI - Feedback (Feedback Portal, Learning) | Not Started | -| 10 | Shopping List & Print | Not Started | -| 11 | Image Strategy (scraped + AI fallback) | Not Started | -| 12 | Polish & Future (variety analysis, budget tracking) | Not Started | +| 2 | Database & Models (Alembic migrations, Pydantic schemas, API endpoints) | **Complete** | +| 3 | Lucky California Scraper (BeautifulSoup + Playwright, ScraperService) | **Complete** | +| 4 | Recipe Engine (CRUD, tagging, search) | Not Started | +| 5 | Meal Planner Engine (generation algorithm, substitutions) | Not Started | +| 6 | SendGrid Email Integration | Not Started | +| 7 | Web UI - Core (Dashboard, Meal Detail, Approval, Pantry) | Not Started | +| 8 | Web UI - Feedback (Feedback Portal, Learning) | Not Started | +| 9 | Shopping List & Print | Not Started | +| 10 | Image Strategy (scraped + AI fallback) | Not Started | +| 11 | Polish & Future (variety analysis, budget tracking) | Not Started | ---