feat: implement Lucky California scraper with Playwright + BeautifulSoup

- Add BaseScraper with rate limiting, retries, session management
- Add LuckyCaliforniaScraper with Playwright for dynamic content
- Add ScraperService to save scraped items to grocery_item table
- Connect /api/admin/scrape to ScraperService
- Update ORIENTATION.md phase table
This commit is contained in:
2026-05-04 20:50:27 -07:00
parent c735d21661
commit 933a0cc9db
6 changed files with 500 additions and 26 deletions
+158
View File
@@ -0,0 +1,158 @@
import logging
from typing import Dict, Any, Optional
from datetime import datetime
from uuid import uuid4
from sqlalchemy.orm import Session
logger = logging.getLogger(__name__)
class ScraperService:
def __init__(self, db: Session):
self.db = db
def run_scrape(self, source: str = "lucky_california", scrape_type: str = "weekly_ad") -> Dict[str, Any]:
from app.scraper import LuckyCaliforniaScraper
from app.models import ScrapeLog, GroceryItem, Ingredient
scrape_log = ScrapeLog(
id=uuid4(),
source=source,
scrape_type=scrape_type,
status="started",
started_at=datetime.now()
)
self.db.add(scrape_log)
self.db.commit()
logger.info(f"Starting {source} {scrape_type} scrape")
try:
scraper = LuckyCaliforniaScraper()
result = scraper.scrape()
scraper.cleanup()
items = result.get("items", [])
saved_count = 0
for item_data in items:
saved_item = self._save_grocery_item(item_data)
if saved_item:
saved_count += 1
scrape_log.status = "success"
scrape_log.items_scraped = saved_count
scrape_log.completed_at = datetime.now()
scrape_log.duration_seconds = int(
(scrape_log.completed_at - scrape_log.started_at).total_seconds()
)
self.db.commit()
logger.info(f"Scrape complete: {saved_count} items saved")
return {
"scrape_id": str(scrape_log.id),
"status": "success",
"items_scraped": saved_count,
"items_found": len(items)
}
except Exception as e:
logger.error(f"Scrape failed: {e}")
scrape_log.status = "failed"
scrape_log.error_message = str(e)
scrape_log.completed_at = datetime.now()
scrape_log.duration_seconds = int(
(scrape_log.completed_at - scrape_log.started_at).total_seconds()
)
self.db.commit()
return {
"scrape_id": str(scrape_log.id),
"status": "failed",
"error": str(e)
}
def _save_grocery_item(self, item_data: Dict[str, Any]) -> Optional[GroceryItem]:
from app.models import GroceryItem, Ingredient
name = item_data.get("name", "").strip()
if not name:
return None
name_lower = name.lower()
ingredient = self.db.query(Ingredient).filter(
Ingredient.name_lower == name_lower
).first()
if not ingredient:
ingredient = Ingredient(
id=uuid4(),
name=name,
name_lower=name_lower,
aisle=item_data.get("aisle"),
typical_price=item_data.get("current_price")
)
self.db.add(ingredient)
self.db.flush()
existing = self.db.query(GroceryItem).filter(
GroceryItem.name == name,
GroceryItem.scraped_url == item_data.get("scraped_url")
).first()
if existing:
existing.current_price = item_data.get("current_price")
existing.is_on_sale = item_data.get("is_on_sale", True)
existing.image_url = item_data.get("image_url")
existing.product_url = item_data.get("product_url")
existing.scraped_at = datetime.now()
self.db.flush()
return existing
grocery_item = GroceryItem(
id=uuid4(),
ingredient_id=ingredient.id,
name=name,
current_price=item_data.get("current_price"),
regular_price=item_data.get("regular_price"),
unit=item_data.get("unit"),
aisle=item_data.get("aisle"),
image_url=item_data.get("image_url"),
product_url=item_data.get("product_url"),
is_on_sale=item_data.get("is_on_sale", True),
sale_start_date=item_data.get("sale_start_date"),
sale_end_date=item_data.get("sale_end_date"),
in_season=item_data.get("in_season", False),
scraped_at=datetime.now(),
scraped_url=item_data.get("scraped_url")
)
self.db.add(grocery_item)
self.db.commit()
self.db.refresh(grocery_item)
return grocery_item
def get_sale_items(self, limit: int = 50) -> list:
from app.models import GroceryItem
items = self.db.query(GroceryItem).filter(
GroceryItem.is_on_sale == True
).order_by(GroceryItem.scraped_at.desc()).limit(limit).all()
return [
{
"id": str(item.id),
"name": item.name,
"current_price": float(item.current_price) if item.current_price else None,
"regular_price": float(item.regular_price) if item.regular_price else None,
"aisle": item.aisle,
"image_url": item.image_url,
"product_url": item.product_url,
"scraped_at": item.scraped_at.isoformat() if item.scraped_at else None
}
for item in items
]