import logging from typing import Dict, Any, Optional from datetime import datetime from uuid import uuid4 from sqlalchemy.orm import Session logger = logging.getLogger(__name__) class ScraperService: def __init__(self, db: Session): self.db = db def run_scrape(self, source: str = "lucky_california", scrape_type: str = "weekly_ad") -> Dict[str, Any]: from app.scraper import LuckyCaliforniaScraper from app.models import ScrapeLog, GroceryItem, Ingredient scrape_log = ScrapeLog( id=uuid4(), source=source, scrape_type=scrape_type, status="started", started_at=datetime.now() ) self.db.add(scrape_log) self.db.commit() logger.info(f"Starting {source} {scrape_type} scrape") try: scraper = LuckyCaliforniaScraper() result = scraper.scrape() scraper.cleanup() items = result.get("items", []) saved_count = 0 for item_data in items: saved_item = self._save_grocery_item(item_data) if saved_item: saved_count += 1 scrape_log.status = "success" scrape_log.items_scraped = saved_count scrape_log.completed_at = datetime.now() scrape_log.duration_seconds = int( (scrape_log.completed_at - scrape_log.started_at).total_seconds() ) self.db.commit() logger.info(f"Scrape complete: {saved_count} items saved") return { "scrape_id": str(scrape_log.id), "status": "success", "items_scraped": saved_count, "items_found": len(items) } except Exception as e: logger.error(f"Scrape failed: {e}") scrape_log.status = "failed" scrape_log.error_message = str(e) scrape_log.completed_at = datetime.now() scrape_log.duration_seconds = int( (scrape_log.completed_at - scrape_log.started_at).total_seconds() ) self.db.commit() return { "scrape_id": str(scrape_log.id), "status": "failed", "error": str(e) } def _save_grocery_item(self, item_data: Dict[str, Any]) -> Optional[GroceryItem]: from app.models import GroceryItem, Ingredient name = item_data.get("name", "").strip() if not name: return None name_lower = name.lower() ingredient = self.db.query(Ingredient).filter( Ingredient.name_lower == name_lower ).first() if not ingredient: ingredient = Ingredient( id=uuid4(), name=name, name_lower=name_lower, aisle=item_data.get("aisle"), typical_price=item_data.get("current_price") ) self.db.add(ingredient) self.db.flush() existing = self.db.query(GroceryItem).filter( GroceryItem.name == name, GroceryItem.scraped_url == item_data.get("scraped_url") ).first() if existing: existing.current_price = item_data.get("current_price") existing.is_on_sale = item_data.get("is_on_sale", True) existing.image_url = item_data.get("image_url") existing.product_url = item_data.get("product_url") existing.scraped_at = datetime.now() self.db.flush() return existing grocery_item = GroceryItem( id=uuid4(), ingredient_id=ingredient.id, name=name, current_price=item_data.get("current_price"), regular_price=item_data.get("regular_price"), unit=item_data.get("unit"), aisle=item_data.get("aisle"), image_url=item_data.get("image_url"), product_url=item_data.get("product_url"), is_on_sale=item_data.get("is_on_sale", True), sale_start_date=item_data.get("sale_start_date"), sale_end_date=item_data.get("sale_end_date"), in_season=item_data.get("in_season", False), scraped_at=datetime.now(), scraped_url=item_data.get("scraped_url") ) self.db.add(grocery_item) self.db.commit() self.db.refresh(grocery_item) return grocery_item def get_sale_items(self, limit: int = 50) -> list: from app.models import GroceryItem items = self.db.query(GroceryItem).filter( GroceryItem.is_on_sale == True ).order_by(GroceryItem.scraped_at.desc()).limit(limit).all() return [ { "id": str(item.id), "name": item.name, "current_price": float(item.current_price) if item.current_price else None, "regular_price": float(item.regular_price) if item.regular_price else None, "aisle": item.aisle, "image_url": item.image_url, "product_url": item.product_url, "scraped_at": item.scraped_at.isoformat() if item.scraped_at else None } for item in items ]