Public Access
feat: implement Lucky California scraper with Playwright + BeautifulSoup
- Add BaseScraper with rate limiting, retries, session management - Add LuckyCaliforniaScraper with Playwright for dynamic content - Add ScraperService to save scraped items to grocery_item table - Connect /api/admin/scrape to ScraperService - Update ORIENTATION.md phase table
This commit is contained in:
@@ -0,0 +1,158 @@
|
||||
import logging
|
||||
from typing import Dict, Any, Optional
|
||||
from datetime import datetime
|
||||
from uuid import uuid4
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class ScraperService:
|
||||
def __init__(self, db: Session):
|
||||
self.db = db
|
||||
|
||||
def run_scrape(self, source: str = "lucky_california", scrape_type: str = "weekly_ad") -> Dict[str, Any]:
|
||||
from app.scraper import LuckyCaliforniaScraper
|
||||
from app.models import ScrapeLog, GroceryItem, Ingredient
|
||||
|
||||
scrape_log = ScrapeLog(
|
||||
id=uuid4(),
|
||||
source=source,
|
||||
scrape_type=scrape_type,
|
||||
status="started",
|
||||
started_at=datetime.now()
|
||||
)
|
||||
self.db.add(scrape_log)
|
||||
self.db.commit()
|
||||
|
||||
logger.info(f"Starting {source} {scrape_type} scrape")
|
||||
|
||||
try:
|
||||
scraper = LuckyCaliforniaScraper()
|
||||
result = scraper.scrape()
|
||||
scraper.cleanup()
|
||||
|
||||
items = result.get("items", [])
|
||||
saved_count = 0
|
||||
|
||||
for item_data in items:
|
||||
saved_item = self._save_grocery_item(item_data)
|
||||
if saved_item:
|
||||
saved_count += 1
|
||||
|
||||
scrape_log.status = "success"
|
||||
scrape_log.items_scraped = saved_count
|
||||
scrape_log.completed_at = datetime.now()
|
||||
scrape_log.duration_seconds = int(
|
||||
(scrape_log.completed_at - scrape_log.started_at).total_seconds()
|
||||
)
|
||||
|
||||
self.db.commit()
|
||||
|
||||
logger.info(f"Scrape complete: {saved_count} items saved")
|
||||
|
||||
return {
|
||||
"scrape_id": str(scrape_log.id),
|
||||
"status": "success",
|
||||
"items_scraped": saved_count,
|
||||
"items_found": len(items)
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Scrape failed: {e}")
|
||||
scrape_log.status = "failed"
|
||||
scrape_log.error_message = str(e)
|
||||
scrape_log.completed_at = datetime.now()
|
||||
scrape_log.duration_seconds = int(
|
||||
(scrape_log.completed_at - scrape_log.started_at).total_seconds()
|
||||
)
|
||||
self.db.commit()
|
||||
|
||||
return {
|
||||
"scrape_id": str(scrape_log.id),
|
||||
"status": "failed",
|
||||
"error": str(e)
|
||||
}
|
||||
|
||||
def _save_grocery_item(self, item_data: Dict[str, Any]) -> Optional[GroceryItem]:
|
||||
from app.models import GroceryItem, Ingredient
|
||||
|
||||
name = item_data.get("name", "").strip()
|
||||
if not name:
|
||||
return None
|
||||
|
||||
name_lower = name.lower()
|
||||
|
||||
ingredient = self.db.query(Ingredient).filter(
|
||||
Ingredient.name_lower == name_lower
|
||||
).first()
|
||||
|
||||
if not ingredient:
|
||||
ingredient = Ingredient(
|
||||
id=uuid4(),
|
||||
name=name,
|
||||
name_lower=name_lower,
|
||||
aisle=item_data.get("aisle"),
|
||||
typical_price=item_data.get("current_price")
|
||||
)
|
||||
self.db.add(ingredient)
|
||||
self.db.flush()
|
||||
|
||||
existing = self.db.query(GroceryItem).filter(
|
||||
GroceryItem.name == name,
|
||||
GroceryItem.scraped_url == item_data.get("scraped_url")
|
||||
).first()
|
||||
|
||||
if existing:
|
||||
existing.current_price = item_data.get("current_price")
|
||||
existing.is_on_sale = item_data.get("is_on_sale", True)
|
||||
existing.image_url = item_data.get("image_url")
|
||||
existing.product_url = item_data.get("product_url")
|
||||
existing.scraped_at = datetime.now()
|
||||
self.db.flush()
|
||||
return existing
|
||||
|
||||
grocery_item = GroceryItem(
|
||||
id=uuid4(),
|
||||
ingredient_id=ingredient.id,
|
||||
name=name,
|
||||
current_price=item_data.get("current_price"),
|
||||
regular_price=item_data.get("regular_price"),
|
||||
unit=item_data.get("unit"),
|
||||
aisle=item_data.get("aisle"),
|
||||
image_url=item_data.get("image_url"),
|
||||
product_url=item_data.get("product_url"),
|
||||
is_on_sale=item_data.get("is_on_sale", True),
|
||||
sale_start_date=item_data.get("sale_start_date"),
|
||||
sale_end_date=item_data.get("sale_end_date"),
|
||||
in_season=item_data.get("in_season", False),
|
||||
scraped_at=datetime.now(),
|
||||
scraped_url=item_data.get("scraped_url")
|
||||
)
|
||||
|
||||
self.db.add(grocery_item)
|
||||
self.db.commit()
|
||||
self.db.refresh(grocery_item)
|
||||
|
||||
return grocery_item
|
||||
|
||||
def get_sale_items(self, limit: int = 50) -> list:
|
||||
from app.models import GroceryItem
|
||||
|
||||
items = self.db.query(GroceryItem).filter(
|
||||
GroceryItem.is_on_sale == True
|
||||
).order_by(GroceryItem.scraped_at.desc()).limit(limit).all()
|
||||
|
||||
return [
|
||||
{
|
||||
"id": str(item.id),
|
||||
"name": item.name,
|
||||
"current_price": float(item.current_price) if item.current_price else None,
|
||||
"regular_price": float(item.regular_price) if item.regular_price else None,
|
||||
"aisle": item.aisle,
|
||||
"image_url": item.image_url,
|
||||
"product_url": item.product_url,
|
||||
"scraped_at": item.scraped_at.isoformat() if item.scraped_at else None
|
||||
}
|
||||
for item in items
|
||||
]
|
||||
Reference in New Issue
Block a user