Public Access
feat: implement Lucky California scraper with Playwright + BeautifulSoup
- Add BaseScraper with rate limiting, retries, session management - Add LuckyCaliforniaScraper with Playwright for dynamic content - Add ScraperService to save scraped items to grocery_item table - Connect /api/admin/scrape to ScraperService - Update ORIENTATION.md phase table
This commit is contained in:
@@ -2,6 +2,7 @@ from fastapi import APIRouter, Depends, HTTPException
|
||||
from sqlalchemy.orm import Session
|
||||
from app.database import get_db
|
||||
from app.models import ScrapeLog, EmailLog, MealPlan
|
||||
from app.services.scraper_service import ScraperService
|
||||
from typing import List, Optional
|
||||
from datetime import datetime, timedelta
|
||||
|
||||
@@ -10,22 +11,10 @@ router = APIRouter()
|
||||
|
||||
@router.post("/scrape")
|
||||
def trigger_scrape(source: str = "lucky_california", scrape_type: str = "weekly_ad", db: Session = Depends(get_db)):
|
||||
scrape_log = ScrapeLog(
|
||||
source=source,
|
||||
scrape_type=scrape_type,
|
||||
status="started",
|
||||
started_at=datetime.now()
|
||||
)
|
||||
db.add(scrape_log)
|
||||
db.commit()
|
||||
db.refresh(scrape_log)
|
||||
scraper_service = ScraperService(db)
|
||||
result = scraper_service.run_scrape(source=source, scrape_type=scrape_type)
|
||||
|
||||
return {
|
||||
"message": "Scrape initiated",
|
||||
"scrape_id": str(scrape_log.id),
|
||||
"source": source,
|
||||
"scrape_type": scrape_type
|
||||
}
|
||||
return result
|
||||
|
||||
|
||||
@router.get("/logs")
|
||||
|
||||
Reference in New Issue
Block a user