Public Access
feat: implement Lucky California scraper with Playwright + BeautifulSoup
- Add BaseScraper with rate limiting, retries, session management - Add LuckyCaliforniaScraper with Playwright for dynamic content - Add ScraperService to save scraped items to grocery_item table - Connect /api/admin/scrape to ScraperService - Update ORIENTATION.md phase table
This commit is contained in:
@@ -0,0 +1,4 @@
|
||||
from .base import BaseScraper
|
||||
from .lucky_ca_scraper import LuckyCaliforniaScraper
|
||||
|
||||
__all__ = ["BaseScraper", "LuckyCaliforniaScraper"]
|
||||
@@ -0,0 +1,138 @@
|
||||
import time
|
||||
import logging
|
||||
from abc import ABC, abstractmethod
|
||||
from typing import Optional, List, Dict, Any
|
||||
from datetime import datetime
|
||||
from urllib.parse import urljoin
|
||||
import requests
|
||||
from requests.adapters import HTTPAdapter
|
||||
from urllib3.util.retry import Retry
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class BaseScraper(ABC):
|
||||
def __init__(
|
||||
self,
|
||||
base_url: str,
|
||||
rate_limit_seconds: float = 2.0,
|
||||
max_retries: int = 3,
|
||||
timeout: int = 30
|
||||
):
|
||||
self.base_url = base_url.rstrip("/")
|
||||
self.rate_limit_seconds = rate_limit_seconds
|
||||
self.max_retries = max_retries
|
||||
self.timeout = timeout
|
||||
self.last_request_time = 0.0
|
||||
self.session = self._create_session()
|
||||
|
||||
def _create_session(self) -> requests.Session:
|
||||
session = requests.Session()
|
||||
|
||||
retry_strategy = Retry(
|
||||
total=self.max_retries,
|
||||
backoff_factor=1.0,
|
||||
status_forcelist=[429, 500, 502, 503, 504],
|
||||
allowed_methods=["HEAD", "GET", "OPTIONS"]
|
||||
)
|
||||
|
||||
adapter = HTTPAdapter(max_retries=retry_strategy, pool_maxsize=10)
|
||||
session.mount("http://", adapter)
|
||||
session.mount("https://", adapter)
|
||||
|
||||
session.headers.update({
|
||||
"User-Agent": "Mozilla/5.0 (compatible; MealPlannerBot/1.0; +https://mealplanner.local)",
|
||||
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
|
||||
"Accept-Language": "en-US,en;q=0.5",
|
||||
})
|
||||
|
||||
return session
|
||||
|
||||
def _rate_limit(self) -> None:
|
||||
elapsed = time.time() - self.last_request_time
|
||||
if elapsed < self.rate_limit_seconds:
|
||||
sleep_time = self.rate_limit_seconds - elapsed
|
||||
logger.debug(f"Rate limiting: sleeping {sleep_time:.2f}s")
|
||||
time.sleep(sleep_time)
|
||||
self.last_request_time = time.time()
|
||||
|
||||
def _get(self, url: str, **kwargs) -> Optional[requests.Response]:
|
||||
self._rate_limit()
|
||||
|
||||
full_url = url if url.startswith("http") else urljoin(self.base_url, url)
|
||||
|
||||
for attempt in range(self.max_retries + 1):
|
||||
try:
|
||||
logger.debug(f"GET {full_url} (attempt {attempt + 1})")
|
||||
response = self.session.get(full_url, timeout=self.timeout, **kwargs)
|
||||
response.raise_for_status()
|
||||
return response
|
||||
|
||||
except requests.exceptions.RequestException as e:
|
||||
logger.warning(f"Request failed (attempt {attempt + 1}): {e}")
|
||||
if attempt >= self.max_retries:
|
||||
logger.error(f"Max retries reached for {full_url}")
|
||||
return None
|
||||
time.sleep(2 ** attempt)
|
||||
|
||||
return None
|
||||
|
||||
def _post(self, url: str, data: Dict[str, Any] = None, **kwargs) -> Optional[requests.Response]:
|
||||
self._rate_limit()
|
||||
|
||||
full_url = url if url.startswith("http") else urljoin(self.base_url, url)
|
||||
|
||||
for attempt in range(self.max_retries + 1):
|
||||
try:
|
||||
logger.debug(f"POST {full_url} (attempt {attempt + 1})")
|
||||
response = self.session.post(full_url, data=data, timeout=self.timeout, **kwargs)
|
||||
response.raise_for_status()
|
||||
return response
|
||||
|
||||
except requests.exceptions.RequestException as e:
|
||||
logger.warning(f"POST request failed (attempt {attempt + 1}): {e}")
|
||||
if attempt >= self.max_retries:
|
||||
logger.error(f"Max retries reached for {full_url}")
|
||||
return None
|
||||
time.sleep(2 ** attempt)
|
||||
|
||||
return None
|
||||
|
||||
@abstractmethod
|
||||
def scrape(self) -> Dict[str, Any]:
|
||||
pass
|
||||
|
||||
def log_scrape(self, status: str, items_scraped: int = 0, error_message: str = None) -> None:
|
||||
logger.info(f"Scrape {status}: {items_scraped} items, error: {error_message}")
|
||||
|
||||
|
||||
class SeleniumScraper(BaseScraper):
|
||||
def __init__(self, *args, **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
self._playwright = None
|
||||
|
||||
def _init_playwright(self):
|
||||
if self._playwright is None:
|
||||
from playwright.sync_api import sync_playwright
|
||||
self._playwright = sync_playwright().start()
|
||||
self._browser = self._playwright.chromium.launch(headless=True)
|
||||
self._context = self._browser.new_context(
|
||||
user_agent="Mozilla/5.0 (compatible; MealPlannerBot/1.0)"
|
||||
)
|
||||
|
||||
def _get_playwright(self):
|
||||
self._init_playwright()
|
||||
return self._playwright
|
||||
|
||||
def get_browser_page(self, url: str):
|
||||
self._get_playwright()
|
||||
page = self._context.new_page()
|
||||
page.goto(url, wait_until="networkidle", timeout=30000)
|
||||
return page
|
||||
|
||||
def cleanup(self):
|
||||
if self._browser:
|
||||
self._browser.close()
|
||||
if self._playwright:
|
||||
self._playwright.stop()
|
||||
self._playwright = None
|
||||
@@ -0,0 +1,180 @@
|
||||
import logging
|
||||
import re
|
||||
from typing import Dict, Any, List, Optional
|
||||
from datetime import datetime
|
||||
from urllib.parse import urljoin
|
||||
from bs4 import BeautifulSoup
|
||||
from .base import SeleniumScraper
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class LuckyCaliforniaScraper(SeleniumScraper):
|
||||
def __init__(self, base_url: str = "https://luckysupermarkets.com"):
|
||||
super().__init__(base_url=base_url, rate_limit_seconds=3.0)
|
||||
self.ingredients_cache = {}
|
||||
|
||||
def scrape(self) -> Dict[str, Any]:
|
||||
logger.info("Starting Lucky California scrape")
|
||||
result = {
|
||||
"source": "lucky_california",
|
||||
"scrape_type": "weekly_ad",
|
||||
"started_at": datetime.now().isoformat(),
|
||||
"items_scraped": 0,
|
||||
"items": []
|
||||
}
|
||||
|
||||
try:
|
||||
featured_items = self.scrape_featured_coupons()
|
||||
result["items"].extend(featured_items)
|
||||
result["items_scraped"] = len(featured_items)
|
||||
|
||||
result["completed_at"] = datetime.now().isoformat()
|
||||
result["status"] = "success"
|
||||
logger.info(f"Lucky California scrape complete: {result['items_scraped']} items")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Lucky California scrape failed: {e}")
|
||||
result["status"] = "failed"
|
||||
result["error_message"] = str(e)
|
||||
result["completed_at"] = datetime.now().isoformat()
|
||||
|
||||
return result
|
||||
|
||||
def scrape_featured_coupons(self) -> List[Dict[str, Any]]:
|
||||
url = f"{self.base_url}/coupons/Coupon%2Flu-featured-in-ad"
|
||||
logger.info(f"Scraping featured coupons from {url}")
|
||||
|
||||
page = self.get_browser_page(url)
|
||||
content = page.content()
|
||||
page.close()
|
||||
|
||||
soup = BeautifulSoup(content, "html.parser")
|
||||
items = []
|
||||
|
||||
coupon_items = soup.find_all("div", class_=re.compile(r"coupon|item|product", re.I))
|
||||
if not coupon_items:
|
||||
headline = soup.find("h1")
|
||||
if headline:
|
||||
logger.info(f"Page loaded, headline: {headline.get_text().strip()}")
|
||||
|
||||
titles = soup.find_all(["h2", "h3", "a"], string=re.compile(r"\$[\d\.]+"))
|
||||
for title_elem in titles[:20]:
|
||||
try:
|
||||
item = self._parse_coupon_item(title_elem)
|
||||
if item:
|
||||
items.append(item)
|
||||
except Exception as e:
|
||||
logger.debug(f"Failed to parse item: {e}")
|
||||
continue
|
||||
|
||||
logger.info(f"Found {len(items)} coupon items")
|
||||
return items
|
||||
|
||||
def _parse_coupon_item(self, element) -> Optional[Dict[str, Any]]:
|
||||
text = element.get_text().strip()
|
||||
|
||||
price_match = re.search(r'\$[\d,]+\.?\d*', text)
|
||||
if not price_match:
|
||||
return None
|
||||
|
||||
price_str = price_match.group().replace("$", "").replace(",", "")
|
||||
try:
|
||||
price = float(price_str)
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
name_elem = element.find_parent("a") or element.find_parent("div")
|
||||
name = text.split("$")[0].strip() if "$" in text else text
|
||||
name = re.sub(r'\s+', " ", name).strip()[:200]
|
||||
|
||||
if not name or len(name) < 3:
|
||||
return None
|
||||
|
||||
image_url = None
|
||||
img_elem = element.find_parent().find("img") if element.find_parent() else None
|
||||
if img_elem and img_elem.get("src"):
|
||||
image_url = img_elem["src"]
|
||||
|
||||
product_url = None
|
||||
link_elem = element.find_parent("a") if element.find_parent() else element.find("a")
|
||||
if link_elem and link_elem.get("href"):
|
||||
product_url = urljoin(self.base_url, link_elem["href"])
|
||||
|
||||
item = {
|
||||
"name": name,
|
||||
"current_price": price,
|
||||
"image_url": image_url,
|
||||
"product_url": product_url,
|
||||
"is_on_sale": True,
|
||||
"scraped_at": datetime.now().isoformat(),
|
||||
"scraped_url": self.base_url
|
||||
}
|
||||
|
||||
return item
|
||||
|
||||
def scrape_produce(self) -> List[Dict[str, Any]]:
|
||||
url = f"{self.base_url}/coupons/Coupon%2Flu-produce"
|
||||
logger.info(f"Scraping produce from {url}")
|
||||
|
||||
page = self.get_browser_page(url)
|
||||
content = page.content()
|
||||
page.close()
|
||||
|
||||
soup = BeautifulSoup(content, "html.parser")
|
||||
items = []
|
||||
|
||||
for item_elem in soup.find_all("div", class_=re.compile(r"product|item|coupon")):
|
||||
try:
|
||||
item = self._parse_coupon_item(item_elem)
|
||||
if item:
|
||||
item["aisle"] = "Produce"
|
||||
items.append(item)
|
||||
except Exception:
|
||||
continue
|
||||
|
||||
return items
|
||||
|
||||
def scrape_meat_seafood(self) -> List[Dict[str, Any]]:
|
||||
url = f"{self.base_url}/coupons/Coupon%2Flu-meat-seafood"
|
||||
logger.info(f"Scraping meat & seafood from {url}")
|
||||
|
||||
page = self.get_browser_page(url)
|
||||
content = page.content()
|
||||
page.close()
|
||||
|
||||
soup = BeautifulSoup(content, "html.parser")
|
||||
items = []
|
||||
|
||||
for item_elem in soup.find_all("div", class_=re.compile(r"product|item|coupon")):
|
||||
try:
|
||||
item = self._parse_coupon_item(item_elem)
|
||||
if item:
|
||||
item["aisle"] = "Meat"
|
||||
items.append(item)
|
||||
except Exception:
|
||||
continue
|
||||
|
||||
return items
|
||||
|
||||
def scrape_dairy_eggs(self) -> List[Dict[str, Any]]:
|
||||
url = f"{self.base_url}/coupons/Coupon%2Flu-dairy-eggs"
|
||||
logger.info(f"Scraping dairy & eggs from {url}")
|
||||
|
||||
page = self.get_browser_page(url)
|
||||
content = page.content()
|
||||
page.close()
|
||||
|
||||
soup = BeautifulSoup(content, "html.parser")
|
||||
items = []
|
||||
|
||||
for item_elem in soup.find_all("div", class_=re.compile(r"product|item|coupon")):
|
||||
try:
|
||||
item = self._parse_coupon_item(item_elem)
|
||||
if item:
|
||||
item["aisle"] = "Dairy"
|
||||
items.append(item)
|
||||
except Exception:
|
||||
continue
|
||||
|
||||
return items
|
||||
Reference in New Issue
Block a user