Public Access
admin
933a0cc9db
- Add BaseScraper with rate limiting, retries, session management - Add LuckyCaliforniaScraper with Playwright for dynamic content - Add ScraperService to save scraped items to grocery_item table - Connect /api/admin/scrape to ScraperService - Update ORIENTATION.md phase table
138 lines
4.8 KiB
Python
138 lines
4.8 KiB
Python
import time
|
|
import logging
|
|
from abc import ABC, abstractmethod
|
|
from typing import Optional, List, Dict, Any
|
|
from datetime import datetime
|
|
from urllib.parse import urljoin
|
|
import requests
|
|
from requests.adapters import HTTPAdapter
|
|
from urllib3.util.retry import Retry
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
class BaseScraper(ABC):
|
|
def __init__(
|
|
self,
|
|
base_url: str,
|
|
rate_limit_seconds: float = 2.0,
|
|
max_retries: int = 3,
|
|
timeout: int = 30
|
|
):
|
|
self.base_url = base_url.rstrip("/")
|
|
self.rate_limit_seconds = rate_limit_seconds
|
|
self.max_retries = max_retries
|
|
self.timeout = timeout
|
|
self.last_request_time = 0.0
|
|
self.session = self._create_session()
|
|
|
|
def _create_session(self) -> requests.Session:
|
|
session = requests.Session()
|
|
|
|
retry_strategy = Retry(
|
|
total=self.max_retries,
|
|
backoff_factor=1.0,
|
|
status_forcelist=[429, 500, 502, 503, 504],
|
|
allowed_methods=["HEAD", "GET", "OPTIONS"]
|
|
)
|
|
|
|
adapter = HTTPAdapter(max_retries=retry_strategy, pool_maxsize=10)
|
|
session.mount("http://", adapter)
|
|
session.mount("https://", adapter)
|
|
|
|
session.headers.update({
|
|
"User-Agent": "Mozilla/5.0 (compatible; MealPlannerBot/1.0; +https://mealplanner.local)",
|
|
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
|
|
"Accept-Language": "en-US,en;q=0.5",
|
|
})
|
|
|
|
return session
|
|
|
|
def _rate_limit(self) -> None:
|
|
elapsed = time.time() - self.last_request_time
|
|
if elapsed < self.rate_limit_seconds:
|
|
sleep_time = self.rate_limit_seconds - elapsed
|
|
logger.debug(f"Rate limiting: sleeping {sleep_time:.2f}s")
|
|
time.sleep(sleep_time)
|
|
self.last_request_time = time.time()
|
|
|
|
def _get(self, url: str, **kwargs) -> Optional[requests.Response]:
|
|
self._rate_limit()
|
|
|
|
full_url = url if url.startswith("http") else urljoin(self.base_url, url)
|
|
|
|
for attempt in range(self.max_retries + 1):
|
|
try:
|
|
logger.debug(f"GET {full_url} (attempt {attempt + 1})")
|
|
response = self.session.get(full_url, timeout=self.timeout, **kwargs)
|
|
response.raise_for_status()
|
|
return response
|
|
|
|
except requests.exceptions.RequestException as e:
|
|
logger.warning(f"Request failed (attempt {attempt + 1}): {e}")
|
|
if attempt >= self.max_retries:
|
|
logger.error(f"Max retries reached for {full_url}")
|
|
return None
|
|
time.sleep(2 ** attempt)
|
|
|
|
return None
|
|
|
|
def _post(self, url: str, data: Dict[str, Any] = None, **kwargs) -> Optional[requests.Response]:
|
|
self._rate_limit()
|
|
|
|
full_url = url if url.startswith("http") else urljoin(self.base_url, url)
|
|
|
|
for attempt in range(self.max_retries + 1):
|
|
try:
|
|
logger.debug(f"POST {full_url} (attempt {attempt + 1})")
|
|
response = self.session.post(full_url, data=data, timeout=self.timeout, **kwargs)
|
|
response.raise_for_status()
|
|
return response
|
|
|
|
except requests.exceptions.RequestException as e:
|
|
logger.warning(f"POST request failed (attempt {attempt + 1}): {e}")
|
|
if attempt >= self.max_retries:
|
|
logger.error(f"Max retries reached for {full_url}")
|
|
return None
|
|
time.sleep(2 ** attempt)
|
|
|
|
return None
|
|
|
|
@abstractmethod
|
|
def scrape(self) -> Dict[str, Any]:
|
|
pass
|
|
|
|
def log_scrape(self, status: str, items_scraped: int = 0, error_message: str = None) -> None:
|
|
logger.info(f"Scrape {status}: {items_scraped} items, error: {error_message}")
|
|
|
|
|
|
class SeleniumScraper(BaseScraper):
|
|
def __init__(self, *args, **kwargs):
|
|
super().__init__(*args, **kwargs)
|
|
self._playwright = None
|
|
|
|
def _init_playwright(self):
|
|
if self._playwright is None:
|
|
from playwright.sync_api import sync_playwright
|
|
self._playwright = sync_playwright().start()
|
|
self._browser = self._playwright.chromium.launch(headless=True)
|
|
self._context = self._browser.new_context(
|
|
user_agent="Mozilla/5.0 (compatible; MealPlannerBot/1.0)"
|
|
)
|
|
|
|
def _get_playwright(self):
|
|
self._init_playwright()
|
|
return self._playwright
|
|
|
|
def get_browser_page(self, url: str):
|
|
self._get_playwright()
|
|
page = self._context.new_page()
|
|
page.goto(url, wait_until="networkidle", timeout=30000)
|
|
return page
|
|
|
|
def cleanup(self):
|
|
if self._browser:
|
|
self._browser.close()
|
|
if self._playwright:
|
|
self._playwright.stop()
|
|
self._playwright = None |