feat: phase r1+r2 recovery + r3-0 swiftly api ingestion

R1 stabilization: pytest harness with transactional db fixture, smoke + alembic + auth + scrape + approval + swiftly tests, github actions ci yaml. Bearer-token admin auth + signed-cookie session for family ui mutations. Async POST /api/admin/scrape (BackgroundTasks, returns 202). Path canonicalization (no /list, /planned suffixes). DATABASE_URL fail-fast on empty. R2 deferred-risk spikes: live lucky california fetch (R2-A), full email+per-voter approval click round trip with single-use enforcement (R2-B, console email backend, sendgrid stub). R3-0 phase 3 redesign: replaced playwright html scraper with requests based swiftly json api client. 17 categories, ~10k products per scrape, upsert by (source, external_id). 401 surfaces actionable token-refresh message via ScrapeLog.error_message. Pre-existing defects fixed: shopping_list.py syntax error blocking app import, MealPlan.votes orphan relationship, JSONB(astext=True) invalid kwarg, missing requests dep, calorie_target schema drift, every SQLEnum needed values_callable, 0001 had empty downgrade(), seed had duplicate ingredient rows. Migrations added: 0003 grocery_item.description, 0004 family_profile. calorie_target, 0005 grocery_item.external_id + source + composite index. Verified: 31/31 pytest green, alembic upgrade->downgrade->upgrade clean, frontend npm run build clean, live scrape 9,960 grocery_item rows in 36s. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
2026-05-05 14:08:19 -07:00
parent b9434967ed
commit 8e89f793d5
58 changed files with 3594 additions and 348 deletions
@@ -0,0 +1,205 @@
+"""
+Pytest fixtures for MealPlanner backend.
+
+DB strategy
+-----------
+- If env ``TEST_DATABASE_URL`` is set, use it (must be a Postgres URL — the
+  current Alembic migrations use ``postgresql.UUID``/``JSONB``/``ARRAY`` types
+  which are not portable to SQLite).
+- Else fall back to ``DATABASE_URL`` if it points at Postgres.
+- Else skip Postgres-only tests (marker: ``requires_postgres``).
+
+Each test using the ``db`` fixture runs inside a SAVEPOINT-style nested
+transaction that rolls back on teardown so tests do not leak state.
+"""
+
+from __future__ import annotations
+
+import os
+import sys
+import pathlib
+import subprocess
+from typing import Iterator
+
+import pytest
+
+# Ensure DATABASE_URL is set BEFORE importing app.config (Settings requires it).
+# We default to the TEST_DATABASE_URL or a sentinel that lets imports succeed;
+# tests that actually need the DB rely on the marker / fixture skip path below.
+_DEFAULT_DSN = "postgresql://mealplanner:password@localhost:5432/mealplanner_test"
+os.environ.setdefault(
+    "DATABASE_URL",
+    os.environ.get("TEST_DATABASE_URL", _DEFAULT_DSN),
+)
+
+# Make backend/ importable when pytest is invoked from repo root.
+BACKEND_ROOT = pathlib.Path(__file__).resolve().parent.parent
+if str(BACKEND_ROOT) not in sys.path:
+    sys.path.insert(0, str(BACKEND_ROOT))
+
+from sqlalchemy import create_engine, text  # noqa: E402
+from sqlalchemy.orm import sessionmaker  # noqa: E402
+from sqlalchemy.exc import OperationalError  # noqa: E402
+
+
+def _resolve_test_dsn() -> str | None:
+    dsn = os.environ.get("TEST_DATABASE_URL") or os.environ.get("DATABASE_URL")
+    if not dsn:
+        return None
+    if not dsn.startswith(("postgresql://", "postgresql+psycopg2://")):
+        return None
+    return dsn
+
+
+def _postgres_reachable(dsn: str) -> bool:
+    try:
+        eng = create_engine(
+            dsn.replace("postgresql://", "postgresql+psycopg2://"),
+            pool_pre_ping=True,
+        )
+        with eng.connect() as conn:
+            conn.execute(text("SELECT 1"))
+        eng.dispose()
+        return True
+    except Exception:
+        return False
+
+
+_DSN = _resolve_test_dsn()
+_PG_AVAILABLE = bool(_DSN) and _postgres_reachable(_DSN)
+
+
+def pytest_collection_modifyitems(config, items):
+    """Skip postgres-only tests when no live Postgres is available."""
+    if _PG_AVAILABLE:
+        return
+    skip_pg = pytest.mark.skip(
+        reason="Postgres not reachable; set TEST_DATABASE_URL to enable."
+    )
+    for item in items:
+        if "requires_postgres" in item.keywords:
+            item.add_marker(skip_pg)
+
+
+def pytest_configure(config):
+    config.addinivalue_line(
+        "markers",
+        "requires_postgres: test needs a live Postgres reachable via TEST_DATABASE_URL",
+    )
+    config.addinivalue_line(
+        "markers",
+        "scraper_offline: parser-only test; uses saved HTML fixture; no "
+        "network or Playwright/Chromium required (CI-safe).",
+    )
+    config.addinivalue_line(
+        "markers",
+        "scraper_live: hits the live grocery website; requires Playwright "
+        "and Chromium; skipped by default in CI.",
+    )
+
+
+# ---------------------------------------------------------------------------
+# Schema bootstrap (session-scoped): run alembic upgrade head once per session.
+# ---------------------------------------------------------------------------
+@pytest.fixture(scope="session")
+def _schema() -> Iterator[None]:
+    if not _PG_AVAILABLE:
+        yield
+        return
+    env = os.environ.copy()
+    env["DATABASE_URL"] = _DSN  # Alembic env.py reads from settings.DATABASE_URL
+    # alembic.ini lives in backend/, run from there.
+    subprocess.run(
+        ["alembic", "upgrade", "head"],
+        cwd=str(BACKEND_ROOT),
+        env=env,
+        check=True,
+    )
+    yield
+    # Best-effort cleanup so re-running the suite locally is idempotent.
+    subprocess.run(
+        ["alembic", "downgrade", "base"],
+        cwd=str(BACKEND_ROOT),
+        env=env,
+        check=False,
+    )
+
+
+@pytest.fixture(scope="session")
+def _engine(_schema):
+    if not _PG_AVAILABLE:
+        yield None
+        return
+    eng = create_engine(
+        _DSN.replace("postgresql://", "postgresql+psycopg2://"),
+        pool_pre_ping=True,
+    )
+    yield eng
+    eng.dispose()
+
+
+@pytest.fixture()
+def db(_engine):
+    """Per-test transactional session that rolls back at teardown."""
+    if _engine is None:
+        pytest.skip("Postgres not reachable")
+    connection = _engine.connect()
+    trans = connection.begin()
+    Session = sessionmaker(bind=connection, autocommit=False, autoflush=False)
+    session = Session()
+    try:
+        yield session
+    finally:
+        session.close()
+        trans.rollback()
+        connection.close()
+
+
+@pytest.fixture()
+def client(db):
+    """TestClient with get_db overridden to yield the test session."""
+    from fastapi.testclient import TestClient
+    from app.main import app
+    from app.database import get_db
+
+    def _override():
+        try:
+            yield db
+        finally:
+            pass
+
+    app.dependency_overrides[get_db] = _override
+    try:
+        with TestClient(app) as c:
+            yield c
+    finally:
+        app.dependency_overrides.pop(get_db, None)
+
+
+@pytest.fixture()
+def client_no_db():
+    """TestClient that does NOT require a live DB — for pure import/wiring smoke."""
+    from fastapi.testclient import TestClient
+    from app.main import app
+    from app.database import get_db
+
+    class _StubSession:
+        def execute(self, *a, **kw):
+            from sqlalchemy.engine import Result  # noqa: F401
+            raise RuntimeError("DB not available in this fixture")
+
+        def query(self, *a, **kw):
+            raise RuntimeError("DB not available in this fixture")
+
+        def close(self):
+            pass
+
+    def _override():
+        yield _StubSession()
+
+    app.dependency_overrides[get_db] = _override
+    try:
+        with TestClient(app) as c:
+            yield c
+    finally:
+        app.dependency_overrides.pop(get_db, None)