From b2cbdd1533c7fd0607b0735a19644c6d9ca432f2 Mon Sep 17 00:00:00 2001
From: Peter Woolery <peter@research.bike>
Date: Thu, 7 May 2026 06:27:27 -0700
Subject: [PATCH] feat: orchestrator step_scrape with retry + stale-data
 fallback

Co-Authored-By: Claude Sonnet 4.6 (1M context) <noreply@anthropic.com>
---
 backend/app/services/orchestrator/steps.py | 62 ++++++++++++++++++++++
 backend/tests/test_orchestrator.py         | 37 +++++++++++++
 2 files changed, 99 insertions(+)
 create mode 100644 backend/app/services/orchestrator/steps.py

diff --git a/backend/app/services/orchestrator/steps.py b/backend/app/services/orchestrator/steps.py
new file mode 100644
index 0000000..a07b415
--- /dev/null
+++ b/backend/app/services/orchestrator/steps.py
@@ -0,0 +1,62 @@
+"""
+Orchestrator step functions.
+
+Each step receives a WeeklyRun row and an open Session. Steps are idempotent:
+a non-null timestamp column causes an immediate return. Steps commit their own
+changes before returning.
+
+All service imports are at module level so tests can monkeypatch via
+`app.services.orchestrator.steps.<name>`.
+"""
+from __future__ import annotations
+
+import logging
+from datetime import datetime, timezone
+from typing import TYPE_CHECKING
+
+from app.config import settings
+from app.models import FamilyMember, FamilyProfile, MealPlan, MealPlanItemStatus
+from app.services.approval import issue_token
+from app.services.email import get_email_backend
+from app.services.orchestrator.alerts import send_admin_alert
+from app.services.planner.generate import generate_meal_plan
+from app.services.scraper_service import ScraperService
+
+if TYPE_CHECKING:
+    from sqlalchemy.orm import Session
+    from app.models import WeeklyRun
+
+logger = logging.getLogger(__name__)
+
+
+def step_scrape(run: "WeeklyRun", db: "Session") -> None:
+    if run.scraped_at is not None:
+        logger.info("step_scrape: already done for %s, skipping", run.week_start_date)
+        return
+
+    service = ScraperService(db)
+    last_result: dict = {}
+    for attempt in (1, 2):
+        last_result = service.run_scrape()
+        if last_result["status"] == "success":
+            run.scraped_at = datetime.now(timezone.utc)
+            run.used_stale_data = False
+            db.commit()
+            logger.info("step_scrape: success on attempt %d", attempt)
+            return
+        logger.warning(
+            "step_scrape: attempt %d failed: %s", attempt, last_result.get("error")
+        )
+
+    # Both attempts failed — mark stale and continue the cycle
+    run.scraped_at = datetime.now(timezone.utc)
+    run.used_stale_data = True
+    db.commit()
+    send_admin_alert(
+        subject=f"Scrape failed week {run.week_start_date}",
+        body=(
+            f"Both scrape attempts failed. Proceeding with stale grocery data.\n"
+            f"Error: {last_result.get('error')}"
+        ),
+    )
+    logger.warning("step_scrape: both attempts failed; marked stale")
diff --git a/backend/tests/test_orchestrator.py b/backend/tests/test_orchestrator.py
index 8e3e3c3..0e84bd4 100644
--- a/backend/tests/test_orchestrator.py
+++ b/backend/tests/test_orchestrator.py
@@ -168,3 +168,40 @@ def test_send_admin_alert_calls_backend(monkeypatch):
     assert len(sent) == 1
     assert "[MealPlanner ALERT] boom" in sent[0]["subject"]
     assert sent[0]["to"] == "admin@example.com"
+
+
+# ---------------------------------------------------------------------------
+# step_scrape tests
+# ---------------------------------------------------------------------------
+
+def test_step_scrape_idempotent(db, weekly_run_scraped):
+    original_ts = weekly_run_scraped.scraped_at
+    from app.services.orchestrator.steps import step_scrape
+    step_scrape(weekly_run_scraped, db)
+    assert weekly_run_scraped.scraped_at == original_ts
+
+
+def test_step_scrape_success(db, weekly_run, monkeypatch):
+    monkeypatch.setattr(
+        "app.services.orchestrator.steps.ScraperService.run_scrape",
+        lambda self, **kw: {"status": "success", "items_scraped": 42},
+    )
+    from app.services.orchestrator.steps import step_scrape
+    step_scrape(weekly_run, db)
+    assert weekly_run.scraped_at is not None
+    assert weekly_run.used_stale_data is False
+
+
+def test_step_scrape_both_fail_marks_stale(db, weekly_run, monkeypatch):
+    monkeypatch.setattr(
+        "app.services.orchestrator.steps.ScraperService.run_scrape",
+        lambda self, **kw: {"status": "failed", "error": "timeout"},
+    )
+    monkeypatch.setattr(
+        "app.services.orchestrator.steps.send_admin_alert",
+        lambda subject, body: None,
+    )
+    from app.services.orchestrator.steps import step_scrape
+    step_scrape(weekly_run, db)
+    assert weekly_run.used_stale_data is True
+    assert weekly_run.scraped_at is not None