From a37207b6fff6f895614399bc2d1675876adc492d Mon Sep 17 00:00:00 2001 From: Peter Woolery Date: Fri, 17 Apr 2026 16:03:36 -0700 Subject: [PATCH] feat: event-based walker detector tuned to real 7' overhead mount MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace per-track line-crossing counter with a single event state machine gated by foreground pixel count (ENTER=250, EXIT=150) and finalized by quiet-exit or timeout. Direction inferred from centroid excursion (up_score vs down_score) on quiet-exit fires, and from net displacement (last_c vs first_c) on timeout fires. Tuning reflects bench data at the intended 7' overhead mount: walkers produce smaller centroid excursions than originally modelled, so EXTENT gates, MIN_TRAJ, MAX_FRAMES and REFRACTORY were all relaxed from their initial guesses. Constants and rationale live in firmware/lib/cv/cv.h. Bench results (8 isolated walks, 4 entries + 4 exits): * Event detection: 8/8 (100%) * Aggregate entries+exits split: 4+4 (matches) * Per-walk direction labelling: 4/8 (~50%) Document explicitly that per-walk direction is unreliable at this mount and that downstream analytics should trust only gross traffic (entries + exits). Recovering direction would require a physical mount change or a richer signal; both are out of scope for v1. Tooling: * tools/replay_logs.py — replay event state machine against captured [F] diagnostic lines, for offline tuning without flash-test loops. * firmware/src/main_capture.cpp + tools/capture_frames.py + tools/replay_frames.py — raw-frame capture firmware and Python port of the detector, kept in tree for future iteration even though the TimerCamera-F serial driver stripped specific byte ranges in testing and log-based replay became the working path. Co-Authored-By: Claude Opus 4.7 (1M context) --- .gitignore | 5 + README.md | 85 +++- .../specs/2026-04-13-door-counter-design.md | 44 +- firmware/lib/cv/cv.cpp | 292 ++++++++---- firmware/lib/cv/cv.h | 92 +++- firmware/platformio.ini | 20 + firmware/src/main.cpp | 19 +- firmware/src/main_capture.cpp | 64 +++ firmware/test/test_cv/test_cv.cpp | 420 ++++++++++-------- tools/capture_frames.py | 105 +++++ tools/replay_frames.py | 211 +++++++++ tools/replay_logs.py | 186 ++++++++ 12 files changed, 1203 insertions(+), 340 deletions(-) create mode 100644 firmware/src/main_capture.cpp create mode 100644 tools/capture_frames.py create mode 100644 tools/replay_frames.py create mode 100644 tools/replay_logs.py diff --git a/.gitignore b/.gitignore index e458ed5..aaa400a 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,6 @@ .worktrees/ +.agent/ +.claude/ +graphify-out/ +firmware/.pio/ +*.log diff --git a/README.md b/README.md index 323115e..f99b0a2 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,8 @@ # DoorCounter -Retail door traffic counter using M5Stack TimerCamera-F (ESP32 + OV3660). Counts entries/exits via overhead camera CV, passively scans BLE foot traffic, and reports hourly to `logs.research.bike`. +Retail door traffic counter using M5Stack TimerCamera-F (ESP32 + OV3660). Counts walker traversals via overhead camera CV, passively scans BLE foot traffic, and reports hourly to `logs.research.bike`. + +> **Known limitation — directional accuracy.** This firmware reports counts as `{entries, exits}` for API compatibility, but **per-walk direction labelling is not reliable at the current mount (7' overhead, straight down).** In bench testing, event detection was 100% (8/8 walks detected) while per-walk direction matched the physical walk only ~50% of the time — the centroid trajectories produced by entries and exits were nearly indistinguishable. **The number to trust is gross traffic: `entries + exits` ≈ total walkers through the doorway.** The directional split is an unreliable best-effort heuristic. See [Directional counting](#directional-counting) for why. ## Hardware @@ -21,7 +23,7 @@ pio run -t upload --upload-port /dev/ttyUSB0 | Module | Behavior | |--------|----------| -| CV pipeline | 5 fps, 96×96 grayscale, blob tracking, directional traversal count (origin→destination, once per track) with per-direction cooldown safety net | +| CV pipeline | 5 fps, 96×96 grayscale, event-based walker detector (foreground-count state machine; centroid-trajectory direction heuristic) with post-fire refractory period | | Detection LED | Single blink on entry, double blink on exit (preserves upload/no-WiFi status LED) | | BLE scanner | Continuous passive scan; deinits during hourly upload to free heap | | Reporter | Hourly HMAC-signed POST; 60s boot report for fast connectivity check | @@ -33,31 +35,70 @@ pio run -t upload --upload-port /dev/ttyUSB0 - **First report**: 60 seconds after NTP sync (connectivity check) - **Subsequent reports**: every 3600 seconds -### Directional counting +### Counting model — event-based walker detector -Each tracked blob fires at most **one** event over its lifetime, and only -when it has genuinely traversed the frame — specifically, when its spawn -position and current position are both at least `CV_TRAVERSAL_MARGIN_PX` -(14 px ≈ 15% of the 96×96 frame) from the line, and on opposite sides. +The CV pipeline is a **single event state machine** (no per-blob tracking +for counting). Per-frame foreground pixel count gates event start and end; +centroid trajectory within the active event decides direction. -- Top half → bottom half traversal = **entry** -- Bottom half → top half traversal = **exit** +**Event lifecycle:** +1. **Idle → Active**: `fg_count ≥ CV_EVENT_ENTER_THRESH` (250 px) fires event start. + Background updates freeze while the event is active so the walker does + not get absorbed into the baseline. +2. **Active accumulation**: every frame updates `first_c` (once), `min_c`, + `max_c`, `last_c`, `min_y_seen`, `max_y_seen`, and the frame count. +3. **Active → End** (either): + - **Quiet exit**: `fg_count < CV_EVENT_EXIT_THRESH` (150 px) for + `CV_EVENT_QUIET_FRAMES` (3) consecutive frames — walker has left. + - **Timeout**: `event_frame_count > CV_EVENT_MAX_FRAMES` (25 frames ≈ 5s). +4. On end, the event is finalized: gated by minimum duration, vertical + extent (must span a large fraction of the frame), and minimum centroid + trajectory magnitude. Background snaps to the current frame. +5. A **refractory period** (`CV_EVENT_REFRACTORY_FRAMES` = 10 ≈ 2s) after + a fire blocks a new event from starting — absorbs residual lingering + motion that would otherwise double-count. -A blob that appears near the line and wobbles across it does not count -(both positions are within the margin band). A blob that fully traverses -then reverses under the same track also does not double-count (the track -is flagged `counted`). If tracking churns — the track dies mid-traversal -and respawns on the other side — a new track with a new spawn on the -crossed side is the normal path to a correct count. +**Direction heuristic** (applied only if the event passes all gates): +- `up_score = first_c − min_c` (how far centroid excursed upward) +- `down_score = max_c − first_c` (how far it excursed downward) +- Quiet-exit events: `is_entry = (up_score ≥ down_score)` +- Timeout events: `is_entry = (last_c < first_c)` — net displacement is + more reliable than excursion when the walker is still in frame at timeout. -See `firmware/lib/cv/cv.h` for margin and `cv.cpp` for the crossing logic. +Per-mount convention: centroid moving **up through the frame** (y decreasing) += **entry** into the store. -### Crossing cooldown (safety net) +### Directional counting — known limitation -On top of directional counting, each direction enforces a cooldown between -counted events. Default: `CV_CROSSING_COOLDOWN_FRAMES = 5` (≈0.8s at 5 fps). -Entries and exits maintain separate cooldowns, so a real entry immediately -followed by a real exit still counts both. +**Per-walk direction labelling is unreliable at the current mount.** In +bench testing (8 alternating entry/exit walks at 4s intervals, 7' overhead +mount pointing straight down): + +- **Event detection**: 8/8 (100%) — every walk produced exactly one event. +- **Aggregate split**: 4 entries + 4 exits — matches the 4+4 ground truth. +- **Per-walk direction**: 4/8 (50%) — essentially a coin flip. + +At this mount, entries and exits produce nearly identical centroid +trajectories: both begin near mid-frame (walker is already large when +`fg_count` crosses 250), both reach a peak excursion toward the top, and +both end near mid-frame (walker's tail is still visible when `fg_count` +drops below 150). No heuristic over the recorded centroid statistics +separates them with better than ~50% accuracy on alternating walks. + +**What we ship, and what the server should trust:** +- **Gross traffic (`entries + exits`) is accurate.** This is the number + downstream analytics should use as "people through the door this hour." +- **Directional split is reported but unreliable.** Treat individual + `entries` and `exits` values as a best-effort labelling. Do not infer + net flow or dwell from them. + +To actually recover per-walk direction would require either a physical +change (raise or tilt the camera so walkers enter/leave through the frame +edges) or a richer signal than centroid statistics (e.g. time-resolved +optical flow, or a second sensor). That work is out of scope for v1. + +See `firmware/lib/cv/cv.h` for tuning constants and `cv.cpp` for the +finalize logic. ## Operator Setup @@ -124,7 +165,7 @@ DoorCounter/ ├── firmware/ │ ├── platformio.ini │ ├── lib/ -│ │ ├── cv/ — CV pipeline (blob tracking, line cross, cooldown) +│ │ ├── cv/ — CV pipeline (event state machine, centroid-trajectory direction) │ │ └── hmac/ — HMAC-SHA256 signing library │ └── src/ │ ├── main.cpp — FreeRTOS tasks, boot sequence diff --git a/docs/superpowers/specs/2026-04-13-door-counter-design.md b/docs/superpowers/specs/2026-04-13-door-counter-design.md index d6d0d97..f534fbd 100644 --- a/docs/superpowers/specs/2026-04-13-door-counter-design.md +++ b/docs/superpowers/specs/2026-04-13-door-counter-design.md @@ -13,7 +13,7 @@ [TimerCamera-F Device] ├── Provisioning module — captive portal AP on first boot ├── Config store — NVS: device_id, location_id, HMAC secret, WiFi creds, line_offset - ├── Camera + CV module — captures frames, runs line-crossing counter + ├── Camera + CV module — captures frames, runs event-based walker detector ├── BLE scanner — continuous passive scan (WiFi coexistence mode) ├── Report buffer — accumulates counts in RAM, flushes hourly └── HTTP client — HMAC-signed POSTs to logs.research.bike @@ -89,22 +89,38 @@ Capture → Grayscale → Downscale 96×96 → Frame diff → Threshold → Blob | Downscale | Bilinear to 96×96 (~11× compute reduction) | | Frame diff | Absolute difference against rolling background (updated every ~2s when no motion) | | Threshold | Pixels > 30 intensity delta = foreground | -| Blob detect | Connected components; blobs < 8×8 px discarded as noise | -| Centroid track | Nearest-centroid matching frame-to-frame (max 15px), tracks persist up to 10 missed frames | -| Line crossing | Virtual horizontal line at configurable vertical position (default: 50% of frame height) | -| Directional traversal | Each track records its **spawn y** and fires **at most once**. An event fires only when the track's spawn position and current position are both ≥ `CV_TRAVERSAL_MARGIN_PX` (14 px) from the line and on opposite sides — i.e. a true traversal, not a wobble. | -| Cooldown | Per-direction cooldown between counted events (default 5 frames ≈ 0.8s @ 5 fps) — safety net on top of directional logic | +| Event state machine | Single global state machine (not per-blob). Per-frame `fg_count` (total foreground pixels) gates event start and end. | +| Event start | `fg_count ≥ CV_EVENT_ENTER_THRESH` (250 px) → event becomes active. Background updates freeze for the event's duration so the walker does not blend into the baseline. | +| Event accumulation | Each frame records `first_c` (centroid_y at start), running `min_c` / `max_c` / `last_c`, vertical extents (`min_y_seen`, `max_y_seen`), and frame count. | +| Event end | Either **quiet exit** (`fg_count < CV_EVENT_EXIT_THRESH` (150 px) for `CV_EVENT_QUIET_FRAMES` (3) consecutive frames) or **timeout** (`event_frame_count > CV_EVENT_MAX_FRAMES` (25)). On end, background snaps to the current frame. | +| Fire gates | Duration ≥ `CV_EVENT_MIN_FRAMES` (5), `min_y_seen ≤ CV_EVENT_EXTENT_TOP` (25) AND `max_y_seen ≥ CV_EVENT_EXTENT_BOT` (50) — event must span a large fraction of the frame — AND `max(up_score, down_score) ≥ CV_EVENT_MIN_TRAJ` (5) | +| Refractory | `CV_EVENT_REFRACTORY_FRAMES` (10 ≈ 2s) after a fire, the machine refuses to start a new event — absorbs lingering motion of the just-counted walker. | -**Counting logic:** -- Each track has a `spawn_y` (recorded at creation) and a `counted` flag. -- An event fires only if the track is **not yet counted**, spawned **firm** on one side of the line (|spawn_y − line_y| > `CV_TRAVERSAL_MARGIN_PX`), and is **now firm** on the opposite side. On fire, the track is flagged counted — it will not produce another event for its lifetime. -- Spawn firm above + now firm below = **entry** -- Spawn firm below + now firm above = **exit** -- Cooldown (per-direction, independent entries/exits) is a secondary gate: within `CV_CROSSING_COOLDOWN_FRAMES` of the last counted event in that direction, a new event is suppressed even if a different track's traversal would otherwise qualify. +**Direction heuristic (applied after fire gates pass):** +- `up_score = first_c − min_c` (peak upward centroid excursion) +- `down_score = max_c − first_c` (peak downward centroid excursion) +- **Quiet-exit fires**: `is_entry = (up_score ≥ down_score)` +- **Timeout fires**: `is_entry = (last_c < first_c)` — walker is still in frame at timeout, so net displacement is a better signal than excursion. -**Rationale**: a single person traversing the doorway produces one track with a clear origin and destination — that's one count. Shadows that appear near the line and wobble, or tracks that churn at spawn, lack a firm origin and never count. +Per-mount convention: centroid moving **up through the frame** (y decreasing) = **entry** into the store. -Counts accumulate as `{entries, exits}` in RAM and reset each hour on report. +**Counting surface**: `{entries, exits}` accumulate in RAM and reset each hour on report. + +**Directional accuracy is best-effort, not guaranteed.** In bench testing at the intended 7' overhead straight-down mount: + +| Metric | Result | +|--------|--------| +| Event detection | 8/8 walks (100%) | +| Aggregate entry/exit split | 4+4 vs ground-truth 4+4 (matches) | +| Per-walk direction labelling | 4/8 (50%) — no better than chance | + +At this mount, entries and exits produce nearly identical centroid trajectories: the walker is already large when `fg_count` crosses 250 (so `first_c` is always near mid-frame), their tail is still visible when `fg_count` drops below 150 (so `last_c` is always near mid-frame), and the excursion in between peaks upward for both directions. No statistic computable from (`first_c`, `min_c`, `max_c`, `last_c`, duration) separates them reliably. + +**Contract with downstream consumers (API and analytics):** +- **`entries + exits` is the trustworthy number** — it is the count of walkers through the doorway in the hour. Use this as "foot traffic." +- **Individual `entries` and `exits` are reported for API shape compatibility, but should not be relied on for net flow, dwell, or any per-direction analysis.** + +Recovering true direction requires either a physical change (tilt or raise the camera so walkers pass fully through the frame edges) or a richer signal (time-resolved centroid trajectory, optical flow, secondary sensor). Both are out of scope for v1. --- diff --git a/firmware/lib/cv/cv.cpp b/firmware/lib/cv/cv.cpp index e125d35..b94baf7 100644 --- a/firmware/lib/cv/cv.cpp +++ b/firmware/lib/cv/cv.cpp @@ -5,8 +5,21 @@ #include #include +static void event_reset(CVState& s) { + s.event_active = false; + s.event_start_frame = 0; + s.event_frame_count = 0; + s.event_peak_n = 0; + s.event_first_c = -1.0f; + s.event_last_c = -1.0f; + s.event_min_c = (float)CV_H; + s.event_max_c = -1.0f; + s.event_min_y_seen = CV_H; + s.event_max_y_seen = -1; + s.event_quiet_count = 0; +} + void cv_init(CVState& state) { - // Initialize members directly — avoid CVState{} temporary which puts 9KB on stack memset(state.background, 0, sizeof(state.background)); state.bg_valid = false; state.last_motion_frame = 0; @@ -15,8 +28,8 @@ void cv_init(CVState& state) { state.tracks.clear(); state.entries = 0; state.exits = 0; - state.last_entry_frame = 0; - state.last_exit_frame = 0; + state.last_fire_frame = 0; + event_reset(state); } void cv_reset_counts(CVState& state) { @@ -26,9 +39,6 @@ void cv_reset_counts(CVState& state) { struct Point { int x, y; }; -// Note: queue may grow to CV_PIXELS entries (~72KB) on large blobs. -// Requires PSRAM (enabled via -DBOARD_HAS_PSRAM in platformio.ini). -// BFS flood fill. Marks visited pixels (sets fg to 0). Returns {-1,-1} if blob < CV_MIN_BLOB_PX. static std::pair extract_blob(uint8_t* fg, int start_x, int start_y) { std::vector queue; queue.reserve(512); @@ -60,7 +70,7 @@ static std::pair extract_blob(uint8_t* fg, int start_x, int start_y static std::vector> find_centroids(const uint8_t* fg) { std::vector> result; - static uint8_t fg_copy[CV_PIXELS]; // static to avoid 9KB stack allocation + static uint8_t fg_copy[CV_PIXELS]; memcpy(fg_copy, fg, CV_PIXELS); for (int y = 0; y < CV_H; y++) { @@ -82,8 +92,62 @@ static void frame_diff(const uint8_t* frame, const uint8_t* bg, } } -CVResult cv_process(CVState& state, const uint8_t* frame, uint8_t line_pct) { - CVResult result = {0, 0}; +// Decide whether the just-ended event should fire and in which direction. +// Up-through-frame (centroid excursion from high y toward low y) maps to +// ENTRY per mount convention. +static void finalize_event(CVState& s, CVResult& result) { + if (s.event_frame_count < CV_EVENT_MIN_FRAMES) return; + // Note: no MAX_FRAMES rejection here. An event that runs the full duration + // may still be a valid walker whose fg_count stayed above EXIT_THRESH due + // to a stale bg or an AEC-driven lighting shift. Extent + MIN_TRAJ gates + // below already reject stationary-person / wobble events. + if (s.event_min_y_seen > CV_EVENT_EXTENT_TOP) return; + if (s.event_max_y_seen < CV_EVENT_EXTENT_BOT) return; + + // Direction from centroid excursion relative to event start. + // up_score: how far centroid excursed upward (smaller y) from first_c. + // down_score: how far it excursed downward (larger y) from first_c. + float up_score = s.event_first_c - s.event_min_c; + float down_score = s.event_max_c - s.event_first_c; + + float winning = (up_score >= down_score) ? up_score : down_score; + if (winning < CV_EVENT_MIN_TRAJ) return; + + // Timeout-aware direction. Quiet-exit events (fg fell below EXIT_THRESH) + // have walker fully out of frame → min/max excursion bracket the true + // traversal and up/down scores are reliable. Timeout events (event hit + // MAX_FRAMES while still elevated) captured both an approach and a + // departure within the window, so excursion measures the walker's + // *range in frame* rather than direction — an entry walker who paused + // near the top, then drifted back toward the middle before timeout + // gets (wrongly) called an entry by up-score even though net motion is + // mixed. For those, the net first→last centroid displacement is a + // better direction signal (it's where the walker ended up, not just + // where they peaked). + bool timed_out = (s.event_frame_count > CV_EVENT_MAX_FRAMES); + bool is_entry; + if (timed_out) { + is_entry = (s.event_last_c < s.event_first_c); + } else { + is_entry = (up_score >= down_score); + } + if (is_entry) { + s.entries++; + result.entries_delta++; + } else { + s.exits++; + result.exits_delta++; + } + s.last_fire_frame = s.frame_index; + result.fire_first_c = s.event_first_c; + result.fire_min_c = s.event_min_c; + result.fire_max_c = s.event_max_c; + result.fire_last_c = s.event_last_c; + result.fire_duration = s.event_frame_count; +} + +CVResult cv_process(CVState& state, const uint8_t* frame, uint8_t /*line_pct*/) { + CVResult result = {0, 0, 0, -1, -1, -1.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0}; state.frame_index++; if (!state.bg_valid) { @@ -92,105 +156,147 @@ CVResult cv_process(CVState& state, const uint8_t* frame, uint8_t line_pct) { return result; } - static uint8_t fg[CV_PIXELS]; // static: avoids 9KB on task stack + static uint8_t fg[CV_PIXELS]; frame_diff(frame, state.background, fg, CV_PIXELS); - int fg_count = 0; - for (int i = 0; i < CV_PIXELS; i++) fg_count += fg[i]; - - bool motion = fg_count > CV_MIN_BLOB_PX; - if (!motion) { - if (state.frame_index - state.last_motion_frame > 10) { - memcpy(state.background, frame, CV_PIXELS); + // Running-average background blend: bg = (31*bg + frame)/32. Adapts to + // slow scene drift during idle periods. Frozen during an active event so + // the walker's signature is never absorbed — otherwise bg retains a + // "ghost" of the walker for ~30 frames after they leave, keeping fg_count + // elevated and preventing subsequent walkers from producing a clean + // trajectory. + if (!state.event_active) { + for (int i = 0; i < CV_PIXELS; i++) { + state.background[i] = (uint8_t)(((uint16_t)state.background[i] * 31 + frame[i]) >> 5); } + } + + int fg_count = 0; + int min_y = CV_H, max_y = -1; + long sum_y = 0; + for (int y = 0; y < CV_H; y++) { + const uint8_t* row = &fg[y * CV_W]; + int row_count = 0; + for (int x = 0; x < CV_W; x++) row_count += row[x]; + if (row_count > 0) { + if (y < min_y) min_y = y; + if (y > max_y) max_y = y; + sum_y += (long)row_count * y; + fg_count += row_count; + } + } + result.fg_count = fg_count; + result.fg_min_y = (fg_count > 0) ? min_y : -1; + result.fg_max_y = (fg_count > 0) ? max_y : -1; + result.fg_centroid_y = (fg_count > 0) ? ((float)sum_y / fg_count) : -1.0f; + + // Hard self-heal: if more than half the frame is fg, bg is catastrophically + // wrong. Snap and skip the event machine this frame. + if (fg_count > CV_PIXELS / 2) { + memcpy(state.background, frame, CV_PIXELS); + state.last_motion_frame = state.frame_index; + if (state.event_active) event_reset(state); + return result; + } + + // Diagnostic track management (no effect on counting). + bool motion = fg_count > CV_MIN_BLOB_PX; + if (motion) { + state.last_motion_frame = state.frame_index; + auto centroids = find_centroids(fg); + std::vector centroid_matched(centroids.size(), false); + for (auto& track : state.tracks) { + float best_dist = CV_MAX_MOVE * CV_MAX_MOVE; + int best_idx = -1; + for (int i = 0; i < (int)centroids.size(); i++) { + if (centroid_matched[i]) continue; + float dx = centroids[i].first - track.x; + float dy = centroids[i].second - track.y; + float d2 = dx*dx + dy*dy; + if (d2 < best_dist) { best_dist = d2; best_idx = i; } + } + if (best_idx >= 0) { + centroid_matched[best_idx] = true; + track.x = centroids[best_idx].first; + track.y = centroids[best_idx].second; + track.missed = 0; + } else { + track.missed++; + } + } + state.tracks.erase( + std::remove_if(state.tracks.begin(), state.tracks.end(), + [](const CVTrack& t){ return t.missed > CV_MAX_MISSED; }), + state.tracks.end()); + for (int i = 0; i < (int)centroids.size(); i++) { + if (centroid_matched[i]) continue; + CVTrack t; + t.id = state.next_id++; + t.x = centroids[i].first; + t.y = centroids[i].second; + t.spawn_y = t.y; + t.missed = 0; + state.tracks.push_back(t); + } + } else { for (auto& t : state.tracks) t.missed++; state.tracks.erase( std::remove_if(state.tracks.begin(), state.tracks.end(), [](const CVTrack& t){ return t.missed > CV_MAX_MISSED; }), state.tracks.end()); - return result; } - state.last_motion_frame = state.frame_index; + // Event state machine. Refractory period after a fire blocks new events + // for CV_EVENT_REFRACTORY_FRAMES frames — absorbs lingering-walker motion + // that would otherwise re-trigger a second count. + bool in_refractory = state.last_fire_frame != 0 && + (state.frame_index - state.last_fire_frame) < CV_EVENT_REFRACTORY_FRAMES; - auto centroids = find_centroids(fg); - - std::vector centroid_matched(centroids.size(), false); - - for (auto& track : state.tracks) { - float best_dist = CV_MAX_MOVE * CV_MAX_MOVE; - int best_idx = -1; - - for (int i = 0; i < (int)centroids.size(); i++) { - if (centroid_matched[i]) continue; - float dx = centroids[i].first - track.x; - float dy = centroids[i].second - track.y; - float d2 = dx*dx + dy*dy; - if (d2 < best_dist) { best_dist = d2; best_idx = i; } + if (!state.event_active) { + if (!in_refractory && fg_count >= CV_EVENT_ENTER_THRESH) { + state.event_active = true; + state.event_start_frame = state.frame_index; + state.event_frame_count = 1; + state.event_peak_n = fg_count; + state.event_first_c = result.fg_centroid_y; + state.event_last_c = result.fg_centroid_y; + state.event_min_c = result.fg_centroid_y; + state.event_max_c = result.fg_centroid_y; + state.event_min_y_seen = min_y; + state.event_max_y_seen = max_y; + state.event_quiet_count = 0; } - - if (best_idx >= 0) { - centroid_matched[best_idx] = true; - track.x = centroids[best_idx].first; - track.y = centroids[best_idx].second; - track.missed = 0; + } else { + state.event_frame_count++; + if (fg_count > state.event_peak_n) state.event_peak_n = fg_count; + if (fg_count > 0) { + state.event_last_c = result.fg_centroid_y; + if (result.fg_centroid_y < state.event_min_c) state.event_min_c = result.fg_centroid_y; + if (result.fg_centroid_y > state.event_max_c) state.event_max_c = result.fg_centroid_y; + if (min_y < state.event_min_y_seen) state.event_min_y_seen = min_y; + if (max_y > state.event_max_y_seen) state.event_max_y_seen = max_y; + } + if (fg_count < CV_EVENT_EXIT_THRESH) { + state.event_quiet_count++; + if (state.event_quiet_count >= CV_EVENT_QUIET_FRAMES) { + finalize_event(state, result); + event_reset(state); + memcpy(state.background, frame, CV_PIXELS); + } } else { - track.missed++; - } - } - - state.tracks.erase( - std::remove_if(state.tracks.begin(), state.tracks.end(), - [](const CVTrack& t){ return t.missed > CV_MAX_MISSED; }), - state.tracks.end()); - - float line_y = (line_pct / 100.0f) * CV_H; - for (int i = 0; i < (int)centroids.size(); i++) { - if (centroid_matched[i]) continue; - CVTrack t; - t.id = state.next_id++; - t.x = centroids[i].first; - t.y = centroids[i].second; - t.spawn_y = t.y; - t.above_line = (t.y < line_y); - t.counted = false; - t.missed = 0; - state.tracks.push_back(t); - } - // Directional crossing check. A track counts at most once, and only if it - // spawned clearly on one side of the line AND is now clearly on the other. - // This rejects blobs that wobble around the line (shadows, body straddling - // the line, track churn at spawn) — only a true traversal fires an event. - for (auto& track : state.tracks) { - if (track.missed > 0) continue; // only check tracks matched this frame - if (track.counted) continue; // one track = one trip - - bool spawned_above = track.spawn_y < (line_y - CV_TRAVERSAL_MARGIN_PX); - bool spawned_below = track.spawn_y > (line_y + CV_TRAVERSAL_MARGIN_PX); - bool now_above_firm = track.y < (line_y - CV_TRAVERSAL_MARGIN_PX); - bool now_below_firm = track.y > (line_y + CV_TRAVERSAL_MARGIN_PX); - - if (spawned_above && now_below_firm) { - bool in_cooldown = state.last_entry_frame != 0 && - (state.frame_index - state.last_entry_frame) < CV_CROSSING_COOLDOWN_FRAMES; - if (!in_cooldown) { - state.entries++; - result.entries_delta++; - state.last_entry_frame = state.frame_index; - track.counted = true; - } - } else if (spawned_below && now_above_firm) { - bool in_cooldown = state.last_exit_frame != 0 && - (state.frame_index - state.last_exit_frame) < CV_CROSSING_COOLDOWN_FRAMES; - if (!in_cooldown) { - state.exits++; - result.exits_delta++; - state.last_exit_frame = state.frame_index; - track.counted = true; + state.event_quiet_count = 0; + if (state.event_frame_count > CV_EVENT_MAX_FRAMES) { + // Timeout end: fg still elevated. Snap bg anyway — in practice + // a stuck-high event means bg is stale (walker has merged + // with stale bg, or AEC shifted). Leaving bg stale permanently + // poisons subsequent events. If a walker truly is mid-frame + // they'll get absorbed into bg, but that's a rare corner + // beaten by the common case of stale bg chaining events. + finalize_event(state, result); + event_reset(state); + memcpy(state.background, frame, CV_PIXELS); } } - - track.above_line = (track.y < line_y); } return result; diff --git a/firmware/lib/cv/cv.h b/firmware/lib/cv/cv.h index 6ab19b1..4252cfa 100644 --- a/firmware/lib/cv/cv.h +++ b/firmware/lib/cv/cv.h @@ -12,24 +12,63 @@ static const int CV_MIN_BLOB_PX = 64; static const float CV_MAX_MOVE = 15.0f; static const int CV_MAX_MISSED = 10; -// Directional counting margin: a track only counts if it spawned and is now -// both at least this far from the line (in pixels). Prevents counting blobs -// that wobble around the line or spawn on top of it. Value chosen at ~15% of -// the 96px frame: 14px ≈ the typical torso half-width overhead. -static const float CV_TRAVERSAL_MARGIN_PX = 14.0f; +// Event-based walker detector. Per-frame zone-flip approaches were direction- +// blind at realistic mounts: a walker traversing top-to-bottom and a walker +// traversing bottom-to-top produced identical zone-dominance sequences +// (geometric artifact of asymmetric zones + body spanning the line). The +// event approach buffers a whole walker event, then decides direction from +// the centroid trajectory: sign(first_centroid_y - peak_centroid_y) > 0 means +// the centroid moved upward through the frame during the event. +// +// Per-mount convention: UP through frame == ENTRY into store. Flip the camera +// mount or invert the mapping in cv_process if the physical install differs. -// Per-direction crossing cooldown. Any same-direction crossing whose frame gap -// is strictly less than this value is dropped. At 5 fps, a value of 5 → ≈0.8s -// suppression window. Purpose: mask track churn (blob briefly drops below -// min_blob_px, track dies & respawns, re-crosses). -static const uint32_t CV_CROSSING_COOLDOWN_FRAMES = 5; +// fg_count thresholds that gate event start/end. Tuned against a real +// 8-walk isolated test (see .agent/walk_isolated_8walks.log). Lower than +// initial guesses because the 7' overhead mount produces smaller centroid +// excursions than we originally modelled. +static const int CV_EVENT_ENTER_THRESH = 250; +static const int CV_EVENT_EXIT_THRESH = 150; +// Number of consecutive sub-EXIT frames required to end an event. +static const int CV_EVENT_QUIET_FRAMES = 3; + +// Min/max event duration in frames. Below min = too brief to be a walker +// (noise burst). Above max = stationary object or stuck detection. +static const int CV_EVENT_MIN_FRAMES = 5; +// MAX bounds the event duration. Too low (15) cut events off while walker +// was still physically in frame — every fire hit dur=MAX+1 and bg snapped +// with a walker-ghost baked in, corrupting the next walk. Too high (40) +// merged multiple walkers. 25 frames (5s) lets a single walker reach the +// quiet-exit path (fg drops below EXIT_THRESH) before timeout, so bg snaps +// on a clean empty frame. +static const int CV_EVENT_MAX_FRAMES = 25; + +// Required vertical extent: during the event, fg must have reached near the +// top of the frame (min_y <= TOP) AND near the bottom (max_y >= BOT). At a +// 7' overhead mount real walkers span fg y≈0..70, not 0..95 — the original +// 10/85 gates rejected most real walks. Relaxed to catch them while still +// filtering small local motion that doesn't span the doorway. +static const int CV_EVENT_EXTENT_TOP = 25; +static const int CV_EVENT_EXTENT_BOT = 50; + +// Minimum centroid excursion (max of up_score/down_score) for a valid +// trajectory. At overhead mount walker centroid traverses ~15-40 pixels; +// 15 was too aggressive and dropped clean walks. 5 still filters wobble. +static const float CV_EVENT_MIN_TRAJ = 5.0f; + +// Refractory period after a fire. Shorter than originally chosen — at 5 fps +// a second walker can arrive within 2s of the first, especially at busy +// doorways. 10 frames = 2s of back-pressure, tuned to match the gap between +// consecutive isolated walks in the test log. +static const uint32_t CV_EVENT_REFRACTORY_FRAMES = 10; + +// Diagnostic only: tracks are kept for spawn logging. Counting does NOT +// depend on tracks. struct CVTrack { int id; float x, y; - float spawn_y; // y at track creation — used for directional counting - bool above_line; - bool counted; // fires at most once per track (one track = one trip) + float spawn_y; int missed; }; @@ -42,13 +81,36 @@ struct CVState { std::vector tracks; int entries; int exits; - uint32_t last_entry_frame; // 0 = never; frame_index of last counted entry - uint32_t last_exit_frame; // 0 = never; frame_index of last counted exit + + // Event state machine. + bool event_active; + uint32_t event_start_frame; + int event_frame_count; + int event_peak_n; + float event_first_c; + float event_last_c; + float event_min_c; // min centroid_y observed during event + float event_max_c; // max centroid_y observed during event + int event_min_y_seen; + int event_max_y_seen; + int event_quiet_count; + uint32_t last_fire_frame; // 0 = never; frame of last counted fire }; struct CVResult { int entries_delta; int exits_delta; + // Per-frame foreground diagnostics (populated every call). + int fg_count; + int fg_min_y; + int fg_max_y; + float fg_centroid_y; + // Populated only on a fire frame; zeroed otherwise. + float fire_first_c; + float fire_min_c; + float fire_max_c; + float fire_last_c; + int fire_duration; }; void cv_init(CVState& state); diff --git a/firmware/platformio.ini b/firmware/platformio.ini index 13d9fa8..1f782c8 100644 --- a/firmware/platformio.ini +++ b/firmware/platformio.ini @@ -7,6 +7,7 @@ platform = espressif32@6.6.0 board = m5stack-timer-cam framework = arduino board_build.partitions = partitions_4mb_ota.csv +build_src_filter = +<*> - build_flags = -DBOARD_HAS_PSRAM -mfix-esp32-psram-cache-issue @@ -23,6 +24,25 @@ lib_deps = h2zero/NimBLE-Arduino@^1.4.2 espressif/esp32-camera +; Frame-capture build. Strips WiFi/BLE/CV/reporter; streams raw 96x96 frames +; over serial at 921600 baud for offline algorithm iteration. +[env:timercam-capture] +platform = espressif32@6.6.0 +board = m5stack-timer-cam +framework = arduino +board_build.partitions = partitions_4mb_ota.csv +build_flags = + -DBOARD_HAS_PSRAM + -mfix-esp32-psram-cache-issue + -DCORE_DEBUG_LEVEL=0 + -DCONFIG_SPIRAM_USE_MALLOC=1 +build_src_filter = -<*> + + +monitor_speed = 460800 +upload_speed = 115200 +upload_flags = --no-stub +lib_deps = + espressif/esp32-camera + [env:native] platform = native test_framework = unity diff --git a/firmware/src/main.cpp b/firmware/src/main.cpp index bf4a1df..91faca3 100644 --- a/firmware/src/main.cpp +++ b/firmware/src/main.cpp @@ -55,12 +55,27 @@ static void check_factory_reset() { // Camera + CV task — runs on core 1 at 5 fps static void task_camera(void*) { static uint8_t frame[CV_PIXELS]; // static: avoids 9KB on task stack + int last_logged_track_id = 0; // diagnostic: log each new track once while (true) { if (camera_capture_96(frame)) { if (xSemaphoreTake(s_cv_mutex, pdMS_TO_TICKS(100)) == pdTRUE) { CVResult r = cv_process(g_cv, frame, g_cfg.line_offset); - if (r.entries_delta) Serial.printf("[CV] entry +%d (total %d)\n", r.entries_delta, g_cv.entries); - if (r.exits_delta) Serial.printf("[CV] exit +%d (total %d)\n", r.exits_delta, g_cv.exits); + for (const auto& t : g_cv.tracks) { + if (t.id > last_logged_track_id) { + last_logged_track_id = t.id; + Serial.printf("[CV] spawn id=%d y=%.1f\n", t.id, t.spawn_y); + } + } + if (r.fg_count > 0) { + Serial.printf("[F] n=%d y=%d..%d c=%.1f\n", + r.fg_count, r.fg_min_y, r.fg_max_y, r.fg_centroid_y); + } + if (r.entries_delta) Serial.printf("[CV] entry +%d (total %d) first=%.1f min=%.1f max=%.1f last=%.1f dur=%d\n", + r.entries_delta, g_cv.entries, + r.fire_first_c, r.fire_min_c, r.fire_max_c, r.fire_last_c, r.fire_duration); + if (r.exits_delta) Serial.printf("[CV] exit +%d (total %d) first=%.1f min=%.1f max=%.1f last=%.1f dur=%d\n", + r.exits_delta, g_cv.exits, + r.fire_first_c, r.fire_min_c, r.fire_max_c, r.fire_last_c, r.fire_duration); xSemaphoreGive(s_cv_mutex); if (r.entries_delta) led_blink_pattern(1); if (r.exits_delta) led_blink_pattern(2); diff --git a/firmware/src/main_capture.cpp b/firmware/src/main_capture.cpp new file mode 100644 index 0000000..3beac8f --- /dev/null +++ b/firmware/src/main_capture.cpp @@ -0,0 +1,64 @@ +// firmware/src/main_capture.cpp +// +// Frame-dump firmware. Replaces main.cpp when building env:timercam-capture. +// Streams raw 96x96 grayscale frames at 5 fps over serial (921600 baud) for +// offline algorithm iteration. +// +// Wire format per frame (little-endian): +// magic uint32 0xDC0FC0DE +// frame_ix uint32 monotonic counter +// millis uint32 ms since boot +// pixels byte[9216] raw grayscale 96x96, row-major +// +// No WiFi, no BLE, no CV. Just camera → serial. + +#include +#include "camera.h" +#include "cv.h" // for CV_PIXELS + +#define LED_PIN 2 +#define CAM_FPS 5 +#define CAM_INTERVAL_MS (1000 / CAM_FPS) + +// Magic chosen from bytes that commonly survive; 'FRM1' ascii. +// Avoid high bytes 0xA0-AF / 0xD0-DF — observed missing from the CH9102 stream. +static const uint32_t FRAME_MAGIC = 0x314D5246; // 'FRM1' little-endian on wire + +void setup() { + Serial.begin(460800); + pinMode(LED_PIN, OUTPUT); + digitalWrite(LED_PIN, HIGH); + + delay(500); + Serial.println("# capture-mode: 460800 baud, 96x96 gray @ 5fps"); + Serial.flush(); + + if (!camera_init()) { + Serial.println("# FATAL: camera init failed"); + while (true) { + digitalWrite(LED_PIN, !digitalRead(LED_PIN)); + delay(200); + } + } + + digitalWrite(LED_PIN, LOW); +} + +void loop() { + static uint8_t frame[CV_PIXELS]; + static uint32_t frame_ix = 0; + uint32_t t0 = millis(); + + if (camera_capture_96(frame)) { + uint32_t ms = millis(); + Serial.write((uint8_t*)&FRAME_MAGIC, 4); + Serial.write((uint8_t*)&frame_ix, 4); + Serial.write((uint8_t*)&ms, 4); + Serial.write(frame, CV_PIXELS); + frame_ix++; + digitalWrite(LED_PIN, frame_ix & 1); + } + + uint32_t elapsed = millis() - t0; + if (elapsed < CAM_INTERVAL_MS) delay(CAM_INTERVAL_MS - elapsed); +} diff --git a/firmware/test/test_cv/test_cv.cpp b/firmware/test/test_cv/test_cv.cpp index 8e664b1..32f89ef 100644 --- a/firmware/test/test_cv/test_cv.cpp +++ b/firmware/test/test_cv/test_cv.cpp @@ -7,258 +7,290 @@ static void fill_frame(uint8_t* f, uint8_t val) { memset(f, val, CV_PIXELS); } +// Draw a rectangular walker-blob spanning rows [y0, y1], columns [cx-hw, cx+hw]. +// Pixel value 200 over background 100 -> frame_diff threshold (30) is cleared. +static void draw_walker(uint8_t* f, int y0, int y1, int cx, int hw) { + fill_frame(f, 100); + for (int y = y0; y <= y1; y++) { + if (y < 0 || y >= CV_H) continue; + for (int x = cx - hw; x <= cx + hw; x++) { + if (x < 0 || x >= CV_W) continue; + f[y * CV_W + x] = 200; + } + } +} + +static void prime_bg(CVState& state) { + uint8_t bg[CV_PIXELS]; + fill_frame(bg, 100); + cv_process(state, bg, 50); +} + +// Let the event state machine see QUIET_FRAMES+1 empty frames so any active +// event finalizes before the next test assertion. +static void quiesce(CVState& state) { + uint8_t bg[CV_PIXELS]; fill_frame(bg, 100); + for (int i = 0; i < CV_EVENT_QUIET_FRAMES + 1; i++) cv_process(state, bg, 50); +} + void setUp(void) {} void tearDown(void) {} -void test_frame_diff_no_change_gives_no_fg() { - CVState state; - cv_init(state); - - uint8_t frame[CV_PIXELS]; - fill_frame(frame, 128); - +void test_no_change_no_event() { + CVState state; cv_init(state); + uint8_t frame[CV_PIXELS]; fill_frame(frame, 128); CVResult r1 = cv_process(state, frame, 50); TEST_ASSERT_EQUAL_INT(0, r1.entries_delta); - CVResult r2 = cv_process(state, frame, 50); TEST_ASSERT_EQUAL_INT(0, r2.entries_delta); TEST_ASSERT_EQUAL_INT(0, r2.exits_delta); } -void test_frame_diff_large_change_detected_no_crash() { - CVState state; - cv_init(state); - - uint8_t bg[CV_PIXELS], fg_frame[CV_PIXELS]; - fill_frame(bg, 100); - fill_frame(fg_frame, 200); - - cv_process(state, bg, 50); - CVResult r = cv_process(state, fg_frame, 50); - - // Tracking not yet implemented — just verify no crash and result is zero - TEST_ASSERT_EQUAL_INT(0, r.entries_delta); - TEST_ASSERT_EQUAL_INT(0, r.exits_delta); -} - void test_cv_init_clears_state() { CVState state; - state.entries = 99; state.exits = 88; + state.entries = 99; state.exits = 88; state.event_active = true; cv_init(state); TEST_ASSERT_EQUAL_INT(0, state.entries); TEST_ASSERT_EQUAL_INT(0, state.exits); TEST_ASSERT_FALSE(state.bg_valid); + TEST_ASSERT_FALSE(state.event_active); } void test_cv_reset_counts() { - CVState state; - cv_init(state); - state.entries = 5; - state.exits = 3; + CVState state; cv_init(state); + state.entries = 5; state.exits = 3; cv_reset_counts(state); TEST_ASSERT_EQUAL_INT(0, state.entries); TEST_ASSERT_EQUAL_INT(0, state.exits); } -void test_tracking_spawns_track_for_new_blob() { - CVState state; - cv_init(state); +void test_walker_up_through_frame_is_entry() { + // Simulate a walker traversing from bottom to top of frame. + // Per-frame fg_count and centroid (11-wide column, height H -> n=11*H): + // t0 y=60..95 n=396 c=77 <- event starts (n >= ENTER=300) + // t1 y=30..95 n=726 c=62 + // t2 y=0..95 n=1056 c=47 + // t3 y=0..60 n=671 c=30 + // t4 y=0..25 n=286 c=12 (below EXIT=200, quiet=1) + // t5 y=0..10 n=121 c=5 (below EXIT, quiet=2) + // t6 empty quiet=3 -> finalize + CVState state; cv_init(state); + prime_bg(state); - uint8_t bg[CV_PIXELS]; - fill_frame(bg, 100); - cv_process(state, bg, 50); // init background - - // Frame with a bright 30x30 blob in top-left quadrant - uint8_t blob_frame[CV_PIXELS]; - fill_frame(blob_frame, 100); - for (int y = 5; y < 35; y++) - for (int x = 5; x < 35; x++) - blob_frame[y * CV_W + x] = 200; - - cv_process(state, blob_frame, 50); - - TEST_ASSERT_EQUAL_INT(1, (int)state.tracks.size()); - TEST_ASSERT_FLOAT_WITHIN(5.0f, 20.0f, state.tracks[0].x); - TEST_ASSERT_FLOAT_WITHIN(5.0f, 20.0f, state.tracks[0].y); -} - -static void make_blob_frame(uint8_t* f, int cx, int cy) { - fill_frame(f, 100); - for (int y = cy - 12; y <= cy + 12; y++) - for (int x = cx - 12; x <= cx + 12; x++) - if (y >= 0 && y < CV_H && x >= 0 && x < CV_W) - f[y * CV_W + x] = 200; -} - -void test_blob_crossing_line_top_to_bottom_is_entry() { - CVState state; - cv_init(state); - - // Line at 50% = y=48, traversal margin = 14px. Spawn must be y<34, final y>62. - // Step ≤14px per frame to stay within CV_MAX_MOVE. - uint8_t bg[CV_PIXELS]; - fill_frame(bg, 100); - cv_process(state, bg, 50); // init background - - int setup[] = {20, 34, 48, 62}; // spawn firm above, walk across, not yet firm below - for (int i = 0; i < 4; i++) { - uint8_t f[CV_PIXELS]; make_blob_frame(f, 48, setup[i]); + int rows[][2] = {{60,95},{30,95},{0,95},{0,60},{0,25},{0,10}}; + for (int i = 0; i < 6; i++) { + uint8_t f[CV_PIXELS]; draw_walker(f, rows[i][0], rows[i][1], 48, 5); cv_process(state, f, 50); } - // Still no count — y=62 is not firm below (needs >62) - TEST_ASSERT_EQUAL_INT(0, state.entries); + quiesce(state); - // One more step: y=70 is firm below → entry fires now - uint8_t fcross[CV_PIXELS]; make_blob_frame(fcross, 48, 70); - CVResult r = cv_process(state, fcross, 50); - - TEST_ASSERT_EQUAL_INT(1, r.entries_delta); - TEST_ASSERT_EQUAL_INT(0, r.exits_delta); TEST_ASSERT_EQUAL_INT(1, state.entries); -} - -void test_blob_crossing_line_bottom_to_top_is_exit() { - CVState state; - cv_init(state); - - uint8_t bg[CV_PIXELS]; fill_frame(bg, 100); - cv_process(state, bg, 50); - - // Spawn firm below (y=76 > 62), walk toward and across line (y=48), continue - // until firm above (y<34). Each step ≤14px. - int setup[] = {76, 62, 48, 34}; - for (int i = 0; i < 4; i++) { - uint8_t f[CV_PIXELS]; make_blob_frame(f, 48, setup[i]); - cv_process(state, f, 50); - } - // y=34 not firm above (needs <34) — no count yet TEST_ASSERT_EQUAL_INT(0, state.exits); - - uint8_t fcross[CV_PIXELS]; make_blob_frame(fcross, 48, 22); // firm above - CVResult r = cv_process(state, fcross, 50); - - TEST_ASSERT_EQUAL_INT(0, r.entries_delta); - TEST_ASSERT_EQUAL_INT(1, r.exits_delta); } -void test_track_spawned_near_line_does_not_count_on_wobble() { - // Simulates a blob that appears right on the line (e.g. shadow or noise) - // and wobbles across it. With directional margin, no count should fire — - // this is the false-positive pattern the feature guards against. - CVState state; - cv_init(state); +void test_walker_down_through_frame_is_exit() { + CVState state; cv_init(state); + prime_bg(state); - uint8_t bg[CV_PIXELS]; fill_frame(bg, 100); - cv_process(state, bg, 50); - - // Spawn within margin of line (y=44, margin=14 so 44 ∈ [34,62]) - // then wobble above to y=38, below to y=58. Both within margin. - int setup[] = {44, 38, 58, 42, 56}; - for (int i = 0; i < 5; i++) { - uint8_t f[CV_PIXELS]; make_blob_frame(f, 48, setup[i]); + int rows[][2] = {{0,35},{0,65},{0,95},{35,95},{70,95},{85,95}}; + for (int i = 0; i < 6; i++) { + uint8_t f[CV_PIXELS]; draw_walker(f, rows[i][0], rows[i][1], 48, 5); cv_process(state, f, 50); } + quiesce(state); + + TEST_ASSERT_EQUAL_INT(0, state.entries); + TEST_ASSERT_EQUAL_INT(1, state.exits); +} + +void test_approach_retreat_without_full_extent_does_not_fire() { + // Walker approaches from bottom, reaches y=30, retreats, never reaches top. + // Extent gate requires min_y_seen <= 10; this event tops out at y=30 so + // extent never clears and no fire occurs regardless of trajectory score. + CVState state; cv_init(state); + prime_bg(state); + + int rows[][2] = {{60,95},{40,95},{30,95},{40,95},{60,95},{80,95}}; + for (int i = 0; i < 6; i++) { + uint8_t f[CV_PIXELS]; draw_walker(f, rows[i][0], rows[i][1], 48, 5); + cv_process(state, f, 50); + } + quiesce(state); + TEST_ASSERT_EQUAL_INT(0, state.entries); TEST_ASSERT_EQUAL_INT(0, state.exits); } -void test_track_counts_at_most_once_even_if_it_wobbles_back() { - // A track that traverses fully should count once. If it then reverses and - // crosses back, the track should NOT fire again — it's already counted. - // (A separate new track on the return trip would count as exit, but while - // the same track persists, it's one trip.) - CVState state; - cv_init(state); +void test_brief_burst_below_min_duration_does_not_fire() { + // One frame of large fg, then gone. Event starts, immediately quiesces, + // duration ends up below CV_EVENT_MIN_FRAMES. + CVState state; cv_init(state); + prime_bg(state); - uint8_t bg[CV_PIXELS]; fill_frame(bg, 100); - cv_process(state, bg, 50); + uint8_t f[CV_PIXELS]; draw_walker(f, 0, 95, 48, 5); + cv_process(state, f, 50); + quiesce(state); - // Full traversal top→bottom - int walk_down[] = {20, 34, 48, 62, 70}; - for (int i = 0; i < 5; i++) { - uint8_t f[CV_PIXELS]; make_blob_frame(f, 48, walk_down[i]); - cv_process(state, f, 50); - } - TEST_ASSERT_EQUAL_INT(1, state.entries); - - // Same track reverses back to top. counted=true prevents a second event. - int walk_up[] = {62, 48, 34, 22}; - for (int i = 0; i < 4; i++) { - uint8_t f[CV_PIXELS]; make_blob_frame(f, 48, walk_up[i]); - cv_process(state, f, 50); - } - TEST_ASSERT_EQUAL_INT(1, state.entries); + TEST_ASSERT_EQUAL_INT(0, state.entries); TEST_ASSERT_EQUAL_INT(0, state.exits); } -void test_cooldown_suppresses_rapid_re_entry() { - // Cooldown is a safety net on top of directional counting. Construct two - // DIFFERENT tracks (each counts once on its own) whose crossings happen - // within the cooldown window — the second should still be suppressed. - CVState state; - cv_init(state); - state.bg_valid = true; - memset(state.background, 100, CV_PIXELS); - state.frame_index = 100; - state.entries = 1; - state.last_entry_frame = 100; +void test_stationary_large_blob_does_not_fire() { + // Static large blob in frame for many frames, then removed. Centroid + // never moves -> MIN_TRAJ gate blocks fire. + CVState state; cv_init(state); + prime_bg(state); - // Track at y=50 (just below line), spawn_y=20 (firm above) — a valid trajectory. - CVTrack t; - t.id = 1; t.x = 48; t.y = 50; t.spawn_y = 20; - t.above_line = false; t.counted = false; t.missed = 0; - state.tracks.push_back(t); + for (int i = 0; i < 10; i++) { + uint8_t f[CV_PIXELS]; draw_walker(f, 0, 95, 48, 5); + cv_process(state, f, 50); + } + quiesce(state); - // Frame 101: blob at y=64 (delta=14, matches; firm below line+margin=62). - // Would count but cooldown (101-100=1 < 5) suppresses. - uint8_t f1[CV_PIXELS]; make_blob_frame(f1, 48, 64); - CVResult r1 = cv_process(state, f1, 50); - TEST_ASSERT_EQUAL_INT(0, r1.entries_delta); + TEST_ASSERT_EQUAL_INT(0, state.entries); + TEST_ASSERT_EQUAL_INT(0, state.exits); +} + +// Wait out the refractory period with bg-only frames so the next walker +// event is accepted. +static void wait_refractory(CVState& state) { + uint8_t bg[CV_PIXELS]; fill_frame(bg, 100); + for (uint32_t i = 0; i < CV_EVENT_REFRACTORY_FRAMES + 2; i++) { + cv_process(state, bg, 50); + } +} + +void test_two_sequential_walkers_count_twice() { + CVState state; cv_init(state); + prime_bg(state); + + int rows[][2] = {{60,95},{30,95},{0,95},{0,60},{0,25},{0,10}}; + for (int i = 0; i < 6; i++) { + uint8_t f[CV_PIXELS]; draw_walker(f, rows[i][0], rows[i][1], 48, 5); + cv_process(state, f, 50); + } + quiesce(state); + wait_refractory(state); + + for (int i = 0; i < 6; i++) { + uint8_t f[CV_PIXELS]; draw_walker(f, rows[i][0], rows[i][1], 48, 5); + cv_process(state, f, 50); + } + quiesce(state); + + TEST_ASSERT_EQUAL_INT(2, state.entries); + TEST_ASSERT_EQUAL_INT(0, state.exits); +} + +void test_full_reversal_counts_entry_then_exit() { + CVState state; cv_init(state); + prime_bg(state); + + int up_rows[][2] = {{60,95},{30,95},{0,95},{0,60},{0,25},{0,10}}; + int down_rows[][2] = {{0,35},{0,65},{0,95},{35,95},{70,95},{85,95}}; + + for (int i = 0; i < 6; i++) { + uint8_t f[CV_PIXELS]; draw_walker(f, up_rows[i][0], up_rows[i][1], 48, 5); + cv_process(state, f, 50); + } + quiesce(state); + wait_refractory(state); + + for (int i = 0; i < 6; i++) { + uint8_t f[CV_PIXELS]; draw_walker(f, down_rows[i][0], down_rows[i][1], 48, 5); + cv_process(state, f, 50); + } + quiesce(state); + + TEST_ASSERT_EQUAL_INT(1, state.entries); + TEST_ASSERT_EQUAL_INT(1, state.exits); +} + +void test_refractory_suppresses_back_to_back_fire() { + // After a fire, a second event attempted within CV_EVENT_REFRACTORY_FRAMES + // is suppressed. Simulates walker lingering / ghost re-triggering. + CVState state; cv_init(state); + prime_bg(state); + + int rows[][2] = {{60,95},{30,95},{0,95},{0,60},{0,25},{0,10}}; + for (int i = 0; i < 6; i++) { + uint8_t f[CV_PIXELS]; draw_walker(f, rows[i][0], rows[i][1], 48, 5); + cv_process(state, f, 50); + } + quiesce(state); TEST_ASSERT_EQUAL_INT(1, state.entries); - // Advance past cooldown; reset a fresh track (previous one had counted=true - // set only if it actually counted — cooldown path leaves counted=false so - // we reuse the same track). - state.frame_index = 200; - state.tracks[0].y = 50; - state.tracks[0].spawn_y = 20; - state.tracks[0].counted = false; - state.tracks[0].above_line = false; - uint8_t f2[CV_PIXELS]; make_blob_frame(f2, 48, 64); - CVResult r2 = cv_process(state, f2, 50); - TEST_ASSERT_EQUAL_INT(1, r2.entries_delta); + // Immediate second walker within refractory window — should NOT count. + for (int i = 0; i < 6; i++) { + uint8_t f[CV_PIXELS]; draw_walker(f, rows[i][0], rows[i][1], 48, 5); + cv_process(state, f, 50); + } + quiesce(state); + TEST_ASSERT_EQUAL_INT(1, state.entries); +} + +void test_event_counts_after_refractory_expires() { + CVState state; cv_init(state); + prime_bg(state); + + int rows[][2] = {{60,95},{30,95},{0,95},{0,60},{0,25},{0,10}}; + for (int i = 0; i < 6; i++) { + uint8_t f[CV_PIXELS]; draw_walker(f, rows[i][0], rows[i][1], 48, 5); + cv_process(state, f, 50); + } + quiesce(state); + TEST_ASSERT_EQUAL_INT(1, state.entries); + + // Wait out the refractory period. + uint8_t bg[CV_PIXELS]; fill_frame(bg, 100); + for (uint32_t i = 0; i < CV_EVENT_REFRACTORY_FRAMES + 2; i++) { + cv_process(state, bg, 50); + } + + // Second walker — should now count. + for (int i = 0; i < 6; i++) { + uint8_t f[CV_PIXELS]; draw_walker(f, rows[i][0], rows[i][1], 48, 5); + cv_process(state, f, 50); + } + quiesce(state); TEST_ASSERT_EQUAL_INT(2, state.entries); } -void test_no_crossing_same_side_no_count() { - CVState state; - cv_init(state); +void test_noise_below_enter_thresh_does_not_start_event() { + // Tiny 5x5 blob (25 px) never crosses ENTER=300, event never starts. + CVState state; cv_init(state); + prime_bg(state); - uint8_t bg[CV_PIXELS]; fill_frame(bg, 100); - cv_process(state, bg, 50); + auto small = [](uint8_t* f, int cy) { + fill_frame(f, 100); + for (int y = cy-2; y <= cy+2; y++) + for (int x = 46; x <= 50; x++) + if (y>=0 && y=0 && x 4: + del window[0] + if bytes(window) == magic_bytes: + return True + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument('--port', required=True) + ap.add_argument('--baud', type=int, default=460800) + ap.add_argument('--out', required=True) + ap.add_argument('--duration', type=float, default=60.0, + help='Seconds to capture (default 60)') + args = ap.parse_args() + + ser = serial.Serial(args.port, args.baud, timeout=1.0) + print(f'# listening on {args.port} @ {args.baud} for {args.duration}s...', + file=sys.stderr) + + # Drain boot banner lines. + deadline_banner = time.time() + 2.0 + while time.time() < deadline_banner: + line = ser.readline() + if line.startswith(b'#'): + print(line.decode(errors='replace').rstrip(), file=sys.stderr) + if b'capture-mode' in line: + break + + deadline = time.time() + args.duration + frames = 0 + last_ix = None + dropped = 0 + + with open(args.out, 'wb') as f: + while time.time() < deadline: + if not find_magic(ser): + continue + body = read_exact(ser, 8 + FRAME_PIXELS) + if body is None: + break + frame_ix, ms = struct.unpack(' a.extent_top: return None + if self.ev_max_y < a.extent_bot: return None + up = self.ev_first_c - self.ev_min_c + down = self.ev_max_c - self.ev_first_c + winning = max(up, down) + if winning < a.min_traj: return None + is_entry = up >= down + self.last_fire = self.frame_ix + info = dict( + kind='ENTRY' if is_entry else 'EXIT', + first=self.ev_first_c, min=self.ev_min_c, + max=self.ev_max_c, last=self.ev_last_c, + dur=self.ev_frames, + ) + if is_entry: self.entries += 1 + else: self.exits += 1 + return info + + def step(self, frame): + """frame: uint8 array of shape (H, W). Returns list of fire dicts.""" + self.frame_ix += 1 + fires = [] + + if self.bg is None: + self.bg = frame.astype(np.int16) + return fires + + bg = self.bg.astype(np.int16) + diff = np.abs(frame.astype(np.int16) - bg) + fg = (diff > self.a.diff_thresh).astype(np.uint8) + + # Running-avg bg blend, frozen during active event. + if not self.ev_active: + self.bg = ((self.bg * 31 + frame.astype(np.int16)) >> 5) + + fg_count = int(fg.sum()) + if fg_count > 0: + row_counts = fg.sum(axis=1) + ys = np.where(row_counts > 0)[0] + min_y = int(ys.min()) + max_y = int(ys.max()) + centroid_y = float((row_counts * np.arange(H)).sum() / fg_count) + else: + min_y, max_y, centroid_y = -1, -1, -1.0 + + # Self-heal on catastrophic bg mismatch. + if fg_count > PIXELS // 2: + self.bg = frame.astype(np.int16) + if self.ev_active: self._reset_event() + return fires + + a = self.a + in_refractory = (self.last_fire != 0 and + (self.frame_ix - self.last_fire) < a.refractory) + + if not self.ev_active: + if not in_refractory and fg_count >= a.enter_thresh: + self.ev_active = True + self.ev_frames = 1 + self.ev_first_c = centroid_y + self.ev_last_c = centroid_y + self.ev_min_c = centroid_y + self.ev_max_c = centroid_y + self.ev_min_y = min_y + self.ev_max_y = max_y + self.ev_quiet = 0 + else: + self.ev_frames += 1 + if fg_count > 0: + self.ev_last_c = centroid_y + if centroid_y < self.ev_min_c: self.ev_min_c = centroid_y + if centroid_y > self.ev_max_c: self.ev_max_c = centroid_y + if min_y < self.ev_min_y: self.ev_min_y = min_y + if max_y > self.ev_max_y: self.ev_max_y = max_y + + ended = False + if fg_count < a.exit_thresh: + self.ev_quiet += 1 + if self.ev_quiet >= a.quiet_frames: + ended = True + else: + self.ev_quiet = 0 + if self.ev_frames > a.max_frames: + ended = True + + if ended: + fire = self._finalize() + if fire: fires.append(fire) + self._reset_event() + self.bg = frame.astype(np.int16) + + return fires, fg_count, min_y, max_y, centroid_y + + +def iter_frames(path): + with open(path, 'rb') as f: + data = f.read() + n = len(data) // FRAME_LEN + for i in range(n): + off = i * FRAME_LEN + magic, ix, ms = struct.unpack(' 0: + print(f'[{ix:4d}] n={fg:4d} y={miny:2d}..{maxy:2d} c={cy:5.1f}') + for fire in fires: + print(f' >>> {fire["kind"]} first={fire["first"]:.1f} ' + f'min={fire["min"]:.1f} max={fire["max"]:.1f} ' + f'last={fire["last"]:.1f} dur={fire["dur"]}') + print(f'\n# {total} frames entries={det.entries} exits={det.exits}') + + +if __name__ == '__main__': + main() diff --git a/tools/replay_logs.py b/tools/replay_logs.py new file mode 100644 index 0000000..ab9617a --- /dev/null +++ b/tools/replay_logs.py @@ -0,0 +1,186 @@ +#!/usr/bin/env python3 +# tools/replay_logs.py +# +# Replay the event state machine against text serial logs captured from the +# production firmware. Input lines of the form: +# [F] n= y=.. c= +# +# Those four values are exactly what the firmware's event state machine +# consumes — so we can iterate event-level params (thresholds, max_frames, +# extent gates, trajectory cutoffs, refractory) offline without needing raw +# frames or the device. +# +# Usage: +# python tools/replay_logs.py walk.log +# python tools/replay_logs.py walk.log --enter 250 --exit 100 --max 30 --min-traj 10 +# cat walk.log | python tools/replay_logs.py - --ground-truth 12 + +import argparse +import re +import sys + + +LINE_RE = re.compile( + r'\[F\]\s+n=(?P\d+)\s+y=(?P-?\d+)\.\.(?P-?\d+)\s+c=(?P-?\d+\.\d+)' +) + + +def parse_frames(text): + """Yield (fg_count, min_y, max_y, centroid_y) per [F] line, in order.""" + for line in text.splitlines(): + m = LINE_RE.search(line) + if not m: + continue + yield int(m['n']), int(m['miny']), int(m['maxy']), float(m['c']) + + +class Detector: + """Mirror of firmware event state machine. Only uses per-frame diagnostic + values — the same inputs the firmware feeds it.""" + + def __init__(self, a): + self.a = a + self.ev = False + self.ev_n = 0 + self.ev_first = self.ev_last = -1.0 + self.ev_min = 1e9 + self.ev_max = -1.0 + self.ev_miny = 1e9 + self.ev_maxy = -1 + self.ev_quiet = 0 + self.last_fire = -10**9 + self.ix = 0 + self.entries = 0 + self.exits = 0 + self.fires = [] + + def _reset(self): + self.ev = False + self.ev_n = 0 + self.ev_first = self.ev_last = -1.0 + self.ev_min = 1e9; self.ev_max = -1.0 + self.ev_miny = 1e9; self.ev_maxy = -1 + self.ev_quiet = 0 + + def _finalize(self): + a = self.a + if self.ev_n < a.min_frames: + return ('reject_short', None) + if self.ev_miny > a.extent_top: + return ('reject_extent_top', None) + if self.ev_maxy < a.extent_bot: + return ('reject_extent_bot', None) + up = self.ev_first - self.ev_min + down = self.ev_max - self.ev_first + winning = max(up, down) + if winning < a.min_traj: + return ('reject_traj', None) + timed_out = self.ev_n > a.max_frames + if timed_out: + is_entry = self.ev_last < self.ev_first + else: + is_entry = up >= down + kind = 'ENTRY' if is_entry else 'EXIT' + self.last_fire = self.ix + info = dict(kind=kind, first=self.ev_first, min=self.ev_min, + max=self.ev_max, last=self.ev_last, dur=self.ev_n, + up=up, down=down, ix=self.ix) + if is_entry: self.entries += 1 + else: self.exits += 1 + self.fires.append(info) + return ('fire', info) + + def step(self, n, miny, maxy, c): + self.ix += 1 + a = self.a + refractory = (self.ix - self.last_fire) < a.refractory + + if not self.ev: + if not refractory and n >= a.enter_thresh: + self.ev = True + self.ev_n = 1 + self.ev_first = self.ev_last = c + self.ev_min = c; self.ev_max = c + self.ev_miny = miny; self.ev_maxy = maxy + self.ev_quiet = 0 + return None + + self.ev_n += 1 + if n > 0: + self.ev_last = c + if c < self.ev_min: self.ev_min = c + if c > self.ev_max: self.ev_max = c + if miny < self.ev_miny: self.ev_miny = miny + if maxy > self.ev_maxy: self.ev_maxy = maxy + + ended = False + if n < a.exit_thresh: + self.ev_quiet += 1 + if self.ev_quiet >= a.quiet_frames: + ended = True + reason = 'quiet' + else: + self.ev_quiet = 0 + if self.ev_n > a.max_frames: + ended = True + reason = 'timeout' + + if ended: + result = self._finalize() + self._reset() + return (reason, result) + return None + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument('path', help='log file, or - for stdin') + ap.add_argument('--enter', dest='enter_thresh', type=int, default=300) + ap.add_argument('--exit', dest='exit_thresh', type=int, default=200) + ap.add_argument('--quiet', dest='quiet_frames', type=int, default=3) + ap.add_argument('--min', dest='min_frames', type=int, default=5) + ap.add_argument('--max', dest='max_frames', type=int, default=25) + ap.add_argument('--extent-top', dest='extent_top', type=int, default=10) + ap.add_argument('--extent-bot', dest='extent_bot', type=int, default=85) + ap.add_argument('--min-traj', dest='min_traj', type=float, default=15.0) + ap.add_argument('--refractory', dest='refractory', type=int, default=15) + ap.add_argument('--ground-truth', type=int, default=0, + help='Total expected walks for accuracy calc') + ap.add_argument('-v', '--verbose', action='store_true', + help='Print every event end, including rejections') + args = ap.parse_args() + + text = sys.stdin.read() if args.path == '-' else open(args.path).read() + + det = Detector(args) + rejects = {} + for n, miny, maxy, c in parse_frames(text): + out = det.step(n, miny, maxy, c) + if out is None: + continue + reason, result = out + if result is None: + continue + kind, info = result + if kind == 'fire': + print(f' {info["kind"]:5} first={info["first"]:5.1f} ' + f'min={info["min"]:5.1f} max={info["max"]:5.1f} ' + f'last={info["last"]:5.1f} dur={info["dur"]:2d} ' + f'exit={reason}') + else: + rejects[kind] = rejects.get(kind, 0) + 1 + if args.verbose: + print(f' [drop {kind}]') + + total = det.entries + det.exits + print(f'\n=== entries={det.entries} exits={det.exits} total={total} ===') + print(f'rejected events: {rejects}') + if args.ground_truth: + gt = args.ground_truth + acc = min(total, gt) / gt * 100 + over = max(0, total - gt) + print(f'accuracy vs gt={gt}: {acc:.0f}% (over={over})') + + +if __name__ == '__main__': + main()