From 9d5b5882319d3425d7ce1b3731f579851ddc6113 Mon Sep 17 00:00:00 2001 From: Peter Woolery Date: Thu, 16 Apr 2026 11:13:50 -0700 Subject: [PATCH] feat: production-ready firmware with BLE memory management, device_id fixes, and docs - Reduce debug level to 1 (errors only) for production builds - Replace BLE pause/resume with full deinit/reinit during HTTP uploads (~25KB freed) - Add 60s boot report delay for fast post-deploy connectivity verification - Add device_id to BLE batch and heartbeat request bodies - Correct API host to http:// (plain HTTP, not HTTPS) - Add HTTP response logging and CV entry/exit serial logging - Create root README.md with operator setup and architecture overview - Update design spec: HMAC format, BLE memory approach, request body shapes, reporting intervals Co-Authored-By: Claude Sonnet 4.6 --- README.md | 107 ++++++++++++++++++ .../specs/2026-04-13-door-counter-design.md | 57 ++++++++-- firmware/platformio.ini | 2 +- firmware/src/ble_scanner.cpp | 13 ++- firmware/src/ble_scanner.h | 6 +- firmware/src/main.cpp | 22 ++-- firmware/src/reporter.cpp | 7 +- firmware/src/reporter.h | 2 +- 8 files changed, 190 insertions(+), 26 deletions(-) create mode 100644 README.md diff --git a/README.md b/README.md new file mode 100644 index 0000000..1755822 --- /dev/null +++ b/README.md @@ -0,0 +1,107 @@ +# DoorCounter + +Retail door traffic counter using M5Stack TimerCamera-F (ESP32 + OV3660). Counts entries/exits via overhead camera CV, passively scans BLE foot traffic, and reports hourly to `logs.research.bike`. + +## Hardware + +- **Device**: M5Stack TimerCamera-F (ESP32-S, OV3660, PSRAM, WiFi/BLE) +- **Mount**: Overhead, camera pointing straight down, centered above doorway +- **Power**: USB (any phone charger) + +## Firmware + +Built with PlatformIO. Target: `timercam`. + +```bash +cd firmware +pio run -t upload --upload-port /dev/ttyUSB0 +``` + +### What it does + +| Module | Behavior | +|--------|----------| +| CV pipeline | 5 fps, 96×96 grayscale, blob tracking, line-crossing count | +| BLE scanner | Continuous passive scan; deinits during hourly upload to free heap | +| Reporter | Hourly HMAC-signed POST; 60s boot report for fast connectivity check | +| Provisioning | Captive portal AP on first boot for WiFi setup | +| OTA | Arduino OTA; operator push via `ota_push.py` | + +### Reporting intervals + +- **First report**: 60 seconds after NTP sync (connectivity check) +- **Subsequent reports**: every 3600 seconds + +## Operator Setup + +### 1. Flash firmware + +```bash +cd firmware +pio run -t upload --upload-port /dev/ttyUSB0 +``` + +### 2. Provision device identity + +```bash +python tools/flash_device.py \ + --port /dev/ttyUSB0 \ + --device-id dc-0042 \ + --location-id retailer-123 \ + --hmac-secret <32-byte-hex> \ + --wifi-ssid "StoreWiFi" \ + --wifi-password "secret" +``` + +WiFi credentials are optional — if omitted, device starts captive portal on boot. + +### 3. OTA updates + +```bash +python tools/ota_push.py \ + --host dc-0042.local \ + --firmware firmware/.pio/build/timercam/firmware.bin +``` + +## End User Setup + +1. Mount device overhead, camera pointing straight down +2. Plug into USB power +3. Connect phone to `DoorCounter-Setup` WiFi +4. Browser opens automatically → enter store WiFi password → done + +**LED indicators**: Red = no WiFi · Blue = counting · Yellow = uploading + +## API + +Endpoint: `http://logs.research.bike` + +| Endpoint | Data | +|----------|------| +| `POST /api/v1/camera/events/batch` | Hourly entry/exit counts | +| `POST /api/v1/events/batch` | Hourly BLE proximity records | +| `POST /api/v1/heartbeat` | Device health (uptime, RSSI, pending records) | + +All requests are HMAC-SHA256 signed. See [design spec](docs/superpowers/specs/2026-04-13-door-counter-design.md) for full API shapes and auth scheme. + +## Project Structure + +``` +DoorCounter/ +├── firmware/ +│ ├── platformio.ini +│ ├── lib/hmac/ — HMAC-SHA256 signing library +│ └── src/ +│ ├── main.cpp — FreeRTOS tasks, boot sequence +│ ├── config.* — NVS read/write +│ ├── provisioning.* — captive portal +│ ├── camera.* — frame capture + CV pipeline +│ ├── ble_scanner.* — BLE passive scan +│ └── reporter.* — hourly batch POST + local buffer +├── tools/ +│ ├── flash_device.py — NVS provisioning script +│ └── ota_push.py — OTA push script +├── docs/superpowers/specs/ +│ └── 2026-04-13-door-counter-design.md +└── server/ — API server (separate deployment) +``` diff --git a/docs/superpowers/specs/2026-04-13-door-counter-design.md b/docs/superpowers/specs/2026-04-13-door-counter-design.md index 209d0fa..aca28f4 100644 --- a/docs/superpowers/specs/2026-04-13-door-counter-design.md +++ b/docs/superpowers/specs/2026-04-13-door-counter-design.md @@ -103,7 +103,7 @@ Counts accumulate as `{entries, exits}` in RAM and reset each hour on report. ## 4. BLE Scanning -Uses ESP32 built-in WiFi+BLE coexistence mode — BLE scans continuously while WiFi remains available. The only pause is a ~3s window during the hourly HTTP POST. +Uses ESP32 built-in WiFi+BLE coexistence mode — BLE scans continuously while WiFi is available. During each hourly upload the BLE stack is fully deinitialized (freeing ~25KB heap for the HTTP client) then reinitialized after the upload completes. - Passive BLE scan, accumulates unique device hashes, near/mid/far counts per hour - Reports to existing `/api/v1/events/batch` endpoint (no server changes needed for BLE) @@ -115,13 +115,15 @@ Uses ESP32 built-in WiFi+BLE coexistence mode — BLE scans continuously while W ### HMAC scheme -Each request includes header: +Each request includes headers: ``` -X-HMAC-Signature: +X-Device-Id: +X-Timestamp: +X-Signature: ``` -Computed as: +Signature computed as: ``` -HMAC-SHA256(secret, device_id + ":" + unix_timestamp + ":" + body_sha256) +HMAC-SHA256(secret, "POST\n" + path + "\n" + timestamp + "\n" + hex(sha256(body))) ``` Timestamp prevents replay attacks. Server validates within ±5 minute window. @@ -142,6 +144,8 @@ Request body: } ``` +Note: `device_id` is sent as the `X-Device-Id` header, not in the body. + Success response: ```json { "status": "ok", "accepted": 1 } @@ -165,10 +169,38 @@ CREATE TABLE camera_records ( Idempotent on `(device_id, period_start)` — duplicate submissions are silently ignored. -### Existing endpoints (unchanged) +### Existing endpoints (body shapes) -- `POST /api/v1/heartbeat` — device health ping, reused as-is -- `POST /api/v1/events/batch` — BLE data, reused as-is +**POST /api/v1/events/batch** (BLE data): +```json +{ + "device_id": "dc-0042", + "location_id": "retailer-123", + "records": [ + { + "period_start": 1712000000, + "period_end": 1712003600, + "unique_devices": 18, + "max_concurrent": 5, + "near_count": 3, + "mid_count": 8, + "far_count": 7 + } + ] +} +``` + +**POST /api/v1/heartbeat**: +```json +{ + "device_id": "dc-0042", + "firmware_version": "1.0.0", + "free_storage_pct": 100, + "wifi_rssi": -65, + "pending_records": 0, + "uptime_seconds": 3661 +} +``` ### Local buffering @@ -230,9 +262,14 @@ DoorCounter/ --- -## 8. Open Questions / Future Work +## 8. Reporting Behavior + +- **Boot report delay**: First report fires 60 seconds after NTP sync (not at the top of the first full hour). This gives a quick connectivity check after deployment. +- **Hourly thereafter**: Reports fire every 3600 seconds. +- **Debug level**: `CORE_DEBUG_LEVEL=1` (errors only) for production builds. + +## 9. Open Questions / Future Work -- Confirm HMAC validation window (±5 min) matches existing server implementation - Line offset calibration: consider a web UI at `.local/config` for adjusting the virtual line position after install - Multi-zone counting (two lines = zone dwell time) — out of scope for v1 - Dashboard / analytics UI — out of scope for v1 diff --git a/firmware/platformio.ini b/firmware/platformio.ini index eca9f14..13d9fa8 100644 --- a/firmware/platformio.ini +++ b/firmware/platformio.ini @@ -10,7 +10,7 @@ board_build.partitions = partitions_4mb_ota.csv build_flags = -DBOARD_HAS_PSRAM -mfix-esp32-psram-cache-issue - -DCORE_DEBUG_LEVEL=3 + -DCORE_DEBUG_LEVEL=1 -DCONFIG_BT_NIMBLE_ENABLED=1 -DCONFIG_SPIRAM_USE_MALLOC=1 -DCONFIG_ARDUINO_LOOP_STACK_SIZE=16384 diff --git a/firmware/src/ble_scanner.cpp b/firmware/src/ble_scanner.cpp index b465ba1..a7525d4 100644 --- a/firmware/src/ble_scanner.cpp +++ b/firmware/src/ble_scanner.cpp @@ -51,6 +51,7 @@ class ScanCallback : public NimBLEAdvertisedDeviceCallbacks { std::lock_guard lock(s_mutex); auto it = s_seen.find(hash); if (it == s_seen.end()) { + Serial.printf("[BLE] new device: %s (rssi %d)\n", hash.c_str(), rssi); s_seen[hash] = {rssi, 1}; } else { it->second.rssi_sum += rssi; @@ -75,9 +76,19 @@ void ble_scanner_start() { s_scan->start(0, nullptr, false); // 0 = continuous } -void ble_scanner_pause() { if (s_scan) s_scan->stop(); } +void ble_scanner_pause() { if (s_scan) s_scan->stop(); } void ble_scanner_resume() { if (s_scan) s_scan->start(0, nullptr, false); } +void ble_scanner_deinit() { + if (s_scan) s_scan->stop(); + s_scan = nullptr; + NimBLEDevice::deinit(true); // frees NimBLE heap (~25KB) +} + +void ble_scanner_reinit() { + ble_scanner_start(); // re-init stack and restart scan +} + BLEHourlyRecord ble_scanner_collect(uint32_t period_start, uint32_t period_end) { // Swap accumulators under lock — minimise time with lock held std::map local_seen; diff --git a/firmware/src/ble_scanner.h b/firmware/src/ble_scanner.h index 6ad35ae..45d8660 100644 --- a/firmware/src/ble_scanner.h +++ b/firmware/src/ble_scanner.h @@ -17,9 +17,13 @@ struct BLEHourlyRecord { // Start continuous passive BLE scan (call once at boot). void ble_scanner_start(); -// Pause scan for ~3s during HTTP upload. +// Pause/resume scan (lightweight — stack stays initialized). void ble_scanner_pause(); void ble_scanner_resume(); +// Full deinit/reinit — frees ~25KB NimBLE heap for SSL, then restarts scan. +void ble_scanner_deinit(); +void ble_scanner_reinit(); + // Collect current hour's record and reset accumulators. Thread-safe. BLEHourlyRecord ble_scanner_collect(uint32_t period_start, uint32_t period_end); diff --git a/firmware/src/main.cpp b/firmware/src/main.cpp index 689f98d..c8189cd 100644 --- a/firmware/src/main.cpp +++ b/firmware/src/main.cpp @@ -18,6 +18,7 @@ #define CAM_FPS 5 #define CAM_INTERVAL_MS (1000 / CAM_FPS) #define REPORT_INTERVAL_S 3600 +#define BOOT_REPORT_DELAY_S 60 // first report fires 60s after NTP sync static DeviceConfig g_cfg; static CVState g_cv; @@ -44,7 +45,9 @@ static void task_camera(void*) { while (true) { if (camera_capture_96(frame)) { if (xSemaphoreTake(s_cv_mutex, pdMS_TO_TICKS(100)) == pdTRUE) { - cv_process(g_cv, frame, g_cfg.line_offset); + CVResult r = cv_process(g_cv, frame, g_cfg.line_offset); + if (r.entries_delta) Serial.printf("[CV] entry +%d (total %d)\n", r.entries_delta, g_cv.entries); + if (r.exits_delta) Serial.printf("[CV] exit +%d (total %d)\n", r.exits_delta, g_cv.exits); xSemaphoreGive(s_cv_mutex); } } @@ -62,8 +65,11 @@ static void task_reporter(void*) { uint32_t now = (uint32_t)(time(nullptr)); if (now < 1700000000UL) continue; // NTP not synced - // First valid timestamp — initialize without reporting - if (last_report_ts == 0) { last_report_ts = now; continue; } + // First valid timestamp — schedule boot report 60s from now + if (last_report_ts == 0) { + last_report_ts = now - (REPORT_INTERVAL_S - BOOT_REPORT_DELAY_S); + continue; + } if ((now - last_report_ts) < REPORT_INTERVAL_S) continue; @@ -71,9 +77,9 @@ static void task_reporter(void*) { uint32_t period_end = now; last_report_ts = now; - // Pause BLE during upload - ble_scanner_pause(); - led_set(true); // yellow indicator (single LED: on = uploading) + // Deinit BLE to free ~25KB heap for SSL handshakes + ble_scanner_deinit(); + led_set(true); // on = uploading CameraHourlyRecord cam_rec; if (xSemaphoreTake(s_cv_mutex, pdMS_TO_TICKS(500)) == pdTRUE) { @@ -82,7 +88,7 @@ static void task_reporter(void*) { xSemaphoreGive(s_cv_mutex); } else { // Failed to acquire — skip this cycle, will report next hour - ble_scanner_resume(); + ble_scanner_reinit(); led_set(false); continue; } @@ -93,7 +99,7 @@ static void task_reporter(void*) { reporter_submit_ble(g_cfg, ble_rec); reporter_heartbeat(g_cfg, millis() / 1000, WiFi.RSSI()); - ble_scanner_resume(); + ble_scanner_reinit(); led_set(false); } } diff --git a/firmware/src/reporter.cpp b/firmware/src/reporter.cpp index 25223f8..a896e71 100644 --- a/firmware/src/reporter.cpp +++ b/firmware/src/reporter.cpp @@ -30,10 +30,6 @@ static bool post_json(const DeviceConfig& cfg, const char* path, const String& b HTTPClient http; String url = String(REPORTER_API_HOST) + path; - // NOTE: Certificate validation is disabled — connection is encrypted but - // server identity is not verified. To enable validation, use WiFiClientSecure - // with setCACert() before calling http.begin(client, url). - // Acceptable for this deployment: devices operate on store WiFi, not public internet. http.begin(url); http.addHeader("Content-Type", "application/json"); http.addHeader("X-Device-Id", cfg.device_id); @@ -42,6 +38,7 @@ static bool post_json(const DeviceConfig& cfg, const char* path, const String& b int code = http.POST(body); http.end(); + Serial.printf("[HTTP] POST %s → %d\n", url.c_str(), code); return (code == 200); } @@ -64,6 +61,7 @@ static String build_camera_batch(const DeviceConfig& cfg, static String build_ble_batch(const DeviceConfig& cfg, const std::vector& recs) { JsonDocument doc; + doc["device_id"] = cfg.device_id; doc["location_id"] = cfg.location_id; JsonArray arr = doc["records"].to(); for (const auto& r : recs) { @@ -151,6 +149,7 @@ void reporter_submit_ble(const DeviceConfig& cfg, const BLEHourlyRecord& rec) { void reporter_heartbeat(const DeviceConfig& cfg, uint32_t uptime_s, int wifi_rssi) { JsonDocument doc; + doc["device_id"] = cfg.device_id; doc["firmware_version"] = "1.0.0"; doc["free_storage_pct"] = 100; doc["wifi_rssi"] = wifi_rssi; diff --git a/firmware/src/reporter.h b/firmware/src/reporter.h index 28c1b55..96e796e 100644 --- a/firmware/src/reporter.h +++ b/firmware/src/reporter.h @@ -12,7 +12,7 @@ struct CameraHourlyRecord { }; static const int REPORTER_MAX_BUFFER = 24; -static const char* REPORTER_API_HOST = "https://logs.research.bike"; +static const char* REPORTER_API_HOST = "http://logs.research.bike"; void reporter_init(); void reporter_submit_camera(const DeviceConfig& cfg, const CameraHourlyRecord& rec);