ESP32: one HTTP client for the pulls; a wake's scene switch skips the scene it leaves

docs/events.md

@@ -393,6 +393,7 @@ an event means:
 | Device commands | all five | `metrics` has no host; the rest are the firmware's (`fos_events.c`) | none: logged and ignored |
 | When a queued event is delivered | the runner thread's message loop | on the task that sent it, before `frameos_nim_send_event` returns; what a render dispatched, right after the render | before `frameos_wasm_event` returns; right after a render |
 | An event aimed at a scene that is not the current one (`target`) | delivered if that scene is in memory | dropped: one resident scene | dropped |
+| An event before the current scene is built | not a state the runner has: scenes are built when selected | the event builds it (`single_scene_host.send`), so a boot's replayed button press reaches the scene it woke; a scene switch is served without building the scene being left | same as the ESP32 |
 | `close` on `reload` / `uploadScenes` | not sent | not sent | — |
 
 A frame on firmware older than `cloud.since` does not know `scene_event`: the

docs/todo.md

@@ -56,28 +56,6 @@ does:
   the image and refuses a board whose table differs, so a bigger NVS strands
   every board already on the old table.
 
-## ESP32: one HTTP client helper for the backend pulls
-
-Seven places in the firmware set up `esp_http_client` by hand for the same
-job — a bearer-authenticated request to the backend or a provider, an
-ETag or a body, a bounded read into a malloc'd buffer: `fos_settings.c`
-(`read_body`), `fos_scenes.c`, `fos_tz.c`, `fos_ota.c` (twice),
-`fos_client.c` (twice) and, since the command queue (PR #525),
-`fos_commands.c` (`post_json`). Each copy is 1 to 2 KB of flash; the queue's
-copy alone was most of what that PR added to the image.
-
-- [ ] **One helper** (`fos_backend_http.c`, say): URL built from the
-  config's backend URL and frame id, the bearer header, optional
-  If-None-Match, optional JSON body, the status, the response ETag, and a
-  body read capped at a caller-given size. Move `read_body` and
-  `collect_etag_handler` (both duplicated in settings and scenes) into it.
-- [ ] Port the pulls one at a time — settings, scenes, commands, tz, the
-  OTA manifest — and compare the firmware size comment on each; expect the
-  set to come out a few KB lighter than before the queue, not just back to
-  even.
-- Keep the image push (`push_image_to_backend`) and the streaming OTA
-  download out of it: they stream bodies, not buffer them.
-
 ## Pre-release manual test sweep
 
 `docs/manual-testing-todo.md` holds the boxes that need hardware: the

embedded/esp32/README.md

@@ -34,6 +34,7 @@ main/                     boot orchestration + platform modules
   fos_scenes.c            /state scene store: combined payload, per-scene split, OOM-restart mark, wipe
   fos_scene_state.c       scene state across deep sleep: one JSON file per scene under /state (host-tested)
   fos_commands.c          the backend's command queue: pulled once per pass and on a nudge, acked by id
+  fos_backend_http.c      the one HTTP client for the small pulls and posts (settings, scenes, commands, tz, OTA manifest)
   fos_schedule.c          schedule evaluation; fos_schedule_catchup.h its catch-up window (host-tested)
   fos_events.c            the ONE routing function for scene events (fos_events_dispatch): every
                           producer — /event/<name>, the schedule, the cloud verbs, the console's

embedded/esp32/main/CMakeLists.txt

@@ -22,6 +22,7 @@ idf_component_register(
         "fos_scenes.c"
         "fos_scene_state.c"
         "fos_commands.c"
+        "fos_backend_http.c"
         "fos_events.c"
         "fos_schedule.c"
         "fos_settings.c"

embedded/esp32/main/fos_backend_http.c

@@ -0,0 +1,139 @@
+/* One HTTP client for the firmware's small pulls and posts. See the header. */
+#include "fos_backend_http.h"
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <strings.h>
+
+#include "esp_crt_bundle.h"
+#include "esp_http_client.h"
+#include "esp_log.h"
+
+#include "fos_config.h"
+#include "fos_mem.h"
+
+static const char *TAG = "fos_http";
+
+/* A body of unknown length (chunked) starts here and doubles up to the cap. */
+#define FOS_HTTP_FIRST_CHUNK (16 * 1024)
+
+static esp_err_t collect_etag_handler(esp_http_client_event_t *evt)
+{
+    if (evt->event_id == HTTP_EVENT_ON_HEADER && strcasecmp(evt->header_key, "ETag") == 0) {
+        fos_http_response_t *res = (fos_http_response_t *)evt->user_data;
+        snprintf(res->etag, sizeof(res->etag), "%s", evt->header_value);
+    }
+    return ESP_OK;
+}
+
+bool fos_backend_url(const char *suffix, char *out, size_t out_len)
+{
+    const fos_config_t *config = fos_config();
+    if (!config->backend_url[0] || config->frame_id == 0) {
+        out[0] = '\0';
+        return false;
+    }
+    int n = snprintf(out, out_len, "%s/api/frames/%lu%s", config->backend_url,
+                     (unsigned long)config->frame_id, suffix ? suffix : "");
+    return n > 0 && (size_t)n < out_len;
+}
+
+void fos_backend_auth(char *out, size_t out_len)
+{
+    snprintf(out, out_len, "Bearer %s", fos_config()->api_key);
+}
+
+void fos_http_response_free(fos_http_response_t *res)
+{
+    free(res->body);
+    res->body = NULL;
+    res->body_len = 0;
+}
+
+/* The body, however it is delimited: Content-Length sizes the buffer once;
+ * a chunked answer grows it up to the cap. */
+static void read_body(esp_http_client_handle_t client, size_t max_body, fos_http_response_t *res)
+{
+    if (res->content_length > (int64_t)max_body) {
+        res->too_large = true;
+        return;
+    }
+    size_t cap = res->content_length > 0 ? (size_t)res->content_length
+                 : max_body < FOS_HTTP_FIRST_CHUNK ? max_body
+                                                   : FOS_HTTP_FIRST_CHUNK;
+    char *buf = fos_big_malloc(cap + 1);
+    if (buf == NULL) return;
+    size_t total = 0;
+    while (true) {
+        if (total == cap) {
+            if (cap >= max_body) {
+                /* More than the cap and still coming. */
+                res->too_large = true;
+                free(buf);
+                return;
+            }
+            size_t new_cap = cap * 2 > max_body ? max_body : cap * 2;
+            char *grown = fos_big_realloc(buf, new_cap + 1);
+            if (grown == NULL) {
+                free(buf);
+                return;
+            }
+            buf = grown;
+            cap = new_cap;
+        }
+        int r = esp_http_client_read(client, buf + total, (int)(cap - total));
+        if (r < 0) {
+            free(buf);
+            return;
+        }
+        if (r == 0) break;
+        total += (size_t)r;
+    }
+    buf[total] = '\0';
+    res->body = buf;
+    res->body_len = total;
+}
+
+esp_err_t fos_http_fetch(const fos_http_request_t *req, size_t max_body, fos_http_response_t *res)
+{
+    memset(res, 0, sizeof(*res));
+    esp_http_client_config_t http_config = {
+        .url = req->url,
+        .method = req->body ? HTTP_METHOD_POST : HTTP_METHOD_GET,
+        .timeout_ms = req->timeout_ms > 0 ? req->timeout_ms : 15000,
+        .crt_bundle_attach = esp_crt_bundle_attach,
+        .buffer_size = req->buffer_size > 0 ? req->buffer_size : 2048,
+        .event_handler = collect_etag_handler,
+        .user_data = res,
+    };
+    esp_http_client_handle_t client = esp_http_client_init(&http_config);
+    if (client == NULL) return ESP_ERR_NO_MEM;
+    if (req->auth) esp_http_client_set_header(client, "Authorization", req->auth);
+    if (req->if_none_match && req->if_none_match[0]) {
+        esp_http_client_set_header(client, "If-None-Match", req->if_none_match);
+    }
+    if (req->no_store) esp_http_client_set_header(client, "Cache-Control", "no-store");
+    if (req->body) esp_http_client_set_header(client, "Content-Type", "application/json");
+
+    int body_len = req->body ? (int)strlen(req->body) : 0;
+    esp_err_t err = esp_http_client_open(client, body_len);
+    if (err != ESP_OK) {
+        ESP_LOGW(TAG, "%s: connect failed: %s", req->url, esp_err_to_name(err));
+        esp_http_client_cleanup(client);
+        return err;
+    }
+    if (body_len > 0 && esp_http_client_write(client, req->body, body_len) != body_len) {
+        esp_http_client_close(client);
+        esp_http_client_cleanup(client);
+        return ESP_FAIL;
+    }
+    res->content_length = esp_http_client_fetch_headers(client);
+    res->status = esp_http_client_get_status_code(client);
+    if (max_body > 0 && res->status != 304 && res->status != 204) {
+        read_body(client, max_body, res);
+    }
+    esp_http_client_close(client);
+    esp_http_client_cleanup(client);
+    return ESP_OK;
+}

embedded/esp32/main/fos_backend_http.h

@@ -0,0 +1,52 @@
+/*
+ * One HTTP client for the firmware's small pulls and posts.
+ *
+ * The settings pull, the scenes pull, the commands pull, the tz slice lookup
+ * and the OTA manifest all used to set up esp_http_client by hand: the same
+ * bearer header, the same optional If-None-Match, the same bounded read into
+ * a malloc'd buffer, the same close/cleanup on every exit — five copies, 1 to
+ * 2 KB of flash each. This is the one copy. Streams stay out of it: the
+ * remote-render bitmap, the OTA image and the image push read or write bodies
+ * as they go.
+ */
+#pragma once
+
+#include <stdbool.h>
+#include <stddef.h>
+#include <stdint.h>
+
+#include "esp_err.h"
+
+typedef struct {
+    const char *url;
+    const char *auth;          /* the Authorization value ("Bearer …"); NULL = none */
+    const char *if_none_match; /* NULL = none */
+    const char *body;          /* JSON to POST; NULL = GET */
+    bool no_store;             /* Cache-Control: no-store (a body that is credentials) */
+    int timeout_ms;            /* 0 = 15 s */
+    int buffer_size;           /* esp_http_client's receive buffer; 0 = 2 KB */
+} fos_http_request_t;
+
+typedef struct {
+    int status;             /* the HTTP status; 0 when no answer came */
+    int64_t content_length; /* as declared; -1 = chunked, read to the end */
+    bool too_large;         /* the body was over the cap and was not read */
+    char etag[96];          /* the response ETag, "" when none */
+    char *body;             /* NUL-terminated; NULL when not read. fos_http_response_free() */
+    size_t body_len;
+} fos_http_response_t;
+
+/* Perform the request. ESP_OK means an answer came (any status, including
+ * 304 and 4xx — look at `res->status`); an error means it never did. The body
+ * is read into `res->body` when `max_body` > 0 and the answer carries one
+ * (not on 304), up to `max_body` bytes: past that it is dropped and
+ * `too_large` set. The buffer is PSRAM where the board has it (fos_mem.h),
+ * so a scene bundle does not land in internal RAM. */
+esp_err_t fos_http_fetch(const fos_http_request_t *req, size_t max_body, fos_http_response_t *res);
+
+void fos_http_response_free(fos_http_response_t *res);
+
+/* `{backend_url}/api/frames/{frame_id}{suffix}`; false when no backend is
+ * configured. And the bearer value for it. */
+bool fos_backend_url(const char *suffix, char *out, size_t out_len);
+void fos_backend_auth(char *out, size_t out_len);

embedded/esp32/main/fos_commands.c

@@ -5,10 +5,9 @@
 #include <string.h>
 
 #include "cJSON.h"
-#include "esp_crt_bundle.h"
-#include "esp_http_client.h"
 #include "esp_log.h"
 
+#include "fos_backend_http.h"
 #include "fos_client.h"
 #include "fos_cloud.h"
 #include "fos_config.h"
@@ -39,48 +38,22 @@ typedef struct {
  * negative value when the request never completed. */
 static int post_json(const char *suffix, const char *body, size_t limit, char **out)
 {
-    const fos_config_t *config = fos_config();
     char url[FOS_URL_LEN + 192];
-    snprintf(url, sizeof(url), "%s/api/frames/%lu/embedded/commands%s", config->backend_url,
-             (unsigned long)config->frame_id, suffix);
+    char path[160];
+    snprintf(path, sizeof(path), "/embedded/commands%s", suffix);
+    fos_backend_url(path, url, sizeof(url));
     char auth[FOS_STR_LEN + 16];
-    snprintf(auth, sizeof(auth), "Bearer %s", config->api_key);
+    fos_backend_auth(auth, sizeof(auth));
+    fos_http_request_t request = {.url = url, .auth = auth, .body = body};
+    fos_http_response_t response;
     if (out) *out = NULL;
-    esp_http_client_config_t http_config = {
-        .url = url,
-        .method = HTTP_METHOD_POST,
-        .timeout_ms = FOS_COMMANDS_TIMEOUT_MS,
-        .crt_bundle_attach = esp_crt_bundle_attach,
-        .buffer_size = 2048,
-    };
-    esp_http_client_handle_t client = esp_http_client_init(&http_config);
-    if (client == NULL) return -1;
-    esp_http_client_set_header(client, "Authorization", auth);
-    esp_http_client_set_header(client, "Content-Type", "application/json");
-    int body_len = (int)strlen(body);
-    int status = -1;
-    if (esp_http_client_open(client, body_len) == ESP_OK &&
-        esp_http_client_write(client, body, body_len) == body_len) {
-        int64_t content_length = esp_http_client_fetch_headers(client);
-        status = esp_http_client_get_status_code(client);
-        if (out && content_length <= (int64_t)limit) {
-            size_t cap = content_length > 0 ? (size_t)content_length : limit;
-            char *buf = malloc(cap + 1);
-            if (buf != NULL) {
-                size_t total = 0;
-                while (total < cap) {
-                    int r = esp_http_client_read(client, buf + total, (int)(cap - total));
-                    if (r <= 0) break;
-                    total += (size_t)r;
-                }
-                buf[total] = '\0';
-                *out = buf;
-            }
-        }
-        esp_http_client_close(client);
+    if (fos_http_fetch(&request, out ? limit : 0, &response) != ESP_OK) return -1;
+    if (out) {
+        *out = response.body; /* ours now */
+    } else {
+        fos_http_response_free(&response);
     }
-    esp_http_client_cleanup(client);
-    return status;
+    return response.status;
 }
 
 /* The ack, with what the frame shows afterwards (the cloud hello's state

embedded/esp32/main/fos_ota.c

@@ -26,6 +26,7 @@
 #include "monocypher.h"
 #include "monocypher-ed25519.h"
 
+#include "fos_backend_http.h"
 #include "fos_cloud.h"
 #include "fos_config.h"
 #include "fos_http.h"
@@ -277,33 +278,25 @@ static esp_err_t ota_fetch_manifest(const ota_source_t *src, ota_manifest_t *out
 {
     *up_to_date = false;
     ESP_LOGI(TAG, "ota (%s): checking manifest %s", src->plane, src->manifest_url);
-    esp_http_client_config_t config = {
+    fos_http_request_t request = {
         .url = src->manifest_url,
+        .auth = src->auth,
         .timeout_ms = 20000,
-        .crt_bundle_attach = esp_crt_bundle_attach,
         .buffer_size = 4096,
     };
-    esp_http_client_handle_t client = esp_http_client_init(&config);
-    if (client == NULL) {
-        ota_log(src, "error", "no-memory");
-        return ESP_ERR_NO_MEM;
-    }
-    esp_http_client_set_header(client, "Authorization", src->auth);
-    esp_err_t err = esp_http_client_open(client, 0);
+    fos_http_response_t response;
+    esp_err_t err = fos_http_fetch(&request, FOS_OTA_MANIFEST_MAX, &response);
     if (err != ESP_OK) {
-        esp_http_client_cleanup(client);
-        ota_log(src, "error", "manifest-connect-failed");
+        ota_log(src, "error", err == ESP_ERR_NO_MEM ? "no-memory" : "manifest-connect-failed");
         return err;
     }
-    int64_t content_length = esp_http_client_fetch_headers(client);
-    int status = esp_http_client_get_status_code(client);
+    int status = response.status;
     /* Dev servers (and some proxies) send chunked responses with no
      * Content-Length — read to EOF under the manifest cap either way. */
-    if (status != 200 || content_length > FOS_OTA_MANIFEST_MAX) {
-        esp_http_client_close(client);
-        esp_http_client_cleanup(client);
+    if (status != 200 || response.too_large) {
+        fos_http_response_free(&response);
         ESP_LOGW(TAG, "ota (%s): manifest HTTP %d (%lld bytes)", src->plane, status,
-                 (long long)content_length);
+                 (long long)response.content_length);
         /* 404 = the release carries no OTA app image for this platform; 409 =
          * it has one but no signature. Both are "nothing to install", and
          * telling them apart in the log is the difference between waiting for
@@ -313,25 +306,13 @@ static esp_err_t ota_fetch_manifest(const ota_source_t *src, ota_manifest_t *out
                                             : "manifest-unavailable");
         return status == 404 ? ESP_ERR_NOT_FOUND : ESP_FAIL;
     }
-    char *body = malloc(FOS_OTA_MANIFEST_MAX + 1);
-    if (body == NULL) {
-        esp_http_client_close(client);
-        esp_http_client_cleanup(client);
+    if (response.body == NULL) {
         ota_log(src, "error", "no-memory");
         return ESP_ERR_NO_MEM;
     }
-    size_t total = 0;
-    while (total < FOS_OTA_MANIFEST_MAX) {
-        int r = esp_http_client_read(client, body + total, FOS_OTA_MANIFEST_MAX - total);
-        if (r <= 0) break;
-        total += (size_t)r;
-    }
-    esp_http_client_close(client);
-    esp_http_client_cleanup(client);
-    body[total] = '\0';
 
-    cJSON *root = cJSON_ParseWithLength(body, total);
-    free(body);
+    cJSON *root = cJSON_ParseWithLength(response.body, response.body_len);
+    fos_http_response_free(&response);
     if (root == NULL) {
         ota_log(src, "error", "manifest-unparseable");
         return ESP_FAIL;