From 228e1252f4f4218a63652a63acc398aeaac90278 Mon Sep 17 00:00:00 2001 From: Alex Hope-O'Connor Date: Thu, 17 Sep 2026 12:10:48 +1000 Subject: [PATCH] test: add portal OTA A/B contract coverage --- .dockerignore | 14 + .github/workflows/ci.yml | 36 +- .gitignore | 2 +- docs/DEVELOPMENT.md | 76 +++- docs/TESTING.md | 88 ++++- platformio.ini | 8 +- scripts/test.sh | 117 ++++++- test/compile-project/platformio.ini | 6 + test/portal-harness/README.md | 22 ++ .../partitions/esp32_ota_4m_no_fs.csv | 12 + test/portal-harness/platformio.ini | 39 +++ test/portal-harness/src/main.cpp | 31 ++ tests/portal-contract/README.md | 4 + tests/portal-contract/compose.ota.yaml | 11 + tests/portal-contract/compose.yaml | 1 + tests/portal-contract/run-portal-contract.sh | 8 +- tests/portal-contract/tests/ota.spec.js | 89 +++++ tools/check-ota-partitions.sh | 66 ++++ tools/lib/portal-hardware-session.sh | 6 + tools/portal-hardware | 329 +++++++++++++++++- tools/tests/test-portal-hardware-cli.sh | 22 ++ 21 files changed, 954 insertions(+), 33 deletions(-) create mode 100644 .dockerignore create mode 100644 test/portal-harness/partitions/esp32_ota_4m_no_fs.csv create mode 100644 tests/portal-contract/compose.ota.yaml create mode 100644 tests/portal-contract/tests/ota.spec.js create mode 100755 tools/check-ota-partitions.sh diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..4afd82d --- /dev/null +++ b/.dockerignore @@ -0,0 +1,14 @@ +# Keep local credentials, build output, and test artifacts out of the Docker +# build context. OTA firmware is mounted read-only by compose.ota.yaml instead +# of copied into an image. +.git +.github +.pio +**/.pio +node_modules +**/node_modules +artifacts +test/portal-station.env +test/.env +platformio.local.ini* +**/platformio.local.ini* diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 079a66f..8dd2b99 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -20,8 +20,13 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - run: bash -n scripts/*.sh tools/portal-hardware tools/lib/*.sh tools/tests/*.sh + - run: bash -n scripts/*.sh tools/check-ota-partitions.sh tools/portal-hardware tools/lib/*.sh tools/tests/*.sh + - run: | + for spec in tests/portal-contract/tests/*.js; do + node --check "$spec" + done - run: ./tools/tests/test-portal-hardware-cli.sh + - run: ./tools/check-ota-partitions.sh - run: ./scripts/check-docs.sh compile-tests: @@ -29,12 +34,35 @@ jobs: strategy: fail-fast: false matrix: - environment: ["esp8266", "esp32"] + include: + - platform: esp8266 + examples: true + ota_fixtures: true + unity: true + - platform: esp32 + examples: true + ota_fixtures: false + unity: true + - platform: esp32-current + examples: false + # Keep current A/B OTA builds in their own worker. PlatformIO uses + # one package-name directory for tool-esptoolpy, so mixing legacy + # 3.0.5 and current 3.3.11 graphs in a single cache is not a + # meaningful compatibility test. + ota_fixtures: true + unity: true steps: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 with: python-version: '3.11' - run: python -m pip install --upgrade platformio==6.1.19 - - run: ./scripts/test.sh compile --platform ${{ matrix.environment }} - - run: ./scripts/test.sh examples --platform ${{ matrix.environment }} + - run: ./scripts/test.sh compile --platform ${{ matrix.platform }} + - if: matrix.unity + run: ./scripts/test.sh unity --platform ${{ matrix.platform }} + - if: matrix.platform == 'esp32-current' + run: ./scripts/test.sh packages --platform esp32-current + - if: matrix.examples + run: ./scripts/test.sh examples --platform ${{ matrix.platform }} + - if: matrix.ota_fixtures + run: ./scripts/test.sh ota-fixtures --platform ${{ matrix.platform }} diff --git a/.gitignore b/.gitignore index 39eab32..ffd0921 100644 --- a/.gitignore +++ b/.gitignore @@ -37,4 +37,4 @@ node_modules/ # Local portal station handoff credentials and optional direct test artifacts /test/portal-station.env /tests/portal-contract/artifacts/ -/artifacts/readme-media/ +/artifacts/ diff --git a/docs/DEVELOPMENT.md b/docs/DEVELOPMENT.md index 3af87d5..98d8c26 100644 --- a/docs/DEVELOPMENT.md +++ b/docs/DEVELOPMENT.md @@ -9,19 +9,57 @@ lib_deps = ## Target pins -The ESP32 test environments pin the pioarduino `51.03.05` platform package, -which selects Arduino-ESP32 3.0.5 / ESP-IDF 5.1.4+. This is a test-target -contract, not a library-manifest dependency: a consuming application chooses -its own `platform` and must validate the complete framework/toolchain stack. -Core 3 Wi-Fi builds need the C++14, `SOC_WIFI_SUPPORTED`, `Network/src`, and -ESP8266-transport ignore settings in this repository's `platformio.ini`; keep -those settings together when adding an ESP32 environment. +WiFiManager currently has two explicit ESP32 test lanes: + +| Lane | pioarduino platform | Purpose | +| --- | --- | --- | +| `esp32` | `51.03.05` / Arduino-ESP32 3.0.5 | temporary compatibility contract | +| `esp32_core_3_3_11` / CLI `esp32-current` | `55.03.311` / Arduino-ESP32 3.3.11 | maintained current validation lane | + +This is a test-target contract, not a library-manifest dependency: a consuming +application chooses its own `platform` and must validate the complete +framework/toolchain stack. Do not let a shared global PlatformIO cache choose +framework metadata or a compiler implicitly, and do not override just the +toolchain to repair a cache mismatch. Each pioarduino platform owns its +matching framework, uploader, and compiler package set. + +Core 3 Wi-Fi builds need the C++14, `SOC_WIFI_SUPPORTED`, and `Network/src` +settings in this repository's `platformio.ini`; keep those settings together +when adding an ESP32 environment. The portal OTA fixture uses the current +3.3.11 lane for ESP32 even while the 3.0.5 compatibility lane remains +available. ESP8266 test environments pin framework commit `521ae60` for the upstream Postmortem large-jump linker fix. The exact rationale and update rule are in the shared [ESP8266 linker-workaround note](https://github.com/alexhopeoconnor/arduino-home-assistant/blob/main/docs/ESP8266-LINKER-WORKAROUND.md). -For the pioarduino release-to-Core mapping and the scoped repair for a stale -global PlatformIO tool package, see [DeviceFramework's toolchain guide](https://github.com/alexhopeoconnor/DeviceFramework/blob/main/docs/TOOLCHAINS.md). +For the pioarduino release-to-Core mapping and cache-collision diagnosis, see +[DeviceFramework's toolchain guide](https://github.com/alexhopeoconnor/DeviceFramework/blob/main/docs/TOOLCHAINS.md). + +`./scripts/test.sh` automatically places the `esp32-current` lane, and +`./tools/portal-hardware ota --platform esp32` places its current A/B fixture, +in a dedicated PlatformIO Core/cache directory, defaulting to +`${XDG_CACHE_HOME:-$HOME/.cache}/wifimanager-platformio/core-3.3.11`. That +keeps pioarduino's package-form `esptool` and generated environment separate +from the legacy 3.0.5 `tool-esptoolpy` graph. Override the location with +`WIFIMANAGER_PLATFORMIO_CORE_DIR`, +`WIFIMANAGER_PLATFORMIO_PACKAGES_DIR`, and +`WIFIMANAGER_PLATFORMIO_CACHE_DIR` when space belongs elsewhere. The first +clean install is several GiB; reserve at least 4 GiB plus cache headroom. It +is a deliberate quarantine, not a reason to delete or override packages in the +shared PlatformIO installation. + +For a disposable cache investigation, point that variable at an exact temporary +directory, run the affected command, inspect the resolved graph, then remove +only that directory: + +```bash +wm_pio_core="$(mktemp -d /tmp/wifimanager-pio-XXXXXX)" +WIFIMANAGER_PLATFORMIO_CORE_DIR="$wm_pio_core" \ + ./scripts/test.sh compile --platform esp32-current +WIFIMANAGER_PLATFORMIO_CORE_DIR="$wm_pio_core" \ + ./scripts/test.sh packages --platform esp32-current +rm -rf -- "$wm_pio_core" +``` Start a release with `bump-version.sh`. It updates package metadata and canonical installation snippets, then creates the changelog section. Replace its generated TODO with the release summary and update any behavioural documentation before running: @@ -31,8 +69,15 @@ Start a release with `bump-version.sh`. It updates package metadata and canonica ./scripts/check-docs.sh ./scripts/test.sh compile --platform esp8266 ./scripts/test.sh compile --platform esp32 +./scripts/test.sh compile --platform esp32-current +./scripts/test.sh unity --platform esp8266 +./scripts/test.sh unity --platform esp32 +./scripts/test.sh unity --platform esp32-current +./scripts/test.sh packages --platform esp32-current ./scripts/test.sh examples --platform esp8266 ./scripts/test.sh examples --platform esp32 +./scripts/test.sh ota-fixtures --platform esp8266 +./scripts/test.sh ota-fixtures --platform esp32-current ./scripts/prepare-release.sh vMAJOR.MINOR.PATCH --tag ``` @@ -57,6 +102,19 @@ for browser/API testing: See [Testing](TESTING.md#docker-portal-contract) for cleanup, artifacts, and optional station handoff credentials. +Run the portal HTTP OTA A/B contract separately when a spare adapter and 4 MB +test board are available. It erases the selected board's flash, serial-flashes +A, and uses the real browser update form to upload B; do not replace its +automatic-reboot assertion with a manual reset: + +```bash +./tools/portal-hardware ota --platform esp8266 --port /dev/serial/by-id/usb-... \ + --client-interface wlx74da385d4165 +``` + +See [Portal HTTP OTA A/B contract](TESTING.md#portal-http-ota-ab-contract) for +the partition, artifact, final-board-state, and adapter rules. + Push the branch and annotated tag. GitHub Actions repeats the board-free compile checks, validates the package, and creates a GitHub Release using that version’s changelog section. The workflow does not publish to the PlatformIO Registry. Back to [documentation](README.md) · [project overview](../README.md). diff --git a/docs/TESTING.md b/docs/TESTING.md index d56aed0..3f57914 100644 --- a/docs/TESTING.md +++ b/docs/TESTING.md @@ -4,10 +4,19 @@ WiFiManager separates repeatable package checks from opt-in tests that flash a real board or join a captive portal. The normal commands never need a board, local Wi-Fi credentials, browser binary, or sibling checkout. -## Clean consumer and example builds +| Physical contract | Transport | Host adapter | Secret source | Required proof | +| --- | --- | --- | --- | --- | +| Portal lifecycle suite | Serial flash + captive-portal HTTP/browser | Named secondary adapter | safe fixture AP password | Unity/lifecycle checks and portal UI/API contract | +| Portal HTTP OTA | WiFiManager multipart `POST /u` | Named secondary adapter | safe fixture AP password | A → rendered browser upload B → automatic reboot → B twice | -The clean-consumer check builds a project that declares only WiFiManager. It -proves the package manifest resolves DFTE, ESPAsyncWebServer, and the correct +The selected secondary adapter is intentionally never used for normal LAN +testing. It is `never-default`, so the host's ordinary route remains intact. + +## Board-free fixture, consumer, and example builds + +The Unity compile check builds WiFiManager's own fixture without a board. The +clean-consumer check builds a project that declares only WiFiManager. Together +they prove the package manifest resolves DFTE, ESPAsyncWebServer, and the correct ESP8266 or ESP32 TCP dependency without a sibling checkout. The runner removes a prior local package link before each check, so dependency resolution uses the current manifest rather than a stale `.pio` copy. @@ -15,13 +24,24 @@ current manifest rather than a stale `.pio` copy. ```bash ./scripts/test.sh compile --platform esp8266 ./scripts/test.sh compile --platform esp32 +./scripts/test.sh compile --platform esp32-current +./scripts/test.sh unity --platform esp8266 +./scripts/test.sh unity --platform esp32 +./scripts/test.sh unity --platform esp32-current ./scripts/test.sh examples --platform esp8266 ./scripts/test.sh examples --platform esp32 +./scripts/test.sh ota-fixtures --platform esp8266 +./scripts/test.sh ota-fixtures --platform esp32-current ``` CI runs these board-free checks for pull requests and pushes to the maintained -branch. It intentionally does not require attached hardware, a local network, -or Docker. +branch. `esp32` remains the explicit Arduino-ESP32 3.0.5 compatibility lane; +`esp32-current` is the clean-consumer Arduino-ESP32 3.3.11 validation lane. +The OTA fixture builds also use 3.3.11 for ESP32 and compile both immutable A +and B images against their tracked OTA partition layout. CI rejects equal A/B +artifacts, an ESP32 image larger than either 0x1F0000-byte app slot, or a +partition-table edit that breaks the required two-slot/no-filesystem layout. These checks +intentionally do not require attached hardware, a local network, or Docker. ## Local hardware lifecycle tests @@ -155,4 +175,62 @@ README tour is readable; it does not change normal browser-contract timing. ESP8266 remains covered by the normal hardware and browser contract but does not produce duplicate README media. +## Portal HTTP OTA A/B contract + +`portal-hardware ota` is a separate opt-in physical test for WiFiManager's +built-in HTTP update path. It exercises the rendered firmware-update page and +its real multipart `POST /u` request; it is not an ArduinoOTA/UDP test. + +```bash +./tools/portal-hardware ota \ + --platform esp8266 \ + --port /dev/serial/by-id/usb-... \ + --client-interface wlx74da385d4165 +``` + +The selected `--client-interface` has exactly the same safety rules as the +normal portal contract: it must be the explicitly named secondary adapter and +cannot be the host default-route interface. The test never attaches that +adapter to a normal station network. The fixture AP uses the safe local +`default1` WPA password; this is an AP-access test, not a claim that `/u` has +HTTP route authentication. + +The runner performs the following complete contract: + +1. Builds immutable A and B fixture images. Their marker is compiled into the + binary, not saved in WiFiManager settings or EEPROM. +2. Checks both ESP32 images against the explicit matching `app0`/`app1` slots; + ESP8266 validates B after A has booted against the exact aligned capacity + passed to `Update.begin()`. +3. Erases the explicitly selected test board's flash, then flashes A over + serial and starts its captive portal. +4. Joins that portal only through the named secondary adapter and requires the + A marker at `/api/test/firmware-marker`. +5. Mounts B read-only into the Playwright container, chooses it in the real + `#wm-ota-file` browser input, and submits the rendered form. +6. Requires the real `POST /u` success response, an automatic portal outage, + automatic restart, and two independent B-marker responses. + +The fixture marker endpoint exists only in `test/portal-harness`; it is not a +WiFiManager library route or a product-firmware pattern. The test does not +issue a manual reset. A board which boots B only after intervention is a +failure, even if B later appears. + +Both fixture images are built with explicit OTA-capable layouts: + +| Platform | Fixture layout | Capacity check | +| --- | --- | --- | +| ESP8266 | `eagle.flash.4m1m.ld` | A's live `ESP.getFreeSketchSpace()` response | +| ESP32 | two `0x1F0000` A/B app slots, no filesystem | tracked CSV `app1` size | + +These are 4 MB fixture layouts (`d1_mini` for ESP8266 and `esp32dev` for +ESP32). Do not run this command against a board with another flash size unless +its matching explicit A/B layout and capacity checks have been added first. + +The final board state is firmware B in the portal-only fixture: it clears +saved station settings on every boot and leaves no developer Wi-Fi credential +on the device. By default the temporary NetworkManager connection is removed +when the test exits. Pass `--keep` only for interactive diagnosis, then run +`./tools/portal-hardware down` to remove that named temporary connection. + Back to [documentation](README.md) · [project overview](../README.md). diff --git a/platformio.ini b/platformio.ini index 3066751..c740fe8 100644 --- a/platformio.ini +++ b/platformio.ini @@ -39,10 +39,16 @@ lib_deps = DeviceFrameworkTemplateEngine=https://github.com/alexhopeoconnor/DFTE.git#v1.2.0 ESP32Async/AsyncTCP@^3.4.9 +; Current ESP32 validation lane. Keep Core 3.0.5 as a named compatibility +; target until its support policy is explicitly retired; never let a shared +; PlatformIO cache choose a framework/toolchain pair implicitly. +[env:esp32_core_3_3_11] +extends = env:esp32 +platform = https://github.com/pioarduino/platform-espressif32/releases/download/55.03.311/platform-espressif32.zip + ; Optional: compile tests with DFTE logs bridged into WiFiManager::log (see README) [env:esp8266_dfte_log] extends = env:esp8266 build_flags = ${env:esp8266.build_flags} -DWM_DFTE_LOGGING - diff --git a/scripts/test.sh b/scripts/test.sh index 0f61ea2..40ec3fc 100755 --- a/scripts/test.sh +++ b/scripts/test.sh @@ -4,15 +4,18 @@ set -euo pipefail usage() { cat <<'USAGE' >&2 Usage: - ./scripts/test.sh compile --platform esp8266|esp32 - ./scripts/test.sh examples --platform esp8266|esp32 - ./scripts/test.sh hardware --platform esp8266|esp32 --port /dev/serial/by-id/... + ./scripts/test.sh compile --platform esp8266|esp32|esp32-current + ./scripts/test.sh unity --platform esp8266|esp32|esp32-current + ./scripts/test.sh examples --platform esp8266|esp32 + ./scripts/test.sh ota-fixtures --platform esp8266|esp32|esp32-current + ./scripts/test.sh packages --platform esp8266|esp32|esp32-current + ./scripts/test.sh hardware --platform esp8266|esp32 --port /dev/serial/by-id/... USAGE exit 2 } mode="${1:-}" -[[ "$mode" == "compile" || "$mode" == "examples" || "$mode" == "hardware" ]] || usage +[[ "$mode" == "compile" || "$mode" == "unity" || "$mode" == "examples" || "$mode" == "ota-fixtures" || "$mode" == "packages" || "$mode" == "hardware" ]] || usage shift platform="" @@ -25,11 +28,76 @@ while [[ $# -gt 0 ]]; do esac done -[[ "$platform" == "esp8266" || "$platform" == "esp32" ]] || usage +case "$platform" in + esp8266|esp32) + environment="$platform" + ;; + esp32-current) + environment="esp32_core_3_3_11" + ;; + *) usage ;; +esac +[[ "$mode" != "examples" || "$platform" != "esp32-current" ]] || { + echo "The current ESP32 lane is a clean-consumer check; examples retain their explicit compatibility environments." >&2 + exit 2 +} +[[ "$mode" != "hardware" || "$platform" != "esp32-current" ]] || { + echo "The current ESP32 lane is a board-free clean-consumer check; use esp32 for the existing Unity hardware suite." >&2 + exit 2 +} +[[ "$mode" != "ota-fixtures" || "$platform" != "esp32" ]] || { + echo "ESP32 OTA fixtures are pinned to the isolated 3.3.11 graph; use --platform esp32-current." >&2 + exit 2 +} [[ "$mode" != "hardware" || -n "$port" ]] || usage [[ "$mode" != "hardware" || -e "$port" ]] || { echo "Serial port not found: $port" >&2; exit 1; } root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" + +pio_for_platform() { + if [[ "$platform" != "esp32-current" ]]; then + pio "$@" + return + fi + + # pioarduino Core 3.3.11's esptool package form cannot safely share a + # PlatformIO Core directory with a legacy Core 3.0.5 tool-esptoolpy + # installation. Keep the current validation lane in a project-owned, + # user-cache location unless the developer deliberately supplies one. + local core_dir packages_dir cache_dir + core_dir="${WIFIMANAGER_PLATFORMIO_CORE_DIR:-${XDG_CACHE_HOME:-$HOME/.cache}/wifimanager-platformio/core-3.3.11}" + packages_dir="${WIFIMANAGER_PLATFORMIO_PACKAGES_DIR:-$core_dir/packages}" + cache_dir="${WIFIMANAGER_PLATFORMIO_CACHE_DIR:-$core_dir/cache}" + install -d -m 700 "$core_dir" "$packages_dir" "$cache_dir" + PLATFORMIO_CORE_DIR="$core_dir" PLATFORMIO_PACKAGES_DIR="$packages_dir" \ + PLATFORMIO_CACHE_DIR="$cache_dir" pio "$@" +} + +assert_ota_fixture_pair() { + local fixture_platform="$1" + local firmware_a firmware_b firmware size capacity + firmware_a="$root/test/portal-harness/.pio/build/${fixture_platform}_ota_a/firmware.bin" + firmware_b="$root/test/portal-harness/.pio/build/${fixture_platform}_ota_b/firmware.bin" + [[ -s "$firmware_a" && -s "$firmware_b" ]] || { + echo "Portal OTA fixture build did not produce both A and B images." >&2 + return 1 + } + if cmp -s "$firmware_a" "$firmware_b"; then + echo "Portal OTA fixture A and B are identical." >&2 + return 1 + fi + if [[ "$platform" == "esp32-current" ]]; then + capacity=$((0x1F0000)) + for firmware in "$firmware_a" "$firmware_b"; do + size="$(wc -c < "$firmware" | tr -d '[:space:]')" + (( size <= capacity )) || { + echo "ESP32 OTA fixture $(basename "$(dirname "$firmware")") is $size bytes; it exceeds the $capacity-byte app slot." >&2 + return 1 + } + done + fi +} + if [[ "$mode" == "hardware" ]]; then # Keep serial flashing and portal-adapter work mutually exclusive. # shellcheck source=tools/lib/portal-hardware-session.sh @@ -44,12 +112,43 @@ if [[ "$mode" == "examples" ]]; then exit 1 fi for example in "${examples[@]}"; do - pio run -d "$example" -e "$platform" /dev/null fi -pio run -d "$root/test/compile-project" -e "$platform" +pio_for_platform run -d "$root/test/compile-project" -e "$environment" echo "WiFiManager consumer compile check passed for $platform" diff --git a/test/compile-project/platformio.ini b/test/compile-project/platformio.ini index 3c6f773..c324420 100644 --- a/test/compile-project/platformio.ini +++ b/test/compile-project/platformio.ini @@ -30,3 +30,9 @@ build_flags = -DWM_LOG_LEVEL=3 lib_deps = ${common.lib_deps} + +; Clean-consumer validation against the maintained ESP32 platform lane. This +; is intentionally separate from the temporary 3.0.5 compatibility target. +[env:esp32_core_3_3_11] +extends = env:esp32 +platform = https://github.com/pioarduino/platform-espressif32/releases/download/55.03.311/platform-espressif32.zip diff --git a/test/portal-harness/README.md b/test/portal-harness/README.md index 889c4f0..58ff17e 100644 --- a/test/portal-harness/README.md +++ b/test/portal-harness/README.md @@ -14,3 +14,25 @@ Use it through the repository runner so a secondary Wi-Fi adapter is explicitly The ESP8266 portal SSID is `WM Contract ESP8266`; the ESP32 SSID is `WM Contract ESP32`. Both use `default1` exclusively for local development tests. The runner cleans up only the temporary connection it creates on the named secondary interface. It refuses to run if that interface is the system default route. + +## A/B portal OTA fixture + +The same fixture has dedicated `*_ota_a` and `*_ota_b` PlatformIO environments +for the physical portal HTTP OTA contract. A and B differ only by a compiled +marker served from the fixture-only `/api/test/firmware-marker` endpoint. That +proves a B boot without trusting saved portal values, EEPROM, or a filename. + +```bash +./tools/portal-hardware ota \ + --platform esp32 \ + --port /dev/serial/by-id/... \ + --client-interface wlx... +``` + +ESP8266 explicitly uses `eagle.flash.4m1m.ld`. ESP32 uses the tracked two-slot +`partitions/esp32_ota_4m_no_fs.csv` layout on the maintained Arduino-ESP32 +3.3.11 fixture lane. Both are 4 MB layouts. The runner builds and preserves A +and B before it touches the board, validates the matching ESP32 slots (or the +live ESP8266 updater capacity), then erases the explicitly selected test board +before serial-flashing A. A successful run leaves B installed in the +portal-only fixture. diff --git a/test/portal-harness/partitions/esp32_ota_4m_no_fs.csv b/test/portal-harness/partitions/esp32_ota_4m_no_fs.csv new file mode 100644 index 0000000..7eabea7 --- /dev/null +++ b/test/portal-harness/partitions/esp32_ota_4m_no_fs.csv @@ -0,0 +1,12 @@ +# 4 MB ESP32 portal-harness OTA test layout. +# +# The fixture compiles portal assets into the application image, so it uses no +# filesystem. app0 and app1 are equal 0x1F0000-byte slots. The portal OTA +# runner validates B against that inactive-slot size before it flashes A or +# submits the browser upload. +# Name, Type, SubType, Offset, Size, Flags +nvs, data, nvs, 0x9000, 0x5000, +otadata, data, ota, 0xe000, 0x2000, +app0, app, ota_0, 0x10000, 0x1F0000, +app1, app, ota_1, 0x200000, 0x1F0000, +coredump, data, coredump,0x3F0000, 0x10000, diff --git a/test/portal-harness/platformio.ini b/test/portal-harness/platformio.ini index 8577977..8ebea3e 100644 --- a/test/portal-harness/platformio.ini +++ b/test/portal-harness/platformio.ini @@ -26,3 +26,42 @@ build_flags = -DSOC_WIFI_SUPPORTED=1 -I${platformio.packages_dir}/framework-arduinoespressif32/libraries/Network/src -DWM_LOG_LEVEL=4 + +; The maintained ESP32 validation lane. Keep this named rather than relying on +; whatever framework/toolchain pair happens to be installed in PlatformIO's +; global package cache. +[env:esp32_core_3_3_11] +extends = env:esp32 +platform = https://github.com/pioarduino/platform-espressif32/releases/download/55.03.311/platform-espressif32.zip + +; OTA contract images deliberately differ only by their compile-time marker. +; ESP8266 needs an OTA-capable linker layout; ESP32 needs a tracked two-slot +; partition table. The hardware runner always flashes A over serial then +; uploads B through the real portal form. +[env:esp8266_ota_a] +extends = env:esp8266 +board_build.ldscript = eagle.flash.4m1m.ld +build_flags = + ${env:esp8266.build_flags} + -DWM_OTA_TEST_IMAGE=\"A\" + +[env:esp8266_ota_b] +extends = env:esp8266 +board_build.ldscript = eagle.flash.4m1m.ld +build_flags = + ${env:esp8266.build_flags} + -DWM_OTA_TEST_IMAGE=\"B\" + +[env:esp32_ota_a] +extends = env:esp32_core_3_3_11 +board_build.partitions = partitions/esp32_ota_4m_no_fs.csv +build_flags = + ${env:esp32_core_3_3_11.build_flags} + -DWM_OTA_TEST_IMAGE=\"A\" + +[env:esp32_ota_b] +extends = env:esp32_core_3_3_11 +board_build.partitions = partitions/esp32_ota_4m_no_fs.csv +build_flags = + ${env:esp32_core_3_3_11.build_flags} + -DWM_OTA_TEST_IMAGE=\"B\" diff --git a/test/portal-harness/src/main.cpp b/test/portal-harness/src/main.cpp index f6797ff..0f55b89 100644 --- a/test/portal-harness/src/main.cpp +++ b/test/portal-harness/src/main.cpp @@ -3,6 +3,14 @@ namespace { +// These markers belong only to the portal hardware fixture. They are compiled +// into the image rather than stored in Wi-FiManager settings, so an A -> B +// assertion proves that the new firmware booted after the updater restarted +// the board. Normal portal-contract builds retain a descriptive fixture value. +#ifndef WM_OTA_TEST_IMAGE +#define WM_OTA_TEST_IMAGE "portal-contract" +#endif + #if defined(ESP8266) constexpr char kPortalSsid[] = "WM Contract ESP8266"; #else @@ -57,6 +65,28 @@ WiFiManagerParameter* const kPortalParameters[] = { &kNotes, }; +void registerOtaTestMarker() { + // setWebServerCallback runs after WiFiManager creates its server and + // before it registers built-in routes. This private fixture endpoint is + // intentionally not a WiFiManager product API. + wifi.setWebServerCallback([]() { + AsyncWebServer* const server = wifi.getServer(); + if (server == nullptr) { + return; + } + + server->on("/api/test/firmware-marker", HTTP_GET, + [](AsyncWebServerRequest* request) { + String response = F("{\"marker\":\""); + response += WM_OTA_TEST_IMAGE; + response += F("\",\"freeSketchSpace\":"); + response += String(ESP.getFreeSketchSpace()); + response += F("}"); + request->send(200, "application/json", response); + }); + }); +} + } // namespace void setup() { @@ -75,6 +105,7 @@ void setup() { IPAddress(192, 168, 4, 1), IPAddress(192, 168, 4, 1), IPAddress(255, 255, 255, 0)); + registerOtaTestMarker(); // Keep custom parameters on their own native Save parameters page. This // lets the browser contract exercise a parameter-only submit without // starting a station connection as part of the regression test. diff --git a/tests/portal-contract/README.md b/tests/portal-contract/README.md index 05d6003..def6717 100644 --- a/tests/portal-contract/README.md +++ b/tests/portal-contract/README.md @@ -9,3 +9,7 @@ the already-routed `192.168.4.1` portal; it never manages host Wi-Fi. The image pins the Playwright package to the matching official browser image. Artifacts, traces, screenshots, JSON results, and the HTML report are written to the output directory printed by the host command. + +`compose.ota.yaml` is an overlay used only by `portal-hardware ota`. It mounts +the already-built B firmware read-only and enables the A/B browser contract. +The ordinary portal contract never receives a firmware artifact. diff --git a/tests/portal-contract/compose.ota.yaml b/tests/portal-contract/compose.ota.yaml new file mode 100644 index 0000000..7219cd5 --- /dev/null +++ b/tests/portal-contract/compose.ota.yaml @@ -0,0 +1,11 @@ +# OTA-only overlay. The normal portal contract neither builds nor mounts a +# firmware image. portal-hardware supplies this absolute path after it has +# built B and copied it into the run's private artifact directory. +services: + portal-contract: + environment: + PORTAL_OTA_FIRMWARE: /firmware/portal-ota-b.bin + PORTAL_OTA_INITIAL_MARKER: ${PORTAL_OTA_INITIAL_MARKER:-A} + PORTAL_OTA_EXPECTED_MARKER: ${PORTAL_OTA_EXPECTED_MARKER:-B} + volumes: + - ${PORTAL_OTA_FIRMWARE_HOST:?OTA firmware path is required}:/firmware/portal-ota-b.bin:ro diff --git a/tests/portal-contract/compose.yaml b/tests/portal-contract/compose.yaml index 1d35666..ac7411f 100644 --- a/tests/portal-contract/compose.yaml +++ b/tests/portal-contract/compose.yaml @@ -17,6 +17,7 @@ services: PORTAL_CUSTOM_PARAMETER_STRESS: ${PORTAL_CUSTOM_PARAMETER_STRESS:-0} PORTAL_PLATFORM: ${PORTAL_PLATFORM:-} PORTAL_CAPTURE_README_MEDIA: ${PORTAL_CAPTURE_README_MEDIA:-0} + PORTAL_TEST_FILE: ${PORTAL_TEST_FILE:-} ARTIFACT_DIR: /artifacts volumes: - ${PORTAL_ARTIFACT_DIR:?portal artifact directory is required}:/artifacts diff --git a/tests/portal-contract/run-portal-contract.sh b/tests/portal-contract/run-portal-contract.sh index ce408b3..bbd606c 100755 --- a/tests/portal-contract/run-portal-contract.sh +++ b/tests/portal-contract/run-portal-contract.sh @@ -1,7 +1,13 @@ #!/usr/bin/env bash set -euo pipefail -/work/node_modules/.bin/playwright test --config /work/playwright.config.js +playwright_args=(test --config /work/playwright.config.js) +if [[ -n "${PORTAL_TEST_FILE:-}" ]]; then + # OTA is a destructive, time-bounded board contract. Run only its browser + # spec instead of allowing ordinary portal tests to consume its AP window. + playwright_args+=("$PORTAL_TEST_FILE") +fi +/work/node_modules/.bin/playwright "${playwright_args[@]}" if [[ "${PORTAL_CAPTURE_README_MEDIA:-0}" == "1" ]]; then /work/render-readme-media.sh diff --git a/tests/portal-contract/tests/ota.spec.js b/tests/portal-contract/tests/ota.spec.js new file mode 100644 index 0000000..dac2ea1 --- /dev/null +++ b/tests/portal-contract/tests/ota.spec.js @@ -0,0 +1,89 @@ +const { test, expect } = require('@playwright/test'); + +const firmware = process.env.PORTAL_OTA_FIRMWARE; +const initialMarker = process.env.PORTAL_OTA_INITIAL_MARKER || 'A'; +const expectedMarker = process.env.PORTAL_OTA_EXPECTED_MARKER || 'B'; + +const sleep = (milliseconds) => new Promise((resolve) => setTimeout(resolve, milliseconds)); + +async function getMarker(request) { + const response = await request.get('/api/test/firmware-marker', { timeout: 4_000 }); + if (!response.ok()) { + throw new Error(`marker endpoint returned HTTP ${response.status()}`); + } + return response.json(); +} + +async function waitForMarker(request, expected, timeout = 75_000) { + let observed; + await expect.poll(async () => { + try { + observed = await getMarker(request); + return observed.marker; + } catch { + return undefined; + } + }, { + timeout, + intervals: [250, 500, 1_000, 1_000], + }).toBe(expected); + return observed; +} + +async function requireRestartOutage(request) { + const deadline = Date.now() + 25_000; + while (Date.now() < deadline) { + try { + const response = await request.get('/api/test/firmware-marker', { timeout: 1_000 }); + if (!response.ok()) { + return; + } + } catch { + return; + } + await sleep(150); + } + throw new Error('The portal never became unavailable after a successful OTA response.'); +} + +test.describe('portal HTTP OTA contract', () => { + test('uploads B through the rendered portal form, requires automatic reboot, and observes B twice', async ({ page, request }) => { + test.skip(!firmware, 'OTA firmware is mounted only for portal-hardware ota.'); + // Initial portal availability, an observed outage, and two fresh B + // responses each have their own bounded waits. Keep the overall budget + // larger than their sum so a valid slow reassociation is not killed by + // Playwright before the fixture contract has concluded. + test.setTimeout(300_000); + + const initial = await waitForMarker(request, initialMarker); + expect(initial.freeSketchSpace).toEqual(expect.any(Number)); + expect(initial.freeSketchSpace).toBeGreaterThan(0); + + // This deliberately uses the rendered UI and its multipart XHR instead of + // posting directly to /u. It therefore covers the real file control, + // submit handling, success JSON, and restart presentation together. + await page.goto('/#/update', { waitUntil: 'networkidle' }); + const input = page.locator('#wm-ota-file'); + await expect(input).toBeVisible(); + await input.setInputFiles(firmware); + + const updateResponse = page.waitForResponse((response) => { + const requestPath = new URL(response.url()).pathname; + return requestPath === '/u' && response.request().method() === 'POST'; + }); + await page.locator('#wm-ota-form button[type="submit"]').click(); + + const response = await updateResponse; + expect(response.status()).toBe(200); + await expect(response.json()).resolves.toMatchObject({ ok: true }); + + // A manual reset is never issued here. Observing the outage is what proves + // WiFiManager's Update.end(true) path restarted the board on its own. + await requireRestartOutage(request); + const first = await waitForMarker(request, expectedMarker); + expect(first.freeSketchSpace).toEqual(expect.any(Number)); + await sleep(1_000); + const second = await waitForMarker(request, expectedMarker); + expect(second.freeSketchSpace).toEqual(expect.any(Number)); + }); +}); diff --git a/tools/check-ota-partitions.sh b/tools/check-ota-partitions.sh new file mode 100755 index 0000000..667784c --- /dev/null +++ b/tools/check-ota-partitions.sh @@ -0,0 +1,66 @@ +#!/usr/bin/env bash +# Verify the tracked ESP32 OTA table used by the portal A/B fixtures. Keep the +# check independent of PlatformIO so CI rejects a layout regression before it +# downloads a framework or a hardware runner erases a board. +set -euo pipefail + +project_dir="$(cd "$(dirname "$0")/.." && pwd)" +table="${1:-$project_dir/test/portal-harness/partitions/esp32_ota_4m_no_fs.csv}" + +[[ $# -le 1 && -r "$table" ]] || { + echo "Usage: $0 [partition-table.csv]" >&2 + exit 2 +} + +partition_row() { + local name="$1" type="$2" subtype="$3" + awk -F, -v name="$name" -v type="$type" -v subtype="$subtype" ' + function trim(value) { + gsub(/^[[:space:]]+|[[:space:]]+$/, "", value) + return value + } + /^[[:space:]]*#/ || NF < 5 { next } + trim($1) == name && trim($2) == type && trim($3) == subtype { + print trim($1) "," trim($2) "," trim($3) "," trim($4) "," trim($5) + } + ' "$table" +} + +require_row() { + local name="$1" type="$2" subtype="$3" offset="$4" size="$5" actual expected + expected="$name,$type,$subtype,$offset,$size" + actual="$(partition_row "$name" "$type" "$subtype")" + [[ "$actual" == "$expected" ]] || { + echo "Expected exactly this OTA partition row: $expected" >&2 + echo "Found: ${actual:-}" >&2 + return 1 + } +} + +require_row nvs data nvs 0x9000 0x5000 +require_row otadata data ota 0xe000 0x2000 +require_row app0 app ota_0 0x10000 0x1F0000 +require_row app1 app ota_1 0x200000 0x1F0000 + +app_count="$(awk -F, ' + function trim(value) { gsub(/^[[:space:]]+|[[:space:]]+$/, "", value); return value } + /^[[:space:]]*#/ || NF < 5 { next } + trim($2) == "app" { count++ } + END { print count + 0 } +' "$table")" +[[ "$app_count" == 2 ]] || { + echo "OTA layout must contain exactly two application partitions, found $app_count." >&2 + exit 1 +} + +if awk -F, ' + function trim(value) { gsub(/^[[:space:]]+|[[:space:]]+$/, "", value); return value } + /^[[:space:]]*#/ || NF < 5 { next } + trim($2) == "data" && trim($3) ~ /^(spiffs|littlefs|fat)$/ { found = 1 } + END { exit found ? 0 : 1 } +' "$table"; then + echo "OTA test layout must not reserve a filesystem partition." >&2 + exit 1 +fi + +echo "ESP32 OTA partition contract passed: $table" diff --git a/tools/lib/portal-hardware-session.sh b/tools/lib/portal-hardware-session.sh index 2d04b88..030d0e7 100755 --- a/tools/lib/portal-hardware-session.sh +++ b/tools/lib/portal-hardware-session.sh @@ -19,11 +19,17 @@ wm_default_route_interface() { wm_acquire_hardware_lock() { local lock_file="${WM_HARDWARE_LOCK_FILE:-/tmp/wifimanager-hardware.lock}" + # `ota` deliberately acquires this before it builds firmware, then calls + # the shared portal-start helper which also acquires it. Keep that nested + # path idempotent so the lock covers the whole A/B contract rather than + # only the serial flash and adapter connection. + [[ "${WM_HARDWARE_LOCK_HELD:-no}" == "yes" ]] && return 0 exec 9>"$lock_file" flock -n 9 || { echo "Another WiFiManager hardware task is already running; wait for it to finish." >&2 return 1 } + WM_HARDWARE_LOCK_HELD=yes } wm_require_client_adapter() { diff --git a/tools/portal-hardware b/tools/portal-hardware index a28172f..62b45d9 100755 --- a/tools/portal-hardware +++ b/tools/portal-hardware @@ -15,11 +15,20 @@ Usage: --client-interface IFACE [--take-over-client-adapter] [--keep] \ [--browser auto|skip] [--station-env PATH] [--output DIRECTORY] \ [--capture-readme-media] [--custom-parameter-stress] + ./tools/portal-hardware ota --platform esp8266|esp32 --port /dev/serial/by-id/... \ + --client-interface IFACE [--take-over-client-adapter] [--keep] \ + [--output DIRECTORY] ./tools/portal-hardware down Only the named client interface may be disconnected or reconfigured. The tool refuses the host default-route interface and preserves the created connection -in a 0600 state file until `down` or normal `run` cleanup. +in a 0600 state file until `down` or normal `run`/`ota` cleanup. + +`ota` is an opt-in physical A/B test. It erases the explicitly selected test +board's flash, serial-flashes image A, then uses the real portal browser form +to upload image B. It requires the automatic restart and two independent +B-marker responses; a manual reset never makes the test pass. On success the +selected board remains on B in the portal-only fixture. USAGE exit 2 } @@ -38,6 +47,11 @@ station_env="" output_dir="" capture_readme_media="no" custom_parameter_stress="no" +ota_environment_a="" +ota_environment_b="" +ota_firmware_a="" +ota_firmware_b="" +ota_browser_prebuilt="no" while [[ $# -gt 0 ]]; do case "$1" in --platform) [[ $# -ge 2 ]] || usage; platform="$2"; shift 2 ;; @@ -63,6 +77,32 @@ require_common() { docker compose version >/dev/null } +pio_for_portal_environment() { + local environment="$1" + shift + + case "$environment" in + # The ESP32 A/B fixture extends the 3.3.11 environment. Its package + # graph must not share a Core directory with the legacy 3.0.5 portal + # fixture, whose tool-esptoolpy package name collides with current + # pioarduino metadata. + esp32_core_3_3_11|esp32_ota_*) + ;; + *) + pio "$@" + return + ;; + esac + + local core_dir packages_dir cache_dir + core_dir="${WIFIMANAGER_PLATFORMIO_CORE_DIR:-${XDG_CACHE_HOME:-$HOME/.cache}/wifimanager-platformio/core-3.3.11}" + packages_dir="${WIFIMANAGER_PLATFORMIO_PACKAGES_DIR:-$core_dir/packages}" + cache_dir="${WIFIMANAGER_PLATFORMIO_CACHE_DIR:-$core_dir/cache}" + install -d -m 700 "$core_dir" "$packages_dir" "$cache_dir" + PLATFORMIO_CORE_DIR="$core_dir" PLATFORMIO_PACKAGES_DIR="$packages_dir" \ + PLATFORMIO_CACHE_DIR="$cache_dir" pio "$@" +} + prepare_output_dir() { if [[ -z "$output_dir" ]]; then if [[ "$capture_readme_media" == "yes" ]]; then @@ -107,6 +147,20 @@ validate_run_arguments() { exit 2 } fi + if [[ "$command_name" == "ota" ]]; then + [[ "$browser" == "auto" ]] || { + echo "OTA coverage always uses the browser; --browser skip is not supported." >&2 + exit 2 + } + [[ -z "$station_env" ]] || { + echo "OTA coverage is portal-only and does not accept --station-env." >&2 + exit 2 + } + [[ "$capture_readme_media" == "no" && "$custom_parameter_stress" == "no" ]] || { + echo "OTA coverage cannot be combined with README media or parameter-stress modes." >&2 + exit 2 + } + fi } write_readme_media_manifest() { @@ -145,7 +199,7 @@ wait_for_portal_ready() { } start_portal_session() { - local ssid + local environment="${1:-$platform}" erase_before_upload="${2:-no}" expected_a_artifact="${3:-}" ssid validate_run_arguments require_common wm_acquire_hardware_lock @@ -154,7 +208,18 @@ start_portal_session() { prepare_output_dir ssid="$(wm_portal_ssid "$platform")" - pio run -d "$root/test/portal-harness" -e "$platform" -t upload --upload-port "$port" + if [[ "$erase_before_upload" == "yes" ]]; then + # OTA selection metadata must not survive from a previous fixture run: + # otherwise a bootloader could select a stale app slot instead of A. + pio_for_portal_environment "$environment" run -d "$root/test/portal-harness" -e "$environment" -t erase --upload-port "$port" + fi + if [[ -n "$expected_a_artifact" ]]; then + ota_assert_a_artifact_matches_build "$expected_a_artifact" + fi + pio_for_portal_environment "$environment" run -d "$root/test/portal-harness" -e "$environment" -t upload --upload-port "$port" + if [[ -n "$expected_a_artifact" ]]; then + ota_assert_a_artifact_matches_build "$expected_a_artifact" + fi if ! wm_create_portal_connection "$client_interface" "$ssid" "default1"; then return 1 fi @@ -173,6 +238,223 @@ start_portal_session() { printf 'Portal connected on %s. Artifacts: %s\n' "$client_interface" "$output_dir" } +ota_environment() { + local image="$1" + printf '%s_ota_%s\n' "$platform" "$image" +} + +ota_marker_field() { + local response="$1" field="$2" + case "$field" in + marker) + sed -n 's/.*"marker"[[:space:]]*:[[:space:]]*"\([^"]*\)".*/\1/p' <<<"$response" + ;; + freeSketchSpace) + sed -n 's/.*"freeSketchSpace"[[:space:]]*:[[:space:]]*\([0-9][0-9]*\).*/\1/p' <<<"$response" + ;; + *) + echo "Unknown OTA marker field: $field" >&2 + return 2 + ;; + esac +} + +ota_fetch_marker() { + curl --interface "$client_interface" --connect-timeout 1 --max-time 2 \ + --silent --show-error http://192.168.4.1/api/test/firmware-marker 2>/dev/null +} + +wait_for_ota_marker() { + local expected="$1" deadline response marker + deadline=$((SECONDS + 75)) + while (( SECONDS < deadline )); do + response="$(ota_fetch_marker || true)" + marker="$(ota_marker_field "$response" marker)" + if [[ "$marker" == "$expected" ]]; then + printf '%s\n' "$response" + return 0 + fi + sleep 0.5 + done + echo "Portal OTA fixture did not report marker '$expected' within 75 seconds." >&2 + return 1 +} + +ota_esp32_partition_row() { + local partition="$1" table="$root/test/portal-harness/partitions/esp32_ota_4m_no_fs.csv" + [[ -r "$table" ]] || { + echo "ESP32 OTA partition table is unreadable: $table" >&2 + return 1 + } + awk -F, -v partition="$partition" ' + function trim(value) { + gsub(/^[[:space:]]+|[[:space:]]+$/, "", value) + return value + } + /^[[:space:]]*#/ || NF < 5 { next } + trim($1) == partition { + printf "%s\t%s\t%s\t%s\n", trim($2), trim($3), trim($4), trim($5) + exit + } + ' "$table" +} + +ota_esp32_require_partition() { + local name="$1" expected_type="$2" expected_subtype="$3" expected_offset="$4" expected_size="$5" + local row type subtype offset size + row="$(ota_esp32_partition_row "$name")" + IFS=$'\t' read -r type subtype offset size <<<"$row" + [[ "$type" == "$expected_type" && "$subtype" == "$expected_subtype" && \ + "$offset" == "$expected_offset" && "$size" == "$expected_size" ]] || { + echo "ESP32 OTA partition '$name' must be $expected_type/$expected_subtype at $expected_offset with size $expected_size; found '${row:-missing}'." >&2 + return 1 + } +} + +ota_esp32_slot_size() { + local table="$root/test/portal-harness/partitions/esp32_ota_4m_no_fs.csv" + local value_a value_b + [[ -r "$table" ]] || { + echo "ESP32 OTA partition table is unreadable: $table" >&2 + return 1 + } + ota_esp32_require_partition nvs data nvs 0x9000 0x5000 + ota_esp32_require_partition otadata data ota 0xe000 0x2000 + ota_esp32_require_partition app0 app ota_0 0x10000 0x1F0000 + ota_esp32_require_partition app1 app ota_1 0x200000 0x1F0000 + value_a="$(ota_esp32_partition_row app0 | awk -F '\t' '{ print $4 }')" + value_b="$(ota_esp32_partition_row app1 | awk -F '\t' '{ print $4 }')" + [[ "$value_a" =~ ^0x[0-9A-Fa-f]+$ && "$value_a" == "$value_b" ]] || { + echo "ESP32 OTA app slots must have equal hexadecimal capacities: app0=$value_a app1=$value_b" >&2 + return 1 + } + printf '%d\n' "$((value_b))" +} + +ota_assert_firmware_fits() { + local firmware="$1" capacity="$2" label="$3" image_size + [[ -n "$firmware" && -s "$firmware" ]] || { + echo "OTA $label firmware artifact is missing." >&2 + return 1 + } + [[ "$capacity" =~ ^[0-9]+$ && "$capacity" -gt 0 ]] || { + echo "OTA capacity is invalid: $capacity" >&2 + return 1 + } + image_size="$(wc -c < "$firmware")" + if (( image_size > capacity )); then + echo "OTA $label image is too large: ${image_size} bytes exceeds ${capacity} bytes." >&2 + return 1 + fi + printf 'OTA %s image fits the available update space: %s <= %s bytes\n' "$label" "$image_size" "$capacity" +} + +prepare_ota_firmware() { + local source_a source_b capacity + ota_environment_a="$(ota_environment a)" + ota_environment_b="$(ota_environment b)" + + # Build and preserve both identities before touching the board. The + # browser mounts B read-only; the A artifact is compared against the + # PlatformIO build used for serial flashing so a later build cannot turn + # the A/B proof into an unrecorded input change. + pio_for_portal_environment "$ota_environment_a" run -d "$root/test/portal-harness" -e "$ota_environment_a" + pio_for_portal_environment "$ota_environment_b" run -d "$root/test/portal-harness" -e "$ota_environment_b" + source_a="$root/test/portal-harness/.pio/build/$ota_environment_a/firmware.bin" + source_b="$root/test/portal-harness/.pio/build/$ota_environment_b/firmware.bin" + [[ -s "$source_a" && -s "$source_b" ]] || { + echo "PlatformIO did not produce both OTA fixture images." >&2 + return 1 + } + if cmp -s "$source_a" "$source_b"; then + echo "PlatformIO produced identical OTA A and B images." >&2 + return 1 + fi + ota_firmware_a="$output_dir/${platform}-portal-ota-a.bin" + ota_firmware_b="$output_dir/${platform}-portal-ota-b.bin" + install -m 600 "$source_a" "$ota_firmware_a" + install -m 600 "$source_b" "$ota_firmware_b" + + if [[ "$platform" == "esp32" ]]; then + "$root/tools/check-ota-partitions.sh" + capacity="$(ota_esp32_slot_size)" + ota_assert_firmware_fits "$ota_firmware_a" "$capacity" A + ota_assert_firmware_fits "$ota_firmware_b" "$capacity" B + fi +} + +ota_assert_a_artifact_matches_build() { + local expected_artifact="$1" source + source="$root/test/portal-harness/.pio/build/$ota_environment_a/firmware.bin" + [[ -s "$expected_artifact" && -s "$source" ]] || { + echo "Prepared OTA A artifact or PlatformIO build output is missing." >&2 + return 1 + } + cmp -s "$expected_artifact" "$source" || { + echo "PlatformIO's A build no longer matches the immutable artifact prepared before serial flashing." >&2 + return 1 + } +} + +validate_running_ota_capacity() { + local marker_response free_sketch_space capacity + [[ "$platform" == "esp8266" ]] || return 0 + marker_response="$(wait_for_ota_marker A)" + free_sketch_space="$(ota_marker_field "$marker_response" freeSketchSpace)" + [[ "$free_sketch_space" =~ ^[0-9]+$ && "$free_sketch_space" -gt 4096 ]] || { + echo "ESP8266 fixture did not report usable free sketch space." >&2 + return 1 + } + # Match WiFiManagerHandlers::handleUpdating(): Update.begin receives this + # aligned value, not the raw ESP.getFreeSketchSpace() number. + capacity=$(( (free_sketch_space - 0x1000) & 0xFFFFF000 )) + ota_assert_firmware_fits "$ota_firmware_b" "$capacity" B +} + +configure_ota_contract_environment() { + export PORTAL_ARTIFACT_DIR="$output_dir" + export LOCAL_UID="$(id -u)" + export LOCAL_GID="$(id -g)" + export PORTAL_BROWSER_MODE=auto + # OTA is its own contract. Do not inherit a developer's unrelated media, + # stress, target, or test-selection setting into a physical firmware run. + export PORTAL_CONTRACT_DOCKER_TARGET=portal-contract + export PORTAL_CAPTURE_README_MEDIA=0 + export PORTAL_CUSTOM_PARAMETER_STRESS=0 + export PORTAL_TEST_FILE=tests/ota.spec.js + export PORTAL_PLATFORM="$platform" + export PORTAL_OTA_FIRMWARE_HOST="$ota_firmware_b" + export PORTAL_OTA_INITIAL_MARKER=A + export PORTAL_OTA_EXPECTED_MARKER=B +} + +ota_compose_files() { + printf '%s\n' \ + -f "$root/tests/portal-contract/compose.yaml" \ + -f "$root/tests/portal-contract/compose.ota.yaml" +} + +prepare_ota_contract_image() { + local -a compose_files + configure_ota_contract_environment + mapfile -t compose_files < <(ota_compose_files) + # Build before A is flashed. A cold Playwright build can otherwise consume + # the finite portal window before the browser has connected. + docker compose "${compose_files[@]}" build portal-contract + ota_browser_prebuilt=yes +} + +run_ota_contract() { + local -a compose_files + configure_ota_contract_environment + mapfile -t compose_files < <(ota_compose_files) + [[ "$ota_browser_prebuilt" == yes ]] || { + echo "Portal OTA browser image was not prepared before A was flashed." >&2 + return 1 + } + docker compose "${compose_files[@]}" run --rm portal-contract +} + finish_portal_session() { wm_load_state wm_remove_connection "$WM_PORTAL_CONNECTION_UUID" @@ -208,6 +490,9 @@ case "$command_name" in export LOCAL_UID="$(id -u)" export LOCAL_GID="$(id -g)" export PORTAL_BROWSER_MODE="$browser" + # A normal run is the full portal suite; do not let a prior OTA shell + # environment restrict it to one spec. + export PORTAL_TEST_FILE='' export PORTAL_CUSTOM_PARAMETER_STRESS=0 if [[ "$custom_parameter_stress" == "yes" ]]; then export PORTAL_CUSTOM_PARAMETER_STRESS=1 @@ -237,5 +522,43 @@ case "$command_name" in printf 'Portal session remains connected; run ./tools/portal-hardware down when finished.\n' fi ;; + ota) + # Acquire the same board/adapter lock before *any* A/B preparation. + # Otherwise two OTA invocations can collide in .pio or create a + # same-second artifact directory before either reaches serial flash. + validate_run_arguments + require_common + wm_acquire_hardware_lock + wm_require_no_active_session + wm_require_client_adapter "$client_interface" "$takeover" + prepare_output_dir + prepare_ota_firmware + prepare_ota_contract_image + start_portal_session "$ota_environment_a" yes "$ota_firmware_a" + cleanup() { + if [[ "$keep" != "yes" ]]; then + finish_portal_session || true + fi + } + trap cleanup EXIT INT TERM + + # A must be visible through the real portal before a B upload can be + # meaningful. On ESP8266 the marker also reports the actual active + # sketch-space limit, which is checked before the browser sees B. + wait_for_ota_marker A >/dev/null + validate_running_ota_capacity + run_ota_contract + + # The browser contract has already required an outage and two B + # observations. Repeat the host-side observation after the container + # exits so the named secondary adapter is also proven to see B. + wait_for_ota_marker B >/dev/null + sleep 1 + wait_for_ota_marker B >/dev/null + printf 'Portal HTTP OTA contract passed. Artifacts: %s\n' "$output_dir" + if [[ "$keep" == "yes" ]]; then + printf 'Portal session remains connected; run ./tools/portal-hardware down when finished.\n' + fi + ;; *) usage ;; esac diff --git a/tools/tests/test-portal-hardware-cli.sh b/tools/tests/test-portal-hardware-cli.sh index 645b239..903da61 100755 --- a/tools/tests/test-portal-hardware-cli.sh +++ b/tools/tests/test-portal-hardware-cli.sh @@ -61,6 +61,20 @@ if "$root/tools/portal-hardware" run --platform esp8266 --port /dev/null \ echo 'browser-skipped custom-parameter stress was accepted' >&2 exit 1 fi +if "$root/tools/portal-hardware" ota --platform esp8266 --port /dev/null \ + --client-interface wlan-client --browser skip >/dev/null 2>&1; then + echo 'browser-skipped OTA was accepted' >&2 + exit 1 +fi +if "$root/tools/portal-hardware" ota --platform esp8266 --port /dev/null \ + --client-interface wlan-client --station-env "$root/test/portal-station.env.example" >/dev/null 2>&1; then + echo 'station handoff inputs were accepted by portal OTA' >&2 + exit 1 +fi +if "$root/scripts/test.sh" ota-fixtures --platform esp32 >/dev/null 2>&1; then + echo 'legacy-cache ESP32 OTA-fixture selector was accepted' >&2 + exit 1 +fi # A failed association must delete the only connection it just created. source "$root/tools/lib/portal-hardware-session.sh" @@ -102,6 +116,14 @@ run_line="$(grep -n " run --rm portal-contract$" "$CALL_LOG" | tail -1 | cut -d: grep -Fq 'wait_for_portal_ready' "$root/tools/portal-hardware" grep -Fq 'api/wifi/scan-status' "$root/tools/portal-hardware" grep -Fq 'PORTAL_CUSTOM_PARAMETER_STRESS' "$root/tools/portal-hardware" +grep -Fq 'compose.ota.yaml' "$root/tools/portal-hardware" +grep -Fq 'wait_for_ota_marker B' "$root/tools/portal-hardware" +grep -Fq 'pio_for_portal_environment "$ota_environment_a"' "$root/tools/portal-hardware" +grep -Fq 'WIFIMANAGER_PLATFORMIO_CORE_DIR' "$root/tools/portal-hardware" +grep -Fq 'assert_ota_fixture_pair' "$root/scripts/test.sh" +grep -Fq 'WiFiManager Unity compile check passed' "$root/scripts/test.sh" +grep -Fq 'eagle.flash.4m1m.ld' "$root/test/portal-harness/platformio.ini" +grep -Fq 'esp32_ota_4m_no_fs.csv' "$root/test/portal-harness/platformio.ini" grep -Fq 'README media GIF exceeds its 2 MiB documentation budget' \ "$root/tests/portal-contract/render-readme-media.sh"