Add MCP server for interacting with meshtastic devices and testing framework / TUI

mcp-server/src/meshtastic_mcp/admin.py

@@ -36,11 +36,18 @@ def _require_confirm(confirm: bool, operation: str) -> None:
 
 
 def _message_to_dict(msg: Any) -> dict[str, Any]:
-    return json_format.MessageToDict(
-        msg,
-        preserving_proto_field_name=True,
-        including_default_value_fields=False,
-    )
+    # `including_default_value_fields` was renamed to
+    # `always_print_fields_with_no_presence` in protobuf 5.26+. Pick whichever
+    # kwarg the installed version accepts so we work against both.
+    kwargs: dict[str, Any] = {"preserving_proto_field_name": True}
+    import inspect
+
+    sig = inspect.signature(json_format.MessageToDict)
+    if "always_print_fields_with_no_presence" in sig.parameters:
+        kwargs["always_print_fields_with_no_presence"] = False
+    elif "including_default_value_fields" in sig.parameters:
+        kwargs["including_default_value_fields"] = False
+    return json_format.MessageToDict(msg, **kwargs)
 
 
 # ---------- owner ----------------------------------------------------------
@@ -291,6 +298,37 @@ def send_text(
     return {"ok": True, "packet_id": packet_id, "destination": destination}
 
 
+# ---------- diagnostics ----------------------------------------------------
+
+
+def set_debug_log_api(enabled: bool, port: str | None = None) -> dict[str, Any]:
+    """Toggle `config.security.debug_log_api_enabled` on the local node.
+
+    When enabled, firmware emits log lines as protobuf `LogRecord` messages
+    over the StreamAPI instead of raw text. meshtastic-python surfaces them
+    on pubsub topic `meshtastic.log.line`, which flows through the SAME
+    SerialInterface our tests already hold open — no `pio device monitor`
+    needed, no port-contention with admin/info calls.
+
+    Firmware gate: `src/SerialConsole.cpp` (`usingProtobufs &&
+    config.security.debug_log_api_enabled`). Setting persists in NVS; it
+    survives reboot. `factory_reset(full=False)` clears it unless it's
+    re-applied after reset.
+
+    Previously-documented concurrency hazard (emitLogRecord sharing the
+    main packet-emission buffers) has been fixed — see `StreamAPI.h`
+    where the log path now owns dedicated `fromRadioScratchLog` /
+    `txBufLog` buffers, and `StreamAPI::emitTxBuffer` +
+    `StreamAPI::emitLogRecord` both serialize their `stream->write`
+    calls via `streamLock`. Leaving the flag on under traffic is safe.
+    """
+    with connect(port=port) as iface:
+        sec = iface.localNode.localConfig.security
+        sec.debug_log_api_enabled = bool(enabled)
+        iface.localNode.writeConfig("security")
+    return {"ok": True, "debug_log_api_enabled": bool(enabled)}
+
+
 # ---------- admin actions --------------------------------------------------
 
 
@@ -315,7 +353,19 @@ def shutdown(
 def factory_reset(
     port: str | None = None, confirm: bool = False, full: bool = False
 ) -> dict[str, Any]:
+    """Tell the node to factory-reset its config.
+
+    Works around a meshtastic-python 2.7.8 bug: `Node.factoryReset(full=True)`
+    internally does `p.factory_reset_config = True` where the field is
+    int32. protobuf 5.x rejects bool→int assignment as a TypeError. We build
+    the AdminMessage directly with int values (1=non-full, 2=full) and call
+    `_sendAdmin` to sidestep the SDK bug entirely.
+    """
     _require_confirm(confirm, "factory_reset")
+    from meshtastic.protobuf import admin_pb2  # type: ignore[import-untyped]
+
     with connect(port=port) as iface:
-        iface.localNode.factoryReset(full=full)
+        msg = admin_pb2.AdminMessage()
+        msg.factory_reset_config = 2 if full else 1
+        iface.localNode._sendAdmin(msg)
     return {"ok": True, "full": full}

mcp-server/src/meshtastic_mcp/cli/__init__.py

@@ -0,0 +1,6 @@
+"""Command-line entry points that sit alongside the MCP server.
+
+Modules here are loaded on-demand by `[project.scripts]` entries in
+`pyproject.toml`. They are NOT imported by `meshtastic_mcp.server` or the
+admin/info tool surface — the MCP server stays pure stdio JSON-RPC.
+"""

mcp-server/src/meshtastic_mcp/cli/_flashlog.py

@@ -0,0 +1,73 @@
+"""Flash progress log tailer for ``meshtastic-mcp-test-tui``.
+
+``pio.py`` / ``hw_tools.py`` tee subprocess output (``pio run -t upload``,
+``esptool erase_flash``, ``nrfutil dfu``, etc.) to ``tests/flash.log``
+line-by-line as it arrives — controlled by the ``MESHTASTIC_MCP_FLASH_LOG``
+env var that ``run-tests.sh`` sets. The TUI tails that file so the operator
+sees live flash progress in the pytest pane instead of 3 minutes of silence
+during ``test_00_bake``.
+
+Separate from ``_fwlog.py`` because that one parses JSONL, this one
+streams plain text lines. Same daemon-thread + EOF-backoff structure.
+"""
+
+from __future__ import annotations
+
+import pathlib
+import threading
+import time
+from typing import Callable
+
+
+class FlashLogTailer(threading.Thread):
+    """Tail a plain-text log file, publish each stripped line via ``post``.
+
+    ``post`` is invoked with a single ``str`` for every new line. Lines are
+    stripped of trailing newlines; empty lines after stripping are dropped.
+
+    The file may not exist yet when this thread starts — it's truncated by
+    ``run-tests.sh`` at session start, but if the tailer races the shell,
+    we tolerate FileNotFoundError for up to ``wait_s`` seconds.
+    """
+
+    def __init__(
+        self,
+        path: pathlib.Path,
+        post: Callable[[str], None],
+        stop: threading.Event,
+        *,
+        wait_s: float = 30.0,
+    ) -> None:
+        super().__init__(daemon=True, name="flashlog-tail")
+        self._path = path
+        self._post = post
+        self._stop = stop
+        self._wait_s = wait_s
+
+    def run(self) -> None:
+        deadline = time.monotonic() + self._wait_s
+        while not self._path.is_file():
+            if self._stop.is_set() or time.monotonic() > deadline:
+                return
+            time.sleep(0.1)
+        try:
+            fh = self._path.open("r", encoding="utf-8", errors="replace")
+        except OSError:
+            return
+        try:
+            while not self._stop.is_set():
+                line = fh.readline()
+                if not line:
+                    time.sleep(0.05)
+                    continue
+                line = line.rstrip("\r\n")
+                if not line:
+                    continue
+                try:
+                    self._post(line)
+                except Exception:
+                    # A post failure (e.g. closed app) is terminal for this
+                    # thread but we still want to close the file handle.
+                    return
+        finally:
+            fh.close()

mcp-server/src/meshtastic_mcp/cli/_fwlog.py

@@ -0,0 +1,95 @@
+"""Firmware log tail worker for ``meshtastic-mcp-test-tui``.
+
+Complements v1's reportlog-tail worker. ``tests/conftest.py`` owns a
+session-scoped autouse fixture (``_firmware_log_stream``) that mirrors
+every ``meshtastic.log.line`` pubsub event to ``tests/fwlog.jsonl`` —
+one JSON object per line:
+
+    {"ts": 1729100000.123, "port": "/dev/cu.usbmodem1101", "line": "..."}
+
+The TUI tails that file from a worker thread; each new line becomes a
+:class:`FirmwareLogLine` message posted to the App. Same pattern as the
+reportlog tail worker — truncate on launch, tolerate missing file for
+30 s, back off at EOF.
+
+Kept in its own module so the (large) ``test_tui.py`` stays focused on
+the Textual App shell.
+"""
+
+from __future__ import annotations
+
+import json
+import pathlib
+import threading
+import time
+from typing import Any, Callable
+
+
+class FirmwareLogTailer(threading.Thread):
+    """Tail ``tests/fwlog.jsonl``, publish parsed records via ``post``.
+
+    ``post`` is the App's ``post_message`` (or any callable that accepts a
+    single payload arg). We pass parsed dicts rather than constructing
+    Textual Message objects here — keeps this module free of the
+    textual dependency so it's unit-testable in a bare venv.
+
+    Parameters
+    ----------
+    path:
+        Path to ``tests/fwlog.jsonl``. The file may not exist yet at
+        startup — pytest only creates it once the session fixture runs.
+    post:
+        Callable invoked with a dict ``{"ts", "port", "line"}`` for every
+        new line parsed from the file.
+    stop:
+        An event the App sets to signal shutdown.
+    wait_s:
+        How long to poll for the file's creation before giving up. Default
+        30 s; pytest collection on a cold cache can be slow.
+    """
+
+    def __init__(
+        self,
+        path: pathlib.Path,
+        post: Callable[[dict[str, Any]], None],
+        stop: threading.Event,
+        *,
+        wait_s: float = 30.0,
+    ) -> None:
+        super().__init__(daemon=True, name="fwlog-tail")
+        self._path = path
+        self._post = post
+        self._stop = stop
+        self._wait_s = wait_s
+
+    def run(self) -> None:
+        deadline = time.monotonic() + self._wait_s
+        while not self._path.is_file():
+            if self._stop.is_set() or time.monotonic() > deadline:
+                return
+            time.sleep(0.1)
+        try:
+            fh = self._path.open("r", encoding="utf-8")
+        except OSError:
+            return
+        try:
+            while not self._stop.is_set():
+                line = fh.readline()
+                if not line:
+                    time.sleep(0.05)
+                    continue
+                line = line.strip()
+                if not line:
+                    continue
+                try:
+                    record = json.loads(line)
+                except json.JSONDecodeError:
+                    continue
+                # Defensive: require the three fields we rely on.
+                if not isinstance(record, dict):
+                    continue
+                if "line" not in record:
+                    continue
+                self._post(record)
+        finally:
+            fh.close()

mcp-server/src/meshtastic_mcp/cli/_history.py

@@ -0,0 +1,127 @@
+"""Cross-run history for ``meshtastic-mcp-test-tui``.
+
+Persists one JSON object per pytest run to
+``mcp-server/tests/.history/runs.jsonl``. The TUI reads the last N
+entries on launch to render a duration sparkline in the header — a
+quick read on whether the suite is slowing down over time.
+
+Schema (keep small; the file can grow for months):
+
+    {"run": 42, "ts": 1729100000.0, "duration_s": 387.2,
+     "passed": 52, "failed": 0, "skipped": 23, "exit_code": 0,
+     "seed": "mcp-user-host"}
+"""
+
+from __future__ import annotations
+
+import json
+import pathlib
+import time
+from dataclasses import asdict, dataclass
+from typing import Iterable
+
+# Sparkline glyphs, low → high. 8 levels is the Unicode convention.
+_SPARK_BLOCKS = "▁▂▃▄▅▆▇█"
+
+
+@dataclass
+class RunRecord:
+    run: int
+    ts: float
+    duration_s: float
+    passed: int
+    failed: int
+    skipped: int
+    exit_code: int
+    seed: str
+
+
+class HistoryStore:
+    """Append-only JSONL store with bounded read.
+
+    Writes are fsynced after each append (the file is tiny; fsync cost
+    is negligible and protects against truncation on a crash).
+    """
+
+    def __init__(self, path: pathlib.Path, *, keep_last: int = 50) -> None:
+        self._path = path
+        self._keep_last = keep_last
+
+    def append(self, record: RunRecord) -> None:
+        try:
+            self._path.parent.mkdir(parents=True, exist_ok=True)
+            with self._path.open("a", encoding="utf-8") as fh:
+                fh.write(json.dumps(asdict(record)) + "\n")
+                fh.flush()
+        except Exception:
+            # Non-fatal: history is cosmetic.
+            pass
+
+    def read_recent(self) -> list[RunRecord]:
+        """Return the last ``keep_last`` records in chronological order."""
+        if not self._path.is_file():
+            return []
+        try:
+            lines = self._path.read_text(encoding="utf-8").splitlines()
+        except OSError:
+            return []
+        out: list[RunRecord] = []
+        # Parse tail-first so we don't waste work on a huge history.
+        for line in lines[-self._keep_last :]:
+            line = line.strip()
+            if not line:
+                continue
+            try:
+                raw = json.loads(line)
+            except json.JSONDecodeError:
+                continue
+            try:
+                out.append(RunRecord(**raw))
+            except TypeError:
+                # Schema drift; skip the record rather than crash.
+                continue
+        return out
+
+    def record_run(
+        self,
+        *,
+        run: int,
+        duration_s: float,
+        passed: int,
+        failed: int,
+        skipped: int,
+        exit_code: int,
+        seed: str,
+    ) -> RunRecord:
+        rec = RunRecord(
+            run=run,
+            ts=time.time(),
+            duration_s=float(duration_s),
+            passed=int(passed),
+            failed=int(failed),
+            skipped=int(skipped),
+            exit_code=int(exit_code),
+            seed=seed,
+        )
+        self.append(rec)
+        return rec
+
+
+def sparkline(values: Iterable[float], *, width: int = 20) -> str:
+    """Render a Unicode block-character sparkline from the last ``width`` values.
+
+    Returns an empty string for empty input so the header handles
+    "no history yet" gracefully.
+    """
+    buf = [v for v in values if v >= 0][-width:]
+    if not buf:
+        return ""
+    lo, hi = min(buf), max(buf)
+    if hi - lo < 1e-9:
+        return _SPARK_BLOCKS[len(_SPARK_BLOCKS) // 2] * len(buf)
+    n = len(_SPARK_BLOCKS) - 1
+    out = []
+    for v in buf:
+        idx = int(round((v - lo) / (hi - lo) * n))
+        out.append(_SPARK_BLOCKS[max(0, min(n, idx))])
+    return "".join(out)

mcp-server/src/meshtastic_mcp/cli/_reproducer.py

@@ -0,0 +1,214 @@
+"""Reproducer bundle builder for ``meshtastic-mcp-test-tui``.
+
+When the operator presses ``x`` on a failed test leaf, we package the
+minimum viable failure context into a tarball under
+``mcp-server/tests/reproducers/``:
+
+::
+
+    repro-<ts>-<short_nodeid>.tar.gz
+      ├── README.md            human-readable overview
+      ├── test_report.json     the failing TestReport event from reportlog
+      ├── fwlog.jsonl          firmware log filtered to the failure window
+      ├── devices.json         per-device device_info + lora config snapshot
+      └── env.json             seed, run #, pytest version, platform, hostname
+
+Separate module so the logic can be unit-tested without Textual. The
+TUI glue is thin — one key binding calls :func:`build_reproducer_bundle`
+with the focused test's state and shows the path in a modal.
+"""
+
+from __future__ import annotations
+
+import io
+import json
+import pathlib
+import platform
+import re
+import socket
+import tarfile
+import time
+from dataclasses import dataclass
+from typing import Any, Iterable
+
+
+@dataclass
+class ReproContext:
+    """Everything :func:`build_reproducer_bundle` needs. Shaped to map
+    cleanly onto the state the TUI already tracks — no extra data
+    collection required at export time."""
+
+    nodeid: str
+    longrepr: str
+    sections: list[tuple[str, str]]
+    start_ts: float | None
+    stop_ts: float | None
+    seed: str
+    run_number: int
+    exit_code: int | None
+    fwlog_path: pathlib.Path
+    output_dir: pathlib.Path
+    extra_device_rows: list[dict[str, Any]]  # [{role, port, info, ...}, ...]
+
+
+def _short_nodeid(nodeid: str) -> str:
+    """Collapse a pytest nodeid into a filename-safe slug (<= 60 chars)."""
+    # Drop the file path prefix; keep test name + parametrization.
+    tail = nodeid.split("::", 1)[-1] if "::" in nodeid else nodeid
+    slug = re.sub(r"[^A-Za-z0-9_.\-]", "_", tail)
+    return slug[:60].strip("_.-") or "test"
+
+
+def _filtered_fwlog(
+    fwlog_path: pathlib.Path,
+    start_ts: float | None,
+    stop_ts: float | None,
+    *,
+    pad_s: float = 5.0,
+) -> bytes:
+    """Return fwlog.jsonl lines whose ``ts`` lies in [start-pad, stop+pad]."""
+    if not fwlog_path.is_file():
+        return b""
+    if start_ts is None or stop_ts is None:
+        # Without a time window, include the whole file — rare; happens
+        # when a test fails in setup before pytest emitted a start ts.
+        try:
+            return fwlog_path.read_bytes()
+        except OSError:
+            return b""
+    lo, hi = start_ts - pad_s, stop_ts + pad_s
+    out = io.BytesIO()
+    try:
+        with fwlog_path.open("r", encoding="utf-8") as fh:
+            for line in fh:
+                stripped = line.strip()
+                if not stripped:
+                    continue
+                try:
+                    record = json.loads(stripped)
+                except json.JSONDecodeError:
+                    continue
+                ts = record.get("ts")
+                if not isinstance(ts, (int, float)):
+                    continue
+                if lo <= ts <= hi:
+                    out.write(line.encode("utf-8"))
+    except OSError:
+        return b""
+    return out.getvalue()
+
+
+def _readme(ctx: ReproContext) -> str:
+    t = time.strftime("%Y-%m-%d %H:%M:%S %Z", time.localtime())
+    return f"""# Reproducer bundle
+
+Exported by `meshtastic-mcp-test-tui` on {t}.
+
+## Failing test
+
+- **nodeid:** `{ctx.nodeid}`
+- **seed:** `{ctx.seed}`
+- **run #:** {ctx.run_number}
+- **suite exit code (at export time):** {ctx.exit_code if ctx.exit_code is not None else "in progress"}
+
+## Files in this archive
+
+| File | Contents |
+|---|---|
+| `test_report.json` | The pytest-reportlog `TestReport` event for the failing test — includes `longrepr`, captured `sections` (stdout/stderr/log), `duration`, `location`, `keywords`. |
+| `fwlog.jsonl` | Firmware log lines (from `meshtastic.log.line` pubsub) filtered to [start−5s, stop+5s] around the test's run window. Each line is `{{ts, port, line}}`. |
+| `devices.json` | Per-device snapshot at export time: `device_info` + `lora` config per detected role. |
+| `env.json` | Python version, platform, hostname, seed, run number. |
+
+## How to triage
+
+1. Open `test_report.json` and read `longrepr` + `sections` — most failures explain themselves there.
+2. If the failure is a mesh/telemetry assertion, `fwlog.jsonl` is where the answer usually lives. Grep for `Error=`, `NAK`, `PKI_UNKNOWN_PUBKEY`, `Skip send`, `Guru Meditation`, or the uptime timestamps around the assertion event.
+3. Compare `devices.json` against the expected state (e.g. `num_nodes >= 2`, `primary_channel == "McpTest"`, `region == "US"`). If fields disagree with the seed-derived USERPREFS profile, the device probably wasn't baked with this session's profile.
+
+## Reproducing locally
+
+```bash
+cd mcp-server
+MESHTASTIC_MCP_SEED='{ctx.seed}' .venv/bin/pytest '{ctx.nodeid}' --tb=long -v
+```
+"""
+
+
+def build_reproducer_bundle(ctx: ReproContext) -> pathlib.Path:
+    """Build a tarball under ``ctx.output_dir`` and return its path.
+
+    Parent dirs are created as needed. Errors during optional sections
+    (devices, env) are swallowed — the bundle is still useful without
+    them; refusing to export because the device poller had a hiccup
+    would be worse than the export missing a file.
+    """
+    ctx.output_dir.mkdir(parents=True, exist_ok=True)
+    ts = int(time.time())
+    slug = _short_nodeid(ctx.nodeid)
+    archive_path = ctx.output_dir / f"repro-{ts}-{slug}.tar.gz"
+
+    with tarfile.open(archive_path, "w:gz") as tar:
+
+        def _add(name: str, data: bytes) -> None:
+            info = tarfile.TarInfo(name=name)
+            info.size = len(data)
+            info.mtime = ts
+            tar.addfile(info, io.BytesIO(data))
+
+        # README
+        _add("README.md", _readme(ctx).encode("utf-8"))
+
+        # test_report.json — reconstruct from the fields the TUI stashes.
+        test_report = {
+            "nodeid": ctx.nodeid,
+            "outcome": "failed",
+            "longrepr": ctx.longrepr,
+            "sections": [list(s) for s in ctx.sections],
+            "start": ctx.start_ts,
+            "stop": ctx.stop_ts,
+        }
+        _add(
+            "test_report.json",
+            json.dumps(test_report, indent=2, default=str).encode("utf-8"),
+        )
+
+        # fwlog.jsonl (filtered)
+        _add("fwlog.jsonl", _filtered_fwlog(ctx.fwlog_path, ctx.start_ts, ctx.stop_ts))
+
+        # devices.json
+        try:
+            devices_payload = json.dumps(
+                ctx.extra_device_rows or [], indent=2, default=str
+            )
+        except Exception:
+            devices_payload = "[]"
+        _add("devices.json", devices_payload.encode("utf-8"))
+
+        # env.json
+        try:
+            from importlib.metadata import version as _pkg_version
+
+            pytest_version = _pkg_version("pytest")
+        except Exception:
+            pytest_version = "unknown"
+        env_payload = {
+            "seed": ctx.seed,
+            "run": ctx.run_number,
+            "exit_code": ctx.exit_code,
+            "export_ts": ts,
+            "python": platform.python_version(),
+            "pytest": pytest_version,
+            "platform": f"{platform.system()} {platform.release()} {platform.machine()}",
+            "hostname": socket.gethostname(),
+        }
+        _add("env.json", json.dumps(env_payload, indent=2).encode("utf-8"))
+
+    return archive_path
+
+
+def iter_entries(archive_path: pathlib.Path) -> Iterable[str]:
+    """Yield member names — used by callers that want to confirm the bundle shape."""
+    with tarfile.open(archive_path, "r:gz") as tar:
+        for m in tar.getmembers():
+            yield m.name

mcp-server/src/meshtastic_mcp/flash.py

@@ -18,7 +18,19 @@ import serial
 
 from . import boards, config, devices, pio, userprefs
 
-ESP32_ARCHES = {"esp32", "esp32s2", "esp32s3", "esp32c3", "esp32c6"}
+# Meshtastic variants use both `esp32s3` and `esp32-s3` style names across
+# variants/*/platformio.ini (no consistency enforced). Accept both spellings.
+ESP32_ARCHES = {
+    "esp32",
+    "esp32s2",
+    "esp32-s2",
+    "esp32s3",
+    "esp32-s3",
+    "esp32c3",
+    "esp32-c3",
+    "esp32c6",
+    "esp32-c6",
+}
 
 
 class FlashError(RuntimeError):
@@ -286,53 +298,142 @@ def update_flash(
     return result
 
 
-def touch_1200bps(port: str, settle_ms: int = 250) -> dict[str, Any]:
+def _do_1200bps_touch(port: str, settle_ms: int, touch_timeout_s: float = 3.0) -> None:
+    """Open port at 1200 baud and close, bounded by a worker thread.
+
+    Both the open and the close can block on a busy CDC device — we wrap the
+    whole thing in a worker so the caller returns in at most `touch_timeout_s`
+    regardless. The touch is signal-only: the USB configuration change to
+    1200 baud alone is enough to trip the Adafruit bootloader's reset, so a
+    worker that's still blocked in the background after timeout has already
+    delivered the signal.
+    """
+    import concurrent.futures
+
+    def _inner() -> None:
+        try:
+            s = serial.Serial(port, 1200)
+        except serial.SerialException as exc:
+            if "No such file" in str(exc) or "could not open" in str(exc).lower():
+                raise
+            return  # other serial errors mid-open are expected during DFU entry
+        try:
+            time.sleep(settle_ms / 1000.0)
+        finally:
+            try:
+                s.close()
+            except Exception:
+                pass
+
+    with concurrent.futures.ThreadPoolExecutor(max_workers=1) as pool:
+        future = pool.submit(_inner)
+        try:
+            future.result(timeout=touch_timeout_s)
+        except concurrent.futures.TimeoutError:
+            pass  # signal already delivered; worker thread leaks harmlessly
+
+
+# Adafruit nRF52 bootloader VID/PID (BOTH RAK4631 and most Feather nRF52 boards).
+# See https://github.com/adafruit/Adafruit_nRF52_Bootloader
+_NRF52_BOOTLOADER_VID = 0x239A
+_NRF52_BOOTLOADER_PIDS = {
+    0x0029,  # Adafruit nRF52 bootloader (generic, used by RAK4631)
+    0x002A,  # Adafruit Feather Express bootloader variant
+    0x4029,  # alt seen on some boards
+}
+
+
+def _find_nrf52_bootloader_port() -> dict[str, Any] | None:
+    """Return a dict for any currently-enumerated nRF52 bootloader port, or None."""
+    for d in devices.list_devices(include_unknown=True):
+        vid_str = d.get("vid")
+        pid_str = d.get("pid")
+        if vid_str is None or pid_str is None:
+            continue
+        try:
+            vid = int(vid_str, 16) if isinstance(vid_str, str) else int(vid_str)
+            pid = int(pid_str, 16) if isinstance(pid_str, str) else int(pid_str)
+        except ValueError:
+            continue
+        if vid == _NRF52_BOOTLOADER_VID and pid in _NRF52_BOOTLOADER_PIDS:
+            return d
+    return None
+
+
+def touch_1200bps(
+    port: str,
+    settle_ms: int = 250,
+    poll_timeout_s: float = 8.0,
+    retries: int = 2,
+) -> dict[str, Any]:
     """Open port at 1200 baud, close immediately — triggers USB CDC bootloader.
 
     Works for: nRF52840 (Adafruit bootloader), ESP32-S3 (native USB download
     mode), RP2040 (when built with 1200bps-reset stdio), Arduino Leonardo/Micro.
 
-    Afterward, polls `list_devices()` for up to 3 seconds to detect a new
-    bootloader port that replaced the original application port.
+    For nRF52 specifically: after the touch, polls for the Adafruit bootloader
+    VID/PID (0x239A / 0x0029) for up to `poll_timeout_s` seconds. Adafruit's
+    bootloader docs note a touch sometimes needs to be repeated, so this
+    retries up to `retries` times. The returned `new_port` is the bootloader
+    port (distinct from the app port) — exactly what's needed for `pio run
+    -t upload` to drive nrfutil.
+
+    For non-nRF52 devices (ESP32-S3, RP2040, Arduino), falls back to
+    "any-new-port appeared" detection.
+
+    Returns `{ok, former_port, new_port, new_port_vid_pid, attempts}`.
     """
-    before_ports = {d["port"] for d in devices.list_devices(include_unknown=True)}
+    before_list = devices.list_devices(include_unknown=True)
+    before_ports = {d["port"] for d in before_list}
 
-    try:
-        s = serial.Serial(port, 1200)
-        # Some drivers need a brief settle before close; others disconnect
-        # immediately when we set 1200 baud. Either is fine.
-        time.sleep(settle_ms / 1000.0)
-        try:
-            s.close()
-        except Exception:
-            pass
-    except serial.SerialException as exc:
-        # Many boards drop the port mid-open when 1200 is set; that's expected.
-        # Only treat "port doesn't exist" as a real error.
-        if "No such file" in str(exc) or "could not open" in str(exc).lower():
-            raise FlashError(f"Cannot open {port}: {exc}") from exc
+    attempts = 0
+    new_port_info: dict[str, Any] | None = None
 
-    # Poll for a new port appearing (bootloader) or the old one disappearing
-    deadline = time.monotonic() + 3.0
-    new_port: str | None = None
-    while time.monotonic() < deadline:
-        time.sleep(0.2)
-        current = {d["port"] for d in devices.list_devices(include_unknown=True)}
-        added = current - before_ports
-        if added:
-            # Prefer a likely-meshtastic port among the newly appeared ones.
-            current_list = devices.list_devices(include_unknown=True)
-            added_records = [d for d in current_list if d["port"] in added]
-            likely = next((d for d in added_records if d["likely_meshtastic"]), None)
-            new_port = (likely or added_records[0])["port"]
+    for attempt in range(1, retries + 1):
+        attempts = attempt
+        _do_1200bps_touch(port, settle_ms=settle_ms, touch_timeout_s=3.0)
+
+        # Poll for either (a) the nRF52 bootloader VID/PID appearing, or
+        # (b) a brand-new port appearing that wasn't there before.
+        deadline = time.monotonic() + poll_timeout_s
+        while time.monotonic() < deadline:
+            time.sleep(0.2)
+
+            bootloader = _find_nrf52_bootloader_port()
+            if bootloader is not None:
+                new_port_info = bootloader
+                break
+
+            current = devices.list_devices(include_unknown=True)
+            current_paths = {d["port"] for d in current}
+            added = current_paths - before_ports
+            if added:
+                added_record = next((d for d in current if d["port"] in added), None)
+                if added_record:
+                    new_port_info = added_record
+                    break
+
+        if new_port_info is not None:
             break
-        if port not in current:
-            # Old port went away entirely; bootloader may have shown up with a
-            # different name. Give it a moment more.
-            continue
+        # No bootloader appeared; try touching again (Adafruit recommends
+        # sometimes requiring two touches for reliability).
+
+    if new_port_info is not None:
+        return {
+            "ok": True,
+            "former_port": port,
+            "new_port": new_port_info["port"],
+            "new_port_vid_pid": (
+                new_port_info.get("vid"),
+                new_port_info.get("pid"),
+            ),
+            "attempts": attempts,
+        }
 
     return {
-        "ok": True,
+        "ok": False,
         "former_port": port,
-        "new_port": new_port,
+        "new_port": None,
+        "new_port_vid_pid": (None, None),
+        "attempts": attempts,
     }

mcp-server/src/meshtastic_mcp/hw_tools.py

@@ -13,7 +13,6 @@ from __future__ import annotations
 
 import re
 import subprocess
-import time
 from pathlib import Path
 from typing import Any, Sequence
 
@@ -34,26 +33,27 @@ def _run(
     timeout: float = _TIMEOUT_LONG,
     cwd: Path | None = None,
 ) -> dict[str, Any]:
-    t0 = time.monotonic()
+    # Shared with pio.run(): if `MESHTASTIC_MCP_FLASH_LOG` is set, each line
+    # of output is tee'd to that file as it arrives so the TUI can show live
+    # esptool/nrfutil/picotool progress instead of 3 minutes of silence.
+    full = [str(binary), *args]
     try:
-        proc = subprocess.run(
-            [str(binary), *args],
-            cwd=str(cwd) if cwd else None,
-            capture_output=True,
-            text=True,
+        rc, stdout, stderr, duration = pio._run_capturing(
+            full,
+            cwd=cwd,
             timeout=timeout,
+            tee_header=f"{binary.name} {' '.join(args)}",
         )
     except subprocess.TimeoutExpired as exc:
         raise ToolError(
             f"{binary.name} {' '.join(args)} timed out after {timeout}s"
         ) from exc
-    duration = time.monotonic() - t0
     return {
-        "exit_code": proc.returncode,
-        "stdout": proc.stdout or "",
-        "stderr": proc.stderr or "",
-        "stdout_tail": pio.tail_lines(proc.stdout or "", 200),
-        "stderr_tail": pio.tail_lines(proc.stderr or "", 200),
+        "exit_code": rc,
+        "stdout": stdout,
+        "stderr": stderr,
+        "stdout_tail": pio.tail_lines(stdout, 200),
+        "stderr_tail": pio.tail_lines(stderr, 200),
         "duration_s": round(duration, 2),
     }
 

mcp-server/src/meshtastic_mcp/pio.py

@@ -3,12 +3,27 @@
 Every PlatformIO interaction in this package funnels through `run()` so we
 have a single place that owns timeouts, buffer sizes, JSON parsing, and the
 "stderr on exit-0 is informational" convention.
+
+`run()` has two execution paths:
+
+* Fast path (default): `subprocess.run(capture_output=True)` — buffered, one
+  return; fine for sub-second pio calls like `pio --version` or
+  `pio project config --json-output`.
+* Streaming path: when the `MESHTASTIC_MCP_FLASH_LOG` env var is set, each
+  output line is tee'd to that file as it arrives via a threaded reader.
+  The TUI tails the file to give live flash progress — otherwise a 3-minute
+  `pio run -t upload` is completely silent to the operator.
+
+`hw_tools.py` shares the streaming helper via `pio._run_capturing()` so
+esptool/nrfutil/picotool output also streams when the env var is set.
 """
 
 from __future__ import annotations
 
 import json
+import os
 import subprocess
+import threading
 import time
 from dataclasses import dataclass
 from pathlib import Path
@@ -55,6 +70,143 @@ class PioResult:
     duration_s: float
 
 
+_FLASH_LOG_ENV = "MESHTASTIC_MCP_FLASH_LOG"
+
+
+def _flash_log_path() -> Path | None:
+    """Return the path to tee subprocess output to, or None if streaming off.
+
+    Controlled by `MESHTASTIC_MCP_FLASH_LOG`. `run-tests.sh` sets this to
+    `tests/flash.log`; the TUI tails that file so `pio run -t upload` shows
+    live progress in the pytest pane.
+    """
+    raw = os.environ.get(_FLASH_LOG_ENV)
+    if not raw:
+        return None
+    return Path(raw)
+
+
+def _run_capturing(
+    argv: Sequence[str],
+    *,
+    cwd: Path | None = None,
+    timeout: float | None = None,
+    tee_header: str | None = None,
+) -> tuple[int, str, str, float]:
+    """Run a subprocess, capture stdout+stderr, optionally tee to the flash log.
+
+    Returns `(returncode, stdout_str, stderr_str, duration_s)`. Raises
+    `subprocess.TimeoutExpired` on timeout (callers map this to their own
+    domain-specific error).
+
+    Fast path: `subprocess.run(capture_output=True)` when no flash log is
+    configured (unchanged behavior).
+
+    Streaming path: `Popen` with line-buffered stdout+stderr pipes; two
+    reader threads accumulate into result strings AND append each line to
+    the flash log file. Stdout and stderr stay separate in the return value
+    (so `stderr_tail` still means stderr), but are interleaved in the log
+    file in the order they arrived — that's what a human wants to read.
+    """
+    log_path = _flash_log_path()
+    t0 = time.monotonic()
+
+    if log_path is None:
+        # Fast path — unchanged.
+        proc = subprocess.run(
+            list(argv),
+            cwd=str(cwd) if cwd else None,
+            capture_output=True,
+            text=True,
+            timeout=timeout,
+        )
+        return (
+            proc.returncode,
+            proc.stdout or "",
+            proc.stderr or "",
+            time.monotonic() - t0,
+        )
+
+    # Streaming path: line-buffered Popen, threaded readers, tee to file.
+    # Ensure parent directory exists so the first tee write doesn't fail.
+    log_path.parent.mkdir(parents=True, exist_ok=True)
+    # Append mode: the TUI truncates on startup, the session may produce
+    # many tee'd commands (erase + flash + factory-reset response), and
+    # we want all of them chronologically in one log.
+    proc = subprocess.Popen(  # noqa: S603
+        list(argv),
+        cwd=str(cwd) if cwd else None,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.PIPE,
+        text=True,
+        bufsize=1,  # line-buffered
+    )
+    stdout_chunks: list[str] = []
+    stderr_chunks: list[str] = []
+    log_lock = threading.Lock()
+
+    def _append_log(line: str) -> None:
+        # Hold the lock briefly to serialize interleaved stdout/stderr writes
+        # so a half-written line from one stream doesn't get garbled by the
+        # other. The `with` + fsync-free write is ~µs per line, negligible.
+        with log_lock:
+            try:
+                with log_path.open("a", encoding="utf-8") as fh:
+                    fh.write(line)
+            except OSError:
+                # Log file disappeared (umount, operator deleted the dir).
+                # Don't let that bubble up — the subprocess output is still
+                # collected in-memory for the return value.
+                pass
+
+    def _tee(stream, sink: list[str]) -> None:
+        try:
+            for line in stream:
+                sink.append(line)
+                _append_log(line)
+        except Exception:
+            pass
+
+    # Header line so the operator can tell commands apart in the log.
+    if tee_header:
+        _append_log(f"\n--- {tee_header} (start)\n")
+
+    assert proc.stdout is not None and proc.stderr is not None
+    t_out = threading.Thread(
+        target=_tee, args=(proc.stdout, stdout_chunks), daemon=True
+    )
+    t_err = threading.Thread(
+        target=_tee, args=(proc.stderr, stderr_chunks), daemon=True
+    )
+    t_out.start()
+    t_err.start()
+
+    # `Popen.wait` with a timeout is the cleanest way to get TimeoutExpired.
+    try:
+        proc.wait(timeout=timeout)
+    except subprocess.TimeoutExpired:
+        proc.kill()
+        proc.wait()
+        # Drain readers before re-raising so we don't leave threads behind.
+        t_out.join(timeout=2)
+        t_err.join(timeout=2)
+        raise
+
+    t_out.join()
+    t_err.join()
+    duration = time.monotonic() - t0
+
+    if tee_header:
+        _append_log(f"--- {tee_header} (exit {proc.returncode} in {duration:.1f}s)\n")
+
+    return (
+        proc.returncode,
+        "".join(stdout_chunks),
+        "".join(stderr_chunks),
+        duration,
+    )
+
+
 def run(
     args: Sequence[str],
     *,
@@ -66,28 +218,28 @@ def run(
 
     `cwd` defaults to the firmware root. `check=True` raises `PioError` on
     non-zero exit; set `check=False` to inspect `returncode` manually.
+
+    If `MESHTASTIC_MCP_FLASH_LOG` is set, output is also tee'd to that file
+    line-by-line as it arrives (for live flash progress in the TUI).
     """
     binary = str(config.pio_bin())
     work_dir = cwd or config.firmware_root()
     full = [binary, *args]
-    t0 = time.monotonic()
     try:
-        proc = subprocess.run(
+        rc, stdout, stderr, duration = _run_capturing(
             full,
-            cwd=str(work_dir),
-            capture_output=True,
-            text=True,
+            cwd=work_dir,
             timeout=timeout,
+            tee_header=f"pio {' '.join(args)}",
         )
     except subprocess.TimeoutExpired as exc:
         raise PioTimeout(f"pio {' '.join(args)} timed out after {timeout}s") from exc
-    duration = time.monotonic() - t0
 
     result = PioResult(
         args=list(args),
-        returncode=proc.returncode,
-        stdout=proc.stdout or "",
-        stderr=proc.stderr or "",
+        returncode=rc,
+        stdout=stdout,
+        stderr=stderr,
         duration_s=duration,
     )
     if check and result.returncode != 0:

mcp-server/src/meshtastic_mcp/server.py

@@ -446,6 +446,26 @@ def set_channel_url(url: str, port: str | None = None) -> dict[str, Any]:
     return admin.set_channel_url(url=url, port=port)
 
 
+@app.tool()
+def set_debug_log_api(enabled: bool, port: str | None = None) -> dict[str, Any]:
+    """Toggle security.debug_log_api_enabled on the local node.
+
+    When true, firmware streams log lines as protobuf `LogRecord` messages
+    over the StreamAPI (topic `meshtastic.log.line` in meshtastic-python)
+    instead of raw text. Lets diagnostic clients capture firmware-side logs
+    through the SAME SerialInterface used for admin/info calls — no
+    separate `pio device monitor` session needed, no exclusive-port-lock
+    conflict. Persists across reboot via NVS; wiped by factory_reset
+    unless re-applied.
+
+    The earlier emitLogRecord race (shared tx buffer) is fixed at the
+    firmware level — the log path has a dedicated scratch + txBuf and
+    both emission paths serialize via a mutex. Safe to leave on under
+    traffic.
+    """
+    return admin.set_debug_log_api(enabled=enabled, port=port)
+
+
 @app.tool()
 def send_text(
     text: str,