From 4630ca4e330c174d77f97813343148879a8ad829 Mon Sep 17 00:00:00 2001 From: Alex Date: Tue, 11 Aug 2026 20:29:35 +0000 Subject: [PATCH 1/4] Do not let a console message kill the radio model The radio model treated any engine message it did not recognise as a desynchronised stream and exited. kConsoleIn (0x06) is sent to every node the moment anything touches the fleet console, so an emulated node died seconds after boot and reported ERROR: radio init failed: -2 which is RadioLib for "chip not found" and points squarely at wiring. It took a packet capture of the socket to see that the chip was fine and the model had gone. Console input belongs to the firmware's serial port, and an emulated node's serial port is the emulator's rather than this socket, so ignoring it is the whole handling. Unknown kinds are now skipped rather than fatal: the framing is length-prefixed and the payload has already been read, so an unrecognised message cannot desynchronise anything - whereas exiting takes the node down for something it did not need. This file was only ever a working copy; it is committed here with the fix. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01TGXGCDQQHLL9Bqo9GBsyrw --- bridge/radioserver.cpp | 321 +++++++++++++++++++++++++++++++++-------- 1 file changed, 257 insertions(+), 64 deletions(-) diff --git a/bridge/radioserver.cpp b/bridge/radioserver.cpp index 964fc26..34ef14b 100644 --- a/bridge/radioserver.cpp +++ b/bridge/radioserver.cpp @@ -1,24 +1,26 @@ -// The radio model, offered on a socket for an emulated MCU to clock. +// The radio model, between an emulated MCU and the RF engine. // -// A native node calls VirtualSX1262 in process, through SimHal. An emulated one -// cannot: the firmware is inside QEMU, and its SPI controller reaches out over -// a socket to whatever is modelling the chip. This is that end. +// A native node reaches VirtualSX1262 in process through SimHal. An emulated one +// cannot: its firmware is inside QEMU, so the chip has to sit outside and be +// reachable from both sides at once. // -// It is deliberately the same chip object either way. Writing a second model -// for the emulated path would give two things to keep in agreement, and the -// first time they drifted every comparison between a native node and an -// emulated one would be measuring our own code rather than MeshCore's. +// QEMU --- SPI transactions ---> this --- frames and ticks ---> engine // -// The protocol is the one QEMU's sx1262 device speaks, and it is small because -// it sits on the hot path of every SPI byte: +// Deliberately the same chip object as the native path. A second model would be +// a second thing to keep in agreement, and the first time the two drifted every +// comparison between a native node and an emulated one would be measuring our +// own code rather than MeshCore's. // -// 0x01 chip select asserted -> beginTransaction() -// 0x02 chip select released -> endTransaction() -// 0x03 one byte out, one back -> transferByte() -// 0x04 read the BUSY line -> one byte, 0 or 1 +// One thing is different from a native node and it matters. There, the bridge +// owns the firmware's execution: a tick runs loop() a millisecond at a time and +// nothing else happens in between. Here the firmware runs inside an emulator on +// its own schedule, so SPI transactions arrive whenever QEMU feels like it, +// while ticks arrive from the engine. Both mutate the chip, so both take a lock, +// and the ordering between them is not reproducible the way a native node's is. +// See the note at the bottom. // // Usage: -// radioserver /run/user/1000/meshbench-radio-7.sock +// radioserver [--bridge host:port] #include "VirtualSX1262.h" @@ -26,14 +28,22 @@ #include #include #include +#include #include +#include +#include +#include +#include +#include +#include #include #include #include namespace { +// The QEMU side. Small because it is on the hot loop of every SPI byte. enum : uint8_t { kCsAssert = 0x01, kCsRelease = 0x02, @@ -41,7 +51,22 @@ enum : uint8_t { kReadBusy = 0x04, }; -// Read exactly n bytes, or say the peer has gone. +// The engine side, shared with the simulator's Go half and with bridge/main.cpp. +constexpr uint8_t kFrame = 0x01; +constexpr uint8_t kTick = 0x02; +constexpr uint8_t kAck = 0x03; +constexpr uint8_t kTxDone = 0x04; +// Console traffic reaches an emulated node over the emulator's own serial +// port, so these two are named here only to be ignored deliberately rather +// than to fall through to the unknown-message path. +constexpr uint8_t kConsoleIn = 0x06; +constexpr uint8_t kChannelBusy = 0x08; +constexpr uint8_t kRadioStats = 0x09; + +VirtualSX1262 gChip; +std::mutex gChipMu; // QEMU and the engine both reach the chip +uint32_t gSimMillis = 0; + bool readAll(int fd, void* buf, size_t n) { auto* p = static_cast(buf); while (n > 0) { @@ -64,19 +89,191 @@ bool writeAll(int fd, const void* buf, size_t n) { return true; } +bool writeMsg(int fd, uint8_t kind, const uint8_t* p, size_t n) { + uint8_t hdr[3] = {kind, (uint8_t)(n >> 8), (uint8_t)n}; + if (!writeAll(fd, hdr, 3)) return false; + return n == 0 || writeAll(fd, p, n); +} + +// Anything the firmware handed its radio goes out to the engine now. +// +// Transmission reaches the channel immediately and is *not* immediately +// complete: the chip stays in transmit until the engine sends kTxDone, exactly +// as a native node does, because that is what stops a node talking over itself. +void drainTx(int bridgeFd) { + if (bridgeFd < 0 || !gChip.hasPendingTx) return; + gChip.hasPendingTx = false; + writeMsg(bridgeFd, kFrame, gChip.pendingTx.data(), gChip.pendingTx.size()); +} + +int dialBridge(const std::string& addr) { + auto colon = addr.rfind(':'); + if (colon == std::string::npos) { + fprintf(stderr, "radioserver: --bridge wants host:port, got %s\n", addr.c_str()); + return -1; + } + std::string host = addr.substr(0, colon), port = addr.substr(colon + 1); + + addrinfo hints{}; + hints.ai_family = AF_UNSPEC; + hints.ai_socktype = SOCK_STREAM; + addrinfo* res = nullptr; + if (getaddrinfo(host.c_str(), port.c_str(), &hints, &res) != 0) { + fprintf(stderr, "radioserver: cannot resolve %s\n", addr.c_str()); + return -1; + } + int fd = -1; + for (addrinfo* a = res; a; a = a->ai_next) { + fd = ::socket(a->ai_family, a->ai_socktype, a->ai_protocol); + if (fd < 0) continue; + if (::connect(fd, a->ai_addr, a->ai_addrlen) == 0) break; + ::close(fd); + fd = -1; + } + freeaddrinfo(res); + if (fd < 0) { + fprintf(stderr, "radioserver: cannot reach the engine at %s\n", addr.c_str()); + return -1; + } + // Frames are small and latency is the whole game here: a tick that waits on + // Nagle is a node that answers late for no reason. + int one = 1; + ::setsockopt(fd, IPPROTO_TCP, TCP_NODELAY, &one, sizeof(one)); + return fd; +} + +// One message from the engine. +bool serviceBridge(int fd) { + uint8_t hdr[3]; + if (!readAll(fd, hdr, 3)) return false; + const uint8_t kind = hdr[0]; + const size_t n = ((size_t)hdr[1] << 8) | hdr[2]; + std::vector payload(n); + if (n && !readAll(fd, payload.data(), n)) return false; + + std::lock_guard lock(gChipMu); + switch (kind) { + case kFrame: + // A packet the channel delivered. Only CRC-passing frames arrive here, + // exactly as on hardware: everything else was recorded and withheld. + gChip.inbox.push_back(std::move(payload)); + break; + + case kTxDone: + gChip.transmitFinished(); + break; + + case kChannelBusy: + if (n >= 1) gChip.setChannelBusy(payload[0] != 0); + break; + + case kTick: { + if (n != 4) break; + uint32_t at = ((uint32_t)payload[0] << 24) | ((uint32_t)payload[1] << 16) | + ((uint32_t)payload[2] << 8) | payload[3]; + // A millisecond at a time, as a native node does. Stepping rather than + // jumping is what keeps the chip's own timeouts behaving: a preamble flag + // that should clear after 66 ms does not, if time arrives in 500 ms + // lumps. + while (gSimMillis < at) { + gSimMillis++; + gChip.tick(gSimMillis); + drainTx(fd); + } + gChip.tick(gSimMillis); + drainTx(fd); + + uint32_t st[4] = {gChip.irqReads(), gChip.busyReads(), gChip.busyMs(), + gChip.spuriousRaises()}; + uint8_t sb[16]; + for (int k = 0; k < 4; k++) { + sb[k * 4 + 0] = (uint8_t)(st[k] >> 24); + sb[k * 4 + 1] = (uint8_t)(st[k] >> 16); + sb[k * 4 + 2] = (uint8_t)(st[k] >> 8); + sb[k * 4 + 3] = (uint8_t)st[k]; + } + writeMsg(fd, kRadioStats, sb, sizeof(sb)); + if (!writeMsg(fd, kAck, payload.data(), 4)) return false; + break; + } + + case kConsoleIn: + // Console input belongs to the firmware's serial port, and an emulated + // node's serial port is the emulator's, not this socket. Ignoring it is + // the whole handling: what matters is that it is not fatal. Treating it + // as unknown killed the radio model the moment anything typed at the + // fleet, and the node then reported "radio init failed: -2" - chip not + // found - which points at wiring rather than at a console. + break; + + default: + // Skipped rather than fatal. The framing is length-prefixed and the + // payload has already been read, so an unrecognised kind costs nothing + // and cannot desynchronise the stream - whereas exiting takes the node + // down for a message it did not need. + fprintf(stderr, "radioserver: ignoring engine message 0x%02x (%zu bytes)\n", + kind, n); + break; + } + return true; +} + +// One message from the emulator. +bool serviceQemu(int fd, uint64_t* transactions, uint64_t* bytes) { + uint8_t tag = 0; + if (!readAll(fd, &tag, 1)) return false; + + std::lock_guard lock(gChipMu); + switch (tag) { + case kCsAssert: + gChip.beginTransaction(); + return true; + + case kCsRelease: + gChip.endTransaction(); + (*transactions)++; + return true; + + case kXfer: { + uint8_t out = 0; + if (!readAll(fd, &out, 1)) return false; + uint8_t in = gChip.transferByte(out); + (*bytes)++; + return writeAll(fd, &in, 1); + } + + case kReadBusy: { + // Always clear, which is what the native path does too: SimHal holds BUSY + // low and VirtualSX1262 does not model the time a real chip spends + // digesting a command. Answering differently here would make an emulated + // node a different radio from a native one, which is the one thing this + // whole arrangement exists to avoid. + uint8_t busy = 0; + return writeAll(fd, &busy, 1); + } + + default: + fprintf(stderr, "radioserver: unknown emulator tag 0x%02x\n", tag); + return false; + } +} + } // namespace int main(int argc, char** argv) { if (argc < 2) { - fprintf(stderr, "usage: %s \n", argv[0]); + fprintf(stderr, "usage: %s [--bridge host:port]\n", argv[0]); return 2; } const char* path = argv[1]; + std::string bridgeAddr; + for (int i = 2; i < argc - 1; i++) { + if (strcmp(argv[i], "--bridge") == 0) bridgeAddr = argv[i + 1]; + } // A broken pipe is an emulator that has exited, which is ordinary. Let the // read fail and tidy up rather than dying on a signal. ::signal(SIGPIPE, SIG_IGN); - ::unlink(path); int srv = ::socket(AF_UNIX, SOCK_STREAM, 0); @@ -91,7 +288,6 @@ int main(int argc, char** argv) { return 1; } strncpy(addr.sun_path, path, sizeof(addr.sun_path) - 1); - if (::bind(srv, (sockaddr*)&addr, sizeof(addr)) < 0) { perror("bind"); return 1; @@ -103,10 +299,23 @@ int main(int argc, char** argv) { printf("radioserver: listening on %s\n", path); fflush(stdout); - VirtualSX1262 chip; + int bridgeFd = -1; + if (!bridgeAddr.empty()) { + bridgeFd = dialBridge(bridgeAddr); + if (bridgeFd < 0) return 1; + printf("radioserver: joined the engine at %s\n", bridgeAddr.c_str()); + fflush(stdout); + } else { + // Worth saying. Without the engine this chip transmits into nowhere and + // never receives, so the firmware comes up and then waits for ever on a + // transmission that cannot complete - which looks like a hang rather than + // like a missing argument. + printf("radioserver: no --bridge, so this node is deaf and mute\n"); + fflush(stdout); + } - int fd = ::accept(srv, nullptr, nullptr); - if (fd < 0) { + int qemuFd = ::accept(srv, nullptr, nullptr); + if (qemuFd < 0) { perror("accept"); return 1; } @@ -114,62 +323,46 @@ int main(int argc, char** argv) { fflush(stdout); uint64_t transactions = 0, bytes = 0; - for (;;) { - uint8_t tag = 0; - if (!readAll(fd, &tag, 1)) break; + pollfd fds[2]; + int n = 0; + fds[n++] = {qemuFd, POLLIN, 0}; + if (bridgeFd >= 0) fds[n++] = {bridgeFd, POLLIN, 0}; - switch (tag) { - case kCsAssert: - chip.beginTransaction(); - break; + if (::poll(fds, n, -1) < 0) break; - case kCsRelease: - chip.endTransaction(); - transactions++; - break; - - case kXfer: { - uint8_t out = 0; - if (!readAll(fd, &out, 1)) goto done; - uint8_t in = chip.transferByte(out); - bytes++; - if (!writeAll(fd, &in, 1)) goto done; - break; - } - - case kReadBusy: { - // Never busy for now. BUSY is asserted by the chip while it digests a - // command, and modelling that needs the simulated clock this process - // does not yet have - see the note below about time. - uint8_t busy = 0; - if (!writeAll(fd, &busy, 1)) goto done; + if (fds[0].revents & (POLLIN | POLLHUP)) { + if (!serviceQemu(qemuFd, &transactions, &bytes)) break; + } + if (bridgeFd >= 0 && (fds[1].revents & (POLLIN | POLLHUP))) { + if (!serviceBridge(bridgeFd)) { + fprintf(stderr, "radioserver: the engine went away\n"); break; } - - default: - fprintf(stderr, "radioserver: unknown tag 0x%02x; the stream has " - "desynchronised, closing\n", tag); - goto done; } } -done: printf("radioserver: %llu transactions, %llu bytes\n", (unsigned long long)transactions, (unsigned long long)bytes); - ::close(fd); + if (bridgeFd >= 0) ::close(bridgeFd); + ::close(qemuFd); ::close(srv); ::unlink(path); return 0; } -// Not here yet, and both are the same missing thing: simulated time. +// What this is not, and it is worth being exact. // -// * BUSY always reads clear. A real chip raises it while it works, and the -// driver waits on it. Answering truthfully means knowing what time it is. -// * Nothing connects this chip to the RF engine, so it transmits into -// nowhere and never receives. VirtualSX1262 already has pendingTx and an -// inbox for exactly that; they need the bridge on the other side. +// A native node runs in lockstep: the engine supplies the clock, the bridge runs +// loop() one millisecond at a time, and the same seed gives the same answer +// every time. Here the engine still supplies the clock to the *chip*, so IRQ +// timing and channel state are engine-relative - but the *firmware* runs inside +// an emulator on wall time, so the instant at which it reads a register is not +// reproducible. // -// Both arrive with the lockstep link, which is what gives an emulated node the -// same clock every native node already runs on. +// The consequence is narrow and real: an emulated node can transmit and receive +// and take part in a mesh, and two runs of one seed will not produce identical +// ledgers. Mixing emulated and native nodes in a measurement therefore costs the +// determinism the native path has. Fixing it means QEMU's -icount with the +// engine driving virtual time, which is the same contract the native bridge +// already implements. From f6b9ca76e10a8566aa8d10bdc1b17e2a12e7b3b4 Mon Sep 17 00:00:00 2001 From: Alex Date: Tue, 11 Aug 2026 21:05:29 +0000 Subject: [PATCH 2/4] Let the radio model listen on TCP as well as a socket path Renode runs on Mono, whose Unix domain socket support has been unreliable for long enough that betting an emulated node on it is a poor trade for one path separator. A leading colon asks for TCP on loopback, and port 0 prints the port it was given - which is what a harness starting several nodes at once needs, rather than picking a number and hoping. QEMU keeps the socket path it already used. Nothing else changes: both emulators reach the same VirtualSX1262 a native node reaches in process. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01TGXGCDQQHLL9Bqo9GBsyrw --- bridge/radioserver.cpp | 80 ++++++++++++++++++++++++++++++------------ 1 file changed, 58 insertions(+), 22 deletions(-) diff --git a/bridge/radioserver.cpp b/bridge/radioserver.cpp index 34ef14b..f0922b9 100644 --- a/bridge/radioserver.cpp +++ b/bridge/radioserver.cpp @@ -274,29 +274,65 @@ int main(int argc, char** argv) { // A broken pipe is an emulator that has exited, which is ordinary. Let the // read fail and tidy up rather than dying on a signal. ::signal(SIGPIPE, SIG_IGN); - ::unlink(path); - int srv = ::socket(AF_UNIX, SOCK_STREAM, 0); - if (srv < 0) { - perror("socket"); - return 1; - } - sockaddr_un addr{}; - addr.sun_family = AF_UNIX; - if (strlen(path) >= sizeof(addr.sun_path)) { - fprintf(stderr, "radioserver: socket path too long: %s\n", path); - return 1; - } - strncpy(addr.sun_path, path, sizeof(addr.sun_path) - 1); - if (::bind(srv, (sockaddr*)&addr, sizeof(addr)) < 0) { - perror("bind"); - return 1; - } - if (::listen(srv, 1) < 0) { - perror("listen"); - return 1; + // Two ways in, because there are two emulators. QEMU is native and takes a + // Unix socket; Renode runs on Mono, whose Unix domain socket support has been + // unreliable for long enough that betting an emulated node on it is a poor + // trade for one path separator. A leading colon asks for TCP on loopback. + const bool useTcp = path[0] == ':'; + int srv = -1; + if (useTcp) { + const int port = atoi(path + 1); + srv = ::socket(AF_INET, SOCK_STREAM, 0); + if (srv < 0) { + perror("socket"); + return 1; + } + int on = 1; + ::setsockopt(srv, SOL_SOCKET, SO_REUSEADDR, &on, sizeof(on)); + sockaddr_in in{}; + in.sin_family = AF_INET; + in.sin_addr.s_addr = htonl(INADDR_LOOPBACK); + in.sin_port = htons((uint16_t)port); + if (::bind(srv, (sockaddr*)&in, sizeof(in)) < 0) { + perror("bind"); + return 1; + } + if (::listen(srv, 1) < 0) { + perror("listen"); + return 1; + } + // The chosen port is printed because port 0 means "any", which is what a + // harness starting several nodes at once wants: it reads the number back + // rather than picking one and hoping. + socklen_t len = sizeof(in); + if (::getsockname(srv, (sockaddr*)&in, &len) == 0) { + printf("radioserver: listening on 127.0.0.1:%d\n", ntohs(in.sin_port)); + } + } else { + ::unlink(path); + srv = ::socket(AF_UNIX, SOCK_STREAM, 0); + if (srv < 0) { + perror("socket"); + return 1; + } + sockaddr_un addr{}; + addr.sun_family = AF_UNIX; + if (strlen(path) >= sizeof(addr.sun_path)) { + fprintf(stderr, "radioserver: socket path too long: %s\n", path); + return 1; + } + strncpy(addr.sun_path, path, sizeof(addr.sun_path) - 1); + if (::bind(srv, (sockaddr*)&addr, sizeof(addr)) < 0) { + perror("bind"); + return 1; + } + if (::listen(srv, 1) < 0) { + perror("listen"); + return 1; + } + printf("radioserver: listening on %s\n", path); } - printf("radioserver: listening on %s\n", path); fflush(stdout); int bridgeFd = -1; @@ -347,7 +383,7 @@ int main(int argc, char** argv) { if (bridgeFd >= 0) ::close(bridgeFd); ::close(qemuFd); ::close(srv); - ::unlink(path); + if (!useTcp) ::unlink(path); return 0; } From e399a752cafcfa2cecb3ba2925eb27129c3caa60 Mon Sep 17 00:00:00 2001 From: Alex Date: Wed, 12 Aug 2026 00:17:13 +0000 Subject: [PATCH 3/4] Answer whether DIO1 is asserted QEMU never needed it - the ESP32 firmware polls the chip's IRQ register over SPI - but an nRF52 waits on the pin, and a pin nothing drives is a node that configures its radio and then sits there. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01TGXGCDQQHLL9Bqo9GBsyrw --- bridge/radioserver.cpp | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/bridge/radioserver.cpp b/bridge/radioserver.cpp index f0922b9..a7f4f29 100644 --- a/bridge/radioserver.cpp +++ b/bridge/radioserver.cpp @@ -49,6 +49,11 @@ enum : uint8_t { kCsRelease = 0x02, kXfer = 0x03, kReadBusy = 0x04, + // Whether DIO1 is asserted. QEMU never needed this - the ESP32 firmware + // polls the chip's IRQ register over SPI - but an nRF52 waits on the pin, + // and a pin nothing drives is a node that configures its radio and then + // sits there for ever. + kReadIrq = 0x05, }; // The engine side, shared with the simulator's Go half and with bridge/main.cpp. @@ -242,6 +247,11 @@ bool serviceQemu(int fd, uint64_t* transactions, uint64_t* bytes) { return writeAll(fd, &in, 1); } + case kReadIrq: { + uint8_t irq = gChip.irqAsserted() ? 1 : 0; + return writeAll(fd, &irq, 1); + } + case kReadBusy: { // Always clear, which is what the native path does too: SimHal holds BUSY // low and VirtualSX1262 does not model the time a real chip spends From b3bc83c7385143998e10d682a18af1ca07ce798a Mon Sep 17 00:00:00 2001 From: Alex Date: Wed, 12 Aug 2026 00:27:49 +0000 Subject: [PATCH 4/4] Trace SPI transactions behind an environment variable The same chip serves a native node, an emulated ESP32 and an emulated nRF52, so when one of them will not bring its radio up, a diff of the three traces says which command got an answer it did not like. That is how the nRF52 chip-select fault was found: 3,320 bytes and zero transactions. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01TGXGCDQQHLL9Bqo9GBsyrw --- bridge/radioserver.cpp | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/bridge/radioserver.cpp b/bridge/radioserver.cpp index a7f4f29..8435f9d 100644 --- a/bridge/radioserver.cpp +++ b/bridge/radioserver.cpp @@ -69,6 +69,10 @@ constexpr uint8_t kChannelBusy = 0x08; constexpr uint8_t kRadioStats = 0x09; VirtualSX1262 gChip; +// MESHCORE_RADIO_TRACE=1 logs every SPI transaction. Off by default: this is +// on the hot path of every byte. +const bool gTracing = getenv("MESHCORE_RADIO_TRACE") != nullptr; +std::vector gTrace; std::mutex gChipMu; // QEMU and the engine both reach the chip uint32_t gSimMillis = 0; @@ -232,11 +236,25 @@ bool serviceQemu(int fd, uint64_t* transactions, uint64_t* bytes) { switch (tag) { case kCsAssert: gChip.beginTransaction(); + gTrace.clear(); return true; case kCsRelease: gChip.endTransaction(); (*transactions)++; + // One line per SPI transaction, opcode first. The point is comparison: + // the same chip serves a native node, an emulated ESP32 and an emulated + // nRF52, so when one of them fails to bring its radio up, a diff of the + // three traces says which command got an answer it did not like. + if (gTracing && !gTrace.empty()) { + fprintf(stderr, "spi:"); + for (size_t i = 0; i < gTrace.size() && i < 24; i++) { + fprintf(stderr, " %02x", gTrace[i]); + } + if (gTrace.size() > 24) fprintf(stderr, " ...(%zu)", gTrace.size()); + fprintf(stderr, "\n"); + fflush(stderr); + } return true; case kXfer: { @@ -244,6 +262,7 @@ bool serviceQemu(int fd, uint64_t* transactions, uint64_t* bytes) { if (!readAll(fd, &out, 1)) return false; uint8_t in = gChip.transferByte(out); (*bytes)++; + if (gTracing) gTrace.push_back(out); return writeAll(fd, &in, 1); }