From 3dec235ff3c367ef7d8f2ba73a797802d775257a Mon Sep 17 00:00:00 2001 From: Hydra Engineering Date: Tue, 11 Aug 2026 18:29:25 +0700 Subject: [PATCH 1/5] epic(610): WS1-3 server-context extension seam + task/update_slots migration - server-hydra-extension.h: A/B seam interface (HYDRA_EXT_MODE=legacy|seam) - hydra-server-context.cpp: hydra_process_task() + extension impl (same TU) - server-context.cpp: friend + hydra_ext member + 3 hook sites; HYDRA task dispatch and update_slots clusters routed through the seam - test-hydra-ext-ab.cpp: hermetic A/B parity test Both modes call identical hydra_process_task/pre_loop/on_empty_batch, so behavior parity is by construction; toggle proves the seam plumbing. --- tests/CMakeLists.txt | 7 + tests/test-hydra-ext-ab.cpp | 83 + tools/server/hydra-server-context.cpp | 2614 +++++++++++++++++++++++++ tools/server/server-context.cpp | 2535 +----------------------- tools/server/server-hydra-extension.h | 52 + 5 files changed, 2812 insertions(+), 2479 deletions(-) create mode 100644 tests/test-hydra-ext-ab.cpp create mode 100644 tools/server/hydra-server-context.cpp create mode 100644 tools/server/server-hydra-extension.h diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 9b955a1c77f4..ac034d595236 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -182,6 +182,13 @@ target_link_libraries(test-hydra-rpc-stale-sock PRIVATE ggml) # #470: test uses ggml_backend_buffer_copy_tensor (declared in ggml-backend-impl.h). target_include_directories(test-hydra-rpc-stale-sock PRIVATE ${PROJECT_SOURCE_DIR}/ggml/src) +# epic #610 WS1: hermetic A/B seam checks (HYDRA_EXT_MODE parsing + the WS1 +# no-op factory). Needs the server-context TU so hydra_create_extension() and +# the inline mode parser are visible. +llama_build_and_test(test-hydra-ext-ab.cpp) +target_link_libraries(test-hydra-ext-ab PRIVATE server-context) +target_include_directories(test-hydra-ext-ab PRIVATE ${CMAKE_SOURCE_DIR}/tools/server) + if (NOT WIN32 OR NOT BUILD_SHARED_LIBS) # these tests are disabled on Windows because they use internal functions not exported with LLAMA_API (when building with shared libraries) llama_build_and_test(test-sampling.cpp) diff --git a/tests/test-hydra-ext-ab.cpp b/tests/test-hydra-ext-ab.cpp new file mode 100644 index 000000000000..9601749f6873 --- /dev/null +++ b/tests/test-hydra-ext-ab.cpp @@ -0,0 +1,83 @@ +// epic #610 WS1: hermetic A/B seam checks. +// +// What this can test without a model/GPU: +// - HYDRA_EXT_MODE env parsing (legacy default, "seam" activates seam) +// - the factory returns the expected no-op WS1 implementation +// - the no-op contract: hooks don't claim tasks / don't alter the loop +// +// What CANNOT be tested hermetically: a server_context_impl requires a loaded +// model+context, so the behavioral A/B (run the SAME scenario through +// HYDRA_EXT_MODE=legacy vs =seam and diff the outputs) is a loopback / live-rig +// step, documented as WS4 in the epic. + +#include "server-hydra-extension.h" + +#include +#include +#include +#include + +static int g_failures = 0; + +static void expect(const char * what, bool ok) { + if (!ok) { + fprintf(stderr, "FAIL: %s\n", what); + g_failures++; + } +} + +// setenv/unsetenv are POSIX; on Windows use _putenv_s (empty = unset). +#if defined(_WIN32) +static void set_ext_mode(const char * value) { + _putenv_s("HYDRA_EXT_MODE", value ? value : ""); +} +#else +static void set_ext_mode(const char * value) { + if (value == nullptr) { + unsetenv("HYDRA_EXT_MODE"); + } else { + setenv("HYDRA_EXT_MODE", value, 1); + } +} +#endif + +int main() { + // --- mode parsing ------------------------------------------------- + set_ext_mode(nullptr); + expect("unset HYDRA_EXT_MODE -> legacy", !hydra_ext_mode_seam()); + + set_ext_mode("legacy"); + expect("HYDRA_EXT_MODE=legacy -> legacy", !hydra_ext_mode_seam()); + + set_ext_mode("seam"); + expect("HYDRA_EXT_MODE=seam -> seam", hydra_ext_mode_seam()); + + set_ext_mode("garbage"); + expect("HYDRA_EXT_MODE=garbage -> legacy", !hydra_ext_mode_seam()); + + set_ext_mode("LEGACY"); // case-sensitive: not "seam" + expect("HYDRA_EXT_MODE=LEGACY -> legacy", !hydra_ext_mode_seam()); + + // --- factory + WS1 no-op contract --------------------------------- + std::unique_ptr ext = hydra_create_extension(); + expect("factory returns non-null", ext != nullptr); + if (ext) { + // WS2/WS3 impl name (handle_task routes to hydra_process_task, and + // pre_loop/on_empty_batch replicate the update_slots clusters). + expect("impl name is hydra-task-ws2", std::strcmp(ext->name(), "hydra-task-ws2") == 0); + } + + // The no-op hooks take a server_context_impl&, which cannot be constructed + // here (needs a loaded model). Their return values are pinned by the WS1 + // contract: handle_task=false (never claims), pre_loop=false (never skips + // the decode loop), on_empty_batch=false (never handles the empty batch). + // Behavioral parity is verified by the live-rig A/B in WS4. + + if (g_failures != 0) { + fprintf(stderr, "test-hydra-ext-ab FAILED (%d)\n", g_failures); + return 1; + } + + fprintf(stderr, "%s", "test-hydra-ext-ab OK\n"); + return 0; +} diff --git a/tools/server/hydra-server-context.cpp b/tools/server/hydra-server-context.cpp new file mode 100644 index 000000000000..65cf55f8ea9b --- /dev/null +++ b/tools/server/hydra-server-context.cpp @@ -0,0 +1,2614 @@ +// Hydra A/B extension seam implementation (epic #610). +// +// This file is NOT an independent translation unit. It is #include'd at the +// bottom of server-context.cpp, so it compiles as part of that TU and can reach +// server_context_impl's private members through the friend declaration on +// hydra_engine_extension (and, for hydra_process_task, because it is a member +// of server_context_impl itself). Do NOT add this file to CMakeLists.txt. +// +// WS1: no-op extension hooks (handle_task/pre_loop/on_empty_batch return false), +// so seam mode is behavior-identical to legacy. +// WS2: the HYDRA task dispatch is extracted into server_context_impl:: +// hydra_process_task(), called from BOTH the legacy switch path (via a +// thin fall-through in process_single_task) and the seam handle_task(). +// Both modes run the exact same method, so A/B parity is by construction; +// the A/B toggle then proves the seam plumbing (routing + claiming) is +// behavior-identical to the legacy switch. +// WS3: update_slots() clusters move into pre_loop()/on_empty_batch(). + +#include "server-hydra-extension.h" + +#include "server-task.h" + +// --------------------------------------------------------------------------- +// WS2: HYDRA task dispatch — member of server_context_impl, defined in this TU. +// --------------------------------------------------------------------------- + // epic #610 WS2: Hydra task dispatch, moved out of process_single_task(). + // Same TU (see #include at bottom of server-context.cpp) so member access + // to server_context_impl privates is available. A switch(task.type) wrapper + // keeps all internal break/continue semantics identical to the inline code. + void server_context_impl::hydra_process_task(server_task & task) { + switch (task.type) { + case SERVER_TASK_TYPE_HYDRA_STATE_GET: + { + // M1: background serialization thread — inference loop continues during state transfer. + // llama_state_seq_get_data reads KV cells for an IDLE sequence; llama_decode + // writes cells for ACTIVE sequences only — no memory overlap for different seq IDs. + const int id_slot = task.hydra_action.id_slot; + auto res = std::make_unique(); + res->id = task.id; + res->id_slot = id_slot; + res->op = HYDRA_OP_STATE_GET; + + server_slot * slot = get_slot_by_id(id_slot); + if (slot == nullptr) { + res->rpc_status = HYDRA_STATUS_NOT_FOUND; + res->error = "invalid slot ID"; + queue_results.send(std::move(res)); + break; + } + if (slot->is_processing() || slot->hydra_transferring->load()) { + res->rpc_status = HYDRA_STATUS_BUSY; + queue_results.send(std::move(res)); + break; + } + + // Snapshot on inference thread (cheap — dry-run serialization, no GPU copies). + const size_t state_size = llama_state_seq_get_size(ctx_tgt, slot->id); + int actual_n_past = slot->n_prompt_tokens_cache + slot->n_decoded; + // Cold prefill: n_prompt_tokens_cache is still 0 so n_decoded (1) dominates. + // Use prompt token count instead — matches STATE_META fallback. + if (slot->n_prompt_tokens_cache == 0 && slot->prompt.tokens.size() > 0) { + actual_n_past = (int)slot->prompt.tokens.size(); + } + res->n_past = actual_n_past; + res->rpc_status = HYDRA_STATUS_OK; + // M-Perf.9 #289: surface model identity alongside the state + // bytes so the Coordinator can record the model that built + // the KV (for cross-model safety on restore). The background + // thread that streams the bytes to the socket can mutate + // res->state_data freely; the model fields are immutable for + // the duration of the response. + res->model_alias = model_name; + res->model_path = params_base.model.path; + if (model_tgt) { + res->tokenizer = llama_model_get_tokenizer_model(model_tgt); + res->model_name = llama_model_get_display_name(model_tgt); + res->model_quant = llama_model_get_quant_label(model_tgt); + res->model_capabilities = llama_model_get_capabilities_bitfield(model_tgt); + } + SRV_INF("hydra: STATE_GET slot=%d n_past=%d state=%.1f MiB — async\n", + id_slot, res->n_past, state_size / (1024.0 * 1024.0)); + + slot->hydra_transferring->store(true); + + // M2: stream directly to socket (zero-copy). + // Runs SYNCHRONOUSLY on the inference thread to avoid + // concurrent ggml-RPC socket access with llama_decode + // on another slot (fixes crash at ggml-rpc.cpp:532). + // The coordinator already does Store Put as fire-and-forget + // so blocking here only delays slot release, not decode. + const int snap_seq_id = slot->id; + llama_context * snap_ctx = ctx_tgt; + // shared_ptr keeps the atomic alive even if the slot is reallocated + std::shared_ptr> flag_ptr = slot->hydra_transferring; + const int hydra_fd = task.hydra_action.hydra_fd; + + // Capture prompt tokens for M1 path header (slot is valid on inference thread) + const llama_tokens prompt_tokens_get = slot->prompt.tokens.get_text_tokens(); + const int32_t n_past_val = res->n_past; + + // Snapshot the most recent native checkpoint so STATE_PUT can + // register it instead of fabricating one at the final position. + // Fabricating at pos_max=n-1 corrupts hybrid/recurrent model + // decode because the recurrent state is one token ahead of the + // decode resume point — it has already processed the final token. + std::vector snapshot_ckpt; + uint8_t hdr_flags = 0x00; + int32_t ckpt_pos_min = 0, ckpt_pos_max = 0; + int64_t ckpt_n_tokens = 0; + if (!slot->prompt.checkpoints.empty()) { + hdr_flags |= 0x01; + const auto & ckpt = slot->prompt.checkpoints.back(); + ckpt_pos_min = ckpt.pos_min; + ckpt_pos_max = ckpt.pos_max; + ckpt_n_tokens = ckpt.n_tokens; + + // Hydra M2-stream double-write fix (#470/#620): serialize the + // recurrent-only capture (data_tgt_recr, PARTIAL_ONLY) instead of + // the full data_tgt. The full live state that follows on the wire + // already carries the attention bytes at the live position, so + // sending the full checkpoint duplicates the attention portion + // (which scales with ctx). The recurrent state is genuinely needed + // at BOTH positions, hence the separate recr-only capture. + // hdr_flags bit 0x02 marks a recurrent-only checkpoint section so + // STATE_PUT/DECODE_APPLY can read it back with matched PARTIAL_ONLY + // flags. Fall back to the full capture when the checkpoint has no + // recr buffer (e.g. it was registered from an old 0x02 blob) — a + // PARTIAL_ONLY read of a full-written buffer is a CUDA memory error. + const bool use_recr = !ckpt.data_tgt_recr.empty(); + if (use_recr) { + hdr_flags |= 0x02; + } + const uint64_t tgt_sz = use_recr ? ckpt.data_tgt_recr.size() : ckpt.data_tgt.size(); + const uint64_t dft_sz = use_recr ? ckpt.data_dft_recr.size() : ckpt.data_dft.size(); + const uint8_t * tgt_ptr = use_recr ? ckpt.data_tgt_recr.data() : ckpt.data_tgt.data(); + const uint8_t * dft_ptr = use_recr ? ckpt.data_dft_recr.data() : ckpt.data_dft.data(); + const size_t ckpt_hdr_sz = 4 + 4 + 8 + 8 + (size_t)tgt_sz + 8 + (size_t)dft_sz; + snapshot_ckpt.resize(ckpt_hdr_sz); + size_t off = 0; + memcpy(snapshot_ckpt.data() + off, &ckpt_pos_min, 4); off += 4; + memcpy(snapshot_ckpt.data() + off, &ckpt_pos_max, 4); off += 4; + memcpy(snapshot_ckpt.data() + off, &ckpt_n_tokens, 8); off += 8; + memcpy(snapshot_ckpt.data() + off, &tgt_sz, 8); off += 8; + if (tgt_sz > 0) { memcpy(snapshot_ckpt.data() + off, tgt_ptr, (size_t)tgt_sz); off += (size_t)tgt_sz; } + memcpy(snapshot_ckpt.data() + off, &dft_sz, 8); off += 8; + if (dft_sz > 0) memcpy(snapshot_ckpt.data() + off, dft_ptr, (size_t)dft_sz); + } + + { + SRV_INF("hydra: STATE_GET streaming (fd=%d state=%.1f MiB)\n", + hydra_fd, state_size / (1024.0 * 1024.0)); + if (hydra_fd >= 0) { + // M2 path: stream v2/v3 blob (header + checkpoint + GPU state) to fd. + // Response header + meta JSON sent first, then v2 header bytes, + // then llama_state_seq_get_data_to_fd writes GPU state directly. + const size_t n_tok = prompt_tokens_get.size(); + const uint32_t hdr_n_tok = (uint32_t)n_tok; + const uint32_t hdr_n_past = (uint32_t)n_past_val; + // 0x03 = v3 blob: checkpoint section carries recurrent-only + // captures (data_tgt_recr, PARTIAL_ONLY). 0x02 = v2 blob: + // checkpoint section carries the full data_tgt. Bumped so a + // mixed-version fleet never misreads a smaller (recr-only) + // checkpoint as a full one. + const uint8_t version_byte = 0x03; + const size_t base_hdr_size = 1 + 4 + 4 + n_tok * sizeof(llama_token) + 1; + const size_t hdr_size = base_hdr_size + snapshot_ckpt.size(); + const size_t total_payload = hdr_size + state_size; + + // Build v2 header buffer + std::vector v2_hdr(hdr_size); + { + size_t off = 0; + memcpy(v2_hdr.data() + off, &version_byte, 1); off += 1; + memcpy(v2_hdr.data() + off, &hdr_n_past, 4); off += 4; + memcpy(v2_hdr.data() + off, &hdr_n_tok, 4); off += 4; + memcpy(v2_hdr.data() + off, prompt_tokens_get.data(), n_tok * sizeof(llama_token)); off += n_tok * sizeof(llama_token); + memcpy(v2_hdr.data() + off, &hdr_flags, 1); off += 1; + if (!snapshot_ckpt.empty()) { + memcpy(v2_hdr.data() + off, snapshot_ckpt.data(), snapshot_ckpt.size()); + off += snapshot_ckpt.size(); + } + } + + { + json meta_j; + meta_j["n_past"] = res->n_past; + meta_j["state_size"] = (uint64_t)state_size; + if (!res->model_alias.empty()) meta_j["model_alias"] = res->model_alias; + if (!res->model_path.empty()) meta_j["model_path"] = res->model_path; + if (!res->tokenizer.empty()) meta_j["tokenizer"] = res->tokenizer; + if (!res->model_name.empty()) meta_j["model_name"] = res->model_name; + if (!res->model_quant.empty()) meta_j["model_quant"] = res->model_quant; + if (res->model_capabilities) meta_j["model_capabilities"] = res->model_capabilities; + const std::string meta_str = meta_j.dump(); + + const uint32_t meta_len = (uint32_t)meta_str.size(); + const uint64_t payload_l = (uint64_t)total_payload; + uint8_t hdr[HYDRA_RES_HEADER_SIZE] = {}; + hdr[0] = HYDRA_STATUS_OK; + hdr[1] = (meta_len) & 0xFF; + hdr[2] = (meta_len >> 8) & 0xFF; + hdr[3] = (meta_len >> 16) & 0xFF; + memcpy(hdr + 4, &payload_l, 8); + hydra_send_all(hydra_fd, hdr, HYDRA_RES_HEADER_SIZE); + hydra_send_all(hydra_fd, meta_str.data(), meta_str.size()); + + // Write v2 blob header before GPU state — STATE_PUT needs tokens + checkpoint + hydra_send_all(hydra_fd, v2_hdr.data(), v2_hdr.size()); + + res->header_sent = true; // META + header + v2-hdr before payload + } + // Stream GPU state to fd (zero-copy from GPU memory) + const size_t streamed = llama_state_seq_get_data_to_fd(snap_ctx, snap_seq_id, hydra_fd, nullptr); + if (streamed != state_size) { + // TOCTOU: state size changed between get_size (header already + // promised state_size bytes) and the stream, or the stream + // failed mid-way. The wire framing is now broken — the only + // safe recovery is to kill the connection. Use shutdown(), + // not close(): the RPC connection loop owns the fd and will + // close it when its next read fails; closing here would race + // (double-close / fd-reuse against unrelated threads). + res->rpc_status = HYDRA_STATUS_ERROR; + res->error = "llama_state_seq_get_data_to_fd streamed " + + std::to_string(streamed) + " B, expected " + + std::to_string(state_size) + " B"; + ::shutdown(hydra_fd, SHUT_RDWR); + } else { + res->streamed_bytes = total_payload; + } + } else { + // M1 path: buffer in memory, RPC thread sends afterwards. + // v3 blob format (0x03): [1B version][4B n_past][4B n_tok][n_tok*4B tokens] + // [1B flags (bit 0 = has_checkpoint)] + // [if flags & 0x01: 4B pos_min | 4B pos_max | 8B n_tokens | 8B tgt_sz | recr_tgt_data | 8B dft_sz | recr_dft_data] + // [raw KV state from llama_state_seq_get_data] + const size_t n_tok = prompt_tokens_get.size(); + const uint32_t hdr_n_tok = (uint32_t)n_tok; + const uint32_t hdr_n_past = (uint32_t)n_past_val; + const uint8_t version_byte = 0x03; + const size_t base_hdr_size = 1 + 4 + 4 + n_tok * sizeof(llama_token) + 1; // version + n_past + n_tok + tokens + flags + const size_t hdr_size = base_hdr_size + snapshot_ckpt.size(); + + // TOCTOU retry: if another slot grew the state between + // get_size (inference thread) and get_data (background thread), + // the copy returns 0. Retry up to 3 times with fresh sizing. + size_t buf_size = hdr_size + state_size; + res->state_data.resize(buf_size); + { + size_t off = 0; + memcpy(res->state_data.data() + off, &version_byte, 1); off += 1; + memcpy(res->state_data.data() + off, &hdr_n_past, 4); off += 4; + memcpy(res->state_data.data() + off, &hdr_n_tok, 4); off += 4; + memcpy(res->state_data.data() + off, prompt_tokens_get.data(), n_tok * sizeof(llama_token)); off += n_tok * sizeof(llama_token); + memcpy(res->state_data.data() + off, &hdr_flags, 1); off += 1; + if (!snapshot_ckpt.empty()) { + memcpy(res->state_data.data() + off, snapshot_ckpt.data(), snapshot_ckpt.size()); + off += snapshot_ckpt.size(); + } + } + + size_t cur_state_size = state_size; + size_t copied = 0; + int retries = 3; + while (retries-- > 0) { + copied = llama_state_seq_get_data( + snap_ctx, res->state_data.data() + hdr_size, cur_state_size, snap_seq_id); + if (copied > 0) break; + // State grew — re-measure and retry + cur_state_size = llama_state_seq_get_size(snap_ctx, snap_seq_id); + res->state_data.resize(hdr_size + cur_state_size); + } + if (copied == 0) { + res->rpc_status = HYDRA_STATUS_ERROR; + res->error = "llama_state_get_data failed after 3 retries"; + res->state_data.clear(); + } + } + flag_ptr->store(false); + // M2 streams to fd (streamed_bytes); M1 buffers into state_data. + const uint64_t out_bytes = (hydra_fd >= 0) + ? res->streamed_bytes + : (uint64_t) res->state_data.size(); + SRV_INF("hydra: STATE_GET done slot=%d rpc_status=%d path=%s bytes=%" PRIu64 "\n", + snap_seq_id, res->rpc_status, + hydra_fd >= 0 ? "M2-stream" : "M1-buffer", out_bytes); + queue_results.send(std::move(res)); + } + + // STATE_GET is synchronous — blocks until KV state is fully + // streamed to the socket. The coordinator's Store Put is + // fire-and-forget, so only slot release is delayed. + } break; + + case SERVER_TASK_TYPE_HYDRA_STATE_PUT: + { + const int id_slot = task.hydra_action.id_slot; + auto res = std::make_unique(); + res->id = task.id; + res->id_slot = id_slot; + res->op = HYDRA_OP_STATE_PUT; + + server_slot * slot = get_slot_by_id(id_slot); + if (slot == nullptr) { + res->rpc_status = HYDRA_STATUS_NOT_FOUND; + res->error = "invalid slot ID"; + queue_results.send(std::move(res)); + break; + } + if (slot->is_processing() || slot->hydra_transferring->load()) { + res->rpc_status = HYDRA_STATUS_BUSY; + queue_results.send(std::move(res)); + break; + } + + // M-Perf.9 #289: populate model identity from resident model. + // model_match = true always (infrastructure only; actual KV + // validation comes when model identity is embedded in the KV header). + res->model_alias = model_name; + res->model_path = params_base.model.path; + res->model_match = true; + if (model_tgt) { + res->tokenizer = llama_model_get_tokenizer_model(model_tgt); + res->model_name = llama_model_get_display_name(model_tgt); + res->model_quant = llama_model_get_quant_label(model_tgt); + res->model_capabilities = llama_model_get_capabilities_bitfield(model_tgt); + } + + // Erase existing checkpoints to avoid collision with restored session state + if (task.hydra_action.erase_existing && !slot->prompt.checkpoints.empty()) { + SLT_INF(*slot, "erasing %zu existing checkpoints before STATE_PUT restore\n", + slot->prompt.checkpoints.size()); + slot->prompt.checkpoints.clear(); + } + + const auto & buf = task.hydra_action.state_data; + + // Detect v2/v3 blob (0x02/0x03 at offset 0) vs legacy format (no version byte). + // v2: [1B version=0x02][4B n_past][4B n_tok][n_tok*4B tokens][1B flags][?ckpt?][KV state] + // v3: same, but the checkpoint section may be a recurrent-only capture + // (hdr_flags bit 0x02 set). Bumped to 0x03 by the M2-stream double-write fix. + const bool is_v2 = buf.size() >= 1 && (buf[0] == 0x02 || buf[0] == 0x03); + + size_t hdr_offset = 0; + int32_t hdr_n_tok = 0; + int32_t hdr_n_past = 0; + bool has_chkpt = false; + bool ckpt_is_recr_only = false; + int32_t ckpt_pos_min_in = 0, ckpt_pos_max_in = 0; + int64_t ckpt_n_tokens_in = 0; + std::vector ckpt_tgt_data, ckpt_dft_data; + + if (is_v2) { + // v2/v3: version at [0], n_past at [1..4], n_tok at [5..8] + if (buf.size() >= 9) { + memcpy(&hdr_n_past, buf.data() + 1, 4); + memcpy(&hdr_n_tok, buf.data() + 5, 4); + } + const size_t token_start = 9; + const size_t token_end = token_start + (size_t)hdr_n_tok * sizeof(llama_token); + hdr_offset = token_end; + if (hdr_offset < buf.size()) { + const uint8_t flags = buf[hdr_offset]; + hdr_offset += 1; // past flags byte + // bit 0x01 = has checkpoint; bit 0x02 = checkpoint section is + // recurrent-only (PARTIAL_ONLY). A v3 blob that fell back to the + // full capture (old-registered checkpoint) leaves 0x02 clear. + ckpt_is_recr_only = (flags & 0x02) != 0; + if (flags & 0x01) { + // Parse checkpoint: 4B pos_min | 4B pos_max | 8B n_tokens | 8B tgt_sz | tgt_data | 8B dft_sz | dft_data + if (hdr_offset + 4 + 4 + 8 + 8 <= buf.size()) { + memcpy(&ckpt_pos_min_in, buf.data() + hdr_offset, 4); hdr_offset += 4; + memcpy(&ckpt_pos_max_in, buf.data() + hdr_offset, 4); hdr_offset += 4; + memcpy(&ckpt_n_tokens_in, buf.data() + hdr_offset, 8); hdr_offset += 8; + uint64_t tgt_sz_in; + memcpy(&tgt_sz_in, buf.data() + hdr_offset, 8); hdr_offset += 8; + if (tgt_sz_in > 0 && hdr_offset + tgt_sz_in <= buf.size()) { + ckpt_tgt_data.assign(buf.data() + hdr_offset, buf.data() + hdr_offset + (size_t)tgt_sz_in); + hdr_offset += (size_t)tgt_sz_in; + } + if (hdr_offset + 8 <= buf.size()) { + uint64_t dft_sz_in; + memcpy(&dft_sz_in, buf.data() + hdr_offset, 8); hdr_offset += 8; + if (dft_sz_in > 0 && hdr_offset + dft_sz_in <= buf.size()) { + ckpt_dft_data.assign(buf.data() + hdr_offset, buf.data() + hdr_offset + (size_t)dft_sz_in); + hdr_offset += (size_t)dft_sz_in; + } + } + has_chkpt = true; + } + } + } + // Restore tokens from token_start + if (hdr_n_tok > 0 && token_start + (size_t)hdr_n_tok * sizeof(llama_token) <= buf.size()) { + slot->prompt.tokens.clear(); + const llama_token * tok_ptr = (const llama_token *)(buf.data() + token_start); + llama_tokens restored_tokens(tok_ptr, tok_ptr + (size_t)hdr_n_tok); + slot->prompt.tokens.insert(restored_tokens); + } + } else { + // Legacy v1 format + if (buf.size() >= 8) { + memcpy(&hdr_n_past, buf.data(), 4); + memcpy(&hdr_n_tok, buf.data() + 4, 4); + hdr_offset = 8 + (size_t)hdr_n_tok * sizeof(llama_token); + } + if (hdr_offset > 0 && hdr_offset <= buf.size()) { + const size_t n_tokens = (size_t)hdr_n_tok; + slot->prompt.tokens.clear(); + if (n_tokens > 0) { + const llama_token * tok_ptr = (const llama_token *)(buf.data() + 8); + llama_tokens restored_tokens(tok_ptr, tok_ptr + n_tokens); + slot->prompt.tokens.insert(restored_tokens); + } + } + } + const bool has_hdr = hdr_offset > 0 && hdr_offset <= buf.size(); + const uint8_t * state_ptr = has_hdr ? buf.data() + hdr_offset : buf.data(); + const size_t state_len = has_hdr ? buf.size() - hdr_offset : buf.size(); + const size_t n_read = llama_state_seq_set_data(ctx_tgt, state_ptr, state_len, slot->id); + if (n_read == 0) { + res->rpc_status = HYDRA_STATUS_ERROR; + res->error = "llama_state_set_data returned 0"; + // Tokens were registered before set_data — clear them so the slot + // is not left poisoned (n_past > 0 with no KV cells → pos_min == -1 + // abort on the next decode that touches this slot). + slot->prompt.tokens.clear(); + slot->prompt.checkpoints.clear(); + slot->n_prompt_tokens_cache = 0; + llama_memory_seq_rm(llama_get_memory(ctx_tgt), slot->id, -1, -1); + } else { + // D4: Inject trailing logits into per-slot buffer instead of the + // shared context-wide llama_get_logits(). This avoids the race where + // another slot's decode clobbers restored logits between STATE_PUT + // and the first sample. + const size_t remaining = state_len - n_read; + const size_t expected_logits = (size_t)llama_vocab_n_tokens(vocab) * sizeof(float); + if (remaining == expected_logits) { + const float * src = (const float *)(state_ptr + n_read); + const size_t n_floats = llama_vocab_n_tokens(vocab); + slot->restored_logits.assign(src, src + n_floats); + slot->logits_valid = true; + SRV_INF("hydra: STATE_PUT slot=%d restored %zu logits to per-slot buffer\n", + id_slot, n_floats); + } + + res->rpc_status = HYDRA_STATUS_OK; + res->restored = true; + res->bytes = (uint64_t)n_read; + // #469 trace: log restored state for cross-flow comparison + SRV_DBG("hydra: STATE_PUT slot=%d RESTORED n_past=%d n_prompt_tok=%d state_bytes=%zu just_restored=true\n", + id_slot, hdr_n_tok, hdr_n_tok, n_read); + { + std::string tok_ids; + for (size_t i = 0; i < std::min(16, slot->prompt.tokens.size()); ++i) { + if (i > 0) tok_ids += ","; + tok_ids += std::to_string(slot->prompt.tokens[i]); + } + SRV_DBG("hydra: STATE_PUT slot=%d first16_tokens=[%s] total=%zu\n", + id_slot, tok_ids.c_str(), slot->prompt.tokens.size()); + } + if (hdr_n_tok > 0) { + slot->n_prompt_tokens_cache = hdr_n_tok; + slot->n_decoded = 0; + res->n_past = hdr_n_tok; + + // Register native checkpoint from the blob (v2) or fabricate one (legacy). + // The native checkpoint has pos_max at n-4 (created before the last + // few prompt tokens were decoded), so loading it rewinds the recurrent + // state to a clean position. The old fabricated checkpoint at (0, n-1) + // puts the recurrent state at the final position — one token ahead of + // where decode must resume — corrupting hybrid/recurrent model output. + slot->prompt.checkpoints.clear(); + if (has_chkpt) { + auto & ckpt = slot->prompt.checkpoints.emplace_back(); + ckpt.n_tokens = ckpt_n_tokens_in; + ckpt.pos_min = ckpt_pos_min_in; + ckpt.pos_max = ckpt_pos_max_in; + // New-format (v3) checkpoints carry a recurrent-only capture — + // route it into data_*_recr and tag is_recr_only so the load + // path uses matched PARTIAL_ONLY flags (plus an attention + // seq_rm at pos_max) instead of the full flags=0 restore. + ckpt.is_recr_only = ckpt_is_recr_only; + if (ckpt_is_recr_only) { + ckpt.data_tgt_recr = std::move(ckpt_tgt_data); + ckpt.data_dft_recr = std::move(ckpt_dft_data); + } else { + ckpt.data_tgt = std::move(ckpt_tgt_data); + ckpt.data_dft = std::move(ckpt_dft_data); + } + SLT_INF(*slot, "STATE_PUT registered native checkpoint (pos_min=%d pos_max=%d n_tokens=%" PRId64 " tgt_sz=%zu recr_only=%d)\n", + ckpt.pos_min, ckpt.pos_max, ckpt.n_tokens, ckpt.size(), (int) ckpt.is_recr_only); + } else { + const auto pos_min = llama_memory_seq_pos_min(llama_get_memory(ctx_tgt), slot->id); + create_checkpoint(*slot, 0, (llama_pos)pos_min, (llama_pos)(hdr_n_tok - 1)); + } + slot->just_restored = true; + } + SRV_INF("hydra: STATE_PUT slot=%d restored=%zu B n_past=%d n_prompt_tok=%d\n", + id_slot, n_read, res->n_past, hdr_n_tok); + } + queue_results.send(std::move(res)); + } break; + + case SERVER_TASK_TYPE_HYDRA_STATE_META: + { + const int id_slot = task.hydra_action.id_slot; + auto res = std::make_unique(); + res->id = task.id; + res->id_slot = id_slot; + res->op = HYDRA_OP_STATE_META; + + server_slot * slot = get_slot_by_id(id_slot); + if (slot == nullptr) { + res->rpc_status = HYDRA_STATUS_NOT_FOUND; + res->error = "invalid slot ID"; + queue_results.send(std::move(res)); + break; + } + // META is safe to serve even while processing or transferring (read-only metadata) + int actual_n_past = slot->n_prompt_tokens_cache + slot->n_decoded; + // For cold prefills n_prompt_tokens_cache is 0 — use prompt token count + if (slot->n_prompt_tokens_cache == 0 && slot->prompt.tokens.size() > 0) { + actual_n_past = (int)slot->prompt.tokens.size(); + } + res->n_past = actual_n_past; + res->is_processing = slot->is_processing(); + res->is_transferring = slot->hydra_transferring->load(); + res->state_size = (uint64_t)llama_state_seq_get_size(ctx_tgt, slot->id); + // M-Perf.9 #289: surface model identity. The Coordinator uses + // these to detect cross-model restores — a slot holding a Mini + // KV cache must never have it decoded by a Balanced-loaded model. + res->model_alias = model_name; + res->model_path = params_base.model.path; + if (model_tgt) { + res->tokenizer = llama_model_get_tokenizer_model(model_tgt); + res->model_name = llama_model_get_display_name(model_tgt); + res->model_quant = llama_model_get_quant_label(model_tgt); + res->model_capabilities = llama_model_get_capabilities_bitfield(model_tgt); + } + // #451: populate progress fields based on slot state + switch (slot->state) { + case SLOT_STATE_PROCESSING_PROMPT: + res->operation = "prefill"; + res->tokens_processed = slot->n_prompt_tokens_processed; + // task->n_tokens() is the total tokens to process (fixed); + // prompt.tokens.size() grows during prefill and is WRONG for total. + res->tokens_total = slot->task ? slot->task->n_tokens() : 0; + if (res->tokens_total > 0) { + res->progress = (float)res->tokens_processed / (float)res->tokens_total; + } + res->elapsed_ms = (slot->t_start_process_prompt > 0) + ? (ggml_time_ms() - slot->t_start_process_prompt) : 0; + break; + case SLOT_STATE_GENERATING: + res->operation = "decode"; + res->tokens_processed = slot->n_decoded; + // n_remaining == -1 is the "unlimited generation" sentinel + // (no finite n_predict). Don't compute progress in that case. + if (slot->n_remaining > 0) { + res->tokens_total = slot->n_decoded + slot->n_remaining; + res->progress = (float)res->tokens_processed / (float)res->tokens_total; + } + res->elapsed_ms = (slot->t_start_generation > 0) + ? (ggml_time_ms() - slot->t_start_generation) : 0; + break; + case SLOT_STATE_IDLE: + res->operation = "idle"; + res->progress = 1.0f; + break; + default: + res->operation = "unknown"; + break; + } + // Handle save/restore operations via hydra_transferring flag. + // Clear any stale progress from the prior state since we're + // now in a transferring context, not the previous operation. + if (slot->hydra_transferring->load()) { + res->operation = "save"; + res->progress = 0.0f; + res->tokens_processed = 0; + res->tokens_total = 0; + res->elapsed_ms = 0; + } + res->rpc_status = HYDRA_STATUS_OK; + queue_results.send(std::move(res)); + } break; + + case SERVER_TASK_TYPE_HYDRA_ENGINE_CONFIGURE: + { + auto res = std::make_unique(); + res->id = task.id; + res->op = HYDRA_OP_CONFIGURE; + res->rpc_status = HYDRA_STATUS_OK; + res->success = true; + + // hydra#406: tiered CONFIGURE (T1/T2/T3). Backward compat: + // a legacy {"state_chunk_size":N} payload is treated as a + // degenerate T1 (the original hydra#334 startup call from + // WorkerSchedulerService.cs:2842). + if (task.hydra_action.config_json.empty()) { + res->tier = "T1"; + SRV_INF("hydra: CONFIGURE (empty payload, slot %d) — T1 no-op\n", + task.hydra_action.id_slot); + queue_results.send(std::move(res)); + break; + } + + json cfg; + try { + cfg = json::parse(task.hydra_action.config_json); + } catch (const std::exception & e) { + res->success = false; + res->rpc_status = HYDRA_STATUS_ERROR; + res->error = std::string("CONFIGURE: invalid config_json: ") + e.what(); + SRV_WRN("hydra: CONFIGURE failed to parse config_json (slot %d): %s\n", + task.hydra_action.id_slot, e.what()); + queue_results.send(std::move(res)); + break; + } + + // Route through the shared classify → apply helper. + // sync=false: T2/T3/T4 are staged for the slot-free moment. + hydra_config_result cfg_result = hydra_apply_config(cfg, /*sync=*/false); + + // hydra#470: report generic (T4) keys that cannot be + // applied BEFORE the tier-0 early return — a payload + // whose keys are all unrecognized/rejected still has + // to surface them (zero silent drops). + res->unrecognized_keys = cfg_result.unrecognized_keys; + res->rejected_keys = cfg_result.rejected_keys; + + if (cfg_result.highest_tier == 0) { + // No recognized keys — still emit a T1 success + // (the legacy {"state_chunk_size":N} case). + res->tier = "T1"; + SRV_INF("hydra: CONFIGURE (no recognized keys, slot %d) — T1 no-op\n", + task.hydra_action.id_slot); + queue_results.send(std::move(res)); + break; + } + + if (!cfg_result.ok) { + res->success = false; + res->rpc_status = HYDRA_STATUS_ERROR; + res->error = "CONFIGURE: " + cfg_result.error; + SRV_WRN("hydra: CONFIGURE apply failed (slot %d): %s\n", + task.hydra_action.id_slot, cfg_result.error.c_str()); + queue_results.send(std::move(res)); + break; + } + + // Build the response from the shared helper's result. + res->tier = hydra_tier_label(cfg_result.highest_tier); + res->params_applied = std::move(cfg_result.params_applied); + res->deferred_keys = std::move(cfg_result.deferred_keys); + res->state_chunk_size_applied = cfg_result.state_chunk_size_applied; + + SRV_INF("hydra: CONFIGURE tier=%s applied=%zu deferred=%zu unrecognized=%zu rejected=%zu (slot %d)\n", + res->tier.c_str(), + res->params_applied.size(), + res->deferred_keys.size(), + res->unrecognized_keys.size(), + res->rejected_keys.size(), + task.hydra_action.id_slot); + queue_results.send(std::move(res)); + } break; + + case SERVER_TASK_TYPE_HYDRA_ENGINE_INFO: + { + auto res = std::make_unique(); + res->id = task.id; + res->op = HYDRA_OP_INFO; + res->rpc_status = HYDRA_STATUS_OK; + // M-Perf.9 #289: advertise the model identity features so + // the Coordinator knows it can send `model` in PREFILL and + // expect model_alias/model_path/tokenizer/model_name/model_quant/model_capabilities + // in META responses. + // `preset_aliases` lists every alias loaded from + // --models-preset (empty when no preset is configured). + json preset_aliases_j = json::array(); + for (const auto & [alias, _path] : preset_alias_to_path) { + preset_aliases_j.push_back(alias); + } + // Hydra #287/#260/#348: two-engine "work together" status + // — see specs/rpc-protocol.md's ENGINE_INFO (0x41) + // contract. pipeline_capable stays false until #287's + // PIPELINE half lands; mode only ever reports + // solo/combined until then. solo_active/rpc_backend_active/ + // peer_reachable/combined_head_attached are independent + // booleans (#348) — replaces the old single "role" string + // and the peer_connected/combined_capable field-aliasing. + const int32_t expert_mode = ctx_tgt ? llama_hydra_get_expert_mode(ctx_tgt) : 0; + // Hydra #383 T1 / #375: advertise "combined" capability when this + // engine is ready to serve in COMBINED mode — either via expert-split + // (hydra_combined_head_attached) or via layer-split (hydra_combined_static). + json capabilities_j = {"prefill", "decode", "state_transfer", + "expert_mode", "quant_swap", + "preset", "tokenizer", "model_name", + "model_quant", "model_capabilities", + "merged_decode"}; + if (hydra_combined_head_attached || hydra_combined_static) { + capabilities_j.push_back("combined"); + } + // In layer-split static mode the engine is always in combined mode; + // in expert-split mode it follows the per-request SET_EXPERT_MODE state. + const std::string mode_str = hydra_combined_static ? "combined" + : (expert_mode == 1 ? "combined" : "solo"); + json info_j = { + {"engine", "llama-server-hydra"}, + {"version", "E1"}, + {"capabilities", capabilities_j}, + {"preset_aliases", preset_aliases_j}, + {"solo_active", hydra_solo_active}, + {"rpc_backend_active", hydra_rpc_backend_active}, + {"mode", mode_str}, + {"split_mode", hydra_split_mode}, + {"peer_addr", hydra_peer}, + {"peer_reachable", hydra_peer_reachable}, + {"layer_split", hydra_combined_pattern}, + {"combined_head_attached", hydra_combined_head_attached || hydra_combined_static}, + {"pipeline_capable", false} + }; + res->info_json = info_j.dump(); + queue_results.send(std::move(res)); + } break; + + case SERVER_TASK_TYPE_HYDRA_ENGINE_PREFILL: + { + const int id_slot = task.hydra_action.id_slot; + auto res = std::make_unique(); + res->id = task.id; + res->op = HYDRA_OP_PREFILL; + + // #451: track timing for PREFILL metrics + const int64_t prefill_start_ms = ggml_time_ms(); + + // Set by the model-resolution block below when a real + // `load_model` swap happens. Used at the response site to + // decide whether the post-prefill model identity is the + // freshly loaded model (swap) or the original (no-swap / + // fallback). + bool model_was_swapped = false; + + server_slot * slot = get_slot_by_id(id_slot); + if (slot == nullptr) { + res->rpc_status = HYDRA_STATUS_NOT_FOUND; + res->error = "invalid slot ID"; + queue_results.send(std::move(res)); + break; + } + + if (slot->is_processing()) { + res->rpc_status = HYDRA_STATUS_BUSY; + res->error = "slot is busy"; + queue_results.send(std::move(res)); + break; + } + + // M-Perf.9 #289: parse the optional `model` key from the + // request body and swap the resident model when the preset + // registry knows the alias. The parse is reused for the + // tokenization step below. Falls back to the resident model + // (with `model_fallback:true` in the response) when the + // alias is unknown or no preset is configured. + json parsed_body; + std::string requested_model; + json hydra_cfg; // optional hydra_config object + bool has_hydra_config = false; + if (!task.hydra_action.request_json.empty()) { + try { + parsed_body = json::parse(task.hydra_action.request_json); + if (parsed_body.is_object() && parsed_body.contains("model") + && parsed_body["model"].is_string()) { + requested_model = parsed_body["model"].get(); + } + // hydra_config: optional config object from Hydra.Core + // containing topology/sampling overrides. When present + // with model_path, it drives the model swap directly + // (bypassing the preset alias lookup). + if (parsed_body.is_object() && parsed_body.contains("hydra_config") + && parsed_body["hydra_config"].is_object()) { + hydra_cfg = parsed_body["hydra_config"]; + has_hydra_config = true; + } + } catch (const std::exception & e) { + res->rpc_status = HYDRA_STATUS_BAD_REQUEST; + res->error = std::string("invalid JSON: ") + e.what(); + queue_results.send(std::move(res)); + break; + } + } + + // Apply hydra_config synchronously when present. + // T1 keys (sampling, n_predict, etc.) are applied in-place. + // T2/T3 keys (n_ctx, cache_type, model_path, split_mode, etc.) + // trigger immediate rebuilds on this task-queue thread. + + // #470: Before applying config, probe all RPC peers for + // reconnection. If a peer restarted since the last request, + // its buffers are gone even though model/params haven't + // changed. Without this probe, the T3 rebuild in + // hydra_apply_config → apply_t3_rebuild would skip (params + // unchanged) and the subsequent graph_compute would fail. + if (ctx_tgt && ggml_backend_rpc_check_any_peer_reconnection()) { + SRV_WRN("%s", "hydra: PREFILL: RPC peer reconnected — forcing T3 rebuild\n"); + ctx_tgt->peer_reconnection_pending = true; + } + + if (has_hydra_config) { + SRV_INF("hydra: PREFILL slot=%d applying hydra_config (%zu keys)\n", + id_slot, hydra_cfg.size()); + hydra_config_result cfg_result = hydra_apply_config(hydra_cfg, /*sync=*/true); + if (!cfg_result.ok) { + res->rpc_status = HYDRA_STATUS_ERROR; + res->error = "hydra_config apply failed: " + cfg_result.error; + SRV_WRN("hydra: PREFILL hydra_config apply failed (slot %d): %s\n", + id_slot, cfg_result.error.c_str()); + queue_results.send(std::move(res)); + break; + } + // If the apply may have rebuilt the slots (T3 + // statics or a T4-only generic config both route + // through apply_t3_rebuild → load_model → + // slots.clear()), track it and re-look-up the slot. + // Without the T4 case (hydra#470) the slot pointer + // captured above would dangle into the prefill. + if (hydra_config_requires_slot_relookup(cfg_result.highest_tier)) { + model_was_swapped = true; + res->model_load_ms = (double)(ggml_time_ms() - prefill_start_ms); + SRV_INF("hydra: PREFILL hydra_config T3/T4 applied model_alias='%s' tokenizer='%s' model_name='%s' quant='%s' caps=0x%x\n", + model_name.empty() ? "?" : model_name.c_str(), + model_tgt ? llama_model_get_tokenizer_model(model_tgt) : "", + model_tgt ? llama_model_get_display_name(model_tgt) : "", + model_tgt ? llama_model_get_quant_label(model_tgt) : "", + model_tgt ? llama_model_get_capabilities_bitfield(model_tgt) : 0); + slot = get_slot_by_id(id_slot); + if (slot == nullptr) { + res->rpc_status = HYDRA_STATUS_NOT_FOUND; + res->error = "slot disappeared after hydra_config T3/T4 rebuild"; + queue_results.send(std::move(res)); + break; + } + } + // When hydra_config carries model_path, the model swap is + // handled by apply_t3_rebuild() above — skip the bare + // model alias lookup below. + if (hydra_cfg.contains("model_path")) { + requested_model.clear(); + } + } + + // Fallback: bare model alias lookup when hydra_config didn't + // handle the model swap (no hydra_config, or no model_path). + if (!requested_model.empty()) { + auto it = preset_alias_to_path.find(requested_model); + if (it == preset_alias_to_path.end()) { + SRV_WRN("hydra: PREFILL model='%s' unknown (preset has %zu alias(es)) — falling back to resident '%s'\n", + requested_model.c_str(), preset_alias_to_path.size(), + model_name.c_str()); + res->model_fallback = true; + } else if (it->second != params_base.model.path) { + SRV_INF("hydra: PREFILL model='%s' swapping %s -> %s\n", + requested_model.c_str(), params_base.model.path.c_str(), + it->second.c_str()); + common_params swapped_params = params_base; + // Apply the target alias's full preset so that + // tensor_buft_overrides, n_gpu_layers, split_mode, + // tensor_split, etc. are replaced — not inherited + // from the source model. Intentionally the FULL + // preset (sampling, chat template, n_ctx, etc. + // included), not just tensor-placement keys: a + // real model swap targets a different model, + // which plausibly needs its own sampling + // defaults/chat template too, not just a new + // memory layout. + auto pit = preset_alias_to_preset.find(requested_model); + if (pit != preset_alias_to_preset.end()) { + // Clear inherited tensor_buft_overrides (padded + // to 4096 by common_params_parse_ex) BEFORE + // apply_to_params, which push_back()'s the new + // preset's entries via CLI handlers. Without + // this, the new entries land after the + // nullptr-terminator and exceed the 4096 limit, + // triggering GGML_ASSERT in + // common_model_params_to_llama (#499 regression). + swapped_params.tensor_buft_overrides.clear(); + try { + // apply_to_params() replays CLI handlers + // (parse_tensor_buffer_overrides, the + // n-cpu-moe std::stoi, two-value option + // parsers) which throw on a malformed + // target preset. Uncaught, that exception + // would escape the task-queue loop and + // kill the task thread — fail the swap + // instead. + pit->second.apply_to_params(swapped_params); + hydra_repad_tensor_buft_overrides(swapped_params, "PREFILL swap"); + } catch (const std::exception & e) { + SRV_WRN("hydra: PREFILL swap preset apply for '%s' failed: %s\n", + requested_model.c_str(), e.what()); + res->rpc_status = HYDRA_STATUS_ERROR; + res->error = std::string("model swap preset apply failed: ") + e.what(); + queue_results.send(std::move(res)); + break; + } + SRV_INF("hydra: PREFILL swap applied preset for '%s' " + "(tensor_buft_overrides=%zu entries)\n", + requested_model.c_str(), + swapped_params.tensor_buft_overrides.size()); + } + swapped_params.model.path = it->second; + // Update the alias so model_name is re-derived + // correctly in load_model() (model_name is set from + // model_alias.first when non-empty). + swapped_params.model_alias = { requested_model }; + // #514: tear down COMBINED state before the + // reload — otherwise the engine loads the + // correct model file but keeps routing tokens + // through the stale peer/expert-binding config, + // collapsing decode throughput. + const bool was_combined = hydra_combined_head_attached || hydra_combined_static; + if (was_combined) { + hydra_teardown_combined_before_reload(); + } + const int64_t model_load_start_ms = ggml_time_ms(); + if (!load_model(swapped_params)) { + res->rpc_status = HYDRA_STATUS_ERROR; + res->error = "model swap to '" + requested_model + "' failed"; + queue_results.send(std::move(res)); + break; + } + if (was_combined) { + hydra_reattach_combined_after_reload(); + } + res->model_load_ms = (double)(ggml_time_ms() - model_load_start_ms); + model_was_swapped = true; + SRV_INF("hydra: PREFILL swap confirmed model_alias='%s' tokenizer='%s' model_name='%s' quant='%s' caps=0x%x model_load_ms=%.1f\n", + swapped_params.model_alias.empty() ? "?" : swapped_params.model_alias.begin()->c_str(), + model_tgt ? llama_model_get_tokenizer_model(model_tgt) : "", + model_tgt ? llama_model_get_display_name(model_tgt) : "", + model_tgt ? llama_model_get_quant_label(model_tgt) : "", + model_tgt ? llama_model_get_capabilities_bitfield(model_tgt) : 0, + res->model_load_ms); + // After load_model, `this` state is reset (new + // slots, new context). Re-look up the slot by id. + slot = get_slot_by_id(id_slot); + if (slot == nullptr) { + res->rpc_status = HYDRA_STATUS_NOT_FOUND; + res->error = "slot disappeared after model swap"; + queue_results.send(std::move(res)); + break; + } + } else { + SRV_DBG("hydra: PREFILL model='%s' already resident, no swap\n", + requested_model.c_str()); + } + } + + // Tokenize from JSON messages if request_json is provided; + // otherwise fall back to pre-tokenized prompt_tokens for back-compat. + std::vector prompt_tokens = std::move(task.hydra_action.prompt_tokens); + if (!parsed_body.is_null()) { + try { + std::vector dummy_files; + json parsed = oaicompat_chat_params_parse(parsed_body, chat_params, dummy_files); + if (!parsed.contains("prompt")) { + res->rpc_status = HYDRA_STATUS_ERROR; + res->error = "chat template produced no prompt"; + queue_results.send(std::move(res)); + break; + } + auto tokenized = tokenize_input_prompts(vocab, mctx, parsed["prompt"], true, true); + if (tokenized.empty()) { + res->rpc_status = HYDRA_STATUS_ERROR; + res->error = "tokenization produced no tokens"; + queue_results.send(std::move(res)); + break; + } + prompt_tokens = tokenized[0].get_tokens(); + } catch (const std::exception & e) { + res->rpc_status = HYDRA_STATUS_ERROR; + res->error = std::string("JSON/tokenization error: ") + e.what(); + queue_results.send(std::move(res)); + break; + } + } + + SRV_INF("hydra: PREFILL slot=%d tokens=%zu\n", id_slot, prompt_tokens.size()); + // #469 trace: log first 16 token IDs for cross-flow comparison + { + std::string tok_ids; + for (size_t i = 0; i < std::min(16, prompt_tokens.size()); ++i) { + if (i > 0) tok_ids += ","; + tok_ids += std::to_string(prompt_tokens[i]); + } + SRV_DBG("hydra: PREFILL slot=%d first16_tokens=[%s] total=%zu\n", + id_slot, tok_ids.c_str(), prompt_tokens.size()); + } + + // Clear existing slot state + slot->prompt_clear(false); + slot->n_prompt_tokens_cache = 0; + slot->n_prompt_tokens_processed = 0; + slot->n_decoded = 0; + + // Insert prompt tokens + if (prompt_tokens.empty()) { + res->rpc_status = HYDRA_STATUS_OK; + res->n_past = 0; + res->state_size = 0; + queue_results.send(std::move(res)); + break; + } + + slot->prompt.tokens.insert(prompt_tokens); + const auto & tokens = slot->prompt.tokens.get_tokens(); + const int n_tokens = (int)tokens.size(); + + // Add BOS if needed (check if slot uses BOS) + int token_offset = 0; + llama_token bos = llama_vocab_bos(vocab); + if (add_bos_token && bos != LLAMA_TOKEN_NULL && (tokens.empty() || tokens[0] != bos)) { + token_offset = 1; + } + + // Decode prompt in batches. Hydra #469 fix: upstream's own + // invariant (see create_checkpoint call in update_slots, + // "we create the checkpoint before calling llama_decode(), + // so the current batch is not yet processed and therefore + // it is not part of the checkpoint") requires the + // checkpoint to be created BEFORE the final token is + // decoded. The previous version of this handler decoded + // the whole prompt first and only afterward claimed (via + // create_checkpoint's pos_max arg, below) that the last + // token was still unprocessed. For hybrid/recurrent (SSM) + // models, whose memory can't be partially rolled back via + // seq_rm, that lie meant a cross-node restore would + // re-decode a token that was already baked into the + // recurrent state — double-applying it and corrupting the + // hidden state. Splitting the loop so the checkpoint is + // captured after n_tokens-1 tokens (matching what + // create_checkpoint's pos_max already claimed) makes the + // claim honest, same as the standard update_slots() path. + const int total_tokens = n_tokens + token_offset; + const int n_ubatch = llama_n_ubatch(ctx_tgt); + const int n_before_last = total_tokens > 1 ? total_tokens - 1 : total_tokens; + bool decode_ok = true; + for (int i = 0; i < n_before_last && decode_ok; i += n_ubatch) { + const int n_tokens_batch = std::min(n_ubatch, n_before_last - i); + common_batch_clear(batch); + for (int j = 0; j < n_tokens_batch; j++) { + const int tok_idx = i + j; + llama_token id; + if (token_offset > 0 && tok_idx == 0) { + id = bos; + } else { + id = tokens[tok_idx - token_offset]; + } + // No token in this phase is the final prompt + // token, so logits are never needed here. + common_batch_add(batch, id, tok_idx, {slot->id}, false); + } + if (llama_decode(ctx_tgt, batch) != 0) { + SRV_ERR("hydra: PREFILL slot=%d llama_decode failed at batch %d\n", id_slot, i); + decode_ok = false; + } + } + + if (!decode_ok) { + res->rpc_status = HYDRA_STATUS_ERROR; + res->error = "llama_decode failed during prefill"; + queue_results.send(std::move(res)); + break; + } + + // Register checkpoint BEFORE decoding the final token, so + // its pos_max claim (n_tokens - 1) is honest. Moved up + // from after the full-prompt decode (see #469 above). + if (n_tokens > 0) { + const auto pos_min = llama_memory_seq_pos_min(llama_get_memory(ctx_tgt), slot->id); + create_checkpoint(*slot, 0, (llama_pos)pos_min, (llama_pos)(n_tokens - 1)); + } + + // Decode the held-back final token (if any) now that the + // checkpoint has captured the state before it. + if (total_tokens > n_before_last) { + common_batch_clear(batch); + const int tok_idx = total_tokens - 1; + llama_token id = (token_offset > 0 && tok_idx == 0) + ? bos + : tokens[tok_idx - token_offset]; + common_batch_add(batch, id, tok_idx, {slot->id}, true); + if (llama_decode(ctx_tgt, batch) != 0) { + SRV_ERR("hydra: PREFILL slot=%d llama_decode failed on final token\n", id_slot); + res->rpc_status = HYDRA_STATUS_ERROR; + res->error = "llama_decode failed during prefill (final token)"; + queue_results.send(std::move(res)); + break; + } + } + + // Update slot tracking + slot->n_prompt_tokens_processed = n_tokens; + slot->n_prompt_tokens_cache = n_tokens; + + // Checkpoint already registered above, before the final + // token was decoded (#469 fix). + + // Build v2/v3 header: [1B version=0x02|0x03][4B n_past][4B n_tok][n_tok*4B tokens][1B flags][?ckpt?] + // Shared by both response paths — M1 embeds it at the head of the + // buffered blob, M2 sends it before streaming the GPU state. + const uint32_t hdr_n_past = (uint32_t)n_tokens; + const uint32_t hdr_n_tok = (uint32_t)(tokens.size()); + uint8_t hdr_flags = 0x00; + std::vector ckpt_buf; + int32_t ckpt_pos_min = 0, ckpt_pos_max = 0; + int64_t ckpt_n_tokens = 0; + if (!slot->prompt.checkpoints.empty()) { + hdr_flags |= 0x01; + const auto & ckpt = slot->prompt.checkpoints.back(); + ckpt_pos_min = ckpt.pos_min; + ckpt_pos_max = ckpt.pos_max; + ckpt_n_tokens = ckpt.n_tokens; + // Hydra M2-stream double-write fix (#470/#620): serialize the + // recurrent-only capture (data_tgt_recr, PARTIAL_ONLY) instead of + // the full data_tgt. The full live state that follows on the wire + // already carries the attention bytes at the live position, so + // sending the full checkpoint duplicates the attention portion + // (which scales with ctx). The recurrent state is genuinely needed + // at BOTH positions, hence the separate recr-only capture. + // hdr_flags bit 0x02 marks a recurrent-only checkpoint section so + // STATE_PUT/DECODE_APPLY can read it back with matched PARTIAL_ONLY + // flags. Fall back to the full capture when the checkpoint has no + // recr buffer (e.g. it was registered from an old 0x02 blob) — a + // PARTIAL_ONLY read of a full-written buffer is a CUDA memory error. + const bool use_recr = !ckpt.data_tgt_recr.empty(); + if (use_recr) { + hdr_flags |= 0x02; + } + const uint64_t tgt_sz = use_recr ? ckpt.data_tgt_recr.size() : ckpt.data_tgt.size(); + const uint64_t dft_sz = use_recr ? ckpt.data_dft_recr.size() : ckpt.data_dft.size(); + const uint8_t * tgt_ptr = use_recr ? ckpt.data_tgt_recr.data() : ckpt.data_tgt.data(); + const uint8_t * dft_ptr = use_recr ? ckpt.data_dft_recr.data() : ckpt.data_dft.data(); + ckpt_buf.resize(4 + 4 + 8 + 8 + (size_t)tgt_sz + 8 + (size_t)dft_sz); + size_t off = 0; + memcpy(ckpt_buf.data() + off, &ckpt_pos_min, 4); off += 4; + memcpy(ckpt_buf.data() + off, &ckpt_pos_max, 4); off += 4; + memcpy(ckpt_buf.data() + off, &ckpt_n_tokens, 8); off += 8; + memcpy(ckpt_buf.data() + off, &tgt_sz, 8); off += 8; + if (tgt_sz > 0) { memcpy(ckpt_buf.data() + off, tgt_ptr, (size_t)tgt_sz); off += (size_t)tgt_sz; } + memcpy(ckpt_buf.data() + off, &dft_sz, 8); off += 8; + if (dft_sz > 0) memcpy(ckpt_buf.data() + off, dft_ptr, (size_t)dft_sz); + } + const size_t base_hdr_size = 1 + 4 + 4 + hdr_n_tok * sizeof(llama_token) + 1; + const size_t v2_size = base_hdr_size + ckpt_buf.size(); + std::vector v2_hdr(v2_size); + { + size_t off = 0; + // 0x03 = v3 blob: checkpoint section carries recurrent-only + // captures (data_tgt_recr, PARTIAL_ONLY). 0x02 = v2 blob: + // checkpoint section carries the full data_tgt. Bumped so a + // mixed-version fleet never misreads a smaller (recr-only) + // checkpoint as a full one. + const uint8_t version_byte = 0x03; + memcpy(v2_hdr.data() + off, &version_byte, 1); off += 1; + memcpy(v2_hdr.data() + off, &hdr_n_past, 4); off += 4; + memcpy(v2_hdr.data() + off, &hdr_n_tok, 4); off += 4; + if (hdr_n_tok > 0) { + const auto & toks = slot->prompt.tokens.get_text_tokens(); + memcpy(v2_hdr.data() + off, toks.data(), toks.size() * sizeof(llama_token)); + off += toks.size() * sizeof(llama_token); + } + memcpy(v2_hdr.data() + off, &hdr_flags, 1); off += 1; + if (!ckpt_buf.empty()) { + memcpy(v2_hdr.data() + off, ckpt_buf.data(), ckpt_buf.size()); + off += ckpt_buf.size(); + } + } + + // Get raw KV state + const size_t state_size = llama_state_seq_get_size(ctx_tgt, slot->id); + + // Snapshot logits NOW into a small buffer. ctx->logits is + // context-global: a concurrent slot decode can overwrite it + // while the M2 state stream is on the wire. Appending + // n_vocab floats gives the decode GPU the activation handoff + // (llama_state_seq_get_data saves KV but not logits), so + // STATE_PUT / DECODE_APPLY can sample immediately. + uint64_t logits_size = 0; + std::vector logits_buf; + { + const int n_vocab = llama_vocab_n_tokens(vocab); + const float * logits_ptr = llama_get_logits(ctx_tgt); + if (logits_ptr && n_vocab > 0) { + logits_size = (uint64_t)n_vocab * sizeof(float); + logits_buf.assign(reinterpret_cast(logits_ptr), + reinterpret_cast(logits_ptr) + (size_t)logits_size); + } + } + + SRV_INF("hydra: PREFILL slot=%d done n_past=%d kv=%zu logits=%" PRIu64 "B total=%zu\n", + id_slot, n_tokens, state_size, logits_size, v2_hdr.size() + state_size + (size_t)logits_size); + + // M-Perf.9 #289: model identity for the slot the prefill + // was just built on. Coordinator uses this to populate + // item.KvModelAlias/Hash and to gate RestoreKvAsync. When + // a `model` swap happened earlier in this handler, the + // post-swap `model_name` / `params_base.model.path` / + // `model` are used. `res->model_fallback` was set by the + // model-resolution block above; we preserve it here. + res->model_alias = model_name; + res->model_path = params_base.model.path; + // res->model_fallback may already be true (alias unknown + // or no preset); only set false when no swap was needed. + if (!model_was_swapped && !res->model_fallback) { + // nothing to do — leave as-is + } + if (model_tgt) { + res->tokenizer = llama_model_get_tokenizer_model(model_tgt); + res->model_name = llama_model_get_display_name(model_tgt); + res->model_quant = llama_model_get_quant_label(model_tgt); + res->model_capabilities = llama_model_get_capabilities_bitfield(model_tgt); + } + + res->rpc_status = HYDRA_STATUS_OK; + res->n_past = n_tokens; + res->state_size = state_size; + res->logits_size = logits_size; + // #451: populate PREFILL metrics + res->prefill_ms = (double)(ggml_time_ms() - prefill_start_ms); + res->prompt_tokens = n_tokens; + res->kv_size = state_size; + if (res->prefill_ms > 0 && n_tokens > 0) { + res->tokens_per_second = (double)n_tokens / (res->prefill_ms / 1000.0); + } + res->cache_tokens = slot->n_prompt_tokens_cache; + + const int hydra_fd = task.hydra_action.hydra_fd; + if (hydra_fd >= 0) { + // M2 path (#470): stream the response straight to the + // socket — 12B header + meta JSON + v2 header, then the + // GPU KV state zero-copy (chunked via cparams.hydra_state_chunk_size), + // then the (small) logits tail. No full-blob RAM buffer: + // at 60-80K context the blob is ~800 MB and grows toward + // 10 GB; buffering it doubled engine peak memory and the + // send only started after compute + full buffer completed. + // Wire layout is byte-identical to M1: payload = + // v2_hdr + KV state + logits, payload_len = the same + // total the coordinator computes from meta. + const size_t total_payload = v2_hdr.size() + state_size + (size_t)logits_size; + + // M2 (#470): pre-compute the wire hash of the whole kv + // segment — v2 header, then [4B magic][4B seq_id] + KV + // state (hash-only pass in wire order), then the logits + // tail. The meta must carry it BEFORE the first payload + // byte goes out (the coordinator forwards it into the + // DECODE frame header, and DECODE_APPLY verifies the + // streamed restore end-to-end). The slot is exclusively + // held by this task, so the state cannot change between + // the hash pass and the stream. + XXH3_state_t * kv_hst = nullptr; + if (hydra_fd >= 0) { + kv_hst = XXH3_createState(); + XXH3_64bits_reset(kv_hst); + XXH3_64bits_update(kv_hst, v2_hdr.data(), v2_hdr.size()); + const size_t hashed = llama_state_seq_hash(ctx_tgt, slot->id, kv_hst); + // state_size (from llama_state_seq_get_size) ALREADY includes the + // [4B magic][4B seq_id] wire header — llama_io_write_dummy counts it. + // llama_state_seq_hash hashes the same [4B magic][4B seq_id] + KV + // bytes, so after the n_bytes() fix hashed == state_size exactly. + // Adding sizeof(uint32_t) + sizeof(llama_seq_id) here double-counted + // the header and killed every PREFILL M2 request (#470). + if (hashed != state_size) { + res->rpc_status = HYDRA_STATUS_ERROR; + res->error = "PREFILL M2: hash pre-pass hashed " + + std::to_string(hashed) + " B, expected " + + std::to_string(state_size) + " B"; + } + if (!logits_buf.empty()) { + XXH3_64bits_update(kv_hst, logits_buf.data(), logits_buf.size()); + } + if (res->rpc_status == HYDRA_STATUS_OK) { + const uint64_t kv_hash = XXH3_64bits_digest(kv_hst); + char hash_hex[17]; + snprintf(hash_hex, sizeof(hash_hex), "%016" PRIx64, kv_hash); + res->kv_hash_str = std::string("xxh3:") + hash_hex; + } + XXH3_freeState(kv_hst); + kv_hst = nullptr; + } + + json meta_j = { + {"n_past", res->n_past}, + {"state_size", res->state_size}, + {"logits_size", res->logits_size} + }; + if (!res->model_alias.empty()) meta_j["model_alias"] = res->model_alias; + if (!res->model_path.empty()) meta_j["model_path"] = res->model_path; + if (!res->tokenizer.empty()) meta_j["tokenizer"] = res->tokenizer; + if (!res->model_name.empty()) meta_j["model_name"] = res->model_name; + if (!res->model_quant.empty()) meta_j["model_quant"] = res->model_quant; + if (res->model_capabilities) meta_j["model_capabilities"] = res->model_capabilities; + meta_j["model_fallback"] = res->model_fallback; + if (res->prefill_ms > 0) meta_j["prefill_ms"] = res->prefill_ms; + if (res->model_load_ms > 0) meta_j["model_load_ms"] = res->model_load_ms; + if (!res->kv_hash_str.empty()) meta_j["kv_hash_str"] = res->kv_hash_str; + const std::string meta_str = meta_j.dump(); + const uint32_t meta_len = (uint32_t)meta_str.size(); + + uint8_t hdr[HYDRA_RES_HEADER_SIZE] = {}; + hdr[0] = HYDRA_STATUS_OK; + hdr[1] = (meta_len) & 0xFF; + hdr[2] = (meta_len >> 8) & 0xFF; + hdr[3] = (meta_len >> 16) & 0xFF; + memcpy(hdr + 4, &total_payload, 8); + if (!hydra_send_all(hydra_fd, hdr, HYDRA_RES_HEADER_SIZE) || + !hydra_send_all(hydra_fd, meta_str.data(), meta_str.size()) || + !hydra_send_all(hydra_fd, v2_hdr.data(), v2_hdr.size())) { + res->rpc_status = HYDRA_STATUS_ERROR; + res->error = "PREFILL M2: response header/meta/v2-hdr send failed"; + ::shutdown(hydra_fd, SHUT_RDWR); + } else { + res->header_sent = true; // META + header + v2-hdr before payload + // Stream GPU state to fd (zero-copy from GPU memory; + // the wire hash was pre-computed above — pass no + // hash state so the io does not double-feed it) + const size_t streamed = llama_state_seq_get_data_to_fd(ctx_tgt, slot->id, hydra_fd, nullptr); + if (streamed != state_size) { + // TOCTOU: state size changed between get_size + // (above) and the stream, or the stream failed + // mid-way. The wire framing is now broken — the + // only safe recovery is to kill the connection. + // shutdown(), not close(): the RPC connection + // loop owns the fd (mirrors STATE_GET M2). + res->rpc_status = HYDRA_STATUS_ERROR; + res->error = "llama_state_seq_get_data_to_fd streamed " + + std::to_string(streamed) + " B, expected " + + std::to_string(state_size) + " B"; + ::shutdown(hydra_fd, SHUT_RDWR); + } else { + // Logits tail after the state stream — PREFILL's + // payload includes logits_size bytes at the end + // (STATE_GET M2 does not send logits). + if (!logits_buf.empty()) { + if (!hydra_send_all(hydra_fd, logits_buf.data(), logits_buf.size())) { + res->rpc_status = HYDRA_STATUS_ERROR; + res->error = "PREFILL M2: logits tail send failed"; + ::shutdown(hydra_fd, SHUT_RDWR); + } + } + if (res->rpc_status == HYDRA_STATUS_OK) { + res->streamed_bytes = (uint64_t)total_payload; + } + } + } + } else { + // M1 path: buffer the full blob in memory; the RPC thread + // sends header + meta + payload afterwards (unchanged). + // v2 blob format (0x02): [1B version][4B n_past][4B n_tok][n_tok*4B tokens] + // [1B flags (bit 0 = has_checkpoint)] + // [if flags & 0x01: 4B pos_min | 4B pos_max | 8B n_tokens | 8B tgt_sz | tgt_data | 8B dft_sz | dft_data] + // [raw KV state from llama_state_seq_get_data] + // [logits (n_vocab * float)] + std::vector v2_blob(v2_hdr.size() + state_size + (size_t)logits_size); + { + size_t off = 0; + memcpy(v2_blob.data() + off, v2_hdr.data(), v2_hdr.size()); + off += v2_hdr.size(); + if (state_size > 0) { + llama_state_seq_get_data(ctx_tgt, v2_blob.data() + off, state_size, slot->id); + } + } + if (!logits_buf.empty()) { + memcpy(v2_blob.data() + v2_hdr.size() + state_size, logits_buf.data(), logits_buf.size()); + } + res->state_data = std::move(v2_blob); + } + // #469 trace: log PREFILL completion with token IDs for cross-flow comparison + SRV_DBG("hydra: PREFILL_DONE slot=%d n_past=%d state_size=%zu logits_size=%zu blob_size=%zu prefill_ms=%.1f\n", + id_slot, n_tokens, state_size, logits_size, + (hydra_fd >= 0) ? v2_hdr.size() + state_size + (size_t)logits_size : res->state_data.size(), + res->prefill_ms); + queue_results.send(std::move(res)); + } break; + + case SERVER_TASK_TYPE_HYDRA_ENGINE_DECODE: + { + // ── Sync phase: Gate A (header-only, no GGUF reads, ~1 ms) ── + // Identity validation, slot reservation, post DECODE_APPLY. + // No model I/O, no KV touched. + const int id_slot = task.hydra_action.id_slot; + const int32_t decode_request_id = task.hydra_action.decode_request_id; + auto res = std::make_unique(); + res->id = task.id; + res->op = HYDRA_OP_DECODE; + res->decode_request_id = decode_request_id; + res->id_slot = id_slot; + + server_slot * slot = get_slot_by_id(id_slot); + if (slot == nullptr) { + res->rpc_status = HYDRA_STATUS_NOT_FOUND; + res->error = "invalid slot ID"; + queue_results.send(std::move(res)); + break; + } + + if (slot->is_processing()) { + res->rpc_status = HYDRA_STATUS_BUSY; + res->error = "slot is busy"; + queue_results.send(std::move(res)); + break; + } + + // Reject if slot is reserved for another decode + if (slot->reserved_for_decode_id != -1 && slot->reserved_for_decode_id != decode_request_id) { + res->rpc_status = HYDRA_STATUS_BUSY; + res->error = "slot reserved for another decode"; + queue_results.send(std::move(res)); + break; + } + + // Parse the merged DECODE JSON header + json decode_req; + try { + decode_req = json::parse(task.hydra_action.decode_json); + } catch (const std::exception & e) { + res->rpc_status = HYDRA_STATUS_BAD_REQUEST; + res->error = std::string("invalid JSON: ") + e.what(); + queue_results.send(std::move(res)); + break; + } + + // ── Gate A: header-only metadata comparison ───────────── + // Compare kv_metadata vs model_metadata from the control + // header. No GGUF reads, no KV touched. + const json & kv_meta = decode_req["kv_metadata"]; + const json & model_meta = decode_req.value("model_metadata", json::object()); + + // Read request identities from header + const std::string req_tokenizer = kv_meta.value("tokenizer", ""); + const std::string req_model_name = kv_meta.value("model_name", ""); + const uint32_t req_capabilities = kv_meta.value("model_capabilities", 0u); + + // Read target identities from header + const std::string tgt_tokenizer = model_meta.value("tokenizer", ""); + const std::string tgt_model_name = model_meta.value("model_name", ""); + + const bool tokenizer_match = (req_tokenizer == tgt_tokenizer); + bool model_name_match = (req_model_name == tgt_model_name); + // #589: cross-node same-model name tolerance. The KV's + // model_name is the display name (GGUF metadata) of the + // file that BUILT the KV — a different build/quant of the + // same model than the decode node's resident file, so + // string equality legitimately fails for the same logical + // model (e.g. kv_metadata carries the source node's + // display name, the decode node reports its resident + // filename). When the header carries the KV's source + // alias (kv_metadata.model_alias) or the resolved request + // alias ("model") and that alias maps through the preset + // table to the resident model path, the KV was built by + // the same logical model — accept. The alias→path check + // is exact (per-node preset INI), so a different model + // (Mini vs Balanced, 27B vs 35B) still maps to a + // different path and is rejected. + if (!model_name_match) { + const std::string kv_alias = kv_meta.value("model_alias", ""); + const std::string hdr_alias = decode_req.value("model", std::string()); + for (const auto & cand : { kv_alias, hdr_alias }) { + if (cand.empty()) { + continue; + } + auto pit = preset_alias_to_path.find(cand); + if (pit != preset_alias_to_path.end() && pit->second == params_base.model.path) { + SRV_INF("hydra: DECODE slot=%d Gate A name fallback — alias '%s' maps to resident path, same logical model\n", + id_slot, cand.c_str()); + model_name_match = true; + break; + } + } + } + const uint32_t capabilities_xor = req_capabilities ^ model_meta.value("model_capabilities", 0u); + + static const char * kCapBitNames[] = {"MTP", "VISION", "REASONING", "TOOL_USE", "CODE"}; + std::vector capabilities_diff_bits; + for (int b = 0; b < 5; b++) { + if (capabilities_xor & (1u << b)) { + capabilities_diff_bits.push_back(kCapBitNames[b]); + } + } + + // MTP(bit0) + VISION(bit1) mismatch → hard reject + const bool valid = tokenizer_match && model_name_match + && !(capabilities_xor & 0x3); + + json match_j = { + {"tokenizer_match", tokenizer_match}, + {"model_name_match", model_name_match}, + {"capabilities_xor", capabilities_xor}, + {"capabilities_diff_bits", capabilities_diff_bits}, + {"model_quant_match", kv_meta.value("model_quant", "") == model_meta.value("model_quant", "")}, + {"model_alias_match", true}, + }; + res->match_json = match_j; + res->match_valid = valid; + + if (!valid) { + res->rpc_status = HYDRA_STATUS_ERROR; + res->error = "model_capabilities_mismatch"; + SRV_WRN("hydra: DECODE slot=%d Gate A reject — tokenizer=%d name=%d caps_xor=0x%x\n", + id_slot, tokenizer_match, model_name_match, capabilities_xor); + queue_results.send(std::move(res)); + break; + } + + // ── Reserve slot ──────────────────────────────────────── + slot->reserved_for_decode_id = decode_request_id; + + SRV_INF("hydra: DECODE slot=%d Gate A pass, reserved for request_id=%d\n", + id_slot, decode_request_id); + + // ── Create decode_result_entry (LOADING state) ───────── + // So GET /v1/decode/{id} returns 202 instead of 404 + // while async DECODE_APPLY is pending. + if (routes_ptr) { + server_routes::decode_result_entry entry; + entry.id_slot = id_slot; + entry.state = server_routes::DECODE_STATE_LOADING; + entry.match_json = match_j; + entry.created_at = std::time(nullptr); + entry.ttl_s = routes_ptr->decode_result_ttl_s; + entry.model_metadata = decode_req.value("model_metadata", json::object()); + entry.model_identity = json::object(); + std::lock_guard lock(routes_ptr->decode_results_mutex); + routes_ptr->decode_results[decode_request_id] = std::move(entry); + routes_ptr->evict_decode_results_locked(); + } + + // ── Send sync validation response ─────────────────────── + res->rpc_status = HYDRA_STATUS_OK; + queue_results.send(std::move(res)); + + // ── Post DECODE_APPLY async task ──────────────────────── + { + server_task apply_task(SERVER_TASK_TYPE_HYDRA_DECODE_APPLY); + apply_task.id = queue_tasks.get_new_id(); + apply_task.hydra_action.id_slot = id_slot; + apply_task.hydra_action.decode_json = std::move(task.hydra_action.decode_json); + apply_task.hydra_action.kv_data = std::move(task.hydra_action.kv_data); + apply_task.hydra_action.decode_request_id = decode_request_id; + queue_tasks.post(std::move(apply_task)); + SRV_INF("hydra: DECODE slot=%d posted DECODE_APPLY (request_id=%d)\n", + id_slot, decode_request_id); + } + } break; + + case SERVER_TASK_TYPE_HYDRA_DECODE_APPLY: + { + // ── Async phase: model swap + Gate B + KV restore + completion ── + const int id_slot = task.hydra_action.id_slot; + const int32_t decode_request_id = task.hydra_action.decode_request_id; + + // Parse the DECODE JSON header (re-parsed for async context) + json decode_req; + try { + decode_req = json::parse(task.hydra_action.decode_json); + } catch (const std::exception & e) { + SRV_WRN("hydra: DECODE_APPLY slot=%d invalid JSON: %s\n", id_slot, e.what()); + // Release reservation on error + server_slot * s = get_slot_by_id(id_slot); + if (s) s->reserved_for_decode_id = -1; + if (routes_ptr) { + server_routes::decode_result_entry entry; + entry.id_slot = id_slot; + entry.error = std::string("DECODE_APPLY JSON parse error: ") + e.what(); + entry.created_at = std::time(nullptr); + entry.ttl_s = routes_ptr->decode_result_ttl_s; + std::lock_guard lock(routes_ptr->decode_results_mutex); + routes_ptr->decode_results[decode_request_id] = std::move(entry); + routes_ptr->evict_decode_results_locked(); + } + break; + } + + const json & kv_meta = decode_req["kv_metadata"]; + const json & model_meta = decode_req.value("model_metadata", json::object()); + + // ── Model swap (if requested model != resident) ───────── + const std::string requested_model = decode_req.value("model", std::string()); + double model_load_ms = 0.0; + bool model_fallback = false; + + if (!requested_model.empty()) { + // #470: resolve the requested alias against the + // T3-CURRENT alias → file map FIRST. The coordinator's + // T3 config (model_path) can load a file the preset INI + // does not associate with the engine's current alias + // (e.g. the dense-27b-combined session T3-loads the + // 27B-Coder file while the alias identity still says + // qwen3.6-35B-balanced). When the requested alias's + // T3-current file == resident, the alias describes the + // resident — swapping to the INI's file would be a + // pointless 73-81s reload + COMBINED teardown/reattach + // that then fails Gate B (header model_metadata of the + // pre-swap resident vs the swapped-in model's identity). + const auto t3it = t3_current_alias_to_path.find(requested_model); + if (t3it != t3_current_alias_to_path.end() && t3it->second == params_base.model.path) { + SRV_INF("hydra: DECODE_APPLY slot=%d model='%s' matches T3-current resident '%s' — no swap\n", + id_slot, requested_model.c_str(), params_base.model.path.c_str()); + } else { + auto it = preset_alias_to_path.find(requested_model); + if (it == preset_alias_to_path.end()) { + SRV_WRN("hydra: DECODE_APPLY slot=%d model='%s' unknown — falling back to resident '%s'\n", + id_slot, requested_model.c_str(), model_name.c_str()); + model_fallback = true; + } else if (it->second != params_base.model.path) { + SRV_INF("hydra: DECODE_APPLY slot=%d model='%s' swapping %s -> %s\n", + id_slot, requested_model.c_str(), params_base.model.path.c_str(), + it->second.c_str()); + common_params swapped_params = params_base; + // Apply the target alias's full preset (same + // treatment, and same intentional full-preset + // scope, as the PREFILL path above). + auto pit = preset_alias_to_preset.find(requested_model); + bool preset_apply_failed = false; + if (pit != preset_alias_to_preset.end()) { + // Same clear+re-pad+try/catch as the PREFILL path. + swapped_params.tensor_buft_overrides.clear(); + try { + pit->second.apply_to_params(swapped_params); + hydra_repad_tensor_buft_overrides(swapped_params, "DECODE_APPLY swap"); + } catch (const std::exception & e) { + SRV_WRN("hydra: DECODE_APPLY slot=%d swap preset apply for '%s' failed: %s\n", + id_slot, requested_model.c_str(), e.what()); + preset_apply_failed = true; + if (routes_ptr) { + server_routes::decode_result_entry entry; + entry.id_slot = id_slot; + entry.error = std::string("model swap preset apply failed: ") + e.what(); + entry.created_at = std::time(nullptr); + entry.ttl_s = routes_ptr->decode_result_ttl_s; + std::lock_guard lock(routes_ptr->decode_results_mutex); + routes_ptr->decode_results[decode_request_id] = std::move(entry); + routes_ptr->evict_decode_results_locked(); + } + } + } + if (preset_apply_failed) { + server_slot * s = get_slot_by_id(id_slot); + if (s) s->reserved_for_decode_id = -1; + break; + } + swapped_params.model.path = it->second; + swapped_params.model_alias = { requested_model }; + // #514: tear down COMBINED state before the + // reload — see hydra_teardown_combined_before_reload(). + const bool was_combined = hydra_combined_head_attached || hydra_combined_static; + if (was_combined) { + hydra_teardown_combined_before_reload(); + } + const int64_t model_load_start_ms = ggml_time_ms(); + if (!load_model(swapped_params)) { + SRV_WRN("hydra: DECODE_APPLY slot=%d model swap to '%s' failed\n", + id_slot, requested_model.c_str()); + server_slot * s = get_slot_by_id(id_slot); + if (s) s->reserved_for_decode_id = -1; + if (routes_ptr) { + server_routes::decode_result_entry entry; + entry.id_slot = id_slot; + entry.error = "model swap to '" + requested_model + "' failed"; + entry.created_at = std::time(nullptr); + entry.ttl_s = routes_ptr->decode_result_ttl_s; + std::lock_guard lock(routes_ptr->decode_results_mutex); + routes_ptr->decode_results[decode_request_id] = std::move(entry); + routes_ptr->evict_decode_results_locked(); + } + break; + } + if (was_combined) { + hydra_reattach_combined_after_reload(); + } + model_load_ms = (double)(ggml_time_ms() - model_load_start_ms); + SRV_INF("hydra: DECODE_APPLY slot=%d swap confirmed model_load_ms=%.1f\n", + id_slot, model_load_ms); + } + } + } + + // ── Gate B: post-load identity check ──────────────────── + // Compare model_metadata from header vs ACTUAL resident GGUF identity. + server_slot * slot = get_slot_by_id(id_slot); + if (slot == nullptr) { + SRV_WRN("hydra: DECODE_APPLY slot=%d disappeared after model swap\n", id_slot); + if (routes_ptr) { + server_routes::decode_result_entry entry; + entry.id_slot = id_slot; + entry.error = "slot disappeared after model swap"; + entry.created_at = std::time(nullptr); + entry.ttl_s = routes_ptr->decode_result_ttl_s; + std::lock_guard lock(routes_ptr->decode_results_mutex); + routes_ptr->decode_results[decode_request_id] = std::move(entry); + routes_ptr->evict_decode_results_locked(); + } + break; + } + + const std::string resident_tokenizer = llama_model_get_tokenizer_model(model_tgt); + const std::string resident_model_name = llama_model_get_display_name(model_tgt); + const std::string resident_model_quant = llama_model_get_quant_label(model_tgt); + const uint32_t resident_capabilities = llama_model_get_capabilities_bitfield(model_tgt); + + const std::string hdr_model_name = model_meta.value("model_name", ""); + const std::string hdr_model_quant = model_meta.value("model_quant", ""); + const uint32_t hdr_capabilities = model_meta.value("model_capabilities", 0u); + + const bool gate_b_tokenizer = (resident_tokenizer == model_meta.value("tokenizer", "")); + const bool gate_b_model_name = (resident_model_name == hdr_model_name); + const uint32_t gate_b_caps_xor = resident_capabilities ^ hdr_capabilities; + + if (!gate_b_tokenizer || !gate_b_model_name || (gate_b_caps_xor & 0x3)) { + SRV_WRN("hydra: DECODE_APPLY slot=%d Gate B reject — tokenizer=%d name=%d caps_xor=0x%x\n", + id_slot, gate_b_tokenizer, gate_b_model_name, gate_b_caps_xor); + slot->reserved_for_decode_id = -1; + if (routes_ptr) { + server_routes::decode_result_entry entry; + entry.id_slot = id_slot; + entry.error = "Gate B identity mismatch after model load"; + entry.match_json = {{"gate_b_tokenizer", gate_b_tokenizer}, {"gate_b_name", gate_b_model_name}, {"gate_b_caps_xor", gate_b_caps_xor}}; + entry.created_at = std::time(nullptr); + entry.ttl_s = routes_ptr->decode_result_ttl_s; + std::lock_guard lock(routes_ptr->decode_results_mutex); + routes_ptr->decode_results[decode_request_id] = std::move(entry); + routes_ptr->evict_decode_results_locked(); + } + break; + } + + if (resident_model_quant != hdr_model_quant) { + SRV_INF("hydra: DECODE_APPLY slot=%d Gate B quant differs (%s → %s) — mix-quant allowed\n", + id_slot, hdr_model_quant.c_str(), resident_model_quant.c_str()); + } + + // ── KV restore ───────────────────────────────────────── + const int64_t restore_start_ms = ggml_time_ms(); + + // M2 (#470): the v2 header arrives pre-parsed (kv_v2_hdr, + // small) and the KV state stream is read directly off + // hydra_fd via llama_state_seq_set_data_from_fd — the engine + // never materializes the full blob (2.3 GB today, 10 GB + // target). M1 (kv_data) is the buffered fallback. + const bool m2_stream = !task.hydra_action.kv_v2_hdr.empty(); + if (!task.hydra_action.kv_data.empty() || m2_stream) { + slot->prompt_clear(false); + slot->n_prompt_tokens_cache = 0; + slot->n_prompt_tokens_processed = 0; + slot->n_decoded = 0; + + // The coordinator may send the v2/v3 blob (header + raw KV) + // or just the raw KV data. Parse the v2 header to extract + // the token list so update_slots()'s n_common decision can + // match incoming tokens against the restored KV — without + // this, prompt.tokens is empty after prompt_clear(), n_past + // computes to 0, and seq_rm(slot, 0, -1) wipes the KV that + // llama_state_seq_set_data just loaded (issue #506). + const uint8_t * kv_ptr = m2_stream + ? task.hydra_action.kv_v2_hdr.data() + : task.hydra_action.kv_data.data(); + size_t kv_len = m2_stream + ? task.hydra_action.kv_v2_hdr.size() + : task.hydra_action.kv_data.size(); + int32_t blob_n_past = 0; + int32_t blob_n_tok = 0; + bool has_chkpt = false; + bool ckpt_is_recr_only = false; + int32_t ckpt_pos_min_in = 0, ckpt_pos_max_in = 0; + int64_t ckpt_n_tokens_in = 0; + std::vector ckpt_tgt_data, ckpt_dft_data; + + // v2 (0x02) blobs carry a full checkpoint; v3 (0x03) blobs may carry + // a recurrent-only checkpoint (hdr_flags bit 0x02). Both share the + // header layout — the M2-stream double-write fix bumped the version. + const bool is_v2 = kv_len >= 1 && (kv_ptr[0] == 0x02 || kv_ptr[0] == 0x03); + if (is_v2 && kv_len >= 9) { + memcpy(&blob_n_past, kv_ptr + 1, 4); + memcpy(&blob_n_tok, kv_ptr + 5, 4); + + const size_t token_start = 9; + const size_t token_end = token_start + (size_t)blob_n_tok * sizeof(llama_token); + if (blob_n_tok > 0 && token_end <= kv_len) { + // Restore token list from v2/v3 blob header + slot->prompt.tokens.clear(); + const llama_token * tok_ptr = (const llama_token *)(kv_ptr + token_start); + llama_tokens restored_tokens(tok_ptr, tok_ptr + (size_t)blob_n_tok); + slot->prompt.tokens.insert(restored_tokens); + SRV_INF("hydra: DECODE_APPLY slot=%d v2/v3 blob: restored %d tokens from header\n", + id_slot, blob_n_tok); + } + + // Skip past v2/v3 header (version + n_past + n_tok + tokens + flags + optional checkpoint) + size_t hdr_offset = token_end; + if (hdr_offset < kv_len) { + const uint8_t flags = kv_ptr[hdr_offset]; + hdr_offset += 1; // past flags byte + // bit 0x01 = has checkpoint; bit 0x02 = recurrent-only (PARTIAL_ONLY) + ckpt_is_recr_only = (flags & 0x02) != 0; + if (flags & 0x01) { + // Capture checkpoint: 4B pos_min | 4B pos_max | 8B n_tokens | 8B tgt_sz | tgt_data | 8B dft_sz | dft_data + // Mirrors the STATE_PUT sibling (~line 3343) — the native + // checkpoint is registered after restore so hybrid/recurrent + // models get their recurrent memory back (KV restored without + // its checkpoint is corrupt). + if (hdr_offset + 4 + 4 + 8 + 8 <= kv_len) { + memcpy(&ckpt_pos_min_in, kv_ptr + hdr_offset, 4); hdr_offset += 4; + memcpy(&ckpt_pos_max_in, kv_ptr + hdr_offset, 4); hdr_offset += 4; + memcpy(&ckpt_n_tokens_in, kv_ptr + hdr_offset, 8); hdr_offset += 8; + uint64_t tgt_sz_in; + memcpy(&tgt_sz_in, kv_ptr + hdr_offset, 8); hdr_offset += 8; + if (tgt_sz_in > 0 && hdr_offset + tgt_sz_in <= kv_len) { + ckpt_tgt_data.assign(kv_ptr + hdr_offset, kv_ptr + hdr_offset + (size_t)tgt_sz_in); + hdr_offset += (size_t)tgt_sz_in; + } + if (hdr_offset + 8 <= kv_len) { + uint64_t dft_sz_in; + memcpy(&dft_sz_in, kv_ptr + hdr_offset, 8); hdr_offset += 8; + if (dft_sz_in > 0 && hdr_offset + dft_sz_in <= kv_len) { + ckpt_dft_data.assign(kv_ptr + hdr_offset, kv_ptr + hdr_offset + (size_t)dft_sz_in); + hdr_offset += (size_t)dft_sz_in; + } + } + has_chkpt = true; + } + } + } + // Advance kv_ptr/kv_len past the v2 header to the raw KV state + if (hdr_offset <= kv_len) { + kv_ptr = kv_ptr + hdr_offset; + kv_len = kv_len - hdr_offset; + } + } + + // M2 (#470): hash the whole kv segment as it streams — + // v2 header first, then every byte the fd restore + // consumes, then the logits tail (wire order). + XXH3_state_t * hst = nullptr; + if (m2_stream) { + hst = XXH3_createState(); + XXH3_64bits_reset(hst); + XXH3_64bits_update(hst, task.hydra_action.kv_v2_hdr.data(), + task.hydra_action.kv_v2_hdr.size()); + } + + size_t status = 0; + if (m2_stream) { + // Stream restore: consumes [4B magic][4B seq_id] + KV + // state off the fd; logits tail is read separately below. + status = llama_state_seq_set_data_from_fd( + ctx_tgt, slot->id, task.hydra_action.hydra_fd, hst); + } else { + status = llama_state_seq_set_data( + ctx_tgt, + kv_ptr, + kv_len, + slot->id); + } + + // llama_state_seq_set_data returns the number of bytes + // read on success (0 means failed to load) — see its + // doc comment in include/llama.h. `status` only counts + // the KV-cache bytes the reader consumed; it does NOT + // include the trailing logits PREFILL_DONE appends + // (see ~line 4098), so status < kv_len is the normal + // case whenever logits are present — compare against + // kv_len here and this false-fails on every restore + // with logits. Matches the STATE_PUT sibling check + // (server-context.cpp ~line 3395: `if (n_read == 0)`). + if (status == 0) { + SRV_WRN("hydra: DECODE_APPLY slot=%d KV restore failed (%d)\n", id_slot, status); + if (hst) { XXH3_freeState(hst); hst = nullptr; } + if (m2_stream) { + // The stream broke mid-way: drop the read side so + // residual unread bytes cannot misalign the next + // request frame. The RPC thread still writes the + // error response (write side stays open). + ::shutdown(task.hydra_action.hydra_fd, SHUT_RD); + } + slot->reserved_for_decode_id = -1; + // Tokens were registered from the v2 header before set_data — + // clear them so the slot is not left poisoned (n_past > 0 + // with no KV cells → pos_min == -1 abort on the next decode + // that touches this slot). Matches the STATE_PUT failure path. + slot->prompt.tokens.clear(); + slot->prompt.checkpoints.clear(); + slot->n_prompt_tokens_cache = 0; + llama_memory_seq_rm(llama_get_memory(ctx_tgt), slot->id, -1, -1); + if (routes_ptr) { + server_routes::decode_result_entry entry; + entry.id_slot = id_slot; + entry.error = "KV restore failed (llama_state_seq_set_data returned " + std::to_string(status) + ")"; + entry.created_at = std::time(nullptr); + entry.ttl_s = routes_ptr->decode_result_ttl_s; + std::lock_guard lock(routes_ptr->decode_results_mutex); + routes_ptr->decode_results[decode_request_id] = std::move(entry); + routes_ptr->evict_decode_results_locked(); + } + break; + } + + // Trailing logits: PREFILL_DONE appends n_vocab floats + // after the KV state (~line 4098) so the decode side + // can sample immediately instead of reading garbage + // after restore. Mirrors STATE_PUT's per-slot + // injection (~line 3405) — DECODE_APPLY was missing + // this step entirely. + { + const size_t expected_logits = (size_t)llama_vocab_n_tokens(vocab) * sizeof(float); + if (m2_stream) { + // Read the logits tail straight off the fd (small). + const size_t remaining = + (size_t)(task.hydra_action.kv_stream_len - status); + if (remaining == expected_logits) { + std::vector logits_buf(remaining); + if (hydra_recv_all(task.hydra_action.hydra_fd, + logits_buf.data(), remaining)) { + XXH3_64bits_update(hst, logits_buf.data(), remaining); + const size_t n_floats = llama_vocab_n_tokens(vocab); + slot->restored_logits.assign( + reinterpret_cast(logits_buf.data()), + reinterpret_cast(logits_buf.data()) + n_floats); + slot->logits_valid = true; + SRV_INF("hydra: DECODE_APPLY slot=%d restored %zu logits to per-slot buffer\n", + id_slot, n_floats); + } else { + SRV_WRN("hydra: DECODE_APPLY slot=%d logits tail read failed\n", id_slot); + } + } + } else { + const size_t remaining = kv_len - status; + if (remaining == expected_logits) { + const float * src = (const float *)(kv_ptr + status); + const size_t n_floats = llama_vocab_n_tokens(vocab); + slot->restored_logits.assign(src, src + n_floats); + slot->logits_valid = true; + SRV_INF("hydra: DECODE_APPLY slot=%d restored %zu logits to per-slot buffer\n", + id_slot, n_floats); + } + } + } + + // M2 wire-hash verification (post-restore — with streaming + // the bytes reach the GPU before a pre-restore hash could + // be computed). On mismatch the slot is cleared so the next + // decode cannot sample corrupt state, and the response + // carries the terminal error for the Coordinator to retry. + if (m2_stream && task.hydra_action.kv_expected_hash != 0) { + const uint64_t computed_kv = XXH3_64bits_digest(hst); + if (computed_kv != task.hydra_action.kv_expected_hash) { + SRV_WRN("hydra: DECODE_APPLY slot=%d SEGMENT_HASH_MISMATCH kv expected=%016" PRIx64 " got=%016" PRIx64 "\n", + id_slot, task.hydra_action.kv_expected_hash, computed_kv); + XXH3_freeState(hst); + hst = nullptr; + slot->reserved_for_decode_id = -1; + slot->prompt.tokens.clear(); + slot->prompt.checkpoints.clear(); + slot->n_prompt_tokens_cache = 0; + llama_memory_seq_rm(llama_get_memory(ctx_tgt), slot->id, -1, -1); + if (routes_ptr) { + server_routes::decode_result_entry entry; + entry.id_slot = id_slot; + entry.error = "KV segment hash mismatch (corrupt stream)"; + entry.created_at = std::time(nullptr); + entry.ttl_s = routes_ptr->decode_result_ttl_s; + std::lock_guard lock(routes_ptr->decode_results_mutex); + routes_ptr->decode_results[decode_request_id] = std::move(entry); + routes_ptr->evict_decode_results_locked(); + } + break; + } + SRV_INF("hydra: DECODE_APPLY slot=%d KV hash verified (%zu + %" PRIu64 " B)\n", + id_slot, task.hydra_action.kv_v2_hdr.size(), + task.hydra_action.kv_stream_len); + } + if (hst) { XXH3_freeState(hst); hst = nullptr; } + + const int n_past = is_v2 ? blob_n_past : kv_meta.value("n_past", 0); + if (n_past > 0) { + // Cache/processed counters come from the same header field + // STATE_PUT reads (hdr_n_tok == blob_n_tok here); PREFILL writes + // both fields as n_tokens so the values are identical today, + // but the two restore paths must read the SAME source. + slot->n_prompt_tokens_cache = is_v2 ? blob_n_tok : n_past; + slot->n_prompt_tokens_processed = is_v2 ? blob_n_tok : n_past; + + // Register the native checkpoint from the blob (v2) or + // fabricate one (legacy) — mirrors STATE_PUT (~line 3447). + // KV restored without its recurrent-memory checkpoint + // corrupts hybrid/recurrent model output. + slot->prompt.checkpoints.clear(); + if (has_chkpt) { + auto & ckpt = slot->prompt.checkpoints.emplace_back(); + ckpt.n_tokens = ckpt_n_tokens_in; + ckpt.pos_min = ckpt_pos_min_in; + ckpt.pos_max = ckpt_pos_max_in; + // New-format (v3) checkpoints carry a recurrent-only capture — + // route into data_*_recr and tag is_recr_only so the load path + // uses matched PARTIAL_ONLY flags (mirrors STATE_PUT). + ckpt.is_recr_only = ckpt_is_recr_only; + if (ckpt_is_recr_only) { + ckpt.data_tgt_recr = std::move(ckpt_tgt_data); + ckpt.data_dft_recr = std::move(ckpt_dft_data); + } else { + ckpt.data_tgt = std::move(ckpt_tgt_data); + ckpt.data_dft = std::move(ckpt_dft_data); + } + SLT_INF(*slot, "DECODE_APPLY registered native checkpoint (pos_min=%d pos_max=%d n_tokens=%" PRId64 " tgt_sz=%zu recr_only=%d)\n", + ckpt.pos_min, ckpt.pos_max, ckpt.n_tokens, ckpt.size(), (int) ckpt.is_recr_only); + } else { + const auto pos_min = llama_memory_seq_pos_min(llama_get_memory(ctx_tgt), slot->id); + create_checkpoint(*slot, 0, (llama_pos)pos_min, (llama_pos)(n_past - 1)); + } + } + slot->just_restored = true; + } + + const double restore_slot_ms = (double)(ggml_time_ms() - restore_start_ms); + const int n_past = slot->n_prompt_tokens_cache + slot->n_decoded; + + SRV_INF("hydra: DECODE_APPLY slot=%d restore=%.1fms n_past=%d model_load_ms=%.1f\n", + id_slot, restore_slot_ms, n_past, model_load_ms); + + // Release reservation — slot is now processing via completion + slot->reserved_for_decode_id = -1; + + // ── Build and post COMPLETION task ────────────────────── + { + json prompt = decode_req["prompt"]; + json cmpl_data; + cmpl_data["stream"] = prompt.value("stream", false); + // #622: the DECODE 0x43 frame has no dedicated stream_options + // channel, but the coordinator always requests usage on the + // merged path (it injects stream_options.include_usage=true on + // its HTTP body). Honor stream_options when the request carries + // it (generation header / prompt segment), otherwise mirror the + // coordinator's injection so the DONE-SSE delta carries usage + // natively and the coordinator's usage-based gate fires. + if (prompt.contains("stream_options") && prompt["stream_options"].is_object()) { + cmpl_data["stream_options"] = prompt["stream_options"]; + } else { + cmpl_data["stream_options"] = json{{"include_usage", true}}; + } + cmpl_data["n_predict"] = prompt.value("n_predict", 256); + cmpl_data["id_slot"] = id_slot; + if (prompt.contains("sampling")) { + const json & samp = prompt["sampling"]; + if (samp.contains("temperature")) cmpl_data["temperature"] = samp["temperature"]; + if (samp.contains("top_p")) cmpl_data["top_p"] = samp["top_p"]; + if (samp.contains("top_k")) cmpl_data["top_k"] = samp["top_k"]; + if (samp.contains("seed")) cmpl_data["seed"] = samp["seed"]; + } + if (prompt.contains("stop")) cmpl_data["stop"] = prompt["stop"]; + + std::string prompt_str; + if (prompt.contains("messages") && !prompt["messages"].is_null()) { + json chat_body; + chat_body["messages"] = prompt["messages"]; + if (prompt.contains("tools")) chat_body["tools"] = prompt["tools"]; + if (prompt.contains("tool_choice")) chat_body["tool_choice"] = prompt["tool_choice"]; + if (prompt.contains("response_format")) chat_body["response_format"] = prompt["response_format"]; + if (prompt.contains("add_generation_prompt")) chat_body["add_generation_prompt"] = prompt["add_generation_prompt"]; + if (prompt.contains("continue_final_message")) chat_body["continue_final_message"] = prompt["continue_final_message"]; + if (prompt.contains("reasoning_format")) chat_body["reasoning_format"] = prompt["reasoning_format"]; + if (prompt.contains("enable_thinking")) chat_body["enable_thinking"] = prompt["enable_thinking"]; + if (prompt.contains("chat_template_kwargs")) chat_body["chat_template_kwargs"] = prompt["chat_template_kwargs"]; + + try { + std::vector dummy_files; + json chat_result = oaicompat_chat_params_parse(chat_body, chat_params, dummy_files); + prompt_str = chat_result.value("prompt", std::string()); + if (chat_result.contains("grammar") && !chat_result["grammar"].is_null()) cmpl_data["grammar"] = chat_result["grammar"]; + if (chat_result.contains("grammar_type")) cmpl_data["grammar_type"] = chat_result["grammar_type"]; + if (chat_result.contains("grammar_lazy")) cmpl_data["grammar_lazy"] = chat_result["grammar_lazy"]; + if (chat_result.contains("grammar_triggers")) cmpl_data["grammar_triggers"] = chat_result["grammar_triggers"]; + if (chat_result.contains("chat_format")) cmpl_data["chat_format"] = chat_result["chat_format"]; + if (chat_result.contains("chat_parser")) cmpl_data["chat_parser"] = chat_result["chat_parser"]; + if (chat_result.contains("generation_prompt")) cmpl_data["generation_prompt"] = chat_result["generation_prompt"]; + if (chat_result.contains("parse_tool_calls")) cmpl_data["parse_tool_calls"] = chat_result["parse_tool_calls"]; + if (chat_result.contains("preserved_tokens")) cmpl_data["preserved_tokens"] = chat_result["preserved_tokens"]; + if (chat_result.contains("reasoning_budget_tokens")) cmpl_data["reasoning_budget_tokens"] = chat_result["reasoning_budget_tokens"]; + if (chat_result.contains("reasoning_budget_start_tag")) cmpl_data["reasoning_budget_start_tag"] = chat_result["reasoning_budget_start_tag"]; + if (chat_result.contains("reasoning_budget_end_tag")) cmpl_data["reasoning_budget_end_tag"] = chat_result["reasoning_budget_end_tag"]; + if (chat_result.contains("reasoning_budget_message")) cmpl_data["reasoning_budget_message"] = chat_result["reasoning_budget_message"]; + if (chat_result.contains("reasoning_control")) cmpl_data["reasoning_control"] = chat_result["reasoning_control"]; + if (chat_result.contains("stop") && chat_result["stop"].is_array()) { + json existing_stops = cmpl_data.value("stop", json::array()); + for (const auto & s : chat_result["stop"]) existing_stops.push_back(s); + cmpl_data["stop"] = existing_stops; + } + } catch (const std::exception & e) { + SRV_WRN("hydra: DECODE_APPLY slot=%d chat template failed: %s\n", id_slot, e.what()); + if (routes_ptr) { + server_routes::decode_result_entry entry; + entry.id_slot = id_slot; + entry.error = std::string("chat template error: ") + e.what(); + entry.created_at = std::time(nullptr); + entry.ttl_s = routes_ptr->decode_result_ttl_s; + std::lock_guard lock(routes_ptr->decode_results_mutex); + routes_ptr->decode_results[decode_request_id] = std::move(entry); + routes_ptr->evict_decode_results_locked(); + } + break; + } + } else { + prompt_str = prompt.value("prompt", std::string()); + } + cmpl_data["prompt"] = prompt_str; + + auto inputs = tokenize_input_prompts(vocab, mctx, prompt_str, true, true); + if (!inputs.empty()) { + const int32_t completion_id = queue_tasks.get_new_id(); + + server_task cmpl_task(SERVER_TASK_TYPE_COMPLETION); + cmpl_task.id = completion_id; + cmpl_task.id_slot = id_slot; + cmpl_task.tokens = std::move(inputs[0]); + cmpl_task.params = server_task::params_from_json_cmpl( + vocab, params_base, get_slot_n_ctx(), params_base.sampling.logit_bias_eog, cmpl_data); + cmpl_task.params.res_type = TASK_RESPONSE_TYPE_OAI_CHAT; + cmpl_task.params.oaicompat_cmpl_id = gen_chatcmplid(); + cmpl_task.params.oaicompat_model = model_name; + + // Mirror server_response_reader::post_task(): the + // consumer thread keeps its own result state so it + // can run result->update() per received result. + task_result_state cmpl_state = cmpl_task.create_state(); + + queue_results.add_waiting_task_id(completion_id); + queue_tasks.post(std::move(cmpl_task)); + SRV_INF("hydra: DECODE_APPLY slot=%d posted COMPLETION (completion_id=%d, request_id=%d)\n", + id_slot, completion_id, decode_request_id); + + // Update state to GENERATING + if (routes_ptr) { + std::lock_guard lk(routes_ptr->decode_results_mutex); + auto dit = routes_ptr->decode_results.find(decode_request_id); + if (dit != routes_ptr->decode_results.end()) { + dit->second.state = server_routes::DECODE_STATE_GENERATING; + dit->second.completion_id = std::to_string(completion_id); + dit->second.stream->completion_task_id = completion_id; + // Capture n_common observability from the slot + dit->second.n_common = slot->n_common; + dit->second.n_prompt_processed = slot->n_prompt_processed; + dit->second.logits_reused = slot->logits_reused; + } + } + + // ── Background consumer ───────────────────────── + // Sole listener on the completion task. Relays + // partial results into the decode_result_entry's + // streaming_queue so GET /v1/decode can stream + // them to the client. Stores the final result + // when generation completes. + if (routes_ptr) { + // Read match_json from the decode_result_entry (set by sync DECODE) + json match_j_bg; + { + std::lock_guard lk(routes_ptr->decode_results_mutex); + auto dit = routes_ptr->decode_results.find(decode_request_id); + if (dit != routes_ptr->decode_results.end()) { + match_j_bg = dit->second.match_json; + } + } + std::thread([this, completion_id, decode_request_id, id_slot, + match_j = std::move(match_j_bg), resident_tokenizer, resident_model_name, + resident_model_quant, resident_capabilities, + oaicompat_model_name = model_name, + model_load_ms, restore_slot_ms, n_past, + &results = queue_results, + states = std::vector{ std::move(cmpl_state) }]() mutable { + std::unordered_set ids = {(int)completion_id}; + bool got_final = false; + + // Loop: receive partials and relay, wait for final + while (!got_final) { + auto res_ptr = results.recv_with_timeout(ids, 120); + if (!res_ptr) { + SRV_WRN("hydra: DECODE_APPLY slot=%d generation timeout (request_id=%d, completion_id=%d)\n", + id_slot, decode_request_id, completion_id); + // Mark stream as finished so GET handler unblocks + { + std::lock_guard lk(routes_ptr->decode_results_mutex); + auto dit = routes_ptr->decode_results.find(decode_request_id); + if (dit != routes_ptr->decode_results.end() && dit->second.stream) { + std::lock_guard slk(dit->second.stream->streaming_mutex); + dit->second.stream->stream_finished = true; + dit->second.stream->streaming_cv.notify_all(); + } + } + return; + } + + // Check if this is a partial or final result + auto * partial = dynamic_cast(res_ptr.get()); + auto * final_r = dynamic_cast(res_ptr.get()); + + // Mirror server_response_reader::next(): run + // update() on every result before handling. + // Populates oaicompat_msg / oaicompat_msg_diffs + // (and sets is_updated, so to_json() won't + // assert on relayed partials). + try { + const size_t idx = res_ptr->index; + GGML_ASSERT(idx < states.size()); + res_ptr->update(states[idx]); + } catch (const std::exception & e) { + // Mirror the standard stream loop's tolerance + // of chat-parse failures (server-context.cpp:7685). + // This is a detached thread: an uncaught + // exception would std::terminate() the whole + // engine. Continue with the unparsed result + // (raw content; reasoning extraction skipped). + SRV_WRN("hydra: DECODE_APPLY slot=%d result update() failed: %s (continuing with unparsed result)\n", + id_slot, e.what()); + if (partial && !partial->is_begin) { + // Keep the relay well-formed: partial + // to_json() asserts is_updated in debug + // builds; with no diffs it emits an empty + // delta, which clients merge harmlessly. + partial->is_updated = true; + } + } + + if (partial && !partial->is_begin) { + // Relay partial to streaming queue + std::lock_guard lk(routes_ptr->decode_results_mutex); + auto dit = routes_ptr->decode_results.find(decode_request_id); + if (dit != routes_ptr->decode_results.end() && dit->second.stream) { + std::lock_guard slk(dit->second.stream->streaming_mutex); + dit->second.stream->streaming_queue.push_back(std::move(res_ptr)); + dit->second.stream->streaming_cv.notify_all(); + } + } else if (final_r) { + // Store final result and mark DONE + got_final = true; + + server_routes::decode_result_entry entry; + entry.id_slot = id_slot; + entry.completion_id = final_r->oaicompat_cmpl_id; + entry.oaicompat_model = oaicompat_model_name; + entry.content = final_r->content; + if (!final_r->oaicompat_msg.reasoning_content.empty()) { + entry.reasoning_content = final_r->oaicompat_msg.reasoning_content; + } + if (!final_r->oaicompat_msg.tool_calls.empty()) { + // Mirror common_chat_msg::to_json_oaicompat() shape so + // GET /v1/decode/:id returns OpenAI-format tool_calls. + json jtool_calls = json::array(); + for (const auto & tool_call : final_r->oaicompat_msg.tool_calls) { + json tc { + {"type", "function"}, + {"function", { + {"name", tool_call.name}, + {"arguments", json(tool_call.arguments)}, + }}, + }; + if (!tool_call.id.empty()) { + tc["id"] = tool_call.id; + } + jtool_calls.push_back(std::move(tc)); + } + entry.tool_calls = std::move(jtool_calls); + } + entry.n_decoded = final_r->n_decoded; + entry.n_prompt_tokens = final_r->n_prompt_tokens; + entry.n_prompt_tokens_cache = final_r->n_prompt_tokens_cache; + entry.timings = final_r->timings; + entry.stop = final_r->stop; + entry.include_usage = final_r->include_usage; + entry.match_json = match_j; + entry.created_at = std::time(nullptr); + entry.ttl_s = routes_ptr->decode_result_ttl_s; + + json metrics = json::object(); + metrics["decode_request_id"] = decode_request_id; + metrics["id_slot"] = id_slot; + metrics["n_past"] = final_r->n_prompt_tokens_cache + final_r->n_decoded; + metrics["decode_ms"] = final_r->timings.predicted_ms; + metrics["prompt_ms"] = final_r->timings.prompt_ms; + metrics["model_load_ms"] = model_load_ms; + metrics["restore_slot_ms"] = restore_slot_ms; + metrics["model_identity"] = { + {"tokenizer", resident_tokenizer}, + {"model_name", resident_model_name}, + {"model_quant", resident_model_quant}, + {"model_capabilities", resident_capabilities} + }; + metrics["match"] = match_j; + metrics["model_fallback"] = false; + // Hydra n_common observability + metrics["n_common"] = entry.n_common; + metrics["n_prompt_processed"] = entry.n_prompt_processed; + metrics["logits_reused"] = entry.logits_reused; + entry.hydra_metrics = metrics; + entry.state = server_routes::DECODE_STATE_DONE; + + // Signal stream finished before storing entry + { + std::lock_guard lk(routes_ptr->decode_results_mutex); + auto dit = routes_ptr->decode_results.find(decode_request_id); + if (dit != routes_ptr->decode_results.end() && dit->second.stream) { + // Transfer streaming state to the new entry + entry.stream = std::move(dit->second.stream); + { + std::lock_guard slk(entry.stream->streaming_mutex); + entry.stream->stream_finished = true; + } + entry.stream->streaming_cv.notify_all(); + } + } + + std::lock_guard lock(routes_ptr->decode_results_mutex); + routes_ptr->decode_results[decode_request_id] = std::move(entry); + routes_ptr->evict_decode_results_locked(); + + SRV_INF("hydra: DECODE_APPLY slot=%d generation complete (request_id=%d, n_decoded=%d)\n", + id_slot, decode_request_id, final_r->n_decoded); + } else { + // is_begin partial — just consume it + } + } + + results.remove_waiting_task_id(completion_id); + }).detach(); + } + } else { + SRV_WRN("hydra: DECODE_APPLY slot=%d tokenization failed\n", id_slot); + } + } + } break; + + case SERVER_TASK_TYPE_HYDRA_ENGINE_SET_EXPERT_MODE: + { + auto res = std::make_unique(); + res->id = task.id; + res->op = HYDRA_OP_SET_EXPERT_MODE; + + // Parse the payload. For backward compatibility, a raw string + // ("solo" or "combined") is accepted. Phase D (C# side) sends + // a JSON payload: {"mode":"combined","peer":"host:port",...}. + std::string requested; + std::string peer_override; + const std::string & raw = task.hydra_action.expert_mode; + if (!raw.empty() && raw[0] == '{') { + try { + json j = json::parse(raw); + requested = j.value("mode", "solo"); + peer_override = j.value("peer", ""); + } catch (...) { + requested = "solo"; + } + } else { + requested = raw; + } + + if (requested != "solo" && requested != "combined") { + res->rpc_status = HYDRA_STATUS_ERROR; + res->success = false; + res->error = "expert_mode must be 'solo' or 'combined'"; + queue_results.send(std::move(res)); + break; + } + + // #29 Phase B: per-request peer switching. If the peer changes, + // clean up the old binding and register the new one. The peer + // info comes from the SET_EXPERT_MODE control-plane payload + // (JSON {"mode":"combined","peer":"host:port"}), NOT from the + // HTTP inference body — keeping control and data separate. + if (!peer_override.empty() && peer_override != hydra_current_peer) { + // Guard: peer switch is unsafe while any slot is decoding. + // sched_reserve() destroys and rebuilds the scheduler, which + // invalidates in-flight decode state across all slots. + bool any_active = false; + for (const auto & s : slots) { + if (s.is_processing()) { any_active = true; break; } + } + if (any_active) { + SRV_WRN("hydra: cannot switch peers — %zu slot(s) are processing, rejecting SET_EXPERT_MODE\n", slots.size()); + res->rpc_status = HYDRA_STATUS_BUSY; + res->success = false; + res->error = "cannot switch peers while slots are processing"; + queue_results.send(std::move(res)); + break; + } + if (!hydra_current_peer.empty()) { + SRV_INF("hydra: switching from peer %s to %s — cleaning up old binding\n", + hydra_current_peer.c_str(), peer_override.c_str()); + ctx_tgt->hydra_remove_combined_rpc_backend(hydra_current_peer.c_str()); + } + hydra_current_peer = peer_override; + } + + // Hydra #383 T1: layer-split (static combined) engines cannot + // switch modes at runtime — the split is baked in at model load. + // "combined" is a no-op (already combined); "solo" is rejected. + if (hydra_combined_static) { + if (requested == "solo") { + res->rpc_status = HYDRA_STATUS_ERROR; + res->success = false; + res->error = "combined_static: this engine loaded in layer-split COMBINED mode; cannot switch to solo at runtime"; + LOG_WRN("srv %12.*s: hydra: SET_EXPERT_MODE solo rejected — engine is combined_static (layer-split)\n", 12, __func__); + queue_results.send(std::move(res)); + break; + } + // requested == "combined": success no-op + res->expert_mode_applied = "combined"; + res->rpc_status = HYDRA_STATUS_OK; + res->success = true; + LOG_INF("srv %12.*s: hydra: SET_EXPERT_MODE combined no-op — engine is combined_static (layer-split)\n", 12, __func__); + queue_results.send(std::move(res)); + break; + } + + // #368 fix: gate on "configured as combined head" (non-empty + // peer addr + OT pattern), NOT on whether the startup + // dual-load succeeded. The rebind path below is fail-open — + // if the peer is still unreachable it stays solo — so + // hydra_combined_head_attached (set only when startup + // succeeded) must NOT block the attempt. Hydra #287/#260/#348 + // intent is preserved: an unconfigured engine (no peer/ + // pattern) still falls back to solo immediately. + const bool want_combined = requested == "combined" && + !hydra_peer.empty() && !hydra_combined_pattern.empty(); + + // #368 (#357 fix): bind-on-activation. Re-bind the peer's + // expert tensors on each SET_EXPERT_MODE("combined") request + // so a peer that was down at boot is picked up on the first + // COMBINED request after it comes up. Fail-open: if the + // rebind fails we stay solo and the Coordinator's + // ReportsSolo path handles it. + bool actually_combined = want_combined; + if (want_combined) { + if (hydra_peer.empty() || hydra_combined_pattern.empty()) { + SRV_WRN("%s\n", "hydra: SET_EXPERT_MODE(combined) but no peer/pattern configured; staying solo"); + actually_combined = false; + } else { + // ggml_backend_rpc_add_server is idempotent — returns + // the existing reg if the peer was registered before. + ggml_backend_reg_t rpc_reg = ggml_backend_reg_by_name("RPC"); + if (!rpc_reg) { + SRV_WRN("%s\n", "hydra: SET_EXPERT_MODE(combined) but RPC backend not available; staying solo"); + actually_combined = false; + } else { + using add_server_fn_t = ggml_backend_reg_t (*)(const char *); + auto add_server_fn = (add_server_fn_t) ggml_backend_reg_get_proc_address(rpc_reg, "ggml_backend_rpc_add_server"); + ggml_backend_reg_t peer_reg = add_server_fn ? add_server_fn(hydra_peer.c_str()) : nullptr; + ggml_backend_dev_t peer_dev = (peer_reg && ggml_backend_reg_dev_count(peer_reg) > 0) ? ggml_backend_reg_dev_get(peer_reg, 0) : nullptr; + if (!peer_dev) { + SRV_WRN("hydra: SET_EXPERT_MODE(combined) but peer %s has no registered device; staying solo\n", + hydra_peer.c_str()); + actually_combined = false; + } else { + int32_t n_bound = llama_hydra_rebind_combined_experts( + ctx_tgt, hydra_peer.c_str(), peer_dev, hydra_combined_pattern.c_str()); + if (n_bound <= 0) { + SRV_WRN("hydra: SET_EXPERT_MODE(combined) rebind on peer %s returned %d; staying solo\n", + hydra_peer.c_str(), n_bound); + actually_combined = false; + } else { + // Peer is up — latch so INFO RPC advertises combined. + hydra_combined_head_attached = true; + } + } + } + } + } + + llama_hydra_set_expert_mode(ctx_tgt, actually_combined ? 1 : 0); + res->expert_mode_applied = actually_combined ? "combined" : "solo"; + + res->rpc_status = HYDRA_STATUS_OK; + res->success = true; + SRV_INF("hydra: SET_EXPERT_MODE requested='%s' applied='%s' (slot %d)\n", + requested.c_str(), res->expert_mode_applied.c_str(), task.hydra_action.id_slot); + queue_results.send(std::move(res)); + } break; + + case SERVER_TASK_TYPE_HYDRA_ENGINE_SWAP_QUANT: + { + auto res = std::make_unique(); + res->id = task.id; + res->op = HYDRA_OP_SWAP_QUANT; + res->rpc_status = HYDRA_STATUS_OK; + res->success = true; + SRV_INF("hydra: SWAP_QUANT quant='%s' pattern='%s' (slot %d)\n", + task.hydra_action.quant_key.c_str(), + task.hydra_action.tensor_pattern.c_str(), + task.hydra_action.id_slot); + queue_results.send(std::move(res)); + } break; + + // M-Perf.9 (#289) / issue #287: PIPELINE_ATTACH is part of the + // two-engine "work together" routing tracked in #287. The + // coordinator wires the request; the engine-side scaffolding + // (--override-tensor local-load, activation passing, COMBINED + // expert mode) is the next deliverable. For now this opcode + // returns NOT_IMPLEMENTED so the wire stays in sync — the + // coordinator will treat that as a fallback to solo mode. + case SERVER_TASK_TYPE_HYDRA_ENGINE_PIPELINE_ATTACH: + { + auto res = std::make_unique(); + res->id = task.id; + res->op = HYDRA_OP_PIPELINE_ATTACH; + res->rpc_status = HYDRA_STATUS_NOT_IMPLEMENTED; + res->success = false; + res->error = "HYDRA_OP_PIPELINE_ATTACH not yet implemented in this build (see issue #287)"; + SRV_WRN("hydra: PIPELINE_ATTACH received (slot %d) — stubbed, issue #287\n", + task.hydra_action.id_slot); + queue_results.send(std::move(res)); + } break; + } + } + + +// --------------------------------------------------------------------------- +// WS1/WS2: the extension object. handle_task() routes HYDRA tasks to the same +// hydra_process_task() method the legacy switch calls — seam == legacy behavior. +// --------------------------------------------------------------------------- +struct hydra_engine_extension : server_hydra_extension { + const char * name() const override { + return "hydra-task-ws2"; + } + + bool handle_task(server_context_impl & impl, server_task & task) override { + switch (task.type) { + case SERVER_TASK_TYPE_HYDRA_STATE_GET: + case SERVER_TASK_TYPE_HYDRA_STATE_PUT: + case SERVER_TASK_TYPE_HYDRA_STATE_META: + case SERVER_TASK_TYPE_HYDRA_ENGINE_CONFIGURE: + case SERVER_TASK_TYPE_HYDRA_ENGINE_INFO: + case SERVER_TASK_TYPE_HYDRA_ENGINE_PREFILL: + case SERVER_TASK_TYPE_HYDRA_ENGINE_DECODE: + case SERVER_TASK_TYPE_HYDRA_DECODE_APPLY: + case SERVER_TASK_TYPE_HYDRA_ENGINE_SET_EXPERT_MODE: + case SERVER_TASK_TYPE_HYDRA_ENGINE_SWAP_QUANT: + case SERVER_TASK_TYPE_HYDRA_ENGINE_PIPELINE_ATTACH: + impl.hydra_process_task(task); + return true; // claimed + default: + return false; // not a Hydra task — fall through to inline dispatch + } + } + + bool pre_loop(server_context_impl & impl) override { + // WS3: handle the slot-free CONFIGURE/T3 moment at the top of + // update_slots(). Mirrors the inline all-idle block; returns true only + // when a staged CONFIGURE was actually applied (so the rest of + // update_slots() is skipped, matching the inline `return;`). + bool all_idle = true; + for (auto & slot : impl.slots) { + if (slot.is_processing() || slot.hydra_transferring->load()) { + all_idle = false; + break; + } + } + if (!all_idle) { + return false; // not the slot-free moment — inline decode body runs + } + + if (impl.ctx_tgt && impl.ctx_tgt->hydra_has_pending_config()) { + SRV_INF("hydra ext: slot-free moment — applying pending CONFIGURE (tier=%s)\n", + impl.ctx_tgt->hydra_get_pending_config_tier().c_str()); + impl.apply_pending_hydra_config(); + return true; + } + if (!impl.ctx_tgt && impl.first_load_pending) { + SRV_INF("%s", "hydra ext: slot-free moment — first load (no context yet)\n"); + impl.apply_pending_hydra_config(); + return true; + } + return false; // nothing staged — inline all-idle block logs + returns + } + + bool on_empty_batch(server_context_impl & impl) override { + // WS3: replicate the inline empty-batch transfer-suppression, including + // the 2s grace window after a transfer ends. Returns true when the + // empty batch was caused by a STATE_GET transfer (suppress the abort); + // false otherwise (inline logic runs, which eventually aborts). + static int64_t hydra_last_transfer_ms = 0; + static int64_t hydra_suppress_count = 0; + + bool any_transferring = false; + for (const auto & s : impl.slots) { + if (s.hydra_transferring && s.hydra_transferring->load()) { + any_transferring = true; + break; + } + } + const int64_t now_ms = ggml_time_us() / 1000; + if (any_transferring) { + hydra_last_transfer_ms = now_ms; + } + if (any_transferring || now_ms - hydra_last_transfer_ms < 2000) { + if (hydra_suppress_count++ % 256 == 0) { + SRV_WRN("hydra ext: empty batch suppressed — transfer %s\n", + any_transferring ? "in flight" : "just ended"); + } + impl.n_empty_consecutive = 0; + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + return true; + } + return false; + } +}; + +std::unique_ptr hydra_create_extension() { + return std::make_unique(); +} diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp index f6c6fbc2b173..bf67e6a19310 100644 --- a/tools/server/server-context.cpp +++ b/tools/server/server-context.cpp @@ -3,6 +3,7 @@ #include "server-chat.h" #include "server-common.h" #include "server-checkpoint-policy.h" +#include "server-hydra-extension.h" #include "server-http.h" #include "server-task.h" #include "server-queue.h" @@ -1053,6 +1054,10 @@ bool hydra_apply_generic_key(common_params & params, const std::string & key, co struct server_context_impl { friend struct server_context; friend struct server_routes; + // epic #610 WS1: the concrete Hydra extension (defined in + // hydra-server-context.cpp, #include'd at the bottom of this TU) needs + // access to the same internals the inline Hydra code uses. + friend struct hydra_engine_extension; public: // only use these pointers outside of this class: @@ -1117,8 +1122,23 @@ struct server_context_impl { // note: chat_params must not be refreshed upon existing sleeping state server_chat_params chat_params; + // epic #610 WS1: Hydra A/B extension seam. + // legacy mode (default): the inline Hydra code in this file runs. + // seam mode (HYDRA_EXT_MODE=seam): the hydra_engine_extension drives the + // same behavior through the server_hydra_extension interface. Both paths + // stay compiled; only one is consulted per run, so the same binary can be + // A/B tested by flipping the env var. + const bool hydra_ext_active = hydra_ext_mode_seam(); + std::unique_ptr hydra_ext; + server_context_impl() { mtmd_helper_log_set(common_log_default_callback, nullptr); + if (hydra_ext_active) { + hydra_ext = hydra_create_extension(); + SRV_INF("hydra ext: seam mode active (HYDRA_EXT_MODE=seam), impl=%s\n", hydra_ext->name()); + } else { + SRV_INF("%s", "hydra ext: legacy mode active (default) - A/B baseline\n"); + } } ~server_context_impl() { @@ -3069,6 +3089,12 @@ struct server_context_impl { } void process_single_task(server_task && task) { + // epic #610 WS1: in seam mode the extension may claim Hydra tasks. + // WS1 impl is a no-op (returns false), so this is a pure A/B switch — + // both modes run the inline dispatch below. + if (hydra_ext_active && hydra_ext && hydra_ext->handle_task(*this, task)) { + return; + } switch (task.type) { case SERVER_TASK_TYPE_COMPLETION: case SERVER_TASK_TYPE_INFILL: @@ -3407,2495 +3433,24 @@ struct server_context_impl { queue_results.send(std::move(res)); } break; - // ── Hydra RPC state-transfer tasks (M1) ────────────────────────── - // All three cases run on the inference thread so llama API access is safe. - // The calling RPC thread blocks on queue_results.recv_with_timeout(). - + // epic #610 WS2: HYDRA task dispatch moved to hydra_process_task() + // (defined in hydra-server-context.cpp). In seam mode the extension + // claims these via handle_task(); in legacy mode this fall-through + // calls the same method. Both modes run identical code. case SERVER_TASK_TYPE_HYDRA_STATE_GET: - { - // M1: background serialization thread — inference loop continues during state transfer. - // llama_state_seq_get_data reads KV cells for an IDLE sequence; llama_decode - // writes cells for ACTIVE sequences only — no memory overlap for different seq IDs. - const int id_slot = task.hydra_action.id_slot; - auto res = std::make_unique(); - res->id = task.id; - res->id_slot = id_slot; - res->op = HYDRA_OP_STATE_GET; - - server_slot * slot = get_slot_by_id(id_slot); - if (slot == nullptr) { - res->rpc_status = HYDRA_STATUS_NOT_FOUND; - res->error = "invalid slot ID"; - queue_results.send(std::move(res)); - break; - } - if (slot->is_processing() || slot->hydra_transferring->load()) { - res->rpc_status = HYDRA_STATUS_BUSY; - queue_results.send(std::move(res)); - break; - } - - // Snapshot on inference thread (cheap — dry-run serialization, no GPU copies). - const size_t state_size = llama_state_seq_get_size(ctx_tgt, slot->id); - int actual_n_past = slot->n_prompt_tokens_cache + slot->n_decoded; - // Cold prefill: n_prompt_tokens_cache is still 0 so n_decoded (1) dominates. - // Use prompt token count instead — matches STATE_META fallback. - if (slot->n_prompt_tokens_cache == 0 && slot->prompt.tokens.size() > 0) { - actual_n_past = (int)slot->prompt.tokens.size(); - } - res->n_past = actual_n_past; - res->rpc_status = HYDRA_STATUS_OK; - // M-Perf.9 #289: surface model identity alongside the state - // bytes so the Coordinator can record the model that built - // the KV (for cross-model safety on restore). The background - // thread that streams the bytes to the socket can mutate - // res->state_data freely; the model fields are immutable for - // the duration of the response. - res->model_alias = model_name; - res->model_path = params_base.model.path; - if (model_tgt) { - res->tokenizer = llama_model_get_tokenizer_model(model_tgt); - res->model_name = llama_model_get_display_name(model_tgt); - res->model_quant = llama_model_get_quant_label(model_tgt); - res->model_capabilities = llama_model_get_capabilities_bitfield(model_tgt); - } - SRV_INF("hydra: STATE_GET slot=%d n_past=%d state=%.1f MiB — async\n", - id_slot, res->n_past, state_size / (1024.0 * 1024.0)); - - slot->hydra_transferring->store(true); - - // M2: stream directly to socket (zero-copy). - // Runs SYNCHRONOUSLY on the inference thread to avoid - // concurrent ggml-RPC socket access with llama_decode - // on another slot (fixes crash at ggml-rpc.cpp:532). - // The coordinator already does Store Put as fire-and-forget - // so blocking here only delays slot release, not decode. - const int snap_seq_id = slot->id; - llama_context * snap_ctx = ctx_tgt; - // shared_ptr keeps the atomic alive even if the slot is reallocated - std::shared_ptr> flag_ptr = slot->hydra_transferring; - const int hydra_fd = task.hydra_action.hydra_fd; - - // Capture prompt tokens for M1 path header (slot is valid on inference thread) - const llama_tokens prompt_tokens_get = slot->prompt.tokens.get_text_tokens(); - const int32_t n_past_val = res->n_past; - - // Snapshot the most recent native checkpoint so STATE_PUT can - // register it instead of fabricating one at the final position. - // Fabricating at pos_max=n-1 corrupts hybrid/recurrent model - // decode because the recurrent state is one token ahead of the - // decode resume point — it has already processed the final token. - std::vector snapshot_ckpt; - uint8_t hdr_flags = 0x00; - int32_t ckpt_pos_min = 0, ckpt_pos_max = 0; - int64_t ckpt_n_tokens = 0; - if (!slot->prompt.checkpoints.empty()) { - hdr_flags |= 0x01; - const auto & ckpt = slot->prompt.checkpoints.back(); - ckpt_pos_min = ckpt.pos_min; - ckpt_pos_max = ckpt.pos_max; - ckpt_n_tokens = ckpt.n_tokens; - - // Hydra M2-stream double-write fix (#470/#620): serialize the - // recurrent-only capture (data_tgt_recr, PARTIAL_ONLY) instead of - // the full data_tgt. The full live state that follows on the wire - // already carries the attention bytes at the live position, so - // sending the full checkpoint duplicates the attention portion - // (which scales with ctx). The recurrent state is genuinely needed - // at BOTH positions, hence the separate recr-only capture. - // hdr_flags bit 0x02 marks a recurrent-only checkpoint section so - // STATE_PUT/DECODE_APPLY can read it back with matched PARTIAL_ONLY - // flags. Fall back to the full capture when the checkpoint has no - // recr buffer (e.g. it was registered from an old 0x02 blob) — a - // PARTIAL_ONLY read of a full-written buffer is a CUDA memory error. - const bool use_recr = !ckpt.data_tgt_recr.empty(); - if (use_recr) { - hdr_flags |= 0x02; - } - const uint64_t tgt_sz = use_recr ? ckpt.data_tgt_recr.size() : ckpt.data_tgt.size(); - const uint64_t dft_sz = use_recr ? ckpt.data_dft_recr.size() : ckpt.data_dft.size(); - const uint8_t * tgt_ptr = use_recr ? ckpt.data_tgt_recr.data() : ckpt.data_tgt.data(); - const uint8_t * dft_ptr = use_recr ? ckpt.data_dft_recr.data() : ckpt.data_dft.data(); - const size_t ckpt_hdr_sz = 4 + 4 + 8 + 8 + (size_t)tgt_sz + 8 + (size_t)dft_sz; - snapshot_ckpt.resize(ckpt_hdr_sz); - size_t off = 0; - memcpy(snapshot_ckpt.data() + off, &ckpt_pos_min, 4); off += 4; - memcpy(snapshot_ckpt.data() + off, &ckpt_pos_max, 4); off += 4; - memcpy(snapshot_ckpt.data() + off, &ckpt_n_tokens, 8); off += 8; - memcpy(snapshot_ckpt.data() + off, &tgt_sz, 8); off += 8; - if (tgt_sz > 0) { memcpy(snapshot_ckpt.data() + off, tgt_ptr, (size_t)tgt_sz); off += (size_t)tgt_sz; } - memcpy(snapshot_ckpt.data() + off, &dft_sz, 8); off += 8; - if (dft_sz > 0) memcpy(snapshot_ckpt.data() + off, dft_ptr, (size_t)dft_sz); - } - - { - SRV_INF("hydra: STATE_GET streaming (fd=%d state=%.1f MiB)\n", - hydra_fd, state_size / (1024.0 * 1024.0)); - if (hydra_fd >= 0) { - // M2 path: stream v2/v3 blob (header + checkpoint + GPU state) to fd. - // Response header + meta JSON sent first, then v2 header bytes, - // then llama_state_seq_get_data_to_fd writes GPU state directly. - const size_t n_tok = prompt_tokens_get.size(); - const uint32_t hdr_n_tok = (uint32_t)n_tok; - const uint32_t hdr_n_past = (uint32_t)n_past_val; - // 0x03 = v3 blob: checkpoint section carries recurrent-only - // captures (data_tgt_recr, PARTIAL_ONLY). 0x02 = v2 blob: - // checkpoint section carries the full data_tgt. Bumped so a - // mixed-version fleet never misreads a smaller (recr-only) - // checkpoint as a full one. - const uint8_t version_byte = 0x03; - const size_t base_hdr_size = 1 + 4 + 4 + n_tok * sizeof(llama_token) + 1; - const size_t hdr_size = base_hdr_size + snapshot_ckpt.size(); - const size_t total_payload = hdr_size + state_size; - - // Build v2 header buffer - std::vector v2_hdr(hdr_size); - { - size_t off = 0; - memcpy(v2_hdr.data() + off, &version_byte, 1); off += 1; - memcpy(v2_hdr.data() + off, &hdr_n_past, 4); off += 4; - memcpy(v2_hdr.data() + off, &hdr_n_tok, 4); off += 4; - memcpy(v2_hdr.data() + off, prompt_tokens_get.data(), n_tok * sizeof(llama_token)); off += n_tok * sizeof(llama_token); - memcpy(v2_hdr.data() + off, &hdr_flags, 1); off += 1; - if (!snapshot_ckpt.empty()) { - memcpy(v2_hdr.data() + off, snapshot_ckpt.data(), snapshot_ckpt.size()); - off += snapshot_ckpt.size(); - } - } - - { - json meta_j; - meta_j["n_past"] = res->n_past; - meta_j["state_size"] = (uint64_t)state_size; - if (!res->model_alias.empty()) meta_j["model_alias"] = res->model_alias; - if (!res->model_path.empty()) meta_j["model_path"] = res->model_path; - if (!res->tokenizer.empty()) meta_j["tokenizer"] = res->tokenizer; - if (!res->model_name.empty()) meta_j["model_name"] = res->model_name; - if (!res->model_quant.empty()) meta_j["model_quant"] = res->model_quant; - if (res->model_capabilities) meta_j["model_capabilities"] = res->model_capabilities; - const std::string meta_str = meta_j.dump(); - - const uint32_t meta_len = (uint32_t)meta_str.size(); - const uint64_t payload_l = (uint64_t)total_payload; - uint8_t hdr[HYDRA_RES_HEADER_SIZE] = {}; - hdr[0] = HYDRA_STATUS_OK; - hdr[1] = (meta_len) & 0xFF; - hdr[2] = (meta_len >> 8) & 0xFF; - hdr[3] = (meta_len >> 16) & 0xFF; - memcpy(hdr + 4, &payload_l, 8); - hydra_send_all(hydra_fd, hdr, HYDRA_RES_HEADER_SIZE); - hydra_send_all(hydra_fd, meta_str.data(), meta_str.size()); - - // Write v2 blob header before GPU state — STATE_PUT needs tokens + checkpoint - hydra_send_all(hydra_fd, v2_hdr.data(), v2_hdr.size()); - - res->header_sent = true; // META + header + v2-hdr before payload - } - // Stream GPU state to fd (zero-copy from GPU memory) - const size_t streamed = llama_state_seq_get_data_to_fd(snap_ctx, snap_seq_id, hydra_fd, nullptr); - if (streamed != state_size) { - // TOCTOU: state size changed between get_size (header already - // promised state_size bytes) and the stream, or the stream - // failed mid-way. The wire framing is now broken — the only - // safe recovery is to kill the connection. Use shutdown(), - // not close(): the RPC connection loop owns the fd and will - // close it when its next read fails; closing here would race - // (double-close / fd-reuse against unrelated threads). - res->rpc_status = HYDRA_STATUS_ERROR; - res->error = "llama_state_seq_get_data_to_fd streamed " + - std::to_string(streamed) + " B, expected " + - std::to_string(state_size) + " B"; - ::shutdown(hydra_fd, SHUT_RDWR); - } else { - res->streamed_bytes = total_payload; - } - } else { - // M1 path: buffer in memory, RPC thread sends afterwards. - // v3 blob format (0x03): [1B version][4B n_past][4B n_tok][n_tok*4B tokens] - // [1B flags (bit 0 = has_checkpoint)] - // [if flags & 0x01: 4B pos_min | 4B pos_max | 8B n_tokens | 8B tgt_sz | recr_tgt_data | 8B dft_sz | recr_dft_data] - // [raw KV state from llama_state_seq_get_data] - const size_t n_tok = prompt_tokens_get.size(); - const uint32_t hdr_n_tok = (uint32_t)n_tok; - const uint32_t hdr_n_past = (uint32_t)n_past_val; - const uint8_t version_byte = 0x03; - const size_t base_hdr_size = 1 + 4 + 4 + n_tok * sizeof(llama_token) + 1; // version + n_past + n_tok + tokens + flags - const size_t hdr_size = base_hdr_size + snapshot_ckpt.size(); - - // TOCTOU retry: if another slot grew the state between - // get_size (inference thread) and get_data (background thread), - // the copy returns 0. Retry up to 3 times with fresh sizing. - size_t buf_size = hdr_size + state_size; - res->state_data.resize(buf_size); - { - size_t off = 0; - memcpy(res->state_data.data() + off, &version_byte, 1); off += 1; - memcpy(res->state_data.data() + off, &hdr_n_past, 4); off += 4; - memcpy(res->state_data.data() + off, &hdr_n_tok, 4); off += 4; - memcpy(res->state_data.data() + off, prompt_tokens_get.data(), n_tok * sizeof(llama_token)); off += n_tok * sizeof(llama_token); - memcpy(res->state_data.data() + off, &hdr_flags, 1); off += 1; - if (!snapshot_ckpt.empty()) { - memcpy(res->state_data.data() + off, snapshot_ckpt.data(), snapshot_ckpt.size()); - off += snapshot_ckpt.size(); - } - } - - size_t cur_state_size = state_size; - size_t copied = 0; - int retries = 3; - while (retries-- > 0) { - copied = llama_state_seq_get_data( - snap_ctx, res->state_data.data() + hdr_size, cur_state_size, snap_seq_id); - if (copied > 0) break; - // State grew — re-measure and retry - cur_state_size = llama_state_seq_get_size(snap_ctx, snap_seq_id); - res->state_data.resize(hdr_size + cur_state_size); - } - if (copied == 0) { - res->rpc_status = HYDRA_STATUS_ERROR; - res->error = "llama_state_get_data failed after 3 retries"; - res->state_data.clear(); - } - } - flag_ptr->store(false); - // M2 streams to fd (streamed_bytes); M1 buffers into state_data. - const uint64_t out_bytes = (hydra_fd >= 0) - ? res->streamed_bytes - : (uint64_t) res->state_data.size(); - SRV_INF("hydra: STATE_GET done slot=%d rpc_status=%d path=%s bytes=%" PRIu64 "\n", - snap_seq_id, res->rpc_status, - hydra_fd >= 0 ? "M2-stream" : "M1-buffer", out_bytes); - queue_results.send(std::move(res)); - } - - // STATE_GET is synchronous — blocks until KV state is fully - // streamed to the socket. The coordinator's Store Put is - // fire-and-forget, so only slot release is delayed. - } break; - case SERVER_TASK_TYPE_HYDRA_STATE_PUT: - { - const int id_slot = task.hydra_action.id_slot; - auto res = std::make_unique(); - res->id = task.id; - res->id_slot = id_slot; - res->op = HYDRA_OP_STATE_PUT; - - server_slot * slot = get_slot_by_id(id_slot); - if (slot == nullptr) { - res->rpc_status = HYDRA_STATUS_NOT_FOUND; - res->error = "invalid slot ID"; - queue_results.send(std::move(res)); - break; - } - if (slot->is_processing() || slot->hydra_transferring->load()) { - res->rpc_status = HYDRA_STATUS_BUSY; - queue_results.send(std::move(res)); - break; - } - - // M-Perf.9 #289: populate model identity from resident model. - // model_match = true always (infrastructure only; actual KV - // validation comes when model identity is embedded in the KV header). - res->model_alias = model_name; - res->model_path = params_base.model.path; - res->model_match = true; - if (model_tgt) { - res->tokenizer = llama_model_get_tokenizer_model(model_tgt); - res->model_name = llama_model_get_display_name(model_tgt); - res->model_quant = llama_model_get_quant_label(model_tgt); - res->model_capabilities = llama_model_get_capabilities_bitfield(model_tgt); - } - - // Erase existing checkpoints to avoid collision with restored session state - if (task.hydra_action.erase_existing && !slot->prompt.checkpoints.empty()) { - SLT_INF(*slot, "erasing %zu existing checkpoints before STATE_PUT restore\n", - slot->prompt.checkpoints.size()); - slot->prompt.checkpoints.clear(); - } - - const auto & buf = task.hydra_action.state_data; - - // Detect v2/v3 blob (0x02/0x03 at offset 0) vs legacy format (no version byte). - // v2: [1B version=0x02][4B n_past][4B n_tok][n_tok*4B tokens][1B flags][?ckpt?][KV state] - // v3: same, but the checkpoint section may be a recurrent-only capture - // (hdr_flags bit 0x02 set). Bumped to 0x03 by the M2-stream double-write fix. - const bool is_v2 = buf.size() >= 1 && (buf[0] == 0x02 || buf[0] == 0x03); - - size_t hdr_offset = 0; - int32_t hdr_n_tok = 0; - int32_t hdr_n_past = 0; - bool has_chkpt = false; - bool ckpt_is_recr_only = false; - int32_t ckpt_pos_min_in = 0, ckpt_pos_max_in = 0; - int64_t ckpt_n_tokens_in = 0; - std::vector ckpt_tgt_data, ckpt_dft_data; - - if (is_v2) { - // v2/v3: version at [0], n_past at [1..4], n_tok at [5..8] - if (buf.size() >= 9) { - memcpy(&hdr_n_past, buf.data() + 1, 4); - memcpy(&hdr_n_tok, buf.data() + 5, 4); - } - const size_t token_start = 9; - const size_t token_end = token_start + (size_t)hdr_n_tok * sizeof(llama_token); - hdr_offset = token_end; - if (hdr_offset < buf.size()) { - const uint8_t flags = buf[hdr_offset]; - hdr_offset += 1; // past flags byte - // bit 0x01 = has checkpoint; bit 0x02 = checkpoint section is - // recurrent-only (PARTIAL_ONLY). A v3 blob that fell back to the - // full capture (old-registered checkpoint) leaves 0x02 clear. - ckpt_is_recr_only = (flags & 0x02) != 0; - if (flags & 0x01) { - // Parse checkpoint: 4B pos_min | 4B pos_max | 8B n_tokens | 8B tgt_sz | tgt_data | 8B dft_sz | dft_data - if (hdr_offset + 4 + 4 + 8 + 8 <= buf.size()) { - memcpy(&ckpt_pos_min_in, buf.data() + hdr_offset, 4); hdr_offset += 4; - memcpy(&ckpt_pos_max_in, buf.data() + hdr_offset, 4); hdr_offset += 4; - memcpy(&ckpt_n_tokens_in, buf.data() + hdr_offset, 8); hdr_offset += 8; - uint64_t tgt_sz_in; - memcpy(&tgt_sz_in, buf.data() + hdr_offset, 8); hdr_offset += 8; - if (tgt_sz_in > 0 && hdr_offset + tgt_sz_in <= buf.size()) { - ckpt_tgt_data.assign(buf.data() + hdr_offset, buf.data() + hdr_offset + (size_t)tgt_sz_in); - hdr_offset += (size_t)tgt_sz_in; - } - if (hdr_offset + 8 <= buf.size()) { - uint64_t dft_sz_in; - memcpy(&dft_sz_in, buf.data() + hdr_offset, 8); hdr_offset += 8; - if (dft_sz_in > 0 && hdr_offset + dft_sz_in <= buf.size()) { - ckpt_dft_data.assign(buf.data() + hdr_offset, buf.data() + hdr_offset + (size_t)dft_sz_in); - hdr_offset += (size_t)dft_sz_in; - } - } - has_chkpt = true; - } - } - } - // Restore tokens from token_start - if (hdr_n_tok > 0 && token_start + (size_t)hdr_n_tok * sizeof(llama_token) <= buf.size()) { - slot->prompt.tokens.clear(); - const llama_token * tok_ptr = (const llama_token *)(buf.data() + token_start); - llama_tokens restored_tokens(tok_ptr, tok_ptr + (size_t)hdr_n_tok); - slot->prompt.tokens.insert(restored_tokens); - } - } else { - // Legacy v1 format - if (buf.size() >= 8) { - memcpy(&hdr_n_past, buf.data(), 4); - memcpy(&hdr_n_tok, buf.data() + 4, 4); - hdr_offset = 8 + (size_t)hdr_n_tok * sizeof(llama_token); - } - if (hdr_offset > 0 && hdr_offset <= buf.size()) { - const size_t n_tokens = (size_t)hdr_n_tok; - slot->prompt.tokens.clear(); - if (n_tokens > 0) { - const llama_token * tok_ptr = (const llama_token *)(buf.data() + 8); - llama_tokens restored_tokens(tok_ptr, tok_ptr + n_tokens); - slot->prompt.tokens.insert(restored_tokens); - } - } - } - const bool has_hdr = hdr_offset > 0 && hdr_offset <= buf.size(); - const uint8_t * state_ptr = has_hdr ? buf.data() + hdr_offset : buf.data(); - const size_t state_len = has_hdr ? buf.size() - hdr_offset : buf.size(); - const size_t n_read = llama_state_seq_set_data(ctx_tgt, state_ptr, state_len, slot->id); - if (n_read == 0) { - res->rpc_status = HYDRA_STATUS_ERROR; - res->error = "llama_state_set_data returned 0"; - // Tokens were registered before set_data — clear them so the slot - // is not left poisoned (n_past > 0 with no KV cells → pos_min == -1 - // abort on the next decode that touches this slot). - slot->prompt.tokens.clear(); - slot->prompt.checkpoints.clear(); - slot->n_prompt_tokens_cache = 0; - llama_memory_seq_rm(llama_get_memory(ctx_tgt), slot->id, -1, -1); - } else { - // D4: Inject trailing logits into per-slot buffer instead of the - // shared context-wide llama_get_logits(). This avoids the race where - // another slot's decode clobbers restored logits between STATE_PUT - // and the first sample. - const size_t remaining = state_len - n_read; - const size_t expected_logits = (size_t)llama_vocab_n_tokens(vocab) * sizeof(float); - if (remaining == expected_logits) { - const float * src = (const float *)(state_ptr + n_read); - const size_t n_floats = llama_vocab_n_tokens(vocab); - slot->restored_logits.assign(src, src + n_floats); - slot->logits_valid = true; - SRV_INF("hydra: STATE_PUT slot=%d restored %zu logits to per-slot buffer\n", - id_slot, n_floats); - } - - res->rpc_status = HYDRA_STATUS_OK; - res->restored = true; - res->bytes = (uint64_t)n_read; - // #469 trace: log restored state for cross-flow comparison - SRV_DBG("hydra: STATE_PUT slot=%d RESTORED n_past=%d n_prompt_tok=%d state_bytes=%zu just_restored=true\n", - id_slot, hdr_n_tok, hdr_n_tok, n_read); - { - std::string tok_ids; - for (size_t i = 0; i < std::min(16, slot->prompt.tokens.size()); ++i) { - if (i > 0) tok_ids += ","; - tok_ids += std::to_string(slot->prompt.tokens[i]); - } - SRV_DBG("hydra: STATE_PUT slot=%d first16_tokens=[%s] total=%zu\n", - id_slot, tok_ids.c_str(), slot->prompt.tokens.size()); - } - if (hdr_n_tok > 0) { - slot->n_prompt_tokens_cache = hdr_n_tok; - slot->n_decoded = 0; - res->n_past = hdr_n_tok; - - // Register native checkpoint from the blob (v2) or fabricate one (legacy). - // The native checkpoint has pos_max at n-4 (created before the last - // few prompt tokens were decoded), so loading it rewinds the recurrent - // state to a clean position. The old fabricated checkpoint at (0, n-1) - // puts the recurrent state at the final position — one token ahead of - // where decode must resume — corrupting hybrid/recurrent model output. - slot->prompt.checkpoints.clear(); - if (has_chkpt) { - auto & ckpt = slot->prompt.checkpoints.emplace_back(); - ckpt.n_tokens = ckpt_n_tokens_in; - ckpt.pos_min = ckpt_pos_min_in; - ckpt.pos_max = ckpt_pos_max_in; - // New-format (v3) checkpoints carry a recurrent-only capture — - // route it into data_*_recr and tag is_recr_only so the load - // path uses matched PARTIAL_ONLY flags (plus an attention - // seq_rm at pos_max) instead of the full flags=0 restore. - ckpt.is_recr_only = ckpt_is_recr_only; - if (ckpt_is_recr_only) { - ckpt.data_tgt_recr = std::move(ckpt_tgt_data); - ckpt.data_dft_recr = std::move(ckpt_dft_data); - } else { - ckpt.data_tgt = std::move(ckpt_tgt_data); - ckpt.data_dft = std::move(ckpt_dft_data); - } - SLT_INF(*slot, "STATE_PUT registered native checkpoint (pos_min=%d pos_max=%d n_tokens=%" PRId64 " tgt_sz=%zu recr_only=%d)\n", - ckpt.pos_min, ckpt.pos_max, ckpt.n_tokens, ckpt.size(), (int) ckpt.is_recr_only); - } else { - const auto pos_min = llama_memory_seq_pos_min(llama_get_memory(ctx_tgt), slot->id); - create_checkpoint(*slot, 0, (llama_pos)pos_min, (llama_pos)(hdr_n_tok - 1)); - } - slot->just_restored = true; - } - SRV_INF("hydra: STATE_PUT slot=%d restored=%zu B n_past=%d n_prompt_tok=%d\n", - id_slot, n_read, res->n_past, hdr_n_tok); - } - queue_results.send(std::move(res)); - } break; - case SERVER_TASK_TYPE_HYDRA_STATE_META: - { - const int id_slot = task.hydra_action.id_slot; - auto res = std::make_unique(); - res->id = task.id; - res->id_slot = id_slot; - res->op = HYDRA_OP_STATE_META; - - server_slot * slot = get_slot_by_id(id_slot); - if (slot == nullptr) { - res->rpc_status = HYDRA_STATUS_NOT_FOUND; - res->error = "invalid slot ID"; - queue_results.send(std::move(res)); - break; - } - // META is safe to serve even while processing or transferring (read-only metadata) - int actual_n_past = slot->n_prompt_tokens_cache + slot->n_decoded; - // For cold prefills n_prompt_tokens_cache is 0 — use prompt token count - if (slot->n_prompt_tokens_cache == 0 && slot->prompt.tokens.size() > 0) { - actual_n_past = (int)slot->prompt.tokens.size(); - } - res->n_past = actual_n_past; - res->is_processing = slot->is_processing(); - res->is_transferring = slot->hydra_transferring->load(); - res->state_size = (uint64_t)llama_state_seq_get_size(ctx_tgt, slot->id); - // M-Perf.9 #289: surface model identity. The Coordinator uses - // these to detect cross-model restores — a slot holding a Mini - // KV cache must never have it decoded by a Balanced-loaded model. - res->model_alias = model_name; - res->model_path = params_base.model.path; - if (model_tgt) { - res->tokenizer = llama_model_get_tokenizer_model(model_tgt); - res->model_name = llama_model_get_display_name(model_tgt); - res->model_quant = llama_model_get_quant_label(model_tgt); - res->model_capabilities = llama_model_get_capabilities_bitfield(model_tgt); - } - // #451: populate progress fields based on slot state - switch (slot->state) { - case SLOT_STATE_PROCESSING_PROMPT: - res->operation = "prefill"; - res->tokens_processed = slot->n_prompt_tokens_processed; - // task->n_tokens() is the total tokens to process (fixed); - // prompt.tokens.size() grows during prefill and is WRONG for total. - res->tokens_total = slot->task ? slot->task->n_tokens() : 0; - if (res->tokens_total > 0) { - res->progress = (float)res->tokens_processed / (float)res->tokens_total; - } - res->elapsed_ms = (slot->t_start_process_prompt > 0) - ? (ggml_time_ms() - slot->t_start_process_prompt) : 0; - break; - case SLOT_STATE_GENERATING: - res->operation = "decode"; - res->tokens_processed = slot->n_decoded; - // n_remaining == -1 is the "unlimited generation" sentinel - // (no finite n_predict). Don't compute progress in that case. - if (slot->n_remaining > 0) { - res->tokens_total = slot->n_decoded + slot->n_remaining; - res->progress = (float)res->tokens_processed / (float)res->tokens_total; - } - res->elapsed_ms = (slot->t_start_generation > 0) - ? (ggml_time_ms() - slot->t_start_generation) : 0; - break; - case SLOT_STATE_IDLE: - res->operation = "idle"; - res->progress = 1.0f; - break; - default: - res->operation = "unknown"; - break; - } - // Handle save/restore operations via hydra_transferring flag. - // Clear any stale progress from the prior state since we're - // now in a transferring context, not the previous operation. - if (slot->hydra_transferring->load()) { - res->operation = "save"; - res->progress = 0.0f; - res->tokens_processed = 0; - res->tokens_total = 0; - res->elapsed_ms = 0; - } - res->rpc_status = HYDRA_STATUS_OK; - queue_results.send(std::move(res)); - } break; - case SERVER_TASK_TYPE_HYDRA_ENGINE_CONFIGURE: - { - auto res = std::make_unique(); - res->id = task.id; - res->op = HYDRA_OP_CONFIGURE; - res->rpc_status = HYDRA_STATUS_OK; - res->success = true; - - // hydra#406: tiered CONFIGURE (T1/T2/T3). Backward compat: - // a legacy {"state_chunk_size":N} payload is treated as a - // degenerate T1 (the original hydra#334 startup call from - // WorkerSchedulerService.cs:2842). - if (task.hydra_action.config_json.empty()) { - res->tier = "T1"; - SRV_INF("hydra: CONFIGURE (empty payload, slot %d) — T1 no-op\n", - task.hydra_action.id_slot); - queue_results.send(std::move(res)); - break; - } - - json cfg; - try { - cfg = json::parse(task.hydra_action.config_json); - } catch (const std::exception & e) { - res->success = false; - res->rpc_status = HYDRA_STATUS_ERROR; - res->error = std::string("CONFIGURE: invalid config_json: ") + e.what(); - SRV_WRN("hydra: CONFIGURE failed to parse config_json (slot %d): %s\n", - task.hydra_action.id_slot, e.what()); - queue_results.send(std::move(res)); - break; - } - - // Route through the shared classify → apply helper. - // sync=false: T2/T3/T4 are staged for the slot-free moment. - hydra_config_result cfg_result = hydra_apply_config(cfg, /*sync=*/false); - - // hydra#470: report generic (T4) keys that cannot be - // applied BEFORE the tier-0 early return — a payload - // whose keys are all unrecognized/rejected still has - // to surface them (zero silent drops). - res->unrecognized_keys = cfg_result.unrecognized_keys; - res->rejected_keys = cfg_result.rejected_keys; - - if (cfg_result.highest_tier == 0) { - // No recognized keys — still emit a T1 success - // (the legacy {"state_chunk_size":N} case). - res->tier = "T1"; - SRV_INF("hydra: CONFIGURE (no recognized keys, slot %d) — T1 no-op\n", - task.hydra_action.id_slot); - queue_results.send(std::move(res)); - break; - } - - if (!cfg_result.ok) { - res->success = false; - res->rpc_status = HYDRA_STATUS_ERROR; - res->error = "CONFIGURE: " + cfg_result.error; - SRV_WRN("hydra: CONFIGURE apply failed (slot %d): %s\n", - task.hydra_action.id_slot, cfg_result.error.c_str()); - queue_results.send(std::move(res)); - break; - } - - // Build the response from the shared helper's result. - res->tier = hydra_tier_label(cfg_result.highest_tier); - res->params_applied = std::move(cfg_result.params_applied); - res->deferred_keys = std::move(cfg_result.deferred_keys); - res->state_chunk_size_applied = cfg_result.state_chunk_size_applied; - - SRV_INF("hydra: CONFIGURE tier=%s applied=%zu deferred=%zu unrecognized=%zu rejected=%zu (slot %d)\n", - res->tier.c_str(), - res->params_applied.size(), - res->deferred_keys.size(), - res->unrecognized_keys.size(), - res->rejected_keys.size(), - task.hydra_action.id_slot); - queue_results.send(std::move(res)); - } break; - case SERVER_TASK_TYPE_HYDRA_ENGINE_INFO: - { - auto res = std::make_unique(); - res->id = task.id; - res->op = HYDRA_OP_INFO; - res->rpc_status = HYDRA_STATUS_OK; - // M-Perf.9 #289: advertise the model identity features so - // the Coordinator knows it can send `model` in PREFILL and - // expect model_alias/model_path/tokenizer/model_name/model_quant/model_capabilities - // in META responses. - // `preset_aliases` lists every alias loaded from - // --models-preset (empty when no preset is configured). - json preset_aliases_j = json::array(); - for (const auto & [alias, _path] : preset_alias_to_path) { - preset_aliases_j.push_back(alias); - } - // Hydra #287/#260/#348: two-engine "work together" status - // — see specs/rpc-protocol.md's ENGINE_INFO (0x41) - // contract. pipeline_capable stays false until #287's - // PIPELINE half lands; mode only ever reports - // solo/combined until then. solo_active/rpc_backend_active/ - // peer_reachable/combined_head_attached are independent - // booleans (#348) — replaces the old single "role" string - // and the peer_connected/combined_capable field-aliasing. - const int32_t expert_mode = ctx_tgt ? llama_hydra_get_expert_mode(ctx_tgt) : 0; - // Hydra #383 T1 / #375: advertise "combined" capability when this - // engine is ready to serve in COMBINED mode — either via expert-split - // (hydra_combined_head_attached) or via layer-split (hydra_combined_static). - json capabilities_j = {"prefill", "decode", "state_transfer", - "expert_mode", "quant_swap", - "preset", "tokenizer", "model_name", - "model_quant", "model_capabilities", - "merged_decode"}; - if (hydra_combined_head_attached || hydra_combined_static) { - capabilities_j.push_back("combined"); - } - // In layer-split static mode the engine is always in combined mode; - // in expert-split mode it follows the per-request SET_EXPERT_MODE state. - const std::string mode_str = hydra_combined_static ? "combined" - : (expert_mode == 1 ? "combined" : "solo"); - json info_j = { - {"engine", "llama-server-hydra"}, - {"version", "E1"}, - {"capabilities", capabilities_j}, - {"preset_aliases", preset_aliases_j}, - {"solo_active", hydra_solo_active}, - {"rpc_backend_active", hydra_rpc_backend_active}, - {"mode", mode_str}, - {"split_mode", hydra_split_mode}, - {"peer_addr", hydra_peer}, - {"peer_reachable", hydra_peer_reachable}, - {"layer_split", hydra_combined_pattern}, - {"combined_head_attached", hydra_combined_head_attached || hydra_combined_static}, - {"pipeline_capable", false} - }; - res->info_json = info_j.dump(); - queue_results.send(std::move(res)); - } break; - case SERVER_TASK_TYPE_HYDRA_ENGINE_PREFILL: - { - const int id_slot = task.hydra_action.id_slot; - auto res = std::make_unique(); - res->id = task.id; - res->op = HYDRA_OP_PREFILL; - - // #451: track timing for PREFILL metrics - const int64_t prefill_start_ms = ggml_time_ms(); - - // Set by the model-resolution block below when a real - // `load_model` swap happens. Used at the response site to - // decide whether the post-prefill model identity is the - // freshly loaded model (swap) or the original (no-swap / - // fallback). - bool model_was_swapped = false; - - server_slot * slot = get_slot_by_id(id_slot); - if (slot == nullptr) { - res->rpc_status = HYDRA_STATUS_NOT_FOUND; - res->error = "invalid slot ID"; - queue_results.send(std::move(res)); - break; - } - - if (slot->is_processing()) { - res->rpc_status = HYDRA_STATUS_BUSY; - res->error = "slot is busy"; - queue_results.send(std::move(res)); - break; - } - - // M-Perf.9 #289: parse the optional `model` key from the - // request body and swap the resident model when the preset - // registry knows the alias. The parse is reused for the - // tokenization step below. Falls back to the resident model - // (with `model_fallback:true` in the response) when the - // alias is unknown or no preset is configured. - json parsed_body; - std::string requested_model; - json hydra_cfg; // optional hydra_config object - bool has_hydra_config = false; - if (!task.hydra_action.request_json.empty()) { - try { - parsed_body = json::parse(task.hydra_action.request_json); - if (parsed_body.is_object() && parsed_body.contains("model") - && parsed_body["model"].is_string()) { - requested_model = parsed_body["model"].get(); - } - // hydra_config: optional config object from Hydra.Core - // containing topology/sampling overrides. When present - // with model_path, it drives the model swap directly - // (bypassing the preset alias lookup). - if (parsed_body.is_object() && parsed_body.contains("hydra_config") - && parsed_body["hydra_config"].is_object()) { - hydra_cfg = parsed_body["hydra_config"]; - has_hydra_config = true; - } - } catch (const std::exception & e) { - res->rpc_status = HYDRA_STATUS_BAD_REQUEST; - res->error = std::string("invalid JSON: ") + e.what(); - queue_results.send(std::move(res)); - break; - } - } - - // Apply hydra_config synchronously when present. - // T1 keys (sampling, n_predict, etc.) are applied in-place. - // T2/T3 keys (n_ctx, cache_type, model_path, split_mode, etc.) - // trigger immediate rebuilds on this task-queue thread. - - // #470: Before applying config, probe all RPC peers for - // reconnection. If a peer restarted since the last request, - // its buffers are gone even though model/params haven't - // changed. Without this probe, the T3 rebuild in - // hydra_apply_config → apply_t3_rebuild would skip (params - // unchanged) and the subsequent graph_compute would fail. - if (ctx_tgt && ggml_backend_rpc_check_any_peer_reconnection()) { - SRV_WRN("%s", "hydra: PREFILL: RPC peer reconnected — forcing T3 rebuild\n"); - ctx_tgt->peer_reconnection_pending = true; - } - - if (has_hydra_config) { - SRV_INF("hydra: PREFILL slot=%d applying hydra_config (%zu keys)\n", - id_slot, hydra_cfg.size()); - hydra_config_result cfg_result = hydra_apply_config(hydra_cfg, /*sync=*/true); - if (!cfg_result.ok) { - res->rpc_status = HYDRA_STATUS_ERROR; - res->error = "hydra_config apply failed: " + cfg_result.error; - SRV_WRN("hydra: PREFILL hydra_config apply failed (slot %d): %s\n", - id_slot, cfg_result.error.c_str()); - queue_results.send(std::move(res)); - break; - } - // If the apply may have rebuilt the slots (T3 - // statics or a T4-only generic config both route - // through apply_t3_rebuild → load_model → - // slots.clear()), track it and re-look-up the slot. - // Without the T4 case (hydra#470) the slot pointer - // captured above would dangle into the prefill. - if (hydra_config_requires_slot_relookup(cfg_result.highest_tier)) { - model_was_swapped = true; - res->model_load_ms = (double)(ggml_time_ms() - prefill_start_ms); - SRV_INF("hydra: PREFILL hydra_config T3/T4 applied model_alias='%s' tokenizer='%s' model_name='%s' quant='%s' caps=0x%x\n", - model_name.empty() ? "?" : model_name.c_str(), - model_tgt ? llama_model_get_tokenizer_model(model_tgt) : "", - model_tgt ? llama_model_get_display_name(model_tgt) : "", - model_tgt ? llama_model_get_quant_label(model_tgt) : "", - model_tgt ? llama_model_get_capabilities_bitfield(model_tgt) : 0); - slot = get_slot_by_id(id_slot); - if (slot == nullptr) { - res->rpc_status = HYDRA_STATUS_NOT_FOUND; - res->error = "slot disappeared after hydra_config T3/T4 rebuild"; - queue_results.send(std::move(res)); - break; - } - } - // When hydra_config carries model_path, the model swap is - // handled by apply_t3_rebuild() above — skip the bare - // model alias lookup below. - if (hydra_cfg.contains("model_path")) { - requested_model.clear(); - } - } - - // Fallback: bare model alias lookup when hydra_config didn't - // handle the model swap (no hydra_config, or no model_path). - if (!requested_model.empty()) { - auto it = preset_alias_to_path.find(requested_model); - if (it == preset_alias_to_path.end()) { - SRV_WRN("hydra: PREFILL model='%s' unknown (preset has %zu alias(es)) — falling back to resident '%s'\n", - requested_model.c_str(), preset_alias_to_path.size(), - model_name.c_str()); - res->model_fallback = true; - } else if (it->second != params_base.model.path) { - SRV_INF("hydra: PREFILL model='%s' swapping %s -> %s\n", - requested_model.c_str(), params_base.model.path.c_str(), - it->second.c_str()); - common_params swapped_params = params_base; - // Apply the target alias's full preset so that - // tensor_buft_overrides, n_gpu_layers, split_mode, - // tensor_split, etc. are replaced — not inherited - // from the source model. Intentionally the FULL - // preset (sampling, chat template, n_ctx, etc. - // included), not just tensor-placement keys: a - // real model swap targets a different model, - // which plausibly needs its own sampling - // defaults/chat template too, not just a new - // memory layout. - auto pit = preset_alias_to_preset.find(requested_model); - if (pit != preset_alias_to_preset.end()) { - // Clear inherited tensor_buft_overrides (padded - // to 4096 by common_params_parse_ex) BEFORE - // apply_to_params, which push_back()'s the new - // preset's entries via CLI handlers. Without - // this, the new entries land after the - // nullptr-terminator and exceed the 4096 limit, - // triggering GGML_ASSERT in - // common_model_params_to_llama (#499 regression). - swapped_params.tensor_buft_overrides.clear(); - try { - // apply_to_params() replays CLI handlers - // (parse_tensor_buffer_overrides, the - // n-cpu-moe std::stoi, two-value option - // parsers) which throw on a malformed - // target preset. Uncaught, that exception - // would escape the task-queue loop and - // kill the task thread — fail the swap - // instead. - pit->second.apply_to_params(swapped_params); - hydra_repad_tensor_buft_overrides(swapped_params, "PREFILL swap"); - } catch (const std::exception & e) { - SRV_WRN("hydra: PREFILL swap preset apply for '%s' failed: %s\n", - requested_model.c_str(), e.what()); - res->rpc_status = HYDRA_STATUS_ERROR; - res->error = std::string("model swap preset apply failed: ") + e.what(); - queue_results.send(std::move(res)); - break; - } - SRV_INF("hydra: PREFILL swap applied preset for '%s' " - "(tensor_buft_overrides=%zu entries)\n", - requested_model.c_str(), - swapped_params.tensor_buft_overrides.size()); - } - swapped_params.model.path = it->second; - // Update the alias so model_name is re-derived - // correctly in load_model() (model_name is set from - // model_alias.first when non-empty). - swapped_params.model_alias = { requested_model }; - // #514: tear down COMBINED state before the - // reload — otherwise the engine loads the - // correct model file but keeps routing tokens - // through the stale peer/expert-binding config, - // collapsing decode throughput. - const bool was_combined = hydra_combined_head_attached || hydra_combined_static; - if (was_combined) { - hydra_teardown_combined_before_reload(); - } - const int64_t model_load_start_ms = ggml_time_ms(); - if (!load_model(swapped_params)) { - res->rpc_status = HYDRA_STATUS_ERROR; - res->error = "model swap to '" + requested_model + "' failed"; - queue_results.send(std::move(res)); - break; - } - if (was_combined) { - hydra_reattach_combined_after_reload(); - } - res->model_load_ms = (double)(ggml_time_ms() - model_load_start_ms); - model_was_swapped = true; - SRV_INF("hydra: PREFILL swap confirmed model_alias='%s' tokenizer='%s' model_name='%s' quant='%s' caps=0x%x model_load_ms=%.1f\n", - swapped_params.model_alias.empty() ? "?" : swapped_params.model_alias.begin()->c_str(), - model_tgt ? llama_model_get_tokenizer_model(model_tgt) : "", - model_tgt ? llama_model_get_display_name(model_tgt) : "", - model_tgt ? llama_model_get_quant_label(model_tgt) : "", - model_tgt ? llama_model_get_capabilities_bitfield(model_tgt) : 0, - res->model_load_ms); - // After load_model, `this` state is reset (new - // slots, new context). Re-look up the slot by id. - slot = get_slot_by_id(id_slot); - if (slot == nullptr) { - res->rpc_status = HYDRA_STATUS_NOT_FOUND; - res->error = "slot disappeared after model swap"; - queue_results.send(std::move(res)); - break; - } - } else { - SRV_DBG("hydra: PREFILL model='%s' already resident, no swap\n", - requested_model.c_str()); - } - } - - // Tokenize from JSON messages if request_json is provided; - // otherwise fall back to pre-tokenized prompt_tokens for back-compat. - std::vector prompt_tokens = std::move(task.hydra_action.prompt_tokens); - if (!parsed_body.is_null()) { - try { - std::vector dummy_files; - json parsed = oaicompat_chat_params_parse(parsed_body, chat_params, dummy_files); - if (!parsed.contains("prompt")) { - res->rpc_status = HYDRA_STATUS_ERROR; - res->error = "chat template produced no prompt"; - queue_results.send(std::move(res)); - break; - } - auto tokenized = tokenize_input_prompts(vocab, mctx, parsed["prompt"], true, true); - if (tokenized.empty()) { - res->rpc_status = HYDRA_STATUS_ERROR; - res->error = "tokenization produced no tokens"; - queue_results.send(std::move(res)); - break; - } - prompt_tokens = tokenized[0].get_tokens(); - } catch (const std::exception & e) { - res->rpc_status = HYDRA_STATUS_ERROR; - res->error = std::string("JSON/tokenization error: ") + e.what(); - queue_results.send(std::move(res)); - break; - } - } - - SRV_INF("hydra: PREFILL slot=%d tokens=%zu\n", id_slot, prompt_tokens.size()); - // #469 trace: log first 16 token IDs for cross-flow comparison - { - std::string tok_ids; - for (size_t i = 0; i < std::min(16, prompt_tokens.size()); ++i) { - if (i > 0) tok_ids += ","; - tok_ids += std::to_string(prompt_tokens[i]); - } - SRV_DBG("hydra: PREFILL slot=%d first16_tokens=[%s] total=%zu\n", - id_slot, tok_ids.c_str(), prompt_tokens.size()); - } - - // Clear existing slot state - slot->prompt_clear(false); - slot->n_prompt_tokens_cache = 0; - slot->n_prompt_tokens_processed = 0; - slot->n_decoded = 0; - - // Insert prompt tokens - if (prompt_tokens.empty()) { - res->rpc_status = HYDRA_STATUS_OK; - res->n_past = 0; - res->state_size = 0; - queue_results.send(std::move(res)); - break; - } - - slot->prompt.tokens.insert(prompt_tokens); - const auto & tokens = slot->prompt.tokens.get_tokens(); - const int n_tokens = (int)tokens.size(); - - // Add BOS if needed (check if slot uses BOS) - int token_offset = 0; - llama_token bos = llama_vocab_bos(vocab); - if (add_bos_token && bos != LLAMA_TOKEN_NULL && (tokens.empty() || tokens[0] != bos)) { - token_offset = 1; - } - - // Decode prompt in batches. Hydra #469 fix: upstream's own - // invariant (see create_checkpoint call in update_slots, - // "we create the checkpoint before calling llama_decode(), - // so the current batch is not yet processed and therefore - // it is not part of the checkpoint") requires the - // checkpoint to be created BEFORE the final token is - // decoded. The previous version of this handler decoded - // the whole prompt first and only afterward claimed (via - // create_checkpoint's pos_max arg, below) that the last - // token was still unprocessed. For hybrid/recurrent (SSM) - // models, whose memory can't be partially rolled back via - // seq_rm, that lie meant a cross-node restore would - // re-decode a token that was already baked into the - // recurrent state — double-applying it and corrupting the - // hidden state. Splitting the loop so the checkpoint is - // captured after n_tokens-1 tokens (matching what - // create_checkpoint's pos_max already claimed) makes the - // claim honest, same as the standard update_slots() path. - const int total_tokens = n_tokens + token_offset; - const int n_ubatch = llama_n_ubatch(ctx_tgt); - const int n_before_last = total_tokens > 1 ? total_tokens - 1 : total_tokens; - bool decode_ok = true; - for (int i = 0; i < n_before_last && decode_ok; i += n_ubatch) { - const int n_tokens_batch = std::min(n_ubatch, n_before_last - i); - common_batch_clear(batch); - for (int j = 0; j < n_tokens_batch; j++) { - const int tok_idx = i + j; - llama_token id; - if (token_offset > 0 && tok_idx == 0) { - id = bos; - } else { - id = tokens[tok_idx - token_offset]; - } - // No token in this phase is the final prompt - // token, so logits are never needed here. - common_batch_add(batch, id, tok_idx, {slot->id}, false); - } - if (llama_decode(ctx_tgt, batch) != 0) { - SRV_ERR("hydra: PREFILL slot=%d llama_decode failed at batch %d\n", id_slot, i); - decode_ok = false; - } - } - - if (!decode_ok) { - res->rpc_status = HYDRA_STATUS_ERROR; - res->error = "llama_decode failed during prefill"; - queue_results.send(std::move(res)); - break; - } - - // Register checkpoint BEFORE decoding the final token, so - // its pos_max claim (n_tokens - 1) is honest. Moved up - // from after the full-prompt decode (see #469 above). - if (n_tokens > 0) { - const auto pos_min = llama_memory_seq_pos_min(llama_get_memory(ctx_tgt), slot->id); - create_checkpoint(*slot, 0, (llama_pos)pos_min, (llama_pos)(n_tokens - 1)); - } - - // Decode the held-back final token (if any) now that the - // checkpoint has captured the state before it. - if (total_tokens > n_before_last) { - common_batch_clear(batch); - const int tok_idx = total_tokens - 1; - llama_token id = (token_offset > 0 && tok_idx == 0) - ? bos - : tokens[tok_idx - token_offset]; - common_batch_add(batch, id, tok_idx, {slot->id}, true); - if (llama_decode(ctx_tgt, batch) != 0) { - SRV_ERR("hydra: PREFILL slot=%d llama_decode failed on final token\n", id_slot); - res->rpc_status = HYDRA_STATUS_ERROR; - res->error = "llama_decode failed during prefill (final token)"; - queue_results.send(std::move(res)); - break; - } - } - - // Update slot tracking - slot->n_prompt_tokens_processed = n_tokens; - slot->n_prompt_tokens_cache = n_tokens; - - // Checkpoint already registered above, before the final - // token was decoded (#469 fix). - - // Build v2/v3 header: [1B version=0x02|0x03][4B n_past][4B n_tok][n_tok*4B tokens][1B flags][?ckpt?] - // Shared by both response paths — M1 embeds it at the head of the - // buffered blob, M2 sends it before streaming the GPU state. - const uint32_t hdr_n_past = (uint32_t)n_tokens; - const uint32_t hdr_n_tok = (uint32_t)(tokens.size()); - uint8_t hdr_flags = 0x00; - std::vector ckpt_buf; - int32_t ckpt_pos_min = 0, ckpt_pos_max = 0; - int64_t ckpt_n_tokens = 0; - if (!slot->prompt.checkpoints.empty()) { - hdr_flags |= 0x01; - const auto & ckpt = slot->prompt.checkpoints.back(); - ckpt_pos_min = ckpt.pos_min; - ckpt_pos_max = ckpt.pos_max; - ckpt_n_tokens = ckpt.n_tokens; - // Hydra M2-stream double-write fix (#470/#620): serialize the - // recurrent-only capture (data_tgt_recr, PARTIAL_ONLY) instead of - // the full data_tgt. The full live state that follows on the wire - // already carries the attention bytes at the live position, so - // sending the full checkpoint duplicates the attention portion - // (which scales with ctx). The recurrent state is genuinely needed - // at BOTH positions, hence the separate recr-only capture. - // hdr_flags bit 0x02 marks a recurrent-only checkpoint section so - // STATE_PUT/DECODE_APPLY can read it back with matched PARTIAL_ONLY - // flags. Fall back to the full capture when the checkpoint has no - // recr buffer (e.g. it was registered from an old 0x02 blob) — a - // PARTIAL_ONLY read of a full-written buffer is a CUDA memory error. - const bool use_recr = !ckpt.data_tgt_recr.empty(); - if (use_recr) { - hdr_flags |= 0x02; - } - const uint64_t tgt_sz = use_recr ? ckpt.data_tgt_recr.size() : ckpt.data_tgt.size(); - const uint64_t dft_sz = use_recr ? ckpt.data_dft_recr.size() : ckpt.data_dft.size(); - const uint8_t * tgt_ptr = use_recr ? ckpt.data_tgt_recr.data() : ckpt.data_tgt.data(); - const uint8_t * dft_ptr = use_recr ? ckpt.data_dft_recr.data() : ckpt.data_dft.data(); - ckpt_buf.resize(4 + 4 + 8 + 8 + (size_t)tgt_sz + 8 + (size_t)dft_sz); - size_t off = 0; - memcpy(ckpt_buf.data() + off, &ckpt_pos_min, 4); off += 4; - memcpy(ckpt_buf.data() + off, &ckpt_pos_max, 4); off += 4; - memcpy(ckpt_buf.data() + off, &ckpt_n_tokens, 8); off += 8; - memcpy(ckpt_buf.data() + off, &tgt_sz, 8); off += 8; - if (tgt_sz > 0) { memcpy(ckpt_buf.data() + off, tgt_ptr, (size_t)tgt_sz); off += (size_t)tgt_sz; } - memcpy(ckpt_buf.data() + off, &dft_sz, 8); off += 8; - if (dft_sz > 0) memcpy(ckpt_buf.data() + off, dft_ptr, (size_t)dft_sz); - } - const size_t base_hdr_size = 1 + 4 + 4 + hdr_n_tok * sizeof(llama_token) + 1; - const size_t v2_size = base_hdr_size + ckpt_buf.size(); - std::vector v2_hdr(v2_size); - { - size_t off = 0; - // 0x03 = v3 blob: checkpoint section carries recurrent-only - // captures (data_tgt_recr, PARTIAL_ONLY). 0x02 = v2 blob: - // checkpoint section carries the full data_tgt. Bumped so a - // mixed-version fleet never misreads a smaller (recr-only) - // checkpoint as a full one. - const uint8_t version_byte = 0x03; - memcpy(v2_hdr.data() + off, &version_byte, 1); off += 1; - memcpy(v2_hdr.data() + off, &hdr_n_past, 4); off += 4; - memcpy(v2_hdr.data() + off, &hdr_n_tok, 4); off += 4; - if (hdr_n_tok > 0) { - const auto & toks = slot->prompt.tokens.get_text_tokens(); - memcpy(v2_hdr.data() + off, toks.data(), toks.size() * sizeof(llama_token)); - off += toks.size() * sizeof(llama_token); - } - memcpy(v2_hdr.data() + off, &hdr_flags, 1); off += 1; - if (!ckpt_buf.empty()) { - memcpy(v2_hdr.data() + off, ckpt_buf.data(), ckpt_buf.size()); - off += ckpt_buf.size(); - } - } - - // Get raw KV state - const size_t state_size = llama_state_seq_get_size(ctx_tgt, slot->id); - - // Snapshot logits NOW into a small buffer. ctx->logits is - // context-global: a concurrent slot decode can overwrite it - // while the M2 state stream is on the wire. Appending - // n_vocab floats gives the decode GPU the activation handoff - // (llama_state_seq_get_data saves KV but not logits), so - // STATE_PUT / DECODE_APPLY can sample immediately. - uint64_t logits_size = 0; - std::vector logits_buf; - { - const int n_vocab = llama_vocab_n_tokens(vocab); - const float * logits_ptr = llama_get_logits(ctx_tgt); - if (logits_ptr && n_vocab > 0) { - logits_size = (uint64_t)n_vocab * sizeof(float); - logits_buf.assign(reinterpret_cast(logits_ptr), - reinterpret_cast(logits_ptr) + (size_t)logits_size); - } - } - - SRV_INF("hydra: PREFILL slot=%d done n_past=%d kv=%zu logits=%" PRIu64 "B total=%zu\n", - id_slot, n_tokens, state_size, logits_size, v2_hdr.size() + state_size + (size_t)logits_size); - - // M-Perf.9 #289: model identity for the slot the prefill - // was just built on. Coordinator uses this to populate - // item.KvModelAlias/Hash and to gate RestoreKvAsync. When - // a `model` swap happened earlier in this handler, the - // post-swap `model_name` / `params_base.model.path` / - // `model` are used. `res->model_fallback` was set by the - // model-resolution block above; we preserve it here. - res->model_alias = model_name; - res->model_path = params_base.model.path; - // res->model_fallback may already be true (alias unknown - // or no preset); only set false when no swap was needed. - if (!model_was_swapped && !res->model_fallback) { - // nothing to do — leave as-is - } - if (model_tgt) { - res->tokenizer = llama_model_get_tokenizer_model(model_tgt); - res->model_name = llama_model_get_display_name(model_tgt); - res->model_quant = llama_model_get_quant_label(model_tgt); - res->model_capabilities = llama_model_get_capabilities_bitfield(model_tgt); - } - - res->rpc_status = HYDRA_STATUS_OK; - res->n_past = n_tokens; - res->state_size = state_size; - res->logits_size = logits_size; - // #451: populate PREFILL metrics - res->prefill_ms = (double)(ggml_time_ms() - prefill_start_ms); - res->prompt_tokens = n_tokens; - res->kv_size = state_size; - if (res->prefill_ms > 0 && n_tokens > 0) { - res->tokens_per_second = (double)n_tokens / (res->prefill_ms / 1000.0); - } - res->cache_tokens = slot->n_prompt_tokens_cache; - - const int hydra_fd = task.hydra_action.hydra_fd; - if (hydra_fd >= 0) { - // M2 path (#470): stream the response straight to the - // socket — 12B header + meta JSON + v2 header, then the - // GPU KV state zero-copy (chunked via cparams.hydra_state_chunk_size), - // then the (small) logits tail. No full-blob RAM buffer: - // at 60-80K context the blob is ~800 MB and grows toward - // 10 GB; buffering it doubled engine peak memory and the - // send only started after compute + full buffer completed. - // Wire layout is byte-identical to M1: payload = - // v2_hdr + KV state + logits, payload_len = the same - // total the coordinator computes from meta. - const size_t total_payload = v2_hdr.size() + state_size + (size_t)logits_size; - - // M2 (#470): pre-compute the wire hash of the whole kv - // segment — v2 header, then [4B magic][4B seq_id] + KV - // state (hash-only pass in wire order), then the logits - // tail. The meta must carry it BEFORE the first payload - // byte goes out (the coordinator forwards it into the - // DECODE frame header, and DECODE_APPLY verifies the - // streamed restore end-to-end). The slot is exclusively - // held by this task, so the state cannot change between - // the hash pass and the stream. - XXH3_state_t * kv_hst = nullptr; - if (hydra_fd >= 0) { - kv_hst = XXH3_createState(); - XXH3_64bits_reset(kv_hst); - XXH3_64bits_update(kv_hst, v2_hdr.data(), v2_hdr.size()); - const size_t hashed = llama_state_seq_hash(ctx_tgt, slot->id, kv_hst); - // state_size (from llama_state_seq_get_size) ALREADY includes the - // [4B magic][4B seq_id] wire header — llama_io_write_dummy counts it. - // llama_state_seq_hash hashes the same [4B magic][4B seq_id] + KV - // bytes, so after the n_bytes() fix hashed == state_size exactly. - // Adding sizeof(uint32_t) + sizeof(llama_seq_id) here double-counted - // the header and killed every PREFILL M2 request (#470). - if (hashed != state_size) { - res->rpc_status = HYDRA_STATUS_ERROR; - res->error = "PREFILL M2: hash pre-pass hashed " + - std::to_string(hashed) + " B, expected " + - std::to_string(state_size) + " B"; - } - if (!logits_buf.empty()) { - XXH3_64bits_update(kv_hst, logits_buf.data(), logits_buf.size()); - } - if (res->rpc_status == HYDRA_STATUS_OK) { - const uint64_t kv_hash = XXH3_64bits_digest(kv_hst); - char hash_hex[17]; - snprintf(hash_hex, sizeof(hash_hex), "%016" PRIx64, kv_hash); - res->kv_hash_str = std::string("xxh3:") + hash_hex; - } - XXH3_freeState(kv_hst); - kv_hst = nullptr; - } - - json meta_j = { - {"n_past", res->n_past}, - {"state_size", res->state_size}, - {"logits_size", res->logits_size} - }; - if (!res->model_alias.empty()) meta_j["model_alias"] = res->model_alias; - if (!res->model_path.empty()) meta_j["model_path"] = res->model_path; - if (!res->tokenizer.empty()) meta_j["tokenizer"] = res->tokenizer; - if (!res->model_name.empty()) meta_j["model_name"] = res->model_name; - if (!res->model_quant.empty()) meta_j["model_quant"] = res->model_quant; - if (res->model_capabilities) meta_j["model_capabilities"] = res->model_capabilities; - meta_j["model_fallback"] = res->model_fallback; - if (res->prefill_ms > 0) meta_j["prefill_ms"] = res->prefill_ms; - if (res->model_load_ms > 0) meta_j["model_load_ms"] = res->model_load_ms; - if (!res->kv_hash_str.empty()) meta_j["kv_hash_str"] = res->kv_hash_str; - const std::string meta_str = meta_j.dump(); - const uint32_t meta_len = (uint32_t)meta_str.size(); - - uint8_t hdr[HYDRA_RES_HEADER_SIZE] = {}; - hdr[0] = HYDRA_STATUS_OK; - hdr[1] = (meta_len) & 0xFF; - hdr[2] = (meta_len >> 8) & 0xFF; - hdr[3] = (meta_len >> 16) & 0xFF; - memcpy(hdr + 4, &total_payload, 8); - if (!hydra_send_all(hydra_fd, hdr, HYDRA_RES_HEADER_SIZE) || - !hydra_send_all(hydra_fd, meta_str.data(), meta_str.size()) || - !hydra_send_all(hydra_fd, v2_hdr.data(), v2_hdr.size())) { - res->rpc_status = HYDRA_STATUS_ERROR; - res->error = "PREFILL M2: response header/meta/v2-hdr send failed"; - ::shutdown(hydra_fd, SHUT_RDWR); - } else { - res->header_sent = true; // META + header + v2-hdr before payload - // Stream GPU state to fd (zero-copy from GPU memory; - // the wire hash was pre-computed above — pass no - // hash state so the io does not double-feed it) - const size_t streamed = llama_state_seq_get_data_to_fd(ctx_tgt, slot->id, hydra_fd, nullptr); - if (streamed != state_size) { - // TOCTOU: state size changed between get_size - // (above) and the stream, or the stream failed - // mid-way. The wire framing is now broken — the - // only safe recovery is to kill the connection. - // shutdown(), not close(): the RPC connection - // loop owns the fd (mirrors STATE_GET M2). - res->rpc_status = HYDRA_STATUS_ERROR; - res->error = "llama_state_seq_get_data_to_fd streamed " + - std::to_string(streamed) + " B, expected " + - std::to_string(state_size) + " B"; - ::shutdown(hydra_fd, SHUT_RDWR); - } else { - // Logits tail after the state stream — PREFILL's - // payload includes logits_size bytes at the end - // (STATE_GET M2 does not send logits). - if (!logits_buf.empty()) { - if (!hydra_send_all(hydra_fd, logits_buf.data(), logits_buf.size())) { - res->rpc_status = HYDRA_STATUS_ERROR; - res->error = "PREFILL M2: logits tail send failed"; - ::shutdown(hydra_fd, SHUT_RDWR); - } - } - if (res->rpc_status == HYDRA_STATUS_OK) { - res->streamed_bytes = (uint64_t)total_payload; - } - } - } - } else { - // M1 path: buffer the full blob in memory; the RPC thread - // sends header + meta + payload afterwards (unchanged). - // v2 blob format (0x02): [1B version][4B n_past][4B n_tok][n_tok*4B tokens] - // [1B flags (bit 0 = has_checkpoint)] - // [if flags & 0x01: 4B pos_min | 4B pos_max | 8B n_tokens | 8B tgt_sz | tgt_data | 8B dft_sz | dft_data] - // [raw KV state from llama_state_seq_get_data] - // [logits (n_vocab * float)] - std::vector v2_blob(v2_hdr.size() + state_size + (size_t)logits_size); - { - size_t off = 0; - memcpy(v2_blob.data() + off, v2_hdr.data(), v2_hdr.size()); - off += v2_hdr.size(); - if (state_size > 0) { - llama_state_seq_get_data(ctx_tgt, v2_blob.data() + off, state_size, slot->id); - } - } - if (!logits_buf.empty()) { - memcpy(v2_blob.data() + v2_hdr.size() + state_size, logits_buf.data(), logits_buf.size()); - } - res->state_data = std::move(v2_blob); - } - // #469 trace: log PREFILL completion with token IDs for cross-flow comparison - SRV_DBG("hydra: PREFILL_DONE slot=%d n_past=%d state_size=%zu logits_size=%zu blob_size=%zu prefill_ms=%.1f\n", - id_slot, n_tokens, state_size, logits_size, - (hydra_fd >= 0) ? v2_hdr.size() + state_size + (size_t)logits_size : res->state_data.size(), - res->prefill_ms); - queue_results.send(std::move(res)); - } break; - case SERVER_TASK_TYPE_HYDRA_ENGINE_DECODE: - { - // ── Sync phase: Gate A (header-only, no GGUF reads, ~1 ms) ── - // Identity validation, slot reservation, post DECODE_APPLY. - // No model I/O, no KV touched. - const int id_slot = task.hydra_action.id_slot; - const int32_t decode_request_id = task.hydra_action.decode_request_id; - auto res = std::make_unique(); - res->id = task.id; - res->op = HYDRA_OP_DECODE; - res->decode_request_id = decode_request_id; - res->id_slot = id_slot; - - server_slot * slot = get_slot_by_id(id_slot); - if (slot == nullptr) { - res->rpc_status = HYDRA_STATUS_NOT_FOUND; - res->error = "invalid slot ID"; - queue_results.send(std::move(res)); - break; - } - - if (slot->is_processing()) { - res->rpc_status = HYDRA_STATUS_BUSY; - res->error = "slot is busy"; - queue_results.send(std::move(res)); - break; - } - - // Reject if slot is reserved for another decode - if (slot->reserved_for_decode_id != -1 && slot->reserved_for_decode_id != decode_request_id) { - res->rpc_status = HYDRA_STATUS_BUSY; - res->error = "slot reserved for another decode"; - queue_results.send(std::move(res)); - break; - } - - // Parse the merged DECODE JSON header - json decode_req; - try { - decode_req = json::parse(task.hydra_action.decode_json); - } catch (const std::exception & e) { - res->rpc_status = HYDRA_STATUS_BAD_REQUEST; - res->error = std::string("invalid JSON: ") + e.what(); - queue_results.send(std::move(res)); - break; - } - - // ── Gate A: header-only metadata comparison ───────────── - // Compare kv_metadata vs model_metadata from the control - // header. No GGUF reads, no KV touched. - const json & kv_meta = decode_req["kv_metadata"]; - const json & model_meta = decode_req.value("model_metadata", json::object()); - - // Read request identities from header - const std::string req_tokenizer = kv_meta.value("tokenizer", ""); - const std::string req_model_name = kv_meta.value("model_name", ""); - const uint32_t req_capabilities = kv_meta.value("model_capabilities", 0u); - - // Read target identities from header - const std::string tgt_tokenizer = model_meta.value("tokenizer", ""); - const std::string tgt_model_name = model_meta.value("model_name", ""); - - const bool tokenizer_match = (req_tokenizer == tgt_tokenizer); - bool model_name_match = (req_model_name == tgt_model_name); - // #589: cross-node same-model name tolerance. The KV's - // model_name is the display name (GGUF metadata) of the - // file that BUILT the KV — a different build/quant of the - // same model than the decode node's resident file, so - // string equality legitimately fails for the same logical - // model (e.g. kv_metadata carries the source node's - // display name, the decode node reports its resident - // filename). When the header carries the KV's source - // alias (kv_metadata.model_alias) or the resolved request - // alias ("model") and that alias maps through the preset - // table to the resident model path, the KV was built by - // the same logical model — accept. The alias→path check - // is exact (per-node preset INI), so a different model - // (Mini vs Balanced, 27B vs 35B) still maps to a - // different path and is rejected. - if (!model_name_match) { - const std::string kv_alias = kv_meta.value("model_alias", ""); - const std::string hdr_alias = decode_req.value("model", std::string()); - for (const auto & cand : { kv_alias, hdr_alias }) { - if (cand.empty()) { - continue; - } - auto pit = preset_alias_to_path.find(cand); - if (pit != preset_alias_to_path.end() && pit->second == params_base.model.path) { - SRV_INF("hydra: DECODE slot=%d Gate A name fallback — alias '%s' maps to resident path, same logical model\n", - id_slot, cand.c_str()); - model_name_match = true; - break; - } - } - } - const uint32_t capabilities_xor = req_capabilities ^ model_meta.value("model_capabilities", 0u); - - static const char * kCapBitNames[] = {"MTP", "VISION", "REASONING", "TOOL_USE", "CODE"}; - std::vector capabilities_diff_bits; - for (int b = 0; b < 5; b++) { - if (capabilities_xor & (1u << b)) { - capabilities_diff_bits.push_back(kCapBitNames[b]); - } - } - - // MTP(bit0) + VISION(bit1) mismatch → hard reject - const bool valid = tokenizer_match && model_name_match - && !(capabilities_xor & 0x3); - - json match_j = { - {"tokenizer_match", tokenizer_match}, - {"model_name_match", model_name_match}, - {"capabilities_xor", capabilities_xor}, - {"capabilities_diff_bits", capabilities_diff_bits}, - {"model_quant_match", kv_meta.value("model_quant", "") == model_meta.value("model_quant", "")}, - {"model_alias_match", true}, - }; - res->match_json = match_j; - res->match_valid = valid; - - if (!valid) { - res->rpc_status = HYDRA_STATUS_ERROR; - res->error = "model_capabilities_mismatch"; - SRV_WRN("hydra: DECODE slot=%d Gate A reject — tokenizer=%d name=%d caps_xor=0x%x\n", - id_slot, tokenizer_match, model_name_match, capabilities_xor); - queue_results.send(std::move(res)); - break; - } - - // ── Reserve slot ──────────────────────────────────────── - slot->reserved_for_decode_id = decode_request_id; - - SRV_INF("hydra: DECODE slot=%d Gate A pass, reserved for request_id=%d\n", - id_slot, decode_request_id); - - // ── Create decode_result_entry (LOADING state) ───────── - // So GET /v1/decode/{id} returns 202 instead of 404 - // while async DECODE_APPLY is pending. - if (routes_ptr) { - server_routes::decode_result_entry entry; - entry.id_slot = id_slot; - entry.state = server_routes::DECODE_STATE_LOADING; - entry.match_json = match_j; - entry.created_at = std::time(nullptr); - entry.ttl_s = routes_ptr->decode_result_ttl_s; - entry.model_metadata = decode_req.value("model_metadata", json::object()); - entry.model_identity = json::object(); - std::lock_guard lock(routes_ptr->decode_results_mutex); - routes_ptr->decode_results[decode_request_id] = std::move(entry); - routes_ptr->evict_decode_results_locked(); - } - - // ── Send sync validation response ─────────────────────── - res->rpc_status = HYDRA_STATUS_OK; - queue_results.send(std::move(res)); - - // ── Post DECODE_APPLY async task ──────────────────────── - { - server_task apply_task(SERVER_TASK_TYPE_HYDRA_DECODE_APPLY); - apply_task.id = queue_tasks.get_new_id(); - apply_task.hydra_action.id_slot = id_slot; - apply_task.hydra_action.decode_json = std::move(task.hydra_action.decode_json); - apply_task.hydra_action.kv_data = std::move(task.hydra_action.kv_data); - apply_task.hydra_action.decode_request_id = decode_request_id; - queue_tasks.post(std::move(apply_task)); - SRV_INF("hydra: DECODE slot=%d posted DECODE_APPLY (request_id=%d)\n", - id_slot, decode_request_id); - } - } break; - case SERVER_TASK_TYPE_HYDRA_DECODE_APPLY: - { - // ── Async phase: model swap + Gate B + KV restore + completion ── - const int id_slot = task.hydra_action.id_slot; - const int32_t decode_request_id = task.hydra_action.decode_request_id; - - // Parse the DECODE JSON header (re-parsed for async context) - json decode_req; - try { - decode_req = json::parse(task.hydra_action.decode_json); - } catch (const std::exception & e) { - SRV_WRN("hydra: DECODE_APPLY slot=%d invalid JSON: %s\n", id_slot, e.what()); - // Release reservation on error - server_slot * s = get_slot_by_id(id_slot); - if (s) s->reserved_for_decode_id = -1; - if (routes_ptr) { - server_routes::decode_result_entry entry; - entry.id_slot = id_slot; - entry.error = std::string("DECODE_APPLY JSON parse error: ") + e.what(); - entry.created_at = std::time(nullptr); - entry.ttl_s = routes_ptr->decode_result_ttl_s; - std::lock_guard lock(routes_ptr->decode_results_mutex); - routes_ptr->decode_results[decode_request_id] = std::move(entry); - routes_ptr->evict_decode_results_locked(); - } - break; - } - - const json & kv_meta = decode_req["kv_metadata"]; - const json & model_meta = decode_req.value("model_metadata", json::object()); - - // ── Model swap (if requested model != resident) ───────── - const std::string requested_model = decode_req.value("model", std::string()); - double model_load_ms = 0.0; - bool model_fallback = false; - - if (!requested_model.empty()) { - // #470: resolve the requested alias against the - // T3-CURRENT alias → file map FIRST. The coordinator's - // T3 config (model_path) can load a file the preset INI - // does not associate with the engine's current alias - // (e.g. the dense-27b-combined session T3-loads the - // 27B-Coder file while the alias identity still says - // qwen3.6-35B-balanced). When the requested alias's - // T3-current file == resident, the alias describes the - // resident — swapping to the INI's file would be a - // pointless 73-81s reload + COMBINED teardown/reattach - // that then fails Gate B (header model_metadata of the - // pre-swap resident vs the swapped-in model's identity). - const auto t3it = t3_current_alias_to_path.find(requested_model); - if (t3it != t3_current_alias_to_path.end() && t3it->second == params_base.model.path) { - SRV_INF("hydra: DECODE_APPLY slot=%d model='%s' matches T3-current resident '%s' — no swap\n", - id_slot, requested_model.c_str(), params_base.model.path.c_str()); - } else { - auto it = preset_alias_to_path.find(requested_model); - if (it == preset_alias_to_path.end()) { - SRV_WRN("hydra: DECODE_APPLY slot=%d model='%s' unknown — falling back to resident '%s'\n", - id_slot, requested_model.c_str(), model_name.c_str()); - model_fallback = true; - } else if (it->second != params_base.model.path) { - SRV_INF("hydra: DECODE_APPLY slot=%d model='%s' swapping %s -> %s\n", - id_slot, requested_model.c_str(), params_base.model.path.c_str(), - it->second.c_str()); - common_params swapped_params = params_base; - // Apply the target alias's full preset (same - // treatment, and same intentional full-preset - // scope, as the PREFILL path above). - auto pit = preset_alias_to_preset.find(requested_model); - bool preset_apply_failed = false; - if (pit != preset_alias_to_preset.end()) { - // Same clear+re-pad+try/catch as the PREFILL path. - swapped_params.tensor_buft_overrides.clear(); - try { - pit->second.apply_to_params(swapped_params); - hydra_repad_tensor_buft_overrides(swapped_params, "DECODE_APPLY swap"); - } catch (const std::exception & e) { - SRV_WRN("hydra: DECODE_APPLY slot=%d swap preset apply for '%s' failed: %s\n", - id_slot, requested_model.c_str(), e.what()); - preset_apply_failed = true; - if (routes_ptr) { - server_routes::decode_result_entry entry; - entry.id_slot = id_slot; - entry.error = std::string("model swap preset apply failed: ") + e.what(); - entry.created_at = std::time(nullptr); - entry.ttl_s = routes_ptr->decode_result_ttl_s; - std::lock_guard lock(routes_ptr->decode_results_mutex); - routes_ptr->decode_results[decode_request_id] = std::move(entry); - routes_ptr->evict_decode_results_locked(); - } - } - } - if (preset_apply_failed) { - server_slot * s = get_slot_by_id(id_slot); - if (s) s->reserved_for_decode_id = -1; - break; - } - swapped_params.model.path = it->second; - swapped_params.model_alias = { requested_model }; - // #514: tear down COMBINED state before the - // reload — see hydra_teardown_combined_before_reload(). - const bool was_combined = hydra_combined_head_attached || hydra_combined_static; - if (was_combined) { - hydra_teardown_combined_before_reload(); - } - const int64_t model_load_start_ms = ggml_time_ms(); - if (!load_model(swapped_params)) { - SRV_WRN("hydra: DECODE_APPLY slot=%d model swap to '%s' failed\n", - id_slot, requested_model.c_str()); - server_slot * s = get_slot_by_id(id_slot); - if (s) s->reserved_for_decode_id = -1; - if (routes_ptr) { - server_routes::decode_result_entry entry; - entry.id_slot = id_slot; - entry.error = "model swap to '" + requested_model + "' failed"; - entry.created_at = std::time(nullptr); - entry.ttl_s = routes_ptr->decode_result_ttl_s; - std::lock_guard lock(routes_ptr->decode_results_mutex); - routes_ptr->decode_results[decode_request_id] = std::move(entry); - routes_ptr->evict_decode_results_locked(); - } - break; - } - if (was_combined) { - hydra_reattach_combined_after_reload(); - } - model_load_ms = (double)(ggml_time_ms() - model_load_start_ms); - SRV_INF("hydra: DECODE_APPLY slot=%d swap confirmed model_load_ms=%.1f\n", - id_slot, model_load_ms); - } - } - } - - // ── Gate B: post-load identity check ──────────────────── - // Compare model_metadata from header vs ACTUAL resident GGUF identity. - server_slot * slot = get_slot_by_id(id_slot); - if (slot == nullptr) { - SRV_WRN("hydra: DECODE_APPLY slot=%d disappeared after model swap\n", id_slot); - if (routes_ptr) { - server_routes::decode_result_entry entry; - entry.id_slot = id_slot; - entry.error = "slot disappeared after model swap"; - entry.created_at = std::time(nullptr); - entry.ttl_s = routes_ptr->decode_result_ttl_s; - std::lock_guard lock(routes_ptr->decode_results_mutex); - routes_ptr->decode_results[decode_request_id] = std::move(entry); - routes_ptr->evict_decode_results_locked(); - } - break; - } - - const std::string resident_tokenizer = llama_model_get_tokenizer_model(model_tgt); - const std::string resident_model_name = llama_model_get_display_name(model_tgt); - const std::string resident_model_quant = llama_model_get_quant_label(model_tgt); - const uint32_t resident_capabilities = llama_model_get_capabilities_bitfield(model_tgt); - - const std::string hdr_model_name = model_meta.value("model_name", ""); - const std::string hdr_model_quant = model_meta.value("model_quant", ""); - const uint32_t hdr_capabilities = model_meta.value("model_capabilities", 0u); - - const bool gate_b_tokenizer = (resident_tokenizer == model_meta.value("tokenizer", "")); - const bool gate_b_model_name = (resident_model_name == hdr_model_name); - const uint32_t gate_b_caps_xor = resident_capabilities ^ hdr_capabilities; - - if (!gate_b_tokenizer || !gate_b_model_name || (gate_b_caps_xor & 0x3)) { - SRV_WRN("hydra: DECODE_APPLY slot=%d Gate B reject — tokenizer=%d name=%d caps_xor=0x%x\n", - id_slot, gate_b_tokenizer, gate_b_model_name, gate_b_caps_xor); - slot->reserved_for_decode_id = -1; - if (routes_ptr) { - server_routes::decode_result_entry entry; - entry.id_slot = id_slot; - entry.error = "Gate B identity mismatch after model load"; - entry.match_json = {{"gate_b_tokenizer", gate_b_tokenizer}, {"gate_b_name", gate_b_model_name}, {"gate_b_caps_xor", gate_b_caps_xor}}; - entry.created_at = std::time(nullptr); - entry.ttl_s = routes_ptr->decode_result_ttl_s; - std::lock_guard lock(routes_ptr->decode_results_mutex); - routes_ptr->decode_results[decode_request_id] = std::move(entry); - routes_ptr->evict_decode_results_locked(); - } - break; - } - - if (resident_model_quant != hdr_model_quant) { - SRV_INF("hydra: DECODE_APPLY slot=%d Gate B quant differs (%s → %s) — mix-quant allowed\n", - id_slot, hdr_model_quant.c_str(), resident_model_quant.c_str()); - } - - // ── KV restore ───────────────────────────────────────── - const int64_t restore_start_ms = ggml_time_ms(); - - // M2 (#470): the v2 header arrives pre-parsed (kv_v2_hdr, - // small) and the KV state stream is read directly off - // hydra_fd via llama_state_seq_set_data_from_fd — the engine - // never materializes the full blob (2.3 GB today, 10 GB - // target). M1 (kv_data) is the buffered fallback. - const bool m2_stream = !task.hydra_action.kv_v2_hdr.empty(); - if (!task.hydra_action.kv_data.empty() || m2_stream) { - slot->prompt_clear(false); - slot->n_prompt_tokens_cache = 0; - slot->n_prompt_tokens_processed = 0; - slot->n_decoded = 0; - - // The coordinator may send the v2/v3 blob (header + raw KV) - // or just the raw KV data. Parse the v2 header to extract - // the token list so update_slots()'s n_common decision can - // match incoming tokens against the restored KV — without - // this, prompt.tokens is empty after prompt_clear(), n_past - // computes to 0, and seq_rm(slot, 0, -1) wipes the KV that - // llama_state_seq_set_data just loaded (issue #506). - const uint8_t * kv_ptr = m2_stream - ? task.hydra_action.kv_v2_hdr.data() - : task.hydra_action.kv_data.data(); - size_t kv_len = m2_stream - ? task.hydra_action.kv_v2_hdr.size() - : task.hydra_action.kv_data.size(); - int32_t blob_n_past = 0; - int32_t blob_n_tok = 0; - bool has_chkpt = false; - bool ckpt_is_recr_only = false; - int32_t ckpt_pos_min_in = 0, ckpt_pos_max_in = 0; - int64_t ckpt_n_tokens_in = 0; - std::vector ckpt_tgt_data, ckpt_dft_data; - - // v2 (0x02) blobs carry a full checkpoint; v3 (0x03) blobs may carry - // a recurrent-only checkpoint (hdr_flags bit 0x02). Both share the - // header layout — the M2-stream double-write fix bumped the version. - const bool is_v2 = kv_len >= 1 && (kv_ptr[0] == 0x02 || kv_ptr[0] == 0x03); - if (is_v2 && kv_len >= 9) { - memcpy(&blob_n_past, kv_ptr + 1, 4); - memcpy(&blob_n_tok, kv_ptr + 5, 4); - - const size_t token_start = 9; - const size_t token_end = token_start + (size_t)blob_n_tok * sizeof(llama_token); - if (blob_n_tok > 0 && token_end <= kv_len) { - // Restore token list from v2/v3 blob header - slot->prompt.tokens.clear(); - const llama_token * tok_ptr = (const llama_token *)(kv_ptr + token_start); - llama_tokens restored_tokens(tok_ptr, tok_ptr + (size_t)blob_n_tok); - slot->prompt.tokens.insert(restored_tokens); - SRV_INF("hydra: DECODE_APPLY slot=%d v2/v3 blob: restored %d tokens from header\n", - id_slot, blob_n_tok); - } - - // Skip past v2/v3 header (version + n_past + n_tok + tokens + flags + optional checkpoint) - size_t hdr_offset = token_end; - if (hdr_offset < kv_len) { - const uint8_t flags = kv_ptr[hdr_offset]; - hdr_offset += 1; // past flags byte - // bit 0x01 = has checkpoint; bit 0x02 = recurrent-only (PARTIAL_ONLY) - ckpt_is_recr_only = (flags & 0x02) != 0; - if (flags & 0x01) { - // Capture checkpoint: 4B pos_min | 4B pos_max | 8B n_tokens | 8B tgt_sz | tgt_data | 8B dft_sz | dft_data - // Mirrors the STATE_PUT sibling (~line 3343) — the native - // checkpoint is registered after restore so hybrid/recurrent - // models get their recurrent memory back (KV restored without - // its checkpoint is corrupt). - if (hdr_offset + 4 + 4 + 8 + 8 <= kv_len) { - memcpy(&ckpt_pos_min_in, kv_ptr + hdr_offset, 4); hdr_offset += 4; - memcpy(&ckpt_pos_max_in, kv_ptr + hdr_offset, 4); hdr_offset += 4; - memcpy(&ckpt_n_tokens_in, kv_ptr + hdr_offset, 8); hdr_offset += 8; - uint64_t tgt_sz_in; - memcpy(&tgt_sz_in, kv_ptr + hdr_offset, 8); hdr_offset += 8; - if (tgt_sz_in > 0 && hdr_offset + tgt_sz_in <= kv_len) { - ckpt_tgt_data.assign(kv_ptr + hdr_offset, kv_ptr + hdr_offset + (size_t)tgt_sz_in); - hdr_offset += (size_t)tgt_sz_in; - } - if (hdr_offset + 8 <= kv_len) { - uint64_t dft_sz_in; - memcpy(&dft_sz_in, kv_ptr + hdr_offset, 8); hdr_offset += 8; - if (dft_sz_in > 0 && hdr_offset + dft_sz_in <= kv_len) { - ckpt_dft_data.assign(kv_ptr + hdr_offset, kv_ptr + hdr_offset + (size_t)dft_sz_in); - hdr_offset += (size_t)dft_sz_in; - } - } - has_chkpt = true; - } - } - } - // Advance kv_ptr/kv_len past the v2 header to the raw KV state - if (hdr_offset <= kv_len) { - kv_ptr = kv_ptr + hdr_offset; - kv_len = kv_len - hdr_offset; - } - } - - // M2 (#470): hash the whole kv segment as it streams — - // v2 header first, then every byte the fd restore - // consumes, then the logits tail (wire order). - XXH3_state_t * hst = nullptr; - if (m2_stream) { - hst = XXH3_createState(); - XXH3_64bits_reset(hst); - XXH3_64bits_update(hst, task.hydra_action.kv_v2_hdr.data(), - task.hydra_action.kv_v2_hdr.size()); - } - - size_t status = 0; - if (m2_stream) { - // Stream restore: consumes [4B magic][4B seq_id] + KV - // state off the fd; logits tail is read separately below. - status = llama_state_seq_set_data_from_fd( - ctx_tgt, slot->id, task.hydra_action.hydra_fd, hst); - } else { - status = llama_state_seq_set_data( - ctx_tgt, - kv_ptr, - kv_len, - slot->id); - } - - // llama_state_seq_set_data returns the number of bytes - // read on success (0 means failed to load) — see its - // doc comment in include/llama.h. `status` only counts - // the KV-cache bytes the reader consumed; it does NOT - // include the trailing logits PREFILL_DONE appends - // (see ~line 4098), so status < kv_len is the normal - // case whenever logits are present — compare against - // kv_len here and this false-fails on every restore - // with logits. Matches the STATE_PUT sibling check - // (server-context.cpp ~line 3395: `if (n_read == 0)`). - if (status == 0) { - SRV_WRN("hydra: DECODE_APPLY slot=%d KV restore failed (%d)\n", id_slot, status); - if (hst) { XXH3_freeState(hst); hst = nullptr; } - if (m2_stream) { - // The stream broke mid-way: drop the read side so - // residual unread bytes cannot misalign the next - // request frame. The RPC thread still writes the - // error response (write side stays open). - ::shutdown(task.hydra_action.hydra_fd, SHUT_RD); - } - slot->reserved_for_decode_id = -1; - // Tokens were registered from the v2 header before set_data — - // clear them so the slot is not left poisoned (n_past > 0 - // with no KV cells → pos_min == -1 abort on the next decode - // that touches this slot). Matches the STATE_PUT failure path. - slot->prompt.tokens.clear(); - slot->prompt.checkpoints.clear(); - slot->n_prompt_tokens_cache = 0; - llama_memory_seq_rm(llama_get_memory(ctx_tgt), slot->id, -1, -1); - if (routes_ptr) { - server_routes::decode_result_entry entry; - entry.id_slot = id_slot; - entry.error = "KV restore failed (llama_state_seq_set_data returned " + std::to_string(status) + ")"; - entry.created_at = std::time(nullptr); - entry.ttl_s = routes_ptr->decode_result_ttl_s; - std::lock_guard lock(routes_ptr->decode_results_mutex); - routes_ptr->decode_results[decode_request_id] = std::move(entry); - routes_ptr->evict_decode_results_locked(); - } - break; - } - - // Trailing logits: PREFILL_DONE appends n_vocab floats - // after the KV state (~line 4098) so the decode side - // can sample immediately instead of reading garbage - // after restore. Mirrors STATE_PUT's per-slot - // injection (~line 3405) — DECODE_APPLY was missing - // this step entirely. - { - const size_t expected_logits = (size_t)llama_vocab_n_tokens(vocab) * sizeof(float); - if (m2_stream) { - // Read the logits tail straight off the fd (small). - const size_t remaining = - (size_t)(task.hydra_action.kv_stream_len - status); - if (remaining == expected_logits) { - std::vector logits_buf(remaining); - if (hydra_recv_all(task.hydra_action.hydra_fd, - logits_buf.data(), remaining)) { - XXH3_64bits_update(hst, logits_buf.data(), remaining); - const size_t n_floats = llama_vocab_n_tokens(vocab); - slot->restored_logits.assign( - reinterpret_cast(logits_buf.data()), - reinterpret_cast(logits_buf.data()) + n_floats); - slot->logits_valid = true; - SRV_INF("hydra: DECODE_APPLY slot=%d restored %zu logits to per-slot buffer\n", - id_slot, n_floats); - } else { - SRV_WRN("hydra: DECODE_APPLY slot=%d logits tail read failed\n", id_slot); - } - } - } else { - const size_t remaining = kv_len - status; - if (remaining == expected_logits) { - const float * src = (const float *)(kv_ptr + status); - const size_t n_floats = llama_vocab_n_tokens(vocab); - slot->restored_logits.assign(src, src + n_floats); - slot->logits_valid = true; - SRV_INF("hydra: DECODE_APPLY slot=%d restored %zu logits to per-slot buffer\n", - id_slot, n_floats); - } - } - } - - // M2 wire-hash verification (post-restore — with streaming - // the bytes reach the GPU before a pre-restore hash could - // be computed). On mismatch the slot is cleared so the next - // decode cannot sample corrupt state, and the response - // carries the terminal error for the Coordinator to retry. - if (m2_stream && task.hydra_action.kv_expected_hash != 0) { - const uint64_t computed_kv = XXH3_64bits_digest(hst); - if (computed_kv != task.hydra_action.kv_expected_hash) { - SRV_WRN("hydra: DECODE_APPLY slot=%d SEGMENT_HASH_MISMATCH kv expected=%016" PRIx64 " got=%016" PRIx64 "\n", - id_slot, task.hydra_action.kv_expected_hash, computed_kv); - XXH3_freeState(hst); - hst = nullptr; - slot->reserved_for_decode_id = -1; - slot->prompt.tokens.clear(); - slot->prompt.checkpoints.clear(); - slot->n_prompt_tokens_cache = 0; - llama_memory_seq_rm(llama_get_memory(ctx_tgt), slot->id, -1, -1); - if (routes_ptr) { - server_routes::decode_result_entry entry; - entry.id_slot = id_slot; - entry.error = "KV segment hash mismatch (corrupt stream)"; - entry.created_at = std::time(nullptr); - entry.ttl_s = routes_ptr->decode_result_ttl_s; - std::lock_guard lock(routes_ptr->decode_results_mutex); - routes_ptr->decode_results[decode_request_id] = std::move(entry); - routes_ptr->evict_decode_results_locked(); - } - break; - } - SRV_INF("hydra: DECODE_APPLY slot=%d KV hash verified (%zu + %" PRIu64 " B)\n", - id_slot, task.hydra_action.kv_v2_hdr.size(), - task.hydra_action.kv_stream_len); - } - if (hst) { XXH3_freeState(hst); hst = nullptr; } - - const int n_past = is_v2 ? blob_n_past : kv_meta.value("n_past", 0); - if (n_past > 0) { - // Cache/processed counters come from the same header field - // STATE_PUT reads (hdr_n_tok == blob_n_tok here); PREFILL writes - // both fields as n_tokens so the values are identical today, - // but the two restore paths must read the SAME source. - slot->n_prompt_tokens_cache = is_v2 ? blob_n_tok : n_past; - slot->n_prompt_tokens_processed = is_v2 ? blob_n_tok : n_past; - - // Register the native checkpoint from the blob (v2) or - // fabricate one (legacy) — mirrors STATE_PUT (~line 3447). - // KV restored without its recurrent-memory checkpoint - // corrupts hybrid/recurrent model output. - slot->prompt.checkpoints.clear(); - if (has_chkpt) { - auto & ckpt = slot->prompt.checkpoints.emplace_back(); - ckpt.n_tokens = ckpt_n_tokens_in; - ckpt.pos_min = ckpt_pos_min_in; - ckpt.pos_max = ckpt_pos_max_in; - // New-format (v3) checkpoints carry a recurrent-only capture — - // route into data_*_recr and tag is_recr_only so the load path - // uses matched PARTIAL_ONLY flags (mirrors STATE_PUT). - ckpt.is_recr_only = ckpt_is_recr_only; - if (ckpt_is_recr_only) { - ckpt.data_tgt_recr = std::move(ckpt_tgt_data); - ckpt.data_dft_recr = std::move(ckpt_dft_data); - } else { - ckpt.data_tgt = std::move(ckpt_tgt_data); - ckpt.data_dft = std::move(ckpt_dft_data); - } - SLT_INF(*slot, "DECODE_APPLY registered native checkpoint (pos_min=%d pos_max=%d n_tokens=%" PRId64 " tgt_sz=%zu recr_only=%d)\n", - ckpt.pos_min, ckpt.pos_max, ckpt.n_tokens, ckpt.size(), (int) ckpt.is_recr_only); - } else { - const auto pos_min = llama_memory_seq_pos_min(llama_get_memory(ctx_tgt), slot->id); - create_checkpoint(*slot, 0, (llama_pos)pos_min, (llama_pos)(n_past - 1)); - } - } - slot->just_restored = true; - } - - const double restore_slot_ms = (double)(ggml_time_ms() - restore_start_ms); - const int n_past = slot->n_prompt_tokens_cache + slot->n_decoded; - - SRV_INF("hydra: DECODE_APPLY slot=%d restore=%.1fms n_past=%d model_load_ms=%.1f\n", - id_slot, restore_slot_ms, n_past, model_load_ms); - - // Release reservation — slot is now processing via completion - slot->reserved_for_decode_id = -1; - - // ── Build and post COMPLETION task ────────────────────── - { - json prompt = decode_req["prompt"]; - json cmpl_data; - cmpl_data["stream"] = prompt.value("stream", false); - // #622: the DECODE 0x43 frame has no dedicated stream_options - // channel, but the coordinator always requests usage on the - // merged path (it injects stream_options.include_usage=true on - // its HTTP body). Honor stream_options when the request carries - // it (generation header / prompt segment), otherwise mirror the - // coordinator's injection so the DONE-SSE delta carries usage - // natively and the coordinator's usage-based gate fires. - if (prompt.contains("stream_options") && prompt["stream_options"].is_object()) { - cmpl_data["stream_options"] = prompt["stream_options"]; - } else { - cmpl_data["stream_options"] = json{{"include_usage", true}}; - } - cmpl_data["n_predict"] = prompt.value("n_predict", 256); - cmpl_data["id_slot"] = id_slot; - if (prompt.contains("sampling")) { - const json & samp = prompt["sampling"]; - if (samp.contains("temperature")) cmpl_data["temperature"] = samp["temperature"]; - if (samp.contains("top_p")) cmpl_data["top_p"] = samp["top_p"]; - if (samp.contains("top_k")) cmpl_data["top_k"] = samp["top_k"]; - if (samp.contains("seed")) cmpl_data["seed"] = samp["seed"]; - } - if (prompt.contains("stop")) cmpl_data["stop"] = prompt["stop"]; - - std::string prompt_str; - if (prompt.contains("messages") && !prompt["messages"].is_null()) { - json chat_body; - chat_body["messages"] = prompt["messages"]; - if (prompt.contains("tools")) chat_body["tools"] = prompt["tools"]; - if (prompt.contains("tool_choice")) chat_body["tool_choice"] = prompt["tool_choice"]; - if (prompt.contains("response_format")) chat_body["response_format"] = prompt["response_format"]; - if (prompt.contains("add_generation_prompt")) chat_body["add_generation_prompt"] = prompt["add_generation_prompt"]; - if (prompt.contains("continue_final_message")) chat_body["continue_final_message"] = prompt["continue_final_message"]; - if (prompt.contains("reasoning_format")) chat_body["reasoning_format"] = prompt["reasoning_format"]; - if (prompt.contains("enable_thinking")) chat_body["enable_thinking"] = prompt["enable_thinking"]; - if (prompt.contains("chat_template_kwargs")) chat_body["chat_template_kwargs"] = prompt["chat_template_kwargs"]; - - try { - std::vector dummy_files; - json chat_result = oaicompat_chat_params_parse(chat_body, chat_params, dummy_files); - prompt_str = chat_result.value("prompt", std::string()); - if (chat_result.contains("grammar") && !chat_result["grammar"].is_null()) cmpl_data["grammar"] = chat_result["grammar"]; - if (chat_result.contains("grammar_type")) cmpl_data["grammar_type"] = chat_result["grammar_type"]; - if (chat_result.contains("grammar_lazy")) cmpl_data["grammar_lazy"] = chat_result["grammar_lazy"]; - if (chat_result.contains("grammar_triggers")) cmpl_data["grammar_triggers"] = chat_result["grammar_triggers"]; - if (chat_result.contains("chat_format")) cmpl_data["chat_format"] = chat_result["chat_format"]; - if (chat_result.contains("chat_parser")) cmpl_data["chat_parser"] = chat_result["chat_parser"]; - if (chat_result.contains("generation_prompt")) cmpl_data["generation_prompt"] = chat_result["generation_prompt"]; - if (chat_result.contains("parse_tool_calls")) cmpl_data["parse_tool_calls"] = chat_result["parse_tool_calls"]; - if (chat_result.contains("preserved_tokens")) cmpl_data["preserved_tokens"] = chat_result["preserved_tokens"]; - if (chat_result.contains("reasoning_budget_tokens")) cmpl_data["reasoning_budget_tokens"] = chat_result["reasoning_budget_tokens"]; - if (chat_result.contains("reasoning_budget_start_tag")) cmpl_data["reasoning_budget_start_tag"] = chat_result["reasoning_budget_start_tag"]; - if (chat_result.contains("reasoning_budget_end_tag")) cmpl_data["reasoning_budget_end_tag"] = chat_result["reasoning_budget_end_tag"]; - if (chat_result.contains("reasoning_budget_message")) cmpl_data["reasoning_budget_message"] = chat_result["reasoning_budget_message"]; - if (chat_result.contains("reasoning_control")) cmpl_data["reasoning_control"] = chat_result["reasoning_control"]; - if (chat_result.contains("stop") && chat_result["stop"].is_array()) { - json existing_stops = cmpl_data.value("stop", json::array()); - for (const auto & s : chat_result["stop"]) existing_stops.push_back(s); - cmpl_data["stop"] = existing_stops; - } - } catch (const std::exception & e) { - SRV_WRN("hydra: DECODE_APPLY slot=%d chat template failed: %s\n", id_slot, e.what()); - if (routes_ptr) { - server_routes::decode_result_entry entry; - entry.id_slot = id_slot; - entry.error = std::string("chat template error: ") + e.what(); - entry.created_at = std::time(nullptr); - entry.ttl_s = routes_ptr->decode_result_ttl_s; - std::lock_guard lock(routes_ptr->decode_results_mutex); - routes_ptr->decode_results[decode_request_id] = std::move(entry); - routes_ptr->evict_decode_results_locked(); - } - break; - } - } else { - prompt_str = prompt.value("prompt", std::string()); - } - cmpl_data["prompt"] = prompt_str; - - auto inputs = tokenize_input_prompts(vocab, mctx, prompt_str, true, true); - if (!inputs.empty()) { - const int32_t completion_id = queue_tasks.get_new_id(); - - server_task cmpl_task(SERVER_TASK_TYPE_COMPLETION); - cmpl_task.id = completion_id; - cmpl_task.id_slot = id_slot; - cmpl_task.tokens = std::move(inputs[0]); - cmpl_task.params = server_task::params_from_json_cmpl( - vocab, params_base, get_slot_n_ctx(), params_base.sampling.logit_bias_eog, cmpl_data); - cmpl_task.params.res_type = TASK_RESPONSE_TYPE_OAI_CHAT; - cmpl_task.params.oaicompat_cmpl_id = gen_chatcmplid(); - cmpl_task.params.oaicompat_model = model_name; - - // Mirror server_response_reader::post_task(): the - // consumer thread keeps its own result state so it - // can run result->update() per received result. - task_result_state cmpl_state = cmpl_task.create_state(); - - queue_results.add_waiting_task_id(completion_id); - queue_tasks.post(std::move(cmpl_task)); - SRV_INF("hydra: DECODE_APPLY slot=%d posted COMPLETION (completion_id=%d, request_id=%d)\n", - id_slot, completion_id, decode_request_id); - - // Update state to GENERATING - if (routes_ptr) { - std::lock_guard lk(routes_ptr->decode_results_mutex); - auto dit = routes_ptr->decode_results.find(decode_request_id); - if (dit != routes_ptr->decode_results.end()) { - dit->second.state = server_routes::DECODE_STATE_GENERATING; - dit->second.completion_id = std::to_string(completion_id); - dit->second.stream->completion_task_id = completion_id; - // Capture n_common observability from the slot - dit->second.n_common = slot->n_common; - dit->second.n_prompt_processed = slot->n_prompt_processed; - dit->second.logits_reused = slot->logits_reused; - } - } - - // ── Background consumer ───────────────────────── - // Sole listener on the completion task. Relays - // partial results into the decode_result_entry's - // streaming_queue so GET /v1/decode can stream - // them to the client. Stores the final result - // when generation completes. - if (routes_ptr) { - // Read match_json from the decode_result_entry (set by sync DECODE) - json match_j_bg; - { - std::lock_guard lk(routes_ptr->decode_results_mutex); - auto dit = routes_ptr->decode_results.find(decode_request_id); - if (dit != routes_ptr->decode_results.end()) { - match_j_bg = dit->second.match_json; - } - } - std::thread([this, completion_id, decode_request_id, id_slot, - match_j = std::move(match_j_bg), resident_tokenizer, resident_model_name, - resident_model_quant, resident_capabilities, - oaicompat_model_name = model_name, - model_load_ms, restore_slot_ms, n_past, - &results = queue_results, - states = std::vector{ std::move(cmpl_state) }]() mutable { - std::unordered_set ids = {(int)completion_id}; - bool got_final = false; - - // Loop: receive partials and relay, wait for final - while (!got_final) { - auto res_ptr = results.recv_with_timeout(ids, 120); - if (!res_ptr) { - SRV_WRN("hydra: DECODE_APPLY slot=%d generation timeout (request_id=%d, completion_id=%d)\n", - id_slot, decode_request_id, completion_id); - // Mark stream as finished so GET handler unblocks - { - std::lock_guard lk(routes_ptr->decode_results_mutex); - auto dit = routes_ptr->decode_results.find(decode_request_id); - if (dit != routes_ptr->decode_results.end() && dit->second.stream) { - std::lock_guard slk(dit->second.stream->streaming_mutex); - dit->second.stream->stream_finished = true; - dit->second.stream->streaming_cv.notify_all(); - } - } - return; - } - - // Check if this is a partial or final result - auto * partial = dynamic_cast(res_ptr.get()); - auto * final_r = dynamic_cast(res_ptr.get()); - - // Mirror server_response_reader::next(): run - // update() on every result before handling. - // Populates oaicompat_msg / oaicompat_msg_diffs - // (and sets is_updated, so to_json() won't - // assert on relayed partials). - try { - const size_t idx = res_ptr->index; - GGML_ASSERT(idx < states.size()); - res_ptr->update(states[idx]); - } catch (const std::exception & e) { - // Mirror the standard stream loop's tolerance - // of chat-parse failures (server-context.cpp:7685). - // This is a detached thread: an uncaught - // exception would std::terminate() the whole - // engine. Continue with the unparsed result - // (raw content; reasoning extraction skipped). - SRV_WRN("hydra: DECODE_APPLY slot=%d result update() failed: %s (continuing with unparsed result)\n", - id_slot, e.what()); - if (partial && !partial->is_begin) { - // Keep the relay well-formed: partial - // to_json() asserts is_updated in debug - // builds; with no diffs it emits an empty - // delta, which clients merge harmlessly. - partial->is_updated = true; - } - } - - if (partial && !partial->is_begin) { - // Relay partial to streaming queue - std::lock_guard lk(routes_ptr->decode_results_mutex); - auto dit = routes_ptr->decode_results.find(decode_request_id); - if (dit != routes_ptr->decode_results.end() && dit->second.stream) { - std::lock_guard slk(dit->second.stream->streaming_mutex); - dit->second.stream->streaming_queue.push_back(std::move(res_ptr)); - dit->second.stream->streaming_cv.notify_all(); - } - } else if (final_r) { - // Store final result and mark DONE - got_final = true; - - server_routes::decode_result_entry entry; - entry.id_slot = id_slot; - entry.completion_id = final_r->oaicompat_cmpl_id; - entry.oaicompat_model = oaicompat_model_name; - entry.content = final_r->content; - if (!final_r->oaicompat_msg.reasoning_content.empty()) { - entry.reasoning_content = final_r->oaicompat_msg.reasoning_content; - } - if (!final_r->oaicompat_msg.tool_calls.empty()) { - // Mirror common_chat_msg::to_json_oaicompat() shape so - // GET /v1/decode/:id returns OpenAI-format tool_calls. - json jtool_calls = json::array(); - for (const auto & tool_call : final_r->oaicompat_msg.tool_calls) { - json tc { - {"type", "function"}, - {"function", { - {"name", tool_call.name}, - {"arguments", json(tool_call.arguments)}, - }}, - }; - if (!tool_call.id.empty()) { - tc["id"] = tool_call.id; - } - jtool_calls.push_back(std::move(tc)); - } - entry.tool_calls = std::move(jtool_calls); - } - entry.n_decoded = final_r->n_decoded; - entry.n_prompt_tokens = final_r->n_prompt_tokens; - entry.n_prompt_tokens_cache = final_r->n_prompt_tokens_cache; - entry.timings = final_r->timings; - entry.stop = final_r->stop; - entry.include_usage = final_r->include_usage; - entry.match_json = match_j; - entry.created_at = std::time(nullptr); - entry.ttl_s = routes_ptr->decode_result_ttl_s; - - json metrics = json::object(); - metrics["decode_request_id"] = decode_request_id; - metrics["id_slot"] = id_slot; - metrics["n_past"] = final_r->n_prompt_tokens_cache + final_r->n_decoded; - metrics["decode_ms"] = final_r->timings.predicted_ms; - metrics["prompt_ms"] = final_r->timings.prompt_ms; - metrics["model_load_ms"] = model_load_ms; - metrics["restore_slot_ms"] = restore_slot_ms; - metrics["model_identity"] = { - {"tokenizer", resident_tokenizer}, - {"model_name", resident_model_name}, - {"model_quant", resident_model_quant}, - {"model_capabilities", resident_capabilities} - }; - metrics["match"] = match_j; - metrics["model_fallback"] = false; - // Hydra n_common observability - metrics["n_common"] = entry.n_common; - metrics["n_prompt_processed"] = entry.n_prompt_processed; - metrics["logits_reused"] = entry.logits_reused; - entry.hydra_metrics = metrics; - entry.state = server_routes::DECODE_STATE_DONE; - - // Signal stream finished before storing entry - { - std::lock_guard lk(routes_ptr->decode_results_mutex); - auto dit = routes_ptr->decode_results.find(decode_request_id); - if (dit != routes_ptr->decode_results.end() && dit->second.stream) { - // Transfer streaming state to the new entry - entry.stream = std::move(dit->second.stream); - { - std::lock_guard slk(entry.stream->streaming_mutex); - entry.stream->stream_finished = true; - } - entry.stream->streaming_cv.notify_all(); - } - } - - std::lock_guard lock(routes_ptr->decode_results_mutex); - routes_ptr->decode_results[decode_request_id] = std::move(entry); - routes_ptr->evict_decode_results_locked(); - - SRV_INF("hydra: DECODE_APPLY slot=%d generation complete (request_id=%d, n_decoded=%d)\n", - id_slot, decode_request_id, final_r->n_decoded); - } else { - // is_begin partial — just consume it - } - } - - results.remove_waiting_task_id(completion_id); - }).detach(); - } - } else { - SRV_WRN("hydra: DECODE_APPLY slot=%d tokenization failed\n", id_slot); - } - } - } break; - case SERVER_TASK_TYPE_HYDRA_ENGINE_SET_EXPERT_MODE: - { - auto res = std::make_unique(); - res->id = task.id; - res->op = HYDRA_OP_SET_EXPERT_MODE; - - // Parse the payload. For backward compatibility, a raw string - // ("solo" or "combined") is accepted. Phase D (C# side) sends - // a JSON payload: {"mode":"combined","peer":"host:port",...}. - std::string requested; - std::string peer_override; - const std::string & raw = task.hydra_action.expert_mode; - if (!raw.empty() && raw[0] == '{') { - try { - json j = json::parse(raw); - requested = j.value("mode", "solo"); - peer_override = j.value("peer", ""); - } catch (...) { - requested = "solo"; - } - } else { - requested = raw; - } - - if (requested != "solo" && requested != "combined") { - res->rpc_status = HYDRA_STATUS_ERROR; - res->success = false; - res->error = "expert_mode must be 'solo' or 'combined'"; - queue_results.send(std::move(res)); - break; - } - - // #29 Phase B: per-request peer switching. If the peer changes, - // clean up the old binding and register the new one. The peer - // info comes from the SET_EXPERT_MODE control-plane payload - // (JSON {"mode":"combined","peer":"host:port"}), NOT from the - // HTTP inference body — keeping control and data separate. - if (!peer_override.empty() && peer_override != hydra_current_peer) { - // Guard: peer switch is unsafe while any slot is decoding. - // sched_reserve() destroys and rebuilds the scheduler, which - // invalidates in-flight decode state across all slots. - bool any_active = false; - for (const auto & s : slots) { - if (s.is_processing()) { any_active = true; break; } - } - if (any_active) { - SRV_WRN("hydra: cannot switch peers — %zu slot(s) are processing, rejecting SET_EXPERT_MODE\n", slots.size()); - res->rpc_status = HYDRA_STATUS_BUSY; - res->success = false; - res->error = "cannot switch peers while slots are processing"; - queue_results.send(std::move(res)); - break; - } - if (!hydra_current_peer.empty()) { - SRV_INF("hydra: switching from peer %s to %s — cleaning up old binding\n", - hydra_current_peer.c_str(), peer_override.c_str()); - ctx_tgt->hydra_remove_combined_rpc_backend(hydra_current_peer.c_str()); - } - hydra_current_peer = peer_override; - } - - // Hydra #383 T1: layer-split (static combined) engines cannot - // switch modes at runtime — the split is baked in at model load. - // "combined" is a no-op (already combined); "solo" is rejected. - if (hydra_combined_static) { - if (requested == "solo") { - res->rpc_status = HYDRA_STATUS_ERROR; - res->success = false; - res->error = "combined_static: this engine loaded in layer-split COMBINED mode; cannot switch to solo at runtime"; - LOG_WRN("srv %12.*s: hydra: SET_EXPERT_MODE solo rejected — engine is combined_static (layer-split)\n", 12, __func__); - queue_results.send(std::move(res)); - break; - } - // requested == "combined": success no-op - res->expert_mode_applied = "combined"; - res->rpc_status = HYDRA_STATUS_OK; - res->success = true; - LOG_INF("srv %12.*s: hydra: SET_EXPERT_MODE combined no-op — engine is combined_static (layer-split)\n", 12, __func__); - queue_results.send(std::move(res)); - break; - } - - // #368 fix: gate on "configured as combined head" (non-empty - // peer addr + OT pattern), NOT on whether the startup - // dual-load succeeded. The rebind path below is fail-open — - // if the peer is still unreachable it stays solo — so - // hydra_combined_head_attached (set only when startup - // succeeded) must NOT block the attempt. Hydra #287/#260/#348 - // intent is preserved: an unconfigured engine (no peer/ - // pattern) still falls back to solo immediately. - const bool want_combined = requested == "combined" && - !hydra_peer.empty() && !hydra_combined_pattern.empty(); - - // #368 (#357 fix): bind-on-activation. Re-bind the peer's - // expert tensors on each SET_EXPERT_MODE("combined") request - // so a peer that was down at boot is picked up on the first - // COMBINED request after it comes up. Fail-open: if the - // rebind fails we stay solo and the Coordinator's - // ReportsSolo path handles it. - bool actually_combined = want_combined; - if (want_combined) { - if (hydra_peer.empty() || hydra_combined_pattern.empty()) { - SRV_WRN("%s\n", "hydra: SET_EXPERT_MODE(combined) but no peer/pattern configured; staying solo"); - actually_combined = false; - } else { - // ggml_backend_rpc_add_server is idempotent — returns - // the existing reg if the peer was registered before. - ggml_backend_reg_t rpc_reg = ggml_backend_reg_by_name("RPC"); - if (!rpc_reg) { - SRV_WRN("%s\n", "hydra: SET_EXPERT_MODE(combined) but RPC backend not available; staying solo"); - actually_combined = false; - } else { - using add_server_fn_t = ggml_backend_reg_t (*)(const char *); - auto add_server_fn = (add_server_fn_t) ggml_backend_reg_get_proc_address(rpc_reg, "ggml_backend_rpc_add_server"); - ggml_backend_reg_t peer_reg = add_server_fn ? add_server_fn(hydra_peer.c_str()) : nullptr; - ggml_backend_dev_t peer_dev = (peer_reg && ggml_backend_reg_dev_count(peer_reg) > 0) ? ggml_backend_reg_dev_get(peer_reg, 0) : nullptr; - if (!peer_dev) { - SRV_WRN("hydra: SET_EXPERT_MODE(combined) but peer %s has no registered device; staying solo\n", - hydra_peer.c_str()); - actually_combined = false; - } else { - int32_t n_bound = llama_hydra_rebind_combined_experts( - ctx_tgt, hydra_peer.c_str(), peer_dev, hydra_combined_pattern.c_str()); - if (n_bound <= 0) { - SRV_WRN("hydra: SET_EXPERT_MODE(combined) rebind on peer %s returned %d; staying solo\n", - hydra_peer.c_str(), n_bound); - actually_combined = false; - } else { - // Peer is up — latch so INFO RPC advertises combined. - hydra_combined_head_attached = true; - } - } - } - } - } - - llama_hydra_set_expert_mode(ctx_tgt, actually_combined ? 1 : 0); - res->expert_mode_applied = actually_combined ? "combined" : "solo"; - - res->rpc_status = HYDRA_STATUS_OK; - res->success = true; - SRV_INF("hydra: SET_EXPERT_MODE requested='%s' applied='%s' (slot %d)\n", - requested.c_str(), res->expert_mode_applied.c_str(), task.hydra_action.id_slot); - queue_results.send(std::move(res)); - } break; - case SERVER_TASK_TYPE_HYDRA_ENGINE_SWAP_QUANT: - { - auto res = std::make_unique(); - res->id = task.id; - res->op = HYDRA_OP_SWAP_QUANT; - res->rpc_status = HYDRA_STATUS_OK; - res->success = true; - SRV_INF("hydra: SWAP_QUANT quant='%s' pattern='%s' (slot %d)\n", - task.hydra_action.quant_key.c_str(), - task.hydra_action.tensor_pattern.c_str(), - task.hydra_action.id_slot); - queue_results.send(std::move(res)); - } break; - - // M-Perf.9 (#289) / issue #287: PIPELINE_ATTACH is part of the - // two-engine "work together" routing tracked in #287. The - // coordinator wires the request; the engine-side scaffolding - // (--override-tensor local-load, activation passing, COMBINED - // expert mode) is the next deliverable. For now this opcode - // returns NOT_IMPLEMENTED so the wire stays in sync — the - // coordinator will treat that as a fallback to solo mode. case SERVER_TASK_TYPE_HYDRA_ENGINE_PIPELINE_ATTACH: - { - auto res = std::make_unique(); - res->id = task.id; - res->op = HYDRA_OP_PIPELINE_ATTACH; - res->rpc_status = HYDRA_STATUS_NOT_IMPLEMENTED; - res->success = false; - res->error = "HYDRA_OP_PIPELINE_ATTACH not yet implemented in this build (see issue #287)"; - SRV_WRN("hydra: PIPELINE_ATTACH received (slot %d) — stubbed, issue #287\n", - task.hydra_action.id_slot); - queue_results.send(std::move(res)); - } break; + hydra_process_task(task); + break; + } } @@ -6364,6 +3919,11 @@ struct server_context_impl { hydra_combined_head_attached = false; } + // epic #610 WS2: HYDRA task dispatch — declared here, defined in + // hydra-server-context.cpp (included at the bottom of this TU). Keeps a + // switch(task.type) wrapper so break/continue semantics are unchanged. + void hydra_process_task(server_task & task); + // Re-attach COMBINED-mode bindings after a model reload, mirroring // hydra_teardown_combined_before_reload() above. Layer-split (static) // just re-enables the mode flag — load_model() already preloaded the @@ -6709,6 +4269,12 @@ struct server_context_impl { } void update_slots() { + // epic #610 WS1: in seam mode the extension may pre-empt the decode + // loop (T3/CONFIGURE/reattach cluster). WS1 impl is a no-op — this is + // a pure A/B switch, both modes run the inline body below. + if (hydra_ext_active && hydra_ext && hydra_ext->pre_loop(*this)) { + return; + } // check if all slots are idle { bool all_idle = true; @@ -7765,7 +5331,13 @@ struct server_context_impl { } if (batch.n_tokens == 0) { - if (++n_empty_consecutive > 3) { + // epic #610 WS1: in seam mode the extension may handle the + // empty-batch case (STATE_GET transfer suppression). WS1 impl is a + // no-op — this is a pure A/B switch, both modes run the inline + // suppression below. + if (hydra_ext_active && hydra_ext && hydra_ext->on_empty_batch(*this)) { + // extension handled the empty batch — skip the inline logic + } else if (++n_empty_consecutive > 3) { // Hydra: a STATE_GET background stream holds the slot (hydra_transferring) // without contributing batch tokens — that is expected, not a stall. // Suppress the abort while a transfer is in flight, and for a short @@ -11584,3 +9156,8 @@ void server_context::start_rpc_server(int port, std::vector) { GGML_UNUSED(port); } #endif // !_WIN32 + +// epic #610 WS1: Hydra extension implementation. Compiled INTO this TU so the +// concrete class can reach server_context_impl private members via the friend +// declaration above. Do NOT add hydra-server-context.cpp to CMakeLists.txt. +#include "hydra-server-context.cpp" diff --git a/tools/server/server-hydra-extension.h b/tools/server/server-hydra-extension.h new file mode 100644 index 000000000000..eaef85586b80 --- /dev/null +++ b/tools/server/server-hydra-extension.h @@ -0,0 +1,52 @@ +// Hydra A/B extension seam (epic #610). +// +// server-context.cpp consults this interface at a few well-defined points so +// Hydra-specific behavior can live in a fork-owned file (hydra-server-context.cpp) +// instead of being woven into upstream server-context.cpp. The concrete +// implementation is #include'd at the bottom of server-context.cpp (same +// translation unit) so it can reach server_context_impl's private members via +// the friend declaration on hydra_engine_extension. +// +// A/B toggle: the HYDRA_EXT_MODE env var selects which implementation drives +// Hydra behavior at runtime, so the SAME binary can be A/B tested: +// HYDRA_EXT_MODE=legacy -> the inline Hydra code in server-context.cpp (default) +// HYDRA_EXT_MODE=seam -> the extension (WS1: no-op, behavior identical) +// Both paths stay compiled; only one is consulted per run. WS4 diffs the same +// scenario through both modes to prove the refactor is behavior-identical. +#pragma once + +#include +#include +#include + +struct server_context_impl; +struct server_task; + +// True when HYDRA_EXT_MODE=seam. Anything else (including unset) = legacy. +inline bool hydra_ext_mode_seam() { + const char * m = std::getenv("HYDRA_EXT_MODE"); + return m && std::strcmp(m, "seam") == 0; +} + +struct server_hydra_extension { + virtual ~server_hydra_extension() = default; + + // Human-readable name of the active implementation (for A/B logging/tests). + virtual const char * name() const = 0; + + // Claim a task from process_single_task(). Return true if fully handled + // (the default dispatch is skipped). The task must NOT be consumed when + // returning false. + virtual bool handle_task(server_context_impl & impl, server_task & task) = 0; + + // Called at the top of update_slots(). Return true to skip the default + // decode loop for this pass (T3 rebuild / CONFIGURE / COMBINED reattach). + virtual bool pre_loop(server_context_impl & impl) = 0; + + // Called when update_slots() finds batch.n_tokens == 0. Return true if the + // empty-batch case was fully handled (STATE_GET transfer suppression). + virtual bool on_empty_batch(server_context_impl & impl) = 0; +}; + +// Factory. Defined in hydra-server-context.cpp. +std::unique_ptr hydra_create_extension(); From 6f82f1c4a87c7a692694d6d28b2cfaae0a2188eb Mon Sep 17 00:00:00 2001 From: Hydra Engineering Date: Tue, 11 Aug 2026 18:38:31 +0700 Subject: [PATCH 2/5] epic(610): WS3.5 move Hydra RPC server out of server-context.cpp The ~1117-line Hydra RPC server (hydra_rpc_ctx, static state handlers, server_context::start_rpc_server, hydra_rpc_bridge, #if !WIN32 guards) moved to hydra-server-context.cpp (same TU via bottom #include). This was the single largest rebase conflict source: upstream added handle_count_tokens etc. exactly where the fork appended the RPC server. --- tools/server/hydra-server-context.cpp | 1124 ++++++++++++++++++++++ tools/server/server-context.cpp | 1273 ------------------------- 2 files changed, 1124 insertions(+), 1273 deletions(-) diff --git a/tools/server/hydra-server-context.cpp b/tools/server/hydra-server-context.cpp index 65cf55f8ea9b..b0c79ac61de2 100644 --- a/tools/server/hydra-server-context.cpp +++ b/tools/server/hydra-server-context.cpp @@ -2612,3 +2612,1127 @@ struct hydra_engine_extension : server_hydra_extension { std::unique_ptr hydra_create_extension() { return std::make_unique(); } + +// --------------------------------------------------------------------------- +// WS3.5: Hydra RPC server (moved from server-context.cpp, epic #610). +// Same TU via the bottom #include, so hydra_rpc_ctx and the static handlers +// can reach server_context_impl privates (friend) and the file-scope +// queue/response objects. Wire format: specs/rpc-protocol.md | server-rpc.h. +// --------------------------------------------------------------------------- +// ═══════════════════════════════════════════════════════════════════════════════ +// Hydra RPC server — KV state transfer (M1: task-queue based) +// Wire format: specs/rpc-protocol.md | constants: server-rpc.h +// Ops implemented: STATE_GET (0x30), STATE_PUT (0x31), STATE_META (0x32) +// M1: All llama API calls routed through task queue (inference thread safe) +// ═══════════════════════════════════════════════════════════════════════════════ + +#if !defined(_WIN32) + +// ── Context for RPC thread — pass to handlers ───────────────────────────────── + +struct hydra_rpc_ctx { + server_queue * queue_tasks = nullptr; + server_response * queue_results = nullptr; +}; + +// ── Low-level I/O helpers ───────────────────────────────────────────────────── + +// Hydra #43: failures here were previously silent — every caller treats a +// `false` return as "give up" but none logged *why*, so a wedged RPC +// response looked identical to a client that vanished. Log once, centrally, +// instead of touching the ~30 call sites. +static bool hydra_recv_all(int fd, void * buf, size_t n) { + char * p = reinterpret_cast(buf); + const size_t total = n; + while (n > 0) { + ssize_t r = ::recv(fd, p, n, 0); + if (r < 0) { + SRV_WRN("hydra rpc: recv failed on fd=%d (%zu/%zu bytes): %s\n", + fd, total - n, total, std::strerror(errno)); + return false; + } + if (r == 0) { + SRV_DBG("hydra rpc: recv EOF on fd=%d (%zu/%zu bytes)\n", fd, total - n, total); + return false; + } + p += r; n -= r; + } + return true; +} + +static bool hydra_send_all(int fd, const void * buf, size_t n) { + const char * p = reinterpret_cast(buf); + const size_t total = n; + while (n > 0) { + ssize_t w = ::send(fd, p, n, MSG_NOSIGNAL); + if (w <= 0) { + SRV_WRN("hydra rpc: send failed on fd=%d (%zu/%zu bytes) w=%zd: %s\n", + fd, total - n, total, w, std::strerror(errno)); + return false; + } + p += w; n -= w; + } + return true; +} + +// Response header: status(1) | meta_len(3 LE uint24) | payload_len(8 LE) — 12 bytes +static void hydra_write_res(int fd, uint8_t status, uint32_t meta_len, uint64_t payload_len) { + uint8_t buf[HYDRA_RES_HEADER_SIZE] = {}; + buf[0] = status; + buf[1] = (meta_len) & 0xFF; + buf[2] = (meta_len >> 8) & 0xFF; + buf[3] = (meta_len >> 16) & 0xFF; + memcpy(buf + 4, &payload_len, 8); // little-endian (x86/arm64) + hydra_send_all(fd, buf, HYDRA_RES_HEADER_SIZE); +} + +// ── Op handlers (M1: dispatch via task queue) ───────────────────────────────── + +// STATE_GET (0x30): Post task, wait for result. +// +// M1 path (hydra_fd < 0): inference thread serializes 800 MB into result buffer; +// RPC thread sends response header + meta JSON + buffer here. +// +// M2 path (hydra_fd = fd): background thread streams GPU→socket directly using +// llama_state_seq_get_data_to_fd; result carries only n_past + streamed_bytes. +// Response header + meta are sent BEFORE the task (we know size from STATE_META), +// so the payload is already on the wire before we even get the result back. +// Actually: we must send header AFTER knowing state_size. So: +// - If M2: we get state_size first from a quick STATE_META query (n_past already known), +// OR we embed state_size in the result from get_size() on the inference thread. +// The inference thread always calls llama_state_seq_get_size (cheap) and stores it +// in res->state_size for M2 so we can send the header before the stream completes. +// +// Timeout: 30s — streaming 800 MB over localhost may take a few seconds. +static void hydra_handle_state_get(int fd, int slot_id, const hydra_rpc_ctx & ctx) { + // Build task — pass fd for M2 zero-copy streaming + server_task task(SERVER_TASK_TYPE_HYDRA_STATE_GET); + task.id = ctx.queue_tasks->get_new_id(); + task.hydra_action.id_slot = slot_id; + task.hydra_action.hydra_fd = fd; // M2: background thread streams here + const int task_id = task.id; + // Register BEFORE posting — server_response::send() silently drops results + // for ids not in waiting_task_ids. + ctx.queue_results->add_waiting_task_id(task_id); + ctx.queue_tasks->wait_until_no_sleep(); + ctx.queue_tasks->post(std::move(task)); + + // Wait for result (n_past + state_size always set; state_data only on M1) + std::unordered_set task_ids = {task_id}; + auto res_ptr = ctx.queue_results->recv_with_timeout(task_ids, 30); // seconds + ctx.queue_results->remove_waiting_task_id(task_id); + if (!res_ptr) { + SRV_WRN("hydra rpc: STATE_GET timeout for slot %d\n", slot_id); + // M2 caveat: the background thread may own the fd (header possibly sent); + // writing an error header here could interleave with the stream. Shut the + // socket down instead so the client unblocks with a clean EOF. + ::shutdown(fd, SHUT_RDWR); + return; + } + + auto * res = dynamic_cast(res_ptr.get()); + if (!res) { + SRV_WRN("hydra rpc: STATE_GET result type mismatch for slot %d\n", slot_id); + hydra_write_res(fd, HYDRA_STATUS_ERROR, 0, 0); + return; + } + + if (res->rpc_status != HYDRA_STATUS_OK) { + if (res->header_sent) { + // M2 failure: header already sent but stream failed; background thread + // shut the socket down — connection loop will close the fd on next read. + // Log and return without sending a second response header. + SRV_WRN("hydra rpc: STATE_GET slot=%d M2 stream failed: %s\n", + slot_id, res->error.c_str()); + return; + } + hydra_write_res(fd, res->rpc_status, 0, 0); + if (!res->error.empty()) { + hydra_send_all(fd, res->error.data(), res->error.size()); + } + return; + } + + if (res->streamed_bytes > 0) { + // M2 path: data already on the wire — response header + meta were sent by background thread. + // Nothing left for RPC thread to do. The protocol framing (header + meta + payload) + // was completed inside llama_io_write_socket / the background thread. + // Note: header was sent AFTER state_size was known (inference thread called get_size). + SRV_INF("hydra rpc: STATE_GET slot=%d M2 streamed %.1f MiB directly\n", + slot_id, res->streamed_bytes / (1024.0 * 1024.0)); + } else { + // M1 path: inference thread buffered 800 MB; send it now. + const uint64_t payload = (uint64_t)res->state_data.size(); + json meta_j; + meta_j["n_past"] = res->n_past; + meta_j["state_size"] = payload; + if (!res->model_alias.empty()) meta_j["model_alias"] = res->model_alias; + if (!res->model_path.empty()) meta_j["model_path"] = res->model_path; + if (!res->tokenizer.empty()) meta_j["tokenizer"] = res->tokenizer; + if (!res->model_name.empty()) meta_j["model_name"] = res->model_name; + if (!res->model_quant.empty()) meta_j["model_quant"] = res->model_quant; + if (res->model_capabilities) meta_j["model_capabilities"] = res->model_capabilities; + const std::string meta_str = meta_j.dump(); + hydra_write_res(fd, HYDRA_STATUS_OK, (uint32_t)meta_str.size(), payload); + hydra_send_all(fd, meta_str.data(), meta_str.size()); + hydra_send_all(fd, res->state_data.data(), (size_t)payload); + SRV_INF("hydra rpc: STATE_GET slot=%d M1 sent %.1f MiB from buffer\n", + slot_id, payload / (1024.0 * 1024.0)); + } +} + +// STATE_PUT (0x31): Receive payload, post task, wait for result, send ack. +static void hydra_handle_state_put(int fd, int slot_id, uint64_t payload_len, const hydra_rpc_ctx & ctx) { + if (payload_len > HYDRA_MAX_STATE_BYTES) { + SRV_WRN("hydra rpc: STATE_PUT payload %" PRIu64 " B exceeds cap %" PRIu64 " B\n", + payload_len, HYDRA_MAX_STATE_BYTES); + hydra_write_res(fd, HYDRA_STATUS_ERROR, 0, 0); + // Drain to keep persistent connection alive + std::vector drain(65536); + for (uint64_t rem = payload_len; rem > 0; ) { + size_t chunk = (size_t)std::min(rem, (uint64_t)drain.size()); + if (!hydra_recv_all(fd, drain.data(), chunk)) break; + rem -= chunk; + } + return; + } + + // Read payload from socket + std::vector buf((size_t)payload_len); + if (!hydra_recv_all(fd, buf.data(), (size_t)payload_len)) { + SRV_WRN("%s", "hydra rpc: STATE_PUT failed to read payload\n"); + hydra_write_res(fd, HYDRA_STATUS_ERROR, 0, 0); + return; + } + + // Post task to inference thread + server_task task(SERVER_TASK_TYPE_HYDRA_STATE_PUT); + task.id = ctx.queue_tasks->get_new_id(); + task.hydra_action.id_slot = slot_id; + task.hydra_action.erase_existing = true; // RPC restore always replaces slot state + task.hydra_action.state_data = std::move(buf); + const int task_id = task.id; + // Register BEFORE posting — results for unregistered ids are dropped. + ctx.queue_results->add_waiting_task_id(task_id); + ctx.queue_tasks->wait_until_no_sleep(); + ctx.queue_tasks->post(std::move(task)); + + // Wait for result from inference thread (30s timeout for large restore) + std::unordered_set task_ids = {task_id}; + auto res_ptr = ctx.queue_results->recv_with_timeout(task_ids, 30); // seconds + ctx.queue_results->remove_waiting_task_id(task_id); + if (!res_ptr) { + SRV_WRN("hydra rpc: STATE_PUT timeout for slot %d\n", slot_id); + hydra_write_res(fd, HYDRA_STATUS_ERROR, 0, 0); + return; + } + + auto * res = dynamic_cast(res_ptr.get()); + if (!res) { + SRV_WRN("hydra rpc: STATE_PUT result type mismatch for slot %d\n", slot_id); + hydra_write_res(fd, HYDRA_STATUS_ERROR, 0, 0); + return; + } + + // Send result back to client + uint8_t rpc_status = res->rpc_status; + if (rpc_status == HYDRA_STATUS_OK) { + json meta_j; + meta_j["restored"] = true; + meta_j["bytes"] = res->bytes; + meta_j["model_match"] = res->model_match; + if (!res->model_alias.empty()) meta_j["model_alias"] = res->model_alias; + if (!res->model_path.empty()) meta_j["model_path"] = res->model_path; + if (!res->tokenizer.empty()) meta_j["tokenizer"] = res->tokenizer; + if (!res->model_name.empty()) meta_j["model_name"] = res->model_name; + if (!res->model_quant.empty()) meta_j["model_quant"] = res->model_quant; + if (res->model_capabilities) meta_j["model_capabilities"] = res->model_capabilities; + const std::string meta_str = meta_j.dump(); + hydra_write_res(fd, HYDRA_STATUS_OK, (uint32_t)meta_str.size(), 0); + hydra_send_all(fd, meta_str.data(), meta_str.size()); + } else { + json err_j; + err_j["error"] = res->error; + const std::string err_str = err_j.dump(); + hydra_write_res(fd, rpc_status, (uint32_t)err_str.size(), 0); + hydra_send_all(fd, err_str.data(), err_str.size()); + } +} + +// STATE_META (0x32): Post task, wait for result, send JSON metadata. +static void hydra_handle_state_meta(int fd, int slot_id, const hydra_rpc_ctx & ctx) { + server_task task(SERVER_TASK_TYPE_HYDRA_STATE_META); + task.id = ctx.queue_tasks->get_new_id(); + task.hydra_action.id_slot = slot_id; + const int task_id = task.id; + // Register BEFORE posting — results for unregistered ids are dropped. + ctx.queue_results->add_waiting_task_id(task_id); + ctx.queue_tasks->wait_until_no_sleep(); + ctx.queue_tasks->post(std::move(task)); + + // Wait for result from inference thread (5s timeout — allows for queue congestion) + std::unordered_set task_ids = {task_id}; + auto res_ptr = ctx.queue_results->recv_with_timeout(task_ids, 5); // seconds + ctx.queue_results->remove_waiting_task_id(task_id); + if (!res_ptr) { + SRV_WRN("hydra rpc: STATE_META timeout for slot %d\n", slot_id); + hydra_write_res(fd, HYDRA_STATUS_ERROR, 0, 0); + return; + } + + auto * res = dynamic_cast(res_ptr.get()); + if (!res) { + SRV_WRN("hydra rpc: STATE_META result type mismatch for slot %d\n", slot_id); + hydra_write_res(fd, HYDRA_STATUS_ERROR, 0, 0); + return; + } + + // Send result back to client + uint8_t rpc_status = res->rpc_status; + if (rpc_status == HYDRA_STATUS_OK) { + json meta_j; + meta_j["slot_id"] = res->id_slot; + meta_j["n_past"] = res->n_past; + meta_j["state_size"] = res->state_size; + meta_j["is_processing"] = res->is_processing; + meta_j["is_transferring"] = res->is_transferring; + if (!res->model_alias.empty()) meta_j["model_alias"] = res->model_alias; + if (!res->model_path.empty()) meta_j["model_path"] = res->model_path; + if (!res->tokenizer.empty()) meta_j["tokenizer"] = res->tokenizer; + if (!res->model_name.empty()) meta_j["model_name"] = res->model_name; + if (!res->model_quant.empty()) meta_j["model_quant"] = res->model_quant; + if (res->model_capabilities) meta_j["model_capabilities"] = res->model_capabilities; + const std::string meta_str = meta_j.dump(); + hydra_write_res(fd, HYDRA_STATUS_OK, (uint32_t)meta_str.size(), 0); + hydra_send_all(fd, meta_str.data(), meta_str.size()); + } else { + hydra_write_res(fd, rpc_status, 0, 0); + } +} + +// ── E1 Engine control handlers ──────────────────────────────────────────────── + +// CONFIGURE (0x33): Read JSON config payload, post task, return success. +static void hydra_handle_configure(int fd, int slot_id, uint64_t payload_len, const hydra_rpc_ctx & ctx) { + std::string config_json(payload_len, '\0'); + if (payload_len > 0 && !hydra_recv_all(fd, config_json.data(), payload_len)) { + hydra_write_res(fd, HYDRA_STATUS_ERROR, 0, 0); + return; + } + + server_task task(SERVER_TASK_TYPE_HYDRA_ENGINE_CONFIGURE); + task.id = ctx.queue_tasks->get_new_id(); + task.hydra_action.id_slot = slot_id; + task.hydra_action.config_json = std::move(config_json); + const int task_id = task.id; + ctx.queue_results->add_waiting_task_id(task_id); + ctx.queue_tasks->wait_until_no_sleep(); + ctx.queue_tasks->post(std::move(task)); + + std::unordered_set task_ids = {task_id}; + auto res_ptr = ctx.queue_results->recv_with_timeout(task_ids, 5); + ctx.queue_results->remove_waiting_task_id(task_id); + if (!res_ptr) { + hydra_write_res(fd, HYDRA_STATUS_ERROR, 0, 0); + return; + } + + auto * res = dynamic_cast(res_ptr.get()); + if (!res || !res->success) { + // hydra#406: on failure, include the error message in the meta so + // the Coordinator can distinguish "drain timeout" from a parse + // error. We still write HYDRA_STATUS_ERROR (0x02) per the wire + // contract — the meta body is for diagnostics only. + if (res && !res->error.empty()) { + json err_j = {{"success", false}, {"error", res->error}}; + if (!res->tier.empty()) err_j["tier"] = res->tier; + const std::string err_str = err_j.dump(); + hydra_write_res(fd, HYDRA_STATUS_ERROR, (uint32_t)err_str.size(), 0); + hydra_send_all(fd, err_str.data(), err_str.size()); + } else { + hydra_write_res(fd, HYDRA_STATUS_ERROR, 0, 0); + } + return; + } + + // hydra#406: tiered CONFIGURE response shape. Always present: success, + // tier, params_applied (T1 keys), deferred_keys (T2/T3 keys). + json meta_j = { + {"success", true}, + {"tier", res->tier.empty() ? std::string("T1") : res->tier}, + {"params_applied", json::object()}, + {"deferred_keys", json::array()}, + }; + for (const auto & kv : res->params_applied) { + meta_j["params_applied"][kv.first] = kv.second; + } + for (const auto & k : res->deferred_keys) { + meta_j["deferred_keys"].push_back(k); + } + // hydra#334: echo the post-clamp value for the state_chunk_size legacy + // path so the Coordinator's existing detection logic still works + // (the same value is also in params_applied, with the dotted key). + if (res->state_chunk_size_applied > 0) { + meta_j["state_chunk_size_applied"] = res->state_chunk_size_applied; + } + const std::string meta_str = meta_j.dump(); + hydra_write_res(fd, HYDRA_STATUS_OK, (uint32_t)meta_str.size(), 0); + hydra_send_all(fd, meta_str.data(), meta_str.size()); +} + +// INFO (0x34): Return engine capabilities as JSON. +static void hydra_handle_info(int fd, int slot_id, const hydra_rpc_ctx & ctx) { + server_task task(SERVER_TASK_TYPE_HYDRA_ENGINE_INFO); + task.id = ctx.queue_tasks->get_new_id(); + task.hydra_action.id_slot = slot_id; + const int task_id = task.id; + ctx.queue_results->add_waiting_task_id(task_id); + ctx.queue_tasks->wait_until_no_sleep(); + ctx.queue_tasks->post(std::move(task)); + + std::unordered_set task_ids = {task_id}; + auto res_ptr = ctx.queue_results->recv_with_timeout(task_ids, 5); + ctx.queue_results->remove_waiting_task_id(task_id); + if (!res_ptr) { + hydra_write_res(fd, HYDRA_STATUS_ERROR, 0, 0); + return; + } + + auto * res = dynamic_cast(res_ptr.get()); + if (!res) { + hydra_write_res(fd, HYDRA_STATUS_ERROR, 0, 0); + return; + } + + const std::string & info_str = res->info_json; + hydra_write_res(fd, HYDRA_STATUS_OK, (uint32_t)info_str.size(), 0); + hydra_send_all(fd, info_str.data(), info_str.size()); +} + +// PREFILL (0x35): Read JSON payload with {"messages": [...]}, +// tokenize internally, run prefill, return n_past + KV state blob. +static void hydra_handle_prefill(int fd, int slot_id, uint64_t payload_len, const hydra_rpc_ctx & ctx) { + if (payload_len == 0) { + hydra_write_res(fd, HYDRA_STATUS_ERROR, 0, 0); + return; + } + + std::string json_str((size_t)payload_len, '\0'); + if (!hydra_recv_all(fd, json_str.data(), (size_t)payload_len)) { + hydra_write_res(fd, HYDRA_STATUS_ERROR, 0, 0); + return; + } + + server_task task(SERVER_TASK_TYPE_HYDRA_ENGINE_PREFILL); + task.id = ctx.queue_tasks->get_new_id(); + task.hydra_action.id_slot = slot_id; + task.hydra_action.request_json = std::move(json_str); + const int task_id = task.id; + ctx.queue_results->add_waiting_task_id(task_id); + ctx.queue_tasks->wait_until_no_sleep(); + ctx.queue_tasks->post(std::move(task)); + + std::unordered_set task_ids = {task_id}; + // Bumped from 60s to 180s. Prefill for 32k+ token prompts exceeds 120s + // (we measured 32s for 22k tokens; 48k ≈ 70s, 100k ≈ 150s+). Long autoregressive + // decode on P100 (28 tok/s) for 4k+ token outputs also exceeds 120s. The C++ + // side was timing out and returning HYDRA_STATUS_ERROR before the C# client + // gave up, surfacing as a 503 from the coordinator even though the model was + // still working. + auto res_ptr = ctx.queue_results->recv_with_timeout(task_ids, 180); + ctx.queue_results->remove_waiting_task_id(task_id); + if (!res_ptr) { + hydra_write_res(fd, HYDRA_STATUS_ERROR, 0, 0); + return; + } + + auto * res = dynamic_cast(res_ptr.get()); + if (!res || res->rpc_status != HYDRA_STATUS_OK) { + hydra_write_res(fd, HYDRA_STATUS_ERROR, 0, 0); + return; + } + + // Return n_past + sizes + model identity in meta; full blob (v2 header + KV + logits) as payload. + // logits_size > 0 signals the decode GPU to inject them into ctx->logits via STATE_PUT. + // M-Perf.9 #289: model identity fields (already populated on res by the PREFILL handler) + // are included so the Coordinator can record which model built the KV. + json meta_j = { + {"n_past", res->n_past}, + {"state_size", res->state_size}, + {"logits_size", res->logits_size} + }; + if (!res->model_alias.empty()) meta_j["model_alias"] = res->model_alias; + if (!res->model_path.empty()) meta_j["model_path"] = res->model_path; + if (!res->tokenizer.empty()) meta_j["tokenizer"] = res->tokenizer; + if (!res->model_name.empty()) meta_j["model_name"] = res->model_name; + if (!res->model_quant.empty()) meta_j["model_quant"] = res->model_quant; + if (res->model_capabilities) meta_j["model_capabilities"] = res->model_capabilities; + meta_j["model_fallback"] = res->model_fallback; + if (res->prefill_ms > 0) meta_j["prefill_ms"] = res->prefill_ms; + if (res->model_load_ms > 0) meta_j["model_load_ms"] = res->model_load_ms; + const std::string meta_str = meta_j.dump(); + const uint64_t total_payload = (uint64_t)res->state_data.size(); + hydra_write_res(fd, HYDRA_STATUS_OK, (uint32_t)meta_str.size(), total_payload); + hydra_send_all(fd, meta_str.data(), meta_str.size()); + if (total_payload > 0) { + hydra_send_all(fd, res->state_data.data(), (size_t)total_payload); + } + SRV_INF("hydra: PREFILL slot=%d sent n_past=%d kv=%" PRIu64 "B logits=%" PRIu64 "B total=%" PRIu64 "B\n", + slot_id, res->n_past, res->state_size, res->logits_size, total_payload); +} + +// DECODE (0x43) — Merged P/D: framed request with async HTTP retrieval. +// Wire format v3 (segmented): +// [4B hdr_len LE] <= 32768 +// [8B hdr_hash LE] xxh3-64 of the hdr JSON bytes that follow +// [hdr_len bytes] control header JSON +// [prompt_len bytes] prompt JSON segment (may be zero-length) +// [kv_len bytes] raw KV blob (may be zero-length) +// +// Control header: +// { "v": 3, "model": "...", "kv_metadata": {...}, "model_metadata": {...}, +// "generation": {...}, "segments": [...] } +// +// Two-phase flow: +// Phase 1 (sync): identity validation + KV restore — waits for inference thread +// Phase 2 (async): background thread posts SERVER_TASK_TYPE_COMPLETION, +// update_slots() drives generation, result stored in decode_results buffer. +// Actual result retrieved via GET /v1/decode/{decode_request_id}. +static void hydra_handle_decode(int fd, int slot_id, uint64_t payload_len, const hydra_rpc_ctx & ctx) { + // ── Read frame header: [4B hdr_len][8B hdr_hash] ────────────────────── + if (payload_len < sizeof(uint32_t) + sizeof(uint64_t)) { + SRV_WRN("%s", "hydra rpc: DECODE payload too small for frame header\n"); + hydra_write_res(fd, HYDRA_STATUS_BAD_REQUEST, 0, 0); + return; + } + + uint32_t hdr_len = 0; + if (!hydra_recv_all(fd, &hdr_len, sizeof(hdr_len))) { + hydra_write_res(fd, HYDRA_STATUS_ERROR, 0, 0); + return; + } + + if (hdr_len > HYDRA_MAX_JSON_HEADER) { + SRV_WRN("hydra rpc: DECODE hdr_len %u B exceeds cap %u B\n", + hdr_len, HYDRA_MAX_JSON_HEADER); + hydra_write_res(fd, HYDRA_STATUS_BAD_REQUEST, 0, 0); + return; + } + + uint64_t hdr_hash = 0; + if (!hydra_recv_all(fd, &hdr_hash, sizeof(hdr_hash))) { + hydra_write_res(fd, HYDRA_STATUS_ERROR, 0, 0); + return; + } + + // ── Read control header JSON ────────────────────────────────────────── + std::string hdr_json_str(hdr_len, '\0'); + if (hdr_len > 0 && !hydra_recv_all(fd, hdr_json_str.data(), hdr_len)) { + hydra_write_res(fd, HYDRA_STATUS_ERROR, 0, 0); + return; + } + + // Verify hdr_hash (xxh3-64 of the JSON bytes) + { + const uint64_t computed = XXH3_64bits(hdr_json_str.data(), hdr_json_str.size()); + if (computed != hdr_hash) { + SRV_WRN("hydra rpc: DECODE HDR_HASH_MISMATCH expected=%016" PRIx64 " got=%016" PRIx64 "\n", + hdr_hash, computed); + hydra_write_res(fd, HYDRA_STATUS_BAD_REQUEST, 0, 0); + return; + } + } + + // Parse control header + json req; + try { + req = json::parse(hdr_json_str); + } catch (const std::exception & e) { + SRV_WRN("hydra rpc: DECODE invalid JSON in control header: %s\n", e.what()); + hydra_write_res(fd, HYDRA_STATUS_BAD_REQUEST, 0, 0); + return; + } + + // Validate version + const int hdr_version = req.value("v", 0); + if (hdr_version < 3) { + SRV_WRN("hydra rpc: DECODE unsupported version %d (need >= 3)\n", hdr_version); + hydra_write_res(fd, HYDRA_STATUS_BAD_REQUEST, 0, 0); + return; + } + + // Validate required fields + if (!req.contains("kv_metadata")) { + SRV_WRN("%s", "hydra rpc: DECODE missing kv_metadata in control header\n"); + hydra_write_res(fd, HYDRA_STATUS_BAD_REQUEST, 0, 0); + return; + } + if (!req.contains("segments") || !req["segments"].is_array()) { + SRV_WRN("%s", "hydra rpc: DECODE missing or invalid segments array\n"); + hydra_write_res(fd, HYDRA_STATUS_BAD_REQUEST, 0, 0); + return; + } + + // ── Parse and validate segment table ────────────────────────────────── + const json & segments = req["segments"]; + const size_t n_segments = segments.size(); + if (n_segments > 3) { + SRV_WRN("hydra rpc: DECODE SEGMENT_TABLE_INVALID: too many segments (%zu)\n", n_segments); + hydra_write_res(fd, HYDRA_STATUS_BAD_REQUEST, 0, 0); + return; + } + + // Each segment: {"id":"prompt"|"kv", "offset":N, "len":N, "hash":"xxh3:HEX"} + uint64_t prompt_len = 0; + uint64_t kv_len = 0; + std::string prompt_hash_str; + std::string kv_hash_str; + uint64_t expected_offset = 0; + for (size_t i = 0; i < n_segments; i++) { + const json & seg = segments[i]; + if (!seg.contains("id") || !seg.contains("offset") || !seg.contains("len") || !seg.contains("hash")) { + SRV_WRN("hydra rpc: DECODE SEGMENT_TABLE_INVALID: segment %zu missing required fields\n", i); + hydra_write_res(fd, HYDRA_STATUS_BAD_REQUEST, 0, 0); + return; + } + const std::string id = seg["id"].get(); + const uint64_t offset = seg["offset"].get(); + const uint64_t len = seg["len"].get(); + const std::string hash = seg["hash"].get(); + + if (offset != expected_offset) { + SRV_WRN("hydra rpc: DECODE SEGMENT_TABLE_INVALID: segment %zu offset=%" PRIu64 " expected=%" PRIu64 "\n", + i, offset, expected_offset); + hydra_write_res(fd, HYDRA_STATUS_BAD_REQUEST, 0, 0); + return; + } + expected_offset = offset + len; + + if (id == "prompt") { + prompt_len = len; + prompt_hash_str = hash; + } else if (id == "kv") { + kv_len = len; + kv_hash_str = hash; + } else { + SRV_WRN("hydra rpc: DECODE SEGMENT_TABLE_INVALID: unknown segment id '%s'\n", id.c_str()); + hydra_write_res(fd, HYDRA_STATUS_BAD_REQUEST, 0, 0); + return; + } + } + + // Verify total segment size matches remaining payload + const uint64_t segments_total = prompt_len + kv_len; + const uint64_t remaining_after_hdr = payload_len - sizeof(uint32_t) - sizeof(uint64_t) - hdr_len; + if (segments_total != remaining_after_hdr) { + SRV_WRN("hydra rpc: DECODE SEGMENT_TABLE_INVALID: segments total %" PRIu64 " != remaining %" PRIu64 "\n", + segments_total, remaining_after_hdr); + hydra_write_res(fd, HYDRA_STATUS_BAD_REQUEST, 0, 0); + return; + } + + // Caps + if (prompt_len > HYDRA_MAX_PROMPT_BYTES) { + SRV_WRN("hydra rpc: DECODE PROMPT_TOO_LARGE %" PRIu64 " > %" PRIu64 "\n", + prompt_len, HYDRA_MAX_PROMPT_BYTES); + hydra_write_res(fd, HYDRA_STATUS_BAD_REQUEST, 0, 0); + return; + } + if (kv_len > HYDRA_MAX_STATE_BYTES) { + SRV_WRN("hydra rpc: DECODE KV_TOO_LARGE %" PRIu64 " > %" PRIu64 "\n", + kv_len, HYDRA_MAX_STATE_BYTES); + hydra_write_res(fd, HYDRA_STATUS_BAD_REQUEST, 0, 0); + return; + } + + // ── Read prompt segment ─────────────────────────────────────────────── + std::vector prompt_data((size_t)prompt_len); + if (prompt_len > 0 && !hydra_recv_all(fd, prompt_data.data(), (size_t)prompt_len)) { + hydra_write_res(fd, HYDRA_STATUS_ERROR, 0, 0); + return; + } + + // ── Read KV segment (may be zero-length) ────────────────────────────── + std::vector kv_data((size_t)kv_len); + if (kv_len > 0 && !hydra_recv_all(fd, kv_data.data(), (size_t)kv_len)) { + hydra_write_res(fd, HYDRA_STATUS_ERROR, 0, 0); + return; + } + + // Verify KV segment hash BEFORE passing to llama_state_seq_set_data + if (kv_len > 0 && !kv_hash_str.empty()) { + // Parse "xxh3:HEX" format + if (kv_hash_str.rfind("xxh3:", 0) == 0) { + const std::string hex_str = kv_hash_str.substr(5); + uint64_t expected_kv_hash = 0; + try { + expected_kv_hash = std::stoull(hex_str, nullptr, 16); + } catch (const std::exception &) { + SRV_WRN("hydra rpc: DECODE invalid KV hash format: %s\n", kv_hash_str.c_str()); + hydra_write_res(fd, HYDRA_STATUS_BAD_REQUEST, 0, 0); + return; + } + const uint64_t computed_kv = XXH3_64bits(kv_data.data(), kv_data.size()); + if (computed_kv != expected_kv_hash) { + SRV_WRN("hydra rpc: DECODE SEGMENT_HASH_MISMATCH kv expected=%016" PRIx64 " got=%016" PRIx64 "\n", + expected_kv_hash, computed_kv); + hydra_write_res(fd, HYDRA_STATUS_BAD_REQUEST, 0, 0); + return; + } + SRV_INF("hydra rpc: DECODE KV hash verified (%" PRIu64 " B)\n", kv_len); + } else { + SRV_WRN("hydra rpc: DECODE unsupported KV hash prefix: %s\n", kv_hash_str.c_str()); + hydra_write_res(fd, HYDRA_STATUS_BAD_REQUEST, 0, 0); + return; + } + } + + // ── Build decode_json from control header + prompt segment ───────────── + // The prompt JSON segment may contain { "prompt": "..." } or { "messages": [...] } + // Merge it into the control header as decode_req["prompt"]. + // Also merge generation params from control header's "generation" key. + json decode_req = req; // control header already has kv_metadata, model, etc. + json prompt_obj; + if (prompt_len > 0) { + try { + prompt_obj = json::parse(std::string(prompt_data.begin(), prompt_data.end())); + } catch (const std::exception & e) { + SRV_WRN("hydra rpc: DECODE invalid prompt segment JSON: %s\n", e.what()); + hydra_write_res(fd, HYDRA_STATUS_BAD_REQUEST, 0, 0); + return; + } + } + // The coordinator sends the prompt segment as the BARE messages array + // (item.Request["messages"].ToString()). The generation-merge below and + // DECODE_APPLY's chat-template path both expect an OBJECT with a + // "messages" key — merging generation keys into an array throws + // nlohmann::type_error, which was silently swallowed by the RPC worker + // pool (the connection leaked, no response written, coordinator timed out + // after 180s). Wrap a bare array so the prompt object matches the + // downstream contract. + if (prompt_obj.is_array()) { + json wrapped; + wrapped["messages"] = std::move(prompt_obj); + prompt_obj = std::move(wrapped); + } + // Merge generation params from control header into prompt object + std::string decode_json_str; + try { + if (req.contains("generation") && req["generation"].is_object()) { + const json & gen = req["generation"]; + for (auto it = gen.begin(); it != gen.end(); ++it) { + if (!prompt_obj.contains(it.key())) { + prompt_obj[it.key()] = it.value(); + } + } + } + decode_req["prompt"] = std::move(prompt_obj); + + decode_json_str = decode_req.dump(); + } catch (const std::exception & e) { + // Never let a malformed prompt object leak the connection: the worker + // pool swallows exceptions and the fd stays open with no response, + // hanging the coordinator until its own timeout. Always write an + // error frame so the caller sees a terminal (retryable-free) result. + SRV_WRN("hydra rpc: DECODE prompt build failed (slot %d): %s\n", slot_id, e.what()); + json err_j = { + {"error", std::string("prompt build failed: ") + e.what()}, + {"decode_request_id", -1}, + }; + const std::string err_str = err_j.dump(); + hydra_write_res(fd, HYDRA_STATUS_BAD_REQUEST, (uint32_t) err_str.size(), 0); + hydra_send_all(fd, err_str.data(), err_str.size()); + return; + } + + // ── Phase 1: sync validate + restore ────────────────────────────────── + const int32_t decode_request_id = ctx.queue_tasks->get_new_id(); + + server_task val_task(SERVER_TASK_TYPE_HYDRA_ENGINE_DECODE); + val_task.id = decode_request_id; + val_task.hydra_action.id_slot = slot_id; + val_task.hydra_action.decode_json = std::move(decode_json_str); + val_task.hydra_action.kv_data = std::move(kv_data); + val_task.hydra_action.decode_request_id = decode_request_id; + ctx.queue_results->add_waiting_task_id(decode_request_id); + ctx.queue_tasks->wait_until_no_sleep(); + ctx.queue_tasks->post(std::move(val_task)); + + // Wait for validation+restore to complete (30s timeout for large KV blobs) + std::unordered_set val_ids = {decode_request_id}; + auto val_res_ptr = ctx.queue_results->recv_with_timeout(val_ids, 30); + ctx.queue_results->remove_waiting_task_id(decode_request_id); + + if (!val_res_ptr) { + SRV_WRN("hydra rpc: DECODE validation timeout for slot %d (request_id=%d)\n", + slot_id, decode_request_id); + json err_j = { + {"error", "validation timeout"}, + {"decode_request_id", decode_request_id}, + }; + const std::string err_str = err_j.dump(); + hydra_write_res(fd, HYDRA_STATUS_ERROR, (uint32_t)err_str.size(), 0); + hydra_send_all(fd, err_str.data(), err_str.size()); + return; + } + + auto * val_res = dynamic_cast(val_res_ptr.get()); + if (!val_res || val_res->rpc_status != HYDRA_STATUS_OK) { + json err_j = { + {"valid", false}, + {"decode_request_id", decode_request_id}, + }; + if (val_res) { + if (!val_res->match_json.is_null()) err_j["match"] = val_res->match_json; + if (!val_res->error.empty()) err_j["reason"] = val_res->error; + err_j["error_code"] = "CAP_MISMATCH"; + } + const std::string err_str = err_j.dump(); + hydra_write_res(fd, HYDRA_STATUS_ERROR, (uint32_t)err_str.size(), 0); + hydra_send_all(fd, err_str.data(), err_str.size()); + return; + } + + // Validation passed — build real success response + json meta_j = { + {"valid", true}, + {"match", val_res->match_json}, + {"decode_request_id", decode_request_id}, + {"n_past_after_restore", val_res->n_past}, + {"restore_slot_ms", val_res->restore_slot_ms}, + }; + const std::string meta_str = meta_j.dump(); + hydra_write_res(fd, HYDRA_STATUS_OK, (uint32_t)meta_str.size(), 0); + hydra_send_all(fd, meta_str.data(), meta_str.size()); + + SRV_INF("hydra: DECODE slot=%d accepted, request_id=%d, restore=%.1fms\n", + slot_id, decode_request_id, val_res->restore_slot_ms); +} + +// SET_EXPERT_MODE (0x37): Read mode string, post task, return success. +static void hydra_handle_set_expert_mode(int fd, int slot_id, uint64_t payload_len, const hydra_rpc_ctx & ctx) { + std::string mode(payload_len, '\0'); + if (payload_len > 0 && !hydra_recv_all(fd, mode.data(), payload_len)) { + hydra_write_res(fd, HYDRA_STATUS_ERROR, 0, 0); + return; + } + + server_task task(SERVER_TASK_TYPE_HYDRA_ENGINE_SET_EXPERT_MODE); + task.id = ctx.queue_tasks->get_new_id(); + task.hydra_action.id_slot = slot_id; + task.hydra_action.expert_mode = std::move(mode); + const int task_id = task.id; + ctx.queue_results->add_waiting_task_id(task_id); + ctx.queue_tasks->wait_until_no_sleep(); + ctx.queue_tasks->post(std::move(task)); + + std::unordered_set task_ids = {task_id}; + auto res_ptr = ctx.queue_results->recv_with_timeout(task_ids, 5); + ctx.queue_results->remove_waiting_task_id(task_id); + if (!res_ptr) { + hydra_write_res(fd, HYDRA_STATUS_ERROR, 0, 0); + return; + } + + auto * res = dynamic_cast(res_ptr.get()); + if (!res || !res->success) { + const std::string err = (res && !res->error.empty()) ? res->error : std::string(); + json err_j = {{"success", false}}; + if (!err.empty()) err_j["error"] = err; + const std::string err_str = err_j.dump(); + hydra_write_res(fd, HYDRA_STATUS_ERROR, (uint32_t)err_str.size(), 0); + hydra_send_all(fd, err_str.data(), err_str.size()); + return; + } + + // Report the ACTUAL mode applied (may be "solo" even though "combined" was + // requested, if this engine never dual-loaded combined experts) — the + // Coordinator's ReportsSolo() reads this key to detect the fallback. + json meta_j = {{"success", true}, {"mode", res->expert_mode_applied}}; + const std::string meta_str = meta_j.dump(); + hydra_write_res(fd, HYDRA_STATUS_OK, (uint32_t)meta_str.size(), 0); + hydra_send_all(fd, meta_str.data(), meta_str.size()); +} + +// SWAP_QUANT (0x38): Read quant_key + tensor_pattern, post task, return success. +static void hydra_handle_swap_quant(int fd, int slot_id, uint64_t payload_len, const hydra_rpc_ctx & ctx) { + if (payload_len < sizeof(uint16_t)) { + hydra_write_res(fd, HYDRA_STATUS_ERROR, 0, 0); + return; + } + + uint16_t quant_key_len = 0; + if (!hydra_recv_all(fd, &quant_key_len, sizeof(quant_key_len))) { + hydra_write_res(fd, HYDRA_STATUS_ERROR, 0, 0); + return; + } + + std::string quant_key(quant_key_len, '\0'); + if (quant_key_len > 0 && !hydra_recv_all(fd, quant_key.data(), quant_key_len)) { + hydra_write_res(fd, HYDRA_STATUS_ERROR, 0, 0); + return; + } + + const uint64_t pattern_len = payload_len - sizeof(uint16_t) - quant_key_len; + std::string tensor_pattern(pattern_len, '\0'); + if (pattern_len > 0 && !hydra_recv_all(fd, tensor_pattern.data(), pattern_len)) { + hydra_write_res(fd, HYDRA_STATUS_ERROR, 0, 0); + return; + } + + server_task task(SERVER_TASK_TYPE_HYDRA_ENGINE_SWAP_QUANT); + task.id = ctx.queue_tasks->get_new_id(); + task.hydra_action.id_slot = slot_id; + task.hydra_action.quant_key = std::move(quant_key); + task.hydra_action.tensor_pattern = std::move(tensor_pattern); + const int task_id = task.id; + ctx.queue_results->add_waiting_task_id(task_id); + ctx.queue_tasks->wait_until_no_sleep(); + ctx.queue_tasks->post(std::move(task)); + + std::unordered_set task_ids = {task_id}; + auto res_ptr = ctx.queue_results->recv_with_timeout(task_ids, 30); + ctx.queue_results->remove_waiting_task_id(task_id); + if (!res_ptr) { + hydra_write_res(fd, HYDRA_STATUS_ERROR, 0, 0); + return; + } + + auto * res = dynamic_cast(res_ptr.get()); + if (!res || !res->success) { + hydra_write_res(fd, HYDRA_STATUS_ERROR, 0, 0); + return; + } + + json meta_j = {{"success", true}}; + const std::string meta_str = meta_j.dump(); + hydra_write_res(fd, HYDRA_STATUS_OK, (uint32_t)meta_str.size(), 0); + hydra_send_all(fd, meta_str.data(), meta_str.size()); +} + +// PIPELINE_ATTACH (0x46): M-Perf.9 (#289) / issue #287 — two-engine "work +// together" routing scaffolding. The C# Coordinator sends the peer address +// and the --override-tensor regex; the engine should load the assigned +// tensor slice from its OWN local model (no weight transfer). This opcode +// is stubbed for now (returns NOT_IMPLEMENTED) — full implementation is +// tracked under issue #287. +static void hydra_handle_pipeline_attach(int fd, int slot_id, uint64_t payload_len, const hydra_rpc_ctx & ctx) { + std::string json_body(payload_len, '\0'); + if (payload_len > 0 && !hydra_recv_all(fd, json_body.data(), payload_len)) { + hydra_write_res(fd, HYDRA_STATUS_ERROR, 0, 0); + return; + } + + server_task task(SERVER_TASK_TYPE_HYDRA_ENGINE_PIPELINE_ATTACH); + task.id = ctx.queue_tasks->get_new_id(); + task.hydra_action.id_slot = slot_id; + task.hydra_action.request_json = std::move(json_body); + const int task_id = task.id; + ctx.queue_results->add_waiting_task_id(task_id); + ctx.queue_tasks->wait_until_no_sleep(); + ctx.queue_tasks->post(std::move(task)); + + std::unordered_set task_ids = {task_id}; + auto res_ptr = ctx.queue_results->recv_with_timeout(task_ids, 5); + ctx.queue_results->remove_waiting_task_id(task_id); + if (!res_ptr) { + hydra_write_res(fd, HYDRA_STATUS_ERROR, 0, 0); + return; + } + + auto * res = dynamic_cast(res_ptr.get()); + // Stubbed: server returns NOT_IMPLEMENTED until issue #287 lands. + // Propagate that status to the client so the Coordinator can + // distinguish "not yet built" from a real error and fall back to solo. + const uint8_t status = (res && res->rpc_status == HYDRA_STATUS_NOT_IMPLEMENTED) + ? HYDRA_STATUS_NOT_IMPLEMENTED : HYDRA_STATUS_ERROR; + json meta_j; + if (res && !res->error.empty()) meta_j["error"] = res->error; + meta_j["success"] = res && res->success; + const std::string meta_str = meta_j.dump(); + hydra_write_res(fd, status, (uint32_t)meta_str.size(), 0); + hydra_send_all(fd, meta_str.data(), meta_str.size()); +} + +// ── Per-connection loop ─────────────────────────────────────────────────────── +// Persistent: one TCP connection handles many sequential requests. + +static void hydra_handle_connection(int fd, const hydra_rpc_ctx & ctx) { + // Set receive timeout to prevent hung connections on stalled clients + struct timeval tv; + tv.tv_sec = 120; // 2 min inactivity timeout + tv.tv_usec = 0; + (void)setsockopt(fd, SOL_SOCKET, SO_RCVTIMEO, &tv, sizeof(tv)); + while (true) { + uint8_t hdr[HYDRA_REQ_HEADER_SIZE]; + if (!hydra_recv_all(fd, hdr, HYDRA_REQ_HEADER_SIZE)) break; + + uint16_t magic = 0; + memcpy(&magic, hdr + 0, 2); + if (magic != HYDRA_MAGIC) { + SRV_WRN("hydra rpc: bad magic 0x%04x — closing connection\n", (unsigned)magic); + break; + } + + const uint8_t op = hdr[2]; + // hdr[3] = flags (reserved, unused in M1) + uint16_t key_len = 0, trace_len = 0; + uint64_t payload_len = 0; + memcpy(&key_len, hdr + 4, 2); + memcpy(&payload_len, hdr + 6, 8); + memcpy(&trace_len, hdr + 14, 2); + + std::string key(key_len, '\0'); + std::string trace_id(trace_len, '\0'); + if (!hydra_recv_all(fd, key.data(), key_len)) break; + if (!hydra_recv_all(fd, trace_id.data(), trace_len)) break; + + // Slot-key parsing: engine-level opcodes (INFO, CONFIGURE, SET_EXPERT_MODE, + // SWAP_QUANT) don't need a valid slot — use slot_id = 0 when the key is + // empty or invalid. Slot-level opcodes (STATE_GET, STATE_PUT, STATE_META, + // PREFILL, DECODE) still require a valid integer key. + int slot_id = -1; + bool is_engine_level_op = (op == HYDRA_OP_INFO || op == HYDRA_OP_CONFIGURE || + op == HYDRA_OP_SET_EXPERT_MODE || op == HYDRA_OP_SWAP_QUANT); + if (key.empty() && is_engine_level_op) { + slot_id = 0; + } else { + try { slot_id = std::stoi(key); } + catch (...) { + SRV_WRN("hydra rpc: invalid slot key '%s'\n", key.c_str()); + hydra_write_res(fd, HYDRA_STATUS_ERROR, 0, 0); + continue; + } + } + + // Dispatch to handler via task queue (no direct slot access) + switch (op) { + case HYDRA_OP_STATE_GET: + SRV_DBG("hydra rpc: STATE_GET slot=%d trace=%s\n", slot_id, trace_id.c_str()); + hydra_handle_state_get(fd, slot_id, ctx); + break; + case HYDRA_OP_STATE_PUT: + SRV_DBG("hydra rpc: STATE_PUT slot=%d payload=%" PRIu64 " trace=%s\n", + slot_id, payload_len, trace_id.c_str()); + hydra_handle_state_put(fd, slot_id, payload_len, ctx); + break; + case HYDRA_OP_STATE_META: + SRV_DBG("hydra rpc: STATE_META slot=%d trace=%s\n", slot_id, trace_id.c_str()); + hydra_handle_state_meta(fd, slot_id, ctx); + break; + case HYDRA_OP_CONFIGURE: + SRV_DBG("hydra rpc: CONFIGURE slot=%d payload=%" PRIu64 " trace=%s\n", + slot_id, payload_len, trace_id.c_str()); + hydra_handle_configure(fd, slot_id, payload_len, ctx); + break; + case HYDRA_OP_INFO: + SRV_DBG("hydra rpc: INFO slot=%d trace=%s\n", slot_id, trace_id.c_str()); + hydra_handle_info(fd, slot_id, ctx); + break; + case HYDRA_OP_PREFILL: + SRV_DBG("hydra rpc: PREFILL slot=%d payload=%" PRIu64 " trace=%s\n", + slot_id, payload_len, trace_id.c_str()); + hydra_handle_prefill(fd, slot_id, payload_len, ctx); + break; + case HYDRA_OP_DECODE: + SRV_DBG("hydra rpc: DECODE slot=%d payload=%" PRIu64 " trace=%s\n", + slot_id, payload_len, trace_id.c_str()); + hydra_handle_decode(fd, slot_id, payload_len, ctx); + break; + case HYDRA_OP_SET_EXPERT_MODE: + SRV_DBG("hydra rpc: SET_EXPERT_MODE slot=%d payload=%" PRIu64 " trace=%s\n", + slot_id, payload_len, trace_id.c_str()); + hydra_handle_set_expert_mode(fd, slot_id, payload_len, ctx); + break; + case HYDRA_OP_SWAP_QUANT: + SRV_DBG("hydra rpc: SWAP_QUANT slot=%d payload=%" PRIu64 " trace=%s\n", + slot_id, payload_len, trace_id.c_str()); + hydra_handle_swap_quant(fd, slot_id, payload_len, ctx); + break; + // M-Perf.9 (#289) / issue #287: PIPELINE_ATTACH (0x46) is the + // two-engine "work together" attach. Stubbed: full impl in #287. + case HYDRA_OP_PIPELINE_ATTACH: + SRV_DBG("hydra rpc: PIPELINE_ATTACH slot=%d payload=%" PRIu64 " trace=%s\n", + slot_id, payload_len, trace_id.c_str()); + hydra_handle_pipeline_attach(fd, slot_id, payload_len, ctx); + break; + default: + SRV_WRN("hydra rpc: unknown op 0x%02x — ignoring\n", (unsigned)op); + hydra_write_res(fd, HYDRA_STATUS_ERROR, 0, 0); + } + } + ::close(fd); +} + +// ── Unified RPC server implementation ─────────────────────────────────────── +// +// `#36` Phase 1: the merged server lives in `tools/llama-engine/hydra_rpc/` +// (fork-isolated). `server_context::start_rpc_server` is a thin adapter that +// builds the settings and delegates to `hydra_rpc::start()`. The Hydra +// protocol entry `hydra_handle_connection` is reached through the +// `hydra_rpc_bridge` trampoline (defined below) — the bridge takes a +// `void*` so the new module can stay decoupled from this file's includes. + +#include "../llama-engine/hydra_rpc/hydra_rpc.h" + +void server_context::start_rpc_server(int port, + std::vector backends) { + if (port <= 0) return; + + // Hydra #43: MUST outlive this function. `hydra_rpc::start()` below + // stores `&ctx` as a raw pointer inside `hydra_rpc::state()`, a + // process-lifetime singleton that every subsequent RPC connection reads + // (from a bounded-thread-pool worker thread) to recover queue_tasks / + // queue_results. An automatic-storage `ctx` here would dangle the + // instant this function returns — a stack-use-after-return that "works" + // until the freed stack slot gets reused, then silently corrupts the + // RPC response path. `start_rpc_server` only ever runs once per process + // (hydra_rpc::start() itself guards double-start), so `static` gives it + // exactly the lifetime the singleton needs. + static hydra_rpc_ctx ctx{}; + if (impl) { + ctx.queue_tasks = &impl->queue_tasks; + ctx.queue_results = &impl->queue_results; + } + + hydra_rpc::settings s; + s.port = port; + s.backends = std::move(backends); + s.hydra_ctx = (ctx.queue_tasks && ctx.queue_results) ? &ctx : nullptr; + s.pool_size = 2; + s.max_queue = 64; + s.host = "0.0.0.0"; + + if (!hydra_rpc::start(s)) { + SRV_ERR("hydra rpc: start() failed on port %d\n", port); + return; + } + + if (s.hydra_ctx) { + SRV_INF("hydra rpc: unified server on 0.0.0.0:%d (ggml-RPC + Hydra protocol)\n", port); + } else { + SRV_INF("hydra rpc: unified server on 0.0.0.0:%d (ggml-RPC only)\n", port); + } +} + +// `hydra_rpc_bridge` — extern "C" trampoline. `hydra_rpc.cpp` calls this +// when the first byte on a new connection is not `RPC_CMD_HELLO`. It +// re-enters the C++ entry point with the typed `hydra_rpc_ctx &`. +// +// Forward-declared with the matching signature so the new +// `tools/llama-engine/hydra_rpc/hydra_rpc.cpp` module can take its +// address without including this heavy header. +extern "C" void hydra_rpc_bridge(int fd, const void * ctx); +extern "C" void hydra_rpc_bridge(int fd, const void * ctx) { + hydra_handle_connection(fd, *static_cast(ctx)); +} + +#else +// Windows: RPC server not implemented — target hardware is Linux-only for M0. +void server_context::start_rpc_server(int port, std::vector) { + if (port > 0) { + SRV_WRN("hydra rpc: not supported on Windows (port %d ignored)\n", port); + } + GGML_UNUSED(port); +} +#endif // !_WIN32 diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp index bf67e6a19310..35658bf4ac8b 100644 --- a/tools/server/server-context.cpp +++ b/tools/server/server-context.cpp @@ -7883,1279 +7883,6 @@ std::unique_ptr server_routes::handle_embeddings_impl(cons return res; } -// ═══════════════════════════════════════════════════════════════════════════════ -// Hydra RPC server — KV state transfer (M1: task-queue based) -// Wire format: specs/rpc-protocol.md | constants: server-rpc.h -// Ops implemented: STATE_GET (0x30), STATE_PUT (0x31), STATE_META (0x32) -// M1: All llama API calls routed through task queue (inference thread safe) -// ═══════════════════════════════════════════════════════════════════════════════ - -#if !defined(_WIN32) - -// ── Context for RPC thread — pass to handlers ───────────────────────────────── - -struct hydra_rpc_ctx { - server_queue * queue_tasks = nullptr; - server_response * queue_results = nullptr; -}; - -// ── Low-level I/O helpers ───────────────────────────────────────────────────── - -// Hydra #43: failures here were previously silent — every caller treats a -// `false` return as "give up" but none logged *why*, so a wedged RPC -// response looked identical to a client that vanished. Log once, centrally, -// instead of touching the ~30 call sites. -static bool hydra_recv_all(int fd, void * buf, size_t n) { - char * p = reinterpret_cast(buf); - const size_t total = n; - while (n > 0) { - ssize_t r = ::recv(fd, p, n, 0); - if (r < 0) { - SRV_WRN("hydra rpc: recv failed on fd=%d (%zu/%zu bytes): %s\n", - fd, total - n, total, std::strerror(errno)); - return false; - } - if (r == 0) { - SRV_DBG("hydra rpc: recv EOF on fd=%d (%zu/%zu bytes)\n", fd, total - n, total); - return false; - } - p += r; n -= r; - } - return true; -} - -static bool hydra_send_all(int fd, const void * buf, size_t n) { - const char * p = reinterpret_cast(buf); - const size_t total = n; - while (n > 0) { - ssize_t w = ::send(fd, p, n, MSG_NOSIGNAL); - if (w <= 0) { - SRV_WRN("hydra rpc: send failed on fd=%d (%zu/%zu bytes) w=%zd: %s\n", - fd, total - n, total, w, std::strerror(errno)); - return false; - } - p += w; n -= w; - } - return true; -} - -// Response header: status(1) | meta_len(3 LE uint24) | payload_len(8 LE) — 12 bytes -static void hydra_write_res(int fd, uint8_t status, uint32_t meta_len, uint64_t payload_len) { - uint8_t buf[HYDRA_RES_HEADER_SIZE] = {}; - buf[0] = status; - buf[1] = (meta_len) & 0xFF; - buf[2] = (meta_len >> 8) & 0xFF; - buf[3] = (meta_len >> 16) & 0xFF; - memcpy(buf + 4, &payload_len, 8); // little-endian (x86/arm64) - hydra_send_all(fd, buf, HYDRA_RES_HEADER_SIZE); -} - -// ── Op handlers (M1: dispatch via task queue) ───────────────────────────────── - -// STATE_GET (0x30): Post task, wait for result. -// -// M1 path (hydra_fd < 0): inference thread serializes 800 MB into result buffer; -// RPC thread sends response header + meta JSON + buffer here. -// -// M2 path (hydra_fd = fd): background thread streams GPU→socket directly using -// llama_state_seq_get_data_to_fd; result carries only n_past + streamed_bytes. -// Response header + meta are sent BEFORE the task (we know size from STATE_META), -// so the payload is already on the wire before we even get the result back. -// Actually: we must send header AFTER knowing state_size. So: -// - If M2: we get state_size first from a quick STATE_META query (n_past already known), -// OR we embed state_size in the result from get_size() on the inference thread. -// The inference thread always calls llama_state_seq_get_size (cheap) and stores it -// in res->state_size for M2 so we can send the header before the stream completes. -// -// Timeout: 30s — streaming 800 MB over localhost may take a few seconds. -static void hydra_handle_state_get(int fd, int slot_id, const hydra_rpc_ctx & ctx) { - // Build task — pass fd for M2 zero-copy streaming - server_task task(SERVER_TASK_TYPE_HYDRA_STATE_GET); - task.id = ctx.queue_tasks->get_new_id(); - task.hydra_action.id_slot = slot_id; - task.hydra_action.hydra_fd = fd; // M2: background thread streams here - const int task_id = task.id; - // Register BEFORE posting — server_response::send() silently drops results - // for ids not in waiting_task_ids. - ctx.queue_results->add_waiting_task_id(task_id); - ctx.queue_tasks->wait_until_no_sleep(); - ctx.queue_tasks->post(std::move(task)); - - // Wait for result (n_past + state_size always set; state_data only on M1) - std::unordered_set task_ids = {task_id}; - auto res_ptr = ctx.queue_results->recv_with_timeout(task_ids, 30); // seconds - ctx.queue_results->remove_waiting_task_id(task_id); - if (!res_ptr) { - SRV_WRN("hydra rpc: STATE_GET timeout for slot %d\n", slot_id); - // M2 caveat: the background thread may own the fd (header possibly sent); - // writing an error header here could interleave with the stream. Shut the - // socket down instead so the client unblocks with a clean EOF. - ::shutdown(fd, SHUT_RDWR); - return; - } - - auto * res = dynamic_cast(res_ptr.get()); - if (!res) { - SRV_WRN("hydra rpc: STATE_GET result type mismatch for slot %d\n", slot_id); - hydra_write_res(fd, HYDRA_STATUS_ERROR, 0, 0); - return; - } - - if (res->rpc_status != HYDRA_STATUS_OK) { - if (res->header_sent) { - // M2 failure: header already sent but stream failed; background thread - // shut the socket down — connection loop will close the fd on next read. - // Log and return without sending a second response header. - SRV_WRN("hydra rpc: STATE_GET slot=%d M2 stream failed: %s\n", - slot_id, res->error.c_str()); - return; - } - hydra_write_res(fd, res->rpc_status, 0, 0); - if (!res->error.empty()) { - hydra_send_all(fd, res->error.data(), res->error.size()); - } - return; - } - - if (res->streamed_bytes > 0) { - // M2 path: data already on the wire — response header + meta were sent by background thread. - // Nothing left for RPC thread to do. The protocol framing (header + meta + payload) - // was completed inside llama_io_write_socket / the background thread. - // Note: header was sent AFTER state_size was known (inference thread called get_size). - SRV_INF("hydra rpc: STATE_GET slot=%d M2 streamed %.1f MiB directly\n", - slot_id, res->streamed_bytes / (1024.0 * 1024.0)); - } else { - // M1 path: inference thread buffered 800 MB; send it now. - const uint64_t payload = (uint64_t)res->state_data.size(); - json meta_j; - meta_j["n_past"] = res->n_past; - meta_j["state_size"] = payload; - if (!res->model_alias.empty()) meta_j["model_alias"] = res->model_alias; - if (!res->model_path.empty()) meta_j["model_path"] = res->model_path; - if (!res->tokenizer.empty()) meta_j["tokenizer"] = res->tokenizer; - if (!res->model_name.empty()) meta_j["model_name"] = res->model_name; - if (!res->model_quant.empty()) meta_j["model_quant"] = res->model_quant; - if (res->model_capabilities) meta_j["model_capabilities"] = res->model_capabilities; - const std::string meta_str = meta_j.dump(); - hydra_write_res(fd, HYDRA_STATUS_OK, (uint32_t)meta_str.size(), payload); - hydra_send_all(fd, meta_str.data(), meta_str.size()); - hydra_send_all(fd, res->state_data.data(), (size_t)payload); - SRV_INF("hydra rpc: STATE_GET slot=%d M1 sent %.1f MiB from buffer\n", - slot_id, payload / (1024.0 * 1024.0)); - } -} - -// STATE_PUT (0x31): Receive payload, post task, wait for result, send ack. -static void hydra_handle_state_put(int fd, int slot_id, uint64_t payload_len, const hydra_rpc_ctx & ctx) { - if (payload_len > HYDRA_MAX_STATE_BYTES) { - SRV_WRN("hydra rpc: STATE_PUT payload %" PRIu64 " B exceeds cap %" PRIu64 " B\n", - payload_len, HYDRA_MAX_STATE_BYTES); - hydra_write_res(fd, HYDRA_STATUS_ERROR, 0, 0); - // Drain to keep persistent connection alive - std::vector drain(65536); - for (uint64_t rem = payload_len; rem > 0; ) { - size_t chunk = (size_t)std::min(rem, (uint64_t)drain.size()); - if (!hydra_recv_all(fd, drain.data(), chunk)) break; - rem -= chunk; - } - return; - } - - // Read payload from socket - std::vector buf((size_t)payload_len); - if (!hydra_recv_all(fd, buf.data(), (size_t)payload_len)) { - SRV_WRN("%s", "hydra rpc: STATE_PUT failed to read payload\n"); - hydra_write_res(fd, HYDRA_STATUS_ERROR, 0, 0); - return; - } - - // Post task to inference thread - server_task task(SERVER_TASK_TYPE_HYDRA_STATE_PUT); - task.id = ctx.queue_tasks->get_new_id(); - task.hydra_action.id_slot = slot_id; - task.hydra_action.erase_existing = true; // RPC restore always replaces slot state - task.hydra_action.state_data = std::move(buf); - const int task_id = task.id; - // Register BEFORE posting — results for unregistered ids are dropped. - ctx.queue_results->add_waiting_task_id(task_id); - ctx.queue_tasks->wait_until_no_sleep(); - ctx.queue_tasks->post(std::move(task)); - - // Wait for result from inference thread (30s timeout for large restore) - std::unordered_set task_ids = {task_id}; - auto res_ptr = ctx.queue_results->recv_with_timeout(task_ids, 30); // seconds - ctx.queue_results->remove_waiting_task_id(task_id); - if (!res_ptr) { - SRV_WRN("hydra rpc: STATE_PUT timeout for slot %d\n", slot_id); - hydra_write_res(fd, HYDRA_STATUS_ERROR, 0, 0); - return; - } - - auto * res = dynamic_cast(res_ptr.get()); - if (!res) { - SRV_WRN("hydra rpc: STATE_PUT result type mismatch for slot %d\n", slot_id); - hydra_write_res(fd, HYDRA_STATUS_ERROR, 0, 0); - return; - } - - // Send result back to client - uint8_t rpc_status = res->rpc_status; - if (rpc_status == HYDRA_STATUS_OK) { - json meta_j; - meta_j["restored"] = true; - meta_j["bytes"] = res->bytes; - meta_j["model_match"] = res->model_match; - if (!res->model_alias.empty()) meta_j["model_alias"] = res->model_alias; - if (!res->model_path.empty()) meta_j["model_path"] = res->model_path; - if (!res->tokenizer.empty()) meta_j["tokenizer"] = res->tokenizer; - if (!res->model_name.empty()) meta_j["model_name"] = res->model_name; - if (!res->model_quant.empty()) meta_j["model_quant"] = res->model_quant; - if (res->model_capabilities) meta_j["model_capabilities"] = res->model_capabilities; - const std::string meta_str = meta_j.dump(); - hydra_write_res(fd, HYDRA_STATUS_OK, (uint32_t)meta_str.size(), 0); - hydra_send_all(fd, meta_str.data(), meta_str.size()); - } else { - json err_j; - err_j["error"] = res->error; - const std::string err_str = err_j.dump(); - hydra_write_res(fd, rpc_status, (uint32_t)err_str.size(), 0); - hydra_send_all(fd, err_str.data(), err_str.size()); - } -} - -// STATE_META (0x32): Post task, wait for result, send JSON metadata. -static void hydra_handle_state_meta(int fd, int slot_id, const hydra_rpc_ctx & ctx) { - server_task task(SERVER_TASK_TYPE_HYDRA_STATE_META); - task.id = ctx.queue_tasks->get_new_id(); - task.hydra_action.id_slot = slot_id; - const int task_id = task.id; - // Register BEFORE posting — results for unregistered ids are dropped. - ctx.queue_results->add_waiting_task_id(task_id); - ctx.queue_tasks->wait_until_no_sleep(); - ctx.queue_tasks->post(std::move(task)); - - // Wait for result from inference thread (5s timeout — allows for queue congestion) - std::unordered_set task_ids = {task_id}; - auto res_ptr = ctx.queue_results->recv_with_timeout(task_ids, 5); // seconds - ctx.queue_results->remove_waiting_task_id(task_id); - if (!res_ptr) { - SRV_WRN("hydra rpc: STATE_META timeout for slot %d\n", slot_id); - hydra_write_res(fd, HYDRA_STATUS_ERROR, 0, 0); - return; - } - - auto * res = dynamic_cast(res_ptr.get()); - if (!res) { - SRV_WRN("hydra rpc: STATE_META result type mismatch for slot %d\n", slot_id); - hydra_write_res(fd, HYDRA_STATUS_ERROR, 0, 0); - return; - } - - // Send result back to client - uint8_t rpc_status = res->rpc_status; - if (rpc_status == HYDRA_STATUS_OK) { - json meta_j; - meta_j["slot_id"] = res->id_slot; - meta_j["n_past"] = res->n_past; - meta_j["state_size"] = res->state_size; - meta_j["is_processing"] = res->is_processing; - meta_j["is_transferring"] = res->is_transferring; - if (!res->model_alias.empty()) meta_j["model_alias"] = res->model_alias; - if (!res->model_path.empty()) meta_j["model_path"] = res->model_path; - if (!res->tokenizer.empty()) meta_j["tokenizer"] = res->tokenizer; - if (!res->model_name.empty()) meta_j["model_name"] = res->model_name; - if (!res->model_quant.empty()) meta_j["model_quant"] = res->model_quant; - if (res->model_capabilities) meta_j["model_capabilities"] = res->model_capabilities; - const std::string meta_str = meta_j.dump(); - hydra_write_res(fd, HYDRA_STATUS_OK, (uint32_t)meta_str.size(), 0); - hydra_send_all(fd, meta_str.data(), meta_str.size()); - } else { - hydra_write_res(fd, rpc_status, 0, 0); - } -} - -// ── E1 Engine control handlers ──────────────────────────────────────────────── - -// CONFIGURE (0x33): Read JSON config payload, post task, return success. -static void hydra_handle_configure(int fd, int slot_id, uint64_t payload_len, const hydra_rpc_ctx & ctx) { - std::string config_json(payload_len, '\0'); - if (payload_len > 0 && !hydra_recv_all(fd, config_json.data(), payload_len)) { - hydra_write_res(fd, HYDRA_STATUS_ERROR, 0, 0); - return; - } - - server_task task(SERVER_TASK_TYPE_HYDRA_ENGINE_CONFIGURE); - task.id = ctx.queue_tasks->get_new_id(); - task.hydra_action.id_slot = slot_id; - task.hydra_action.config_json = std::move(config_json); - const int task_id = task.id; - ctx.queue_results->add_waiting_task_id(task_id); - ctx.queue_tasks->wait_until_no_sleep(); - ctx.queue_tasks->post(std::move(task)); - - std::unordered_set task_ids = {task_id}; - auto res_ptr = ctx.queue_results->recv_with_timeout(task_ids, 5); - ctx.queue_results->remove_waiting_task_id(task_id); - if (!res_ptr) { - hydra_write_res(fd, HYDRA_STATUS_ERROR, 0, 0); - return; - } - - auto * res = dynamic_cast(res_ptr.get()); - if (!res || !res->success) { - // hydra#406: on failure, include the error message in the meta so - // the Coordinator can distinguish "drain timeout" from a parse - // error. We still write HYDRA_STATUS_ERROR (0x02) per the wire - // contract — the meta body is for diagnostics only. - if (res && !res->error.empty()) { - json err_j = {{"success", false}, {"error", res->error}}; - if (!res->tier.empty()) err_j["tier"] = res->tier; - const std::string err_str = err_j.dump(); - hydra_write_res(fd, HYDRA_STATUS_ERROR, (uint32_t)err_str.size(), 0); - hydra_send_all(fd, err_str.data(), err_str.size()); - } else { - hydra_write_res(fd, HYDRA_STATUS_ERROR, 0, 0); - } - return; - } - - // hydra#406: tiered CONFIGURE response shape. Always present: success, - // tier, params_applied (T1 keys), deferred_keys (T2/T3 keys). - json meta_j = { - {"success", true}, - {"tier", res->tier.empty() ? std::string("T1") : res->tier}, - {"params_applied", json::object()}, - {"deferred_keys", json::array()}, - }; - for (const auto & kv : res->params_applied) { - meta_j["params_applied"][kv.first] = kv.second; - } - for (const auto & k : res->deferred_keys) { - meta_j["deferred_keys"].push_back(k); - } - // hydra#334: echo the post-clamp value for the state_chunk_size legacy - // path so the Coordinator's existing detection logic still works - // (the same value is also in params_applied, with the dotted key). - if (res->state_chunk_size_applied > 0) { - meta_j["state_chunk_size_applied"] = res->state_chunk_size_applied; - } - const std::string meta_str = meta_j.dump(); - hydra_write_res(fd, HYDRA_STATUS_OK, (uint32_t)meta_str.size(), 0); - hydra_send_all(fd, meta_str.data(), meta_str.size()); -} - -// INFO (0x34): Return engine capabilities as JSON. -static void hydra_handle_info(int fd, int slot_id, const hydra_rpc_ctx & ctx) { - server_task task(SERVER_TASK_TYPE_HYDRA_ENGINE_INFO); - task.id = ctx.queue_tasks->get_new_id(); - task.hydra_action.id_slot = slot_id; - const int task_id = task.id; - ctx.queue_results->add_waiting_task_id(task_id); - ctx.queue_tasks->wait_until_no_sleep(); - ctx.queue_tasks->post(std::move(task)); - - std::unordered_set task_ids = {task_id}; - auto res_ptr = ctx.queue_results->recv_with_timeout(task_ids, 5); - ctx.queue_results->remove_waiting_task_id(task_id); - if (!res_ptr) { - hydra_write_res(fd, HYDRA_STATUS_ERROR, 0, 0); - return; - } - - auto * res = dynamic_cast(res_ptr.get()); - if (!res) { - hydra_write_res(fd, HYDRA_STATUS_ERROR, 0, 0); - return; - } - - const std::string & info_str = res->info_json; - hydra_write_res(fd, HYDRA_STATUS_OK, (uint32_t)info_str.size(), 0); - hydra_send_all(fd, info_str.data(), info_str.size()); -} - -// PREFILL (0x35): Read JSON payload with {"messages": [...]}, -// tokenize internally, run prefill, return n_past + KV state blob. -static void hydra_handle_prefill(int fd, int slot_id, uint64_t payload_len, const hydra_rpc_ctx & ctx) { - if (payload_len == 0) { - hydra_write_res(fd, HYDRA_STATUS_ERROR, 0, 0); - return; - } - - std::string json_str((size_t)payload_len, '\0'); - if (!hydra_recv_all(fd, json_str.data(), (size_t)payload_len)) { - hydra_write_res(fd, HYDRA_STATUS_ERROR, 0, 0); - return; - } - - server_task task(SERVER_TASK_TYPE_HYDRA_ENGINE_PREFILL); - task.id = ctx.queue_tasks->get_new_id(); - task.hydra_action.id_slot = slot_id; - task.hydra_action.hydra_fd = fd; // M2 (#470): task thread streams the response here - task.hydra_action.request_json = std::move(json_str); - const int task_id = task.id; - ctx.queue_results->add_waiting_task_id(task_id); - ctx.queue_tasks->wait_until_no_sleep(); - ctx.queue_tasks->post(std::move(task)); - - std::unordered_set task_ids = {task_id}; - // Bumped from 60s to 180s, then 180s to 600s (#470). Prefill for 32k+ - // token prompts exceeds 120s (we measured 32s for 22k tokens; 48k ≈ 70s, - // 100k ≈ 150s+). Long autoregressive decode on P100 (28 tok/s) for 4k+ - // token outputs also exceeds 120s. On top of compute, the M2 stream must - // push the whole KV blob (≈800 MB today, 10 GB target) over the socket - // before this wait returns — the old 180s budget raced that transfer and - // dropped the connection mid-frame (coordinator then read garbage framing - // like 'RPC payload length out of range'). 600s covers compute + transfer - // with headroom; the Coordinator enforces an idle-based budget client-side. - auto res_ptr = ctx.queue_results->recv_with_timeout(task_ids, 600); - ctx.queue_results->remove_waiting_task_id(task_id); - if (!res_ptr) { - // #470: the M2 stream task owns the fd (header + KV already streaming); - // writing an error header here could interleave with the stream. Mirror - // the STATE_GET M2 caveat (see hydra_handle_state_get): shut the socket - // down so the client unblocks with a clean EOF and the worker frees - // immediately (SO_SNDTIMEO bounds the send-side park meanwhile). - SRV_WRN("hydra rpc: PREFILL timeout for slot %d (task_id=%d)\n", slot_id, task_id); - ::shutdown(fd, SHUT_RDWR); - return; - } - - auto * res = dynamic_cast(res_ptr.get()); - if (!res || res->rpc_status != HYDRA_STATUS_OK) { - if (res && res->header_sent) { - // M2 failure: header + meta already on the wire but the stream - // failed; the task thread shut the socket down — the connection - // loop will close the fd on its next read. Do NOT write a second - // response header (would interleave with nothing — socket is - // shut — but must not emit a second frame either). - SRV_WRN("hydra rpc: PREFILL slot=%d M2 stream failed: %s\n", - slot_id, res->error.c_str()); - return; - } - hydra_write_res(fd, HYDRA_STATUS_ERROR, 0, 0); - return; - } - - if (res->streamed_bytes > 0) { - // M2 path: the task thread already wrote the response header + meta + - // v2 header + KV state + logits straight to the socket. Nothing left - // for the RPC thread to do — protocol framing was completed on the - // task thread. (mirrors STATE_GET M2 handling) - SRV_INF("hydra rpc: PREFILL slot=%d M2 streamed %.1f MiB directly\n", - slot_id, res->streamed_bytes / (1024.0 * 1024.0)); - return; - } - - // M1 path: RPC thread sends header + meta + buffered payload (unchanged). - // Return n_past + sizes + model identity in meta; full blob (v2 header + KV + logits) as payload. - // logits_size > 0 signals the decode GPU to inject them into ctx->logits via STATE_PUT. - // M-Perf.9 #289: model identity fields (already populated on res by the PREFILL handler) - // are included so the Coordinator can record which model built the KV. - json meta_j = { - {"n_past", res->n_past}, - {"state_size", res->state_size}, - {"logits_size", res->logits_size} - }; - if (!res->model_alias.empty()) meta_j["model_alias"] = res->model_alias; - if (!res->model_path.empty()) meta_j["model_path"] = res->model_path; - if (!res->tokenizer.empty()) meta_j["tokenizer"] = res->tokenizer; - if (!res->model_name.empty()) meta_j["model_name"] = res->model_name; - if (!res->model_quant.empty()) meta_j["model_quant"] = res->model_quant; - if (res->model_capabilities) meta_j["model_capabilities"] = res->model_capabilities; - meta_j["model_fallback"] = res->model_fallback; - if (res->prefill_ms > 0) meta_j["prefill_ms"] = res->prefill_ms; - if (res->model_load_ms > 0) meta_j["model_load_ms"] = res->model_load_ms; - const std::string meta_str = meta_j.dump(); - const uint64_t total_payload = (uint64_t)res->state_data.size(); - hydra_write_res(fd, HYDRA_STATUS_OK, (uint32_t)meta_str.size(), total_payload); - hydra_send_all(fd, meta_str.data(), meta_str.size()); - if (total_payload > 0) { - hydra_send_all(fd, res->state_data.data(), (size_t)total_payload); - } - SRV_INF("hydra: PREFILL slot=%d sent n_past=%d kv=%" PRIu64 "B logits=%" PRIu64 "B total=%" PRIu64 "B\n", - slot_id, res->n_past, res->state_size, res->logits_size, total_payload); -} - -// DECODE (0x43) — Merged P/D: framed request with async HTTP retrieval. -// Wire format v3 (segmented): -// [4B hdr_len LE] <= 32768 -// [8B hdr_hash LE] xxh3-64 of the hdr JSON bytes that follow -// [hdr_len bytes] control header JSON -// [prompt_len bytes] prompt JSON segment (may be zero-length) -// [kv_len bytes] raw KV blob (may be zero-length) -// -// Control header: -// { "v": 3, "model": "...", "kv_metadata": {...}, "model_metadata": {...}, -// "generation": {...}, "segments": [...] } -// -// Two-phase flow: -// Phase 1 (sync): identity validation + KV restore — waits for inference thread -// Phase 2 (async): background thread posts SERVER_TASK_TYPE_COMPLETION, -// update_slots() drives generation, result stored in decode_results buffer. -// Actual result retrieved via GET /v1/decode/{decode_request_id}. -static void hydra_handle_decode(int fd, int slot_id, uint64_t payload_len, const hydra_rpc_ctx & ctx) { - // ── Read frame header: [4B hdr_len][8B hdr_hash] ────────────────────── - if (payload_len < sizeof(uint32_t) + sizeof(uint64_t)) { - SRV_WRN("%s", "hydra rpc: DECODE payload too small for frame header\n"); - hydra_write_res(fd, HYDRA_STATUS_BAD_REQUEST, 0, 0); - return; - } - - uint32_t hdr_len = 0; - if (!hydra_recv_all(fd, &hdr_len, sizeof(hdr_len))) { - hydra_write_res(fd, HYDRA_STATUS_ERROR, 0, 0); - return; - } - - if (hdr_len > HYDRA_MAX_JSON_HEADER) { - SRV_WRN("hydra rpc: DECODE hdr_len %u B exceeds cap %u B\n", - hdr_len, HYDRA_MAX_JSON_HEADER); - hydra_write_res(fd, HYDRA_STATUS_BAD_REQUEST, 0, 0); - return; - } - - uint64_t hdr_hash = 0; - if (!hydra_recv_all(fd, &hdr_hash, sizeof(hdr_hash))) { - hydra_write_res(fd, HYDRA_STATUS_ERROR, 0, 0); - return; - } - - // ── Read control header JSON ────────────────────────────────────────── - std::string hdr_json_str(hdr_len, '\0'); - if (hdr_len > 0 && !hydra_recv_all(fd, hdr_json_str.data(), hdr_len)) { - hydra_write_res(fd, HYDRA_STATUS_ERROR, 0, 0); - return; - } - - // Verify hdr_hash (xxh3-64 of the JSON bytes) - { - const uint64_t computed = XXH3_64bits(hdr_json_str.data(), hdr_json_str.size()); - if (computed != hdr_hash) { - SRV_WRN("hydra rpc: DECODE HDR_HASH_MISMATCH expected=%016" PRIx64 " got=%016" PRIx64 "\n", - hdr_hash, computed); - hydra_write_res(fd, HYDRA_STATUS_BAD_REQUEST, 0, 0); - return; - } - } - - // Parse control header - json req; - try { - req = json::parse(hdr_json_str); - } catch (const std::exception & e) { - SRV_WRN("hydra rpc: DECODE invalid JSON in control header: %s\n", e.what()); - hydra_write_res(fd, HYDRA_STATUS_BAD_REQUEST, 0, 0); - return; - } - - // Validate version - const int hdr_version = req.value("v", 0); - if (hdr_version < 3) { - SRV_WRN("hydra rpc: DECODE unsupported version %d (need >= 3)\n", hdr_version); - hydra_write_res(fd, HYDRA_STATUS_BAD_REQUEST, 0, 0); - return; - } - - // Validate required fields - if (!req.contains("kv_metadata")) { - SRV_WRN("%s", "hydra rpc: DECODE missing kv_metadata in control header\n"); - hydra_write_res(fd, HYDRA_STATUS_BAD_REQUEST, 0, 0); - return; - } - if (!req.contains("segments") || !req["segments"].is_array()) { - SRV_WRN("%s", "hydra rpc: DECODE missing or invalid segments array\n"); - hydra_write_res(fd, HYDRA_STATUS_BAD_REQUEST, 0, 0); - return; - } - - // ── Parse and validate segment table ────────────────────────────────── - const json & segments = req["segments"]; - const size_t n_segments = segments.size(); - if (n_segments > 3) { - SRV_WRN("hydra rpc: DECODE SEGMENT_TABLE_INVALID: too many segments (%zu)\n", n_segments); - hydra_write_res(fd, HYDRA_STATUS_BAD_REQUEST, 0, 0); - return; - } - - // Each segment: {"id":"prompt"|"kv", "offset":N, "len":N, "hash":"xxh3:HEX"} - uint64_t prompt_len = 0; - uint64_t kv_len = 0; - std::string prompt_hash_str; - std::string kv_hash_str; - uint64_t expected_offset = 0; - for (size_t i = 0; i < n_segments; i++) { - const json & seg = segments[i]; - if (!seg.contains("id") || !seg.contains("offset") || !seg.contains("len") || !seg.contains("hash")) { - SRV_WRN("hydra rpc: DECODE SEGMENT_TABLE_INVALID: segment %zu missing required fields\n", i); - hydra_write_res(fd, HYDRA_STATUS_BAD_REQUEST, 0, 0); - return; - } - const std::string id = seg["id"].get(); - const uint64_t offset = seg["offset"].get(); - const uint64_t len = seg["len"].get(); - const std::string hash = seg["hash"].get(); - - if (offset != expected_offset) { - SRV_WRN("hydra rpc: DECODE SEGMENT_TABLE_INVALID: segment %zu offset=%" PRIu64 " expected=%" PRIu64 "\n", - i, offset, expected_offset); - hydra_write_res(fd, HYDRA_STATUS_BAD_REQUEST, 0, 0); - return; - } - expected_offset = offset + len; - - if (id == "prompt") { - prompt_len = len; - prompt_hash_str = hash; - } else if (id == "kv") { - kv_len = len; - kv_hash_str = hash; - } else { - SRV_WRN("hydra rpc: DECODE SEGMENT_TABLE_INVALID: unknown segment id '%s'\n", id.c_str()); - hydra_write_res(fd, HYDRA_STATUS_BAD_REQUEST, 0, 0); - return; - } - } - - // Verify total segment size matches remaining payload - const uint64_t segments_total = prompt_len + kv_len; - const uint64_t remaining_after_hdr = payload_len - sizeof(uint32_t) - sizeof(uint64_t) - hdr_len; - if (segments_total != remaining_after_hdr) { - SRV_WRN("hydra rpc: DECODE SEGMENT_TABLE_INVALID: segments total %" PRIu64 " != remaining %" PRIu64 "\n", - segments_total, remaining_after_hdr); - hydra_write_res(fd, HYDRA_STATUS_BAD_REQUEST, 0, 0); - return; - } - - // Caps - if (prompt_len > HYDRA_MAX_PROMPT_BYTES) { - SRV_WRN("hydra rpc: DECODE PROMPT_TOO_LARGE %" PRIu64 " > %" PRIu64 "\n", - prompt_len, HYDRA_MAX_PROMPT_BYTES); - hydra_write_res(fd, HYDRA_STATUS_BAD_REQUEST, 0, 0); - return; - } - if (kv_len > HYDRA_MAX_STATE_BYTES) { - SRV_WRN("hydra rpc: DECODE KV_TOO_LARGE %" PRIu64 " > %" PRIu64 "\n", - kv_len, HYDRA_MAX_STATE_BYTES); - hydra_write_res(fd, HYDRA_STATUS_BAD_REQUEST, 0, 0); - return; - } - - // ── Read prompt segment ─────────────────────────────────────────────── - std::vector prompt_data((size_t)prompt_len); - if (prompt_len > 0 && !hydra_recv_all(fd, prompt_data.data(), (size_t)prompt_len)) { - hydra_write_res(fd, HYDRA_STATUS_ERROR, 0, 0); - return; - } - - // ── Read KV segment (may be zero-length) ────────────────────────────── - // M2 (#470): the KV blob (2.3 GB today, 10 GB target) is never materialized. - // The v2 blob header (small: version + n_past + n_tok + tokens + flags + - // checkpoint) is read here; the remaining stream (magic + seq_id + KV state - // + logits) stays on the fd and is consumed by the DECODE task thread via - // llama_state_seq_set_data_from_fd. The wire hash (xxh3-64 over the whole - // kv segment) is verified post-restore in the task thread — with streaming - // the bytes reach the GPU before a pre-restore hash could be computed. - uint64_t expected_kv_hash = 0; - bool has_kv_hash = false; - if (kv_len > 0 && !kv_hash_str.empty()) { - if (kv_hash_str.rfind("xxh3:", 0) != 0) { - SRV_WRN("hydra rpc: DECODE unsupported KV hash prefix: %s\n", kv_hash_str.c_str()); - hydra_write_res(fd, HYDRA_STATUS_BAD_REQUEST, 0, 0); - return; - } - try { - expected_kv_hash = std::stoull(kv_hash_str.substr(5), nullptr, 16); - has_kv_hash = true; - } catch (const std::exception &) { - SRV_WRN("hydra rpc: DECODE invalid KV hash format: %s\n", kv_hash_str.c_str()); - hydra_write_res(fd, HYDRA_STATUS_BAD_REQUEST, 0, 0); - return; - } - } - - std::vector kv_data; // M1 fallback (legacy non-v2 blobs) - std::vector kv_v2_hdr; // M2: parsed v2 header (small) - uint64_t kv_stream_len = 0; // M2: bytes remaining on fd after the header - if (kv_len > 0) { - uint8_t version_byte = 0; - if (!hydra_recv_all(fd, &version_byte, 1)) { - hydra_write_res(fd, HYDRA_STATUS_ERROR, 0, 0); - return; - } - if (version_byte != 0x02 && version_byte != 0x03) { - // Legacy non-v2/v3 blob — buffered M1 path (pre-#470 behavior). - // Replay the consumed version byte into the buffer. - kv_data.resize((size_t)kv_len); - kv_data[0] = version_byte; - if (kv_len > 1 && !hydra_recv_all(fd, kv_data.data() + 1, (size_t)kv_len - 1)) { - hydra_write_res(fd, HYDRA_STATUS_ERROR, 0, 0); - return; - } - if (has_kv_hash) { - const uint64_t computed_kv = XXH3_64bits(kv_data.data(), kv_data.size()); - if (computed_kv != expected_kv_hash) { - SRV_WRN("hydra rpc: DECODE SEGMENT_HASH_MISMATCH kv expected=%016" PRIx64 " got=%016" PRIx64 "\n", - expected_kv_hash, computed_kv); - hydra_write_res(fd, HYDRA_STATUS_BAD_REQUEST, 0, 0); - return; - } - SRV_INF("hydra rpc: DECODE KV hash verified (%" PRIu64 " B)\n", kv_len); - } - } else { - // M2: parse the v2 header incrementally (all small reads) and leave - // the state stream on the fd for the task thread. - kv_v2_hdr.push_back(version_byte); - if (kv_len < 9) { - SRV_WRN("hydra rpc: DECODE KV segment too small for v2 header (%" PRIu64 " B)\n", kv_len); - hydra_write_res(fd, HYDRA_STATUS_BAD_REQUEST, 0, 0); - return; - } - uint32_t n_past_in = 0, n_tok_in = 0; - if (!hydra_recv_all(fd, &n_past_in, 4) || !hydra_recv_all(fd, &n_tok_in, 4)) { - hydra_write_res(fd, HYDRA_STATUS_ERROR, 0, 0); - return; - } - kv_v2_hdr.insert(kv_v2_hdr.end(), (const uint8_t *)&n_past_in, (const uint8_t *)&n_past_in + 4); - kv_v2_hdr.insert(kv_v2_hdr.end(), (const uint8_t *)&n_tok_in, (const uint8_t *)&n_tok_in + 4); - - const size_t tokens_bytes = (size_t)n_tok_in * sizeof(llama_token); - if (9 + tokens_bytes + 1 > kv_len) { - SRV_WRN("hydra rpc: DECODE v2 header tokens exceed kv_len (%" PRIu64 " B)\n", kv_len); - hydra_write_res(fd, HYDRA_STATUS_BAD_REQUEST, 0, 0); - return; - } - std::vector tokens_buf(tokens_bytes); - if (tokens_bytes > 0 && !hydra_recv_all(fd, tokens_buf.data(), tokens_bytes)) { - hydra_write_res(fd, HYDRA_STATUS_ERROR, 0, 0); - return; - } - kv_v2_hdr.insert(kv_v2_hdr.end(), tokens_buf.begin(), tokens_buf.end()); - - uint8_t flags = 0; - if (!hydra_recv_all(fd, &flags, 1)) { - hydra_write_res(fd, HYDRA_STATUS_ERROR, 0, 0); - return; - } - kv_v2_hdr.push_back(flags); - - if (flags & 0x01) { - // Checkpoint: 4B pos_min | 4B pos_max | 8B n_tokens | 8B tgt_sz | - // tgt_data | 8B dft_sz | dft_data (mirrors the DECODE_APPLY parse). - if (kv_v2_hdr.size() + 24 > kv_len) { - SRV_WRN("hydra rpc: DECODE v2 checkpoint header exceeds kv_len (%" PRIu64 " B)\n", kv_len); - hydra_write_res(fd, HYDRA_STATUS_BAD_REQUEST, 0, 0); - return; - } - uint8_t ckpt_fixed[24]; // pos_min + pos_max + n_tokens + tgt_sz - if (!hydra_recv_all(fd, ckpt_fixed, sizeof(ckpt_fixed))) { - hydra_write_res(fd, HYDRA_STATUS_ERROR, 0, 0); - return; - } - kv_v2_hdr.insert(kv_v2_hdr.end(), ckpt_fixed, ckpt_fixed + sizeof(ckpt_fixed)); - uint64_t tgt_sz_in = 0; - memcpy(&tgt_sz_in, ckpt_fixed + 16, 8); - if (kv_v2_hdr.size() + (size_t)tgt_sz_in + 8 > kv_len) { - SRV_WRN("hydra rpc: DECODE v2 checkpoint payload exceeds kv_len (%" PRIu64 " B)\n", kv_len); - hydra_write_res(fd, HYDRA_STATUS_BAD_REQUEST, 0, 0); - return; - } - if (tgt_sz_in > 0) { - std::vector tgt_buf((size_t)tgt_sz_in); - if (!hydra_recv_all(fd, tgt_buf.data(), (size_t)tgt_sz_in)) { - hydra_write_res(fd, HYDRA_STATUS_ERROR, 0, 0); - return; - } - kv_v2_hdr.insert(kv_v2_hdr.end(), tgt_buf.begin(), tgt_buf.end()); - } - uint8_t dft_sz_buf[8]; - if (!hydra_recv_all(fd, dft_sz_buf, sizeof(dft_sz_buf))) { - hydra_write_res(fd, HYDRA_STATUS_ERROR, 0, 0); - return; - } - kv_v2_hdr.insert(kv_v2_hdr.end(), dft_sz_buf, dft_sz_buf + sizeof(dft_sz_buf)); - uint64_t dft_sz_in = 0; - memcpy(&dft_sz_in, dft_sz_buf, 8); - if (kv_v2_hdr.size() + (size_t)dft_sz_in > kv_len) { - SRV_WRN("hydra rpc: DECODE v2 checkpoint payload exceeds kv_len (%" PRIu64 " B)\n", kv_len); - hydra_write_res(fd, HYDRA_STATUS_BAD_REQUEST, 0, 0); - return; - } - if (dft_sz_in > 0) { - std::vector dft_buf((size_t)dft_sz_in); - if (!hydra_recv_all(fd, dft_buf.data(), (size_t)dft_sz_in)) { - hydra_write_res(fd, HYDRA_STATUS_ERROR, 0, 0); - return; - } - kv_v2_hdr.insert(kv_v2_hdr.end(), dft_buf.begin(), dft_buf.end()); - } - } - - kv_stream_len = kv_len - kv_v2_hdr.size(); - // The stream must at least hold the [4B magic][4B seq_id] framing. - if (kv_stream_len < 8) { - SRV_WRN("hydra rpc: DECODE v2 stream too small (%" PRIu64 " B after header)\n", kv_stream_len); - hydra_write_res(fd, HYDRA_STATUS_BAD_REQUEST, 0, 0); - return; - } - } - } - - // ── Build decode_json from control header + prompt segment ───────────── - // The prompt JSON segment may contain { "prompt": "..." } or { "messages": [...] } - // Merge it into the control header as decode_req["prompt"]. - // Also merge generation params from control header's "generation" key. - json decode_req = req; // control header already has kv_metadata, model, etc. - json prompt_obj; - if (prompt_len > 0) { - try { - prompt_obj = json::parse(std::string(prompt_data.begin(), prompt_data.end())); - } catch (const std::exception & e) { - SRV_WRN("hydra rpc: DECODE invalid prompt segment JSON: %s\n", e.what()); - hydra_write_res(fd, HYDRA_STATUS_BAD_REQUEST, 0, 0); - return; - } - } - // The coordinator sends the prompt segment as the BARE messages array - // (item.Request["messages"].ToString()). The generation-merge below and - // DECODE_APPLY's chat-template path both expect an OBJECT with a - // "messages" key — merging generation keys into an array throws - // nlohmann::type_error, which was silently swallowed by the RPC worker - // pool (the connection leaked, no response written, coordinator timed out - // after 180s). Wrap a bare array so the prompt object matches the - // downstream contract. - if (prompt_obj.is_array()) { - json wrapped; - wrapped["messages"] = std::move(prompt_obj); - prompt_obj = std::move(wrapped); - } - // Merge generation params from control header into prompt object - std::string decode_json_str; - try { - if (req.contains("generation") && req["generation"].is_object()) { - const json & gen = req["generation"]; - for (auto it = gen.begin(); it != gen.end(); ++it) { - if (!prompt_obj.contains(it.key())) { - prompt_obj[it.key()] = it.value(); - } - } - } - decode_req["prompt"] = std::move(prompt_obj); - - decode_json_str = decode_req.dump(); - } catch (const std::exception & e) { - // Never let a malformed prompt object leak the connection: the worker - // pool swallows exceptions and the fd stays open with no response, - // hanging the coordinator until its own timeout. Always write an - // error frame so the caller sees a terminal (retryable-free) result. - SRV_WRN("hydra rpc: DECODE prompt build failed (slot %d): %s\n", slot_id, e.what()); - json err_j = { - {"error", std::string("prompt build failed: ") + e.what()}, - {"decode_request_id", -1}, - }; - const std::string err_str = err_j.dump(); - hydra_write_res(fd, HYDRA_STATUS_BAD_REQUEST, (uint32_t) err_str.size(), 0); - hydra_send_all(fd, err_str.data(), err_str.size()); - return; - } - - // ── Phase 1: sync validate + restore ────────────────────────────────── - const int32_t decode_request_id = ctx.queue_tasks->get_new_id(); - - server_task val_task(SERVER_TASK_TYPE_HYDRA_ENGINE_DECODE); - val_task.id = decode_request_id; - val_task.hydra_action.id_slot = slot_id; - val_task.hydra_action.decode_json = std::move(decode_json_str); - val_task.hydra_action.kv_data = std::move(kv_data); - // M2 (#470): the KV state stream stays on the fd and is consumed by the - // task thread (llama_state_seq_set_data_from_fd) — no full-blob buffer. - val_task.hydra_action.hydra_fd = fd; - val_task.hydra_action.kv_v2_hdr = std::move(kv_v2_hdr); - val_task.hydra_action.kv_stream_len = kv_stream_len; - val_task.hydra_action.kv_expected_hash = has_kv_hash ? expected_kv_hash : 0; - val_task.hydra_action.decode_request_id = decode_request_id; - ctx.queue_results->add_waiting_task_id(decode_request_id); - ctx.queue_tasks->wait_until_no_sleep(); - ctx.queue_tasks->post(std::move(val_task)); - - // Wait for validation+restore to complete. Raised 30s -> 600s (#470): the - // M2 restore streams the whole KV blob (2.3 GB today, 10 GB target) off the - // fd inside the task, so compute + transfer must fit the wait. The - // Coordinator enforces its own idle-based client budget. - std::unordered_set val_ids = {decode_request_id}; - auto val_res_ptr = ctx.queue_results->recv_with_timeout(val_ids, 600); - ctx.queue_results->remove_waiting_task_id(decode_request_id); - - if (!val_res_ptr) { - SRV_WRN("hydra rpc: DECODE validation timeout for slot %d (request_id=%d)\n", - slot_id, decode_request_id); - // #470: same M2 caveat as STATE_GET/PREFILL — the restore task may own - // the fd mid-stream; shut the socket down instead of writing an error - // header that could interleave. Client unblocks with a clean EOF, the - // worker frees immediately. - ::shutdown(fd, SHUT_RDWR); - return; - } - - auto * val_res = dynamic_cast(val_res_ptr.get()); - if (!val_res || val_res->rpc_status != HYDRA_STATUS_OK) { - json err_j = { - {"valid", false}, - {"decode_request_id", decode_request_id}, - }; - if (val_res) { - if (!val_res->match_json.is_null()) err_j["match"] = val_res->match_json; - if (!val_res->error.empty()) err_j["reason"] = val_res->error; - err_j["error_code"] = "CAP_MISMATCH"; - } - const std::string err_str = err_j.dump(); - hydra_write_res(fd, HYDRA_STATUS_ERROR, (uint32_t)err_str.size(), 0); - hydra_send_all(fd, err_str.data(), err_str.size()); - return; - } - - // Validation passed — build real success response - json meta_j = { - {"valid", true}, - {"match", val_res->match_json}, - {"decode_request_id", decode_request_id}, - {"n_past_after_restore", val_res->n_past}, - {"restore_slot_ms", val_res->restore_slot_ms}, - }; - const std::string meta_str = meta_j.dump(); - hydra_write_res(fd, HYDRA_STATUS_OK, (uint32_t)meta_str.size(), 0); - hydra_send_all(fd, meta_str.data(), meta_str.size()); - - SRV_INF("hydra: DECODE slot=%d accepted, request_id=%d, restore=%.1fms\n", - slot_id, decode_request_id, val_res->restore_slot_ms); -} - -// SET_EXPERT_MODE (0x37): Read mode string, post task, return success. -static void hydra_handle_set_expert_mode(int fd, int slot_id, uint64_t payload_len, const hydra_rpc_ctx & ctx) { - std::string mode(payload_len, '\0'); - if (payload_len > 0 && !hydra_recv_all(fd, mode.data(), payload_len)) { - hydra_write_res(fd, HYDRA_STATUS_ERROR, 0, 0); - return; - } - - server_task task(SERVER_TASK_TYPE_HYDRA_ENGINE_SET_EXPERT_MODE); - task.id = ctx.queue_tasks->get_new_id(); - task.hydra_action.id_slot = slot_id; - task.hydra_action.expert_mode = std::move(mode); - const int task_id = task.id; - ctx.queue_results->add_waiting_task_id(task_id); - ctx.queue_tasks->wait_until_no_sleep(); - ctx.queue_tasks->post(std::move(task)); - - std::unordered_set task_ids = {task_id}; - auto res_ptr = ctx.queue_results->recv_with_timeout(task_ids, 5); - ctx.queue_results->remove_waiting_task_id(task_id); - if (!res_ptr) { - hydra_write_res(fd, HYDRA_STATUS_ERROR, 0, 0); - return; - } - - auto * res = dynamic_cast(res_ptr.get()); - if (!res || !res->success) { - const std::string err = (res && !res->error.empty()) ? res->error : std::string(); - json err_j = {{"success", false}}; - if (!err.empty()) err_j["error"] = err; - const std::string err_str = err_j.dump(); - hydra_write_res(fd, HYDRA_STATUS_ERROR, (uint32_t)err_str.size(), 0); - hydra_send_all(fd, err_str.data(), err_str.size()); - return; - } - - // Report the ACTUAL mode applied (may be "solo" even though "combined" was - // requested, if this engine never dual-loaded combined experts) — the - // Coordinator's ReportsSolo() reads this key to detect the fallback. - json meta_j = {{"success", true}, {"mode", res->expert_mode_applied}}; - const std::string meta_str = meta_j.dump(); - hydra_write_res(fd, HYDRA_STATUS_OK, (uint32_t)meta_str.size(), 0); - hydra_send_all(fd, meta_str.data(), meta_str.size()); -} - -// SWAP_QUANT (0x38): Read quant_key + tensor_pattern, post task, return success. -static void hydra_handle_swap_quant(int fd, int slot_id, uint64_t payload_len, const hydra_rpc_ctx & ctx) { - if (payload_len < sizeof(uint16_t)) { - hydra_write_res(fd, HYDRA_STATUS_ERROR, 0, 0); - return; - } - - uint16_t quant_key_len = 0; - if (!hydra_recv_all(fd, &quant_key_len, sizeof(quant_key_len))) { - hydra_write_res(fd, HYDRA_STATUS_ERROR, 0, 0); - return; - } - - std::string quant_key(quant_key_len, '\0'); - if (quant_key_len > 0 && !hydra_recv_all(fd, quant_key.data(), quant_key_len)) { - hydra_write_res(fd, HYDRA_STATUS_ERROR, 0, 0); - return; - } - - const uint64_t pattern_len = payload_len - sizeof(uint16_t) - quant_key_len; - std::string tensor_pattern(pattern_len, '\0'); - if (pattern_len > 0 && !hydra_recv_all(fd, tensor_pattern.data(), pattern_len)) { - hydra_write_res(fd, HYDRA_STATUS_ERROR, 0, 0); - return; - } - - server_task task(SERVER_TASK_TYPE_HYDRA_ENGINE_SWAP_QUANT); - task.id = ctx.queue_tasks->get_new_id(); - task.hydra_action.id_slot = slot_id; - task.hydra_action.quant_key = std::move(quant_key); - task.hydra_action.tensor_pattern = std::move(tensor_pattern); - const int task_id = task.id; - ctx.queue_results->add_waiting_task_id(task_id); - ctx.queue_tasks->wait_until_no_sleep(); - ctx.queue_tasks->post(std::move(task)); - - std::unordered_set task_ids = {task_id}; - auto res_ptr = ctx.queue_results->recv_with_timeout(task_ids, 30); - ctx.queue_results->remove_waiting_task_id(task_id); - if (!res_ptr) { - hydra_write_res(fd, HYDRA_STATUS_ERROR, 0, 0); - return; - } - - auto * res = dynamic_cast(res_ptr.get()); - if (!res || !res->success) { - hydra_write_res(fd, HYDRA_STATUS_ERROR, 0, 0); - return; - } - - json meta_j = {{"success", true}}; - const std::string meta_str = meta_j.dump(); - hydra_write_res(fd, HYDRA_STATUS_OK, (uint32_t)meta_str.size(), 0); - hydra_send_all(fd, meta_str.data(), meta_str.size()); -} - -// PIPELINE_ATTACH (0x46): M-Perf.9 (#289) / issue #287 — two-engine "work -// together" routing scaffolding. The C# Coordinator sends the peer address -// and the --override-tensor regex; the engine should load the assigned -// tensor slice from its OWN local model (no weight transfer). This opcode -// is stubbed for now (returns NOT_IMPLEMENTED) — full implementation is -// tracked under issue #287. -static void hydra_handle_pipeline_attach(int fd, int slot_id, uint64_t payload_len, const hydra_rpc_ctx & ctx) { - std::string json_body(payload_len, '\0'); - if (payload_len > 0 && !hydra_recv_all(fd, json_body.data(), payload_len)) { - hydra_write_res(fd, HYDRA_STATUS_ERROR, 0, 0); - return; - } - - server_task task(SERVER_TASK_TYPE_HYDRA_ENGINE_PIPELINE_ATTACH); - task.id = ctx.queue_tasks->get_new_id(); - task.hydra_action.id_slot = slot_id; - task.hydra_action.request_json = std::move(json_body); - const int task_id = task.id; - ctx.queue_results->add_waiting_task_id(task_id); - ctx.queue_tasks->wait_until_no_sleep(); - ctx.queue_tasks->post(std::move(task)); - - std::unordered_set task_ids = {task_id}; - auto res_ptr = ctx.queue_results->recv_with_timeout(task_ids, 5); - ctx.queue_results->remove_waiting_task_id(task_id); - if (!res_ptr) { - hydra_write_res(fd, HYDRA_STATUS_ERROR, 0, 0); - return; - } - - auto * res = dynamic_cast(res_ptr.get()); - // Stubbed: server returns NOT_IMPLEMENTED until issue #287 lands. - // Propagate that status to the client so the Coordinator can - // distinguish "not yet built" from a real error and fall back to solo. - const uint8_t status = (res && res->rpc_status == HYDRA_STATUS_NOT_IMPLEMENTED) - ? HYDRA_STATUS_NOT_IMPLEMENTED : HYDRA_STATUS_ERROR; - json meta_j; - if (res && !res->error.empty()) meta_j["error"] = res->error; - meta_j["success"] = res && res->success; - const std::string meta_str = meta_j.dump(); - hydra_write_res(fd, status, (uint32_t)meta_str.size(), 0); - hydra_send_all(fd, meta_str.data(), meta_str.size()); -} - -// ── Per-connection loop ─────────────────────────────────────────────────────── -// Persistent: one TCP connection handles many sequential requests. - -static void hydra_handle_connection(int fd, const hydra_rpc_ctx & ctx) { - // Set receive timeout to prevent hung connections on stalled clients - struct timeval tv; - tv.tv_sec = 120; // 2 min inactivity timeout - tv.tv_usec = 0; - (void)setsockopt(fd, SOL_SOCKET, SO_RCVTIMEO, &tv, sizeof(tv)); - while (true) { - uint8_t hdr[HYDRA_REQ_HEADER_SIZE]; - if (!hydra_recv_all(fd, hdr, HYDRA_REQ_HEADER_SIZE)) break; - - uint16_t magic = 0; - memcpy(&magic, hdr + 0, 2); - if (magic != HYDRA_MAGIC) { - SRV_WRN("hydra rpc: bad magic 0x%04x — closing connection\n", (unsigned)magic); - break; - } - - const uint8_t op = hdr[2]; - // hdr[3] = flags (reserved, unused in M1) - uint16_t key_len = 0, trace_len = 0; - uint64_t payload_len = 0; - memcpy(&key_len, hdr + 4, 2); - memcpy(&payload_len, hdr + 6, 8); - memcpy(&trace_len, hdr + 14, 2); - - std::string key(key_len, '\0'); - std::string trace_id(trace_len, '\0'); - if (!hydra_recv_all(fd, key.data(), key_len)) break; - if (!hydra_recv_all(fd, trace_id.data(), trace_len)) break; - - // Slot-key parsing: engine-level opcodes (INFO, CONFIGURE, SET_EXPERT_MODE, - // SWAP_QUANT) don't need a valid slot — use slot_id = 0 when the key is - // empty or invalid. Slot-level opcodes (STATE_GET, STATE_PUT, STATE_META, - // PREFILL, DECODE) still require a valid integer key. - int slot_id = -1; - bool is_engine_level_op = (op == HYDRA_OP_INFO || op == HYDRA_OP_CONFIGURE || - op == HYDRA_OP_SET_EXPERT_MODE || op == HYDRA_OP_SWAP_QUANT); - if (key.empty() && is_engine_level_op) { - slot_id = 0; - } else { - try { slot_id = std::stoi(key); } - catch (...) { - SRV_WRN("hydra rpc: invalid slot key '%s'\n", key.c_str()); - hydra_write_res(fd, HYDRA_STATUS_ERROR, 0, 0); - continue; - } - } - - // Dispatch to handler via task queue (no direct slot access) - switch (op) { - case HYDRA_OP_STATE_GET: - SRV_DBG("hydra rpc: STATE_GET slot=%d trace=%s\n", slot_id, trace_id.c_str()); - hydra_handle_state_get(fd, slot_id, ctx); - break; - case HYDRA_OP_STATE_PUT: - SRV_DBG("hydra rpc: STATE_PUT slot=%d payload=%" PRIu64 " trace=%s\n", - slot_id, payload_len, trace_id.c_str()); - hydra_handle_state_put(fd, slot_id, payload_len, ctx); - break; - case HYDRA_OP_STATE_META: - SRV_DBG("hydra rpc: STATE_META slot=%d trace=%s\n", slot_id, trace_id.c_str()); - hydra_handle_state_meta(fd, slot_id, ctx); - break; - case HYDRA_OP_CONFIGURE: - SRV_DBG("hydra rpc: CONFIGURE slot=%d payload=%" PRIu64 " trace=%s\n", - slot_id, payload_len, trace_id.c_str()); - hydra_handle_configure(fd, slot_id, payload_len, ctx); - break; - case HYDRA_OP_INFO: - SRV_DBG("hydra rpc: INFO slot=%d trace=%s\n", slot_id, trace_id.c_str()); - hydra_handle_info(fd, slot_id, ctx); - break; - case HYDRA_OP_PREFILL: - SRV_DBG("hydra rpc: PREFILL slot=%d payload=%" PRIu64 " trace=%s\n", - slot_id, payload_len, trace_id.c_str()); - hydra_handle_prefill(fd, slot_id, payload_len, ctx); - break; - case HYDRA_OP_DECODE: - SRV_DBG("hydra rpc: DECODE slot=%d payload=%" PRIu64 " trace=%s\n", - slot_id, payload_len, trace_id.c_str()); - hydra_handle_decode(fd, slot_id, payload_len, ctx); - break; - case HYDRA_OP_SET_EXPERT_MODE: - SRV_DBG("hydra rpc: SET_EXPERT_MODE slot=%d payload=%" PRIu64 " trace=%s\n", - slot_id, payload_len, trace_id.c_str()); - hydra_handle_set_expert_mode(fd, slot_id, payload_len, ctx); - break; - case HYDRA_OP_SWAP_QUANT: - SRV_DBG("hydra rpc: SWAP_QUANT slot=%d payload=%" PRIu64 " trace=%s\n", - slot_id, payload_len, trace_id.c_str()); - hydra_handle_swap_quant(fd, slot_id, payload_len, ctx); - break; - // M-Perf.9 (#289) / issue #287: PIPELINE_ATTACH (0x46) is the - // two-engine "work together" attach. Stubbed: full impl in #287. - case HYDRA_OP_PIPELINE_ATTACH: - SRV_DBG("hydra rpc: PIPELINE_ATTACH slot=%d payload=%" PRIu64 " trace=%s\n", - slot_id, payload_len, trace_id.c_str()); - hydra_handle_pipeline_attach(fd, slot_id, payload_len, ctx); - break; - default: - SRV_WRN("hydra rpc: unknown op 0x%02x — ignoring\n", (unsigned)op); - hydra_write_res(fd, HYDRA_STATUS_ERROR, 0, 0); - } - } - ::close(fd); -} - -// ── Unified RPC server implementation ─────────────────────────────────────── -// -// `#36` Phase 1: the merged server lives in `tools/llama-engine/hydra_rpc/` -// (fork-isolated). `server_context::start_rpc_server` is a thin adapter that -// builds the settings and delegates to `hydra_rpc::start()`. The Hydra -// protocol entry `hydra_handle_connection` is reached through the -// `hydra_rpc_bridge` trampoline (defined below) — the bridge takes a -// `void*` so the new module can stay decoupled from this file's includes. - -#include "../llama-engine/hydra_rpc/hydra_rpc.h" - -void server_context::start_rpc_server(int port, - std::vector backends) { - if (port <= 0) return; - - // Hydra #43: MUST outlive this function. `hydra_rpc::start()` below - // stores `&ctx` as a raw pointer inside `hydra_rpc::state()`, a - // process-lifetime singleton that every subsequent RPC connection reads - // (from a bounded-thread-pool worker thread) to recover queue_tasks / - // queue_results. An automatic-storage `ctx` here would dangle the - // instant this function returns — a stack-use-after-return that "works" - // until the freed stack slot gets reused, then silently corrupts the - // RPC response path. `start_rpc_server` only ever runs once per process - // (hydra_rpc::start() itself guards double-start), so `static` gives it - // exactly the lifetime the singleton needs. - static hydra_rpc_ctx ctx{}; - if (impl) { - ctx.queue_tasks = &impl->queue_tasks; - ctx.queue_results = &impl->queue_results; - } - - hydra_rpc::settings s; - s.port = port; - s.backends = std::move(backends); - s.hydra_ctx = (ctx.queue_tasks && ctx.queue_results) ? &ctx : nullptr; - s.pool_size = 2; - s.max_queue = 64; - s.host = "0.0.0.0"; - - if (!hydra_rpc::start(s)) { - SRV_ERR("hydra rpc: start() failed on port %d\n", port); - return; - } - - if (s.hydra_ctx) { - SRV_INF("hydra rpc: unified server on 0.0.0.0:%d (ggml-RPC + Hydra protocol)\n", port); - } else { - SRV_INF("hydra rpc: unified server on 0.0.0.0:%d (ggml-RPC only)\n", port); - } -} - -// `hydra_rpc_bridge` — extern "C" trampoline. `hydra_rpc.cpp` calls this -// when the first byte on a new connection is not `RPC_CMD_HELLO`. It -// re-enters the C++ entry point with the typed `hydra_rpc_ctx &`. -// -// Forward-declared with the matching signature so the new -// `tools/llama-engine/hydra_rpc/hydra_rpc.cpp` module can take its -// address without including this heavy header. -extern "C" void hydra_rpc_bridge(int fd, const void * ctx); -extern "C" void hydra_rpc_bridge(int fd, const void * ctx) { - hydra_handle_connection(fd, *static_cast(ctx)); -} - -#else -// Windows: RPC server not implemented — target hardware is Linux-only for M0. -void server_context::start_rpc_server(int port, std::vector) { - if (port > 0) { - SRV_WRN("hydra rpc: not supported on Windows (port %d ignored)\n", port); - } - GGML_UNUSED(port); -} -#endif // !_WIN32 // epic #610 WS1: Hydra extension implementation. Compiled INTO this TU so the // concrete class can reach server_context_impl private members via the friend From 546d3f8fc3df382946a27923daba51a56fc3a362 Mon Sep 17 00:00:00 2001 From: Hydra Engineering Date: Tue, 11 Aug 2026 18:48:13 +0700 Subject: [PATCH 3/5] epic(610): WS3.5 move T1/T2/T3 config + COMBINED helper methods out Moved hydra_classify_config_key, hydra_tier_label, hydra_apply_t1_config, hydra_apply_t3_mutators, hydra_parse_cache_type, hydra_register_rpc_servers, hydra_teardown_combined_before_reload, hydra_reattach_combined_after_reload, hydra_repad_tensor_buft_overrides, apply_pending_hydra_config, apply_t2_rebuild, apply_t3_rebuild to hydra-server-context.cpp (same TU). server-context.cpp keeps only in-class declarations. ~770 lines out of the upstream file. --- tools/server/hydra-server-context.cpp | 794 ++++++++++++++ tools/server/server-context.cpp | 1441 +------------------------ 2 files changed, 799 insertions(+), 1436 deletions(-) diff --git a/tools/server/hydra-server-context.cpp b/tools/server/hydra-server-context.cpp index b0c79ac61de2..9b06956e5bee 100644 --- a/tools/server/hydra-server-context.cpp +++ b/tools/server/hydra-server-context.cpp @@ -3736,3 +3736,797 @@ void server_context::start_rpc_server(int port, std::vector) { GGML_UNUSED(port); } #endif // !_WIN32 + +// --- WS3.5 moved helper methods --- + + bool server_context_impl::apply_t3_rebuild() { + bool is_first_load = !ctx_tgt; + + // Track the last override_tensor string that was actually + // applied so we can detect "nothing changed" on subsequent + // calls and skip the expensive unload+reload cycle. + static std::string old_override_applied; + + common_params old_params = params_base; + common_params swapped_params = params_base; + + // Read the staged T3 statics and apply them to swapped_params. + if (llama_hydra_get_pending_n_gpu_layers() >= 0) { + swapped_params.n_gpu_layers = llama_hydra_get_pending_n_gpu_layers(); + } + // n_cpu_moe is informational only — the actual MoE expert + // offload is done via override_tensor (parsed below into + // tensor_buft_overrides). The standard common_params struct + // has no n_cpu_moe field; we just log the staged value for + // operator visibility. + if (llama_hydra_get_pending_n_cpu_moe() >= 0) { + SRV_INF("hydra: T3 rebuild: staged n_cpu_moe=%d (informational; expert routing via override_tensor)\n", + llama_hydra_get_pending_n_cpu_moe()); + } + const char * path = llama_hydra_get_pending_model_path(); + if (path && *path) { + swapped_params.model.path = path; + } + const char * mode = llama_hydra_get_pending_split_mode(); + if (mode && *mode) { + std::string m(mode); + if (m == "none") swapped_params.split_mode = LLAMA_SPLIT_MODE_NONE; + else if (m == "layer") swapped_params.split_mode = LLAMA_SPLIT_MODE_LAYER; + else if (m == "row") swapped_params.split_mode = LLAMA_SPLIT_MODE_ROW; + else SRV_WRN("hydra: T3 split_mode='%s' unknown; keeping current\n", m.c_str()); + } + const size_t n_split = llama_hydra_get_pending_tensor_split_count(); + if (n_split > 0) { + const float * split = llama_hydra_get_pending_tensor_split(); + // common_params::tensor_split is a fixed-size array. + const size_t cap = sizeof(swapped_params.tensor_split) / + sizeof(swapped_params.tensor_split[0]); + const size_t n = n_split < cap ? n_split : cap; + for (size_t i = 0; i < n; i++) { + swapped_params.tensor_split[i] = split[i]; + } + // Zero the rest so the engine doesn't see stale values. + for (size_t i = n; i < cap; i++) { + swapped_params.tensor_split[i] = 0.0f; + } + } + const char * override = llama_hydra_get_pending_override_tensor(); + if (override && *override) { + // Wire-shape: comma-separated "pattern=buft" pairs (e.g. + // "blk.*.ffn_*_exps.weight=CPU"). The C++ side stores + // these as a vector. + // Buft names are looked up + // via ggml_backend_dev_buffer_type() + ggml_backend_buft_name() + // (mirrors common/arg.cpp:parse_tensor_buffer_overrides). + ggml_backend_load_all(); + std::map buft_list; + for (size_t i = 0; i < ggml_backend_dev_count(); ++i) { + auto * dev = ggml_backend_dev_get(i); + auto * buft = ggml_backend_dev_buffer_type(dev); + if (buft) { + buft_list[std::string(ggml_backend_buft_name(buft))] = buft; + } + } + // CPU is the common case (MoE expert routing) — also lookup + // explicitly since some backends may not register the CPU buft. + buft_list["CPU"] = ggml_backend_cpu_buffer_type(); + + // Keep pattern strings alive for the lifetime of the + // process — entry.pattern is a const char* that must not + // dangle. Matches the safe pattern in common/arg.cpp. + static std::list buft_override_patterns; + + std::vector staged; + + const std::string ovr(override); + size_t start = 0; + while (start < ovr.size()) { + size_t comma = ovr.find(',', start); + std::string part = ovr.substr(start, comma == std::string::npos ? std::string::npos : comma - start); + size_t eq = part.find('='); + if (eq != std::string::npos) { + std::string pattern = part.substr(0, eq); + std::string buft_name = part.substr(eq + 1); + auto it = buft_list.find(buft_name); + if (it != buft_list.end()) { + buft_override_patterns.push_back(pattern); + llama_model_tensor_buft_override entry; + entry.pattern = buft_override_patterns.back().c_str(); + entry.buft = it->second; + staged.push_back(entry); + } else { + SRV_WRN("%s", "hydra: T3 rebuild: override_tensor buft name not in registered list; skipping pattern\n"); + } + } + if (comma == std::string::npos) break; + start = comma + 1; + } + + // Install the staged patterns *in place of* the base ones instead of + // appending to them. + // + // common_params_parse_ex() (common/arg.cpp) unconditionally pads this + // vector out to llama_max_tensor_buft_overrides() entries of + // {nullptr, nullptr}, so by the time we get here the real CLI overrides + // sit at the head and the rest is terminator padding. push_back() would + // land *behind* that padding, which breaks twice over: + // 1. common_model_params_to_llama() asserts that back().pattern is + // nullptr, so the engine aborts before the model loads; + // 2. even without that assert, llama_model_loader stops scanning at + // the first nullptr pattern, so appended entries are never read — + // the override would be silently dropped and the MoE experts would + // land on the GPU. + // Replacing also matches the sibling fields handled above: model.path, + // split_mode, n_gpu_layers and tensor_split are all overwritten by the + // staged T3 config rather than merged into it. + const size_t ntbo = llama_max_tensor_buft_overrides(); + if (staged.empty()) { + // Nothing resolved (every buft name was unknown). Wiping the base + // overrides here would silently change how the model is placed, so + // keep them and make the no-op explicit. + SRV_WRN("%s", "hydra: T3 rebuild: staged override_tensor resolved to no usable patterns; keeping base overrides\n"); + } else { + if (staged.size() + 1 > ntbo) { + SRV_WRN("hydra: T3 rebuild: %zu override_tensor patterns exceed the %zu-entry limit; keeping the first %zu\n", + staged.size(), ntbo, ntbo - 1); + staged.resize(ntbo - 1); + } + // assign() re-establishes the full terminator padding, so everything + // from staged.size() onward is {nullptr, nullptr}. + swapped_params.tensor_buft_overrides.assign(ntbo, llama_model_tensor_buft_override{ nullptr, nullptr }); + for (size_t i = 0; i < staged.size(); ++i) { + swapped_params.tensor_buft_overrides[i] = staged[i]; + } + } + } + + // Early-exit: if the model and all T3-relevant params are + // identical to what is already loaded, skip the expensive + // unload+reload cycle. Without this, every COMPLETION + // request that carries hydra_config triggers a full model + // swap even when nothing changed (the coordinator sends the + // same config on every decode request). + if (!is_first_load) { + const char * cur_override = llama_hydra_get_pending_override_tensor(); + bool params_unchanged = + swapped_params.model.path == old_params.model.path && + swapped_params.n_gpu_layers == old_params.n_gpu_layers && + swapped_params.split_mode == old_params.split_mode && + ((cur_override == nullptr && old_override_applied.empty()) || + (cur_override && old_override_applied == cur_override)); + if (params_unchanged) { + // T3 overrides (override_tensor, split_mode) were staged by + // the COMPLETION hydra_config path. But the model reload is + // being skipped. Clear the staged override so the next decode + // uses the current tensor placement (not the staged override). + llama_hydra_set_override_tensor(ctx_tgt, nullptr); + SRV_INF("%s", "hydra: T3 rebuild: model and params unchanged — skipping reload, cleared staged overrides\n"); + return true; + } + } + + // COMBINED-mode teardown BEFORE the model reload — see + // hydra_teardown_combined_before_reload() above. + const bool was_combined = hydra_combined_head_attached || hydra_combined_static; + if (!is_first_load && was_combined) { + hydra_teardown_combined_before_reload(); + } + + // Register any new RPC peer devices before load_model() so the + // peer's device exists in the global ggml backend registry when + // common_init_from_params() tries to place tensors per + // tensor_split/split_mode. Only genuinely new endpoints are + // registered (hydra_register_rpc_servers tracks already-registered + // endpoints to avoid unsafe repeated registration). + if (!g_pending_rpc_servers.empty()) { + json rpc_arr = json::array(); + for (const auto & s : g_pending_rpc_servers) { + rpc_arr.push_back(s); + } + hydra_register_rpc_servers(rpc_arr); + g_pending_rpc_servers.clear(); + } + + // Full model reload. load_model() handles the unload of the + // current model, the load of the new model, the new context + // creation, the MTP/draft paths, and the slot rebuild. + // NOTE: load_model() does `params_base = params` internally + // (line 844), so after a successful load params_base reflects + // swapped_params — no explicit reassignment needed by us. + // + // #507: Skip the fit_params probe during T3 rebuild. The probe + // does a full model-structure load with no_alloc=true to measure + // GPU memory — expensive (~45-90s) and unnecessary here because: + // (a) we just freed VRAM by destroying the old model, (b) the new + // model's requirements are known (same or smaller), (c) a controlled + // inference server has predictable VRAM. Disabling saves ~1 min. + swapped_params.fit_params = false; + if (!load_model(swapped_params)) { + if (is_first_load) { + SRV_WRN("%s", "hydra: T3 first load failed — engine stays empty\n"); + return false; + } + SRV_ERR("hydra: T3 reload to '%s' failed (load_model returned false); " + "rolling back to old model\n", + swapped_params.model.path.c_str()); + if (!load_model(old_params)) { + SRV_ERR("%s", "hydra: T3 rollback also failed — engine in unrecoverable state\n"); + GGML_ABORT("hydra: T3 rollback failed (cannot reload old model). " + "Engine exiting to prevent serving with corrupted state."); + } + SRV_INF("hydra: T3 rollback succeeded — restored old model '%s'\n", + old_params.model.path.c_str()); + return false; + } + + // Record the override_tensor that was just applied so the + // next call can skip the reload if nothing changed. + { + const char * cur = llama_hydra_get_pending_override_tensor(); + old_override_applied = cur ? cur : ""; + } + + // T3 reload confirmed. Log model identity for traceability. + SRV_INF("hydra: T3 reload confirmed model_alias='%s' tokenizer='%s' model_name='%s' quant='%s' caps=0x%x model_path='%s'\n", + swapped_params.model_alias.empty() ? "?" : swapped_params.model_alias.begin()->c_str(), + model_tgt ? llama_model_get_tokenizer_model(model_tgt) : "", + model_tgt ? llama_model_get_display_name(model_tgt) : "", + model_tgt ? llama_model_get_quant_label(model_tgt) : "", + model_tgt ? llama_model_get_capabilities_bitfield(model_tgt) : 0, + swapped_params.model.path.c_str()); + + // COMBINED-mode reattach AFTER the model reload — see + // hydra_reattach_combined_after_reload() above. + if (was_combined) { + hydra_reattach_combined_after_reload(); + } + + SRV_INF("hydra: T3 rebuild applied (model='%s', split_mode=%d, n_gpu_layers=%d, slots=%zu)\n", + params_base.model.path.c_str(), (int) params_base.split_mode, + params_base.n_gpu_layers, slots.size()); + return true; + } + + void server_context_impl::hydra_repad_tensor_buft_overrides(common_params & p, const char * ctx_label) { + const size_t ntbo = llama_max_tensor_buft_overrides(); + if (p.tensor_buft_overrides.size() + 1 > ntbo) { + SRV_WRN("hydra: %s: %zu tensor_buft_overrides exceed the %zu-entry limit; keeping the first %zu\n", + ctx_label, p.tensor_buft_overrides.size(), ntbo, ntbo - 1); + p.tensor_buft_overrides.resize(ntbo - 1); + } + p.tensor_buft_overrides.resize(ntbo, llama_model_tensor_buft_override{ nullptr, nullptr }); + } + + void server_context_impl::hydra_reattach_combined_after_reload() { + SRV_INF("%s", "hydra: re-attaching COMBINED on new model\n"); + if (hydra_combined_static) { + llama_hydra_set_expert_mode(ctx_tgt, 1); + } else if (!hydra_peer.empty() && !hydra_combined_pattern.empty()) { + if (llama_hydra_peer_reachable(hydra_peer.c_str())) { + ggml_backend_reg_t rpc_reg = ggml_backend_reg_by_name("RPC"); + if (rpc_reg) { + using add_server_fn_t = ggml_backend_reg_t (*)(const char *); + auto add_server_fn = (add_server_fn_t) ggml_backend_reg_get_proc_address(rpc_reg, "ggml_backend_rpc_add_server"); + ggml_backend_reg_t peer_reg = add_server_fn ? add_server_fn(hydra_peer.c_str()) : nullptr; + ggml_backend_dev_t peer_dev = (peer_reg && ggml_backend_reg_dev_count(peer_reg) > 0) ? ggml_backend_reg_dev_get(peer_reg, 0) : nullptr; + if (peer_dev) { + int32_t n_bound = llama_hydra_rebind_combined_experts( + ctx_tgt, hydra_peer.c_str(), peer_dev, hydra_combined_pattern.c_str()); + if (n_bound > 0) { + hydra_combined_head_attached = true; + llama_hydra_set_expert_mode(ctx_tgt, 1); + SRV_INF("hydra: COMBINED re-attached on peer %s (%d layers bound)\n", + hydra_peer.c_str(), n_bound); + } else { + SRV_WRN("hydra: rebind returned %d; staying solo\n", n_bound); + } + } else { + SRV_WRN("hydra: peer %s has no device; staying solo\n", hydra_peer.c_str()); + } + } else { + SRV_WRN("%s\n", "hydra: RPC backend not available; staying solo"); + } + } else { + SRV_WRN("hydra: peer %s unreachable; staying solo\n", hydra_peer.c_str()); + } + } + } + + void server_context_impl::hydra_teardown_combined_before_reload() { + SRV_INF("hydra: tearing down COMBINED before model reload (was head_attached=%d, static=%d)\n", + (int) hydra_combined_head_attached, (int) hydra_combined_static); + llama_hydra_set_expert_mode(ctx_tgt, 0); + if (!hydra_current_peer.empty()) { + ctx_tgt->hydra_remove_combined_rpc_backend(hydra_current_peer.c_str()); + } + llama_hydra_clear_combined_bindings(ctx_tgt, hydra_peer.c_str()); + hydra_combined_head_attached = false; + } + + void server_context_impl::hydra_register_rpc_servers(const json & servers_arr) { + static std::set registered; + + if (!servers_arr.is_array() || servers_arr.empty()) { + return; + } + + ggml_backend_load_all(); + ggml_backend_reg_t rpc_reg = ggml_backend_reg_by_name("RPC"); + if (!rpc_reg) { + SRV_WRN("%s", "hydra: rpc_servers: RPC backend not available\n"); + return; + } + + typedef ggml_backend_reg_t (*ggml_backend_rpc_add_server_t)(const char * endpoint); + auto add_server_fn = (ggml_backend_rpc_add_server_t) + ggml_backend_reg_get_proc_address(rpc_reg, "ggml_backend_rpc_add_server"); + if (!add_server_fn) { + SRV_WRN("%s", "hydra: rpc_servers: ggml_backend_rpc_add_server not found\n"); + return; + } + + for (const auto & v : servers_arr) { + if (!v.is_string()) continue; + const std::string endpoint = v.get(); + if (endpoint.empty()) continue; + if (registered.count(endpoint)) { + SRV_DBG("hydra: rpc_servers: endpoint '%s' already registered, skipping\n", + endpoint.c_str()); + continue; + } + ggml_backend_reg_t reg = add_server_fn(endpoint.c_str()); + if (reg) { + ggml_backend_register(reg); + registered.insert(endpoint); + SRV_INF("hydra: rpc_servers: registered endpoint '%s'\n", endpoint.c_str()); + } else { + SRV_WRN("hydra: rpc_servers: failed to register endpoint '%s'\n", + endpoint.c_str()); + } + } + } + + bool server_context_impl::apply_t2_rebuild(const std::string & pending_json) { + if (!ctx_tgt || !model_tgt) return false; + + json cfg; + try { + cfg = json::parse(pending_json); + } catch (const std::exception & e) { + SRV_WRN("hydra: T2 apply: invalid JSON in pending_config: %s\n", e.what()); + return false; + } + + // Snapshot the old params for rollback. params_base is the + // canonical "what's in effect" state; restoring it plus a + // recreate-cycle is the rollback path. + common_params old_params = params_base; + + // Update params_base with the T2 keys. Each is optional; + // absence means "leave unchanged". + if (cfg.contains("n_ctx") && cfg["n_ctx"].is_number_integer()) { + const int32_t n_ctx = cfg["n_ctx"].get(); + // Clamp to the model's training ctx. The wire spec does + // not require a reject-on-too-large (the engine's own + // check below does that); we clamp and report. + const int32_t max_ctx = (int32_t) llama_model_n_ctx_train(model_tgt); + if (n_ctx > max_ctx) { + SRV_WRN("hydra: T2 n_ctx=%d exceeds model_n_ctx_train=%d; clamping\n", + n_ctx, max_ctx); + params_base.n_ctx = max_ctx; + } else { + params_base.n_ctx = n_ctx; + } + } + if (cfg.contains("cache_type_k") && cfg["cache_type_k"].is_string()) { + const std::string & s = cfg["cache_type_k"].get_ref(); + ggml_type t = hydra_parse_cache_type(s); + if (t == GGML_TYPE_COUNT) { + SRV_WRN("hydra: T2 cache_type_k='%s' unparseable; ignoring\n", s.c_str()); + } else { + params_base.cache_type_k = t; + } + } + if (cfg.contains("cache_type_v") && cfg["cache_type_v"].is_string()) { + const std::string & s = cfg["cache_type_v"].get_ref(); + ggml_type t = hydra_parse_cache_type(s); + if (t == GGML_TYPE_COUNT) { + SRV_WRN("hydra: T2 cache_type_v='%s' unparseable; ignoring\n", s.c_str()); + } else { + params_base.cache_type_v = t; + } + } + if (cfg.contains("rope_freq_base") && cfg["rope_freq_base"].is_number()) { + params_base.rope_freq_base = cfg["rope_freq_base"].get(); + } + if (cfg.contains("rope_freq_scale") && cfg["rope_freq_scale"].is_number()) { + params_base.rope_freq_scale = cfg["rope_freq_scale"].get(); + } + if (cfg.contains("yarn_ext_factor") && cfg["yarn_ext_factor"].is_number()) { + params_base.yarn_ext_factor = cfg["yarn_ext_factor"].get(); + } + if (cfg.contains("yarn_attn_factor") && cfg["yarn_attn_factor"].is_number()) { + params_base.yarn_attn_factor = cfg["yarn_attn_factor"].get(); + } + if (cfg.contains("yarn_beta_fast") && cfg["yarn_beta_fast"].is_number()) { + params_base.yarn_beta_fast = cfg["yarn_beta_fast"].get(); + } + if (cfg.contains("yarn_beta_slow") && cfg["yarn_beta_slow"].is_number()) { + params_base.yarn_beta_slow = cfg["yarn_beta_slow"].get(); + } + if (cfg.contains("yarn_orig_ctx") && cfg["yarn_orig_ctx"].is_number_integer()) { + params_base.yarn_orig_ctx = cfg["yarn_orig_ctx"].get(); + } + + // Free the live context. KV cache is destroyed; this is the + // T2 cost. The model is kept (T2 is context-only). + llama_free(ctx_tgt); + if (ctx_dft) { + llama_free(ctx_dft.get()); + ctx_dft.reset(); + } + + // Build new cparams from the updated params_base. This is + // the same call site load_model() uses internally. + auto cparams = common_context_params_to_llama(params_base); + + // Recreate the context with the new cparams. + ctx_tgt = llama_new_context_with_model(model_tgt, cparams); + if (!ctx_tgt) { + // Rollback: rebuild with the old params_base. The old + // params must work (we just freed and recreated the + // context with them). If they don't, the engine is in + // a bad state — abort. + SRV_WRN("hydra: T2 rebuild failed with n_ctx=%d cache_type=%d/%d; " + "rolling back to old params\n", + params_base.n_ctx, (int) params_base.cache_type_k, + (int) params_base.cache_type_v); + params_base = old_params; + auto cparams_old = common_context_params_to_llama(params_base); + ctx_tgt = llama_new_context_with_model(model_tgt, cparams_old); + if (!ctx_tgt) { + GGML_ABORT("hydra: T2 rollback failed (cannot rebuild context with old params). " + "Engine exiting to prevent serving with corrupted state."); + } + return false; + } + + // Re-init per-slot samplers. The old samplers were bound to + // the now-freed context; common_sampler_init() on the new + // model picks up the (possibly changed) sampling config. + for (auto & slot : slots) { + slot.smpl.reset(common_sampler_init(model_tgt, params_base.sampling)); + } + + n_ctx = llama_n_ctx(ctx_tgt); + SRV_INF("hydra: T2 rebuild applied (n_ctx=%d, cache=%d/%d, slots=%zu)\n", + n_ctx, (int) params_base.cache_type_k, + (int) params_base.cache_type_v, slots.size()); + return true; + } + + ggml_type server_context_impl::hydra_parse_cache_type(const std::string & s) { + if (s.empty()) return GGML_TYPE_COUNT; + for (int i = 0; i < GGML_TYPE_COUNT; i++) { + ggml_type t = (ggml_type) i; + if (strcmp(ggml_type_name(t), s.c_str()) == 0) return t; + } + return GGML_TYPE_COUNT; + } + + bool server_context_impl::apply_pending_hydra_config() { + const bool is_first_load = !ctx_tgt; + if (is_first_load) { + if (!first_load_pending) { + return false; + } + // Don't check hydra_has_pending_config — ctx_tgt doesn't exist yet. + // The T3 statics were staged by hydra_apply_t3_mutators() in the + // CONFIGURE handler. Set a default tier for the rebuild path. + } else if (!ctx_tgt->hydra_has_pending_config()) { + return false; + } + + // 1. Drain timeout — skipped for first load (no ctx_tgt timestamp). + std::string tier; + std::string pending_json; + + if (is_first_load) { + tier = "T3"; + // pending_json stays empty — T3 statics are staged in global + // overrides, not in pending_config (ctx_tgt doesn't exist yet). + } else { + constexpr time_t k_drain_timeout_default = 300; + time_t now = std::time(nullptr); + time_t elapsed = now - ctx_tgt->hydra_get_pending_config_set_at(); + int env_timeout = 0; + if (const char * e = getenv("HYDRA_COORD_PROFILE_SWITCH_DRAIN_TIMEOUT")) { + env_timeout = atoi(e); + } + time_t drain_timeout = env_timeout > 0 ? env_timeout : k_drain_timeout_default; + if (elapsed > drain_timeout) { + SRV_WRN("hydra: pending config drain timeout (elapsed=%lld, limit=%lld) — discarding, " + "tier='%s' payload_size=%zu\n", + (long long) elapsed, (long long) drain_timeout, + ctx_tgt->hydra_get_pending_config_tier().c_str(), + ctx_tgt->hydra_get_pending_config().size()); + ctx_tgt->hydra_clear_pending_config(); + llama_hydra_clear_pending_t3(); + return false; + } + + tier = ctx_tgt->hydra_get_pending_config_tier(); + pending_json = ctx_tgt->hydra_get_pending_config(); + SRV_INF("hydra: applying pending config (tier='%s', age=%llds, payload_size=%zu)\n", + tier.c_str(), (long long) elapsed, pending_json.size()); + } + + bool ok = true; + + // 2. T2 work: free + rebuild context with the new cparams. + // Skipped when tier is T3 (T3's load_model() handles both). + if (tier == "T2") { + if (!apply_t2_rebuild(pending_json)) { + SRV_ERR("%s", "hydra: T2 rebuild failed; engine continues with old context\n"); + ok = false; + } + } + + // 3. T3 work: full model reload with the staged T3 statics. + // load_model() handles the unload+reload cycle. COMBINED-mode + // expert bindings are torn down before the reload and re- + // attached after, in the same pattern as SET_EXPERT_MODE. + if (tier == "T3") { + if (!apply_t3_rebuild()) { + SRV_ERR("%s", "hydra: T3 rebuild failed; engine continues with old model\n"); + ok = false; + } else { + // P1-6: T3 model changed — the cached server_context_meta + // (model_path, split_mode, tensor_split, chat_params, …) + // is now stale. Refresh it on the task-queue thread + // (safe — runs during the drain window when no slots are + // processing and no new requests are being dispatched). + if (routes_ptr) { + routes_ptr->refresh_meta(); + } + // P0-1 (#49): after deferred first-load, apply staged capabilities + // so ENGINE_INFO(0x41) and COMBINED-mode logic work correctly. + if (is_first_load) { + hydra_rpc_backend_active = bootstrap_rpc_active; + hydra_peer = bootstrap_peer; + hydra_peer_reachable = bootstrap_peer_reachable; + hydra_combined_pattern = bootstrap_pattern; + hydra_split_mode = bootstrap_split_mode; + if (bootstrap_combined_static) { + hydra_combined_static = true; + SRV_INF("%s", "P0-1: deferred first-load — combined_static mode activated\n"); + } + // Register local tensors and enable shared-backend compute + // lock so the model can serve inbound RPC requests. + if (model_tgt && ctx_tgt) { + llama_hydra_register_local_tensors_for_rpc(ctx_tgt); + llama_hydra_enable_shared_backend_compute_lock(); + } + // Update the RPC server's compute backends now that the + // model is loaded. The RPC server was started with empty + // backends (head-bootstrap mode); now populate it. + if (ctx_tgt) { + std::vector backends(8); + size_t n = llama_hydra_get_compute_backends(ctx_tgt, backends.data(), backends.size()); + if (n > backends.size()) { + backends.resize(n); + n = llama_hydra_get_compute_backends(ctx_tgt, backends.data(), backends.size()); + } + backends.resize(n); + hydra_rpc::update_backends(backends); + SRV_INF("P0-1: updated RPC backends to %zu compute device(s)\n", backends.size()); + } + SRV_INF("%s", "hydra-engine ready — model loaded via CONFIGURE T3\n"); + } + } + } + + // 4. Clear the staged state regardless of success. On failure + // the rollback in apply_t{2,3}_rebuild has restored the + // previous state; clearing the staged state prevents the + // next slot-free moment from re-attempting the same rebuild. + if (is_first_load) { + first_load_pending = false; + } else { + ctx_tgt->hydra_clear_pending_config(); + } + llama_hydra_clear_pending_t3(); + return ok; + } + + void server_context_impl::hydra_apply_t3_mutators(llama_context * ctx, const json & cfg, std::vector & deferred_keys) { + if (cfg.contains("n_gpu_layers") && cfg["n_gpu_layers"].is_number_integer()) { + llama_hydra_set_pending_n_gpu_layers(cfg["n_gpu_layers"].get()); + deferred_keys.push_back("n_gpu_layers"); + } + if (cfg.contains("n_cpu_moe") && cfg["n_cpu_moe"].is_number_integer()) { + llama_hydra_set_pending_n_cpu_moe(cfg["n_cpu_moe"].get()); + deferred_keys.push_back("n_cpu_moe"); + } + if (cfg.contains("override_tensor") && cfg["override_tensor"].is_string()) { + llama_hydra_set_override_tensor(ctx, cfg["override_tensor"].get().c_str()); + deferred_keys.push_back("override_tensor"); + } + if (cfg.contains("split_mode") && cfg["split_mode"].is_string()) { + std::vector split; + if (cfg.contains("tensor_split") && cfg["tensor_split"].is_array()) { + for (const auto & v : cfg["tensor_split"]) { + if (v.is_number()) split.push_back(v.get()); + } + } + llama_hydra_set_split_mode(ctx, cfg["split_mode"].get().c_str(), + split.empty() ? nullptr : split.data(), split.size()); + deferred_keys.push_back("split_mode"); + if (!split.empty()) deferred_keys.push_back("tensor_split"); + } else if (cfg.contains("tensor_split") && cfg["tensor_split"].is_array()) { + // tensor_split without split_mode is meaningless; record it as + // deferred and let the apply step surface the missing mode. + deferred_keys.push_back("tensor_split"); + } + if (cfg.contains("model") && cfg["model"].is_object() && + cfg["model"].contains("path") && cfg["model"]["path"].is_string()) { + llama_hydra_set_pending_model_path(cfg["model"]["path"].get().c_str()); + deferred_keys.push_back("model.path"); + } else if (cfg.contains("model") && cfg["model"].is_string()) { + // legacy shorthand: {"model": "/path/to.gguf"} + llama_hydra_set_pending_model_path(cfg["model"].get().c_str()); + deferred_keys.push_back("model"); + } + // hydra_config flat key: {"model_path": "/path/to.gguf"} + if (cfg.contains("model_path") && cfg["model_path"].is_string()) { + llama_hydra_set_pending_model_path(cfg["model_path"].get().c_str()); + deferred_keys.push_back("model_path"); + } + // hydra_config: {"rpc_servers": ["host1:port1", "host2:port2"]} + // Stored in a static for apply_t3_rebuild() to consume before + // load_model(). The actual ggml backend registration happens in + // hydra_register_rpc_servers() called from apply_t3_rebuild(). + if (cfg.contains("rpc_servers") && cfg["rpc_servers"].is_array()) { + g_pending_rpc_servers.clear(); + for (const auto & v : cfg["rpc_servers"]) { + if (v.is_string()) { + g_pending_rpc_servers.push_back(v.get()); + } + } + deferred_keys.push_back("rpc_servers"); + } + } + + bool server_context_impl::hydra_apply_t1_config(common_params & params, llama_context * ctx, const json & cfg, std::map & params_applied) { + // sampling.* — set on the common_params, which the next launch_slot + // will pick up when re-initializing the slot's common_sampler. + if (cfg.contains("sampling") && cfg["sampling"].is_object()) { + const json & s = cfg["sampling"]; + #define COPY_FLOAT(field) \ + if (s.contains(#field) && s[#field].is_number()) { \ + params.sampling.field = s[#field].get(); \ + params_applied["sampling." #field] = params.sampling.field; \ + } + #define COPY_INT(field) \ + if (s.contains(#field) && s[#field].is_number()) { \ + params.sampling.field = s[#field].get(); \ + params_applied["sampling." #field] = params.sampling.field; \ + } + COPY_FLOAT(temp) + COPY_FLOAT(top_p) + COPY_FLOAT(min_p) + COPY_FLOAT(penalty_repeat) + COPY_INT(top_k) + COPY_INT(seed) + #undef COPY_FLOAT + #undef COPY_INT + } + // n_predict + if (cfg.contains("n_predict") && cfg["n_predict"].is_number_integer()) { + params.n_predict = cfg["n_predict"].get(); + params_applied["n_predict"] = params.n_predict; + } else if (cfg.contains("n_predict") && !cfg["n_predict"].is_number_integer()) { + SRV_WRN("%s", "hydra: CONFIGURE n_predict must be an integer\n"); + return false; + } + // n_keep + if (cfg.contains("n_keep") && cfg["n_keep"].is_number_integer()) { + params.n_keep = cfg["n_keep"].get(); + params_applied["n_keep"] = params.n_keep; + } else if (cfg.contains("n_keep") && !cfg["n_keep"].is_number_integer()) { + SRV_WRN("%s", "hydra: CONFIGURE n_keep must be an integer\n"); + return false; + } + // seed (top-level — sets the sampler's seed via common_params::sampling). + // common_params itself has no top-level seed; common_params_sampling does. + if (cfg.contains("seed") && cfg["seed"].is_number_unsigned()) { + params.sampling.seed = cfg["seed"].get(); + params_applied["seed"] = params.sampling.seed; + } else if (cfg.contains("seed") && cfg["seed"].is_number_integer()) { + params.sampling.seed = (uint32_t) cfg["seed"].get(); + params_applied["seed"] = params.sampling.seed; + } else if (cfg.contains("seed") && !cfg["seed"].is_number()) { + SRV_WRN("%s", "hydra: CONFIGURE seed must be a number\n"); + return false; + } + // antiprompt — full replacement (matches the existing semantics + // of CLI --reverse-prompt) + if (cfg.contains("antiprompt") && cfg["antiprompt"].is_array()) { + std::vector new_antiprompt; + for (const auto & v : cfg["antiprompt"]) { + if (!v.is_string()) { + SRV_WRN("%s", "hydra: CONFIGURE antiprompt entries must be strings\n"); + return false; + } + new_antiprompt.push_back(v.get()); + } + params.antiprompt = std::move(new_antiprompt); + params_applied["antiprompt"] = params.antiprompt; + } else if (cfg.contains("antiprompt") && !cfg["antiprompt"].is_array()) { + SRV_WRN("%s", "hydra: CONFIGURE antiprompt must be an array of strings\n"); + return false; + } + // state_chunk_size — apply via the existing llama_hydra API (clamps + // and echoes the post-clamp value) + if (cfg.contains("state_chunk_size") && cfg["state_chunk_size"].is_number_unsigned()) { + const size_t bytes = cfg["state_chunk_size"].get(); + if (ctx) { + llama_hydra_set_state_chunk_size(ctx, bytes); + } + const size_t applied = ctx ? llama_hydra_get_state_chunk_size(ctx) : llama_hydra_clamp_state_chunk_size(bytes); + params_applied["state_chunk_size"] = (uint64_t) applied; + } else if (cfg.contains("state_chunk_size") && !cfg["state_chunk_size"].is_number_unsigned()) { + SRV_WRN("%s", "hydra: CONFIGURE state_chunk_size must be a non-negative integer\n"); + return false; + } + return true; + } + + int server_context_impl::hydra_classify_config_key(const std::string & key) { + // T1: sampling nested keys + if (key == "sampling.temp" || + key == "sampling.top_p" || + key == "sampling.top_k" || + key == "sampling.min_p" || + key == "sampling.penalty_repeat" || + key == "sampling.seed") { + return 1; + } + // T1: top-level fields + if (key == "n_predict" || + key == "n_keep" || + key == "seed" || + key == "antiprompt" || + key == "state_chunk_size") { + return 1; + } + // T2: context-level (KV cache / RoPE / ctx) + if (key == "n_ctx" || + key == "cache_type_k" || + key == "cache_type_v" || + key.rfind("rope_", 0) == 0) { + return 2; + } + // T3: model-level (offload / placement / model) + if (key == "n_gpu_layers" || + key == "n_cpu_moe" || + key == "override_tensor" || + key == "split_mode" || + key == "tensor_split" || + key == "model_path" || // hydra_config: absolute GGUF path + key == "rpc_servers" || // hydra_config: RPC peer endpoints to register + key == "model.path" || + key == "model") { // legacy alias for { "model": { "path": ... } } + return 3; + } + return 0; // unknown + } + + const char * server_context_impl::hydra_tier_label(int tier) { + switch (tier) { + case 1: return "T1"; + case 2: return "T2"; + case 3: return "T3"; + default: return "T1"; // 0 (no recognized keys) → degenerate T1 + } + } diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp index 35658bf4ac8b..b7022f6a719d 100644 --- a/tools/server/server-context.cpp +++ b/tools/server/server-context.cpp @@ -2746,872 +2746,6 @@ struct server_context_impl { cur.pos_max, cur.n_tokens, (float) cur.size() / 1024 / 1024); } - // Apply the T1 keys from `cfg` to `params` and to the live context (for - // state_chunk_size). Echoes each applied key + post-clamp value into - // `params_applied`. Returns true on success; false if a key's value is - // of the wrong type (which is reported back to the caller — the request - // is malformed and we don't want to apply a partial set). - bool hydra_apply_t1_config(common_params & params, llama_context * ctx, - const json & cfg, - std::map & params_applied) { - // sampling.* — set on the common_params, which the next launch_slot - // will pick up when re-initializing the slot's common_sampler. - if (cfg.contains("sampling") && cfg["sampling"].is_object()) { - const json & s = cfg["sampling"]; - #define COPY_FLOAT(field) \ - if (s.contains(#field) && s[#field].is_number()) { \ - params.sampling.field = s[#field].get(); \ - params_applied["sampling." #field] = params.sampling.field; \ - } - #define COPY_INT(field) \ - if (s.contains(#field) && s[#field].is_number()) { \ - params.sampling.field = s[#field].get(); \ - params_applied["sampling." #field] = params.sampling.field; \ - } - COPY_FLOAT(temp) - COPY_FLOAT(top_p) - COPY_FLOAT(min_p) - COPY_FLOAT(penalty_repeat) - COPY_INT(top_k) - COPY_INT(seed) - #undef COPY_FLOAT - #undef COPY_INT - } - // n_predict - if (cfg.contains("n_predict") && cfg["n_predict"].is_number_integer()) { - params.n_predict = cfg["n_predict"].get(); - params_applied["n_predict"] = params.n_predict; - } else if (cfg.contains("n_predict") && !cfg["n_predict"].is_number_integer()) { - SRV_WRN("%s", "hydra: CONFIGURE n_predict must be an integer\n"); - return false; - } - // n_keep - if (cfg.contains("n_keep") && cfg["n_keep"].is_number_integer()) { - params.n_keep = cfg["n_keep"].get(); - params_applied["n_keep"] = params.n_keep; - } else if (cfg.contains("n_keep") && !cfg["n_keep"].is_number_integer()) { - SRV_WRN("%s", "hydra: CONFIGURE n_keep must be an integer\n"); - return false; - } - // seed (top-level — sets the sampler's seed via common_params::sampling). - // common_params itself has no top-level seed; common_params_sampling does. - if (cfg.contains("seed") && cfg["seed"].is_number_unsigned()) { - params.sampling.seed = cfg["seed"].get(); - params_applied["seed"] = params.sampling.seed; - } else if (cfg.contains("seed") && cfg["seed"].is_number_integer()) { - params.sampling.seed = (uint32_t) cfg["seed"].get(); - params_applied["seed"] = params.sampling.seed; - } else if (cfg.contains("seed") && !cfg["seed"].is_number()) { - SRV_WRN("%s", "hydra: CONFIGURE seed must be a number\n"); - return false; - } - // antiprompt — full replacement (matches the existing semantics - // of CLI --reverse-prompt) - if (cfg.contains("antiprompt") && cfg["antiprompt"].is_array()) { - std::vector new_antiprompt; - for (const auto & v : cfg["antiprompt"]) { - if (!v.is_string()) { - SRV_WRN("%s", "hydra: CONFIGURE antiprompt entries must be strings\n"); - return false; - } - new_antiprompt.push_back(v.get()); - } - params.antiprompt = std::move(new_antiprompt); - params_applied["antiprompt"] = params.antiprompt; - } else if (cfg.contains("antiprompt") && !cfg["antiprompt"].is_array()) { - SRV_WRN("%s", "hydra: CONFIGURE antiprompt must be an array of strings\n"); - return false; - } - // state_chunk_size — apply via the existing llama_hydra API (clamps - // and echoes the post-clamp value) - if (cfg.contains("state_chunk_size") && cfg["state_chunk_size"].is_number_unsigned()) { - const size_t bytes = cfg["state_chunk_size"].get(); - if (ctx) { - llama_hydra_set_state_chunk_size(ctx, bytes); - } - const size_t applied = ctx ? llama_hydra_get_state_chunk_size(ctx) : llama_hydra_clamp_state_chunk_size(bytes); - params_applied["state_chunk_size"] = (uint64_t) applied; - } else if (cfg.contains("state_chunk_size") && !cfg["state_chunk_size"].is_number_unsigned()) { - SRV_WRN("%s", "hydra: CONFIGURE state_chunk_size must be a non-negative integer\n"); - return false; - } - return true; - } - - // Apply the T3 mutators immediately. The "staging" is: the statics in - // llama-hydra.cpp + the T3 keys in pending_config JSON. The actual - // model reload happens later, in the slot-free trigger. - void hydra_apply_t3_mutators(llama_context * ctx, const json & cfg, - std::vector & deferred_keys) { - if (cfg.contains("n_gpu_layers") && cfg["n_gpu_layers"].is_number_integer()) { - llama_hydra_set_pending_n_gpu_layers(cfg["n_gpu_layers"].get()); - deferred_keys.push_back("n_gpu_layers"); - } - if (cfg.contains("n_cpu_moe") && cfg["n_cpu_moe"].is_number_integer()) { - llama_hydra_set_pending_n_cpu_moe(cfg["n_cpu_moe"].get()); - deferred_keys.push_back("n_cpu_moe"); - } - if (cfg.contains("override_tensor") && cfg["override_tensor"].is_string()) { - llama_hydra_set_override_tensor(ctx, cfg["override_tensor"].get().c_str()); - deferred_keys.push_back("override_tensor"); - } - if (cfg.contains("split_mode") && cfg["split_mode"].is_string()) { - std::vector split; - if (cfg.contains("tensor_split") && cfg["tensor_split"].is_array()) { - for (const auto & v : cfg["tensor_split"]) { - if (v.is_number()) split.push_back(v.get()); - } - } - llama_hydra_set_split_mode(ctx, cfg["split_mode"].get().c_str(), - split.empty() ? nullptr : split.data(), split.size()); - deferred_keys.push_back("split_mode"); - if (!split.empty()) deferred_keys.push_back("tensor_split"); - } else if (cfg.contains("tensor_split") && cfg["tensor_split"].is_array()) { - // tensor_split without split_mode is meaningless; record it as - // deferred and let the apply step surface the missing mode. - deferred_keys.push_back("tensor_split"); - } - if (cfg.contains("model") && cfg["model"].is_object() && - cfg["model"].contains("path") && cfg["model"]["path"].is_string()) { - llama_hydra_set_pending_model_path(cfg["model"]["path"].get().c_str()); - deferred_keys.push_back("model.path"); - } else if (cfg.contains("model") && cfg["model"].is_string()) { - // legacy shorthand: {"model": "/path/to.gguf"} - llama_hydra_set_pending_model_path(cfg["model"].get().c_str()); - deferred_keys.push_back("model"); - } - // hydra_config flat key: {"model_path": "/path/to.gguf"} - if (cfg.contains("model_path") && cfg["model_path"].is_string()) { - llama_hydra_set_pending_model_path(cfg["model_path"].get().c_str()); - deferred_keys.push_back("model_path"); - } - // hydra_config: {"rpc_servers": ["host1:port1", "host2:port2"]} - // Stored in a static for apply_t3_rebuild() to consume before - // load_model(). The actual ggml backend registration happens in - // hydra_register_rpc_servers() called from apply_t3_rebuild(). - if (cfg.contains("rpc_servers") && cfg["rpc_servers"].is_array()) { - g_pending_rpc_servers.clear(); - for (const auto & v : cfg["rpc_servers"]) { - if (v.is_string()) { - g_pending_rpc_servers.push_back(v.get()); - } - } - deferred_keys.push_back("rpc_servers"); - } - } - - // Shared helper: classify config keys, apply T1 immediately, and either - // stage (sync=false) or synchronously apply (sync=true) T2/T3. - // - // sync=false (CONFIGURE path): T2/T3 are staged via hydra_set_pending_config() - // + hydra_apply_t3_mutators() for later application at the slot-free moment. - // - // sync=true (PREFILL / HTTP decode path): T2/T3 are applied immediately - // via apply_t2_rebuild() / apply_t3_rebuild(). The caller already owns - // the task-queue thread context, so synchronous application is safe. - // - // Returns a structured result so callers can build CONFIGURE responses - // or handle errors uniformly. - struct hydra_config_result { - int highest_tier = 0; - std::map params_applied; - std::vector deferred_keys; - json t2t3_subset = json::object(); - bool ok = true; - std::string error; - uint64_t state_chunk_size_applied = 0; - // hydra#470: generic (T4) keys that cannot take effect. Echoed to - // the Coordinator via the CONFIGURE response so nothing is silent. - std::vector unrecognized_keys; // no llama.cpp arg-table entry - std::vector rejected_keys; // startup-only / flag / two-value arg - }; - - hydra_config_result hydra_apply_config(const json & cfg, bool sync) { - hydra_config_result result; - - // 1. Classify every top-level key. T1 → apply now; T2/T3/T4 → - // defer (stage) or apply synchronously depending on `sync`. - // T4 (generic) keys are additionally validated against the - // llama.cpp arg table here so the CONFIGURE response can report - // unrecognized/rejected keys before the deferred apply runs. - // T2 + appliable-T4 keys are staged together into the reload - // config (g_pending_reload_config) so neither the context-reload - // path nor the model-reload path strands them. - json t1_subset = json::object(); - json reload_subset = json::object(); - for (auto it = cfg.begin(); it != cfg.end(); ++it) { - const std::string key = it.key(); - int tier = hydra_classify_config_key(key); - if (tier == 1) { - t1_subset[key] = it.value(); - } else if (tier == 2) { - result.t2t3_subset[key] = it.value(); - result.deferred_keys.push_back(key); - reload_subset[key] = it.value(); - } else if (tier == 3) { - result.t2t3_subset[key] = it.value(); - result.deferred_keys.push_back(key); - // T3 keys are staged via hydra_apply_t3_mutators() statics, - // not the reload-config JSON. - } else { - // T4: generic-arg pass-through. Classify against the arg - // table; only appliable keys are staged for the deferred - // slot-free moment. The rest are reported loudly. - const hydra_generic_key_status st = hydra_classify_generic_key(key); - if (st == hydra_generic_key_status::APPLIABLE) { - result.t2t3_subset[key] = it.value(); - result.deferred_keys.push_back(key); - reload_subset[key] = it.value(); - } else if (st == hydra_generic_key_status::DENIED) { - SRV_WRN("hydra: CONFIGURE key '%s' cannot change at reload (startup-only/flag arg) — rejected\n", - key.c_str()); - result.rejected_keys.push_back(key); - } else { - SRV_WRN("hydra: CONFIGURE key '%s' is not a known llama.cpp argument — unrecognized, value ignored\n", - key.c_str()); - result.unrecognized_keys.push_back(key); - } - } - if (tier > result.highest_tier) result.highest_tier = tier; - } - // Stage the reload config (T2 + appliable-T4 keys) for the deferred - // slot-free moment. Unconditional overwrite = absolute state: a - // superseding CONFIGURE without T2/T4 keys must clear whatever an - // earlier CONFIGURE staged, or the stale keys would be applied on - // the next unrelated reload. - g_pending_reload_config = reload_subset.dump(); - // The "sampling" object may contain unlisted nested keys - // (e.g. penalty_last_n, mirostat) — route the whole object - // through T1 when present. - if (cfg.contains("sampling") && cfg["sampling"].is_object()) { - t1_subset["sampling"] = cfg["sampling"]; - if (result.highest_tier < 1) result.highest_tier = 1; - } - // model.path is nested — the legacy {"model": {...}} form - // is recognized by hydra_classify_config_key returning 3 - // for the bare "model" key. If the bare "model" is set - // and is an object with a "path", route it as T3. - if (cfg.contains("model")) { - if (cfg["model"].is_object()) { - result.t2t3_subset["model"] = cfg["model"]; - if (std::find(result.deferred_keys.begin(), result.deferred_keys.end(), "model") - == result.deferred_keys.end()) { - result.deferred_keys.push_back("model"); - } - if (result.highest_tier < 3) result.highest_tier = 3; - } else if (cfg["model"].is_string()) { - result.t2t3_subset["model"] = cfg["model"]; - if (std::find(result.deferred_keys.begin(), result.deferred_keys.end(), "model") - == result.deferred_keys.end()) { - result.deferred_keys.push_back("model"); - } - if (result.highest_tier < 3) result.highest_tier = 3; - } - } - - if (result.highest_tier == 0) { - // No recognized keys — caller decides whether to treat as - // a no-op or surface an error. - return result; - } - - // 2. Apply T1 keys in-place. - if (!t1_subset.empty()) { - if (!hydra_apply_t1_config(params_base, ctx_tgt, t1_subset, result.params_applied)) { - result.ok = false; - result.error = "T1 key has wrong type (see log)"; - return result; - } - // Capture state_chunk_size for callers that need it (CONFIGURE). - auto it = result.params_applied.find("state_chunk_size"); - if (it != result.params_applied.end() && it->second.is_number_unsigned()) { - result.state_chunk_size_applied = it->second.get(); - } - } - - // 3. T2/T3/T4 handling — diverges based on sync flag. - if (!result.t2t3_subset.empty()) { - if (sync) { - // Synchronous mode (PREFILL / HTTP decode): apply now - // on the task-queue thread. The caller owns this thread - // context so blocking is safe. - if (result.highest_tier >= 3) { - // T3 (model reload) also covers a T4-only config: - // load_model() recreates the context and the - // speculative/draft state, so every generic key - // takes effect. hydra_apply_t3_mutators() is a no-op - // when no T3 keys are staged. - hydra_apply_t3_mutators(ctx_tgt, result.t2t3_subset, result.deferred_keys); - // #470: force rebuild if a peer reconnection was detected - // during a prior graph_compute — the peer's buffers are gone - // even though model/params haven't changed. - const bool reconn_force = (ctx_tgt && ctx_tgt->peer_reconnection_pending); - if (reconn_force) { - ctx_tgt->peer_reconnection_pending = false; - SRV_WRN("%s", "hydra: PREFILL handler: peer reconnection pending — forcing T3 rebuild\n"); - } - if (!apply_t3_rebuild(reconn_force)) { - result.ok = false; - result.error = "T3 rebuild failed"; - return result; - } - } else if (result.highest_tier == 2) { - if (!apply_t2_rebuild(result.t2t3_subset.dump())) { - result.ok = false; - result.error = "T2 rebuild failed"; - return result; - } - } - // Clear T3 staged statics — they were consumed by the - // sync apply and must not leak into a later deferred path. - llama_hydra_clear_pending_t3(); - } else { - // Stage mode (CONFIGURE): record the mutators and store - // pending_config for application at the next slot-free moment. - if (result.highest_tier >= 3) { - hydra_apply_t3_mutators(ctx_tgt, result.t2t3_subset, result.deferred_keys); - } - if (ctx_tgt) { - ctx_tgt->hydra_set_pending_config( - result.t2t3_subset.dump(), hydra_tier_label(result.highest_tier)); - } else { - // First load: ctx_tgt is null, so apply_pending_hydra_config() - // and update_slots() can't trigger. Set the flag so the - // task-queue thread runs apply_t3_rebuild() at the next - // slot-free moment. - first_load_pending = true; - SRV_INF("%s", "hydra: config staged for first load (no context yet)\n"); - } - } - } - - return result; - } - - void process_single_task(server_task && task) { - // epic #610 WS1: in seam mode the extension may claim Hydra tasks. - // WS1 impl is a no-op (returns false), so this is a pure A/B switch — - // both modes run the inline dispatch below. - if (hydra_ext_active && hydra_ext && hydra_ext->handle_task(*this, task)) { - return; - } - switch (task.type) { - case SERVER_TASK_TYPE_COMPLETION: - case SERVER_TASK_TYPE_INFILL: - case SERVER_TASK_TYPE_EMBEDDING: - case SERVER_TASK_TYPE_RERANK: - { - // special case: if input is provided via CLI, tokenize it first - // otherwise, no need to tokenize as it's already done inside the HTTP thread - if (task.cli) { - if (!tokenize_cli_input(task)) { - break; - } - } - - // Hydra config from HTTP decode path: apply synchronously - // on the task-queue thread before any slot scheduling or - // generation work. This is safe because we own this thread; - // the previous attempt applied on the httplib worker thread - // and raced the main queue (reverted in ebbbe1116). - if (!task.hydra_config_json.empty()) { - json hydra_cfg; - try { - hydra_cfg = json::parse(task.hydra_config_json); - } catch (const std::exception & e) { - SRV_WRN("hydra: COMPLETION hydra_config parse failed: %s\n", e.what()); - } - if (!hydra_cfg.is_null() && hydra_cfg.is_object()) { - SRV_INF("hydra: COMPLETION applying hydra_config (%zu keys)\n", - hydra_cfg.size()); - hydra_config_result cfg_result = hydra_apply_config(hydra_cfg, /*sync=*/true); - if (!cfg_result.ok) { - SRV_WRN("hydra: COMPLETION hydra_config apply failed: %s\n", - cfg_result.error.c_str()); - } - // After T3 rebuild, model/slots are reset. - // The slot lookup below will pick up the new state. - } - } - - const int id_slot = task.id_slot; - const int id_task = task.id; - - server_slot * slot = id_slot != -1 ? get_slot_by_id(id_slot) : get_available_slot(task); - - // - // slot scheduling logic - // - - if (slot == nullptr) { - // if no slot is available, we defer this task for processing later - SRV_DBG("no slot is available, defer task, id_task = %d\n", id_task); - queue_tasks.defer(std::move(task)); - break; - } - - if (slot->is_processing() || slot->hydra_transferring->load()) { - // if requested slot is unavailable, we defer this task for processing later - SRV_DBG("requested slot is unavailable, defer task, id_task = %d\n", id_task); - queue_tasks.defer(std::move(task)); - break; - } - - if (task.is_parent()) { - // try getting free slots for all child tasks - size_t n_child_tasks = task.child_tasks.size(); - std::vector child_slots = get_free_slots(n_child_tasks, slot->id); - if (child_slots.size() < n_child_tasks) { - SRV_DBG("not enough free slots for child tasks, n_free = %zu, n_children = %zu, defer task, id_task = %d\n", child_slots.size(), n_child_tasks, id_task); - queue_tasks.defer(std::move(task)); - break; - } - if (!launch_slots_with_parent_task(*slot, child_slots, std::move(task))) { - SRV_ERR("failed to launch slot with parent task, id_task = %d\n", id_task); - break; // drop the task - } - } else if (!launch_slot_with_task(*slot, std::move(task))) { - SRV_ERR("failed to launch slot with task, id_task = %d\n", id_task); - break; // drop the task - } - - if (params_base.cache_idle_slots) { - for (auto & s : slots) { - if (!s.is_processing() && !s.hydra_transferring->load()) { - slot_save_and_clear(s); - } - } - } - } break; - case SERVER_TASK_TYPE_CANCEL: - { - // release slot linked with the task id - for (auto & slot : slots) { - if (slot.task && slot.task->id == task.id_target) { - slot.release(); - break; - } - } - } break; - case SERVER_TASK_TYPE_CONTROL: - { - auto res = std::make_unique(); - res->id = task.id; - - server_slot * slot = get_slot_by_cmpl_id(task.params.control_cmpl_id); - if (slot == nullptr) { - res->success = false; - res->message = "no active completion for this id"; - queue_results.send(std::move(res)); - break; - } - - if (task.params.control_action == "reasoning_end") { - // the budget sampler only exists when reasoning control was armed - if (!slot->task->params.sampling.reasoning_control) { - res->success = false; - res->message = "reasoning control not enabled for this completion"; - queue_results.send(std::move(res)); - break; - } - // act on the live slot mid generation, never defer - common_sampler_reasoning_budget_force(slot->smpl.get()); - res->success = true; - } else { - res->success = false; - res->message = "unknown control action"; - } - - queue_results.send(std::move(res)); - } break; - case SERVER_TASK_TYPE_NEXT_RESPONSE: - { - // do nothing - } break; - case SERVER_TASK_TYPE_METRICS: - { - json slots_data = json::array(); - - int n_idle_slots = 0; - int n_processing_slots = 0; - - for (server_slot & slot : slots) { - json slot_data = slot.to_json(slots_debug == 0); - - if (slot.is_processing() || slot.hydra_transferring->load()) { - n_processing_slots++; - } else { - n_idle_slots++; - } - - slots_data.push_back(slot_data); - } - SRV_DBG("n_idle_slots = %d, n_processing_slots = %d\n", n_idle_slots, n_processing_slots); - - auto res = std::make_unique(); - res->id = task.id; - res->slots_data = std::move(slots_data); - res->n_idle_slots = n_idle_slots; - res->n_processing_slots = n_processing_slots; - res->n_tasks_deferred = queue_tasks.queue_tasks_deferred_size(); - res->t_start = metrics.t_start; - - res->n_prompt_tokens_processed_total = metrics.n_prompt_tokens_processed_total; - res->t_prompt_processing_total = metrics.t_prompt_processing_total; - res->n_tokens_predicted_total = metrics.n_tokens_predicted_total; - res->t_tokens_generation_total = metrics.t_tokens_generation_total; - - res->n_tokens_max = metrics.n_tokens_max; - - res->n_prompt_tokens_processed = metrics.n_prompt_tokens_processed; - res->t_prompt_processing = metrics.t_prompt_processing; - res->n_tokens_predicted = metrics.n_tokens_predicted; - res->t_tokens_generation = metrics.t_tokens_generation; - - res->n_decode_total = metrics.n_decode_total; - res->n_busy_slots_total = metrics.n_busy_slots_total; - - if (task.metrics_reset_bucket) { - metrics.reset_bucket(); - } - queue_results.send(std::move(res)); - } break; - case SERVER_TASK_TYPE_SLOT_SAVE: - { - if (!check_no_mtmd(task.id)) { - break; - } - - const int id_slot = task.slot_action.id_slot; - server_slot * slot = get_slot_by_id(id_slot); - if (slot == nullptr) { - send_error(task, "Invalid slot ID", ERROR_TYPE_INVALID_REQUEST); - break; - } - if (slot->is_processing()) { - // if requested slot is unavailable, we defer this task for processing later - SRV_DBG("requested slot is unavailable, defer task, id_task = %d\n", task.id); - queue_tasks.defer(std::move(task)); - break; - } - - const size_t token_count = slot->prompt.tokens.size(); - const int64_t t_start = ggml_time_us(); - - std::string filename = task.slot_action.filename; - std::string filepath = task.slot_action.filepath; - - const llama_tokens & tokens = slot->prompt.tokens.get_tokens(); - const size_t nwrite = llama_state_seq_save_file(ctx_tgt, filepath.c_str(), slot->id, tokens.data(), token_count); - - const int64_t t_end = ggml_time_us(); - const double t_save_ms = (t_end - t_start) / 1000.0; - - auto res = std::make_unique(); - res->id = task.id; - res->id_slot = id_slot; - res->filename = filename; - res->is_save = true; - res->n_tokens = token_count; - res->n_bytes = nwrite; - res->t_ms = t_save_ms; - queue_results.send(std::move(res)); - } break; - case SERVER_TASK_TYPE_SLOT_RESTORE: - { - if (!check_no_mtmd(task.id)) break; - const int id_slot = task.slot_action.id_slot; - server_slot * slot = get_slot_by_id(id_slot); - if (slot == nullptr) { - send_error(task, "Invalid slot ID", ERROR_TYPE_INVALID_REQUEST); - break; - } - if (slot->is_processing()) { - // if requested slot is unavailable, we defer this task for processing later - SRV_DBG("requested slot is unavailable, defer task, id_task = %d\n", task.id); - queue_tasks.defer(std::move(task)); - break; - } - - const int64_t t_start = ggml_time_us(); - - std::string filename = task.slot_action.filename; - std::string filepath = task.slot_action.filepath; - - llama_tokens tokens; - tokens.resize(slot->n_ctx); - size_t token_count = 0; - size_t nread = llama_state_seq_load_file(ctx_tgt, filepath.c_str(), slot->id, tokens.data(), tokens.size(), &token_count); - if (nread == 0) { - slot->prompt.tokens.clear(); // KV may already been invalidated? - send_error(task, "Unable to restore slot, no available space in KV cache or invalid slot save file", ERROR_TYPE_INVALID_REQUEST); - break; - } - tokens.resize(token_count); - slot->prompt.tokens.clear(); - slot->prompt.tokens.insert(tokens); - - const int64_t t_end = ggml_time_us(); - const double t_restore_ms = (t_end - t_start) / 1000.0; - - auto res = std::make_unique(); - res->id = task.id; - res->id_slot = id_slot; - res->filename = filename; - res->is_save = false; - res->n_tokens = token_count; - res->n_bytes = nread; - res->t_ms = t_restore_ms; - queue_results.send(std::move(res)); - } break; - case SERVER_TASK_TYPE_SLOT_ERASE: - { - if (!check_no_mtmd(task.id)) { - break; - } - const int id_slot = task.slot_action.id_slot; - server_slot * slot = get_slot_by_id(id_slot); - if (slot == nullptr) { - send_error(task, "Invalid slot ID", ERROR_TYPE_INVALID_REQUEST); - break; - } - if (slot->is_processing() || slot->hydra_transferring->load()) { - // if requested slot is unavailable, we defer this task for processing later - SRV_DBG("requested slot is unavailable, defer task, id_task = %d\n", task.id); - queue_tasks.defer(std::move(task)); - break; - } - - // Erase token cache - const size_t n_erased = slot->prompt.tokens.size(); - - slot->prompt_clear(false); - - auto res = std::make_unique(); - res->id = task.id; - res->id_slot = id_slot; - res->n_erased = n_erased; - queue_results.send(std::move(res)); - } break; - case SERVER_TASK_TYPE_GET_LORA: - { - // TODO @ngxson : make lora_adapters a dedicated member of server_context - auto & loras = params_base.lora_adapters; - auto res = std::make_unique(); - res->id = task.id; - for (size_t i = 0; i < loras.size(); ++i) { - auto & lora = loras[i]; - std::string alora_invocation_string = ""; - const uint64_t n_alora_tokens = llama_adapter_get_alora_n_invocation_tokens(lora.ptr); - llama_tokens alora_invocation_tokens; - if (n_alora_tokens) { - const llama_token * alora_tokens = llama_adapter_get_alora_invocation_tokens(lora.ptr); - for (uint64_t j = 0; j < n_alora_tokens; ++j) { - alora_invocation_string += common_token_to_piece(vocab, alora_tokens[j]); - alora_invocation_tokens.push_back(alora_tokens[j]); - } - } - res->loras.push_back(server_task_result_get_lora::lora{ - lora, - alora_invocation_string, - alora_invocation_tokens, - }); - } - queue_results.send(std::move(res)); - } break; - case SERVER_TASK_TYPE_SET_LORA: - { - auto new_loras = construct_lora_list(task.set_lora); - // logging - for (size_t i = 0; i < new_loras.size(); ++i) { - SRV_INF("set lora adapter idx=%zu scale=%f\n", i, new_loras[i].scale); - } - // TODO @ngxson : make lora_adapters a dedicated member of server_context - params_base.lora_adapters = new_loras; - auto res = std::make_unique(); - res->id = task.id; - queue_results.send(std::move(res)); - } break; - - // epic #610 WS2: HYDRA task dispatch moved to hydra_process_task() - // (defined in hydra-server-context.cpp). In seam mode the extension - // claims these via handle_task(); in legacy mode this fall-through - // calls the same method. Both modes run identical code. - case SERVER_TASK_TYPE_HYDRA_STATE_GET: - case SERVER_TASK_TYPE_HYDRA_STATE_PUT: - case SERVER_TASK_TYPE_HYDRA_STATE_META: - case SERVER_TASK_TYPE_HYDRA_ENGINE_CONFIGURE: - case SERVER_TASK_TYPE_HYDRA_ENGINE_INFO: - case SERVER_TASK_TYPE_HYDRA_ENGINE_PREFILL: - case SERVER_TASK_TYPE_HYDRA_ENGINE_DECODE: - case SERVER_TASK_TYPE_HYDRA_DECODE_APPLY: - case SERVER_TASK_TYPE_HYDRA_ENGINE_SET_EXPERT_MODE: - case SERVER_TASK_TYPE_HYDRA_ENGINE_SWAP_QUANT: - case SERVER_TASK_TYPE_HYDRA_ENGINE_PIPELINE_ATTACH: - hydra_process_task(task); - break; - - } - } - - // Hydra #406 (Phase 2b follow-up): apply the staged T2/T3 CONFIGURE - // rebuild. Called from update_slots() when the slot-free moment - // arrives (all slots idle, no slot is hydra_transferring). - // - // The flow: - // 1. Check drain timeout (HYDRA_COORD_PROFILE_SWITCH_DRAIN_TIMEOUT, - // default 300s). On timeout, discard the staged config and - // return — the next INFO call surfaces the cleared state. - // 2. T2 work (free + rebuild context). Skipped when tier is T3 - // (T3's load_model() rebuilds the context as a side effect). - // 3. T3 work (full model reload). Uses the staged T3 statics - // (override_tensor / split_mode / tensor_split / n_gpu_layers - // / n_cpu_moe / model.path) and falls through to load_model() - // for the actual unload+reload cycle. COMBINED-mode bindings - // are torn down before the reload and re-attached after. - // 4. Clear the staged state (T3 statics + pending_config JSON). - // - // On any failure: rollback to the pre-apply params_base and rebuild - // from there. The exception path (GGML_ABORT) is reserved for the - // catastrophic case where the rollback itself fails — the engine - // would be unable to serve in any state and must exit. - bool apply_pending_hydra_config() { - const bool is_first_load = !ctx_tgt; - if (is_first_load) { - if (!first_load_pending) { - return false; - } - // Don't check hydra_has_pending_config — ctx_tgt doesn't exist yet. - // The T3 statics were staged by hydra_apply_t3_mutators() in the - // CONFIGURE handler. Set a default tier for the rebuild path. - } else if (!ctx_tgt->hydra_has_pending_config()) { - return false; - } - - // 1. Drain timeout — skipped for first load (no ctx_tgt timestamp). - std::string tier; - std::string pending_json; - - if (is_first_load) { - tier = "T3"; - // pending_json stays empty — T3 statics are staged in global - // overrides, not in pending_config (ctx_tgt doesn't exist yet). - } else { - constexpr time_t k_drain_timeout_default = 300; - time_t now = std::time(nullptr); - time_t elapsed = now - ctx_tgt->hydra_get_pending_config_set_at(); - int env_timeout = 0; - if (const char * e = getenv("HYDRA_COORD_PROFILE_SWITCH_DRAIN_TIMEOUT")) { - env_timeout = atoi(e); - } - time_t drain_timeout = env_timeout > 0 ? env_timeout : k_drain_timeout_default; - if (elapsed > drain_timeout) { - SRV_WRN("hydra: pending config drain timeout (elapsed=%lld, limit=%lld) — discarding, " - "tier='%s' payload_size=%zu\n", - (long long) elapsed, (long long) drain_timeout, - ctx_tgt->hydra_get_pending_config_tier().c_str(), - ctx_tgt->hydra_get_pending_config().size()); - ctx_tgt->hydra_clear_pending_config(); - llama_hydra_clear_pending_t3(); - // hydra#470: the staged generic (T4) subset must not - // survive the discard — a stale config would be applied - // on the next unrelated reload. - g_pending_reload_config.clear(); - return false; - } - - tier = ctx_tgt->hydra_get_pending_config_tier(); - pending_json = ctx_tgt->hydra_get_pending_config(); - SRV_INF("hydra: applying pending config (tier='%s', age=%llds, payload_size=%zu)\n", - tier.c_str(), (long long) elapsed, pending_json.size()); - } - - bool ok = true; - - // 2. T2 work: free + rebuild context with the new cparams. - // Skipped when tier is T3 (T3's load_model() handles both). - if (tier == "T2") { - if (!apply_t2_rebuild(pending_json)) { - SRV_ERR("%s", "hydra: T2 rebuild failed; engine continues with old context\n"); - ok = false; - } - } - - // 3. T3 work: full model reload with the staged T3 statics. - // load_model() handles the unload+reload cycle. COMBINED-mode - // expert bindings are torn down before the reload and re- - // attached after, in the same pattern as SET_EXPERT_MODE. - // T4 (generic-arg) configs route here too: a model reload is - // the only apply that makes EVERY generic key take effect - // (speculative types need load_model's MTP/draft setup). - if (tier == "T3" || tier == "T4") { - // #470: force rebuild if a peer reconnection was detected - const bool reconn_force = (ctx_tgt && ctx_tgt->peer_reconnection_pending); - if (reconn_force) { - ctx_tgt->peer_reconnection_pending = false; - SRV_WRN("%s", "hydra: apply_pending: peer reconnection pending — forcing T3 rebuild\n"); - } - if (!apply_t3_rebuild(reconn_force)) { - SRV_ERR("%s", "hydra: T3 rebuild failed; engine continues with old model\n"); - ok = false; - } else { - // P1-6: T3 model changed — the cached server_context_meta - // (model_path, split_mode, tensor_split, chat_params, …) - // is now stale. Refresh it on the task-queue thread - // (safe — runs during the drain window when no slots are - // processing and no new requests are being dispatched). - if (routes_ptr) { - routes_ptr->refresh_meta(); - } - // P0-1 (#49): after deferred first-load, apply staged capabilities - // so ENGINE_INFO(0x41) and COMBINED-mode logic work correctly. - if (is_first_load) { - hydra_rpc_backend_active = bootstrap_rpc_active; - hydra_peer = bootstrap_peer; - hydra_peer_reachable = bootstrap_peer_reachable; - hydra_combined_pattern = bootstrap_pattern; - hydra_split_mode = bootstrap_split_mode; - if (bootstrap_combined_static) { - hydra_combined_static = true; - SRV_INF("%s", "P0-1: deferred first-load — combined_static mode activated\n"); - } - // Register local tensors and enable shared-backend compute - // lock so the model can serve inbound RPC requests. - if (model_tgt && ctx_tgt) { - llama_hydra_register_local_tensors_for_rpc(ctx_tgt); - llama_hydra_enable_shared_backend_compute_lock(); - } - // Update the RPC server's compute backends now that the - // model is loaded. The RPC server was started with empty - // backends (head-bootstrap mode); now populate it. - if (ctx_tgt) { - std::vector backends(8); - size_t n = llama_hydra_get_compute_backends(ctx_tgt, backends.data(), backends.size()); - if (n > backends.size()) { - backends.resize(n); - n = llama_hydra_get_compute_backends(ctx_tgt, backends.data(), backends.size()); - } - backends.resize(n); - hydra_rpc::update_backends(backends); - SRV_INF("P0-1: updated RPC backends to %zu compute device(s)\n", backends.size()); - } - SRV_INF("%s", "hydra-engine ready — model loaded via CONFIGURE T3\n"); - } - } - } - - // 4. Clear the staged state regardless of success. On failure - // the rollback in apply_t{2,3}_rebuild has restored the - // previous state; clearing the staged state prevents the - // next slot-free moment from re-attempting the same rebuild. - if (is_first_load) { - first_load_pending = false; - } else { - ctx_tgt->hydra_clear_pending_config(); - } - llama_hydra_clear_pending_t3(); - return ok; - } // T2 rebuild: free the live llama_context, rebuild llama_context_params // from the updated params_base (n_ctx / cache_type_k / cache_type_v / @@ -3623,205 +2757,8 @@ struct server_context_impl { // There is no public ggml_parse_type() in upstream llama.cpp, so // we iterate ggml_type_traits via ggml_get_type_traits() and // match on ggml_type_name(). - static ggml_type hydra_parse_cache_type(const std::string & s) { - if (s.empty()) return GGML_TYPE_COUNT; - for (int i = 0; i < GGML_TYPE_COUNT; i++) { - ggml_type t = (ggml_type) i; - if (strcmp(ggml_type_name(t), s.c_str()) == 0) return t; - } - return GGML_TYPE_COUNT; - } +static ggml_type hydra_parse_cache_type(const std::string & s); - // hydra#470: apply the staged context-level (T2) + generic (T4) keys - // from `cfg` to `params`. Shared by apply_t2_rebuild() (params_base, - // before the context rebuild) and apply_t3_rebuild() (swapped_params, - // before load_model()) so a mixed T2+T4 payload is handled identically - // on both reload paths — the model-reload path consumes the same staged - // config as the context-reload path and no T2 key is stranded. - // - // The explicit T2 keys keep their custom semantics (n_ctx clamp to - // model_n_ctx_train, cache_type validation); every other key goes - // through the llama.cpp arg table (hydra_apply_generic_key). A - // present-but-unusable value is logged with SRV_WRN — never silent. - void hydra_apply_staged_context_keys(common_params & params, const json & cfg) { - // ── explicit T2 keys (context-level); each is optional, absence - // means "leave unchanged" ── - if (cfg.contains("n_ctx") && cfg["n_ctx"].is_number_integer()) { - const int32_t n_ctx = cfg["n_ctx"].get(); - if (model_tgt) { - // Clamp to the model's training ctx. The wire spec does - // not require a reject-on-too-large; we clamp and report. - // At first load (model_tgt null) the value is used as-is - // and llama.cpp validates it during context creation. - const int32_t max_ctx = (int32_t) llama_model_n_ctx_train(model_tgt); - if (n_ctx > max_ctx) { - SRV_WRN("hydra: n_ctx=%d exceeds model_n_ctx_train=%d; clamping\n", - n_ctx, max_ctx); - params.n_ctx = max_ctx; - } else { - params.n_ctx = n_ctx; - } - } else { - params.n_ctx = n_ctx; - } - } - if (cfg.contains("cache_type_k") && cfg["cache_type_k"].is_string()) { - const std::string & s = cfg["cache_type_k"].get_ref(); - ggml_type t = hydra_parse_cache_type(s); - if (t == GGML_TYPE_COUNT) { - SRV_WRN("hydra: cache_type_k='%s' unparseable; ignoring\n", s.c_str()); - } else { - params.cache_type_k = t; - } - } - if (cfg.contains("cache_type_v") && cfg["cache_type_v"].is_string()) { - const std::string & s = cfg["cache_type_v"].get_ref(); - ggml_type t = hydra_parse_cache_type(s); - if (t == GGML_TYPE_COUNT) { - SRV_WRN("hydra: cache_type_v='%s' unparseable; ignoring\n", s.c_str()); - } else { - params.cache_type_v = t; - } - } - if (cfg.contains("rope_freq_base") && cfg["rope_freq_base"].is_number()) { - params.rope_freq_base = cfg["rope_freq_base"].get(); - } - if (cfg.contains("rope_freq_scale") && cfg["rope_freq_scale"].is_number()) { - params.rope_freq_scale = cfg["rope_freq_scale"].get(); - } - if (cfg.contains("yarn_ext_factor") && cfg["yarn_ext_factor"].is_number()) { - params.yarn_ext_factor = cfg["yarn_ext_factor"].get(); - } - if (cfg.contains("yarn_attn_factor") && cfg["yarn_attn_factor"].is_number()) { - params.yarn_attn_factor = cfg["yarn_attn_factor"].get(); - } - if (cfg.contains("yarn_beta_fast") && cfg["yarn_beta_fast"].is_number()) { - params.yarn_beta_fast = cfg["yarn_beta_fast"].get(); - } - if (cfg.contains("yarn_beta_slow") && cfg["yarn_beta_slow"].is_number()) { - params.yarn_beta_slow = cfg["yarn_beta_slow"].get(); - } - if (cfg.contains("yarn_orig_ctx") && cfg["yarn_orig_ctx"].is_number_integer()) { - params.yarn_orig_ctx = cfg["yarn_orig_ctx"].get(); - } - - // ── generic (T4) fallback: every key the explicit T2 code above - // did NOT handle is applied via the llama.cpp arg table. This - // also covers special-classified keys with no explicit handling - // (rope_scale / rope_scaling classify as T2 via the rope_* - // prefix but have no dedicated code above). ── - static const std::set t2_handled = { - "n_ctx", "cache_type_k", "cache_type_v", - "rope_freq_base", "rope_freq_scale", - "yarn_ext_factor", "yarn_attn_factor", - "yarn_beta_fast", "yarn_beta_slow", "yarn_orig_ctx", - }; - for (auto it = cfg.begin(); it != cfg.end(); ++it) { - const std::string & key = it.key(); - if (t2_handled.count(key) > 0) { - continue; // explicit T2 code above - } - if (!hydra_apply_generic_key(params, key, it.value())) { - SRV_WRN("hydra: staged context key '%s' not applied (see log)\n", key.c_str()); - } - } - - // ── present-but-unusable T2 values must be loud, not silent ── - for (const auto & key : t2_handled) { - if (!cfg.contains(key)) { - continue; - } - const json & v = cfg[key]; - const bool usable = - (key == "n_ctx" || key == "yarn_orig_ctx") ? v.is_number_integer() : - (key == "cache_type_k" || key == "cache_type_v") ? v.is_string() : - v.is_number(); - if (!usable) { - SRV_WRN("hydra: T2 key '%s' has unusable value (JSON type %s) — value ignored\n", - key.c_str(), v.type_name()); - } - } - } - - bool apply_t2_rebuild(const std::string & pending_json) { - if (!ctx_tgt || !model_tgt) return false; - - json cfg; - try { - cfg = json::parse(pending_json); - } catch (const std::exception & e) { - SRV_WRN("hydra: T2 apply: invalid JSON in pending_config: %s\n", e.what()); - return false; - } - - // Snapshot the old params for rollback. params_base is the - // canonical "what's in effect" state; restoring it plus a - // recreate-cycle is the rollback path. - common_params old_params = params_base; - - // hydra#470: apply the staged context-level + generic keys to - // params_base (explicit T2 keys + arg-table fallback). Shared with - // apply_t3_rebuild() so mixed T2+T4 payloads are handled identically - // on both reload paths. - hydra_apply_staged_context_keys(params_base, cfg); - // The staged reload config was consumed above (its keys ride the - // pending_config JSON); clear the static so a later T3 reload does - // not re-apply stale values. The "last applied" marker is updated - // only on the success path below. - const std::string consumed_reload_config = g_pending_reload_config; - g_pending_reload_config.clear(); - - // Free the live context. KV cache is destroyed; this is the - // T2 cost. The model is kept (T2 is context-only). - llama_free(ctx_tgt); - if (ctx_dft) { - llama_free(ctx_dft.get()); - ctx_dft.reset(); - } - - // Build new cparams from the updated params_base. This is - // the same call site load_model() uses internally. - auto cparams = common_context_params_to_llama(params_base); - - // Recreate the context with the new cparams. - ctx_tgt = llama_new_context_with_model(model_tgt, cparams); - if (!ctx_tgt) { - // Rollback: rebuild with the old params_base. The old - // params must work (we just freed and recreated the - // context with them). If they don't, the engine is in - // a bad state — abort. - SRV_WRN("hydra: T2 rebuild failed with n_ctx=%d cache_type=%d/%d; " - "rolling back to old params\n", - params_base.n_ctx, (int) params_base.cache_type_k, - (int) params_base.cache_type_v); - params_base = old_params; - auto cparams_old = common_context_params_to_llama(params_base); - ctx_tgt = llama_new_context_with_model(model_tgt, cparams_old); - if (!ctx_tgt) { - GGML_ABORT("hydra: T2 rollback failed (cannot rebuild context with old params). " - "Engine exiting to prevent serving with corrupted state."); - } - return false; - } - - // Re-init per-slot samplers. The old samplers were bound to - // the now-freed context; common_sampler_init() on the new - // model picks up the (possibly changed) sampling config. - for (auto & slot : slots) { - slot.smpl.reset(common_sampler_init(model_tgt, params_base.sampling)); - } - - n_ctx = llama_n_ctx(ctx_tgt); - // hydra#470: record the staged reload config as applied. A later - // T3-tier config carrying the same staged keys can then skip the - // model reload via the early-exit (the values are already in - // params_base, which feeds swapped_params). - g_last_reload_config_applied = consumed_reload_config; - SRV_INF("hydra: T2 rebuild applied (n_ctx=%d, cache=%d/%d, slots=%zu)\n", - n_ctx, (int) params_base.cache_type_k, - (int) params_base.cache_type_v, slots.size()); - return true; - } // Register new RPC peer devices into the global ggml backend registry. // Called from apply_t3_rebuild() before load_model() so the new peer's @@ -3831,48 +2768,7 @@ struct server_context_impl { // Repeated registration is NOT safe/idempotent in the underlying API, // so we track already-registered endpoints in a static set and only // register genuinely new ones. - static void hydra_register_rpc_servers(const json & servers_arr) { - static std::set registered; - - if (!servers_arr.is_array() || servers_arr.empty()) { - return; - } - - ggml_backend_load_all(); - ggml_backend_reg_t rpc_reg = ggml_backend_reg_by_name("RPC"); - if (!rpc_reg) { - SRV_WRN("%s", "hydra: rpc_servers: RPC backend not available\n"); - return; - } - - typedef ggml_backend_reg_t (*ggml_backend_rpc_add_server_t)(const char * endpoint); - auto add_server_fn = (ggml_backend_rpc_add_server_t) - ggml_backend_reg_get_proc_address(rpc_reg, "ggml_backend_rpc_add_server"); - if (!add_server_fn) { - SRV_WRN("%s", "hydra: rpc_servers: ggml_backend_rpc_add_server not found\n"); - return; - } - - for (const auto & v : servers_arr) { - if (!v.is_string()) continue; - const std::string endpoint = v.get(); - if (endpoint.empty()) continue; - if (registered.count(endpoint)) { - SRV_DBG("hydra: rpc_servers: endpoint '%s' already registered, skipping\n", - endpoint.c_str()); - continue; - } - ggml_backend_reg_t reg = add_server_fn(endpoint.c_str()); - if (reg) { - ggml_backend_register(reg); - registered.insert(endpoint); - SRV_INF("hydra: rpc_servers: registered endpoint '%s'\n", endpoint.c_str()); - } else { - SRV_WRN("hydra: rpc_servers: failed to register endpoint '%s'\n", - endpoint.c_str()); - } - } - } +static void hydra_register_rpc_servers(const json & servers_arr); // #470: refresh the T3-current alias → file map after a successful // model load. The map's keys are the engine's own identity aliases @@ -3908,16 +2804,7 @@ struct server_context_impl { // skip this entirely: the engine loaded the correct model file but kept // routing tokens through the old COMBINED config, which is #514 // (throughput collapses to ~2-4 tok/s after a dynamic model swap). - void hydra_teardown_combined_before_reload() { - SRV_INF("hydra: tearing down COMBINED before model reload (was head_attached=%d, static=%d)\n", - (int) hydra_combined_head_attached, (int) hydra_combined_static); - llama_hydra_set_expert_mode(ctx_tgt, 0); - if (!hydra_current_peer.empty()) { - ctx_tgt->hydra_remove_combined_rpc_backend(hydra_current_peer.c_str()); - } - llama_hydra_clear_combined_bindings(ctx_tgt, hydra_peer.c_str()); - hydra_combined_head_attached = false; - } +void hydra_teardown_combined_before_reload(); // epic #610 WS2: HYDRA task dispatch — declared here, defined in // hydra-server-context.cpp (included at the bottom of this TU). Keeps a @@ -3931,40 +2818,7 @@ struct server_context_impl { // peer's RPC device and rebinds the expert tensors; same fail-open // pattern as SET_EXPERT_MODE — if the peer is unreachable, the engine // stays solo and the coordinator's solo-fallback path handles it. - void hydra_reattach_combined_after_reload() { - SRV_INF("%s", "hydra: re-attaching COMBINED on new model\n"); - if (hydra_combined_static) { - llama_hydra_set_expert_mode(ctx_tgt, 1); - } else if (!hydra_peer.empty() && !hydra_combined_pattern.empty()) { - if (llama_hydra_peer_reachable(hydra_peer.c_str())) { - ggml_backend_reg_t rpc_reg = ggml_backend_reg_by_name("RPC"); - if (rpc_reg) { - using add_server_fn_t = ggml_backend_reg_t (*)(const char *); - auto add_server_fn = (add_server_fn_t) ggml_backend_reg_get_proc_address(rpc_reg, "ggml_backend_rpc_add_server"); - ggml_backend_reg_t peer_reg = add_server_fn ? add_server_fn(hydra_peer.c_str()) : nullptr; - ggml_backend_dev_t peer_dev = (peer_reg && ggml_backend_reg_dev_count(peer_reg) > 0) ? ggml_backend_reg_dev_get(peer_reg, 0) : nullptr; - if (peer_dev) { - int32_t n_bound = llama_hydra_rebind_combined_experts( - ctx_tgt, hydra_peer.c_str(), peer_dev, hydra_combined_pattern.c_str()); - if (n_bound > 0) { - hydra_combined_head_attached = true; - llama_hydra_set_expert_mode(ctx_tgt, 1); - SRV_INF("hydra: COMBINED re-attached on peer %s (%d layers bound)\n", - hydra_peer.c_str(), n_bound); - } else { - SRV_WRN("hydra: rebind returned %d; staying solo\n", n_bound); - } - } else { - SRV_WRN("hydra: peer %s has no device; staying solo\n", hydra_peer.c_str()); - } - } else { - SRV_WRN("%s\n", "hydra: RPC backend not available; staying solo"); - } - } else { - SRV_WRN("hydra: peer %s unreachable; staying solo\n", hydra_peer.c_str()); - } - } - } +void hydra_reattach_combined_after_reload(); // Re-pad tensor_buft_overrides to the nullptr-terminated capacity // llama_max_tensor_buft_overrides() after a preset's apply_to_params() @@ -3974,15 +2828,7 @@ struct server_context_impl { // has (below) — apply_to_params() push_backs unconditionally, so an // unusually large preset could otherwise overflow the same 4096-entry // limit this whole clear/re-pad dance exists to respect. - void hydra_repad_tensor_buft_overrides(common_params & p, const char * ctx_label) { - const size_t ntbo = llama_max_tensor_buft_overrides(); - if (p.tensor_buft_overrides.size() + 1 > ntbo) { - SRV_WRN("hydra: %s: %zu tensor_buft_overrides exceed the %zu-entry limit; keeping the first %zu\n", - ctx_label, p.tensor_buft_overrides.size(), ntbo, ntbo - 1); - p.tensor_buft_overrides.resize(ntbo - 1); - } - p.tensor_buft_overrides.resize(ntbo, llama_model_tensor_buft_override{ nullptr, nullptr }); - } +void hydra_repad_tensor_buft_overrides(common_params & p, const char * ctx_label); // T3 rebuild: full model reload. Uses the staged T3 statics // (override_tensor, split_mode, tensor_split, n_gpu_layers, @@ -3990,283 +2836,6 @@ struct server_context_impl { // Falls through to load_model() for the actual unload+reload // cycle (which handles mmproj, MTP/draft, slot rebuild, etc.). // On failure: rollback by reloading the old params_base. - bool apply_t3_rebuild(bool force = false) { - bool is_first_load = !ctx_tgt; - - // Track the last override_tensor string that was actually - // applied so we can detect "nothing changed" on subsequent - // calls and skip the expensive unload+reload cycle. - static std::string old_override_applied; - - common_params old_params = params_base; - common_params swapped_params = params_base; - - // Read the staged T3 statics and apply them to swapped_params. - if (llama_hydra_get_pending_n_gpu_layers() >= 0) { - swapped_params.n_gpu_layers = llama_hydra_get_pending_n_gpu_layers(); - } - // n_cpu_moe is informational only — the actual MoE expert - // offload is done via override_tensor (parsed below into - // tensor_buft_overrides). The standard common_params struct - // has no n_cpu_moe field; we just log the staged value for - // operator visibility. - if (llama_hydra_get_pending_n_cpu_moe() >= 0) { - SRV_INF("hydra: T3 rebuild: staged n_cpu_moe=%d (informational; expert routing via override_tensor)\n", - llama_hydra_get_pending_n_cpu_moe()); - } - const char * path = llama_hydra_get_pending_model_path(); - if (path && *path) { - swapped_params.model.path = path; - } - const char * mode = llama_hydra_get_pending_split_mode(); - if (mode && *mode) { - std::string m(mode); - if (m == "none") swapped_params.split_mode = LLAMA_SPLIT_MODE_NONE; - else if (m == "layer") swapped_params.split_mode = LLAMA_SPLIT_MODE_LAYER; - else if (m == "row") swapped_params.split_mode = LLAMA_SPLIT_MODE_ROW; - else SRV_WRN("hydra: T3 split_mode='%s' unknown; keeping current\n", m.c_str()); - } - const size_t n_split = llama_hydra_get_pending_tensor_split_count(); - if (n_split > 0) { - const float * split = llama_hydra_get_pending_tensor_split(); - // common_params::tensor_split is a fixed-size array. - const size_t cap = sizeof(swapped_params.tensor_split) / - sizeof(swapped_params.tensor_split[0]); - const size_t n = n_split < cap ? n_split : cap; - for (size_t i = 0; i < n; i++) { - swapped_params.tensor_split[i] = split[i]; - } - // Zero the rest so the engine doesn't see stale values. - for (size_t i = n; i < cap; i++) { - swapped_params.tensor_split[i] = 0.0f; - } - } - const char * override = llama_hydra_get_pending_override_tensor(); - if (override && *override) { - // Wire-shape: comma-separated "pattern=buft" pairs (e.g. - // "blk.*.ffn_*_exps.weight=CPU"). The C++ side stores - // these as a vector. - // Buft names are looked up - // via ggml_backend_dev_buffer_type() + ggml_backend_buft_name() - // (mirrors common/arg.cpp:parse_tensor_buffer_overrides). - ggml_backend_load_all(); - std::map buft_list; - for (size_t i = 0; i < ggml_backend_dev_count(); ++i) { - auto * dev = ggml_backend_dev_get(i); - auto * buft = ggml_backend_dev_buffer_type(dev); - if (buft) { - buft_list[std::string(ggml_backend_buft_name(buft))] = buft; - } - } - // CPU is the common case (MoE expert routing) — also lookup - // explicitly since some backends may not register the CPU buft. - buft_list["CPU"] = ggml_backend_cpu_buffer_type(); - - // Keep pattern strings alive for the lifetime of the - // process — entry.pattern is a const char* that must not - // dangle. Matches the safe pattern in common/arg.cpp. - static std::list buft_override_patterns; - - std::vector staged; - - const std::string ovr(override); - size_t start = 0; - while (start < ovr.size()) { - size_t comma = ovr.find(',', start); - std::string part = ovr.substr(start, comma == std::string::npos ? std::string::npos : comma - start); - size_t eq = part.find('='); - if (eq != std::string::npos) { - std::string pattern = part.substr(0, eq); - std::string buft_name = part.substr(eq + 1); - auto it = buft_list.find(buft_name); - if (it != buft_list.end()) { - buft_override_patterns.push_back(pattern); - llama_model_tensor_buft_override entry; - entry.pattern = buft_override_patterns.back().c_str(); - entry.buft = it->second; - staged.push_back(entry); - } else { - SRV_WRN("%s", "hydra: T3 rebuild: override_tensor buft name not in registered list; skipping pattern\n"); - } - } - if (comma == std::string::npos) break; - start = comma + 1; - } - - // Install the staged patterns *in place of* the base ones instead of - // appending to them. - // - // common_params_parse_ex() (common/arg.cpp) unconditionally pads this - // vector out to llama_max_tensor_buft_overrides() entries of - // {nullptr, nullptr}, so by the time we get here the real CLI overrides - // sit at the head and the rest is terminator padding. push_back() would - // land *behind* that padding, which breaks twice over: - // 1. common_model_params_to_llama() asserts that back().pattern is - // nullptr, so the engine aborts before the model loads; - // 2. even without that assert, llama_model_loader stops scanning at - // the first nullptr pattern, so appended entries are never read — - // the override would be silently dropped and the MoE experts would - // land on the GPU. - // Replacing also matches the sibling fields handled above: model.path, - // split_mode, n_gpu_layers and tensor_split are all overwritten by the - // staged T3 config rather than merged into it. - const size_t ntbo = llama_max_tensor_buft_overrides(); - if (staged.empty()) { - // Nothing resolved (every buft name was unknown). Wiping the base - // overrides here would silently change how the model is placed, so - // keep them and make the no-op explicit. - SRV_WRN("%s", "hydra: T3 rebuild: staged override_tensor resolved to no usable patterns; keeping base overrides\n"); - } else { - if (staged.size() + 1 > ntbo) { - SRV_WRN("hydra: T3 rebuild: %zu override_tensor patterns exceed the %zu-entry limit; keeping the first %zu\n", - staged.size(), ntbo, ntbo - 1); - staged.resize(ntbo - 1); - } - // assign() re-establishes the full terminator padding, so everything - // from staged.size() onward is {nullptr, nullptr}. - swapped_params.tensor_buft_overrides.assign(ntbo, llama_model_tensor_buft_override{ nullptr, nullptr }); - for (size_t i = 0; i < staged.size(); ++i) { - swapped_params.tensor_buft_overrides[i] = staged[i]; - } - } - } - - // hydra#470: apply the staged reload config (context-level T2 keys - // + generic T4 keys) to swapped_params BEFORE any model (re)load — - // they must land in common_params before load_model() consumes them - // (n_ctx, cache types, speculative types, ...). The staged JSON is - // consumed here (cleared), so a later T2/T3 apply does not re-apply - // stale values. Mixed T2+T4 payloads are applied by the same helper - // the context-reload path uses — nothing is stranded. - std::string staged_reload = g_pending_reload_config; - g_pending_reload_config.clear(); - if (!staged_reload.empty()) { - json reload_cfg; - try { - reload_cfg = json::parse(staged_reload); - } catch (const std::exception & e) { - SRV_WRN("hydra: T3 rebuild: staged reload config failed to parse: %s\n", e.what()); - staged_reload.clear(); - } - if (!reload_cfg.is_null()) { - hydra_apply_staged_context_keys(swapped_params, reload_cfg); - } - } - - // Early-exit: if the model, all T3-relevant params AND the staged - // reload config (T2 + T4 keys) are identical to what is already - // loaded, skip the expensive unload+reload cycle. Without this, - // every COMPLETION request that carries hydra_config triggers a - // full model swap even when nothing changed (the coordinator sends - // the same config on every decode request). The comparison is - // against the last staged dump that was actually loaded — identical - // config → identical dump → skip; a changed T2 or T4 key → forced - // reload (no masked changes). - const bool reload_unchanged = (staged_reload == g_last_reload_config_applied); - if (!is_first_load) { - const char * cur_override = llama_hydra_get_pending_override_tensor(); - bool params_unchanged = - swapped_params.model.path == old_params.model.path && - swapped_params.n_gpu_layers == old_params.n_gpu_layers && - swapped_params.split_mode == old_params.split_mode && - ((cur_override == nullptr && old_override_applied.empty()) || - (cur_override && old_override_applied == cur_override)); - if (params_unchanged && reload_unchanged && !force) { - // T3 overrides (override_tensor, split_mode) were staged by - // the COMPLETION hydra_config path. But the model reload is - // being skipped. Clear the staged override so the next decode - // uses the current tensor placement (not the staged override). - llama_hydra_set_override_tensor(ctx_tgt, nullptr); - SRV_INF("%s", "hydra: T3 rebuild: model and params unchanged — skipping reload, cleared staged overrides\n"); - return true; - } - } - - // COMBINED-mode teardown BEFORE the model reload — see - // hydra_teardown_combined_before_reload() above. - const bool was_combined = hydra_combined_head_attached || hydra_combined_static; - if (!is_first_load && was_combined) { - hydra_teardown_combined_before_reload(); - } - - // Register any new RPC peer devices before load_model() so the - // peer's device exists in the global ggml backend registry when - // common_init_from_params() tries to place tensors per - // tensor_split/split_mode. Only genuinely new endpoints are - // registered (hydra_register_rpc_servers tracks already-registered - // endpoints to avoid unsafe repeated registration). - if (!g_pending_rpc_servers.empty()) { - json rpc_arr = json::array(); - for (const auto & s : g_pending_rpc_servers) { - rpc_arr.push_back(s); - } - hydra_register_rpc_servers(rpc_arr); - g_pending_rpc_servers.clear(); - } - - // Full model reload. load_model() handles the unload of the - // current model, the load of the new model, the new context - // creation, the MTP/draft paths, and the slot rebuild. - // NOTE: load_model() does `params_base = params` internally - // (line 844), so after a successful load params_base reflects - // swapped_params — no explicit reassignment needed by us. - // - // #507: Skip the fit_params probe during T3 rebuild. The probe - // does a full model-structure load with no_alloc=true to measure - // GPU memory — expensive (~45-90s) and unnecessary here because: - // (a) we just freed VRAM by destroying the old model, (b) the new - // model's requirements are known (same or smaller), (c) a controlled - // inference server has predictable VRAM. Disabling saves ~1 min. - swapped_params.fit_params = false; - if (!load_model(swapped_params)) { - if (is_first_load) { - SRV_WRN("%s", "hydra: T3 first load failed — engine stays empty\n"); - return false; - } - SRV_ERR("hydra: T3 reload to '%s' failed (load_model returned false); " - "rolling back to old model\n", - swapped_params.model.path.c_str()); - if (!load_model(old_params)) { - SRV_ERR("%s", "hydra: T3 rollback also failed — engine in unrecoverable state\n"); - GGML_ABORT("hydra: T3 rollback failed (cannot reload old model). " - "Engine exiting to prevent serving with corrupted state."); - } - SRV_INF("hydra: T3 rollback succeeded — restored old model '%s'\n", - old_params.model.path.c_str()); - return false; - } - - // Record the override_tensor that was just applied so the - // next call can skip the reload if nothing changed. - { - const char * cur = llama_hydra_get_pending_override_tensor(); - old_override_applied = cur ? cur : ""; - // hydra#470: also record the staged reload config (T2+T4 keys) - // that was just loaded, so a repeated identical CONFIGURE/decode - // payload skips the reload (early-exit above). - g_last_reload_config_applied = staged_reload; - } - - // T3 reload confirmed. Log model identity for traceability. - SRV_INF("hydra: T3 reload confirmed model_alias='%s' tokenizer='%s' model_name='%s' quant='%s' caps=0x%x model_path='%s'\n", - swapped_params.model_alias.empty() ? "?" : swapped_params.model_alias.begin()->c_str(), - model_tgt ? llama_model_get_tokenizer_model(model_tgt) : "", - model_tgt ? llama_model_get_display_name(model_tgt) : "", - model_tgt ? llama_model_get_quant_label(model_tgt) : "", - model_tgt ? llama_model_get_capabilities_bitfield(model_tgt) : 0, - swapped_params.model.path.c_str()); - - // COMBINED-mode reattach AFTER the model reload — see - // hydra_reattach_combined_after_reload() above. - if (was_combined) { - hydra_reattach_combined_after_reload(); - } - - SRV_INF("hydra: T3 rebuild applied (model='%s', split_mode=%d, n_gpu_layers=%d, slots=%zu)\n", - params_base.model.path.c_str(), (int) params_base.split_mode, - params_base.n_gpu_layers, slots.size()); - return true; - } void update_slots() { // epic #610 WS1: in seam mode the extension may pre-empt the decode From 9c50a568ff6fdcba308cad8daac73dd032040e76 Mon Sep 17 00:00:00 2001 From: Hydra Engineering Date: Tue, 11 Aug 2026 22:06:01 +0700 Subject: [PATCH 4/5] epic(610): WS5 flip default to seam (HYDRA_EXT_MODE no longer required) HYDRA_EXT_MODE now defaults to the extension (seam) implementation; only an explicit HYDRA_EXT_MODE=legacy opts into the inline Hydra code for A/B. Inverts the pre-WS5 default (legacy). Verified: all 9 hermetic tests pass in default/legacy/seam; live-server /props + /health identical across modes. Inline-path deletion in update_slots stays gated on WS4 (live-GPU parity) so the A/B baseline remains available until real decode/COMBINED is proven. --- tests/test-hydra-ext-ab.cpp | 14 +++++++------- tools/server/server-context.cpp | 4 ++-- tools/server/server-hydra-extension.h | 8 ++++---- 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/tests/test-hydra-ext-ab.cpp b/tests/test-hydra-ext-ab.cpp index 9601749f6873..a6a21fdb5bb4 100644 --- a/tests/test-hydra-ext-ab.cpp +++ b/tests/test-hydra-ext-ab.cpp @@ -42,21 +42,21 @@ static void set_ext_mode(const char * value) { #endif int main() { - // --- mode parsing ------------------------------------------------- + // --- mode parsing (WS5: default = seam unless explicitly legacy) ---- set_ext_mode(nullptr); - expect("unset HYDRA_EXT_MODE -> legacy", !hydra_ext_mode_seam()); + expect("unset HYDRA_EXT_MODE -> seam (default)", hydra_ext_mode_seam()); set_ext_mode("legacy"); - expect("HYDRA_EXT_MODE=legacy -> legacy", !hydra_ext_mode_seam()); + expect("HYDRA_EXT_MODE=legacy -> legacy", !hydra_ext_mode_seam()); set_ext_mode("seam"); - expect("HYDRA_EXT_MODE=seam -> seam", hydra_ext_mode_seam()); + expect("HYDRA_EXT_MODE=seam -> seam", hydra_ext_mode_seam()); set_ext_mode("garbage"); - expect("HYDRA_EXT_MODE=garbage -> legacy", !hydra_ext_mode_seam()); + expect("HYDRA_EXT_MODE=garbage -> seam (default)", hydra_ext_mode_seam()); - set_ext_mode("LEGACY"); // case-sensitive: not "seam" - expect("HYDRA_EXT_MODE=LEGACY -> legacy", !hydra_ext_mode_seam()); + set_ext_mode("LEGACY"); // case-sensitive: not "legacy" + expect("HYDRA_EXT_MODE=LEGACY -> seam (default)", hydra_ext_mode_seam()); // --- factory + WS1 no-op contract --------------------------------- std::unique_ptr ext = hydra_create_extension(); diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp index b7022f6a719d..8309b49d2ad8 100644 --- a/tools/server/server-context.cpp +++ b/tools/server/server-context.cpp @@ -1135,9 +1135,9 @@ struct server_context_impl { mtmd_helper_log_set(common_log_default_callback, nullptr); if (hydra_ext_active) { hydra_ext = hydra_create_extension(); - SRV_INF("hydra ext: seam mode active (HYDRA_EXT_MODE=seam), impl=%s\n", hydra_ext->name()); + SRV_INF("hydra ext: seam mode active (default), impl=%s\n", hydra_ext->name()); } else { - SRV_INF("%s", "hydra ext: legacy mode active (default) - A/B baseline\n"); + SRV_INF("%s", "hydra ext: legacy mode active (HYDRA_EXT_MODE=legacy) - A/B baseline\n"); } } diff --git a/tools/server/server-hydra-extension.h b/tools/server/server-hydra-extension.h index eaef85586b80..cc1b0adacd5f 100644 --- a/tools/server/server-hydra-extension.h +++ b/tools/server/server-hydra-extension.h @@ -9,8 +9,8 @@ // // A/B toggle: the HYDRA_EXT_MODE env var selects which implementation drives // Hydra behavior at runtime, so the SAME binary can be A/B tested: -// HYDRA_EXT_MODE=legacy -> the inline Hydra code in server-context.cpp (default) -// HYDRA_EXT_MODE=seam -> the extension (WS1: no-op, behavior identical) +// HYDRA_EXT_MODE=seam -> the extension (default since WS5) +// HYDRA_EXT_MODE=legacy -> the inline Hydra code in server-context.cpp // Both paths stay compiled; only one is consulted per run. WS4 diffs the same // scenario through both modes to prove the refactor is behavior-identical. #pragma once @@ -22,10 +22,10 @@ struct server_context_impl; struct server_task; -// True when HYDRA_EXT_MODE=seam. Anything else (including unset) = legacy. +// True when HYDRA_EXT_MODE is NOT "legacy" (i.e. default = seam since WS5). inline bool hydra_ext_mode_seam() { const char * m = std::getenv("HYDRA_EXT_MODE"); - return m && std::strcmp(m, "seam") == 0; + return !(m && std::strcmp(m, "legacy") == 0); } struct server_hydra_extension { From ecfbfda11275e2108e0e332365989989d2197f78 Mon Sep 17 00:00:00 2001 From: Hydra Engineering Date: Fri, 21 Aug 2026 13:27:32 +0700 Subject: [PATCH 5/5] fix(610): restore generic dispatch + config-apply dropped by rebase extraction MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The regex-based conflict resolution during the hydra-fork rebase over-deleted two things it shouldn't have: - process_single_task() in server-context.cpp: this is upstream's generic task dispatch (COMPLETION/CANCEL/CONTROL/etc.), not Hydra-specific. Only the Hydra task-type case belongs extracted (now hydra_process_task(), defined in hydra-server-context.cpp, called from the generic switch's fall-through). Restored the generic function; verified no duplicate definition exists between the two files (grep-checked call sites match forward declarations 1:1). - hydra_apply_config() in hydra-server-context.cpp: the T1/T2/T3/T4 config tiering logic that process_single_task's hydra_config_json path and CONFIGURE both call — was referenced but never defined post-extraction, which would fail to link. Restored from hydra-fork upstream with the server_context_impl:: qualification the extraction requires. Reviewed line-by-line against hydra-fork's originals and the call sites in both files; C++ compiles clean (link still needs the fork's ggml-rpc submodule, not available standalone in this worktree — matches the prior build report). Co-Authored-By: Claude Sonnet 5 --- tools/server/hydra-server-context.cpp | 165 ++++++++++- tools/server/server-context.cpp | 397 ++++++++++++++++++++++++++ 2 files changed, 561 insertions(+), 1 deletion(-) diff --git a/tools/server/hydra-server-context.cpp b/tools/server/hydra-server-context.cpp index 9b06956e5bee..80107ee5daaf 100644 --- a/tools/server/hydra-server-context.cpp +++ b/tools/server/hydra-server-context.cpp @@ -2519,6 +2519,169 @@ // --------------------------------------------------------------------------- + server_context_impl::hydra_config_result server_context_impl::hydra_apply_config(const json & cfg, bool sync) { + server_context_impl::hydra_config_result result; + + // 1. Classify every top-level key. T1 → apply now; T2/T3/T4 → + // defer (stage) or apply synchronously depending on `sync`. + // T4 (generic) keys are additionally validated against the + // llama.cpp arg table here so the CONFIGURE response can report + // unrecognized/rejected keys before the deferred apply runs. + // T2 + appliable-T4 keys are staged together into the reload + // config (g_pending_reload_config) so neither the context-reload + // path nor the model-reload path strands them. + json t1_subset = json::object(); + json reload_subset = json::object(); + for (auto it = cfg.begin(); it != cfg.end(); ++it) { + const std::string key = it.key(); + int tier = hydra_classify_config_key(key); + if (tier == 1) { + t1_subset[key] = it.value(); + } else if (tier == 2) { + result.t2t3_subset[key] = it.value(); + result.deferred_keys.push_back(key); + reload_subset[key] = it.value(); + } else if (tier == 3) { + result.t2t3_subset[key] = it.value(); + result.deferred_keys.push_back(key); + // T3 keys are staged via hydra_apply_t3_mutators() statics, + // not the reload-config JSON. + } else { + // T4: generic-arg pass-through. Classify against the arg + // table; only appliable keys are staged for the deferred + // slot-free moment. The rest are reported loudly. + const hydra_generic_key_status st = hydra_classify_generic_key(key); + if (st == hydra_generic_key_status::APPLIABLE) { + result.t2t3_subset[key] = it.value(); + result.deferred_keys.push_back(key); + reload_subset[key] = it.value(); + } else if (st == hydra_generic_key_status::DENIED) { + SRV_WRN("hydra: CONFIGURE key '%s' cannot change at reload (startup-only/flag arg) — rejected\n", + key.c_str()); + result.rejected_keys.push_back(key); + } else { + SRV_WRN("hydra: CONFIGURE key '%s' is not a known llama.cpp argument — unrecognized, value ignored\n", + key.c_str()); + result.unrecognized_keys.push_back(key); + } + } + if (tier > result.highest_tier) result.highest_tier = tier; + } + // Stage the reload config (T2 + appliable-T4 keys) for the deferred + // slot-free moment. Unconditional overwrite = absolute state: a + // superseding CONFIGURE without T2/T4 keys must clear whatever an + // earlier CONFIGURE staged, or the stale keys would be applied on + // the next unrelated reload. + g_pending_reload_config = reload_subset.dump(); + // The "sampling" object may contain unlisted nested keys + // (e.g. penalty_last_n, mirostat) — route the whole object + // through T1 when present. + if (cfg.contains("sampling") && cfg["sampling"].is_object()) { + t1_subset["sampling"] = cfg["sampling"]; + if (result.highest_tier < 1) result.highest_tier = 1; + } + // model.path is nested — the legacy {"model": {...}} form + // is recognized by hydra_classify_config_key returning 3 + // for the bare "model" key. If the bare "model" is set + // and is an object with a "path", route it as T3. + if (cfg.contains("model")) { + if (cfg["model"].is_object()) { + result.t2t3_subset["model"] = cfg["model"]; + if (std::find(result.deferred_keys.begin(), result.deferred_keys.end(), "model") + == result.deferred_keys.end()) { + result.deferred_keys.push_back("model"); + } + if (result.highest_tier < 3) result.highest_tier = 3; + } else if (cfg["model"].is_string()) { + result.t2t3_subset["model"] = cfg["model"]; + if (std::find(result.deferred_keys.begin(), result.deferred_keys.end(), "model") + == result.deferred_keys.end()) { + result.deferred_keys.push_back("model"); + } + if (result.highest_tier < 3) result.highest_tier = 3; + } + } + + if (result.highest_tier == 0) { + // No recognized keys — caller decides whether to treat as + // a no-op or surface an error. + return result; + } + + // 2. Apply T1 keys in-place. + if (!t1_subset.empty()) { + if (!hydra_apply_t1_config(params_base, ctx_tgt, t1_subset, result.params_applied)) { + result.ok = false; + result.error = "T1 key has wrong type (see log)"; + return result; + } + // Capture state_chunk_size for callers that need it (CONFIGURE). + auto it = result.params_applied.find("state_chunk_size"); + if (it != result.params_applied.end() && it->second.is_number_unsigned()) { + result.state_chunk_size_applied = it->second.get(); + } + } + + // 3. T2/T3/T4 handling — diverges based on sync flag. + if (!result.t2t3_subset.empty()) { + if (sync) { + // Synchronous mode (PREFILL / HTTP decode): apply now + // on the task-queue thread. The caller owns this thread + // context so blocking is safe. + if (result.highest_tier >= 3) { + // T3 (model reload) also covers a T4-only config: + // load_model() recreates the context and the + // speculative/draft state, so every generic key + // takes effect. hydra_apply_t3_mutators() is a no-op + // when no T3 keys are staged. + hydra_apply_t3_mutators(ctx_tgt, result.t2t3_subset, result.deferred_keys); + // #470: force rebuild if a peer reconnection was detected + // during a prior graph_compute — the peer's buffers are gone + // even though model/params haven't changed. + const bool reconn_force = (ctx_tgt && ctx_tgt->peer_reconnection_pending); + if (reconn_force) { + ctx_tgt->peer_reconnection_pending = false; + SRV_WRN("%s", "hydra: PREFILL handler: peer reconnection pending — forcing T3 rebuild\n"); + } + if (!apply_t3_rebuild(reconn_force)) { + result.ok = false; + result.error = "T3 rebuild failed"; + return result; + } + } else if (result.highest_tier == 2) { + if (!apply_t2_rebuild(result.t2t3_subset.dump())) { + result.ok = false; + result.error = "T2 rebuild failed"; + return result; + } + } + // Clear T3 staged statics — they were consumed by the + // sync apply and must not leak into a later deferred path. + llama_hydra_clear_pending_t3(); + } else { + // Stage mode (CONFIGURE): record the mutators and store + // pending_config for application at the next slot-free moment. + if (result.highest_tier >= 3) { + hydra_apply_t3_mutators(ctx_tgt, result.t2t3_subset, result.deferred_keys); + } + if (ctx_tgt) { + ctx_tgt->hydra_set_pending_config( + result.t2t3_subset.dump(), hydra_tier_label(result.highest_tier)); + } else { + // First load: ctx_tgt is null, so apply_pending_hydra_config() + // and update_slots() can't trigger. Set the flag so the + // task-queue thread runs apply_t3_rebuild() at the next + // slot-free moment. + first_load_pending = true; + SRV_INF("%s", "hydra: config staged for first load (no context yet)\n"); + } + } + } + + return result; + } + + // WS1/WS2: the extension object. handle_task() routes HYDRA tasks to the same // hydra_process_task() method the legacy switch calls — seam == legacy behavior. // --------------------------------------------------------------------------- @@ -3739,7 +3902,7 @@ void server_context::start_rpc_server(int port, std::vector) { // --- WS3.5 moved helper methods --- - bool server_context_impl::apply_t3_rebuild() { + bool server_context_impl::apply_t3_rebuild(bool force) { bool is_first_load = !ctx_tgt; // Track the last override_tensor string that was actually diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp index 8309b49d2ad8..c01e546ba6a6 100644 --- a/tools/server/server-context.cpp +++ b/tools/server/server-context.cpp @@ -4326,6 +4326,403 @@ void hydra_repad_tensor_buft_overrides(common_params & p, const char * ctx_label server_response_reader get_response_reader() { return server_response_reader(queue_tasks, queue_results, HTTP_POLLING_SECONDS); } + + void process_single_task(server_task && task) { + // epic #610 WS1: in seam mode the extension may claim Hydra tasks. + // WS1 impl is a no-op (returns false), so this is a pure A/B switch — + // both modes run the inline dispatch below. + if (hydra_ext_active && hydra_ext && hydra_ext->handle_task(*this, task)) { + return; + } + switch (task.type) { + case SERVER_TASK_TYPE_COMPLETION: + case SERVER_TASK_TYPE_INFILL: + case SERVER_TASK_TYPE_EMBEDDING: + case SERVER_TASK_TYPE_RERANK: + { + // special case: if input is provided via CLI, tokenize it first + // otherwise, no need to tokenize as it's already done inside the HTTP thread + if (task.cli) { + if (!tokenize_cli_input(task)) { + break; + } + } + + // Hydra config from HTTP decode path: apply synchronously + // on the task-queue thread before any slot scheduling or + // generation work. This is safe because we own this thread; + // the previous attempt applied on the httplib worker thread + // and raced the main queue (reverted in ebbbe1116). + if (!task.hydra_config_json.empty()) { + json hydra_cfg; + try { + hydra_cfg = json::parse(task.hydra_config_json); + } catch (const std::exception & e) { + SRV_WRN("hydra: COMPLETION hydra_config parse failed: %s\n", e.what()); + } + if (!hydra_cfg.is_null() && hydra_cfg.is_object()) { + SRV_INF("hydra: COMPLETION applying hydra_config (%zu keys)\n", + hydra_cfg.size()); + hydra_config_result cfg_result = hydra_apply_config(hydra_cfg, /*sync=*/true); + if (!cfg_result.ok) { + SRV_WRN("hydra: COMPLETION hydra_config apply failed: %s\n", + cfg_result.error.c_str()); + } + // After T3 rebuild, model/slots are reset. + // The slot lookup below will pick up the new state. + } + } + + const int id_slot = task.id_slot; + const int id_task = task.id; + + server_slot * slot = id_slot != -1 ? get_slot_by_id(id_slot) : get_available_slot(task); + + // + // slot scheduling logic + // + + if (slot == nullptr) { + // if no slot is available, we defer this task for processing later + SRV_DBG("no slot is available, defer task, id_task = %d\n", id_task); + queue_tasks.defer(std::move(task)); + break; + } + + if (slot->is_processing() || slot->hydra_transferring->load()) { + // if requested slot is unavailable, we defer this task for processing later + SRV_DBG("requested slot is unavailable, defer task, id_task = %d\n", id_task); + queue_tasks.defer(std::move(task)); + break; + } + + if (task.is_parent()) { + // try getting free slots for all child tasks + size_t n_child_tasks = task.child_tasks.size(); + std::vector child_slots = get_free_slots(n_child_tasks, slot->id); + if (child_slots.size() < n_child_tasks) { + SRV_DBG("not enough free slots for child tasks, n_free = %zu, n_children = %zu, defer task, id_task = %d\n", child_slots.size(), n_child_tasks, id_task); + queue_tasks.defer(std::move(task)); + break; + } + if (!launch_slots_with_parent_task(*slot, child_slots, std::move(task))) { + SRV_ERR("failed to launch slot with parent task, id_task = %d\n", id_task); + break; // drop the task + } + } else if (!launch_slot_with_task(*slot, std::move(task))) { + SRV_ERR("failed to launch slot with task, id_task = %d\n", id_task); + break; // drop the task + } + + if (params_base.cache_idle_slots) { + for (auto & s : slots) { + if (!s.is_processing() && !s.hydra_transferring->load()) { + slot_save_and_clear(s); + } + } + } + } break; + case SERVER_TASK_TYPE_CANCEL: + { + // release slot linked with the task id + for (auto & slot : slots) { + if (slot.task && slot.task->id == task.id_target) { + slot.release(); + break; + } + } + } break; + case SERVER_TASK_TYPE_CONTROL: + { + auto res = std::make_unique(); + res->id = task.id; + + server_slot * slot = get_slot_by_cmpl_id(task.params.control_cmpl_id); + if (slot == nullptr) { + res->success = false; + res->message = "no active completion for this id"; + queue_results.send(std::move(res)); + break; + } + + if (task.params.control_action == "reasoning_end") { + // the budget sampler only exists when reasoning control was armed + if (!slot->task->params.sampling.reasoning_control) { + res->success = false; + res->message = "reasoning control not enabled for this completion"; + queue_results.send(std::move(res)); + break; + } + // act on the live slot mid generation, never defer + common_sampler_reasoning_budget_force(slot->smpl.get()); + res->success = true; + } else { + res->success = false; + res->message = "unknown control action"; + } + + queue_results.send(std::move(res)); + } break; + case SERVER_TASK_TYPE_NEXT_RESPONSE: + { + // do nothing + } break; + case SERVER_TASK_TYPE_METRICS: + { + json slots_data = json::array(); + + int n_idle_slots = 0; + int n_processing_slots = 0; + + for (server_slot & slot : slots) { + json slot_data = slot.to_json(slots_debug == 0); + + if (slot.is_processing() || slot.hydra_transferring->load()) { + n_processing_slots++; + } else { + n_idle_slots++; + } + + slots_data.push_back(slot_data); + } + SRV_DBG("n_idle_slots = %d, n_processing_slots = %d\n", n_idle_slots, n_processing_slots); + + auto res = std::make_unique(); + res->id = task.id; + res->slots_data = std::move(slots_data); + res->n_idle_slots = n_idle_slots; + res->n_processing_slots = n_processing_slots; + res->n_tasks_deferred = queue_tasks.queue_tasks_deferred_size(); + res->t_start = metrics.t_start; + + res->n_prompt_tokens_processed_total = metrics.n_prompt_tokens_processed_total; + res->t_prompt_processing_total = metrics.t_prompt_processing_total; + res->n_tokens_predicted_total = metrics.n_tokens_predicted_total; + res->t_tokens_generation_total = metrics.t_tokens_generation_total; + + res->n_tokens_max = metrics.n_tokens_max; + + res->n_prompt_tokens_processed = metrics.n_prompt_tokens_processed; + res->t_prompt_processing = metrics.t_prompt_processing; + res->n_tokens_predicted = metrics.n_tokens_predicted; + res->t_tokens_generation = metrics.t_tokens_generation; + + res->n_decode_total = metrics.n_decode_total; + res->n_busy_slots_total = metrics.n_busy_slots_total; + + if (task.metrics_reset_bucket) { + metrics.reset_bucket(); + } + queue_results.send(std::move(res)); + } break; + case SERVER_TASK_TYPE_SLOT_SAVE: + { + if (!check_no_mtmd(task.id)) { + break; + } + + const int id_slot = task.slot_action.id_slot; + server_slot * slot = get_slot_by_id(id_slot); + if (slot == nullptr) { + send_error(task, "Invalid slot ID", ERROR_TYPE_INVALID_REQUEST); + break; + } + if (slot->is_processing()) { + // if requested slot is unavailable, we defer this task for processing later + SRV_DBG("requested slot is unavailable, defer task, id_task = %d\n", task.id); + queue_tasks.defer(std::move(task)); + break; + } + + const size_t token_count = slot->prompt.tokens.size(); + const int64_t t_start = ggml_time_us(); + + std::string filename = task.slot_action.filename; + std::string filepath = task.slot_action.filepath; + + const llama_tokens & tokens = slot->prompt.tokens.get_tokens(); + const size_t nwrite = llama_state_seq_save_file(ctx_tgt, filepath.c_str(), slot->id, tokens.data(), token_count); + + const int64_t t_end = ggml_time_us(); + const double t_save_ms = (t_end - t_start) / 1000.0; + + auto res = std::make_unique(); + res->id = task.id; + res->id_slot = id_slot; + res->filename = filename; + res->is_save = true; + res->n_tokens = token_count; + res->n_bytes = nwrite; + res->t_ms = t_save_ms; + queue_results.send(std::move(res)); + } break; + case SERVER_TASK_TYPE_SLOT_RESTORE: + { + if (!check_no_mtmd(task.id)) break; + const int id_slot = task.slot_action.id_slot; + server_slot * slot = get_slot_by_id(id_slot); + if (slot == nullptr) { + send_error(task, "Invalid slot ID", ERROR_TYPE_INVALID_REQUEST); + break; + } + if (slot->is_processing()) { + // if requested slot is unavailable, we defer this task for processing later + SRV_DBG("requested slot is unavailable, defer task, id_task = %d\n", task.id); + queue_tasks.defer(std::move(task)); + break; + } + + const int64_t t_start = ggml_time_us(); + + std::string filename = task.slot_action.filename; + std::string filepath = task.slot_action.filepath; + + llama_tokens tokens; + tokens.resize(slot->n_ctx); + size_t token_count = 0; + size_t nread = llama_state_seq_load_file(ctx_tgt, filepath.c_str(), slot->id, tokens.data(), tokens.size(), &token_count); + if (nread == 0) { + slot->prompt.tokens.clear(); // KV may already been invalidated? + send_error(task, "Unable to restore slot, no available space in KV cache or invalid slot save file", ERROR_TYPE_INVALID_REQUEST); + break; + } + tokens.resize(token_count); + slot->prompt.tokens.clear(); + slot->prompt.tokens.insert(tokens); + + const int64_t t_end = ggml_time_us(); + const double t_restore_ms = (t_end - t_start) / 1000.0; + + auto res = std::make_unique(); + res->id = task.id; + res->id_slot = id_slot; + res->filename = filename; + res->is_save = false; + res->n_tokens = token_count; + res->n_bytes = nread; + res->t_ms = t_restore_ms; + queue_results.send(std::move(res)); + } break; + case SERVER_TASK_TYPE_SLOT_ERASE: + { + if (!check_no_mtmd(task.id)) { + break; + } + const int id_slot = task.slot_action.id_slot; + server_slot * slot = get_slot_by_id(id_slot); + if (slot == nullptr) { + send_error(task, "Invalid slot ID", ERROR_TYPE_INVALID_REQUEST); + break; + } + if (slot->is_processing() || slot->hydra_transferring->load()) { + // if requested slot is unavailable, we defer this task for processing later + SRV_DBG("requested slot is unavailable, defer task, id_task = %d\n", task.id); + queue_tasks.defer(std::move(task)); + break; + } + + // Erase token cache + const size_t n_erased = slot->prompt.tokens.size(); + + slot->prompt_clear(false); + + auto res = std::make_unique(); + res->id = task.id; + res->id_slot = id_slot; + res->n_erased = n_erased; + queue_results.send(std::move(res)); + } break; + case SERVER_TASK_TYPE_GET_LORA: + { + // TODO @ngxson : make lora_adapters a dedicated member of server_context + auto & loras = params_base.lora_adapters; + auto res = std::make_unique(); + res->id = task.id; + for (size_t i = 0; i < loras.size(); ++i) { + auto & lora = loras[i]; + std::string alora_invocation_string = ""; + const uint64_t n_alora_tokens = llama_adapter_get_alora_n_invocation_tokens(lora.ptr); + llama_tokens alora_invocation_tokens; + if (n_alora_tokens) { + const llama_token * alora_tokens = llama_adapter_get_alora_invocation_tokens(lora.ptr); + for (uint64_t j = 0; j < n_alora_tokens; ++j) { + alora_invocation_string += common_token_to_piece(vocab, alora_tokens[j]); + alora_invocation_tokens.push_back(alora_tokens[j]); + } + } + res->loras.push_back(server_task_result_get_lora::lora{ + lora, + alora_invocation_string, + alora_invocation_tokens, + }); + } + queue_results.send(std::move(res)); + } break; + case SERVER_TASK_TYPE_SET_LORA: + { + auto new_loras = construct_lora_list(task.set_lora); + // logging + for (size_t i = 0; i < new_loras.size(); ++i) { + SRV_INF("set lora adapter idx=%zu scale=%f\n", i, new_loras[i].scale); + } + // TODO @ngxson : make lora_adapters a dedicated member of server_context + params_base.lora_adapters = new_loras; + auto res = std::make_unique(); + res->id = task.id; + queue_results.send(std::move(res)); + } break; + + // epic #610 WS2: HYDRA task dispatch moved to hydra_process_task() + // (defined in hydra-server-context.cpp). In seam mode the extension + // claims these via handle_task(); in legacy mode this fall-through + // calls the same method. Both modes run identical code. + case SERVER_TASK_TYPE_HYDRA_STATE_GET: + case SERVER_TASK_TYPE_HYDRA_STATE_PUT: + case SERVER_TASK_TYPE_HYDRA_STATE_META: + case SERVER_TASK_TYPE_HYDRA_ENGINE_CONFIGURE: + case SERVER_TASK_TYPE_HYDRA_ENGINE_INFO: + case SERVER_TASK_TYPE_HYDRA_ENGINE_PREFILL: + case SERVER_TASK_TYPE_HYDRA_ENGINE_DECODE: + case SERVER_TASK_TYPE_HYDRA_DECODE_APPLY: + case SERVER_TASK_TYPE_HYDRA_ENGINE_SET_EXPERT_MODE: + case SERVER_TASK_TYPE_HYDRA_ENGINE_SWAP_QUANT: + case SERVER_TASK_TYPE_HYDRA_ENGINE_PIPELINE_ATTACH: + hydra_process_task(task); + break; + + } + } + + // epic #610 WS3.5: Hydra helper functions — defined in hydra-server-context.cpp + // (same TU via bottom #include). Declarations here so the compiler resolves + // the out-of-class definitions. + static int hydra_classify_config_key(const std::string & key); + static const char * hydra_tier_label(int tier); + + bool hydra_apply_t1_config(common_params & params, llama_context * ctx, + const json & cfg, + std::map & params_applied); + + void hydra_apply_t3_mutators(llama_context * ctx, const json & cfg, + std::vector & deferred_keys); + + struct hydra_config_result { + int highest_tier = 0; + std::map params_applied; + std::vector deferred_keys; + json t2t3_subset = json::object(); + bool ok = true; + std::string error; + uint64_t state_chunk_size_applied = 0; + std::vector unrecognized_keys; + std::vector rejected_keys; + }; + + hydra_config_result hydra_apply_config(const json & cfg, bool sync); + + bool apply_pending_hydra_config(); + bool apply_t2_rebuild(const std::string & pending_json); + bool apply_t3_rebuild(bool force = false); }; //