diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 9b955a1c77f4..ac034d595236 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -182,6 +182,13 @@ target_link_libraries(test-hydra-rpc-stale-sock PRIVATE ggml) # #470: test uses ggml_backend_buffer_copy_tensor (declared in ggml-backend-impl.h). target_include_directories(test-hydra-rpc-stale-sock PRIVATE ${PROJECT_SOURCE_DIR}/ggml/src) +# epic #610 WS1: hermetic A/B seam checks (HYDRA_EXT_MODE parsing + the WS1 +# no-op factory). Needs the server-context TU so hydra_create_extension() and +# the inline mode parser are visible. +llama_build_and_test(test-hydra-ext-ab.cpp) +target_link_libraries(test-hydra-ext-ab PRIVATE server-context) +target_include_directories(test-hydra-ext-ab PRIVATE ${CMAKE_SOURCE_DIR}/tools/server) + if (NOT WIN32 OR NOT BUILD_SHARED_LIBS) # these tests are disabled on Windows because they use internal functions not exported with LLAMA_API (when building with shared libraries) llama_build_and_test(test-sampling.cpp) diff --git a/tests/test-hydra-ext-ab.cpp b/tests/test-hydra-ext-ab.cpp new file mode 100644 index 000000000000..a6a21fdb5bb4 --- /dev/null +++ b/tests/test-hydra-ext-ab.cpp @@ -0,0 +1,83 @@ +// epic #610 WS1: hermetic A/B seam checks. +// +// What this can test without a model/GPU: +// - HYDRA_EXT_MODE env parsing (legacy default, "seam" activates seam) +// - the factory returns the expected no-op WS1 implementation +// - the no-op contract: hooks don't claim tasks / don't alter the loop +// +// What CANNOT be tested hermetically: a server_context_impl requires a loaded +// model+context, so the behavioral A/B (run the SAME scenario through +// HYDRA_EXT_MODE=legacy vs =seam and diff the outputs) is a loopback / live-rig +// step, documented as WS4 in the epic. + +#include "server-hydra-extension.h" + +#include +#include +#include +#include + +static int g_failures = 0; + +static void expect(const char * what, bool ok) { + if (!ok) { + fprintf(stderr, "FAIL: %s\n", what); + g_failures++; + } +} + +// setenv/unsetenv are POSIX; on Windows use _putenv_s (empty = unset). +#if defined(_WIN32) +static void set_ext_mode(const char * value) { + _putenv_s("HYDRA_EXT_MODE", value ? value : ""); +} +#else +static void set_ext_mode(const char * value) { + if (value == nullptr) { + unsetenv("HYDRA_EXT_MODE"); + } else { + setenv("HYDRA_EXT_MODE", value, 1); + } +} +#endif + +int main() { + // --- mode parsing (WS5: default = seam unless explicitly legacy) ---- + set_ext_mode(nullptr); + expect("unset HYDRA_EXT_MODE -> seam (default)", hydra_ext_mode_seam()); + + set_ext_mode("legacy"); + expect("HYDRA_EXT_MODE=legacy -> legacy", !hydra_ext_mode_seam()); + + set_ext_mode("seam"); + expect("HYDRA_EXT_MODE=seam -> seam", hydra_ext_mode_seam()); + + set_ext_mode("garbage"); + expect("HYDRA_EXT_MODE=garbage -> seam (default)", hydra_ext_mode_seam()); + + set_ext_mode("LEGACY"); // case-sensitive: not "legacy" + expect("HYDRA_EXT_MODE=LEGACY -> seam (default)", hydra_ext_mode_seam()); + + // --- factory + WS1 no-op contract --------------------------------- + std::unique_ptr ext = hydra_create_extension(); + expect("factory returns non-null", ext != nullptr); + if (ext) { + // WS2/WS3 impl name (handle_task routes to hydra_process_task, and + // pre_loop/on_empty_batch replicate the update_slots clusters). + expect("impl name is hydra-task-ws2", std::strcmp(ext->name(), "hydra-task-ws2") == 0); + } + + // The no-op hooks take a server_context_impl&, which cannot be constructed + // here (needs a loaded model). Their return values are pinned by the WS1 + // contract: handle_task=false (never claims), pre_loop=false (never skips + // the decode loop), on_empty_batch=false (never handles the empty batch). + // Behavioral parity is verified by the live-rig A/B in WS4. + + if (g_failures != 0) { + fprintf(stderr, "test-hydra-ext-ab FAILED (%d)\n", g_failures); + return 1; + } + + fprintf(stderr, "%s", "test-hydra-ext-ab OK\n"); + return 0; +} diff --git a/tools/server/hydra-server-context.cpp b/tools/server/hydra-server-context.cpp new file mode 100644 index 000000000000..80107ee5daaf --- /dev/null +++ b/tools/server/hydra-server-context.cpp @@ -0,0 +1,4695 @@ +// Hydra A/B extension seam implementation (epic #610). +// +// This file is NOT an independent translation unit. It is #include'd at the +// bottom of server-context.cpp, so it compiles as part of that TU and can reach +// server_context_impl's private members through the friend declaration on +// hydra_engine_extension (and, for hydra_process_task, because it is a member +// of server_context_impl itself). Do NOT add this file to CMakeLists.txt. +// +// WS1: no-op extension hooks (handle_task/pre_loop/on_empty_batch return false), +// so seam mode is behavior-identical to legacy. +// WS2: the HYDRA task dispatch is extracted into server_context_impl:: +// hydra_process_task(), called from BOTH the legacy switch path (via a +// thin fall-through in process_single_task) and the seam handle_task(). +// Both modes run the exact same method, so A/B parity is by construction; +// the A/B toggle then proves the seam plumbing (routing + claiming) is +// behavior-identical to the legacy switch. +// WS3: update_slots() clusters move into pre_loop()/on_empty_batch(). + +#include "server-hydra-extension.h" + +#include "server-task.h" + +// --------------------------------------------------------------------------- +// WS2: HYDRA task dispatch — member of server_context_impl, defined in this TU. +// --------------------------------------------------------------------------- + // epic #610 WS2: Hydra task dispatch, moved out of process_single_task(). + // Same TU (see #include at bottom of server-context.cpp) so member access + // to server_context_impl privates is available. A switch(task.type) wrapper + // keeps all internal break/continue semantics identical to the inline code. + void server_context_impl::hydra_process_task(server_task & task) { + switch (task.type) { + case SERVER_TASK_TYPE_HYDRA_STATE_GET: + { + // M1: background serialization thread — inference loop continues during state transfer. + // llama_state_seq_get_data reads KV cells for an IDLE sequence; llama_decode + // writes cells for ACTIVE sequences only — no memory overlap for different seq IDs. + const int id_slot = task.hydra_action.id_slot; + auto res = std::make_unique(); + res->id = task.id; + res->id_slot = id_slot; + res->op = HYDRA_OP_STATE_GET; + + server_slot * slot = get_slot_by_id(id_slot); + if (slot == nullptr) { + res->rpc_status = HYDRA_STATUS_NOT_FOUND; + res->error = "invalid slot ID"; + queue_results.send(std::move(res)); + break; + } + if (slot->is_processing() || slot->hydra_transferring->load()) { + res->rpc_status = HYDRA_STATUS_BUSY; + queue_results.send(std::move(res)); + break; + } + + // Snapshot on inference thread (cheap — dry-run serialization, no GPU copies). + const size_t state_size = llama_state_seq_get_size(ctx_tgt, slot->id); + int actual_n_past = slot->n_prompt_tokens_cache + slot->n_decoded; + // Cold prefill: n_prompt_tokens_cache is still 0 so n_decoded (1) dominates. + // Use prompt token count instead — matches STATE_META fallback. + if (slot->n_prompt_tokens_cache == 0 && slot->prompt.tokens.size() > 0) { + actual_n_past = (int)slot->prompt.tokens.size(); + } + res->n_past = actual_n_past; + res->rpc_status = HYDRA_STATUS_OK; + // M-Perf.9 #289: surface model identity alongside the state + // bytes so the Coordinator can record the model that built + // the KV (for cross-model safety on restore). The background + // thread that streams the bytes to the socket can mutate + // res->state_data freely; the model fields are immutable for + // the duration of the response. + res->model_alias = model_name; + res->model_path = params_base.model.path; + if (model_tgt) { + res->tokenizer = llama_model_get_tokenizer_model(model_tgt); + res->model_name = llama_model_get_display_name(model_tgt); + res->model_quant = llama_model_get_quant_label(model_tgt); + res->model_capabilities = llama_model_get_capabilities_bitfield(model_tgt); + } + SRV_INF("hydra: STATE_GET slot=%d n_past=%d state=%.1f MiB — async\n", + id_slot, res->n_past, state_size / (1024.0 * 1024.0)); + + slot->hydra_transferring->store(true); + + // M2: stream directly to socket (zero-copy). + // Runs SYNCHRONOUSLY on the inference thread to avoid + // concurrent ggml-RPC socket access with llama_decode + // on another slot (fixes crash at ggml-rpc.cpp:532). + // The coordinator already does Store Put as fire-and-forget + // so blocking here only delays slot release, not decode. + const int snap_seq_id = slot->id; + llama_context * snap_ctx = ctx_tgt; + // shared_ptr keeps the atomic alive even if the slot is reallocated + std::shared_ptr> flag_ptr = slot->hydra_transferring; + const int hydra_fd = task.hydra_action.hydra_fd; + + // Capture prompt tokens for M1 path header (slot is valid on inference thread) + const llama_tokens prompt_tokens_get = slot->prompt.tokens.get_text_tokens(); + const int32_t n_past_val = res->n_past; + + // Snapshot the most recent native checkpoint so STATE_PUT can + // register it instead of fabricating one at the final position. + // Fabricating at pos_max=n-1 corrupts hybrid/recurrent model + // decode because the recurrent state is one token ahead of the + // decode resume point — it has already processed the final token. + std::vector snapshot_ckpt; + uint8_t hdr_flags = 0x00; + int32_t ckpt_pos_min = 0, ckpt_pos_max = 0; + int64_t ckpt_n_tokens = 0; + if (!slot->prompt.checkpoints.empty()) { + hdr_flags |= 0x01; + const auto & ckpt = slot->prompt.checkpoints.back(); + ckpt_pos_min = ckpt.pos_min; + ckpt_pos_max = ckpt.pos_max; + ckpt_n_tokens = ckpt.n_tokens; + + // Hydra M2-stream double-write fix (#470/#620): serialize the + // recurrent-only capture (data_tgt_recr, PARTIAL_ONLY) instead of + // the full data_tgt. The full live state that follows on the wire + // already carries the attention bytes at the live position, so + // sending the full checkpoint duplicates the attention portion + // (which scales with ctx). The recurrent state is genuinely needed + // at BOTH positions, hence the separate recr-only capture. + // hdr_flags bit 0x02 marks a recurrent-only checkpoint section so + // STATE_PUT/DECODE_APPLY can read it back with matched PARTIAL_ONLY + // flags. Fall back to the full capture when the checkpoint has no + // recr buffer (e.g. it was registered from an old 0x02 blob) — a + // PARTIAL_ONLY read of a full-written buffer is a CUDA memory error. + const bool use_recr = !ckpt.data_tgt_recr.empty(); + if (use_recr) { + hdr_flags |= 0x02; + } + const uint64_t tgt_sz = use_recr ? ckpt.data_tgt_recr.size() : ckpt.data_tgt.size(); + const uint64_t dft_sz = use_recr ? ckpt.data_dft_recr.size() : ckpt.data_dft.size(); + const uint8_t * tgt_ptr = use_recr ? ckpt.data_tgt_recr.data() : ckpt.data_tgt.data(); + const uint8_t * dft_ptr = use_recr ? ckpt.data_dft_recr.data() : ckpt.data_dft.data(); + const size_t ckpt_hdr_sz = 4 + 4 + 8 + 8 + (size_t)tgt_sz + 8 + (size_t)dft_sz; + snapshot_ckpt.resize(ckpt_hdr_sz); + size_t off = 0; + memcpy(snapshot_ckpt.data() + off, &ckpt_pos_min, 4); off += 4; + memcpy(snapshot_ckpt.data() + off, &ckpt_pos_max, 4); off += 4; + memcpy(snapshot_ckpt.data() + off, &ckpt_n_tokens, 8); off += 8; + memcpy(snapshot_ckpt.data() + off, &tgt_sz, 8); off += 8; + if (tgt_sz > 0) { memcpy(snapshot_ckpt.data() + off, tgt_ptr, (size_t)tgt_sz); off += (size_t)tgt_sz; } + memcpy(snapshot_ckpt.data() + off, &dft_sz, 8); off += 8; + if (dft_sz > 0) memcpy(snapshot_ckpt.data() + off, dft_ptr, (size_t)dft_sz); + } + + { + SRV_INF("hydra: STATE_GET streaming (fd=%d state=%.1f MiB)\n", + hydra_fd, state_size / (1024.0 * 1024.0)); + if (hydra_fd >= 0) { + // M2 path: stream v2/v3 blob (header + checkpoint + GPU state) to fd. + // Response header + meta JSON sent first, then v2 header bytes, + // then llama_state_seq_get_data_to_fd writes GPU state directly. + const size_t n_tok = prompt_tokens_get.size(); + const uint32_t hdr_n_tok = (uint32_t)n_tok; + const uint32_t hdr_n_past = (uint32_t)n_past_val; + // 0x03 = v3 blob: checkpoint section carries recurrent-only + // captures (data_tgt_recr, PARTIAL_ONLY). 0x02 = v2 blob: + // checkpoint section carries the full data_tgt. Bumped so a + // mixed-version fleet never misreads a smaller (recr-only) + // checkpoint as a full one. + const uint8_t version_byte = 0x03; + const size_t base_hdr_size = 1 + 4 + 4 + n_tok * sizeof(llama_token) + 1; + const size_t hdr_size = base_hdr_size + snapshot_ckpt.size(); + const size_t total_payload = hdr_size + state_size; + + // Build v2 header buffer + std::vector v2_hdr(hdr_size); + { + size_t off = 0; + memcpy(v2_hdr.data() + off, &version_byte, 1); off += 1; + memcpy(v2_hdr.data() + off, &hdr_n_past, 4); off += 4; + memcpy(v2_hdr.data() + off, &hdr_n_tok, 4); off += 4; + memcpy(v2_hdr.data() + off, prompt_tokens_get.data(), n_tok * sizeof(llama_token)); off += n_tok * sizeof(llama_token); + memcpy(v2_hdr.data() + off, &hdr_flags, 1); off += 1; + if (!snapshot_ckpt.empty()) { + memcpy(v2_hdr.data() + off, snapshot_ckpt.data(), snapshot_ckpt.size()); + off += snapshot_ckpt.size(); + } + } + + { + json meta_j; + meta_j["n_past"] = res->n_past; + meta_j["state_size"] = (uint64_t)state_size; + if (!res->model_alias.empty()) meta_j["model_alias"] = res->model_alias; + if (!res->model_path.empty()) meta_j["model_path"] = res->model_path; + if (!res->tokenizer.empty()) meta_j["tokenizer"] = res->tokenizer; + if (!res->model_name.empty()) meta_j["model_name"] = res->model_name; + if (!res->model_quant.empty()) meta_j["model_quant"] = res->model_quant; + if (res->model_capabilities) meta_j["model_capabilities"] = res->model_capabilities; + const std::string meta_str = meta_j.dump(); + + const uint32_t meta_len = (uint32_t)meta_str.size(); + const uint64_t payload_l = (uint64_t)total_payload; + uint8_t hdr[HYDRA_RES_HEADER_SIZE] = {}; + hdr[0] = HYDRA_STATUS_OK; + hdr[1] = (meta_len) & 0xFF; + hdr[2] = (meta_len >> 8) & 0xFF; + hdr[3] = (meta_len >> 16) & 0xFF; + memcpy(hdr + 4, &payload_l, 8); + hydra_send_all(hydra_fd, hdr, HYDRA_RES_HEADER_SIZE); + hydra_send_all(hydra_fd, meta_str.data(), meta_str.size()); + + // Write v2 blob header before GPU state — STATE_PUT needs tokens + checkpoint + hydra_send_all(hydra_fd, v2_hdr.data(), v2_hdr.size()); + + res->header_sent = true; // META + header + v2-hdr before payload + } + // Stream GPU state to fd (zero-copy from GPU memory) + const size_t streamed = llama_state_seq_get_data_to_fd(snap_ctx, snap_seq_id, hydra_fd, nullptr); + if (streamed != state_size) { + // TOCTOU: state size changed between get_size (header already + // promised state_size bytes) and the stream, or the stream + // failed mid-way. The wire framing is now broken — the only + // safe recovery is to kill the connection. Use shutdown(), + // not close(): the RPC connection loop owns the fd and will + // close it when its next read fails; closing here would race + // (double-close / fd-reuse against unrelated threads). + res->rpc_status = HYDRA_STATUS_ERROR; + res->error = "llama_state_seq_get_data_to_fd streamed " + + std::to_string(streamed) + " B, expected " + + std::to_string(state_size) + " B"; + ::shutdown(hydra_fd, SHUT_RDWR); + } else { + res->streamed_bytes = total_payload; + } + } else { + // M1 path: buffer in memory, RPC thread sends afterwards. + // v3 blob format (0x03): [1B version][4B n_past][4B n_tok][n_tok*4B tokens] + // [1B flags (bit 0 = has_checkpoint)] + // [if flags & 0x01: 4B pos_min | 4B pos_max | 8B n_tokens | 8B tgt_sz | recr_tgt_data | 8B dft_sz | recr_dft_data] + // [raw KV state from llama_state_seq_get_data] + const size_t n_tok = prompt_tokens_get.size(); + const uint32_t hdr_n_tok = (uint32_t)n_tok; + const uint32_t hdr_n_past = (uint32_t)n_past_val; + const uint8_t version_byte = 0x03; + const size_t base_hdr_size = 1 + 4 + 4 + n_tok * sizeof(llama_token) + 1; // version + n_past + n_tok + tokens + flags + const size_t hdr_size = base_hdr_size + snapshot_ckpt.size(); + + // TOCTOU retry: if another slot grew the state between + // get_size (inference thread) and get_data (background thread), + // the copy returns 0. Retry up to 3 times with fresh sizing. + size_t buf_size = hdr_size + state_size; + res->state_data.resize(buf_size); + { + size_t off = 0; + memcpy(res->state_data.data() + off, &version_byte, 1); off += 1; + memcpy(res->state_data.data() + off, &hdr_n_past, 4); off += 4; + memcpy(res->state_data.data() + off, &hdr_n_tok, 4); off += 4; + memcpy(res->state_data.data() + off, prompt_tokens_get.data(), n_tok * sizeof(llama_token)); off += n_tok * sizeof(llama_token); + memcpy(res->state_data.data() + off, &hdr_flags, 1); off += 1; + if (!snapshot_ckpt.empty()) { + memcpy(res->state_data.data() + off, snapshot_ckpt.data(), snapshot_ckpt.size()); + off += snapshot_ckpt.size(); + } + } + + size_t cur_state_size = state_size; + size_t copied = 0; + int retries = 3; + while (retries-- > 0) { + copied = llama_state_seq_get_data( + snap_ctx, res->state_data.data() + hdr_size, cur_state_size, snap_seq_id); + if (copied > 0) break; + // State grew — re-measure and retry + cur_state_size = llama_state_seq_get_size(snap_ctx, snap_seq_id); + res->state_data.resize(hdr_size + cur_state_size); + } + if (copied == 0) { + res->rpc_status = HYDRA_STATUS_ERROR; + res->error = "llama_state_get_data failed after 3 retries"; + res->state_data.clear(); + } + } + flag_ptr->store(false); + // M2 streams to fd (streamed_bytes); M1 buffers into state_data. + const uint64_t out_bytes = (hydra_fd >= 0) + ? res->streamed_bytes + : (uint64_t) res->state_data.size(); + SRV_INF("hydra: STATE_GET done slot=%d rpc_status=%d path=%s bytes=%" PRIu64 "\n", + snap_seq_id, res->rpc_status, + hydra_fd >= 0 ? "M2-stream" : "M1-buffer", out_bytes); + queue_results.send(std::move(res)); + } + + // STATE_GET is synchronous — blocks until KV state is fully + // streamed to the socket. The coordinator's Store Put is + // fire-and-forget, so only slot release is delayed. + } break; + + case SERVER_TASK_TYPE_HYDRA_STATE_PUT: + { + const int id_slot = task.hydra_action.id_slot; + auto res = std::make_unique(); + res->id = task.id; + res->id_slot = id_slot; + res->op = HYDRA_OP_STATE_PUT; + + server_slot * slot = get_slot_by_id(id_slot); + if (slot == nullptr) { + res->rpc_status = HYDRA_STATUS_NOT_FOUND; + res->error = "invalid slot ID"; + queue_results.send(std::move(res)); + break; + } + if (slot->is_processing() || slot->hydra_transferring->load()) { + res->rpc_status = HYDRA_STATUS_BUSY; + queue_results.send(std::move(res)); + break; + } + + // M-Perf.9 #289: populate model identity from resident model. + // model_match = true always (infrastructure only; actual KV + // validation comes when model identity is embedded in the KV header). + res->model_alias = model_name; + res->model_path = params_base.model.path; + res->model_match = true; + if (model_tgt) { + res->tokenizer = llama_model_get_tokenizer_model(model_tgt); + res->model_name = llama_model_get_display_name(model_tgt); + res->model_quant = llama_model_get_quant_label(model_tgt); + res->model_capabilities = llama_model_get_capabilities_bitfield(model_tgt); + } + + // Erase existing checkpoints to avoid collision with restored session state + if (task.hydra_action.erase_existing && !slot->prompt.checkpoints.empty()) { + SLT_INF(*slot, "erasing %zu existing checkpoints before STATE_PUT restore\n", + slot->prompt.checkpoints.size()); + slot->prompt.checkpoints.clear(); + } + + const auto & buf = task.hydra_action.state_data; + + // Detect v2/v3 blob (0x02/0x03 at offset 0) vs legacy format (no version byte). + // v2: [1B version=0x02][4B n_past][4B n_tok][n_tok*4B tokens][1B flags][?ckpt?][KV state] + // v3: same, but the checkpoint section may be a recurrent-only capture + // (hdr_flags bit 0x02 set). Bumped to 0x03 by the M2-stream double-write fix. + const bool is_v2 = buf.size() >= 1 && (buf[0] == 0x02 || buf[0] == 0x03); + + size_t hdr_offset = 0; + int32_t hdr_n_tok = 0; + int32_t hdr_n_past = 0; + bool has_chkpt = false; + bool ckpt_is_recr_only = false; + int32_t ckpt_pos_min_in = 0, ckpt_pos_max_in = 0; + int64_t ckpt_n_tokens_in = 0; + std::vector ckpt_tgt_data, ckpt_dft_data; + + if (is_v2) { + // v2/v3: version at [0], n_past at [1..4], n_tok at [5..8] + if (buf.size() >= 9) { + memcpy(&hdr_n_past, buf.data() + 1, 4); + memcpy(&hdr_n_tok, buf.data() + 5, 4); + } + const size_t token_start = 9; + const size_t token_end = token_start + (size_t)hdr_n_tok * sizeof(llama_token); + hdr_offset = token_end; + if (hdr_offset < buf.size()) { + const uint8_t flags = buf[hdr_offset]; + hdr_offset += 1; // past flags byte + // bit 0x01 = has checkpoint; bit 0x02 = checkpoint section is + // recurrent-only (PARTIAL_ONLY). A v3 blob that fell back to the + // full capture (old-registered checkpoint) leaves 0x02 clear. + ckpt_is_recr_only = (flags & 0x02) != 0; + if (flags & 0x01) { + // Parse checkpoint: 4B pos_min | 4B pos_max | 8B n_tokens | 8B tgt_sz | tgt_data | 8B dft_sz | dft_data + if (hdr_offset + 4 + 4 + 8 + 8 <= buf.size()) { + memcpy(&ckpt_pos_min_in, buf.data() + hdr_offset, 4); hdr_offset += 4; + memcpy(&ckpt_pos_max_in, buf.data() + hdr_offset, 4); hdr_offset += 4; + memcpy(&ckpt_n_tokens_in, buf.data() + hdr_offset, 8); hdr_offset += 8; + uint64_t tgt_sz_in; + memcpy(&tgt_sz_in, buf.data() + hdr_offset, 8); hdr_offset += 8; + if (tgt_sz_in > 0 && hdr_offset + tgt_sz_in <= buf.size()) { + ckpt_tgt_data.assign(buf.data() + hdr_offset, buf.data() + hdr_offset + (size_t)tgt_sz_in); + hdr_offset += (size_t)tgt_sz_in; + } + if (hdr_offset + 8 <= buf.size()) { + uint64_t dft_sz_in; + memcpy(&dft_sz_in, buf.data() + hdr_offset, 8); hdr_offset += 8; + if (dft_sz_in > 0 && hdr_offset + dft_sz_in <= buf.size()) { + ckpt_dft_data.assign(buf.data() + hdr_offset, buf.data() + hdr_offset + (size_t)dft_sz_in); + hdr_offset += (size_t)dft_sz_in; + } + } + has_chkpt = true; + } + } + } + // Restore tokens from token_start + if (hdr_n_tok > 0 && token_start + (size_t)hdr_n_tok * sizeof(llama_token) <= buf.size()) { + slot->prompt.tokens.clear(); + const llama_token * tok_ptr = (const llama_token *)(buf.data() + token_start); + llama_tokens restored_tokens(tok_ptr, tok_ptr + (size_t)hdr_n_tok); + slot->prompt.tokens.insert(restored_tokens); + } + } else { + // Legacy v1 format + if (buf.size() >= 8) { + memcpy(&hdr_n_past, buf.data(), 4); + memcpy(&hdr_n_tok, buf.data() + 4, 4); + hdr_offset = 8 + (size_t)hdr_n_tok * sizeof(llama_token); + } + if (hdr_offset > 0 && hdr_offset <= buf.size()) { + const size_t n_tokens = (size_t)hdr_n_tok; + slot->prompt.tokens.clear(); + if (n_tokens > 0) { + const llama_token * tok_ptr = (const llama_token *)(buf.data() + 8); + llama_tokens restored_tokens(tok_ptr, tok_ptr + n_tokens); + slot->prompt.tokens.insert(restored_tokens); + } + } + } + const bool has_hdr = hdr_offset > 0 && hdr_offset <= buf.size(); + const uint8_t * state_ptr = has_hdr ? buf.data() + hdr_offset : buf.data(); + const size_t state_len = has_hdr ? buf.size() - hdr_offset : buf.size(); + const size_t n_read = llama_state_seq_set_data(ctx_tgt, state_ptr, state_len, slot->id); + if (n_read == 0) { + res->rpc_status = HYDRA_STATUS_ERROR; + res->error = "llama_state_set_data returned 0"; + // Tokens were registered before set_data — clear them so the slot + // is not left poisoned (n_past > 0 with no KV cells → pos_min == -1 + // abort on the next decode that touches this slot). + slot->prompt.tokens.clear(); + slot->prompt.checkpoints.clear(); + slot->n_prompt_tokens_cache = 0; + llama_memory_seq_rm(llama_get_memory(ctx_tgt), slot->id, -1, -1); + } else { + // D4: Inject trailing logits into per-slot buffer instead of the + // shared context-wide llama_get_logits(). This avoids the race where + // another slot's decode clobbers restored logits between STATE_PUT + // and the first sample. + const size_t remaining = state_len - n_read; + const size_t expected_logits = (size_t)llama_vocab_n_tokens(vocab) * sizeof(float); + if (remaining == expected_logits) { + const float * src = (const float *)(state_ptr + n_read); + const size_t n_floats = llama_vocab_n_tokens(vocab); + slot->restored_logits.assign(src, src + n_floats); + slot->logits_valid = true; + SRV_INF("hydra: STATE_PUT slot=%d restored %zu logits to per-slot buffer\n", + id_slot, n_floats); + } + + res->rpc_status = HYDRA_STATUS_OK; + res->restored = true; + res->bytes = (uint64_t)n_read; + // #469 trace: log restored state for cross-flow comparison + SRV_DBG("hydra: STATE_PUT slot=%d RESTORED n_past=%d n_prompt_tok=%d state_bytes=%zu just_restored=true\n", + id_slot, hdr_n_tok, hdr_n_tok, n_read); + { + std::string tok_ids; + for (size_t i = 0; i < std::min(16, slot->prompt.tokens.size()); ++i) { + if (i > 0) tok_ids += ","; + tok_ids += std::to_string(slot->prompt.tokens[i]); + } + SRV_DBG("hydra: STATE_PUT slot=%d first16_tokens=[%s] total=%zu\n", + id_slot, tok_ids.c_str(), slot->prompt.tokens.size()); + } + if (hdr_n_tok > 0) { + slot->n_prompt_tokens_cache = hdr_n_tok; + slot->n_decoded = 0; + res->n_past = hdr_n_tok; + + // Register native checkpoint from the blob (v2) or fabricate one (legacy). + // The native checkpoint has pos_max at n-4 (created before the last + // few prompt tokens were decoded), so loading it rewinds the recurrent + // state to a clean position. The old fabricated checkpoint at (0, n-1) + // puts the recurrent state at the final position — one token ahead of + // where decode must resume — corrupting hybrid/recurrent model output. + slot->prompt.checkpoints.clear(); + if (has_chkpt) { + auto & ckpt = slot->prompt.checkpoints.emplace_back(); + ckpt.n_tokens = ckpt_n_tokens_in; + ckpt.pos_min = ckpt_pos_min_in; + ckpt.pos_max = ckpt_pos_max_in; + // New-format (v3) checkpoints carry a recurrent-only capture — + // route it into data_*_recr and tag is_recr_only so the load + // path uses matched PARTIAL_ONLY flags (plus an attention + // seq_rm at pos_max) instead of the full flags=0 restore. + ckpt.is_recr_only = ckpt_is_recr_only; + if (ckpt_is_recr_only) { + ckpt.data_tgt_recr = std::move(ckpt_tgt_data); + ckpt.data_dft_recr = std::move(ckpt_dft_data); + } else { + ckpt.data_tgt = std::move(ckpt_tgt_data); + ckpt.data_dft = std::move(ckpt_dft_data); + } + SLT_INF(*slot, "STATE_PUT registered native checkpoint (pos_min=%d pos_max=%d n_tokens=%" PRId64 " tgt_sz=%zu recr_only=%d)\n", + ckpt.pos_min, ckpt.pos_max, ckpt.n_tokens, ckpt.size(), (int) ckpt.is_recr_only); + } else { + const auto pos_min = llama_memory_seq_pos_min(llama_get_memory(ctx_tgt), slot->id); + create_checkpoint(*slot, 0, (llama_pos)pos_min, (llama_pos)(hdr_n_tok - 1)); + } + slot->just_restored = true; + } + SRV_INF("hydra: STATE_PUT slot=%d restored=%zu B n_past=%d n_prompt_tok=%d\n", + id_slot, n_read, res->n_past, hdr_n_tok); + } + queue_results.send(std::move(res)); + } break; + + case SERVER_TASK_TYPE_HYDRA_STATE_META: + { + const int id_slot = task.hydra_action.id_slot; + auto res = std::make_unique(); + res->id = task.id; + res->id_slot = id_slot; + res->op = HYDRA_OP_STATE_META; + + server_slot * slot = get_slot_by_id(id_slot); + if (slot == nullptr) { + res->rpc_status = HYDRA_STATUS_NOT_FOUND; + res->error = "invalid slot ID"; + queue_results.send(std::move(res)); + break; + } + // META is safe to serve even while processing or transferring (read-only metadata) + int actual_n_past = slot->n_prompt_tokens_cache + slot->n_decoded; + // For cold prefills n_prompt_tokens_cache is 0 — use prompt token count + if (slot->n_prompt_tokens_cache == 0 && slot->prompt.tokens.size() > 0) { + actual_n_past = (int)slot->prompt.tokens.size(); + } + res->n_past = actual_n_past; + res->is_processing = slot->is_processing(); + res->is_transferring = slot->hydra_transferring->load(); + res->state_size = (uint64_t)llama_state_seq_get_size(ctx_tgt, slot->id); + // M-Perf.9 #289: surface model identity. The Coordinator uses + // these to detect cross-model restores — a slot holding a Mini + // KV cache must never have it decoded by a Balanced-loaded model. + res->model_alias = model_name; + res->model_path = params_base.model.path; + if (model_tgt) { + res->tokenizer = llama_model_get_tokenizer_model(model_tgt); + res->model_name = llama_model_get_display_name(model_tgt); + res->model_quant = llama_model_get_quant_label(model_tgt); + res->model_capabilities = llama_model_get_capabilities_bitfield(model_tgt); + } + // #451: populate progress fields based on slot state + switch (slot->state) { + case SLOT_STATE_PROCESSING_PROMPT: + res->operation = "prefill"; + res->tokens_processed = slot->n_prompt_tokens_processed; + // task->n_tokens() is the total tokens to process (fixed); + // prompt.tokens.size() grows during prefill and is WRONG for total. + res->tokens_total = slot->task ? slot->task->n_tokens() : 0; + if (res->tokens_total > 0) { + res->progress = (float)res->tokens_processed / (float)res->tokens_total; + } + res->elapsed_ms = (slot->t_start_process_prompt > 0) + ? (ggml_time_ms() - slot->t_start_process_prompt) : 0; + break; + case SLOT_STATE_GENERATING: + res->operation = "decode"; + res->tokens_processed = slot->n_decoded; + // n_remaining == -1 is the "unlimited generation" sentinel + // (no finite n_predict). Don't compute progress in that case. + if (slot->n_remaining > 0) { + res->tokens_total = slot->n_decoded + slot->n_remaining; + res->progress = (float)res->tokens_processed / (float)res->tokens_total; + } + res->elapsed_ms = (slot->t_start_generation > 0) + ? (ggml_time_ms() - slot->t_start_generation) : 0; + break; + case SLOT_STATE_IDLE: + res->operation = "idle"; + res->progress = 1.0f; + break; + default: + res->operation = "unknown"; + break; + } + // Handle save/restore operations via hydra_transferring flag. + // Clear any stale progress from the prior state since we're + // now in a transferring context, not the previous operation. + if (slot->hydra_transferring->load()) { + res->operation = "save"; + res->progress = 0.0f; + res->tokens_processed = 0; + res->tokens_total = 0; + res->elapsed_ms = 0; + } + res->rpc_status = HYDRA_STATUS_OK; + queue_results.send(std::move(res)); + } break; + + case SERVER_TASK_TYPE_HYDRA_ENGINE_CONFIGURE: + { + auto res = std::make_unique(); + res->id = task.id; + res->op = HYDRA_OP_CONFIGURE; + res->rpc_status = HYDRA_STATUS_OK; + res->success = true; + + // hydra#406: tiered CONFIGURE (T1/T2/T3). Backward compat: + // a legacy {"state_chunk_size":N} payload is treated as a + // degenerate T1 (the original hydra#334 startup call from + // WorkerSchedulerService.cs:2842). + if (task.hydra_action.config_json.empty()) { + res->tier = "T1"; + SRV_INF("hydra: CONFIGURE (empty payload, slot %d) — T1 no-op\n", + task.hydra_action.id_slot); + queue_results.send(std::move(res)); + break; + } + + json cfg; + try { + cfg = json::parse(task.hydra_action.config_json); + } catch (const std::exception & e) { + res->success = false; + res->rpc_status = HYDRA_STATUS_ERROR; + res->error = std::string("CONFIGURE: invalid config_json: ") + e.what(); + SRV_WRN("hydra: CONFIGURE failed to parse config_json (slot %d): %s\n", + task.hydra_action.id_slot, e.what()); + queue_results.send(std::move(res)); + break; + } + + // Route through the shared classify → apply helper. + // sync=false: T2/T3/T4 are staged for the slot-free moment. + hydra_config_result cfg_result = hydra_apply_config(cfg, /*sync=*/false); + + // hydra#470: report generic (T4) keys that cannot be + // applied BEFORE the tier-0 early return — a payload + // whose keys are all unrecognized/rejected still has + // to surface them (zero silent drops). + res->unrecognized_keys = cfg_result.unrecognized_keys; + res->rejected_keys = cfg_result.rejected_keys; + + if (cfg_result.highest_tier == 0) { + // No recognized keys — still emit a T1 success + // (the legacy {"state_chunk_size":N} case). + res->tier = "T1"; + SRV_INF("hydra: CONFIGURE (no recognized keys, slot %d) — T1 no-op\n", + task.hydra_action.id_slot); + queue_results.send(std::move(res)); + break; + } + + if (!cfg_result.ok) { + res->success = false; + res->rpc_status = HYDRA_STATUS_ERROR; + res->error = "CONFIGURE: " + cfg_result.error; + SRV_WRN("hydra: CONFIGURE apply failed (slot %d): %s\n", + task.hydra_action.id_slot, cfg_result.error.c_str()); + queue_results.send(std::move(res)); + break; + } + + // Build the response from the shared helper's result. + res->tier = hydra_tier_label(cfg_result.highest_tier); + res->params_applied = std::move(cfg_result.params_applied); + res->deferred_keys = std::move(cfg_result.deferred_keys); + res->state_chunk_size_applied = cfg_result.state_chunk_size_applied; + + SRV_INF("hydra: CONFIGURE tier=%s applied=%zu deferred=%zu unrecognized=%zu rejected=%zu (slot %d)\n", + res->tier.c_str(), + res->params_applied.size(), + res->deferred_keys.size(), + res->unrecognized_keys.size(), + res->rejected_keys.size(), + task.hydra_action.id_slot); + queue_results.send(std::move(res)); + } break; + + case SERVER_TASK_TYPE_HYDRA_ENGINE_INFO: + { + auto res = std::make_unique(); + res->id = task.id; + res->op = HYDRA_OP_INFO; + res->rpc_status = HYDRA_STATUS_OK; + // M-Perf.9 #289: advertise the model identity features so + // the Coordinator knows it can send `model` in PREFILL and + // expect model_alias/model_path/tokenizer/model_name/model_quant/model_capabilities + // in META responses. + // `preset_aliases` lists every alias loaded from + // --models-preset (empty when no preset is configured). + json preset_aliases_j = json::array(); + for (const auto & [alias, _path] : preset_alias_to_path) { + preset_aliases_j.push_back(alias); + } + // Hydra #287/#260/#348: two-engine "work together" status + // — see specs/rpc-protocol.md's ENGINE_INFO (0x41) + // contract. pipeline_capable stays false until #287's + // PIPELINE half lands; mode only ever reports + // solo/combined until then. solo_active/rpc_backend_active/ + // peer_reachable/combined_head_attached are independent + // booleans (#348) — replaces the old single "role" string + // and the peer_connected/combined_capable field-aliasing. + const int32_t expert_mode = ctx_tgt ? llama_hydra_get_expert_mode(ctx_tgt) : 0; + // Hydra #383 T1 / #375: advertise "combined" capability when this + // engine is ready to serve in COMBINED mode — either via expert-split + // (hydra_combined_head_attached) or via layer-split (hydra_combined_static). + json capabilities_j = {"prefill", "decode", "state_transfer", + "expert_mode", "quant_swap", + "preset", "tokenizer", "model_name", + "model_quant", "model_capabilities", + "merged_decode"}; + if (hydra_combined_head_attached || hydra_combined_static) { + capabilities_j.push_back("combined"); + } + // In layer-split static mode the engine is always in combined mode; + // in expert-split mode it follows the per-request SET_EXPERT_MODE state. + const std::string mode_str = hydra_combined_static ? "combined" + : (expert_mode == 1 ? "combined" : "solo"); + json info_j = { + {"engine", "llama-server-hydra"}, + {"version", "E1"}, + {"capabilities", capabilities_j}, + {"preset_aliases", preset_aliases_j}, + {"solo_active", hydra_solo_active}, + {"rpc_backend_active", hydra_rpc_backend_active}, + {"mode", mode_str}, + {"split_mode", hydra_split_mode}, + {"peer_addr", hydra_peer}, + {"peer_reachable", hydra_peer_reachable}, + {"layer_split", hydra_combined_pattern}, + {"combined_head_attached", hydra_combined_head_attached || hydra_combined_static}, + {"pipeline_capable", false} + }; + res->info_json = info_j.dump(); + queue_results.send(std::move(res)); + } break; + + case SERVER_TASK_TYPE_HYDRA_ENGINE_PREFILL: + { + const int id_slot = task.hydra_action.id_slot; + auto res = std::make_unique(); + res->id = task.id; + res->op = HYDRA_OP_PREFILL; + + // #451: track timing for PREFILL metrics + const int64_t prefill_start_ms = ggml_time_ms(); + + // Set by the model-resolution block below when a real + // `load_model` swap happens. Used at the response site to + // decide whether the post-prefill model identity is the + // freshly loaded model (swap) or the original (no-swap / + // fallback). + bool model_was_swapped = false; + + server_slot * slot = get_slot_by_id(id_slot); + if (slot == nullptr) { + res->rpc_status = HYDRA_STATUS_NOT_FOUND; + res->error = "invalid slot ID"; + queue_results.send(std::move(res)); + break; + } + + if (slot->is_processing()) { + res->rpc_status = HYDRA_STATUS_BUSY; + res->error = "slot is busy"; + queue_results.send(std::move(res)); + break; + } + + // M-Perf.9 #289: parse the optional `model` key from the + // request body and swap the resident model when the preset + // registry knows the alias. The parse is reused for the + // tokenization step below. Falls back to the resident model + // (with `model_fallback:true` in the response) when the + // alias is unknown or no preset is configured. + json parsed_body; + std::string requested_model; + json hydra_cfg; // optional hydra_config object + bool has_hydra_config = false; + if (!task.hydra_action.request_json.empty()) { + try { + parsed_body = json::parse(task.hydra_action.request_json); + if (parsed_body.is_object() && parsed_body.contains("model") + && parsed_body["model"].is_string()) { + requested_model = parsed_body["model"].get(); + } + // hydra_config: optional config object from Hydra.Core + // containing topology/sampling overrides. When present + // with model_path, it drives the model swap directly + // (bypassing the preset alias lookup). + if (parsed_body.is_object() && parsed_body.contains("hydra_config") + && parsed_body["hydra_config"].is_object()) { + hydra_cfg = parsed_body["hydra_config"]; + has_hydra_config = true; + } + } catch (const std::exception & e) { + res->rpc_status = HYDRA_STATUS_BAD_REQUEST; + res->error = std::string("invalid JSON: ") + e.what(); + queue_results.send(std::move(res)); + break; + } + } + + // Apply hydra_config synchronously when present. + // T1 keys (sampling, n_predict, etc.) are applied in-place. + // T2/T3 keys (n_ctx, cache_type, model_path, split_mode, etc.) + // trigger immediate rebuilds on this task-queue thread. + + // #470: Before applying config, probe all RPC peers for + // reconnection. If a peer restarted since the last request, + // its buffers are gone even though model/params haven't + // changed. Without this probe, the T3 rebuild in + // hydra_apply_config → apply_t3_rebuild would skip (params + // unchanged) and the subsequent graph_compute would fail. + if (ctx_tgt && ggml_backend_rpc_check_any_peer_reconnection()) { + SRV_WRN("%s", "hydra: PREFILL: RPC peer reconnected — forcing T3 rebuild\n"); + ctx_tgt->peer_reconnection_pending = true; + } + + if (has_hydra_config) { + SRV_INF("hydra: PREFILL slot=%d applying hydra_config (%zu keys)\n", + id_slot, hydra_cfg.size()); + hydra_config_result cfg_result = hydra_apply_config(hydra_cfg, /*sync=*/true); + if (!cfg_result.ok) { + res->rpc_status = HYDRA_STATUS_ERROR; + res->error = "hydra_config apply failed: " + cfg_result.error; + SRV_WRN("hydra: PREFILL hydra_config apply failed (slot %d): %s\n", + id_slot, cfg_result.error.c_str()); + queue_results.send(std::move(res)); + break; + } + // If the apply may have rebuilt the slots (T3 + // statics or a T4-only generic config both route + // through apply_t3_rebuild → load_model → + // slots.clear()), track it and re-look-up the slot. + // Without the T4 case (hydra#470) the slot pointer + // captured above would dangle into the prefill. + if (hydra_config_requires_slot_relookup(cfg_result.highest_tier)) { + model_was_swapped = true; + res->model_load_ms = (double)(ggml_time_ms() - prefill_start_ms); + SRV_INF("hydra: PREFILL hydra_config T3/T4 applied model_alias='%s' tokenizer='%s' model_name='%s' quant='%s' caps=0x%x\n", + model_name.empty() ? "?" : model_name.c_str(), + model_tgt ? llama_model_get_tokenizer_model(model_tgt) : "", + model_tgt ? llama_model_get_display_name(model_tgt) : "", + model_tgt ? llama_model_get_quant_label(model_tgt) : "", + model_tgt ? llama_model_get_capabilities_bitfield(model_tgt) : 0); + slot = get_slot_by_id(id_slot); + if (slot == nullptr) { + res->rpc_status = HYDRA_STATUS_NOT_FOUND; + res->error = "slot disappeared after hydra_config T3/T4 rebuild"; + queue_results.send(std::move(res)); + break; + } + } + // When hydra_config carries model_path, the model swap is + // handled by apply_t3_rebuild() above — skip the bare + // model alias lookup below. + if (hydra_cfg.contains("model_path")) { + requested_model.clear(); + } + } + + // Fallback: bare model alias lookup when hydra_config didn't + // handle the model swap (no hydra_config, or no model_path). + if (!requested_model.empty()) { + auto it = preset_alias_to_path.find(requested_model); + if (it == preset_alias_to_path.end()) { + SRV_WRN("hydra: PREFILL model='%s' unknown (preset has %zu alias(es)) — falling back to resident '%s'\n", + requested_model.c_str(), preset_alias_to_path.size(), + model_name.c_str()); + res->model_fallback = true; + } else if (it->second != params_base.model.path) { + SRV_INF("hydra: PREFILL model='%s' swapping %s -> %s\n", + requested_model.c_str(), params_base.model.path.c_str(), + it->second.c_str()); + common_params swapped_params = params_base; + // Apply the target alias's full preset so that + // tensor_buft_overrides, n_gpu_layers, split_mode, + // tensor_split, etc. are replaced — not inherited + // from the source model. Intentionally the FULL + // preset (sampling, chat template, n_ctx, etc. + // included), not just tensor-placement keys: a + // real model swap targets a different model, + // which plausibly needs its own sampling + // defaults/chat template too, not just a new + // memory layout. + auto pit = preset_alias_to_preset.find(requested_model); + if (pit != preset_alias_to_preset.end()) { + // Clear inherited tensor_buft_overrides (padded + // to 4096 by common_params_parse_ex) BEFORE + // apply_to_params, which push_back()'s the new + // preset's entries via CLI handlers. Without + // this, the new entries land after the + // nullptr-terminator and exceed the 4096 limit, + // triggering GGML_ASSERT in + // common_model_params_to_llama (#499 regression). + swapped_params.tensor_buft_overrides.clear(); + try { + // apply_to_params() replays CLI handlers + // (parse_tensor_buffer_overrides, the + // n-cpu-moe std::stoi, two-value option + // parsers) which throw on a malformed + // target preset. Uncaught, that exception + // would escape the task-queue loop and + // kill the task thread — fail the swap + // instead. + pit->second.apply_to_params(swapped_params); + hydra_repad_tensor_buft_overrides(swapped_params, "PREFILL swap"); + } catch (const std::exception & e) { + SRV_WRN("hydra: PREFILL swap preset apply for '%s' failed: %s\n", + requested_model.c_str(), e.what()); + res->rpc_status = HYDRA_STATUS_ERROR; + res->error = std::string("model swap preset apply failed: ") + e.what(); + queue_results.send(std::move(res)); + break; + } + SRV_INF("hydra: PREFILL swap applied preset for '%s' " + "(tensor_buft_overrides=%zu entries)\n", + requested_model.c_str(), + swapped_params.tensor_buft_overrides.size()); + } + swapped_params.model.path = it->second; + // Update the alias so model_name is re-derived + // correctly in load_model() (model_name is set from + // model_alias.first when non-empty). + swapped_params.model_alias = { requested_model }; + // #514: tear down COMBINED state before the + // reload — otherwise the engine loads the + // correct model file but keeps routing tokens + // through the stale peer/expert-binding config, + // collapsing decode throughput. + const bool was_combined = hydra_combined_head_attached || hydra_combined_static; + if (was_combined) { + hydra_teardown_combined_before_reload(); + } + const int64_t model_load_start_ms = ggml_time_ms(); + if (!load_model(swapped_params)) { + res->rpc_status = HYDRA_STATUS_ERROR; + res->error = "model swap to '" + requested_model + "' failed"; + queue_results.send(std::move(res)); + break; + } + if (was_combined) { + hydra_reattach_combined_after_reload(); + } + res->model_load_ms = (double)(ggml_time_ms() - model_load_start_ms); + model_was_swapped = true; + SRV_INF("hydra: PREFILL swap confirmed model_alias='%s' tokenizer='%s' model_name='%s' quant='%s' caps=0x%x model_load_ms=%.1f\n", + swapped_params.model_alias.empty() ? "?" : swapped_params.model_alias.begin()->c_str(), + model_tgt ? llama_model_get_tokenizer_model(model_tgt) : "", + model_tgt ? llama_model_get_display_name(model_tgt) : "", + model_tgt ? llama_model_get_quant_label(model_tgt) : "", + model_tgt ? llama_model_get_capabilities_bitfield(model_tgt) : 0, + res->model_load_ms); + // After load_model, `this` state is reset (new + // slots, new context). Re-look up the slot by id. + slot = get_slot_by_id(id_slot); + if (slot == nullptr) { + res->rpc_status = HYDRA_STATUS_NOT_FOUND; + res->error = "slot disappeared after model swap"; + queue_results.send(std::move(res)); + break; + } + } else { + SRV_DBG("hydra: PREFILL model='%s' already resident, no swap\n", + requested_model.c_str()); + } + } + + // Tokenize from JSON messages if request_json is provided; + // otherwise fall back to pre-tokenized prompt_tokens for back-compat. + std::vector prompt_tokens = std::move(task.hydra_action.prompt_tokens); + if (!parsed_body.is_null()) { + try { + std::vector dummy_files; + json parsed = oaicompat_chat_params_parse(parsed_body, chat_params, dummy_files); + if (!parsed.contains("prompt")) { + res->rpc_status = HYDRA_STATUS_ERROR; + res->error = "chat template produced no prompt"; + queue_results.send(std::move(res)); + break; + } + auto tokenized = tokenize_input_prompts(vocab, mctx, parsed["prompt"], true, true); + if (tokenized.empty()) { + res->rpc_status = HYDRA_STATUS_ERROR; + res->error = "tokenization produced no tokens"; + queue_results.send(std::move(res)); + break; + } + prompt_tokens = tokenized[0].get_tokens(); + } catch (const std::exception & e) { + res->rpc_status = HYDRA_STATUS_ERROR; + res->error = std::string("JSON/tokenization error: ") + e.what(); + queue_results.send(std::move(res)); + break; + } + } + + SRV_INF("hydra: PREFILL slot=%d tokens=%zu\n", id_slot, prompt_tokens.size()); + // #469 trace: log first 16 token IDs for cross-flow comparison + { + std::string tok_ids; + for (size_t i = 0; i < std::min(16, prompt_tokens.size()); ++i) { + if (i > 0) tok_ids += ","; + tok_ids += std::to_string(prompt_tokens[i]); + } + SRV_DBG("hydra: PREFILL slot=%d first16_tokens=[%s] total=%zu\n", + id_slot, tok_ids.c_str(), prompt_tokens.size()); + } + + // Clear existing slot state + slot->prompt_clear(false); + slot->n_prompt_tokens_cache = 0; + slot->n_prompt_tokens_processed = 0; + slot->n_decoded = 0; + + // Insert prompt tokens + if (prompt_tokens.empty()) { + res->rpc_status = HYDRA_STATUS_OK; + res->n_past = 0; + res->state_size = 0; + queue_results.send(std::move(res)); + break; + } + + slot->prompt.tokens.insert(prompt_tokens); + const auto & tokens = slot->prompt.tokens.get_tokens(); + const int n_tokens = (int)tokens.size(); + + // Add BOS if needed (check if slot uses BOS) + int token_offset = 0; + llama_token bos = llama_vocab_bos(vocab); + if (add_bos_token && bos != LLAMA_TOKEN_NULL && (tokens.empty() || tokens[0] != bos)) { + token_offset = 1; + } + + // Decode prompt in batches. Hydra #469 fix: upstream's own + // invariant (see create_checkpoint call in update_slots, + // "we create the checkpoint before calling llama_decode(), + // so the current batch is not yet processed and therefore + // it is not part of the checkpoint") requires the + // checkpoint to be created BEFORE the final token is + // decoded. The previous version of this handler decoded + // the whole prompt first and only afterward claimed (via + // create_checkpoint's pos_max arg, below) that the last + // token was still unprocessed. For hybrid/recurrent (SSM) + // models, whose memory can't be partially rolled back via + // seq_rm, that lie meant a cross-node restore would + // re-decode a token that was already baked into the + // recurrent state — double-applying it and corrupting the + // hidden state. Splitting the loop so the checkpoint is + // captured after n_tokens-1 tokens (matching what + // create_checkpoint's pos_max already claimed) makes the + // claim honest, same as the standard update_slots() path. + const int total_tokens = n_tokens + token_offset; + const int n_ubatch = llama_n_ubatch(ctx_tgt); + const int n_before_last = total_tokens > 1 ? total_tokens - 1 : total_tokens; + bool decode_ok = true; + for (int i = 0; i < n_before_last && decode_ok; i += n_ubatch) { + const int n_tokens_batch = std::min(n_ubatch, n_before_last - i); + common_batch_clear(batch); + for (int j = 0; j < n_tokens_batch; j++) { + const int tok_idx = i + j; + llama_token id; + if (token_offset > 0 && tok_idx == 0) { + id = bos; + } else { + id = tokens[tok_idx - token_offset]; + } + // No token in this phase is the final prompt + // token, so logits are never needed here. + common_batch_add(batch, id, tok_idx, {slot->id}, false); + } + if (llama_decode(ctx_tgt, batch) != 0) { + SRV_ERR("hydra: PREFILL slot=%d llama_decode failed at batch %d\n", id_slot, i); + decode_ok = false; + } + } + + if (!decode_ok) { + res->rpc_status = HYDRA_STATUS_ERROR; + res->error = "llama_decode failed during prefill"; + queue_results.send(std::move(res)); + break; + } + + // Register checkpoint BEFORE decoding the final token, so + // its pos_max claim (n_tokens - 1) is honest. Moved up + // from after the full-prompt decode (see #469 above). + if (n_tokens > 0) { + const auto pos_min = llama_memory_seq_pos_min(llama_get_memory(ctx_tgt), slot->id); + create_checkpoint(*slot, 0, (llama_pos)pos_min, (llama_pos)(n_tokens - 1)); + } + + // Decode the held-back final token (if any) now that the + // checkpoint has captured the state before it. + if (total_tokens > n_before_last) { + common_batch_clear(batch); + const int tok_idx = total_tokens - 1; + llama_token id = (token_offset > 0 && tok_idx == 0) + ? bos + : tokens[tok_idx - token_offset]; + common_batch_add(batch, id, tok_idx, {slot->id}, true); + if (llama_decode(ctx_tgt, batch) != 0) { + SRV_ERR("hydra: PREFILL slot=%d llama_decode failed on final token\n", id_slot); + res->rpc_status = HYDRA_STATUS_ERROR; + res->error = "llama_decode failed during prefill (final token)"; + queue_results.send(std::move(res)); + break; + } + } + + // Update slot tracking + slot->n_prompt_tokens_processed = n_tokens; + slot->n_prompt_tokens_cache = n_tokens; + + // Checkpoint already registered above, before the final + // token was decoded (#469 fix). + + // Build v2/v3 header: [1B version=0x02|0x03][4B n_past][4B n_tok][n_tok*4B tokens][1B flags][?ckpt?] + // Shared by both response paths — M1 embeds it at the head of the + // buffered blob, M2 sends it before streaming the GPU state. + const uint32_t hdr_n_past = (uint32_t)n_tokens; + const uint32_t hdr_n_tok = (uint32_t)(tokens.size()); + uint8_t hdr_flags = 0x00; + std::vector ckpt_buf; + int32_t ckpt_pos_min = 0, ckpt_pos_max = 0; + int64_t ckpt_n_tokens = 0; + if (!slot->prompt.checkpoints.empty()) { + hdr_flags |= 0x01; + const auto & ckpt = slot->prompt.checkpoints.back(); + ckpt_pos_min = ckpt.pos_min; + ckpt_pos_max = ckpt.pos_max; + ckpt_n_tokens = ckpt.n_tokens; + // Hydra M2-stream double-write fix (#470/#620): serialize the + // recurrent-only capture (data_tgt_recr, PARTIAL_ONLY) instead of + // the full data_tgt. The full live state that follows on the wire + // already carries the attention bytes at the live position, so + // sending the full checkpoint duplicates the attention portion + // (which scales with ctx). The recurrent state is genuinely needed + // at BOTH positions, hence the separate recr-only capture. + // hdr_flags bit 0x02 marks a recurrent-only checkpoint section so + // STATE_PUT/DECODE_APPLY can read it back with matched PARTIAL_ONLY + // flags. Fall back to the full capture when the checkpoint has no + // recr buffer (e.g. it was registered from an old 0x02 blob) — a + // PARTIAL_ONLY read of a full-written buffer is a CUDA memory error. + const bool use_recr = !ckpt.data_tgt_recr.empty(); + if (use_recr) { + hdr_flags |= 0x02; + } + const uint64_t tgt_sz = use_recr ? ckpt.data_tgt_recr.size() : ckpt.data_tgt.size(); + const uint64_t dft_sz = use_recr ? ckpt.data_dft_recr.size() : ckpt.data_dft.size(); + const uint8_t * tgt_ptr = use_recr ? ckpt.data_tgt_recr.data() : ckpt.data_tgt.data(); + const uint8_t * dft_ptr = use_recr ? ckpt.data_dft_recr.data() : ckpt.data_dft.data(); + ckpt_buf.resize(4 + 4 + 8 + 8 + (size_t)tgt_sz + 8 + (size_t)dft_sz); + size_t off = 0; + memcpy(ckpt_buf.data() + off, &ckpt_pos_min, 4); off += 4; + memcpy(ckpt_buf.data() + off, &ckpt_pos_max, 4); off += 4; + memcpy(ckpt_buf.data() + off, &ckpt_n_tokens, 8); off += 8; + memcpy(ckpt_buf.data() + off, &tgt_sz, 8); off += 8; + if (tgt_sz > 0) { memcpy(ckpt_buf.data() + off, tgt_ptr, (size_t)tgt_sz); off += (size_t)tgt_sz; } + memcpy(ckpt_buf.data() + off, &dft_sz, 8); off += 8; + if (dft_sz > 0) memcpy(ckpt_buf.data() + off, dft_ptr, (size_t)dft_sz); + } + const size_t base_hdr_size = 1 + 4 + 4 + hdr_n_tok * sizeof(llama_token) + 1; + const size_t v2_size = base_hdr_size + ckpt_buf.size(); + std::vector v2_hdr(v2_size); + { + size_t off = 0; + // 0x03 = v3 blob: checkpoint section carries recurrent-only + // captures (data_tgt_recr, PARTIAL_ONLY). 0x02 = v2 blob: + // checkpoint section carries the full data_tgt. Bumped so a + // mixed-version fleet never misreads a smaller (recr-only) + // checkpoint as a full one. + const uint8_t version_byte = 0x03; + memcpy(v2_hdr.data() + off, &version_byte, 1); off += 1; + memcpy(v2_hdr.data() + off, &hdr_n_past, 4); off += 4; + memcpy(v2_hdr.data() + off, &hdr_n_tok, 4); off += 4; + if (hdr_n_tok > 0) { + const auto & toks = slot->prompt.tokens.get_text_tokens(); + memcpy(v2_hdr.data() + off, toks.data(), toks.size() * sizeof(llama_token)); + off += toks.size() * sizeof(llama_token); + } + memcpy(v2_hdr.data() + off, &hdr_flags, 1); off += 1; + if (!ckpt_buf.empty()) { + memcpy(v2_hdr.data() + off, ckpt_buf.data(), ckpt_buf.size()); + off += ckpt_buf.size(); + } + } + + // Get raw KV state + const size_t state_size = llama_state_seq_get_size(ctx_tgt, slot->id); + + // Snapshot logits NOW into a small buffer. ctx->logits is + // context-global: a concurrent slot decode can overwrite it + // while the M2 state stream is on the wire. Appending + // n_vocab floats gives the decode GPU the activation handoff + // (llama_state_seq_get_data saves KV but not logits), so + // STATE_PUT / DECODE_APPLY can sample immediately. + uint64_t logits_size = 0; + std::vector logits_buf; + { + const int n_vocab = llama_vocab_n_tokens(vocab); + const float * logits_ptr = llama_get_logits(ctx_tgt); + if (logits_ptr && n_vocab > 0) { + logits_size = (uint64_t)n_vocab * sizeof(float); + logits_buf.assign(reinterpret_cast(logits_ptr), + reinterpret_cast(logits_ptr) + (size_t)logits_size); + } + } + + SRV_INF("hydra: PREFILL slot=%d done n_past=%d kv=%zu logits=%" PRIu64 "B total=%zu\n", + id_slot, n_tokens, state_size, logits_size, v2_hdr.size() + state_size + (size_t)logits_size); + + // M-Perf.9 #289: model identity for the slot the prefill + // was just built on. Coordinator uses this to populate + // item.KvModelAlias/Hash and to gate RestoreKvAsync. When + // a `model` swap happened earlier in this handler, the + // post-swap `model_name` / `params_base.model.path` / + // `model` are used. `res->model_fallback` was set by the + // model-resolution block above; we preserve it here. + res->model_alias = model_name; + res->model_path = params_base.model.path; + // res->model_fallback may already be true (alias unknown + // or no preset); only set false when no swap was needed. + if (!model_was_swapped && !res->model_fallback) { + // nothing to do — leave as-is + } + if (model_tgt) { + res->tokenizer = llama_model_get_tokenizer_model(model_tgt); + res->model_name = llama_model_get_display_name(model_tgt); + res->model_quant = llama_model_get_quant_label(model_tgt); + res->model_capabilities = llama_model_get_capabilities_bitfield(model_tgt); + } + + res->rpc_status = HYDRA_STATUS_OK; + res->n_past = n_tokens; + res->state_size = state_size; + res->logits_size = logits_size; + // #451: populate PREFILL metrics + res->prefill_ms = (double)(ggml_time_ms() - prefill_start_ms); + res->prompt_tokens = n_tokens; + res->kv_size = state_size; + if (res->prefill_ms > 0 && n_tokens > 0) { + res->tokens_per_second = (double)n_tokens / (res->prefill_ms / 1000.0); + } + res->cache_tokens = slot->n_prompt_tokens_cache; + + const int hydra_fd = task.hydra_action.hydra_fd; + if (hydra_fd >= 0) { + // M2 path (#470): stream the response straight to the + // socket — 12B header + meta JSON + v2 header, then the + // GPU KV state zero-copy (chunked via cparams.hydra_state_chunk_size), + // then the (small) logits tail. No full-blob RAM buffer: + // at 60-80K context the blob is ~800 MB and grows toward + // 10 GB; buffering it doubled engine peak memory and the + // send only started after compute + full buffer completed. + // Wire layout is byte-identical to M1: payload = + // v2_hdr + KV state + logits, payload_len = the same + // total the coordinator computes from meta. + const size_t total_payload = v2_hdr.size() + state_size + (size_t)logits_size; + + // M2 (#470): pre-compute the wire hash of the whole kv + // segment — v2 header, then [4B magic][4B seq_id] + KV + // state (hash-only pass in wire order), then the logits + // tail. The meta must carry it BEFORE the first payload + // byte goes out (the coordinator forwards it into the + // DECODE frame header, and DECODE_APPLY verifies the + // streamed restore end-to-end). The slot is exclusively + // held by this task, so the state cannot change between + // the hash pass and the stream. + XXH3_state_t * kv_hst = nullptr; + if (hydra_fd >= 0) { + kv_hst = XXH3_createState(); + XXH3_64bits_reset(kv_hst); + XXH3_64bits_update(kv_hst, v2_hdr.data(), v2_hdr.size()); + const size_t hashed = llama_state_seq_hash(ctx_tgt, slot->id, kv_hst); + // state_size (from llama_state_seq_get_size) ALREADY includes the + // [4B magic][4B seq_id] wire header — llama_io_write_dummy counts it. + // llama_state_seq_hash hashes the same [4B magic][4B seq_id] + KV + // bytes, so after the n_bytes() fix hashed == state_size exactly. + // Adding sizeof(uint32_t) + sizeof(llama_seq_id) here double-counted + // the header and killed every PREFILL M2 request (#470). + if (hashed != state_size) { + res->rpc_status = HYDRA_STATUS_ERROR; + res->error = "PREFILL M2: hash pre-pass hashed " + + std::to_string(hashed) + " B, expected " + + std::to_string(state_size) + " B"; + } + if (!logits_buf.empty()) { + XXH3_64bits_update(kv_hst, logits_buf.data(), logits_buf.size()); + } + if (res->rpc_status == HYDRA_STATUS_OK) { + const uint64_t kv_hash = XXH3_64bits_digest(kv_hst); + char hash_hex[17]; + snprintf(hash_hex, sizeof(hash_hex), "%016" PRIx64, kv_hash); + res->kv_hash_str = std::string("xxh3:") + hash_hex; + } + XXH3_freeState(kv_hst); + kv_hst = nullptr; + } + + json meta_j = { + {"n_past", res->n_past}, + {"state_size", res->state_size}, + {"logits_size", res->logits_size} + }; + if (!res->model_alias.empty()) meta_j["model_alias"] = res->model_alias; + if (!res->model_path.empty()) meta_j["model_path"] = res->model_path; + if (!res->tokenizer.empty()) meta_j["tokenizer"] = res->tokenizer; + if (!res->model_name.empty()) meta_j["model_name"] = res->model_name; + if (!res->model_quant.empty()) meta_j["model_quant"] = res->model_quant; + if (res->model_capabilities) meta_j["model_capabilities"] = res->model_capabilities; + meta_j["model_fallback"] = res->model_fallback; + if (res->prefill_ms > 0) meta_j["prefill_ms"] = res->prefill_ms; + if (res->model_load_ms > 0) meta_j["model_load_ms"] = res->model_load_ms; + if (!res->kv_hash_str.empty()) meta_j["kv_hash_str"] = res->kv_hash_str; + const std::string meta_str = meta_j.dump(); + const uint32_t meta_len = (uint32_t)meta_str.size(); + + uint8_t hdr[HYDRA_RES_HEADER_SIZE] = {}; + hdr[0] = HYDRA_STATUS_OK; + hdr[1] = (meta_len) & 0xFF; + hdr[2] = (meta_len >> 8) & 0xFF; + hdr[3] = (meta_len >> 16) & 0xFF; + memcpy(hdr + 4, &total_payload, 8); + if (!hydra_send_all(hydra_fd, hdr, HYDRA_RES_HEADER_SIZE) || + !hydra_send_all(hydra_fd, meta_str.data(), meta_str.size()) || + !hydra_send_all(hydra_fd, v2_hdr.data(), v2_hdr.size())) { + res->rpc_status = HYDRA_STATUS_ERROR; + res->error = "PREFILL M2: response header/meta/v2-hdr send failed"; + ::shutdown(hydra_fd, SHUT_RDWR); + } else { + res->header_sent = true; // META + header + v2-hdr before payload + // Stream GPU state to fd (zero-copy from GPU memory; + // the wire hash was pre-computed above — pass no + // hash state so the io does not double-feed it) + const size_t streamed = llama_state_seq_get_data_to_fd(ctx_tgt, slot->id, hydra_fd, nullptr); + if (streamed != state_size) { + // TOCTOU: state size changed between get_size + // (above) and the stream, or the stream failed + // mid-way. The wire framing is now broken — the + // only safe recovery is to kill the connection. + // shutdown(), not close(): the RPC connection + // loop owns the fd (mirrors STATE_GET M2). + res->rpc_status = HYDRA_STATUS_ERROR; + res->error = "llama_state_seq_get_data_to_fd streamed " + + std::to_string(streamed) + " B, expected " + + std::to_string(state_size) + " B"; + ::shutdown(hydra_fd, SHUT_RDWR); + } else { + // Logits tail after the state stream — PREFILL's + // payload includes logits_size bytes at the end + // (STATE_GET M2 does not send logits). + if (!logits_buf.empty()) { + if (!hydra_send_all(hydra_fd, logits_buf.data(), logits_buf.size())) { + res->rpc_status = HYDRA_STATUS_ERROR; + res->error = "PREFILL M2: logits tail send failed"; + ::shutdown(hydra_fd, SHUT_RDWR); + } + } + if (res->rpc_status == HYDRA_STATUS_OK) { + res->streamed_bytes = (uint64_t)total_payload; + } + } + } + } else { + // M1 path: buffer the full blob in memory; the RPC thread + // sends header + meta + payload afterwards (unchanged). + // v2 blob format (0x02): [1B version][4B n_past][4B n_tok][n_tok*4B tokens] + // [1B flags (bit 0 = has_checkpoint)] + // [if flags & 0x01: 4B pos_min | 4B pos_max | 8B n_tokens | 8B tgt_sz | tgt_data | 8B dft_sz | dft_data] + // [raw KV state from llama_state_seq_get_data] + // [logits (n_vocab * float)] + std::vector v2_blob(v2_hdr.size() + state_size + (size_t)logits_size); + { + size_t off = 0; + memcpy(v2_blob.data() + off, v2_hdr.data(), v2_hdr.size()); + off += v2_hdr.size(); + if (state_size > 0) { + llama_state_seq_get_data(ctx_tgt, v2_blob.data() + off, state_size, slot->id); + } + } + if (!logits_buf.empty()) { + memcpy(v2_blob.data() + v2_hdr.size() + state_size, logits_buf.data(), logits_buf.size()); + } + res->state_data = std::move(v2_blob); + } + // #469 trace: log PREFILL completion with token IDs for cross-flow comparison + SRV_DBG("hydra: PREFILL_DONE slot=%d n_past=%d state_size=%zu logits_size=%zu blob_size=%zu prefill_ms=%.1f\n", + id_slot, n_tokens, state_size, logits_size, + (hydra_fd >= 0) ? v2_hdr.size() + state_size + (size_t)logits_size : res->state_data.size(), + res->prefill_ms); + queue_results.send(std::move(res)); + } break; + + case SERVER_TASK_TYPE_HYDRA_ENGINE_DECODE: + { + // ── Sync phase: Gate A (header-only, no GGUF reads, ~1 ms) ── + // Identity validation, slot reservation, post DECODE_APPLY. + // No model I/O, no KV touched. + const int id_slot = task.hydra_action.id_slot; + const int32_t decode_request_id = task.hydra_action.decode_request_id; + auto res = std::make_unique(); + res->id = task.id; + res->op = HYDRA_OP_DECODE; + res->decode_request_id = decode_request_id; + res->id_slot = id_slot; + + server_slot * slot = get_slot_by_id(id_slot); + if (slot == nullptr) { + res->rpc_status = HYDRA_STATUS_NOT_FOUND; + res->error = "invalid slot ID"; + queue_results.send(std::move(res)); + break; + } + + if (slot->is_processing()) { + res->rpc_status = HYDRA_STATUS_BUSY; + res->error = "slot is busy"; + queue_results.send(std::move(res)); + break; + } + + // Reject if slot is reserved for another decode + if (slot->reserved_for_decode_id != -1 && slot->reserved_for_decode_id != decode_request_id) { + res->rpc_status = HYDRA_STATUS_BUSY; + res->error = "slot reserved for another decode"; + queue_results.send(std::move(res)); + break; + } + + // Parse the merged DECODE JSON header + json decode_req; + try { + decode_req = json::parse(task.hydra_action.decode_json); + } catch (const std::exception & e) { + res->rpc_status = HYDRA_STATUS_BAD_REQUEST; + res->error = std::string("invalid JSON: ") + e.what(); + queue_results.send(std::move(res)); + break; + } + + // ── Gate A: header-only metadata comparison ───────────── + // Compare kv_metadata vs model_metadata from the control + // header. No GGUF reads, no KV touched. + const json & kv_meta = decode_req["kv_metadata"]; + const json & model_meta = decode_req.value("model_metadata", json::object()); + + // Read request identities from header + const std::string req_tokenizer = kv_meta.value("tokenizer", ""); + const std::string req_model_name = kv_meta.value("model_name", ""); + const uint32_t req_capabilities = kv_meta.value("model_capabilities", 0u); + + // Read target identities from header + const std::string tgt_tokenizer = model_meta.value("tokenizer", ""); + const std::string tgt_model_name = model_meta.value("model_name", ""); + + const bool tokenizer_match = (req_tokenizer == tgt_tokenizer); + bool model_name_match = (req_model_name == tgt_model_name); + // #589: cross-node same-model name tolerance. The KV's + // model_name is the display name (GGUF metadata) of the + // file that BUILT the KV — a different build/quant of the + // same model than the decode node's resident file, so + // string equality legitimately fails for the same logical + // model (e.g. kv_metadata carries the source node's + // display name, the decode node reports its resident + // filename). When the header carries the KV's source + // alias (kv_metadata.model_alias) or the resolved request + // alias ("model") and that alias maps through the preset + // table to the resident model path, the KV was built by + // the same logical model — accept. The alias→path check + // is exact (per-node preset INI), so a different model + // (Mini vs Balanced, 27B vs 35B) still maps to a + // different path and is rejected. + if (!model_name_match) { + const std::string kv_alias = kv_meta.value("model_alias", ""); + const std::string hdr_alias = decode_req.value("model", std::string()); + for (const auto & cand : { kv_alias, hdr_alias }) { + if (cand.empty()) { + continue; + } + auto pit = preset_alias_to_path.find(cand); + if (pit != preset_alias_to_path.end() && pit->second == params_base.model.path) { + SRV_INF("hydra: DECODE slot=%d Gate A name fallback — alias '%s' maps to resident path, same logical model\n", + id_slot, cand.c_str()); + model_name_match = true; + break; + } + } + } + const uint32_t capabilities_xor = req_capabilities ^ model_meta.value("model_capabilities", 0u); + + static const char * kCapBitNames[] = {"MTP", "VISION", "REASONING", "TOOL_USE", "CODE"}; + std::vector capabilities_diff_bits; + for (int b = 0; b < 5; b++) { + if (capabilities_xor & (1u << b)) { + capabilities_diff_bits.push_back(kCapBitNames[b]); + } + } + + // MTP(bit0) + VISION(bit1) mismatch → hard reject + const bool valid = tokenizer_match && model_name_match + && !(capabilities_xor & 0x3); + + json match_j = { + {"tokenizer_match", tokenizer_match}, + {"model_name_match", model_name_match}, + {"capabilities_xor", capabilities_xor}, + {"capabilities_diff_bits", capabilities_diff_bits}, + {"model_quant_match", kv_meta.value("model_quant", "") == model_meta.value("model_quant", "")}, + {"model_alias_match", true}, + }; + res->match_json = match_j; + res->match_valid = valid; + + if (!valid) { + res->rpc_status = HYDRA_STATUS_ERROR; + res->error = "model_capabilities_mismatch"; + SRV_WRN("hydra: DECODE slot=%d Gate A reject — tokenizer=%d name=%d caps_xor=0x%x\n", + id_slot, tokenizer_match, model_name_match, capabilities_xor); + queue_results.send(std::move(res)); + break; + } + + // ── Reserve slot ──────────────────────────────────────── + slot->reserved_for_decode_id = decode_request_id; + + SRV_INF("hydra: DECODE slot=%d Gate A pass, reserved for request_id=%d\n", + id_slot, decode_request_id); + + // ── Create decode_result_entry (LOADING state) ───────── + // So GET /v1/decode/{id} returns 202 instead of 404 + // while async DECODE_APPLY is pending. + if (routes_ptr) { + server_routes::decode_result_entry entry; + entry.id_slot = id_slot; + entry.state = server_routes::DECODE_STATE_LOADING; + entry.match_json = match_j; + entry.created_at = std::time(nullptr); + entry.ttl_s = routes_ptr->decode_result_ttl_s; + entry.model_metadata = decode_req.value("model_metadata", json::object()); + entry.model_identity = json::object(); + std::lock_guard lock(routes_ptr->decode_results_mutex); + routes_ptr->decode_results[decode_request_id] = std::move(entry); + routes_ptr->evict_decode_results_locked(); + } + + // ── Send sync validation response ─────────────────────── + res->rpc_status = HYDRA_STATUS_OK; + queue_results.send(std::move(res)); + + // ── Post DECODE_APPLY async task ──────────────────────── + { + server_task apply_task(SERVER_TASK_TYPE_HYDRA_DECODE_APPLY); + apply_task.id = queue_tasks.get_new_id(); + apply_task.hydra_action.id_slot = id_slot; + apply_task.hydra_action.decode_json = std::move(task.hydra_action.decode_json); + apply_task.hydra_action.kv_data = std::move(task.hydra_action.kv_data); + apply_task.hydra_action.decode_request_id = decode_request_id; + queue_tasks.post(std::move(apply_task)); + SRV_INF("hydra: DECODE slot=%d posted DECODE_APPLY (request_id=%d)\n", + id_slot, decode_request_id); + } + } break; + + case SERVER_TASK_TYPE_HYDRA_DECODE_APPLY: + { + // ── Async phase: model swap + Gate B + KV restore + completion ── + const int id_slot = task.hydra_action.id_slot; + const int32_t decode_request_id = task.hydra_action.decode_request_id; + + // Parse the DECODE JSON header (re-parsed for async context) + json decode_req; + try { + decode_req = json::parse(task.hydra_action.decode_json); + } catch (const std::exception & e) { + SRV_WRN("hydra: DECODE_APPLY slot=%d invalid JSON: %s\n", id_slot, e.what()); + // Release reservation on error + server_slot * s = get_slot_by_id(id_slot); + if (s) s->reserved_for_decode_id = -1; + if (routes_ptr) { + server_routes::decode_result_entry entry; + entry.id_slot = id_slot; + entry.error = std::string("DECODE_APPLY JSON parse error: ") + e.what(); + entry.created_at = std::time(nullptr); + entry.ttl_s = routes_ptr->decode_result_ttl_s; + std::lock_guard lock(routes_ptr->decode_results_mutex); + routes_ptr->decode_results[decode_request_id] = std::move(entry); + routes_ptr->evict_decode_results_locked(); + } + break; + } + + const json & kv_meta = decode_req["kv_metadata"]; + const json & model_meta = decode_req.value("model_metadata", json::object()); + + // ── Model swap (if requested model != resident) ───────── + const std::string requested_model = decode_req.value("model", std::string()); + double model_load_ms = 0.0; + bool model_fallback = false; + + if (!requested_model.empty()) { + // #470: resolve the requested alias against the + // T3-CURRENT alias → file map FIRST. The coordinator's + // T3 config (model_path) can load a file the preset INI + // does not associate with the engine's current alias + // (e.g. the dense-27b-combined session T3-loads the + // 27B-Coder file while the alias identity still says + // qwen3.6-35B-balanced). When the requested alias's + // T3-current file == resident, the alias describes the + // resident — swapping to the INI's file would be a + // pointless 73-81s reload + COMBINED teardown/reattach + // that then fails Gate B (header model_metadata of the + // pre-swap resident vs the swapped-in model's identity). + const auto t3it = t3_current_alias_to_path.find(requested_model); + if (t3it != t3_current_alias_to_path.end() && t3it->second == params_base.model.path) { + SRV_INF("hydra: DECODE_APPLY slot=%d model='%s' matches T3-current resident '%s' — no swap\n", + id_slot, requested_model.c_str(), params_base.model.path.c_str()); + } else { + auto it = preset_alias_to_path.find(requested_model); + if (it == preset_alias_to_path.end()) { + SRV_WRN("hydra: DECODE_APPLY slot=%d model='%s' unknown — falling back to resident '%s'\n", + id_slot, requested_model.c_str(), model_name.c_str()); + model_fallback = true; + } else if (it->second != params_base.model.path) { + SRV_INF("hydra: DECODE_APPLY slot=%d model='%s' swapping %s -> %s\n", + id_slot, requested_model.c_str(), params_base.model.path.c_str(), + it->second.c_str()); + common_params swapped_params = params_base; + // Apply the target alias's full preset (same + // treatment, and same intentional full-preset + // scope, as the PREFILL path above). + auto pit = preset_alias_to_preset.find(requested_model); + bool preset_apply_failed = false; + if (pit != preset_alias_to_preset.end()) { + // Same clear+re-pad+try/catch as the PREFILL path. + swapped_params.tensor_buft_overrides.clear(); + try { + pit->second.apply_to_params(swapped_params); + hydra_repad_tensor_buft_overrides(swapped_params, "DECODE_APPLY swap"); + } catch (const std::exception & e) { + SRV_WRN("hydra: DECODE_APPLY slot=%d swap preset apply for '%s' failed: %s\n", + id_slot, requested_model.c_str(), e.what()); + preset_apply_failed = true; + if (routes_ptr) { + server_routes::decode_result_entry entry; + entry.id_slot = id_slot; + entry.error = std::string("model swap preset apply failed: ") + e.what(); + entry.created_at = std::time(nullptr); + entry.ttl_s = routes_ptr->decode_result_ttl_s; + std::lock_guard lock(routes_ptr->decode_results_mutex); + routes_ptr->decode_results[decode_request_id] = std::move(entry); + routes_ptr->evict_decode_results_locked(); + } + } + } + if (preset_apply_failed) { + server_slot * s = get_slot_by_id(id_slot); + if (s) s->reserved_for_decode_id = -1; + break; + } + swapped_params.model.path = it->second; + swapped_params.model_alias = { requested_model }; + // #514: tear down COMBINED state before the + // reload — see hydra_teardown_combined_before_reload(). + const bool was_combined = hydra_combined_head_attached || hydra_combined_static; + if (was_combined) { + hydra_teardown_combined_before_reload(); + } + const int64_t model_load_start_ms = ggml_time_ms(); + if (!load_model(swapped_params)) { + SRV_WRN("hydra: DECODE_APPLY slot=%d model swap to '%s' failed\n", + id_slot, requested_model.c_str()); + server_slot * s = get_slot_by_id(id_slot); + if (s) s->reserved_for_decode_id = -1; + if (routes_ptr) { + server_routes::decode_result_entry entry; + entry.id_slot = id_slot; + entry.error = "model swap to '" + requested_model + "' failed"; + entry.created_at = std::time(nullptr); + entry.ttl_s = routes_ptr->decode_result_ttl_s; + std::lock_guard lock(routes_ptr->decode_results_mutex); + routes_ptr->decode_results[decode_request_id] = std::move(entry); + routes_ptr->evict_decode_results_locked(); + } + break; + } + if (was_combined) { + hydra_reattach_combined_after_reload(); + } + model_load_ms = (double)(ggml_time_ms() - model_load_start_ms); + SRV_INF("hydra: DECODE_APPLY slot=%d swap confirmed model_load_ms=%.1f\n", + id_slot, model_load_ms); + } + } + } + + // ── Gate B: post-load identity check ──────────────────── + // Compare model_metadata from header vs ACTUAL resident GGUF identity. + server_slot * slot = get_slot_by_id(id_slot); + if (slot == nullptr) { + SRV_WRN("hydra: DECODE_APPLY slot=%d disappeared after model swap\n", id_slot); + if (routes_ptr) { + server_routes::decode_result_entry entry; + entry.id_slot = id_slot; + entry.error = "slot disappeared after model swap"; + entry.created_at = std::time(nullptr); + entry.ttl_s = routes_ptr->decode_result_ttl_s; + std::lock_guard lock(routes_ptr->decode_results_mutex); + routes_ptr->decode_results[decode_request_id] = std::move(entry); + routes_ptr->evict_decode_results_locked(); + } + break; + } + + const std::string resident_tokenizer = llama_model_get_tokenizer_model(model_tgt); + const std::string resident_model_name = llama_model_get_display_name(model_tgt); + const std::string resident_model_quant = llama_model_get_quant_label(model_tgt); + const uint32_t resident_capabilities = llama_model_get_capabilities_bitfield(model_tgt); + + const std::string hdr_model_name = model_meta.value("model_name", ""); + const std::string hdr_model_quant = model_meta.value("model_quant", ""); + const uint32_t hdr_capabilities = model_meta.value("model_capabilities", 0u); + + const bool gate_b_tokenizer = (resident_tokenizer == model_meta.value("tokenizer", "")); + const bool gate_b_model_name = (resident_model_name == hdr_model_name); + const uint32_t gate_b_caps_xor = resident_capabilities ^ hdr_capabilities; + + if (!gate_b_tokenizer || !gate_b_model_name || (gate_b_caps_xor & 0x3)) { + SRV_WRN("hydra: DECODE_APPLY slot=%d Gate B reject — tokenizer=%d name=%d caps_xor=0x%x\n", + id_slot, gate_b_tokenizer, gate_b_model_name, gate_b_caps_xor); + slot->reserved_for_decode_id = -1; + if (routes_ptr) { + server_routes::decode_result_entry entry; + entry.id_slot = id_slot; + entry.error = "Gate B identity mismatch after model load"; + entry.match_json = {{"gate_b_tokenizer", gate_b_tokenizer}, {"gate_b_name", gate_b_model_name}, {"gate_b_caps_xor", gate_b_caps_xor}}; + entry.created_at = std::time(nullptr); + entry.ttl_s = routes_ptr->decode_result_ttl_s; + std::lock_guard lock(routes_ptr->decode_results_mutex); + routes_ptr->decode_results[decode_request_id] = std::move(entry); + routes_ptr->evict_decode_results_locked(); + } + break; + } + + if (resident_model_quant != hdr_model_quant) { + SRV_INF("hydra: DECODE_APPLY slot=%d Gate B quant differs (%s → %s) — mix-quant allowed\n", + id_slot, hdr_model_quant.c_str(), resident_model_quant.c_str()); + } + + // ── KV restore ───────────────────────────────────────── + const int64_t restore_start_ms = ggml_time_ms(); + + // M2 (#470): the v2 header arrives pre-parsed (kv_v2_hdr, + // small) and the KV state stream is read directly off + // hydra_fd via llama_state_seq_set_data_from_fd — the engine + // never materializes the full blob (2.3 GB today, 10 GB + // target). M1 (kv_data) is the buffered fallback. + const bool m2_stream = !task.hydra_action.kv_v2_hdr.empty(); + if (!task.hydra_action.kv_data.empty() || m2_stream) { + slot->prompt_clear(false); + slot->n_prompt_tokens_cache = 0; + slot->n_prompt_tokens_processed = 0; + slot->n_decoded = 0; + + // The coordinator may send the v2/v3 blob (header + raw KV) + // or just the raw KV data. Parse the v2 header to extract + // the token list so update_slots()'s n_common decision can + // match incoming tokens against the restored KV — without + // this, prompt.tokens is empty after prompt_clear(), n_past + // computes to 0, and seq_rm(slot, 0, -1) wipes the KV that + // llama_state_seq_set_data just loaded (issue #506). + const uint8_t * kv_ptr = m2_stream + ? task.hydra_action.kv_v2_hdr.data() + : task.hydra_action.kv_data.data(); + size_t kv_len = m2_stream + ? task.hydra_action.kv_v2_hdr.size() + : task.hydra_action.kv_data.size(); + int32_t blob_n_past = 0; + int32_t blob_n_tok = 0; + bool has_chkpt = false; + bool ckpt_is_recr_only = false; + int32_t ckpt_pos_min_in = 0, ckpt_pos_max_in = 0; + int64_t ckpt_n_tokens_in = 0; + std::vector ckpt_tgt_data, ckpt_dft_data; + + // v2 (0x02) blobs carry a full checkpoint; v3 (0x03) blobs may carry + // a recurrent-only checkpoint (hdr_flags bit 0x02). Both share the + // header layout — the M2-stream double-write fix bumped the version. + const bool is_v2 = kv_len >= 1 && (kv_ptr[0] == 0x02 || kv_ptr[0] == 0x03); + if (is_v2 && kv_len >= 9) { + memcpy(&blob_n_past, kv_ptr + 1, 4); + memcpy(&blob_n_tok, kv_ptr + 5, 4); + + const size_t token_start = 9; + const size_t token_end = token_start + (size_t)blob_n_tok * sizeof(llama_token); + if (blob_n_tok > 0 && token_end <= kv_len) { + // Restore token list from v2/v3 blob header + slot->prompt.tokens.clear(); + const llama_token * tok_ptr = (const llama_token *)(kv_ptr + token_start); + llama_tokens restored_tokens(tok_ptr, tok_ptr + (size_t)blob_n_tok); + slot->prompt.tokens.insert(restored_tokens); + SRV_INF("hydra: DECODE_APPLY slot=%d v2/v3 blob: restored %d tokens from header\n", + id_slot, blob_n_tok); + } + + // Skip past v2/v3 header (version + n_past + n_tok + tokens + flags + optional checkpoint) + size_t hdr_offset = token_end; + if (hdr_offset < kv_len) { + const uint8_t flags = kv_ptr[hdr_offset]; + hdr_offset += 1; // past flags byte + // bit 0x01 = has checkpoint; bit 0x02 = recurrent-only (PARTIAL_ONLY) + ckpt_is_recr_only = (flags & 0x02) != 0; + if (flags & 0x01) { + // Capture checkpoint: 4B pos_min | 4B pos_max | 8B n_tokens | 8B tgt_sz | tgt_data | 8B dft_sz | dft_data + // Mirrors the STATE_PUT sibling (~line 3343) — the native + // checkpoint is registered after restore so hybrid/recurrent + // models get their recurrent memory back (KV restored without + // its checkpoint is corrupt). + if (hdr_offset + 4 + 4 + 8 + 8 <= kv_len) { + memcpy(&ckpt_pos_min_in, kv_ptr + hdr_offset, 4); hdr_offset += 4; + memcpy(&ckpt_pos_max_in, kv_ptr + hdr_offset, 4); hdr_offset += 4; + memcpy(&ckpt_n_tokens_in, kv_ptr + hdr_offset, 8); hdr_offset += 8; + uint64_t tgt_sz_in; + memcpy(&tgt_sz_in, kv_ptr + hdr_offset, 8); hdr_offset += 8; + if (tgt_sz_in > 0 && hdr_offset + tgt_sz_in <= kv_len) { + ckpt_tgt_data.assign(kv_ptr + hdr_offset, kv_ptr + hdr_offset + (size_t)tgt_sz_in); + hdr_offset += (size_t)tgt_sz_in; + } + if (hdr_offset + 8 <= kv_len) { + uint64_t dft_sz_in; + memcpy(&dft_sz_in, kv_ptr + hdr_offset, 8); hdr_offset += 8; + if (dft_sz_in > 0 && hdr_offset + dft_sz_in <= kv_len) { + ckpt_dft_data.assign(kv_ptr + hdr_offset, kv_ptr + hdr_offset + (size_t)dft_sz_in); + hdr_offset += (size_t)dft_sz_in; + } + } + has_chkpt = true; + } + } + } + // Advance kv_ptr/kv_len past the v2 header to the raw KV state + if (hdr_offset <= kv_len) { + kv_ptr = kv_ptr + hdr_offset; + kv_len = kv_len - hdr_offset; + } + } + + // M2 (#470): hash the whole kv segment as it streams — + // v2 header first, then every byte the fd restore + // consumes, then the logits tail (wire order). + XXH3_state_t * hst = nullptr; + if (m2_stream) { + hst = XXH3_createState(); + XXH3_64bits_reset(hst); + XXH3_64bits_update(hst, task.hydra_action.kv_v2_hdr.data(), + task.hydra_action.kv_v2_hdr.size()); + } + + size_t status = 0; + if (m2_stream) { + // Stream restore: consumes [4B magic][4B seq_id] + KV + // state off the fd; logits tail is read separately below. + status = llama_state_seq_set_data_from_fd( + ctx_tgt, slot->id, task.hydra_action.hydra_fd, hst); + } else { + status = llama_state_seq_set_data( + ctx_tgt, + kv_ptr, + kv_len, + slot->id); + } + + // llama_state_seq_set_data returns the number of bytes + // read on success (0 means failed to load) — see its + // doc comment in include/llama.h. `status` only counts + // the KV-cache bytes the reader consumed; it does NOT + // include the trailing logits PREFILL_DONE appends + // (see ~line 4098), so status < kv_len is the normal + // case whenever logits are present — compare against + // kv_len here and this false-fails on every restore + // with logits. Matches the STATE_PUT sibling check + // (server-context.cpp ~line 3395: `if (n_read == 0)`). + if (status == 0) { + SRV_WRN("hydra: DECODE_APPLY slot=%d KV restore failed (%d)\n", id_slot, status); + if (hst) { XXH3_freeState(hst); hst = nullptr; } + if (m2_stream) { + // The stream broke mid-way: drop the read side so + // residual unread bytes cannot misalign the next + // request frame. The RPC thread still writes the + // error response (write side stays open). + ::shutdown(task.hydra_action.hydra_fd, SHUT_RD); + } + slot->reserved_for_decode_id = -1; + // Tokens were registered from the v2 header before set_data — + // clear them so the slot is not left poisoned (n_past > 0 + // with no KV cells → pos_min == -1 abort on the next decode + // that touches this slot). Matches the STATE_PUT failure path. + slot->prompt.tokens.clear(); + slot->prompt.checkpoints.clear(); + slot->n_prompt_tokens_cache = 0; + llama_memory_seq_rm(llama_get_memory(ctx_tgt), slot->id, -1, -1); + if (routes_ptr) { + server_routes::decode_result_entry entry; + entry.id_slot = id_slot; + entry.error = "KV restore failed (llama_state_seq_set_data returned " + std::to_string(status) + ")"; + entry.created_at = std::time(nullptr); + entry.ttl_s = routes_ptr->decode_result_ttl_s; + std::lock_guard lock(routes_ptr->decode_results_mutex); + routes_ptr->decode_results[decode_request_id] = std::move(entry); + routes_ptr->evict_decode_results_locked(); + } + break; + } + + // Trailing logits: PREFILL_DONE appends n_vocab floats + // after the KV state (~line 4098) so the decode side + // can sample immediately instead of reading garbage + // after restore. Mirrors STATE_PUT's per-slot + // injection (~line 3405) — DECODE_APPLY was missing + // this step entirely. + { + const size_t expected_logits = (size_t)llama_vocab_n_tokens(vocab) * sizeof(float); + if (m2_stream) { + // Read the logits tail straight off the fd (small). + const size_t remaining = + (size_t)(task.hydra_action.kv_stream_len - status); + if (remaining == expected_logits) { + std::vector logits_buf(remaining); + if (hydra_recv_all(task.hydra_action.hydra_fd, + logits_buf.data(), remaining)) { + XXH3_64bits_update(hst, logits_buf.data(), remaining); + const size_t n_floats = llama_vocab_n_tokens(vocab); + slot->restored_logits.assign( + reinterpret_cast(logits_buf.data()), + reinterpret_cast(logits_buf.data()) + n_floats); + slot->logits_valid = true; + SRV_INF("hydra: DECODE_APPLY slot=%d restored %zu logits to per-slot buffer\n", + id_slot, n_floats); + } else { + SRV_WRN("hydra: DECODE_APPLY slot=%d logits tail read failed\n", id_slot); + } + } + } else { + const size_t remaining = kv_len - status; + if (remaining == expected_logits) { + const float * src = (const float *)(kv_ptr + status); + const size_t n_floats = llama_vocab_n_tokens(vocab); + slot->restored_logits.assign(src, src + n_floats); + slot->logits_valid = true; + SRV_INF("hydra: DECODE_APPLY slot=%d restored %zu logits to per-slot buffer\n", + id_slot, n_floats); + } + } + } + + // M2 wire-hash verification (post-restore — with streaming + // the bytes reach the GPU before a pre-restore hash could + // be computed). On mismatch the slot is cleared so the next + // decode cannot sample corrupt state, and the response + // carries the terminal error for the Coordinator to retry. + if (m2_stream && task.hydra_action.kv_expected_hash != 0) { + const uint64_t computed_kv = XXH3_64bits_digest(hst); + if (computed_kv != task.hydra_action.kv_expected_hash) { + SRV_WRN("hydra: DECODE_APPLY slot=%d SEGMENT_HASH_MISMATCH kv expected=%016" PRIx64 " got=%016" PRIx64 "\n", + id_slot, task.hydra_action.kv_expected_hash, computed_kv); + XXH3_freeState(hst); + hst = nullptr; + slot->reserved_for_decode_id = -1; + slot->prompt.tokens.clear(); + slot->prompt.checkpoints.clear(); + slot->n_prompt_tokens_cache = 0; + llama_memory_seq_rm(llama_get_memory(ctx_tgt), slot->id, -1, -1); + if (routes_ptr) { + server_routes::decode_result_entry entry; + entry.id_slot = id_slot; + entry.error = "KV segment hash mismatch (corrupt stream)"; + entry.created_at = std::time(nullptr); + entry.ttl_s = routes_ptr->decode_result_ttl_s; + std::lock_guard lock(routes_ptr->decode_results_mutex); + routes_ptr->decode_results[decode_request_id] = std::move(entry); + routes_ptr->evict_decode_results_locked(); + } + break; + } + SRV_INF("hydra: DECODE_APPLY slot=%d KV hash verified (%zu + %" PRIu64 " B)\n", + id_slot, task.hydra_action.kv_v2_hdr.size(), + task.hydra_action.kv_stream_len); + } + if (hst) { XXH3_freeState(hst); hst = nullptr; } + + const int n_past = is_v2 ? blob_n_past : kv_meta.value("n_past", 0); + if (n_past > 0) { + // Cache/processed counters come from the same header field + // STATE_PUT reads (hdr_n_tok == blob_n_tok here); PREFILL writes + // both fields as n_tokens so the values are identical today, + // but the two restore paths must read the SAME source. + slot->n_prompt_tokens_cache = is_v2 ? blob_n_tok : n_past; + slot->n_prompt_tokens_processed = is_v2 ? blob_n_tok : n_past; + + // Register the native checkpoint from the blob (v2) or + // fabricate one (legacy) — mirrors STATE_PUT (~line 3447). + // KV restored without its recurrent-memory checkpoint + // corrupts hybrid/recurrent model output. + slot->prompt.checkpoints.clear(); + if (has_chkpt) { + auto & ckpt = slot->prompt.checkpoints.emplace_back(); + ckpt.n_tokens = ckpt_n_tokens_in; + ckpt.pos_min = ckpt_pos_min_in; + ckpt.pos_max = ckpt_pos_max_in; + // New-format (v3) checkpoints carry a recurrent-only capture — + // route into data_*_recr and tag is_recr_only so the load path + // uses matched PARTIAL_ONLY flags (mirrors STATE_PUT). + ckpt.is_recr_only = ckpt_is_recr_only; + if (ckpt_is_recr_only) { + ckpt.data_tgt_recr = std::move(ckpt_tgt_data); + ckpt.data_dft_recr = std::move(ckpt_dft_data); + } else { + ckpt.data_tgt = std::move(ckpt_tgt_data); + ckpt.data_dft = std::move(ckpt_dft_data); + } + SLT_INF(*slot, "DECODE_APPLY registered native checkpoint (pos_min=%d pos_max=%d n_tokens=%" PRId64 " tgt_sz=%zu recr_only=%d)\n", + ckpt.pos_min, ckpt.pos_max, ckpt.n_tokens, ckpt.size(), (int) ckpt.is_recr_only); + } else { + const auto pos_min = llama_memory_seq_pos_min(llama_get_memory(ctx_tgt), slot->id); + create_checkpoint(*slot, 0, (llama_pos)pos_min, (llama_pos)(n_past - 1)); + } + } + slot->just_restored = true; + } + + const double restore_slot_ms = (double)(ggml_time_ms() - restore_start_ms); + const int n_past = slot->n_prompt_tokens_cache + slot->n_decoded; + + SRV_INF("hydra: DECODE_APPLY slot=%d restore=%.1fms n_past=%d model_load_ms=%.1f\n", + id_slot, restore_slot_ms, n_past, model_load_ms); + + // Release reservation — slot is now processing via completion + slot->reserved_for_decode_id = -1; + + // ── Build and post COMPLETION task ────────────────────── + { + json prompt = decode_req["prompt"]; + json cmpl_data; + cmpl_data["stream"] = prompt.value("stream", false); + // #622: the DECODE 0x43 frame has no dedicated stream_options + // channel, but the coordinator always requests usage on the + // merged path (it injects stream_options.include_usage=true on + // its HTTP body). Honor stream_options when the request carries + // it (generation header / prompt segment), otherwise mirror the + // coordinator's injection so the DONE-SSE delta carries usage + // natively and the coordinator's usage-based gate fires. + if (prompt.contains("stream_options") && prompt["stream_options"].is_object()) { + cmpl_data["stream_options"] = prompt["stream_options"]; + } else { + cmpl_data["stream_options"] = json{{"include_usage", true}}; + } + cmpl_data["n_predict"] = prompt.value("n_predict", 256); + cmpl_data["id_slot"] = id_slot; + if (prompt.contains("sampling")) { + const json & samp = prompt["sampling"]; + if (samp.contains("temperature")) cmpl_data["temperature"] = samp["temperature"]; + if (samp.contains("top_p")) cmpl_data["top_p"] = samp["top_p"]; + if (samp.contains("top_k")) cmpl_data["top_k"] = samp["top_k"]; + if (samp.contains("seed")) cmpl_data["seed"] = samp["seed"]; + } + if (prompt.contains("stop")) cmpl_data["stop"] = prompt["stop"]; + + std::string prompt_str; + if (prompt.contains("messages") && !prompt["messages"].is_null()) { + json chat_body; + chat_body["messages"] = prompt["messages"]; + if (prompt.contains("tools")) chat_body["tools"] = prompt["tools"]; + if (prompt.contains("tool_choice")) chat_body["tool_choice"] = prompt["tool_choice"]; + if (prompt.contains("response_format")) chat_body["response_format"] = prompt["response_format"]; + if (prompt.contains("add_generation_prompt")) chat_body["add_generation_prompt"] = prompt["add_generation_prompt"]; + if (prompt.contains("continue_final_message")) chat_body["continue_final_message"] = prompt["continue_final_message"]; + if (prompt.contains("reasoning_format")) chat_body["reasoning_format"] = prompt["reasoning_format"]; + if (prompt.contains("enable_thinking")) chat_body["enable_thinking"] = prompt["enable_thinking"]; + if (prompt.contains("chat_template_kwargs")) chat_body["chat_template_kwargs"] = prompt["chat_template_kwargs"]; + + try { + std::vector dummy_files; + json chat_result = oaicompat_chat_params_parse(chat_body, chat_params, dummy_files); + prompt_str = chat_result.value("prompt", std::string()); + if (chat_result.contains("grammar") && !chat_result["grammar"].is_null()) cmpl_data["grammar"] = chat_result["grammar"]; + if (chat_result.contains("grammar_type")) cmpl_data["grammar_type"] = chat_result["grammar_type"]; + if (chat_result.contains("grammar_lazy")) cmpl_data["grammar_lazy"] = chat_result["grammar_lazy"]; + if (chat_result.contains("grammar_triggers")) cmpl_data["grammar_triggers"] = chat_result["grammar_triggers"]; + if (chat_result.contains("chat_format")) cmpl_data["chat_format"] = chat_result["chat_format"]; + if (chat_result.contains("chat_parser")) cmpl_data["chat_parser"] = chat_result["chat_parser"]; + if (chat_result.contains("generation_prompt")) cmpl_data["generation_prompt"] = chat_result["generation_prompt"]; + if (chat_result.contains("parse_tool_calls")) cmpl_data["parse_tool_calls"] = chat_result["parse_tool_calls"]; + if (chat_result.contains("preserved_tokens")) cmpl_data["preserved_tokens"] = chat_result["preserved_tokens"]; + if (chat_result.contains("reasoning_budget_tokens")) cmpl_data["reasoning_budget_tokens"] = chat_result["reasoning_budget_tokens"]; + if (chat_result.contains("reasoning_budget_start_tag")) cmpl_data["reasoning_budget_start_tag"] = chat_result["reasoning_budget_start_tag"]; + if (chat_result.contains("reasoning_budget_end_tag")) cmpl_data["reasoning_budget_end_tag"] = chat_result["reasoning_budget_end_tag"]; + if (chat_result.contains("reasoning_budget_message")) cmpl_data["reasoning_budget_message"] = chat_result["reasoning_budget_message"]; + if (chat_result.contains("reasoning_control")) cmpl_data["reasoning_control"] = chat_result["reasoning_control"]; + if (chat_result.contains("stop") && chat_result["stop"].is_array()) { + json existing_stops = cmpl_data.value("stop", json::array()); + for (const auto & s : chat_result["stop"]) existing_stops.push_back(s); + cmpl_data["stop"] = existing_stops; + } + } catch (const std::exception & e) { + SRV_WRN("hydra: DECODE_APPLY slot=%d chat template failed: %s\n", id_slot, e.what()); + if (routes_ptr) { + server_routes::decode_result_entry entry; + entry.id_slot = id_slot; + entry.error = std::string("chat template error: ") + e.what(); + entry.created_at = std::time(nullptr); + entry.ttl_s = routes_ptr->decode_result_ttl_s; + std::lock_guard lock(routes_ptr->decode_results_mutex); + routes_ptr->decode_results[decode_request_id] = std::move(entry); + routes_ptr->evict_decode_results_locked(); + } + break; + } + } else { + prompt_str = prompt.value("prompt", std::string()); + } + cmpl_data["prompt"] = prompt_str; + + auto inputs = tokenize_input_prompts(vocab, mctx, prompt_str, true, true); + if (!inputs.empty()) { + const int32_t completion_id = queue_tasks.get_new_id(); + + server_task cmpl_task(SERVER_TASK_TYPE_COMPLETION); + cmpl_task.id = completion_id; + cmpl_task.id_slot = id_slot; + cmpl_task.tokens = std::move(inputs[0]); + cmpl_task.params = server_task::params_from_json_cmpl( + vocab, params_base, get_slot_n_ctx(), params_base.sampling.logit_bias_eog, cmpl_data); + cmpl_task.params.res_type = TASK_RESPONSE_TYPE_OAI_CHAT; + cmpl_task.params.oaicompat_cmpl_id = gen_chatcmplid(); + cmpl_task.params.oaicompat_model = model_name; + + // Mirror server_response_reader::post_task(): the + // consumer thread keeps its own result state so it + // can run result->update() per received result. + task_result_state cmpl_state = cmpl_task.create_state(); + + queue_results.add_waiting_task_id(completion_id); + queue_tasks.post(std::move(cmpl_task)); + SRV_INF("hydra: DECODE_APPLY slot=%d posted COMPLETION (completion_id=%d, request_id=%d)\n", + id_slot, completion_id, decode_request_id); + + // Update state to GENERATING + if (routes_ptr) { + std::lock_guard lk(routes_ptr->decode_results_mutex); + auto dit = routes_ptr->decode_results.find(decode_request_id); + if (dit != routes_ptr->decode_results.end()) { + dit->second.state = server_routes::DECODE_STATE_GENERATING; + dit->second.completion_id = std::to_string(completion_id); + dit->second.stream->completion_task_id = completion_id; + // Capture n_common observability from the slot + dit->second.n_common = slot->n_common; + dit->second.n_prompt_processed = slot->n_prompt_processed; + dit->second.logits_reused = slot->logits_reused; + } + } + + // ── Background consumer ───────────────────────── + // Sole listener on the completion task. Relays + // partial results into the decode_result_entry's + // streaming_queue so GET /v1/decode can stream + // them to the client. Stores the final result + // when generation completes. + if (routes_ptr) { + // Read match_json from the decode_result_entry (set by sync DECODE) + json match_j_bg; + { + std::lock_guard lk(routes_ptr->decode_results_mutex); + auto dit = routes_ptr->decode_results.find(decode_request_id); + if (dit != routes_ptr->decode_results.end()) { + match_j_bg = dit->second.match_json; + } + } + std::thread([this, completion_id, decode_request_id, id_slot, + match_j = std::move(match_j_bg), resident_tokenizer, resident_model_name, + resident_model_quant, resident_capabilities, + oaicompat_model_name = model_name, + model_load_ms, restore_slot_ms, n_past, + &results = queue_results, + states = std::vector{ std::move(cmpl_state) }]() mutable { + std::unordered_set ids = {(int)completion_id}; + bool got_final = false; + + // Loop: receive partials and relay, wait for final + while (!got_final) { + auto res_ptr = results.recv_with_timeout(ids, 120); + if (!res_ptr) { + SRV_WRN("hydra: DECODE_APPLY slot=%d generation timeout (request_id=%d, completion_id=%d)\n", + id_slot, decode_request_id, completion_id); + // Mark stream as finished so GET handler unblocks + { + std::lock_guard lk(routes_ptr->decode_results_mutex); + auto dit = routes_ptr->decode_results.find(decode_request_id); + if (dit != routes_ptr->decode_results.end() && dit->second.stream) { + std::lock_guard slk(dit->second.stream->streaming_mutex); + dit->second.stream->stream_finished = true; + dit->second.stream->streaming_cv.notify_all(); + } + } + return; + } + + // Check if this is a partial or final result + auto * partial = dynamic_cast(res_ptr.get()); + auto * final_r = dynamic_cast(res_ptr.get()); + + // Mirror server_response_reader::next(): run + // update() on every result before handling. + // Populates oaicompat_msg / oaicompat_msg_diffs + // (and sets is_updated, so to_json() won't + // assert on relayed partials). + try { + const size_t idx = res_ptr->index; + GGML_ASSERT(idx < states.size()); + res_ptr->update(states[idx]); + } catch (const std::exception & e) { + // Mirror the standard stream loop's tolerance + // of chat-parse failures (server-context.cpp:7685). + // This is a detached thread: an uncaught + // exception would std::terminate() the whole + // engine. Continue with the unparsed result + // (raw content; reasoning extraction skipped). + SRV_WRN("hydra: DECODE_APPLY slot=%d result update() failed: %s (continuing with unparsed result)\n", + id_slot, e.what()); + if (partial && !partial->is_begin) { + // Keep the relay well-formed: partial + // to_json() asserts is_updated in debug + // builds; with no diffs it emits an empty + // delta, which clients merge harmlessly. + partial->is_updated = true; + } + } + + if (partial && !partial->is_begin) { + // Relay partial to streaming queue + std::lock_guard lk(routes_ptr->decode_results_mutex); + auto dit = routes_ptr->decode_results.find(decode_request_id); + if (dit != routes_ptr->decode_results.end() && dit->second.stream) { + std::lock_guard slk(dit->second.stream->streaming_mutex); + dit->second.stream->streaming_queue.push_back(std::move(res_ptr)); + dit->second.stream->streaming_cv.notify_all(); + } + } else if (final_r) { + // Store final result and mark DONE + got_final = true; + + server_routes::decode_result_entry entry; + entry.id_slot = id_slot; + entry.completion_id = final_r->oaicompat_cmpl_id; + entry.oaicompat_model = oaicompat_model_name; + entry.content = final_r->content; + if (!final_r->oaicompat_msg.reasoning_content.empty()) { + entry.reasoning_content = final_r->oaicompat_msg.reasoning_content; + } + if (!final_r->oaicompat_msg.tool_calls.empty()) { + // Mirror common_chat_msg::to_json_oaicompat() shape so + // GET /v1/decode/:id returns OpenAI-format tool_calls. + json jtool_calls = json::array(); + for (const auto & tool_call : final_r->oaicompat_msg.tool_calls) { + json tc { + {"type", "function"}, + {"function", { + {"name", tool_call.name}, + {"arguments", json(tool_call.arguments)}, + }}, + }; + if (!tool_call.id.empty()) { + tc["id"] = tool_call.id; + } + jtool_calls.push_back(std::move(tc)); + } + entry.tool_calls = std::move(jtool_calls); + } + entry.n_decoded = final_r->n_decoded; + entry.n_prompt_tokens = final_r->n_prompt_tokens; + entry.n_prompt_tokens_cache = final_r->n_prompt_tokens_cache; + entry.timings = final_r->timings; + entry.stop = final_r->stop; + entry.include_usage = final_r->include_usage; + entry.match_json = match_j; + entry.created_at = std::time(nullptr); + entry.ttl_s = routes_ptr->decode_result_ttl_s; + + json metrics = json::object(); + metrics["decode_request_id"] = decode_request_id; + metrics["id_slot"] = id_slot; + metrics["n_past"] = final_r->n_prompt_tokens_cache + final_r->n_decoded; + metrics["decode_ms"] = final_r->timings.predicted_ms; + metrics["prompt_ms"] = final_r->timings.prompt_ms; + metrics["model_load_ms"] = model_load_ms; + metrics["restore_slot_ms"] = restore_slot_ms; + metrics["model_identity"] = { + {"tokenizer", resident_tokenizer}, + {"model_name", resident_model_name}, + {"model_quant", resident_model_quant}, + {"model_capabilities", resident_capabilities} + }; + metrics["match"] = match_j; + metrics["model_fallback"] = false; + // Hydra n_common observability + metrics["n_common"] = entry.n_common; + metrics["n_prompt_processed"] = entry.n_prompt_processed; + metrics["logits_reused"] = entry.logits_reused; + entry.hydra_metrics = metrics; + entry.state = server_routes::DECODE_STATE_DONE; + + // Signal stream finished before storing entry + { + std::lock_guard lk(routes_ptr->decode_results_mutex); + auto dit = routes_ptr->decode_results.find(decode_request_id); + if (dit != routes_ptr->decode_results.end() && dit->second.stream) { + // Transfer streaming state to the new entry + entry.stream = std::move(dit->second.stream); + { + std::lock_guard slk(entry.stream->streaming_mutex); + entry.stream->stream_finished = true; + } + entry.stream->streaming_cv.notify_all(); + } + } + + std::lock_guard lock(routes_ptr->decode_results_mutex); + routes_ptr->decode_results[decode_request_id] = std::move(entry); + routes_ptr->evict_decode_results_locked(); + + SRV_INF("hydra: DECODE_APPLY slot=%d generation complete (request_id=%d, n_decoded=%d)\n", + id_slot, decode_request_id, final_r->n_decoded); + } else { + // is_begin partial — just consume it + } + } + + results.remove_waiting_task_id(completion_id); + }).detach(); + } + } else { + SRV_WRN("hydra: DECODE_APPLY slot=%d tokenization failed\n", id_slot); + } + } + } break; + + case SERVER_TASK_TYPE_HYDRA_ENGINE_SET_EXPERT_MODE: + { + auto res = std::make_unique(); + res->id = task.id; + res->op = HYDRA_OP_SET_EXPERT_MODE; + + // Parse the payload. For backward compatibility, a raw string + // ("solo" or "combined") is accepted. Phase D (C# side) sends + // a JSON payload: {"mode":"combined","peer":"host:port",...}. + std::string requested; + std::string peer_override; + const std::string & raw = task.hydra_action.expert_mode; + if (!raw.empty() && raw[0] == '{') { + try { + json j = json::parse(raw); + requested = j.value("mode", "solo"); + peer_override = j.value("peer", ""); + } catch (...) { + requested = "solo"; + } + } else { + requested = raw; + } + + if (requested != "solo" && requested != "combined") { + res->rpc_status = HYDRA_STATUS_ERROR; + res->success = false; + res->error = "expert_mode must be 'solo' or 'combined'"; + queue_results.send(std::move(res)); + break; + } + + // #29 Phase B: per-request peer switching. If the peer changes, + // clean up the old binding and register the new one. The peer + // info comes from the SET_EXPERT_MODE control-plane payload + // (JSON {"mode":"combined","peer":"host:port"}), NOT from the + // HTTP inference body — keeping control and data separate. + if (!peer_override.empty() && peer_override != hydra_current_peer) { + // Guard: peer switch is unsafe while any slot is decoding. + // sched_reserve() destroys and rebuilds the scheduler, which + // invalidates in-flight decode state across all slots. + bool any_active = false; + for (const auto & s : slots) { + if (s.is_processing()) { any_active = true; break; } + } + if (any_active) { + SRV_WRN("hydra: cannot switch peers — %zu slot(s) are processing, rejecting SET_EXPERT_MODE\n", slots.size()); + res->rpc_status = HYDRA_STATUS_BUSY; + res->success = false; + res->error = "cannot switch peers while slots are processing"; + queue_results.send(std::move(res)); + break; + } + if (!hydra_current_peer.empty()) { + SRV_INF("hydra: switching from peer %s to %s — cleaning up old binding\n", + hydra_current_peer.c_str(), peer_override.c_str()); + ctx_tgt->hydra_remove_combined_rpc_backend(hydra_current_peer.c_str()); + } + hydra_current_peer = peer_override; + } + + // Hydra #383 T1: layer-split (static combined) engines cannot + // switch modes at runtime — the split is baked in at model load. + // "combined" is a no-op (already combined); "solo" is rejected. + if (hydra_combined_static) { + if (requested == "solo") { + res->rpc_status = HYDRA_STATUS_ERROR; + res->success = false; + res->error = "combined_static: this engine loaded in layer-split COMBINED mode; cannot switch to solo at runtime"; + LOG_WRN("srv %12.*s: hydra: SET_EXPERT_MODE solo rejected — engine is combined_static (layer-split)\n", 12, __func__); + queue_results.send(std::move(res)); + break; + } + // requested == "combined": success no-op + res->expert_mode_applied = "combined"; + res->rpc_status = HYDRA_STATUS_OK; + res->success = true; + LOG_INF("srv %12.*s: hydra: SET_EXPERT_MODE combined no-op — engine is combined_static (layer-split)\n", 12, __func__); + queue_results.send(std::move(res)); + break; + } + + // #368 fix: gate on "configured as combined head" (non-empty + // peer addr + OT pattern), NOT on whether the startup + // dual-load succeeded. The rebind path below is fail-open — + // if the peer is still unreachable it stays solo — so + // hydra_combined_head_attached (set only when startup + // succeeded) must NOT block the attempt. Hydra #287/#260/#348 + // intent is preserved: an unconfigured engine (no peer/ + // pattern) still falls back to solo immediately. + const bool want_combined = requested == "combined" && + !hydra_peer.empty() && !hydra_combined_pattern.empty(); + + // #368 (#357 fix): bind-on-activation. Re-bind the peer's + // expert tensors on each SET_EXPERT_MODE("combined") request + // so a peer that was down at boot is picked up on the first + // COMBINED request after it comes up. Fail-open: if the + // rebind fails we stay solo and the Coordinator's + // ReportsSolo path handles it. + bool actually_combined = want_combined; + if (want_combined) { + if (hydra_peer.empty() || hydra_combined_pattern.empty()) { + SRV_WRN("%s\n", "hydra: SET_EXPERT_MODE(combined) but no peer/pattern configured; staying solo"); + actually_combined = false; + } else { + // ggml_backend_rpc_add_server is idempotent — returns + // the existing reg if the peer was registered before. + ggml_backend_reg_t rpc_reg = ggml_backend_reg_by_name("RPC"); + if (!rpc_reg) { + SRV_WRN("%s\n", "hydra: SET_EXPERT_MODE(combined) but RPC backend not available; staying solo"); + actually_combined = false; + } else { + using add_server_fn_t = ggml_backend_reg_t (*)(const char *); + auto add_server_fn = (add_server_fn_t) ggml_backend_reg_get_proc_address(rpc_reg, "ggml_backend_rpc_add_server"); + ggml_backend_reg_t peer_reg = add_server_fn ? add_server_fn(hydra_peer.c_str()) : nullptr; + ggml_backend_dev_t peer_dev = (peer_reg && ggml_backend_reg_dev_count(peer_reg) > 0) ? ggml_backend_reg_dev_get(peer_reg, 0) : nullptr; + if (!peer_dev) { + SRV_WRN("hydra: SET_EXPERT_MODE(combined) but peer %s has no registered device; staying solo\n", + hydra_peer.c_str()); + actually_combined = false; + } else { + int32_t n_bound = llama_hydra_rebind_combined_experts( + ctx_tgt, hydra_peer.c_str(), peer_dev, hydra_combined_pattern.c_str()); + if (n_bound <= 0) { + SRV_WRN("hydra: SET_EXPERT_MODE(combined) rebind on peer %s returned %d; staying solo\n", + hydra_peer.c_str(), n_bound); + actually_combined = false; + } else { + // Peer is up — latch so INFO RPC advertises combined. + hydra_combined_head_attached = true; + } + } + } + } + } + + llama_hydra_set_expert_mode(ctx_tgt, actually_combined ? 1 : 0); + res->expert_mode_applied = actually_combined ? "combined" : "solo"; + + res->rpc_status = HYDRA_STATUS_OK; + res->success = true; + SRV_INF("hydra: SET_EXPERT_MODE requested='%s' applied='%s' (slot %d)\n", + requested.c_str(), res->expert_mode_applied.c_str(), task.hydra_action.id_slot); + queue_results.send(std::move(res)); + } break; + + case SERVER_TASK_TYPE_HYDRA_ENGINE_SWAP_QUANT: + { + auto res = std::make_unique(); + res->id = task.id; + res->op = HYDRA_OP_SWAP_QUANT; + res->rpc_status = HYDRA_STATUS_OK; + res->success = true; + SRV_INF("hydra: SWAP_QUANT quant='%s' pattern='%s' (slot %d)\n", + task.hydra_action.quant_key.c_str(), + task.hydra_action.tensor_pattern.c_str(), + task.hydra_action.id_slot); + queue_results.send(std::move(res)); + } break; + + // M-Perf.9 (#289) / issue #287: PIPELINE_ATTACH is part of the + // two-engine "work together" routing tracked in #287. The + // coordinator wires the request; the engine-side scaffolding + // (--override-tensor local-load, activation passing, COMBINED + // expert mode) is the next deliverable. For now this opcode + // returns NOT_IMPLEMENTED so the wire stays in sync — the + // coordinator will treat that as a fallback to solo mode. + case SERVER_TASK_TYPE_HYDRA_ENGINE_PIPELINE_ATTACH: + { + auto res = std::make_unique(); + res->id = task.id; + res->op = HYDRA_OP_PIPELINE_ATTACH; + res->rpc_status = HYDRA_STATUS_NOT_IMPLEMENTED; + res->success = false; + res->error = "HYDRA_OP_PIPELINE_ATTACH not yet implemented in this build (see issue #287)"; + SRV_WRN("hydra: PIPELINE_ATTACH received (slot %d) — stubbed, issue #287\n", + task.hydra_action.id_slot); + queue_results.send(std::move(res)); + } break; + } + } + + +// --------------------------------------------------------------------------- + server_context_impl::hydra_config_result server_context_impl::hydra_apply_config(const json & cfg, bool sync) { + server_context_impl::hydra_config_result result; + + // 1. Classify every top-level key. T1 → apply now; T2/T3/T4 → + // defer (stage) or apply synchronously depending on `sync`. + // T4 (generic) keys are additionally validated against the + // llama.cpp arg table here so the CONFIGURE response can report + // unrecognized/rejected keys before the deferred apply runs. + // T2 + appliable-T4 keys are staged together into the reload + // config (g_pending_reload_config) so neither the context-reload + // path nor the model-reload path strands them. + json t1_subset = json::object(); + json reload_subset = json::object(); + for (auto it = cfg.begin(); it != cfg.end(); ++it) { + const std::string key = it.key(); + int tier = hydra_classify_config_key(key); + if (tier == 1) { + t1_subset[key] = it.value(); + } else if (tier == 2) { + result.t2t3_subset[key] = it.value(); + result.deferred_keys.push_back(key); + reload_subset[key] = it.value(); + } else if (tier == 3) { + result.t2t3_subset[key] = it.value(); + result.deferred_keys.push_back(key); + // T3 keys are staged via hydra_apply_t3_mutators() statics, + // not the reload-config JSON. + } else { + // T4: generic-arg pass-through. Classify against the arg + // table; only appliable keys are staged for the deferred + // slot-free moment. The rest are reported loudly. + const hydra_generic_key_status st = hydra_classify_generic_key(key); + if (st == hydra_generic_key_status::APPLIABLE) { + result.t2t3_subset[key] = it.value(); + result.deferred_keys.push_back(key); + reload_subset[key] = it.value(); + } else if (st == hydra_generic_key_status::DENIED) { + SRV_WRN("hydra: CONFIGURE key '%s' cannot change at reload (startup-only/flag arg) — rejected\n", + key.c_str()); + result.rejected_keys.push_back(key); + } else { + SRV_WRN("hydra: CONFIGURE key '%s' is not a known llama.cpp argument — unrecognized, value ignored\n", + key.c_str()); + result.unrecognized_keys.push_back(key); + } + } + if (tier > result.highest_tier) result.highest_tier = tier; + } + // Stage the reload config (T2 + appliable-T4 keys) for the deferred + // slot-free moment. Unconditional overwrite = absolute state: a + // superseding CONFIGURE without T2/T4 keys must clear whatever an + // earlier CONFIGURE staged, or the stale keys would be applied on + // the next unrelated reload. + g_pending_reload_config = reload_subset.dump(); + // The "sampling" object may contain unlisted nested keys + // (e.g. penalty_last_n, mirostat) — route the whole object + // through T1 when present. + if (cfg.contains("sampling") && cfg["sampling"].is_object()) { + t1_subset["sampling"] = cfg["sampling"]; + if (result.highest_tier < 1) result.highest_tier = 1; + } + // model.path is nested — the legacy {"model": {...}} form + // is recognized by hydra_classify_config_key returning 3 + // for the bare "model" key. If the bare "model" is set + // and is an object with a "path", route it as T3. + if (cfg.contains("model")) { + if (cfg["model"].is_object()) { + result.t2t3_subset["model"] = cfg["model"]; + if (std::find(result.deferred_keys.begin(), result.deferred_keys.end(), "model") + == result.deferred_keys.end()) { + result.deferred_keys.push_back("model"); + } + if (result.highest_tier < 3) result.highest_tier = 3; + } else if (cfg["model"].is_string()) { + result.t2t3_subset["model"] = cfg["model"]; + if (std::find(result.deferred_keys.begin(), result.deferred_keys.end(), "model") + == result.deferred_keys.end()) { + result.deferred_keys.push_back("model"); + } + if (result.highest_tier < 3) result.highest_tier = 3; + } + } + + if (result.highest_tier == 0) { + // No recognized keys — caller decides whether to treat as + // a no-op or surface an error. + return result; + } + + // 2. Apply T1 keys in-place. + if (!t1_subset.empty()) { + if (!hydra_apply_t1_config(params_base, ctx_tgt, t1_subset, result.params_applied)) { + result.ok = false; + result.error = "T1 key has wrong type (see log)"; + return result; + } + // Capture state_chunk_size for callers that need it (CONFIGURE). + auto it = result.params_applied.find("state_chunk_size"); + if (it != result.params_applied.end() && it->second.is_number_unsigned()) { + result.state_chunk_size_applied = it->second.get(); + } + } + + // 3. T2/T3/T4 handling — diverges based on sync flag. + if (!result.t2t3_subset.empty()) { + if (sync) { + // Synchronous mode (PREFILL / HTTP decode): apply now + // on the task-queue thread. The caller owns this thread + // context so blocking is safe. + if (result.highest_tier >= 3) { + // T3 (model reload) also covers a T4-only config: + // load_model() recreates the context and the + // speculative/draft state, so every generic key + // takes effect. hydra_apply_t3_mutators() is a no-op + // when no T3 keys are staged. + hydra_apply_t3_mutators(ctx_tgt, result.t2t3_subset, result.deferred_keys); + // #470: force rebuild if a peer reconnection was detected + // during a prior graph_compute — the peer's buffers are gone + // even though model/params haven't changed. + const bool reconn_force = (ctx_tgt && ctx_tgt->peer_reconnection_pending); + if (reconn_force) { + ctx_tgt->peer_reconnection_pending = false; + SRV_WRN("%s", "hydra: PREFILL handler: peer reconnection pending — forcing T3 rebuild\n"); + } + if (!apply_t3_rebuild(reconn_force)) { + result.ok = false; + result.error = "T3 rebuild failed"; + return result; + } + } else if (result.highest_tier == 2) { + if (!apply_t2_rebuild(result.t2t3_subset.dump())) { + result.ok = false; + result.error = "T2 rebuild failed"; + return result; + } + } + // Clear T3 staged statics — they were consumed by the + // sync apply and must not leak into a later deferred path. + llama_hydra_clear_pending_t3(); + } else { + // Stage mode (CONFIGURE): record the mutators and store + // pending_config for application at the next slot-free moment. + if (result.highest_tier >= 3) { + hydra_apply_t3_mutators(ctx_tgt, result.t2t3_subset, result.deferred_keys); + } + if (ctx_tgt) { + ctx_tgt->hydra_set_pending_config( + result.t2t3_subset.dump(), hydra_tier_label(result.highest_tier)); + } else { + // First load: ctx_tgt is null, so apply_pending_hydra_config() + // and update_slots() can't trigger. Set the flag so the + // task-queue thread runs apply_t3_rebuild() at the next + // slot-free moment. + first_load_pending = true; + SRV_INF("%s", "hydra: config staged for first load (no context yet)\n"); + } + } + } + + return result; + } + + +// WS1/WS2: the extension object. handle_task() routes HYDRA tasks to the same +// hydra_process_task() method the legacy switch calls — seam == legacy behavior. +// --------------------------------------------------------------------------- +struct hydra_engine_extension : server_hydra_extension { + const char * name() const override { + return "hydra-task-ws2"; + } + + bool handle_task(server_context_impl & impl, server_task & task) override { + switch (task.type) { + case SERVER_TASK_TYPE_HYDRA_STATE_GET: + case SERVER_TASK_TYPE_HYDRA_STATE_PUT: + case SERVER_TASK_TYPE_HYDRA_STATE_META: + case SERVER_TASK_TYPE_HYDRA_ENGINE_CONFIGURE: + case SERVER_TASK_TYPE_HYDRA_ENGINE_INFO: + case SERVER_TASK_TYPE_HYDRA_ENGINE_PREFILL: + case SERVER_TASK_TYPE_HYDRA_ENGINE_DECODE: + case SERVER_TASK_TYPE_HYDRA_DECODE_APPLY: + case SERVER_TASK_TYPE_HYDRA_ENGINE_SET_EXPERT_MODE: + case SERVER_TASK_TYPE_HYDRA_ENGINE_SWAP_QUANT: + case SERVER_TASK_TYPE_HYDRA_ENGINE_PIPELINE_ATTACH: + impl.hydra_process_task(task); + return true; // claimed + default: + return false; // not a Hydra task — fall through to inline dispatch + } + } + + bool pre_loop(server_context_impl & impl) override { + // WS3: handle the slot-free CONFIGURE/T3 moment at the top of + // update_slots(). Mirrors the inline all-idle block; returns true only + // when a staged CONFIGURE was actually applied (so the rest of + // update_slots() is skipped, matching the inline `return;`). + bool all_idle = true; + for (auto & slot : impl.slots) { + if (slot.is_processing() || slot.hydra_transferring->load()) { + all_idle = false; + break; + } + } + if (!all_idle) { + return false; // not the slot-free moment — inline decode body runs + } + + if (impl.ctx_tgt && impl.ctx_tgt->hydra_has_pending_config()) { + SRV_INF("hydra ext: slot-free moment — applying pending CONFIGURE (tier=%s)\n", + impl.ctx_tgt->hydra_get_pending_config_tier().c_str()); + impl.apply_pending_hydra_config(); + return true; + } + if (!impl.ctx_tgt && impl.first_load_pending) { + SRV_INF("%s", "hydra ext: slot-free moment — first load (no context yet)\n"); + impl.apply_pending_hydra_config(); + return true; + } + return false; // nothing staged — inline all-idle block logs + returns + } + + bool on_empty_batch(server_context_impl & impl) override { + // WS3: replicate the inline empty-batch transfer-suppression, including + // the 2s grace window after a transfer ends. Returns true when the + // empty batch was caused by a STATE_GET transfer (suppress the abort); + // false otherwise (inline logic runs, which eventually aborts). + static int64_t hydra_last_transfer_ms = 0; + static int64_t hydra_suppress_count = 0; + + bool any_transferring = false; + for (const auto & s : impl.slots) { + if (s.hydra_transferring && s.hydra_transferring->load()) { + any_transferring = true; + break; + } + } + const int64_t now_ms = ggml_time_us() / 1000; + if (any_transferring) { + hydra_last_transfer_ms = now_ms; + } + if (any_transferring || now_ms - hydra_last_transfer_ms < 2000) { + if (hydra_suppress_count++ % 256 == 0) { + SRV_WRN("hydra ext: empty batch suppressed — transfer %s\n", + any_transferring ? "in flight" : "just ended"); + } + impl.n_empty_consecutive = 0; + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + return true; + } + return false; + } +}; + +std::unique_ptr hydra_create_extension() { + return std::make_unique(); +} + +// --------------------------------------------------------------------------- +// WS3.5: Hydra RPC server (moved from server-context.cpp, epic #610). +// Same TU via the bottom #include, so hydra_rpc_ctx and the static handlers +// can reach server_context_impl privates (friend) and the file-scope +// queue/response objects. Wire format: specs/rpc-protocol.md | server-rpc.h. +// --------------------------------------------------------------------------- +// ═══════════════════════════════════════════════════════════════════════════════ +// Hydra RPC server — KV state transfer (M1: task-queue based) +// Wire format: specs/rpc-protocol.md | constants: server-rpc.h +// Ops implemented: STATE_GET (0x30), STATE_PUT (0x31), STATE_META (0x32) +// M1: All llama API calls routed through task queue (inference thread safe) +// ═══════════════════════════════════════════════════════════════════════════════ + +#if !defined(_WIN32) + +// ── Context for RPC thread — pass to handlers ───────────────────────────────── + +struct hydra_rpc_ctx { + server_queue * queue_tasks = nullptr; + server_response * queue_results = nullptr; +}; + +// ── Low-level I/O helpers ───────────────────────────────────────────────────── + +// Hydra #43: failures here were previously silent — every caller treats a +// `false` return as "give up" but none logged *why*, so a wedged RPC +// response looked identical to a client that vanished. Log once, centrally, +// instead of touching the ~30 call sites. +static bool hydra_recv_all(int fd, void * buf, size_t n) { + char * p = reinterpret_cast(buf); + const size_t total = n; + while (n > 0) { + ssize_t r = ::recv(fd, p, n, 0); + if (r < 0) { + SRV_WRN("hydra rpc: recv failed on fd=%d (%zu/%zu bytes): %s\n", + fd, total - n, total, std::strerror(errno)); + return false; + } + if (r == 0) { + SRV_DBG("hydra rpc: recv EOF on fd=%d (%zu/%zu bytes)\n", fd, total - n, total); + return false; + } + p += r; n -= r; + } + return true; +} + +static bool hydra_send_all(int fd, const void * buf, size_t n) { + const char * p = reinterpret_cast(buf); + const size_t total = n; + while (n > 0) { + ssize_t w = ::send(fd, p, n, MSG_NOSIGNAL); + if (w <= 0) { + SRV_WRN("hydra rpc: send failed on fd=%d (%zu/%zu bytes) w=%zd: %s\n", + fd, total - n, total, w, std::strerror(errno)); + return false; + } + p += w; n -= w; + } + return true; +} + +// Response header: status(1) | meta_len(3 LE uint24) | payload_len(8 LE) — 12 bytes +static void hydra_write_res(int fd, uint8_t status, uint32_t meta_len, uint64_t payload_len) { + uint8_t buf[HYDRA_RES_HEADER_SIZE] = {}; + buf[0] = status; + buf[1] = (meta_len) & 0xFF; + buf[2] = (meta_len >> 8) & 0xFF; + buf[3] = (meta_len >> 16) & 0xFF; + memcpy(buf + 4, &payload_len, 8); // little-endian (x86/arm64) + hydra_send_all(fd, buf, HYDRA_RES_HEADER_SIZE); +} + +// ── Op handlers (M1: dispatch via task queue) ───────────────────────────────── + +// STATE_GET (0x30): Post task, wait for result. +// +// M1 path (hydra_fd < 0): inference thread serializes 800 MB into result buffer; +// RPC thread sends response header + meta JSON + buffer here. +// +// M2 path (hydra_fd = fd): background thread streams GPU→socket directly using +// llama_state_seq_get_data_to_fd; result carries only n_past + streamed_bytes. +// Response header + meta are sent BEFORE the task (we know size from STATE_META), +// so the payload is already on the wire before we even get the result back. +// Actually: we must send header AFTER knowing state_size. So: +// - If M2: we get state_size first from a quick STATE_META query (n_past already known), +// OR we embed state_size in the result from get_size() on the inference thread. +// The inference thread always calls llama_state_seq_get_size (cheap) and stores it +// in res->state_size for M2 so we can send the header before the stream completes. +// +// Timeout: 30s — streaming 800 MB over localhost may take a few seconds. +static void hydra_handle_state_get(int fd, int slot_id, const hydra_rpc_ctx & ctx) { + // Build task — pass fd for M2 zero-copy streaming + server_task task(SERVER_TASK_TYPE_HYDRA_STATE_GET); + task.id = ctx.queue_tasks->get_new_id(); + task.hydra_action.id_slot = slot_id; + task.hydra_action.hydra_fd = fd; // M2: background thread streams here + const int task_id = task.id; + // Register BEFORE posting — server_response::send() silently drops results + // for ids not in waiting_task_ids. + ctx.queue_results->add_waiting_task_id(task_id); + ctx.queue_tasks->wait_until_no_sleep(); + ctx.queue_tasks->post(std::move(task)); + + // Wait for result (n_past + state_size always set; state_data only on M1) + std::unordered_set task_ids = {task_id}; + auto res_ptr = ctx.queue_results->recv_with_timeout(task_ids, 30); // seconds + ctx.queue_results->remove_waiting_task_id(task_id); + if (!res_ptr) { + SRV_WRN("hydra rpc: STATE_GET timeout for slot %d\n", slot_id); + // M2 caveat: the background thread may own the fd (header possibly sent); + // writing an error header here could interleave with the stream. Shut the + // socket down instead so the client unblocks with a clean EOF. + ::shutdown(fd, SHUT_RDWR); + return; + } + + auto * res = dynamic_cast(res_ptr.get()); + if (!res) { + SRV_WRN("hydra rpc: STATE_GET result type mismatch for slot %d\n", slot_id); + hydra_write_res(fd, HYDRA_STATUS_ERROR, 0, 0); + return; + } + + if (res->rpc_status != HYDRA_STATUS_OK) { + if (res->header_sent) { + // M2 failure: header already sent but stream failed; background thread + // shut the socket down — connection loop will close the fd on next read. + // Log and return without sending a second response header. + SRV_WRN("hydra rpc: STATE_GET slot=%d M2 stream failed: %s\n", + slot_id, res->error.c_str()); + return; + } + hydra_write_res(fd, res->rpc_status, 0, 0); + if (!res->error.empty()) { + hydra_send_all(fd, res->error.data(), res->error.size()); + } + return; + } + + if (res->streamed_bytes > 0) { + // M2 path: data already on the wire — response header + meta were sent by background thread. + // Nothing left for RPC thread to do. The protocol framing (header + meta + payload) + // was completed inside llama_io_write_socket / the background thread. + // Note: header was sent AFTER state_size was known (inference thread called get_size). + SRV_INF("hydra rpc: STATE_GET slot=%d M2 streamed %.1f MiB directly\n", + slot_id, res->streamed_bytes / (1024.0 * 1024.0)); + } else { + // M1 path: inference thread buffered 800 MB; send it now. + const uint64_t payload = (uint64_t)res->state_data.size(); + json meta_j; + meta_j["n_past"] = res->n_past; + meta_j["state_size"] = payload; + if (!res->model_alias.empty()) meta_j["model_alias"] = res->model_alias; + if (!res->model_path.empty()) meta_j["model_path"] = res->model_path; + if (!res->tokenizer.empty()) meta_j["tokenizer"] = res->tokenizer; + if (!res->model_name.empty()) meta_j["model_name"] = res->model_name; + if (!res->model_quant.empty()) meta_j["model_quant"] = res->model_quant; + if (res->model_capabilities) meta_j["model_capabilities"] = res->model_capabilities; + const std::string meta_str = meta_j.dump(); + hydra_write_res(fd, HYDRA_STATUS_OK, (uint32_t)meta_str.size(), payload); + hydra_send_all(fd, meta_str.data(), meta_str.size()); + hydra_send_all(fd, res->state_data.data(), (size_t)payload); + SRV_INF("hydra rpc: STATE_GET slot=%d M1 sent %.1f MiB from buffer\n", + slot_id, payload / (1024.0 * 1024.0)); + } +} + +// STATE_PUT (0x31): Receive payload, post task, wait for result, send ack. +static void hydra_handle_state_put(int fd, int slot_id, uint64_t payload_len, const hydra_rpc_ctx & ctx) { + if (payload_len > HYDRA_MAX_STATE_BYTES) { + SRV_WRN("hydra rpc: STATE_PUT payload %" PRIu64 " B exceeds cap %" PRIu64 " B\n", + payload_len, HYDRA_MAX_STATE_BYTES); + hydra_write_res(fd, HYDRA_STATUS_ERROR, 0, 0); + // Drain to keep persistent connection alive + std::vector drain(65536); + for (uint64_t rem = payload_len; rem > 0; ) { + size_t chunk = (size_t)std::min(rem, (uint64_t)drain.size()); + if (!hydra_recv_all(fd, drain.data(), chunk)) break; + rem -= chunk; + } + return; + } + + // Read payload from socket + std::vector buf((size_t)payload_len); + if (!hydra_recv_all(fd, buf.data(), (size_t)payload_len)) { + SRV_WRN("%s", "hydra rpc: STATE_PUT failed to read payload\n"); + hydra_write_res(fd, HYDRA_STATUS_ERROR, 0, 0); + return; + } + + // Post task to inference thread + server_task task(SERVER_TASK_TYPE_HYDRA_STATE_PUT); + task.id = ctx.queue_tasks->get_new_id(); + task.hydra_action.id_slot = slot_id; + task.hydra_action.erase_existing = true; // RPC restore always replaces slot state + task.hydra_action.state_data = std::move(buf); + const int task_id = task.id; + // Register BEFORE posting — results for unregistered ids are dropped. + ctx.queue_results->add_waiting_task_id(task_id); + ctx.queue_tasks->wait_until_no_sleep(); + ctx.queue_tasks->post(std::move(task)); + + // Wait for result from inference thread (30s timeout for large restore) + std::unordered_set task_ids = {task_id}; + auto res_ptr = ctx.queue_results->recv_with_timeout(task_ids, 30); // seconds + ctx.queue_results->remove_waiting_task_id(task_id); + if (!res_ptr) { + SRV_WRN("hydra rpc: STATE_PUT timeout for slot %d\n", slot_id); + hydra_write_res(fd, HYDRA_STATUS_ERROR, 0, 0); + return; + } + + auto * res = dynamic_cast(res_ptr.get()); + if (!res) { + SRV_WRN("hydra rpc: STATE_PUT result type mismatch for slot %d\n", slot_id); + hydra_write_res(fd, HYDRA_STATUS_ERROR, 0, 0); + return; + } + + // Send result back to client + uint8_t rpc_status = res->rpc_status; + if (rpc_status == HYDRA_STATUS_OK) { + json meta_j; + meta_j["restored"] = true; + meta_j["bytes"] = res->bytes; + meta_j["model_match"] = res->model_match; + if (!res->model_alias.empty()) meta_j["model_alias"] = res->model_alias; + if (!res->model_path.empty()) meta_j["model_path"] = res->model_path; + if (!res->tokenizer.empty()) meta_j["tokenizer"] = res->tokenizer; + if (!res->model_name.empty()) meta_j["model_name"] = res->model_name; + if (!res->model_quant.empty()) meta_j["model_quant"] = res->model_quant; + if (res->model_capabilities) meta_j["model_capabilities"] = res->model_capabilities; + const std::string meta_str = meta_j.dump(); + hydra_write_res(fd, HYDRA_STATUS_OK, (uint32_t)meta_str.size(), 0); + hydra_send_all(fd, meta_str.data(), meta_str.size()); + } else { + json err_j; + err_j["error"] = res->error; + const std::string err_str = err_j.dump(); + hydra_write_res(fd, rpc_status, (uint32_t)err_str.size(), 0); + hydra_send_all(fd, err_str.data(), err_str.size()); + } +} + +// STATE_META (0x32): Post task, wait for result, send JSON metadata. +static void hydra_handle_state_meta(int fd, int slot_id, const hydra_rpc_ctx & ctx) { + server_task task(SERVER_TASK_TYPE_HYDRA_STATE_META); + task.id = ctx.queue_tasks->get_new_id(); + task.hydra_action.id_slot = slot_id; + const int task_id = task.id; + // Register BEFORE posting — results for unregistered ids are dropped. + ctx.queue_results->add_waiting_task_id(task_id); + ctx.queue_tasks->wait_until_no_sleep(); + ctx.queue_tasks->post(std::move(task)); + + // Wait for result from inference thread (5s timeout — allows for queue congestion) + std::unordered_set task_ids = {task_id}; + auto res_ptr = ctx.queue_results->recv_with_timeout(task_ids, 5); // seconds + ctx.queue_results->remove_waiting_task_id(task_id); + if (!res_ptr) { + SRV_WRN("hydra rpc: STATE_META timeout for slot %d\n", slot_id); + hydra_write_res(fd, HYDRA_STATUS_ERROR, 0, 0); + return; + } + + auto * res = dynamic_cast(res_ptr.get()); + if (!res) { + SRV_WRN("hydra rpc: STATE_META result type mismatch for slot %d\n", slot_id); + hydra_write_res(fd, HYDRA_STATUS_ERROR, 0, 0); + return; + } + + // Send result back to client + uint8_t rpc_status = res->rpc_status; + if (rpc_status == HYDRA_STATUS_OK) { + json meta_j; + meta_j["slot_id"] = res->id_slot; + meta_j["n_past"] = res->n_past; + meta_j["state_size"] = res->state_size; + meta_j["is_processing"] = res->is_processing; + meta_j["is_transferring"] = res->is_transferring; + if (!res->model_alias.empty()) meta_j["model_alias"] = res->model_alias; + if (!res->model_path.empty()) meta_j["model_path"] = res->model_path; + if (!res->tokenizer.empty()) meta_j["tokenizer"] = res->tokenizer; + if (!res->model_name.empty()) meta_j["model_name"] = res->model_name; + if (!res->model_quant.empty()) meta_j["model_quant"] = res->model_quant; + if (res->model_capabilities) meta_j["model_capabilities"] = res->model_capabilities; + const std::string meta_str = meta_j.dump(); + hydra_write_res(fd, HYDRA_STATUS_OK, (uint32_t)meta_str.size(), 0); + hydra_send_all(fd, meta_str.data(), meta_str.size()); + } else { + hydra_write_res(fd, rpc_status, 0, 0); + } +} + +// ── E1 Engine control handlers ──────────────────────────────────────────────── + +// CONFIGURE (0x33): Read JSON config payload, post task, return success. +static void hydra_handle_configure(int fd, int slot_id, uint64_t payload_len, const hydra_rpc_ctx & ctx) { + std::string config_json(payload_len, '\0'); + if (payload_len > 0 && !hydra_recv_all(fd, config_json.data(), payload_len)) { + hydra_write_res(fd, HYDRA_STATUS_ERROR, 0, 0); + return; + } + + server_task task(SERVER_TASK_TYPE_HYDRA_ENGINE_CONFIGURE); + task.id = ctx.queue_tasks->get_new_id(); + task.hydra_action.id_slot = slot_id; + task.hydra_action.config_json = std::move(config_json); + const int task_id = task.id; + ctx.queue_results->add_waiting_task_id(task_id); + ctx.queue_tasks->wait_until_no_sleep(); + ctx.queue_tasks->post(std::move(task)); + + std::unordered_set task_ids = {task_id}; + auto res_ptr = ctx.queue_results->recv_with_timeout(task_ids, 5); + ctx.queue_results->remove_waiting_task_id(task_id); + if (!res_ptr) { + hydra_write_res(fd, HYDRA_STATUS_ERROR, 0, 0); + return; + } + + auto * res = dynamic_cast(res_ptr.get()); + if (!res || !res->success) { + // hydra#406: on failure, include the error message in the meta so + // the Coordinator can distinguish "drain timeout" from a parse + // error. We still write HYDRA_STATUS_ERROR (0x02) per the wire + // contract — the meta body is for diagnostics only. + if (res && !res->error.empty()) { + json err_j = {{"success", false}, {"error", res->error}}; + if (!res->tier.empty()) err_j["tier"] = res->tier; + const std::string err_str = err_j.dump(); + hydra_write_res(fd, HYDRA_STATUS_ERROR, (uint32_t)err_str.size(), 0); + hydra_send_all(fd, err_str.data(), err_str.size()); + } else { + hydra_write_res(fd, HYDRA_STATUS_ERROR, 0, 0); + } + return; + } + + // hydra#406: tiered CONFIGURE response shape. Always present: success, + // tier, params_applied (T1 keys), deferred_keys (T2/T3 keys). + json meta_j = { + {"success", true}, + {"tier", res->tier.empty() ? std::string("T1") : res->tier}, + {"params_applied", json::object()}, + {"deferred_keys", json::array()}, + }; + for (const auto & kv : res->params_applied) { + meta_j["params_applied"][kv.first] = kv.second; + } + for (const auto & k : res->deferred_keys) { + meta_j["deferred_keys"].push_back(k); + } + // hydra#334: echo the post-clamp value for the state_chunk_size legacy + // path so the Coordinator's existing detection logic still works + // (the same value is also in params_applied, with the dotted key). + if (res->state_chunk_size_applied > 0) { + meta_j["state_chunk_size_applied"] = res->state_chunk_size_applied; + } + const std::string meta_str = meta_j.dump(); + hydra_write_res(fd, HYDRA_STATUS_OK, (uint32_t)meta_str.size(), 0); + hydra_send_all(fd, meta_str.data(), meta_str.size()); +} + +// INFO (0x34): Return engine capabilities as JSON. +static void hydra_handle_info(int fd, int slot_id, const hydra_rpc_ctx & ctx) { + server_task task(SERVER_TASK_TYPE_HYDRA_ENGINE_INFO); + task.id = ctx.queue_tasks->get_new_id(); + task.hydra_action.id_slot = slot_id; + const int task_id = task.id; + ctx.queue_results->add_waiting_task_id(task_id); + ctx.queue_tasks->wait_until_no_sleep(); + ctx.queue_tasks->post(std::move(task)); + + std::unordered_set task_ids = {task_id}; + auto res_ptr = ctx.queue_results->recv_with_timeout(task_ids, 5); + ctx.queue_results->remove_waiting_task_id(task_id); + if (!res_ptr) { + hydra_write_res(fd, HYDRA_STATUS_ERROR, 0, 0); + return; + } + + auto * res = dynamic_cast(res_ptr.get()); + if (!res) { + hydra_write_res(fd, HYDRA_STATUS_ERROR, 0, 0); + return; + } + + const std::string & info_str = res->info_json; + hydra_write_res(fd, HYDRA_STATUS_OK, (uint32_t)info_str.size(), 0); + hydra_send_all(fd, info_str.data(), info_str.size()); +} + +// PREFILL (0x35): Read JSON payload with {"messages": [...]}, +// tokenize internally, run prefill, return n_past + KV state blob. +static void hydra_handle_prefill(int fd, int slot_id, uint64_t payload_len, const hydra_rpc_ctx & ctx) { + if (payload_len == 0) { + hydra_write_res(fd, HYDRA_STATUS_ERROR, 0, 0); + return; + } + + std::string json_str((size_t)payload_len, '\0'); + if (!hydra_recv_all(fd, json_str.data(), (size_t)payload_len)) { + hydra_write_res(fd, HYDRA_STATUS_ERROR, 0, 0); + return; + } + + server_task task(SERVER_TASK_TYPE_HYDRA_ENGINE_PREFILL); + task.id = ctx.queue_tasks->get_new_id(); + task.hydra_action.id_slot = slot_id; + task.hydra_action.request_json = std::move(json_str); + const int task_id = task.id; + ctx.queue_results->add_waiting_task_id(task_id); + ctx.queue_tasks->wait_until_no_sleep(); + ctx.queue_tasks->post(std::move(task)); + + std::unordered_set task_ids = {task_id}; + // Bumped from 60s to 180s. Prefill for 32k+ token prompts exceeds 120s + // (we measured 32s for 22k tokens; 48k ≈ 70s, 100k ≈ 150s+). Long autoregressive + // decode on P100 (28 tok/s) for 4k+ token outputs also exceeds 120s. The C++ + // side was timing out and returning HYDRA_STATUS_ERROR before the C# client + // gave up, surfacing as a 503 from the coordinator even though the model was + // still working. + auto res_ptr = ctx.queue_results->recv_with_timeout(task_ids, 180); + ctx.queue_results->remove_waiting_task_id(task_id); + if (!res_ptr) { + hydra_write_res(fd, HYDRA_STATUS_ERROR, 0, 0); + return; + } + + auto * res = dynamic_cast(res_ptr.get()); + if (!res || res->rpc_status != HYDRA_STATUS_OK) { + hydra_write_res(fd, HYDRA_STATUS_ERROR, 0, 0); + return; + } + + // Return n_past + sizes + model identity in meta; full blob (v2 header + KV + logits) as payload. + // logits_size > 0 signals the decode GPU to inject them into ctx->logits via STATE_PUT. + // M-Perf.9 #289: model identity fields (already populated on res by the PREFILL handler) + // are included so the Coordinator can record which model built the KV. + json meta_j = { + {"n_past", res->n_past}, + {"state_size", res->state_size}, + {"logits_size", res->logits_size} + }; + if (!res->model_alias.empty()) meta_j["model_alias"] = res->model_alias; + if (!res->model_path.empty()) meta_j["model_path"] = res->model_path; + if (!res->tokenizer.empty()) meta_j["tokenizer"] = res->tokenizer; + if (!res->model_name.empty()) meta_j["model_name"] = res->model_name; + if (!res->model_quant.empty()) meta_j["model_quant"] = res->model_quant; + if (res->model_capabilities) meta_j["model_capabilities"] = res->model_capabilities; + meta_j["model_fallback"] = res->model_fallback; + if (res->prefill_ms > 0) meta_j["prefill_ms"] = res->prefill_ms; + if (res->model_load_ms > 0) meta_j["model_load_ms"] = res->model_load_ms; + const std::string meta_str = meta_j.dump(); + const uint64_t total_payload = (uint64_t)res->state_data.size(); + hydra_write_res(fd, HYDRA_STATUS_OK, (uint32_t)meta_str.size(), total_payload); + hydra_send_all(fd, meta_str.data(), meta_str.size()); + if (total_payload > 0) { + hydra_send_all(fd, res->state_data.data(), (size_t)total_payload); + } + SRV_INF("hydra: PREFILL slot=%d sent n_past=%d kv=%" PRIu64 "B logits=%" PRIu64 "B total=%" PRIu64 "B\n", + slot_id, res->n_past, res->state_size, res->logits_size, total_payload); +} + +// DECODE (0x43) — Merged P/D: framed request with async HTTP retrieval. +// Wire format v3 (segmented): +// [4B hdr_len LE] <= 32768 +// [8B hdr_hash LE] xxh3-64 of the hdr JSON bytes that follow +// [hdr_len bytes] control header JSON +// [prompt_len bytes] prompt JSON segment (may be zero-length) +// [kv_len bytes] raw KV blob (may be zero-length) +// +// Control header: +// { "v": 3, "model": "...", "kv_metadata": {...}, "model_metadata": {...}, +// "generation": {...}, "segments": [...] } +// +// Two-phase flow: +// Phase 1 (sync): identity validation + KV restore — waits for inference thread +// Phase 2 (async): background thread posts SERVER_TASK_TYPE_COMPLETION, +// update_slots() drives generation, result stored in decode_results buffer. +// Actual result retrieved via GET /v1/decode/{decode_request_id}. +static void hydra_handle_decode(int fd, int slot_id, uint64_t payload_len, const hydra_rpc_ctx & ctx) { + // ── Read frame header: [4B hdr_len][8B hdr_hash] ────────────────────── + if (payload_len < sizeof(uint32_t) + sizeof(uint64_t)) { + SRV_WRN("%s", "hydra rpc: DECODE payload too small for frame header\n"); + hydra_write_res(fd, HYDRA_STATUS_BAD_REQUEST, 0, 0); + return; + } + + uint32_t hdr_len = 0; + if (!hydra_recv_all(fd, &hdr_len, sizeof(hdr_len))) { + hydra_write_res(fd, HYDRA_STATUS_ERROR, 0, 0); + return; + } + + if (hdr_len > HYDRA_MAX_JSON_HEADER) { + SRV_WRN("hydra rpc: DECODE hdr_len %u B exceeds cap %u B\n", + hdr_len, HYDRA_MAX_JSON_HEADER); + hydra_write_res(fd, HYDRA_STATUS_BAD_REQUEST, 0, 0); + return; + } + + uint64_t hdr_hash = 0; + if (!hydra_recv_all(fd, &hdr_hash, sizeof(hdr_hash))) { + hydra_write_res(fd, HYDRA_STATUS_ERROR, 0, 0); + return; + } + + // ── Read control header JSON ────────────────────────────────────────── + std::string hdr_json_str(hdr_len, '\0'); + if (hdr_len > 0 && !hydra_recv_all(fd, hdr_json_str.data(), hdr_len)) { + hydra_write_res(fd, HYDRA_STATUS_ERROR, 0, 0); + return; + } + + // Verify hdr_hash (xxh3-64 of the JSON bytes) + { + const uint64_t computed = XXH3_64bits(hdr_json_str.data(), hdr_json_str.size()); + if (computed != hdr_hash) { + SRV_WRN("hydra rpc: DECODE HDR_HASH_MISMATCH expected=%016" PRIx64 " got=%016" PRIx64 "\n", + hdr_hash, computed); + hydra_write_res(fd, HYDRA_STATUS_BAD_REQUEST, 0, 0); + return; + } + } + + // Parse control header + json req; + try { + req = json::parse(hdr_json_str); + } catch (const std::exception & e) { + SRV_WRN("hydra rpc: DECODE invalid JSON in control header: %s\n", e.what()); + hydra_write_res(fd, HYDRA_STATUS_BAD_REQUEST, 0, 0); + return; + } + + // Validate version + const int hdr_version = req.value("v", 0); + if (hdr_version < 3) { + SRV_WRN("hydra rpc: DECODE unsupported version %d (need >= 3)\n", hdr_version); + hydra_write_res(fd, HYDRA_STATUS_BAD_REQUEST, 0, 0); + return; + } + + // Validate required fields + if (!req.contains("kv_metadata")) { + SRV_WRN("%s", "hydra rpc: DECODE missing kv_metadata in control header\n"); + hydra_write_res(fd, HYDRA_STATUS_BAD_REQUEST, 0, 0); + return; + } + if (!req.contains("segments") || !req["segments"].is_array()) { + SRV_WRN("%s", "hydra rpc: DECODE missing or invalid segments array\n"); + hydra_write_res(fd, HYDRA_STATUS_BAD_REQUEST, 0, 0); + return; + } + + // ── Parse and validate segment table ────────────────────────────────── + const json & segments = req["segments"]; + const size_t n_segments = segments.size(); + if (n_segments > 3) { + SRV_WRN("hydra rpc: DECODE SEGMENT_TABLE_INVALID: too many segments (%zu)\n", n_segments); + hydra_write_res(fd, HYDRA_STATUS_BAD_REQUEST, 0, 0); + return; + } + + // Each segment: {"id":"prompt"|"kv", "offset":N, "len":N, "hash":"xxh3:HEX"} + uint64_t prompt_len = 0; + uint64_t kv_len = 0; + std::string prompt_hash_str; + std::string kv_hash_str; + uint64_t expected_offset = 0; + for (size_t i = 0; i < n_segments; i++) { + const json & seg = segments[i]; + if (!seg.contains("id") || !seg.contains("offset") || !seg.contains("len") || !seg.contains("hash")) { + SRV_WRN("hydra rpc: DECODE SEGMENT_TABLE_INVALID: segment %zu missing required fields\n", i); + hydra_write_res(fd, HYDRA_STATUS_BAD_REQUEST, 0, 0); + return; + } + const std::string id = seg["id"].get(); + const uint64_t offset = seg["offset"].get(); + const uint64_t len = seg["len"].get(); + const std::string hash = seg["hash"].get(); + + if (offset != expected_offset) { + SRV_WRN("hydra rpc: DECODE SEGMENT_TABLE_INVALID: segment %zu offset=%" PRIu64 " expected=%" PRIu64 "\n", + i, offset, expected_offset); + hydra_write_res(fd, HYDRA_STATUS_BAD_REQUEST, 0, 0); + return; + } + expected_offset = offset + len; + + if (id == "prompt") { + prompt_len = len; + prompt_hash_str = hash; + } else if (id == "kv") { + kv_len = len; + kv_hash_str = hash; + } else { + SRV_WRN("hydra rpc: DECODE SEGMENT_TABLE_INVALID: unknown segment id '%s'\n", id.c_str()); + hydra_write_res(fd, HYDRA_STATUS_BAD_REQUEST, 0, 0); + return; + } + } + + // Verify total segment size matches remaining payload + const uint64_t segments_total = prompt_len + kv_len; + const uint64_t remaining_after_hdr = payload_len - sizeof(uint32_t) - sizeof(uint64_t) - hdr_len; + if (segments_total != remaining_after_hdr) { + SRV_WRN("hydra rpc: DECODE SEGMENT_TABLE_INVALID: segments total %" PRIu64 " != remaining %" PRIu64 "\n", + segments_total, remaining_after_hdr); + hydra_write_res(fd, HYDRA_STATUS_BAD_REQUEST, 0, 0); + return; + } + + // Caps + if (prompt_len > HYDRA_MAX_PROMPT_BYTES) { + SRV_WRN("hydra rpc: DECODE PROMPT_TOO_LARGE %" PRIu64 " > %" PRIu64 "\n", + prompt_len, HYDRA_MAX_PROMPT_BYTES); + hydra_write_res(fd, HYDRA_STATUS_BAD_REQUEST, 0, 0); + return; + } + if (kv_len > HYDRA_MAX_STATE_BYTES) { + SRV_WRN("hydra rpc: DECODE KV_TOO_LARGE %" PRIu64 " > %" PRIu64 "\n", + kv_len, HYDRA_MAX_STATE_BYTES); + hydra_write_res(fd, HYDRA_STATUS_BAD_REQUEST, 0, 0); + return; + } + + // ── Read prompt segment ─────────────────────────────────────────────── + std::vector prompt_data((size_t)prompt_len); + if (prompt_len > 0 && !hydra_recv_all(fd, prompt_data.data(), (size_t)prompt_len)) { + hydra_write_res(fd, HYDRA_STATUS_ERROR, 0, 0); + return; + } + + // ── Read KV segment (may be zero-length) ────────────────────────────── + std::vector kv_data((size_t)kv_len); + if (kv_len > 0 && !hydra_recv_all(fd, kv_data.data(), (size_t)kv_len)) { + hydra_write_res(fd, HYDRA_STATUS_ERROR, 0, 0); + return; + } + + // Verify KV segment hash BEFORE passing to llama_state_seq_set_data + if (kv_len > 0 && !kv_hash_str.empty()) { + // Parse "xxh3:HEX" format + if (kv_hash_str.rfind("xxh3:", 0) == 0) { + const std::string hex_str = kv_hash_str.substr(5); + uint64_t expected_kv_hash = 0; + try { + expected_kv_hash = std::stoull(hex_str, nullptr, 16); + } catch (const std::exception &) { + SRV_WRN("hydra rpc: DECODE invalid KV hash format: %s\n", kv_hash_str.c_str()); + hydra_write_res(fd, HYDRA_STATUS_BAD_REQUEST, 0, 0); + return; + } + const uint64_t computed_kv = XXH3_64bits(kv_data.data(), kv_data.size()); + if (computed_kv != expected_kv_hash) { + SRV_WRN("hydra rpc: DECODE SEGMENT_HASH_MISMATCH kv expected=%016" PRIx64 " got=%016" PRIx64 "\n", + expected_kv_hash, computed_kv); + hydra_write_res(fd, HYDRA_STATUS_BAD_REQUEST, 0, 0); + return; + } + SRV_INF("hydra rpc: DECODE KV hash verified (%" PRIu64 " B)\n", kv_len); + } else { + SRV_WRN("hydra rpc: DECODE unsupported KV hash prefix: %s\n", kv_hash_str.c_str()); + hydra_write_res(fd, HYDRA_STATUS_BAD_REQUEST, 0, 0); + return; + } + } + + // ── Build decode_json from control header + prompt segment ───────────── + // The prompt JSON segment may contain { "prompt": "..." } or { "messages": [...] } + // Merge it into the control header as decode_req["prompt"]. + // Also merge generation params from control header's "generation" key. + json decode_req = req; // control header already has kv_metadata, model, etc. + json prompt_obj; + if (prompt_len > 0) { + try { + prompt_obj = json::parse(std::string(prompt_data.begin(), prompt_data.end())); + } catch (const std::exception & e) { + SRV_WRN("hydra rpc: DECODE invalid prompt segment JSON: %s\n", e.what()); + hydra_write_res(fd, HYDRA_STATUS_BAD_REQUEST, 0, 0); + return; + } + } + // The coordinator sends the prompt segment as the BARE messages array + // (item.Request["messages"].ToString()). The generation-merge below and + // DECODE_APPLY's chat-template path both expect an OBJECT with a + // "messages" key — merging generation keys into an array throws + // nlohmann::type_error, which was silently swallowed by the RPC worker + // pool (the connection leaked, no response written, coordinator timed out + // after 180s). Wrap a bare array so the prompt object matches the + // downstream contract. + if (prompt_obj.is_array()) { + json wrapped; + wrapped["messages"] = std::move(prompt_obj); + prompt_obj = std::move(wrapped); + } + // Merge generation params from control header into prompt object + std::string decode_json_str; + try { + if (req.contains("generation") && req["generation"].is_object()) { + const json & gen = req["generation"]; + for (auto it = gen.begin(); it != gen.end(); ++it) { + if (!prompt_obj.contains(it.key())) { + prompt_obj[it.key()] = it.value(); + } + } + } + decode_req["prompt"] = std::move(prompt_obj); + + decode_json_str = decode_req.dump(); + } catch (const std::exception & e) { + // Never let a malformed prompt object leak the connection: the worker + // pool swallows exceptions and the fd stays open with no response, + // hanging the coordinator until its own timeout. Always write an + // error frame so the caller sees a terminal (retryable-free) result. + SRV_WRN("hydra rpc: DECODE prompt build failed (slot %d): %s\n", slot_id, e.what()); + json err_j = { + {"error", std::string("prompt build failed: ") + e.what()}, + {"decode_request_id", -1}, + }; + const std::string err_str = err_j.dump(); + hydra_write_res(fd, HYDRA_STATUS_BAD_REQUEST, (uint32_t) err_str.size(), 0); + hydra_send_all(fd, err_str.data(), err_str.size()); + return; + } + + // ── Phase 1: sync validate + restore ────────────────────────────────── + const int32_t decode_request_id = ctx.queue_tasks->get_new_id(); + + server_task val_task(SERVER_TASK_TYPE_HYDRA_ENGINE_DECODE); + val_task.id = decode_request_id; + val_task.hydra_action.id_slot = slot_id; + val_task.hydra_action.decode_json = std::move(decode_json_str); + val_task.hydra_action.kv_data = std::move(kv_data); + val_task.hydra_action.decode_request_id = decode_request_id; + ctx.queue_results->add_waiting_task_id(decode_request_id); + ctx.queue_tasks->wait_until_no_sleep(); + ctx.queue_tasks->post(std::move(val_task)); + + // Wait for validation+restore to complete (30s timeout for large KV blobs) + std::unordered_set val_ids = {decode_request_id}; + auto val_res_ptr = ctx.queue_results->recv_with_timeout(val_ids, 30); + ctx.queue_results->remove_waiting_task_id(decode_request_id); + + if (!val_res_ptr) { + SRV_WRN("hydra rpc: DECODE validation timeout for slot %d (request_id=%d)\n", + slot_id, decode_request_id); + json err_j = { + {"error", "validation timeout"}, + {"decode_request_id", decode_request_id}, + }; + const std::string err_str = err_j.dump(); + hydra_write_res(fd, HYDRA_STATUS_ERROR, (uint32_t)err_str.size(), 0); + hydra_send_all(fd, err_str.data(), err_str.size()); + return; + } + + auto * val_res = dynamic_cast(val_res_ptr.get()); + if (!val_res || val_res->rpc_status != HYDRA_STATUS_OK) { + json err_j = { + {"valid", false}, + {"decode_request_id", decode_request_id}, + }; + if (val_res) { + if (!val_res->match_json.is_null()) err_j["match"] = val_res->match_json; + if (!val_res->error.empty()) err_j["reason"] = val_res->error; + err_j["error_code"] = "CAP_MISMATCH"; + } + const std::string err_str = err_j.dump(); + hydra_write_res(fd, HYDRA_STATUS_ERROR, (uint32_t)err_str.size(), 0); + hydra_send_all(fd, err_str.data(), err_str.size()); + return; + } + + // Validation passed — build real success response + json meta_j = { + {"valid", true}, + {"match", val_res->match_json}, + {"decode_request_id", decode_request_id}, + {"n_past_after_restore", val_res->n_past}, + {"restore_slot_ms", val_res->restore_slot_ms}, + }; + const std::string meta_str = meta_j.dump(); + hydra_write_res(fd, HYDRA_STATUS_OK, (uint32_t)meta_str.size(), 0); + hydra_send_all(fd, meta_str.data(), meta_str.size()); + + SRV_INF("hydra: DECODE slot=%d accepted, request_id=%d, restore=%.1fms\n", + slot_id, decode_request_id, val_res->restore_slot_ms); +} + +// SET_EXPERT_MODE (0x37): Read mode string, post task, return success. +static void hydra_handle_set_expert_mode(int fd, int slot_id, uint64_t payload_len, const hydra_rpc_ctx & ctx) { + std::string mode(payload_len, '\0'); + if (payload_len > 0 && !hydra_recv_all(fd, mode.data(), payload_len)) { + hydra_write_res(fd, HYDRA_STATUS_ERROR, 0, 0); + return; + } + + server_task task(SERVER_TASK_TYPE_HYDRA_ENGINE_SET_EXPERT_MODE); + task.id = ctx.queue_tasks->get_new_id(); + task.hydra_action.id_slot = slot_id; + task.hydra_action.expert_mode = std::move(mode); + const int task_id = task.id; + ctx.queue_results->add_waiting_task_id(task_id); + ctx.queue_tasks->wait_until_no_sleep(); + ctx.queue_tasks->post(std::move(task)); + + std::unordered_set task_ids = {task_id}; + auto res_ptr = ctx.queue_results->recv_with_timeout(task_ids, 5); + ctx.queue_results->remove_waiting_task_id(task_id); + if (!res_ptr) { + hydra_write_res(fd, HYDRA_STATUS_ERROR, 0, 0); + return; + } + + auto * res = dynamic_cast(res_ptr.get()); + if (!res || !res->success) { + const std::string err = (res && !res->error.empty()) ? res->error : std::string(); + json err_j = {{"success", false}}; + if (!err.empty()) err_j["error"] = err; + const std::string err_str = err_j.dump(); + hydra_write_res(fd, HYDRA_STATUS_ERROR, (uint32_t)err_str.size(), 0); + hydra_send_all(fd, err_str.data(), err_str.size()); + return; + } + + // Report the ACTUAL mode applied (may be "solo" even though "combined" was + // requested, if this engine never dual-loaded combined experts) — the + // Coordinator's ReportsSolo() reads this key to detect the fallback. + json meta_j = {{"success", true}, {"mode", res->expert_mode_applied}}; + const std::string meta_str = meta_j.dump(); + hydra_write_res(fd, HYDRA_STATUS_OK, (uint32_t)meta_str.size(), 0); + hydra_send_all(fd, meta_str.data(), meta_str.size()); +} + +// SWAP_QUANT (0x38): Read quant_key + tensor_pattern, post task, return success. +static void hydra_handle_swap_quant(int fd, int slot_id, uint64_t payload_len, const hydra_rpc_ctx & ctx) { + if (payload_len < sizeof(uint16_t)) { + hydra_write_res(fd, HYDRA_STATUS_ERROR, 0, 0); + return; + } + + uint16_t quant_key_len = 0; + if (!hydra_recv_all(fd, &quant_key_len, sizeof(quant_key_len))) { + hydra_write_res(fd, HYDRA_STATUS_ERROR, 0, 0); + return; + } + + std::string quant_key(quant_key_len, '\0'); + if (quant_key_len > 0 && !hydra_recv_all(fd, quant_key.data(), quant_key_len)) { + hydra_write_res(fd, HYDRA_STATUS_ERROR, 0, 0); + return; + } + + const uint64_t pattern_len = payload_len - sizeof(uint16_t) - quant_key_len; + std::string tensor_pattern(pattern_len, '\0'); + if (pattern_len > 0 && !hydra_recv_all(fd, tensor_pattern.data(), pattern_len)) { + hydra_write_res(fd, HYDRA_STATUS_ERROR, 0, 0); + return; + } + + server_task task(SERVER_TASK_TYPE_HYDRA_ENGINE_SWAP_QUANT); + task.id = ctx.queue_tasks->get_new_id(); + task.hydra_action.id_slot = slot_id; + task.hydra_action.quant_key = std::move(quant_key); + task.hydra_action.tensor_pattern = std::move(tensor_pattern); + const int task_id = task.id; + ctx.queue_results->add_waiting_task_id(task_id); + ctx.queue_tasks->wait_until_no_sleep(); + ctx.queue_tasks->post(std::move(task)); + + std::unordered_set task_ids = {task_id}; + auto res_ptr = ctx.queue_results->recv_with_timeout(task_ids, 30); + ctx.queue_results->remove_waiting_task_id(task_id); + if (!res_ptr) { + hydra_write_res(fd, HYDRA_STATUS_ERROR, 0, 0); + return; + } + + auto * res = dynamic_cast(res_ptr.get()); + if (!res || !res->success) { + hydra_write_res(fd, HYDRA_STATUS_ERROR, 0, 0); + return; + } + + json meta_j = {{"success", true}}; + const std::string meta_str = meta_j.dump(); + hydra_write_res(fd, HYDRA_STATUS_OK, (uint32_t)meta_str.size(), 0); + hydra_send_all(fd, meta_str.data(), meta_str.size()); +} + +// PIPELINE_ATTACH (0x46): M-Perf.9 (#289) / issue #287 — two-engine "work +// together" routing scaffolding. The C# Coordinator sends the peer address +// and the --override-tensor regex; the engine should load the assigned +// tensor slice from its OWN local model (no weight transfer). This opcode +// is stubbed for now (returns NOT_IMPLEMENTED) — full implementation is +// tracked under issue #287. +static void hydra_handle_pipeline_attach(int fd, int slot_id, uint64_t payload_len, const hydra_rpc_ctx & ctx) { + std::string json_body(payload_len, '\0'); + if (payload_len > 0 && !hydra_recv_all(fd, json_body.data(), payload_len)) { + hydra_write_res(fd, HYDRA_STATUS_ERROR, 0, 0); + return; + } + + server_task task(SERVER_TASK_TYPE_HYDRA_ENGINE_PIPELINE_ATTACH); + task.id = ctx.queue_tasks->get_new_id(); + task.hydra_action.id_slot = slot_id; + task.hydra_action.request_json = std::move(json_body); + const int task_id = task.id; + ctx.queue_results->add_waiting_task_id(task_id); + ctx.queue_tasks->wait_until_no_sleep(); + ctx.queue_tasks->post(std::move(task)); + + std::unordered_set task_ids = {task_id}; + auto res_ptr = ctx.queue_results->recv_with_timeout(task_ids, 5); + ctx.queue_results->remove_waiting_task_id(task_id); + if (!res_ptr) { + hydra_write_res(fd, HYDRA_STATUS_ERROR, 0, 0); + return; + } + + auto * res = dynamic_cast(res_ptr.get()); + // Stubbed: server returns NOT_IMPLEMENTED until issue #287 lands. + // Propagate that status to the client so the Coordinator can + // distinguish "not yet built" from a real error and fall back to solo. + const uint8_t status = (res && res->rpc_status == HYDRA_STATUS_NOT_IMPLEMENTED) + ? HYDRA_STATUS_NOT_IMPLEMENTED : HYDRA_STATUS_ERROR; + json meta_j; + if (res && !res->error.empty()) meta_j["error"] = res->error; + meta_j["success"] = res && res->success; + const std::string meta_str = meta_j.dump(); + hydra_write_res(fd, status, (uint32_t)meta_str.size(), 0); + hydra_send_all(fd, meta_str.data(), meta_str.size()); +} + +// ── Per-connection loop ─────────────────────────────────────────────────────── +// Persistent: one TCP connection handles many sequential requests. + +static void hydra_handle_connection(int fd, const hydra_rpc_ctx & ctx) { + // Set receive timeout to prevent hung connections on stalled clients + struct timeval tv; + tv.tv_sec = 120; // 2 min inactivity timeout + tv.tv_usec = 0; + (void)setsockopt(fd, SOL_SOCKET, SO_RCVTIMEO, &tv, sizeof(tv)); + while (true) { + uint8_t hdr[HYDRA_REQ_HEADER_SIZE]; + if (!hydra_recv_all(fd, hdr, HYDRA_REQ_HEADER_SIZE)) break; + + uint16_t magic = 0; + memcpy(&magic, hdr + 0, 2); + if (magic != HYDRA_MAGIC) { + SRV_WRN("hydra rpc: bad magic 0x%04x — closing connection\n", (unsigned)magic); + break; + } + + const uint8_t op = hdr[2]; + // hdr[3] = flags (reserved, unused in M1) + uint16_t key_len = 0, trace_len = 0; + uint64_t payload_len = 0; + memcpy(&key_len, hdr + 4, 2); + memcpy(&payload_len, hdr + 6, 8); + memcpy(&trace_len, hdr + 14, 2); + + std::string key(key_len, '\0'); + std::string trace_id(trace_len, '\0'); + if (!hydra_recv_all(fd, key.data(), key_len)) break; + if (!hydra_recv_all(fd, trace_id.data(), trace_len)) break; + + // Slot-key parsing: engine-level opcodes (INFO, CONFIGURE, SET_EXPERT_MODE, + // SWAP_QUANT) don't need a valid slot — use slot_id = 0 when the key is + // empty or invalid. Slot-level opcodes (STATE_GET, STATE_PUT, STATE_META, + // PREFILL, DECODE) still require a valid integer key. + int slot_id = -1; + bool is_engine_level_op = (op == HYDRA_OP_INFO || op == HYDRA_OP_CONFIGURE || + op == HYDRA_OP_SET_EXPERT_MODE || op == HYDRA_OP_SWAP_QUANT); + if (key.empty() && is_engine_level_op) { + slot_id = 0; + } else { + try { slot_id = std::stoi(key); } + catch (...) { + SRV_WRN("hydra rpc: invalid slot key '%s'\n", key.c_str()); + hydra_write_res(fd, HYDRA_STATUS_ERROR, 0, 0); + continue; + } + } + + // Dispatch to handler via task queue (no direct slot access) + switch (op) { + case HYDRA_OP_STATE_GET: + SRV_DBG("hydra rpc: STATE_GET slot=%d trace=%s\n", slot_id, trace_id.c_str()); + hydra_handle_state_get(fd, slot_id, ctx); + break; + case HYDRA_OP_STATE_PUT: + SRV_DBG("hydra rpc: STATE_PUT slot=%d payload=%" PRIu64 " trace=%s\n", + slot_id, payload_len, trace_id.c_str()); + hydra_handle_state_put(fd, slot_id, payload_len, ctx); + break; + case HYDRA_OP_STATE_META: + SRV_DBG("hydra rpc: STATE_META slot=%d trace=%s\n", slot_id, trace_id.c_str()); + hydra_handle_state_meta(fd, slot_id, ctx); + break; + case HYDRA_OP_CONFIGURE: + SRV_DBG("hydra rpc: CONFIGURE slot=%d payload=%" PRIu64 " trace=%s\n", + slot_id, payload_len, trace_id.c_str()); + hydra_handle_configure(fd, slot_id, payload_len, ctx); + break; + case HYDRA_OP_INFO: + SRV_DBG("hydra rpc: INFO slot=%d trace=%s\n", slot_id, trace_id.c_str()); + hydra_handle_info(fd, slot_id, ctx); + break; + case HYDRA_OP_PREFILL: + SRV_DBG("hydra rpc: PREFILL slot=%d payload=%" PRIu64 " trace=%s\n", + slot_id, payload_len, trace_id.c_str()); + hydra_handle_prefill(fd, slot_id, payload_len, ctx); + break; + case HYDRA_OP_DECODE: + SRV_DBG("hydra rpc: DECODE slot=%d payload=%" PRIu64 " trace=%s\n", + slot_id, payload_len, trace_id.c_str()); + hydra_handle_decode(fd, slot_id, payload_len, ctx); + break; + case HYDRA_OP_SET_EXPERT_MODE: + SRV_DBG("hydra rpc: SET_EXPERT_MODE slot=%d payload=%" PRIu64 " trace=%s\n", + slot_id, payload_len, trace_id.c_str()); + hydra_handle_set_expert_mode(fd, slot_id, payload_len, ctx); + break; + case HYDRA_OP_SWAP_QUANT: + SRV_DBG("hydra rpc: SWAP_QUANT slot=%d payload=%" PRIu64 " trace=%s\n", + slot_id, payload_len, trace_id.c_str()); + hydra_handle_swap_quant(fd, slot_id, payload_len, ctx); + break; + // M-Perf.9 (#289) / issue #287: PIPELINE_ATTACH (0x46) is the + // two-engine "work together" attach. Stubbed: full impl in #287. + case HYDRA_OP_PIPELINE_ATTACH: + SRV_DBG("hydra rpc: PIPELINE_ATTACH slot=%d payload=%" PRIu64 " trace=%s\n", + slot_id, payload_len, trace_id.c_str()); + hydra_handle_pipeline_attach(fd, slot_id, payload_len, ctx); + break; + default: + SRV_WRN("hydra rpc: unknown op 0x%02x — ignoring\n", (unsigned)op); + hydra_write_res(fd, HYDRA_STATUS_ERROR, 0, 0); + } + } + ::close(fd); +} + +// ── Unified RPC server implementation ─────────────────────────────────────── +// +// `#36` Phase 1: the merged server lives in `tools/llama-engine/hydra_rpc/` +// (fork-isolated). `server_context::start_rpc_server` is a thin adapter that +// builds the settings and delegates to `hydra_rpc::start()`. The Hydra +// protocol entry `hydra_handle_connection` is reached through the +// `hydra_rpc_bridge` trampoline (defined below) — the bridge takes a +// `void*` so the new module can stay decoupled from this file's includes. + +#include "../llama-engine/hydra_rpc/hydra_rpc.h" + +void server_context::start_rpc_server(int port, + std::vector backends) { + if (port <= 0) return; + + // Hydra #43: MUST outlive this function. `hydra_rpc::start()` below + // stores `&ctx` as a raw pointer inside `hydra_rpc::state()`, a + // process-lifetime singleton that every subsequent RPC connection reads + // (from a bounded-thread-pool worker thread) to recover queue_tasks / + // queue_results. An automatic-storage `ctx` here would dangle the + // instant this function returns — a stack-use-after-return that "works" + // until the freed stack slot gets reused, then silently corrupts the + // RPC response path. `start_rpc_server` only ever runs once per process + // (hydra_rpc::start() itself guards double-start), so `static` gives it + // exactly the lifetime the singleton needs. + static hydra_rpc_ctx ctx{}; + if (impl) { + ctx.queue_tasks = &impl->queue_tasks; + ctx.queue_results = &impl->queue_results; + } + + hydra_rpc::settings s; + s.port = port; + s.backends = std::move(backends); + s.hydra_ctx = (ctx.queue_tasks && ctx.queue_results) ? &ctx : nullptr; + s.pool_size = 2; + s.max_queue = 64; + s.host = "0.0.0.0"; + + if (!hydra_rpc::start(s)) { + SRV_ERR("hydra rpc: start() failed on port %d\n", port); + return; + } + + if (s.hydra_ctx) { + SRV_INF("hydra rpc: unified server on 0.0.0.0:%d (ggml-RPC + Hydra protocol)\n", port); + } else { + SRV_INF("hydra rpc: unified server on 0.0.0.0:%d (ggml-RPC only)\n", port); + } +} + +// `hydra_rpc_bridge` — extern "C" trampoline. `hydra_rpc.cpp` calls this +// when the first byte on a new connection is not `RPC_CMD_HELLO`. It +// re-enters the C++ entry point with the typed `hydra_rpc_ctx &`. +// +// Forward-declared with the matching signature so the new +// `tools/llama-engine/hydra_rpc/hydra_rpc.cpp` module can take its +// address without including this heavy header. +extern "C" void hydra_rpc_bridge(int fd, const void * ctx); +extern "C" void hydra_rpc_bridge(int fd, const void * ctx) { + hydra_handle_connection(fd, *static_cast(ctx)); +} + +#else +// Windows: RPC server not implemented — target hardware is Linux-only for M0. +void server_context::start_rpc_server(int port, std::vector) { + if (port > 0) { + SRV_WRN("hydra rpc: not supported on Windows (port %d ignored)\n", port); + } + GGML_UNUSED(port); +} +#endif // !_WIN32 + +// --- WS3.5 moved helper methods --- + + bool server_context_impl::apply_t3_rebuild(bool force) { + bool is_first_load = !ctx_tgt; + + // Track the last override_tensor string that was actually + // applied so we can detect "nothing changed" on subsequent + // calls and skip the expensive unload+reload cycle. + static std::string old_override_applied; + + common_params old_params = params_base; + common_params swapped_params = params_base; + + // Read the staged T3 statics and apply them to swapped_params. + if (llama_hydra_get_pending_n_gpu_layers() >= 0) { + swapped_params.n_gpu_layers = llama_hydra_get_pending_n_gpu_layers(); + } + // n_cpu_moe is informational only — the actual MoE expert + // offload is done via override_tensor (parsed below into + // tensor_buft_overrides). The standard common_params struct + // has no n_cpu_moe field; we just log the staged value for + // operator visibility. + if (llama_hydra_get_pending_n_cpu_moe() >= 0) { + SRV_INF("hydra: T3 rebuild: staged n_cpu_moe=%d (informational; expert routing via override_tensor)\n", + llama_hydra_get_pending_n_cpu_moe()); + } + const char * path = llama_hydra_get_pending_model_path(); + if (path && *path) { + swapped_params.model.path = path; + } + const char * mode = llama_hydra_get_pending_split_mode(); + if (mode && *mode) { + std::string m(mode); + if (m == "none") swapped_params.split_mode = LLAMA_SPLIT_MODE_NONE; + else if (m == "layer") swapped_params.split_mode = LLAMA_SPLIT_MODE_LAYER; + else if (m == "row") swapped_params.split_mode = LLAMA_SPLIT_MODE_ROW; + else SRV_WRN("hydra: T3 split_mode='%s' unknown; keeping current\n", m.c_str()); + } + const size_t n_split = llama_hydra_get_pending_tensor_split_count(); + if (n_split > 0) { + const float * split = llama_hydra_get_pending_tensor_split(); + // common_params::tensor_split is a fixed-size array. + const size_t cap = sizeof(swapped_params.tensor_split) / + sizeof(swapped_params.tensor_split[0]); + const size_t n = n_split < cap ? n_split : cap; + for (size_t i = 0; i < n; i++) { + swapped_params.tensor_split[i] = split[i]; + } + // Zero the rest so the engine doesn't see stale values. + for (size_t i = n; i < cap; i++) { + swapped_params.tensor_split[i] = 0.0f; + } + } + const char * override = llama_hydra_get_pending_override_tensor(); + if (override && *override) { + // Wire-shape: comma-separated "pattern=buft" pairs (e.g. + // "blk.*.ffn_*_exps.weight=CPU"). The C++ side stores + // these as a vector. + // Buft names are looked up + // via ggml_backend_dev_buffer_type() + ggml_backend_buft_name() + // (mirrors common/arg.cpp:parse_tensor_buffer_overrides). + ggml_backend_load_all(); + std::map buft_list; + for (size_t i = 0; i < ggml_backend_dev_count(); ++i) { + auto * dev = ggml_backend_dev_get(i); + auto * buft = ggml_backend_dev_buffer_type(dev); + if (buft) { + buft_list[std::string(ggml_backend_buft_name(buft))] = buft; + } + } + // CPU is the common case (MoE expert routing) — also lookup + // explicitly since some backends may not register the CPU buft. + buft_list["CPU"] = ggml_backend_cpu_buffer_type(); + + // Keep pattern strings alive for the lifetime of the + // process — entry.pattern is a const char* that must not + // dangle. Matches the safe pattern in common/arg.cpp. + static std::list buft_override_patterns; + + std::vector staged; + + const std::string ovr(override); + size_t start = 0; + while (start < ovr.size()) { + size_t comma = ovr.find(',', start); + std::string part = ovr.substr(start, comma == std::string::npos ? std::string::npos : comma - start); + size_t eq = part.find('='); + if (eq != std::string::npos) { + std::string pattern = part.substr(0, eq); + std::string buft_name = part.substr(eq + 1); + auto it = buft_list.find(buft_name); + if (it != buft_list.end()) { + buft_override_patterns.push_back(pattern); + llama_model_tensor_buft_override entry; + entry.pattern = buft_override_patterns.back().c_str(); + entry.buft = it->second; + staged.push_back(entry); + } else { + SRV_WRN("%s", "hydra: T3 rebuild: override_tensor buft name not in registered list; skipping pattern\n"); + } + } + if (comma == std::string::npos) break; + start = comma + 1; + } + + // Install the staged patterns *in place of* the base ones instead of + // appending to them. + // + // common_params_parse_ex() (common/arg.cpp) unconditionally pads this + // vector out to llama_max_tensor_buft_overrides() entries of + // {nullptr, nullptr}, so by the time we get here the real CLI overrides + // sit at the head and the rest is terminator padding. push_back() would + // land *behind* that padding, which breaks twice over: + // 1. common_model_params_to_llama() asserts that back().pattern is + // nullptr, so the engine aborts before the model loads; + // 2. even without that assert, llama_model_loader stops scanning at + // the first nullptr pattern, so appended entries are never read — + // the override would be silently dropped and the MoE experts would + // land on the GPU. + // Replacing also matches the sibling fields handled above: model.path, + // split_mode, n_gpu_layers and tensor_split are all overwritten by the + // staged T3 config rather than merged into it. + const size_t ntbo = llama_max_tensor_buft_overrides(); + if (staged.empty()) { + // Nothing resolved (every buft name was unknown). Wiping the base + // overrides here would silently change how the model is placed, so + // keep them and make the no-op explicit. + SRV_WRN("%s", "hydra: T3 rebuild: staged override_tensor resolved to no usable patterns; keeping base overrides\n"); + } else { + if (staged.size() + 1 > ntbo) { + SRV_WRN("hydra: T3 rebuild: %zu override_tensor patterns exceed the %zu-entry limit; keeping the first %zu\n", + staged.size(), ntbo, ntbo - 1); + staged.resize(ntbo - 1); + } + // assign() re-establishes the full terminator padding, so everything + // from staged.size() onward is {nullptr, nullptr}. + swapped_params.tensor_buft_overrides.assign(ntbo, llama_model_tensor_buft_override{ nullptr, nullptr }); + for (size_t i = 0; i < staged.size(); ++i) { + swapped_params.tensor_buft_overrides[i] = staged[i]; + } + } + } + + // Early-exit: if the model and all T3-relevant params are + // identical to what is already loaded, skip the expensive + // unload+reload cycle. Without this, every COMPLETION + // request that carries hydra_config triggers a full model + // swap even when nothing changed (the coordinator sends the + // same config on every decode request). + if (!is_first_load) { + const char * cur_override = llama_hydra_get_pending_override_tensor(); + bool params_unchanged = + swapped_params.model.path == old_params.model.path && + swapped_params.n_gpu_layers == old_params.n_gpu_layers && + swapped_params.split_mode == old_params.split_mode && + ((cur_override == nullptr && old_override_applied.empty()) || + (cur_override && old_override_applied == cur_override)); + if (params_unchanged) { + // T3 overrides (override_tensor, split_mode) were staged by + // the COMPLETION hydra_config path. But the model reload is + // being skipped. Clear the staged override so the next decode + // uses the current tensor placement (not the staged override). + llama_hydra_set_override_tensor(ctx_tgt, nullptr); + SRV_INF("%s", "hydra: T3 rebuild: model and params unchanged — skipping reload, cleared staged overrides\n"); + return true; + } + } + + // COMBINED-mode teardown BEFORE the model reload — see + // hydra_teardown_combined_before_reload() above. + const bool was_combined = hydra_combined_head_attached || hydra_combined_static; + if (!is_first_load && was_combined) { + hydra_teardown_combined_before_reload(); + } + + // Register any new RPC peer devices before load_model() so the + // peer's device exists in the global ggml backend registry when + // common_init_from_params() tries to place tensors per + // tensor_split/split_mode. Only genuinely new endpoints are + // registered (hydra_register_rpc_servers tracks already-registered + // endpoints to avoid unsafe repeated registration). + if (!g_pending_rpc_servers.empty()) { + json rpc_arr = json::array(); + for (const auto & s : g_pending_rpc_servers) { + rpc_arr.push_back(s); + } + hydra_register_rpc_servers(rpc_arr); + g_pending_rpc_servers.clear(); + } + + // Full model reload. load_model() handles the unload of the + // current model, the load of the new model, the new context + // creation, the MTP/draft paths, and the slot rebuild. + // NOTE: load_model() does `params_base = params` internally + // (line 844), so after a successful load params_base reflects + // swapped_params — no explicit reassignment needed by us. + // + // #507: Skip the fit_params probe during T3 rebuild. The probe + // does a full model-structure load with no_alloc=true to measure + // GPU memory — expensive (~45-90s) and unnecessary here because: + // (a) we just freed VRAM by destroying the old model, (b) the new + // model's requirements are known (same or smaller), (c) a controlled + // inference server has predictable VRAM. Disabling saves ~1 min. + swapped_params.fit_params = false; + if (!load_model(swapped_params)) { + if (is_first_load) { + SRV_WRN("%s", "hydra: T3 first load failed — engine stays empty\n"); + return false; + } + SRV_ERR("hydra: T3 reload to '%s' failed (load_model returned false); " + "rolling back to old model\n", + swapped_params.model.path.c_str()); + if (!load_model(old_params)) { + SRV_ERR("%s", "hydra: T3 rollback also failed — engine in unrecoverable state\n"); + GGML_ABORT("hydra: T3 rollback failed (cannot reload old model). " + "Engine exiting to prevent serving with corrupted state."); + } + SRV_INF("hydra: T3 rollback succeeded — restored old model '%s'\n", + old_params.model.path.c_str()); + return false; + } + + // Record the override_tensor that was just applied so the + // next call can skip the reload if nothing changed. + { + const char * cur = llama_hydra_get_pending_override_tensor(); + old_override_applied = cur ? cur : ""; + } + + // T3 reload confirmed. Log model identity for traceability. + SRV_INF("hydra: T3 reload confirmed model_alias='%s' tokenizer='%s' model_name='%s' quant='%s' caps=0x%x model_path='%s'\n", + swapped_params.model_alias.empty() ? "?" : swapped_params.model_alias.begin()->c_str(), + model_tgt ? llama_model_get_tokenizer_model(model_tgt) : "", + model_tgt ? llama_model_get_display_name(model_tgt) : "", + model_tgt ? llama_model_get_quant_label(model_tgt) : "", + model_tgt ? llama_model_get_capabilities_bitfield(model_tgt) : 0, + swapped_params.model.path.c_str()); + + // COMBINED-mode reattach AFTER the model reload — see + // hydra_reattach_combined_after_reload() above. + if (was_combined) { + hydra_reattach_combined_after_reload(); + } + + SRV_INF("hydra: T3 rebuild applied (model='%s', split_mode=%d, n_gpu_layers=%d, slots=%zu)\n", + params_base.model.path.c_str(), (int) params_base.split_mode, + params_base.n_gpu_layers, slots.size()); + return true; + } + + void server_context_impl::hydra_repad_tensor_buft_overrides(common_params & p, const char * ctx_label) { + const size_t ntbo = llama_max_tensor_buft_overrides(); + if (p.tensor_buft_overrides.size() + 1 > ntbo) { + SRV_WRN("hydra: %s: %zu tensor_buft_overrides exceed the %zu-entry limit; keeping the first %zu\n", + ctx_label, p.tensor_buft_overrides.size(), ntbo, ntbo - 1); + p.tensor_buft_overrides.resize(ntbo - 1); + } + p.tensor_buft_overrides.resize(ntbo, llama_model_tensor_buft_override{ nullptr, nullptr }); + } + + void server_context_impl::hydra_reattach_combined_after_reload() { + SRV_INF("%s", "hydra: re-attaching COMBINED on new model\n"); + if (hydra_combined_static) { + llama_hydra_set_expert_mode(ctx_tgt, 1); + } else if (!hydra_peer.empty() && !hydra_combined_pattern.empty()) { + if (llama_hydra_peer_reachable(hydra_peer.c_str())) { + ggml_backend_reg_t rpc_reg = ggml_backend_reg_by_name("RPC"); + if (rpc_reg) { + using add_server_fn_t = ggml_backend_reg_t (*)(const char *); + auto add_server_fn = (add_server_fn_t) ggml_backend_reg_get_proc_address(rpc_reg, "ggml_backend_rpc_add_server"); + ggml_backend_reg_t peer_reg = add_server_fn ? add_server_fn(hydra_peer.c_str()) : nullptr; + ggml_backend_dev_t peer_dev = (peer_reg && ggml_backend_reg_dev_count(peer_reg) > 0) ? ggml_backend_reg_dev_get(peer_reg, 0) : nullptr; + if (peer_dev) { + int32_t n_bound = llama_hydra_rebind_combined_experts( + ctx_tgt, hydra_peer.c_str(), peer_dev, hydra_combined_pattern.c_str()); + if (n_bound > 0) { + hydra_combined_head_attached = true; + llama_hydra_set_expert_mode(ctx_tgt, 1); + SRV_INF("hydra: COMBINED re-attached on peer %s (%d layers bound)\n", + hydra_peer.c_str(), n_bound); + } else { + SRV_WRN("hydra: rebind returned %d; staying solo\n", n_bound); + } + } else { + SRV_WRN("hydra: peer %s has no device; staying solo\n", hydra_peer.c_str()); + } + } else { + SRV_WRN("%s\n", "hydra: RPC backend not available; staying solo"); + } + } else { + SRV_WRN("hydra: peer %s unreachable; staying solo\n", hydra_peer.c_str()); + } + } + } + + void server_context_impl::hydra_teardown_combined_before_reload() { + SRV_INF("hydra: tearing down COMBINED before model reload (was head_attached=%d, static=%d)\n", + (int) hydra_combined_head_attached, (int) hydra_combined_static); + llama_hydra_set_expert_mode(ctx_tgt, 0); + if (!hydra_current_peer.empty()) { + ctx_tgt->hydra_remove_combined_rpc_backend(hydra_current_peer.c_str()); + } + llama_hydra_clear_combined_bindings(ctx_tgt, hydra_peer.c_str()); + hydra_combined_head_attached = false; + } + + void server_context_impl::hydra_register_rpc_servers(const json & servers_arr) { + static std::set registered; + + if (!servers_arr.is_array() || servers_arr.empty()) { + return; + } + + ggml_backend_load_all(); + ggml_backend_reg_t rpc_reg = ggml_backend_reg_by_name("RPC"); + if (!rpc_reg) { + SRV_WRN("%s", "hydra: rpc_servers: RPC backend not available\n"); + return; + } + + typedef ggml_backend_reg_t (*ggml_backend_rpc_add_server_t)(const char * endpoint); + auto add_server_fn = (ggml_backend_rpc_add_server_t) + ggml_backend_reg_get_proc_address(rpc_reg, "ggml_backend_rpc_add_server"); + if (!add_server_fn) { + SRV_WRN("%s", "hydra: rpc_servers: ggml_backend_rpc_add_server not found\n"); + return; + } + + for (const auto & v : servers_arr) { + if (!v.is_string()) continue; + const std::string endpoint = v.get(); + if (endpoint.empty()) continue; + if (registered.count(endpoint)) { + SRV_DBG("hydra: rpc_servers: endpoint '%s' already registered, skipping\n", + endpoint.c_str()); + continue; + } + ggml_backend_reg_t reg = add_server_fn(endpoint.c_str()); + if (reg) { + ggml_backend_register(reg); + registered.insert(endpoint); + SRV_INF("hydra: rpc_servers: registered endpoint '%s'\n", endpoint.c_str()); + } else { + SRV_WRN("hydra: rpc_servers: failed to register endpoint '%s'\n", + endpoint.c_str()); + } + } + } + + bool server_context_impl::apply_t2_rebuild(const std::string & pending_json) { + if (!ctx_tgt || !model_tgt) return false; + + json cfg; + try { + cfg = json::parse(pending_json); + } catch (const std::exception & e) { + SRV_WRN("hydra: T2 apply: invalid JSON in pending_config: %s\n", e.what()); + return false; + } + + // Snapshot the old params for rollback. params_base is the + // canonical "what's in effect" state; restoring it plus a + // recreate-cycle is the rollback path. + common_params old_params = params_base; + + // Update params_base with the T2 keys. Each is optional; + // absence means "leave unchanged". + if (cfg.contains("n_ctx") && cfg["n_ctx"].is_number_integer()) { + const int32_t n_ctx = cfg["n_ctx"].get(); + // Clamp to the model's training ctx. The wire spec does + // not require a reject-on-too-large (the engine's own + // check below does that); we clamp and report. + const int32_t max_ctx = (int32_t) llama_model_n_ctx_train(model_tgt); + if (n_ctx > max_ctx) { + SRV_WRN("hydra: T2 n_ctx=%d exceeds model_n_ctx_train=%d; clamping\n", + n_ctx, max_ctx); + params_base.n_ctx = max_ctx; + } else { + params_base.n_ctx = n_ctx; + } + } + if (cfg.contains("cache_type_k") && cfg["cache_type_k"].is_string()) { + const std::string & s = cfg["cache_type_k"].get_ref(); + ggml_type t = hydra_parse_cache_type(s); + if (t == GGML_TYPE_COUNT) { + SRV_WRN("hydra: T2 cache_type_k='%s' unparseable; ignoring\n", s.c_str()); + } else { + params_base.cache_type_k = t; + } + } + if (cfg.contains("cache_type_v") && cfg["cache_type_v"].is_string()) { + const std::string & s = cfg["cache_type_v"].get_ref(); + ggml_type t = hydra_parse_cache_type(s); + if (t == GGML_TYPE_COUNT) { + SRV_WRN("hydra: T2 cache_type_v='%s' unparseable; ignoring\n", s.c_str()); + } else { + params_base.cache_type_v = t; + } + } + if (cfg.contains("rope_freq_base") && cfg["rope_freq_base"].is_number()) { + params_base.rope_freq_base = cfg["rope_freq_base"].get(); + } + if (cfg.contains("rope_freq_scale") && cfg["rope_freq_scale"].is_number()) { + params_base.rope_freq_scale = cfg["rope_freq_scale"].get(); + } + if (cfg.contains("yarn_ext_factor") && cfg["yarn_ext_factor"].is_number()) { + params_base.yarn_ext_factor = cfg["yarn_ext_factor"].get(); + } + if (cfg.contains("yarn_attn_factor") && cfg["yarn_attn_factor"].is_number()) { + params_base.yarn_attn_factor = cfg["yarn_attn_factor"].get(); + } + if (cfg.contains("yarn_beta_fast") && cfg["yarn_beta_fast"].is_number()) { + params_base.yarn_beta_fast = cfg["yarn_beta_fast"].get(); + } + if (cfg.contains("yarn_beta_slow") && cfg["yarn_beta_slow"].is_number()) { + params_base.yarn_beta_slow = cfg["yarn_beta_slow"].get(); + } + if (cfg.contains("yarn_orig_ctx") && cfg["yarn_orig_ctx"].is_number_integer()) { + params_base.yarn_orig_ctx = cfg["yarn_orig_ctx"].get(); + } + + // Free the live context. KV cache is destroyed; this is the + // T2 cost. The model is kept (T2 is context-only). + llama_free(ctx_tgt); + if (ctx_dft) { + llama_free(ctx_dft.get()); + ctx_dft.reset(); + } + + // Build new cparams from the updated params_base. This is + // the same call site load_model() uses internally. + auto cparams = common_context_params_to_llama(params_base); + + // Recreate the context with the new cparams. + ctx_tgt = llama_new_context_with_model(model_tgt, cparams); + if (!ctx_tgt) { + // Rollback: rebuild with the old params_base. The old + // params must work (we just freed and recreated the + // context with them). If they don't, the engine is in + // a bad state — abort. + SRV_WRN("hydra: T2 rebuild failed with n_ctx=%d cache_type=%d/%d; " + "rolling back to old params\n", + params_base.n_ctx, (int) params_base.cache_type_k, + (int) params_base.cache_type_v); + params_base = old_params; + auto cparams_old = common_context_params_to_llama(params_base); + ctx_tgt = llama_new_context_with_model(model_tgt, cparams_old); + if (!ctx_tgt) { + GGML_ABORT("hydra: T2 rollback failed (cannot rebuild context with old params). " + "Engine exiting to prevent serving with corrupted state."); + } + return false; + } + + // Re-init per-slot samplers. The old samplers were bound to + // the now-freed context; common_sampler_init() on the new + // model picks up the (possibly changed) sampling config. + for (auto & slot : slots) { + slot.smpl.reset(common_sampler_init(model_tgt, params_base.sampling)); + } + + n_ctx = llama_n_ctx(ctx_tgt); + SRV_INF("hydra: T2 rebuild applied (n_ctx=%d, cache=%d/%d, slots=%zu)\n", + n_ctx, (int) params_base.cache_type_k, + (int) params_base.cache_type_v, slots.size()); + return true; + } + + ggml_type server_context_impl::hydra_parse_cache_type(const std::string & s) { + if (s.empty()) return GGML_TYPE_COUNT; + for (int i = 0; i < GGML_TYPE_COUNT; i++) { + ggml_type t = (ggml_type) i; + if (strcmp(ggml_type_name(t), s.c_str()) == 0) return t; + } + return GGML_TYPE_COUNT; + } + + bool server_context_impl::apply_pending_hydra_config() { + const bool is_first_load = !ctx_tgt; + if (is_first_load) { + if (!first_load_pending) { + return false; + } + // Don't check hydra_has_pending_config — ctx_tgt doesn't exist yet. + // The T3 statics were staged by hydra_apply_t3_mutators() in the + // CONFIGURE handler. Set a default tier for the rebuild path. + } else if (!ctx_tgt->hydra_has_pending_config()) { + return false; + } + + // 1. Drain timeout — skipped for first load (no ctx_tgt timestamp). + std::string tier; + std::string pending_json; + + if (is_first_load) { + tier = "T3"; + // pending_json stays empty — T3 statics are staged in global + // overrides, not in pending_config (ctx_tgt doesn't exist yet). + } else { + constexpr time_t k_drain_timeout_default = 300; + time_t now = std::time(nullptr); + time_t elapsed = now - ctx_tgt->hydra_get_pending_config_set_at(); + int env_timeout = 0; + if (const char * e = getenv("HYDRA_COORD_PROFILE_SWITCH_DRAIN_TIMEOUT")) { + env_timeout = atoi(e); + } + time_t drain_timeout = env_timeout > 0 ? env_timeout : k_drain_timeout_default; + if (elapsed > drain_timeout) { + SRV_WRN("hydra: pending config drain timeout (elapsed=%lld, limit=%lld) — discarding, " + "tier='%s' payload_size=%zu\n", + (long long) elapsed, (long long) drain_timeout, + ctx_tgt->hydra_get_pending_config_tier().c_str(), + ctx_tgt->hydra_get_pending_config().size()); + ctx_tgt->hydra_clear_pending_config(); + llama_hydra_clear_pending_t3(); + return false; + } + + tier = ctx_tgt->hydra_get_pending_config_tier(); + pending_json = ctx_tgt->hydra_get_pending_config(); + SRV_INF("hydra: applying pending config (tier='%s', age=%llds, payload_size=%zu)\n", + tier.c_str(), (long long) elapsed, pending_json.size()); + } + + bool ok = true; + + // 2. T2 work: free + rebuild context with the new cparams. + // Skipped when tier is T3 (T3's load_model() handles both). + if (tier == "T2") { + if (!apply_t2_rebuild(pending_json)) { + SRV_ERR("%s", "hydra: T2 rebuild failed; engine continues with old context\n"); + ok = false; + } + } + + // 3. T3 work: full model reload with the staged T3 statics. + // load_model() handles the unload+reload cycle. COMBINED-mode + // expert bindings are torn down before the reload and re- + // attached after, in the same pattern as SET_EXPERT_MODE. + if (tier == "T3") { + if (!apply_t3_rebuild()) { + SRV_ERR("%s", "hydra: T3 rebuild failed; engine continues with old model\n"); + ok = false; + } else { + // P1-6: T3 model changed — the cached server_context_meta + // (model_path, split_mode, tensor_split, chat_params, …) + // is now stale. Refresh it on the task-queue thread + // (safe — runs during the drain window when no slots are + // processing and no new requests are being dispatched). + if (routes_ptr) { + routes_ptr->refresh_meta(); + } + // P0-1 (#49): after deferred first-load, apply staged capabilities + // so ENGINE_INFO(0x41) and COMBINED-mode logic work correctly. + if (is_first_load) { + hydra_rpc_backend_active = bootstrap_rpc_active; + hydra_peer = bootstrap_peer; + hydra_peer_reachable = bootstrap_peer_reachable; + hydra_combined_pattern = bootstrap_pattern; + hydra_split_mode = bootstrap_split_mode; + if (bootstrap_combined_static) { + hydra_combined_static = true; + SRV_INF("%s", "P0-1: deferred first-load — combined_static mode activated\n"); + } + // Register local tensors and enable shared-backend compute + // lock so the model can serve inbound RPC requests. + if (model_tgt && ctx_tgt) { + llama_hydra_register_local_tensors_for_rpc(ctx_tgt); + llama_hydra_enable_shared_backend_compute_lock(); + } + // Update the RPC server's compute backends now that the + // model is loaded. The RPC server was started with empty + // backends (head-bootstrap mode); now populate it. + if (ctx_tgt) { + std::vector backends(8); + size_t n = llama_hydra_get_compute_backends(ctx_tgt, backends.data(), backends.size()); + if (n > backends.size()) { + backends.resize(n); + n = llama_hydra_get_compute_backends(ctx_tgt, backends.data(), backends.size()); + } + backends.resize(n); + hydra_rpc::update_backends(backends); + SRV_INF("P0-1: updated RPC backends to %zu compute device(s)\n", backends.size()); + } + SRV_INF("%s", "hydra-engine ready — model loaded via CONFIGURE T3\n"); + } + } + } + + // 4. Clear the staged state regardless of success. On failure + // the rollback in apply_t{2,3}_rebuild has restored the + // previous state; clearing the staged state prevents the + // next slot-free moment from re-attempting the same rebuild. + if (is_first_load) { + first_load_pending = false; + } else { + ctx_tgt->hydra_clear_pending_config(); + } + llama_hydra_clear_pending_t3(); + return ok; + } + + void server_context_impl::hydra_apply_t3_mutators(llama_context * ctx, const json & cfg, std::vector & deferred_keys) { + if (cfg.contains("n_gpu_layers") && cfg["n_gpu_layers"].is_number_integer()) { + llama_hydra_set_pending_n_gpu_layers(cfg["n_gpu_layers"].get()); + deferred_keys.push_back("n_gpu_layers"); + } + if (cfg.contains("n_cpu_moe") && cfg["n_cpu_moe"].is_number_integer()) { + llama_hydra_set_pending_n_cpu_moe(cfg["n_cpu_moe"].get()); + deferred_keys.push_back("n_cpu_moe"); + } + if (cfg.contains("override_tensor") && cfg["override_tensor"].is_string()) { + llama_hydra_set_override_tensor(ctx, cfg["override_tensor"].get().c_str()); + deferred_keys.push_back("override_tensor"); + } + if (cfg.contains("split_mode") && cfg["split_mode"].is_string()) { + std::vector split; + if (cfg.contains("tensor_split") && cfg["tensor_split"].is_array()) { + for (const auto & v : cfg["tensor_split"]) { + if (v.is_number()) split.push_back(v.get()); + } + } + llama_hydra_set_split_mode(ctx, cfg["split_mode"].get().c_str(), + split.empty() ? nullptr : split.data(), split.size()); + deferred_keys.push_back("split_mode"); + if (!split.empty()) deferred_keys.push_back("tensor_split"); + } else if (cfg.contains("tensor_split") && cfg["tensor_split"].is_array()) { + // tensor_split without split_mode is meaningless; record it as + // deferred and let the apply step surface the missing mode. + deferred_keys.push_back("tensor_split"); + } + if (cfg.contains("model") && cfg["model"].is_object() && + cfg["model"].contains("path") && cfg["model"]["path"].is_string()) { + llama_hydra_set_pending_model_path(cfg["model"]["path"].get().c_str()); + deferred_keys.push_back("model.path"); + } else if (cfg.contains("model") && cfg["model"].is_string()) { + // legacy shorthand: {"model": "/path/to.gguf"} + llama_hydra_set_pending_model_path(cfg["model"].get().c_str()); + deferred_keys.push_back("model"); + } + // hydra_config flat key: {"model_path": "/path/to.gguf"} + if (cfg.contains("model_path") && cfg["model_path"].is_string()) { + llama_hydra_set_pending_model_path(cfg["model_path"].get().c_str()); + deferred_keys.push_back("model_path"); + } + // hydra_config: {"rpc_servers": ["host1:port1", "host2:port2"]} + // Stored in a static for apply_t3_rebuild() to consume before + // load_model(). The actual ggml backend registration happens in + // hydra_register_rpc_servers() called from apply_t3_rebuild(). + if (cfg.contains("rpc_servers") && cfg["rpc_servers"].is_array()) { + g_pending_rpc_servers.clear(); + for (const auto & v : cfg["rpc_servers"]) { + if (v.is_string()) { + g_pending_rpc_servers.push_back(v.get()); + } + } + deferred_keys.push_back("rpc_servers"); + } + } + + bool server_context_impl::hydra_apply_t1_config(common_params & params, llama_context * ctx, const json & cfg, std::map & params_applied) { + // sampling.* — set on the common_params, which the next launch_slot + // will pick up when re-initializing the slot's common_sampler. + if (cfg.contains("sampling") && cfg["sampling"].is_object()) { + const json & s = cfg["sampling"]; + #define COPY_FLOAT(field) \ + if (s.contains(#field) && s[#field].is_number()) { \ + params.sampling.field = s[#field].get(); \ + params_applied["sampling." #field] = params.sampling.field; \ + } + #define COPY_INT(field) \ + if (s.contains(#field) && s[#field].is_number()) { \ + params.sampling.field = s[#field].get(); \ + params_applied["sampling." #field] = params.sampling.field; \ + } + COPY_FLOAT(temp) + COPY_FLOAT(top_p) + COPY_FLOAT(min_p) + COPY_FLOAT(penalty_repeat) + COPY_INT(top_k) + COPY_INT(seed) + #undef COPY_FLOAT + #undef COPY_INT + } + // n_predict + if (cfg.contains("n_predict") && cfg["n_predict"].is_number_integer()) { + params.n_predict = cfg["n_predict"].get(); + params_applied["n_predict"] = params.n_predict; + } else if (cfg.contains("n_predict") && !cfg["n_predict"].is_number_integer()) { + SRV_WRN("%s", "hydra: CONFIGURE n_predict must be an integer\n"); + return false; + } + // n_keep + if (cfg.contains("n_keep") && cfg["n_keep"].is_number_integer()) { + params.n_keep = cfg["n_keep"].get(); + params_applied["n_keep"] = params.n_keep; + } else if (cfg.contains("n_keep") && !cfg["n_keep"].is_number_integer()) { + SRV_WRN("%s", "hydra: CONFIGURE n_keep must be an integer\n"); + return false; + } + // seed (top-level — sets the sampler's seed via common_params::sampling). + // common_params itself has no top-level seed; common_params_sampling does. + if (cfg.contains("seed") && cfg["seed"].is_number_unsigned()) { + params.sampling.seed = cfg["seed"].get(); + params_applied["seed"] = params.sampling.seed; + } else if (cfg.contains("seed") && cfg["seed"].is_number_integer()) { + params.sampling.seed = (uint32_t) cfg["seed"].get(); + params_applied["seed"] = params.sampling.seed; + } else if (cfg.contains("seed") && !cfg["seed"].is_number()) { + SRV_WRN("%s", "hydra: CONFIGURE seed must be a number\n"); + return false; + } + // antiprompt — full replacement (matches the existing semantics + // of CLI --reverse-prompt) + if (cfg.contains("antiprompt") && cfg["antiprompt"].is_array()) { + std::vector new_antiprompt; + for (const auto & v : cfg["antiprompt"]) { + if (!v.is_string()) { + SRV_WRN("%s", "hydra: CONFIGURE antiprompt entries must be strings\n"); + return false; + } + new_antiprompt.push_back(v.get()); + } + params.antiprompt = std::move(new_antiprompt); + params_applied["antiprompt"] = params.antiprompt; + } else if (cfg.contains("antiprompt") && !cfg["antiprompt"].is_array()) { + SRV_WRN("%s", "hydra: CONFIGURE antiprompt must be an array of strings\n"); + return false; + } + // state_chunk_size — apply via the existing llama_hydra API (clamps + // and echoes the post-clamp value) + if (cfg.contains("state_chunk_size") && cfg["state_chunk_size"].is_number_unsigned()) { + const size_t bytes = cfg["state_chunk_size"].get(); + if (ctx) { + llama_hydra_set_state_chunk_size(ctx, bytes); + } + const size_t applied = ctx ? llama_hydra_get_state_chunk_size(ctx) : llama_hydra_clamp_state_chunk_size(bytes); + params_applied["state_chunk_size"] = (uint64_t) applied; + } else if (cfg.contains("state_chunk_size") && !cfg["state_chunk_size"].is_number_unsigned()) { + SRV_WRN("%s", "hydra: CONFIGURE state_chunk_size must be a non-negative integer\n"); + return false; + } + return true; + } + + int server_context_impl::hydra_classify_config_key(const std::string & key) { + // T1: sampling nested keys + if (key == "sampling.temp" || + key == "sampling.top_p" || + key == "sampling.top_k" || + key == "sampling.min_p" || + key == "sampling.penalty_repeat" || + key == "sampling.seed") { + return 1; + } + // T1: top-level fields + if (key == "n_predict" || + key == "n_keep" || + key == "seed" || + key == "antiprompt" || + key == "state_chunk_size") { + return 1; + } + // T2: context-level (KV cache / RoPE / ctx) + if (key == "n_ctx" || + key == "cache_type_k" || + key == "cache_type_v" || + key.rfind("rope_", 0) == 0) { + return 2; + } + // T3: model-level (offload / placement / model) + if (key == "n_gpu_layers" || + key == "n_cpu_moe" || + key == "override_tensor" || + key == "split_mode" || + key == "tensor_split" || + key == "model_path" || // hydra_config: absolute GGUF path + key == "rpc_servers" || // hydra_config: RPC peer endpoints to register + key == "model.path" || + key == "model") { // legacy alias for { "model": { "path": ... } } + return 3; + } + return 0; // unknown + } + + const char * server_context_impl::hydra_tier_label(int tier) { + switch (tier) { + case 1: return "T1"; + case 2: return "T2"; + case 3: return "T3"; + default: return "T1"; // 0 (no recognized keys) → degenerate T1 + } + } diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp index f6c6fbc2b173..c01e546ba6a6 100644 --- a/tools/server/server-context.cpp +++ b/tools/server/server-context.cpp @@ -3,6 +3,7 @@ #include "server-chat.h" #include "server-common.h" #include "server-checkpoint-policy.h" +#include "server-hydra-extension.h" #include "server-http.h" #include "server-task.h" #include "server-queue.h" @@ -1053,6 +1054,10 @@ bool hydra_apply_generic_key(common_params & params, const std::string & key, co struct server_context_impl { friend struct server_context; friend struct server_routes; + // epic #610 WS1: the concrete Hydra extension (defined in + // hydra-server-context.cpp, #include'd at the bottom of this TU) needs + // access to the same internals the inline Hydra code uses. + friend struct hydra_engine_extension; public: // only use these pointers outside of this class: @@ -1117,8 +1122,23 @@ struct server_context_impl { // note: chat_params must not be refreshed upon existing sleeping state server_chat_params chat_params; + // epic #610 WS1: Hydra A/B extension seam. + // legacy mode (default): the inline Hydra code in this file runs. + // seam mode (HYDRA_EXT_MODE=seam): the hydra_engine_extension drives the + // same behavior through the server_hydra_extension interface. Both paths + // stay compiled; only one is consulted per run, so the same binary can be + // A/B tested by flipping the env var. + const bool hydra_ext_active = hydra_ext_mode_seam(); + std::unique_ptr hydra_ext; + server_context_impl() { mtmd_helper_log_set(common_log_default_callback, nullptr); + if (hydra_ext_active) { + hydra_ext = hydra_create_extension(); + SRV_INF("hydra ext: seam mode active (default), impl=%s\n", hydra_ext->name()); + } else { + SRV_INF("%s", "hydra ext: legacy mode active (HYDRA_EXT_MODE=legacy) - A/B baseline\n"); + } } ~server_context_impl() { @@ -2726,8861 +2746,4111 @@ struct server_context_impl { cur.pos_max, cur.n_tokens, (float) cur.size() / 1024 / 1024); } - // Apply the T1 keys from `cfg` to `params` and to the live context (for - // state_chunk_size). Echoes each applied key + post-clamp value into - // `params_applied`. Returns true on success; false if a key's value is - // of the wrong type (which is reported back to the caller — the request - // is malformed and we don't want to apply a partial set). - bool hydra_apply_t1_config(common_params & params, llama_context * ctx, - const json & cfg, - std::map & params_applied) { - // sampling.* — set on the common_params, which the next launch_slot - // will pick up when re-initializing the slot's common_sampler. - if (cfg.contains("sampling") && cfg["sampling"].is_object()) { - const json & s = cfg["sampling"]; - #define COPY_FLOAT(field) \ - if (s.contains(#field) && s[#field].is_number()) { \ - params.sampling.field = s[#field].get(); \ - params_applied["sampling." #field] = params.sampling.field; \ - } - #define COPY_INT(field) \ - if (s.contains(#field) && s[#field].is_number()) { \ - params.sampling.field = s[#field].get(); \ - params_applied["sampling." #field] = params.sampling.field; \ - } - COPY_FLOAT(temp) - COPY_FLOAT(top_p) - COPY_FLOAT(min_p) - COPY_FLOAT(penalty_repeat) - COPY_INT(top_k) - COPY_INT(seed) - #undef COPY_FLOAT - #undef COPY_INT - } - // n_predict - if (cfg.contains("n_predict") && cfg["n_predict"].is_number_integer()) { - params.n_predict = cfg["n_predict"].get(); - params_applied["n_predict"] = params.n_predict; - } else if (cfg.contains("n_predict") && !cfg["n_predict"].is_number_integer()) { - SRV_WRN("%s", "hydra: CONFIGURE n_predict must be an integer\n"); - return false; - } - // n_keep - if (cfg.contains("n_keep") && cfg["n_keep"].is_number_integer()) { - params.n_keep = cfg["n_keep"].get(); - params_applied["n_keep"] = params.n_keep; - } else if (cfg.contains("n_keep") && !cfg["n_keep"].is_number_integer()) { - SRV_WRN("%s", "hydra: CONFIGURE n_keep must be an integer\n"); - return false; + + // T2 rebuild: free the live llama_context, rebuild llama_context_params + // from the updated params_base (n_ctx / cache_type_k / cache_type_v / + // RoPE / YaRN), recreate the context, re-init per-slot samplers. + // On failure: rebuild with the old params_base (rollback). + // + // Helper: parse a wire-shape cache_type string ("f16" / "q8_0" / ...) + // to a ggml_type. The wire spec uses llama.cpp's ggml type names. + // There is no public ggml_parse_type() in upstream llama.cpp, so + // we iterate ggml_type_traits via ggml_get_type_traits() and + // match on ggml_type_name(). +static ggml_type hydra_parse_cache_type(const std::string & s); + + + // Register new RPC peer devices into the global ggml backend registry. + // Called from apply_t3_rebuild() before load_model() so the new peer's + // device exists when common_init_from_params() tries to place tensors + // per tensor_split/split_mode. + // + // Repeated registration is NOT safe/idempotent in the underlying API, + // so we track already-registered endpoints in a static set and only + // register genuinely new ones. +static void hydra_register_rpc_servers(const json & servers_arr); + + // #470: refresh the T3-current alias → file map after a successful + // model load. The map's keys are the engine's own identity aliases + // (model_name + model_aliases, as recomputed by load_model()); the + // value is the resident file. Called from load_model() right after the + // identity members are re-derived, so the map always tracks the CURRENT + // resident — covering boot, apply_t3_rebuild(), the bare-alias swap + // paths and T3 rollback alike. + // + // Why this map exists: preset_alias_to_path is the static INI mapping, + // but the coordinator's T3 config (hydra_config.model_path) can load a + // file that the INI does NOT associate with the engine's current alias + // (e.g. the dense-27b-combined session T3-loads the 27B-Coder file + // while the engine's identity still says qwen3.6-35B-balanced from an + // earlier SOLO session). DECODE_APPLY must know that the requested + // alias already refers to the resident before it decides to swap. + void hydra_t3_record_current_alias_to_path() { + t3_current_alias_to_path.clear(); + const std::string & resident = params_base.model.path; + t3_current_alias_to_path[model_name] = resident; + for (const auto & alias : model_aliases) { + t3_current_alias_to_path[alias] = resident; } - // seed (top-level — sets the sampler's seed via common_params::sampling). - // common_params itself has no top-level seed; common_params_sampling does. - if (cfg.contains("seed") && cfg["seed"].is_number_unsigned()) { - params.sampling.seed = cfg["seed"].get(); - params_applied["seed"] = params.sampling.seed; - } else if (cfg.contains("seed") && cfg["seed"].is_number_integer()) { - params.sampling.seed = (uint32_t) cfg["seed"].get(); - params_applied["seed"] = params.sampling.seed; - } else if (cfg.contains("seed") && !cfg["seed"].is_number()) { - SRV_WRN("%s", "hydra: CONFIGURE seed must be a number\n"); - return false; + SRV_DBG("hydra: T3-current alias→file map recorded %zu alias(es) → '%s'\n", + t3_current_alias_to_path.size(), resident.c_str()); + } + + // Tear down COMBINED-mode RPC peer bindings before a model reload. + // Shared by apply_t3_rebuild() and the bare-alias swap paths (PREFILL, + // DECODE_APPLY, server-context.cpp ~3800 / ~4310). Must run BEFORE + // load_model() — otherwise the new ctx_tgt (post-reload) inherits a + // stale binding to the old peer's device. The bare-alias paths used to + // skip this entirely: the engine loaded the correct model file but kept + // routing tokens through the old COMBINED config, which is #514 + // (throughput collapses to ~2-4 tok/s after a dynamic model swap). +void hydra_teardown_combined_before_reload(); + + // epic #610 WS2: HYDRA task dispatch — declared here, defined in + // hydra-server-context.cpp (included at the bottom of this TU). Keeps a + // switch(task.type) wrapper so break/continue semantics are unchanged. + void hydra_process_task(server_task & task); + + // Re-attach COMBINED-mode bindings after a model reload, mirroring + // hydra_teardown_combined_before_reload() above. Layer-split (static) + // just re-enables the mode flag — load_model() already preloaded the + // peer device with the new tensor_split. Expert-split re-resolves the + // peer's RPC device and rebinds the expert tensors; same fail-open + // pattern as SET_EXPERT_MODE — if the peer is unreachable, the engine + // stays solo and the coordinator's solo-fallback path handles it. +void hydra_reattach_combined_after_reload(); + + // Re-pad tensor_buft_overrides to the nullptr-terminated capacity + // llama_max_tensor_buft_overrides() after a preset's apply_to_params() + // has push_back()'d entries onto a freshly-cleared vector (see the + // bare-alias swap paths, server-context.cpp ~3800 / ~4310). Caps at + // the limit with the same guard the override_tensor T3 path already + // has (below) — apply_to_params() push_backs unconditionally, so an + // unusually large preset could otherwise overflow the same 4096-entry + // limit this whole clear/re-pad dance exists to respect. +void hydra_repad_tensor_buft_overrides(common_params & p, const char * ctx_label); + + // T3 rebuild: full model reload. Uses the staged T3 statics + // (override_tensor, split_mode, tensor_split, n_gpu_layers, + // n_cpu_moe, model.path) populated by hydra_apply_t3_mutators(). + // Falls through to load_model() for the actual unload+reload + // cycle (which handles mmproj, MTP/draft, slot rebuild, etc.). + // On failure: rollback by reloading the old params_base. + + void update_slots() { + // epic #610 WS1: in seam mode the extension may pre-empt the decode + // loop (T3/CONFIGURE/reattach cluster). WS1 impl is a no-op — this is + // a pure A/B switch, both modes run the inline body below. + if (hydra_ext_active && hydra_ext && hydra_ext->pre_loop(*this)) { + return; } - // antiprompt — full replacement (matches the existing semantics - // of CLI --reverse-prompt) - if (cfg.contains("antiprompt") && cfg["antiprompt"].is_array()) { - std::vector new_antiprompt; - for (const auto & v : cfg["antiprompt"]) { - if (!v.is_string()) { - SRV_WRN("%s", "hydra: CONFIGURE antiprompt entries must be strings\n"); - return false; + // check if all slots are idle + { + bool all_idle = true; + + for (auto & slot : slots) { + if (slot.is_processing() || slot.hydra_transferring->load()) { + all_idle = false; + break; } - new_antiprompt.push_back(v.get()); } - params.antiprompt = std::move(new_antiprompt); - params_applied["antiprompt"] = params.antiprompt; - } else if (cfg.contains("antiprompt") && !cfg["antiprompt"].is_array()) { - SRV_WRN("%s", "hydra: CONFIGURE antiprompt must be an array of strings\n"); - return false; - } - // state_chunk_size — apply via the existing llama_hydra API (clamps - // and echoes the post-clamp value) - if (cfg.contains("state_chunk_size") && cfg["state_chunk_size"].is_number_unsigned()) { - const size_t bytes = cfg["state_chunk_size"].get(); - if (ctx) { - llama_hydra_set_state_chunk_size(ctx, bytes); - } - const size_t applied = ctx ? llama_hydra_get_state_chunk_size(ctx) : llama_hydra_clamp_state_chunk_size(bytes); - params_applied["state_chunk_size"] = (uint64_t) applied; - } else if (cfg.contains("state_chunk_size") && !cfg["state_chunk_size"].is_number_unsigned()) { - SRV_WRN("%s", "hydra: CONFIGURE state_chunk_size must be a non-negative integer\n"); - return false; - } - return true; - } - // Apply the T3 mutators immediately. The "staging" is: the statics in - // llama-hydra.cpp + the T3 keys in pending_config JSON. The actual - // model reload happens later, in the slot-free trigger. - void hydra_apply_t3_mutators(llama_context * ctx, const json & cfg, - std::vector & deferred_keys) { - if (cfg.contains("n_gpu_layers") && cfg["n_gpu_layers"].is_number_integer()) { - llama_hydra_set_pending_n_gpu_layers(cfg["n_gpu_layers"].get()); - deferred_keys.push_back("n_gpu_layers"); - } - if (cfg.contains("n_cpu_moe") && cfg["n_cpu_moe"].is_number_integer()) { - llama_hydra_set_pending_n_cpu_moe(cfg["n_cpu_moe"].get()); - deferred_keys.push_back("n_cpu_moe"); - } - if (cfg.contains("override_tensor") && cfg["override_tensor"].is_string()) { - llama_hydra_set_override_tensor(ctx, cfg["override_tensor"].get().c_str()); - deferred_keys.push_back("override_tensor"); - } - if (cfg.contains("split_mode") && cfg["split_mode"].is_string()) { - std::vector split; - if (cfg.contains("tensor_split") && cfg["tensor_split"].is_array()) { - for (const auto & v : cfg["tensor_split"]) { - if (v.is_number()) split.push_back(v.get()); + if (all_idle) { + SRV_INF("%s", "all slots are idle\n"); + + // Hydra #406: slot-free moment — if a tiered CONFIGURE + // staged a T2/T3 rebuild, run the apply step now. The + // apply step (in apply_pending_hydra_config below) does + // the actual T2 context rebuild and/or T3 model reload, + // then clears the staged state. The low-level helper + // llama_hydra_apply_pending_config() is a no-op once the + // staged state has been cleared. + if (ctx_tgt && ctx_tgt->hydra_has_pending_config()) { + SRV_INF("hydra: slot-free moment — applying pending CONFIGURE (tier=%s)\n", + ctx_tgt->hydra_get_pending_config_tier().c_str()); + apply_pending_hydra_config(); + } else if (!ctx_tgt && first_load_pending) { + SRV_INF("%s", "hydra: slot-free moment — first load (no context yet)\n"); + apply_pending_hydra_config(); } - } - llama_hydra_set_split_mode(ctx, cfg["split_mode"].get().c_str(), - split.empty() ? nullptr : split.data(), split.size()); - deferred_keys.push_back("split_mode"); - if (!split.empty()) deferred_keys.push_back("tensor_split"); - } else if (cfg.contains("tensor_split") && cfg["tensor_split"].is_array()) { - // tensor_split without split_mode is meaningless; record it as - // deferred and let the apply step surface the missing mode. - deferred_keys.push_back("tensor_split"); - } - if (cfg.contains("model") && cfg["model"].is_object() && - cfg["model"].contains("path") && cfg["model"]["path"].is_string()) { - llama_hydra_set_pending_model_path(cfg["model"]["path"].get().c_str()); - deferred_keys.push_back("model.path"); - } else if (cfg.contains("model") && cfg["model"].is_string()) { - // legacy shorthand: {"model": "/path/to.gguf"} - llama_hydra_set_pending_model_path(cfg["model"].get().c_str()); - deferred_keys.push_back("model"); - } - // hydra_config flat key: {"model_path": "/path/to.gguf"} - if (cfg.contains("model_path") && cfg["model_path"].is_string()) { - llama_hydra_set_pending_model_path(cfg["model_path"].get().c_str()); - deferred_keys.push_back("model_path"); - } - // hydra_config: {"rpc_servers": ["host1:port1", "host2:port2"]} - // Stored in a static for apply_t3_rebuild() to consume before - // load_model(). The actual ggml backend registration happens in - // hydra_register_rpc_servers() called from apply_t3_rebuild(). - if (cfg.contains("rpc_servers") && cfg["rpc_servers"].is_array()) { - g_pending_rpc_servers.clear(); - for (const auto & v : cfg["rpc_servers"]) { - if (v.is_string()) { - g_pending_rpc_servers.push_back(v.get()); + + // #470 Option B: check if a peer reconnection was detected + // during graph_compute. If so, trigger a T3 rebuild to + // re-provision model layers on the fresh peer. + if (ctx_tgt && ctx_tgt->peer_reconnection_pending) { + ctx_tgt->peer_reconnection_pending = false; + SRV_WRN("%s", "hydra: peer reconnection detected — triggering T3 rebuild\n"); + // Force a T3 rebuild even if model config hasn't changed. + // The peer's buffers are gone, so we need to re-push. + if (!apply_t3_rebuild(true)) { + SRV_ERR("%s", "hydra: T3 rebuild after peer reconnection failed\n"); + } } + + return; } - deferred_keys.push_back("rpc_servers"); } - } - // Shared helper: classify config keys, apply T1 immediately, and either - // stage (sync=false) or synchronously apply (sync=true) T2/T3. - // - // sync=false (CONFIGURE path): T2/T3 are staged via hydra_set_pending_config() - // + hydra_apply_t3_mutators() for later application at the slot-free moment. - // - // sync=true (PREFILL / HTTP decode path): T2/T3 are applied immediately - // via apply_t2_rebuild() / apply_t3_rebuild(). The caller already owns - // the task-queue thread context, so synchronous application is safe. - // - // Returns a structured result so callers can build CONFIGURE responses - // or handle errors uniformly. - struct hydra_config_result { - int highest_tier = 0; - std::map params_applied; - std::vector deferred_keys; - json t2t3_subset = json::object(); - bool ok = true; - std::string error; - uint64_t state_chunk_size_applied = 0; - // hydra#470: generic (T4) keys that cannot take effect. Echoed to - // the Coordinator via the CONFIGURE response so nothing is silent. - std::vector unrecognized_keys; // no llama.cpp arg-table entry - std::vector rejected_keys; // startup-only / flag / two-value arg - }; + { + SRV_DBG("%s", "posting NEXT_RESPONSE\n"); - hydra_config_result hydra_apply_config(const json & cfg, bool sync) { - hydra_config_result result; - - // 1. Classify every top-level key. T1 → apply now; T2/T3/T4 → - // defer (stage) or apply synchronously depending on `sync`. - // T4 (generic) keys are additionally validated against the - // llama.cpp arg table here so the CONFIGURE response can report - // unrecognized/rejected keys before the deferred apply runs. - // T2 + appliable-T4 keys are staged together into the reload - // config (g_pending_reload_config) so neither the context-reload - // path nor the model-reload path strands them. - json t1_subset = json::object(); - json reload_subset = json::object(); - for (auto it = cfg.begin(); it != cfg.end(); ++it) { - const std::string key = it.key(); - int tier = hydra_classify_config_key(key); - if (tier == 1) { - t1_subset[key] = it.value(); - } else if (tier == 2) { - result.t2t3_subset[key] = it.value(); - result.deferred_keys.push_back(key); - reload_subset[key] = it.value(); - } else if (tier == 3) { - result.t2t3_subset[key] = it.value(); - result.deferred_keys.push_back(key); - // T3 keys are staged via hydra_apply_t3_mutators() statics, - // not the reload-config JSON. - } else { - // T4: generic-arg pass-through. Classify against the arg - // table; only appliable keys are staged for the deferred - // slot-free moment. The rest are reported loudly. - const hydra_generic_key_status st = hydra_classify_generic_key(key); - if (st == hydra_generic_key_status::APPLIABLE) { - result.t2t3_subset[key] = it.value(); - result.deferred_keys.push_back(key); - reload_subset[key] = it.value(); - } else if (st == hydra_generic_key_status::DENIED) { - SRV_WRN("hydra: CONFIGURE key '%s' cannot change at reload (startup-only/flag arg) — rejected\n", - key.c_str()); - result.rejected_keys.push_back(key); - } else { - SRV_WRN("hydra: CONFIGURE key '%s' is not a known llama.cpp argument — unrecognized, value ignored\n", - key.c_str()); - result.unrecognized_keys.push_back(key); + server_task task(SERVER_TASK_TYPE_NEXT_RESPONSE); + task.id = queue_tasks.get_new_id(); + queue_tasks.post(std::move(task)); + } + + // apply context-shift if needed + // TODO: simplify and improve + for (server_slot & slot : slots) { + if (slot.state == SLOT_STATE_GENERATING && slot.prompt.n_tokens() + 1 >= slot.n_ctx) { + if (!params_base.ctx_shift) { + // this check is redundant (for good) + // we should never get here, because generation should already stopped in process_token() + send_error(slot, "context shift is disabled", ERROR_TYPE_SERVER); + slot.release(); + continue; } - } - if (tier > result.highest_tier) result.highest_tier = tier; - } - // Stage the reload config (T2 + appliable-T4 keys) for the deferred - // slot-free moment. Unconditional overwrite = absolute state: a - // superseding CONFIGURE without T2/T4 keys must clear whatever an - // earlier CONFIGURE staged, or the stale keys would be applied on - // the next unrelated reload. - g_pending_reload_config = reload_subset.dump(); - // The "sampling" object may contain unlisted nested keys - // (e.g. penalty_last_n, mirostat) — route the whole object - // through T1 when present. - if (cfg.contains("sampling") && cfg["sampling"].is_object()) { - t1_subset["sampling"] = cfg["sampling"]; - if (result.highest_tier < 1) result.highest_tier = 1; - } - // model.path is nested — the legacy {"model": {...}} form - // is recognized by hydra_classify_config_key returning 3 - // for the bare "model" key. If the bare "model" is set - // and is an object with a "path", route it as T3. - if (cfg.contains("model")) { - if (cfg["model"].is_object()) { - result.t2t3_subset["model"] = cfg["model"]; - if (std::find(result.deferred_keys.begin(), result.deferred_keys.end(), "model") - == result.deferred_keys.end()) { - result.deferred_keys.push_back("model"); + + if (mctx) { + // we should never reach this because params_base.ctx_shift is automatically disabled if mmproj is loaded + // we don't support ctx_shift because an image chunk may contains multiple tokens + GGML_ABORT("not supported by multimodal"); } - if (result.highest_tier < 3) result.highest_tier = 3; - } else if (cfg["model"].is_string()) { - result.t2t3_subset["model"] = cfg["model"]; - if (std::find(result.deferred_keys.begin(), result.deferred_keys.end(), "model") - == result.deferred_keys.end()) { - result.deferred_keys.push_back("model"); + + if (slot.task->is_parent() || slot.task->is_child()) { + send_error(slot, "context shift cannot be used for shared prompt", ERROR_TYPE_SERVER); + slot.release(); + continue; } - if (result.highest_tier < 3) result.highest_tier = 3; - } - } - - if (result.highest_tier == 0) { - // No recognized keys — caller decides whether to treat as - // a no-op or surface an error. - return result; - } - - // 2. Apply T1 keys in-place. - if (!t1_subset.empty()) { - if (!hydra_apply_t1_config(params_base, ctx_tgt, t1_subset, result.params_applied)) { - result.ok = false; - result.error = "T1 key has wrong type (see log)"; - return result; - } - // Capture state_chunk_size for callers that need it (CONFIGURE). - auto it = result.params_applied.find("state_chunk_size"); - if (it != result.params_applied.end() && it->second.is_number_unsigned()) { - result.state_chunk_size_applied = it->second.get(); - } - } - - // 3. T2/T3/T4 handling — diverges based on sync flag. - if (!result.t2t3_subset.empty()) { - if (sync) { - // Synchronous mode (PREFILL / HTTP decode): apply now - // on the task-queue thread. The caller owns this thread - // context so blocking is safe. - if (result.highest_tier >= 3) { - // T3 (model reload) also covers a T4-only config: - // load_model() recreates the context and the - // speculative/draft state, so every generic key - // takes effect. hydra_apply_t3_mutators() is a no-op - // when no T3 keys are staged. - hydra_apply_t3_mutators(ctx_tgt, result.t2t3_subset, result.deferred_keys); - // #470: force rebuild if a peer reconnection was detected - // during a prior graph_compute — the peer's buffers are gone - // even though model/params haven't changed. - const bool reconn_force = (ctx_tgt && ctx_tgt->peer_reconnection_pending); - if (reconn_force) { - ctx_tgt->peer_reconnection_pending = false; - SRV_WRN("%s", "hydra: PREFILL handler: peer reconnection pending — forcing T3 rebuild\n"); - } - if (!apply_t3_rebuild(reconn_force)) { - result.ok = false; - result.error = "T3 rebuild failed"; - return result; - } - } else if (result.highest_tier == 2) { - if (!apply_t2_rebuild(result.t2t3_subset.dump())) { - result.ok = false; - result.error = "T2 rebuild failed"; - return result; - } + + // Shift context + int n_keep = slot.task->params.n_keep < 0 ? slot.task->n_tokens() : slot.task->params.n_keep; + + if (add_bos_token) { + n_keep += 1; } - // Clear T3 staged statics — they were consumed by the - // sync apply and must not leak into a later deferred path. - llama_hydra_clear_pending_t3(); - } else { - // Stage mode (CONFIGURE): record the mutators and store - // pending_config for application at the next slot-free moment. - if (result.highest_tier >= 3) { - hydra_apply_t3_mutators(ctx_tgt, result.t2t3_subset, result.deferred_keys); + + n_keep = std::min(slot.n_ctx - 4, n_keep); + + const int n_left = slot.prompt.n_tokens() - n_keep; + const int n_discard = slot.task->params.n_discard ? slot.task->params.n_discard : (n_left / 2); + + SLT_WRN(slot, "slot context shift, n_keep = %d, n_left = %d, n_discard = %d\n", n_keep, n_left, n_discard); + + common_context_seq_rm (ctx_tgt, slot.id, n_keep , n_keep + n_discard); + common_context_seq_add(ctx_tgt, slot.id, n_keep + n_discard, slot.prompt.n_tokens(), -n_discard); + + // D4: seq_rm invalidates restored logits + slot.logits_valid = false; + slot.restored_logits.clear(); + + if (ctx_dft) { + common_context_seq_rm (ctx_dft.get(), slot.id, n_keep , n_keep + n_discard); + common_context_seq_add(ctx_dft.get(), slot.id, n_keep + n_discard, slot.prompt.tokens.pos_next(), -n_discard); } - if (ctx_tgt) { - ctx_tgt->hydra_set_pending_config( - result.t2t3_subset.dump(), hydra_tier_label(result.highest_tier)); - } else { - // First load: ctx_tgt is null, so apply_pending_hydra_config() - // and update_slots() can't trigger. Set the flag so the - // task-queue thread runs apply_t3_rebuild() at the next - // slot-free moment. - first_load_pending = true; - SRV_INF("%s", "hydra: config staged for first load (no context yet)\n"); + + // add generated tokens to cache + // ref: https://github.com/ggml-org/llama.cpp/pull/16818#discussion_r2473269481 + { + GGML_ASSERT(!slot.prompt.tokens.has_mtmd); + + llama_tokens new_tokens = slot.prompt.tokens.get_tokens(); // copy + for (size_t i = n_keep + n_discard; i < new_tokens.size(); i++) { + new_tokens[i - n_discard] = new_tokens[i]; + } + + new_tokens.resize(slot.prompt.tokens.size() - n_discard); + + slot.prompt.tokens.clear(); + slot.prompt.tokens.insert(new_tokens); } + + slot.truncated = true; } } - return result; - } + // start populating the batch for this iteration + common_batch_clear(batch); - void process_single_task(server_task && task) { - switch (task.type) { - case SERVER_TASK_TYPE_COMPLETION: - case SERVER_TASK_TYPE_INFILL: - case SERVER_TASK_TYPE_EMBEDDING: - case SERVER_TASK_TYPE_RERANK: - { - // special case: if input is provided via CLI, tokenize it first - // otherwise, no need to tokenize as it's already done inside the HTTP thread - if (task.cli) { - if (!tokenize_cli_input(task)) { - break; - } - } + // track if given slot can be batched with slots already in the batch + server_slot * slot_batched = nullptr; - // Hydra config from HTTP decode path: apply synchronously - // on the task-queue thread before any slot scheduling or - // generation work. This is safe because we own this thread; - // the previous attempt applied on the httplib worker thread - // and raced the main queue (reverted in ebbbe1116). - if (!task.hydra_config_json.empty()) { - json hydra_cfg; - try { - hydra_cfg = json::parse(task.hydra_config_json); - } catch (const std::exception & e) { - SRV_WRN("hydra: COMPLETION hydra_config parse failed: %s\n", e.what()); - } - if (!hydra_cfg.is_null() && hydra_cfg.is_object()) { - SRV_INF("hydra: COMPLETION applying hydra_config (%zu keys)\n", - hydra_cfg.size()); - hydra_config_result cfg_result = hydra_apply_config(hydra_cfg, /*sync=*/true); - if (!cfg_result.ok) { - SRV_WRN("hydra: COMPLETION hydra_config apply failed: %s\n", - cfg_result.error.c_str()); - } - // After T3 rebuild, model/slots are reset. - // The slot lookup below will pick up the new state. - } - } - - const int id_slot = task.id_slot; - const int id_task = task.id; + std::vector generating; + std::vector drafting; - server_slot * slot = id_slot != -1 ? get_slot_by_id(id_slot) : get_available_slot(task); + // determine which slots are generating and drafting + for (auto & slot : slots) { + if (slot.state != SLOT_STATE_GENERATING) { + continue; + } - // - // slot scheduling logic - // + // check if we can batch this slot with the previous one + if (!slot_batched) { + slot_batched = &slot; + } else if (!slot_batched->can_batch_with(slot)) { + continue; + } - if (slot == nullptr) { - // if no slot is available, we defer this task for processing later - SRV_DBG("no slot is available, defer task, id_task = %d\n", id_task); - queue_tasks.defer(std::move(task)); - break; - } + generating.push_back(&slot); - if (slot->is_processing() || slot->hydra_transferring->load()) { - // if requested slot is unavailable, we defer this task for processing later - SRV_DBG("requested slot is unavailable, defer task, id_task = %d\n", id_task); - queue_tasks.defer(std::move(task)); - break; - } + if (spec) { + common_speculative_get_draft_params(spec.get(), slot.id).drafting = false; - if (task.is_parent()) { - // try getting free slots for all child tasks - size_t n_child_tasks = task.child_tasks.size(); - std::vector child_slots = get_free_slots(n_child_tasks, slot->id); - if (child_slots.size() < n_child_tasks) { - SRV_DBG("not enough free slots for child tasks, n_free = %zu, n_children = %zu, defer task, id_task = %d\n", child_slots.size(), n_child_tasks, id_task); - queue_tasks.defer(std::move(task)); - break; - } - if (!launch_slots_with_parent_task(*slot, child_slots, std::move(task))) { - SRV_ERR("failed to launch slot with parent task, id_task = %d\n", id_task); - break; // drop the task - } - } else if (!launch_slot_with_task(*slot, std::move(task))) { - SRV_ERR("failed to launch slot with task, id_task = %d\n", id_task); - break; // drop the task - } + const bool use_ckpt_tgt = ctx_tgt_seq_rm_type == COMMON_CONTEXT_SEQ_RM_TYPE_FULL; + const bool use_ckpt_dft = ctx_dft_seq_rm_type == COMMON_CONTEXT_SEQ_RM_TYPE_FULL; - if (params_base.cache_idle_slots) { - for (auto & s : slots) { - if (!s.is_processing() && !s.hydra_transferring->load()) { - slot_save_and_clear(s); - } - } - } - } break; - case SERVER_TASK_TYPE_CANCEL: - { - // release slot linked with the task id - for (auto & slot : slots) { - if (slot.task && slot.task->id == task.id_target) { - slot.release(); - break; - } - } - } break; - case SERVER_TASK_TYPE_CONTROL: - { - auto res = std::make_unique(); - res->id = task.id; + const int n_draft_max = slot.get_n_draft_max(); - server_slot * slot = get_slot_by_cmpl_id(task.params.control_cmpl_id); - if (slot == nullptr) { - res->success = false; - res->message = "no active completion for this id"; - queue_results.send(std::move(res)); - break; - } + if (n_draft_max > 0) { + GGML_ASSERT(slot.can_speculate()); - if (task.params.control_action == "reasoning_end") { - // the budget sampler only exists when reasoning control was armed - if (!slot->task->params.sampling.reasoning_control) { - res->success = false; - res->message = "reasoning control not enabled for this completion"; - queue_results.send(std::move(res)); - break; + if (!slot.spec_draft.empty()) { + // we have a previous (partial) draft to reuse + if (use_ckpt_tgt) { + GGML_ASSERT(!slot.spec_ckpt.empty()); } - // act on the live slot mid generation, never defer - common_sampler_reasoning_budget_force(slot->smpl.get()); - res->success = true; } else { - res->success = false; - res->message = "unknown control action"; - } + GGML_ASSERT(slot.spec_i_batch.empty()); - queue_results.send(std::move(res)); - } break; - case SERVER_TASK_TYPE_NEXT_RESPONSE: - { - // do nothing - } break; - case SERVER_TASK_TYPE_METRICS: - { - json slots_data = json::array(); + slot.spec_ckpt.update_pos( + slot.prompt.n_tokens(), + llama_memory_seq_pos_min(llama_get_memory(ctx_tgt), slot.id), + llama_memory_seq_pos_max(llama_get_memory(ctx_tgt), slot.id)); - int n_idle_slots = 0; - int n_processing_slots = 0; + if (use_ckpt_dft) { + slot.spec_ckpt.update_dft(ctx_dft.get(), slot.id, LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY | LLAMA_STATE_SEQ_FLAGS_ON_DEVICE); + } - for (server_slot & slot : slots) { - json slot_data = slot.to_json(slots_debug == 0); + slot.spec_prompt = slot.prompt.tokens.get_text_tokens(); - if (slot.is_processing() || slot.hydra_transferring->load()) { - n_processing_slots++; - } else { - n_idle_slots++; - } + common_speculative_get_draft_params(spec.get(), slot.id) = { + /* .drafting = */ true, + /* .n_max = */ n_draft_max, + /* .n_past = */ slot.prompt.n_tokens(), + /* .id_last = */ slot.sampled, + /* .prompt = */ &slot.spec_prompt, + /* .result = */ &slot.spec_draft, + }; - slots_data.push_back(slot_data); + drafting.push_back(&slot); } - SRV_DBG("n_idle_slots = %d, n_processing_slots = %d\n", n_idle_slots, n_processing_slots); + } + } + } - auto res = std::make_unique(); - res->id = task.id; - res->slots_data = std::move(slots_data); - res->n_idle_slots = n_idle_slots; - res->n_processing_slots = n_processing_slots; - res->n_tasks_deferred = queue_tasks.queue_tasks_deferred_size(); - res->t_start = metrics.t_start; + // generate the actual drafts (if any) + { + common_speculative_draft(spec.get()); + } - res->n_prompt_tokens_processed_total = metrics.n_prompt_tokens_processed_total; - res->t_prompt_processing_total = metrics.t_prompt_processing_total; - res->n_tokens_predicted_total = metrics.n_tokens_predicted_total; - res->t_tokens_generation_total = metrics.t_tokens_generation_total; + // make checkpoints if needed + for (auto * slot_ptr : drafting) { + auto & slot = *slot_ptr; - res->n_tokens_max = metrics.n_tokens_max; + auto & draft = slot.spec_draft; + auto & ckpt = slot.spec_ckpt; - res->n_prompt_tokens_processed = metrics.n_prompt_tokens_processed; - res->t_prompt_processing = metrics.t_prompt_processing; - res->n_tokens_predicted = metrics.n_tokens_predicted; - res->t_tokens_generation = metrics.t_tokens_generation; + slot.n_draft_total += draft.size(); - res->n_decode_total = metrics.n_decode_total; - res->n_busy_slots_total = metrics.n_busy_slots_total; + // TODO: avoid restoring the draft context and re-evaluating the drafted tokens when not needed [TAG_SPEC_AVOID_DRAFT_REEVAL] + const bool use_ckpt_dft = ctx_dft_seq_rm_type == COMMON_CONTEXT_SEQ_RM_TYPE_FULL; - if (task.metrics_reset_bucket) { - metrics.reset_bucket(); - } - queue_results.send(std::move(res)); - } break; - case SERVER_TASK_TYPE_SLOT_SAVE: - { - if (!check_no_mtmd(task.id)) { - break; - } + if (ctx_dft) { + if (use_ckpt_dft) { + ckpt.load_dft(ctx_dft.get(), slot.id, LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY | LLAMA_STATE_SEQ_FLAGS_ON_DEVICE); + } - const int id_slot = task.slot_action.id_slot; - server_slot * slot = get_slot_by_id(id_slot); - if (slot == nullptr) { - send_error(task, "Invalid slot ID", ERROR_TYPE_INVALID_REQUEST); - break; - } - if (slot->is_processing()) { - // if requested slot is unavailable, we defer this task for processing later - SRV_DBG("requested slot is unavailable, defer task, id_task = %d\n", task.id); - queue_tasks.defer(std::move(task)); - break; - } + common_context_seq_rm(ctx_dft.get(), slot.id, ckpt.pos_max + 1, -1); + } - const size_t token_count = slot->prompt.tokens.size(); - const int64_t t_start = ggml_time_us(); + if (!draft.empty()) { + const bool use_ckpt_tgt = + ctx_tgt_seq_rm_type == COMMON_CONTEXT_SEQ_RM_TYPE_FULL || + (ctx_tgt_seq_rm_type == COMMON_CONTEXT_SEQ_RM_TYPE_RS && draft.size() > llama_n_rs_seq(ctx_tgt)); - std::string filename = task.slot_action.filename; - std::string filepath = task.slot_action.filepath; + const bool use_ckpt_dft = + (ctx_dft_seq_rm_type == COMMON_CONTEXT_SEQ_RM_TYPE_RS && draft.size() > llama_n_rs_seq(ctx_dft.get())); - const llama_tokens & tokens = slot->prompt.tokens.get_tokens(); - const size_t nwrite = llama_state_seq_save_file(ctx_tgt, filepath.c_str(), slot->id, tokens.data(), token_count); + if (use_ckpt_tgt) { + //const int64_t t_start = ggml_time_us(); - const int64_t t_end = ggml_time_us(); - const double t_save_ms = (t_end - t_start) / 1000.0; + ckpt.update_tgt(ctx_tgt, slot.id, LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY | LLAMA_STATE_SEQ_FLAGS_ON_DEVICE); - auto res = std::make_unique(); - res->id = task.id; - res->id_slot = id_slot; - res->filename = filename; - res->is_save = true; - res->n_tokens = token_count; - res->n_bytes = nwrite; - res->t_ms = t_save_ms; - queue_results.send(std::move(res)); - } break; - case SERVER_TASK_TYPE_SLOT_RESTORE: - { - if (!check_no_mtmd(task.id)) break; - const int id_slot = task.slot_action.id_slot; - server_slot * slot = get_slot_by_id(id_slot); - if (slot == nullptr) { - send_error(task, "Invalid slot ID", ERROR_TYPE_INVALID_REQUEST); - break; - } - if (slot->is_processing()) { - // if requested slot is unavailable, we defer this task for processing later - SRV_DBG("requested slot is unavailable, defer task, id_task = %d\n", task.id); - queue_tasks.defer(std::move(task)); - break; - } - - const int64_t t_start = ggml_time_us(); + //const int64_t t_total = ggml_time_us() - t_start; + //printf("checkpoint total: %f ms\n", t_total / 1000.0); - std::string filename = task.slot_action.filename; - std::string filepath = task.slot_action.filepath; + SLT_DBG(slot, "created speculative checkpoint (pos_min = %d, pos_max = %d, n_tokens = %d, size = %.3f MiB, draft = %.3f MiB)\n", + ckpt.pos_min, ckpt.pos_max, slot.prompt.n_tokens(), + (float) ckpt.size() / 1024 / 1024, + (float) ckpt.data_dft.size() / 1024 / 1024); + } - llama_tokens tokens; - tokens.resize(slot->n_ctx); - size_t token_count = 0; - size_t nread = llama_state_seq_load_file(ctx_tgt, filepath.c_str(), slot->id, tokens.data(), tokens.size(), &token_count); - if (nread == 0) { - slot->prompt.tokens.clear(); // KV may already been invalidated? - send_error(task, "Unable to restore slot, no available space in KV cache or invalid slot save file", ERROR_TYPE_INVALID_REQUEST); - break; - } - tokens.resize(token_count); - slot->prompt.tokens.clear(); - slot->prompt.tokens.insert(tokens); + if (use_ckpt_dft) { + ckpt.update_dft(ctx_dft.get(), slot.id, LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY | LLAMA_STATE_SEQ_FLAGS_ON_DEVICE); + } + } + } - const int64_t t_end = ggml_time_us(); - const double t_restore_ms = (t_end - t_start) / 1000.0; + // update the batch with the sampled/drafted tokens + for (auto * slot_ptr : generating) { + auto & slot = *slot_ptr; - auto res = std::make_unique(); - res->id = task.id; - res->id_slot = id_slot; - res->filename = filename; - res->is_save = false; - res->n_tokens = token_count; - res->n_bytes = nread; - res->t_ms = t_restore_ms; - queue_results.send(std::move(res)); - } break; - case SERVER_TASK_TYPE_SLOT_ERASE: - { - if (!check_no_mtmd(task.id)) { - break; - } - const int id_slot = task.slot_action.id_slot; - server_slot * slot = get_slot_by_id(id_slot); - if (slot == nullptr) { - send_error(task, "Invalid slot ID", ERROR_TYPE_INVALID_REQUEST); - break; - } - if (slot->is_processing() || slot->hydra_transferring->load()) { - // if requested slot is unavailable, we defer this task for processing later - SRV_DBG("requested slot is unavailable, defer task, id_task = %d\n", task.id); - queue_tasks.defer(std::move(task)); - break; - } + slot.update_batch(batch); + } - // Erase token cache - const size_t n_erased = slot->prompt.tokens.size(); + // process in chunks of params.n_batch + int32_t n_batch = llama_n_batch(ctx_tgt); + int32_t n_ubatch = llama_n_ubatch(ctx_tgt); - slot->prompt_clear(false); + float alora_scale = -1.0f; + size_t alora_disabled_id = 0; - auto res = std::make_unique(); - res->id = task.id; - res->id_slot = id_slot; - res->n_erased = n_erased; - queue_results.send(std::move(res)); - } break; - case SERVER_TASK_TYPE_GET_LORA: - { - // TODO @ngxson : make lora_adapters a dedicated member of server_context - auto & loras = params_base.lora_adapters; - auto res = std::make_unique(); - res->id = task.id; - for (size_t i = 0; i < loras.size(); ++i) { - auto & lora = loras[i]; - std::string alora_invocation_string = ""; - const uint64_t n_alora_tokens = llama_adapter_get_alora_n_invocation_tokens(lora.ptr); - llama_tokens alora_invocation_tokens; - if (n_alora_tokens) { - const llama_token * alora_tokens = llama_adapter_get_alora_invocation_tokens(lora.ptr); - for (uint64_t j = 0; j < n_alora_tokens; ++j) { - alora_invocation_string += common_token_to_piece(vocab, alora_tokens[j]); - alora_invocation_tokens.push_back(alora_tokens[j]); - } - } - res->loras.push_back(server_task_result_get_lora::lora{ - lora, - alora_invocation_string, - alora_invocation_tokens, - }); - } - queue_results.send(std::move(res)); - } break; - case SERVER_TASK_TYPE_SET_LORA: - { - auto new_loras = construct_lora_list(task.set_lora); - // logging - for (size_t i = 0; i < new_loras.size(); ++i) { - SRV_INF("set lora adapter idx=%zu scale=%f\n", i, new_loras[i].scale); - } - // TODO @ngxson : make lora_adapters a dedicated member of server_context - params_base.lora_adapters = new_loras; - auto res = std::make_unique(); - res->id = task.id; - queue_results.send(std::move(res)); - } break; + // next, batch any pending prompts without exceeding n_batch + if (params_base.cont_batching || batch.n_tokens == 0) { + for (auto & slot : slots) { + if (!slot.is_processing()) { + continue; + } - // ── Hydra RPC state-transfer tasks (M1) ────────────────────────── - // All three cases run on the inference thread so llama API access is safe. - // The calling RPC thread blocks on queue_results.recv_with_timeout(). + // check if we can batch this slot with the previous one + if (slot_batched && !slot_batched->can_batch_with(slot)) { + continue; + } - case SERVER_TASK_TYPE_HYDRA_STATE_GET: - { - // M1: background serialization thread — inference loop continues during state transfer. - // llama_state_seq_get_data reads KV cells for an IDLE sequence; llama_decode - // writes cells for ACTIVE sequences only — no memory overlap for different seq IDs. - const int id_slot = task.hydra_action.id_slot; - auto res = std::make_unique(); - res->id = task.id; - res->id_slot = id_slot; - res->op = HYDRA_OP_STATE_GET; + // check if this is a child slot + if (slot.state == SLOT_STATE_WAIT_OTHER) { + SLT_DBG(slot, "%s", "waiting for parent slot to complete\n"); + continue; + } - server_slot * slot = get_slot_by_id(id_slot); - if (slot == nullptr) { - res->rpc_status = HYDRA_STATUS_NOT_FOUND; - res->error = "invalid slot ID"; - queue_results.send(std::move(res)); - break; - } - if (slot->is_processing() || slot->hydra_transferring->load()) { - res->rpc_status = HYDRA_STATUS_BUSY; - queue_results.send(std::move(res)); - break; - } + // this slot still has a prompt to be processed + if (slot.state == SLOT_STATE_PROCESSING_PROMPT || slot.state == SLOT_STATE_STARTED) { + const auto & input_tokens = slot.task->tokens; - // Snapshot on inference thread (cheap — dry-run serialization, no GPU copies). - const size_t state_size = llama_state_seq_get_size(ctx_tgt, slot->id); - int actual_n_past = slot->n_prompt_tokens_cache + slot->n_decoded; - // Cold prefill: n_prompt_tokens_cache is still 0 so n_decoded (1) dominates. - // Use prompt token count instead — matches STATE_META fallback. - if (slot->n_prompt_tokens_cache == 0 && slot->prompt.tokens.size() > 0) { - actual_n_past = (int)slot->prompt.tokens.size(); - } - res->n_past = actual_n_past; - res->rpc_status = HYDRA_STATUS_OK; - // M-Perf.9 #289: surface model identity alongside the state - // bytes so the Coordinator can record the model that built - // the KV (for cross-model safety on restore). The background - // thread that streams the bytes to the socket can mutate - // res->state_data freely; the model fields are immutable for - // the duration of the response. - res->model_alias = model_name; - res->model_path = params_base.model.path; - if (model_tgt) { - res->tokenizer = llama_model_get_tokenizer_model(model_tgt); - res->model_name = llama_model_get_display_name(model_tgt); - res->model_quant = llama_model_get_quant_label(model_tgt); - res->model_capabilities = llama_model_get_capabilities_bitfield(model_tgt); - } - SRV_INF("hydra: STATE_GET slot=%d n_past=%d state=%.1f MiB — async\n", - id_slot, res->n_past, state_size / (1024.0 * 1024.0)); - - slot->hydra_transferring->store(true); - - // M2: stream directly to socket (zero-copy). - // Runs SYNCHRONOUSLY on the inference thread to avoid - // concurrent ggml-RPC socket access with llama_decode - // on another slot (fixes crash at ggml-rpc.cpp:532). - // The coordinator already does Store Put as fire-and-forget - // so blocking here only delays slot release, not decode. - const int snap_seq_id = slot->id; - llama_context * snap_ctx = ctx_tgt; - // shared_ptr keeps the atomic alive even if the slot is reallocated - std::shared_ptr> flag_ptr = slot->hydra_transferring; - const int hydra_fd = task.hydra_action.hydra_fd; - - // Capture prompt tokens for M1 path header (slot is valid on inference thread) - const llama_tokens prompt_tokens_get = slot->prompt.tokens.get_text_tokens(); - const int32_t n_past_val = res->n_past; - - // Snapshot the most recent native checkpoint so STATE_PUT can - // register it instead of fabricating one at the final position. - // Fabricating at pos_max=n-1 corrupts hybrid/recurrent model - // decode because the recurrent state is one token ahead of the - // decode resume point — it has already processed the final token. - std::vector snapshot_ckpt; - uint8_t hdr_flags = 0x00; - int32_t ckpt_pos_min = 0, ckpt_pos_max = 0; - int64_t ckpt_n_tokens = 0; - if (!slot->prompt.checkpoints.empty()) { - hdr_flags |= 0x01; - const auto & ckpt = slot->prompt.checkpoints.back(); - ckpt_pos_min = ckpt.pos_min; - ckpt_pos_max = ckpt.pos_max; - ckpt_n_tokens = ckpt.n_tokens; - - // Hydra M2-stream double-write fix (#470/#620): serialize the - // recurrent-only capture (data_tgt_recr, PARTIAL_ONLY) instead of - // the full data_tgt. The full live state that follows on the wire - // already carries the attention bytes at the live position, so - // sending the full checkpoint duplicates the attention portion - // (which scales with ctx). The recurrent state is genuinely needed - // at BOTH positions, hence the separate recr-only capture. - // hdr_flags bit 0x02 marks a recurrent-only checkpoint section so - // STATE_PUT/DECODE_APPLY can read it back with matched PARTIAL_ONLY - // flags. Fall back to the full capture when the checkpoint has no - // recr buffer (e.g. it was registered from an old 0x02 blob) — a - // PARTIAL_ONLY read of a full-written buffer is a CUDA memory error. - const bool use_recr = !ckpt.data_tgt_recr.empty(); - if (use_recr) { - hdr_flags |= 0x02; + // #469 trace: log input tokens for cross-flow comparison + { + std::string tok_ids; + for (size_t i = 0; i < std::min(16, input_tokens.size()); ++i) { + if (i > 0) tok_ids += ","; + tok_ids += std::to_string(input_tokens[i]); } - const uint64_t tgt_sz = use_recr ? ckpt.data_tgt_recr.size() : ckpt.data_tgt.size(); - const uint64_t dft_sz = use_recr ? ckpt.data_dft_recr.size() : ckpt.data_dft.size(); - const uint8_t * tgt_ptr = use_recr ? ckpt.data_tgt_recr.data() : ckpt.data_tgt.data(); - const uint8_t * dft_ptr = use_recr ? ckpt.data_dft_recr.data() : ckpt.data_dft.data(); - const size_t ckpt_hdr_sz = 4 + 4 + 8 + 8 + (size_t)tgt_sz + 8 + (size_t)dft_sz; - snapshot_ckpt.resize(ckpt_hdr_sz); - size_t off = 0; - memcpy(snapshot_ckpt.data() + off, &ckpt_pos_min, 4); off += 4; - memcpy(snapshot_ckpt.data() + off, &ckpt_pos_max, 4); off += 4; - memcpy(snapshot_ckpt.data() + off, &ckpt_n_tokens, 8); off += 8; - memcpy(snapshot_ckpt.data() + off, &tgt_sz, 8); off += 8; - if (tgt_sz > 0) { memcpy(snapshot_ckpt.data() + off, tgt_ptr, (size_t)tgt_sz); off += (size_t)tgt_sz; } - memcpy(snapshot_ckpt.data() + off, &dft_sz, 8); off += 8; - if (dft_sz > 0) memcpy(snapshot_ckpt.data() + off, dft_ptr, (size_t)dft_sz); + SLT_DBG(slot, "#PD-TRACE HTTP_COMPLETION slot=%d input_tokens_first16=[%s] input_total=%zu cached=%d just_restored=%d\n", + slot.id, tok_ids.c_str(), input_tokens.size(), + slot.n_prompt_tokens_cache, slot.just_restored); } - { - SRV_INF("hydra: STATE_GET streaming (fd=%d state=%.1f MiB)\n", - hydra_fd, state_size / (1024.0 * 1024.0)); - if (hydra_fd >= 0) { - // M2 path: stream v2/v3 blob (header + checkpoint + GPU state) to fd. - // Response header + meta JSON sent first, then v2 header bytes, - // then llama_state_seq_get_data_to_fd writes GPU state directly. - const size_t n_tok = prompt_tokens_get.size(); - const uint32_t hdr_n_tok = (uint32_t)n_tok; - const uint32_t hdr_n_past = (uint32_t)n_past_val; - // 0x03 = v3 blob: checkpoint section carries recurrent-only - // captures (data_tgt_recr, PARTIAL_ONLY). 0x02 = v2 blob: - // checkpoint section carries the full data_tgt. Bumped so a - // mixed-version fleet never misreads a smaller (recr-only) - // checkpoint as a full one. - const uint8_t version_byte = 0x03; - const size_t base_hdr_size = 1 + 4 + 4 + n_tok * sizeof(llama_token) + 1; - const size_t hdr_size = base_hdr_size + snapshot_ckpt.size(); - const size_t total_payload = hdr_size + state_size; - - // Build v2 header buffer - std::vector v2_hdr(hdr_size); - { - size_t off = 0; - memcpy(v2_hdr.data() + off, &version_byte, 1); off += 1; - memcpy(v2_hdr.data() + off, &hdr_n_past, 4); off += 4; - memcpy(v2_hdr.data() + off, &hdr_n_tok, 4); off += 4; - memcpy(v2_hdr.data() + off, prompt_tokens_get.data(), n_tok * sizeof(llama_token)); off += n_tok * sizeof(llama_token); - memcpy(v2_hdr.data() + off, &hdr_flags, 1); off += 1; - if (!snapshot_ckpt.empty()) { - memcpy(v2_hdr.data() + off, snapshot_ckpt.data(), snapshot_ckpt.size()); - off += snapshot_ckpt.size(); - } - } + // used to determine the number of tokens added to the batch for the current slot + const auto n_tokens_prev = batch.n_tokens; - { - json meta_j; - meta_j["n_past"] = res->n_past; - meta_j["state_size"] = (uint64_t)state_size; - if (!res->model_alias.empty()) meta_j["model_alias"] = res->model_alias; - if (!res->model_path.empty()) meta_j["model_path"] = res->model_path; - if (!res->tokenizer.empty()) meta_j["tokenizer"] = res->tokenizer; - if (!res->model_name.empty()) meta_j["model_name"] = res->model_name; - if (!res->model_quant.empty()) meta_j["model_quant"] = res->model_quant; - if (res->model_capabilities) meta_j["model_capabilities"] = res->model_capabilities; - const std::string meta_str = meta_j.dump(); - - const uint32_t meta_len = (uint32_t)meta_str.size(); - const uint64_t payload_l = (uint64_t)total_payload; - uint8_t hdr[HYDRA_RES_HEADER_SIZE] = {}; - hdr[0] = HYDRA_STATUS_OK; - hdr[1] = (meta_len) & 0xFF; - hdr[2] = (meta_len >> 8) & 0xFF; - hdr[3] = (meta_len >> 16) & 0xFF; - memcpy(hdr + 4, &payload_l, 8); - hydra_send_all(hydra_fd, hdr, HYDRA_RES_HEADER_SIZE); - hydra_send_all(hydra_fd, meta_str.data(), meta_str.size()); - - // Write v2 blob header before GPU state — STATE_PUT needs tokens + checkpoint - hydra_send_all(hydra_fd, v2_hdr.data(), v2_hdr.size()); - - res->header_sent = true; // META + header + v2-hdr before payload - } - // Stream GPU state to fd (zero-copy from GPU memory) - const size_t streamed = llama_state_seq_get_data_to_fd(snap_ctx, snap_seq_id, hydra_fd, nullptr); - if (streamed != state_size) { - // TOCTOU: state size changed between get_size (header already - // promised state_size bytes) and the stream, or the stream - // failed mid-way. The wire framing is now broken — the only - // safe recovery is to kill the connection. Use shutdown(), - // not close(): the RPC connection loop owns the fd and will - // close it when its next read fails; closing here would race - // (double-close / fd-reuse against unrelated threads). - res->rpc_status = HYDRA_STATUS_ERROR; - res->error = "llama_state_seq_get_data_to_fd streamed " + - std::to_string(streamed) + " B, expected " + - std::to_string(state_size) + " B"; - ::shutdown(hydra_fd, SHUT_RDWR); - } else { - res->streamed_bytes = total_payload; - } - } else { - // M1 path: buffer in memory, RPC thread sends afterwards. - // v3 blob format (0x03): [1B version][4B n_past][4B n_tok][n_tok*4B tokens] - // [1B flags (bit 0 = has_checkpoint)] - // [if flags & 0x01: 4B pos_min | 4B pos_max | 8B n_tokens | 8B tgt_sz | recr_tgt_data | 8B dft_sz | recr_dft_data] - // [raw KV state from llama_state_seq_get_data] - const size_t n_tok = prompt_tokens_get.size(); - const uint32_t hdr_n_tok = (uint32_t)n_tok; - const uint32_t hdr_n_past = (uint32_t)n_past_val; - const uint8_t version_byte = 0x03; - const size_t base_hdr_size = 1 + 4 + 4 + n_tok * sizeof(llama_token) + 1; // version + n_past + n_tok + tokens + flags - const size_t hdr_size = base_hdr_size + snapshot_ckpt.size(); - - // TOCTOU retry: if another slot grew the state between - // get_size (inference thread) and get_data (background thread), - // the copy returns 0. Retry up to 3 times with fresh sizing. - size_t buf_size = hdr_size + state_size; - res->state_data.resize(buf_size); - { - size_t off = 0; - memcpy(res->state_data.data() + off, &version_byte, 1); off += 1; - memcpy(res->state_data.data() + off, &hdr_n_past, 4); off += 4; - memcpy(res->state_data.data() + off, &hdr_n_tok, 4); off += 4; - memcpy(res->state_data.data() + off, prompt_tokens_get.data(), n_tok * sizeof(llama_token)); off += n_tok * sizeof(llama_token); - memcpy(res->state_data.data() + off, &hdr_flags, 1); off += 1; - if (!snapshot_ckpt.empty()) { - memcpy(res->state_data.data() + off, snapshot_ckpt.data(), snapshot_ckpt.size()); - off += snapshot_ckpt.size(); - } - } - - size_t cur_state_size = state_size; - size_t copied = 0; - int retries = 3; - while (retries-- > 0) { - copied = llama_state_seq_get_data( - snap_ctx, res->state_data.data() + hdr_size, cur_state_size, snap_seq_id); - if (copied > 0) break; - // State grew — re-measure and retry - cur_state_size = llama_state_seq_get_size(snap_ctx, snap_seq_id); - res->state_data.resize(hdr_size + cur_state_size); - } - if (copied == 0) { - res->rpc_status = HYDRA_STATUS_ERROR; - res->error = "llama_state_get_data failed after 3 retries"; - res->state_data.clear(); - } - } - flag_ptr->store(false); - // M2 streams to fd (streamed_bytes); M1 buffers into state_data. - const uint64_t out_bytes = (hydra_fd >= 0) - ? res->streamed_bytes - : (uint64_t) res->state_data.size(); - SRV_INF("hydra: STATE_GET done slot=%d rpc_status=%d path=%s bytes=%" PRIu64 "\n", - snap_seq_id, res->rpc_status, - hydra_fd >= 0 ? "M2-stream" : "M1-buffer", out_bytes); - queue_results.send(std::move(res)); - } + // TODO: maybe move branch to outside of this loop in the future + if (slot.state == SLOT_STATE_STARTED) { + slot.t_start_process_prompt = ggml_time_us(); + slot.t_start_generation = 0; - // STATE_GET is synchronous — blocks until KV state is fully - // streamed to the socket. The coordinator's Store Put is - // fire-and-forget, so only slot release is delayed. - } break; + slot.state = SLOT_STATE_PROCESSING_PROMPT; - case SERVER_TASK_TYPE_HYDRA_STATE_PUT: - { - const int id_slot = task.hydra_action.id_slot; - auto res = std::make_unique(); - res->id = task.id; - res->id_slot = id_slot; - res->op = HYDRA_OP_STATE_PUT; + SLT_TRC(slot, "new prompt, n_ctx_slot = %d, n_keep = %d, task.n_tokens = %d\n", + slot.n_ctx, slot.task->params.n_keep, slot.task->n_tokens()); - server_slot * slot = get_slot_by_id(id_slot); - if (slot == nullptr) { - res->rpc_status = HYDRA_STATUS_NOT_FOUND; - res->error = "invalid slot ID"; - queue_results.send(std::move(res)); - break; - } - if (slot->is_processing() || slot->hydra_transferring->load()) { - res->rpc_status = HYDRA_STATUS_BUSY; - queue_results.send(std::move(res)); - break; - } + // print prompt tokens (for debugging) + /*if (1) { + // first 16 tokens (avoid flooding logs) + for (int i = 0; i < std::min(16, input_tokens.size()); i++) { + SLT_DBG(slot, "prompt token %3d: %6d '%s'\n", i, input_tokens[i], common_token_to_piece(ctx_tgt, input_tokens[i]).c_str()); + } + } else { + // all + for (int i = 0; i < (int) input_tokens.size(); i++) { + SLT_DBG(slot, "prompt token %3d: %6d '%s'\n", i, input_tokens[i], common_token_to_piece(ctx_tgt, input_tokens[i]).c_str()); + } + }*/ - // M-Perf.9 #289: populate model identity from resident model. - // model_match = true always (infrastructure only; actual KV - // validation comes when model identity is embedded in the KV header). - res->model_alias = model_name; - res->model_path = params_base.model.path; - res->model_match = true; - if (model_tgt) { - res->tokenizer = llama_model_get_tokenizer_model(model_tgt); - res->model_name = llama_model_get_display_name(model_tgt); - res->model_quant = llama_model_get_quant_label(model_tgt); - res->model_capabilities = llama_model_get_capabilities_bitfield(model_tgt); - } + // keep track how many tokens we can reuse from the previous state + int n_past = 0; - // Erase existing checkpoints to avoid collision with restored session state - if (task.hydra_action.erase_existing && !slot->prompt.checkpoints.empty()) { - SLT_INF(*slot, "erasing %zu existing checkpoints before STATE_PUT restore\n", - slot->prompt.checkpoints.size()); - slot->prompt.checkpoints.clear(); - } + // empty prompt passed -> release the slot and send empty response + if (input_tokens.empty()) { + SLT_WRN(slot, "%s", "empty prompt - releasing slot\n"); - const auto & buf = task.hydra_action.state_data; - - // Detect v2/v3 blob (0x02/0x03 at offset 0) vs legacy format (no version byte). - // v2: [1B version=0x02][4B n_past][4B n_tok][n_tok*4B tokens][1B flags][?ckpt?][KV state] - // v3: same, but the checkpoint section may be a recurrent-only capture - // (hdr_flags bit 0x02 set). Bumped to 0x03 by the M2-stream double-write fix. - const bool is_v2 = buf.size() >= 1 && (buf[0] == 0x02 || buf[0] == 0x03); - - size_t hdr_offset = 0; - int32_t hdr_n_tok = 0; - int32_t hdr_n_past = 0; - bool has_chkpt = false; - bool ckpt_is_recr_only = false; - int32_t ckpt_pos_min_in = 0, ckpt_pos_max_in = 0; - int64_t ckpt_n_tokens_in = 0; - std::vector ckpt_tgt_data, ckpt_dft_data; - - if (is_v2) { - // v2/v3: version at [0], n_past at [1..4], n_tok at [5..8] - if (buf.size() >= 9) { - memcpy(&hdr_n_past, buf.data() + 1, 4); - memcpy(&hdr_n_tok, buf.data() + 5, 4); - } - const size_t token_start = 9; - const size_t token_end = token_start + (size_t)hdr_n_tok * sizeof(llama_token); - hdr_offset = token_end; - if (hdr_offset < buf.size()) { - const uint8_t flags = buf[hdr_offset]; - hdr_offset += 1; // past flags byte - // bit 0x01 = has checkpoint; bit 0x02 = checkpoint section is - // recurrent-only (PARTIAL_ONLY). A v3 blob that fell back to the - // full capture (old-registered checkpoint) leaves 0x02 clear. - ckpt_is_recr_only = (flags & 0x02) != 0; - if (flags & 0x01) { - // Parse checkpoint: 4B pos_min | 4B pos_max | 8B n_tokens | 8B tgt_sz | tgt_data | 8B dft_sz | dft_data - if (hdr_offset + 4 + 4 + 8 + 8 <= buf.size()) { - memcpy(&ckpt_pos_min_in, buf.data() + hdr_offset, 4); hdr_offset += 4; - memcpy(&ckpt_pos_max_in, buf.data() + hdr_offset, 4); hdr_offset += 4; - memcpy(&ckpt_n_tokens_in, buf.data() + hdr_offset, 8); hdr_offset += 8; - uint64_t tgt_sz_in; - memcpy(&tgt_sz_in, buf.data() + hdr_offset, 8); hdr_offset += 8; - if (tgt_sz_in > 0 && hdr_offset + tgt_sz_in <= buf.size()) { - ckpt_tgt_data.assign(buf.data() + hdr_offset, buf.data() + hdr_offset + (size_t)tgt_sz_in); - hdr_offset += (size_t)tgt_sz_in; - } - if (hdr_offset + 8 <= buf.size()) { - uint64_t dft_sz_in; - memcpy(&dft_sz_in, buf.data() + hdr_offset, 8); hdr_offset += 8; - if (dft_sz_in > 0 && hdr_offset + dft_sz_in <= buf.size()) { - ckpt_dft_data.assign(buf.data() + hdr_offset, buf.data() + hdr_offset + (size_t)dft_sz_in); - hdr_offset += (size_t)dft_sz_in; - } - } - has_chkpt = true; - } - } - } - // Restore tokens from token_start - if (hdr_n_tok > 0 && token_start + (size_t)hdr_n_tok * sizeof(llama_token) <= buf.size()) { - slot->prompt.tokens.clear(); - const llama_token * tok_ptr = (const llama_token *)(buf.data() + token_start); - llama_tokens restored_tokens(tok_ptr, tok_ptr + (size_t)hdr_n_tok); - slot->prompt.tokens.insert(restored_tokens); - } - } else { - // Legacy v1 format - if (buf.size() >= 8) { - memcpy(&hdr_n_past, buf.data(), 4); - memcpy(&hdr_n_tok, buf.data() + 4, 4); - hdr_offset = 8 + (size_t)hdr_n_tok * sizeof(llama_token); - } - if (hdr_offset > 0 && hdr_offset <= buf.size()) { - const size_t n_tokens = (size_t)hdr_n_tok; - slot->prompt.tokens.clear(); - if (n_tokens > 0) { - const llama_token * tok_ptr = (const llama_token *)(buf.data() + 8); - llama_tokens restored_tokens(tok_ptr, tok_ptr + n_tokens); - slot->prompt.tokens.insert(restored_tokens); - } - } - } - const bool has_hdr = hdr_offset > 0 && hdr_offset <= buf.size(); - const uint8_t * state_ptr = has_hdr ? buf.data() + hdr_offset : buf.data(); - const size_t state_len = has_hdr ? buf.size() - hdr_offset : buf.size(); - const size_t n_read = llama_state_seq_set_data(ctx_tgt, state_ptr, state_len, slot->id); - if (n_read == 0) { - res->rpc_status = HYDRA_STATUS_ERROR; - res->error = "llama_state_set_data returned 0"; - // Tokens were registered before set_data — clear them so the slot - // is not left poisoned (n_past > 0 with no KV cells → pos_min == -1 - // abort on the next decode that touches this slot). - slot->prompt.tokens.clear(); - slot->prompt.checkpoints.clear(); - slot->n_prompt_tokens_cache = 0; - llama_memory_seq_rm(llama_get_memory(ctx_tgt), slot->id, -1, -1); - } else { - // D4: Inject trailing logits into per-slot buffer instead of the - // shared context-wide llama_get_logits(). This avoids the race where - // another slot's decode clobbers restored logits between STATE_PUT - // and the first sample. - const size_t remaining = state_len - n_read; - const size_t expected_logits = (size_t)llama_vocab_n_tokens(vocab) * sizeof(float); - if (remaining == expected_logits) { - const float * src = (const float *)(state_ptr + n_read); - const size_t n_floats = llama_vocab_n_tokens(vocab); - slot->restored_logits.assign(src, src + n_floats); - slot->logits_valid = true; - SRV_INF("hydra: STATE_PUT slot=%d restored %zu logits to per-slot buffer\n", - id_slot, n_floats); - } + slot.print_timings(); + send_final_response(slot); + slot.release(); - res->rpc_status = HYDRA_STATUS_OK; - res->restored = true; - res->bytes = (uint64_t)n_read; - // #469 trace: log restored state for cross-flow comparison - SRV_DBG("hydra: STATE_PUT slot=%d RESTORED n_past=%d n_prompt_tok=%d state_bytes=%zu just_restored=true\n", - id_slot, hdr_n_tok, hdr_n_tok, n_read); - { - std::string tok_ids; - for (size_t i = 0; i < std::min(16, slot->prompt.tokens.size()); ++i) { - if (i > 0) tok_ids += ","; - tok_ids += std::to_string(slot->prompt.tokens[i]); - } - SRV_DBG("hydra: STATE_PUT slot=%d first16_tokens=[%s] total=%zu\n", - id_slot, tok_ids.c_str(), slot->prompt.tokens.size()); + continue; } - if (hdr_n_tok > 0) { - slot->n_prompt_tokens_cache = hdr_n_tok; - slot->n_decoded = 0; - res->n_past = hdr_n_tok; - - // Register native checkpoint from the blob (v2) or fabricate one (legacy). - // The native checkpoint has pos_max at n-4 (created before the last - // few prompt tokens were decoded), so loading it rewinds the recurrent - // state to a clean position. The old fabricated checkpoint at (0, n-1) - // puts the recurrent state at the final position — one token ahead of - // where decode must resume — corrupting hybrid/recurrent model output. - slot->prompt.checkpoints.clear(); - if (has_chkpt) { - auto & ckpt = slot->prompt.checkpoints.emplace_back(); - ckpt.n_tokens = ckpt_n_tokens_in; - ckpt.pos_min = ckpt_pos_min_in; - ckpt.pos_max = ckpt_pos_max_in; - // New-format (v3) checkpoints carry a recurrent-only capture — - // route it into data_*_recr and tag is_recr_only so the load - // path uses matched PARTIAL_ONLY flags (plus an attention - // seq_rm at pos_max) instead of the full flags=0 restore. - ckpt.is_recr_only = ckpt_is_recr_only; - if (ckpt_is_recr_only) { - ckpt.data_tgt_recr = std::move(ckpt_tgt_data); - ckpt.data_dft_recr = std::move(ckpt_dft_data); - } else { - ckpt.data_tgt = std::move(ckpt_tgt_data); - ckpt.data_dft = std::move(ckpt_dft_data); - } - SLT_INF(*slot, "STATE_PUT registered native checkpoint (pos_min=%d pos_max=%d n_tokens=%" PRId64 " tgt_sz=%zu recr_only=%d)\n", - ckpt.pos_min, ckpt.pos_max, ckpt.n_tokens, ckpt.size(), (int) ckpt.is_recr_only); - } else { - const auto pos_min = llama_memory_seq_pos_min(llama_get_memory(ctx_tgt), slot->id); - create_checkpoint(*slot, 0, (llama_pos)pos_min, (llama_pos)(hdr_n_tok - 1)); - } - slot->just_restored = true; + + // TODO: support memory-less logits computation + if (slot.task->need_logits() && !llama_get_memory(ctx_tgt)) { + send_error(slot, "the current context does not logits computation. skipping", ERROR_TYPE_SERVER); + slot.release(); + continue; } - SRV_INF("hydra: STATE_PUT slot=%d restored=%zu B n_past=%d n_prompt_tok=%d\n", - id_slot, n_read, res->n_past, hdr_n_tok); - } - queue_results.send(std::move(res)); - } break; - case SERVER_TASK_TYPE_HYDRA_STATE_META: - { - const int id_slot = task.hydra_action.id_slot; - auto res = std::make_unique(); - res->id = task.id; - res->id_slot = id_slot; - res->op = HYDRA_OP_STATE_META; + if (!slot.can_split()) { + if (slot.task->n_tokens() > n_ubatch) { + send_error(slot, + string_format( + "input (%d tokens) is too large to process. increase the physical batch " + "size (current batch size: %d)", + slot.task->n_tokens(), n_ubatch), + ERROR_TYPE_SERVER); + slot.release(); + continue; + } - server_slot * slot = get_slot_by_id(id_slot); - if (slot == nullptr) { - res->rpc_status = HYDRA_STATUS_NOT_FOUND; - res->error = "invalid slot ID"; - queue_results.send(std::move(res)); - break; - } - // META is safe to serve even while processing or transferring (read-only metadata) - int actual_n_past = slot->n_prompt_tokens_cache + slot->n_decoded; - // For cold prefills n_prompt_tokens_cache is 0 — use prompt token count - if (slot->n_prompt_tokens_cache == 0 && slot->prompt.tokens.size() > 0) { - actual_n_past = (int)slot->prompt.tokens.size(); - } - res->n_past = actual_n_past; - res->is_processing = slot->is_processing(); - res->is_transferring = slot->hydra_transferring->load(); - res->state_size = (uint64_t)llama_state_seq_get_size(ctx_tgt, slot->id); - // M-Perf.9 #289: surface model identity. The Coordinator uses - // these to detect cross-model restores — a slot holding a Mini - // KV cache must never have it decoded by a Balanced-loaded model. - res->model_alias = model_name; - res->model_path = params_base.model.path; - if (model_tgt) { - res->tokenizer = llama_model_get_tokenizer_model(model_tgt); - res->model_name = llama_model_get_display_name(model_tgt); - res->model_quant = llama_model_get_quant_label(model_tgt); - res->model_capabilities = llama_model_get_capabilities_bitfield(model_tgt); - } - // #451: populate progress fields based on slot state - switch (slot->state) { - case SLOT_STATE_PROCESSING_PROMPT: - res->operation = "prefill"; - res->tokens_processed = slot->n_prompt_tokens_processed; - // task->n_tokens() is the total tokens to process (fixed); - // prompt.tokens.size() grows during prefill and is WRONG for total. - res->tokens_total = slot->task ? slot->task->n_tokens() : 0; - if (res->tokens_total > 0) { - res->progress = (float)res->tokens_processed / (float)res->tokens_total; + if (slot.task->n_tokens() > slot.n_ctx) { + send_error( + slot, + string_format( + "input (%d tokens) is larger than the max context size (%d tokens). skipping", + slot.task->n_tokens(), slot.n_ctx), + ERROR_TYPE_EXCEED_CONTEXT_SIZE); + slot.release(); + continue; } - res->elapsed_ms = (slot->t_start_process_prompt > 0) - ? (ggml_time_ms() - slot->t_start_process_prompt) : 0; - break; - case SLOT_STATE_GENERATING: - res->operation = "decode"; - res->tokens_processed = slot->n_decoded; - // n_remaining == -1 is the "unlimited generation" sentinel - // (no finite n_predict). Don't compute progress in that case. - if (slot->n_remaining > 0) { - res->tokens_total = slot->n_decoded + slot->n_remaining; - res->progress = (float)res->tokens_processed / (float)res->tokens_total; + } else { + if (slot.task->n_tokens() >= slot.n_ctx) { + send_error(slot, + string_format("request (%d tokens) exceeds the available context size (%d " + "tokens), try increasing it", + slot.task->n_tokens(), slot.n_ctx), + ERROR_TYPE_EXCEED_CONTEXT_SIZE); + slot.release(); + continue; } - res->elapsed_ms = (slot->t_start_generation > 0) - ? (ggml_time_ms() - slot->t_start_generation) : 0; - break; - case SLOT_STATE_IDLE: - res->operation = "idle"; - res->progress = 1.0f; - break; - default: - res->operation = "unknown"; - break; - } - // Handle save/restore operations via hydra_transferring flag. - // Clear any stale progress from the prior state since we're - // now in a transferring context, not the previous operation. - if (slot->hydra_transferring->load()) { - res->operation = "save"; - res->progress = 0.0f; - res->tokens_processed = 0; - res->tokens_total = 0; - res->elapsed_ms = 0; - } - res->rpc_status = HYDRA_STATUS_OK; - queue_results.send(std::move(res)); - } break; - - case SERVER_TASK_TYPE_HYDRA_ENGINE_CONFIGURE: - { - auto res = std::make_unique(); - res->id = task.id; - res->op = HYDRA_OP_CONFIGURE; - res->rpc_status = HYDRA_STATUS_OK; - res->success = true; - - // hydra#406: tiered CONFIGURE (T1/T2/T3). Backward compat: - // a legacy {"state_chunk_size":N} payload is treated as a - // degenerate T1 (the original hydra#334 startup call from - // WorkerSchedulerService.cs:2842). - if (task.hydra_action.config_json.empty()) { - res->tier = "T1"; - SRV_INF("hydra: CONFIGURE (empty payload, slot %d) — T1 no-op\n", - task.hydra_action.id_slot); - queue_results.send(std::move(res)); - break; - } - json cfg; - try { - cfg = json::parse(task.hydra_action.config_json); - } catch (const std::exception & e) { - res->success = false; - res->rpc_status = HYDRA_STATUS_ERROR; - res->error = std::string("CONFIGURE: invalid config_json: ") + e.what(); - SRV_WRN("hydra: CONFIGURE failed to parse config_json (slot %d): %s\n", - task.hydra_action.id_slot, e.what()); - queue_results.send(std::move(res)); - break; - } + if (slot.task->params.cache_prompt) { + // reuse any previously computed tokens that are common with the new prompt + n_past = slot.prompt.tokens.get_common_prefix(input_tokens); - // Route through the shared classify → apply helper. - // sync=false: T2/T3/T4 are staged for the slot-free moment. - hydra_config_result cfg_result = hydra_apply_config(cfg, /*sync=*/false); - - // hydra#470: report generic (T4) keys that cannot be - // applied BEFORE the tier-0 early return — a payload - // whose keys are all unrecognized/rejected still has - // to surface them (zero silent drops). - res->unrecognized_keys = cfg_result.unrecognized_keys; - res->rejected_keys = cfg_result.rejected_keys; - - if (cfg_result.highest_tier == 0) { - // No recognized keys — still emit a T1 success - // (the legacy {"state_chunk_size":N} case). - res->tier = "T1"; - SRV_INF("hydra: CONFIGURE (no recognized keys, slot %d) — T1 no-op\n", - task.hydra_action.id_slot); - queue_results.send(std::move(res)); - break; - } + // #469 trace: log common prefix for cross-flow comparison + SLT_DBG(slot, "#PD-TRACE COMMON_PREFIX slot=%d n_past=%d cached=%d input_total=%zu just_restored=%d\n", + slot.id, n_past, slot.n_prompt_tokens_cache, input_tokens.size(), slot.just_restored); + { + // Log cached tokens if any + if (slot.n_prompt_tokens_cache > 0) { + std::string cached_ids; + for (int i = 0; i < std::min(16, slot.n_prompt_tokens_cache); ++i) { + if (i > 0) cached_ids += ","; + cached_ids += std::to_string(slot.prompt.tokens[i]); + } + SLT_DBG(slot, "#PD-TRACE CACHED_TOKENS slot=%d first16=[%s] total=%d\n", + slot.id, cached_ids.c_str(), slot.n_prompt_tokens_cache); + } + // Log first mismatch point when common prefix < input size + if (n_past < (int)input_tokens.size()) { + if (n_past < (int)slot.prompt.tokens.size()) { + SLT_WRN(slot, "#PD-TRACE MISMATCH slot=%d n_past=%d cached=%d input_total=%zu stored_tok[%d]=%d input_tok[%d]=%d\n", + slot.id, n_past, slot.n_prompt_tokens_cache, input_tokens.size(), + n_past, slot.prompt.tokens[n_past], + n_past, input_tokens[n_past]); + } else { + SLT_WRN(slot, "#PD-TRACE MISMATCH slot=%d n_past=%d cached=%d input_total=%zu stored_size=%zu input exceeds stored\n", + slot.id, n_past, slot.n_prompt_tokens_cache, input_tokens.size(), + slot.prompt.tokens.size()); + } + // Log first 16 of both token lists for comparison + { + std::string stored_ids, input_ids; + for (int i = 0; i < std::min(16, (int)slot.prompt.tokens.size()); ++i) { + if (i > 0) stored_ids += ","; + stored_ids += std::to_string(slot.prompt.tokens[i]); + } + for (size_t i = 0; i < std::min(16, input_tokens.size()); ++i) { + if (i > 0) input_ids += ","; + input_ids += std::to_string(input_tokens[i]); + } + SLT_WRN(slot, "#PD-TRACE MISMATCH_STORED slot=%d first16=[%s] total=%zu\n", + slot.id, stored_ids.c_str(), slot.prompt.tokens.size()); + SLT_WRN(slot, "#PD-TRACE MISMATCH_INPUT slot=%d first16=[%s] total=%zu\n", + slot.id, input_ids.c_str(), input_tokens.size()); + } + } + } - if (!cfg_result.ok) { - res->success = false; - res->rpc_status = HYDRA_STATUS_ERROR; - res->error = "CONFIGURE: " + cfg_result.error; - SRV_WRN("hydra: CONFIGURE apply failed (slot %d): %s\n", - task.hydra_action.id_slot, cfg_result.error.c_str()); - queue_results.send(std::move(res)); - break; - } + // ── Hydra n_common decision rule ────────────── + // n_slot = tokens resident in KV (blob, prior turn, or prefill). + // n_new = tokens in the incoming prompt. + // n_common = length of common prefix (== n_past before alora). + // + // SAFETY: Every slot must produce at least one batch row to + // reach common_sampler_sample() (server-context.cpp:6591). + // The batch-assembly loop [6170] adds tokens in + // [slot.prompt.n_tokens(), n_new). When n_past == n_new + // (zero-prompt decode) that range is empty, leaving the slot + // with no batch row, no slot.i_batch, and a failed + // GGML_ASSERT(batch.n_tokens > 0) at [6240]. All branches + // therefore set n_past < n_new to ensure at least one token + // enters the batch; for the logits_reused path the restored + // logits are injected before sampling (line 6578), so the + // model-computed logits for that row are safely overwritten. + { + const int n_slot = (int) slot.prompt.tokens.size(); + const int n_new = (int) input_tokens.size(); + const int n_common_val = n_past; // before alora adjustment + const bool logits_valid = slot.logits_valid; - // Build the response from the shared helper's result. - res->tier = hydra_tier_label(cfg_result.highest_tier); - res->params_applied = std::move(cfg_result.params_applied); - res->deferred_keys = std::move(cfg_result.deferred_keys); - res->state_chunk_size_applied = cfg_result.state_chunk_size_applied; - - SRV_INF("hydra: CONFIGURE tier=%s applied=%zu deferred=%zu unrecognized=%zu rejected=%zu (slot %d)\n", - res->tier.c_str(), - res->params_applied.size(), - res->deferred_keys.size(), - res->unrecognized_keys.size(), - res->rejected_keys.size(), - task.hydra_action.id_slot); - queue_results.send(std::move(res)); - } break; + slot.n_common = n_common_val; + slot.logits_reused = false; + slot.n_prompt_processed = 0; - case SERVER_TASK_TYPE_HYDRA_ENGINE_INFO: - { - auto res = std::make_unique(); - res->id = task.id; - res->op = HYDRA_OP_INFO; - res->rpc_status = HYDRA_STATUS_OK; - // M-Perf.9 #289: advertise the model identity features so - // the Coordinator knows it can send `model` in PREFILL and - // expect model_alias/model_path/tokenizer/model_name/model_quant/model_capabilities - // in META responses. - // `preset_aliases` lists every alias loaded from - // --models-preset (empty when no preset is configured). - json preset_aliases_j = json::array(); - for (const auto & [alias, _path] : preset_alias_to_path) { - preset_aliases_j.push_back(alias); - } - // Hydra #287/#260/#348: two-engine "work together" status - // — see specs/rpc-protocol.md's ENGINE_INFO (0x41) - // contract. pipeline_capable stays false until #287's - // PIPELINE half lands; mode only ever reports - // solo/combined until then. solo_active/rpc_backend_active/ - // peer_reachable/combined_head_attached are independent - // booleans (#348) — replaces the old single "role" string - // and the peer_connected/combined_capable field-aliasing. - const int32_t expert_mode = ctx_tgt ? llama_hydra_get_expert_mode(ctx_tgt) : 0; - // Hydra #383 T1 / #375: advertise "combined" capability when this - // engine is ready to serve in COMBINED mode — either via expert-split - // (hydra_combined_head_attached) or via layer-split (hydra_combined_static). - json capabilities_j = {"prefill", "decode", "state_transfer", - "expert_mode", "quant_swap", - "preset", "tokenizer", "model_name", - "model_quant", "model_capabilities", - "merged_decode"}; - if (hydra_combined_head_attached || hydra_combined_static) { - capabilities_j.push_back("combined"); - } - // In layer-split static mode the engine is always in combined mode; - // in expert-split mode it follows the per-request SET_EXPERT_MODE state. - const std::string mode_str = hydra_combined_static ? "combined" - : (expert_mode == 1 ? "combined" : "solo"); - json info_j = { - {"engine", "llama-server-hydra"}, - {"version", "E1"}, - {"capabilities", capabilities_j}, - {"preset_aliases", preset_aliases_j}, - {"solo_active", hydra_solo_active}, - {"rpc_backend_active", hydra_rpc_backend_active}, - {"mode", mode_str}, - {"split_mode", hydra_split_mode}, - {"peer_addr", hydra_peer}, - {"peer_reachable", hydra_peer_reachable}, - {"layer_split", hydra_combined_pattern}, - {"combined_head_attached", hydra_combined_head_attached || hydra_combined_static}, - {"pipeline_capable", false} - }; - res->info_json = info_j.dump(); - queue_results.send(std::move(res)); - } break; + SLT_DBG(slot, "#PD-TRACE N_COMMON slot=%d n_slot=%d n_new=%d n_common=%d logits_valid=%d just_restored=%d\n", + slot.id, n_slot, n_new, n_common_val, (int) logits_valid, (int) slot.just_restored); - case SERVER_TASK_TYPE_HYDRA_ENGINE_PREFILL: - { - const int id_slot = task.hydra_action.id_slot; - auto res = std::make_unique(); - res->id = task.id; - res->op = HYDRA_OP_PREFILL; + if (n_common_val == n_new && logits_valid) { + // Full prompt matches resident KV and restored logits are + // valid. Re-decode the final token so the slot gets a + // batch row (required to reach common_sampler_sample); + // the restored logits are injected before sampling, + // overwriting the model-computed logits for that row. + // NOTE: n_past cannot be set to n_new here because the + // batch-assembly loop at [6170] only adds tokens in + // [slot.prompt.n_tokens(), n_new) — with n_past == n_new + // zero tokens would be added, leaving the slot with no + // batch row, no slot.i_batch, and a failed assertion at + // GGML_ASSERT(batch.n_tokens > 0) [6240]. + n_past = n_new - 1; + slot.logits_reused = true; + slot.n_prompt_processed = 1; + SLT_INF(slot, "#PD-TRACE N_COMMON zero-prompt decode n_common=%d logits_reused=true (1-token batch row)\n", n_common_val); + } else if (n_common_val == n_new && !logits_valid) { + // Full prompt matches but restored logits are stale or + // absent. Re-decode the final token to regenerate logits + // (the "1-token trick"). DEFAULT for warm/COMBINED. + n_past = n_common_val - 1; + slot.n_prompt_processed = 1; + SLT_INF(slot, "#PD-TRACE N_COMMON 1-token re-decode n_common=%d logits_reused=false\n", n_common_val); + } else if (n_common_val < n_slot) { + // Partial match — trim KV from divergence point. + // Discard restored logits FIRST (any seq_rm invalidates them). + slot.logits_valid = false; + slot.logits_reused = false; + n_past = n_common_val; + SLT_INF(slot, "#PD-TRACE N_COMMON trim n_common=%d < n_slot=%d, logits discarded\n", n_common_val, n_slot); + } else { + // Normal: n_common < n_new. Process [n_common, n_new). + n_past = n_common_val; + slot.logits_reused = false; + } + } - // #451: track timing for PREFILL metrics - const int64_t prefill_start_ms = ggml_time_ms(); + // if there is an alora invoked, don't cache after the invocation start + if (slot.alora_invocation_start > 0) { + SLT_DBG(slot, "only caching to alora invocation start (n_past = %d, alora_invocation_start = %d)\n", n_past, slot.alora_invocation_start); + n_past = std::min(n_past, slot.alora_invocation_start - 1); + } - // Set by the model-resolution block below when a real - // `load_model` swap happens. Used at the response site to - // decide whether the post-prefill model identity is the - // freshly loaded model (swap) or the original (no-swap / - // fallback). - bool model_was_swapped = false; + const auto n_cache_reuse = slot.task->params.n_cache_reuse; - server_slot * slot = get_slot_by_id(id_slot); - if (slot == nullptr) { - res->rpc_status = HYDRA_STATUS_NOT_FOUND; - res->error = "invalid slot ID"; - queue_results.send(std::move(res)); - break; - } + const bool can_cache_reuse = + llama_memory_can_shift(llama_get_memory(ctx_tgt)) && + !slot.prompt.tokens.has_mtmd; - if (slot->is_processing()) { - res->rpc_status = HYDRA_STATUS_BUSY; - res->error = "slot is busy"; - queue_results.send(std::move(res)); - break; - } + if (!can_cache_reuse && n_cache_reuse > 0) { + SLT_WRN(slot, "cache reuse is not supported - ignoring n_cache_reuse = %d\n", n_cache_reuse); + } - // M-Perf.9 #289: parse the optional `model` key from the - // request body and swap the resident model when the preset - // registry knows the alias. The parse is reused for the - // tokenization step below. Falls back to the resident model - // (with `model_fallback:true` in the response) when the - // alias is unknown or no preset is configured. - json parsed_body; - std::string requested_model; - json hydra_cfg; // optional hydra_config object - bool has_hydra_config = false; - if (!task.hydra_action.request_json.empty()) { - try { - parsed_body = json::parse(task.hydra_action.request_json); - if (parsed_body.is_object() && parsed_body.contains("model") - && parsed_body["model"].is_string()) { - requested_model = parsed_body["model"].get(); - } - // hydra_config: optional config object from Hydra.Core - // containing topology/sampling overrides. When present - // with model_path, it drives the model swap directly - // (bypassing the preset alias lookup). - if (parsed_body.is_object() && parsed_body.contains("hydra_config") - && parsed_body["hydra_config"].is_object()) { - hydra_cfg = parsed_body["hydra_config"]; - has_hydra_config = true; - } - } catch (const std::exception & e) { - res->rpc_status = HYDRA_STATUS_BAD_REQUEST; - res->error = std::string("invalid JSON: ") + e.what(); - queue_results.send(std::move(res)); - break; - } - } + // reuse chunks from the cached prompt by shifting their KV cache in the new position + if (can_cache_reuse && n_cache_reuse > 0) { + GGML_ASSERT(!slot.prompt.tokens.has_mtmd); - // Apply hydra_config synchronously when present. - // T1 keys (sampling, n_predict, etc.) are applied in-place. - // T2/T3 keys (n_ctx, cache_type, model_path, split_mode, etc.) - // trigger immediate rebuilds on this task-queue thread. - - // #470: Before applying config, probe all RPC peers for - // reconnection. If a peer restarted since the last request, - // its buffers are gone even though model/params haven't - // changed. Without this probe, the T3 rebuild in - // hydra_apply_config → apply_t3_rebuild would skip (params - // unchanged) and the subsequent graph_compute would fail. - if (ctx_tgt && ggml_backend_rpc_check_any_peer_reconnection()) { - SRV_WRN("%s", "hydra: PREFILL: RPC peer reconnected — forcing T3 rebuild\n"); - ctx_tgt->peer_reconnection_pending = true; - } + size_t head_c = n_past; // cache + size_t head_p = n_past; // current prompt - if (has_hydra_config) { - SRV_INF("hydra: PREFILL slot=%d applying hydra_config (%zu keys)\n", - id_slot, hydra_cfg.size()); - hydra_config_result cfg_result = hydra_apply_config(hydra_cfg, /*sync=*/true); - if (!cfg_result.ok) { - res->rpc_status = HYDRA_STATUS_ERROR; - res->error = "hydra_config apply failed: " + cfg_result.error; - SRV_WRN("hydra: PREFILL hydra_config apply failed (slot %d): %s\n", - id_slot, cfg_result.error.c_str()); - queue_results.send(std::move(res)); - break; - } - // If the apply may have rebuilt the slots (T3 - // statics or a T4-only generic config both route - // through apply_t3_rebuild → load_model → - // slots.clear()), track it and re-look-up the slot. - // Without the T4 case (hydra#470) the slot pointer - // captured above would dangle into the prefill. - if (hydra_config_requires_slot_relookup(cfg_result.highest_tier)) { - model_was_swapped = true; - res->model_load_ms = (double)(ggml_time_ms() - prefill_start_ms); - SRV_INF("hydra: PREFILL hydra_config T3/T4 applied model_alias='%s' tokenizer='%s' model_name='%s' quant='%s' caps=0x%x\n", - model_name.empty() ? "?" : model_name.c_str(), - model_tgt ? llama_model_get_tokenizer_model(model_tgt) : "", - model_tgt ? llama_model_get_display_name(model_tgt) : "", - model_tgt ? llama_model_get_quant_label(model_tgt) : "", - model_tgt ? llama_model_get_capabilities_bitfield(model_tgt) : 0); - slot = get_slot_by_id(id_slot); - if (slot == nullptr) { - res->rpc_status = HYDRA_STATUS_NOT_FOUND; - res->error = "slot disappeared after hydra_config T3/T4 rebuild"; - queue_results.send(std::move(res)); - break; - } - } - // When hydra_config carries model_path, the model swap is - // handled by apply_t3_rebuild() above — skip the bare - // model alias lookup below. - if (hydra_cfg.contains("model_path")) { - requested_model.clear(); - } - } + if (mctx) { + // we should never reach this + GGML_ABORT("not supported by multimodal"); + } - // Fallback: bare model alias lookup when hydra_config didn't - // handle the model swap (no hydra_config, or no model_path). - if (!requested_model.empty()) { - auto it = preset_alias_to_path.find(requested_model); - if (it == preset_alias_to_path.end()) { - SRV_WRN("hydra: PREFILL model='%s' unknown (preset has %zu alias(es)) — falling back to resident '%s'\n", - requested_model.c_str(), preset_alias_to_path.size(), - model_name.c_str()); - res->model_fallback = true; - } else if (it->second != params_base.model.path) { - SRV_INF("hydra: PREFILL model='%s' swapping %s -> %s\n", - requested_model.c_str(), params_base.model.path.c_str(), - it->second.c_str()); - common_params swapped_params = params_base; - // Apply the target alias's full preset so that - // tensor_buft_overrides, n_gpu_layers, split_mode, - // tensor_split, etc. are replaced — not inherited - // from the source model. Intentionally the FULL - // preset (sampling, chat template, n_ctx, etc. - // included), not just tensor-placement keys: a - // real model swap targets a different model, - // which plausibly needs its own sampling - // defaults/chat template too, not just a new - // memory layout. - auto pit = preset_alias_to_preset.find(requested_model); - if (pit != preset_alias_to_preset.end()) { - // Clear inherited tensor_buft_overrides (padded - // to 4096 by common_params_parse_ex) BEFORE - // apply_to_params, which push_back()'s the new - // preset's entries via CLI handlers. Without - // this, the new entries land after the - // nullptr-terminator and exceed the 4096 limit, - // triggering GGML_ASSERT in - // common_model_params_to_llama (#499 regression). - swapped_params.tensor_buft_overrides.clear(); - try { - // apply_to_params() replays CLI handlers - // (parse_tensor_buffer_overrides, the - // n-cpu-moe std::stoi, two-value option - // parsers) which throw on a malformed - // target preset. Uncaught, that exception - // would escape the task-queue loop and - // kill the task thread — fail the swap - // instead. - pit->second.apply_to_params(swapped_params); - hydra_repad_tensor_buft_overrides(swapped_params, "PREFILL swap"); - } catch (const std::exception & e) { - SRV_WRN("hydra: PREFILL swap preset apply for '%s' failed: %s\n", - requested_model.c_str(), e.what()); - res->rpc_status = HYDRA_STATUS_ERROR; - res->error = std::string("model swap preset apply failed: ") + e.what(); - queue_results.send(std::move(res)); - break; - } - SRV_INF("hydra: PREFILL swap applied preset for '%s' " - "(tensor_buft_overrides=%zu entries)\n", - requested_model.c_str(), - swapped_params.tensor_buft_overrides.size()); - } - swapped_params.model.path = it->second; - // Update the alias so model_name is re-derived - // correctly in load_model() (model_name is set from - // model_alias.first when non-empty). - swapped_params.model_alias = { requested_model }; - // #514: tear down COMBINED state before the - // reload — otherwise the engine loads the - // correct model file but keeps routing tokens - // through the stale peer/expert-binding config, - // collapsing decode throughput. - const bool was_combined = hydra_combined_head_attached || hydra_combined_static; - if (was_combined) { - hydra_teardown_combined_before_reload(); - } - const int64_t model_load_start_ms = ggml_time_ms(); - if (!load_model(swapped_params)) { - res->rpc_status = HYDRA_STATUS_ERROR; - res->error = "model swap to '" + requested_model + "' failed"; - queue_results.send(std::move(res)); - break; - } - if (was_combined) { - hydra_reattach_combined_after_reload(); - } - res->model_load_ms = (double)(ggml_time_ms() - model_load_start_ms); - model_was_swapped = true; - SRV_INF("hydra: PREFILL swap confirmed model_alias='%s' tokenizer='%s' model_name='%s' quant='%s' caps=0x%x model_load_ms=%.1f\n", - swapped_params.model_alias.empty() ? "?" : swapped_params.model_alias.begin()->c_str(), - model_tgt ? llama_model_get_tokenizer_model(model_tgt) : "", - model_tgt ? llama_model_get_display_name(model_tgt) : "", - model_tgt ? llama_model_get_quant_label(model_tgt) : "", - model_tgt ? llama_model_get_capabilities_bitfield(model_tgt) : 0, - res->model_load_ms); - // After load_model, `this` state is reset (new - // slots, new context). Re-look up the slot by id. - slot = get_slot_by_id(id_slot); - if (slot == nullptr) { - res->rpc_status = HYDRA_STATUS_NOT_FOUND; - res->error = "slot disappeared after model swap"; - queue_results.send(std::move(res)); - break; - } - } else { - SRV_DBG("hydra: PREFILL model='%s' already resident, no swap\n", - requested_model.c_str()); - } - } + SLT_DBG(slot, "trying to reuse chunks with size > %d, n_past = %d\n", n_cache_reuse, n_past); - // Tokenize from JSON messages if request_json is provided; - // otherwise fall back to pre-tokenized prompt_tokens for back-compat. - std::vector prompt_tokens = std::move(task.hydra_action.prompt_tokens); - if (!parsed_body.is_null()) { - try { - std::vector dummy_files; - json parsed = oaicompat_chat_params_parse(parsed_body, chat_params, dummy_files); - if (!parsed.contains("prompt")) { - res->rpc_status = HYDRA_STATUS_ERROR; - res->error = "chat template produced no prompt"; - queue_results.send(std::move(res)); - break; - } - auto tokenized = tokenize_input_prompts(vocab, mctx, parsed["prompt"], true, true); - if (tokenized.empty()) { - res->rpc_status = HYDRA_STATUS_ERROR; - res->error = "tokenization produced no tokens"; - queue_results.send(std::move(res)); - break; - } - prompt_tokens = tokenized[0].get_tokens(); - } catch (const std::exception & e) { - res->rpc_status = HYDRA_STATUS_ERROR; - res->error = std::string("JSON/tokenization error: ") + e.what(); - queue_results.send(std::move(res)); - break; - } - } + while (head_c < slot.prompt.tokens.size() && + head_p < input_tokens.size()) { - SRV_INF("hydra: PREFILL slot=%d tokens=%zu\n", id_slot, prompt_tokens.size()); - // #469 trace: log first 16 token IDs for cross-flow comparison - { - std::string tok_ids; - for (size_t i = 0; i < std::min(16, prompt_tokens.size()); ++i) { - if (i > 0) tok_ids += ","; - tok_ids += std::to_string(prompt_tokens[i]); - } - SRV_DBG("hydra: PREFILL slot=%d first16_tokens=[%s] total=%zu\n", - id_slot, tok_ids.c_str(), prompt_tokens.size()); - } + size_t n_match = 0; + while (head_c + n_match < slot.prompt.tokens.size() && + head_p + n_match < input_tokens.size() && + slot.prompt.tokens[head_c + n_match] == input_tokens[head_p + n_match]) { + n_match++; + } - // Clear existing slot state - slot->prompt_clear(false); - slot->n_prompt_tokens_cache = 0; - slot->n_prompt_tokens_processed = 0; - slot->n_decoded = 0; - - // Insert prompt tokens - if (prompt_tokens.empty()) { - res->rpc_status = HYDRA_STATUS_OK; - res->n_past = 0; - res->state_size = 0; - queue_results.send(std::move(res)); - break; - } - - slot->prompt.tokens.insert(prompt_tokens); - const auto & tokens = slot->prompt.tokens.get_tokens(); - const int n_tokens = (int)tokens.size(); - - // Add BOS if needed (check if slot uses BOS) - int token_offset = 0; - llama_token bos = llama_vocab_bos(vocab); - if (add_bos_token && bos != LLAMA_TOKEN_NULL && (tokens.empty() || tokens[0] != bos)) { - token_offset = 1; - } - - // Decode prompt in batches. Hydra #469 fix: upstream's own - // invariant (see create_checkpoint call in update_slots, - // "we create the checkpoint before calling llama_decode(), - // so the current batch is not yet processed and therefore - // it is not part of the checkpoint") requires the - // checkpoint to be created BEFORE the final token is - // decoded. The previous version of this handler decoded - // the whole prompt first and only afterward claimed (via - // create_checkpoint's pos_max arg, below) that the last - // token was still unprocessed. For hybrid/recurrent (SSM) - // models, whose memory can't be partially rolled back via - // seq_rm, that lie meant a cross-node restore would - // re-decode a token that was already baked into the - // recurrent state — double-applying it and corrupting the - // hidden state. Splitting the loop so the checkpoint is - // captured after n_tokens-1 tokens (matching what - // create_checkpoint's pos_max already claimed) makes the - // claim honest, same as the standard update_slots() path. - const int total_tokens = n_tokens + token_offset; - const int n_ubatch = llama_n_ubatch(ctx_tgt); - const int n_before_last = total_tokens > 1 ? total_tokens - 1 : total_tokens; - bool decode_ok = true; - for (int i = 0; i < n_before_last && decode_ok; i += n_ubatch) { - const int n_tokens_batch = std::min(n_ubatch, n_before_last - i); - common_batch_clear(batch); - for (int j = 0; j < n_tokens_batch; j++) { - const int tok_idx = i + j; - llama_token id; - if (token_offset > 0 && tok_idx == 0) { - id = bos; - } else { - id = tokens[tok_idx - token_offset]; - } - // No token in this phase is the final prompt - // token, so logits are never needed here. - common_batch_add(batch, id, tok_idx, {slot->id}, false); - } - if (llama_decode(ctx_tgt, batch) != 0) { - SRV_ERR("hydra: PREFILL slot=%d llama_decode failed at batch %d\n", id_slot, i); - decode_ok = false; - } - } - - if (!decode_ok) { - res->rpc_status = HYDRA_STATUS_ERROR; - res->error = "llama_decode failed during prefill"; - queue_results.send(std::move(res)); - break; - } - - // Register checkpoint BEFORE decoding the final token, so - // its pos_max claim (n_tokens - 1) is honest. Moved up - // from after the full-prompt decode (see #469 above). - if (n_tokens > 0) { - const auto pos_min = llama_memory_seq_pos_min(llama_get_memory(ctx_tgt), slot->id); - create_checkpoint(*slot, 0, (llama_pos)pos_min, (llama_pos)(n_tokens - 1)); - } + if (n_match >= (size_t) n_cache_reuse) { + SLT_TRC(slot, "reusing chunk with size %zu, shifting KV cache [%zu, %zu) -> [%zu, %zu)\n", n_match, head_c, head_c + n_match, head_p, head_p + n_match); + //for (size_t i = head_p; i < head_p + n_match; i++) { + // SLT_DBG(slot, "cache token %3zu: %6d '%s'\n", i, prompt_tokens[i], common_token_to_piece(ctx_tgt, prompt_tokens[i]).c_str()); + //} - // Decode the held-back final token (if any) now that the - // checkpoint has captured the state before it. - if (total_tokens > n_before_last) { - common_batch_clear(batch); - const int tok_idx = total_tokens - 1; - llama_token id = (token_offset > 0 && tok_idx == 0) - ? bos - : tokens[tok_idx - token_offset]; - common_batch_add(batch, id, tok_idx, {slot->id}, true); - if (llama_decode(ctx_tgt, batch) != 0) { - SRV_ERR("hydra: PREFILL slot=%d llama_decode failed on final token\n", id_slot); - res->rpc_status = HYDRA_STATUS_ERROR; - res->error = "llama_decode failed during prefill (final token)"; - queue_results.send(std::move(res)); - break; - } - } + const int64_t kv_shift = (int64_t) head_p - (int64_t) head_c; - // Update slot tracking - slot->n_prompt_tokens_processed = n_tokens; - slot->n_prompt_tokens_cache = n_tokens; - - // Checkpoint already registered above, before the final - // token was decoded (#469 fix). - - // Build v2/v3 header: [1B version=0x02|0x03][4B n_past][4B n_tok][n_tok*4B tokens][1B flags][?ckpt?] - // Shared by both response paths — M1 embeds it at the head of the - // buffered blob, M2 sends it before streaming the GPU state. - const uint32_t hdr_n_past = (uint32_t)n_tokens; - const uint32_t hdr_n_tok = (uint32_t)(tokens.size()); - uint8_t hdr_flags = 0x00; - std::vector ckpt_buf; - int32_t ckpt_pos_min = 0, ckpt_pos_max = 0; - int64_t ckpt_n_tokens = 0; - if (!slot->prompt.checkpoints.empty()) { - hdr_flags |= 0x01; - const auto & ckpt = slot->prompt.checkpoints.back(); - ckpt_pos_min = ckpt.pos_min; - ckpt_pos_max = ckpt.pos_max; - ckpt_n_tokens = ckpt.n_tokens; - // Hydra M2-stream double-write fix (#470/#620): serialize the - // recurrent-only capture (data_tgt_recr, PARTIAL_ONLY) instead of - // the full data_tgt. The full live state that follows on the wire - // already carries the attention bytes at the live position, so - // sending the full checkpoint duplicates the attention portion - // (which scales with ctx). The recurrent state is genuinely needed - // at BOTH positions, hence the separate recr-only capture. - // hdr_flags bit 0x02 marks a recurrent-only checkpoint section so - // STATE_PUT/DECODE_APPLY can read it back with matched PARTIAL_ONLY - // flags. Fall back to the full capture when the checkpoint has no - // recr buffer (e.g. it was registered from an old 0x02 blob) — a - // PARTIAL_ONLY read of a full-written buffer is a CUDA memory error. - const bool use_recr = !ckpt.data_tgt_recr.empty(); - if (use_recr) { - hdr_flags |= 0x02; - } - const uint64_t tgt_sz = use_recr ? ckpt.data_tgt_recr.size() : ckpt.data_tgt.size(); - const uint64_t dft_sz = use_recr ? ckpt.data_dft_recr.size() : ckpt.data_dft.size(); - const uint8_t * tgt_ptr = use_recr ? ckpt.data_tgt_recr.data() : ckpt.data_tgt.data(); - const uint8_t * dft_ptr = use_recr ? ckpt.data_dft_recr.data() : ckpt.data_dft.data(); - ckpt_buf.resize(4 + 4 + 8 + 8 + (size_t)tgt_sz + 8 + (size_t)dft_sz); - size_t off = 0; - memcpy(ckpt_buf.data() + off, &ckpt_pos_min, 4); off += 4; - memcpy(ckpt_buf.data() + off, &ckpt_pos_max, 4); off += 4; - memcpy(ckpt_buf.data() + off, &ckpt_n_tokens, 8); off += 8; - memcpy(ckpt_buf.data() + off, &tgt_sz, 8); off += 8; - if (tgt_sz > 0) { memcpy(ckpt_buf.data() + off, tgt_ptr, (size_t)tgt_sz); off += (size_t)tgt_sz; } - memcpy(ckpt_buf.data() + off, &dft_sz, 8); off += 8; - if (dft_sz > 0) memcpy(ckpt_buf.data() + off, dft_ptr, (size_t)dft_sz); - } - const size_t base_hdr_size = 1 + 4 + 4 + hdr_n_tok * sizeof(llama_token) + 1; - const size_t v2_size = base_hdr_size + ckpt_buf.size(); - std::vector v2_hdr(v2_size); - { - size_t off = 0; - // 0x03 = v3 blob: checkpoint section carries recurrent-only - // captures (data_tgt_recr, PARTIAL_ONLY). 0x02 = v2 blob: - // checkpoint section carries the full data_tgt. Bumped so a - // mixed-version fleet never misreads a smaller (recr-only) - // checkpoint as a full one. - const uint8_t version_byte = 0x03; - memcpy(v2_hdr.data() + off, &version_byte, 1); off += 1; - memcpy(v2_hdr.data() + off, &hdr_n_past, 4); off += 4; - memcpy(v2_hdr.data() + off, &hdr_n_tok, 4); off += 4; - if (hdr_n_tok > 0) { - const auto & toks = slot->prompt.tokens.get_text_tokens(); - memcpy(v2_hdr.data() + off, toks.data(), toks.size() * sizeof(llama_token)); - off += toks.size() * sizeof(llama_token); - } - memcpy(v2_hdr.data() + off, &hdr_flags, 1); off += 1; - if (!ckpt_buf.empty()) { - memcpy(v2_hdr.data() + off, ckpt_buf.data(), ckpt_buf.size()); - off += ckpt_buf.size(); - } - } + common_context_seq_rm (ctx_tgt, slot.id, head_p, head_c); + common_context_seq_add(ctx_tgt, slot.id, head_c, head_c + n_match, kv_shift); - // Get raw KV state - const size_t state_size = llama_state_seq_get_size(ctx_tgt, slot->id); - - // Snapshot logits NOW into a small buffer. ctx->logits is - // context-global: a concurrent slot decode can overwrite it - // while the M2 state stream is on the wire. Appending - // n_vocab floats gives the decode GPU the activation handoff - // (llama_state_seq_get_data saves KV but not logits), so - // STATE_PUT / DECODE_APPLY can sample immediately. - uint64_t logits_size = 0; - std::vector logits_buf; - { - const int n_vocab = llama_vocab_n_tokens(vocab); - const float * logits_ptr = llama_get_logits(ctx_tgt); - if (logits_ptr && n_vocab > 0) { - logits_size = (uint64_t)n_vocab * sizeof(float); - logits_buf.assign(reinterpret_cast(logits_ptr), - reinterpret_cast(logits_ptr) + (size_t)logits_size); - } - } + if (ctx_dft) { + common_context_seq_rm (ctx_dft.get(), slot.id, head_p, head_c); + common_context_seq_add(ctx_dft.get(), slot.id, head_c, head_c + n_match, kv_shift); + } - SRV_INF("hydra: PREFILL slot=%d done n_past=%d kv=%zu logits=%" PRIu64 "B total=%zu\n", - id_slot, n_tokens, state_size, logits_size, v2_hdr.size() + state_size + (size_t)logits_size); - - // M-Perf.9 #289: model identity for the slot the prefill - // was just built on. Coordinator uses this to populate - // item.KvModelAlias/Hash and to gate RestoreKvAsync. When - // a `model` swap happened earlier in this handler, the - // post-swap `model_name` / `params_base.model.path` / - // `model` are used. `res->model_fallback` was set by the - // model-resolution block above; we preserve it here. - res->model_alias = model_name; - res->model_path = params_base.model.path; - // res->model_fallback may already be true (alias unknown - // or no preset); only set false when no swap was needed. - if (!model_was_swapped && !res->model_fallback) { - // nothing to do — leave as-is - } - if (model_tgt) { - res->tokenizer = llama_model_get_tokenizer_model(model_tgt); - res->model_name = llama_model_get_display_name(model_tgt); - res->model_quant = llama_model_get_quant_label(model_tgt); - res->model_capabilities = llama_model_get_capabilities_bitfield(model_tgt); - } + for (size_t i = 0; i < n_match; i++) { + slot.prompt.tokens.set_token(head_p + i, slot.prompt.tokens[head_c + i]); + n_past++; + } - res->rpc_status = HYDRA_STATUS_OK; - res->n_past = n_tokens; - res->state_size = state_size; - res->logits_size = logits_size; - // #451: populate PREFILL metrics - res->prefill_ms = (double)(ggml_time_ms() - prefill_start_ms); - res->prompt_tokens = n_tokens; - res->kv_size = state_size; - if (res->prefill_ms > 0 && n_tokens > 0) { - res->tokens_per_second = (double)n_tokens / (res->prefill_ms / 1000.0); - } - res->cache_tokens = slot->n_prompt_tokens_cache; - - const int hydra_fd = task.hydra_action.hydra_fd; - if (hydra_fd >= 0) { - // M2 path (#470): stream the response straight to the - // socket — 12B header + meta JSON + v2 header, then the - // GPU KV state zero-copy (chunked via cparams.hydra_state_chunk_size), - // then the (small) logits tail. No full-blob RAM buffer: - // at 60-80K context the blob is ~800 MB and grows toward - // 10 GB; buffering it doubled engine peak memory and the - // send only started after compute + full buffer completed. - // Wire layout is byte-identical to M1: payload = - // v2_hdr + KV state + logits, payload_len = the same - // total the coordinator computes from meta. - const size_t total_payload = v2_hdr.size() + state_size + (size_t)logits_size; - - // M2 (#470): pre-compute the wire hash of the whole kv - // segment — v2 header, then [4B magic][4B seq_id] + KV - // state (hash-only pass in wire order), then the logits - // tail. The meta must carry it BEFORE the first payload - // byte goes out (the coordinator forwards it into the - // DECODE frame header, and DECODE_APPLY verifies the - // streamed restore end-to-end). The slot is exclusively - // held by this task, so the state cannot change between - // the hash pass and the stream. - XXH3_state_t * kv_hst = nullptr; - if (hydra_fd >= 0) { - kv_hst = XXH3_createState(); - XXH3_64bits_reset(kv_hst); - XXH3_64bits_update(kv_hst, v2_hdr.data(), v2_hdr.size()); - const size_t hashed = llama_state_seq_hash(ctx_tgt, slot->id, kv_hst); - // state_size (from llama_state_seq_get_size) ALREADY includes the - // [4B magic][4B seq_id] wire header — llama_io_write_dummy counts it. - // llama_state_seq_hash hashes the same [4B magic][4B seq_id] + KV - // bytes, so after the n_bytes() fix hashed == state_size exactly. - // Adding sizeof(uint32_t) + sizeof(llama_seq_id) here double-counted - // the header and killed every PREFILL M2 request (#470). - if (hashed != state_size) { - res->rpc_status = HYDRA_STATUS_ERROR; - res->error = "PREFILL M2: hash pre-pass hashed " + - std::to_string(hashed) + " B, expected " + - std::to_string(state_size) + " B"; - } - if (!logits_buf.empty()) { - XXH3_64bits_update(kv_hst, logits_buf.data(), logits_buf.size()); - } - if (res->rpc_status == HYDRA_STATUS_OK) { - const uint64_t kv_hash = XXH3_64bits_digest(kv_hst); - char hash_hex[17]; - snprintf(hash_hex, sizeof(hash_hex), "%016" PRIx64, kv_hash); - res->kv_hash_str = std::string("xxh3:") + hash_hex; - } - XXH3_freeState(kv_hst); - kv_hst = nullptr; - } + head_c += n_match; + head_p += n_match; + } else { + head_c += 1; + } + } - json meta_j = { - {"n_past", res->n_past}, - {"state_size", res->state_size}, - {"logits_size", res->logits_size} - }; - if (!res->model_alias.empty()) meta_j["model_alias"] = res->model_alias; - if (!res->model_path.empty()) meta_j["model_path"] = res->model_path; - if (!res->tokenizer.empty()) meta_j["tokenizer"] = res->tokenizer; - if (!res->model_name.empty()) meta_j["model_name"] = res->model_name; - if (!res->model_quant.empty()) meta_j["model_quant"] = res->model_quant; - if (res->model_capabilities) meta_j["model_capabilities"] = res->model_capabilities; - meta_j["model_fallback"] = res->model_fallback; - if (res->prefill_ms > 0) meta_j["prefill_ms"] = res->prefill_ms; - if (res->model_load_ms > 0) meta_j["model_load_ms"] = res->model_load_ms; - if (!res->kv_hash_str.empty()) meta_j["kv_hash_str"] = res->kv_hash_str; - const std::string meta_str = meta_j.dump(); - const uint32_t meta_len = (uint32_t)meta_str.size(); - - uint8_t hdr[HYDRA_RES_HEADER_SIZE] = {}; - hdr[0] = HYDRA_STATUS_OK; - hdr[1] = (meta_len) & 0xFF; - hdr[2] = (meta_len >> 8) & 0xFF; - hdr[3] = (meta_len >> 16) & 0xFF; - memcpy(hdr + 4, &total_payload, 8); - if (!hydra_send_all(hydra_fd, hdr, HYDRA_RES_HEADER_SIZE) || - !hydra_send_all(hydra_fd, meta_str.data(), meta_str.size()) || - !hydra_send_all(hydra_fd, v2_hdr.data(), v2_hdr.size())) { - res->rpc_status = HYDRA_STATUS_ERROR; - res->error = "PREFILL M2: response header/meta/v2-hdr send failed"; - ::shutdown(hydra_fd, SHUT_RDWR); - } else { - res->header_sent = true; // META + header + v2-hdr before payload - // Stream GPU state to fd (zero-copy from GPU memory; - // the wire hash was pre-computed above — pass no - // hash state so the io does not double-feed it) - const size_t streamed = llama_state_seq_get_data_to_fd(ctx_tgt, slot->id, hydra_fd, nullptr); - if (streamed != state_size) { - // TOCTOU: state size changed between get_size - // (above) and the stream, or the stream failed - // mid-way. The wire framing is now broken — the - // only safe recovery is to kill the connection. - // shutdown(), not close(): the RPC connection - // loop owns the fd (mirrors STATE_GET M2). - res->rpc_status = HYDRA_STATUS_ERROR; - res->error = "llama_state_seq_get_data_to_fd streamed " + - std::to_string(streamed) + " B, expected " + - std::to_string(state_size) + " B"; - ::shutdown(hydra_fd, SHUT_RDWR); + SLT_DBG(slot, "after context reuse, new n_past = %d\n", n_past); + } } else { - // Logits tail after the state stream — PREFILL's - // payload includes logits_size bytes at the end - // (STATE_GET M2 does not send logits). - if (!logits_buf.empty()) { - if (!hydra_send_all(hydra_fd, logits_buf.data(), logits_buf.size())) { - res->rpc_status = HYDRA_STATUS_ERROR; - res->error = "PREFILL M2: logits tail send failed"; - ::shutdown(hydra_fd, SHUT_RDWR); - } - } - if (res->rpc_status == HYDRA_STATUS_OK) { - res->streamed_bytes = (uint64_t)total_payload; - } - } - } - } else { - // M1 path: buffer the full blob in memory; the RPC thread - // sends header + meta + payload afterwards (unchanged). - // v2 blob format (0x02): [1B version][4B n_past][4B n_tok][n_tok*4B tokens] - // [1B flags (bit 0 = has_checkpoint)] - // [if flags & 0x01: 4B pos_min | 4B pos_max | 8B n_tokens | 8B tgt_sz | tgt_data | 8B dft_sz | dft_data] - // [raw KV state from llama_state_seq_get_data] - // [logits (n_vocab * float)] - std::vector v2_blob(v2_hdr.size() + state_size + (size_t)logits_size); - { - size_t off = 0; - memcpy(v2_blob.data() + off, v2_hdr.data(), v2_hdr.size()); - off += v2_hdr.size(); - if (state_size > 0) { - llama_state_seq_get_data(ctx_tgt, v2_blob.data() + off, state_size, slot->id); + // if we don't cache the prompt, we have to remove all previous tokens + n_past = 0; } - } - if (!logits_buf.empty()) { - memcpy(v2_blob.data() + v2_hdr.size() + state_size, logits_buf.data(), logits_buf.size()); - } - res->state_data = std::move(v2_blob); - } - // #469 trace: log PREFILL completion with token IDs for cross-flow comparison - SRV_DBG("hydra: PREFILL_DONE slot=%d n_past=%d state_size=%zu logits_size=%zu blob_size=%zu prefill_ms=%.1f\n", - id_slot, n_tokens, state_size, logits_size, - (hydra_fd >= 0) ? v2_hdr.size() + state_size + (size_t)logits_size : res->state_data.size(), - res->prefill_ms); - queue_results.send(std::move(res)); - } break; - - case SERVER_TASK_TYPE_HYDRA_ENGINE_DECODE: - { - // ── Sync phase: Gate A (header-only, no GGUF reads, ~1 ms) ── - // Identity validation, slot reservation, post DECODE_APPLY. - // No model I/O, no KV touched. - const int id_slot = task.hydra_action.id_slot; - const int32_t decode_request_id = task.hydra_action.decode_request_id; - auto res = std::make_unique(); - res->id = task.id; - res->op = HYDRA_OP_DECODE; - res->decode_request_id = decode_request_id; - res->id_slot = id_slot; - - server_slot * slot = get_slot_by_id(id_slot); - if (slot == nullptr) { - res->rpc_status = HYDRA_STATUS_NOT_FOUND; - res->error = "invalid slot ID"; - queue_results.send(std::move(res)); - break; - } - if (slot->is_processing()) { - res->rpc_status = HYDRA_STATUS_BUSY; - res->error = "slot is busy"; - queue_results.send(std::move(res)); - break; - } + llama_pos pos_next = slot.prompt.tokens.pos_next(n_past); - // Reject if slot is reserved for another decode - if (slot->reserved_for_decode_id != -1 && slot->reserved_for_decode_id != decode_request_id) { - res->rpc_status = HYDRA_STATUS_BUSY; - res->error = "slot reserved for another decode"; - queue_results.send(std::move(res)); - break; - } + // the largest pos_min required for a checkpoint to be useful + const auto pos_min_thold = std::max(0, pos_next - n_swa - 1); - // Parse the merged DECODE JSON header - json decode_req; - try { - decode_req = json::parse(task.hydra_action.decode_json); - } catch (const std::exception & e) { - res->rpc_status = HYDRA_STATUS_BAD_REQUEST; - res->error = std::string("invalid JSON: ") + e.what(); - queue_results.send(std::move(res)); - break; - } + if (n_past > 0 && n_past <= slot.prompt.n_tokens()) { + const auto pos_min = llama_memory_seq_pos_min(llama_get_memory(ctx_tgt), slot.id); + if (pos_min == -1) { + SLT_ERR(slot, "n_past = %d, slot.prompt.tokens.size() = %d, seq_id = %d, pos_min = %d\n", n_past, (int) slot.prompt.tokens.size(), slot.id, pos_min); + GGML_ABORT("pos_min == -1, but n_past > 0 - should not happen: https://github.com/ggml-org/llama.cpp/pull/13833#discussion_r2116181237"); + } - // ── Gate A: header-only metadata comparison ───────────── - // Compare kv_metadata vs model_metadata from the control - // header. No GGUF reads, no KV touched. - const json & kv_meta = decode_req["kv_metadata"]; - const json & model_meta = decode_req.value("model_metadata", json::object()); - - // Read request identities from header - const std::string req_tokenizer = kv_meta.value("tokenizer", ""); - const std::string req_model_name = kv_meta.value("model_name", ""); - const uint32_t req_capabilities = kv_meta.value("model_capabilities", 0u); - - // Read target identities from header - const std::string tgt_tokenizer = model_meta.value("tokenizer", ""); - const std::string tgt_model_name = model_meta.value("model_name", ""); - - const bool tokenizer_match = (req_tokenizer == tgt_tokenizer); - bool model_name_match = (req_model_name == tgt_model_name); - // #589: cross-node same-model name tolerance. The KV's - // model_name is the display name (GGUF metadata) of the - // file that BUILT the KV — a different build/quant of the - // same model than the decode node's resident file, so - // string equality legitimately fails for the same logical - // model (e.g. kv_metadata carries the source node's - // display name, the decode node reports its resident - // filename). When the header carries the KV's source - // alias (kv_metadata.model_alias) or the resolved request - // alias ("model") and that alias maps through the preset - // table to the resident model path, the KV was built by - // the same logical model — accept. The alias→path check - // is exact (per-node preset INI), so a different model - // (Mini vs Balanced, 27B vs 35B) still maps to a - // different path and is rejected. - if (!model_name_match) { - const std::string kv_alias = kv_meta.value("model_alias", ""); - const std::string hdr_alias = decode_req.value("model", std::string()); - for (const auto & cand : { kv_alias, hdr_alias }) { - if (cand.empty()) { - continue; - } - auto pit = preset_alias_to_path.find(cand); - if (pit != preset_alias_to_path.end() && pit->second == params_base.model.path) { - SRV_INF("hydra: DECODE slot=%d Gate A name fallback — alias '%s' maps to resident path, same logical model\n", - id_slot, cand.c_str()); - model_name_match = true; - break; - } - } - } - const uint32_t capabilities_xor = req_capabilities ^ model_meta.value("model_capabilities", 0u); + // Hydra #641: post-decode KV restore (STATE_PUT / merged DECODE) freezes the + // slot's checkpoint at PREFILL end — checkpoints are only created during prompt + // processing, and the prompt loop breaks 4+n_ubatch/4 tokens early — so on the + // NEXT continuation that stale early checkpoint matches (is_rec: pos_max <= pos_next) + // and load_tgt() overwrites the whole sequence state (attention + SSM) with the + // old snapshot, re-prefilling ~800-1400 already-cached tokens (5.5s on RTX, 51s on + // P100 for the warm-affinity turn 2). A pure extension — the whole cache is a + // strict prefix of the new prompt and memory really ends at pos_next-1 — must not + // enter the checkpoint search. Logic is pinned by + // tests/test-hydra-checkpoint-policy.cpp (see server_should_rewind_to_checkpoint). + const auto pos_max_mem = llama_memory_seq_pos_max(llama_get_memory(ctx_tgt), slot.id); + const bool no_rewind_needed = !server_should_rewind_to_checkpoint( + n_past, + (llama_pos) slot.prompt.n_tokens(), + (llama_pos) slot.task->n_tokens(), + pos_next, + pos_max_mem); - static const char * kCapBitNames[] = {"MTP", "VISION", "REASONING", "TOOL_USE", "CODE"}; - std::vector capabilities_diff_bits; - for (int b = 0; b < 5; b++) { - if (capabilities_xor & (1u << b)) { - capabilities_diff_bits.push_back(kCapBitNames[b]); - } - } + // when the prompt prefix does not match, print the tokens around the mismatch + // this is useful for debugging prompt caching + if (slots_debug) { + const int np0 = std::max(n_past - 4, 0); + const int np1 = std::min(n_past + 6, std::min(slot.prompt.tokens.size(), slot.task->tokens.size())); - // MTP(bit0) + VISION(bit1) mismatch → hard reject - const bool valid = tokenizer_match && model_name_match - && !(capabilities_xor & 0x3); - - json match_j = { - {"tokenizer_match", tokenizer_match}, - {"model_name_match", model_name_match}, - {"capabilities_xor", capabilities_xor}, - {"capabilities_diff_bits", capabilities_diff_bits}, - {"model_quant_match", kv_meta.value("model_quant", "") == model_meta.value("model_quant", "")}, - {"model_alias_match", true}, - }; - res->match_json = match_j; - res->match_valid = valid; - - if (!valid) { - res->rpc_status = HYDRA_STATUS_ERROR; - res->error = "model_capabilities_mismatch"; - SRV_WRN("hydra: DECODE slot=%d Gate A reject — tokenizer=%d name=%d caps_xor=0x%x\n", - id_slot, tokenizer_match, model_name_match, capabilities_xor); - queue_results.send(std::move(res)); - break; - } + std::stringstream ss0; + std::stringstream ss1; - // ── Reserve slot ──────────────────────────────────────── - slot->reserved_for_decode_id = decode_request_id; - - SRV_INF("hydra: DECODE slot=%d Gate A pass, reserved for request_id=%d\n", - id_slot, decode_request_id); - - // ── Create decode_result_entry (LOADING state) ───────── - // So GET /v1/decode/{id} returns 202 instead of 404 - // while async DECODE_APPLY is pending. - if (routes_ptr) { - server_routes::decode_result_entry entry; - entry.id_slot = id_slot; - entry.state = server_routes::DECODE_STATE_LOADING; - entry.match_json = match_j; - entry.created_at = std::time(nullptr); - entry.ttl_s = routes_ptr->decode_result_ttl_s; - entry.model_metadata = decode_req.value("model_metadata", json::object()); - entry.model_identity = json::object(); - std::lock_guard lock(routes_ptr->decode_results_mutex); - routes_ptr->decode_results[decode_request_id] = std::move(entry); - routes_ptr->evict_decode_results_locked(); - } + std::stringstream st0; + std::stringstream st1; - // ── Send sync validation response ─────────────────────── - res->rpc_status = HYDRA_STATUS_OK; - queue_results.send(std::move(res)); + ss0 << "old: ... "; + ss1 << "new: ... "; - // ── Post DECODE_APPLY async task ──────────────────────── - { - server_task apply_task(SERVER_TASK_TYPE_HYDRA_DECODE_APPLY); - apply_task.id = queue_tasks.get_new_id(); - apply_task.hydra_action.id_slot = id_slot; - apply_task.hydra_action.decode_json = std::move(task.hydra_action.decode_json); - apply_task.hydra_action.kv_data = std::move(task.hydra_action.kv_data); - apply_task.hydra_action.decode_request_id = decode_request_id; - queue_tasks.post(std::move(apply_task)); - SRV_INF("hydra: DECODE slot=%d posted DECODE_APPLY (request_id=%d)\n", - id_slot, decode_request_id); - } - } break; + for (int i = np0; i < np1; i++) { + if (i == n_past) { + ss0 << " | "; + ss1 << " | "; + } - case SERVER_TASK_TYPE_HYDRA_DECODE_APPLY: - { - // ── Async phase: model swap + Gate B + KV restore + completion ── - const int id_slot = task.hydra_action.id_slot; - const int32_t decode_request_id = task.hydra_action.decode_request_id; - - // Parse the DECODE JSON header (re-parsed for async context) - json decode_req; - try { - decode_req = json::parse(task.hydra_action.decode_json); - } catch (const std::exception & e) { - SRV_WRN("hydra: DECODE_APPLY slot=%d invalid JSON: %s\n", id_slot, e.what()); - // Release reservation on error - server_slot * s = get_slot_by_id(id_slot); - if (s) s->reserved_for_decode_id = -1; - if (routes_ptr) { - server_routes::decode_result_entry entry; - entry.id_slot = id_slot; - entry.error = std::string("DECODE_APPLY JSON parse error: ") + e.what(); - entry.created_at = std::time(nullptr); - entry.ttl_s = routes_ptr->decode_result_ttl_s; - std::lock_guard lock(routes_ptr->decode_results_mutex); - routes_ptr->decode_results[decode_request_id] = std::move(entry); - routes_ptr->evict_decode_results_locked(); - } - break; - } + { + const auto token = slot.prompt.tokens[i]; + const auto piece = token != LLAMA_TOKEN_NULL ? common_token_to_piece(ctx_tgt, token) : "[mtmd]"; + ss0 << piece; + st0 << std::setw(8) << token; + } - const json & kv_meta = decode_req["kv_metadata"]; - const json & model_meta = decode_req.value("model_metadata", json::object()); - - // ── Model swap (if requested model != resident) ───────── - const std::string requested_model = decode_req.value("model", std::string()); - double model_load_ms = 0.0; - bool model_fallback = false; - - if (!requested_model.empty()) { - // #470: resolve the requested alias against the - // T3-CURRENT alias → file map FIRST. The coordinator's - // T3 config (model_path) can load a file the preset INI - // does not associate with the engine's current alias - // (e.g. the dense-27b-combined session T3-loads the - // 27B-Coder file while the alias identity still says - // qwen3.6-35B-balanced). When the requested alias's - // T3-current file == resident, the alias describes the - // resident — swapping to the INI's file would be a - // pointless 73-81s reload + COMBINED teardown/reattach - // that then fails Gate B (header model_metadata of the - // pre-swap resident vs the swapped-in model's identity). - const auto t3it = t3_current_alias_to_path.find(requested_model); - if (t3it != t3_current_alias_to_path.end() && t3it->second == params_base.model.path) { - SRV_INF("hydra: DECODE_APPLY slot=%d model='%s' matches T3-current resident '%s' — no swap\n", - id_slot, requested_model.c_str(), params_base.model.path.c_str()); - } else { - auto it = preset_alias_to_path.find(requested_model); - if (it == preset_alias_to_path.end()) { - SRV_WRN("hydra: DECODE_APPLY slot=%d model='%s' unknown — falling back to resident '%s'\n", - id_slot, requested_model.c_str(), model_name.c_str()); - model_fallback = true; - } else if (it->second != params_base.model.path) { - SRV_INF("hydra: DECODE_APPLY slot=%d model='%s' swapping %s -> %s\n", - id_slot, requested_model.c_str(), params_base.model.path.c_str(), - it->second.c_str()); - common_params swapped_params = params_base; - // Apply the target alias's full preset (same - // treatment, and same intentional full-preset - // scope, as the PREFILL path above). - auto pit = preset_alias_to_preset.find(requested_model); - bool preset_apply_failed = false; - if (pit != preset_alias_to_preset.end()) { - // Same clear+re-pad+try/catch as the PREFILL path. - swapped_params.tensor_buft_overrides.clear(); - try { - pit->second.apply_to_params(swapped_params); - hydra_repad_tensor_buft_overrides(swapped_params, "DECODE_APPLY swap"); - } catch (const std::exception & e) { - SRV_WRN("hydra: DECODE_APPLY slot=%d swap preset apply for '%s' failed: %s\n", - id_slot, requested_model.c_str(), e.what()); - preset_apply_failed = true; - if (routes_ptr) { - server_routes::decode_result_entry entry; - entry.id_slot = id_slot; - entry.error = std::string("model swap preset apply failed: ") + e.what(); - entry.created_at = std::time(nullptr); - entry.ttl_s = routes_ptr->decode_result_ttl_s; - std::lock_guard lock(routes_ptr->decode_results_mutex); - routes_ptr->decode_results[decode_request_id] = std::move(entry); - routes_ptr->evict_decode_results_locked(); + { + const auto token = slot.task->tokens[i]; + const auto piece = token != LLAMA_TOKEN_NULL ? common_token_to_piece(ctx_tgt, token) : "[mtmd]"; + ss1 << piece; + st1 << std::setw(8) << token; } } - } - if (preset_apply_failed) { - server_slot * s = get_slot_by_id(id_slot); - if (s) s->reserved_for_decode_id = -1; - break; - } - swapped_params.model.path = it->second; - swapped_params.model_alias = { requested_model }; - // #514: tear down COMBINED state before the - // reload — see hydra_teardown_combined_before_reload(). - const bool was_combined = hydra_combined_head_attached || hydra_combined_static; - if (was_combined) { - hydra_teardown_combined_before_reload(); - } - const int64_t model_load_start_ms = ggml_time_ms(); - if (!load_model(swapped_params)) { - SRV_WRN("hydra: DECODE_APPLY slot=%d model swap to '%s' failed\n", - id_slot, requested_model.c_str()); - server_slot * s = get_slot_by_id(id_slot); - if (s) s->reserved_for_decode_id = -1; - if (routes_ptr) { - server_routes::decode_result_entry entry; - entry.id_slot = id_slot; - entry.error = "model swap to '" + requested_model + "' failed"; - entry.created_at = std::time(nullptr); - entry.ttl_s = routes_ptr->decode_result_ttl_s; - std::lock_guard lock(routes_ptr->decode_results_mutex); - routes_ptr->decode_results[decode_request_id] = std::move(entry); - routes_ptr->evict_decode_results_locked(); - } - break; - } - if (was_combined) { - hydra_reattach_combined_after_reload(); - } - model_load_ms = (double)(ggml_time_ms() - model_load_start_ms); - SRV_INF("hydra: DECODE_APPLY slot=%d swap confirmed model_load_ms=%.1f\n", - id_slot, model_load_ms); - } - } - } - - // ── Gate B: post-load identity check ──────────────────── - // Compare model_metadata from header vs ACTUAL resident GGUF identity. - server_slot * slot = get_slot_by_id(id_slot); - if (slot == nullptr) { - SRV_WRN("hydra: DECODE_APPLY slot=%d disappeared after model swap\n", id_slot); - if (routes_ptr) { - server_routes::decode_result_entry entry; - entry.id_slot = id_slot; - entry.error = "slot disappeared after model swap"; - entry.created_at = std::time(nullptr); - entry.ttl_s = routes_ptr->decode_result_ttl_s; - std::lock_guard lock(routes_ptr->decode_results_mutex); - routes_ptr->decode_results[decode_request_id] = std::move(entry); - routes_ptr->evict_decode_results_locked(); - } - break; - } - - const std::string resident_tokenizer = llama_model_get_tokenizer_model(model_tgt); - const std::string resident_model_name = llama_model_get_display_name(model_tgt); - const std::string resident_model_quant = llama_model_get_quant_label(model_tgt); - const uint32_t resident_capabilities = llama_model_get_capabilities_bitfield(model_tgt); - - const std::string hdr_model_name = model_meta.value("model_name", ""); - const std::string hdr_model_quant = model_meta.value("model_quant", ""); - const uint32_t hdr_capabilities = model_meta.value("model_capabilities", 0u); - - const bool gate_b_tokenizer = (resident_tokenizer == model_meta.value("tokenizer", "")); - const bool gate_b_model_name = (resident_model_name == hdr_model_name); - const uint32_t gate_b_caps_xor = resident_capabilities ^ hdr_capabilities; - - if (!gate_b_tokenizer || !gate_b_model_name || (gate_b_caps_xor & 0x3)) { - SRV_WRN("hydra: DECODE_APPLY slot=%d Gate B reject — tokenizer=%d name=%d caps_xor=0x%x\n", - id_slot, gate_b_tokenizer, gate_b_model_name, gate_b_caps_xor); - slot->reserved_for_decode_id = -1; - if (routes_ptr) { - server_routes::decode_result_entry entry; - entry.id_slot = id_slot; - entry.error = "Gate B identity mismatch after model load"; - entry.match_json = {{"gate_b_tokenizer", gate_b_tokenizer}, {"gate_b_name", gate_b_model_name}, {"gate_b_caps_xor", gate_b_caps_xor}}; - entry.created_at = std::time(nullptr); - entry.ttl_s = routes_ptr->decode_result_ttl_s; - std::lock_guard lock(routes_ptr->decode_results_mutex); - routes_ptr->decode_results[decode_request_id] = std::move(entry); - routes_ptr->evict_decode_results_locked(); - } - break; - } - if (resident_model_quant != hdr_model_quant) { - SRV_INF("hydra: DECODE_APPLY slot=%d Gate B quant differs (%s → %s) — mix-quant allowed\n", - id_slot, hdr_model_quant.c_str(), resident_model_quant.c_str()); - } + SLT_WRN(slot, "%s\n", ss0.str().c_str()); + SLT_WRN(slot, "%s\n", ss1.str().c_str()); - // ── KV restore ───────────────────────────────────────── - const int64_t restore_start_ms = ggml_time_ms(); - - // M2 (#470): the v2 header arrives pre-parsed (kv_v2_hdr, - // small) and the KV state stream is read directly off - // hydra_fd via llama_state_seq_set_data_from_fd — the engine - // never materializes the full blob (2.3 GB today, 10 GB - // target). M1 (kv_data) is the buffered fallback. - const bool m2_stream = !task.hydra_action.kv_v2_hdr.empty(); - if (!task.hydra_action.kv_data.empty() || m2_stream) { - slot->prompt_clear(false); - slot->n_prompt_tokens_cache = 0; - slot->n_prompt_tokens_processed = 0; - slot->n_decoded = 0; - - // The coordinator may send the v2/v3 blob (header + raw KV) - // or just the raw KV data. Parse the v2 header to extract - // the token list so update_slots()'s n_common decision can - // match incoming tokens against the restored KV — without - // this, prompt.tokens is empty after prompt_clear(), n_past - // computes to 0, and seq_rm(slot, 0, -1) wipes the KV that - // llama_state_seq_set_data just loaded (issue #506). - const uint8_t * kv_ptr = m2_stream - ? task.hydra_action.kv_v2_hdr.data() - : task.hydra_action.kv_data.data(); - size_t kv_len = m2_stream - ? task.hydra_action.kv_v2_hdr.size() - : task.hydra_action.kv_data.size(); - int32_t blob_n_past = 0; - int32_t blob_n_tok = 0; - bool has_chkpt = false; - bool ckpt_is_recr_only = false; - int32_t ckpt_pos_min_in = 0, ckpt_pos_max_in = 0; - int64_t ckpt_n_tokens_in = 0; - std::vector ckpt_tgt_data, ckpt_dft_data; - - // v2 (0x02) blobs carry a full checkpoint; v3 (0x03) blobs may carry - // a recurrent-only checkpoint (hdr_flags bit 0x02). Both share the - // header layout — the M2-stream double-write fix bumped the version. - const bool is_v2 = kv_len >= 1 && (kv_ptr[0] == 0x02 || kv_ptr[0] == 0x03); - if (is_v2 && kv_len >= 9) { - memcpy(&blob_n_past, kv_ptr + 1, 4); - memcpy(&blob_n_tok, kv_ptr + 5, 4); - - const size_t token_start = 9; - const size_t token_end = token_start + (size_t)blob_n_tok * sizeof(llama_token); - if (blob_n_tok > 0 && token_end <= kv_len) { - // Restore token list from v2/v3 blob header - slot->prompt.tokens.clear(); - const llama_token * tok_ptr = (const llama_token *)(kv_ptr + token_start); - llama_tokens restored_tokens(tok_ptr, tok_ptr + (size_t)blob_n_tok); - slot->prompt.tokens.insert(restored_tokens); - SRV_INF("hydra: DECODE_APPLY slot=%d v2/v3 blob: restored %d tokens from header\n", - id_slot, blob_n_tok); - } + SLT_WRN(slot, "%s\n", st0.str().c_str()); + SLT_WRN(slot, "%s\n", st1.str().c_str()); + } - // Skip past v2/v3 header (version + n_past + n_tok + tokens + flags + optional checkpoint) - size_t hdr_offset = token_end; - if (hdr_offset < kv_len) { - const uint8_t flags = kv_ptr[hdr_offset]; - hdr_offset += 1; // past flags byte - // bit 0x01 = has checkpoint; bit 0x02 = recurrent-only (PARTIAL_ONLY) - ckpt_is_recr_only = (flags & 0x02) != 0; - if (flags & 0x01) { - // Capture checkpoint: 4B pos_min | 4B pos_max | 8B n_tokens | 8B tgt_sz | tgt_data | 8B dft_sz | dft_data - // Mirrors the STATE_PUT sibling (~line 3343) — the native - // checkpoint is registered after restore so hybrid/recurrent - // models get their recurrent memory back (KV restored without - // its checkpoint is corrupt). - if (hdr_offset + 4 + 4 + 8 + 8 <= kv_len) { - memcpy(&ckpt_pos_min_in, kv_ptr + hdr_offset, 4); hdr_offset += 4; - memcpy(&ckpt_pos_max_in, kv_ptr + hdr_offset, 4); hdr_offset += 4; - memcpy(&ckpt_n_tokens_in, kv_ptr + hdr_offset, 8); hdr_offset += 8; - uint64_t tgt_sz_in; - memcpy(&tgt_sz_in, kv_ptr + hdr_offset, 8); hdr_offset += 8; - if (tgt_sz_in > 0 && hdr_offset + tgt_sz_in <= kv_len) { - ckpt_tgt_data.assign(kv_ptr + hdr_offset, kv_ptr + hdr_offset + (size_t)tgt_sz_in); - hdr_offset += (size_t)tgt_sz_in; - } - if (hdr_offset + 8 <= kv_len) { - uint64_t dft_sz_in; - memcpy(&dft_sz_in, kv_ptr + hdr_offset, 8); hdr_offset += 8; - if (dft_sz_in > 0 && hdr_offset + dft_sz_in <= kv_len) { - ckpt_dft_data.assign(kv_ptr + hdr_offset, kv_ptr + hdr_offset + (size_t)dft_sz_in); - hdr_offset += (size_t)dft_sz_in; + if (pos_min >= pos_min_thold && !no_rewind_needed) { + // For recurrent/hybrid models (e.g. Qwen3.x MTP) a checkpoint's + // pos_min equals the full sequence length, so the usual + // `pos_min < pos_min_thold` test is perpetually false → every turn + // force-re-prefills. Match on pos_max <= pos_next instead so cached + // KV is reused. Ref: ik_llama.cpp#1762 (port). + const bool is_rec = llama_model_is_recurrent(model_tgt) || + llama_model_is_hybrid(model_tgt); + // search for a context checkpoint + const auto it = std::find_if( + slot.prompt.checkpoints.rbegin(), + slot.prompt.checkpoints.rend(), + [&, func_name = __func__](const auto & cur) { + // guarantee that a checkpoint will result in at least one token being processed [TAG_PROMPT_LOGITS] + LOG_INF("slot %12.*s: id %2d | task %d | Checking checkpoint with [%d, %d] against %d...\n", 12, + func_name, (slot).id, ((slot).task ? (slot).task->id : -1), cur.pos_min, cur.pos_max, pos_min_thold); + if (is_rec) { + return cur.pos_max <= pos_next; } + return cur.pos_min < pos_min_thold || cur.pos_min == 0; } - has_chkpt = true; - } - } - } - // Advance kv_ptr/kv_len past the v2 header to the raw KV state - if (hdr_offset <= kv_len) { - kv_ptr = kv_ptr + hdr_offset; - kv_len = kv_len - hdr_offset; - } - } - - // M2 (#470): hash the whole kv segment as it streams — - // v2 header first, then every byte the fd restore - // consumes, then the logits tail (wire order). - XXH3_state_t * hst = nullptr; - if (m2_stream) { - hst = XXH3_createState(); - XXH3_64bits_reset(hst); - XXH3_64bits_update(hst, task.hydra_action.kv_v2_hdr.data(), - task.hydra_action.kv_v2_hdr.size()); - } + ); - size_t status = 0; - if (m2_stream) { - // Stream restore: consumes [4B magic][4B seq_id] + KV - // state off the fd; logits tail is read separately below. - status = llama_state_seq_set_data_from_fd( - ctx_tgt, slot->id, task.hydra_action.hydra_fd, hst); - } else { - status = llama_state_seq_set_data( - ctx_tgt, - kv_ptr, - kv_len, - slot->id); - } + bool do_reset = it == slot.prompt.checkpoints.rend(); - // llama_state_seq_set_data returns the number of bytes - // read on success (0 means failed to load) — see its - // doc comment in include/llama.h. `status` only counts - // the KV-cache bytes the reader consumed; it does NOT - // include the trailing logits PREFILL_DONE appends - // (see ~line 4098), so status < kv_len is the normal - // case whenever logits are present — compare against - // kv_len here and this false-fails on every restore - // with logits. Matches the STATE_PUT sibling check - // (server-context.cpp ~line 3395: `if (n_read == 0)`). - if (status == 0) { - SRV_WRN("hydra: DECODE_APPLY slot=%d KV restore failed (%d)\n", id_slot, status); - if (hst) { XXH3_freeState(hst); hst = nullptr; } - if (m2_stream) { - // The stream broke mid-way: drop the read side so - // residual unread bytes cannot misalign the next - // request frame. The RPC thread still writes the - // error response (write side stays open). - ::shutdown(task.hydra_action.hydra_fd, SHUT_RD); - } - slot->reserved_for_decode_id = -1; - // Tokens were registered from the v2 header before set_data — - // clear them so the slot is not left poisoned (n_past > 0 - // with no KV cells → pos_min == -1 abort on the next decode - // that touches this slot). Matches the STATE_PUT failure path. - slot->prompt.tokens.clear(); - slot->prompt.checkpoints.clear(); - slot->n_prompt_tokens_cache = 0; - llama_memory_seq_rm(llama_get_memory(ctx_tgt), slot->id, -1, -1); - if (routes_ptr) { - server_routes::decode_result_entry entry; - entry.id_slot = id_slot; - entry.error = "KV restore failed (llama_state_seq_set_data returned " + std::to_string(status) + ")"; - entry.created_at = std::time(nullptr); - entry.ttl_s = routes_ptr->decode_result_ttl_s; - std::lock_guard lock(routes_ptr->decode_results_mutex); - routes_ptr->decode_results[decode_request_id] = std::move(entry); - routes_ptr->evict_decode_results_locked(); - } - break; - } + // #469 trace: log checkpoint search result and just_restored decision + SLT_DBG(slot, "#PD-TRACE CHECKPOINT_SEARCH slot=%d do_reset=%d just_restored=%d n_past=%d checkpoints=%zu pos_next=%d\n", + slot.id, do_reset, slot.just_restored, n_past, slot.prompt.checkpoints.size(), pos_next); - // Trailing logits: PREFILL_DONE appends n_vocab floats - // after the KV state (~line 4098) so the decode side - // can sample immediately instead of reading garbage - // after restore. Mirrors STATE_PUT's per-slot - // injection (~line 3405) — DECODE_APPLY was missing - // this step entirely. - { - const size_t expected_logits = (size_t)llama_vocab_n_tokens(vocab) * sizeof(float); - if (m2_stream) { - // Read the logits tail straight off the fd (small). - const size_t remaining = - (size_t)(task.hydra_action.kv_stream_len - status); - if (remaining == expected_logits) { - std::vector logits_buf(remaining); - if (hydra_recv_all(task.hydra_action.hydra_fd, - logits_buf.data(), remaining)) { - XXH3_64bits_update(hst, logits_buf.data(), remaining); - const size_t n_floats = llama_vocab_n_tokens(vocab); - slot->restored_logits.assign( - reinterpret_cast(logits_buf.data()), - reinterpret_cast(logits_buf.data()) + n_floats); - slot->logits_valid = true; - SRV_INF("hydra: DECODE_APPLY slot=%d restored %zu logits to per-slot buffer\n", - id_slot, n_floats); - } else { - SRV_WRN("hydra: DECODE_APPLY slot=%d logits tail read failed\n", id_slot); + // For slots restored via STATE_PUT (full context state), + // skip the checkpoint search entirely. The restored state + // already has the correct KV cache + logits. The checkpoint + // check is needed for in-server reuse across turns, not for + // cross-node migration where the full state is restored. + if (do_reset && slot.just_restored && n_past > 0) { + SLT_WRN(slot, "STATE_PUT restored slot — using cached n_past=%d, skipping checkpoint check\n", n_past); + do_reset = false; + pos_next = n_past; + slot.just_restored = false; } - } - } else { - const size_t remaining = kv_len - status; - if (remaining == expected_logits) { - const float * src = (const float *)(kv_ptr + status); - const size_t n_floats = llama_vocab_n_tokens(vocab); - slot->restored_logits.assign(src, src + n_floats); - slot->logits_valid = true; - SRV_INF("hydra: DECODE_APPLY slot=%d restored %zu logits to per-slot buffer\n", - id_slot, n_floats); - } - } - } - // M2 wire-hash verification (post-restore — with streaming - // the bytes reach the GPU before a pre-restore hash could - // be computed). On mismatch the slot is cleared so the next - // decode cannot sample corrupt state, and the response - // carries the terminal error for the Coordinator to retry. - if (m2_stream && task.hydra_action.kv_expected_hash != 0) { - const uint64_t computed_kv = XXH3_64bits_digest(hst); - if (computed_kv != task.hydra_action.kv_expected_hash) { - SRV_WRN("hydra: DECODE_APPLY slot=%d SEGMENT_HASH_MISMATCH kv expected=%016" PRIx64 " got=%016" PRIx64 "\n", - id_slot, task.hydra_action.kv_expected_hash, computed_kv); - XXH3_freeState(hst); - hst = nullptr; - slot->reserved_for_decode_id = -1; - slot->prompt.tokens.clear(); - slot->prompt.checkpoints.clear(); - slot->n_prompt_tokens_cache = 0; - llama_memory_seq_rm(llama_get_memory(ctx_tgt), slot->id, -1, -1); - if (routes_ptr) { - server_routes::decode_result_entry entry; - entry.id_slot = id_slot; - entry.error = "KV segment hash mismatch (corrupt stream)"; - entry.created_at = std::time(nullptr); - entry.ttl_s = routes_ptr->decode_result_ttl_s; - std::lock_guard lock(routes_ptr->decode_results_mutex); - routes_ptr->decode_results[decode_request_id] = std::move(entry); - routes_ptr->evict_decode_results_locked(); - } - break; - } - SRV_INF("hydra: DECODE_APPLY slot=%d KV hash verified (%zu + %" PRIu64 " B)\n", - id_slot, task.hydra_action.kv_v2_hdr.size(), - task.hydra_action.kv_stream_len); - } - if (hst) { XXH3_freeState(hst); hst = nullptr; } - - const int n_past = is_v2 ? blob_n_past : kv_meta.value("n_past", 0); - if (n_past > 0) { - // Cache/processed counters come from the same header field - // STATE_PUT reads (hdr_n_tok == blob_n_tok here); PREFILL writes - // both fields as n_tokens so the values are identical today, - // but the two restore paths must read the SAME source. - slot->n_prompt_tokens_cache = is_v2 ? blob_n_tok : n_past; - slot->n_prompt_tokens_processed = is_v2 ? blob_n_tok : n_past; - - // Register the native checkpoint from the blob (v2) or - // fabricate one (legacy) — mirrors STATE_PUT (~line 3447). - // KV restored without its recurrent-memory checkpoint - // corrupts hybrid/recurrent model output. - slot->prompt.checkpoints.clear(); - if (has_chkpt) { - auto & ckpt = slot->prompt.checkpoints.emplace_back(); - ckpt.n_tokens = ckpt_n_tokens_in; - ckpt.pos_min = ckpt_pos_min_in; - ckpt.pos_max = ckpt_pos_max_in; - // New-format (v3) checkpoints carry a recurrent-only capture — - // route into data_*_recr and tag is_recr_only so the load path - // uses matched PARTIAL_ONLY flags (mirrors STATE_PUT). - ckpt.is_recr_only = ckpt_is_recr_only; - if (ckpt_is_recr_only) { - ckpt.data_tgt_recr = std::move(ckpt_tgt_data); - ckpt.data_dft_recr = std::move(ckpt_dft_data); - } else { - ckpt.data_tgt = std::move(ckpt_tgt_data); - ckpt.data_dft = std::move(ckpt_dft_data); - } - SLT_INF(*slot, "DECODE_APPLY registered native checkpoint (pos_min=%d pos_max=%d n_tokens=%" PRId64 " tgt_sz=%zu recr_only=%d)\n", - ckpt.pos_min, ckpt.pos_max, ckpt.n_tokens, ckpt.size(), (int) ckpt.is_recr_only); - } else { - const auto pos_min = llama_memory_seq_pos_min(llama_get_memory(ctx_tgt), slot->id); - create_checkpoint(*slot, 0, (llama_pos)pos_min, (llama_pos)(n_past - 1)); - } - } - slot->just_restored = true; - } - - const double restore_slot_ms = (double)(ggml_time_ms() - restore_start_ms); - const int n_past = slot->n_prompt_tokens_cache + slot->n_decoded; + if (!do_reset) { + if (it != slot.prompt.checkpoints.rend()) { + // restore the context checkpoint + if (it->is_recr_only) { + // Hydra M2 wire checkpoint (v3): the buffer holds a + // recurrent-only (PARTIAL_ONLY) capture. The recurrent + // (SSM) state is restored with matched PARTIAL_ONLY flags + // (flag symmetry — a PARTIAL_ONLY buffer physically lacks + // the mem_attn bytes). The attention cache must be trimmed + // at pos_max first: after the full live-state restore it + // still holds cells past the checkpoint position, and a + // full restore has no attention bytes to overwrite them. + // seq_rm is called on the ATTENTION cache directly — the + // blanket llama_memory_hybrid::seq_rm would wipe the + // recurrent cell first (n_rs_seq == 0 rollback path). + if (llama_model_is_hybrid(model_tgt)) { + ((llama_memory_hybrid *) llama_get_memory(ctx_tgt))->get_mem_attn()->seq_rm(slot.id, it->pos_max, -1); + } + it->load_tgt_recr(ctx_tgt, slot.id); - SRV_INF("hydra: DECODE_APPLY slot=%d restore=%.1fms n_past=%d model_load_ms=%.1f\n", - id_slot, restore_slot_ms, n_past, model_load_ms); + // Mirror for the draft (MTP) context when enabled. + if (ctx_dft && llama_model_is_hybrid(model_tgt)) { + ((llama_memory_hybrid *) llama_get_memory(ctx_dft.get()))->get_mem_attn()->seq_rm(slot.id, it->pos_max, -1); + } + it->load_dft_recr(ctx_dft.get(), slot.id); + } else { + it->load_tgt(ctx_tgt, slot.id, 0); + it->load_dft(ctx_dft.get(), slot.id, 0); + } - // Release reservation — slot is now processing via completion - slot->reserved_for_decode_id = -1; + pos_next = std::min(pos_next, std::max(it->pos_min + 1, it->pos_max)); + n_past = std::min(slot.prompt.tokens.size_up_to_pos(pos_next), (size_t) it->n_tokens); + SLT_WRN(slot, "restored context checkpoint (pos_min = %d, pos_max = %d, n_tokens = %" PRId64 ", n_past = %d, size = %.3f MiB)\n", it->pos_min, it->pos_max, it->n_tokens, n_past, (float) it->size() / 1024 / 1024); + // One-shot: STATE_PUT flag consumed on first successful match + slot.just_restored = false; + } + // else: just_restored override — KV already in place via STATE_PUT, + // pos_next/n_past set above; no checkpoint to load from iterator. + } - // ── Build and post COMPLETION task ────────────────────── - { - json prompt = decode_req["prompt"]; - json cmpl_data; - cmpl_data["stream"] = prompt.value("stream", false); - // #622: the DECODE 0x43 frame has no dedicated stream_options - // channel, but the coordinator always requests usage on the - // merged path (it injects stream_options.include_usage=true on - // its HTTP body). Honor stream_options when the request carries - // it (generation header / prompt segment), otherwise mirror the - // coordinator's injection so the DONE-SSE delta carries usage - // natively and the coordinator's usage-based gate fires. - if (prompt.contains("stream_options") && prompt["stream_options"].is_object()) { - cmpl_data["stream_options"] = prompt["stream_options"]; - } else { - cmpl_data["stream_options"] = json{{"include_usage", true}}; - } - cmpl_data["n_predict"] = prompt.value("n_predict", 256); - cmpl_data["id_slot"] = id_slot; - if (prompt.contains("sampling")) { - const json & samp = prompt["sampling"]; - if (samp.contains("temperature")) cmpl_data["temperature"] = samp["temperature"]; - if (samp.contains("top_p")) cmpl_data["top_p"] = samp["top_p"]; - if (samp.contains("top_k")) cmpl_data["top_k"] = samp["top_k"]; - if (samp.contains("seed")) cmpl_data["seed"] = samp["seed"]; - } - if (prompt.contains("stop")) cmpl_data["stop"] = prompt["stop"]; - - std::string prompt_str; - if (prompt.contains("messages") && !prompt["messages"].is_null()) { - json chat_body; - chat_body["messages"] = prompt["messages"]; - if (prompt.contains("tools")) chat_body["tools"] = prompt["tools"]; - if (prompt.contains("tool_choice")) chat_body["tool_choice"] = prompt["tool_choice"]; - if (prompt.contains("response_format")) chat_body["response_format"] = prompt["response_format"]; - if (prompt.contains("add_generation_prompt")) chat_body["add_generation_prompt"] = prompt["add_generation_prompt"]; - if (prompt.contains("continue_final_message")) chat_body["continue_final_message"] = prompt["continue_final_message"]; - if (prompt.contains("reasoning_format")) chat_body["reasoning_format"] = prompt["reasoning_format"]; - if (prompt.contains("enable_thinking")) chat_body["enable_thinking"] = prompt["enable_thinking"]; - if (prompt.contains("chat_template_kwargs")) chat_body["chat_template_kwargs"] = prompt["chat_template_kwargs"]; - - try { - std::vector dummy_files; - json chat_result = oaicompat_chat_params_parse(chat_body, chat_params, dummy_files); - prompt_str = chat_result.value("prompt", std::string()); - if (chat_result.contains("grammar") && !chat_result["grammar"].is_null()) cmpl_data["grammar"] = chat_result["grammar"]; - if (chat_result.contains("grammar_type")) cmpl_data["grammar_type"] = chat_result["grammar_type"]; - if (chat_result.contains("grammar_lazy")) cmpl_data["grammar_lazy"] = chat_result["grammar_lazy"]; - if (chat_result.contains("grammar_triggers")) cmpl_data["grammar_triggers"] = chat_result["grammar_triggers"]; - if (chat_result.contains("chat_format")) cmpl_data["chat_format"] = chat_result["chat_format"]; - if (chat_result.contains("chat_parser")) cmpl_data["chat_parser"] = chat_result["chat_parser"]; - if (chat_result.contains("generation_prompt")) cmpl_data["generation_prompt"] = chat_result["generation_prompt"]; - if (chat_result.contains("parse_tool_calls")) cmpl_data["parse_tool_calls"] = chat_result["parse_tool_calls"]; - if (chat_result.contains("preserved_tokens")) cmpl_data["preserved_tokens"] = chat_result["preserved_tokens"]; - if (chat_result.contains("reasoning_budget_tokens")) cmpl_data["reasoning_budget_tokens"] = chat_result["reasoning_budget_tokens"]; - if (chat_result.contains("reasoning_budget_start_tag")) cmpl_data["reasoning_budget_start_tag"] = chat_result["reasoning_budget_start_tag"]; - if (chat_result.contains("reasoning_budget_end_tag")) cmpl_data["reasoning_budget_end_tag"] = chat_result["reasoning_budget_end_tag"]; - if (chat_result.contains("reasoning_budget_message")) cmpl_data["reasoning_budget_message"] = chat_result["reasoning_budget_message"]; - if (chat_result.contains("reasoning_control")) cmpl_data["reasoning_control"] = chat_result["reasoning_control"]; - if (chat_result.contains("stop") && chat_result["stop"].is_array()) { - json existing_stops = cmpl_data.value("stop", json::array()); - for (const auto & s : chat_result["stop"]) existing_stops.push_back(s); - cmpl_data["stop"] = existing_stops; - } - } catch (const std::exception & e) { - SRV_WRN("hydra: DECODE_APPLY slot=%d chat template failed: %s\n", id_slot, e.what()); - if (routes_ptr) { - server_routes::decode_result_entry entry; - entry.id_slot = id_slot; - entry.error = std::string("chat template error: ") + e.what(); - entry.created_at = std::time(nullptr); - entry.ttl_s = routes_ptr->decode_result_ttl_s; - std::lock_guard lock(routes_ptr->decode_results_mutex); - routes_ptr->decode_results[decode_request_id] = std::move(entry); - routes_ptr->evict_decode_results_locked(); - } - break; - } - } else { - prompt_str = prompt.value("prompt", std::string()); - } - cmpl_data["prompt"] = prompt_str; - - auto inputs = tokenize_input_prompts(vocab, mctx, prompt_str, true, true); - if (!inputs.empty()) { - const int32_t completion_id = queue_tasks.get_new_id(); - - server_task cmpl_task(SERVER_TASK_TYPE_COMPLETION); - cmpl_task.id = completion_id; - cmpl_task.id_slot = id_slot; - cmpl_task.tokens = std::move(inputs[0]); - cmpl_task.params = server_task::params_from_json_cmpl( - vocab, params_base, get_slot_n_ctx(), params_base.sampling.logit_bias_eog, cmpl_data); - cmpl_task.params.res_type = TASK_RESPONSE_TYPE_OAI_CHAT; - cmpl_task.params.oaicompat_cmpl_id = gen_chatcmplid(); - cmpl_task.params.oaicompat_model = model_name; - - // Mirror server_response_reader::post_task(): the - // consumer thread keeps its own result state so it - // can run result->update() per received result. - task_result_state cmpl_state = cmpl_task.create_state(); - - queue_results.add_waiting_task_id(completion_id); - queue_tasks.post(std::move(cmpl_task)); - SRV_INF("hydra: DECODE_APPLY slot=%d posted COMPLETION (completion_id=%d, request_id=%d)\n", - id_slot, completion_id, decode_request_id); - - // Update state to GENERATING - if (routes_ptr) { - std::lock_guard lk(routes_ptr->decode_results_mutex); - auto dit = routes_ptr->decode_results.find(decode_request_id); - if (dit != routes_ptr->decode_results.end()) { - dit->second.state = server_routes::DECODE_STATE_GENERATING; - dit->second.completion_id = std::to_string(completion_id); - dit->second.stream->completion_task_id = completion_id; - // Capture n_common observability from the slot - dit->second.n_common = slot->n_common; - dit->second.n_prompt_processed = slot->n_prompt_processed; - dit->second.logits_reused = slot->logits_reused; + if (do_reset) { + SLT_WRN(slot, "forcing full prompt re-processing due to lack of cache data (likely due to SWA or hybrid/recurrent memory, see %s)\n", + "https://github.com/ggml-org/llama.cpp/pull/13194#issuecomment-2868343055"); + pos_next = 0; + n_past = 0; + } + } else if (pos_min >= pos_min_thold) { + // #641: pure extension — the whole cached sequence is a strict prefix of + // the new prompt and memory ends exactly at pos_next - 1, so no rewind is + // needed and the stale PREFILL-end checkpoint must not be loaded on top of + // the restored state (which would re-prefill already-cached tokens). + SLT_INF(slot, "no rewind needed — memory at pos_next-1 (n_past = %d, prompt = %d, task = %d, pos_next = %d, pos_max_mem = %d); skipping checkpoint search\n", + n_past, (int) slot.prompt.n_tokens(), (int) slot.task->n_tokens(), (int) pos_next, (int) pos_max_mem); + // consume the one-shot STATE_PUT flag so it can't leak into a later turn + slot.just_restored = false; } } - // ── Background consumer ───────────────────────── - // Sole listener on the completion task. Relays - // partial results into the decode_result_entry's - // streaming_queue so GET /v1/decode can stream - // them to the client. Stores the final result - // when generation completes. - if (routes_ptr) { - // Read match_json from the decode_result_entry (set by sync DECODE) - json match_j_bg; - { - std::lock_guard lk(routes_ptr->decode_results_mutex); - auto dit = routes_ptr->decode_results.find(decode_request_id); - if (dit != routes_ptr->decode_results.end()) { - match_j_bg = dit->second.match_json; + { + // erase any checkpoints with pos_max > pos_next + for (auto it = slot.prompt.checkpoints.begin(); it != slot.prompt.checkpoints.end();) { + const auto & cur = *it; + if (cur.pos_max > pos_next) { + SLT_WRN(slot, "erased invalidated context checkpoint (pos_min = %d, pos_max = %d, n_tokens = %" PRId64 ", n_swa = %d, pos_next = %d, size = %.3f MiB)\n", cur.pos_min, cur.pos_max, cur.n_tokens, n_swa, pos_next, (float) cur.size() / 1024 / 1024); + it = slot.prompt.checkpoints.erase(it); + } else { + ++it; } } - std::thread([this, completion_id, decode_request_id, id_slot, - match_j = std::move(match_j_bg), resident_tokenizer, resident_model_name, - resident_model_quant, resident_capabilities, - oaicompat_model_name = model_name, - model_load_ms, restore_slot_ms, n_past, - &results = queue_results, - states = std::vector{ std::move(cmpl_state) }]() mutable { - std::unordered_set ids = {(int)completion_id}; - bool got_final = false; - - // Loop: receive partials and relay, wait for final - while (!got_final) { - auto res_ptr = results.recv_with_timeout(ids, 120); - if (!res_ptr) { - SRV_WRN("hydra: DECODE_APPLY slot=%d generation timeout (request_id=%d, completion_id=%d)\n", - id_slot, decode_request_id, completion_id); - // Mark stream as finished so GET handler unblocks - { - std::lock_guard lk(routes_ptr->decode_results_mutex); - auto dit = routes_ptr->decode_results.find(decode_request_id); - if (dit != routes_ptr->decode_results.end() && dit->second.stream) { - std::lock_guard slk(dit->second.stream->streaming_mutex); - dit->second.stream->stream_finished = true; - dit->second.stream->streaming_cv.notify_all(); - } - } - return; - } - - // Check if this is a partial or final result - auto * partial = dynamic_cast(res_ptr.get()); - auto * final_r = dynamic_cast(res_ptr.get()); - - // Mirror server_response_reader::next(): run - // update() on every result before handling. - // Populates oaicompat_msg / oaicompat_msg_diffs - // (and sets is_updated, so to_json() won't - // assert on relayed partials). - try { - const size_t idx = res_ptr->index; - GGML_ASSERT(idx < states.size()); - res_ptr->update(states[idx]); - } catch (const std::exception & e) { - // Mirror the standard stream loop's tolerance - // of chat-parse failures (server-context.cpp:7685). - // This is a detached thread: an uncaught - // exception would std::terminate() the whole - // engine. Continue with the unparsed result - // (raw content; reasoning extraction skipped). - SRV_WRN("hydra: DECODE_APPLY slot=%d result update() failed: %s (continuing with unparsed result)\n", - id_slot, e.what()); - if (partial && !partial->is_begin) { - // Keep the relay well-formed: partial - // to_json() asserts is_updated in debug - // builds; with no diffs it emits an empty - // delta, which clients merge harmlessly. - partial->is_updated = true; - } - } + } + } - if (partial && !partial->is_begin) { - // Relay partial to streaming queue - std::lock_guard lk(routes_ptr->decode_results_mutex); - auto dit = routes_ptr->decode_results.find(decode_request_id); - if (dit != routes_ptr->decode_results.end() && dit->second.stream) { - std::lock_guard slk(dit->second.stream->streaming_mutex); - dit->second.stream->streaming_queue.push_back(std::move(res_ptr)); - dit->second.stream->streaming_cv.notify_all(); - } - } else if (final_r) { - // Store final result and mark DONE - got_final = true; - - server_routes::decode_result_entry entry; - entry.id_slot = id_slot; - entry.completion_id = final_r->oaicompat_cmpl_id; - entry.oaicompat_model = oaicompat_model_name; - entry.content = final_r->content; - if (!final_r->oaicompat_msg.reasoning_content.empty()) { - entry.reasoning_content = final_r->oaicompat_msg.reasoning_content; - } - if (!final_r->oaicompat_msg.tool_calls.empty()) { - // Mirror common_chat_msg::to_json_oaicompat() shape so - // GET /v1/decode/:id returns OpenAI-format tool_calls. - json jtool_calls = json::array(); - for (const auto & tool_call : final_r->oaicompat_msg.tool_calls) { - json tc { - {"type", "function"}, - {"function", { - {"name", tool_call.name}, - {"arguments", json(tool_call.arguments)}, - }}, - }; - if (!tool_call.id.empty()) { - tc["id"] = tool_call.id; - } - jtool_calls.push_back(std::move(tc)); - } - entry.tool_calls = std::move(jtool_calls); - } - entry.n_decoded = final_r->n_decoded; - entry.n_prompt_tokens = final_r->n_prompt_tokens; - entry.n_prompt_tokens_cache = final_r->n_prompt_tokens_cache; - entry.timings = final_r->timings; - entry.stop = final_r->stop; - entry.include_usage = final_r->include_usage; - entry.match_json = match_j; - entry.created_at = std::time(nullptr); - entry.ttl_s = routes_ptr->decode_result_ttl_s; - - json metrics = json::object(); - metrics["decode_request_id"] = decode_request_id; - metrics["id_slot"] = id_slot; - metrics["n_past"] = final_r->n_prompt_tokens_cache + final_r->n_decoded; - metrics["decode_ms"] = final_r->timings.predicted_ms; - metrics["prompt_ms"] = final_r->timings.prompt_ms; - metrics["model_load_ms"] = model_load_ms; - metrics["restore_slot_ms"] = restore_slot_ms; - metrics["model_identity"] = { - {"tokenizer", resident_tokenizer}, - {"model_name", resident_model_name}, - {"model_quant", resident_model_quant}, - {"model_capabilities", resident_capabilities} - }; - metrics["match"] = match_j; - metrics["model_fallback"] = false; - // Hydra n_common observability - metrics["n_common"] = entry.n_common; - metrics["n_prompt_processed"] = entry.n_prompt_processed; - metrics["logits_reused"] = entry.logits_reused; - entry.hydra_metrics = metrics; - entry.state = server_routes::DECODE_STATE_DONE; - - // Signal stream finished before storing entry - { - std::lock_guard lk(routes_ptr->decode_results_mutex); - auto dit = routes_ptr->decode_results.find(decode_request_id); - if (dit != routes_ptr->decode_results.end() && dit->second.stream) { - // Transfer streaming state to the new entry - entry.stream = std::move(dit->second.stream); - { - std::lock_guard slk(entry.stream->streaming_mutex); - entry.stream->stream_finished = true; - } - entry.stream->streaming_cv.notify_all(); - } - } + // [TAG_PROMPT_LOGITS] + // When logits_reused is true (zero-prompt decode from restored + // logits), skip this guard — the entire prompt is cached. + if (!slot.logits_reused && n_past == slot.task->n_tokens() && n_past > 0) { + SLT_WRN(slot, "need to evaluate at least 1 token for each active slot (n_past = %d, task.n_tokens() = %d)\n", n_past, slot.task->n_tokens()); + n_past--; + SLT_WRN(slot, "n_past was set to %d\n", n_past); + } - std::lock_guard lock(routes_ptr->decode_results_mutex); - routes_ptr->decode_results[decode_request_id] = std::move(entry); - routes_ptr->evict_decode_results_locked(); + slot.n_prompt_tokens_cache = n_past; + slot.n_prompt_tokens_processed = 0; - SRV_INF("hydra: DECODE_APPLY slot=%d generation complete (request_id=%d, n_decoded=%d)\n", - id_slot, decode_request_id, final_r->n_decoded); - } else { - // is_begin partial — just consume it - } - } + slot.prompt.tokens.keep_first(n_past); - results.remove_waiting_task_id(completion_id); - }).detach(); + // this is to signal the client that the request has started processing + if (slot.task->params.stream) { + if (slot.task->params.return_progress) { + // send initial 0% progress update if needed + send_partial_response(slot, {}, true); + } else { + // otherwise, for streaming without progress, signal HTTP to send the headers (i.e. 200 status) + send_partial_response(slot, {}, false, true); } - } else { - SRV_WRN("hydra: DECODE_APPLY slot=%d tokenization failed\n", id_slot); } } - } break; - case SERVER_TASK_TYPE_HYDRA_ENGINE_SET_EXPERT_MODE: - { - auto res = std::make_unique(); - res->id = task.id; - res->op = HYDRA_OP_SET_EXPERT_MODE; - - // Parse the payload. For backward compatibility, a raw string - // ("solo" or "combined") is accepted. Phase D (C# side) sends - // a JSON payload: {"mode":"combined","peer":"host:port",...}. - std::string requested; - std::string peer_override; - const std::string & raw = task.hydra_action.expert_mode; - if (!raw.empty() && raw[0] == '{') { - try { - json j = json::parse(raw); - requested = j.value("mode", "solo"); - peer_override = j.value("peer", ""); - } catch (...) { - requested = "solo"; + if (!slot.can_split()) { + // cannot fit the prompt in the current batch - will try next iter + if (batch.n_tokens + slot.task->n_tokens() > n_batch) { + continue; } - } else { - requested = raw; } - if (requested != "solo" && requested != "combined") { - res->rpc_status = HYDRA_STATUS_ERROR; - res->success = false; - res->error = "expert_mode must be 'solo' or 'combined'"; - queue_results.send(std::move(res)); - break; + const int64_t t_current = ggml_time_us(); + slot.t_prompt_processing = (t_current - slot.t_start_process_prompt) / 1e3; + slot.print_timings_pp(); + + // truncate any tokens that are beyond n_past for this slot + const llama_pos p0 = slot.prompt.tokens.pos_next(); + + SLT_TRC(slot, "cached n_tokens = %d, memory_seq_rm [%d, end)\n", slot.prompt.n_tokens(), p0); + + common_context_seq_rm(ctx_tgt, slot.id, p0, -1); + if (ctx_dft) { + common_context_seq_rm(ctx_dft.get(), slot.id, p0, -1); } - // #29 Phase B: per-request peer switching. If the peer changes, - // clean up the old binding and register the new one. The peer - // info comes from the SET_EXPERT_MODE control-plane payload - // (JSON {"mode":"combined","peer":"host:port"}), NOT from the - // HTTP inference body — keeping control and data separate. - if (!peer_override.empty() && peer_override != hydra_current_peer) { - // Guard: peer switch is unsafe while any slot is decoding. - // sched_reserve() destroys and rebuilds the scheduler, which - // invalidates in-flight decode state across all slots. - bool any_active = false; - for (const auto & s : slots) { - if (s.is_processing()) { any_active = true; break; } - } - if (any_active) { - SRV_WRN("hydra: cannot switch peers — %zu slot(s) are processing, rejecting SET_EXPERT_MODE\n", slots.size()); - res->rpc_status = HYDRA_STATUS_BUSY; - res->success = false; - res->error = "cannot switch peers while slots are processing"; - queue_results.send(std::move(res)); - break; - } - if (!hydra_current_peer.empty()) { - SRV_INF("hydra: switching from peer %s to %s — cleaning up old binding\n", - hydra_current_peer.c_str(), peer_override.c_str()); - ctx_tgt->hydra_remove_combined_rpc_backend(hydra_current_peer.c_str()); - } - hydra_current_peer = peer_override; + // If using an alora, there may be uncached tokens that come + // before the invocation sequence. When this happens, the + // tokens before the invocation sequence need to be + // processed without the adapter in a separate batch, then + // the adapter needs to be enabled for the remaining tokens. + if (lora_all_alora(slot.lora) && slot.alora_invocation_start - 1 > slot.prompt.n_tokens()) { + SLT_DBG(slot, "processing pre-alora tokens without the adapter (n_tokens = %d, alora_invocation_start = %d)\n", slot.prompt.n_tokens(), slot.alora_invocation_start); + const auto & enabled_loras = lora_get_enabled_ids(slot.lora); + GGML_ASSERT(enabled_loras.size() == 1); + alora_scale = slot.lora[enabled_loras[0]].scale; + slot.lora[enabled_loras[0]].scale = 0.0f; + alora_disabled_id = enabled_loras[0]; } - // Hydra #383 T1: layer-split (static combined) engines cannot - // switch modes at runtime — the split is baked in at model load. - // "combined" is a no-op (already combined); "solo" is rejected. - if (hydra_combined_static) { - if (requested == "solo") { - res->rpc_status = HYDRA_STATUS_ERROR; - res->success = false; - res->error = "combined_static: this engine loaded in layer-split COMBINED mode; cannot switch to solo at runtime"; - LOG_WRN("srv %12.*s: hydra: SET_EXPERT_MODE solo rejected — engine is combined_static (layer-split)\n", 12, __func__); - queue_results.send(std::move(res)); - break; + // make a checkpoint of the parts of the memory that cannot be rolled back. + // checkpoints are created only if (see server_should_create_checkpoint): + // - the model does not support partial sequence removal + // - the model uses SWA (and we are not using `swa_full`) + // - the model supports partial sequence removal but only up to a fixed bound + // - the model is recurrent/hybrid (see below) + // Hydra: when the binary RPC port is enabled this server participates in + // cross-node KV migration. The restore target may not support rollback + // (e.g. it reports SEQ_RM_TYPE_FULL for the same model), so create native + // checkpoints regardless of the local seq_rm verdict — STATE_GET ships + // the latest checkpoint in the v2 blob, and without one the receiver + // fabricates a checkpoint at the final position, which corrupts + // hybrid/recurrent decode (recurrent state ends up past the resume point). + // Hydra (#316): the generic seq_rm probe in common_context_can_seq_rm() + // reports PART (not RS) for this hybrid arch's mixed attention/recurrent + // memory, since llama_n_rs_seq() is 0 and the smoke-test removal succeeds. + // That left checkpoint creation gated on rpc_port (only true for in-cluster + // nodes), so a standalone server with no RPC peer never created checkpoints + // and every cache-search below (which already special-cases is_rec, see + // ik_llama.cpp#1762) found nothing to restore — forcing a full re-prefill + // on every request. Recurrent/hybrid models need checkpoints on their own + // merits, independent of rpc_port. + // Hydra (#8): the gate is extracted to server_should_create_checkpoint() + // and pinned by tests/test-hydra-checkpoint-policy.cpp so the is_rec term + // can't be silently dropped — doing so breaks hybrid KV-cache restore. + const bool is_rec = llama_model_is_recurrent(model_tgt) || + llama_model_is_hybrid(model_tgt); + bool do_checkpoint = server_should_create_checkpoint( + params_base.n_ctx_checkpoints, + slot.task->type == SERVER_TASK_TYPE_COMPLETION, + ctx_tgt_seq_rm_type, + n_swa, + is_rec, + params_base.rpc_port); + + bool has_mtmd = false; + + // check if we should process the image + while (slot.prompt.n_tokens() < slot.task->n_tokens() && input_tokens[slot.prompt.n_tokens()] == LLAMA_TOKEN_NULL) { + // process the image + size_t n_tokens_out = 0; + int32_t res = input_tokens.process_chunk(ctx_tgt, mctx, slot.prompt.n_tokens(), slot.prompt.tokens.pos_next(), slot.id, n_tokens_out); + if (res != 0) { + SLT_ERR(slot, "failed to process image, res = %d\n", res); + send_error(slot, "failed to process image", ERROR_TYPE_SERVER); + slot.release(); + continue; } - // requested == "combined": success no-op - res->expert_mode_applied = "combined"; - res->rpc_status = HYDRA_STATUS_OK; - res->success = true; - LOG_INF("srv %12.*s: hydra: SET_EXPERT_MODE combined no-op — engine is combined_static (layer-split)\n", 12, __func__); - queue_results.send(std::move(res)); - break; - } - // #368 fix: gate on "configured as combined head" (non-empty - // peer addr + OT pattern), NOT on whether the startup - // dual-load succeeded. The rebind path below is fail-open — - // if the peer is still unreachable it stays solo — so - // hydra_combined_head_attached (set only when startup - // succeeded) must NOT block the attempt. Hydra #287/#260/#348 - // intent is preserved: an unconfigured engine (no peer/ - // pattern) still falls back to solo immediately. - const bool want_combined = requested == "combined" && - !hydra_peer.empty() && !hydra_combined_pattern.empty(); - - // #368 (#357 fix): bind-on-activation. Re-bind the peer's - // expert tensors on each SET_EXPERT_MODE("combined") request - // so a peer that was down at boot is picked up on the first - // COMBINED request after it comes up. Fail-open: if the - // rebind fails we stay solo and the Coordinator's - // ReportsSolo path handles it. - bool actually_combined = want_combined; - if (want_combined) { - if (hydra_peer.empty() || hydra_combined_pattern.empty()) { - SRV_WRN("%s\n", "hydra: SET_EXPERT_MODE(combined) but no peer/pattern configured; staying solo"); - actually_combined = false; - } else { - // ggml_backend_rpc_add_server is idempotent — returns - // the existing reg if the peer was registered before. - ggml_backend_reg_t rpc_reg = ggml_backend_reg_by_name("RPC"); - if (!rpc_reg) { - SRV_WRN("%s\n", "hydra: SET_EXPERT_MODE(combined) but RPC backend not available; staying solo"); - actually_combined = false; - } else { - using add_server_fn_t = ggml_backend_reg_t (*)(const char *); - auto add_server_fn = (add_server_fn_t) ggml_backend_reg_get_proc_address(rpc_reg, "ggml_backend_rpc_add_server"); - ggml_backend_reg_t peer_reg = add_server_fn ? add_server_fn(hydra_peer.c_str()) : nullptr; - ggml_backend_dev_t peer_dev = (peer_reg && ggml_backend_reg_dev_count(peer_reg) > 0) ? ggml_backend_reg_dev_get(peer_reg, 0) : nullptr; - if (!peer_dev) { - SRV_WRN("hydra: SET_EXPERT_MODE(combined) but peer %s has no registered device; staying solo\n", - hydra_peer.c_str()); - actually_combined = false; - } else { - int32_t n_bound = llama_hydra_rebind_combined_experts( - ctx_tgt, hydra_peer.c_str(), peer_dev, hydra_combined_pattern.c_str()); - if (n_bound <= 0) { - SRV_WRN("hydra: SET_EXPERT_MODE(combined) rebind on peer %s returned %d; staying solo\n", - hydra_peer.c_str(), n_bound); - actually_combined = false; - } else { - // Peer is up — latch so INFO RPC advertises combined. - hydra_combined_head_attached = true; - } - } + if (ctx_dft) { + // TODO: in the future, figure out how to infuse target embeddings to the images + // for now, we skip this for simplicity + // maybe we simply need to call `common_speculative_process()` on the mtmd batches in the `process_chunk` above? + res = input_tokens.process_chunk(ctx_dft.get(), mctx, slot.prompt.n_tokens(), slot.prompt.tokens.pos_next(), slot.id, n_tokens_out); + if (res != 0) { + GGML_ABORT("failed to process multi-modal data on draft context\n"); } } - } - - llama_hydra_set_expert_mode(ctx_tgt, actually_combined ? 1 : 0); - res->expert_mode_applied = actually_combined ? "combined" : "solo"; - res->rpc_status = HYDRA_STATUS_OK; - res->success = true; - SRV_INF("hydra: SET_EXPERT_MODE requested='%s' applied='%s' (slot %d)\n", - requested.c_str(), res->expert_mode_applied.c_str(), task.hydra_action.id_slot); - queue_results.send(std::move(res)); - } break; + slot.n_prompt_tokens_processed += n_tokens_out; - case SERVER_TASK_TYPE_HYDRA_ENGINE_SWAP_QUANT: - { - auto res = std::make_unique(); - res->id = task.id; - res->op = HYDRA_OP_SWAP_QUANT; - res->rpc_status = HYDRA_STATUS_OK; - res->success = true; - SRV_INF("hydra: SWAP_QUANT quant='%s' pattern='%s' (slot %d)\n", - task.hydra_action.quant_key.c_str(), - task.hydra_action.tensor_pattern.c_str(), - task.hydra_action.id_slot); - queue_results.send(std::move(res)); - } break; + // add the image chunk to cache + { + const auto & chunk = input_tokens.find_chunk(slot.prompt.n_tokens()); + slot.prompt.tokens.push_back(chunk.get()); // copy + } - // M-Perf.9 (#289) / issue #287: PIPELINE_ATTACH is part of the - // two-engine "work together" routing tracked in #287. The - // coordinator wires the request; the engine-side scaffolding - // (--override-tensor local-load, activation passing, COMBINED - // expert mode) is the next deliverable. For now this opcode - // returns NOT_IMPLEMENTED so the wire stays in sync — the - // coordinator will treat that as a fallback to solo mode. - case SERVER_TASK_TYPE_HYDRA_ENGINE_PIPELINE_ATTACH: - { - auto res = std::make_unique(); - res->id = task.id; - res->op = HYDRA_OP_PIPELINE_ATTACH; - res->rpc_status = HYDRA_STATUS_NOT_IMPLEMENTED; - res->success = false; - res->error = "HYDRA_OP_PIPELINE_ATTACH not yet implemented in this build (see issue #287)"; - SRV_WRN("hydra: PIPELINE_ATTACH received (slot %d) — stubbed, issue #287\n", - task.hydra_action.id_slot); - queue_results.send(std::move(res)); - } break; - } - } + has_mtmd = true; + } - // Hydra #406 (Phase 2b follow-up): apply the staged T2/T3 CONFIGURE - // rebuild. Called from update_slots() when the slot-free moment - // arrives (all slots idle, no slot is hydra_transferring). - // - // The flow: - // 1. Check drain timeout (HYDRA_COORD_PROFILE_SWITCH_DRAIN_TIMEOUT, - // default 300s). On timeout, discard the staged config and - // return — the next INFO call surfaces the cleared state. - // 2. T2 work (free + rebuild context). Skipped when tier is T3 - // (T3's load_model() rebuilds the context as a side effect). - // 3. T3 work (full model reload). Uses the staged T3 statics - // (override_tensor / split_mode / tensor_split / n_gpu_layers - // / n_cpu_moe / model.path) and falls through to load_model() - // for the actual unload+reload cycle. COMBINED-mode bindings - // are torn down before the reload and re-attached after. - // 4. Clear the staged state (T3 statics + pending_config JSON). - // - // On any failure: rollback to the pre-apply params_base and rebuild - // from there. The exception path (GGML_ABORT) is reserved for the - // catastrophic case where the rollback itself fails — the engine - // would be unable to serve in any state and must exit. - bool apply_pending_hydra_config() { - const bool is_first_load = !ctx_tgt; - if (is_first_load) { - if (!first_load_pending) { - return false; - } - // Don't check hydra_has_pending_config — ctx_tgt doesn't exist yet. - // The T3 statics were staged by hydra_apply_t3_mutators() in the - // CONFIGURE handler. Set a default tier for the rebuild path. - } else if (!ctx_tgt->hydra_has_pending_config()) { - return false; - } + const int32_t n_before_user = slot.task->params.n_before_user; + const bool n_before_user_known = n_before_user > 0; - // 1. Drain timeout — skipped for first load (no ctx_tgt timestamp). - std::string tier; - std::string pending_json; + // add prompt tokens for processing in the current batch + while (slot.prompt.n_tokens() < slot.task->n_tokens() && batch.n_tokens < n_batch) { + // get next token to process + llama_token cur_tok = input_tokens[slot.prompt.n_tokens()]; + if (cur_tok == LLAMA_TOKEN_NULL) { + break; // end of text chunk + } - if (is_first_load) { - tier = "T3"; - // pending_json stays empty — T3 statics are staged in global - // overrides, not in pending_config (ctx_tgt doesn't exist yet). - } else { - constexpr time_t k_drain_timeout_default = 300; - time_t now = std::time(nullptr); - time_t elapsed = now - ctx_tgt->hydra_get_pending_config_set_at(); - int env_timeout = 0; - if (const char * e = getenv("HYDRA_COORD_PROFILE_SWITCH_DRAIN_TIMEOUT")) { - env_timeout = atoi(e); - } - time_t drain_timeout = env_timeout > 0 ? env_timeout : k_drain_timeout_default; - if (elapsed > drain_timeout) { - SRV_WRN("hydra: pending config drain timeout (elapsed=%lld, limit=%lld) — discarding, " - "tier='%s' payload_size=%zu\n", - (long long) elapsed, (long long) drain_timeout, - ctx_tgt->hydra_get_pending_config_tier().c_str(), - ctx_tgt->hydra_get_pending_config().size()); - ctx_tgt->hydra_clear_pending_config(); - llama_hydra_clear_pending_t3(); - // hydra#470: the staged generic (T4) subset must not - // survive the discard — a stale config would be applied - // on the next unrelated reload. - g_pending_reload_config.clear(); - return false; - } + // if this is an alora request with pre-invocation + // tokens that are not cached, we need to stop filling + // this batch at those pre-invocation tokens. + if (alora_scale > 0 && slot.prompt.n_tokens() == slot.alora_invocation_start - 1) { + SLT_DBG(slot, "stop prompt batch filling at (n_tokens = %d, alora_invocation_start = %d)\n", slot.prompt.n_tokens(), slot.alora_invocation_start); + break; + } - tier = ctx_tgt->hydra_get_pending_config_tier(); - pending_json = ctx_tgt->hydra_get_pending_config(); - SRV_INF("hydra: applying pending config (tier='%s', age=%llds, payload_size=%zu)\n", - tier.c_str(), (long long) elapsed, pending_json.size()); - } + // embedding requires all tokens in the batch to be output; + // MTP also wants logits at every prompt position so the + // streaming hook can mirror t_h_nextn into ctx_dft. + common_batch_add(batch, + cur_tok, + slot.prompt.tokens.pos_next(), + { slot.id }, + slot.need_embd()); + slot.prompt.tokens.push_back(cur_tok); - bool ok = true; + slot.n_prompt_tokens_processed++; - // 2. T2 work: free + rebuild context with the new cparams. - // Skipped when tier is T3 (T3's load_model() handles both). - if (tier == "T2") { - if (!apply_t2_rebuild(pending_json)) { - SRV_ERR("%s", "hydra: T2 rebuild failed; engine continues with old context\n"); - ok = false; - } - } - - // 3. T3 work: full model reload with the staged T3 statics. - // load_model() handles the unload+reload cycle. COMBINED-mode - // expert bindings are torn down before the reload and re- - // attached after, in the same pattern as SET_EXPERT_MODE. - // T4 (generic-arg) configs route here too: a model reload is - // the only apply that makes EVERY generic key take effect - // (speculative types need load_model's MTP/draft setup). - if (tier == "T3" || tier == "T4") { - // #470: force rebuild if a peer reconnection was detected - const bool reconn_force = (ctx_tgt && ctx_tgt->peer_reconnection_pending); - if (reconn_force) { - ctx_tgt->peer_reconnection_pending = false; - SRV_WRN("%s", "hydra: apply_pending: peer reconnection pending — forcing T3 rebuild\n"); - } - if (!apply_t3_rebuild(reconn_force)) { - SRV_ERR("%s", "hydra: T3 rebuild failed; engine continues with old model\n"); - ok = false; - } else { - // P1-6: T3 model changed — the cached server_context_meta - // (model_path, split_mode, tensor_split, chat_params, …) - // is now stale. Refresh it on the task-queue thread - // (safe — runs during the drain window when no slots are - // processing and no new requests are being dispatched). - if (routes_ptr) { - routes_ptr->refresh_meta(); - } - // P0-1 (#49): after deferred first-load, apply staged capabilities - // so ENGINE_INFO(0x41) and COMBINED-mode logic work correctly. - if (is_first_load) { - hydra_rpc_backend_active = bootstrap_rpc_active; - hydra_peer = bootstrap_peer; - hydra_peer_reachable = bootstrap_peer_reachable; - hydra_combined_pattern = bootstrap_pattern; - hydra_split_mode = bootstrap_split_mode; - if (bootstrap_combined_static) { - hydra_combined_static = true; - SRV_INF("%s", "P0-1: deferred first-load — combined_static mode activated\n"); + // stop the prompt batch exactly before the latest user input, so a checkpoint + // can be created after the previous messages + if (n_before_user_known && + slot.prompt.n_tokens() == n_before_user) { + break; + } + + // process the last few tokens of the prompt separately in order to allow for a checkpoint to be created. + // create checkpoints that many tokens before the end of the prompt: + // - 4 + n_ubatch + // - 4 + // ref: https://github.com/ggml-org/llama.cpp/pull/20288 + if (do_checkpoint) { + static const int checkpoint_offsets[] = {4 + n_ubatch, 4}; + + bool should_break = false; + for (int offset : checkpoint_offsets) { + const int n_last = std::min(n_batch, offset); + if (slot.task->n_tokens() == slot.prompt.n_tokens() + n_last) { + should_break = true; + break; + } + } + if (should_break) { + break; + } + } } - // Register local tensors and enable shared-backend compute - // lock so the model can serve inbound RPC requests. - if (model_tgt && ctx_tgt) { - llama_hydra_register_local_tensors_for_rpc(ctx_tgt); - llama_hydra_enable_shared_backend_compute_lock(); + + // the number of tokens added to the batch for the current slot + const auto n_tokens_cur = batch.n_tokens - n_tokens_prev; + + // Track total prompt tokens processed for n_common observability + if (!slot.logits_reused) { + slot.n_prompt_processed += n_tokens_cur; } - // Update the RPC server's compute backends now that the - // model is loaded. The RPC server was started with empty - // backends (head-bootstrap mode); now populate it. - if (ctx_tgt) { - std::vector backends(8); - size_t n = llama_hydra_get_compute_backends(ctx_tgt, backends.data(), backends.size()); - if (n > backends.size()) { - backends.resize(n); - n = llama_hydra_get_compute_backends(ctx_tgt, backends.data(), backends.size()); + + const bool near_prompt_end = slot.task->n_tokens() < slot.prompt.n_tokens() + n_ubatch; + + // entire prompt has been processed + if (slot.prompt.n_tokens() == slot.task->n_tokens()) { + slot.state = SLOT_STATE_DONE_PROMPT; + + GGML_ASSERT(batch.n_tokens > 0); + + // extract the logits only for the last token + batch.logits[batch.n_tokens - 1] = true; + + slot.n_decoded = 0; + slot.i_batch = batch.n_tokens - 1; + + slot.init_sampler(); + } else { + // skip ordinary mid-prompt checkpoints + if (!n_before_user_known && !near_prompt_end) { + do_checkpoint = false; } - backends.resize(n); - hydra_rpc::update_backends(backends); - SRV_INF("P0-1: updated RPC backends to %zu compute device(s)\n", backends.size()); } - SRV_INF("%s", "hydra-engine ready — model loaded via CONFIGURE T3\n"); - } - } - } - // 4. Clear the staged state regardless of success. On failure - // the rollback in apply_t{2,3}_rebuild has restored the - // previous state; clearing the staged state prevents the - // next slot-free moment from re-attempting the same rebuild. - if (is_first_load) { - first_load_pending = false; - } else { - ctx_tgt->hydra_clear_pending_config(); - } - llama_hydra_clear_pending_t3(); - return ok; - } - - // T2 rebuild: free the live llama_context, rebuild llama_context_params - // from the updated params_base (n_ctx / cache_type_k / cache_type_v / - // RoPE / YaRN), recreate the context, re-init per-slot samplers. - // On failure: rebuild with the old params_base (rollback). - // - // Helper: parse a wire-shape cache_type string ("f16" / "q8_0" / ...) - // to a ggml_type. The wire spec uses llama.cpp's ggml type names. - // There is no public ggml_parse_type() in upstream llama.cpp, so - // we iterate ggml_type_traits via ggml_get_type_traits() and - // match on ggml_type_name(). - static ggml_type hydra_parse_cache_type(const std::string & s) { - if (s.empty()) return GGML_TYPE_COUNT; - for (int i = 0; i < GGML_TYPE_COUNT; i++) { - ggml_type t = (ggml_type) i; - if (strcmp(ggml_type_name(t), s.c_str()) == 0) return t; - } - return GGML_TYPE_COUNT; - } + const auto pos_min = llama_memory_seq_pos_min(llama_get_memory(ctx_tgt), slot.id); + const auto pos_max = llama_memory_seq_pos_max(llama_get_memory(ctx_tgt), slot.id); - // hydra#470: apply the staged context-level (T2) + generic (T4) keys - // from `cfg` to `params`. Shared by apply_t2_rebuild() (params_base, - // before the context rebuild) and apply_t3_rebuild() (swapped_params, - // before load_model()) so a mixed T2+T4 payload is handled identically - // on both reload paths — the model-reload path consumes the same staged - // config as the context-reload path and no T2 key is stranded. - // - // The explicit T2 keys keep their custom semantics (n_ctx clamp to - // model_n_ctx_train, cache_type validation); every other key goes - // through the llama.cpp arg table (hydra_apply_generic_key). A - // present-but-unusable value is logged with SRV_WRN — never silent. - void hydra_apply_staged_context_keys(common_params & params, const json & cfg) { - // ── explicit T2 keys (context-level); each is optional, absence - // means "leave unchanged" ── - if (cfg.contains("n_ctx") && cfg["n_ctx"].is_number_integer()) { - const int32_t n_ctx = cfg["n_ctx"].get(); - if (model_tgt) { - // Clamp to the model's training ctx. The wire spec does - // not require a reject-on-too-large; we clamp and report. - // At first load (model_tgt null) the value is used as-is - // and llama.cpp validates it during context creation. - const int32_t max_ctx = (int32_t) llama_model_n_ctx_train(model_tgt); - if (n_ctx > max_ctx) { - SRV_WRN("hydra: n_ctx=%d exceeds model_n_ctx_train=%d; clamping\n", - n_ctx, max_ctx); - params.n_ctx = max_ctx; - } else { - params.n_ctx = n_ctx; - } - } else { - params.n_ctx = n_ctx; - } - } - if (cfg.contains("cache_type_k") && cfg["cache_type_k"].is_string()) { - const std::string & s = cfg["cache_type_k"].get_ref(); - ggml_type t = hydra_parse_cache_type(s); - if (t == GGML_TYPE_COUNT) { - SRV_WRN("hydra: cache_type_k='%s' unparseable; ignoring\n", s.c_str()); - } else { - params.cache_type_k = t; - } - } - if (cfg.contains("cache_type_v") && cfg["cache_type_v"].is_string()) { - const std::string & s = cfg["cache_type_v"].get_ref(); - ggml_type t = hydra_parse_cache_type(s); - if (t == GGML_TYPE_COUNT) { - SRV_WRN("hydra: cache_type_v='%s' unparseable; ignoring\n", s.c_str()); - } else { - params.cache_type_v = t; - } - } - if (cfg.contains("rope_freq_base") && cfg["rope_freq_base"].is_number()) { - params.rope_freq_base = cfg["rope_freq_base"].get(); - } - if (cfg.contains("rope_freq_scale") && cfg["rope_freq_scale"].is_number()) { - params.rope_freq_scale = cfg["rope_freq_scale"].get(); - } - if (cfg.contains("yarn_ext_factor") && cfg["yarn_ext_factor"].is_number()) { - params.yarn_ext_factor = cfg["yarn_ext_factor"].get(); - } - if (cfg.contains("yarn_attn_factor") && cfg["yarn_attn_factor"].is_number()) { - params.yarn_attn_factor = cfg["yarn_attn_factor"].get(); - } - if (cfg.contains("yarn_beta_fast") && cfg["yarn_beta_fast"].is_number()) { - params.yarn_beta_fast = cfg["yarn_beta_fast"].get(); - } - if (cfg.contains("yarn_beta_slow") && cfg["yarn_beta_slow"].is_number()) { - params.yarn_beta_slow = cfg["yarn_beta_slow"].get(); - } - if (cfg.contains("yarn_orig_ctx") && cfg["yarn_orig_ctx"].is_number_integer()) { - params.yarn_orig_ctx = cfg["yarn_orig_ctx"].get(); - } + // checkpoints are created before the current batch is decoded, so + // their token position is the batch start rather than the prompt end + const int32_t n_tokens_start = slot.prompt.n_tokens() - n_tokens_cur; - // ── generic (T4) fallback: every key the explicit T2 code above - // did NOT handle is applied via the llama.cpp arg table. This - // also covers special-classified keys with no explicit handling - // (rope_scale / rope_scaling classify as T2 via the rope_* - // prefix but have no dedicated code above). ── - static const std::set t2_handled = { - "n_ctx", "cache_type_k", "cache_type_v", - "rope_freq_base", "rope_freq_scale", - "yarn_ext_factor", "yarn_attn_factor", - "yarn_beta_fast", "yarn_beta_slow", "yarn_orig_ctx", - }; - for (auto it = cfg.begin(); it != cfg.end(); ++it) { - const std::string & key = it.key(); - if (t2_handled.count(key) > 0) { - continue; // explicit T2 code above - } - if (!hydra_apply_generic_key(params, key, it.value())) { - SRV_WRN("hydra: staged context key '%s' not applied (see log)\n", key.c_str()); - } - } + { + const bool is_on_user = + n_before_user_known && + n_tokens_start == n_before_user; - // ── present-but-unusable T2 values must be loud, not silent ── - for (const auto & key : t2_handled) { - if (!cfg.contains(key)) { - continue; - } - const json & v = cfg[key]; - const bool usable = - (key == "n_ctx" || key == "yarn_orig_ctx") ? v.is_number_integer() : - (key == "cache_type_k" || key == "cache_type_v") ? v.is_string() : - v.is_number(); - if (!usable) { - SRV_WRN("hydra: T2 key '%s' has unusable value (JSON type %s) — value ignored\n", - key.c_str(), v.type_name()); - } - } - } + const bool is_after_user = + n_before_user_known && + n_tokens_start > n_before_user; - bool apply_t2_rebuild(const std::string & pending_json) { - if (!ctx_tgt || !model_tgt) return false; + const bool is_allowed = + !n_before_user_known || + is_on_user || + (is_after_user && near_prompt_end); - json cfg; - try { - cfg = json::parse(pending_json); - } catch (const std::exception & e) { - SRV_WRN("hydra: T2 apply: invalid JSON in pending_config: %s\n", e.what()); - return false; - } + if (do_checkpoint && !is_allowed) { + do_checkpoint = false; + } + } - // Snapshot the old params for rollback. params_base is the - // canonical "what's in effect" state; restoring it plus a - // recreate-cycle is the rollback path. - common_params old_params = params_base; - - // hydra#470: apply the staged context-level + generic keys to - // params_base (explicit T2 keys + arg-table fallback). Shared with - // apply_t3_rebuild() so mixed T2+T4 payloads are handled identically - // on both reload paths. - hydra_apply_staged_context_keys(params_base, cfg); - // The staged reload config was consumed above (its keys ride the - // pending_config JSON); clear the static so a later T3 reload does - // not re-apply stale values. The "last applied" marker is updated - // only on the success path below. - const std::string consumed_reload_config = g_pending_reload_config; - g_pending_reload_config.clear(); - - // Free the live context. KV cache is destroyed; this is the - // T2 cost. The model is kept (T2 is context-only). - llama_free(ctx_tgt); - if (ctx_dft) { - llama_free(ctx_dft.get()); - ctx_dft.reset(); - } + // nothing to checkpoint yet + // TODO: is this check needed? + if (do_checkpoint && pos_min < 0) { + do_checkpoint = false; + } - // Build new cparams from the updated params_base. This is - // the same call site load_model() uses internally. - auto cparams = common_context_params_to_llama(params_base); - - // Recreate the context with the new cparams. - ctx_tgt = llama_new_context_with_model(model_tgt, cparams); - if (!ctx_tgt) { - // Rollback: rebuild with the old params_base. The old - // params must work (we just freed and recreated the - // context with them). If they don't, the engine is in - // a bad state — abort. - SRV_WRN("hydra: T2 rebuild failed with n_ctx=%d cache_type=%d/%d; " - "rolling back to old params\n", - params_base.n_ctx, (int) params_base.cache_type_k, - (int) params_base.cache_type_v); - params_base = old_params; - auto cparams_old = common_context_params_to_llama(params_base); - ctx_tgt = llama_new_context_with_model(model_tgt, cparams_old); - if (!ctx_tgt) { - GGML_ABORT("hydra: T2 rollback failed (cannot rebuild context with old params). " - "Engine exiting to prevent serving with corrupted state."); - } - return false; - } + // do not checkpoint after mtmd chunks + do_checkpoint = do_checkpoint && !has_mtmd; - // Re-init per-slot samplers. The old samplers were bound to - // the now-freed context; common_sampler_init() on the new - // model picks up the (possibly changed) sampling config. - for (auto & slot : slots) { - slot.smpl.reset(common_sampler_init(model_tgt, params_base.sampling)); - } + // no need to create checkpoints that are too close together. + // For recurrent/hybrid models, use a much smaller minimum spacing so short + // follow-up turns still get a checkpoint to resume from. Ref: ik_llama.cpp#1762. + const int eff_checkpoint_min_step = + (llama_model_is_recurrent(model_tgt) || llama_model_is_hybrid(model_tgt)) + ? std::min(params_base.checkpoint_min_step, 4) + : params_base.checkpoint_min_step; + do_checkpoint = do_checkpoint && (slot.prompt.checkpoints.empty() || n_tokens_start > slot.prompt.checkpoints.back().n_tokens + eff_checkpoint_min_step); + SLT_DBG(slot, "main/do_checkpoint = %s, pos_min = %d, pos_max = %d\n", do_checkpoint ? "yes" : "no", pos_min, pos_max); - n_ctx = llama_n_ctx(ctx_tgt); - // hydra#470: record the staged reload config as applied. A later - // T3-tier config carrying the same staged keys can then skip the - // model reload via the early-exit (the values are already in - // params_base, which feeds swapped_params). - g_last_reload_config_applied = consumed_reload_config; - SRV_INF("hydra: T2 rebuild applied (n_ctx=%d, cache=%d/%d, slots=%zu)\n", - n_ctx, (int) params_base.cache_type_k, - (int) params_base.cache_type_v, slots.size()); - return true; - } + // note: we create the checkpoint before calling llama_decode(), so the current batch is not + // yet processed and therefore it is not part of the checkpoint. + if (do_checkpoint) { + create_checkpoint(slot, n_tokens_cur, pos_min, pos_max); + } + } - // Register new RPC peer devices into the global ggml backend registry. - // Called from apply_t3_rebuild() before load_model() so the new peer's - // device exists when common_init_from_params() tries to place tensors - // per tensor_split/split_mode. - // - // Repeated registration is NOT safe/idempotent in the underlying API, - // so we track already-registered endpoints in a static set and only - // register genuinely new ones. - static void hydra_register_rpc_servers(const json & servers_arr) { - static std::set registered; + if (!slot_batched) { + slot_batched = &slot; + } - if (!servers_arr.is_array() || servers_arr.empty()) { - return; + if (batch.n_tokens >= n_batch) { + break; + } + } } - ggml_backend_load_all(); - ggml_backend_reg_t rpc_reg = ggml_backend_reg_by_name("RPC"); - if (!rpc_reg) { - SRV_WRN("%s", "hydra: rpc_servers: RPC backend not available\n"); - return; - } + SRV_DBG("decoding batch, n_tokens = %d\n", batch.n_tokens); - typedef ggml_backend_reg_t (*ggml_backend_rpc_add_server_t)(const char * endpoint); - auto add_server_fn = (ggml_backend_rpc_add_server_t) - ggml_backend_reg_get_proc_address(rpc_reg, "ggml_backend_rpc_add_server"); - if (!add_server_fn) { - SRV_WRN("%s", "hydra: rpc_servers: ggml_backend_rpc_add_server not found\n"); - return; - } + auto accept_special_token = [&](server_slot & slot, llama_token token) { + return params_base.special || + slot.task->params.sampling.preserved_tokens.find(token) != slot.task->params.sampling.preserved_tokens.end(); + }; - for (const auto & v : servers_arr) { - if (!v.is_string()) continue; - const std::string endpoint = v.get(); - if (endpoint.empty()) continue; - if (registered.count(endpoint)) { - SRV_DBG("hydra: rpc_servers: endpoint '%s' already registered, skipping\n", - endpoint.c_str()); - continue; - } - ggml_backend_reg_t reg = add_server_fn(endpoint.c_str()); - if (reg) { - ggml_backend_register(reg); - registered.insert(endpoint); - SRV_INF("hydra: rpc_servers: registered endpoint '%s'\n", endpoint.c_str()); - } else { - SRV_WRN("hydra: rpc_servers: failed to register endpoint '%s'\n", - endpoint.c_str()); + if (slot_batched) { + // apply lora, only need to do it once per batch + common_set_adapter_lora(ctx_tgt, slot_batched->lora); + + // if the lora is temporarily disabled for an alora, re-enable it + // for next time + if (alora_scale > 0.0f) { + SRV_DBG("re-enabling alora with scale %f\n", alora_scale); + slot_batched->lora[alora_disabled_id].scale = alora_scale; } - } - } - // #470: refresh the T3-current alias → file map after a successful - // model load. The map's keys are the engine's own identity aliases - // (model_name + model_aliases, as recomputed by load_model()); the - // value is the resident file. Called from load_model() right after the - // identity members are re-derived, so the map always tracks the CURRENT - // resident — covering boot, apply_t3_rebuild(), the bare-alias swap - // paths and T3 rollback alike. - // - // Why this map exists: preset_alias_to_path is the static INI mapping, - // but the coordinator's T3 config (hydra_config.model_path) can load a - // file that the INI does NOT associate with the engine's current alias - // (e.g. the dense-27b-combined session T3-loads the 27B-Coder file - // while the engine's identity still says qwen3.6-35B-balanced from an - // earlier SOLO session). DECODE_APPLY must know that the requested - // alias already refers to the resident before it decides to swap. - void hydra_t3_record_current_alias_to_path() { - t3_current_alias_to_path.clear(); - const std::string & resident = params_base.model.path; - t3_current_alias_to_path[model_name] = resident; - for (const auto & alias : model_aliases) { - t3_current_alias_to_path[alias] = resident; + llama_set_embeddings(ctx_tgt, slot_batched->need_embd()); } - SRV_DBG("hydra: T3-current alias→file map recorded %zu alias(es) → '%s'\n", - t3_current_alias_to_path.size(), resident.c_str()); - } - // Tear down COMBINED-mode RPC peer bindings before a model reload. - // Shared by apply_t3_rebuild() and the bare-alias swap paths (PREFILL, - // DECODE_APPLY, server-context.cpp ~3800 / ~4310). Must run BEFORE - // load_model() — otherwise the new ctx_tgt (post-reload) inherits a - // stale binding to the old peer's device. The bare-alias paths used to - // skip this entirely: the engine loaded the correct model file but kept - // routing tokens through the old COMBINED config, which is #514 - // (throughput collapses to ~2-4 tok/s after a dynamic model swap). - void hydra_teardown_combined_before_reload() { - SRV_INF("hydra: tearing down COMBINED before model reload (was head_attached=%d, static=%d)\n", - (int) hydra_combined_head_attached, (int) hydra_combined_static); - llama_hydra_set_expert_mode(ctx_tgt, 0); - if (!hydra_current_peer.empty()) { - ctx_tgt->hydra_remove_combined_rpc_backend(hydra_current_peer.c_str()); - } - llama_hydra_clear_combined_bindings(ctx_tgt, hydra_peer.c_str()); - hydra_combined_head_attached = false; - } - - // Re-attach COMBINED-mode bindings after a model reload, mirroring - // hydra_teardown_combined_before_reload() above. Layer-split (static) - // just re-enables the mode flag — load_model() already preloaded the - // peer device with the new tensor_split. Expert-split re-resolves the - // peer's RPC device and rebinds the expert tensors; same fail-open - // pattern as SET_EXPERT_MODE — if the peer is unreachable, the engine - // stays solo and the coordinator's solo-fallback path handles it. - void hydra_reattach_combined_after_reload() { - SRV_INF("%s", "hydra: re-attaching COMBINED on new model\n"); - if (hydra_combined_static) { - llama_hydra_set_expert_mode(ctx_tgt, 1); - } else if (!hydra_peer.empty() && !hydra_combined_pattern.empty()) { - if (llama_hydra_peer_reachable(hydra_peer.c_str())) { - ggml_backend_reg_t rpc_reg = ggml_backend_reg_by_name("RPC"); - if (rpc_reg) { - using add_server_fn_t = ggml_backend_reg_t (*)(const char *); - auto add_server_fn = (add_server_fn_t) ggml_backend_reg_get_proc_address(rpc_reg, "ggml_backend_rpc_add_server"); - ggml_backend_reg_t peer_reg = add_server_fn ? add_server_fn(hydra_peer.c_str()) : nullptr; - ggml_backend_dev_t peer_dev = (peer_reg && ggml_backend_reg_dev_count(peer_reg) > 0) ? ggml_backend_reg_dev_get(peer_reg, 0) : nullptr; - if (peer_dev) { - int32_t n_bound = llama_hydra_rebind_combined_experts( - ctx_tgt, hydra_peer.c_str(), peer_dev, hydra_combined_pattern.c_str()); - if (n_bound > 0) { - hydra_combined_head_attached = true; - llama_hydra_set_expert_mode(ctx_tgt, 1); - SRV_INF("hydra: COMBINED re-attached on peer %s (%d layers bound)\n", - hydra_peer.c_str(), n_bound); - } else { - SRV_WRN("hydra: rebind returned %d; staying solo\n", n_bound); - } - } else { - SRV_WRN("hydra: peer %s has no device; staying solo\n", hydra_peer.c_str()); + if (batch.n_tokens == 0) { + // epic #610 WS1: in seam mode the extension may handle the + // empty-batch case (STATE_GET transfer suppression). WS1 impl is a + // no-op — this is a pure A/B switch, both modes run the inline + // suppression below. + if (hydra_ext_active && hydra_ext && hydra_ext->on_empty_batch(*this)) { + // extension handled the empty batch — skip the inline logic + } else if (++n_empty_consecutive > 3) { + // Hydra: a STATE_GET background stream holds the slot (hydra_transferring) + // without contributing batch tokens — that is expected, not a stall. + // Suppress the abort while a transfer is in flight, and for a short + // grace window after it ends (the flag clears a few loop iterations + // before the queue delivers the releasing task — without the grace + // window those tail iterations trip the abort). + static int64_t hydra_last_transfer_ms = 0; + static int64_t hydra_suppress_count = 0; + bool any_transferring = false; + for (const auto & s : slots) { + if (s.hydra_transferring && s.hydra_transferring->load()) { + any_transferring = true; + break; } - } else { - SRV_WRN("%s\n", "hydra: RPC backend not available; staying solo"); } - } else { - SRV_WRN("hydra: peer %s unreachable; staying solo\n", hydra_peer.c_str()); - } - } - } - - // Re-pad tensor_buft_overrides to the nullptr-terminated capacity - // llama_max_tensor_buft_overrides() after a preset's apply_to_params() - // has push_back()'d entries onto a freshly-cleared vector (see the - // bare-alias swap paths, server-context.cpp ~3800 / ~4310). Caps at - // the limit with the same guard the override_tensor T3 path already - // has (below) — apply_to_params() push_backs unconditionally, so an - // unusually large preset could otherwise overflow the same 4096-entry - // limit this whole clear/re-pad dance exists to respect. - void hydra_repad_tensor_buft_overrides(common_params & p, const char * ctx_label) { - const size_t ntbo = llama_max_tensor_buft_overrides(); - if (p.tensor_buft_overrides.size() + 1 > ntbo) { - SRV_WRN("hydra: %s: %zu tensor_buft_overrides exceed the %zu-entry limit; keeping the first %zu\n", - ctx_label, p.tensor_buft_overrides.size(), ntbo, ntbo - 1); - p.tensor_buft_overrides.resize(ntbo - 1); - } - p.tensor_buft_overrides.resize(ntbo, llama_model_tensor_buft_override{ nullptr, nullptr }); - } - - // T3 rebuild: full model reload. Uses the staged T3 statics - // (override_tensor, split_mode, tensor_split, n_gpu_layers, - // n_cpu_moe, model.path) populated by hydra_apply_t3_mutators(). - // Falls through to load_model() for the actual unload+reload - // cycle (which handles mmproj, MTP/draft, slot rebuild, etc.). - // On failure: rollback by reloading the old params_base. - bool apply_t3_rebuild(bool force = false) { - bool is_first_load = !ctx_tgt; - - // Track the last override_tensor string that was actually - // applied so we can detect "nothing changed" on subsequent - // calls and skip the expensive unload+reload cycle. - static std::string old_override_applied; - - common_params old_params = params_base; - common_params swapped_params = params_base; - - // Read the staged T3 statics and apply them to swapped_params. - if (llama_hydra_get_pending_n_gpu_layers() >= 0) { - swapped_params.n_gpu_layers = llama_hydra_get_pending_n_gpu_layers(); - } - // n_cpu_moe is informational only — the actual MoE expert - // offload is done via override_tensor (parsed below into - // tensor_buft_overrides). The standard common_params struct - // has no n_cpu_moe field; we just log the staged value for - // operator visibility. - if (llama_hydra_get_pending_n_cpu_moe() >= 0) { - SRV_INF("hydra: T3 rebuild: staged n_cpu_moe=%d (informational; expert routing via override_tensor)\n", - llama_hydra_get_pending_n_cpu_moe()); - } - const char * path = llama_hydra_get_pending_model_path(); - if (path && *path) { - swapped_params.model.path = path; - } - const char * mode = llama_hydra_get_pending_split_mode(); - if (mode && *mode) { - std::string m(mode); - if (m == "none") swapped_params.split_mode = LLAMA_SPLIT_MODE_NONE; - else if (m == "layer") swapped_params.split_mode = LLAMA_SPLIT_MODE_LAYER; - else if (m == "row") swapped_params.split_mode = LLAMA_SPLIT_MODE_ROW; - else SRV_WRN("hydra: T3 split_mode='%s' unknown; keeping current\n", m.c_str()); - } - const size_t n_split = llama_hydra_get_pending_tensor_split_count(); - if (n_split > 0) { - const float * split = llama_hydra_get_pending_tensor_split(); - // common_params::tensor_split is a fixed-size array. - const size_t cap = sizeof(swapped_params.tensor_split) / - sizeof(swapped_params.tensor_split[0]); - const size_t n = n_split < cap ? n_split : cap; - for (size_t i = 0; i < n; i++) { - swapped_params.tensor_split[i] = split[i]; - } - // Zero the rest so the engine doesn't see stale values. - for (size_t i = n; i < cap; i++) { - swapped_params.tensor_split[i] = 0.0f; - } - } - const char * override = llama_hydra_get_pending_override_tensor(); - if (override && *override) { - // Wire-shape: comma-separated "pattern=buft" pairs (e.g. - // "blk.*.ffn_*_exps.weight=CPU"). The C++ side stores - // these as a vector. - // Buft names are looked up - // via ggml_backend_dev_buffer_type() + ggml_backend_buft_name() - // (mirrors common/arg.cpp:parse_tensor_buffer_overrides). - ggml_backend_load_all(); - std::map buft_list; - for (size_t i = 0; i < ggml_backend_dev_count(); ++i) { - auto * dev = ggml_backend_dev_get(i); - auto * buft = ggml_backend_dev_buffer_type(dev); - if (buft) { - buft_list[std::string(ggml_backend_buft_name(buft))] = buft; + const int64_t now_ms = ggml_time_us() / 1000; + if (any_transferring) { + hydra_last_transfer_ms = now_ms; } - } - // CPU is the common case (MoE expert routing) — also lookup - // explicitly since some backends may not register the CPU buft. - buft_list["CPU"] = ggml_backend_cpu_buffer_type(); - - // Keep pattern strings alive for the lifetime of the - // process — entry.pattern is a const char* that must not - // dangle. Matches the safe pattern in common/arg.cpp. - static std::list buft_override_patterns; - - std::vector staged; - - const std::string ovr(override); - size_t start = 0; - while (start < ovr.size()) { - size_t comma = ovr.find(',', start); - std::string part = ovr.substr(start, comma == std::string::npos ? std::string::npos : comma - start); - size_t eq = part.find('='); - if (eq != std::string::npos) { - std::string pattern = part.substr(0, eq); - std::string buft_name = part.substr(eq + 1); - auto it = buft_list.find(buft_name); - if (it != buft_list.end()) { - buft_override_patterns.push_back(pattern); - llama_model_tensor_buft_override entry; - entry.pattern = buft_override_patterns.back().c_str(); - entry.buft = it->second; - staged.push_back(entry); - } else { - SRV_WRN("%s", "hydra: T3 rebuild: override_tensor buft name not in registered list; skipping pattern\n"); + if (any_transferring || now_ms - hydra_last_transfer_ms < 2000) { + // rate-limit: this branch runs in a hot loop — log once per 256 suppressions + if (hydra_suppress_count++ % 256 == 0) { + SRV_WRN("empty batch threshold exceeded (n_empty=%d, suppressed=%" PRId64 ") — hydra transfer %s, suppressing abort\n", + n_empty_consecutive, hydra_suppress_count, + any_transferring ? "in flight" : "just ended"); } - } - if (comma == std::string::npos) break; - start = comma + 1; - } - - // Install the staged patterns *in place of* the base ones instead of - // appending to them. - // - // common_params_parse_ex() (common/arg.cpp) unconditionally pads this - // vector out to llama_max_tensor_buft_overrides() entries of - // {nullptr, nullptr}, so by the time we get here the real CLI overrides - // sit at the head and the rest is terminator padding. push_back() would - // land *behind* that padding, which breaks twice over: - // 1. common_model_params_to_llama() asserts that back().pattern is - // nullptr, so the engine aborts before the model loads; - // 2. even without that assert, llama_model_loader stops scanning at - // the first nullptr pattern, so appended entries are never read — - // the override would be silently dropped and the MoE experts would - // land on the GPU. - // Replacing also matches the sibling fields handled above: model.path, - // split_mode, n_gpu_layers and tensor_split are all overwritten by the - // staged T3 config rather than merged into it. - const size_t ntbo = llama_max_tensor_buft_overrides(); - if (staged.empty()) { - // Nothing resolved (every buft name was unknown). Wiping the base - // overrides here would silently change how the model is placed, so - // keep them and make the no-op explicit. - SRV_WRN("%s", "hydra: T3 rebuild: staged override_tensor resolved to no usable patterns; keeping base overrides\n"); - } else { - if (staged.size() + 1 > ntbo) { - SRV_WRN("hydra: T3 rebuild: %zu override_tensor patterns exceed the %zu-entry limit; keeping the first %zu\n", - staged.size(), ntbo, ntbo - 1); - staged.resize(ntbo - 1); - } - // assign() re-establishes the full terminator padding, so everything - // from staged.size() onward is {nullptr, nullptr}. - swapped_params.tensor_buft_overrides.assign(ntbo, llama_model_tensor_buft_override{ nullptr, nullptr }); - for (size_t i = 0; i < staged.size(); ++i) { - swapped_params.tensor_buft_overrides[i] = staged[i]; + n_empty_consecutive = 0; + // avoid hot-spinning while the transfer holds the slot + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + } else { + SRV_WRN("%s", "no tokens to decode\n"); + GGML_ABORT("fatal error - please provide logs and repro in %s\n", "https://github.com/ggml-org/llama.cpp/pull/20277"); } } + } else { + n_empty_consecutive = 0; } - // hydra#470: apply the staged reload config (context-level T2 keys - // + generic T4 keys) to swapped_params BEFORE any model (re)load — - // they must land in common_params before load_model() consumes them - // (n_ctx, cache types, speculative types, ...). The staged JSON is - // consumed here (cleared), so a later T2/T3 apply does not re-apply - // stale values. Mixed T2+T4 payloads are applied by the same helper - // the context-reload path uses — nothing is stranded. - std::string staged_reload = g_pending_reload_config; - g_pending_reload_config.clear(); - if (!staged_reload.empty()) { - json reload_cfg; - try { - reload_cfg = json::parse(staged_reload); - } catch (const std::exception & e) { - SRV_WRN("hydra: T3 rebuild: staged reload config failed to parse: %s\n", e.what()); - staged_reload.clear(); - } - if (!reload_cfg.is_null()) { - hydra_apply_staged_context_keys(swapped_params, reload_cfg); - } - } - - // Early-exit: if the model, all T3-relevant params AND the staged - // reload config (T2 + T4 keys) are identical to what is already - // loaded, skip the expensive unload+reload cycle. Without this, - // every COMPLETION request that carries hydra_config triggers a - // full model swap even when nothing changed (the coordinator sends - // the same config on every decode request). The comparison is - // against the last staged dump that was actually loaded — identical - // config → identical dump → skip; a changed T2 or T4 key → forced - // reload (no masked changes). - const bool reload_unchanged = (staged_reload == g_last_reload_config_applied); - if (!is_first_load) { - const char * cur_override = llama_hydra_get_pending_override_tensor(); - bool params_unchanged = - swapped_params.model.path == old_params.model.path && - swapped_params.n_gpu_layers == old_params.n_gpu_layers && - swapped_params.split_mode == old_params.split_mode && - ((cur_override == nullptr && old_override_applied.empty()) || - (cur_override && old_override_applied == cur_override)); - if (params_unchanged && reload_unchanged && !force) { - // T3 overrides (override_tensor, split_mode) were staged by - // the COMPLETION hydra_config path. But the model reload is - // being skipped. Clear the staged override so the next decode - // uses the current tensor placement (not the staged override). - llama_hydra_set_override_tensor(ctx_tgt, nullptr); - SRV_INF("%s", "hydra: T3 rebuild: model and params unchanged — skipping reload, cleared staged overrides\n"); - return true; - } - } + int32_t i_next = 0; - // COMBINED-mode teardown BEFORE the model reload — see - // hydra_teardown_combined_before_reload() above. - const bool was_combined = hydra_combined_head_attached || hydra_combined_static; - if (!is_first_load && was_combined) { - hydra_teardown_combined_before_reload(); - } - - // Register any new RPC peer devices before load_model() so the - // peer's device exists in the global ggml backend registry when - // common_init_from_params() tries to place tensors per - // tensor_split/split_mode. Only genuinely new endpoints are - // registered (hydra_register_rpc_servers tracks already-registered - // endpoints to avoid unsafe repeated registration). - if (!g_pending_rpc_servers.empty()) { - json rpc_arr = json::array(); - for (const auto & s : g_pending_rpc_servers) { - rpc_arr.push_back(s); - } - hydra_register_rpc_servers(rpc_arr); - g_pending_rpc_servers.clear(); - } - - // Full model reload. load_model() handles the unload of the - // current model, the load of the new model, the new context - // creation, the MTP/draft paths, and the slot rebuild. - // NOTE: load_model() does `params_base = params` internally - // (line 844), so after a successful load params_base reflects - // swapped_params — no explicit reassignment needed by us. - // - // #507: Skip the fit_params probe during T3 rebuild. The probe - // does a full model-structure load with no_alloc=true to measure - // GPU memory — expensive (~45-90s) and unnecessary here because: - // (a) we just freed VRAM by destroying the old model, (b) the new - // model's requirements are known (same or smaller), (c) a controlled - // inference server has predictable VRAM. Disabling saves ~1 min. - swapped_params.fit_params = false; - if (!load_model(swapped_params)) { - if (is_first_load) { - SRV_WRN("%s", "hydra: T3 first load failed — engine stays empty\n"); - return false; - } - SRV_ERR("hydra: T3 reload to '%s' failed (load_model returned false); " - "rolling back to old model\n", - swapped_params.model.path.c_str()); - if (!load_model(old_params)) { - SRV_ERR("%s", "hydra: T3 rollback also failed — engine in unrecoverable state\n"); - GGML_ABORT("hydra: T3 rollback failed (cannot reload old model). " - "Engine exiting to prevent serving with corrupted state."); - } - SRV_INF("hydra: T3 rollback succeeded — restored old model '%s'\n", - old_params.model.path.c_str()); - return false; - } + // process the created batch of tokens + for (int32_t i = 0; i < batch.n_tokens; i = i_next) { + const int32_t n_tokens = std::min(n_batch, batch.n_tokens - i); - // Record the override_tensor that was just applied so the - // next call can skip the reload if nothing changed. - { - const char * cur = llama_hydra_get_pending_override_tensor(); - old_override_applied = cur ? cur : ""; - // hydra#470: also record the staged reload config (T2+T4 keys) - // that was just loaded, so a repeated identical CONFIGURE/decode - // payload skips the reload (early-exit above). - g_last_reload_config_applied = staged_reload; - } - - // T3 reload confirmed. Log model identity for traceability. - SRV_INF("hydra: T3 reload confirmed model_alias='%s' tokenizer='%s' model_name='%s' quant='%s' caps=0x%x model_path='%s'\n", - swapped_params.model_alias.empty() ? "?" : swapped_params.model_alias.begin()->c_str(), - model_tgt ? llama_model_get_tokenizer_model(model_tgt) : "", - model_tgt ? llama_model_get_display_name(model_tgt) : "", - model_tgt ? llama_model_get_quant_label(model_tgt) : "", - model_tgt ? llama_model_get_capabilities_bitfield(model_tgt) : 0, - swapped_params.model.path.c_str()); - - // COMBINED-mode reattach AFTER the model reload — see - // hydra_reattach_combined_after_reload() above. - if (was_combined) { - hydra_reattach_combined_after_reload(); - } - - SRV_INF("hydra: T3 rebuild applied (model='%s', split_mode=%d, n_gpu_layers=%d, slots=%zu)\n", - params_base.model.path.c_str(), (int) params_base.split_mode, - params_base.n_gpu_layers, slots.size()); - return true; - } + llama_batch batch_view = { + n_tokens, + batch.token + i, + nullptr, + batch.pos + i, + batch.n_seq_id + i, + batch.seq_id + i, + batch.logits + i, + }; - void update_slots() { - // check if all slots are idle - { - bool all_idle = true; + const int ret = llama_decode(ctx_tgt, batch_view); - for (auto & slot : slots) { - if (slot.is_processing() || slot.hydra_transferring->load()) { - all_idle = false; - break; - } - } + metrics.on_decoded(slots); - if (all_idle) { - SRV_INF("%s", "all slots are idle\n"); + if (ret != 0) { + { + std::string err; - // Hydra #406: slot-free moment — if a tiered CONFIGURE - // staged a T2/T3 rebuild, run the apply step now. The - // apply step (in apply_pending_hydra_config below) does - // the actual T2 context rebuild and/or T3 model reload, - // then clears the staged state. The low-level helper - // llama_hydra_apply_pending_config() is a no-op once the - // staged state has been cleared. - if (ctx_tgt && ctx_tgt->hydra_has_pending_config()) { - SRV_INF("hydra: slot-free moment — applying pending CONFIGURE (tier=%s)\n", - ctx_tgt->hydra_get_pending_config_tier().c_str()); - apply_pending_hydra_config(); - } else if (!ctx_tgt && first_load_pending) { - SRV_INF("%s", "hydra: slot-free moment — first load (no context yet)\n"); - apply_pending_hydra_config(); - } + if (n_batch == 1 && ret == 1) { + // TODO: try to terminate only the largest active slot/sequence and continue with the rest + // need to remove the tokens from the current batch too + err = "Context size has been exceeded."; + } - // #470 Option B: check if a peer reconnection was detected - // during graph_compute. If so, trigger a T3 rebuild to - // re-provision model layers on the fresh peer. - if (ctx_tgt && ctx_tgt->peer_reconnection_pending) { - ctx_tgt->peer_reconnection_pending = false; - SRV_WRN("%s", "hydra: peer reconnection detected — triggering T3 rebuild\n"); - // Force a T3 rebuild even if model config hasn't changed. - // The peer's buffers are gone, so we need to re-push. - if (!apply_t3_rebuild(true)) { - SRV_ERR("%s", "hydra: T3 rebuild after peer reconnection failed\n"); + if (ret == -1) { + err = "Invalid input batch."; } - } - return; - } - } + if (ret < -1) { + // TODO: update slot state based on llama_memory_seq_pos_min() and llama_memory_seq_pos_max() + err = "Compute error."; + } - { - SRV_DBG("%s", "posting NEXT_RESPONSE\n"); + // TODO: handle ret == 2 (abort) when we start aborting - server_task task(SERVER_TASK_TYPE_NEXT_RESPONSE); - task.id = queue_tasks.get_new_id(); - queue_tasks.post(std::move(task)); - } + if (!err.empty()) { + SRV_ERR("%s i = %d, n_batch = %d, ret = %d\n", err.c_str(), i, n_batch, ret); - // apply context-shift if needed - // TODO: simplify and improve - for (server_slot & slot : slots) { - if (slot.state == SLOT_STATE_GENERATING && slot.prompt.n_tokens() + 1 >= slot.n_ctx) { - if (!params_base.ctx_shift) { - // this check is redundant (for good) - // we should never get here, because generation should already stopped in process_token() - send_error(slot, "context shift is disabled", ERROR_TYPE_SERVER); - slot.release(); - continue; - } + for (auto & slot : slots) { + if (slot.is_processing() || slot.hydra_transferring->load()) { + send_error(slot, err); + slot.release(); - if (mctx) { - // we should never reach this because params_base.ctx_shift is automatically disabled if mmproj is loaded - // we don't support ctx_shift because an image chunk may contains multiple tokens - GGML_ABORT("not supported by multimodal"); - } + // note: it's complicated to keep track of how much of the current batch has been + // processed before the error occurred, so we simply clear the entire context + slot.prompt_clear(false); + } + } - if (slot.task->is_parent() || slot.task->is_child()) { - send_error(slot, "context shift cannot be used for shared prompt", ERROR_TYPE_SERVER); - slot.release(); - continue; + break; + } } - // Shift context - int n_keep = slot.task->params.n_keep < 0 ? slot.task->n_tokens() : slot.task->params.n_keep; - - if (add_bos_token) { - n_keep += 1; + // retry with half the batch size to try to find a free slot in the KV cache + if (!try_clear_idle_slots()) { + n_batch /= 2; } - n_keep = std::min(slot.n_ctx - 4, n_keep); + SRV_WRN("failed to find free space in the KV cache, retrying with smaller batch size, i = %d, n_batch = %d, ret = %d\n", i, n_batch, ret); - const int n_left = slot.prompt.n_tokens() - n_keep; - const int n_discard = slot.task->params.n_discard ? slot.task->params.n_discard : (n_left / 2); + continue; // continue loop of n_batch + } - SLT_WRN(slot, "slot context shift, n_keep = %d, n_left = %d, n_discard = %d\n", n_keep, n_left, n_discard); + // TODO: avoid restoring the draft context and re-evaluating the drafted tokens when not needed [TAG_SPEC_AVOID_DRAFT_REEVAL] + // for now, always re-evaluate for simplicity + // ref: https://github.com/ggml-org/llama.cpp/pull/22728#issuecomment-4400925384 + // + // | spec type | need re-eval | + // | --- | --- | + // | draft model | no | because the draft model does not use embeddings from the target + // | MTP (std) | yes | + // | MTP Gemma4 | no | because the KV cache is shared + // | Eagle3 | yes | + // | DFlash | yes | https://github.com/ggml-org/llama.cpp/pull/22728#issuecomment-4405406982 + // + // note: this logic is now moved in `common_speculative_process()` + // keeping the sketch here until for a bit, until the logic is finalized + // + //if (ctx_dft) { + // // TODO: update as needed for MTP, Eagle3, etc. + // const bool need_tgt_embd = false; - common_context_seq_rm (ctx_tgt, slot.id, n_keep , n_keep + n_discard); - common_context_seq_add(ctx_tgt, slot.id, n_keep + n_discard, slot.prompt.n_tokens(), -n_discard); + // if (need_tgt_embd) { + // llama_synchronize(ctx_tgt); + // } - // D4: seq_rm invalidates restored logits - slot.logits_valid = false; - slot.restored_logits.clear(); + // // the logic here varies depending on the speculative decoding method + // // - some draft contexts require embeddings from the target context, others don't + // // - some draft contexts involve an encoder step to transform the target embeddings to draft embeddings + // // TODO: extract this in a function ? + // { + // // TODO: hook the embeddings from the last target batch here + // if (llama_model_has_encoder(model_dft.get())) { + // //llama_encode(ctx_dft, ...); - if (ctx_dft) { - common_context_seq_rm (ctx_dft.get(), slot.id, n_keep , n_keep + n_discard); - common_context_seq_add(ctx_dft.get(), slot.id, n_keep + n_discard, slot.prompt.tokens.pos_next(), -n_discard); - } + // GGML_ABORT("not implemented yet\n"); + // } - // add generated tokens to cache - // ref: https://github.com/ggml-org/llama.cpp/pull/16818#discussion_r2473269481 - { - GGML_ASSERT(!slot.prompt.tokens.has_mtmd); - - llama_tokens new_tokens = slot.prompt.tokens.get_tokens(); // copy - for (size_t i = n_keep + n_discard; i < new_tokens.size(); i++) { - new_tokens[i - n_discard] = new_tokens[i]; - } + // const int ret = llama_decode(ctx_dft.get(), batch_view); - new_tokens.resize(slot.prompt.tokens.size() - n_discard); + // if (ret != 0) { + // SRV_ERR("failed to decode draft batch, ret = %d\n", ret); - slot.prompt.tokens.clear(); - slot.prompt.tokens.insert(new_tokens); - } + // // TODO: handle error + // break; + // } + // } + //} + if (!common_speculative_process(spec.get(), batch_view)) { + SRV_ERR("%s", "failed to process speculative batch\n"); - slot.truncated = true; + // TODO: handle error + break; } - } - // start populating the batch for this iteration - common_batch_clear(batch); + // move the head of the batch forward with the number of tokens we just processed + i_next = i + n_tokens; - // track if given slot can be batched with slots already in the batch - server_slot * slot_batched = nullptr; + // on successful decode, restore the original batch size + n_batch = llama_n_batch(ctx_tgt); - std::vector generating; - std::vector drafting; + // handle `n_cmpl > 1` tasks - when the main prompt is processed, activate all child tasks too + for (auto & slot : slots) { + if (slot.state == SLOT_STATE_DONE_PROMPT && slot.task->is_parent()) { + std::vector children; + for (auto & other : slots) { + if (other.state == SLOT_STATE_WAIT_OTHER && slot.task->id == other.task->id_parent) { + children.push_back(&other); + } + } - // determine which slots are generating and drafting - for (auto & slot : slots) { - if (slot.state != SLOT_STATE_GENERATING) { - continue; - } + // all children slots should already launched by launch_slots_with_parent_task() + // copy state to the child slots + for (auto & child : children) { + SLT_INF(slot, " - copying state to child %d\n", child->id); - // check if we can batch this slot with the previous one - if (!slot_batched) { - slot_batched = &slot; - } else if (!slot_batched->can_batch_with(slot)) { - continue; - } + GGML_ASSERT(child->state == SLOT_STATE_WAIT_OTHER); - generating.push_back(&slot); + slot.copy_state_to(*child); + child->state = SLOT_STATE_DONE_PROMPT; + } + } + } - if (spec) { - common_speculative_get_draft_params(spec.get(), slot.id).drafting = false; + for (auto & slot : slots) { + // optionally send prompt processing progress + if (slot.state == SLOT_STATE_PROCESSING_PROMPT || slot.state == SLOT_STATE_DONE_PROMPT) { + if (slot.task->params.stream && slot.task->params.return_progress) { + send_partial_response(slot, {}, true); + } + } - const bool use_ckpt_tgt = ctx_tgt_seq_rm_type == COMMON_CONTEXT_SEQ_RM_TYPE_FULL; - const bool use_ckpt_dft = ctx_dft_seq_rm_type == COMMON_CONTEXT_SEQ_RM_TYPE_FULL; + if (slot.i_batch < (int) i || slot.i_batch >= (int) (i + n_tokens)) { + continue; // continue loop of slots + } - const int n_draft_max = slot.get_n_draft_max(); + if (slot.state == SLOT_STATE_DONE_PROMPT) { + if (slot.task->type == SERVER_TASK_TYPE_EMBEDDING) { + // prompt evaluated for embedding + send_embedding(slot, batch_view); + slot.release(); + slot.i_batch = -1; + continue; // continue loop of slots + } - if (n_draft_max > 0) { - GGML_ASSERT(slot.can_speculate()); + if (slot.task->type == SERVER_TASK_TYPE_RERANK) { + send_rerank(slot, batch_view); + slot.release(); + slot.i_batch = -1; + continue; // continue loop of slots + } - if (!slot.spec_draft.empty()) { - // we have a previous (partial) draft to reuse - if (use_ckpt_tgt) { - GGML_ASSERT(!slot.spec_ckpt.empty()); - } - } else { - GGML_ASSERT(slot.spec_i_batch.empty()); + GGML_ASSERT(slot.task->need_sampling()); - slot.spec_ckpt.update_pos( - slot.prompt.n_tokens(), - llama_memory_seq_pos_min(llama_get_memory(ctx_tgt), slot.id), - llama_memory_seq_pos_max(llama_get_memory(ctx_tgt), slot.id)); + // prompt evaluated for next-token prediction + slot.state = SLOT_STATE_GENERATING; - if (use_ckpt_dft) { - slot.spec_ckpt.update_dft(ctx_dft.get(), slot.id, LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY | LLAMA_STATE_SEQ_FLAGS_ON_DEVICE); - } + if (slot.can_speculate()) { + common_speculative_begin(spec.get(), slot.id, slot.prompt.tokens.get_text_tokens()); + } + } else if (slot.state != SLOT_STATE_GENERATING) { + continue; // continue loop of slots + } - slot.spec_prompt = slot.prompt.tokens.get_text_tokens(); + if (slot.can_speculate() && !slot.spec_draft.empty()) { + continue; // sample using speculative decoding + } - common_speculative_get_draft_params(spec.get(), slot.id) = { - /* .drafting = */ true, - /* .n_max = */ n_draft_max, - /* .n_past = */ slot.prompt.n_tokens(), - /* .id_last = */ slot.sampled, - /* .prompt = */ &slot.spec_prompt, - /* .result = */ &slot.spec_draft, - }; + const int tok_idx = slot.i_batch - i; - drafting.push_back(&slot); + // D4: Inject per-slot restored logits into the correct batch row before + // first sample. The sampler reads via llama_get_logits_ith(ctx, tok_idx), + // so we must write to that exact row — not row 0. + if (slot.logits_valid && !slot.restored_logits.empty() && slot.n_decoded == 0) { + float * row_logits = llama_get_logits_ith(slot.ctx_tgt, tok_idx); + if (row_logits) { + const size_t n_floats = slot.restored_logits.size(); + memcpy(row_logits, slot.restored_logits.data(), n_floats * sizeof(float)); + SLT_INF(slot, "consumed %zu restored logits into row %d (tok_idx)\n", n_floats, tok_idx); + } else { + SLT_WRN(slot, "restored logits skipped: llama_get_logits_ith returned null for row %d\n", tok_idx); } + slot.restored_logits.clear(); + slot.logits_valid = false; } - } - } - // generate the actual drafts (if any) - { - common_speculative_draft(spec.get()); - } + llama_token id = common_sampler_sample(slot.smpl.get(), slot.ctx_tgt, tok_idx); - // make checkpoints if needed - for (auto * slot_ptr : drafting) { - auto & slot = *slot_ptr; + slot.i_batch = -1; - auto & draft = slot.spec_draft; - auto & ckpt = slot.spec_ckpt; + common_sampler_accept(slot.smpl.get(), id, true); - slot.n_draft_total += draft.size(); + // here we have synchronized the llama_context (due to the sampling above), so we can do time measurement + const int64_t t_current = ggml_time_us(); - // TODO: avoid restoring the draft context and re-evaluating the drafted tokens when not needed [TAG_SPEC_AVOID_DRAFT_REEVAL] - const bool use_ckpt_dft = ctx_dft_seq_rm_type == COMMON_CONTEXT_SEQ_RM_TYPE_FULL; + slot.n_decoded += 1; - if (ctx_dft) { - if (use_ckpt_dft) { - ckpt.load_dft(ctx_dft.get(), slot.id, LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY | LLAMA_STATE_SEQ_FLAGS_ON_DEVICE); + if (slot.n_decoded == 1) { + slot.t_start_generation = t_current; + slot.t_prompt_processing = (slot.t_start_generation - slot.t_start_process_prompt) / 1e3; + metrics.on_prompt_eval(slot); } - common_context_seq_rm(ctx_dft.get(), slot.id, ckpt.pos_max + 1, -1); - } - - if (!draft.empty()) { - const bool use_ckpt_tgt = - ctx_tgt_seq_rm_type == COMMON_CONTEXT_SEQ_RM_TYPE_FULL || - (ctx_tgt_seq_rm_type == COMMON_CONTEXT_SEQ_RM_TYPE_RS && draft.size() > llama_n_rs_seq(ctx_tgt)); - - const bool use_ckpt_dft = - (ctx_dft_seq_rm_type == COMMON_CONTEXT_SEQ_RM_TYPE_RS && draft.size() > llama_n_rs_seq(ctx_dft.get())); + slot.t_token_generation = std::max(1, t_current - slot.t_start_generation) / 1e3; - if (use_ckpt_tgt) { - //const int64_t t_start = ggml_time_us(); + completion_token_output result; + result.tok = id; + result.text_to_send = common_token_to_piece(slot.ctx_tgt, result.tok, accept_special_token(slot, result.tok)); + result.prob = 1.0f; // TODO: set it here instead of doing inside populate_token_probs - ckpt.update_tgt(ctx_tgt, slot.id, LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY | LLAMA_STATE_SEQ_FLAGS_ON_DEVICE); + if (slot.task->params.sampling.n_probs > 0) { + populate_token_probs(slot, result, slot.task->params.post_sampling_probs, params_base.special, tok_idx); + } - //const int64_t t_total = ggml_time_us() - t_start; - //printf("checkpoint total: %f ms\n", t_total / 1000.0); + if (!process_token(result, slot)) { + // release slot because of stop condition + slot.print_timings(); + send_final_response(slot); + metrics.on_prediction(slot); + slot.release(); - SLT_DBG(slot, "created speculative checkpoint (pos_min = %d, pos_max = %d, n_tokens = %d, size = %.3f MiB, draft = %.3f MiB)\n", - ckpt.pos_min, ckpt.pos_max, slot.prompt.n_tokens(), - (float) ckpt.size() / 1024 / 1024, - (float) ckpt.data_dft.size() / 1024 / 1024); + continue; } - if (use_ckpt_dft) { - ckpt.update_dft(ctx_dft.get(), slot.id, LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY | LLAMA_STATE_SEQ_FLAGS_ON_DEVICE); - } + slot.print_timings_tg(); } - } - - // update the batch with the sampled/drafted tokens - for (auto * slot_ptr : generating) { - auto & slot = *slot_ptr; - - slot.update_batch(batch); - } - - // process in chunks of params.n_batch - int32_t n_batch = llama_n_batch(ctx_tgt); - int32_t n_ubatch = llama_n_ubatch(ctx_tgt); - - float alora_scale = -1.0f; - size_t alora_disabled_id = 0; - // next, batch any pending prompts without exceeding n_batch - if (params_base.cont_batching || batch.n_tokens == 0) { + // speculative decoding - main model sample and accept for (auto & slot : slots) { - if (!slot.is_processing()) { + if (slot.state != SLOT_STATE_GENERATING || !slot.can_speculate() || slot.spec_draft.empty()) { continue; } - // check if we can batch this slot with the previous one - if (slot_batched && !slot_batched->can_batch_with(slot)) { - continue; - } + // save the original draft size + const size_t n_draft = slot.spec_draft.size(); - // check if this is a child slot - if (slot.state == SLOT_STATE_WAIT_OTHER) { - SLT_DBG(slot, "%s", "waiting for parent slot to complete\n"); - continue; - } + GGML_ASSERT(n_draft > 0); - // this slot still has a prompt to be processed - if (slot.state == SLOT_STATE_PROCESSING_PROMPT || slot.state == SLOT_STATE_STARTED) { - const auto & input_tokens = slot.task->tokens; + // verify and try to accept the draft + { + // save the sampler sampler state in case we need to restore it + common_sampler_ptr smpl_save(common_sampler_clone(slot.smpl.get())); - // #469 trace: log input tokens for cross-flow comparison - { - std::string tok_ids; - for (size_t i = 0; i < std::min(16, input_tokens.size()); ++i) { - if (i > 0) tok_ids += ","; - tok_ids += std::to_string(input_tokens[i]); - } - SLT_DBG(slot, "#PD-TRACE HTTP_COMPLETION slot=%d input_tokens_first16=[%s] input_total=%zu cached=%d just_restored=%d\n", - slot.id, tok_ids.c_str(), input_tokens.size(), - slot.n_prompt_tokens_cache, slot.just_restored); - } - - // used to determine the number of tokens added to the batch for the current slot - const auto n_tokens_prev = batch.n_tokens; + GGML_ASSERT(slot.spec_i_batch.size() == n_draft + 1); + auto accepted = common_sampler_sample_and_accept_n(slot.smpl.get(), slot.ctx_tgt, slot.spec_i_batch, slot.spec_draft); + slot.spec_i_batch.clear(); - // TODO: maybe move branch to outside of this loop in the future - if (slot.state == SLOT_STATE_STARTED) { - slot.t_start_process_prompt = ggml_time_us(); - slot.t_start_generation = 0; + GGML_ASSERT(accepted.size() >= 1); - slot.state = SLOT_STATE_PROCESSING_PROMPT; + const uint32_t n_rollback = slot.spec_draft.size() + 1 - accepted.size(); - SLT_TRC(slot, "new prompt, n_ctx_slot = %d, n_keep = %d, task.n_tokens = %d\n", - slot.n_ctx, slot.task->params.n_keep, slot.task->n_tokens()); + const bool use_ckpt_tgt = + ctx_tgt_seq_rm_type == COMMON_CONTEXT_SEQ_RM_TYPE_FULL || + (ctx_tgt_seq_rm_type == COMMON_CONTEXT_SEQ_RM_TYPE_RS && n_rollback > llama_n_rs_seq(ctx_tgt)); - // print prompt tokens (for debugging) - /*if (1) { - // first 16 tokens (avoid flooding logs) - for (int i = 0; i < std::min(16, input_tokens.size()); i++) { - SLT_DBG(slot, "prompt token %3d: %6d '%s'\n", i, input_tokens[i], common_token_to_piece(ctx_tgt, input_tokens[i]).c_str()); - } - } else { - // all - for (int i = 0; i < (int) input_tokens.size(); i++) { - SLT_DBG(slot, "prompt token %3d: %6d '%s'\n", i, input_tokens[i], common_token_to_piece(ctx_tgt, input_tokens[i]).c_str()); + // check for partial draft acceptance + if (n_rollback > 0) { + if (use_ckpt_tgt) { + if (trace > 0) { + SLT_INF(slot, "accepted %2zu/%2zu draft tokens (restore checkpoint)\n", accepted.size() - 1, slot.spec_draft.size()); } - }*/ - - // keep track how many tokens we can reuse from the previous state - int n_past = 0; - // empty prompt passed -> release the slot and send empty response - if (input_tokens.empty()) { - SLT_WRN(slot, "%s", "empty prompt - releasing slot\n"); + // partial acceptance is not supported by the context -> truncate the draft and restore the state + slot.spec_draft = std::move(accepted); - slot.print_timings(); - send_final_response(slot); - slot.release(); + const auto & ckpt = slot.spec_ckpt; - continue; - } + SLT_DBG(slot, "restoring speculative checkpoint (pos_min = %d, pos_max = %d, size = %zu)\n", ckpt.pos_min, ckpt.pos_max, ckpt.size()); - // TODO: support memory-less logits computation - if (slot.task->need_logits() && !llama_get_memory(ctx_tgt)) { - send_error(slot, "the current context does not logits computation. skipping", ERROR_TYPE_SERVER); - slot.release(); - continue; - } + { + ckpt.load_tgt(slot.ctx_tgt, slot.id, LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY | LLAMA_STATE_SEQ_FLAGS_ON_DEVICE); - if (!slot.can_split()) { - if (slot.task->n_tokens() > n_ubatch) { - send_error(slot, - string_format( - "input (%d tokens) is too large to process. increase the physical batch " - "size (current batch size: %d)", - slot.task->n_tokens(), n_ubatch), - ERROR_TYPE_SERVER); - slot.release(); - continue; + common_context_seq_rm(slot.ctx_tgt, slot.id, ckpt.pos_max + 1, -1); } - if (slot.task->n_tokens() > slot.n_ctx) { - send_error( - slot, - string_format( - "input (%d tokens) is larger than the max context size (%d tokens). skipping", - slot.task->n_tokens(), slot.n_ctx), - ERROR_TYPE_EXCEED_CONTEXT_SIZE); - slot.release(); - continue; - } - } else { - if (slot.task->n_tokens() >= slot.n_ctx) { - send_error(slot, - string_format("request (%d tokens) exceeds the available context size (%d " - "tokens), try increasing it", - slot.task->n_tokens(), slot.n_ctx), - ERROR_TYPE_EXCEED_CONTEXT_SIZE); - slot.release(); - continue; + if (slot.ctx_dft) { + ckpt.load_dft(slot.ctx_dft, slot.id, LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY | LLAMA_STATE_SEQ_FLAGS_ON_DEVICE); + + common_context_seq_rm(slot.ctx_dft, slot.id, ckpt.pos_max + 1, -1); } - if (slot.task->params.cache_prompt) { - // reuse any previously computed tokens that are common with the new prompt - n_past = slot.prompt.tokens.get_common_prefix(input_tokens); + slot.prompt.tokens.keep_first(ckpt.n_tokens); + slot.smpl = std::move(smpl_save); - // #469 trace: log common prefix for cross-flow comparison - SLT_DBG(slot, "#PD-TRACE COMMON_PREFIX slot=%d n_past=%d cached=%d input_total=%zu just_restored=%d\n", - slot.id, n_past, slot.n_prompt_tokens_cache, input_tokens.size(), slot.just_restored); - { - // Log cached tokens if any - if (slot.n_prompt_tokens_cache > 0) { - std::string cached_ids; - for (int i = 0; i < std::min(16, slot.n_prompt_tokens_cache); ++i) { - if (i > 0) cached_ids += ","; - cached_ids += std::to_string(slot.prompt.tokens[i]); - } - SLT_DBG(slot, "#PD-TRACE CACHED_TOKENS slot=%d first16=[%s] total=%d\n", - slot.id, cached_ids.c_str(), slot.n_prompt_tokens_cache); - } - // Log first mismatch point when common prefix < input size - if (n_past < (int)input_tokens.size()) { - if (n_past < (int)slot.prompt.tokens.size()) { - SLT_WRN(slot, "#PD-TRACE MISMATCH slot=%d n_past=%d cached=%d input_total=%zu stored_tok[%d]=%d input_tok[%d]=%d\n", - slot.id, n_past, slot.n_prompt_tokens_cache, input_tokens.size(), - n_past, slot.prompt.tokens[n_past], - n_past, input_tokens[n_past]); - } else { - SLT_WRN(slot, "#PD-TRACE MISMATCH slot=%d n_past=%d cached=%d input_total=%zu stored_size=%zu input exceeds stored\n", - slot.id, n_past, slot.n_prompt_tokens_cache, input_tokens.size(), - slot.prompt.tokens.size()); - } - // Log first 16 of both token lists for comparison - { - std::string stored_ids, input_ids; - for (int i = 0; i < std::min(16, (int)slot.prompt.tokens.size()); ++i) { - if (i > 0) stored_ids += ","; - stored_ids += std::to_string(slot.prompt.tokens[i]); - } - for (size_t i = 0; i < std::min(16, input_tokens.size()); ++i) { - if (i > 0) input_ids += ","; - input_ids += std::to_string(input_tokens[i]); - } - SLT_WRN(slot, "#PD-TRACE MISMATCH_STORED slot=%d first16=[%s] total=%zu\n", - slot.id, stored_ids.c_str(), slot.prompt.tokens.size()); - SLT_WRN(slot, "#PD-TRACE MISMATCH_INPUT slot=%d first16=[%s] total=%zu\n", - slot.id, input_ids.c_str(), input_tokens.size()); - } - } - } + continue; + } + } - // ── Hydra n_common decision rule ────────────── - // n_slot = tokens resident in KV (blob, prior turn, or prefill). - // n_new = tokens in the incoming prompt. - // n_common = length of common prefix (== n_past before alora). - // - // SAFETY: Every slot must produce at least one batch row to - // reach common_sampler_sample() (server-context.cpp:6591). - // The batch-assembly loop [6170] adds tokens in - // [slot.prompt.n_tokens(), n_new). When n_past == n_new - // (zero-prompt decode) that range is empty, leaving the slot - // with no batch row, no slot.i_batch, and a failed - // GGML_ASSERT(batch.n_tokens > 0) at [6240]. All branches - // therefore set n_past < n_new to ensure at least one token - // enters the batch; for the logits_reused path the restored - // logits are injected before sampling (line 6578), so the - // model-computed logits for that row are safely overwritten. - { - const int n_slot = (int) slot.prompt.tokens.size(); - const int n_new = (int) input_tokens.size(); - const int n_common_val = n_past; // before alora adjustment - const bool logits_valid = slot.logits_valid; + if (trace > 0) { + SLT_INF(slot, "accepted %2zu/%2zu draft tokens\n", accepted.size() - 1, n_draft); + } - slot.n_common = n_common_val; - slot.logits_reused = false; - slot.n_prompt_processed = 0; + common_speculative_accept(spec.get(), slot.id, accepted.size() - 1); - SLT_DBG(slot, "#PD-TRACE N_COMMON slot=%d n_slot=%d n_new=%d n_common=%d logits_valid=%d just_restored=%d\n", - slot.id, n_slot, n_new, n_common_val, (int) logits_valid, (int) slot.just_restored); + slot.spec_draft = std::move(accepted); + } - if (n_common_val == n_new && logits_valid) { - // Full prompt matches resident KV and restored logits are - // valid. Re-decode the final token so the slot gets a - // batch row (required to reach common_sampler_sample); - // the restored logits are injected before sampling, - // overwriting the model-computed logits for that row. - // NOTE: n_past cannot be set to n_new here because the - // batch-assembly loop at [6170] only adds tokens in - // [slot.prompt.n_tokens(), n_new) — with n_past == n_new - // zero tokens would be added, leaving the slot with no - // batch row, no slot.i_batch, and a failed assertion at - // GGML_ASSERT(batch.n_tokens > 0) [6240]. - n_past = n_new - 1; - slot.logits_reused = true; - slot.n_prompt_processed = 1; - SLT_INF(slot, "#PD-TRACE N_COMMON zero-prompt decode n_common=%d logits_reused=true (1-token batch row)\n", n_common_val); - } else if (n_common_val == n_new && !logits_valid) { - // Full prompt matches but restored logits are stale or - // absent. Re-decode the final token to regenerate logits - // (the "1-token trick"). DEFAULT for warm/COMBINED. - n_past = n_common_val - 1; - slot.n_prompt_processed = 1; - SLT_INF(slot, "#PD-TRACE N_COMMON 1-token re-decode n_common=%d logits_reused=false\n", n_common_val); - } else if (n_common_val < n_slot) { - // Partial match — trim KV from divergence point. - // Discard restored logits FIRST (any seq_rm invalidates them). - slot.logits_valid = false; - slot.logits_reused = false; - n_past = n_common_val; - SLT_INF(slot, "#PD-TRACE N_COMMON trim n_common=%d < n_slot=%d, logits discarded\n", n_common_val, n_slot); - } else { - // Normal: n_common < n_new. Process [n_common, n_new). - n_past = n_common_val; - slot.logits_reused = false; - } - } + const int64_t t_current = ggml_time_us(); - // if there is an alora invoked, don't cache after the invocation start - if (slot.alora_invocation_start > 0) { - SLT_DBG(slot, "only caching to alora invocation start (n_past = %d, alora_invocation_start = %d)\n", n_past, slot.alora_invocation_start); - n_past = std::min(n_past, slot.alora_invocation_start - 1); - } + const auto ids = std::move(slot.spec_draft); - const auto n_cache_reuse = slot.task->params.n_cache_reuse; + slot.t_token_generation = std::max(1, t_current - slot.t_start_generation) / 1e3; - const bool can_cache_reuse = - llama_memory_can_shift(llama_get_memory(ctx_tgt)) && - !slot.prompt.tokens.has_mtmd; + // update how many tokens out of those tested were accepted + slot.n_draft_accepted += ids.size() - 1; - if (!can_cache_reuse && n_cache_reuse > 0) { - SLT_WRN(slot, "cache reuse is not supported - ignoring n_cache_reuse = %d\n", n_cache_reuse); - } + // add accepted tokens to the prompt + slot.prompt.tokens.keep_first(slot.prompt.n_tokens() - n_draft); + slot.prompt.tokens.insert({ids.begin(), ids.end() - 1}); - // reuse chunks from the cached prompt by shifting their KV cache in the new position - if (can_cache_reuse && n_cache_reuse > 0) { - GGML_ASSERT(!slot.prompt.tokens.has_mtmd); + slot.sampled = ids.back(); // last accepted token + SLT_DBG(slot, "add accepted tokens: sampled=%d, ids.size=%zu, n_draft=%zu\n", slot.sampled, ids.size(), n_draft); - size_t head_c = n_past; // cache - size_t head_p = n_past; // current prompt + common_context_seq_rm(slot.ctx_tgt, slot.id, slot.prompt.tokens.pos_next(), -1); + if (slot.ctx_dft) { + common_context_seq_rm(slot.ctx_dft, slot.id, slot.prompt.tokens.pos_next(), -1); + } - if (mctx) { - // we should never reach this - GGML_ABORT("not supported by multimodal"); - } + for (size_t i = 0; i < ids.size(); ++i) { + completion_token_output result; - SLT_DBG(slot, "trying to reuse chunks with size > %d, n_past = %d\n", n_cache_reuse, n_past); + result.tok = ids[i]; + result.text_to_send = common_token_to_piece(slot.ctx_tgt, result.tok, accept_special_token(slot, result.tok)); + result.prob = 1.0f; // set later - while (head_c < slot.prompt.tokens.size() && - head_p < input_tokens.size()) { + // TODO: set result.probs - size_t n_match = 0; - while (head_c + n_match < slot.prompt.tokens.size() && - head_p + n_match < input_tokens.size() && - slot.prompt.tokens[head_c + n_match] == input_tokens[head_p + n_match]) { - n_match++; - } + slot.n_decoded += 1; - if (n_match >= (size_t) n_cache_reuse) { - SLT_TRC(slot, "reusing chunk with size %zu, shifting KV cache [%zu, %zu) -> [%zu, %zu)\n", n_match, head_c, head_c + n_match, head_p, head_p + n_match); - //for (size_t i = head_p; i < head_p + n_match; i++) { - // SLT_DBG(slot, "cache token %3zu: %6d '%s'\n", i, prompt_tokens[i], common_token_to_piece(ctx_tgt, prompt_tokens[i]).c_str()); - //} + if (!process_token(result, slot)) { + slot.print_timings(); + send_final_response(slot); + metrics.on_prediction(slot); + slot.release(); - const int64_t kv_shift = (int64_t) head_p - (int64_t) head_c; + break; + } + } - common_context_seq_rm (ctx_tgt, slot.id, head_p, head_c); - common_context_seq_add(ctx_tgt, slot.id, head_c, head_c + n_match, kv_shift); + slot.print_timings_tg(); - if (ctx_dft) { - common_context_seq_rm (ctx_dft.get(), slot.id, head_p, head_c); - common_context_seq_add(ctx_dft.get(), slot.id, head_c, head_c + n_match, kv_shift); - } + SLT_DBG(slot, "accepted %d/%d draft tokens, new n_tokens = %d\n", (int) ids.size() - 1, (int) n_draft, slot.prompt.n_tokens()); + } + } - for (size_t i = 0; i < n_match; i++) { - slot.prompt.tokens.set_token(head_p + i, slot.prompt.tokens[head_c + i]); - n_past++; - } + SRV_DBG("%s", "run slots completed\n"); + } - head_c += n_match; - head_p += n_match; - } else { - head_c += 1; - } - } + int get_slot_n_ctx() { + return slots.back().n_ctx; + } - SLT_DBG(slot, "after context reuse, new n_past = %d\n", n_past); - } - } else { - // if we don't cache the prompt, we have to remove all previous tokens - n_past = 0; - } + server_response_reader get_response_reader() { + return server_response_reader(queue_tasks, queue_results, HTTP_POLLING_SECONDS); + } - llama_pos pos_next = slot.prompt.tokens.pos_next(n_past); + void process_single_task(server_task && task) { + // epic #610 WS1: in seam mode the extension may claim Hydra tasks. + // WS1 impl is a no-op (returns false), so this is a pure A/B switch — + // both modes run the inline dispatch below. + if (hydra_ext_active && hydra_ext && hydra_ext->handle_task(*this, task)) { + return; + } + switch (task.type) { + case SERVER_TASK_TYPE_COMPLETION: + case SERVER_TASK_TYPE_INFILL: + case SERVER_TASK_TYPE_EMBEDDING: + case SERVER_TASK_TYPE_RERANK: + { + // special case: if input is provided via CLI, tokenize it first + // otherwise, no need to tokenize as it's already done inside the HTTP thread + if (task.cli) { + if (!tokenize_cli_input(task)) { + break; + } + } - // the largest pos_min required for a checkpoint to be useful - const auto pos_min_thold = std::max(0, pos_next - n_swa - 1); + // Hydra config from HTTP decode path: apply synchronously + // on the task-queue thread before any slot scheduling or + // generation work. This is safe because we own this thread; + // the previous attempt applied on the httplib worker thread + // and raced the main queue (reverted in ebbbe1116). + if (!task.hydra_config_json.empty()) { + json hydra_cfg; + try { + hydra_cfg = json::parse(task.hydra_config_json); + } catch (const std::exception & e) { + SRV_WRN("hydra: COMPLETION hydra_config parse failed: %s\n", e.what()); + } + if (!hydra_cfg.is_null() && hydra_cfg.is_object()) { + SRV_INF("hydra: COMPLETION applying hydra_config (%zu keys)\n", + hydra_cfg.size()); + hydra_config_result cfg_result = hydra_apply_config(hydra_cfg, /*sync=*/true); + if (!cfg_result.ok) { + SRV_WRN("hydra: COMPLETION hydra_config apply failed: %s\n", + cfg_result.error.c_str()); + } + // After T3 rebuild, model/slots are reset. + // The slot lookup below will pick up the new state. + } + } - if (n_past > 0 && n_past <= slot.prompt.n_tokens()) { - const auto pos_min = llama_memory_seq_pos_min(llama_get_memory(ctx_tgt), slot.id); - if (pos_min == -1) { - SLT_ERR(slot, "n_past = %d, slot.prompt.tokens.size() = %d, seq_id = %d, pos_min = %d\n", n_past, (int) slot.prompt.tokens.size(), slot.id, pos_min); - GGML_ABORT("pos_min == -1, but n_past > 0 - should not happen: https://github.com/ggml-org/llama.cpp/pull/13833#discussion_r2116181237"); - } + const int id_slot = task.id_slot; + const int id_task = task.id; - // Hydra #641: post-decode KV restore (STATE_PUT / merged DECODE) freezes the - // slot's checkpoint at PREFILL end — checkpoints are only created during prompt - // processing, and the prompt loop breaks 4+n_ubatch/4 tokens early — so on the - // NEXT continuation that stale early checkpoint matches (is_rec: pos_max <= pos_next) - // and load_tgt() overwrites the whole sequence state (attention + SSM) with the - // old snapshot, re-prefilling ~800-1400 already-cached tokens (5.5s on RTX, 51s on - // P100 for the warm-affinity turn 2). A pure extension — the whole cache is a - // strict prefix of the new prompt and memory really ends at pos_next-1 — must not - // enter the checkpoint search. Logic is pinned by - // tests/test-hydra-checkpoint-policy.cpp (see server_should_rewind_to_checkpoint). - const auto pos_max_mem = llama_memory_seq_pos_max(llama_get_memory(ctx_tgt), slot.id); - const bool no_rewind_needed = !server_should_rewind_to_checkpoint( - n_past, - (llama_pos) slot.prompt.n_tokens(), - (llama_pos) slot.task->n_tokens(), - pos_next, - pos_max_mem); + server_slot * slot = id_slot != -1 ? get_slot_by_id(id_slot) : get_available_slot(task); - // when the prompt prefix does not match, print the tokens around the mismatch - // this is useful for debugging prompt caching - if (slots_debug) { - const int np0 = std::max(n_past - 4, 0); - const int np1 = std::min(n_past + 6, std::min(slot.prompt.tokens.size(), slot.task->tokens.size())); + // + // slot scheduling logic + // - std::stringstream ss0; - std::stringstream ss1; + if (slot == nullptr) { + // if no slot is available, we defer this task for processing later + SRV_DBG("no slot is available, defer task, id_task = %d\n", id_task); + queue_tasks.defer(std::move(task)); + break; + } - std::stringstream st0; - std::stringstream st1; + if (slot->is_processing() || slot->hydra_transferring->load()) { + // if requested slot is unavailable, we defer this task for processing later + SRV_DBG("requested slot is unavailable, defer task, id_task = %d\n", id_task); + queue_tasks.defer(std::move(task)); + break; + } - ss0 << "old: ... "; - ss1 << "new: ... "; + if (task.is_parent()) { + // try getting free slots for all child tasks + size_t n_child_tasks = task.child_tasks.size(); + std::vector child_slots = get_free_slots(n_child_tasks, slot->id); + if (child_slots.size() < n_child_tasks) { + SRV_DBG("not enough free slots for child tasks, n_free = %zu, n_children = %zu, defer task, id_task = %d\n", child_slots.size(), n_child_tasks, id_task); + queue_tasks.defer(std::move(task)); + break; + } + if (!launch_slots_with_parent_task(*slot, child_slots, std::move(task))) { + SRV_ERR("failed to launch slot with parent task, id_task = %d\n", id_task); + break; // drop the task + } + } else if (!launch_slot_with_task(*slot, std::move(task))) { + SRV_ERR("failed to launch slot with task, id_task = %d\n", id_task); + break; // drop the task + } - for (int i = np0; i < np1; i++) { - if (i == n_past) { - ss0 << " | "; - ss1 << " | "; - } + if (params_base.cache_idle_slots) { + for (auto & s : slots) { + if (!s.is_processing() && !s.hydra_transferring->load()) { + slot_save_and_clear(s); + } + } + } + } break; + case SERVER_TASK_TYPE_CANCEL: + { + // release slot linked with the task id + for (auto & slot : slots) { + if (slot.task && slot.task->id == task.id_target) { + slot.release(); + break; + } + } + } break; + case SERVER_TASK_TYPE_CONTROL: + { + auto res = std::make_unique(); + res->id = task.id; - { - const auto token = slot.prompt.tokens[i]; - const auto piece = token != LLAMA_TOKEN_NULL ? common_token_to_piece(ctx_tgt, token) : "[mtmd]"; - ss0 << piece; - st0 << std::setw(8) << token; - } + server_slot * slot = get_slot_by_cmpl_id(task.params.control_cmpl_id); + if (slot == nullptr) { + res->success = false; + res->message = "no active completion for this id"; + queue_results.send(std::move(res)); + break; + } - { - const auto token = slot.task->tokens[i]; - const auto piece = token != LLAMA_TOKEN_NULL ? common_token_to_piece(ctx_tgt, token) : "[mtmd]"; - ss1 << piece; - st1 << std::setw(8) << token; - } - } + if (task.params.control_action == "reasoning_end") { + // the budget sampler only exists when reasoning control was armed + if (!slot->task->params.sampling.reasoning_control) { + res->success = false; + res->message = "reasoning control not enabled for this completion"; + queue_results.send(std::move(res)); + break; + } + // act on the live slot mid generation, never defer + common_sampler_reasoning_budget_force(slot->smpl.get()); + res->success = true; + } else { + res->success = false; + res->message = "unknown control action"; + } - SLT_WRN(slot, "%s\n", ss0.str().c_str()); - SLT_WRN(slot, "%s\n", ss1.str().c_str()); + queue_results.send(std::move(res)); + } break; + case SERVER_TASK_TYPE_NEXT_RESPONSE: + { + // do nothing + } break; + case SERVER_TASK_TYPE_METRICS: + { + json slots_data = json::array(); - SLT_WRN(slot, "%s\n", st0.str().c_str()); - SLT_WRN(slot, "%s\n", st1.str().c_str()); - } + int n_idle_slots = 0; + int n_processing_slots = 0; - if (pos_min >= pos_min_thold && !no_rewind_needed) { - // For recurrent/hybrid models (e.g. Qwen3.x MTP) a checkpoint's - // pos_min equals the full sequence length, so the usual - // `pos_min < pos_min_thold` test is perpetually false → every turn - // force-re-prefills. Match on pos_max <= pos_next instead so cached - // KV is reused. Ref: ik_llama.cpp#1762 (port). - const bool is_rec = llama_model_is_recurrent(model_tgt) || - llama_model_is_hybrid(model_tgt); - // search for a context checkpoint - const auto it = std::find_if( - slot.prompt.checkpoints.rbegin(), - slot.prompt.checkpoints.rend(), - [&, func_name = __func__](const auto & cur) { - // guarantee that a checkpoint will result in at least one token being processed [TAG_PROMPT_LOGITS] - LOG_INF("slot %12.*s: id %2d | task %d | Checking checkpoint with [%d, %d] against %d...\n", 12, - func_name, (slot).id, ((slot).task ? (slot).task->id : -1), cur.pos_min, cur.pos_max, pos_min_thold); - if (is_rec) { - return cur.pos_max <= pos_next; - } - return cur.pos_min < pos_min_thold || cur.pos_min == 0; - } - ); + for (server_slot & slot : slots) { + json slot_data = slot.to_json(slots_debug == 0); - bool do_reset = it == slot.prompt.checkpoints.rend(); + if (slot.is_processing() || slot.hydra_transferring->load()) { + n_processing_slots++; + } else { + n_idle_slots++; + } - // #469 trace: log checkpoint search result and just_restored decision - SLT_DBG(slot, "#PD-TRACE CHECKPOINT_SEARCH slot=%d do_reset=%d just_restored=%d n_past=%d checkpoints=%zu pos_next=%d\n", - slot.id, do_reset, slot.just_restored, n_past, slot.prompt.checkpoints.size(), pos_next); + slots_data.push_back(slot_data); + } + SRV_DBG("n_idle_slots = %d, n_processing_slots = %d\n", n_idle_slots, n_processing_slots); - // For slots restored via STATE_PUT (full context state), - // skip the checkpoint search entirely. The restored state - // already has the correct KV cache + logits. The checkpoint - // check is needed for in-server reuse across turns, not for - // cross-node migration where the full state is restored. - if (do_reset && slot.just_restored && n_past > 0) { - SLT_WRN(slot, "STATE_PUT restored slot — using cached n_past=%d, skipping checkpoint check\n", n_past); - do_reset = false; - pos_next = n_past; - slot.just_restored = false; - } + auto res = std::make_unique(); + res->id = task.id; + res->slots_data = std::move(slots_data); + res->n_idle_slots = n_idle_slots; + res->n_processing_slots = n_processing_slots; + res->n_tasks_deferred = queue_tasks.queue_tasks_deferred_size(); + res->t_start = metrics.t_start; - if (!do_reset) { - if (it != slot.prompt.checkpoints.rend()) { - // restore the context checkpoint - if (it->is_recr_only) { - // Hydra M2 wire checkpoint (v3): the buffer holds a - // recurrent-only (PARTIAL_ONLY) capture. The recurrent - // (SSM) state is restored with matched PARTIAL_ONLY flags - // (flag symmetry — a PARTIAL_ONLY buffer physically lacks - // the mem_attn bytes). The attention cache must be trimmed - // at pos_max first: after the full live-state restore it - // still holds cells past the checkpoint position, and a - // full restore has no attention bytes to overwrite them. - // seq_rm is called on the ATTENTION cache directly — the - // blanket llama_memory_hybrid::seq_rm would wipe the - // recurrent cell first (n_rs_seq == 0 rollback path). - if (llama_model_is_hybrid(model_tgt)) { - ((llama_memory_hybrid *) llama_get_memory(ctx_tgt))->get_mem_attn()->seq_rm(slot.id, it->pos_max, -1); - } - it->load_tgt_recr(ctx_tgt, slot.id); + res->n_prompt_tokens_processed_total = metrics.n_prompt_tokens_processed_total; + res->t_prompt_processing_total = metrics.t_prompt_processing_total; + res->n_tokens_predicted_total = metrics.n_tokens_predicted_total; + res->t_tokens_generation_total = metrics.t_tokens_generation_total; - // Mirror for the draft (MTP) context when enabled. - if (ctx_dft && llama_model_is_hybrid(model_tgt)) { - ((llama_memory_hybrid *) llama_get_memory(ctx_dft.get()))->get_mem_attn()->seq_rm(slot.id, it->pos_max, -1); - } - it->load_dft_recr(ctx_dft.get(), slot.id); - } else { - it->load_tgt(ctx_tgt, slot.id, 0); - it->load_dft(ctx_dft.get(), slot.id, 0); - } + res->n_tokens_max = metrics.n_tokens_max; - pos_next = std::min(pos_next, std::max(it->pos_min + 1, it->pos_max)); - n_past = std::min(slot.prompt.tokens.size_up_to_pos(pos_next), (size_t) it->n_tokens); - SLT_WRN(slot, "restored context checkpoint (pos_min = %d, pos_max = %d, n_tokens = %" PRId64 ", n_past = %d, size = %.3f MiB)\n", it->pos_min, it->pos_max, it->n_tokens, n_past, (float) it->size() / 1024 / 1024); - // One-shot: STATE_PUT flag consumed on first successful match - slot.just_restored = false; - } - // else: just_restored override — KV already in place via STATE_PUT, - // pos_next/n_past set above; no checkpoint to load from iterator. - } + res->n_prompt_tokens_processed = metrics.n_prompt_tokens_processed; + res->t_prompt_processing = metrics.t_prompt_processing; + res->n_tokens_predicted = metrics.n_tokens_predicted; + res->t_tokens_generation = metrics.t_tokens_generation; - if (do_reset) { - SLT_WRN(slot, "forcing full prompt re-processing due to lack of cache data (likely due to SWA or hybrid/recurrent memory, see %s)\n", - "https://github.com/ggml-org/llama.cpp/pull/13194#issuecomment-2868343055"); - pos_next = 0; - n_past = 0; - } - } else if (pos_min >= pos_min_thold) { - // #641: pure extension — the whole cached sequence is a strict prefix of - // the new prompt and memory ends exactly at pos_next - 1, so no rewind is - // needed and the stale PREFILL-end checkpoint must not be loaded on top of - // the restored state (which would re-prefill already-cached tokens). - SLT_INF(slot, "no rewind needed — memory at pos_next-1 (n_past = %d, prompt = %d, task = %d, pos_next = %d, pos_max_mem = %d); skipping checkpoint search\n", - n_past, (int) slot.prompt.n_tokens(), (int) slot.task->n_tokens(), (int) pos_next, (int) pos_max_mem); - // consume the one-shot STATE_PUT flag so it can't leak into a later turn - slot.just_restored = false; - } - } + res->n_decode_total = metrics.n_decode_total; + res->n_busy_slots_total = metrics.n_busy_slots_total; - { - // erase any checkpoints with pos_max > pos_next - for (auto it = slot.prompt.checkpoints.begin(); it != slot.prompt.checkpoints.end();) { - const auto & cur = *it; - if (cur.pos_max > pos_next) { - SLT_WRN(slot, "erased invalidated context checkpoint (pos_min = %d, pos_max = %d, n_tokens = %" PRId64 ", n_swa = %d, pos_next = %d, size = %.3f MiB)\n", cur.pos_min, cur.pos_max, cur.n_tokens, n_swa, pos_next, (float) cur.size() / 1024 / 1024); - it = slot.prompt.checkpoints.erase(it); - } else { - ++it; - } - } - } - } + if (task.metrics_reset_bucket) { + metrics.reset_bucket(); + } + queue_results.send(std::move(res)); + } break; + case SERVER_TASK_TYPE_SLOT_SAVE: + { + if (!check_no_mtmd(task.id)) { + break; + } - // [TAG_PROMPT_LOGITS] - // When logits_reused is true (zero-prompt decode from restored - // logits), skip this guard — the entire prompt is cached. - if (!slot.logits_reused && n_past == slot.task->n_tokens() && n_past > 0) { - SLT_WRN(slot, "need to evaluate at least 1 token for each active slot (n_past = %d, task.n_tokens() = %d)\n", n_past, slot.task->n_tokens()); - n_past--; - SLT_WRN(slot, "n_past was set to %d\n", n_past); - } + const int id_slot = task.slot_action.id_slot; + server_slot * slot = get_slot_by_id(id_slot); + if (slot == nullptr) { + send_error(task, "Invalid slot ID", ERROR_TYPE_INVALID_REQUEST); + break; + } + if (slot->is_processing()) { + // if requested slot is unavailable, we defer this task for processing later + SRV_DBG("requested slot is unavailable, defer task, id_task = %d\n", task.id); + queue_tasks.defer(std::move(task)); + break; + } - slot.n_prompt_tokens_cache = n_past; - slot.n_prompt_tokens_processed = 0; + const size_t token_count = slot->prompt.tokens.size(); + const int64_t t_start = ggml_time_us(); - slot.prompt.tokens.keep_first(n_past); + std::string filename = task.slot_action.filename; + std::string filepath = task.slot_action.filepath; - // this is to signal the client that the request has started processing - if (slot.task->params.stream) { - if (slot.task->params.return_progress) { - // send initial 0% progress update if needed - send_partial_response(slot, {}, true); - } else { - // otherwise, for streaming without progress, signal HTTP to send the headers (i.e. 200 status) - send_partial_response(slot, {}, false, true); - } - } - } + const llama_tokens & tokens = slot->prompt.tokens.get_tokens(); + const size_t nwrite = llama_state_seq_save_file(ctx_tgt, filepath.c_str(), slot->id, tokens.data(), token_count); - if (!slot.can_split()) { - // cannot fit the prompt in the current batch - will try next iter - if (batch.n_tokens + slot.task->n_tokens() > n_batch) { - continue; - } - } + const int64_t t_end = ggml_time_us(); + const double t_save_ms = (t_end - t_start) / 1000.0; - const int64_t t_current = ggml_time_us(); - slot.t_prompt_processing = (t_current - slot.t_start_process_prompt) / 1e3; - slot.print_timings_pp(); + auto res = std::make_unique(); + res->id = task.id; + res->id_slot = id_slot; + res->filename = filename; + res->is_save = true; + res->n_tokens = token_count; + res->n_bytes = nwrite; + res->t_ms = t_save_ms; + queue_results.send(std::move(res)); + } break; + case SERVER_TASK_TYPE_SLOT_RESTORE: + { + if (!check_no_mtmd(task.id)) break; + const int id_slot = task.slot_action.id_slot; + server_slot * slot = get_slot_by_id(id_slot); + if (slot == nullptr) { + send_error(task, "Invalid slot ID", ERROR_TYPE_INVALID_REQUEST); + break; + } + if (slot->is_processing()) { + // if requested slot is unavailable, we defer this task for processing later + SRV_DBG("requested slot is unavailable, defer task, id_task = %d\n", task.id); + queue_tasks.defer(std::move(task)); + break; + } - // truncate any tokens that are beyond n_past for this slot - const llama_pos p0 = slot.prompt.tokens.pos_next(); + const int64_t t_start = ggml_time_us(); - SLT_TRC(slot, "cached n_tokens = %d, memory_seq_rm [%d, end)\n", slot.prompt.n_tokens(), p0); + std::string filename = task.slot_action.filename; + std::string filepath = task.slot_action.filepath; - common_context_seq_rm(ctx_tgt, slot.id, p0, -1); - if (ctx_dft) { - common_context_seq_rm(ctx_dft.get(), slot.id, p0, -1); + llama_tokens tokens; + tokens.resize(slot->n_ctx); + size_t token_count = 0; + size_t nread = llama_state_seq_load_file(ctx_tgt, filepath.c_str(), slot->id, tokens.data(), tokens.size(), &token_count); + if (nread == 0) { + slot->prompt.tokens.clear(); // KV may already been invalidated? + send_error(task, "Unable to restore slot, no available space in KV cache or invalid slot save file", ERROR_TYPE_INVALID_REQUEST); + break; } + tokens.resize(token_count); + slot->prompt.tokens.clear(); + slot->prompt.tokens.insert(tokens); - // If using an alora, there may be uncached tokens that come - // before the invocation sequence. When this happens, the - // tokens before the invocation sequence need to be - // processed without the adapter in a separate batch, then - // the adapter needs to be enabled for the remaining tokens. - if (lora_all_alora(slot.lora) && slot.alora_invocation_start - 1 > slot.prompt.n_tokens()) { - SLT_DBG(slot, "processing pre-alora tokens without the adapter (n_tokens = %d, alora_invocation_start = %d)\n", slot.prompt.n_tokens(), slot.alora_invocation_start); - const auto & enabled_loras = lora_get_enabled_ids(slot.lora); - GGML_ASSERT(enabled_loras.size() == 1); - alora_scale = slot.lora[enabled_loras[0]].scale; - slot.lora[enabled_loras[0]].scale = 0.0f; - alora_disabled_id = enabled_loras[0]; - } + const int64_t t_end = ggml_time_us(); + const double t_restore_ms = (t_end - t_start) / 1000.0; - // make a checkpoint of the parts of the memory that cannot be rolled back. - // checkpoints are created only if (see server_should_create_checkpoint): - // - the model does not support partial sequence removal - // - the model uses SWA (and we are not using `swa_full`) - // - the model supports partial sequence removal but only up to a fixed bound - // - the model is recurrent/hybrid (see below) - // Hydra: when the binary RPC port is enabled this server participates in - // cross-node KV migration. The restore target may not support rollback - // (e.g. it reports SEQ_RM_TYPE_FULL for the same model), so create native - // checkpoints regardless of the local seq_rm verdict — STATE_GET ships - // the latest checkpoint in the v2 blob, and without one the receiver - // fabricates a checkpoint at the final position, which corrupts - // hybrid/recurrent decode (recurrent state ends up past the resume point). - // Hydra (#316): the generic seq_rm probe in common_context_can_seq_rm() - // reports PART (not RS) for this hybrid arch's mixed attention/recurrent - // memory, since llama_n_rs_seq() is 0 and the smoke-test removal succeeds. - // That left checkpoint creation gated on rpc_port (only true for in-cluster - // nodes), so a standalone server with no RPC peer never created checkpoints - // and every cache-search below (which already special-cases is_rec, see - // ik_llama.cpp#1762) found nothing to restore — forcing a full re-prefill - // on every request. Recurrent/hybrid models need checkpoints on their own - // merits, independent of rpc_port. - // Hydra (#8): the gate is extracted to server_should_create_checkpoint() - // and pinned by tests/test-hydra-checkpoint-policy.cpp so the is_rec term - // can't be silently dropped — doing so breaks hybrid KV-cache restore. - const bool is_rec = llama_model_is_recurrent(model_tgt) || - llama_model_is_hybrid(model_tgt); - bool do_checkpoint = server_should_create_checkpoint( - params_base.n_ctx_checkpoints, - slot.task->type == SERVER_TASK_TYPE_COMPLETION, - ctx_tgt_seq_rm_type, - n_swa, - is_rec, - params_base.rpc_port); + auto res = std::make_unique(); + res->id = task.id; + res->id_slot = id_slot; + res->filename = filename; + res->is_save = false; + res->n_tokens = token_count; + res->n_bytes = nread; + res->t_ms = t_restore_ms; + queue_results.send(std::move(res)); + } break; + case SERVER_TASK_TYPE_SLOT_ERASE: + { + if (!check_no_mtmd(task.id)) { + break; + } + const int id_slot = task.slot_action.id_slot; + server_slot * slot = get_slot_by_id(id_slot); + if (slot == nullptr) { + send_error(task, "Invalid slot ID", ERROR_TYPE_INVALID_REQUEST); + break; + } + if (slot->is_processing() || slot->hydra_transferring->load()) { + // if requested slot is unavailable, we defer this task for processing later + SRV_DBG("requested slot is unavailable, defer task, id_task = %d\n", task.id); + queue_tasks.defer(std::move(task)); + break; + } - bool has_mtmd = false; + // Erase token cache + const size_t n_erased = slot->prompt.tokens.size(); - // check if we should process the image - while (slot.prompt.n_tokens() < slot.task->n_tokens() && input_tokens[slot.prompt.n_tokens()] == LLAMA_TOKEN_NULL) { - // process the image - size_t n_tokens_out = 0; - int32_t res = input_tokens.process_chunk(ctx_tgt, mctx, slot.prompt.n_tokens(), slot.prompt.tokens.pos_next(), slot.id, n_tokens_out); - if (res != 0) { - SLT_ERR(slot, "failed to process image, res = %d\n", res); - send_error(slot, "failed to process image", ERROR_TYPE_SERVER); - slot.release(); - continue; - } + slot->prompt_clear(false); - if (ctx_dft) { - // TODO: in the future, figure out how to infuse target embeddings to the images - // for now, we skip this for simplicity - // maybe we simply need to call `common_speculative_process()` on the mtmd batches in the `process_chunk` above? - res = input_tokens.process_chunk(ctx_dft.get(), mctx, slot.prompt.n_tokens(), slot.prompt.tokens.pos_next(), slot.id, n_tokens_out); - if (res != 0) { - GGML_ABORT("failed to process multi-modal data on draft context\n"); + auto res = std::make_unique(); + res->id = task.id; + res->id_slot = id_slot; + res->n_erased = n_erased; + queue_results.send(std::move(res)); + } break; + case SERVER_TASK_TYPE_GET_LORA: + { + // TODO @ngxson : make lora_adapters a dedicated member of server_context + auto & loras = params_base.lora_adapters; + auto res = std::make_unique(); + res->id = task.id; + for (size_t i = 0; i < loras.size(); ++i) { + auto & lora = loras[i]; + std::string alora_invocation_string = ""; + const uint64_t n_alora_tokens = llama_adapter_get_alora_n_invocation_tokens(lora.ptr); + llama_tokens alora_invocation_tokens; + if (n_alora_tokens) { + const llama_token * alora_tokens = llama_adapter_get_alora_invocation_tokens(lora.ptr); + for (uint64_t j = 0; j < n_alora_tokens; ++j) { + alora_invocation_string += common_token_to_piece(vocab, alora_tokens[j]); + alora_invocation_tokens.push_back(alora_tokens[j]); } } + res->loras.push_back(server_task_result_get_lora::lora{ + lora, + alora_invocation_string, + alora_invocation_tokens, + }); + } + queue_results.send(std::move(res)); + } break; + case SERVER_TASK_TYPE_SET_LORA: + { + auto new_loras = construct_lora_list(task.set_lora); + // logging + for (size_t i = 0; i < new_loras.size(); ++i) { + SRV_INF("set lora adapter idx=%zu scale=%f\n", i, new_loras[i].scale); + } + // TODO @ngxson : make lora_adapters a dedicated member of server_context + params_base.lora_adapters = new_loras; + auto res = std::make_unique(); + res->id = task.id; + queue_results.send(std::move(res)); + } break; - slot.n_prompt_tokens_processed += n_tokens_out; + // epic #610 WS2: HYDRA task dispatch moved to hydra_process_task() + // (defined in hydra-server-context.cpp). In seam mode the extension + // claims these via handle_task(); in legacy mode this fall-through + // calls the same method. Both modes run identical code. + case SERVER_TASK_TYPE_HYDRA_STATE_GET: + case SERVER_TASK_TYPE_HYDRA_STATE_PUT: + case SERVER_TASK_TYPE_HYDRA_STATE_META: + case SERVER_TASK_TYPE_HYDRA_ENGINE_CONFIGURE: + case SERVER_TASK_TYPE_HYDRA_ENGINE_INFO: + case SERVER_TASK_TYPE_HYDRA_ENGINE_PREFILL: + case SERVER_TASK_TYPE_HYDRA_ENGINE_DECODE: + case SERVER_TASK_TYPE_HYDRA_DECODE_APPLY: + case SERVER_TASK_TYPE_HYDRA_ENGINE_SET_EXPERT_MODE: + case SERVER_TASK_TYPE_HYDRA_ENGINE_SWAP_QUANT: + case SERVER_TASK_TYPE_HYDRA_ENGINE_PIPELINE_ATTACH: + hydra_process_task(task); + break; - // add the image chunk to cache - { - const auto & chunk = input_tokens.find_chunk(slot.prompt.n_tokens()); - slot.prompt.tokens.push_back(chunk.get()); // copy - } + } + } - has_mtmd = true; - } + // epic #610 WS3.5: Hydra helper functions — defined in hydra-server-context.cpp + // (same TU via bottom #include). Declarations here so the compiler resolves + // the out-of-class definitions. + static int hydra_classify_config_key(const std::string & key); + static const char * hydra_tier_label(int tier); - const int32_t n_before_user = slot.task->params.n_before_user; - const bool n_before_user_known = n_before_user > 0; + bool hydra_apply_t1_config(common_params & params, llama_context * ctx, + const json & cfg, + std::map & params_applied); - // add prompt tokens for processing in the current batch - while (slot.prompt.n_tokens() < slot.task->n_tokens() && batch.n_tokens < n_batch) { - // get next token to process - llama_token cur_tok = input_tokens[slot.prompt.n_tokens()]; - if (cur_tok == LLAMA_TOKEN_NULL) { - break; // end of text chunk - } + void hydra_apply_t3_mutators(llama_context * ctx, const json & cfg, + std::vector & deferred_keys); - // if this is an alora request with pre-invocation - // tokens that are not cached, we need to stop filling - // this batch at those pre-invocation tokens. - if (alora_scale > 0 && slot.prompt.n_tokens() == slot.alora_invocation_start - 1) { - SLT_DBG(slot, "stop prompt batch filling at (n_tokens = %d, alora_invocation_start = %d)\n", slot.prompt.n_tokens(), slot.alora_invocation_start); - break; - } + struct hydra_config_result { + int highest_tier = 0; + std::map params_applied; + std::vector deferred_keys; + json t2t3_subset = json::object(); + bool ok = true; + std::string error; + uint64_t state_chunk_size_applied = 0; + std::vector unrecognized_keys; + std::vector rejected_keys; + }; - // embedding requires all tokens in the batch to be output; - // MTP also wants logits at every prompt position so the - // streaming hook can mirror t_h_nextn into ctx_dft. - common_batch_add(batch, - cur_tok, - slot.prompt.tokens.pos_next(), - { slot.id }, - slot.need_embd()); - slot.prompt.tokens.push_back(cur_tok); + hydra_config_result hydra_apply_config(const json & cfg, bool sync); - slot.n_prompt_tokens_processed++; + bool apply_pending_hydra_config(); + bool apply_t2_rebuild(const std::string & pending_json); + bool apply_t3_rebuild(bool force = false); +}; - // stop the prompt batch exactly before the latest user input, so a checkpoint - // can be created after the previous messages - if (n_before_user_known && - slot.prompt.n_tokens() == n_before_user) { - break; - } +// +// server_context (public API) +// - // process the last few tokens of the prompt separately in order to allow for a checkpoint to be created. - // create checkpoints that many tokens before the end of the prompt: - // - 4 + n_ubatch - // - 4 - // ref: https://github.com/ggml-org/llama.cpp/pull/20288 - if (do_checkpoint) { - static const int checkpoint_offsets[] = {4 + n_ubatch, 4}; +server_context::server_context() : impl(new server_context_impl()) {} +server_context::~server_context() = default; - bool should_break = false; - for (int offset : checkpoint_offsets) { - const int n_last = std::min(n_batch, offset); - if (slot.task->n_tokens() == slot.prompt.n_tokens() + n_last) { - should_break = true; - break; - } - } - if (should_break) { - break; - } - } - } +bool server_context::load_model(common_params & params) { + return impl->load_model(params); +} - // the number of tokens added to the batch for the current slot - const auto n_tokens_cur = batch.n_tokens - n_tokens_prev; +void server_context::start_loop() { + auto & params = impl->params_base; + impl->queue_tasks.start_loop(params.sleep_idle_seconds * 1000); +} - // Track total prompt tokens processed for n_common observability - if (!slot.logits_reused) { - slot.n_prompt_processed += n_tokens_cur; - } +void server_context::terminate() { + impl->queue_tasks.terminate(); +} - const bool near_prompt_end = slot.task->n_tokens() < slot.prompt.n_tokens() + n_ubatch; +llama_context * server_context::get_llama_context() const { + return impl->ctx_tgt; +} - // entire prompt has been processed - if (slot.prompt.n_tokens() == slot.task->n_tokens()) { - slot.state = SLOT_STATE_DONE_PROMPT; +void server_context::set_hydra_capabilities(bool rpc_backend_active, const std::string & peer, + bool peer_reachable, const std::string & combined_pattern, const std::string & split_mode) { + impl->hydra_rpc_backend_active = rpc_backend_active; + impl->hydra_peer = peer; + impl->hydra_peer_reachable = peer_reachable; + impl->hydra_combined_pattern = combined_pattern; + impl->hydra_split_mode = split_mode; +} - GGML_ASSERT(batch.n_tokens > 0); +void server_context::set_hydra_combined_head_attached(bool attached) { + impl->hydra_combined_head_attached = attached; +} - // extract the logits only for the last token - batch.logits[batch.n_tokens - 1] = true; +void server_context::set_hydra_combined_static(bool is_static) { + impl->hydra_combined_static = is_static; +} - slot.n_decoded = 0; - slot.i_batch = batch.n_tokens - 1; +server_response_reader server_context::get_response_reader() { + return impl->get_response_reader(); +} - slot.init_sampler(); - } else { - // skip ordinary mid-prompt checkpoints - if (!n_before_user_known && !near_prompt_end) { - do_checkpoint = false; - } - } +server_context_meta server_context::get_meta() const { + auto bos_id = llama_vocab_bos(impl->vocab); + auto eos_id = llama_vocab_eos(impl->vocab); + auto bos_token_str = bos_id != LLAMA_TOKEN_NULL ? common_token_to_piece(impl->ctx_tgt, bos_id, true) : ""; + auto eos_token_str = eos_id != LLAMA_TOKEN_NULL ? common_token_to_piece(impl->ctx_tgt, eos_id, true) : ""; - const auto pos_min = llama_memory_seq_pos_min(llama_get_memory(ctx_tgt), slot.id); - const auto pos_max = llama_memory_seq_pos_max(llama_get_memory(ctx_tgt), slot.id); + return server_context_meta { + /* build_info */ std::string(llama_build_info()), + /* model_name */ impl->model_name, + /* model_aliases */ impl->model_aliases, + /* model_tags */ impl->model_tags, + /* model_path */ impl->params_base.model.path, + /* has_mtmd */ impl->mctx != nullptr, + /* has_inp_image */ impl->chat_params.allow_image, + /* has_inp_audio */ impl->chat_params.allow_audio, + /* json_ui_settings */ impl->json_ui_settings, + /* json_webui_settings */ impl->json_webui_settings, // Deprecated + /* slot_n_ctx */ impl->get_slot_n_ctx(), + /* pooling_type */ llama_pooling_type(impl->ctx_tgt), - // checkpoints are created before the current batch is decoded, so - // their token position is the batch start rather than the prompt end - const int32_t n_tokens_start = slot.prompt.n_tokens() - n_tokens_cur; + /* chat_params */ impl->chat_params, + /* chat_template_caps */ common_chat_templates_get_caps(impl->chat_params.tmpls.get()), - { - const bool is_on_user = - n_before_user_known && - n_tokens_start == n_before_user; + /* bos_token_str */ bos_token_str, + /* eos_token_str */ eos_token_str, + /* fim_pre_token */ llama_vocab_fim_pre(impl->vocab), + /* fim_sub_token */ llama_vocab_fim_suf(impl->vocab), + /* fim_mid_token */ llama_vocab_fim_mid(impl->vocab), + /* fim_pad_token */ llama_vocab_fim_pad(impl->vocab), + /* fim_rep_token */ llama_vocab_fim_rep(impl->vocab), + /* fim_sep_token */ llama_vocab_fim_sep(impl->vocab), - const bool is_after_user = - n_before_user_known && - n_tokens_start > n_before_user; + /* logit_bias_eog */ impl->params_base.sampling.logit_bias_eog, - const bool is_allowed = - !n_before_user_known || - is_on_user || - (is_after_user && near_prompt_end); + /* model_vocab_type */ llama_vocab_type(impl->vocab), + /* model_vocab_n_tokens */ llama_vocab_n_tokens(impl->vocab), + /* model_n_ctx_train */ llama_model_n_ctx_train(impl->model_tgt), + /* model_n_embd_inp */ llama_model_n_embd(impl->model_tgt), + /* model_n_params */ llama_model_n_params(impl->model_tgt), + /* model_size */ llama_model_size(impl->model_tgt), + /* split_mode */ impl->params_base.split_mode, + /* tensor_split */ std::vector(impl->params_base.tensor_split, impl->params_base.tensor_split + 128), + }; +} - if (do_checkpoint && !is_allowed) { - do_checkpoint = false; - } - } - // nothing to checkpoint yet - // TODO: is this check needed? - if (do_checkpoint && pos_min < 0) { - do_checkpoint = false; - } - // do not checkpoint after mtmd chunks - do_checkpoint = do_checkpoint && !has_mtmd; +// generator-like API for HTTP response generation +// may have bypass_sleep = true if the task does not use ctx_server +struct server_res_generator : server_http_res { + server_response_reader rd; + server_res_generator(server_queue & queue_tasks, server_response & queue_results, int sleep_idle_seconds, bool bypass_sleep = false) + : rd(queue_tasks, queue_results, HTTP_POLLING_SECONDS) { + // fast path in case sleeping is disabled + bypass_sleep |= sleep_idle_seconds < 0; + if (!bypass_sleep) { + queue_tasks.wait_until_no_sleep(); + } + } + void ok(const json & response_data) { + status = 200; + data = safe_json_to_str(response_data); + } + void error(const json & error_data) { + status = json_value(error_data, "code", 500); + data = safe_json_to_str({{ "error", error_data }}); + } +}; - // no need to create checkpoints that are too close together. - // For recurrent/hybrid models, use a much smaller minimum spacing so short - // follow-up turns still get a checkpoint to resume from. Ref: ik_llama.cpp#1762. - const int eff_checkpoint_min_step = - (llama_model_is_recurrent(model_tgt) || llama_model_is_hybrid(model_tgt)) - ? std::min(params_base.checkpoint_min_step, 4) - : params_base.checkpoint_min_step; - do_checkpoint = do_checkpoint && (slot.prompt.checkpoints.empty() || n_tokens_start > slot.prompt.checkpoints.back().n_tokens + eff_checkpoint_min_step); - SLT_DBG(slot, "main/do_checkpoint = %s, pos_min = %d, pos_max = %d\n", do_checkpoint ? "yes" : "no", pos_min, pos_max); +void server_context::on_sleeping_changed(std::function callback) { + impl->queue_tasks.on_sleeping_state(std::move(callback)); +} - // note: we create the checkpoint before calling llama_decode(), so the current batch is not - // yet processed and therefore it is not part of the checkpoint. - if (do_checkpoint) { - create_checkpoint(slot, n_tokens_cur, pos_min, pos_max); - } - } +void server_context::set_routes_ptr(server_routes * routes) { + impl->routes_ptr = routes; +} - if (!slot_batched) { - slot_batched = &slot; - } +bool server_context::bootstrap_init() { + return impl->bootstrap_init(); +} - if (batch.n_tokens >= n_batch) { - break; - } - } - } +void server_context::set_bootstrap_capabilities(bool rpc_active, const std::string & peer, + bool peer_reachable, const std::string & pattern, + const std::string & split_mode, bool combined_static) { + impl->bootstrap_rpc_active = rpc_active; + impl->bootstrap_peer = peer; + impl->bootstrap_peer_reachable = peer_reachable; + impl->bootstrap_pattern = pattern; + impl->bootstrap_split_mode = split_mode; + impl->bootstrap_combined_static = combined_static; +} - SRV_DBG("decoding batch, n_tokens = %d\n", batch.n_tokens); +// compute the number of tokens before the last user message in the prompt +static int32_t prompt_get_n_before_user( + const json & message_spans, + const std::string & prompt, + const std::vector & files, + const llama_vocab * vocab, + mtmd_context * mctx) { + int32_t result = -1; + int32_t byte_pos = -1; - auto accept_special_token = [&](server_slot & slot, llama_token token) { - return params_base.special || - slot.task->params.sampling.preserved_tokens.find(token) != slot.task->params.sampling.preserved_tokens.end(); - }; + for (const auto & span : message_spans) { + const std::string role = json_value(span, "role", std::string()); - if (slot_batched) { - // apply lora, only need to do it once per batch - common_set_adapter_lora(ctx_tgt, slot_batched->lora); + if (role == "user") { + byte_pos = json_value(span, "pos", -1); + } + } - // if the lora is temporarily disabled for an alora, re-enable it - // for next time - if (alora_scale > 0.0f) { - SRV_DBG("re-enabling alora with scale %f\n", alora_scale); - slot_batched->lora[alora_disabled_id].scale = alora_scale; - } + if (byte_pos >= 0) { + GGML_ASSERT((size_t) byte_pos <= prompt.size()); - llama_set_embeddings(ctx_tgt, slot_batched->need_embd()); + const std::string prefix = prompt.substr(0, (size_t) byte_pos); + + const std::string marker = get_media_marker(); + size_t n_prefix_media = 0; + for (size_t pos = 0; (pos = prefix.find(marker, pos)) != std::string::npos; pos += marker.size()) { + n_prefix_media++; } - if (batch.n_tokens == 0) { - if (++n_empty_consecutive > 3) { - // Hydra: a STATE_GET background stream holds the slot (hydra_transferring) - // without contributing batch tokens — that is expected, not a stall. - // Suppress the abort while a transfer is in flight, and for a short - // grace window after it ends (the flag clears a few loop iterations - // before the queue delivers the releasing task — without the grace - // window those tail iterations trip the abort). - static int64_t hydra_last_transfer_ms = 0; - static int64_t hydra_suppress_count = 0; - bool any_transferring = false; - for (const auto & s : slots) { - if (s.hydra_transferring && s.hydra_transferring->load()) { - any_transferring = true; - break; - } - } - const int64_t now_ms = ggml_time_us() / 1000; - if (any_transferring) { - hydra_last_transfer_ms = now_ms; - } - if (any_transferring || now_ms - hydra_last_transfer_ms < 2000) { - // rate-limit: this branch runs in a hot loop — log once per 256 suppressions - if (hydra_suppress_count++ % 256 == 0) { - SRV_WRN("empty batch threshold exceeded (n_empty=%d, suppressed=%" PRId64 ") — hydra transfer %s, suppressing abort\n", - n_empty_consecutive, hydra_suppress_count, - any_transferring ? "in flight" : "just ended"); - } - n_empty_consecutive = 0; - // avoid hot-spinning while the transfer holds the slot - std::this_thread::sleep_for(std::chrono::milliseconds(1)); - } else { - SRV_WRN("%s", "no tokens to decode\n"); - GGML_ABORT("fatal error - please provide logs and repro in %s\n", "https://github.com/ggml-org/llama.cpp/pull/20277"); - } - } + GGML_ASSERT(n_prefix_media <= files.size()); + + if (mctx != nullptr && n_prefix_media > 0) { + // TODO: this makes a copy - avoid it + std::vector prefix_files(files.begin(), files.begin() + n_prefix_media); + + result = (int32_t) process_mtmd_prompt(mctx, prefix, prefix_files).size(); } else { - n_empty_consecutive = 0; + result = (int32_t) tokenize_input_prompts(vocab, nullptr, prefix, true, true)[0].size(); } - int32_t i_next = 0; + SRV_TRC("message_spans: last user message: byte_pos=%d, media=%zu, n_before_user=%d\n", + byte_pos, n_prefix_media, result); + } - // process the created batch of tokens - for (int32_t i = 0; i < batch.n_tokens; i = i_next) { - const int32_t n_tokens = std::min(n_batch, batch.n_tokens - i); + return result; +} - llama_batch batch_view = { - n_tokens, - batch.token + i, - nullptr, - batch.pos + i, - batch.n_seq_id + i, - batch.seq_id + i, - batch.logits + i, - }; - const int ret = llama_decode(ctx_tgt, batch_view); +// +// server_routes +// - metrics.on_decoded(slots); +std::unique_ptr server_routes::handle_completions_impl( + const server_http_req & req, + server_task_type type, + const json & data, + const std::vector & files, + task_response_type res_type, + const std::string & hydra_config_json) { + GGML_ASSERT(type == SERVER_TASK_TYPE_COMPLETION || type == SERVER_TASK_TYPE_INFILL); - if (ret != 0) { - { - std::string err; + std::shared_lock meta_lock(meta_mutex); - if (n_batch == 1 && ret == 1) { - // TODO: try to terminate only the largest active slot/sequence and continue with the rest - // need to remove the tokens from the current batch too - err = "Context size has been exceeded."; - } + // P0-1 (#49): null-meta guard — meta is null until update_meta() is + // called after model load. Return 503 instead of crashing. + if (!meta) { + auto res = create_response(); + res->error(format_error_response("model not loaded — waiting for CONFIGURE", + ERROR_TYPE_NOT_SUPPORTED)); + return res; + } - if (ret == -1) { - err = "Invalid input batch."; - } + auto res = create_response(); + auto completion_id = gen_chatcmplid(); + auto & rd = res->rd; + auto & params = this->params; - if (ret < -1) { - // TODO: update slot state based on llama_memory_seq_pos_min() and llama_memory_seq_pos_max() - err = "Compute error."; - } + try { + std::vector tasks; - // TODO: handle ret == 2 (abort) when we start aborting + const auto & prompt = data.at("prompt"); + // TODO: this log can become very long, put it behind a flag or think about a more compact format + //SRV_DBG("Prompt: %s\n", prompt.is_string() ? prompt.get().c_str() : prompt.dump(2).c_str()); - if (!err.empty()) { - SRV_ERR("%s i = %d, n_batch = %d, ret = %d\n", err.c_str(), i, n_batch, ret); + // process prompt + std::vector inputs; - for (auto & slot : slots) { - if (slot.is_processing() || slot.hydra_transferring->load()) { - send_error(slot, err); - slot.release(); + if (res_type != TASK_RESPONSE_TYPE_NONE && ctx_server.mctx != nullptr) { + // This is the case used by OAI compatible chat path with MTMD. TODO It can be moved to the path below. + inputs.push_back(process_mtmd_prompt(ctx_server.mctx, prompt.get(), files)); + } else { + // Everything else, including multimodal completions. + inputs = tokenize_input_prompts(ctx_server.vocab, ctx_server.mctx, prompt, true, true); + } - // note: it's complicated to keep track of how much of the current batch has been - // processed before the error occurred, so we simply clear the entire context - slot.prompt_clear(false); - } - } + // tasks.reserve(inputs.size()); // TODO: this is inaccurate due to child tasks - break; - } - } + for (size_t i = 0; i < inputs.size(); i++) { + server_task task = server_task(type); - // retry with half the batch size to try to find a free slot in the KV cache - if (!try_clear_idle_slots()) { - n_batch /= 2; - } + task.id = rd.get_new_id(); - SRV_WRN("failed to find free space in the KV cache, retrying with smaller batch size, i = %d, n_batch = %d, ret = %d\n", i, n_batch, ret); + task.tokens = std::move(inputs[i]); + task.params = server_task::params_from_json_cmpl( + ctx_server.vocab, + params, + meta->slot_n_ctx, + meta->logit_bias_eog, + data); - continue; // continue loop of n_batch + const auto message_spans = json_value(data, "message_spans", json::array()); + if (prompt.is_string() && message_spans.is_array()) { + task.params.n_before_user = + prompt_get_n_before_user( + message_spans, + prompt.get(), + files, + ctx_server.vocab, + ctx_server.mctx); } - // TODO: avoid restoring the draft context and re-evaluating the drafted tokens when not needed [TAG_SPEC_AVOID_DRAFT_REEVAL] - // for now, always re-evaluate for simplicity - // ref: https://github.com/ggml-org/llama.cpp/pull/22728#issuecomment-4400925384 - // - // | spec type | need re-eval | - // | --- | --- | - // | draft model | no | because the draft model does not use embeddings from the target - // | MTP (std) | yes | - // | MTP Gemma4 | no | because the KV cache is shared - // | Eagle3 | yes | - // | DFlash | yes | https://github.com/ggml-org/llama.cpp/pull/22728#issuecomment-4405406982 - // - // note: this logic is now moved in `common_speculative_process()` - // keeping the sketch here until for a bit, until the logic is finalized - // - //if (ctx_dft) { - // // TODO: update as needed for MTP, Eagle3, etc. - // const bool need_tgt_embd = false; - - // if (need_tgt_embd) { - // llama_synchronize(ctx_tgt); - // } - - // // the logic here varies depending on the speculative decoding method - // // - some draft contexts require embeddings from the target context, others don't - // // - some draft contexts involve an encoder step to transform the target embeddings to draft embeddings - // // TODO: extract this in a function ? - // { - // // TODO: hook the embeddings from the last target batch here - // if (llama_model_has_encoder(model_dft.get())) { - // //llama_encode(ctx_dft, ...); - - // GGML_ABORT("not implemented yet\n"); - // } - - // const int ret = llama_decode(ctx_dft.get(), batch_view); - - // if (ret != 0) { - // SRV_ERR("failed to decode draft batch, ret = %d\n", ret); + task.id_slot = json_value(data, "id_slot", -1); - // // TODO: handle error - // break; - // } - // } - //} - if (!common_speculative_process(spec.get(), batch_view)) { - SRV_ERR("%s", "failed to process speculative batch\n"); + // OAI-compat + task.params.res_type = res_type; + task.params.oaicompat_cmpl_id = completion_id; + task.params.oaicompat_model = meta->model_name; - // TODO: handle error - break; + // Attach hydra_config (if any) so the task-queue thread can + // apply it synchronously before processing the completion. + if (!hydra_config_json.empty()) { + task.hydra_config_json = hydra_config_json; } - // move the head of the batch forward with the number of tokens we just processed - i_next = i + n_tokens; - - // on successful decode, restore the original batch size - n_batch = llama_n_batch(ctx_tgt); + // prepare child tasks + if (task.params.n_cmpl > 1) { + int n_children = task.params.n_cmpl - 1; + for (int j = 0; j < n_children; j++) { + task.add_child(task.id, rd.get_new_id()); + } + } - // handle `n_cmpl > 1` tasks - when the main prompt is processed, activate all child tasks too - for (auto & slot : slots) { - if (slot.state == SLOT_STATE_DONE_PROMPT && slot.task->is_parent()) { - std::vector children; - for (auto & other : slots) { - if (other.state == SLOT_STATE_WAIT_OTHER && slot.task->id == other.task->id_parent) { - children.push_back(&other); - } - } + tasks.push_back(std::move(task)); + } - // all children slots should already launched by launch_slots_with_parent_task() - // copy state to the child slots - for (auto & child : children) { - SLT_INF(slot, " - copying state to child %d\n", child->id); + rd.post_tasks(std::move(tasks)); + } catch (const std::exception & e) { + res->error(format_error_response(e.what(), ERROR_TYPE_INVALID_REQUEST)); + return res; + } - GGML_ASSERT(child->state == SLOT_STATE_WAIT_OTHER); + bool stream = json_value(data, "stream", false); - slot.copy_state_to(*child); - child->state = SLOT_STATE_DONE_PROMPT; + if (!stream) { + // non-stream, wait for the results + auto all_results = rd.wait_for_all(req.should_stop); + if (all_results.is_terminated) { + return res; // connection is closed + } else if (all_results.error) { + res->error(all_results.error->to_json()); + return res; + } else { + json arr = json::array(); + for (auto & res : all_results.results) { + auto * cmpl = dynamic_cast(res.get()); + GGML_ASSERT(cmpl != nullptr); + // P1-5/P1-6: build metrics inline from the (always-current) + // meta rather than reading a class-level field that could + // leak between concurrent requests. + { + auto split_mode_str = [](enum llama_split_mode m) -> const char * { + switch (m) { + case LLAMA_SPLIT_MODE_NONE: return "none"; + case LLAMA_SPLIT_MODE_LAYER: return "layer"; + case LLAMA_SPLIT_MODE_ROW: return "row"; + default: return "none"; + } + }; + json metrics = json::object(); + metrics["model_path"] = meta->model_path; + metrics["split_mode"] = split_mode_str(meta->split_mode); + metrics["t3_reloaded"] = false; + metrics["t3_reload_ms"] = 0.0; + json ts_arr = json::array(); + for (const auto & v : meta->tensor_split) { + if (v != 0.0f) { + ts_arr.push_back(v); + } else { + break; + } } + metrics["tensor_split"] = ts_arr; + cmpl->hydra_metrics = metrics; } + arr.push_back(cmpl->to_json()); } - - for (auto & slot : slots) { - // optionally send prompt processing progress - if (slot.state == SLOT_STATE_PROCESSING_PROMPT || slot.state == SLOT_STATE_DONE_PROMPT) { - if (slot.task->params.stream && slot.task->params.return_progress) { - send_partial_response(slot, {}, true); - } - } - - if (slot.i_batch < (int) i || slot.i_batch >= (int) (i + n_tokens)) { - continue; // continue loop of slots + GGML_ASSERT(!arr.empty() && "empty results"); + if (arr.size() == 1) { + // if single request, return single object instead of array + res->ok(arr[0]); + } else if (res_type == TASK_RESPONSE_TYPE_OAI_CHAT || res_type == TASK_RESPONSE_TYPE_OAI_CMPL) { + // if multiple results in OAI format, we need to re-format them + json & choices = arr[0]["choices"]; + for (size_t i = 1; i < arr.size(); i++) { + choices.push_back(std::move(arr[i]["choices"][0])); } + res->ok(arr[0]); + } else { + // multi-results, non-OAI compat + res->ok(arr); + } + } + } else { + // in streaming mode, the first error must be treated as non-stream response + // this is to match the OAI API behavior + // ref: https://github.com/ggml-org/llama.cpp/pull/16486#discussion_r2419657309 + auto first_result = rd.next(req.should_stop); + if (first_result == nullptr) { + GGML_ASSERT(req.should_stop()); + return res; // connection is closed + } - if (slot.state == SLOT_STATE_DONE_PROMPT) { - if (slot.task->type == SERVER_TASK_TYPE_EMBEDDING) { - // prompt evaluated for embedding - send_embedding(slot, batch_view); - slot.release(); - slot.i_batch = -1; - continue; // continue loop of slots - } + if (first_result->is_error()) { + res->error(first_result->to_json()); + return res; + } - if (slot.task->type == SERVER_TASK_TYPE_RERANK) { - send_rerank(slot, batch_view); - slot.release(); - slot.i_batch = -1; - continue; // continue loop of slots - } - - GGML_ASSERT(slot.task->need_sampling()); - - // prompt evaluated for next-token prediction - slot.state = SLOT_STATE_GENERATING; + GGML_ASSERT( + dynamic_cast(first_result.get()) != nullptr || + dynamic_cast (first_result.get()) != nullptr + ); - if (slot.can_speculate()) { - common_speculative_begin(spec.get(), slot.id, slot.prompt.tokens.get_text_tokens()); - } - } else if (slot.state != SLOT_STATE_GENERATING) { - continue; // continue loop of slots + // next responses are streamed + // to be sent immediately + json first_result_json = first_result->to_json(); + if (first_result_json == nullptr) { + res->data = ""; // simply send HTTP headers and status code + } else if (res_type == TASK_RESPONSE_TYPE_ANTHROPIC) { + res->data = format_anthropic_sse(first_result_json); + } else if (res_type == TASK_RESPONSE_TYPE_OAI_RESP) { + res->data = format_oai_resp_sse(first_result_json); + } else { + res->data = format_oai_sse(first_result_json); + } + res->status = 200; + res->content_type = "text/event-stream"; + res->next = [res_this = res.get(), res_type, &req, ¶ms](std::string & output) -> bool { + static auto format_error = [](task_response_type res_type, const json & res_json) { + if (res_type == TASK_RESPONSE_TYPE_ANTHROPIC) { + return format_anthropic_sse({ + {"event", "error"}, + {"data", res_json}, + }); + } else { + return format_oai_sse(json {{ "error", res_json }}); } + }; - if (slot.can_speculate() && !slot.spec_draft.empty()) { - continue; // sample using speculative decoding + try { + if (req.should_stop()) { + SRV_DBG("%s", "stopping streaming due to should_stop condition\n"); + return false; // should_stop condition met } - const int tok_idx = slot.i_batch - i; - - // D4: Inject per-slot restored logits into the correct batch row before - // first sample. The sampler reads via llama_get_logits_ith(ctx, tok_idx), - // so we must write to that exact row — not row 0. - if (slot.logits_valid && !slot.restored_logits.empty() && slot.n_decoded == 0) { - float * row_logits = llama_get_logits_ith(slot.ctx_tgt, tok_idx); - if (row_logits) { - const size_t n_floats = slot.restored_logits.size(); - memcpy(row_logits, slot.restored_logits.data(), n_floats * sizeof(float)); - SLT_INF(slot, "consumed %zu restored logits into row %d (tok_idx)\n", n_floats, tok_idx); - } else { - SLT_WRN(slot, "restored logits skipped: llama_get_logits_ith returned null for row %d\n", tok_idx); - } - slot.restored_logits.clear(); - slot.logits_valid = false; + if (!res_this->data.empty()) { + // flush the first chunk + output = std::move(res_this->data); + res_this->data.clear(); + return true; } - llama_token id = common_sampler_sample(slot.smpl.get(), slot.ctx_tgt, tok_idx); - - slot.i_batch = -1; - - common_sampler_accept(slot.smpl.get(), id, true); - - // here we have synchronized the llama_context (due to the sampling above), so we can do time measurement - const int64_t t_current = ggml_time_us(); + server_response_reader & rd = res_this->rd; - slot.n_decoded += 1; + // check if there is more data + if (!rd.has_next()) { + switch (res_type) { + case TASK_RESPONSE_TYPE_NONE: + case TASK_RESPONSE_TYPE_OAI_RESP: + case TASK_RESPONSE_TYPE_ANTHROPIC: + output = ""; + break; - if (slot.n_decoded == 1) { - slot.t_start_generation = t_current; - slot.t_prompt_processing = (slot.t_start_generation - slot.t_start_process_prompt) / 1e3; - metrics.on_prompt_eval(slot); + default: + output = "data: [DONE]\n\n"; + break; + } + SRV_DBG("%s", "all results received, terminating stream\n"); + return false; // no more data, terminate } - slot.t_token_generation = std::max(1, t_current - slot.t_start_generation) / 1e3; - - completion_token_output result; - result.tok = id; - result.text_to_send = common_token_to_piece(slot.ctx_tgt, result.tok, accept_special_token(slot, result.tok)); - result.prob = 1.0f; // TODO: set it here instead of doing inside populate_token_probs + // receive subsequent results + bool timeout = false; + int64_t start_time = ggml_time_ms(); + auto result = rd.next([&timeout, &req, &start_time, ¶ms]() { + if (req.should_stop()) { + return true; // should_stop condition met + } else if (params.sse_ping_interval > 0 && ggml_time_ms() - start_time > (int64_t)params.sse_ping_interval * 1000) { + timeout = true; + return true; // timeout + } + return false; + }); - if (slot.task->params.sampling.n_probs > 0) { - populate_token_probs(slot, result, slot.task->params.post_sampling_probs, params_base.special, tok_idx); + if (timeout) { + // some clients may time out (e.g. undici) will time out if no data is received for a while, so we need to send a ping to keep the connection alive + SRV_DBG("%s", "sending SSE ping\n"); + output = ":\n\n"; + return true; } - if (!process_token(result, slot)) { - // release slot because of stop condition - slot.print_timings(); - send_final_response(slot); - metrics.on_prediction(slot); - slot.release(); - - continue; + if (result == nullptr) { + SRV_DBG("%s", "stopping streaming due to should_stop condition\n"); + GGML_ASSERT(req.should_stop()); + return false; // should_stop condition met } - slot.print_timings_tg(); - } - - // speculative decoding - main model sample and accept - for (auto & slot : slots) { - if (slot.state != SLOT_STATE_GENERATING || !slot.can_speculate() || slot.spec_draft.empty()) { - continue; + // send the results + if (result->is_error()) { + json res_json = result->to_json(); + output = format_error(res_type, res_json); + SRV_DBG("%s", "error received during streaming, terminating stream\n"); + return false; // terminate on error + } else { + GGML_ASSERT( + dynamic_cast(result.get()) != nullptr + || dynamic_cast(result.get()) != nullptr + ); + json res_json = result->to_json(); + if (res_type == TASK_RESPONSE_TYPE_ANTHROPIC) { + output = format_anthropic_sse(res_json); + } else if (res_type == TASK_RESPONSE_TYPE_OAI_RESP) { + output = format_oai_resp_sse(res_json); + } else { + output = format_oai_sse(res_json); + } } - // save the original draft size - const size_t n_draft = slot.spec_draft.size(); - - GGML_ASSERT(n_draft > 0); - - // verify and try to accept the draft - { - // save the sampler sampler state in case we need to restore it - common_sampler_ptr smpl_save(common_sampler_clone(slot.smpl.get())); + // has next data, continue + return true; - GGML_ASSERT(slot.spec_i_batch.size() == n_draft + 1); - auto accepted = common_sampler_sample_and_accept_n(slot.smpl.get(), slot.ctx_tgt, slot.spec_i_batch, slot.spec_draft); - slot.spec_i_batch.clear(); + } catch (const std::exception & e) { + json error_json = format_error_response(e.what(), ERROR_TYPE_SERVER); + output = format_error(res_type, error_json); - GGML_ASSERT(accepted.size() >= 1); + // terminate on exception + return false; + } + }; + } - const uint32_t n_rollback = slot.spec_draft.size() + 1 - accepted.size(); + return res; +} - const bool use_ckpt_tgt = - ctx_tgt_seq_rm_type == COMMON_CONTEXT_SEQ_RM_TYPE_FULL || - (ctx_tgt_seq_rm_type == COMMON_CONTEXT_SEQ_RM_TYPE_RS && n_rollback > llama_n_rs_seq(ctx_tgt)); +std::unique_ptr server_routes::create_response(bool bypass_sleep) { + return std::make_unique(queue_tasks, queue_results, params.sleep_idle_seconds, bypass_sleep); +} - // check for partial draft acceptance - if (n_rollback > 0) { - if (use_ckpt_tgt) { - if (trace > 0) { - SLT_INF(slot, "accepted %2zu/%2zu draft tokens (restore checkpoint)\n", accepted.size() - 1, slot.spec_draft.size()); - } +void server_routes::evict_decode_results_locked() { + // Evict by TTL + const int64_t now = std::time(nullptr); + for (auto it = decode_results.begin(); it != decode_results.end(); ) { + if (now - it->second.created_at >= it->second.ttl_s) { + it = decode_results.erase(it); + } else { + ++it; + } + } + // Evict oldest by insertion order when over capacity + while ((int)decode_results.size() > decode_result_max) { + decode_results.erase(decode_results.begin()); + } +} - // partial acceptance is not supported by the context -> truncate the draft and restore the state - slot.spec_draft = std::move(accepted); +server_routes::server_routes(const common_params & params, server_context & ctx_server) + : params(params), + ctx_server_outer(ctx_server), + ctx_server(*ctx_server.impl), + queue_tasks(ctx_server.impl->queue_tasks), + queue_results(ctx_server.impl->queue_results) { + // Merged DECODE result buffer config from env vars + if (const char * e = getenv("HYDRA_DECODE_RESULT_TTL_S")) { + decode_result_ttl_s = std::max(1, atoi(e)); + } else { + decode_result_ttl_s = HYDRA_DECODE_RESULT_TTL_S_DEFAULT; + } + if (const char * e = getenv("HYDRA_DECODE_RESULT_MAX")) { + decode_result_max = std::max(1, atoi(e)); + } else { + decode_result_max = HYDRA_DECODE_RESULT_MAX_DEFAULT; + } + init_routes(); +} - const auto & ckpt = slot.spec_ckpt; +void server_routes::init_routes() { + // IMPORTANT: all lambda functions must start with create_response() + // this is to ensure that the server_res_generator can handle sleeping case correctly - SLT_DBG(slot, "restoring speculative checkpoint (pos_min = %d, pos_max = %d, size = %zu)\n", ckpt.pos_min, ckpt.pos_max, ckpt.size()); + this->get_health = [this](const server_http_req &) { + // error and loading states are handled by middleware + auto res = create_response(true); - { - ckpt.load_tgt(slot.ctx_tgt, slot.id, LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY | LLAMA_STATE_SEQ_FLAGS_ON_DEVICE); + // this endpoint can be accessed during sleeping + // the next LOC is to avoid someone accidentally use ctx_server + bool ctx_server; // do NOT delete this line + GGML_UNUSED(ctx_server); - common_context_seq_rm(slot.ctx_tgt, slot.id, ckpt.pos_max + 1, -1); - } + res->ok({{"status", "ok"}}); + return res; + }; - if (slot.ctx_dft) { - ckpt.load_dft(slot.ctx_dft, slot.id, LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY | LLAMA_STATE_SEQ_FLAGS_ON_DEVICE); - - common_context_seq_rm(slot.ctx_dft, slot.id, ckpt.pos_max + 1, -1); - } - - slot.prompt.tokens.keep_first(ckpt.n_tokens); - slot.smpl = std::move(smpl_save); - - continue; - } - } - - if (trace > 0) { - SLT_INF(slot, "accepted %2zu/%2zu draft tokens\n", accepted.size() - 1, n_draft); - } - - common_speculative_accept(spec.get(), slot.id, accepted.size() - 1); + this->get_metrics = [this](const server_http_req & req) { + auto res = create_response(); + if (!params.endpoint_metrics) { + res->error(format_error_response("This server does not support metrics endpoint. Start it with `--metrics`", ERROR_TYPE_NOT_SUPPORTED)); + return res; + } - slot.spec_draft = std::move(accepted); - } + // request slots data using task queue + { + server_task task(SERVER_TASK_TYPE_METRICS); + task.id = res->rd.get_new_id(); + res->rd.post_task(std::move(task), true); // high-priority task + } - const int64_t t_current = ggml_time_us(); + // get the result + auto result = res->rd.next(req.should_stop); + if (!result) { + // connection was closed + GGML_ASSERT(req.should_stop()); + return res; + } - const auto ids = std::move(slot.spec_draft); + if (result->is_error()) { + res->error(result->to_json()); + return res; + } - slot.t_token_generation = std::max(1, t_current - slot.t_start_generation) / 1e3; + // TODO: get rid of this dynamic_cast + auto res_task = dynamic_cast(result.get()); + GGML_ASSERT(res_task != nullptr); - // update how many tokens out of those tested were accepted - slot.n_draft_accepted += ids.size() - 1; + // metrics definition: https://prometheus.io/docs/practices/naming/#metric-names + json all_metrics_def = json { + {"counter", {{ + {"name", "prompt_tokens_total"}, + {"help", "Number of prompt tokens processed."}, + {"value", (uint64_t) res_task->n_prompt_tokens_processed_total} + }, { + {"name", "prompt_seconds_total"}, + {"help", "Prompt process time"}, + {"value", (uint64_t) res_task->t_prompt_processing_total / 1.e3} + }, { + {"name", "tokens_predicted_total"}, + {"help", "Number of generation tokens processed."}, + {"value", (uint64_t) res_task->n_tokens_predicted_total} + }, { + {"name", "tokens_predicted_seconds_total"}, + {"help", "Predict process time"}, + {"value", (uint64_t) res_task->t_tokens_generation_total / 1.e3} + }, { + {"name", "n_decode_total"}, + {"help", "Total number of llama_decode() calls"}, + {"value", res_task->n_decode_total} + }, { + {"name", "n_tokens_max"}, + {"help", "Largest observed n_tokens."}, + {"value", res_task->n_tokens_max} + }}}, + {"gauge", {{ + {"name", "prompt_tokens_seconds"}, + {"help", "Average prompt throughput in tokens/s."}, + {"value", res_task->n_prompt_tokens_processed ? 1.e3 / res_task->t_prompt_processing * res_task->n_prompt_tokens_processed : 0.} + },{ + {"name", "predicted_tokens_seconds"}, + {"help", "Average generation throughput in tokens/s."}, + {"value", res_task->n_tokens_predicted ? 1.e3 / res_task->t_tokens_generation * res_task->n_tokens_predicted : 0.} + },{ + {"name", "requests_processing"}, + {"help", "Number of requests processing."}, + {"value", (uint64_t) res_task->n_processing_slots} + },{ + {"name", "requests_deferred"}, + {"help", "Number of requests deferred."}, + {"value", (uint64_t) res_task->n_tasks_deferred} + },{ + {"name", "n_busy_slots_per_decode"}, + {"help", "Average number of busy slots per llama_decode() call"}, + {"value", (float) res_task->n_busy_slots_total / std::max((float) res_task->n_decode_total, 1.f)} + }}} + }; - // add accepted tokens to the prompt - slot.prompt.tokens.keep_first(slot.prompt.n_tokens() - n_draft); - slot.prompt.tokens.insert({ids.begin(), ids.end() - 1}); + std::stringstream prometheus; - slot.sampled = ids.back(); // last accepted token - SLT_DBG(slot, "add accepted tokens: sampled=%d, ids.size=%zu, n_draft=%zu\n", slot.sampled, ids.size(), n_draft); + for (const auto & el : all_metrics_def.items()) { + const auto & type = el.key(); + const auto & metrics_def = el.value(); - common_context_seq_rm(slot.ctx_tgt, slot.id, slot.prompt.tokens.pos_next(), -1); - if (slot.ctx_dft) { - common_context_seq_rm(slot.ctx_dft, slot.id, slot.prompt.tokens.pos_next(), -1); - } + for (const auto & metric_def : metrics_def) { + const std::string name = metric_def.at("name"); + const std::string help = metric_def.at("help"); - for (size_t i = 0; i < ids.size(); ++i) { - completion_token_output result; + auto value = json_value(metric_def, "value", 0.); + prometheus << "# HELP llamacpp:" << name << " " << help << "\n" + << "# TYPE llamacpp:" << name << " " << type << "\n" + << "llamacpp:" << name << " " << value << "\n"; + } + } - result.tok = ids[i]; - result.text_to_send = common_token_to_piece(slot.ctx_tgt, result.tok, accept_special_token(slot, result.tok)); - result.prob = 1.0f; // set later + res->headers["Process-Start-Time-Unix"] = std::to_string(res_task->t_start); + res->content_type = "text/plain; version=0.0.4"; + res->status = 200; + res->data = prometheus.str(); + return res; + }; - // TODO: set result.probs + this->get_slots = [this](const server_http_req & req) { + auto res = create_response(); + if (!params.endpoint_slots) { + res->error(format_error_response("This server does not support slots endpoint. Start it with `--slots`", ERROR_TYPE_NOT_SUPPORTED)); + return res; + } - slot.n_decoded += 1; + // request slots data using task queue + { + server_task task(SERVER_TASK_TYPE_METRICS); + task.id = res->rd.get_new_id(); + res->rd.post_task(std::move(task), true); // high-priority task + } - if (!process_token(result, slot)) { - slot.print_timings(); - send_final_response(slot); - metrics.on_prediction(slot); - slot.release(); + // get the result + auto result = res->rd.next(req.should_stop); + if (!result) { + // connection was closed + GGML_ASSERT(req.should_stop()); + return res; + } - break; - } - } + if (result->is_error()) { + res->error(result->to_json()); + return res; + } - slot.print_timings_tg(); + // TODO: get rid of this dynamic_cast + auto * res_task = dynamic_cast(result.get()); + GGML_ASSERT(res_task != nullptr); - SLT_DBG(slot, "accepted %d/%d draft tokens, new n_tokens = %d\n", (int) ids.size() - 1, (int) n_draft, slot.prompt.n_tokens()); + // optionally return "fail_on_no_slot" error + if (!req.get_param("fail_on_no_slot").empty()) { + if (res_task->n_idle_slots == 0) { + res->error(format_error_response("no slot available", ERROR_TYPE_UNAVAILABLE)); + return res; } } - SRV_DBG("%s", "run slots completed\n"); - } - - int get_slot_n_ctx() { - return slots.back().n_ctx; - } - - server_response_reader get_response_reader() { - return server_response_reader(queue_tasks, queue_results, HTTP_POLLING_SECONDS); - } -}; - -// -// server_context (public API) -// - -server_context::server_context() : impl(new server_context_impl()) {} -server_context::~server_context() = default; - -bool server_context::load_model(common_params & params) { - return impl->load_model(params); -} - -void server_context::start_loop() { - auto & params = impl->params_base; - impl->queue_tasks.start_loop(params.sleep_idle_seconds * 1000); -} + res->ok(res_task->slots_data); + return res; + }; -void server_context::terminate() { - impl->queue_tasks.terminate(); -} + this->post_slots = [this](const server_http_req & req) { + auto res = create_response(); + if (params.slot_save_path.empty()) { + res->error(format_error_response("This server does not support slots action. Start it with `--slot-save-path`", ERROR_TYPE_NOT_SUPPORTED)); + return res; + } -llama_context * server_context::get_llama_context() const { - return impl->ctx_tgt; -} + std::string id_slot_str = req.get_param("id_slot"); -void server_context::set_hydra_capabilities(bool rpc_backend_active, const std::string & peer, - bool peer_reachable, const std::string & combined_pattern, const std::string & split_mode) { - impl->hydra_rpc_backend_active = rpc_backend_active; - impl->hydra_peer = peer; - impl->hydra_peer_reachable = peer_reachable; - impl->hydra_combined_pattern = combined_pattern; - impl->hydra_split_mode = split_mode; -} + int id_slot; + try { + id_slot = std::stoi(id_slot_str); + } catch (const std::exception &) { + res->error(format_error_response("Invalid slot ID", ERROR_TYPE_INVALID_REQUEST)); + return res; + } -void server_context::set_hydra_combined_head_attached(bool attached) { - impl->hydra_combined_head_attached = attached; -} + std::string action = req.get_param("action"); -void server_context::set_hydra_combined_static(bool is_static) { - impl->hydra_combined_static = is_static; -} + if (action == "save") { + return handle_slots_save(req, id_slot); + } + if (action == "restore") { + return handle_slots_restore(req, id_slot); + } + if (action == "erase") { + return handle_slots_erase(req, id_slot); + } -server_response_reader server_context::get_response_reader() { - return impl->get_response_reader(); -} + res->error(format_error_response("Invalid action", ERROR_TYPE_INVALID_REQUEST)); + return res; + }; -server_context_meta server_context::get_meta() const { - auto bos_id = llama_vocab_bos(impl->vocab); - auto eos_id = llama_vocab_eos(impl->vocab); - auto bos_token_str = bos_id != LLAMA_TOKEN_NULL ? common_token_to_piece(impl->ctx_tgt, bos_id, true) : ""; - auto eos_token_str = eos_id != LLAMA_TOKEN_NULL ? common_token_to_piece(impl->ctx_tgt, eos_id, true) : ""; - - return server_context_meta { - /* build_info */ std::string(llama_build_info()), - /* model_name */ impl->model_name, - /* model_aliases */ impl->model_aliases, - /* model_tags */ impl->model_tags, - /* model_path */ impl->params_base.model.path, - /* has_mtmd */ impl->mctx != nullptr, - /* has_inp_image */ impl->chat_params.allow_image, - /* has_inp_audio */ impl->chat_params.allow_audio, - /* json_ui_settings */ impl->json_ui_settings, - /* json_webui_settings */ impl->json_webui_settings, // Deprecated - /* slot_n_ctx */ impl->get_slot_n_ctx(), - /* pooling_type */ llama_pooling_type(impl->ctx_tgt), - - /* chat_params */ impl->chat_params, - /* chat_template_caps */ common_chat_templates_get_caps(impl->chat_params.tmpls.get()), - - /* bos_token_str */ bos_token_str, - /* eos_token_str */ eos_token_str, - /* fim_pre_token */ llama_vocab_fim_pre(impl->vocab), - /* fim_sub_token */ llama_vocab_fim_suf(impl->vocab), - /* fim_mid_token */ llama_vocab_fim_mid(impl->vocab), - /* fim_pad_token */ llama_vocab_fim_pad(impl->vocab), - /* fim_rep_token */ llama_vocab_fim_rep(impl->vocab), - /* fim_sep_token */ llama_vocab_fim_sep(impl->vocab), - - /* logit_bias_eog */ impl->params_base.sampling.logit_bias_eog, - - /* model_vocab_type */ llama_vocab_type(impl->vocab), - /* model_vocab_n_tokens */ llama_vocab_n_tokens(impl->vocab), - /* model_n_ctx_train */ llama_model_n_ctx_train(impl->model_tgt), - /* model_n_embd_inp */ llama_model_n_embd(impl->model_tgt), - /* model_n_params */ llama_model_n_params(impl->model_tgt), - /* model_size */ llama_model_size(impl->model_tgt), - /* split_mode */ impl->params_base.split_mode, - /* tensor_split */ std::vector(impl->params_base.tensor_split, impl->params_base.tensor_split + 128), - }; -} - - - -// generator-like API for HTTP response generation -// may have bypass_sleep = true if the task does not use ctx_server -struct server_res_generator : server_http_res { - server_response_reader rd; - server_res_generator(server_queue & queue_tasks, server_response & queue_results, int sleep_idle_seconds, bool bypass_sleep = false) - : rd(queue_tasks, queue_results, HTTP_POLLING_SECONDS) { - // fast path in case sleeping is disabled - bypass_sleep |= sleep_idle_seconds < 0; - if (!bypass_sleep) { - queue_tasks.wait_until_no_sleep(); - } - } - void ok(const json & response_data) { - status = 200; - data = safe_json_to_str(response_data); - } - void error(const json & error_data) { - status = json_value(error_data, "code", 500); - data = safe_json_to_str({{ "error", error_data }}); - } -}; - -void server_context::on_sleeping_changed(std::function callback) { - impl->queue_tasks.on_sleeping_state(std::move(callback)); -} - -void server_context::set_routes_ptr(server_routes * routes) { - impl->routes_ptr = routes; -} - -bool server_context::bootstrap_init() { - return impl->bootstrap_init(); -} - -void server_context::set_bootstrap_capabilities(bool rpc_active, const std::string & peer, - bool peer_reachable, const std::string & pattern, - const std::string & split_mode, bool combined_static) { - impl->bootstrap_rpc_active = rpc_active; - impl->bootstrap_peer = peer; - impl->bootstrap_peer_reachable = peer_reachable; - impl->bootstrap_pattern = pattern; - impl->bootstrap_split_mode = split_mode; - impl->bootstrap_combined_static = combined_static; -} - -// compute the number of tokens before the last user message in the prompt -static int32_t prompt_get_n_before_user( - const json & message_spans, - const std::string & prompt, - const std::vector & files, - const llama_vocab * vocab, - mtmd_context * mctx) { - int32_t result = -1; - int32_t byte_pos = -1; - - for (const auto & span : message_spans) { - const std::string role = json_value(span, "role", std::string()); - - if (role == "user") { - byte_pos = json_value(span, "pos", -1); - } - } - - if (byte_pos >= 0) { - GGML_ASSERT((size_t) byte_pos <= prompt.size()); - - const std::string prefix = prompt.substr(0, (size_t) byte_pos); - - const std::string marker = get_media_marker(); - size_t n_prefix_media = 0; - for (size_t pos = 0; (pos = prefix.find(marker, pos)) != std::string::npos; pos += marker.size()) { - n_prefix_media++; - } - - GGML_ASSERT(n_prefix_media <= files.size()); - - if (mctx != nullptr && n_prefix_media > 0) { - // TODO: this makes a copy - avoid it - std::vector prefix_files(files.begin(), files.begin() + n_prefix_media); - - result = (int32_t) process_mtmd_prompt(mctx, prefix, prefix_files).size(); - } else { - result = (int32_t) tokenize_input_prompts(vocab, nullptr, prefix, true, true)[0].size(); - } - - SRV_TRC("message_spans: last user message: byte_pos=%d, media=%zu, n_before_user=%d\n", - byte_pos, n_prefix_media, result); - } - - return result; -} - - -// -// server_routes -// - -std::unique_ptr server_routes::handle_completions_impl( - const server_http_req & req, - server_task_type type, - const json & data, - const std::vector & files, - task_response_type res_type, - const std::string & hydra_config_json) { - GGML_ASSERT(type == SERVER_TASK_TYPE_COMPLETION || type == SERVER_TASK_TYPE_INFILL); - - std::shared_lock meta_lock(meta_mutex); - - // P0-1 (#49): null-meta guard — meta is null until update_meta() is - // called after model load. Return 503 instead of crashing. - if (!meta) { - auto res = create_response(); - res->error(format_error_response("model not loaded — waiting for CONFIGURE", - ERROR_TYPE_NOT_SUPPORTED)); - return res; - } - - auto res = create_response(); - auto completion_id = gen_chatcmplid(); - auto & rd = res->rd; - auto & params = this->params; - - try { - std::vector tasks; - - const auto & prompt = data.at("prompt"); - // TODO: this log can become very long, put it behind a flag or think about a more compact format - //SRV_DBG("Prompt: %s\n", prompt.is_string() ? prompt.get().c_str() : prompt.dump(2).c_str()); - - // process prompt - std::vector inputs; - - if (res_type != TASK_RESPONSE_TYPE_NONE && ctx_server.mctx != nullptr) { - // This is the case used by OAI compatible chat path with MTMD. TODO It can be moved to the path below. - inputs.push_back(process_mtmd_prompt(ctx_server.mctx, prompt.get(), files)); - } else { - // Everything else, including multimodal completions. - inputs = tokenize_input_prompts(ctx_server.vocab, ctx_server.mctx, prompt, true, true); - } - - // tasks.reserve(inputs.size()); // TODO: this is inaccurate due to child tasks - - for (size_t i = 0; i < inputs.size(); i++) { - server_task task = server_task(type); - - task.id = rd.get_new_id(); - - task.tokens = std::move(inputs[i]); - task.params = server_task::params_from_json_cmpl( - ctx_server.vocab, - params, - meta->slot_n_ctx, - meta->logit_bias_eog, - data); - - const auto message_spans = json_value(data, "message_spans", json::array()); - if (prompt.is_string() && message_spans.is_array()) { - task.params.n_before_user = - prompt_get_n_before_user( - message_spans, - prompt.get(), - files, - ctx_server.vocab, - ctx_server.mctx); - } - - task.id_slot = json_value(data, "id_slot", -1); - - // OAI-compat - task.params.res_type = res_type; - task.params.oaicompat_cmpl_id = completion_id; - task.params.oaicompat_model = meta->model_name; - - // Attach hydra_config (if any) so the task-queue thread can - // apply it synchronously before processing the completion. - if (!hydra_config_json.empty()) { - task.hydra_config_json = hydra_config_json; - } - - // prepare child tasks - if (task.params.n_cmpl > 1) { - int n_children = task.params.n_cmpl - 1; - for (int j = 0; j < n_children; j++) { - task.add_child(task.id, rd.get_new_id()); - } - } - - tasks.push_back(std::move(task)); - } - - rd.post_tasks(std::move(tasks)); - } catch (const std::exception & e) { - res->error(format_error_response(e.what(), ERROR_TYPE_INVALID_REQUEST)); - return res; - } - - bool stream = json_value(data, "stream", false); - - if (!stream) { - // non-stream, wait for the results - auto all_results = rd.wait_for_all(req.should_stop); - if (all_results.is_terminated) { - return res; // connection is closed - } else if (all_results.error) { - res->error(all_results.error->to_json()); - return res; - } else { - json arr = json::array(); - for (auto & res : all_results.results) { - auto * cmpl = dynamic_cast(res.get()); - GGML_ASSERT(cmpl != nullptr); - // P1-5/P1-6: build metrics inline from the (always-current) - // meta rather than reading a class-level field that could - // leak between concurrent requests. - { - auto split_mode_str = [](enum llama_split_mode m) -> const char * { - switch (m) { - case LLAMA_SPLIT_MODE_NONE: return "none"; - case LLAMA_SPLIT_MODE_LAYER: return "layer"; - case LLAMA_SPLIT_MODE_ROW: return "row"; - default: return "none"; - } - }; - json metrics = json::object(); - metrics["model_path"] = meta->model_path; - metrics["split_mode"] = split_mode_str(meta->split_mode); - metrics["t3_reloaded"] = false; - metrics["t3_reload_ms"] = 0.0; - json ts_arr = json::array(); - for (const auto & v : meta->tensor_split) { - if (v != 0.0f) { - ts_arr.push_back(v); - } else { - break; - } - } - metrics["tensor_split"] = ts_arr; - cmpl->hydra_metrics = metrics; - } - arr.push_back(cmpl->to_json()); - } - GGML_ASSERT(!arr.empty() && "empty results"); - if (arr.size() == 1) { - // if single request, return single object instead of array - res->ok(arr[0]); - } else if (res_type == TASK_RESPONSE_TYPE_OAI_CHAT || res_type == TASK_RESPONSE_TYPE_OAI_CMPL) { - // if multiple results in OAI format, we need to re-format them - json & choices = arr[0]["choices"]; - for (size_t i = 1; i < arr.size(); i++) { - choices.push_back(std::move(arr[i]["choices"][0])); - } - res->ok(arr[0]); - } else { - // multi-results, non-OAI compat - res->ok(arr); - } - } - } else { - // in streaming mode, the first error must be treated as non-stream response - // this is to match the OAI API behavior - // ref: https://github.com/ggml-org/llama.cpp/pull/16486#discussion_r2419657309 - auto first_result = rd.next(req.should_stop); - if (first_result == nullptr) { - GGML_ASSERT(req.should_stop()); - return res; // connection is closed - } - - if (first_result->is_error()) { - res->error(first_result->to_json()); - return res; - } - - GGML_ASSERT( - dynamic_cast(first_result.get()) != nullptr || - dynamic_cast (first_result.get()) != nullptr - ); - - // next responses are streamed - // to be sent immediately - json first_result_json = first_result->to_json(); - if (first_result_json == nullptr) { - res->data = ""; // simply send HTTP headers and status code - } else if (res_type == TASK_RESPONSE_TYPE_ANTHROPIC) { - res->data = format_anthropic_sse(first_result_json); - } else if (res_type == TASK_RESPONSE_TYPE_OAI_RESP) { - res->data = format_oai_resp_sse(first_result_json); - } else { - res->data = format_oai_sse(first_result_json); - } - res->status = 200; - res->content_type = "text/event-stream"; - res->next = [res_this = res.get(), res_type, &req, ¶ms](std::string & output) -> bool { - static auto format_error = [](task_response_type res_type, const json & res_json) { - if (res_type == TASK_RESPONSE_TYPE_ANTHROPIC) { - return format_anthropic_sse({ - {"event", "error"}, - {"data", res_json}, - }); - } else { - return format_oai_sse(json {{ "error", res_json }}); - } - }; - - try { - if (req.should_stop()) { - SRV_DBG("%s", "stopping streaming due to should_stop condition\n"); - return false; // should_stop condition met - } - - if (!res_this->data.empty()) { - // flush the first chunk - output = std::move(res_this->data); - res_this->data.clear(); - return true; - } - - server_response_reader & rd = res_this->rd; - - // check if there is more data - if (!rd.has_next()) { - switch (res_type) { - case TASK_RESPONSE_TYPE_NONE: - case TASK_RESPONSE_TYPE_OAI_RESP: - case TASK_RESPONSE_TYPE_ANTHROPIC: - output = ""; - break; - - default: - output = "data: [DONE]\n\n"; - break; - } - SRV_DBG("%s", "all results received, terminating stream\n"); - return false; // no more data, terminate - } - - // receive subsequent results - bool timeout = false; - int64_t start_time = ggml_time_ms(); - auto result = rd.next([&timeout, &req, &start_time, ¶ms]() { - if (req.should_stop()) { - return true; // should_stop condition met - } else if (params.sse_ping_interval > 0 && ggml_time_ms() - start_time > (int64_t)params.sse_ping_interval * 1000) { - timeout = true; - return true; // timeout - } - return false; - }); - - if (timeout) { - // some clients may time out (e.g. undici) will time out if no data is received for a while, so we need to send a ping to keep the connection alive - SRV_DBG("%s", "sending SSE ping\n"); - output = ":\n\n"; - return true; - } - - if (result == nullptr) { - SRV_DBG("%s", "stopping streaming due to should_stop condition\n"); - GGML_ASSERT(req.should_stop()); - return false; // should_stop condition met - } - - // send the results - if (result->is_error()) { - json res_json = result->to_json(); - output = format_error(res_type, res_json); - SRV_DBG("%s", "error received during streaming, terminating stream\n"); - return false; // terminate on error - } else { - GGML_ASSERT( - dynamic_cast(result.get()) != nullptr - || dynamic_cast(result.get()) != nullptr - ); - json res_json = result->to_json(); - if (res_type == TASK_RESPONSE_TYPE_ANTHROPIC) { - output = format_anthropic_sse(res_json); - } else if (res_type == TASK_RESPONSE_TYPE_OAI_RESP) { - output = format_oai_resp_sse(res_json); - } else { - output = format_oai_sse(res_json); - } - } - - // has next data, continue - return true; - - } catch (const std::exception & e) { - json error_json = format_error_response(e.what(), ERROR_TYPE_SERVER); - output = format_error(res_type, error_json); - - // terminate on exception - return false; - } - }; - } - - return res; -} - -std::unique_ptr server_routes::create_response(bool bypass_sleep) { - return std::make_unique(queue_tasks, queue_results, params.sleep_idle_seconds, bypass_sleep); -} - -void server_routes::evict_decode_results_locked() { - // Evict by TTL - const int64_t now = std::time(nullptr); - for (auto it = decode_results.begin(); it != decode_results.end(); ) { - if (now - it->second.created_at >= it->second.ttl_s) { - it = decode_results.erase(it); - } else { - ++it; - } - } - // Evict oldest by insertion order when over capacity - while ((int)decode_results.size() > decode_result_max) { - decode_results.erase(decode_results.begin()); - } -} - -server_routes::server_routes(const common_params & params, server_context & ctx_server) - : params(params), - ctx_server_outer(ctx_server), - ctx_server(*ctx_server.impl), - queue_tasks(ctx_server.impl->queue_tasks), - queue_results(ctx_server.impl->queue_results) { - // Merged DECODE result buffer config from env vars - if (const char * e = getenv("HYDRA_DECODE_RESULT_TTL_S")) { - decode_result_ttl_s = std::max(1, atoi(e)); - } else { - decode_result_ttl_s = HYDRA_DECODE_RESULT_TTL_S_DEFAULT; - } - if (const char * e = getenv("HYDRA_DECODE_RESULT_MAX")) { - decode_result_max = std::max(1, atoi(e)); - } else { - decode_result_max = HYDRA_DECODE_RESULT_MAX_DEFAULT; - } - init_routes(); -} - -void server_routes::init_routes() { - // IMPORTANT: all lambda functions must start with create_response() - // this is to ensure that the server_res_generator can handle sleeping case correctly - - this->get_health = [this](const server_http_req &) { - // error and loading states are handled by middleware - auto res = create_response(true); - - // this endpoint can be accessed during sleeping - // the next LOC is to avoid someone accidentally use ctx_server - bool ctx_server; // do NOT delete this line - GGML_UNUSED(ctx_server); - - res->ok({{"status", "ok"}}); - return res; - }; - - this->get_metrics = [this](const server_http_req & req) { - auto res = create_response(); - if (!params.endpoint_metrics) { - res->error(format_error_response("This server does not support metrics endpoint. Start it with `--metrics`", ERROR_TYPE_NOT_SUPPORTED)); - return res; - } - - // request slots data using task queue - { - server_task task(SERVER_TASK_TYPE_METRICS); - task.id = res->rd.get_new_id(); - res->rd.post_task(std::move(task), true); // high-priority task - } - - // get the result - auto result = res->rd.next(req.should_stop); - if (!result) { - // connection was closed - GGML_ASSERT(req.should_stop()); - return res; - } - - if (result->is_error()) { - res->error(result->to_json()); - return res; - } - - // TODO: get rid of this dynamic_cast - auto res_task = dynamic_cast(result.get()); - GGML_ASSERT(res_task != nullptr); - - // metrics definition: https://prometheus.io/docs/practices/naming/#metric-names - json all_metrics_def = json { - {"counter", {{ - {"name", "prompt_tokens_total"}, - {"help", "Number of prompt tokens processed."}, - {"value", (uint64_t) res_task->n_prompt_tokens_processed_total} - }, { - {"name", "prompt_seconds_total"}, - {"help", "Prompt process time"}, - {"value", (uint64_t) res_task->t_prompt_processing_total / 1.e3} - }, { - {"name", "tokens_predicted_total"}, - {"help", "Number of generation tokens processed."}, - {"value", (uint64_t) res_task->n_tokens_predicted_total} - }, { - {"name", "tokens_predicted_seconds_total"}, - {"help", "Predict process time"}, - {"value", (uint64_t) res_task->t_tokens_generation_total / 1.e3} - }, { - {"name", "n_decode_total"}, - {"help", "Total number of llama_decode() calls"}, - {"value", res_task->n_decode_total} - }, { - {"name", "n_tokens_max"}, - {"help", "Largest observed n_tokens."}, - {"value", res_task->n_tokens_max} - }}}, - {"gauge", {{ - {"name", "prompt_tokens_seconds"}, - {"help", "Average prompt throughput in tokens/s."}, - {"value", res_task->n_prompt_tokens_processed ? 1.e3 / res_task->t_prompt_processing * res_task->n_prompt_tokens_processed : 0.} - },{ - {"name", "predicted_tokens_seconds"}, - {"help", "Average generation throughput in tokens/s."}, - {"value", res_task->n_tokens_predicted ? 1.e3 / res_task->t_tokens_generation * res_task->n_tokens_predicted : 0.} - },{ - {"name", "requests_processing"}, - {"help", "Number of requests processing."}, - {"value", (uint64_t) res_task->n_processing_slots} - },{ - {"name", "requests_deferred"}, - {"help", "Number of requests deferred."}, - {"value", (uint64_t) res_task->n_tasks_deferred} - },{ - {"name", "n_busy_slots_per_decode"}, - {"help", "Average number of busy slots per llama_decode() call"}, - {"value", (float) res_task->n_busy_slots_total / std::max((float) res_task->n_decode_total, 1.f)} - }}} - }; - - std::stringstream prometheus; - - for (const auto & el : all_metrics_def.items()) { - const auto & type = el.key(); - const auto & metrics_def = el.value(); - - for (const auto & metric_def : metrics_def) { - const std::string name = metric_def.at("name"); - const std::string help = metric_def.at("help"); - - auto value = json_value(metric_def, "value", 0.); - prometheus << "# HELP llamacpp:" << name << " " << help << "\n" - << "# TYPE llamacpp:" << name << " " << type << "\n" - << "llamacpp:" << name << " " << value << "\n"; - } - } - - res->headers["Process-Start-Time-Unix"] = std::to_string(res_task->t_start); - res->content_type = "text/plain; version=0.0.4"; - res->status = 200; - res->data = prometheus.str(); - return res; - }; - - this->get_slots = [this](const server_http_req & req) { - auto res = create_response(); - if (!params.endpoint_slots) { - res->error(format_error_response("This server does not support slots endpoint. Start it with `--slots`", ERROR_TYPE_NOT_SUPPORTED)); - return res; - } - - // request slots data using task queue - { - server_task task(SERVER_TASK_TYPE_METRICS); - task.id = res->rd.get_new_id(); - res->rd.post_task(std::move(task), true); // high-priority task - } - - // get the result - auto result = res->rd.next(req.should_stop); - if (!result) { - // connection was closed - GGML_ASSERT(req.should_stop()); - return res; - } - - if (result->is_error()) { - res->error(result->to_json()); - return res; - } - - // TODO: get rid of this dynamic_cast - auto * res_task = dynamic_cast(result.get()); - GGML_ASSERT(res_task != nullptr); - - // optionally return "fail_on_no_slot" error - if (!req.get_param("fail_on_no_slot").empty()) { - if (res_task->n_idle_slots == 0) { - res->error(format_error_response("no slot available", ERROR_TYPE_UNAVAILABLE)); - return res; - } - } - - res->ok(res_task->slots_data); - return res; - }; - - this->post_slots = [this](const server_http_req & req) { - auto res = create_response(); - if (params.slot_save_path.empty()) { - res->error(format_error_response("This server does not support slots action. Start it with `--slot-save-path`", ERROR_TYPE_NOT_SUPPORTED)); - return res; - } - - std::string id_slot_str = req.get_param("id_slot"); - - int id_slot; - try { - id_slot = std::stoi(id_slot_str); - } catch (const std::exception &) { - res->error(format_error_response("Invalid slot ID", ERROR_TYPE_INVALID_REQUEST)); - return res; - } - - std::string action = req.get_param("action"); - - if (action == "save") { - return handle_slots_save(req, id_slot); - } - if (action == "restore") { - return handle_slots_restore(req, id_slot); - } - if (action == "erase") { - return handle_slots_erase(req, id_slot); - } - - res->error(format_error_response("Invalid action", ERROR_TYPE_INVALID_REQUEST)); - return res; - }; - - // ── Hydra state streaming (M0.0) ─────────────────────────────────────── - this->get_state = [this](const server_http_req & req) { - auto res = create_response(); - int id_slot; - try { - id_slot = std::stoi(req.get_param("id_slot")); - } catch (const std::exception &) { - res->error(format_error_response("Invalid slot ID", ERROR_TYPE_INVALID_REQUEST)); - return res; - } - server_task task(SERVER_TASK_TYPE_HYDRA_STATE_GET); - task.id = res->rd.get_new_id(); - task.hydra_action.id_slot = id_slot; - task.hydra_action.hydra_fd = -1; - res->rd.post_task(std::move(task)); - auto result = res->rd.next(req.should_stop); - if (!result) { - GGML_ASSERT(req.should_stop()); - return res; - } - if (result->is_error()) { - res->error(result->to_json()); - return res; - } - auto * hr = dynamic_cast(result.get()); - GGML_ASSERT(hr != nullptr); - if (hr->rpc_status != HYDRA_STATUS_OK) { - res->status = hr->rpc_status == HYDRA_STATUS_NOT_FOUND ? 404 : 503; - res->data = hr->error.empty() ? "" : hr->error; - return res; - } - res->content_type = "application/octet-stream"; - res->headers["X-Hydra-State-Size"] = std::to_string(hr->state_data.size()); - res->headers["X-Hydra-N-Past"] = std::to_string(hr->n_past); - res->data.assign((const char*)hr->state_data.data(), hr->state_data.size()); - return res; - }; - this->put_state = [this](const server_http_req & req) { - auto res = create_response(); - int id_slot; - try { - id_slot = std::stoi(req.get_param("id_slot")); - } catch (const std::exception &) { - res->error(format_error_response("Invalid slot ID", ERROR_TYPE_INVALID_REQUEST)); - return res; - } - server_task task(SERVER_TASK_TYPE_HYDRA_STATE_PUT); - task.id = res->rd.get_new_id(); - task.hydra_action.id_slot = id_slot; - task.hydra_action.erase_existing = req.get_param("erase_existing") == "true"; - task.hydra_action.state_data.assign(req.body.begin(), req.body.end()); - res->rd.post_task(std::move(task)); - auto result = res->rd.next(req.should_stop); - if (!result) { - GGML_ASSERT(req.should_stop()); - return res; - } - if (result->is_error()) { - res->error(result->to_json()); - return res; - } - auto * hr = dynamic_cast(result.get()); - GGML_ASSERT(hr != nullptr); - if (hr->rpc_status != HYDRA_STATUS_OK) { - res->status = 503; - res->data = hr->error.empty() ? "restore failed" : hr->error; - return res; - } - res->ok(json{ - {"restored", hr->restored}, - {"n_past", hr->n_past}, - {"bytes", hr->bytes}, - }); - return res; - }; - this->get_state_meta = [this](const server_http_req & req) { - auto res = create_response(); - int id_slot; - try { - id_slot = std::stoi(req.get_param("id_slot")); - } catch (const std::exception &) { - res->error(format_error_response("Invalid slot ID", ERROR_TYPE_INVALID_REQUEST)); - return res; - } - server_task task(SERVER_TASK_TYPE_HYDRA_STATE_META); - task.id = res->rd.get_new_id(); - task.hydra_action.id_slot = id_slot; - res->rd.post_task(std::move(task)); - auto result = res->rd.next(req.should_stop); - if (!result) { - GGML_ASSERT(req.should_stop()); - return res; - } - if (result->is_error()) { - res->error(result->to_json()); - return res; - } - auto * hr = dynamic_cast(result.get()); - GGML_ASSERT(hr != nullptr); - if (hr->rpc_status != HYDRA_STATUS_OK) { - res->status = hr->rpc_status == HYDRA_STATUS_NOT_FOUND ? 404 : 503; - res->data = hr->error.empty() ? "" : hr->error; - return res; - } - res->ok(json{ - {"slot_id", hr->id_slot}, - {"n_past", hr->n_past}, - {"state_size", (uint64_t)hr->state_size}, - {"is_processing", hr->is_processing}, - {"is_transferring", hr->is_transferring}, - {"operation", hr->operation}, - {"progress", hr->progress}, - {"tokens_processed", hr->tokens_processed}, - {"tokens_total", hr->tokens_total}, - {"elapsed_ms", hr->elapsed_ms}, - // #470/A7: model identity — the Coordinator's merged-decode Gate A - // compares these against kv_metadata. Without them the engine's - // model_metadata came back empty and every COMBINED merged decode - // was rejected (tokenizer/name mismatch) before KV restore. - {"model_alias", hr->model_alias}, - {"model_path", hr->model_path}, - {"tokenizer", hr->tokenizer}, - {"model_name", hr->model_name}, - {"model_quant", hr->model_quant}, - {"model_capabilities", hr->model_capabilities}, - }); - return res; - }; - - this->get_props = [this](const server_http_req &) { - auto res = create_response(true); - std::shared_lock meta_lock(meta_mutex); - - // this endpoint can be accessed during sleeping - // the next LOC is to avoid someone accidentally use ctx_server - bool ctx_server; // do NOT delete this line - GGML_UNUSED(ctx_server); - - // P0-1 (#49): null-meta guard - if (!meta) { - res->error(format_error_response("model not loaded — waiting for CONFIGURE", - ERROR_TYPE_NOT_SUPPORTED)); - return res; - } - - task_params tparams; - tparams.sampling = params.sampling; - json default_generation_settings_for_props = json { - { "params", tparams.to_json(true) }, - { "n_ctx", meta->slot_n_ctx }, - }; - - std::string tmpl_default = common_chat_templates_source(meta->chat_params.tmpls.get(), ""); - std::string tmpl_tools = common_chat_templates_source(meta->chat_params.tmpls.get(), "tool_use"); - - json props = { - { "default_generation_settings", default_generation_settings_for_props }, - { "total_slots", params.n_parallel }, - { "model_alias", meta->model_name }, - { "model_path", meta->model_path }, - { "modalities", json { - {"vision", meta->has_inp_image}, - {"audio", meta->has_inp_audio}, - } }, - { "media_marker", get_media_marker() }, - { "endpoint_slots", params.endpoint_slots }, - { "endpoint_props", params.endpoint_props }, - { "endpoint_metrics", params.endpoint_metrics }, - // New keys - { "ui", params.ui }, - { "ui_settings", meta->json_ui_settings }, - // Deprecated: use ui/ui_settings instead (kept for backward compat) - { "webui", params.webui }, - { "webui_settings", meta->json_webui_settings }, - { "chat_template", tmpl_default }, - { "chat_template_caps", meta->chat_template_caps }, - { "bos_token", meta->bos_token_str }, - { "eos_token", meta->eos_token_str }, - { "build_info", meta->build_info }, - { "is_sleeping", queue_tasks.is_sleeping() }, - { "cors_proxy_enabled", params.ui_mcp_proxy || params.webui_mcp_proxy }, - }; - if (params.use_jinja) { - if (!tmpl_tools.empty()) { - props["chat_template_tool_use"] = tmpl_tools; - } - } - res->ok(props); - return res; - }; - - this->post_props = [this](const server_http_req &) { - auto res = create_response(); - if (!params.endpoint_props) { - res->error(format_error_response("This server does not support changing global properties. Start it with `--props`", ERROR_TYPE_NOT_SUPPORTED)); - return res; - } - // update any props here - - res->ok({{ "success", true }}); - return res; - }; - - this->post_infill = [this](const server_http_req & req) { - auto res = create_response(); - // P0-1 (#49): null-meta guard - { - std::shared_lock meta_lock(meta_mutex); - if (!meta) { - res->error(format_error_response("model not loaded — waiting for CONFIGURE", - ERROR_TYPE_NOT_SUPPORTED)); - return res; - } - } - - // Validate input and compute infill prompt — these read meta->slot_n_ctx. - // Scope the shared_lock so it releases before handle_completions_impl - // takes its own lock (recursive shared_lock is UB on std::shared_mutex). - json data; - std::vector files; - { - std::shared_lock meta_lock(meta_mutex); - - // check model compatibility - std::string err; - if (llama_vocab_fim_pre(ctx_server.vocab) == LLAMA_TOKEN_NULL) { - err += "prefix token is missing. "; - } - if (llama_vocab_fim_suf(ctx_server.vocab) == LLAMA_TOKEN_NULL) { - err += "suffix token is missing. "; - } - if (llama_vocab_fim_mid(ctx_server.vocab) == LLAMA_TOKEN_NULL) { - err += "middle token is missing. "; - } - if (!err.empty()) { - res->error(format_error_response(string_format("Infill is not supported by this model: %s", err.c_str()), ERROR_TYPE_NOT_SUPPORTED)); - return res; - } - - // validate input - data = json::parse(req.body); - if (data.contains("prompt") && !data.at("prompt").is_string()) { - // prompt is optional - res->error(format_error_response("\"prompt\" must be a string", ERROR_TYPE_INVALID_REQUEST)); - } - - if (!data.contains("input_prefix")) { - res->error(format_error_response("\"input_prefix\" is required", ERROR_TYPE_INVALID_REQUEST)); - } - - if (!data.contains("input_suffix")) { - res->error(format_error_response("\"input_suffix\" is required", ERROR_TYPE_INVALID_REQUEST)); - } - - if (data.contains("input_extra") && !data.at("input_extra").is_array()) { - // input_extra is optional - res->error(format_error_response("\"input_extra\" must be an array of {\"filename\": string, \"text\": string}", ERROR_TYPE_INVALID_REQUEST)); - return res; - } - - json input_extra = json_value(data, "input_extra", json::array()); - for (const auto & chunk : input_extra) { - // { "text": string, "filename": string } - if (!chunk.contains("text") || !chunk.at("text").is_string()) { - res->error(format_error_response("extra_context chunk must contain a \"text\" field with a string value", ERROR_TYPE_INVALID_REQUEST)); - return res; - } - // filename is optional - if (chunk.contains("filename") && !chunk.at("filename").is_string()) { - res->error(format_error_response("extra_context chunk's \"filename\" field must be a string", ERROR_TYPE_INVALID_REQUEST)); - return res; - } - } - data["input_extra"] = input_extra; // default to empty array if it's not exist - - std::string prompt = json_value(data, "prompt", std::string()); - std::vector tokenized_prompts = tokenize_input_prompts(ctx_server.vocab, ctx_server.mctx, prompt, false, true); - SRV_DBG("creating infill tasks, n_prompts = %d\n", (int) tokenized_prompts.size()); - data["prompt"] = format_prompt_infill( - ctx_server.vocab, - data.at("input_prefix"), - data.at("input_suffix"), - data.at("input_extra"), - params.n_batch, - params.n_predict, - meta->slot_n_ctx, - params.spm_infill, - tokenized_prompts[0].get_tokens() // TODO: this could maybe be multimodal. - ); - } - // meta_lock released here - - return handle_completions_impl( - req, - SERVER_TASK_TYPE_INFILL, - data, - files, - TASK_RESPONSE_TYPE_NONE); // infill is not OAI compatible - }; - - this->post_completions = [this](const server_http_req & req) { - auto res = create_response(); - std::vector files; // dummy - const json body = json::parse(req.body); - return handle_completions_impl( - req, - SERVER_TASK_TYPE_COMPLETION, - body, - files, - TASK_RESPONSE_TYPE_NONE); - }; - - this->post_completions_oai = [this](const server_http_req & req) { - auto res = create_response(); - std::vector files; // dummy - const json body = json::parse(req.body); - return handle_completions_impl( - req, - SERVER_TASK_TYPE_COMPLETION, - body, - files, - TASK_RESPONSE_TYPE_OAI_CMPL); - }; - - this->post_chat_completions = [this](const server_http_req & req) { - auto res = create_response(); - // P0-1 (#49): null-meta guard — meta is null until update_meta() is - // called after model load. Return 503 instead of crashing. - { - std::shared_lock meta_lock(meta_mutex); - if (!meta) { - res->error(format_error_response("model not loaded — waiting for CONFIGURE", - ERROR_TYPE_NOT_SUPPORTED)); - return res; - } - } - // Extract optional hydra_config from the raw body before it's - // consumed by oaicompat_chat_params_parse(). The config is applied - // on the task-queue thread (not here on the httplib worker thread) - // to avoid racing the main inference loop. - std::string hydra_config_str; - { - json raw_body = json::parse(req.body); - if (raw_body.is_object() && raw_body.contains("hydra_config") - && raw_body["hydra_config"].is_object()) { - hydra_config_str = raw_body["hydra_config"].dump(); - } - } - json body_parsed; - std::vector files; - { - std::shared_lock meta_lock(meta_mutex); - json body = json::parse(req.body); - body_parsed = oaicompat_chat_params_parse(body, meta->chat_params, files); - } - // meta_lock released before handle_completions_impl (which takes its own) - - return handle_completions_impl( - req, - SERVER_TASK_TYPE_COMPLETION, - body_parsed, - files, - TASK_RESPONSE_TYPE_OAI_CHAT, - hydra_config_str); - }; - - this->post_control = [this](const server_http_req & req) { - auto res = create_response(); - const json body = json::parse(req.body); - - const std::string cmpl_id = json_value(body, "id", std::string()); - const std::string action = json_value(body, "action", std::string()); - if (cmpl_id.empty()) { - res->error(format_error_response("missing completion id", ERROR_TYPE_INVALID_REQUEST)); - return res; - } - if (action != "reasoning_end") { - res->error(format_error_response("unknown control action", ERROR_TYPE_INVALID_REQUEST)); - return res; - } - - auto & rd = res->rd; - { - server_task task(SERVER_TASK_TYPE_CONTROL); - task.id = rd.get_new_id(); - task.params.control_cmpl_id = cmpl_id; - task.params.control_action = action; - rd.post_task(std::move(task)); - } - - auto result = rd.next(req.should_stop); - if (!result) { - GGML_ASSERT(req.should_stop()); - return res; - } - if (result->is_error()) { - res->error(result->to_json()); - return res; - } - res->ok(result->to_json()); - return res; - }; - - this->post_responses_oai = [this](const server_http_req & req) { - auto res = create_response(); - json body_parsed; - std::vector files; - { - std::shared_lock meta_lock(meta_mutex); - // P0-1 (#49): null-meta guard - if (!meta) { - res->error(format_error_response("model not loaded — waiting for CONFIGURE", - ERROR_TYPE_NOT_SUPPORTED)); - return res; - } - json body = server_chat_convert_responses_to_chatcmpl(json::parse(req.body)); - SRV_DBG("%s\n", "Request converted: OpenAI Responses -> OpenAI Chat Completions"); - SRV_DBG("converted request: %s\n", body.dump().c_str()); - body_parsed = oaicompat_chat_params_parse(body, meta->chat_params, files); - } - // meta_lock released before handle_completions_impl - - return handle_completions_impl( - req, - SERVER_TASK_TYPE_COMPLETION, - body_parsed, - files, - TASK_RESPONSE_TYPE_OAI_RESP); - }; - - this->post_transcriptions_oai = [this](const server_http_req & req) { - auto res = create_response(); - json body_parsed; - std::vector files; - { - std::shared_lock meta_lock(meta_mutex); - - // P0-1 (#49): null-meta guard - if (!meta) { - res->error(format_error_response("model not loaded — waiting for CONFIGURE", - ERROR_TYPE_NOT_SUPPORTED)); - return res; - } - - if (!meta->has_mtmd || !meta->chat_params.allow_audio) { - res->error(format_error_response("The current model does not support audio input.", ERROR_TYPE_NOT_SUPPORTED)); - return res; - } - - json body = convert_transcriptions_to_chatcmpl( - json::parse(req.body), - meta->chat_params.tmpls.get(), - req.files, - files); - SRV_DBG("%s\n", "Request converted: OpenAI Transcriptions -> OpenAI Chat Completions"); - SRV_DBG("converted request: %s\n", body.dump().c_str()); - body_parsed = oaicompat_chat_params_parse(body, meta->chat_params, files); - } - // meta_lock released before handle_completions_impl - - return handle_completions_impl( - req, - SERVER_TASK_TYPE_COMPLETION, - body_parsed, - files, - TASK_RESPONSE_TYPE_OAI_ASR); - }; - - this->post_anthropic_messages = [this](const server_http_req & req) { - auto res = create_response(); - json body_parsed; - std::vector files; - { - std::shared_lock meta_lock(meta_mutex); - // P0-1 (#49): null-meta guard - if (!meta) { - res->error(format_error_response("model not loaded — waiting for CONFIGURE", - ERROR_TYPE_NOT_SUPPORTED)); - return res; - } - json body = server_chat_convert_anthropic_to_oai(json::parse(req.body)); - SRV_DBG("%s\n", "Request converted: Anthropic -> OpenAI Chat Completions"); - SRV_DBG("converted request: %s\n", body.dump().c_str()); - body_parsed = oaicompat_chat_params_parse(body, meta->chat_params, files); - } - // meta_lock released before handle_completions_impl - - return handle_completions_impl( - req, - SERVER_TASK_TYPE_COMPLETION, - body_parsed, - files, - TASK_RESPONSE_TYPE_ANTHROPIC); - }; - - this->post_anthropic_count_tokens = [this](const server_http_req & req) { - auto res = create_response(); - std::shared_lock meta_lock(meta_mutex); - // P0-1 (#49): null-meta guard - if (!meta) { - res->error(format_error_response("model not loaded — waiting for CONFIGURE", - ERROR_TYPE_NOT_SUPPORTED)); - return res; - } - std::vector files; - json body = server_chat_convert_anthropic_to_oai(json::parse(req.body)); - SRV_DBG("%s\n", "Request converted: Anthropic -> OpenAI Chat Completions"); - SRV_DBG("converted request: %s\n", body.dump().c_str()); - json body_parsed = oaicompat_chat_params_parse( - body, - meta->chat_params, - files); - - json prompt = body_parsed.at("prompt"); - llama_tokens tokens = tokenize_mixed(ctx_server.vocab, prompt, true, true); - res->ok({{"input_tokens", static_cast(tokens.size())}}); - return res; - }; - - // same with handle_chat_completions, but without inference part - this->post_apply_template = [this](const server_http_req & req) { - auto res = create_response(); - std::shared_lock meta_lock(meta_mutex); - // P0-1 (#49): null-meta guard - if (!meta) { - res->error(format_error_response("model not loaded — waiting for CONFIGURE", - ERROR_TYPE_NOT_SUPPORTED)); - return res; - } - std::vector files; // dummy, unused - json body = json::parse(req.body); - json data = oaicompat_chat_params_parse( - body, - meta->chat_params, - files); - res->ok({{ "prompt", std::move(data.at("prompt")) }}); - return res; - }; - - this->get_models = [this](const server_http_req &) { - auto res = create_response(true); - std::shared_lock meta_lock(meta_mutex); - - // this endpoint can be accessed during sleeping - // the next LOC is to avoid someone accidentally use ctx_server - bool ctx_server; // do NOT delete this line - GGML_UNUSED(ctx_server); - - // P0-1 (#49): null-meta guard - if (!meta) { - res->error(format_error_response("model not loaded — waiting for CONFIGURE", - ERROR_TYPE_NOT_SUPPORTED)); - return res; - } - - json models = { - {"models", { - { - {"name", meta->model_name}, - {"model", meta->model_name}, - {"modified_at", ""}, - {"size", ""}, - {"digest", ""}, // dummy value, llama.cpp does not support managing model file's hash - {"type", "model"}, - {"description", ""}, - {"tags", {""}}, - {"capabilities", meta->has_mtmd ? json({"completion","multimodal"}) : json({"completion"})}, - {"parameters", ""}, - {"details", { - {"parent_model", ""}, - {"format", "gguf"}, - {"family", ""}, - {"families", {""}}, - {"parameter_size", ""}, - {"quantization_level", ""} - }} - } - }}, - {"object", "list"}, - {"data", { - get_model_info(), - }} - }; - - res->ok(models); - return res; - }; - - this->post_tokenize = [this](const server_http_req & req) { - auto res = create_response(); - const json body = json::parse(req.body); - json tokens_response = json::array(); - if (body.count("content") != 0) { - const bool add_special = json_value(body, "add_special", false); - const bool parse_special = json_value(body, "parse_special", true); - const bool with_pieces = json_value(body, "with_pieces", false); - - llama_tokens tokens = tokenize_mixed(ctx_server.vocab, body.at("content"), add_special, parse_special); - - if (with_pieces) { - for (const auto& token : tokens) { - std::string piece = common_token_to_piece(ctx_server.vocab, token); - json piece_json; - - // Check if the piece is valid UTF-8 - if (is_valid_utf8(piece)) { - piece_json = piece; - } else { - // If not valid UTF-8, store as array of byte values - piece_json = json::array(); - for (unsigned char c : piece) { - piece_json.push_back(static_cast(c)); - } - } - - tokens_response.push_back({ - {"id", token}, - {"piece", piece_json} - }); - } - } else { - tokens_response = tokens; - } - } - - res->ok(json{{"tokens", std::move(tokens_response)}}); - return res; - }; - - this->post_detokenize = [this](const server_http_req & req) { - auto res = create_response(); - const json body = json::parse(req.body); - - std::string content; - if (body.count("tokens") != 0) { - const llama_tokens tokens = body.at("tokens"); - content = tokens_to_str(ctx_server.vocab, tokens); - } - - res->ok(json{{"content", std::move(content)}}); - return res; - }; - - this->post_embeddings = [this](const server_http_req & req) { - return handle_embeddings_impl(req, TASK_RESPONSE_TYPE_NONE); - }; - - this->post_embeddings_oai = [this](const server_http_req & req) { - return handle_embeddings_impl(req, TASK_RESPONSE_TYPE_OAI_EMBD); - }; - - this->post_rerank = [this](const server_http_req & req) { + // ── Hydra state streaming (M0.0) ─────────────────────────────────────── + this->get_state = [this](const server_http_req & req) { auto res = create_response(); - std::shared_lock meta_lock(meta_mutex); - // P0-1 (#49): null-meta guard - if (!meta) { - res->error(format_error_response("model not loaded — waiting for CONFIGURE", - ERROR_TYPE_NOT_SUPPORTED)); - return res; - } - if (!params.embedding || params.pooling_type != LLAMA_POOLING_TYPE_RANK) { - res->error(format_error_response("This server does not support reranking. Start it with `--reranking`", ERROR_TYPE_NOT_SUPPORTED)); + int id_slot; + try { + id_slot = std::stoi(req.get_param("id_slot")); + } catch (const std::exception &) { + res->error(format_error_response("Invalid slot ID", ERROR_TYPE_INVALID_REQUEST)); return res; } - - const json body = json::parse(req.body); - - // if true, use TEI API format, otherwise use Jina API format - // Jina: https://jina.ai/reranker/ - // TEI: https://huggingface.github.io/text-embeddings-inference/#/Text%20Embeddings%20Inference/rerank - bool is_tei_format = body.contains("texts"); - - json query; - if (body.count("query") == 1) { - query = body.at("query"); - if (!query.is_string()) { - res->error(format_error_response("\"query\" must be a string", ERROR_TYPE_INVALID_REQUEST)); - return res; - } - } else { - res->error(format_error_response("\"query\" must be provided", ERROR_TYPE_INVALID_REQUEST)); + server_task task(SERVER_TASK_TYPE_HYDRA_STATE_GET); + task.id = res->rd.get_new_id(); + task.hydra_action.id_slot = id_slot; + task.hydra_action.hydra_fd = -1; + res->rd.post_task(std::move(task)); + auto result = res->rd.next(req.should_stop); + if (!result) { + GGML_ASSERT(req.should_stop()); return res; } - - std::vector documents = json_value(body, "documents", - json_value(body, "texts", std::vector())); - if (documents.empty()) { - res->error(format_error_response("\"documents\" must be a non-empty string array", ERROR_TYPE_INVALID_REQUEST)); + if (result->is_error()) { + res->error(result->to_json()); return res; } - - int top_n = json_value(body, "top_n", (int)documents.size()); - - // create and queue the task - json responses = json::array(); - auto & rd = res->rd; - { - std::vector tasks; - tasks.reserve(documents.size()); - for (size_t i = 0; i < documents.size(); i++) { - auto tmp = format_prompt_rerank(ctx_server.model_tgt, ctx_server.vocab, ctx_server.mctx, query, documents[i]); - server_task task = server_task(SERVER_TASK_TYPE_RERANK); - task.id = rd.get_new_id(); - task.tokens = std::move(tmp); - tasks.push_back(std::move(task)); - } - rd.post_tasks(std::move(tasks)); - } - - // wait for the results - auto all_results = rd.wait_for_all(req.should_stop); - - // collect results - if (all_results.is_terminated) { - return res; // connection is closed - } else if (all_results.error) { - res->error(all_results.error->to_json()); + auto * hr = dynamic_cast(result.get()); + GGML_ASSERT(hr != nullptr); + if (hr->rpc_status != HYDRA_STATUS_OK) { + res->status = hr->rpc_status == HYDRA_STATUS_NOT_FOUND ? 404 : 503; + res->data = hr->error.empty() ? "" : hr->error; return res; - } else { - for (auto & res : all_results.results) { - GGML_ASSERT(dynamic_cast(res.get()) != nullptr); - responses.push_back(res->to_json()); - } } - - // write JSON response - json root = format_response_rerank( - body, - meta->model_name, - responses, - is_tei_format, - documents, - top_n); - - res->ok(root); + res->content_type = "application/octet-stream"; + res->headers["X-Hydra-State-Size"] = std::to_string(hr->state_data.size()); + res->headers["X-Hydra-N-Past"] = std::to_string(hr->n_past); + res->data.assign((const char*)hr->state_data.data(), hr->state_data.size()); return res; }; - - this->get_lora_adapters = [this](const server_http_req & req) { + this->put_state = [this](const server_http_req & req) { auto res = create_response(); - - auto & rd = res->rd; - { - server_task task(SERVER_TASK_TYPE_GET_LORA); - task.id = rd.get_new_id(); - rd.post_task(std::move(task)); + int id_slot; + try { + id_slot = std::stoi(req.get_param("id_slot")); + } catch (const std::exception &) { + res->error(format_error_response("Invalid slot ID", ERROR_TYPE_INVALID_REQUEST)); + return res; } - - // get the result - auto result = rd.next(req.should_stop); + server_task task(SERVER_TASK_TYPE_HYDRA_STATE_PUT); + task.id = res->rd.get_new_id(); + task.hydra_action.id_slot = id_slot; + task.hydra_action.erase_existing = req.get_param("erase_existing") == "true"; + task.hydra_action.state_data.assign(req.body.begin(), req.body.end()); + res->rd.post_task(std::move(task)); + auto result = res->rd.next(req.should_stop); if (!result) { - // connection was closed GGML_ASSERT(req.should_stop()); return res; } - if (result->is_error()) { res->error(result->to_json()); return res; } - - GGML_ASSERT(dynamic_cast(result.get()) != nullptr); - res->ok(result->to_json()); + auto * hr = dynamic_cast(result.get()); + GGML_ASSERT(hr != nullptr); + if (hr->rpc_status != HYDRA_STATUS_OK) { + res->status = 503; + res->data = hr->error.empty() ? "restore failed" : hr->error; + return res; + } + res->ok(json{ + {"restored", hr->restored}, + {"n_past", hr->n_past}, + {"bytes", hr->bytes}, + }); return res; }; - - this->post_lora_adapters = [this](const server_http_req & req) { + this->get_state_meta = [this](const server_http_req & req) { auto res = create_response(); - const json body = json::parse(req.body); - if (!body.is_array()) { - res->error(format_error_response("Request body must be an array", ERROR_TYPE_INVALID_REQUEST)); + int id_slot; + try { + id_slot = std::stoi(req.get_param("id_slot")); + } catch (const std::exception &) { + res->error(format_error_response("Invalid slot ID", ERROR_TYPE_INVALID_REQUEST)); return res; } - - auto & rd = res->rd; - { - server_task task(SERVER_TASK_TYPE_SET_LORA); - task.id = rd.get_new_id(); - task.set_lora = parse_lora_request(body); - rd.post_task(std::move(task)); - } - - // get the result - auto result = rd.next(req.should_stop); + server_task task(SERVER_TASK_TYPE_HYDRA_STATE_META); + task.id = res->rd.get_new_id(); + task.hydra_action.id_slot = id_slot; + res->rd.post_task(std::move(task)); + auto result = res->rd.next(req.should_stop); if (!result) { - // connection was closed GGML_ASSERT(req.should_stop()); return res; } - if (result->is_error()) { - res->error(result->to_json()); - return res; - } - - GGML_ASSERT(dynamic_cast(result.get()) != nullptr); - res->ok(result->to_json()); - return res; - }; - - // ── Merged DECODE result retrieval ───────────────────────────────────── - this->get_decode_result = [this](const server_http_req & req) { - auto res = create_response(true); - int32_t decode_request_id; - try { - decode_request_id = std::stoi(req.get_param("decode_request_id")); - } catch (const std::exception &) { - res->error(format_error_response("Invalid decode_request_id", ERROR_TYPE_INVALID_REQUEST)); - return res; - } - - std::unique_lock lock(decode_results_mutex); - evict_decode_results_locked(); - auto it = decode_results.find(decode_request_id); - if (it == decode_results.end()) { - lock.unlock(); - res->error(format_error_response("decode_request_id not found or expired", ERROR_TYPE_NOT_FOUND)); + res->error(result->to_json()); return res; } - - // Read fields we need before unlocking - const auto entry_state = it->second.state; - const auto entry_error = it->second.error; - const int32_t id_slot = it->second.id_slot; - const std::string completion_id = it->second.completion_id; - const std::string oaicompat_model = it->second.oaicompat_model; - const std::string content = it->second.content; - const std::string reasoning_content = it->second.reasoning_content; - const json tool_calls = it->second.tool_calls; - const int32_t n_decoded = it->second.n_decoded; - const int32_t n_prompt_tokens = it->second.n_prompt_tokens; - const int32_t n_prompt_tokens_cache = it->second.n_prompt_tokens_cache; - const result_timings timings = it->second.timings; - const stop_type stop = it->second.stop; - const bool include_usage = it->second.include_usage; - const json hydra_metrics = it->second.hydra_metrics; - const json match_json = it->second.match_json; - const double model_load_ms = it->second.model_load_ms; - const double restore_slot_ms = it->second.restore_slot_ms; - const json model_identity = it->second.model_identity; - lock.unlock(); - - // ── Terminal error ───────────────────────────────────────────────── - if (!entry_error.empty()) { - { - std::lock_guard lk(decode_results_mutex); - decode_results.erase(decode_request_id); - } - json err_j = { - {"error", entry_error}, - {"error_code", "DECODE_FAILED"}, - {"match", match_json}, - }; - res->error(format_error_response(entry_error, ERROR_TYPE_INVALID_REQUEST)); + auto * hr = dynamic_cast(result.get()); + GGML_ASSERT(hr != nullptr); + if (hr->rpc_status != HYDRA_STATUS_OK) { + res->status = hr->rpc_status == HYDRA_STATUS_NOT_FOUND ? 404 : 503; + res->data = hr->error.empty() ? "" : hr->error; return res; } + res->ok(json{ + {"slot_id", hr->id_slot}, + {"n_past", hr->n_past}, + {"state_size", (uint64_t)hr->state_size}, + {"is_processing", hr->is_processing}, + {"is_transferring", hr->is_transferring}, + {"operation", hr->operation}, + {"progress", hr->progress}, + {"tokens_processed", hr->tokens_processed}, + {"tokens_total", hr->tokens_total}, + {"elapsed_ms", hr->elapsed_ms}, + // #470/A7: model identity — the Coordinator's merged-decode Gate A + // compares these against kv_metadata. Without them the engine's + // model_metadata came back empty and every COMBINED merged decode + // was rejected (tokenizer/name mismatch) before KV restore. + {"model_alias", hr->model_alias}, + {"model_path", hr->model_path}, + {"tokenizer", hr->tokenizer}, + {"model_name", hr->model_name}, + {"model_quant", hr->model_quant}, + {"model_capabilities", hr->model_capabilities}, + }); + return res; + }; - // httplib's own Headers map is case-insensitive - // (detail::case_ignore::hash, httplib.h), but server-http.cpp - // get_headers() copies it into a plain case-sensitive - // std::map, so match the Accept header - // case-insensitively here — otherwise clients sending - // "Accept: text/event-stream" never reach the SSE branches. - const bool stream = [&]() { - for (const auto & [hname, hval] : req.headers) { - if (hval.find("text/event-stream") == std::string::npos) { - continue; - } - if (hname.size() != 6) { - continue; - } - bool is_accept = true; - for (size_t i = 0; i < 6; i++) { - char c = hname[i]; - if (c >= 'A' && c <= 'Z') { - c = (char)(c - 'A' + 'a'); - } - if (c != "accept"[i]) { - is_accept = false; - break; - } - } - if (is_accept) { - return true; - } - } - return false; - }(); + this->get_props = [this](const server_http_req &) { + auto res = create_response(true); + std::shared_lock meta_lock(meta_mutex); - // ── In-progress states → 202 ────────────────────────────────────── - if (entry_state == server_routes::DECODE_STATE_LOADING || - entry_state == server_routes::DECODE_STATE_RESTORING) { - json state_j = { - {"state", entry_state == server_routes::DECODE_STATE_LOADING ? "loading" : "restoring"}, - {"decode_request_id", decode_request_id}, - {"id_slot", id_slot}, - {"model_load_ms", model_load_ms}, - {"restore_slot_ms", restore_slot_ms}, - {"match", match_json}, - }; - res->status = 202; - res->data = safe_json_to_str(state_j); + // this endpoint can be accessed during sleeping + // the next LOC is to avoid someone accidentally use ctx_server + bool ctx_server; // do NOT delete this line + GGML_UNUSED(ctx_server); + + // P0-1 (#49): null-meta guard + if (!meta) { + res->error(format_error_response("model not loaded — waiting for CONFIGURE", + ERROR_TYPE_NOT_SUPPORTED)); return res; } - // ── GENERATING + SSE → stream partials from relay queue ───────── - if (entry_state == server_routes::DECODE_STATE_GENERATING && stream) { - // Set up SSE response; partials arrive via the streaming_queue - // populated by the background consumer thread. - res->status = 200; - res->content_type = "text/event-stream"; - res->data = ""; // no initial chunk — send headers immediately - - res->next = [this, res_this = res.get(), decode_request_id, &req, sent_final = false]( - std::string & output) mutable -> bool { - try { - if (req.should_stop()) { - return false; - } - - std::unique_lock lock(decode_results_mutex); - auto it = decode_results.find(decode_request_id); - if (it == decode_results.end()) { - output = "data: [DONE]\n\n"; - return false; - } - auto & entry = it->second; - - // Drain streaming queue - if (entry.stream) { - std::lock_guard slk(entry.stream->streaming_mutex); - if (!entry.stream->streaming_queue.empty()) { - auto result = std::move(entry.stream->streaming_queue.front()); - entry.stream->streaming_queue.pop_front(); - lock.unlock(); - - if (result->is_error()) { - json err_j = format_error_response("generation error", ERROR_TYPE_SERVER); - output = format_oai_sse(json{{"error", err_j}}); - return false; - } - json j = result->to_json(); - if (j.is_null()) { - // is_begin partial — skip - return true; - } - output = format_oai_sse(j); - return true; - } - } + task_params tparams; + tparams.sampling = params.sampling; + json default_generation_settings_for_props = json { + { "params", tparams.to_json(true) }, + { "n_ctx", meta->slot_n_ctx }, + }; - // Queue empty — stream finished: emit the final DONE delta - // exactly once, then terminate with [DONE]. Without it a - // client attached during GENERATING never sees the final - // finish_reason / usage / hydra_metrics. Content is - // deliberately NOT repeated: the relay already streamed - // content/reasoning_content/tool_calls incrementally via - // the partial deltas, so this is OpenAI's empty final - // chunk ({"delta": {...}, "finish_reason": ...}) — echoing - // full content/tool_calls again would make concat-based - // clients see output twice. (The DONE+SSE single-delta - // branch below keeps full content: that one fires for - // attach-after-DONE clients that saw no partials.) - if (entry.stream && entry.stream->stream_finished) { - if (!sent_final && entry.state == server_routes::DECODE_STATE_DONE) { - sent_final = true; - std::time_t t = std::time(0); - json delta { - {"choices", json::array({ - json { - {"finish_reason", entry.stop == STOP_TYPE_WORD || entry.stop == STOP_TYPE_EOS - ? (entry.tool_calls.empty() ? "stop" : "tool_calls") - : "length"}, - {"index", 0}, - {"delta", json{{"role", "assistant"}, {"content", ""}}}, - }, - })}, - {"created", t}, - {"id", entry.completion_id}, - {"model", entry.oaicompat_model}, - {"system_fingerprint", std::string(llama_build_info())}, - {"object", "chat.completion.chunk"}, - }; - if (entry.include_usage) { - delta["usage"] = json { - {"completion_tokens", entry.n_decoded}, - {"prompt_tokens", entry.n_prompt_tokens}, - {"total_tokens", entry.n_decoded + entry.n_prompt_tokens}, - {"prompt_tokens_details", json{{"cached_tokens", entry.n_prompt_tokens_cache}}}, - }; - } - if (!entry.hydra_metrics.is_null()) { - delta["hydra_metrics"] = entry.hydra_metrics; - } - output = format_oai_sse(delta); - return true; - } - output = "data: [DONE]\n\n"; - return false; - } + std::string tmpl_default = common_chat_templates_source(meta->chat_params.tmpls.get(), ""); + std::string tmpl_tools = common_chat_templates_source(meta->chat_params.tmpls.get(), "tool_use"); - // Wait for next partial with ping interval - if (entry.stream) { - entry.stream->streaming_cv.wait_for(lock, std::chrono::seconds(30)); - } - return true; // loop again + json props = { + { "default_generation_settings", default_generation_settings_for_props }, + { "total_slots", params.n_parallel }, + { "model_alias", meta->model_name }, + { "model_path", meta->model_path }, + { "modalities", json { + {"vision", meta->has_inp_image}, + {"audio", meta->has_inp_audio}, + } }, + { "media_marker", get_media_marker() }, + { "endpoint_slots", params.endpoint_slots }, + { "endpoint_props", params.endpoint_props }, + { "endpoint_metrics", params.endpoint_metrics }, + // New keys + { "ui", params.ui }, + { "ui_settings", meta->json_ui_settings }, + // Deprecated: use ui/ui_settings instead (kept for backward compat) + { "webui", params.webui }, + { "webui_settings", meta->json_webui_settings }, + { "chat_template", tmpl_default }, + { "chat_template_caps", meta->chat_template_caps }, + { "bos_token", meta->bos_token_str }, + { "eos_token", meta->eos_token_str }, + { "build_info", meta->build_info }, + { "is_sleeping", queue_tasks.is_sleeping() }, + { "cors_proxy_enabled", params.ui_mcp_proxy || params.webui_mcp_proxy }, + }; + if (params.use_jinja) { + if (!tmpl_tools.empty()) { + props["chat_template_tool_use"] = tmpl_tools; + } + } + res->ok(props); + return res; + }; - } catch (const std::exception & e) { - json err_j = format_error_response(e.what(), ERROR_TYPE_SERVER); - output = format_oai_sse(json{{"error", err_j}}); - return false; - } - }; + this->post_props = [this](const server_http_req &) { + auto res = create_response(); + if (!params.endpoint_props) { + res->error(format_error_response("This server does not support changing global properties. Start it with `--props`", ERROR_TYPE_NOT_SUPPORTED)); return res; } + // update any props here - // ── DONE → return full result ───────────────────────────────────── - if (entry_state == server_routes::DECODE_STATE_DONE || !content.empty()) { - if (stream) { - // SSE streaming: send full result as a single delta, then finish - std::time_t t = std::time(0); - json delta { - {"choices", json::array({ - json { - {"finish_reason", stop == STOP_TYPE_WORD || stop == STOP_TYPE_EOS - ? (tool_calls.empty() ? "stop" : "tool_calls") - : "length"}, - {"index", 0}, - {"delta", json{{"role", "assistant"}, {"content", content}}}, - }, - })}, - {"created", t}, - {"id", completion_id}, - {"model", oaicompat_model}, - {"system_fingerprint", std::string(llama_build_info())}, - {"object", "chat.completion.chunk"}, - }; + res->ok({{ "success", true }}); + return res; + }; - if (!reasoning_content.empty()) { - delta["choices"][0]["delta"]["reasoning_content"] = reasoning_content; - } + this->post_infill = [this](const server_http_req & req) { + auto res = create_response(); + // P0-1 (#49): null-meta guard + { + std::shared_lock meta_lock(meta_mutex); + if (!meta) { + res->error(format_error_response("model not loaded — waiting for CONFIGURE", + ERROR_TYPE_NOT_SUPPORTED)); + return res; + } + } - if (!tool_calls.empty()) { - delta["choices"][0]["delta"]["tool_calls"] = tool_calls; - } + // Validate input and compute infill prompt — these read meta->slot_n_ctx. + // Scope the shared_lock so it releases before handle_completions_impl + // takes its own lock (recursive shared_lock is UB on std::shared_mutex). + json data; + std::vector files; + { + std::shared_lock meta_lock(meta_mutex); - if (include_usage) { - delta["usage"] = json { - {"completion_tokens", n_decoded}, - {"prompt_tokens", n_prompt_tokens}, - {"total_tokens", n_decoded + n_prompt_tokens}, - {"prompt_tokens_details", json{{"cached_tokens", n_prompt_tokens_cache}}}, - }; - } - if (!hydra_metrics.is_null()) { - delta["hydra_metrics"] = hydra_metrics; - } + // check model compatibility + std::string err; + if (llama_vocab_fim_pre(ctx_server.vocab) == LLAMA_TOKEN_NULL) { + err += "prefix token is missing. "; + } + if (llama_vocab_fim_suf(ctx_server.vocab) == LLAMA_TOKEN_NULL) { + err += "suffix token is missing. "; + } + if (llama_vocab_fim_mid(ctx_server.vocab) == LLAMA_TOKEN_NULL) { + err += "middle token is missing. "; + } + if (!err.empty()) { + res->error(format_error_response(string_format("Infill is not supported by this model: %s", err.c_str()), ERROR_TYPE_NOT_SUPPORTED)); + return res; + } - res->status = 200; - res->content_type = "text/event-stream"; - res->data = format_oai_sse(delta); - } else { - // Buffered: full OAI chat completion response - json message; - message["role"] = "assistant"; - message["content"] = content; - if (!reasoning_content.empty()) { - message["reasoning_content"] = reasoning_content; + // validate input + data = json::parse(req.body); + if (data.contains("prompt") && !data.at("prompt").is_string()) { + // prompt is optional + res->error(format_error_response("\"prompt\" must be a string", ERROR_TYPE_INVALID_REQUEST)); + } + + if (!data.contains("input_prefix")) { + res->error(format_error_response("\"input_prefix\" is required", ERROR_TYPE_INVALID_REQUEST)); + } + + if (!data.contains("input_suffix")) { + res->error(format_error_response("\"input_suffix\" is required", ERROR_TYPE_INVALID_REQUEST)); + } + + if (data.contains("input_extra") && !data.at("input_extra").is_array()) { + // input_extra is optional + res->error(format_error_response("\"input_extra\" must be an array of {\"filename\": string, \"text\": string}", ERROR_TYPE_INVALID_REQUEST)); + return res; + } + + json input_extra = json_value(data, "input_extra", json::array()); + for (const auto & chunk : input_extra) { + // { "text": string, "filename": string } + if (!chunk.contains("text") || !chunk.at("text").is_string()) { + res->error(format_error_response("extra_context chunk must contain a \"text\" field with a string value", ERROR_TYPE_INVALID_REQUEST)); + return res; } - if (!tool_calls.empty()) { - message["tool_calls"] = tool_calls; + // filename is optional + if (chunk.contains("filename") && !chunk.at("filename").is_string()) { + res->error(format_error_response("extra_context chunk's \"filename\" field must be a string", ERROR_TYPE_INVALID_REQUEST)); + return res; } + } + data["input_extra"] = input_extra; // default to empty array if it's not exist - json choice { - {"finish_reason", stop == STOP_TYPE_WORD || stop == STOP_TYPE_EOS - ? (tool_calls.empty() ? "stop" : "tool_calls") - : "length"}, - {"index", 0}, - {"message", message}, - }; + std::string prompt = json_value(data, "prompt", std::string()); + std::vector tokenized_prompts = tokenize_input_prompts(ctx_server.vocab, ctx_server.mctx, prompt, false, true); + SRV_DBG("creating infill tasks, n_prompts = %d\n", (int) tokenized_prompts.size()); + data["prompt"] = format_prompt_infill( + ctx_server.vocab, + data.at("input_prefix"), + data.at("input_suffix"), + data.at("input_extra"), + params.n_batch, + params.n_predict, + meta->slot_n_ctx, + params.spm_infill, + tokenized_prompts[0].get_tokens() // TODO: this could maybe be multimodal. + ); + } + // meta_lock released here - json oai_response { - {"choices", json::array({choice})}, - {"created", std::time(0)}, - {"model", oaicompat_model}, - {"system_fingerprint", std::string(llama_build_info())}, - {"object", "chat.completion"}, - {"usage", { - {"completion_tokens", n_decoded}, - {"prompt_tokens", n_prompt_tokens}, - {"total_tokens", n_decoded + n_prompt_tokens}, - {"prompt_tokens_details", json{{"cached_tokens", n_prompt_tokens_cache}}}, - }}, - {"id", completion_id}, - {"id_slot", id_slot}, - {"timings", timings.to_json()}, - }; - if (!hydra_metrics.is_null()) { - oai_response["hydra_metrics"] = hydra_metrics; - } + return handle_completions_impl( + req, + SERVER_TASK_TYPE_INFILL, + data, + files, + TASK_RESPONSE_TYPE_NONE); // infill is not OAI compatible + }; + + this->post_completions = [this](const server_http_req & req) { + auto res = create_response(); + std::vector files; // dummy + const json body = json::parse(req.body); + return handle_completions_impl( + req, + SERVER_TASK_TYPE_COMPLETION, + body, + files, + TASK_RESPONSE_TYPE_NONE); + }; + + this->post_completions_oai = [this](const server_http_req & req) { + auto res = create_response(); + std::vector files; // dummy + const json body = json::parse(req.body); + return handle_completions_impl( + req, + SERVER_TASK_TYPE_COMPLETION, + body, + files, + TASK_RESPONSE_TYPE_OAI_CMPL); + }; + + this->post_chat_completions = [this](const server_http_req & req) { + auto res = create_response(); + // P0-1 (#49): null-meta guard — meta is null until update_meta() is + // called after model load. Return 503 instead of crashing. + { + std::shared_lock meta_lock(meta_mutex); + if (!meta) { + res->error(format_error_response("model not loaded — waiting for CONFIGURE", + ERROR_TYPE_NOT_SUPPORTED)); + return res; + } + } + // Extract optional hydra_config from the raw body before it's + // consumed by oaicompat_chat_params_parse(). The config is applied + // on the task-queue thread (not here on the httplib worker thread) + // to avoid racing the main inference loop. + std::string hydra_config_str; + { + json raw_body = json::parse(req.body); + if (raw_body.is_object() && raw_body.contains("hydra_config") + && raw_body["hydra_config"].is_object()) { + hydra_config_str = raw_body["hydra_config"].dump(); + } + } + json body_parsed; + std::vector files; + { + std::shared_lock meta_lock(meta_mutex); + json body = json::parse(req.body); + body_parsed = oaicompat_chat_params_parse(body, meta->chat_params, files); + } + // meta_lock released before handle_completions_impl (which takes its own) + + return handle_completions_impl( + req, + SERVER_TASK_TYPE_COMPLETION, + body_parsed, + files, + TASK_RESPONSE_TYPE_OAI_CHAT, + hydra_config_str); + }; + + this->post_control = [this](const server_http_req & req) { + auto res = create_response(); + const json body = json::parse(req.body); - res->ok(oai_response); - } + const std::string cmpl_id = json_value(body, "id", std::string()); + const std::string action = json_value(body, "action", std::string()); + if (cmpl_id.empty()) { + res->error(format_error_response("missing completion id", ERROR_TYPE_INVALID_REQUEST)); return res; } - - // Fallback: no content yet - res->error(format_error_response("decode_request_id not ready", ERROR_TYPE_NOT_FOUND)); - return res; - }; - - this->delete_decode_result = [this](const server_http_req & req) { - auto res = create_response(true); - int32_t decode_request_id; - try { - decode_request_id = std::stoi(req.get_param("decode_request_id")); - } catch (const std::exception &) { - res->error(format_error_response("Invalid decode_request_id", ERROR_TYPE_INVALID_REQUEST)); + if (action != "reasoning_end") { + res->error(format_error_response("unknown control action", ERROR_TYPE_INVALID_REQUEST)); return res; } - int32_t id_slot = -1; - int32_t task_id_to_cancel = -1; + auto & rd = res->rd; { - std::unique_lock lock(decode_results_mutex); - auto it = decode_results.find(decode_request_id); - if (it == decode_results.end()) { - lock.unlock(); - res->error(format_error_response("decode_request_id not found or expired", ERROR_TYPE_NOT_FOUND)); - return res; - } - id_slot = it->second.id_slot; - task_id_to_cancel = it->second.stream ? it->second.stream->completion_task_id.load() : -1; - - // Signal streaming queue to finish so any waiting GET handler unblocks - if (it->second.stream) { - std::lock_guard slk(it->second.stream->streaming_mutex); - it->second.stream->stream_finished = true; - it->second.stream->streaming_cv.notify_all(); - } - - decode_results.erase(it); + server_task task(SERVER_TASK_TYPE_CONTROL); + task.id = rd.get_new_id(); + task.params.control_cmpl_id = cmpl_id; + task.params.control_action = action; + rd.post_task(std::move(task)); } - // Deterministically cancel the running completion task via the task queue. - // SERVER_TASK_TYPE_CANCEL causes the inference thread to release the slot. - if (task_id_to_cancel > 0) { - server_task cancel_task(SERVER_TASK_TYPE_CANCEL); - cancel_task.id = queue_tasks.get_new_id(); - cancel_task.id_target = task_id_to_cancel; - queue_tasks.post(std::move(cancel_task), true); - SRV_INF("hydra: DECODE_CANCEL id=%d slot=%d completion_task=%d (cancel posted)\n", - decode_request_id, id_slot, task_id_to_cancel); - } else { - SRV_INF("hydra: DECODE_CANCEL id=%d slot=%d (no active completion)\n", - decode_request_id, id_slot); + auto result = rd.next(req.should_stop); + if (!result) { + GGML_ASSERT(req.should_stop()); + return res; } - - res->ok(json{{"cancelled", true}, {"decode_request_id", decode_request_id}}); + if (result->is_error()) { + res->error(result->to_json()); + return res; + } + res->ok(result->to_json()); return res; }; -} - -json server_routes::get_model_info() const { - std::shared_lock meta_lock(meta_mutex); - // P0-1 (#49): null-meta guard — called from get_models and other paths - if (!meta) { - return json{{"error", "model not loaded — waiting for CONFIGURE"}}; - } + this->post_responses_oai = [this](const server_http_req & req) { + auto res = create_response(); + json body_parsed; + std::vector files; + { + std::shared_lock meta_lock(meta_mutex); + // P0-1 (#49): null-meta guard + if (!meta) { + res->error(format_error_response("model not loaded — waiting for CONFIGURE", + ERROR_TYPE_NOT_SUPPORTED)); + return res; + } + json body = server_chat_convert_responses_to_chatcmpl(json::parse(req.body)); + SRV_DBG("%s\n", "Request converted: OpenAI Responses -> OpenAI Chat Completions"); + SRV_DBG("converted request: %s\n", body.dump().c_str()); + body_parsed = oaicompat_chat_params_parse(body, meta->chat_params, files); + } + // meta_lock released before handle_completions_impl - return json { - {"id", meta->model_name}, - {"aliases", meta->model_aliases}, - {"tags", meta->model_tags}, - {"object", "model"}, - {"created", std::time(0)}, - {"owned_by", "llamacpp"}, - {"meta", { - {"vocab_type", meta->model_vocab_type}, - {"n_vocab", meta->model_vocab_n_tokens}, - {"n_ctx", meta->slot_n_ctx}, - {"n_ctx_train", meta->model_n_ctx_train}, - {"n_embd", meta->model_n_embd_inp}, - {"n_params", meta->model_n_params}, - {"size", meta->model_size}, - }}, + return handle_completions_impl( + req, + SERVER_TASK_TYPE_COMPLETION, + body_parsed, + files, + TASK_RESPONSE_TYPE_OAI_RESP); }; -} - -std::unique_ptr server_routes::handle_slots_save(const server_http_req & req, int id_slot) { - auto res = create_response(); - const json request_data = json::parse(req.body); - std::string filename = request_data.at("filename"); - if (!fs_validate_filename(filename)) { - res->error(format_error_response("Invalid filename", ERROR_TYPE_INVALID_REQUEST)); - return res; - } - std::string filepath = params.slot_save_path + filename; - - auto & rd = res->rd; - { - server_task task(SERVER_TASK_TYPE_SLOT_SAVE); - task.id = rd.get_new_id(); - task.slot_action.id_slot = id_slot; - task.slot_action.filename = filename; - task.slot_action.filepath = filepath; - rd.post_task(std::move(task)); - } - - auto result = rd.next(req.should_stop); - if (!result) { - // connection was closed - GGML_ASSERT(req.should_stop()); - return res; - } - if (result->is_error()) { - res->error(result->to_json()); - return res; - } + this->post_transcriptions_oai = [this](const server_http_req & req) { + auto res = create_response(); + json body_parsed; + std::vector files; + { + std::shared_lock meta_lock(meta_mutex); - res->ok(result->to_json()); - return res; -} + // P0-1 (#49): null-meta guard + if (!meta) { + res->error(format_error_response("model not loaded — waiting for CONFIGURE", + ERROR_TYPE_NOT_SUPPORTED)); + return res; + } -std::unique_ptr server_routes::handle_slots_restore(const server_http_req & req, int id_slot) { - auto res = create_response(); - const json request_data = json::parse(req.body); - std::string filename = request_data.at("filename"); - if (!fs_validate_filename(filename)) { - res->error(format_error_response("Invalid filename", ERROR_TYPE_INVALID_REQUEST)); - return res; - } - std::string filepath = params.slot_save_path + filename; + if (!meta->has_mtmd || !meta->chat_params.allow_audio) { + res->error(format_error_response("The current model does not support audio input.", ERROR_TYPE_NOT_SUPPORTED)); + return res; + } - auto & rd = res->rd; - { - server_task task(SERVER_TASK_TYPE_SLOT_RESTORE); - task.id = rd.get_new_id(); - task.slot_action.id_slot = id_slot; - task.slot_action.filename = filename; - task.slot_action.filepath = filepath; - rd.post_task(std::move(task)); - } + json body = convert_transcriptions_to_chatcmpl( + json::parse(req.body), + meta->chat_params.tmpls.get(), + req.files, + files); + SRV_DBG("%s\n", "Request converted: OpenAI Transcriptions -> OpenAI Chat Completions"); + SRV_DBG("converted request: %s\n", body.dump().c_str()); + body_parsed = oaicompat_chat_params_parse(body, meta->chat_params, files); + } + // meta_lock released before handle_completions_impl - auto result = rd.next(req.should_stop); - if (!result) { - // connection was closed - GGML_ASSERT(req.should_stop()); - return res; - } + return handle_completions_impl( + req, + SERVER_TASK_TYPE_COMPLETION, + body_parsed, + files, + TASK_RESPONSE_TYPE_OAI_ASR); + }; - if (result->is_error()) { - res->error(result->to_json()); - return res; - } + this->post_anthropic_messages = [this](const server_http_req & req) { + auto res = create_response(); + json body_parsed; + std::vector files; + { + std::shared_lock meta_lock(meta_mutex); + // P0-1 (#49): null-meta guard + if (!meta) { + res->error(format_error_response("model not loaded — waiting for CONFIGURE", + ERROR_TYPE_NOT_SUPPORTED)); + return res; + } + json body = server_chat_convert_anthropic_to_oai(json::parse(req.body)); + SRV_DBG("%s\n", "Request converted: Anthropic -> OpenAI Chat Completions"); + SRV_DBG("converted request: %s\n", body.dump().c_str()); + body_parsed = oaicompat_chat_params_parse(body, meta->chat_params, files); + } + // meta_lock released before handle_completions_impl - GGML_ASSERT(dynamic_cast(result.get()) != nullptr); - res->ok(result->to_json()); - return res; -} + return handle_completions_impl( + req, + SERVER_TASK_TYPE_COMPLETION, + body_parsed, + files, + TASK_RESPONSE_TYPE_ANTHROPIC); + }; -std::unique_ptr server_routes::handle_slots_erase(const server_http_req & req, int id_slot) { - auto res = create_response(); - auto & rd = res->rd; - { - server_task task(SERVER_TASK_TYPE_SLOT_ERASE); - task.id = rd.get_new_id(); - task.slot_action.id_slot = id_slot; - rd.post_task(std::move(task)); - } + this->post_anthropic_count_tokens = [this](const server_http_req & req) { + auto res = create_response(); + std::shared_lock meta_lock(meta_mutex); + // P0-1 (#49): null-meta guard + if (!meta) { + res->error(format_error_response("model not loaded — waiting for CONFIGURE", + ERROR_TYPE_NOT_SUPPORTED)); + return res; + } + std::vector files; + json body = server_chat_convert_anthropic_to_oai(json::parse(req.body)); + SRV_DBG("%s\n", "Request converted: Anthropic -> OpenAI Chat Completions"); + SRV_DBG("converted request: %s\n", body.dump().c_str()); + json body_parsed = oaicompat_chat_params_parse( + body, + meta->chat_params, + files); - auto result = rd.next(req.should_stop); - if (!result) { - // connection was closed - GGML_ASSERT(req.should_stop()); + json prompt = body_parsed.at("prompt"); + llama_tokens tokens = tokenize_mixed(ctx_server.vocab, prompt, true, true); + res->ok({{"input_tokens", static_cast(tokens.size())}}); return res; - } + }; - if (result->is_error()) { - res->error(result->to_json()); + // same with handle_chat_completions, but without inference part + this->post_apply_template = [this](const server_http_req & req) { + auto res = create_response(); + std::shared_lock meta_lock(meta_mutex); + // P0-1 (#49): null-meta guard + if (!meta) { + res->error(format_error_response("model not loaded — waiting for CONFIGURE", + ERROR_TYPE_NOT_SUPPORTED)); + return res; + } + std::vector files; // dummy, unused + json body = json::parse(req.body); + json data = oaicompat_chat_params_parse( + body, + meta->chat_params, + files); + res->ok({{ "prompt", std::move(data.at("prompt")) }}); return res; - } + }; - GGML_ASSERT(dynamic_cast(result.get()) != nullptr); - res->ok(result->to_json()); - return res; -} + this->get_models = [this](const server_http_req &) { + auto res = create_response(true); + std::shared_lock meta_lock(meta_mutex); -std::unique_ptr server_routes::handle_embeddings_impl(const server_http_req & req, task_response_type res_type) { - std::shared_lock meta_lock(meta_mutex); + // this endpoint can be accessed during sleeping + // the next LOC is to avoid someone accidentally use ctx_server + bool ctx_server; // do NOT delete this line + GGML_UNUSED(ctx_server); - auto res = create_response(); - // P0-1 (#49): null-meta guard - if (!meta) { - res->error(format_error_response("model not loaded — waiting for CONFIGURE", - ERROR_TYPE_NOT_SUPPORTED)); - return res; - } - if (!params.embedding) { - res->error(format_error_response("This server does not support embeddings. Start it with `--embeddings`", ERROR_TYPE_NOT_SUPPORTED)); - return res; - } + // P0-1 (#49): null-meta guard + if (!meta) { + res->error(format_error_response("model not loaded — waiting for CONFIGURE", + ERROR_TYPE_NOT_SUPPORTED)); + return res; + } - if (res_type != TASK_RESPONSE_TYPE_NONE && meta->pooling_type == LLAMA_POOLING_TYPE_NONE) { - res->error(format_error_response("Pooling type 'none' is not OAI compatible. Please use a different pooling type", ERROR_TYPE_INVALID_REQUEST)); + json models = { + {"models", { + { + {"name", meta->model_name}, + {"model", meta->model_name}, + {"modified_at", ""}, + {"size", ""}, + {"digest", ""}, // dummy value, llama.cpp does not support managing model file's hash + {"type", "model"}, + {"description", ""}, + {"tags", {""}}, + {"capabilities", meta->has_mtmd ? json({"completion","multimodal"}) : json({"completion"})}, + {"parameters", ""}, + {"details", { + {"parent_model", ""}, + {"format", "gguf"}, + {"family", ""}, + {"families", {""}}, + {"parameter_size", ""}, + {"quantization_level", ""} + }} + } + }}, + {"object", "list"}, + {"data", { + get_model_info(), + }} + }; + + res->ok(models); return res; - } + }; - const json body = json::parse(req.body); + this->post_tokenize = [this](const server_http_req & req) { + auto res = create_response(); + const json body = json::parse(req.body); + json tokens_response = json::array(); + if (body.count("content") != 0) { + const bool add_special = json_value(body, "add_special", false); + const bool parse_special = json_value(body, "parse_special", true); + const bool with_pieces = json_value(body, "with_pieces", false); - // for the shape of input/content, see tokenize_input_prompts() - json prompt; - if (body.count("input") != 0) { - prompt = body.at("input"); - } else if (body.contains("content")) { - res_type = TASK_RESPONSE_TYPE_NONE; // "content" field is not OAI compatible - prompt = body.at("content"); - } else { - res->error(format_error_response("\"input\" or \"content\" must be provided", ERROR_TYPE_INVALID_REQUEST)); - return res; - } + llama_tokens tokens = tokenize_mixed(ctx_server.vocab, body.at("content"), add_special, parse_special); - bool use_base64 = false; - if (body.count("encoding_format") != 0) { - const std::string & format = body.at("encoding_format"); - if (format == "base64") { - use_base64 = true; - } else if (format != "float") { - res->error(format_error_response("The format to return the embeddings in. Can be either float or base64", ERROR_TYPE_INVALID_REQUEST)); - return res; - } - } + if (with_pieces) { + for (const auto& token : tokens) { + std::string piece = common_token_to_piece(ctx_server.vocab, token); + json piece_json; - auto tokenized_prompts = tokenize_input_prompts(ctx_server.vocab, ctx_server.mctx, prompt, true, true); - for (const auto & tokens : tokenized_prompts) { - // this check is necessary for models that do not add BOS token to the input - if (tokens.empty()) { - res->error(format_error_response("Input content cannot be empty", ERROR_TYPE_INVALID_REQUEST)); - return res; - } - } + // Check if the piece is valid UTF-8 + if (is_valid_utf8(piece)) { + piece_json = piece; + } else { + // If not valid UTF-8, store as array of byte values + piece_json = json::array(); + for (unsigned char c : piece) { + piece_json.push_back(static_cast(c)); + } + } - int embd_normalize = params.embd_normalize; - if (body.count("embd_normalize") != 0) { - embd_normalize = body.at("embd_normalize"); - if (meta->pooling_type == LLAMA_POOLING_TYPE_NONE) { - SRV_DBG("embd_normalize is not supported by pooling type %d, ignoring it\n", meta->pooling_type); + tokens_response.push_back({ + {"id", token}, + {"piece", piece_json} + }); + } + } else { + tokens_response = tokens; + } } - } - - // create and queue the task - json responses = json::array(); - auto & rd = res->rd; - { - std::vector tasks; - for (size_t i = 0; i < tokenized_prompts.size(); i++) { - server_task task = server_task(SERVER_TASK_TYPE_EMBEDDING); - task.id = rd.get_new_id(); - task.tokens = std::move(tokenized_prompts[i]); + res->ok(json{{"tokens", std::move(tokens_response)}}); + return res; + }; - // OAI-compat - task.params.res_type = res_type; - task.params.embd_normalize = embd_normalize; + this->post_detokenize = [this](const server_http_req & req) { + auto res = create_response(); + const json body = json::parse(req.body); - tasks.push_back(std::move(task)); + std::string content; + if (body.count("tokens") != 0) { + const llama_tokens tokens = body.at("tokens"); + content = tokens_to_str(ctx_server.vocab, tokens); } - rd.post_tasks(std::move(tasks)); - } - // wait for the results - auto all_results = rd.wait_for_all(req.should_stop); - - // collect results - if (all_results.is_terminated) { - return res; // connection is closed - } else if (all_results.error) { - res->error(all_results.error->to_json()); + res->ok(json{{"content", std::move(content)}}); return res; - } else { - for (auto & res : all_results.results) { - GGML_ASSERT(dynamic_cast(res.get()) != nullptr); - responses.push_back(res->to_json()); - } - } + }; - // write JSON response - json root = res_type == TASK_RESPONSE_TYPE_OAI_EMBD - ? format_embeddings_response_oaicompat(body, meta->model_name, responses, use_base64) - : json(responses); - res->ok(root); - return res; -} + this->post_embeddings = [this](const server_http_req & req) { + return handle_embeddings_impl(req, TASK_RESPONSE_TYPE_NONE); + }; -// ═══════════════════════════════════════════════════════════════════════════════ -// Hydra RPC server — KV state transfer (M1: task-queue based) -// Wire format: specs/rpc-protocol.md | constants: server-rpc.h -// Ops implemented: STATE_GET (0x30), STATE_PUT (0x31), STATE_META (0x32) -// M1: All llama API calls routed through task queue (inference thread safe) -// ═══════════════════════════════════════════════════════════════════════════════ + this->post_embeddings_oai = [this](const server_http_req & req) { + return handle_embeddings_impl(req, TASK_RESPONSE_TYPE_OAI_EMBD); + }; -#if !defined(_WIN32) + this->post_rerank = [this](const server_http_req & req) { + auto res = create_response(); + std::shared_lock meta_lock(meta_mutex); + // P0-1 (#49): null-meta guard + if (!meta) { + res->error(format_error_response("model not loaded — waiting for CONFIGURE", + ERROR_TYPE_NOT_SUPPORTED)); + return res; + } + if (!params.embedding || params.pooling_type != LLAMA_POOLING_TYPE_RANK) { + res->error(format_error_response("This server does not support reranking. Start it with `--reranking`", ERROR_TYPE_NOT_SUPPORTED)); + return res; + } -// ── Context for RPC thread — pass to handlers ───────────────────────────────── + const json body = json::parse(req.body); -struct hydra_rpc_ctx { - server_queue * queue_tasks = nullptr; - server_response * queue_results = nullptr; -}; + // if true, use TEI API format, otherwise use Jina API format + // Jina: https://jina.ai/reranker/ + // TEI: https://huggingface.github.io/text-embeddings-inference/#/Text%20Embeddings%20Inference/rerank + bool is_tei_format = body.contains("texts"); -// ── Low-level I/O helpers ───────────────────────────────────────────────────── - -// Hydra #43: failures here were previously silent — every caller treats a -// `false` return as "give up" but none logged *why*, so a wedged RPC -// response looked identical to a client that vanished. Log once, centrally, -// instead of touching the ~30 call sites. -static bool hydra_recv_all(int fd, void * buf, size_t n) { - char * p = reinterpret_cast(buf); - const size_t total = n; - while (n > 0) { - ssize_t r = ::recv(fd, p, n, 0); - if (r < 0) { - SRV_WRN("hydra rpc: recv failed on fd=%d (%zu/%zu bytes): %s\n", - fd, total - n, total, std::strerror(errno)); - return false; - } - if (r == 0) { - SRV_DBG("hydra rpc: recv EOF on fd=%d (%zu/%zu bytes)\n", fd, total - n, total); - return false; + json query; + if (body.count("query") == 1) { + query = body.at("query"); + if (!query.is_string()) { + res->error(format_error_response("\"query\" must be a string", ERROR_TYPE_INVALID_REQUEST)); + return res; + } + } else { + res->error(format_error_response("\"query\" must be provided", ERROR_TYPE_INVALID_REQUEST)); + return res; } - p += r; n -= r; - } - return true; -} -static bool hydra_send_all(int fd, const void * buf, size_t n) { - const char * p = reinterpret_cast(buf); - const size_t total = n; - while (n > 0) { - ssize_t w = ::send(fd, p, n, MSG_NOSIGNAL); - if (w <= 0) { - SRV_WRN("hydra rpc: send failed on fd=%d (%zu/%zu bytes) w=%zd: %s\n", - fd, total - n, total, w, std::strerror(errno)); - return false; + std::vector documents = json_value(body, "documents", + json_value(body, "texts", std::vector())); + if (documents.empty()) { + res->error(format_error_response("\"documents\" must be a non-empty string array", ERROR_TYPE_INVALID_REQUEST)); + return res; } - p += w; n -= w; - } - return true; -} -// Response header: status(1) | meta_len(3 LE uint24) | payload_len(8 LE) — 12 bytes -static void hydra_write_res(int fd, uint8_t status, uint32_t meta_len, uint64_t payload_len) { - uint8_t buf[HYDRA_RES_HEADER_SIZE] = {}; - buf[0] = status; - buf[1] = (meta_len) & 0xFF; - buf[2] = (meta_len >> 8) & 0xFF; - buf[3] = (meta_len >> 16) & 0xFF; - memcpy(buf + 4, &payload_len, 8); // little-endian (x86/arm64) - hydra_send_all(fd, buf, HYDRA_RES_HEADER_SIZE); -} - -// ── Op handlers (M1: dispatch via task queue) ───────────────────────────────── - -// STATE_GET (0x30): Post task, wait for result. -// -// M1 path (hydra_fd < 0): inference thread serializes 800 MB into result buffer; -// RPC thread sends response header + meta JSON + buffer here. -// -// M2 path (hydra_fd = fd): background thread streams GPU→socket directly using -// llama_state_seq_get_data_to_fd; result carries only n_past + streamed_bytes. -// Response header + meta are sent BEFORE the task (we know size from STATE_META), -// so the payload is already on the wire before we even get the result back. -// Actually: we must send header AFTER knowing state_size. So: -// - If M2: we get state_size first from a quick STATE_META query (n_past already known), -// OR we embed state_size in the result from get_size() on the inference thread. -// The inference thread always calls llama_state_seq_get_size (cheap) and stores it -// in res->state_size for M2 so we can send the header before the stream completes. -// -// Timeout: 30s — streaming 800 MB over localhost may take a few seconds. -static void hydra_handle_state_get(int fd, int slot_id, const hydra_rpc_ctx & ctx) { - // Build task — pass fd for M2 zero-copy streaming - server_task task(SERVER_TASK_TYPE_HYDRA_STATE_GET); - task.id = ctx.queue_tasks->get_new_id(); - task.hydra_action.id_slot = slot_id; - task.hydra_action.hydra_fd = fd; // M2: background thread streams here - const int task_id = task.id; - // Register BEFORE posting — server_response::send() silently drops results - // for ids not in waiting_task_ids. - ctx.queue_results->add_waiting_task_id(task_id); - ctx.queue_tasks->wait_until_no_sleep(); - ctx.queue_tasks->post(std::move(task)); - - // Wait for result (n_past + state_size always set; state_data only on M1) - std::unordered_set task_ids = {task_id}; - auto res_ptr = ctx.queue_results->recv_with_timeout(task_ids, 30); // seconds - ctx.queue_results->remove_waiting_task_id(task_id); - if (!res_ptr) { - SRV_WRN("hydra rpc: STATE_GET timeout for slot %d\n", slot_id); - // M2 caveat: the background thread may own the fd (header possibly sent); - // writing an error header here could interleave with the stream. Shut the - // socket down instead so the client unblocks with a clean EOF. - ::shutdown(fd, SHUT_RDWR); - return; - } - - auto * res = dynamic_cast(res_ptr.get()); - if (!res) { - SRV_WRN("hydra rpc: STATE_GET result type mismatch for slot %d\n", slot_id); - hydra_write_res(fd, HYDRA_STATUS_ERROR, 0, 0); - return; - } + int top_n = json_value(body, "top_n", (int)documents.size()); - if (res->rpc_status != HYDRA_STATUS_OK) { - if (res->header_sent) { - // M2 failure: header already sent but stream failed; background thread - // shut the socket down — connection loop will close the fd on next read. - // Log and return without sending a second response header. - SRV_WRN("hydra rpc: STATE_GET slot=%d M2 stream failed: %s\n", - slot_id, res->error.c_str()); - return; - } - hydra_write_res(fd, res->rpc_status, 0, 0); - if (!res->error.empty()) { - hydra_send_all(fd, res->error.data(), res->error.size()); + // create and queue the task + json responses = json::array(); + auto & rd = res->rd; + { + std::vector tasks; + tasks.reserve(documents.size()); + for (size_t i = 0; i < documents.size(); i++) { + auto tmp = format_prompt_rerank(ctx_server.model_tgt, ctx_server.vocab, ctx_server.mctx, query, documents[i]); + server_task task = server_task(SERVER_TASK_TYPE_RERANK); + task.id = rd.get_new_id(); + task.tokens = std::move(tmp); + tasks.push_back(std::move(task)); + } + rd.post_tasks(std::move(tasks)); } - return; - } - if (res->streamed_bytes > 0) { - // M2 path: data already on the wire — response header + meta were sent by background thread. - // Nothing left for RPC thread to do. The protocol framing (header + meta + payload) - // was completed inside llama_io_write_socket / the background thread. - // Note: header was sent AFTER state_size was known (inference thread called get_size). - SRV_INF("hydra rpc: STATE_GET slot=%d M2 streamed %.1f MiB directly\n", - slot_id, res->streamed_bytes / (1024.0 * 1024.0)); - } else { - // M1 path: inference thread buffered 800 MB; send it now. - const uint64_t payload = (uint64_t)res->state_data.size(); - json meta_j; - meta_j["n_past"] = res->n_past; - meta_j["state_size"] = payload; - if (!res->model_alias.empty()) meta_j["model_alias"] = res->model_alias; - if (!res->model_path.empty()) meta_j["model_path"] = res->model_path; - if (!res->tokenizer.empty()) meta_j["tokenizer"] = res->tokenizer; - if (!res->model_name.empty()) meta_j["model_name"] = res->model_name; - if (!res->model_quant.empty()) meta_j["model_quant"] = res->model_quant; - if (res->model_capabilities) meta_j["model_capabilities"] = res->model_capabilities; - const std::string meta_str = meta_j.dump(); - hydra_write_res(fd, HYDRA_STATUS_OK, (uint32_t)meta_str.size(), payload); - hydra_send_all(fd, meta_str.data(), meta_str.size()); - hydra_send_all(fd, res->state_data.data(), (size_t)payload); - SRV_INF("hydra rpc: STATE_GET slot=%d M1 sent %.1f MiB from buffer\n", - slot_id, payload / (1024.0 * 1024.0)); - } -} + // wait for the results + auto all_results = rd.wait_for_all(req.should_stop); -// STATE_PUT (0x31): Receive payload, post task, wait for result, send ack. -static void hydra_handle_state_put(int fd, int slot_id, uint64_t payload_len, const hydra_rpc_ctx & ctx) { - if (payload_len > HYDRA_MAX_STATE_BYTES) { - SRV_WRN("hydra rpc: STATE_PUT payload %" PRIu64 " B exceeds cap %" PRIu64 " B\n", - payload_len, HYDRA_MAX_STATE_BYTES); - hydra_write_res(fd, HYDRA_STATUS_ERROR, 0, 0); - // Drain to keep persistent connection alive - std::vector drain(65536); - for (uint64_t rem = payload_len; rem > 0; ) { - size_t chunk = (size_t)std::min(rem, (uint64_t)drain.size()); - if (!hydra_recv_all(fd, drain.data(), chunk)) break; - rem -= chunk; - } - return; - } + // collect results + if (all_results.is_terminated) { + return res; // connection is closed + } else if (all_results.error) { + res->error(all_results.error->to_json()); + return res; + } else { + for (auto & res : all_results.results) { + GGML_ASSERT(dynamic_cast(res.get()) != nullptr); + responses.push_back(res->to_json()); + } + } - // Read payload from socket - std::vector buf((size_t)payload_len); - if (!hydra_recv_all(fd, buf.data(), (size_t)payload_len)) { - SRV_WRN("%s", "hydra rpc: STATE_PUT failed to read payload\n"); - hydra_write_res(fd, HYDRA_STATUS_ERROR, 0, 0); - return; - } + // write JSON response + json root = format_response_rerank( + body, + meta->model_name, + responses, + is_tei_format, + documents, + top_n); - // Post task to inference thread - server_task task(SERVER_TASK_TYPE_HYDRA_STATE_PUT); - task.id = ctx.queue_tasks->get_new_id(); - task.hydra_action.id_slot = slot_id; - task.hydra_action.erase_existing = true; // RPC restore always replaces slot state - task.hydra_action.state_data = std::move(buf); - const int task_id = task.id; - // Register BEFORE posting — results for unregistered ids are dropped. - ctx.queue_results->add_waiting_task_id(task_id); - ctx.queue_tasks->wait_until_no_sleep(); - ctx.queue_tasks->post(std::move(task)); - - // Wait for result from inference thread (30s timeout for large restore) - std::unordered_set task_ids = {task_id}; - auto res_ptr = ctx.queue_results->recv_with_timeout(task_ids, 30); // seconds - ctx.queue_results->remove_waiting_task_id(task_id); - if (!res_ptr) { - SRV_WRN("hydra rpc: STATE_PUT timeout for slot %d\n", slot_id); - hydra_write_res(fd, HYDRA_STATUS_ERROR, 0, 0); - return; - } + res->ok(root); + return res; + }; - auto * res = dynamic_cast(res_ptr.get()); - if (!res) { - SRV_WRN("hydra rpc: STATE_PUT result type mismatch for slot %d\n", slot_id); - hydra_write_res(fd, HYDRA_STATUS_ERROR, 0, 0); - return; - } + this->get_lora_adapters = [this](const server_http_req & req) { + auto res = create_response(); - // Send result back to client - uint8_t rpc_status = res->rpc_status; - if (rpc_status == HYDRA_STATUS_OK) { - json meta_j; - meta_j["restored"] = true; - meta_j["bytes"] = res->bytes; - meta_j["model_match"] = res->model_match; - if (!res->model_alias.empty()) meta_j["model_alias"] = res->model_alias; - if (!res->model_path.empty()) meta_j["model_path"] = res->model_path; - if (!res->tokenizer.empty()) meta_j["tokenizer"] = res->tokenizer; - if (!res->model_name.empty()) meta_j["model_name"] = res->model_name; - if (!res->model_quant.empty()) meta_j["model_quant"] = res->model_quant; - if (res->model_capabilities) meta_j["model_capabilities"] = res->model_capabilities; - const std::string meta_str = meta_j.dump(); - hydra_write_res(fd, HYDRA_STATUS_OK, (uint32_t)meta_str.size(), 0); - hydra_send_all(fd, meta_str.data(), meta_str.size()); - } else { - json err_j; - err_j["error"] = res->error; - const std::string err_str = err_j.dump(); - hydra_write_res(fd, rpc_status, (uint32_t)err_str.size(), 0); - hydra_send_all(fd, err_str.data(), err_str.size()); - } -} + auto & rd = res->rd; + { + server_task task(SERVER_TASK_TYPE_GET_LORA); + task.id = rd.get_new_id(); + rd.post_task(std::move(task)); + } -// STATE_META (0x32): Post task, wait for result, send JSON metadata. -static void hydra_handle_state_meta(int fd, int slot_id, const hydra_rpc_ctx & ctx) { - server_task task(SERVER_TASK_TYPE_HYDRA_STATE_META); - task.id = ctx.queue_tasks->get_new_id(); - task.hydra_action.id_slot = slot_id; - const int task_id = task.id; - // Register BEFORE posting — results for unregistered ids are dropped. - ctx.queue_results->add_waiting_task_id(task_id); - ctx.queue_tasks->wait_until_no_sleep(); - ctx.queue_tasks->post(std::move(task)); - - // Wait for result from inference thread (5s timeout — allows for queue congestion) - std::unordered_set task_ids = {task_id}; - auto res_ptr = ctx.queue_results->recv_with_timeout(task_ids, 5); // seconds - ctx.queue_results->remove_waiting_task_id(task_id); - if (!res_ptr) { - SRV_WRN("hydra rpc: STATE_META timeout for slot %d\n", slot_id); - hydra_write_res(fd, HYDRA_STATUS_ERROR, 0, 0); - return; - } + // get the result + auto result = rd.next(req.should_stop); + if (!result) { + // connection was closed + GGML_ASSERT(req.should_stop()); + return res; + } - auto * res = dynamic_cast(res_ptr.get()); - if (!res) { - SRV_WRN("hydra rpc: STATE_META result type mismatch for slot %d\n", slot_id); - hydra_write_res(fd, HYDRA_STATUS_ERROR, 0, 0); - return; - } + if (result->is_error()) { + res->error(result->to_json()); + return res; + } - // Send result back to client - uint8_t rpc_status = res->rpc_status; - if (rpc_status == HYDRA_STATUS_OK) { - json meta_j; - meta_j["slot_id"] = res->id_slot; - meta_j["n_past"] = res->n_past; - meta_j["state_size"] = res->state_size; - meta_j["is_processing"] = res->is_processing; - meta_j["is_transferring"] = res->is_transferring; - if (!res->model_alias.empty()) meta_j["model_alias"] = res->model_alias; - if (!res->model_path.empty()) meta_j["model_path"] = res->model_path; - if (!res->tokenizer.empty()) meta_j["tokenizer"] = res->tokenizer; - if (!res->model_name.empty()) meta_j["model_name"] = res->model_name; - if (!res->model_quant.empty()) meta_j["model_quant"] = res->model_quant; - if (res->model_capabilities) meta_j["model_capabilities"] = res->model_capabilities; - const std::string meta_str = meta_j.dump(); - hydra_write_res(fd, HYDRA_STATUS_OK, (uint32_t)meta_str.size(), 0); - hydra_send_all(fd, meta_str.data(), meta_str.size()); - } else { - hydra_write_res(fd, rpc_status, 0, 0); - } -} + GGML_ASSERT(dynamic_cast(result.get()) != nullptr); + res->ok(result->to_json()); + return res; + }; -// ── E1 Engine control handlers ──────────────────────────────────────────────── + this->post_lora_adapters = [this](const server_http_req & req) { + auto res = create_response(); + const json body = json::parse(req.body); + if (!body.is_array()) { + res->error(format_error_response("Request body must be an array", ERROR_TYPE_INVALID_REQUEST)); + return res; + } -// CONFIGURE (0x33): Read JSON config payload, post task, return success. -static void hydra_handle_configure(int fd, int slot_id, uint64_t payload_len, const hydra_rpc_ctx & ctx) { - std::string config_json(payload_len, '\0'); - if (payload_len > 0 && !hydra_recv_all(fd, config_json.data(), payload_len)) { - hydra_write_res(fd, HYDRA_STATUS_ERROR, 0, 0); - return; - } + auto & rd = res->rd; + { + server_task task(SERVER_TASK_TYPE_SET_LORA); + task.id = rd.get_new_id(); + task.set_lora = parse_lora_request(body); + rd.post_task(std::move(task)); + } - server_task task(SERVER_TASK_TYPE_HYDRA_ENGINE_CONFIGURE); - task.id = ctx.queue_tasks->get_new_id(); - task.hydra_action.id_slot = slot_id; - task.hydra_action.config_json = std::move(config_json); - const int task_id = task.id; - ctx.queue_results->add_waiting_task_id(task_id); - ctx.queue_tasks->wait_until_no_sleep(); - ctx.queue_tasks->post(std::move(task)); - - std::unordered_set task_ids = {task_id}; - auto res_ptr = ctx.queue_results->recv_with_timeout(task_ids, 5); - ctx.queue_results->remove_waiting_task_id(task_id); - if (!res_ptr) { - hydra_write_res(fd, HYDRA_STATUS_ERROR, 0, 0); - return; - } + // get the result + auto result = rd.next(req.should_stop); + if (!result) { + // connection was closed + GGML_ASSERT(req.should_stop()); + return res; + } - auto * res = dynamic_cast(res_ptr.get()); - if (!res || !res->success) { - // hydra#406: on failure, include the error message in the meta so - // the Coordinator can distinguish "drain timeout" from a parse - // error. We still write HYDRA_STATUS_ERROR (0x02) per the wire - // contract — the meta body is for diagnostics only. - if (res && !res->error.empty()) { - json err_j = {{"success", false}, {"error", res->error}}; - if (!res->tier.empty()) err_j["tier"] = res->tier; - const std::string err_str = err_j.dump(); - hydra_write_res(fd, HYDRA_STATUS_ERROR, (uint32_t)err_str.size(), 0); - hydra_send_all(fd, err_str.data(), err_str.size()); - } else { - hydra_write_res(fd, HYDRA_STATUS_ERROR, 0, 0); + if (result->is_error()) { + res->error(result->to_json()); + return res; } - return; - } - // hydra#406: tiered CONFIGURE response shape. Always present: success, - // tier, params_applied (T1 keys), deferred_keys (T2/T3 keys). - json meta_j = { - {"success", true}, - {"tier", res->tier.empty() ? std::string("T1") : res->tier}, - {"params_applied", json::object()}, - {"deferred_keys", json::array()}, + GGML_ASSERT(dynamic_cast(result.get()) != nullptr); + res->ok(result->to_json()); + return res; }; - for (const auto & kv : res->params_applied) { - meta_j["params_applied"][kv.first] = kv.second; - } - for (const auto & k : res->deferred_keys) { - meta_j["deferred_keys"].push_back(k); - } - // hydra#334: echo the post-clamp value for the state_chunk_size legacy - // path so the Coordinator's existing detection logic still works - // (the same value is also in params_applied, with the dotted key). - if (res->state_chunk_size_applied > 0) { - meta_j["state_chunk_size_applied"] = res->state_chunk_size_applied; - } - const std::string meta_str = meta_j.dump(); - hydra_write_res(fd, HYDRA_STATUS_OK, (uint32_t)meta_str.size(), 0); - hydra_send_all(fd, meta_str.data(), meta_str.size()); -} - -// INFO (0x34): Return engine capabilities as JSON. -static void hydra_handle_info(int fd, int slot_id, const hydra_rpc_ctx & ctx) { - server_task task(SERVER_TASK_TYPE_HYDRA_ENGINE_INFO); - task.id = ctx.queue_tasks->get_new_id(); - task.hydra_action.id_slot = slot_id; - const int task_id = task.id; - ctx.queue_results->add_waiting_task_id(task_id); - ctx.queue_tasks->wait_until_no_sleep(); - ctx.queue_tasks->post(std::move(task)); - - std::unordered_set task_ids = {task_id}; - auto res_ptr = ctx.queue_results->recv_with_timeout(task_ids, 5); - ctx.queue_results->remove_waiting_task_id(task_id); - if (!res_ptr) { - hydra_write_res(fd, HYDRA_STATUS_ERROR, 0, 0); - return; - } - auto * res = dynamic_cast(res_ptr.get()); - if (!res) { - hydra_write_res(fd, HYDRA_STATUS_ERROR, 0, 0); - return; - } + // ── Merged DECODE result retrieval ───────────────────────────────────── + this->get_decode_result = [this](const server_http_req & req) { + auto res = create_response(true); + int32_t decode_request_id; + try { + decode_request_id = std::stoi(req.get_param("decode_request_id")); + } catch (const std::exception &) { + res->error(format_error_response("Invalid decode_request_id", ERROR_TYPE_INVALID_REQUEST)); + return res; + } - const std::string & info_str = res->info_json; - hydra_write_res(fd, HYDRA_STATUS_OK, (uint32_t)info_str.size(), 0); - hydra_send_all(fd, info_str.data(), info_str.size()); -} + std::unique_lock lock(decode_results_mutex); + evict_decode_results_locked(); + auto it = decode_results.find(decode_request_id); + if (it == decode_results.end()) { + lock.unlock(); + res->error(format_error_response("decode_request_id not found or expired", ERROR_TYPE_NOT_FOUND)); + return res; + } -// PREFILL (0x35): Read JSON payload with {"messages": [...]}, -// tokenize internally, run prefill, return n_past + KV state blob. -static void hydra_handle_prefill(int fd, int slot_id, uint64_t payload_len, const hydra_rpc_ctx & ctx) { - if (payload_len == 0) { - hydra_write_res(fd, HYDRA_STATUS_ERROR, 0, 0); - return; - } + // Read fields we need before unlocking + const auto entry_state = it->second.state; + const auto entry_error = it->second.error; + const int32_t id_slot = it->second.id_slot; + const std::string completion_id = it->second.completion_id; + const std::string oaicompat_model = it->second.oaicompat_model; + const std::string content = it->second.content; + const std::string reasoning_content = it->second.reasoning_content; + const json tool_calls = it->second.tool_calls; + const int32_t n_decoded = it->second.n_decoded; + const int32_t n_prompt_tokens = it->second.n_prompt_tokens; + const int32_t n_prompt_tokens_cache = it->second.n_prompt_tokens_cache; + const result_timings timings = it->second.timings; + const stop_type stop = it->second.stop; + const bool include_usage = it->second.include_usage; + const json hydra_metrics = it->second.hydra_metrics; + const json match_json = it->second.match_json; + const double model_load_ms = it->second.model_load_ms; + const double restore_slot_ms = it->second.restore_slot_ms; + const json model_identity = it->second.model_identity; + lock.unlock(); - std::string json_str((size_t)payload_len, '\0'); - if (!hydra_recv_all(fd, json_str.data(), (size_t)payload_len)) { - hydra_write_res(fd, HYDRA_STATUS_ERROR, 0, 0); - return; - } + // ── Terminal error ───────────────────────────────────────────────── + if (!entry_error.empty()) { + { + std::lock_guard lk(decode_results_mutex); + decode_results.erase(decode_request_id); + } + json err_j = { + {"error", entry_error}, + {"error_code", "DECODE_FAILED"}, + {"match", match_json}, + }; + res->error(format_error_response(entry_error, ERROR_TYPE_INVALID_REQUEST)); + return res; + } - server_task task(SERVER_TASK_TYPE_HYDRA_ENGINE_PREFILL); - task.id = ctx.queue_tasks->get_new_id(); - task.hydra_action.id_slot = slot_id; - task.hydra_action.hydra_fd = fd; // M2 (#470): task thread streams the response here - task.hydra_action.request_json = std::move(json_str); - const int task_id = task.id; - ctx.queue_results->add_waiting_task_id(task_id); - ctx.queue_tasks->wait_until_no_sleep(); - ctx.queue_tasks->post(std::move(task)); - - std::unordered_set task_ids = {task_id}; - // Bumped from 60s to 180s, then 180s to 600s (#470). Prefill for 32k+ - // token prompts exceeds 120s (we measured 32s for 22k tokens; 48k ≈ 70s, - // 100k ≈ 150s+). Long autoregressive decode on P100 (28 tok/s) for 4k+ - // token outputs also exceeds 120s. On top of compute, the M2 stream must - // push the whole KV blob (≈800 MB today, 10 GB target) over the socket - // before this wait returns — the old 180s budget raced that transfer and - // dropped the connection mid-frame (coordinator then read garbage framing - // like 'RPC payload length out of range'). 600s covers compute + transfer - // with headroom; the Coordinator enforces an idle-based budget client-side. - auto res_ptr = ctx.queue_results->recv_with_timeout(task_ids, 600); - ctx.queue_results->remove_waiting_task_id(task_id); - if (!res_ptr) { - // #470: the M2 stream task owns the fd (header + KV already streaming); - // writing an error header here could interleave with the stream. Mirror - // the STATE_GET M2 caveat (see hydra_handle_state_get): shut the socket - // down so the client unblocks with a clean EOF and the worker frees - // immediately (SO_SNDTIMEO bounds the send-side park meanwhile). - SRV_WRN("hydra rpc: PREFILL timeout for slot %d (task_id=%d)\n", slot_id, task_id); - ::shutdown(fd, SHUT_RDWR); - return; - } + // httplib's own Headers map is case-insensitive + // (detail::case_ignore::hash, httplib.h), but server-http.cpp + // get_headers() copies it into a plain case-sensitive + // std::map, so match the Accept header + // case-insensitively here — otherwise clients sending + // "Accept: text/event-stream" never reach the SSE branches. + const bool stream = [&]() { + for (const auto & [hname, hval] : req.headers) { + if (hval.find("text/event-stream") == std::string::npos) { + continue; + } + if (hname.size() != 6) { + continue; + } + bool is_accept = true; + for (size_t i = 0; i < 6; i++) { + char c = hname[i]; + if (c >= 'A' && c <= 'Z') { + c = (char)(c - 'A' + 'a'); + } + if (c != "accept"[i]) { + is_accept = false; + break; + } + } + if (is_accept) { + return true; + } + } + return false; + }(); - auto * res = dynamic_cast(res_ptr.get()); - if (!res || res->rpc_status != HYDRA_STATUS_OK) { - if (res && res->header_sent) { - // M2 failure: header + meta already on the wire but the stream - // failed; the task thread shut the socket down — the connection - // loop will close the fd on its next read. Do NOT write a second - // response header (would interleave with nothing — socket is - // shut — but must not emit a second frame either). - SRV_WRN("hydra rpc: PREFILL slot=%d M2 stream failed: %s\n", - slot_id, res->error.c_str()); - return; + // ── In-progress states → 202 ────────────────────────────────────── + if (entry_state == server_routes::DECODE_STATE_LOADING || + entry_state == server_routes::DECODE_STATE_RESTORING) { + json state_j = { + {"state", entry_state == server_routes::DECODE_STATE_LOADING ? "loading" : "restoring"}, + {"decode_request_id", decode_request_id}, + {"id_slot", id_slot}, + {"model_load_ms", model_load_ms}, + {"restore_slot_ms", restore_slot_ms}, + {"match", match_json}, + }; + res->status = 202; + res->data = safe_json_to_str(state_j); + return res; } - hydra_write_res(fd, HYDRA_STATUS_ERROR, 0, 0); - return; - } - if (res->streamed_bytes > 0) { - // M2 path: the task thread already wrote the response header + meta + - // v2 header + KV state + logits straight to the socket. Nothing left - // for the RPC thread to do — protocol framing was completed on the - // task thread. (mirrors STATE_GET M2 handling) - SRV_INF("hydra rpc: PREFILL slot=%d M2 streamed %.1f MiB directly\n", - slot_id, res->streamed_bytes / (1024.0 * 1024.0)); - return; - } + // ── GENERATING + SSE → stream partials from relay queue ───────── + if (entry_state == server_routes::DECODE_STATE_GENERATING && stream) { + // Set up SSE response; partials arrive via the streaming_queue + // populated by the background consumer thread. + res->status = 200; + res->content_type = "text/event-stream"; + res->data = ""; // no initial chunk — send headers immediately - // M1 path: RPC thread sends header + meta + buffered payload (unchanged). - // Return n_past + sizes + model identity in meta; full blob (v2 header + KV + logits) as payload. - // logits_size > 0 signals the decode GPU to inject them into ctx->logits via STATE_PUT. - // M-Perf.9 #289: model identity fields (already populated on res by the PREFILL handler) - // are included so the Coordinator can record which model built the KV. - json meta_j = { - {"n_past", res->n_past}, - {"state_size", res->state_size}, - {"logits_size", res->logits_size} - }; - if (!res->model_alias.empty()) meta_j["model_alias"] = res->model_alias; - if (!res->model_path.empty()) meta_j["model_path"] = res->model_path; - if (!res->tokenizer.empty()) meta_j["tokenizer"] = res->tokenizer; - if (!res->model_name.empty()) meta_j["model_name"] = res->model_name; - if (!res->model_quant.empty()) meta_j["model_quant"] = res->model_quant; - if (res->model_capabilities) meta_j["model_capabilities"] = res->model_capabilities; - meta_j["model_fallback"] = res->model_fallback; - if (res->prefill_ms > 0) meta_j["prefill_ms"] = res->prefill_ms; - if (res->model_load_ms > 0) meta_j["model_load_ms"] = res->model_load_ms; - const std::string meta_str = meta_j.dump(); - const uint64_t total_payload = (uint64_t)res->state_data.size(); - hydra_write_res(fd, HYDRA_STATUS_OK, (uint32_t)meta_str.size(), total_payload); - hydra_send_all(fd, meta_str.data(), meta_str.size()); - if (total_payload > 0) { - hydra_send_all(fd, res->state_data.data(), (size_t)total_payload); - } - SRV_INF("hydra: PREFILL slot=%d sent n_past=%d kv=%" PRIu64 "B logits=%" PRIu64 "B total=%" PRIu64 "B\n", - slot_id, res->n_past, res->state_size, res->logits_size, total_payload); -} + res->next = [this, res_this = res.get(), decode_request_id, &req, sent_final = false]( + std::string & output) mutable -> bool { + try { + if (req.should_stop()) { + return false; + } -// DECODE (0x43) — Merged P/D: framed request with async HTTP retrieval. -// Wire format v3 (segmented): -// [4B hdr_len LE] <= 32768 -// [8B hdr_hash LE] xxh3-64 of the hdr JSON bytes that follow -// [hdr_len bytes] control header JSON -// [prompt_len bytes] prompt JSON segment (may be zero-length) -// [kv_len bytes] raw KV blob (may be zero-length) -// -// Control header: -// { "v": 3, "model": "...", "kv_metadata": {...}, "model_metadata": {...}, -// "generation": {...}, "segments": [...] } -// -// Two-phase flow: -// Phase 1 (sync): identity validation + KV restore — waits for inference thread -// Phase 2 (async): background thread posts SERVER_TASK_TYPE_COMPLETION, -// update_slots() drives generation, result stored in decode_results buffer. -// Actual result retrieved via GET /v1/decode/{decode_request_id}. -static void hydra_handle_decode(int fd, int slot_id, uint64_t payload_len, const hydra_rpc_ctx & ctx) { - // ── Read frame header: [4B hdr_len][8B hdr_hash] ────────────────────── - if (payload_len < sizeof(uint32_t) + sizeof(uint64_t)) { - SRV_WRN("%s", "hydra rpc: DECODE payload too small for frame header\n"); - hydra_write_res(fd, HYDRA_STATUS_BAD_REQUEST, 0, 0); - return; - } + std::unique_lock lock(decode_results_mutex); + auto it = decode_results.find(decode_request_id); + if (it == decode_results.end()) { + output = "data: [DONE]\n\n"; + return false; + } + auto & entry = it->second; - uint32_t hdr_len = 0; - if (!hydra_recv_all(fd, &hdr_len, sizeof(hdr_len))) { - hydra_write_res(fd, HYDRA_STATUS_ERROR, 0, 0); - return; - } + // Drain streaming queue + if (entry.stream) { + std::lock_guard slk(entry.stream->streaming_mutex); + if (!entry.stream->streaming_queue.empty()) { + auto result = std::move(entry.stream->streaming_queue.front()); + entry.stream->streaming_queue.pop_front(); + lock.unlock(); - if (hdr_len > HYDRA_MAX_JSON_HEADER) { - SRV_WRN("hydra rpc: DECODE hdr_len %u B exceeds cap %u B\n", - hdr_len, HYDRA_MAX_JSON_HEADER); - hydra_write_res(fd, HYDRA_STATUS_BAD_REQUEST, 0, 0); - return; - } + if (result->is_error()) { + json err_j = format_error_response("generation error", ERROR_TYPE_SERVER); + output = format_oai_sse(json{{"error", err_j}}); + return false; + } + json j = result->to_json(); + if (j.is_null()) { + // is_begin partial — skip + return true; + } + output = format_oai_sse(j); + return true; + } + } - uint64_t hdr_hash = 0; - if (!hydra_recv_all(fd, &hdr_hash, sizeof(hdr_hash))) { - hydra_write_res(fd, HYDRA_STATUS_ERROR, 0, 0); - return; - } + // Queue empty — stream finished: emit the final DONE delta + // exactly once, then terminate with [DONE]. Without it a + // client attached during GENERATING never sees the final + // finish_reason / usage / hydra_metrics. Content is + // deliberately NOT repeated: the relay already streamed + // content/reasoning_content/tool_calls incrementally via + // the partial deltas, so this is OpenAI's empty final + // chunk ({"delta": {...}, "finish_reason": ...}) — echoing + // full content/tool_calls again would make concat-based + // clients see output twice. (The DONE+SSE single-delta + // branch below keeps full content: that one fires for + // attach-after-DONE clients that saw no partials.) + if (entry.stream && entry.stream->stream_finished) { + if (!sent_final && entry.state == server_routes::DECODE_STATE_DONE) { + sent_final = true; + std::time_t t = std::time(0); + json delta { + {"choices", json::array({ + json { + {"finish_reason", entry.stop == STOP_TYPE_WORD || entry.stop == STOP_TYPE_EOS + ? (entry.tool_calls.empty() ? "stop" : "tool_calls") + : "length"}, + {"index", 0}, + {"delta", json{{"role", "assistant"}, {"content", ""}}}, + }, + })}, + {"created", t}, + {"id", entry.completion_id}, + {"model", entry.oaicompat_model}, + {"system_fingerprint", std::string(llama_build_info())}, + {"object", "chat.completion.chunk"}, + }; + if (entry.include_usage) { + delta["usage"] = json { + {"completion_tokens", entry.n_decoded}, + {"prompt_tokens", entry.n_prompt_tokens}, + {"total_tokens", entry.n_decoded + entry.n_prompt_tokens}, + {"prompt_tokens_details", json{{"cached_tokens", entry.n_prompt_tokens_cache}}}, + }; + } + if (!entry.hydra_metrics.is_null()) { + delta["hydra_metrics"] = entry.hydra_metrics; + } + output = format_oai_sse(delta); + return true; + } + output = "data: [DONE]\n\n"; + return false; + } - // ── Read control header JSON ────────────────────────────────────────── - std::string hdr_json_str(hdr_len, '\0'); - if (hdr_len > 0 && !hydra_recv_all(fd, hdr_json_str.data(), hdr_len)) { - hydra_write_res(fd, HYDRA_STATUS_ERROR, 0, 0); - return; - } + // Wait for next partial with ping interval + if (entry.stream) { + entry.stream->streaming_cv.wait_for(lock, std::chrono::seconds(30)); + } + return true; // loop again - // Verify hdr_hash (xxh3-64 of the JSON bytes) - { - const uint64_t computed = XXH3_64bits(hdr_json_str.data(), hdr_json_str.size()); - if (computed != hdr_hash) { - SRV_WRN("hydra rpc: DECODE HDR_HASH_MISMATCH expected=%016" PRIx64 " got=%016" PRIx64 "\n", - hdr_hash, computed); - hydra_write_res(fd, HYDRA_STATUS_BAD_REQUEST, 0, 0); - return; + } catch (const std::exception & e) { + json err_j = format_error_response(e.what(), ERROR_TYPE_SERVER); + output = format_oai_sse(json{{"error", err_j}}); + return false; + } + }; + return res; } - } - - // Parse control header - json req; - try { - req = json::parse(hdr_json_str); - } catch (const std::exception & e) { - SRV_WRN("hydra rpc: DECODE invalid JSON in control header: %s\n", e.what()); - hydra_write_res(fd, HYDRA_STATUS_BAD_REQUEST, 0, 0); - return; - } - // Validate version - const int hdr_version = req.value("v", 0); - if (hdr_version < 3) { - SRV_WRN("hydra rpc: DECODE unsupported version %d (need >= 3)\n", hdr_version); - hydra_write_res(fd, HYDRA_STATUS_BAD_REQUEST, 0, 0); - return; - } + // ── DONE → return full result ───────────────────────────────────── + if (entry_state == server_routes::DECODE_STATE_DONE || !content.empty()) { + if (stream) { + // SSE streaming: send full result as a single delta, then finish + std::time_t t = std::time(0); + json delta { + {"choices", json::array({ + json { + {"finish_reason", stop == STOP_TYPE_WORD || stop == STOP_TYPE_EOS + ? (tool_calls.empty() ? "stop" : "tool_calls") + : "length"}, + {"index", 0}, + {"delta", json{{"role", "assistant"}, {"content", content}}}, + }, + })}, + {"created", t}, + {"id", completion_id}, + {"model", oaicompat_model}, + {"system_fingerprint", std::string(llama_build_info())}, + {"object", "chat.completion.chunk"}, + }; - // Validate required fields - if (!req.contains("kv_metadata")) { - SRV_WRN("%s", "hydra rpc: DECODE missing kv_metadata in control header\n"); - hydra_write_res(fd, HYDRA_STATUS_BAD_REQUEST, 0, 0); - return; - } - if (!req.contains("segments") || !req["segments"].is_array()) { - SRV_WRN("%s", "hydra rpc: DECODE missing or invalid segments array\n"); - hydra_write_res(fd, HYDRA_STATUS_BAD_REQUEST, 0, 0); - return; - } + if (!reasoning_content.empty()) { + delta["choices"][0]["delta"]["reasoning_content"] = reasoning_content; + } - // ── Parse and validate segment table ────────────────────────────────── - const json & segments = req["segments"]; - const size_t n_segments = segments.size(); - if (n_segments > 3) { - SRV_WRN("hydra rpc: DECODE SEGMENT_TABLE_INVALID: too many segments (%zu)\n", n_segments); - hydra_write_res(fd, HYDRA_STATUS_BAD_REQUEST, 0, 0); - return; - } + if (!tool_calls.empty()) { + delta["choices"][0]["delta"]["tool_calls"] = tool_calls; + } - // Each segment: {"id":"prompt"|"kv", "offset":N, "len":N, "hash":"xxh3:HEX"} - uint64_t prompt_len = 0; - uint64_t kv_len = 0; - std::string prompt_hash_str; - std::string kv_hash_str; - uint64_t expected_offset = 0; - for (size_t i = 0; i < n_segments; i++) { - const json & seg = segments[i]; - if (!seg.contains("id") || !seg.contains("offset") || !seg.contains("len") || !seg.contains("hash")) { - SRV_WRN("hydra rpc: DECODE SEGMENT_TABLE_INVALID: segment %zu missing required fields\n", i); - hydra_write_res(fd, HYDRA_STATUS_BAD_REQUEST, 0, 0); - return; - } - const std::string id = seg["id"].get(); - const uint64_t offset = seg["offset"].get(); - const uint64_t len = seg["len"].get(); - const std::string hash = seg["hash"].get(); + if (include_usage) { + delta["usage"] = json { + {"completion_tokens", n_decoded}, + {"prompt_tokens", n_prompt_tokens}, + {"total_tokens", n_decoded + n_prompt_tokens}, + {"prompt_tokens_details", json{{"cached_tokens", n_prompt_tokens_cache}}}, + }; + } + if (!hydra_metrics.is_null()) { + delta["hydra_metrics"] = hydra_metrics; + } - if (offset != expected_offset) { - SRV_WRN("hydra rpc: DECODE SEGMENT_TABLE_INVALID: segment %zu offset=%" PRIu64 " expected=%" PRIu64 "\n", - i, offset, expected_offset); - hydra_write_res(fd, HYDRA_STATUS_BAD_REQUEST, 0, 0); - return; - } - expected_offset = offset + len; + res->status = 200; + res->content_type = "text/event-stream"; + res->data = format_oai_sse(delta); + } else { + // Buffered: full OAI chat completion response + json message; + message["role"] = "assistant"; + message["content"] = content; + if (!reasoning_content.empty()) { + message["reasoning_content"] = reasoning_content; + } + if (!tool_calls.empty()) { + message["tool_calls"] = tool_calls; + } - if (id == "prompt") { - prompt_len = len; - prompt_hash_str = hash; - } else if (id == "kv") { - kv_len = len; - kv_hash_str = hash; - } else { - SRV_WRN("hydra rpc: DECODE SEGMENT_TABLE_INVALID: unknown segment id '%s'\n", id.c_str()); - hydra_write_res(fd, HYDRA_STATUS_BAD_REQUEST, 0, 0); - return; - } - } + json choice { + {"finish_reason", stop == STOP_TYPE_WORD || stop == STOP_TYPE_EOS + ? (tool_calls.empty() ? "stop" : "tool_calls") + : "length"}, + {"index", 0}, + {"message", message}, + }; - // Verify total segment size matches remaining payload - const uint64_t segments_total = prompt_len + kv_len; - const uint64_t remaining_after_hdr = payload_len - sizeof(uint32_t) - sizeof(uint64_t) - hdr_len; - if (segments_total != remaining_after_hdr) { - SRV_WRN("hydra rpc: DECODE SEGMENT_TABLE_INVALID: segments total %" PRIu64 " != remaining %" PRIu64 "\n", - segments_total, remaining_after_hdr); - hydra_write_res(fd, HYDRA_STATUS_BAD_REQUEST, 0, 0); - return; - } + json oai_response { + {"choices", json::array({choice})}, + {"created", std::time(0)}, + {"model", oaicompat_model}, + {"system_fingerprint", std::string(llama_build_info())}, + {"object", "chat.completion"}, + {"usage", { + {"completion_tokens", n_decoded}, + {"prompt_tokens", n_prompt_tokens}, + {"total_tokens", n_decoded + n_prompt_tokens}, + {"prompt_tokens_details", json{{"cached_tokens", n_prompt_tokens_cache}}}, + }}, + {"id", completion_id}, + {"id_slot", id_slot}, + {"timings", timings.to_json()}, + }; + if (!hydra_metrics.is_null()) { + oai_response["hydra_metrics"] = hydra_metrics; + } - // Caps - if (prompt_len > HYDRA_MAX_PROMPT_BYTES) { - SRV_WRN("hydra rpc: DECODE PROMPT_TOO_LARGE %" PRIu64 " > %" PRIu64 "\n", - prompt_len, HYDRA_MAX_PROMPT_BYTES); - hydra_write_res(fd, HYDRA_STATUS_BAD_REQUEST, 0, 0); - return; - } - if (kv_len > HYDRA_MAX_STATE_BYTES) { - SRV_WRN("hydra rpc: DECODE KV_TOO_LARGE %" PRIu64 " > %" PRIu64 "\n", - kv_len, HYDRA_MAX_STATE_BYTES); - hydra_write_res(fd, HYDRA_STATUS_BAD_REQUEST, 0, 0); - return; - } + res->ok(oai_response); + } + return res; + } - // ── Read prompt segment ─────────────────────────────────────────────── - std::vector prompt_data((size_t)prompt_len); - if (prompt_len > 0 && !hydra_recv_all(fd, prompt_data.data(), (size_t)prompt_len)) { - hydra_write_res(fd, HYDRA_STATUS_ERROR, 0, 0); - return; - } + // Fallback: no content yet + res->error(format_error_response("decode_request_id not ready", ERROR_TYPE_NOT_FOUND)); + return res; + }; - // ── Read KV segment (may be zero-length) ────────────────────────────── - // M2 (#470): the KV blob (2.3 GB today, 10 GB target) is never materialized. - // The v2 blob header (small: version + n_past + n_tok + tokens + flags + - // checkpoint) is read here; the remaining stream (magic + seq_id + KV state - // + logits) stays on the fd and is consumed by the DECODE task thread via - // llama_state_seq_set_data_from_fd. The wire hash (xxh3-64 over the whole - // kv segment) is verified post-restore in the task thread — with streaming - // the bytes reach the GPU before a pre-restore hash could be computed. - uint64_t expected_kv_hash = 0; - bool has_kv_hash = false; - if (kv_len > 0 && !kv_hash_str.empty()) { - if (kv_hash_str.rfind("xxh3:", 0) != 0) { - SRV_WRN("hydra rpc: DECODE unsupported KV hash prefix: %s\n", kv_hash_str.c_str()); - hydra_write_res(fd, HYDRA_STATUS_BAD_REQUEST, 0, 0); - return; - } + this->delete_decode_result = [this](const server_http_req & req) { + auto res = create_response(true); + int32_t decode_request_id; try { - expected_kv_hash = std::stoull(kv_hash_str.substr(5), nullptr, 16); - has_kv_hash = true; + decode_request_id = std::stoi(req.get_param("decode_request_id")); } catch (const std::exception &) { - SRV_WRN("hydra rpc: DECODE invalid KV hash format: %s\n", kv_hash_str.c_str()); - hydra_write_res(fd, HYDRA_STATUS_BAD_REQUEST, 0, 0); - return; - } - } - - std::vector kv_data; // M1 fallback (legacy non-v2 blobs) - std::vector kv_v2_hdr; // M2: parsed v2 header (small) - uint64_t kv_stream_len = 0; // M2: bytes remaining on fd after the header - if (kv_len > 0) { - uint8_t version_byte = 0; - if (!hydra_recv_all(fd, &version_byte, 1)) { - hydra_write_res(fd, HYDRA_STATUS_ERROR, 0, 0); - return; + res->error(format_error_response("Invalid decode_request_id", ERROR_TYPE_INVALID_REQUEST)); + return res; } - if (version_byte != 0x02 && version_byte != 0x03) { - // Legacy non-v2/v3 blob — buffered M1 path (pre-#470 behavior). - // Replay the consumed version byte into the buffer. - kv_data.resize((size_t)kv_len); - kv_data[0] = version_byte; - if (kv_len > 1 && !hydra_recv_all(fd, kv_data.data() + 1, (size_t)kv_len - 1)) { - hydra_write_res(fd, HYDRA_STATUS_ERROR, 0, 0); - return; - } - if (has_kv_hash) { - const uint64_t computed_kv = XXH3_64bits(kv_data.data(), kv_data.size()); - if (computed_kv != expected_kv_hash) { - SRV_WRN("hydra rpc: DECODE SEGMENT_HASH_MISMATCH kv expected=%016" PRIx64 " got=%016" PRIx64 "\n", - expected_kv_hash, computed_kv); - hydra_write_res(fd, HYDRA_STATUS_BAD_REQUEST, 0, 0); - return; - } - SRV_INF("hydra rpc: DECODE KV hash verified (%" PRIu64 " B)\n", kv_len); - } - } else { - // M2: parse the v2 header incrementally (all small reads) and leave - // the state stream on the fd for the task thread. - kv_v2_hdr.push_back(version_byte); - if (kv_len < 9) { - SRV_WRN("hydra rpc: DECODE KV segment too small for v2 header (%" PRIu64 " B)\n", kv_len); - hydra_write_res(fd, HYDRA_STATUS_BAD_REQUEST, 0, 0); - return; - } - uint32_t n_past_in = 0, n_tok_in = 0; - if (!hydra_recv_all(fd, &n_past_in, 4) || !hydra_recv_all(fd, &n_tok_in, 4)) { - hydra_write_res(fd, HYDRA_STATUS_ERROR, 0, 0); - return; - } - kv_v2_hdr.insert(kv_v2_hdr.end(), (const uint8_t *)&n_past_in, (const uint8_t *)&n_past_in + 4); - kv_v2_hdr.insert(kv_v2_hdr.end(), (const uint8_t *)&n_tok_in, (const uint8_t *)&n_tok_in + 4); - - const size_t tokens_bytes = (size_t)n_tok_in * sizeof(llama_token); - if (9 + tokens_bytes + 1 > kv_len) { - SRV_WRN("hydra rpc: DECODE v2 header tokens exceed kv_len (%" PRIu64 " B)\n", kv_len); - hydra_write_res(fd, HYDRA_STATUS_BAD_REQUEST, 0, 0); - return; - } - std::vector tokens_buf(tokens_bytes); - if (tokens_bytes > 0 && !hydra_recv_all(fd, tokens_buf.data(), tokens_bytes)) { - hydra_write_res(fd, HYDRA_STATUS_ERROR, 0, 0); - return; - } - kv_v2_hdr.insert(kv_v2_hdr.end(), tokens_buf.begin(), tokens_buf.end()); - - uint8_t flags = 0; - if (!hydra_recv_all(fd, &flags, 1)) { - hydra_write_res(fd, HYDRA_STATUS_ERROR, 0, 0); - return; - } - kv_v2_hdr.push_back(flags); - if (flags & 0x01) { - // Checkpoint: 4B pos_min | 4B pos_max | 8B n_tokens | 8B tgt_sz | - // tgt_data | 8B dft_sz | dft_data (mirrors the DECODE_APPLY parse). - if (kv_v2_hdr.size() + 24 > kv_len) { - SRV_WRN("hydra rpc: DECODE v2 checkpoint header exceeds kv_len (%" PRIu64 " B)\n", kv_len); - hydra_write_res(fd, HYDRA_STATUS_BAD_REQUEST, 0, 0); - return; - } - uint8_t ckpt_fixed[24]; // pos_min + pos_max + n_tokens + tgt_sz - if (!hydra_recv_all(fd, ckpt_fixed, sizeof(ckpt_fixed))) { - hydra_write_res(fd, HYDRA_STATUS_ERROR, 0, 0); - return; - } - kv_v2_hdr.insert(kv_v2_hdr.end(), ckpt_fixed, ckpt_fixed + sizeof(ckpt_fixed)); - uint64_t tgt_sz_in = 0; - memcpy(&tgt_sz_in, ckpt_fixed + 16, 8); - if (kv_v2_hdr.size() + (size_t)tgt_sz_in + 8 > kv_len) { - SRV_WRN("hydra rpc: DECODE v2 checkpoint payload exceeds kv_len (%" PRIu64 " B)\n", kv_len); - hydra_write_res(fd, HYDRA_STATUS_BAD_REQUEST, 0, 0); - return; - } - if (tgt_sz_in > 0) { - std::vector tgt_buf((size_t)tgt_sz_in); - if (!hydra_recv_all(fd, tgt_buf.data(), (size_t)tgt_sz_in)) { - hydra_write_res(fd, HYDRA_STATUS_ERROR, 0, 0); - return; - } - kv_v2_hdr.insert(kv_v2_hdr.end(), tgt_buf.begin(), tgt_buf.end()); - } - uint8_t dft_sz_buf[8]; - if (!hydra_recv_all(fd, dft_sz_buf, sizeof(dft_sz_buf))) { - hydra_write_res(fd, HYDRA_STATUS_ERROR, 0, 0); - return; - } - kv_v2_hdr.insert(kv_v2_hdr.end(), dft_sz_buf, dft_sz_buf + sizeof(dft_sz_buf)); - uint64_t dft_sz_in = 0; - memcpy(&dft_sz_in, dft_sz_buf, 8); - if (kv_v2_hdr.size() + (size_t)dft_sz_in > kv_len) { - SRV_WRN("hydra rpc: DECODE v2 checkpoint payload exceeds kv_len (%" PRIu64 " B)\n", kv_len); - hydra_write_res(fd, HYDRA_STATUS_BAD_REQUEST, 0, 0); - return; - } - if (dft_sz_in > 0) { - std::vector dft_buf((size_t)dft_sz_in); - if (!hydra_recv_all(fd, dft_buf.data(), (size_t)dft_sz_in)) { - hydra_write_res(fd, HYDRA_STATUS_ERROR, 0, 0); - return; - } - kv_v2_hdr.insert(kv_v2_hdr.end(), dft_buf.begin(), dft_buf.end()); - } + int32_t id_slot = -1; + int32_t task_id_to_cancel = -1; + { + std::unique_lock lock(decode_results_mutex); + auto it = decode_results.find(decode_request_id); + if (it == decode_results.end()) { + lock.unlock(); + res->error(format_error_response("decode_request_id not found or expired", ERROR_TYPE_NOT_FOUND)); + return res; } + id_slot = it->second.id_slot; + task_id_to_cancel = it->second.stream ? it->second.stream->completion_task_id.load() : -1; - kv_stream_len = kv_len - kv_v2_hdr.size(); - // The stream must at least hold the [4B magic][4B seq_id] framing. - if (kv_stream_len < 8) { - SRV_WRN("hydra rpc: DECODE v2 stream too small (%" PRIu64 " B after header)\n", kv_stream_len); - hydra_write_res(fd, HYDRA_STATUS_BAD_REQUEST, 0, 0); - return; + // Signal streaming queue to finish so any waiting GET handler unblocks + if (it->second.stream) { + std::lock_guard slk(it->second.stream->streaming_mutex); + it->second.stream->stream_finished = true; + it->second.stream->streaming_cv.notify_all(); } - } - } - // ── Build decode_json from control header + prompt segment ───────────── - // The prompt JSON segment may contain { "prompt": "..." } or { "messages": [...] } - // Merge it into the control header as decode_req["prompt"]. - // Also merge generation params from control header's "generation" key. - json decode_req = req; // control header already has kv_metadata, model, etc. - json prompt_obj; - if (prompt_len > 0) { - try { - prompt_obj = json::parse(std::string(prompt_data.begin(), prompt_data.end())); - } catch (const std::exception & e) { - SRV_WRN("hydra rpc: DECODE invalid prompt segment JSON: %s\n", e.what()); - hydra_write_res(fd, HYDRA_STATUS_BAD_REQUEST, 0, 0); - return; + decode_results.erase(it); } - } - // The coordinator sends the prompt segment as the BARE messages array - // (item.Request["messages"].ToString()). The generation-merge below and - // DECODE_APPLY's chat-template path both expect an OBJECT with a - // "messages" key — merging generation keys into an array throws - // nlohmann::type_error, which was silently swallowed by the RPC worker - // pool (the connection leaked, no response written, coordinator timed out - // after 180s). Wrap a bare array so the prompt object matches the - // downstream contract. - if (prompt_obj.is_array()) { - json wrapped; - wrapped["messages"] = std::move(prompt_obj); - prompt_obj = std::move(wrapped); - } - // Merge generation params from control header into prompt object - std::string decode_json_str; - try { - if (req.contains("generation") && req["generation"].is_object()) { - const json & gen = req["generation"]; - for (auto it = gen.begin(); it != gen.end(); ++it) { - if (!prompt_obj.contains(it.key())) { - prompt_obj[it.key()] = it.value(); - } - } + + // Deterministically cancel the running completion task via the task queue. + // SERVER_TASK_TYPE_CANCEL causes the inference thread to release the slot. + if (task_id_to_cancel > 0) { + server_task cancel_task(SERVER_TASK_TYPE_CANCEL); + cancel_task.id = queue_tasks.get_new_id(); + cancel_task.id_target = task_id_to_cancel; + queue_tasks.post(std::move(cancel_task), true); + SRV_INF("hydra: DECODE_CANCEL id=%d slot=%d completion_task=%d (cancel posted)\n", + decode_request_id, id_slot, task_id_to_cancel); + } else { + SRV_INF("hydra: DECODE_CANCEL id=%d slot=%d (no active completion)\n", + decode_request_id, id_slot); } - decode_req["prompt"] = std::move(prompt_obj); - decode_json_str = decode_req.dump(); - } catch (const std::exception & e) { - // Never let a malformed prompt object leak the connection: the worker - // pool swallows exceptions and the fd stays open with no response, - // hanging the coordinator until its own timeout. Always write an - // error frame so the caller sees a terminal (retryable-free) result. - SRV_WRN("hydra rpc: DECODE prompt build failed (slot %d): %s\n", slot_id, e.what()); - json err_j = { - {"error", std::string("prompt build failed: ") + e.what()}, - {"decode_request_id", -1}, - }; - const std::string err_str = err_j.dump(); - hydra_write_res(fd, HYDRA_STATUS_BAD_REQUEST, (uint32_t) err_str.size(), 0); - hydra_send_all(fd, err_str.data(), err_str.size()); - return; - } + res->ok(json{{"cancelled", true}, {"decode_request_id", decode_request_id}}); + return res; + }; +} - // ── Phase 1: sync validate + restore ────────────────────────────────── - const int32_t decode_request_id = ctx.queue_tasks->get_new_id(); - - server_task val_task(SERVER_TASK_TYPE_HYDRA_ENGINE_DECODE); - val_task.id = decode_request_id; - val_task.hydra_action.id_slot = slot_id; - val_task.hydra_action.decode_json = std::move(decode_json_str); - val_task.hydra_action.kv_data = std::move(kv_data); - // M2 (#470): the KV state stream stays on the fd and is consumed by the - // task thread (llama_state_seq_set_data_from_fd) — no full-blob buffer. - val_task.hydra_action.hydra_fd = fd; - val_task.hydra_action.kv_v2_hdr = std::move(kv_v2_hdr); - val_task.hydra_action.kv_stream_len = kv_stream_len; - val_task.hydra_action.kv_expected_hash = has_kv_hash ? expected_kv_hash : 0; - val_task.hydra_action.decode_request_id = decode_request_id; - ctx.queue_results->add_waiting_task_id(decode_request_id); - ctx.queue_tasks->wait_until_no_sleep(); - ctx.queue_tasks->post(std::move(val_task)); - - // Wait for validation+restore to complete. Raised 30s -> 600s (#470): the - // M2 restore streams the whole KV blob (2.3 GB today, 10 GB target) off the - // fd inside the task, so compute + transfer must fit the wait. The - // Coordinator enforces its own idle-based client budget. - std::unordered_set val_ids = {decode_request_id}; - auto val_res_ptr = ctx.queue_results->recv_with_timeout(val_ids, 600); - ctx.queue_results->remove_waiting_task_id(decode_request_id); - - if (!val_res_ptr) { - SRV_WRN("hydra rpc: DECODE validation timeout for slot %d (request_id=%d)\n", - slot_id, decode_request_id); - // #470: same M2 caveat as STATE_GET/PREFILL — the restore task may own - // the fd mid-stream; shut the socket down instead of writing an error - // header that could interleave. Client unblocks with a clean EOF, the - // worker frees immediately. - ::shutdown(fd, SHUT_RDWR); - return; - } +json server_routes::get_model_info() const { + std::shared_lock meta_lock(meta_mutex); - auto * val_res = dynamic_cast(val_res_ptr.get()); - if (!val_res || val_res->rpc_status != HYDRA_STATUS_OK) { - json err_j = { - {"valid", false}, - {"decode_request_id", decode_request_id}, - }; - if (val_res) { - if (!val_res->match_json.is_null()) err_j["match"] = val_res->match_json; - if (!val_res->error.empty()) err_j["reason"] = val_res->error; - err_j["error_code"] = "CAP_MISMATCH"; - } - const std::string err_str = err_j.dump(); - hydra_write_res(fd, HYDRA_STATUS_ERROR, (uint32_t)err_str.size(), 0); - hydra_send_all(fd, err_str.data(), err_str.size()); - return; + // P0-1 (#49): null-meta guard — called from get_models and other paths + if (!meta) { + return json{{"error", "model not loaded — waiting for CONFIGURE"}}; } - // Validation passed — build real success response - json meta_j = { - {"valid", true}, - {"match", val_res->match_json}, - {"decode_request_id", decode_request_id}, - {"n_past_after_restore", val_res->n_past}, - {"restore_slot_ms", val_res->restore_slot_ms}, + return json { + {"id", meta->model_name}, + {"aliases", meta->model_aliases}, + {"tags", meta->model_tags}, + {"object", "model"}, + {"created", std::time(0)}, + {"owned_by", "llamacpp"}, + {"meta", { + {"vocab_type", meta->model_vocab_type}, + {"n_vocab", meta->model_vocab_n_tokens}, + {"n_ctx", meta->slot_n_ctx}, + {"n_ctx_train", meta->model_n_ctx_train}, + {"n_embd", meta->model_n_embd_inp}, + {"n_params", meta->model_n_params}, + {"size", meta->model_size}, + }}, }; - const std::string meta_str = meta_j.dump(); - hydra_write_res(fd, HYDRA_STATUS_OK, (uint32_t)meta_str.size(), 0); - hydra_send_all(fd, meta_str.data(), meta_str.size()); - - SRV_INF("hydra: DECODE slot=%d accepted, request_id=%d, restore=%.1fms\n", - slot_id, decode_request_id, val_res->restore_slot_ms); } -// SET_EXPERT_MODE (0x37): Read mode string, post task, return success. -static void hydra_handle_set_expert_mode(int fd, int slot_id, uint64_t payload_len, const hydra_rpc_ctx & ctx) { - std::string mode(payload_len, '\0'); - if (payload_len > 0 && !hydra_recv_all(fd, mode.data(), payload_len)) { - hydra_write_res(fd, HYDRA_STATUS_ERROR, 0, 0); - return; +std::unique_ptr server_routes::handle_slots_save(const server_http_req & req, int id_slot) { + auto res = create_response(); + const json request_data = json::parse(req.body); + std::string filename = request_data.at("filename"); + if (!fs_validate_filename(filename)) { + res->error(format_error_response("Invalid filename", ERROR_TYPE_INVALID_REQUEST)); + return res; + } + std::string filepath = params.slot_save_path + filename; + + auto & rd = res->rd; + { + server_task task(SERVER_TASK_TYPE_SLOT_SAVE); + task.id = rd.get_new_id(); + task.slot_action.id_slot = id_slot; + task.slot_action.filename = filename; + task.slot_action.filepath = filepath; + rd.post_task(std::move(task)); } - server_task task(SERVER_TASK_TYPE_HYDRA_ENGINE_SET_EXPERT_MODE); - task.id = ctx.queue_tasks->get_new_id(); - task.hydra_action.id_slot = slot_id; - task.hydra_action.expert_mode = std::move(mode); - const int task_id = task.id; - ctx.queue_results->add_waiting_task_id(task_id); - ctx.queue_tasks->wait_until_no_sleep(); - ctx.queue_tasks->post(std::move(task)); - - std::unordered_set task_ids = {task_id}; - auto res_ptr = ctx.queue_results->recv_with_timeout(task_ids, 5); - ctx.queue_results->remove_waiting_task_id(task_id); - if (!res_ptr) { - hydra_write_res(fd, HYDRA_STATUS_ERROR, 0, 0); - return; + auto result = rd.next(req.should_stop); + if (!result) { + // connection was closed + GGML_ASSERT(req.should_stop()); + return res; } - auto * res = dynamic_cast(res_ptr.get()); - if (!res || !res->success) { - const std::string err = (res && !res->error.empty()) ? res->error : std::string(); - json err_j = {{"success", false}}; - if (!err.empty()) err_j["error"] = err; - const std::string err_str = err_j.dump(); - hydra_write_res(fd, HYDRA_STATUS_ERROR, (uint32_t)err_str.size(), 0); - hydra_send_all(fd, err_str.data(), err_str.size()); - return; + if (result->is_error()) { + res->error(result->to_json()); + return res; } - // Report the ACTUAL mode applied (may be "solo" even though "combined" was - // requested, if this engine never dual-loaded combined experts) — the - // Coordinator's ReportsSolo() reads this key to detect the fallback. - json meta_j = {{"success", true}, {"mode", res->expert_mode_applied}}; - const std::string meta_str = meta_j.dump(); - hydra_write_res(fd, HYDRA_STATUS_OK, (uint32_t)meta_str.size(), 0); - hydra_send_all(fd, meta_str.data(), meta_str.size()); + res->ok(result->to_json()); + return res; } -// SWAP_QUANT (0x38): Read quant_key + tensor_pattern, post task, return success. -static void hydra_handle_swap_quant(int fd, int slot_id, uint64_t payload_len, const hydra_rpc_ctx & ctx) { - if (payload_len < sizeof(uint16_t)) { - hydra_write_res(fd, HYDRA_STATUS_ERROR, 0, 0); - return; +std::unique_ptr server_routes::handle_slots_restore(const server_http_req & req, int id_slot) { + auto res = create_response(); + const json request_data = json::parse(req.body); + std::string filename = request_data.at("filename"); + if (!fs_validate_filename(filename)) { + res->error(format_error_response("Invalid filename", ERROR_TYPE_INVALID_REQUEST)); + return res; + } + std::string filepath = params.slot_save_path + filename; + + auto & rd = res->rd; + { + server_task task(SERVER_TASK_TYPE_SLOT_RESTORE); + task.id = rd.get_new_id(); + task.slot_action.id_slot = id_slot; + task.slot_action.filename = filename; + task.slot_action.filepath = filepath; + rd.post_task(std::move(task)); } - uint16_t quant_key_len = 0; - if (!hydra_recv_all(fd, &quant_key_len, sizeof(quant_key_len))) { - hydra_write_res(fd, HYDRA_STATUS_ERROR, 0, 0); - return; + auto result = rd.next(req.should_stop); + if (!result) { + // connection was closed + GGML_ASSERT(req.should_stop()); + return res; } - std::string quant_key(quant_key_len, '\0'); - if (quant_key_len > 0 && !hydra_recv_all(fd, quant_key.data(), quant_key_len)) { - hydra_write_res(fd, HYDRA_STATUS_ERROR, 0, 0); - return; + if (result->is_error()) { + res->error(result->to_json()); + return res; } - const uint64_t pattern_len = payload_len - sizeof(uint16_t) - quant_key_len; - std::string tensor_pattern(pattern_len, '\0'); - if (pattern_len > 0 && !hydra_recv_all(fd, tensor_pattern.data(), pattern_len)) { - hydra_write_res(fd, HYDRA_STATUS_ERROR, 0, 0); - return; + GGML_ASSERT(dynamic_cast(result.get()) != nullptr); + res->ok(result->to_json()); + return res; +} + +std::unique_ptr server_routes::handle_slots_erase(const server_http_req & req, int id_slot) { + auto res = create_response(); + auto & rd = res->rd; + { + server_task task(SERVER_TASK_TYPE_SLOT_ERASE); + task.id = rd.get_new_id(); + task.slot_action.id_slot = id_slot; + rd.post_task(std::move(task)); } - server_task task(SERVER_TASK_TYPE_HYDRA_ENGINE_SWAP_QUANT); - task.id = ctx.queue_tasks->get_new_id(); - task.hydra_action.id_slot = slot_id; - task.hydra_action.quant_key = std::move(quant_key); - task.hydra_action.tensor_pattern = std::move(tensor_pattern); - const int task_id = task.id; - ctx.queue_results->add_waiting_task_id(task_id); - ctx.queue_tasks->wait_until_no_sleep(); - ctx.queue_tasks->post(std::move(task)); - - std::unordered_set task_ids = {task_id}; - auto res_ptr = ctx.queue_results->recv_with_timeout(task_ids, 30); - ctx.queue_results->remove_waiting_task_id(task_id); - if (!res_ptr) { - hydra_write_res(fd, HYDRA_STATUS_ERROR, 0, 0); - return; + auto result = rd.next(req.should_stop); + if (!result) { + // connection was closed + GGML_ASSERT(req.should_stop()); + return res; } - auto * res = dynamic_cast(res_ptr.get()); - if (!res || !res->success) { - hydra_write_res(fd, HYDRA_STATUS_ERROR, 0, 0); - return; + if (result->is_error()) { + res->error(result->to_json()); + return res; } - json meta_j = {{"success", true}}; - const std::string meta_str = meta_j.dump(); - hydra_write_res(fd, HYDRA_STATUS_OK, (uint32_t)meta_str.size(), 0); - hydra_send_all(fd, meta_str.data(), meta_str.size()); + GGML_ASSERT(dynamic_cast(result.get()) != nullptr); + res->ok(result->to_json()); + return res; } -// PIPELINE_ATTACH (0x46): M-Perf.9 (#289) / issue #287 — two-engine "work -// together" routing scaffolding. The C# Coordinator sends the peer address -// and the --override-tensor regex; the engine should load the assigned -// tensor slice from its OWN local model (no weight transfer). This opcode -// is stubbed for now (returns NOT_IMPLEMENTED) — full implementation is -// tracked under issue #287. -static void hydra_handle_pipeline_attach(int fd, int slot_id, uint64_t payload_len, const hydra_rpc_ctx & ctx) { - std::string json_body(payload_len, '\0'); - if (payload_len > 0 && !hydra_recv_all(fd, json_body.data(), payload_len)) { - hydra_write_res(fd, HYDRA_STATUS_ERROR, 0, 0); - return; +std::unique_ptr server_routes::handle_embeddings_impl(const server_http_req & req, task_response_type res_type) { + std::shared_lock meta_lock(meta_mutex); + + auto res = create_response(); + // P0-1 (#49): null-meta guard + if (!meta) { + res->error(format_error_response("model not loaded — waiting for CONFIGURE", + ERROR_TYPE_NOT_SUPPORTED)); + return res; + } + if (!params.embedding) { + res->error(format_error_response("This server does not support embeddings. Start it with `--embeddings`", ERROR_TYPE_NOT_SUPPORTED)); + return res; } - server_task task(SERVER_TASK_TYPE_HYDRA_ENGINE_PIPELINE_ATTACH); - task.id = ctx.queue_tasks->get_new_id(); - task.hydra_action.id_slot = slot_id; - task.hydra_action.request_json = std::move(json_body); - const int task_id = task.id; - ctx.queue_results->add_waiting_task_id(task_id); - ctx.queue_tasks->wait_until_no_sleep(); - ctx.queue_tasks->post(std::move(task)); - - std::unordered_set task_ids = {task_id}; - auto res_ptr = ctx.queue_results->recv_with_timeout(task_ids, 5); - ctx.queue_results->remove_waiting_task_id(task_id); - if (!res_ptr) { - hydra_write_res(fd, HYDRA_STATUS_ERROR, 0, 0); - return; + if (res_type != TASK_RESPONSE_TYPE_NONE && meta->pooling_type == LLAMA_POOLING_TYPE_NONE) { + res->error(format_error_response("Pooling type 'none' is not OAI compatible. Please use a different pooling type", ERROR_TYPE_INVALID_REQUEST)); + return res; } - auto * res = dynamic_cast(res_ptr.get()); - // Stubbed: server returns NOT_IMPLEMENTED until issue #287 lands. - // Propagate that status to the client so the Coordinator can - // distinguish "not yet built" from a real error and fall back to solo. - const uint8_t status = (res && res->rpc_status == HYDRA_STATUS_NOT_IMPLEMENTED) - ? HYDRA_STATUS_NOT_IMPLEMENTED : HYDRA_STATUS_ERROR; - json meta_j; - if (res && !res->error.empty()) meta_j["error"] = res->error; - meta_j["success"] = res && res->success; - const std::string meta_str = meta_j.dump(); - hydra_write_res(fd, status, (uint32_t)meta_str.size(), 0); - hydra_send_all(fd, meta_str.data(), meta_str.size()); -} + const json body = json::parse(req.body); -// ── Per-connection loop ─────────────────────────────────────────────────────── -// Persistent: one TCP connection handles many sequential requests. - -static void hydra_handle_connection(int fd, const hydra_rpc_ctx & ctx) { - // Set receive timeout to prevent hung connections on stalled clients - struct timeval tv; - tv.tv_sec = 120; // 2 min inactivity timeout - tv.tv_usec = 0; - (void)setsockopt(fd, SOL_SOCKET, SO_RCVTIMEO, &tv, sizeof(tv)); - while (true) { - uint8_t hdr[HYDRA_REQ_HEADER_SIZE]; - if (!hydra_recv_all(fd, hdr, HYDRA_REQ_HEADER_SIZE)) break; - - uint16_t magic = 0; - memcpy(&magic, hdr + 0, 2); - if (magic != HYDRA_MAGIC) { - SRV_WRN("hydra rpc: bad magic 0x%04x — closing connection\n", (unsigned)magic); - break; - } - - const uint8_t op = hdr[2]; - // hdr[3] = flags (reserved, unused in M1) - uint16_t key_len = 0, trace_len = 0; - uint64_t payload_len = 0; - memcpy(&key_len, hdr + 4, 2); - memcpy(&payload_len, hdr + 6, 8); - memcpy(&trace_len, hdr + 14, 2); - - std::string key(key_len, '\0'); - std::string trace_id(trace_len, '\0'); - if (!hydra_recv_all(fd, key.data(), key_len)) break; - if (!hydra_recv_all(fd, trace_id.data(), trace_len)) break; - - // Slot-key parsing: engine-level opcodes (INFO, CONFIGURE, SET_EXPERT_MODE, - // SWAP_QUANT) don't need a valid slot — use slot_id = 0 when the key is - // empty or invalid. Slot-level opcodes (STATE_GET, STATE_PUT, STATE_META, - // PREFILL, DECODE) still require a valid integer key. - int slot_id = -1; - bool is_engine_level_op = (op == HYDRA_OP_INFO || op == HYDRA_OP_CONFIGURE || - op == HYDRA_OP_SET_EXPERT_MODE || op == HYDRA_OP_SWAP_QUANT); - if (key.empty() && is_engine_level_op) { - slot_id = 0; - } else { - try { slot_id = std::stoi(key); } - catch (...) { - SRV_WRN("hydra rpc: invalid slot key '%s'\n", key.c_str()); - hydra_write_res(fd, HYDRA_STATUS_ERROR, 0, 0); - continue; - } + // for the shape of input/content, see tokenize_input_prompts() + json prompt; + if (body.count("input") != 0) { + prompt = body.at("input"); + } else if (body.contains("content")) { + res_type = TASK_RESPONSE_TYPE_NONE; // "content" field is not OAI compatible + prompt = body.at("content"); + } else { + res->error(format_error_response("\"input\" or \"content\" must be provided", ERROR_TYPE_INVALID_REQUEST)); + return res; + } + + bool use_base64 = false; + if (body.count("encoding_format") != 0) { + const std::string & format = body.at("encoding_format"); + if (format == "base64") { + use_base64 = true; + } else if (format != "float") { + res->error(format_error_response("The format to return the embeddings in. Can be either float or base64", ERROR_TYPE_INVALID_REQUEST)); + return res; } + } - // Dispatch to handler via task queue (no direct slot access) - switch (op) { - case HYDRA_OP_STATE_GET: - SRV_DBG("hydra rpc: STATE_GET slot=%d trace=%s\n", slot_id, trace_id.c_str()); - hydra_handle_state_get(fd, slot_id, ctx); - break; - case HYDRA_OP_STATE_PUT: - SRV_DBG("hydra rpc: STATE_PUT slot=%d payload=%" PRIu64 " trace=%s\n", - slot_id, payload_len, trace_id.c_str()); - hydra_handle_state_put(fd, slot_id, payload_len, ctx); - break; - case HYDRA_OP_STATE_META: - SRV_DBG("hydra rpc: STATE_META slot=%d trace=%s\n", slot_id, trace_id.c_str()); - hydra_handle_state_meta(fd, slot_id, ctx); - break; - case HYDRA_OP_CONFIGURE: - SRV_DBG("hydra rpc: CONFIGURE slot=%d payload=%" PRIu64 " trace=%s\n", - slot_id, payload_len, trace_id.c_str()); - hydra_handle_configure(fd, slot_id, payload_len, ctx); - break; - case HYDRA_OP_INFO: - SRV_DBG("hydra rpc: INFO slot=%d trace=%s\n", slot_id, trace_id.c_str()); - hydra_handle_info(fd, slot_id, ctx); - break; - case HYDRA_OP_PREFILL: - SRV_DBG("hydra rpc: PREFILL slot=%d payload=%" PRIu64 " trace=%s\n", - slot_id, payload_len, trace_id.c_str()); - hydra_handle_prefill(fd, slot_id, payload_len, ctx); - break; - case HYDRA_OP_DECODE: - SRV_DBG("hydra rpc: DECODE slot=%d payload=%" PRIu64 " trace=%s\n", - slot_id, payload_len, trace_id.c_str()); - hydra_handle_decode(fd, slot_id, payload_len, ctx); - break; - case HYDRA_OP_SET_EXPERT_MODE: - SRV_DBG("hydra rpc: SET_EXPERT_MODE slot=%d payload=%" PRIu64 " trace=%s\n", - slot_id, payload_len, trace_id.c_str()); - hydra_handle_set_expert_mode(fd, slot_id, payload_len, ctx); - break; - case HYDRA_OP_SWAP_QUANT: - SRV_DBG("hydra rpc: SWAP_QUANT slot=%d payload=%" PRIu64 " trace=%s\n", - slot_id, payload_len, trace_id.c_str()); - hydra_handle_swap_quant(fd, slot_id, payload_len, ctx); - break; - // M-Perf.9 (#289) / issue #287: PIPELINE_ATTACH (0x46) is the - // two-engine "work together" attach. Stubbed: full impl in #287. - case HYDRA_OP_PIPELINE_ATTACH: - SRV_DBG("hydra rpc: PIPELINE_ATTACH slot=%d payload=%" PRIu64 " trace=%s\n", - slot_id, payload_len, trace_id.c_str()); - hydra_handle_pipeline_attach(fd, slot_id, payload_len, ctx); - break; - default: - SRV_WRN("hydra rpc: unknown op 0x%02x — ignoring\n", (unsigned)op); - hydra_write_res(fd, HYDRA_STATUS_ERROR, 0, 0); + auto tokenized_prompts = tokenize_input_prompts(ctx_server.vocab, ctx_server.mctx, prompt, true, true); + for (const auto & tokens : tokenized_prompts) { + // this check is necessary for models that do not add BOS token to the input + if (tokens.empty()) { + res->error(format_error_response("Input content cannot be empty", ERROR_TYPE_INVALID_REQUEST)); + return res; } } - ::close(fd); -} -// ── Unified RPC server implementation ─────────────────────────────────────── -// -// `#36` Phase 1: the merged server lives in `tools/llama-engine/hydra_rpc/` -// (fork-isolated). `server_context::start_rpc_server` is a thin adapter that -// builds the settings and delegates to `hydra_rpc::start()`. The Hydra -// protocol entry `hydra_handle_connection` is reached through the -// `hydra_rpc_bridge` trampoline (defined below) — the bridge takes a -// `void*` so the new module can stay decoupled from this file's includes. + int embd_normalize = params.embd_normalize; + if (body.count("embd_normalize") != 0) { + embd_normalize = body.at("embd_normalize"); + if (meta->pooling_type == LLAMA_POOLING_TYPE_NONE) { + SRV_DBG("embd_normalize is not supported by pooling type %d, ignoring it\n", meta->pooling_type); + } + } -#include "../llama-engine/hydra_rpc/hydra_rpc.h" + // create and queue the task + json responses = json::array(); + auto & rd = res->rd; + { + std::vector tasks; + for (size_t i = 0; i < tokenized_prompts.size(); i++) { + server_task task = server_task(SERVER_TASK_TYPE_EMBEDDING); -void server_context::start_rpc_server(int port, - std::vector backends) { - if (port <= 0) return; - - // Hydra #43: MUST outlive this function. `hydra_rpc::start()` below - // stores `&ctx` as a raw pointer inside `hydra_rpc::state()`, a - // process-lifetime singleton that every subsequent RPC connection reads - // (from a bounded-thread-pool worker thread) to recover queue_tasks / - // queue_results. An automatic-storage `ctx` here would dangle the - // instant this function returns — a stack-use-after-return that "works" - // until the freed stack slot gets reused, then silently corrupts the - // RPC response path. `start_rpc_server` only ever runs once per process - // (hydra_rpc::start() itself guards double-start), so `static` gives it - // exactly the lifetime the singleton needs. - static hydra_rpc_ctx ctx{}; - if (impl) { - ctx.queue_tasks = &impl->queue_tasks; - ctx.queue_results = &impl->queue_results; - } + task.id = rd.get_new_id(); + task.tokens = std::move(tokenized_prompts[i]); + + // OAI-compat + task.params.res_type = res_type; + task.params.embd_normalize = embd_normalize; - hydra_rpc::settings s; - s.port = port; - s.backends = std::move(backends); - s.hydra_ctx = (ctx.queue_tasks && ctx.queue_results) ? &ctx : nullptr; - s.pool_size = 2; - s.max_queue = 64; - s.host = "0.0.0.0"; - - if (!hydra_rpc::start(s)) { - SRV_ERR("hydra rpc: start() failed on port %d\n", port); - return; + tasks.push_back(std::move(task)); + } + rd.post_tasks(std::move(tasks)); } - if (s.hydra_ctx) { - SRV_INF("hydra rpc: unified server on 0.0.0.0:%d (ggml-RPC + Hydra protocol)\n", port); + // wait for the results + auto all_results = rd.wait_for_all(req.should_stop); + + // collect results + if (all_results.is_terminated) { + return res; // connection is closed + } else if (all_results.error) { + res->error(all_results.error->to_json()); + return res; } else { - SRV_INF("hydra rpc: unified server on 0.0.0.0:%d (ggml-RPC only)\n", port); + for (auto & res : all_results.results) { + GGML_ASSERT(dynamic_cast(res.get()) != nullptr); + responses.push_back(res->to_json()); + } } -} -// `hydra_rpc_bridge` — extern "C" trampoline. `hydra_rpc.cpp` calls this -// when the first byte on a new connection is not `RPC_CMD_HELLO`. It -// re-enters the C++ entry point with the typed `hydra_rpc_ctx &`. -// -// Forward-declared with the matching signature so the new -// `tools/llama-engine/hydra_rpc/hydra_rpc.cpp` module can take its -// address without including this heavy header. -extern "C" void hydra_rpc_bridge(int fd, const void * ctx); -extern "C" void hydra_rpc_bridge(int fd, const void * ctx) { - hydra_handle_connection(fd, *static_cast(ctx)); + // write JSON response + json root = res_type == TASK_RESPONSE_TYPE_OAI_EMBD + ? format_embeddings_response_oaicompat(body, meta->model_name, responses, use_base64) + : json(responses); + res->ok(root); + return res; } -#else -// Windows: RPC server not implemented — target hardware is Linux-only for M0. -void server_context::start_rpc_server(int port, std::vector) { - if (port > 0) { - SRV_WRN("hydra rpc: not supported on Windows (port %d ignored)\n", port); - } - GGML_UNUSED(port); -} -#endif // !_WIN32 + +// epic #610 WS1: Hydra extension implementation. Compiled INTO this TU so the +// concrete class can reach server_context_impl private members via the friend +// declaration above. Do NOT add hydra-server-context.cpp to CMakeLists.txt. +#include "hydra-server-context.cpp" diff --git a/tools/server/server-hydra-extension.h b/tools/server/server-hydra-extension.h new file mode 100644 index 000000000000..cc1b0adacd5f --- /dev/null +++ b/tools/server/server-hydra-extension.h @@ -0,0 +1,52 @@ +// Hydra A/B extension seam (epic #610). +// +// server-context.cpp consults this interface at a few well-defined points so +// Hydra-specific behavior can live in a fork-owned file (hydra-server-context.cpp) +// instead of being woven into upstream server-context.cpp. The concrete +// implementation is #include'd at the bottom of server-context.cpp (same +// translation unit) so it can reach server_context_impl's private members via +// the friend declaration on hydra_engine_extension. +// +// A/B toggle: the HYDRA_EXT_MODE env var selects which implementation drives +// Hydra behavior at runtime, so the SAME binary can be A/B tested: +// HYDRA_EXT_MODE=seam -> the extension (default since WS5) +// HYDRA_EXT_MODE=legacy -> the inline Hydra code in server-context.cpp +// Both paths stay compiled; only one is consulted per run. WS4 diffs the same +// scenario through both modes to prove the refactor is behavior-identical. +#pragma once + +#include +#include +#include + +struct server_context_impl; +struct server_task; + +// True when HYDRA_EXT_MODE is NOT "legacy" (i.e. default = seam since WS5). +inline bool hydra_ext_mode_seam() { + const char * m = std::getenv("HYDRA_EXT_MODE"); + return !(m && std::strcmp(m, "legacy") == 0); +} + +struct server_hydra_extension { + virtual ~server_hydra_extension() = default; + + // Human-readable name of the active implementation (for A/B logging/tests). + virtual const char * name() const = 0; + + // Claim a task from process_single_task(). Return true if fully handled + // (the default dispatch is skipped). The task must NOT be consumed when + // returning false. + virtual bool handle_task(server_context_impl & impl, server_task & task) = 0; + + // Called at the top of update_slots(). Return true to skip the default + // decode loop for this pass (T3 rebuild / CONFIGURE / COMBINED reattach). + virtual bool pre_loop(server_context_impl & impl) = 0; + + // Called when update_slots() finds batch.n_tokens == 0. Return true if the + // empty-batch case was fully handled (STATE_GET transfer suppression). + virtual bool on_empty_batch(server_context_impl & impl) = 0; +}; + +// Factory. Defined in hydra-server-context.cpp. +std::unique_ptr hydra_create_extension();