Commit 8c07cee
Changed files (3)
ext
elelem
llama
ext/elelem/llama/elelem_llama.cpp
@@ -2,16 +2,19 @@
// inference from Ruby (Fiddle). All volatile C++/by-value types are handled here,
// compiled against the vendored headers, so Ruby only sees simple signatures:
// JSON in (OpenAI-style messages + tools), JSON out (content + tool_calls). The
-// model + chat templates are opened once and kept resident on an opaque handle; a
-// fresh context per generate keeps each request statelessly isolated.
+// model, chat templates and context are opened once and kept resident on an
+// opaque handle; clearing the KV cache at the start of each generate keeps every
+// request statelessly isolated without paying to rebuild the context.
#include "llama.h"
#include "chat.h"
+#include "common.h" // common_cpu_get_num_math (also reached via chat.h; explicit so it survives header churn)
#include <nlohmann/json.hpp>
#include <algorithm>
#include <cctype>
#include <chrono>
#include <cstdlib>
#include <cstring>
+#include <memory>
#include <string>
#include <vector>
@@ -37,15 +40,20 @@ static bool g_backend = false;
// llama.cpp/ggml log to stderr by default; gate that behind LOG_LEVEL so
// GGUF model loads aren't noisy unless a caller opts in, matching the
// verbosity Elelem.logger is configured with in lib/elelem.rb.
+// Read once: LOG_LEVEL can't change mid-process, and this ran per log line.
static enum ggml_log_level el_log_threshold() {
- const char *level = std::getenv("LOG_LEVEL");
- if (!level) return GGML_LOG_LEVEL_WARN;
- std::string v(level);
- std::transform(v.begin(), v.end(), v.begin(), ::tolower);
- if (v == "debug") return GGML_LOG_LEVEL_DEBUG;
- if (v == "info") return GGML_LOG_LEVEL_INFO;
- if (v == "error") return GGML_LOG_LEVEL_ERROR;
- return GGML_LOG_LEVEL_WARN;
+ static const enum ggml_log_level cached = [] {
+ const char *level = std::getenv("LOG_LEVEL");
+ if (!level) return GGML_LOG_LEVEL_WARN;
+ std::string v(level);
+ std::transform(v.begin(), v.end(), v.begin(),
+ [](unsigned char c) { return (char) std::tolower(c); });
+ if (v == "debug") return GGML_LOG_LEVEL_DEBUG;
+ if (v == "info") return GGML_LOG_LEVEL_INFO;
+ if (v == "error") return GGML_LOG_LEVEL_ERROR;
+ return GGML_LOG_LEVEL_WARN;
+ }();
+ return cached;
}
static void el_log_callback(enum ggml_log_level level, const char *text, void * /*user_data*/) {
@@ -67,23 +75,20 @@ static const int EL_MIN_CTX = 4096;
// unsupported size), retry at half the size down to EL_MIN_CTX; this is
// correct for every architecture because it measures the actual allocation
// instead of predicting it.
-static llama_context *el_init_context(llama_model *model, int n_ctx, int n_threads, uint32_t *out_n_ctx) {
+static llama_context *el_init_context(llama_model *model, int n_ctx, int n_threads) {
uint32_t requested = n_ctx > 0 ? (uint32_t) n_ctx : 0;
llama_context_params cp = llama_context_default_params();
cp.n_ctx = requested;
cp.n_threads = n_threads;
cp.n_threads_batch = n_threads;
llama_context *ctx = llama_init_from_model(model, cp);
- if (ctx) {
- *out_n_ctx = requested > 0 ? requested : llama_model_n_ctx_train(model);
- return ctx;
- }
+ if (ctx) return ctx;
if (requested == 0) return nullptr; // "from model" failed; no size to halve
for (uint32_t size = requested / 2; size >= EL_MIN_CTX; size /= 2) {
cp.n_ctx = size;
ctx = llama_init_from_model(model, cp);
- if (ctx) { *out_n_ctx = size; return ctx; }
+ if (ctx) return ctx;
}
return nullptr;
}
@@ -113,11 +118,17 @@ void *el_open(const char *path, int n_gpu_layers, int n_ctx, int n_threads, floa
try {
auto tmpls = common_chat_templates_init(model, "");
- uint32_t actual_n_ctx = 0;
- llama_context *ctx = el_init_context(model, n_ctx, threads, &actual_n_ctx);
+ llama_context *ctx = el_init_context(model, n_ctx, threads);
if (!ctx) { llama_model_free(model); return nullptr; }
- return new el_handle{model, std::move(tmpls), ctx, (int) actual_n_ctx, threads, temp, (uint32_t) seed};
+ // Ask the context what it actually allocated rather than assuming the
+ // request was honoured -- llama.cpp may clamp or round n_ctx down, and
+ // this value is the sole gate on the decode budget in el_generate_impl.
+ // Per-sequence, not total: we decode one sequence (llama_batch_get_one),
+ // and llama.cpp splits the pool as n_ctx_seq = n_ctx / n_seq_max. The
+ // two are equal at the default n_seq_max=1, so this only matters if
+ // that ever changes -- at which point n_ctx would overstate our room.
+ return new el_handle{model, std::move(tmpls), ctx, (int) llama_n_ctx_seq(ctx), threads, temp, (uint32_t) seed};
} catch (const std::exception &) {
llama_model_free(model);
return nullptr;
@@ -168,6 +179,22 @@ static std::vector<common_chat_tool> build_tools(const json &arr) {
return out;
}
+// Shape every early return like a normal result so the Ruby side (gguf.rb)
+// can read "error" without special-casing missing keys.
+static const char *el_error(std::string &buf, const std::string &message) {
+ json result = {
+ {"content", ""},
+ {"reasoning", ""},
+ {"tool_calls", json::array()},
+ {"fallback", false},
+ {"harmony_tag_fallback", false},
+ {"truncated", false},
+ {"error", message}
+ };
+ buf = result.dump();
+ return buf.c_str();
+}
+
// Returns a JSON string {"content": "...", "tool_calls": [{id,name,arguments}]}.
// The buffer is valid until the next el_generate call on this thread.
// Every throwing call in here (nlohmann::json parsing, jinja template
@@ -183,8 +210,6 @@ static const char *el_generate_impl(void *handle, const char *messages_json, con
static thread_local std::string buf; // result JSON; valid until the next call on this thread
common_chat_templates_inputs inputs;
- inputs.messages = build_msgs(json::parse(messages_json));
- if (tools_json && *tools_json) inputs.tools = build_tools(json::parse(tools_json));
inputs.add_generation_prompt = true;
inputs.use_jinja = true;
// Must be set before templates_apply -- it bakes extract_reasoning into the
@@ -193,11 +218,22 @@ static const char *el_generate_impl(void *handle, const char *messages_json, con
// it). Without this, thinking-tag models (e.g. GLM's <think>...</think>)
// leave reasoning text inline in content instead of reasoning_content.
inputs.reasoning_format = COMMON_REASONING_FORMAT_AUTO;
- common_chat_params cparams = common_chat_templates_apply(h->tmpls.get(), inputs);
+
+ // Malformed request JSON and an unsupported chat template are *caller*
+ // errors, not backend failures: handle them here so they return an error
+ // result without marking the handle dead (which would strand a perfectly
+ // healthy model+context for the rest of the process). Only errors that
+ // escape to el_generate are treated as fatal to the device.
+ common_chat_params cparams;
+ try {
+ inputs.messages = build_msgs(json::parse(messages_json));
+ if (tools_json && *tools_json) inputs.tools = build_tools(json::parse(tools_json));
+ cparams = common_chat_templates_apply(h->tmpls.get(), inputs);
+ } catch (const std::exception &e) {
+ return el_error(buf, std::string("invalid request or chat template: ") + e.what());
+ }
json result;
- result["content"] = "";
- result["tool_calls"] = json::array();
// ctx is persistent on the handle (see el_open); clear its KV cache so
// each call is still stateless from the model's point of view, just
@@ -207,64 +243,106 @@ static const char *el_generate_impl(void *handle, const char *messages_json, con
llama_memory_clear(llama_get_memory(ctx), true);
auto t1 = std::chrono::steady_clock::now();
- llama_sampler *smpl = llama_sampler_chain_init(llama_sampler_chain_default_params());
+ // unique_ptr because everything between here and the decode loop can throw
+ // (vector allocation, json assignment); a raw pointer freed at the bottom
+ // would leak the whole sampler chain on that path. Deliberately per-call:
+ // llama_sampler_init_dist carries RNG state, so hoisting this onto the
+ // handle would make seeded runs non-reproducible across calls.
+ std::unique_ptr<llama_sampler, decltype(&llama_sampler_free)> smpl(
+ llama_sampler_chain_init(llama_sampler_chain_default_params()), llama_sampler_free);
if (h->temp <= 0.0f) {
- llama_sampler_chain_add(smpl, llama_sampler_init_greedy());
+ llama_sampler_chain_add(smpl.get(), llama_sampler_init_greedy());
} else {
- llama_sampler_chain_add(smpl, llama_sampler_init_top_k(40));
- llama_sampler_chain_add(smpl, llama_sampler_init_top_p(0.95f, 1));
- llama_sampler_chain_add(smpl, llama_sampler_init_temp(h->temp));
- llama_sampler_chain_add(smpl, llama_sampler_init_dist(h->seed));
+ llama_sampler_chain_add(smpl.get(), llama_sampler_init_top_k(40));
+ llama_sampler_chain_add(smpl.get(), llama_sampler_init_top_p(0.95f, 1));
+ llama_sampler_chain_add(smpl.get(), llama_sampler_init_temp(h->temp));
+ llama_sampler_chain_add(smpl.get(), llama_sampler_init_dist(h->seed));
}
const std::string &prompt = cparams.prompt;
int n_prompt = -llama_tokenize(vocab, prompt.c_str(), (int32_t) prompt.size(), nullptr, 0, true, true);
+ // A non-positive count means the template produced an empty prompt (or
+ // tokenization failed). Sampling with an empty context asserts inside
+ // llama.cpp and takes the process down, so stop here instead.
+ if (n_prompt <= 0) return el_error(buf, "empty prompt after applying chat template");
+
+ // The prompt must leave room for at least one generated token. Without
+ // this, budget below goes <= 0 and the caller sees an empty reply
+ // misreported as "hit max_tokens" rather than "context overflow".
+ if (n_prompt >= h->n_ctx) {
+ return el_error(buf, "prompt of " + std::to_string(n_prompt) +
+ " tokens exceeds context window of " + std::to_string(h->n_ctx));
+ }
+
std::vector<llama_token> tokens(n_prompt);
llama_tokenize(vocab, prompt.c_str(), (int32_t) prompt.size(), tokens.data(), (int32_t) tokens.size(), true, true);
// Ingest the prompt in n_batch-sized chunks (a full prompt commonly exceeds one
- // batch, which llama_decode asserts against).
+ // batch, which llama_decode asserts against). A failure here leaves the KV
+ // cache holding only part of the prompt -- sampling from that yields a reply
+ // conditioned on a truncated conversation, so report it rather than continuing.
const int n_batch = (int) llama_n_batch(ctx);
for (int i = 0; i < n_prompt; i += n_batch) {
int n = std::min(n_batch, n_prompt - i);
- if (llama_decode(ctx, llama_batch_get_one(tokens.data() + i, n)) != 0) break;
+ if (llama_decode(ctx, llama_batch_get_one(tokens.data() + i, n)) != 0) {
+ return el_error(buf, "failed to evaluate prompt (llama_decode failed at token " +
+ std::to_string(i) + " of " + std::to_string(n_prompt) + ")");
+ }
}
auto t2 = std::chrono::steady_clock::now();
- // The caller's max_tokens is a policy choice (how long one reply may
- // run), not a hardware quantity -- but decoding past what's left in the
- // context window is never correct regardless of what was requested, so
- // that part of the ceiling is calculated, not configured.
- int budget = std::min(max_tokens, h->n_ctx - n_prompt);
+ // Decoding past what's left in the context window is never correct, so the
+ // hardware half of the ceiling is always calculated, never configured.
+ // max_tokens <= 0 means "auto", the same convention n_ctx and n_threads use
+ // in el_open.
+ //
+ // Auto spends at most half the remaining window on a single reply rather
+ // than all of it. Conversations here are append-only (see Conversation),
+ // so a reply that consumed the entire window would leave the next turn's
+ // prompt with no room and hard-fail the agent loop at the n_prompt guard
+ // above. Half keeps replies long enough to be useful while letting the
+ // conversation continue -- the optimal value for a multi-turn agent is not
+ // the largest single reply. An explicit max_tokens is still honoured (and
+ // still clamped to the real headroom, which is never negotiable).
+ int headroom = h->n_ctx - n_prompt;
+ int budget = max_tokens > 0 ? std::min(max_tokens, headroom) : std::max(1, headroom / 2);
std::string output;
char piece[512];
int n_decoded = 0;
+ bool eog = false;
+ bool decode_failed = false;
for (int t = 0; t < budget; t++) {
- llama_token id = llama_sampler_sample(smpl, ctx, -1);
- if (llama_vocab_is_eog(vocab, id)) break;
+ llama_token id = llama_sampler_sample(smpl.get(), ctx, -1);
+ if (llama_vocab_is_eog(vocab, id)) { eog = true; break; }
int np = llama_token_to_piece(vocab, id, piece, (int32_t) sizeof(piece), 0, true);
if (np > 0) output.append(piece, np);
n_decoded++;
// Advance the KV cache so we can sample the next token. Skip it on the last
// planned iteration -- that forward pass would never be sampled from.
- if (t + 1 < budget && llama_decode(ctx, llama_batch_get_one(&id, 1)) != 0) break;
+ if (t + 1 < budget && llama_decode(ctx, llama_batch_get_one(&id, 1)) != 0) {
+ decode_failed = true;
+ break;
+ }
}
auto t3 = std::chrono::steady_clock::now();
- llama_sampler_free(smpl);
-
auto ms = [](auto a, auto b) { return std::chrono::duration<double, std::milli>(b - a).count(); };
result["ms_reset"] = ms(t0, t1);
result["ms_prefill"] = ms(t1, t2);
result["ms_decode"] = ms(t2, t3);
result["n_prompt"] = n_prompt;
result["n_decoded"] = n_decoded;
- // n_decoded hits the budget without an EOG token when generation was cut
- // off mid-thought (e.g. the whole budget spent on reasoning) rather than
- // finishing naturally -- surface it so callers don't mistake silence for
- // "the model had nothing to say".
- result["truncated"] = n_decoded >= budget;
+ // Truncated means "stopped without reaching an end-of-generation token" --
+ // the budget ran out mid-thought (e.g. spent entirely on reasoning) rather
+ // than the model finishing naturally. Keyed off the EOG flag rather than
+ // n_decoded >= budget so a decode failure isn't reported as a clean stop.
+ result["truncated"] = !eog;
+ result["budget"] = budget;
+ // A mid-generation decode failure still returns whatever was produced, but
+ // the caller needs to know the reply was cut short by an error rather than
+ // by policy.
+ if (decode_failed) result["error"] = "llama_decode failed during generation";
// The parse rules live in a PEG arena that templates_apply serialized into
// cparams.parser; common_chat_parse forwards params.parser to the PEG engine,
@@ -296,6 +374,14 @@ static const char *el_generate_impl(void *handle, const char *messages_json, con
// so the strict parser misses it. If tools were offered and nothing parsed, pull
// out the first JSON object that names a real tool. result["fallback"] records
// whether this fired -- a diagnostic for judging if a model needs the crutch.
+ auto known_tool = [&inputs](const std::string &name) {
+ if (name.empty()) return false;
+ for (const auto &tool : inputs.tools) {
+ if (tool.name == name) return true;
+ }
+ return false;
+ };
+
bool fallback_used = false;
if (parsed.tool_calls.empty() && !inputs.tools.empty()) {
size_t a = output.find('{'), b = output.rfind('}');
@@ -303,9 +389,7 @@ static const char *el_generate_impl(void *handle, const char *messages_json, con
try {
json j = json::parse(output.substr(a, b - a + 1));
std::string name = j.value("name", "");
- bool known = false;
- for (const auto &tool : inputs.tools) known |= (tool.name == name);
- if (known && j.contains("arguments")) {
+ if (known_tool(name) && j.contains("arguments")) {
common_chat_tool_call tc;
tc.name = name;
tc.arguments = as_json_string(j["arguments"]);
@@ -343,10 +427,7 @@ static const char *el_generate_impl(void *handle, const char *messages_json, con
std::string name = output.substr(name_start, name_end - name_start);
size_t m = output.find(msg_tag, name_end);
- bool known = false;
- for (const auto &tool : inputs.tools) known |= (tool.name == name);
-
- if (known && m != std::string::npos) {
+ if (known_tool(name) && m != std::string::npos) {
size_t arg_start = m + msg_tag.length();
// Assumes a single trailing JSON object (b is the last '}' in
// the whole output, not scoped to this call) -- fine while a
@@ -376,6 +457,7 @@ static const char *el_generate_impl(void *handle, const char *messages_json, con
result["harmony_tag_fallback"] = harmony_tag_fallback_used;
result["content"] = parsed.content;
result["reasoning"] = parsed.reasoning_content;
+ result["tool_calls"] = json::array(); // always present, even when empty
int i = 0;
for (const auto &tc : parsed.tool_calls) {
result["tool_calls"].push_back({
@@ -392,6 +474,13 @@ static const char *el_generate_impl(void *handle, const char *messages_json, con
const char *el_generate(void *handle, const char *messages_json, const char *tools_json, int max_tokens) {
static thread_local std::string errbuf;
+ if (!handle) return el_error(errbuf, "null handle");
+ if (!messages_json) return el_error(errbuf, "null messages");
+ // Once the device is lost every subsequent call fails the same way, and
+ // some of those failures abort the process rather than throwing. Refuse up
+ // front so one lost GPU degrades to clean errors instead of a crash.
+ if (((el_handle *) handle)->dead) return el_error(errbuf, "handle is unusable after a previous fatal error");
+
try {
return el_generate_impl(handle, messages_json, tools_json, max_tokens);
} catch (const std::exception &e) {
@@ -402,15 +491,11 @@ const char *el_generate(void *handle, const char *messages_json, const char *too
// a caller-side try/catch in el_close ever runs. Mark the handle
// dead so el_close knows to skip teardown instead of trying (and
// failing) to catch the uncatchable.
- if (handle) ((el_handle *) handle)->dead = true;
- json result = {{"content", ""}, {"tool_calls", json::array()}, {"reasoning", ""}, {"fallback", false}, {"error", e.what()}};
- errbuf = result.dump();
- return errbuf.c_str();
+ ((el_handle *) handle)->dead = true;
+ return el_error(errbuf, e.what());
} catch (...) {
- if (handle) ((el_handle *) handle)->dead = true;
- json result = {{"content", ""}, {"tool_calls", json::array()}, {"reasoning", ""}, {"fallback", false}, {"error", "unknown exception in el_generate"}};
- errbuf = result.dump();
- return errbuf.c_str();
+ ((el_handle *) handle)->dead = true;
+ return el_error(errbuf, "unknown exception in el_generate");
}
}
@@ -430,7 +515,11 @@ void el_close(void *handle) {
llama_free(h->ctx);
llama_model_free(h->model);
delete h;
- llama_backend_free();
+ // Deliberately no llama_backend_free(): the backend is process-global and
+ // shared by every handle, so freeing it here would pull it out from under
+ // any other live handle, and g_backend would still read true -- so a
+ // subsequent el_open would skip re-init and run against freed state. It is
+ // reclaimed by the OS at exit.
}
} // extern "C"
lib/elelem/llama/client.rb
@@ -29,7 +29,10 @@ module Elelem
end
end
- def initialize(model:, n_ctx: 0, n_threads: 0, max_tokens: 512, n_gpu_layers: 0, temp: 0.7, seed: -1)
+ # n_ctx, n_threads and max_tokens all take 0 to mean "auto": the native
+ # side derives the best value from the model and the hardware it is
+ # actually running on, rather than making every caller guess.
+ def initialize(model:, n_ctx: 0, n_threads: 0, max_tokens: 0, n_gpu_layers: 0, temp: 0.7, seed: -1)
@max_tokens = max_tokens
@handle = self.class.functions[:open].call(model, n_gpu_layers, n_ctx, n_threads, temp, seed)
raise "gguf: failed to load model at #{model}" if @handle.null?
@@ -58,11 +61,17 @@ module Elelem
block&.call(type: "thinking", text: reasoning) unless reasoning.empty?
content = result["content"].to_s
- if result["truncated"] && content.empty? && result.fetch("tool_calls", []).empty?
- Elelem.logger.warn("gguf: hit max_tokens (#{result["n_decoded"]}) before producing a reply")
- content = "[no reply: ran out of output tokens before finishing]"
- elsif result["error"] && content.empty?
+ # Check error before truncation: a prompt that overflows the context
+ # window also comes back empty and truncated, and reporting that as
+ # "hit max_tokens" points at the wrong knob.
+ if result["error"] && content.empty?
content = "[error: #{result["error"]}]"
+ elsif result["truncated"] && content.empty? && result.fetch("tool_calls", []).empty?
+ # Name the limit that actually bound: with max_tokens: 0 there is no
+ # max_tokens, and the budget came from the free context instead.
+ limit = @max_tokens.positive? ? "max_tokens" : "available context"
+ Elelem.logger.warn("gguf: hit #{limit} (#{result["n_decoded"]}/#{result["budget"]}) before producing a reply")
+ content = "[no reply: ran out of output tokens before finishing]"
end
block&.call(type: "saying", text: content) unless content.empty?
lib/elelem/llama/plugin.rb
@@ -5,16 +5,17 @@ Elelem::Providers.register(:gguf) do
Elelem::Net::GGUF.new(
model: ENV.fetch("GGUF_MODEL", File.expand_path("~/.agents/models/Qwen3.8-27B-UD-Q4_K_XL.gguf")),
- # n_ctx/n_threads/n_gpu_layers are calculated, not configured: the native
- # shim asks llama.cpp for the model's trained max context (falling back to
- # smaller sizes if it doesn't fit) and picks thread count from the host's
- # physical cores; every layer is offloaded whenever a GPU backend is
- # present. Convention over configuration -- there's no better answer a
+ # n_ctx/n_threads/n_gpu_layers/max_tokens are calculated, not configured:
+ # the native shim asks llama.cpp for the model's trained max context
+ # (falling back to smaller sizes if it doesn't fit) and picks thread count
+ # from the host's physical cores; every layer is offloaded whenever a GPU
+ # backend is present, and a reply may use whatever context the prompt
+ # leaves free. Convention over configuration -- there's no better answer a
# user could supply than what the hardware and model already determine.
n_ctx: 0,
n_threads: 0,
n_gpu_layers: gpu ? 999 : 0,
- max_tokens: Integer(ENV.fetch("GGUF_MAX_TOKENS", "4096")),
+ max_tokens: Integer(ENV.fetch("GGUF_MAX_TOKENS", "0")),
temp: Float(ENV.fetch("GGUF_TEMP", "0.7")),
seed: Integer(ENV.fetch("GGUF_SEED", "-1"))
)