gguf-provider
1// Thin, stable C façade over llama.cpp + its common_chat layer, for in-process
2// inference from Ruby (Fiddle). All volatile C++/by-value types are handled here,
3// compiled against the vendored headers, so Ruby only sees simple signatures:
4// JSON in (OpenAI-style messages + tools), JSON out (content + tool_calls). The
5// model + chat templates are opened once and kept resident on an opaque handle; a
6// fresh context per generate keeps each request statelessly isolated.
7#include "llama.h"
8#include "chat.h"
9#include <nlohmann/json.hpp>
10#include <algorithm>
11#include <cctype>
12#include <chrono>
13#include <cstdlib>
14#include <cstring>
15#include <string>
16#include <vector>
17
18using json = nlohmann::ordered_json;
19
20extern "C" {
21
22struct el_handle {
23 llama_model *model;
24 common_chat_templates_ptr tmpls;
25 llama_context *ctx; // persistent across el_generate calls; memory is
26 // cleared at the start of each call so behavior
27 // stays identical to a fresh context per call
28 int n_ctx;
29 int n_threads;
30 float temp; // <= 0 => greedy (deterministic); used by evals
31 uint32_t seed;
32};
33
34static bool g_backend = false;
35
36// llama.cpp/ggml log to stderr by default; gate that behind LOG_LEVEL so
37// GGUF model loads aren't noisy unless a caller opts in, matching the
38// verbosity Elelem.logger is configured with in lib/elelem.rb.
39static enum ggml_log_level el_log_threshold() {
40 const char *level = std::getenv("LOG_LEVEL");
41 if (!level) return GGML_LOG_LEVEL_WARN;
42 std::string v(level);
43 std::transform(v.begin(), v.end(), v.begin(), ::tolower);
44 if (v == "debug") return GGML_LOG_LEVEL_DEBUG;
45 if (v == "info") return GGML_LOG_LEVEL_INFO;
46 if (v == "error") return GGML_LOG_LEVEL_ERROR;
47 return GGML_LOG_LEVEL_WARN;
48}
49
50static void el_log_callback(enum ggml_log_level level, const char *text, void * /*user_data*/) {
51 if (level < el_log_threshold()) return;
52 fputs(text, stderr);
53}
54
55// temp <= 0 selects greedy/deterministic sampling; seed is the RNG seed for the
56// sampled path (both surfaced so callers -- notably the eval harness -- can pin them).
57void *el_open(const char *path, int n_gpu_layers, int n_ctx, int n_threads, float temp, int seed) {
58 if (!g_backend) {
59 llama_log_set(el_log_callback, nullptr);
60 llama_backend_init();
61 g_backend = true;
62 }
63
64 llama_model_params mp = llama_model_default_params();
65 mp.n_gpu_layers = n_gpu_layers;
66 llama_model *model = llama_model_load_from_file(path, mp);
67 if (!model) return nullptr;
68
69 // common_chat_templates_init parses the model's embedded Jinja chat
70 // template; a malformed/unsupported template throws instead of
71 // returning null, which would otherwise abort the whole process (see
72 // el_generate's comment on the FFI boundary).
73 try {
74 auto tmpls = common_chat_templates_init(model, "");
75
76 llama_context_params cp = llama_context_default_params();
77 cp.n_ctx = (uint32_t) n_ctx;
78 cp.n_threads = n_threads;
79 cp.n_threads_batch = n_threads;
80 llama_context *ctx = llama_init_from_model(model, cp);
81 if (!ctx) { llama_model_free(model); return nullptr; }
82
83 return new el_handle{model, std::move(tmpls), ctx, n_ctx, n_threads, temp, (uint32_t) seed};
84 } catch (const std::exception &) {
85 llama_model_free(model);
86 return nullptr;
87 }
88}
89
90// Tool-call arguments arrive as either a JSON string or an object; llama.cpp's
91// common_chat wants a string either way.
92static std::string as_json_string(const json &v) {
93 return v.is_string() ? v.get<std::string>() : v.dump();
94}
95
96static std::vector<common_chat_msg> build_msgs(const json &arr) {
97 std::vector<common_chat_msg> out;
98 for (const auto &m : arr) {
99 common_chat_msg cm;
100 cm.role = m.value("role", "user");
101 cm.content = m.value("content", "");
102 cm.tool_name = m.value("tool_name", "");
103 cm.tool_call_id = m.value("tool_call_id", "");
104 if (m.contains("tool_calls")) {
105 for (const auto &tc : m["tool_calls"]) {
106 common_chat_tool_call c;
107 c.id = tc.value("id", "");
108 c.name = tc.value("name", "");
109 if (tc.contains("arguments")) {
110 const auto &a = tc["arguments"];
111 c.arguments = as_json_string(a);
112 }
113 cm.tool_calls.push_back(c);
114 }
115 }
116 out.push_back(cm);
117 }
118 return out;
119}
120
121static std::vector<common_chat_tool> build_tools(const json &arr) {
122 std::vector<common_chat_tool> out;
123 for (const auto &t : arr) {
124 const json &fn = t.contains("function") ? t["function"] : t;
125 common_chat_tool ct;
126 ct.name = fn.value("name", "");
127 ct.description = fn.value("description", "");
128 ct.parameters = fn.contains("parameters") ? fn["parameters"].dump() : "{}";
129 out.push_back(ct);
130 }
131 return out;
132}
133
134// Returns a JSON string {"content": "...", "tool_calls": [{id,name,arguments}]}.
135// The buffer is valid until the next el_generate call on this thread.
136// Every throwing call in here (nlohmann::json parsing, jinja template
137// application, the chat PEG parser) is wrapped by the caller, el_generate,
138// in one top-level try/catch: any C++ exception that unwinds past this
139// function crosses the Fiddle FFI boundary into Ruby and aborts the whole
140// process (std::terminate, not a catchable Ruby exception). A malformed
141// request or an unfamiliar template family (new model = new chat_template)
142// must degrade to an error result, never crash the caller.
143static const char *el_generate_impl(void *handle, const char *messages_json, const char *tools_json, int max_tokens) {
144 auto *h = (el_handle *) handle;
145 const llama_vocab *vocab = llama_model_get_vocab(h->model);
146 static thread_local std::string buf; // result JSON; valid until the next call on this thread
147
148 common_chat_templates_inputs inputs;
149 inputs.messages = build_msgs(json::parse(messages_json));
150 if (tools_json && *tools_json) inputs.tools = build_tools(json::parse(tools_json));
151 inputs.add_generation_prompt = true;
152 inputs.use_jinja = true;
153 // Must be set before templates_apply -- it bakes extract_reasoning into the
154 // parser grammar templates_apply builds (see common_chat_parser_params below,
155 // which only controls parse-time behavior for formats that already support
156 // it). Without this, thinking-tag models (e.g. GLM's <think>...</think>)
157 // leave reasoning text inline in content instead of reasoning_content.
158 inputs.reasoning_format = COMMON_REASONING_FORMAT_AUTO;
159 common_chat_params cparams = common_chat_templates_apply(h->tmpls.get(), inputs);
160
161 json result;
162 result["content"] = "";
163 result["tool_calls"] = json::array();
164
165 // ctx is persistent on the handle (see el_open); clear its KV cache so
166 // each call is still stateless from the model's point of view, just
167 // without paying context-alloc/graph-reserve cost every time.
168 llama_context *ctx = h->ctx;
169 auto t0 = std::chrono::steady_clock::now();
170 llama_memory_clear(llama_get_memory(ctx), true);
171 auto t1 = std::chrono::steady_clock::now();
172
173 llama_sampler *smpl = llama_sampler_chain_init(llama_sampler_chain_default_params());
174 if (h->temp <= 0.0f) {
175 llama_sampler_chain_add(smpl, llama_sampler_init_greedy());
176 } else {
177 llama_sampler_chain_add(smpl, llama_sampler_init_top_k(40));
178 llama_sampler_chain_add(smpl, llama_sampler_init_top_p(0.95f, 1));
179 llama_sampler_chain_add(smpl, llama_sampler_init_temp(h->temp));
180 llama_sampler_chain_add(smpl, llama_sampler_init_dist(h->seed));
181 }
182
183 const std::string &prompt = cparams.prompt;
184 int n_prompt = -llama_tokenize(vocab, prompt.c_str(), (int32_t) prompt.size(), nullptr, 0, true, true);
185 std::vector<llama_token> tokens(n_prompt);
186 llama_tokenize(vocab, prompt.c_str(), (int32_t) prompt.size(), tokens.data(), (int32_t) tokens.size(), true, true);
187
188 // Ingest the prompt in n_batch-sized chunks (a full prompt commonly exceeds one
189 // batch, which llama_decode asserts against).
190 const int n_batch = (int) llama_n_batch(ctx);
191 for (int i = 0; i < n_prompt; i += n_batch) {
192 int n = std::min(n_batch, n_prompt - i);
193 if (llama_decode(ctx, llama_batch_get_one(tokens.data() + i, n)) != 0) break;
194 }
195 auto t2 = std::chrono::steady_clock::now();
196
197 std::string output;
198 char piece[512];
199 int n_decoded = 0;
200 for (int t = 0; t < max_tokens; t++) {
201 llama_token id = llama_sampler_sample(smpl, ctx, -1);
202 if (llama_vocab_is_eog(vocab, id)) break;
203 int np = llama_token_to_piece(vocab, id, piece, (int32_t) sizeof(piece), 0, true);
204 if (np > 0) output.append(piece, np);
205 n_decoded++;
206 // Advance the KV cache so we can sample the next token. Skip it on the last
207 // planned iteration -- that forward pass would never be sampled from.
208 if (t + 1 < max_tokens && llama_decode(ctx, llama_batch_get_one(&id, 1)) != 0) break;
209 }
210 auto t3 = std::chrono::steady_clock::now();
211
212 llama_sampler_free(smpl);
213
214 auto ms = [](auto a, auto b) { return std::chrono::duration<double, std::milli>(b - a).count(); };
215 result["ms_reset"] = ms(t0, t1);
216 result["ms_prefill"] = ms(t1, t2);
217 result["ms_decode"] = ms(t2, t3);
218 result["n_prompt"] = n_prompt;
219 result["n_decoded"] = n_decoded;
220
221 // The parse rules live in a PEG arena that templates_apply serialized into
222 // cparams.parser; common_chat_parse forwards params.parser to the PEG engine,
223 // so it must be deserialized here or every format returns content-only (no
224 // tool calls). The converting ctor carries format + generation_prompt.
225 //
226 // common_chat_parse throws std::runtime_error when the model's raw output
227 // doesn't match its own template's expected grammar (seen with gpt-oss's
228 // harmony format on malformed/truncated generations). That exception can't
229 // cross the Fiddle FFI boundary -- it aborts the whole Ruby process -- so
230 // treat a parse failure the same as "nothing parsed": fall through to the
231 // lenient fallback below with the raw text kept as content.
232 common_chat_msg parsed;
233 parsed.content = output;
234 try {
235 common_chat_parser_params pp(cparams);
236 // AUTO routes <think>/harmony-analysis text into reasoning_content
237 // instead of leaving it inline in content (the default, NONE, does not
238 // split it out at all -- see gpt-oss's <|channel|>analysis<|message|>).
239 pp.reasoning_format = COMMON_REASONING_FORMAT_AUTO;
240 if (!cparams.parser.empty()) pp.parser.load(cparams.parser);
241 parsed = common_chat_parse(output, false, pp);
242 } catch (const std::exception &) {
243 parsed.content = output;
244 }
245
246 // Lenient fallback: small/quantized models often emit a bare {"name","arguments"}
247 // tool-call JSON (frequently fenced) instead of the template's exact tag syntax,
248 // so the strict parser misses it. If tools were offered and nothing parsed, pull
249 // out the first JSON object that names a real tool. result["fallback"] records
250 // whether this fired -- a diagnostic for judging if a model needs the crutch.
251 bool fallback_used = false;
252 if (parsed.tool_calls.empty() && !inputs.tools.empty()) {
253 size_t a = output.find('{'), b = output.rfind('}');
254 if (a != std::string::npos && b != std::string::npos && b > a) {
255 try {
256 json j = json::parse(output.substr(a, b - a + 1));
257 std::string name = j.value("name", "");
258 bool known = false;
259 for (const auto &tool : inputs.tools) known |= (tool.name == name);
260 if (known && j.contains("arguments")) {
261 common_chat_tool_call tc;
262 tc.name = name;
263 tc.arguments = as_json_string(j["arguments"]);
264 parsed.tool_calls.push_back(tc);
265 parsed.content.clear();
266 fallback_used = true;
267 }
268 } catch (...) { /* not a tool call; leave content as-is */ }
269 }
270 }
271
272 // Harmony fallback: gpt-oss sometimes emits a malformed header --
273 // e.g. a doubled "<|channel|>commentary" before "<|constrain|>json", or a
274 // missing space -- that the PEG grammar in common_chat_parse rejects
275 // outright (logged upstream as "unparsed peg-native output"), leaving
276 // tool_calls empty and the whole raw header+JSON sitting in content. The
277 // JSON-sniffing fallback above can't help: the tool name lives in the
278 // "to=functions.NAME" tag, not in the trailing JSON object, which here is
279 // bare arguments. Scrape the tag instead of trying to normalize every way
280 // the header tags can be malformed. Search for the *last* "to=functions."
281 // that is actually followed by "<|message|>" on the same call, since
282 // model prose can hallucinate an earlier, unrelated "to=functions." (e.g.
283 // mid-sentence speculation) before the real one.
284 bool harmony_tag_fallback_used = false;
285 if (parsed.tool_calls.empty() && !inputs.tools.empty()) {
286 static const std::string tag = "to=functions.";
287 static const std::string msg_tag = "<|message|>";
288 size_t search_from = output.size();
289 for (;;) {
290 size_t t = output.rfind(tag, search_from);
291 if (t == std::string::npos) break;
292 size_t name_start = t + tag.length();
293 size_t name_end = name_start;
294 while (name_end < output.size() && (isalnum((unsigned char) output[name_end]) || output[name_end] == '_')) name_end++;
295 std::string name = output.substr(name_start, name_end - name_start);
296
297 size_t m = output.find(msg_tag, name_end);
298 bool known = false;
299 for (const auto &tool : inputs.tools) known |= (tool.name == name);
300
301 if (known && m != std::string::npos) {
302 size_t arg_start = m + msg_tag.length();
303 // Assumes a single trailing JSON object (b is the last '}' in
304 // the whole output, not scoped to this call) -- fine while a
305 // generation carries at most one malformed tool call, but two
306 // such calls or trailing prose containing '}' would make this
307 // span both and fail to parse, silently giving up rather than
308 // producing a wrong call.
309 size_t a = output.find('{', arg_start), b = output.rfind('}');
310 if (a != std::string::npos && b != std::string::npos && b > a) {
311 try {
312 json args = json::parse(output.substr(a, b - a + 1));
313 common_chat_tool_call tc;
314 tc.name = name;
315 tc.arguments = as_json_string(args);
316 parsed.tool_calls.push_back(tc);
317 parsed.content.clear();
318 harmony_tag_fallback_used = true;
319 } catch (...) { /* bare JSON didn't parse; give up on this tag */ }
320 }
321 }
322 if (harmony_tag_fallback_used || t == 0) break;
323 search_from = t - 1;
324 }
325 }
326
327 result["fallback"] = fallback_used;
328 result["harmony_tag_fallback"] = harmony_tag_fallback_used;
329 result["content"] = parsed.content;
330 result["reasoning"] = parsed.reasoning_content;
331 int i = 0;
332 for (const auto &tc : parsed.tool_calls) {
333 result["tool_calls"].push_back({
334 {"id", tc.id.empty() ? "call_" + std::to_string(i) : tc.id},
335 {"name", tc.name},
336 {"arguments", tc.arguments}
337 });
338 i++;
339 }
340
341 buf = result.dump();
342 return buf.c_str();
343}
344
345const char *el_generate(void *handle, const char *messages_json, const char *tools_json, int max_tokens) {
346 static thread_local std::string errbuf;
347 try {
348 return el_generate_impl(handle, messages_json, tools_json, max_tokens);
349 } catch (const std::exception &e) {
350 json result = {{"content", ""}, {"tool_calls", json::array()}, {"reasoning", ""}, {"fallback", false}, {"error", e.what()}};
351 errbuf = result.dump();
352 return errbuf.c_str();
353 } catch (...) {
354 json result = {{"content", ""}, {"tool_calls", json::array()}, {"reasoning", ""}, {"fallback", false}, {"error", "unknown exception in el_generate"}};
355 errbuf = result.dump();
356 return errbuf.c_str();
357 }
358}
359
360void el_close(void *handle) {
361 auto *h = (el_handle *) handle;
362 if (!h) return;
363 llama_free(h->ctx);
364 llama_model_free(h->model);
365 delete h;
366}
367
368} // extern "C"