Commit `d140464`

mo khan <mo@mokhan.ca>

2026-09-05 06:36:03

fix: close the native GGUF handle on exit and surface max_tokens truncation

el_close existed in the shim but was never bound or called from Ruby, so the loaded model/context/backend were never freed -- wire it up via at_exit. Guard the teardown with try/catch: if a prior generate call lost the GPU device (observed as a real, reproducible Vulkan ErrorDeviceLost under memory pressure on long decodes), tearing down the context throws from inside the backend and would otherwise abort the whole process on a clean exit path. Also: when generation hits max_tokens without ever reaching content (e.g. the entire budget spent on reasoning), the native side now reports `truncated`, and gguf.rb turns that into a visible message instead of silently returning nothing. Claude-Session: https://claude.ai/code/session_01FpbgyAMtPEkDbo2kx78qR6

main

1 parent 762d326

Changed files (2)

ext

elelem

llama

elelem_llama.cpp

lib

elelem

llama

client.rb

ext/elelem/llama/elelem_llama.cpp

@@ -253,6 +253,11 @@ static const char *el_generate_impl(void *handle, const char *messages_json, con
     result["ms_decode"] = ms(t2, t3);
     result["n_prompt"] = n_prompt;
     result["n_decoded"] = n_decoded;
+    // n_decoded hits max_tokens without an EOG token when generation was cut
+    // off mid-thought (e.g. the whole budget spent on reasoning) rather than
+    // finishing naturally -- surface it so callers don't mistake silence for
+    // "the model had nothing to say".
+    result["truncated"] = n_decoded >= max_tokens;
 
     // The parse rules live in a PEG arena that templates_apply serialized into
     // cparams.parser; common_chat_parse forwards params.parser to the PEG engine,
@@ -396,9 +401,17 @@ const char *el_generate(void *handle, const char *messages_json, const char *too
 void el_close(void *handle) {
     auto *h = (el_handle *) handle;
     if (!h) return;
-    llama_free(h->ctx);
-    llama_model_free(h->model);
+    // A prior generate call may have already lost the GPU device (e.g. a
+    // Vulkan ErrorDeviceLost from a driver-side OOM); tearing down a context
+    // in that state throws from deep inside the backend, and an exception
+    // can't cross back out through this C ABI boundary without aborting the
+    // whole process (same hazard as el_generate, see its comment above).
+    try {
+        llama_free(h->ctx);
+        llama_model_free(h->model);
+    } catch (...) { /* device already gone; nothing left to clean up */ }
     delete h;
+    llama_backend_free();
 }
 
 }  // extern "C"

lib/elelem/llama/client.rb

@@ -23,7 +23,8 @@ module Elelem
           lib = Fiddle.dlopen(SHIM)
           {
             open: Fiddle::Function.new(lib["el_open"], [V, I, I, I, F, I], V),
-            generate: Fiddle::Function.new(lib["el_generate"], [V, V, V, I], V)
+            generate: Fiddle::Function.new(lib["el_generate"], [V, V, V, I], V),
+            close: Fiddle::Function.new(lib["el_close"], [V], Fiddle::TYPE_VOID)
           }
         end
       end
@@ -32,6 +33,8 @@ module Elelem
         @max_tokens = max_tokens
         @handle = self.class.functions[:open].call(model, n_gpu_layers, n_ctx, n_threads, temp, seed)
         raise "gguf: failed to load model at #{model}" if @handle.null?
+
+        at_exit { self.class.functions[:close].call(@handle) }
       end
 
       def fetch(messages, tools = [], &block)
@@ -55,6 +58,10 @@ module Elelem
         block&.call(type: "thinking", text: reasoning) unless reasoning.empty?
 
         content = result["content"].to_s
+        if result["truncated"] && content.empty? && result.fetch("tool_calls", []).empty?
+          Elelem.logger.warn("gguf: hit max_tokens (#{result["n_decoded"]}) before producing a reply")
+          content = "[no reply: ran out of output tokens before finishing]"
+        end
         block&.call(type: "saying", text: content) unless content.empty?
 
         result.fetch("tool_calls", []).map do |call|

Commit d140464

Commit `d140464`