Commit `a088590`

mo khan <mo@mokhan.ca>

2026-08-27 07:03:56

feat: in-process GGUF provider backed by bundled llama.cpp

Load a local GGUF model directly inside the elelem process -- no subprocess, no HTTP server -- via a thin C shim over llama.cpp bound with stdlib Fiddle. - ext/elelem_llama/elelem_llama.cpp: ~95-line C facade that owns llama.cpp's volatile by-value structs, exposing stable el_open/el_generate/el_close. - ext/elelem_llama/extconf.rb: cmake-builds vendored llama.cpp (CPU) + the shim into lib/elelem/native with an $ORIGIN rpath (self-contained). Registered via spec.extensions so `gem install elelem` compiles it; `rake compile` runs it. - lib/elelem/plugins/gguf.rb: :gguf provider; renders the model chat template and streams tokens as `saying` events. Config via GGUF_MODEL / GGUF_N_CTX / GGUF_THREADS / GGUF_MAX_TOKENS. - vendor/llama.cpp: pinned mainline submodule (d7a2074), shipped in the gem. - fiddle added to gemspec runtime deps. CPU only for now; Vulkan/GPU and local-model tool-calling are deferred. Claude-Session: https://claude.ai/code/session_01UDKgb5gaG9Xmn3DViHRnJ7

gguf-provider

1 parent 82f66d2

Changed files (9)

ext

elelem_llama

elelem_llama.cpp

extconf.rb

lib

elelem

plugins

gguf.rb

vendor

llama.cpp

.gitignore

.gitmodules

elelem.gemspec

Gemfile.lock

Rakefile

ext/elelem_llama/elelem_llama.cpp

@@ -0,0 +1,94 @@
+// Thin, stable C façade over llama.cpp for in-process inference from Ruby (Fiddle).
+// All volatile by-value structs (llama_model_params, llama_context_params,
+// llama_batch, ...) are handled here in C, compiled against the vendored llama.h,
+// so Ruby only ever sees simple, stable signatures that don't change when
+// llama.cpp bumps. Model stays resident; a fresh context per generate call keeps
+// each request stateless (no KV bleed between turns).
+#include "llama.h"
+#include <string>
+#include <vector>
+
+extern "C" {
+
+typedef void (*el_token_cb)(const char *piece);
+
+static bool g_backend = false;
+
+// Load a model and keep it resident. n_gpu_layers 0 = CPU. Returns model* or NULL.
+void *el_open(const char *path, int n_gpu_layers) {
+    if (!g_backend) { llama_backend_init(); g_backend = true; }
+    llama_model_params mp = llama_model_default_params();
+    mp.n_gpu_layers = n_gpu_layers;
+    return (void *) llama_model_load_from_file(path, mp);
+}
+
+// Render the message array through the model's chat template, then stream the
+// generated pieces to cb. Returns tokens generated, or -1 on error.
+int el_generate(void *model_, const char **roles, const char **contents, int n_msgs,
+                int n_ctx, int n_threads, int max_tokens, el_token_cb cb) {
+    llama_model *model = (llama_model *) model_;
+    const llama_vocab *vocab = llama_model_get_vocab(model);
+
+    std::vector<llama_chat_message> msgs(n_msgs);
+    for (int i = 0; i < n_msgs; i++) { msgs[i].role = roles[i]; msgs[i].content = contents[i]; }
+
+    std::string text;
+    const char *tmpl = llama_model_chat_template(model, nullptr);
+    if (tmpl && n_msgs > 0) {
+        int need = llama_chat_apply_template(tmpl, msgs.data(), n_msgs, true, nullptr, 0);
+        if (need > 0) {
+            std::vector<char> buf(need);
+            int n = llama_chat_apply_template(tmpl, msgs.data(), n_msgs, true, buf.data(), (int32_t) buf.size());
+            if (n > 0) text.assign(buf.data(), n);
+        }
+    }
+    if (text.empty()) {  // no template: crude fallback
+        for (int i = 0; i < n_msgs; i++) { text += roles[i]; text += ": "; text += contents[i]; text += "\n"; }
+        text += "assistant:";
+    }
+
+    llama_context_params cp = llama_context_default_params();
+    cp.n_ctx = (uint32_t) n_ctx;
+    cp.n_threads = n_threads;
+    cp.n_threads_batch = n_threads;
+    llama_context *ctx = llama_init_from_model(model, cp);
+    if (!ctx) return -1;
+
+    int n_prompt = -llama_tokenize(vocab, text.c_str(), (int32_t) text.size(), nullptr, 0, true, true);
+    std::vector<llama_token> tokens(n_prompt);
+    if (llama_tokenize(vocab, text.c_str(), (int32_t) text.size(), tokens.data(),
+                       (int32_t) tokens.size(), true, true) < 0) {
+        llama_free(ctx);
+        return -1;
+    }
+
+    llama_sampler *smpl = llama_sampler_chain_init(llama_sampler_chain_default_params());
+    llama_sampler_chain_add(smpl, llama_sampler_init_top_k(40));
+    llama_sampler_chain_add(smpl, llama_sampler_init_top_p(0.95f, 1));
+    llama_sampler_chain_add(smpl, llama_sampler_init_temp(0.7f));
+    llama_sampler_chain_add(smpl, llama_sampler_init_dist(LLAMA_DEFAULT_SEED));
+
+    llama_batch batch = llama_batch_get_one(tokens.data(), (int32_t) tokens.size());
+    llama_token cur = 0;
+    char piece[512];
+    int generated = 0;
+
+    for (int i = 0; i < max_tokens; i++) {
+        if (llama_decode(ctx, batch) != 0) break;
+        llama_token id = llama_sampler_sample(smpl, ctx, -1);
+        if (llama_vocab_is_eog(vocab, id)) break;
+        int np = llama_token_to_piece(vocab, id, piece, (int32_t) sizeof(piece), 0, true);
+        if (np > 0 && cb) { std::string s(piece, np); cb(s.c_str()); }
+        generated++;
+        cur = id;                                   // must outlive the decode below
+        batch = llama_batch_get_one(&cur, 1);
+    }
+
+    llama_sampler_free(smpl);
+    llama_free(ctx);
+    return generated;
+}
+
+void el_close(void *model) { if (model) llama_model_free((llama_model *) model); }
+
+}  // extern "C"

ext/elelem_llama/extconf.rb

@@ -0,0 +1,67 @@
+# frozen_string_literal: true
+
+# Builds the vendored llama.cpp (CPU shared libs) and our thin C shim at gem
+# install time, installing the shared objects into lib/elelem/native/ so the
+# :gguf provider can Fiddle.dlopen them and run a model in-process. llama.cpp is
+# a CMake project (not an mkmf extension), so we drive cmake ourselves and then
+# emit a stub Makefile to satisfy RubyGems' `make && make install` step.
+require "fileutils"
+
+EXT_DIR   = __dir__
+GEM_ROOT  = File.expand_path("../..", EXT_DIR)
+VENDOR    = File.join(GEM_ROOT, "vendor", "llama.cpp")
+BUILD_DIR = File.join(EXT_DIR, "build")
+NATIVE    = File.join(GEM_ROOT, "lib", "elelem", "native")
+
+def run(*cmd)
+  warn "elelem: + #{cmd.join(' ')}"
+  system(*cmd) || abort("elelem: build step failed: #{cmd.join(' ')}")
+end
+
+unless File.exist?(File.join(VENDOR, "CMakeLists.txt"))
+  abort "elelem: vendored llama.cpp missing at #{VENDOR}\n" \
+        "        run: git submodule update --init --recursive"
+end
+
+# 1. Configure + build only libllama (CPU). Examples/tests/tools/server off keeps
+#    the build fast; GPU backends come later.
+run("cmake", "-S", VENDOR, "-B", BUILD_DIR,
+    "-DCMAKE_BUILD_TYPE=Release",
+    "-DBUILD_SHARED_LIBS=ON",
+    # Bake $ORIGIN rpath into the libs so, once co-located in lib/elelem/native,
+    # libllama finds its ggml siblings there -- not in this throwaway build dir.
+    "-DCMAKE_BUILD_WITH_INSTALL_RPATH=ON",
+    "-DCMAKE_INSTALL_RPATH=$ORIGIN",
+    "-DGGML_VULKAN=OFF",
+    "-DLLAMA_CURL=OFF",
+    "-DLLAMA_BUILD_TESTS=OFF",
+    "-DLLAMA_BUILD_EXAMPLES=OFF",
+    "-DLLAMA_BUILD_TOOLS=OFF",
+    "-DLLAMA_BUILD_SERVER=OFF")
+run("cmake", "--build", BUILD_DIR, "--target", "llama", "--parallel")
+
+# 2. Copy the shared objects (preserving SONAME symlinks) next to where the shim
+#    will live, so the shim's $ORIGIN rpath resolves libllama/libggml at runtime.
+FileUtils.mkdir_p(NATIVE)
+libdir = File.join(BUILD_DIR, "bin")
+run("sh", "-c", "cp -a #{libdir}/*.so* #{NATIVE}/")
+
+# 3. Compile the shim against the vendored headers, link libllama, rpath $ORIGIN.
+cxx = ENV["CXX"] || "c++"
+run(cxx, "-std=c++17", "-O2", "-shared", "-fPIC",
+    "-I", File.join(VENDOR, "include"),
+    "-I", File.join(VENDOR, "ggml", "include"),
+    File.join(EXT_DIR, "elelem_llama.cpp"),
+    "-o", File.join(NATIVE, "libelelem_llama.so"),
+    "-L", libdir, "-lllama",
+    "-Wl,-rpath,$ORIGIN")
+
+# 4. Stub Makefile so `make` / `make install` succeed (work is already done).
+File.write(File.join(EXT_DIR, "Makefile"), <<~MAKE)
+  all:
+  clean:
+  install:
+  .PHONY: all clean install
+MAKE
+
+warn "elelem: built llama.cpp + shim into #{NATIVE}"

lib/elelem/plugins/gguf.rb

@@ -0,0 +1,81 @@
+# frozen_string_literal: true
+
+# In-process GGUF provider: loads a local model inside the elelem process via a
+# tiny C shim over llama.cpp (see ../gguf/elelem_llama.cpp), bound with stdlib
+# Fiddle. No subprocess, no HTTP server, no third-party gem.
+require "fiddle"
+
+module Elelem
+  module GGUF
+    SHIM = File.expand_path("../native/libelelem_llama.so", __dir__)
+    V = Fiddle::TYPE_VOIDP
+    I = Fiddle::TYPE_INT
+
+    class Client
+      def initialize(model_path:, n_ctx: 4096, n_threads: 16, max_tokens: 512)
+        @n_ctx = n_ctx
+        @n_threads = n_threads
+        @max_tokens = max_tokens
+
+        lib = Fiddle.dlopen(SHIM)
+        @open     = Fiddle::Function.new(lib["el_open"], [V, I], V)
+        @generate = Fiddle::Function.new(lib["el_generate"], [V, V, V, I, I, I, I, V], I)
+        @close    = Fiddle::Function.new(lib["el_close"], [V], Fiddle::TYPE_VOID)
+
+        @model = @open.call(model_path, 0) # n_gpu_layers 0 = CPU (GPU later)
+        raise "gguf: failed to load model at #{model_path}" if @model.null?
+      end
+
+      # elelem provider contract: fetch(messages, tools=[]) { |event| } -> tool_calls
+      def fetch(messages, _tools = [], &block)
+        usable = messages.select { |m| m[:content] && !m[:content].to_s.empty? }
+        roles, keep_roles = str_array(usable.map { |m| role_of(m[:role]) })
+        contents, keep_contents = str_array(usable.map { |m| m[:content].to_s })
+
+        callback = Fiddle::Closure::BlockCaller.new(Fiddle::TYPE_VOID, [V]) do |cstr|
+          block&.call(type: "saying", text: Fiddle::Pointer.new(cstr).to_s)
+        end
+
+        @generate.call(@model, roles, contents, usable.length,
+                       @n_ctx, @n_threads, @max_tokens, callback)
+        keep_roles.clear # hold the malloc'd strings alive until the call returns
+        keep_contents.clear
+        [] # tool-calling from the local model is deferred (phase 2)
+      end
+
+      private
+
+      def role_of(role)
+        r = role.to_s
+        %w[system user assistant].include?(r) ? r : "user"
+      end
+
+      # Build a C `const char**` from Ruby strings; returns [pointer_array, keep]
+      # where `keep` holds the per-string buffers so GC can't free them mid-call.
+      def str_array(strings)
+        cstrs = strings.map { |s| cstr(s) }
+        arr = Fiddle::Pointer.malloc([strings.size, 1].max * Fiddle::SIZEOF_VOIDP, Fiddle::RUBY_FREE)
+        cstrs.each_with_index do |ptr, i|
+          arr[i * Fiddle::SIZEOF_VOIDP, Fiddle::SIZEOF_VOIDP] = [ptr.to_i].pack("J")
+        end
+        [arr, cstrs]
+      end
+
+      def cstr(str)
+        bytes = "#{str}\0".b
+        ptr = Fiddle::Pointer.malloc(bytes.bytesize, Fiddle::RUBY_FREE)
+        ptr[0, bytes.bytesize] = bytes
+        ptr
+      end
+    end
+  end
+end
+
+Elelem::Providers.register(:gguf) do
+  Elelem::GGUF::Client.new(
+    model_path: ENV.fetch("GGUF_MODEL", File.expand_path("~/models/Qwen2.5-Coder-7B-Instruct-Q4_K_M.gguf")),
+    n_ctx: Integer(ENV.fetch("GGUF_N_CTX", "4096")),
+    n_threads: Integer(ENV.fetch("GGUF_THREADS", "16")),
+    max_tokens: Integer(ENV.fetch("GGUF_MAX_TOKENS", "512"))
+  )
+end

vendor/llama.cpp

@@ -0,0 +1,1 @@
+Subproject commit d7a2074112d27649303fa107eb8c94db1ee435f3