Commit 51f8f25
Changed files (19)
ext
elelem
ggml
lib
elelem
spec
elelem
fixtures
ext/elelem/ggml/elelem.c
@@ -0,0 +1,319 @@
+#include <math.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include "ggml.h"
+#include "ggml-cpu.h"
+#include "gguf.h"
+
+#define EL_COMPUTE_MEM_FLOOR (256 * 1024 * 1024)
+#define EL_COMPUTE_MEM_PER_TOKEN_PER_LAYER (256 * 1024)
+
+struct el_hparams {
+ char arch[32];
+ int32_t n_layer;
+ int32_t n_embd;
+ int32_t n_head;
+ int32_t n_head_kv;
+ int32_t n_ff;
+ int32_t n_vocab;
+ float rope_freq_base;
+ float rms_norm_eps;
+};
+
+struct el_handle {
+ struct gguf_context *gguf;
+ struct ggml_context *weights;
+ struct el_hparams hparams;
+ int n_threads;
+ char version[256];
+};
+
+static int32_t el_kv_i32(struct gguf_context *gguf, const char *key, int32_t fallback) {
+ int64_t id = gguf_find_key(gguf, key);
+ if (id < 0) return fallback;
+
+ switch (gguf_get_kv_type(gguf, id)) {
+ case GGUF_TYPE_UINT8: return gguf_get_val_u8(gguf, id);
+ case GGUF_TYPE_INT8: return gguf_get_val_i8(gguf, id);
+ case GGUF_TYPE_UINT16: return gguf_get_val_u16(gguf, id);
+ case GGUF_TYPE_INT16: return gguf_get_val_i16(gguf, id);
+ case GGUF_TYPE_UINT32: return (int32_t) gguf_get_val_u32(gguf, id);
+ case GGUF_TYPE_INT32: return gguf_get_val_i32(gguf, id);
+ case GGUF_TYPE_UINT64: return (int32_t) gguf_get_val_u64(gguf, id);
+ case GGUF_TYPE_INT64: return (int32_t) gguf_get_val_i64(gguf, id);
+ default: return fallback;
+ }
+}
+
+static float el_kv_f32(struct gguf_context *gguf, const char *key, float fallback) {
+ int64_t id = gguf_find_key(gguf, key);
+ if (id < 0) return fallback;
+
+ switch (gguf_get_kv_type(gguf, id)) {
+ case GGUF_TYPE_FLOAT32: return gguf_get_val_f32(gguf, id);
+ case GGUF_TYPE_FLOAT64: return (float) gguf_get_val_f64(gguf, id);
+ default: return fallback;
+ }
+}
+
+static void el_load_hparams(struct gguf_context *gguf, struct el_hparams *hp) {
+ memset(hp, 0, sizeof(*hp));
+
+ int64_t arch_id = gguf_find_key(gguf, "general.architecture");
+ const char *arch = arch_id >= 0 ? gguf_get_val_str(gguf, arch_id) : "unknown";
+ snprintf(hp->arch, sizeof(hp->arch), "%s", arch);
+
+ char key[128];
+
+ snprintf(key, sizeof(key), "%s.block_count", arch);
+ hp->n_layer = el_kv_i32(gguf, key, 0);
+
+ snprintf(key, sizeof(key), "%s.embedding_length", arch);
+ hp->n_embd = el_kv_i32(gguf, key, 0);
+
+ snprintf(key, sizeof(key), "%s.attention.head_count", arch);
+ hp->n_head = el_kv_i32(gguf, key, 0);
+
+ snprintf(key, sizeof(key), "%s.attention.head_count_kv", arch);
+ hp->n_head_kv = el_kv_i32(gguf, key, hp->n_head);
+
+ snprintf(key, sizeof(key), "%s.feed_forward_length", arch);
+ hp->n_ff = el_kv_i32(gguf, key, 0);
+
+ snprintf(key, sizeof(key), "%s.rope.freq_base", arch);
+ hp->rope_freq_base = el_kv_f32(gguf, key, 10000.0f);
+
+ snprintf(key, sizeof(key), "%s.attention.layer_norm_rms_epsilon", arch);
+ hp->rms_norm_eps = el_kv_f32(gguf, key, 1e-6f);
+
+ int64_t tok_id = gguf_find_key(gguf, "tokenizer.ggml.tokens");
+ hp->n_vocab = tok_id >= 0 ? (int32_t) gguf_get_arr_n(gguf, tok_id) : 0;
+}
+
+void *el_open(const char *model_path, int n_gpu_layers, int n_ctx, int n_threads) {
+ (void) n_gpu_layers;
+ (void) n_ctx;
+
+ struct ggml_context *weights = NULL;
+ struct gguf_init_params params = {
+ .no_alloc = false,
+ .ctx = &weights,
+ };
+
+ struct gguf_context *gguf = gguf_init_from_file(model_path, params);
+ if (!gguf) return NULL;
+
+ struct el_handle *h = malloc(sizeof(struct el_handle));
+ if (!h) {
+ gguf_free(gguf);
+ return NULL;
+ }
+
+ h->gguf = gguf;
+ h->weights = weights;
+ h->n_threads = n_threads > 0 ? n_threads : 4;
+ el_load_hparams(gguf, &h->hparams);
+
+ snprintf(h->version, sizeof(h->version),
+ "elelem-ggml arch=%s n_layer=%d n_embd=%d n_head=%d n_head_kv=%d n_ff=%d n_vocab=%d",
+ h->hparams.arch, h->hparams.n_layer, h->hparams.n_embd,
+ h->hparams.n_head, h->hparams.n_head_kv, h->hparams.n_ff, h->hparams.n_vocab);
+
+ return h;
+}
+
+const char *el_version(void *handle) {
+ if (!handle) return "";
+ struct el_handle *h = handle;
+ return h->version;
+}
+
+static struct ggml_tensor *el_weight(struct ggml_context *weights, const char *fmt, int layer) {
+ char name[128];
+ if (layer >= 0) {
+ snprintf(name, sizeof(name), fmt, layer);
+ } else {
+ snprintf(name, sizeof(name), "%s", fmt);
+ }
+ return ggml_get_tensor(weights, name);
+}
+
+static struct ggml_tensor *el_rms_norm(struct ggml_context *ctx, struct ggml_tensor *cur,
+ struct ggml_tensor *weight, float eps) {
+ cur = ggml_rms_norm(ctx, cur, eps);
+ cur = ggml_mul(ctx, cur, weight);
+ return cur;
+}
+
+static int32_t el_forward_argmax(struct el_handle *h, const int32_t *tokens, int32_t n_tokens) {
+ struct el_hparams *hp = &h->hparams;
+
+ const int64_t n_embd_head = hp->n_embd / hp->n_head;
+
+ size_t linear_term = (size_t) n_tokens * hp->n_layer * EL_COMPUTE_MEM_PER_TOKEN_PER_LAYER;
+ size_t attn_term = (size_t) n_tokens * n_tokens * hp->n_head * hp->n_layer * sizeof(float) * 2;
+ size_t mem_size = linear_term + attn_term;
+ if (mem_size < EL_COMPUTE_MEM_FLOOR) mem_size = EL_COMPUTE_MEM_FLOOR;
+
+ struct ggml_context *ctx = ggml_init((struct ggml_init_params) {
+ .mem_size = mem_size,
+ .mem_buffer = NULL,
+ .no_alloc = false,
+ });
+ if (!ctx) return -1;
+
+ struct ggml_tensor *tok_embd = el_weight(h->weights, "token_embd.weight", -1);
+ struct ggml_tensor *output_norm = el_weight(h->weights, "output_norm.weight", -1);
+ struct ggml_tensor *output = el_weight(h->weights, "output.weight", -1);
+ if (!tok_embd || !output_norm || !output) {
+ ggml_free(ctx);
+ return -1;
+ }
+
+ struct ggml_tensor *inp_tokens = ggml_new_tensor_1d(ctx, GGML_TYPE_I32, n_tokens);
+ memcpy(inp_tokens->data, tokens, n_tokens * sizeof(int32_t));
+
+ struct ggml_tensor *inp_pos = ggml_new_tensor_1d(ctx, GGML_TYPE_I32, n_tokens);
+ for (int32_t i = 0; i < n_tokens; i++) {
+ ((int32_t *) inp_pos->data)[i] = i;
+ }
+
+ struct ggml_tensor *kq_mask = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, n_tokens, n_tokens);
+ float *mask = (float *) kq_mask->data;
+ for (int32_t q = 0; q < n_tokens; q++) {
+ for (int32_t k = 0; k < n_tokens; k++) {
+ mask[q * n_tokens + k] = k <= q ? 0.0f : -INFINITY;
+ }
+ }
+
+ struct ggml_tensor *cur = ggml_get_rows(ctx, tok_embd, inp_tokens);
+
+ for (int32_t il = 0; il < hp->n_layer; il++) {
+ struct ggml_tensor *inp_sa = cur;
+
+ struct ggml_tensor *attn_norm = el_weight(h->weights, "blk.%d.attn_norm.weight", il);
+ struct ggml_tensor *wq = el_weight(h->weights, "blk.%d.attn_q.weight", il);
+ struct ggml_tensor *bq = el_weight(h->weights, "blk.%d.attn_q.bias", il);
+ struct ggml_tensor *wk = el_weight(h->weights, "blk.%d.attn_k.weight", il);
+ struct ggml_tensor *bk = el_weight(h->weights, "blk.%d.attn_k.bias", il);
+ struct ggml_tensor *wv = el_weight(h->weights, "blk.%d.attn_v.weight", il);
+ struct ggml_tensor *bv = el_weight(h->weights, "blk.%d.attn_v.bias", il);
+ struct ggml_tensor *wo = el_weight(h->weights, "blk.%d.attn_output.weight", il);
+ struct ggml_tensor *ffn_norm = el_weight(h->weights, "blk.%d.ffn_norm.weight", il);
+ struct ggml_tensor *w_gate = el_weight(h->weights, "blk.%d.ffn_gate.weight", il);
+ struct ggml_tensor *w_up = el_weight(h->weights, "blk.%d.ffn_up.weight", il);
+ struct ggml_tensor *w_down = el_weight(h->weights, "blk.%d.ffn_down.weight", il);
+
+ if (!attn_norm || !wq || !bq || !wk || !bk || !wv || !bv || !wo ||
+ !ffn_norm || !w_gate || !w_up || !w_down) {
+ ggml_free(ctx);
+ return -1;
+ }
+
+ cur = el_rms_norm(ctx, cur, attn_norm, hp->rms_norm_eps);
+
+ struct ggml_tensor *q = ggml_add(ctx, ggml_mul_mat(ctx, wq, cur), bq);
+ struct ggml_tensor *k = ggml_add(ctx, ggml_mul_mat(ctx, wk, cur), bk);
+ struct ggml_tensor *v = ggml_add(ctx, ggml_mul_mat(ctx, wv, cur), bv);
+
+ q = ggml_reshape_3d(ctx, q, n_embd_head, hp->n_head, n_tokens);
+ k = ggml_reshape_3d(ctx, k, n_embd_head, hp->n_head_kv, n_tokens);
+ v = ggml_reshape_3d(ctx, v, n_embd_head, hp->n_head_kv, n_tokens);
+
+ q = ggml_rope_ext(ctx, q, inp_pos, NULL, n_embd_head, GGML_ROPE_TYPE_NEOX, 0,
+ hp->rope_freq_base, 1.0f, 0.0f, 1.0f, 0.0f, 0.0f);
+ k = ggml_rope_ext(ctx, k, inp_pos, NULL, n_embd_head, GGML_ROPE_TYPE_NEOX, 0,
+ hp->rope_freq_base, 1.0f, 0.0f, 1.0f, 0.0f, 0.0f);
+
+ q = ggml_permute(ctx, q, 0, 2, 1, 3);
+ k = ggml_permute(ctx, k, 0, 2, 1, 3);
+ v = ggml_permute(ctx, v, 0, 2, 1, 3);
+ v = ggml_cont(ctx, ggml_transpose(ctx, v));
+
+ struct ggml_tensor *kq = ggml_mul_mat(ctx, k, q);
+ kq = ggml_soft_max_ext(ctx, kq, kq_mask, 1.0f / sqrtf((float) n_embd_head), 0.0f);
+
+ struct ggml_tensor *kqv = ggml_mul_mat(ctx, v, kq);
+ kqv = ggml_permute(ctx, kqv, 0, 2, 1, 3);
+ kqv = ggml_cont_2d(ctx, kqv, n_embd_head * hp->n_head, n_tokens);
+
+ cur = ggml_mul_mat(ctx, wo, kqv);
+ cur = ggml_add(ctx, cur, inp_sa);
+
+ struct ggml_tensor *ffn_inp = cur;
+
+ cur = el_rms_norm(ctx, ffn_inp, ffn_norm, hp->rms_norm_eps);
+
+ struct ggml_tensor *gate = ggml_silu(ctx, ggml_mul_mat(ctx, w_gate, cur));
+ struct ggml_tensor *up = ggml_mul_mat(ctx, w_up, cur);
+ cur = ggml_mul(ctx, gate, up);
+ cur = ggml_mul_mat(ctx, w_down, cur);
+
+ cur = ggml_add(ctx, cur, ffn_inp);
+ }
+
+ cur = el_rms_norm(ctx, cur, output_norm, hp->rms_norm_eps);
+ cur = ggml_mul_mat(ctx, output, cur);
+
+ struct ggml_cgraph *gf = ggml_new_graph(ctx);
+ ggml_build_forward_expand(gf, cur);
+
+ ggml_graph_compute_with_ctx(ctx, gf, h->n_threads);
+
+ const float *logits = (const float *) cur->data;
+ const float *last_row = logits + (int64_t)(n_tokens - 1) * hp->n_vocab;
+
+ int32_t best_id = 0;
+ float best_logit = last_row[0];
+ for (int32_t i = 1; i < hp->n_vocab; i++) {
+ if (last_row[i] > best_logit) {
+ best_logit = last_row[i];
+ best_id = i;
+ }
+ }
+
+ ggml_free(ctx);
+ return best_id;
+}
+
+int32_t el_generate(void *handle, const int32_t *prompt, int32_t n_prompt,
+ int32_t max_tokens, int32_t eos_token_id, int32_t *out_ids) {
+ struct el_handle *h = handle;
+
+ int32_t capacity = n_prompt + max_tokens;
+ int32_t *tokens = malloc(sizeof(int32_t) * capacity);
+ if (!tokens) return -1;
+
+ memcpy(tokens, prompt, sizeof(int32_t) * n_prompt);
+
+ int32_t n_tokens = n_prompt;
+ int32_t n_generated = 0;
+
+ while (n_generated < max_tokens) {
+ int32_t next = el_forward_argmax(h, tokens, n_tokens);
+ if (next < 0) {
+ free(tokens);
+ return -1;
+ }
+
+ tokens[n_tokens] = next;
+ n_tokens++;
+ out_ids[n_generated] = next;
+ n_generated++;
+
+ if (next == eos_token_id) break;
+ }
+
+ free(tokens);
+ return n_generated;
+}
+
+void el_close(void *handle) {
+ if (!handle) return;
+ struct el_handle *h = handle;
+ if (h->weights) ggml_free(h->weights);
+ if (h->gguf) gguf_free(h->gguf);
+ free(h);
+}
ext/elelem/ggml/elelem.cpp
@@ -1,61 +0,0 @@
-// Minimal shim over raw ggml (ggml-org/ggml), exposed to Ruby via Fiddle.
-//
-// Unlike llama.cpp, ggml has no GGUF-to-chat-model inference built in: no
-// tokenizer, no chat templates, no sampler chain, no tool-calling. This shim
-// currently only proves out the ggml_context/tensor lifecycle end to end
-// (el_open allocates a ggml context, el_close frees it) so the Ruby <-> C++
-// plumbing (Provider -> Fiddle -> this shim) can be exercised.
-//
-// TODO(real inference): to make this a working provider, this file needs to:
-// 1. Parse a GGUF file directly (ggml/include/gguf.h) to read tensor data
-// and hyperparameters -- there is no shared "model" abstraction like
-// llama_model, so this is manual per-architecture work.
-// 2. Build the computation graph for one transformer architecture (see
-// ggml/examples/gpt-2 or gpt-j for a worked reference) using
-// ggml_build_forward_expand + ggml_graph_compute.
-// 3. Implement tokenization (ggml itself doesn't provide a BPE/SPM
-// tokenizer -- llama.cpp vendors its own for this reason).
-// 4. Implement sampling (top-k/top-p/temp) over the output logits tensor.
-//
-// This is meaningfully more work than elelem-llama or elelem-candle, since
-// both of those depend on libraries that already include a full inference
-// stack; ggml is the tensor library underneath that stack, not the stack.
-
-#include <cstdlib>
-#include <cstring>
-#include <string>
-
-#include "ggml.h"
-
-struct el_handle {
- ggml_context *ctx;
-};
-
-extern "C" void *el_open(const char *model_path, int /*n_gpu_layers*/, int /*n_ctx*/, int /*n_threads*/) {
- (void) model_path; // TODO: gguf_init_from_file once GGUF parsing is wired up
-
- ggml_init_params params = {};
- params.mem_size = 16 * 1024 * 1024;
- params.mem_buffer = nullptr;
- params.no_alloc = false;
-
- ggml_context *ctx = ggml_init(params);
- if (!ctx) return nullptr;
-
- el_handle *h = new el_handle{ctx};
- return h;
-}
-
-extern "C" const char *el_version(void *handle) {
- (void) handle;
- static thread_local std::string buf;
- buf = "elelem-ggml shim over ggml (no model loaded -- scaffold only)";
- return buf.c_str();
-}
-
-extern "C" void el_close(void *handle) {
- if (!handle) return;
- el_handle *h = static_cast<el_handle *>(handle);
- if (h->ctx) ggml_free(h->ctx);
- delete h;
-}
ext/elelem/ggml/extconf.rb
@@ -19,10 +19,6 @@ unless File.exist?(File.join(VENDOR, "CMakeLists.txt"))
abort "elelem-ggml: vendored ggml missing at #{VENDOR}\n run: git submodule update --init --recursive"
end
-# CPU-only for now (GGML_NATIVE is ON by default -> host CPU SIMD via
-# -march=native). GPU backends (Vulkan/CUDA/HIP) can follow the same
-# detect_backend pattern as elelem-llama's extconf.rb once this skeleton
-# actually loads a model.
run(
"cmake", "-S", VENDOR, "-B", BUILD_DIR,
"-DCMAKE_BUILD_TYPE=Release",
@@ -30,7 +26,7 @@ run(
"-DGGML_BUILD_TESTS=OFF",
"-DGGML_BUILD_EXAMPLES=OFF"
)
-run("cmake", "--build", BUILD_DIR, "--target", "ggml", "--parallel")
+run("cmake", "--build", BUILD_DIR, "--target", "ggml", "--target", "ggml-cpu", "--parallel")
libdir = File.join(BUILD_DIR, "src") # ggml's CMake puts libggml*.so here (not bin/, unlike llama.cpp)
@@ -38,12 +34,12 @@ FileUtils.mkdir_p(NATIVE)
FileUtils.rm_f(Dir.glob(File.join(NATIVE, "*.so*")))
FileUtils.cp(Dir.glob(File.join(libdir, "*.so*")), NATIVE)
-cxx = ENV["CXX"] || "c++"
-run(cxx, "-std=c++17", "-O2", "-Wall", "-Wextra", "-shared", "-fPIC",
+cc = ENV["CC"] || "cc"
+run(cc, "-std=c11", "-O2", "-Wall", "-Wextra", "-shared", "-fPIC",
"-I", File.join(VENDOR, "include"),
- File.join(EXT_DIR, "elelem.cpp"),
+ File.join(EXT_DIR, "elelem.c"),
"-o", File.join(NATIVE, "libelelem_ggml.so"),
- "-L", libdir, "-lggml", "-lggml-base",
+ "-L", libdir, "-lggml", "-lggml-base", "-lggml-cpu",
"-Wl,-rpath,$ORIGIN")
File.write(File.join(EXT_DIR, "Makefile"), <<~MAKE)
lib/elelem/ggml/chat_ml.rb
@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module Elelem
+ module Ggml
+ class ChatMl
+ DEFAULT_SYSTEM = "You are Qwen, created by Alibaba Cloud. You are a helpful assistant."
+
+ def self.prompt(messages)
+ messages = with_default_system(messages)
+ turns = messages.map { |m| "<|im_start|>#{m[:role] || m['role']}\n#{m[:content] || m['content']}<|im_end|>\n" }
+ "#{turns.join}<|im_start|>assistant\n"
+ end
+
+ def self.with_default_system(messages)
+ first = messages.first
+ return messages if first && (first[:role] || first["role"]) == "system"
+
+ [{ role: "system", content: DEFAULT_SYSTEM }] + messages
+ end
+ private_class_method :with_default_system
+ end
+ end
+end
lib/elelem/ggml/gguf.rb
@@ -0,0 +1,82 @@
+# frozen_string_literal: true
+
+module Elelem
+ module Ggml
+ class Gguf
+ TYPE_SIZES = {
+ 0 => 1, 1 => 1, 2 => 2, 3 => 2, 4 => 4, 5 => 4, 6 => 4, 7 => 1,
+ 10 => 8, 11 => 8, 12 => 8
+ }.freeze
+ TYPE_UNPACK = {
+ 0 => "C", 1 => "c", 2 => "S<", 3 => "s<", 4 => "L<", 5 => "l<",
+ 6 => "e", 7 => "C", 10 => "Q<", 11 => "q<", 12 => "E"
+ }.freeze
+ TYPE_STRING = 8
+ TYPE_ARRAY = 9
+
+ def self.metadata(path)
+ File.open(path, "rb") { |io| new(io).metadata }
+ end
+
+ def initialize(io)
+ @io = io
+ end
+
+ def metadata
+ magic = @io.read(4)
+ raise "not a GGUF file" unless magic == "GGUF"
+
+ @io.read(4)
+ read_u64
+ n_kv = read_u64
+
+ n_kv.times.to_h { [read_string, read_value] }
+ end
+
+ private
+
+ def read_value
+ read_typed_value(read_i32)
+ end
+
+ def read_typed_value(type)
+ case type
+ when TYPE_STRING then read_string
+ when TYPE_ARRAY then read_array
+ else read_scalar(type)
+ end
+ end
+
+ def read_array
+ element_type = read_i32
+ n = read_u64
+
+ if element_type == TYPE_STRING
+ Array.new(n) { read_string }
+ else
+ size = TYPE_SIZES.fetch(element_type)
+ fmt = TYPE_UNPACK.fetch(element_type)
+ @io.read(size * n).unpack(fmt * n)
+ end
+ end
+
+ def read_scalar(type)
+ size = TYPE_SIZES.fetch(type)
+ fmt = TYPE_UNPACK.fetch(type)
+ @io.read(size).unpack1(fmt)
+ end
+
+ def read_string
+ @io.read(read_u64)
+ end
+
+ def read_u64
+ @io.read(8).unpack1("Q<")
+ end
+
+ def read_i32
+ @io.read(4).unpack1("l<")
+ end
+ end
+ end
+end
lib/elelem/ggml/provider.rb
@@ -14,24 +14,47 @@ module Elelem
{
open: Fiddle::Function.new(lib["el_open"], [V, I, I, I], V),
version: Fiddle::Function.new(lib["el_version"], [V], V),
+ generate: Fiddle::Function.new(lib["el_generate"], [V, V, I, I, I, V], I),
close: Fiddle::Function.new(lib["el_close"], [V], Fiddle::TYPE_VOID)
}
end
end
- def initialize(model:, n_ctx: 0, n_threads: 0, n_gpu_layers: 0)
+ def self.close_at_exit(handle)
+ @open_handles ||= []
+ @open_handles << handle
+ return if @exit_hook_installed
+
+ @exit_hook_installed = true
+ at_exit { @open_handles.each { |h| functions[:close].call(h) } }
+ end
+
+ def initialize(model:, n_ctx: 0, n_threads: 0, n_gpu_layers: 0, max_tokens: 256)
@handle = self.class.functions[:open].call(model, n_gpu_layers, n_ctx, n_threads)
- raise "ggml: failed to init context for #{model}" if @handle.null?
+ raise "ggml: failed to load model at #{model}" if @handle.null?
+
+ self.class.close_at_exit(@handle)
- at_exit { self.class.functions[:close].call(@handle) }
+ @max_tokens = max_tokens
+ metadata = Gguf.metadata(model)
+ @tokenizer = Tokenizer.new(metadata)
+ @eos_token_id = metadata.fetch("tokenizer.ggml.eos_token_id")
end
- def fetch(_messages, _tools = [], &block)
- # TODO: real GGUF loading + forward pass + sampling (see elelem.cpp).
- # This proves the FFI round-trip so Provider wiring can be tested end
- # to end before real inference exists.
- ptr = self.class.functions[:version].call(@handle)
- text = "[elelem-ggml scaffold] #{Fiddle::Pointer.new(ptr).to_s}"
+ def fetch(messages, _tools = [], &block)
+ prompt_ids = @tokenizer.encode(ChatMl.prompt(messages))
+
+ packed_prompt = prompt_ids.pack("l*")
+ out = " " * (4 * @max_tokens)
+ n_generated = self.class.functions[:generate].call(
+ @handle, packed_prompt, prompt_ids.length, @max_tokens, @eos_token_id, out
+ )
+ raise "ggml: generation failed" if n_generated.negative?
+
+ generated_ids = out.unpack("l*").first(n_generated)
+ generated_ids.pop if generated_ids.last == @eos_token_id
+ text = @tokenizer.decode(generated_ids)
+
block&.call(type: "saying", text: text)
[]
end
lib/elelem/ggml/tokenizer.rb
@@ -0,0 +1,95 @@
+# frozen_string_literal: true
+
+module Elelem
+ module Ggml
+ class Tokenizer
+ SPLIT_PATTERN = /(?:'[sS]|'[tT]|'[rR][eE]|'[vV][eE]|'[mM]|'[lL][lL]|'[dD])|[^\r\n\p{L}\p{N}]?\p{L}+|\p{N}| ?[^\s\p{L}\p{N}]+[\r\n]*|\s*[\r\n]+|\s+(?!\S)|\s+/.freeze
+
+ def self.byte_encoder
+ @byte_encoder ||= begin
+ printable = (0x21..0x7E).to_a + (0xA1..0xAC).to_a + (0xAE..0xFF).to_a
+ table = {}
+ printable.each { |b| table[b] = b }
+ next_code = 256
+ (0..255).each do |b|
+ next if table.key?(b)
+
+ table[b] = next_code
+ next_code += 1
+ end
+ table.transform_values { |cp| [cp].pack("U") }
+ end
+ end
+
+ def self.byte_decoder
+ @byte_decoder ||= byte_encoder.to_h { |byte, char| [char, byte] }
+ end
+
+ def initialize(metadata)
+ @tokens = metadata.fetch("tokenizer.ggml.tokens").map { |t| t.dup.force_encoding("UTF-8") }
+ @token_ids = @tokens.each_with_index.to_h
+ @merge_ranks = metadata.fetch("tokenizer.ggml.merges").each_with_index.to_h do |line, rank|
+ [line.force_encoding("UTF-8").split(" ", 2), rank]
+ end
+ @special_ids = special_token_ids(metadata)
+ end
+
+ def encode(text)
+ text.scan(/<\|[^|]*\|>|[^<]+|</m).flat_map do |chunk|
+ if @special_ids.key?(chunk)
+ [@special_ids[chunk]]
+ else
+ encode_ordinary(chunk)
+ end
+ end
+ end
+
+ def decode(ids)
+ bytes = ids.flat_map { |id| (@tokens[id] || "").each_char.map { |c| self.class.byte_decoder.fetch(c) } }
+ bytes.pack("C*").force_encoding("UTF-8")
+ end
+
+ private
+
+ def special_token_ids(metadata)
+ types = metadata["tokenizer.ggml.token_type"] || []
+ @tokens.each_with_index.each_with_object({}) do |(tok, id), memo|
+ memo[tok] = id if types[id] == 3
+ end
+ end
+
+ def encode_ordinary(text)
+ text.scan(SPLIT_PATTERN).flat_map { |piece| bpe(to_byte_string(piece)) }
+ end
+
+ def to_byte_string(piece)
+ piece.b.each_byte.map { |b| self.class.byte_encoder.fetch(b) }.join
+ end
+
+ def bpe(piece)
+ symbols = piece.each_char.to_a
+ return [] if symbols.empty?
+
+ loop do
+ best_rank = nil
+ best_index = nil
+
+ (symbols.length - 1).times do |i|
+ rank = @merge_ranks[[symbols[i], symbols[i + 1]]]
+ next unless rank
+ next unless best_rank.nil? || rank < best_rank
+
+ best_rank = rank
+ best_index = i
+ end
+
+ break unless best_index
+
+ symbols[best_index, 2] = symbols[best_index] + symbols[best_index + 1]
+ end
+
+ symbols.map { |s| @token_ids.fetch(s) }
+ end
+ end
+ end
+end
lib/elelem/ggml.rb
@@ -5,6 +5,9 @@ require "json"
require "elelem"
require_relative "ggml/version"
+require_relative "ggml/gguf"
+require_relative "ggml/tokenizer"
+require_relative "ggml/chat_ml"
require_relative "ggml/provider"
Elelem.configure do |config|
spec/elelem/ggml/chat_ml_spec.rb
@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+RSpec.describe Elelem::Ggml::ChatMl do
+ describe ".prompt" do
+ it "injects the default system message when none is given" do
+ prompt = described_class.prompt([{ role: "user", content: "hi" }])
+
+ expect(prompt).to eq(
+ "<|im_start|>system\n" \
+ "You are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\n" \
+ "<|im_start|>user\nhi<|im_end|>\n" \
+ "<|im_start|>assistant\n"
+ )
+ end
+
+ it "preserves a caller-supplied system message" do
+ messages = [{ role: "system", content: "Be terse." }, { role: "user", content: "hi" }]
+
+ prompt = described_class.prompt(messages)
+
+ expect(prompt).to eq("<|im_start|>system\nBe terse.<|im_end|>\n<|im_start|>user\nhi<|im_end|>\n<|im_start|>assistant\n")
+ end
+ end
+
+ describe "tokenized against the real vocab" do
+ it "produces the exact pinned prompt token sequence" do
+ fixture = File.expand_path("../../fixtures/qwen2-vocab.gguf", __dir__)
+ metadata = Elelem::Ggml::Gguf.metadata(fixture)
+ tokenizer = Elelem::Ggml::Tokenizer.new(metadata)
+
+ prompt = described_class.prompt([{ role: "user", content: "hi" }])
+ ids = tokenizer.encode(prompt)
+
+ expect(ids).to eq([
+ 151644, 8948, 198, 2610, 525, 1207, 16948, 11, 3465, 553, 54364, 14817, 13,
+ 1446, 525, 264, 10950, 17847, 13, 151645, 198, 151644, 872, 198, 6023, 151645,
+ 198, 151644, 77091, 198
+ ])
+ end
+ end
+end
spec/elelem/ggml/provider_spec.rb
@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+RSpec.describe Elelem::Ggml::Provider do
+ describe ".new" do
+ it "raises when the model path cannot be loaded" do
+ expect { described_class.new(model: "/nonexistent/model.gguf") }.to raise_error(/failed to load/)
+ end
+
+ it "reports the model hyperparameters via version", :real_model do
+ model = ENV.fetch("ELELEM_GGML_TEST_MODEL")
+ handle = described_class.functions[:open].call(model, 0, 0, 0)
+
+ text = Fiddle::Pointer.new(described_class.functions[:version].call(handle)).to_s
+
+ expect(text).to include("arch=qwen2")
+ expect(text).to include("n_layer=24")
+ expect(text).to include("n_embd=896")
+ expect(text).to include("n_head=14")
+ expect(text).to include("n_head_kv=2")
+ expect(text).to include("n_ff=4864")
+ expect(text).to include("n_vocab=151936")
+
+ described_class.functions[:close].call(handle)
+ end
+ end
+
+ describe "#fetch", :real_model do
+ it "generates a greedy completion for a simple prompt" do
+ provider = described_class.new(model: ENV.fetch("ELELEM_GGML_TEST_MODEL"), max_tokens: 8)
+ messages = []
+
+ provider.fetch([{ role: "user", content: "hi" }]) { |msg| messages << msg }
+
+ expect(messages.first[:type]).to eq("saying")
+ expect(messages.first[:text]).to start_with("Hello")
+ end
+ end
+end
spec/elelem/ggml/tokenizer_spec.rb
@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+RSpec.describe Elelem::Ggml::Tokenizer do
+ let(:fixture) { File.expand_path("../../fixtures/qwen2-vocab.gguf", __dir__) }
+ let(:metadata) { Elelem::Ggml::Gguf.metadata(fixture) }
+ let(:tokenizer) { described_class.new(metadata) }
+
+ describe "#encode" do
+ it "encodes a single-token word" do
+ expect(tokenizer.encode("hi")).to eq([6023])
+ end
+
+ it "encodes a ChatML-templated prompt" do
+ prompt = "<|im_start|>user\nhi<|im_end|>\n<|im_start|>assistant\n"
+
+ expect(tokenizer.encode(prompt)).to eq([151644, 872, 198, 6023, 151645, 198, 151644, 77091, 198])
+ end
+
+ it "encodes a sentence requiring multi-step BPE merges" do
+ expect(tokenizer.encode("Hello world, this is a test.")).to eq([9707, 1879, 11, 419, 374, 264, 1273, 13])
+ end
+
+ it "encodes text spanning multiple lines" do
+ expect(tokenizer.encode("line one\nline two")).to eq([1056, 825, 198, 1056, 1378])
+ end
+
+ it "encodes runs of indentation-like whitespace" do
+ expect(tokenizer.encode(" indented spaces")).to eq([220, 1257, 15864, 256, 12621])
+ end
+ end
+
+ describe "#decode" do
+ it "round-trips a sentence back to its original text" do
+ ids = tokenizer.encode("Hello world, this is a test.")
+
+ expect(tokenizer.decode(ids)).to eq("Hello world, this is a test.")
+ end
+
+ it "round-trips non-ASCII text as UTF-8" do
+ ids = tokenizer.encode("こんにちは")
+
+ decoded = tokenizer.decode(ids)
+
+ expect(decoded).to eq("こんにちは")
+ expect(decoded.encoding).to eq(Encoding::UTF_8)
+ end
+ end
+end
spec/fixtures/qwen2-vocab.gguf
Binary file
spec/spec_helper.rb
@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+require "fiddle"
+require_relative "../lib/elelem/ggml/version"
+require_relative "../lib/elelem/ggml/provider"
+require_relative "../lib/elelem/ggml/gguf"
+require_relative "../lib/elelem/ggml/tokenizer"
+require_relative "../lib/elelem/ggml/chat_ml"
+
+RSpec.configure do |config|
+ config.disable_monkey_patching!
+ config.filter_run_excluding real_model: true unless ENV["ELELEM_GGML_TEST_MODEL"]
+
+ config.expect_with :rspec do |c|
+ c.syntax = :expect
+ end
+end
.rspec
@@ -0,0 +1,1 @@
+--require spec_helper
elelem-ggml.gemspec
@@ -21,13 +21,14 @@ Gem::Specification.new do |spec|
"LICENSE.txt",
"lib/elelem/ggml.rb",
"lib/elelem/ggml/version.rb",
+ "lib/elelem/ggml/gguf.rb",
+ "lib/elelem/ggml/tokenizer.rb",
+ "lib/elelem/ggml/chat_ml.rb",
"lib/elelem/ggml/provider.rb",
"exe/elelem-ggml",
- "ext/elelem/ggml/elelem.cpp",
+ "ext/elelem/ggml/elelem.c",
"ext/elelem/ggml/extconf.rb",
]
- # Only what extconf.rb's cmake build actually needs to configure and
- # compile: the ggml library itself, no tests/examples/docs.
vendor_exclude = %r{\Avendor/ggml/(tests|docs|examples|\.github|ci)/}
spec.files += Dir["vendor/ggml/**/*"].select do |path|
File.file?(path) && path !~ vendor_exclude
Gemfile
@@ -6,3 +6,4 @@ gemspec name: "elelem-ggml"
gem "irb"
gem "rake", "~> 13.0"
+gem "rspec", "~> 3.0"
Gemfile.lock
@@ -0,0 +1,129 @@
+PATH
+ remote: .
+ specs:
+ elelem-ggml (0.1.0)
+ elelem (~> 0.11)
+ fiddle (~> 1.1)
+ json (~> 3.0)
+
+GEM
+ remote: https://rubygems.org/
+ specs:
+ bigdecimal (4.1.2)
+ diff-lcs (1.6.2)
+ elelem (0.11.0)
+ erb (~> 6.0)
+ forwardable (~> 1.4)
+ io-console (~> 0.9)
+ json (~> 3.0)
+ json_schemer (~> 2.5)
+ logger (~> 1.7)
+ open3 (~> 0.2)
+ optparse (~> 0.8)
+ pathname (~> 0.5)
+ reline (~> 0.7)
+ shellwords (~> 0.2)
+ uri (~> 1.0)
+ erb (6.0.7)
+ fiddle (1.1.8)
+ forwardable (1.4.0)
+ hana (1.3.7)
+ io-console (0.9.2)
+ irb (1.18.0)
+ pp (>= 0.6.0)
+ prism (>= 1.3.0)
+ rdoc (>= 4.0.0)
+ reline (>= 0.4.2)
+ json (3.0.1)
+ json_schemer (2.5.0)
+ bigdecimal
+ hana (~> 1.3)
+ regexp_parser (~> 2.0)
+ simpleidn (~> 0.2)
+ logger (1.7.0)
+ open3 (0.2.1)
+ optparse (0.8.1)
+ pathname (0.5.0)
+ pp (0.6.4)
+ prettyprint
+ prettyprint (0.2.0)
+ prism (1.9.0)
+ rake (13.4.2)
+ rbs (4.2.0)
+ logger
+ prism (>= 1.6.0)
+ tsort
+ rdoc (8.0.0)
+ erb
+ prism (>= 1.6.0)
+ rbs (>= 4.0.0)
+ tsort
+ regexp_parser (2.12.0)
+ reline (0.7.0)
+ io-console (~> 0.5)
+ rspec (3.13.2)
+ rspec-core (~> 3.13.0)
+ rspec-expectations (~> 3.13.0)
+ rspec-mocks (~> 3.13.0)
+ rspec-core (3.13.6)
+ rspec-support (~> 3.13.0)
+ rspec-expectations (3.13.5)
+ diff-lcs (>= 1.2.0, < 2.0)
+ rspec-support (~> 3.13.0)
+ rspec-mocks (3.13.8)
+ diff-lcs (>= 1.2.0, < 2.0)
+ rspec-support (~> 3.13.0)
+ rspec-support (3.13.7)
+ shellwords (0.2.2)
+ simpleidn (0.3.0)
+ tsort (0.2.0)
+ uri (1.1.1)
+
+PLATFORMS
+ ruby
+ x86_64-linux
+
+DEPENDENCIES
+ elelem-ggml!
+ irb
+ rake (~> 13.0)
+ rspec (~> 3.0)
+
+CHECKSUMS
+ bigdecimal (4.1.2) sha256=53d217666027eab4280346fba98e7d5b66baaae1b9c3c1c0ffe89d48188a3fbd
+ bundler (4.0.20) sha256=7978a8ac648767f5e635bc522445b79e80a52b907a39a36c2d8085ed6bc762ae
+ diff-lcs (1.6.2)
+ elelem (0.11.0) sha256=552cafb092320e3896b8e07572d322fe2b94f0f10d965bf7b993b74517a6f744
+ elelem-ggml (0.1.0)
+ erb (6.0.7) sha256=c5ca6dc25b0ef974a44dc8f59fe847577122483b1968a38dec305c60bf91ee92
+ fiddle (1.1.8) sha256=7fa8ee3627271497f3add5503acdbc3f40b32f610fc1cf49634f083ef3f32eee
+ forwardable (1.4.0) sha256=f1cd40cc9812937980e1c76f1aa053660990a7c9b6a98fc37d945468afcce838
+ hana (1.3.7) sha256=5425db42d651fea08859811c29d20446f16af196308162894db208cac5ce9b0d
+ io-console (0.9.2) sha256=efa74f891dd03c0939a931dfc6e74c2813d904763d456ea9762b0525e748db08
+ irb (1.18.0) sha256=de9454a0703a54704b9811a5ef31a60c86949fbf4013fcf244fabc7c775248e3
+ json (3.0.1) sha256=c80f74a3570c3a310cd41c526904e7ef8363abdece4432f31fe97d812bc4a39e
+ json_schemer (2.5.0) sha256=2f01fb4cce721a4e08dd068fc2030cffd0702a7f333f1ea2be6e8991f00ae396
+ logger (1.7.0) sha256=196edec7cc44b66cfb40f9755ce11b392f21f7967696af15d274dde7edff0203
+ open3 (0.2.1) sha256=8e2d7d2113526351201438c1aa35c8139f0141c9e8913baa007c898973bf3952
+ optparse (0.8.1) sha256=42bea10d53907ccff4f080a69991441d611fbf8733b60ed1ce9ee365ce03bd1a
+ pathname (0.5.0) sha256=d5a331784f6e1f2fefb31c2ff0b8855aabfb661d807284ead9fa47b883d81623
+ pp (0.6.4) sha256=dfcb0fce700c41456265922884f9fe195d7fbb0674a3578e6c0f69588e82b570
+ prettyprint (0.2.0) sha256=2bc9e15581a94742064a3cc8b0fb9d45aae3d03a1baa6ef80922627a0766f193
+ prism (1.9.0) sha256=7b530c6a9f92c24300014919c9dcbc055bf4cdf51ec30aed099b06cd6674ef85
+ rake (13.4.2) sha256=cb825b2bd5f1f8e91ca37bddb4b9aaf345551b4731da62949be002fa89283701
+ rbs (4.2.0) sha256=51f7b886dcc05bc09e10b901daa6a81829f6adc03101d6ca9ea4aac6103e0674
+ rdoc (8.0.0) sha256=03bf8c08a9639658855a0cfd77c0abca8325c227693f7f33f82957811348c469
+ regexp_parser (2.12.0) sha256=35a916a1d63190ab5c9009457136ae5f3c0c7512d60291d0d1378ba18ce08ebb
+ reline (0.7.0) sha256=5b012d8e55dbf9d450f12bde2cf7d15ff546ae80b3f8f3b30e570d431815583d
+ rspec (3.13.2)
+ rspec-core (3.13.6)
+ rspec-expectations (3.13.5)
+ rspec-mocks (3.13.8)
+ rspec-support (3.13.7)
+ shellwords (0.2.2) sha256=b8695a791de2f71472de5abdc3f4332f6535a4177f55d8f99e7e44266cd32f94
+ simpleidn (0.3.0) sha256=12ca730bed2f3db04d11e9bfd1bca3e11fb37f55b21eb2e9793fb5814bf54d03
+ tsort (0.2.0) sha256=9650a793f6859a43b6641671278f79cfead60ac714148aabe4e3f0060480089f
+ uri (1.1.1) sha256=379fa58d27ffb1387eaada68c749d1426738bd0f654d812fcc07e7568f5c57c6
+
+BUNDLED WITH
+ 4.0.20
Rakefile
@@ -1,10 +1,13 @@
# frozen_string_literal: true
require "bundler/gem_tasks"
+require "rspec/core/rake_task"
desc "Build the in-process ggml extension (the same build `gem install` runs)"
task :compile do
ruby "ext/elelem/ggml/extconf.rb"
end
-task default: %i[]
+RSpec::Core::RakeTask.new(:spec)
+
+task default: %i[spec]
README.md
@@ -5,26 +5,18 @@ directly by [ggml-org/ggml](https://github.com/ggml-org/ggml), the tensor
library llama.cpp itself is built on -- vendored as a git submodule at
`vendor/ggml`, same pattern as `elelem-llama`'s `vendor/llama.cpp`.
-## Status: scaffold only
-
-This is currently a skeleton, and meaningfully less complete than
-`elelem-llama`. ggml is a tensor library, not an LLM inference stack: it has
-no GGUF-to-chat-model loader, no tokenizer, no chat templates, no sampler
-chain, no tool-calling -- all of the things llama.cpp itself builds on top of
-ggml. `el_open`/`el_close` in `ext/elelem/ggml/elelem.cpp` currently only
-prove out a `ggml_context` lifecycle round trip over Fiddle; there is no
-model loading or inference yet.
-
-See the TODOs in `ext/elelem/ggml/elelem.cpp` for what real inference would
-require:
-
-1. Parse a GGUF file directly (`ggml/include/gguf.h`) -- there's no shared
- "model" abstraction, so this is manual per-architecture work.
-2. Build the computation graph for one transformer architecture (see
- `vendor/ggml/examples/gpt-2` or `gpt-j` for worked references) via
- `ggml_build_forward_expand` + `ggml_graph_compute`.
-3. Implement tokenization yourself (ggml doesn't ship a tokenizer).
-4. Implement sampling (top-k/top-p/temp) over the output logits.
+## Status: Qwen2 inference works end to end
+
+The C extension (`ext/elelem/ggml/elelem.c`) loads a GGUF file directly via
+`gguf.h`, builds a Qwen2 transformer graph by hand (RMSNorm, GQA attention
+with NEOX RoPE, SwiGLU FFN), and runs greedy generation with no KV cache. ggml
+itself has no tokenizer, so BPE tokenization and the ChatML template live in
+Ruby (`lib/elelem/ggml/tokenizer.rb`, `lib/elelem/ggml/chat_ml.rb`), parsing
+vocab and merges straight out of the GGUF metadata.
+
+Other architectures, a KV cache, real sampling (top-k/top-p/temp), and
+tool-calling are not implemented yet -- see `spec/` for what's currently
+verified.
## Setup
@@ -38,5 +30,5 @@ bundle exec rake compile
```
export ELELEM_GGML_MODEL=/path/to/model.gguf
-bundle exec exe/elelem-ggml ask "hi"
+ruby -Ilib exe/elelem-ggml ask "hi"
```