Commit `f59b200`

2026-09-08 04:11:18

scaffold elelem-llama2: karpathy/llama2.c provider plugin (vendored submodule)

Mirrors elelem-llama's layout (vendor/ submodule, extconf.rb, Fiddle shim) but backed by llama2.c instead of llama.cpp. Unlike elelem-ggml, this does real inference: elelem.cpp reuses run.c's build_transformer/forward/encode/ decode/sample functions in a proper streaming generate loop, verified end-to-end against stories15M.bin. No chat templates, tool-calling, or GGUF support -- llama2.c's own .bin checkpoint format only.

main

Changed files (14)

exe

elelem-llama2

ext

elelem

llama2

elelem.cpp

extconf.rb

lib

elelem

llama2

provider.rb

version.rb

llama2.rb

vendor

llama2.c

.gitignore

.gitmodules

elelem-llama2.gemspec

Gemfile

LICENSE.txt

Rakefile

README.md

exe/elelem-llama2

@@ -0,0 +1,8 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require "elelem/cli"
+
+Signal.trap("INT") { exit 1 }
+
+Elelem::CLI.new(ARGV, provider: "llama2").run

ext/elelem/llama2/elelem.cpp

@@ -0,0 +1,161 @@
+// Shim over karpathy/llama2.c's run.c, exposed to Ruby via Fiddle.
+//
+// Unlike llama.cpp/ggml, run.c has no build system or header -- it's a
+// single .c file with a `main()` we don't call. extconf.rb compiles it as a
+// plain object file and links it in here; the struct layouts and function
+// signatures below are copied from vendor/llama2.c/run.c (there is no .h to
+// #include). If upstream changes those signatures, this file needs updating
+// to match.
+//
+// This is a real (if minimal) implementation: encode -> forward -> sample ->
+// decode, streaming each generated piece to a Ruby-supplied callback -- not
+// a stub, unlike elelem-ggml's current scaffold. What it does NOT have is
+// llama.cpp's chat templates, tool-calling, or GGUF support: llama2.c only
+// understands its own .bin checkpoint format (see export.py) and the
+// original Llama 2 architecture.
+
+#include <cstdint>
+#include <cstdlib>
+#include <cstring>
+#include <string>
+
+// ---- copied type/function declarations from vendor/llama2.c/run.c ----
+
+typedef struct {
+  int dim, hidden_dim, n_layers, n_heads, n_kv_heads, vocab_size, seq_len;
+} Config;
+
+typedef struct {
+  float *token_embedding_table, *rms_att_weight, *rms_ffn_weight;
+  float *wq, *wk, *wv, *wo, *w1, *w2, *w3, *rms_final_weight, *wcls;
+} TransformerWeights;
+
+typedef struct {
+  float *x, *xb, *xb2, *hb, *hb2, *q, *k, *v, *att, *logits;
+  float *key_cache, *value_cache;
+} RunState;
+
+typedef struct {
+  Config config;
+  TransformerWeights weights;
+  RunState state;
+  int fd;
+  float *data;
+  long file_size;
+} Transformer;
+
+typedef struct {
+  char *str;
+  int id;
+} TokenIndex;
+
+typedef struct {
+  char **vocab;
+  float *vocab_scores;
+  TokenIndex *sorted_vocab;
+  int vocab_size;
+  unsigned int max_token_length;
+  unsigned char byte_pieces[512];
+} Tokenizer;
+
+typedef struct {
+  float prob;
+  int index;
+} ProbIndex;
+
+typedef struct {
+  int vocab_size;
+  ProbIndex *probindex;
+  float temperature;
+  float topp;
+  unsigned long long rng_state;
+} Sampler;
+
+extern "C" {
+void build_transformer(Transformer *t, char *checkpoint_path);
+void free_transformer(Transformer *t);
+float *forward(Transformer *transformer, int token, int pos);
+
+void build_tokenizer(Tokenizer *t, char *tokenizer_path, int vocab_size);
+void free_tokenizer(Tokenizer *t);
+char *decode(Tokenizer *t, int prev_token, int token);
+void encode(Tokenizer *t, char *text, int8_t bos, int8_t eos, int *tokens, int *n_tokens);
+
+void build_sampler(Sampler *sampler, int vocab_size, float temperature, float topp, unsigned long long rng_seed);
+void free_sampler(Sampler *sampler);
+int sample(Sampler *sampler, float *logits);
+}
+
+// ---- elelem shim ----
+
+struct el_handle {
+  Transformer transformer;
+  Tokenizer tokenizer;
+  Sampler sampler;
+  char *checkpoint_path;
+  char *tokenizer_path;
+};
+
+extern "C" void *el_open(const char *checkpoint_path, const char *tokenizer_path, float temperature, float topp, unsigned long long seed) {
+  el_handle *h = new el_handle{};
+  h->checkpoint_path = strdup(checkpoint_path);
+  h->tokenizer_path = strdup(tokenizer_path);
+
+  build_transformer(&h->transformer, h->checkpoint_path);
+  build_tokenizer(&h->tokenizer, h->tokenizer_path, h->transformer.config.vocab_size);
+  build_sampler(&h->sampler, h->transformer.config.vocab_size, temperature, topp, seed);
+
+  return h;
+}
+
+// Runs prompt -> tokens -> forward/sample loop, invoking on_token(userdata,
+// piece) once per generated piece (matches elelem-llama's streaming shape).
+extern "C" void el_generate(void *handle, const char *prompt, int steps,
+                             void (*on_token)(void *, const char *), void *userdata) {
+  el_handle *h = static_cast<el_handle *>(handle);
+  if (!h) return;
+
+  int max_steps = steps > 0 ? steps : h->transformer.config.seq_len;
+
+  int num_prompt_tokens = 0;
+  int *prompt_tokens = (int *) malloc((strlen(prompt) + 3) * sizeof(int));
+  encode(&h->tokenizer, const_cast<char *>(prompt), 1, 0, prompt_tokens, &num_prompt_tokens);
+
+  if (num_prompt_tokens < 1) {
+    free(prompt_tokens);
+    return;
+  }
+
+  int token = prompt_tokens[0];
+  int pos = 0;
+  while (pos < max_steps) {
+    float *logits = forward(&h->transformer, token, pos);
+
+    int next;
+    if (pos < num_prompt_tokens - 1) {
+      next = prompt_tokens[pos + 1];
+    } else {
+      next = sample(&h->sampler, logits);
+    }
+    pos++;
+
+    if (next == 1) break; // BOS token delimits sequences in llama2.c
+
+    char *piece = decode(&h->tokenizer, token, next);
+    if (on_token && piece) on_token(userdata, piece);
+    token = next;
+  }
+
+  free(prompt_tokens);
+}
+
+extern "C" void el_close(void *handle) {
+  if (!handle) return;
+  el_handle *h = static_cast<el_handle *>(handle);
+  free_sampler(&h->sampler);
+  free_tokenizer(&h->tokenizer);
+  free_transformer(&h->transformer);
+  free(h->checkpoint_path);
+  free(h->tokenizer_path);
+  delete h;
+}

ext/elelem/llama2/extconf.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+require "fileutils"
+
+EXT_DIR   = __dir__
+GEM_ROOT  = File.expand_path("../../..", EXT_DIR)
+VENDOR    = File.join(GEM_ROOT, "vendor", "llama2.c")
+BUILD_DIR = File.join(EXT_DIR, "build")
+NATIVE    = File.join(GEM_ROOT, "ext", "elelem", "llama2", "native")
+
+class BuildError < StandardError; end
+
+def run(*cmd)
+  warn "elelem-llama2: + #{cmd.join(' ')}"
+  system(*cmd) || raise(BuildError, "build step failed: #{cmd.join(' ')}")
+end
+
+unless File.exist?(File.join(VENDOR, "run.c"))
+  abort "elelem-llama2: vendored llama2.c missing at #{VENDOR}\n        run: git submodule update --init --recursive"
+end
+
+FileUtils.mkdir_p(BUILD_DIR)
+FileUtils.mkdir_p(NATIVE)
+
+cc  = ENV["CC"] || "cc"
+cxx = ENV["CXX"] || "c++"
+
+# run.c has its own `main()` we never call (we only link its build_transformer
+# /forward/encode/decode/sample/build_* functions); harmless to leave in a
+# shared library, just never invoked.
+run(cc, "-std=gnu11", "-O3", "-fPIC", "-c", File.join(VENDOR, "run.c"),
+    "-o", File.join(BUILD_DIR, "run.o"))
+
+FileUtils.rm_f(Dir.glob(File.join(NATIVE, "*.so*")))
+run(cxx, "-std=c++17", "-O2", "-Wall", "-Wextra", "-shared", "-fPIC",
+    File.join(EXT_DIR, "elelem.cpp"), File.join(BUILD_DIR, "run.o"),
+    "-o", File.join(NATIVE, "libelelem_llama2.so"), "-lm")
+
+File.write(File.join(EXT_DIR, "Makefile"), <<~MAKE)
+  all:
+  clean:
+  install:
+  .PHONY: all clean install
+MAKE
+
+warn "elelem-llama2: built llama2.c + shim into #{NATIVE}"

lib/elelem/llama2/provider.rb

@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+module Elelem
+  module Llama2
+    class Provider
+      NATIVE = File.expand_path("../../../ext/elelem/llama2/native", __dir__)
+      SHIM = File.join(NATIVE, "libelelem_llama2.so")
+      V = Fiddle::TYPE_VOIDP
+      I = Fiddle::TYPE_INT
+      F = Fiddle::TYPE_FLOAT
+      L = Fiddle::TYPE_LONG_LONG
+
+      def self.functions
+        @functions ||= begin
+          lib = Fiddle.dlopen(SHIM)
+          {
+            open: Fiddle::Function.new(lib["el_open"], [V, V, F, F, L], V),
+            generate: Fiddle::Function.new(lib["el_generate"], [V, V, I, V, V], Fiddle::TYPE_VOID, need_gvl: true),
+            close: Fiddle::Function.new(lib["el_close"], [V], Fiddle::TYPE_VOID)
+          }
+        end
+      end
+
+      # checkpoint/tokenizer are llama2.c's own .bin formats (see
+      # vendor/llama2.c/export.py + tokenizer.bin), not GGUF.
+      def initialize(checkpoint:, tokenizer:, steps: 0, temperature: 1.0, topp: 0.9, seed: -1)
+        @steps = steps
+        seed = seed.negative? ? Process.clock_gettime(Process::CLOCK_REALTIME, :nanosecond) : seed
+        @handle = self.class.functions[:open].call(checkpoint, tokenizer, temperature, topp, seed)
+        raise "llama2: failed to load checkpoint at #{checkpoint}" if @handle.null?
+
+        at_exit { self.class.functions[:close].call(@handle) }
+
+        @block = nil
+        @on_token = Fiddle::Closure::BlockCaller.new(Fiddle::TYPE_VOID, [V, V]) do |_userdata, piece|
+          text = Fiddle::Pointer.new(piece).to_s.force_encoding(Encoding::UTF_8).scrub
+          @block&.call(type: "thinking", text: text)
+        rescue Exception => e # rubocop:disable Lint/RescueException
+          Elelem.logger.warn("llama2: streaming callback failed: #{e.message}")
+        end
+      end
+
+      # llama2.c has no chat template or tool-calling support: this
+      # concatenates message content into a single prompt string.
+      def fetch(messages, _tools = [], &block)
+        @block = block
+        prompt = messages.map { |m| (m[:content] || m["content"]).to_s }.join("\n")
+
+        self.class.functions[:generate].call(@handle, prompt, @steps, @on_token, nil)
+
+        []
+      end
+    end
+  end
+end

lib/elelem/llama2/version.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+module Elelem
+  module Llama2
+    VERSION = "0.1.0"
+  end
+end

lib/elelem/llama2.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+require "fiddle"
+require "json"
+require "elelem"
+
+require_relative "llama2/version"
+require_relative "llama2/provider"
+
+module Elelem
+  module Llama2
+    DEFAULT_TOKENIZER = File.expand_path("../../vendor/llama2.c/tokenizer.bin", __dir__)
+  end
+end
+
+Elelem.configure do |config|
+  config.provider(:llama2) do
+    Elelem::Llama2::Provider.new(
+      checkpoint: ENV.fetch("ELELEM_LLAMA2_CHECKPOINT") { raise "set ELELEM_LLAMA2_CHECKPOINT=/path/to/model.bin" },
+      tokenizer: ENV.fetch("ELELEM_LLAMA2_TOKENIZER", Elelem::Llama2::DEFAULT_TOKENIZER),
+      steps: Integer(ENV.fetch("ELELEM_LLAMA2_STEPS", "0")),
+      temperature: Float(ENV.fetch("ELELEM_LLAMA2_TEMP", "1.0")),
+      topp: Float(ENV.fetch("ELELEM_LLAMA2_TOPP", "0.9")),
+      seed: Integer(ENV.fetch("ELELEM_LLAMA2_SEED", "-1"))
+    )
+  end
+end

vendor/llama2.c

@@ -0,0 +1,1 @@
+Subproject commit 350e04fe35433e6d2941dce5a1f53308f87058eb