Commit 408739c

@@ -20,6 +20,7 @@ struct el_hparams {
   int32_t n_head_kv;
   int32_t n_ff;
   int32_t n_vocab;
+  int32_t n_rot;
   float rope_freq_base;
   float rms_norm_eps;
 };
@@ -95,6 +96,9 @@ static void el_load_hparams(struct gguf_context *gguf, struct el_hparams *hp, in
   snprintf(key, sizeof(key), "%s.rope.freq_base", arch);
   hp->rope_freq_base = el_kv_f32(gguf, key, 10000.0f);
 
+  snprintf(key, sizeof(key), "%s.rope.dimension_count", arch);
+  hp->n_rot = el_kv_i32(gguf, key, hp->n_head > 0 ? hp->n_embd / hp->n_head : 0);
+
   snprintf(key, sizeof(key), "%s.attention.layer_norm_rms_epsilon", arch);
   hp->rms_norm_eps = el_kv_f32(gguf, key, 1e-6f);
 
@@ -229,6 +233,7 @@ static int32_t el_forward_argmax(struct el_handle *h, const int32_t *new_tokens,
   struct ggml_tensor *tok_embd = el_weight(h->weights, "token_embd.weight", -1);
   struct ggml_tensor *output_norm = el_weight(h->weights, "output_norm.weight", -1);
   struct ggml_tensor *output = el_weight(h->weights, "output.weight", -1);
+  if (!output) output = tok_embd;
   if (!tok_embd || !output_norm || !output) {
     ggml_free(ctx);
     return -1;
@@ -270,7 +275,7 @@ static int32_t el_forward_argmax(struct el_handle *h, const int32_t *new_tokens,
     struct ggml_tensor *w_up = el_weight(h->weights, "blk.%d.ffn_up.weight", il);
     struct ggml_tensor *w_down = el_weight(h->weights, "blk.%d.ffn_down.weight", il);
 
-    if (!attn_norm || !wq || !bq || !wk || !bk || !wv || !bv || !wo ||
+    if (!attn_norm || !wq || !wk || !wv || !wo ||
         !ffn_norm || !w_gate || !w_up || !w_down) {
       ggml_free(ctx);
       return -1;
@@ -278,17 +283,20 @@ static int32_t el_forward_argmax(struct el_handle *h, const int32_t *new_tokens,
 
     cur = el_rms_norm(ctx, cur, attn_norm, hp->rms_norm_eps);
 
-    struct ggml_tensor *q = ggml_add(ctx, ggml_mul_mat(ctx, wq, cur), bq);
-    struct ggml_tensor *k = ggml_add(ctx, ggml_mul_mat(ctx, wk, cur), bk);
-    struct ggml_tensor *v = ggml_add(ctx, ggml_mul_mat(ctx, wv, cur), bv);
+    struct ggml_tensor *q = ggml_mul_mat(ctx, wq, cur);
+    struct ggml_tensor *k = ggml_mul_mat(ctx, wk, cur);
+    struct ggml_tensor *v = ggml_mul_mat(ctx, wv, cur);
+    if (bq) q = ggml_add(ctx, q, bq);
+    if (bk) k = ggml_add(ctx, k, bk);
+    if (bv) v = ggml_add(ctx, v, bv);
 
     q = ggml_reshape_3d(ctx, q, n_embd_head, hp->n_head, n_new);
     k = ggml_reshape_3d(ctx, k, n_embd_head, hp->n_head_kv, n_new);
     v = ggml_reshape_3d(ctx, v, n_embd_head, hp->n_head_kv, n_new);
 
-    q = ggml_rope_ext(ctx, q, inp_pos, NULL, n_embd_head, GGML_ROPE_TYPE_NEOX, 0,
+    q = ggml_rope_ext(ctx, q, inp_pos, NULL, hp->n_rot, GGML_ROPE_TYPE_NEOX, 0,
                        hp->rope_freq_base, 1.0f, 0.0f, 1.0f, 0.0f, 0.0f);
-    k = ggml_rope_ext(ctx, k, inp_pos, NULL, n_embd_head, GGML_ROPE_TYPE_NEOX, 0,
+    k = ggml_rope_ext(ctx, k, inp_pos, NULL, hp->n_rot, GGML_ROPE_TYPE_NEOX, 0,
                        hp->rope_freq_base, 1.0f, 0.0f, 1.0f, 0.0f, 0.0f);
 
     struct ggml_tensor *k_cache = h->kv.k[il];

@@ -12,11 +12,25 @@ require_relative "ggml/provider"
 
 Elelem.configure do |config|
   config.provider(:ggml) do
+    default_model = -> {
+      dir = File.expand_path("~/.agents/models")
+      link = File.join(dir, "default.gguf")
+      return link if File.symlink?(link) || File.exist?(link)
+
+      models = Dir[File.join(dir, "*.gguf")].sort
+      case models.length
+      when 1 then models.first
+      when 0 then raise "no .gguf model found in #{dir}. Set ELELEM_GGUF_MODEL=/path/to/model.gguf"
+      else raise "multiple .gguf models found in #{dir}. Set ELELEM_GGUF_MODEL=/path/to/model.gguf or symlink #{link} to the one to use by default"
+      end
+    }
+
     Elelem::Ggml::Provider.new(
-      model: ENV.fetch("ELELEM_GGML_MODEL") { raise "set ELELEM_GGML_MODEL=/path/to/model.gguf" },
+      model: ENV.fetch("ELELEM_GGUF_MODEL") { default_model.call },
       n_ctx: 0,
       n_threads: 0,
-      n_gpu_layers: 0
+      n_gpu_layers: 0,
+      max_tokens: Integer(ENV.fetch("ELELEM_GGUF_MAX_TOKENS", "256"))
     )
   end
 end

@@ -35,4 +35,35 @@ RSpec.describe Elelem::Ggml::Provider do
       expect(messages.first[:text]).to start_with("Hello")
     end
   end
+
+  describe "loading a model with no attention bias and tied embeddings" do
+    let(:model) { File.expand_path("../../fixtures/tinystories-656k.gguf", __dir__) }
+
+    it "loads and reports llama-family hyperparameters" do
+      handle = described_class.functions[:open].call(model, 0, 0, 0)
+
+      text = Fiddle::Pointer.new(described_class.functions[:version].call(handle)).to_s
+
+      expect(text).to include("arch=llama")
+      expect(text).to include("n_layer=2")
+      expect(text).to include("n_embd=128")
+      expect(text).to include("n_vocab=2048")
+
+      described_class.functions[:close].call(handle)
+    end
+
+    it "produces deterministic ids as a regression canary (NEOX rope; llama-arch actually needs NORMAL, unsupported)" do
+      handle = described_class.functions[:open].call(model, 0, 0, 0)
+      prompt = [1, 3].pack("l*")
+      out = " " * (4 * 20)
+
+      n = described_class.functions[:generate].call(handle, prompt, 2, 20, 2, out)
+
+      expect(out.unpack("l*").first(n)).to eq(
+        [299, 286, 229, 521, 776, 115, 56, 696, 183, 92, 730, 532, 215, 1653, 544, 2041, 215, 93, 319, 115]
+      )
+
+      described_class.functions[:close].call(handle)
+    end
+  end
 end

@@ -18,8 +18,15 @@ ChatML template live in Ruby (`lib/elelem/ggml/tokenizer.rb`,
 `lib/elelem/ggml/chat_ml.rb`), parsing vocab and merges straight out of the
 GGUF metadata.
 
-Other architectures, real sampling (top-k/top-p/temp), and tool-calling are
-not implemented yet -- see `spec/` for what's currently verified.
+This is a minimal, readable reference implementation of transformer
+inference on ggml -- CPU-only, dense Qwen2-family models only, greedy
+decoding only, no tool-calling. That scope is deliberate: the whole point is
+a forward pass small enough to read start to finish and know exactly what's
+happening to your tokens. For GPU backends, real sampling, tool-calling, or
+broad architecture support, use
+[elelem-llama](https://src.mokhan.ca/elelem/llama) instead.
+
+See `spec/` for what's currently verified.
 
 ## Setup
 
@@ -31,7 +38,10 @@ bundle exec rake compile
 
 ## Usage
 
+Set `ELELEM_GGUF_MODEL=/path/to/model.gguf`, or place a `.gguf` file (or a
+`default.gguf` symlink, if you keep more than one) in `~/.agents/models/`.
+
 ```
-export ELELEM_GGML_MODEL=/path/to/model.gguf
+export ELELEM_GGUF_MODEL=/path/to/model.gguf
 ruby -Ilib exe/elelem-ggml ask "hi"
 ```