Commit 34072a7

@@ -11,8 +11,8 @@ NATIVE    = File.join(GEM_ROOT, "ext", "elelem", "ggml", "native")
 class BuildError < StandardError; end
 
 def run(*cmd)
-  warn "elelem-ggml: + #{cmd.join(' ')}"
-  system(*cmd) || raise(BuildError, "build step failed: #{cmd.join(' ')}")
+  warn "elelem-ggml: + #{cmd.join(" ")}"
+  system(*cmd) || raise(BuildError, "build step failed: #{cmd.join(" ")}")
 end
 
 unless File.exist?(File.join(VENDOR, "CMakeLists.txt"))

@@ -7,7 +7,7 @@ module Elelem
 
       def self.prompt(messages)
         messages = with_default_system(messages)
-        turns = messages.map { |m| "<|im_start|>#{m[:role] || m['role']}\n#{m[:content] || m['content']}<|im_end|>\n" }
+        turns = messages.map { |m| "<|im_start|>#{m[:role] || m["role"]}\n#{m[:content] || m["content"]}<|im_end|>\n" }
         "#{turns.join}<|im_start|>assistant\n"
       end
 

@@ -3,7 +3,7 @@
 module Elelem
   module Ggml
     class Tokenizer
-      SPLIT_PATTERN = /(?:'[sS]|'[tT]|'[rR][eE]|'[vV][eE]|'[mM]|'[lL][lL]|'[dD])|[^\r\n\p{L}\p{N}]?\p{L}+|\p{N}| ?[^\s\p{L}\p{N}]+[\r\n]*|\s*[\r\n]+|\s+(?!\S)|\s+/.freeze
+      SPLIT_PATTERN = /(?:'[sS]|'[tT]|'[rR][eE]|'[vV][eE]|'[mM]|'[lL][lL]|'[dD])|[^\r\n\p{L}\p{N}]?\p{L}+|\p{N}| ?[^\s\p{L}\p{N}]+[\r\n]*|\s*[\r\n]+|\s+(?!\S)|\s+/ # rubocop:disable Layout/LineLength
 
       def self.byte_encoder
         @byte_encoder ||= begin
@@ -53,7 +53,7 @@ module Elelem
 
       def special_token_ids(metadata)
         types = metadata["tokenizer.ggml.token_type"] || []
-        @tokens.each_with_index.each_with_object({}) do |(tok, id), memo|
+        @tokens.each_with_index.with_object({}) do |(tok, id), memo|
           memo[tok] = id if types[id] == 3
         end
       end

@@ -12,16 +12,17 @@ require_relative "ggml/provider"
 
 Elelem.configure do |config|
   config.provider(:ggml) do
-    default_model = -> {
+    default_model = lambda {
       dir = File.expand_path("~/.agents/models")
       link = File.join(dir, "default.gguf")
       return link if File.symlink?(link) || File.exist?(link)
 
-      models = Dir[File.join(dir, "*.gguf")].sort
+      models = Dir[File.join(dir, "*.gguf")]
       case models.length
       when 1 then models.first
       when 0 then raise "no .gguf model found in #{dir}. Set ELELEM_GGUF_MODEL=/path/to/model.gguf"
-      else raise "multiple .gguf models found in #{dir}. Set ELELEM_GGUF_MODEL=/path/to/model.gguf or symlink #{link} to the one to use by default"
+      else raise "multiple .gguf models found in #{dir}. Set ELELEM_GGUF_MODEL=/path/to/model.gguf or symlink " \
+                 "#{link} to the one to use by default"
       end
     }
 

@@ -18,7 +18,9 @@ RSpec.describe Elelem::Ggml::ChatMl do
 
       prompt = described_class.prompt(messages)
 
-      expect(prompt).to eq("<|im_start|>system\nBe terse.<|im_end|>\n<|im_start|>user\nhi<|im_end|>\n<|im_start|>assistant\n")
+      expect(prompt).to eq(
+        "<|im_start|>system\nBe terse.<|im_end|>\n<|im_start|>user\nhi<|im_end|>\n<|im_start|>assistant\n"
+      )
     end
   end
 
@@ -32,10 +34,10 @@ RSpec.describe Elelem::Ggml::ChatMl do
       ids = tokenizer.encode(prompt)
 
       expect(ids).to eq([
-        151644, 8948, 198, 2610, 525, 1207, 16948, 11, 3465, 553, 54364, 14817, 13,
-        1446, 525, 264, 10950, 17847, 13, 151645, 198, 151644, 872, 198, 6023, 151645,
-        198, 151644, 77091, 198
-      ])
+                          151_644, 8948, 198, 2610, 525, 1207, 16_948, 11, 3465, 553, 54_364, 14_817, 13,
+                          1446, 525, 264, 10_950, 17_847, 13, 151_645, 198, 151_644, 872, 198, 6023, 151_645,
+                          198, 151_644, 77_091, 198
+                        ])
     end
   end
 end

@@ -13,7 +13,7 @@ RSpec.describe Elelem::Ggml::Tokenizer do
     it "encodes a ChatML-templated prompt" do
       prompt = "<|im_start|>user\nhi<|im_end|>\n<|im_start|>assistant\n"
 
-      expect(tokenizer.encode(prompt)).to eq([151644, 872, 198, 6023, 151645, 198, 151644, 77091, 198])
+      expect(tokenizer.encode(prompt)).to eq([151_644, 872, 198, 6023, 151_645, 198, 151_644, 77_091, 198])
     end
 
     it "encodes a sentence requiring multi-step BPE merges" do
@@ -25,7 +25,7 @@ RSpec.describe Elelem::Ggml::Tokenizer do
     end
 
     it "encodes runs of indentation-like whitespace" do
-      expect(tokenizer.encode("  indented   spaces")).to eq([220, 1257, 15864, 256, 12621])
+      expect(tokenizer.encode("  indented   spaces")).to eq([220, 1257, 15_864, 256, 12_621])
     end
   end