gguf-provider
1# frozen_string_literal: true
2
3module Elelem
4 module Evals
5 class NullTerminal < Elelem::Terminal
6 def initialize
7 super(quiet: true)
8 end
9
10 def interactive?
11 false
12 end
13 end
14
15 def self.gguf(model: ENV.fetch("GGUF_MODEL", File.expand_path("~/.elelem/models/gpt-oss-20b-MXFP4.gguf")))
16 @gguf ||= begin
17 gpu = %w[vulkan cuda metal].include?(Elelem::Net::GGUF.backend)
18 Elelem::Net::GGUF.new(
19 model: File.expand_path(model),
20 n_ctx: Integer(ENV.fetch("GGUF_N_CTX", "16384")),
21 n_gpu_layers: Integer(ENV.fetch("GGUF_N_GPU_LAYERS", gpu ? "999" : "0")),
22 max_tokens: Integer(ENV.fetch("GGUF_MAX_TOKENS", "4096")),
23 temp: 0.0,
24 seed: 42
25 )
26 end
27 end
28
29 def self.client
30 gguf
31 end
32
33 class Case
34 CASES = File.expand_path("cases", __dir__)
35 DEFAULT_MAX_TURNS = 30
36
37 def self.load_all(dir = CASES)
38 Dir["#{dir}/*.yml"].sort.flat_map { |file| load_file(file) }
39 end
40
41 def self.load_file(file)
42 group = File.basename(file, ".yml")
43 YAML.safe_load_file(file).map { |attrs| new(group: group, **symbolize(attrs)) }
44 end
45
46 def self.symbolize(hash)
47 (hash || {}).transform_keys(&:to_sym)
48 end
49
50 attr_reader :id, :group, :fixture, :setup, :turns, :expect
51
52 def initialize(id:, fixture:, turns:, group: nil, setup: [], expect: {})
53 @id = id
54 @group = group
55 @fixture = fixture
56 @setup = Array(setup)
57 @turns = Array(turns)
58 @expect = self.class.symbolize(expect)
59 end
60
61 def max_turns
62 @expect.fetch(:max_turns, DEFAULT_MAX_TURNS)
63 end
64
65 def to_s
66 "#{group}/#{id}"
67 end
68 end
69
70 class BoundedClient
71 TurnLimitExceeded = Class.new(StandardError)
72
73 attr_reader :turns
74
75 def initialize(client, max_turns:)
76 @client = client
77 @max_turns = max_turns
78 @turns = 0
79 end
80
81 def fetch(messages, tools = [], &block)
82 @turns += 1
83 raise TurnLimitExceeded, "turn limit #{@max_turns} exceeded" if @turns > @max_turns
84
85 @client.fetch(messages, tools, &block)
86 end
87 end
88
89 class Assertions
90 def initialize(expect)
91 @expect = expect || {}
92 end
93
94 def failures(workspace:, response:)
95 [
96 verify_failure(workspace),
97 *file_failures(workspace),
98 *response_failures(response)
99 ].compact
100 end
101
102 private
103
104 def verify_failure(workspace)
105 command = @expect[:verify]
106 return unless command
107
108 result = workspace.sh(command)
109 return if result[:exit_status].zero?
110
111 "verify failed (exit #{result[:exit_status]}): #{command}"
112 end
113
114 def file_failures(workspace)
115 (@expect[:files] || {}).flat_map do |path, rules|
116 content = workspace.read(path)
117 next ["#{path} missing"] unless content
118
119 missing = Array(rules["contains"]).reject { |text| content.include?(text) }
120 present = Array(rules["not_contains"]).select { |text| content.include?(text) }
121
122 missing.map { |text| "#{path} missing #{text.inspect}" } +
123 present.map { |text| "#{path} still contains #{text.inspect}" }
124 end
125 end
126
127 def response_failures(response)
128 text = response.to_s
129
130 Array(@expect[:response_contains]).reject { |s| text.include?(s) }.map { |s| "response missing #{s.inspect}" } +
131 Array(@expect[:response_not_contains]).select { |s| text.include?(s) }.map { |s| "response contains #{s.inspect}" } +
132 Array(@expect[:response_matches]).reject { |p| Regexp.new(p).match?(text) }.map { |p| "response does not match /#{p}/" }
133 end
134 end
135
136 Result = Data.define(:id, :group, :failures, :response) do
137 def passed?
138 failures.empty?
139 end
140 end
141
142 CHAMPION = File.expand_path("../../lib/elelem/prompts/default.erb", __dir__)
143
144 class Runner
145 def initialize(prompt:, client: -> { Evals.client })
146 @prompt = prompt
147 @client = client
148 end
149
150 def run(kase)
151 response = nil
152 bounded = BoundedClient.new(@client.call, max_turns: kase.max_turns)
153
154 Workspace.open(fixture: kase.fixture, setup: kase.setup) do |workspace|
155 workspace.chdir do
156 agent = build_agent(bounded)
157 kase.turns.each { |turn| response = agent.turn(turn) }
158
159 failures = Assertions.new(kase.expect).failures(workspace: workspace, response: response)
160
161 Result.new(id: kase.id, group: kase.group, failures: failures, response: response.to_s)
162 end
163 end
164 end
165
166 private
167
168 def build_agent(client)
169 agent = Elelem::Agent.new(client, terminal: NullTerminal.new, system_prompt: @prompt)
170 Elelem::Plugins.setup!(agent)
171 agent
172 end
173 end
174
175 class Workspace
176 SetupFailed = Class.new(StandardError)
177
178 FIXTURES = File.expand_path("fixtures", __dir__)
179
180 def self.open(fixture:, setup: [])
181 dir = Dir.mktmpdir("elelem-evals-")
182 workspace = new(dir, fixture)
183 workspace.prepare(setup)
184 yield workspace
185 ensure
186 FileUtils.remove_entry(dir) if dir && File.directory?(dir)
187 end
188
189 attr_reader :path
190
191 def initialize(path, fixture)
192 @path = path
193 @fixture = fixture
194 end
195
196 def prepare(setup)
197 FileUtils.cp_r("#{FIXTURES}/#{@fixture}/.", @path)
198 chdir { setup.each { |command| run_setup(command) } }
199 end
200
201 def chdir(&block)
202 Dir.chdir(@path) { Bundler.with_unbundled_env(&block) }
203 end
204
205 def sh(command)
206 Elelem.sh("bash", args: ["-c", command], cwd: @path)
207 end
208
209 def read(relative)
210 file = File.join(@path, relative)
211 File.exist?(file) ? File.read(file) : nil
212 end
213
214 private
215
216 def run_setup(command)
217 result = sh(command)
218 return if result[:exit_status].zero?
219
220 raise SetupFailed, "setup command failed (exit #{result[:exit_status]}): #{command}\n#{result[:content]}"
221 end
222 end
223 end
224end
225
226RSpec.describe "eval cases" do
227 runner = Elelem::Evals::Runner.new(prompt: File.read(Elelem::Evals::CHAMPION))
228
229 Elelem::Evals::Case.load_all.each do |kase|
230 it kase.to_s do
231 result = runner.run(kase)
232
233 expect(result.failures).to be_empty, -> { "#{kase}\n #{result.failures.join("\n ")}\n\n#{result.response}" }
234 end
235 end
236end