Commit 250b525

mo khan <[email protected]>
2026-09-08 01:41:27
fix: finish client->provider rename and reuse provider across cases
evals.rb still required the pre-rename evals/bounded_client file, breaking every invocation. Runner also rebuilt (and reloaded the model for) a fresh provider on every case instead of reusing one, exhausting GPU memory after a few cases.
main
1 parent 1c823d1
Changed files (2)
lib
elelem
evals
runner.rb
evals.rb
lib/elelem/evals/runner.rb
@@ -11,7 +11,7 @@ module Elelem
 
       def run(kase)
         response = nil
-        bounded = BoundedProvider.new(@provider.call, max_turns: kase.max_turns)
+        bounded = BoundedProvider.new(provider, max_turns: kase.max_turns)
 
         Workspace.open(fixtures: @fixtures, fixture: kase.fixture, setup: kase.setup) do |workspace|
           workspace.chdir do
@@ -27,6 +27,10 @@ module Elelem
 
       private
 
+      def provider
+        @provider_instance ||= @provider.call
+      end
+
       def build_agent(provider)
         agent = Elelem::Agent.new(provider, system_prompt: @prompt)
         Elelem::Config.apply(agent)
lib/elelem/evals.rb
@@ -5,7 +5,7 @@ require "elelem/builtins"
 
 require_relative "evals/version"
 require_relative "evals/case"
-require_relative "evals/bounded_client"
+require_relative "evals/bounded_provider"
 require_relative "evals/assertions"
 require_relative "evals/workspace"
 require_relative "evals/result"