Commit 5be4e1a
Changed files (3)
lib
elelem
lib/elelem/evals/bounded_client.rb → lib/elelem/evals/bounded_provider.rb
@@ -2,13 +2,13 @@
module Elelem
module Evals
- class BoundedClient
+ class BoundedProvider
TurnLimitExceeded = Class.new(StandardError)
attr_reader :turns
- def initialize(client, max_turns:)
- @client = client
+ def initialize(provider, max_turns:)
+ @provider = provider
@max_turns = max_turns
@turns = 0
end
@@ -17,7 +17,7 @@ module Elelem
@turns += 1
raise TurnLimitExceeded, "turn limit #{@max_turns} exceeded" if @turns > @max_turns
- @client.fetch(messages, tools, &block)
+ @provider.fetch(messages, tools, &block)
end
end
end
lib/elelem/evals/cli.rb
@@ -17,8 +17,8 @@ module Elelem
return list(cases) if options[:list]
- client = -> { Elelem::Config.build_client(options[:provider]) }
- runner = Runner.new(fixtures: options[:fixtures], client: client, prompt: options[:prompt])
+ provider = -> { Elelem::Config.build_provider(options[:provider]) }
+ runner = Runner.new(fixtures: options[:fixtures], provider: provider, prompt: options[:prompt])
reporter = Reporter.new
cases.each do |kase|
lib/elelem/evals/runner.rb
@@ -3,15 +3,15 @@
module Elelem
module Evals
class Runner
- def initialize(fixtures:, client:, prompt: nil)
+ def initialize(fixtures:, provider:, prompt: nil)
@fixtures = fixtures
- @client = client
+ @provider = provider
@prompt = prompt
end
def run(kase)
response = nil
- bounded = BoundedClient.new(@client.call, max_turns: kase.max_turns)
+ bounded = BoundedProvider.new(@provider.call, max_turns: kase.max_turns)
Workspace.open(fixtures: @fixtures, fixture: kase.fixture, setup: kase.setup) do |workspace|
workspace.chdir do
@@ -27,8 +27,8 @@ module Elelem
private
- def build_agent(client)
- agent = Elelem::Agent.new(client, system_prompt: @prompt)
+ def build_agent(provider)
+ agent = Elelem::Agent.new(provider, system_prompt: @prompt)
Elelem::Config.apply(agent)
agent
end