Commit 6e3cb59
Changed files (6)
lib
elelem
lib/elelem/evals/assertions.rb
@@ -43,9 +43,13 @@ module Elelem
def response_failures(response)
text = response.to_s
- Array(@expect[:response_contains]).reject { |s| text.include?(s) }.map { |s| "response missing #{s.inspect}" } +
- Array(@expect[:response_not_contains]).select { |s| text.include?(s) }.map { |s| "response contains #{s.inspect}" } +
- Array(@expect[:response_matches]).reject { |p| Regexp.new(p).match?(text) }.map { |p| "response does not match /#{p}/" }
+ missing = Array(@expect[:response_contains]).reject { |s| text.include?(s) }
+ present = Array(@expect[:response_not_contains]).select { |s| text.include?(s) }
+ unmatched = Array(@expect[:response_matches]).reject { |p| Regexp.new(p).match?(text) }
+
+ missing.map { |s| "response missing #{s.inspect}" } +
+ present.map { |s| "response contains #{s.inspect}" } +
+ unmatched.map { |p| "response does not match /#{p}/" }
end
end
end
lib/elelem/evals/bounded_provider.rb
@@ -3,7 +3,8 @@
module Elelem
module Evals
class BoundedProvider
- TurnLimitExceeded = Class.new(StandardError)
+ class TurnLimitExceeded < StandardError
+ end
attr_reader :turns
@@ -13,11 +14,11 @@ module Elelem
@turns = 0
end
- def fetch(messages, tools = [], &block)
+ def fetch(messages, tools = [], &)
@turns += 1
raise TurnLimitExceeded, "turn limit #{@max_turns} exceeded" if @turns > @max_turns
- @provider.fetch(messages, tools, &block)
+ @provider.fetch(messages, tools, &)
end
end
end
lib/elelem/evals/case.rb
@@ -8,7 +8,7 @@ module Elelem
DEFAULT_MAX_TURNS = 30
def self.load_all(dir)
- Dir["#{dir}/*.yml"].sort.flat_map { |file| load_file(file) }
+ Dir["#{dir}/*.yml"].flat_map { |file| load_file(file) }
end
def self.load_file(file)
lib/elelem/evals/cli.rb
@@ -33,7 +33,7 @@ module Elelem
private
def list(cases)
- cases.each { |kase| puts kase.to_s }
+ cases.each { |kase| puts kase }
end
def match?(kase, filter)
lib/elelem/evals/runner.rb
@@ -28,7 +28,7 @@ module Elelem
private
def provider
- @provider_instance ||= @provider.call
+ @provider ||= @provider.call
end
def build_session(provider)
lib/elelem/evals/workspace.rb
@@ -7,7 +7,8 @@ require "tmpdir"
module Elelem
module Evals
class Workspace
- SetupFailed = Class.new(StandardError)
+ class SetupFailed < StandardError
+ end
def self.open(fixtures:, fixture:, setup: [])
dir = Dir.mktmpdir("elelem-evals-")
@@ -31,8 +32,8 @@ module Elelem
chdir { setup.each { |command| run_setup(command) } }
end
- def chdir(&block)
- Dir.chdir(@path) { Bundler.with_unbundled_env(&block) }
+ def chdir(&)
+ Dir.chdir(@path) { Bundler.with_unbundled_env(&) }
end
def sh(command)