Commit `3f682f8`

2026-09-07 03:51:37

feat: scaffold elelem-evals gem

Ports the eval harness out of elelem core's spec suite into its own gem with a real exe/elelem-evals CLI (-p provider, -c case glob, -l list, -P prompt), targeting any registered Elelem::Providers provider instead of only running under rspec. Claude-Session: https://claude.ai/code/session_01DUuj4amvRrPxDnHPHBkvz5

main

Changed files (52)

bin

console

setup

cases

build.yml

create.yml

edit.yml

fix.yml

git.yml

holdout.yml

implement.yml

json.yml

ops.yml

refactor.yml

search.yml

shell.yml

unix.yml

voice.yml

exe

elelem-evals

fixtures

blank

README.md

broken

lib

calc.rb

test.rb

config

loader.rb

parser.rb

app.rb

greeter

lib.rb

main.rb

hello

hello.rb

logs

requests.jsonl

metadata

session.rb

token.rb

user.rb

roman

roman.rb

roman_test.rb

sizes

data

report.csv

lib

app.rb

README.md

lib

elelem

evals

assertions.rb

bounded_client.rb

case.rb

cli.rb

null_terminal.rb

reporter.rb

result.rb

runner.rb

version.rb

workspace.rb

evals.rb

.gitignore

elelem-evals.gemspec

Gemfile

Gemfile.lock

LICENSE.txt

Rakefile

bin/console

@@ -0,0 +1,8 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require "bundler/setup"
+require "elelem/evals"
+require "irb"
+
+IRB.start(__FILE__)

bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

cases/build.yml

@@ -0,0 +1,10 @@
+# Defends: run the tests, interpret the failure, and DON'T edit when only asked to diagnose.
+- id: diagnose-failing-test
+  fixture: broken
+  turns:
+    - run test.rb and explain what is broken, but do not change any files
+  expect:
+    response_contains: ["add"]
+    files:
+      lib/calc.rb:
+        contains: ["a - b"]

cases/create.yml

@@ -0,0 +1,11 @@
+# Defends: scaffold a new program from scratch and run it.
+- id: fizzbuzz-from-scratch
+  fixture: blank
+  turns:
+    - create fizzbuzz.rb that prints the numbers 1 to 15 one per line, but Fizz for
+      multiples of 3, Buzz for multiples of 5, and FizzBuzz for multiples of 15; then run it
+  expect:
+    verify: '[ "$(ruby fizzbuzz.rb | sed -n 3p)" = "Fizz" ] && [ "$(ruby fizzbuzz.rb | sed -n 5p)" = "Buzz" ] && [ "$(ruby fizzbuzz.rb | sed -n 15p)" = "FizzBuzz" ]'
+    files:
+      fizzbuzz.rb:
+        contains: ["FizzBuzz"]

cases/edit.yml

@@ -0,0 +1,36 @@
+# Defends: "Change a file with write: read it, then write the full new contents."
+# The read half is asserted directly; keeping app_metadata proves the whole file
+# was rewritten, not just the new method appended blindly.
+- id: read-before-write
+  fixture: metadata
+  turns:
+    - add a user_id method to the User class in user.rb that returns the row's "id" value
+  expect:
+    verify: ruby -e 'require "./user"; u = User.new({"id" => 7, "app_metadata" => {"a" => 1}}); raise unless u.user_id == 7 && u.app_metadata == {"a" => 1}'
+    files:
+      user.rb:
+        contains: ["user_id", "app_metadata"]
+
+# Defends: "Use `sed` only for a trivial single-line substitution."
+- id: restructure-with-write
+  fixture: metadata
+  turns:
+    - wrap the User class in user.rb in a module named Auth, keeping the class body
+      unchanged
+  expect:
+    verify: ruby -e 'require "./user"; Auth::User'
+    files:
+      user.rb:
+        contains: ["module Auth"]
+
+- id: multi-line-edit
+  fixture: hello
+  turns:
+    - 'change greet so it prints "Goodbye, #{name}." with a period instead of an
+      exclamation mark, and call it with "friend" instead of "world"'
+  expect:
+    verify: ruby hello.rb | grep -qxF "Goodbye, friend."
+    files:
+      hello.rb:
+        contains: ["Goodbye"]
+        not_contains: ["Hello"]

cases/fix.yml

@@ -0,0 +1,9 @@
+- id: fix-failing-test
+  fixture: broken
+  turns:
+    - run test.rb and fix whatever is broken
+  expect:
+    verify: ruby test.rb
+    files:
+      lib/calc.rb:
+        contains: ["a + b"]

cases/git.yml

@@ -0,0 +1,27 @@
+- id: commit-50-72
+  fixture: hello
+  setup:
+    - git init -q
+    - git config user.email [email protected]
+    - git config user.name evals
+    - git add -A
+    - git commit -q -m "initial commit"
+    - printf 'greet("again")\n' >> hello.rb
+  turns:
+    - commit the current changes with a message that follows the 50/72 rule
+  expect:
+    verify: test -z "$(git status --porcelain)" && test "$(git log -1 --format=%s | wc -c)" -le 51
+
+# Defends: query git history and report, without editing.
+- id: last-commit-subject
+  fixture: hello
+  setup:
+    - git init -q
+    - git config user.email [email protected]
+    - git config user.name evals
+    - git add -A
+    - git commit -q -m "Add greeting script"
+  turns:
+    - what is the subject line of the most recent git commit?
+  expect:
+    response_contains: ["Add greeting script"]

cases/holdout.yml

@@ -0,0 +1,62 @@
+# Held out from the improver: it never sees these failures, but a regression here
+# still blocks a promotion -- a guard against prompts that overfit the visible set.
+- id: rename-method-holdout
+  fixture: hello
+  turns:
+    - rename the greet method to hail, including the call at the bottom
+  expect:
+    verify: ruby hello.rb
+    files:
+      hello.rb:
+        contains: ["hail"]
+        not_contains: ["greet"]
+
+# Paraphrased twin of edit/read-before-write, held out so the improver can't
+# overfit the read-then-write instruction to one fixture.
+- id: read-before-write-holdout
+  fixture: metadata
+  turns:
+    - add an issuer method to the Token class in token.rb that returns the "iss" claim
+  expect:
+    verify: ruby -e 'require "./token"; t = Token.new({"iss" => "acme", "app_metadata" => {"a" => 1}}); raise unless t.issuer == "acme" && t.to_h["app_metadata"] == {"a" => 1}'
+    files:
+      token.rb:
+        contains: ["issuer", "to_h"]
+
+# Paraphrased twin of voice/concise-answer: holds the conciseness bound out of
+# the improver's view so terseness can't overfit to one prompt.
+- id: concise-answer-holdout
+  fixture: metadata
+  turns:
+    - which file defines the Token class?
+  expect:
+    response_contains: ["token.rb"]
+    response_matches: ['\A[\s\S]{0,400}\z']
+
+- id: locate-error-holdout
+  fixture: config
+  turns:
+    - which file raises the "Failed to load configuration" error?
+  expect:
+    response_contains: ["config/loader.rb"]
+
+- id: locate-class
+  fixture: config
+  turns:
+    - which file defines the Loader class?
+  expect:
+    response_contains: ["config/loader.rb"]
+
+- id: count-ruby-files
+  fixture: metadata
+  turns:
+    - how many ruby files are in this project?
+  expect:
+    response_matches: ['\b3\b']
+
+- id: locate-parser
+  fixture: config
+  turns:
+    - which file defines the Parser class?
+  expect:
+    response_contains: ["config/parser.rb"]

cases/implement.yml

@@ -0,0 +1,10 @@
+# Defends: implement a function from a stub until its test passes (Exercism/HumanEval style).
+- id: roman-numerals
+  fixture: roman
+  turns:
+    - implement to_roman in roman.rb so that `ruby roman_test.rb` passes
+  expect:
+    verify: ruby roman_test.rb
+    files:
+      roman.rb:
+        not_contains: ["NotImplementedError"]

cases/json.yml

@@ -0,0 +1,7 @@
+- id: summarize-json-log
+  fixture: logs
+  turns:
+    - which path returned the most 5xx responses in requests.jsonl, and how many?
+  expect:
+    response_contains: ["/scim/v2/Users"]
+    response_matches: ['/scim/v2/Users[\s\S]{0,120}3']

cases/ops.yml

@@ -0,0 +1,11 @@
+# Shell-verified ops tasks. No privileged or destructive commands yet -- these
+# run in the same tmpdir workspace as every other case.
+- id: count-500s
+  fixture: logs
+  turns:
+    - count how many lines in requests.jsonl have response_code 500 and write
+      just that number to count.txt
+  expect:
+    verify: 'test "$(cat count.txt)" = "$(grep -c ''"response_code":500'' requests.jsonl)"'
+    files:
+      count.txt: {}

cases/refactor.yml

@@ -0,0 +1,14 @@
+# Defends: a multi-file edit that renames a symbol and its caller, behavior intact.
+- id: rename-across-files
+  fixture: greeter
+  turns:
+    - rename the greeting method to salutation everywhere, including its caller
+  expect:
+    verify: ruby main.rb
+    files:
+      lib.rb:
+        contains: ["salutation"]
+        not_contains: ["def greeting"]
+      main.rb:
+        contains: ["salutation"]
+        not_contains: ["greeting"]

cases/search.yml

@@ -0,0 +1,30 @@
+# Defends: locate where an error string originates by searching, then answer.
+- id: locate-error-string
+  fixture: config
+  turns:
+    - this app raises 'Failed to load configuration'. which file in this project raises it?
+  expect:
+    response_contains: ["config/loader.rb"]
+
+# Defends: "- files: `fd -e rb .`"
+- id: count-ruby-files
+  fixture: metadata
+  turns:
+    - how many ruby files are in this project?
+  expect:
+    response_matches: ['\b3\b']
+
+# Defends: "- text: `rg -n "pattern" .`"
+- id: grep-with-ripgrep
+  fixture: config
+  turns:
+    - search the codebase for the text "Failed to load configuration" and tell me which file contains it
+  expect:
+    response_contains: ["config/loader.rb"]
+
+- id: list-references
+  fixture: metadata
+  turns:
+    - which files reference app_metadata?
+  expect:
+    response_contains: ["user.rb", "token.rb"]

cases/shell.yml

@@ -0,0 +1,17 @@
+- id: largest-tracked-file
+  fixture: sizes
+  setup:
+    - for i in $(seq 1 400); do printf '%d,widget-%d,%d\n' "$i" "$i" $((i * 7)) >> data/report.csv; done
+    - git init -q
+    - git config user.email [email protected]
+    - git config user.name evals
+    - git add -A
+    - git commit -q -m "initial commit"
+    # .git/description is never read by git, so padding it is a safe stand-in
+    # for the pack files that dominate a real repository's byte counts.
+    - for i in $(seq 1 1200); do printf 'padding line %d\n' "$i" >> .git/description; done
+  turns:
+    - what is the biggest tracked file in this repository?
+  expect:
+    response_contains: ["report.csv"]
+    response_not_contains: [".git/"]

cases/unix.yml

@@ -0,0 +1,23 @@
+# Defends: answer with a shell pipeline over tracked files, not by editing.
+- id: count-files-with-string
+  fixture: metadata
+  turns:
+    - write the number of ruby files that contain the text app_metadata to count.txt
+  expect:
+    verify: '[ "$(cat count.txt)" = "2" ]'
+
+# Defends: portable byte size (BSD `stat -f%z` vs GNU `stat -c%s`).
+- id: byte-size-portable
+  fixture: sizes
+  turns:
+    - write the size of README.md in bytes to size.txt
+  expect:
+    verify: '[ "$(cat size.txt)" = "$(wc -c < README.md | tr -d " ")" ]'
+
+# Defends: aggregate a value across many files with a pipeline.
+- id: total-ruby-lines
+  fixture: metadata
+  turns:
+    - write the total number of lines across all ruby files in this project to total.txt
+  expect:
+    verify: '[ "$(cat total.txt)" = "$(cat *.rb | wc -l | tr -d " ")" ]'

cases/voice.yml

@@ -0,0 +1,11 @@
+# Defends: "Answer in as few words as the task allows -- no preamble, no filler."
+# The right answer is a single filename. The length bound passes a concise reply
+# that names the file and stops, but fails a padded, preamble-laden one -- so the
+# voice line earns its place and the ablator won't strip it.
+- id: concise-answer
+  fixture: config
+  turns:
+    - which file defines the Parser class?
+  expect:
+    response_contains: ["config/parser.rb"]
+    response_matches: ['\A[\s\S]{0,400}\z']

exe/elelem-evals

@@ -0,0 +1,6 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require "elelem/evals"
+
+Elelem::Evals::CLI.run(ARGV)

fixtures/blank/README.md

@@ -0,0 +1,3 @@
+# Scratch
+
+An empty workspace for building something new.

fixtures/broken/lib/calc.rb

@@ -0,0 +1,3 @@
+def add(a, b)
+  a - b
+end

fixtures/broken/test.rb

@@ -0,0 +1,5 @@
+require_relative "lib/calc"
+
+raise "add(2, 3) should be 5, got #{add(2, 3)}" unless add(2, 3) == 5
+
+puts "ok"

fixtures/config/config/loader.rb

@@ -0,0 +1,11 @@
+class Loader
+  def initialize(path)
+    @path = path
+  end
+
+  def call
+    raise "Failed to load configuration: #{@path}" unless File.exist?(@path)
+
+    Parser.new(File.read(@path)).call
+  end
+end

fixtures/config/config/parser.rb

@@ -0,0 +1,9 @@
+class Parser
+  def initialize(text)
+    @text = text
+  end
+
+  def call
+    @text.lines.map { |line| line.split(":", 2) }.to_h
+  end
+end

fixtures/config/app.rb

@@ -0,0 +1,3 @@
+require_relative "config/loader"
+
+Loader.new("app.yml").call

fixtures/greeter/lib.rb

@@ -0,0 +1,3 @@
+def greeting(name)
+  "Hi, #{name}"
+end

fixtures/greeter/main.rb

@@ -0,0 +1,3 @@
+require_relative "lib"
+
+puts greeting("world")

fixtures/hello/hello.rb

@@ -0,0 +1,5 @@
+def greet(name)
+  puts "Hello, #{name}!"
+end
+
+greet("world")

fixtures/logs/requests.jsonl

@@ -0,0 +1,12 @@
+{"path":"/scim/v2/Users","response_code":200}
+{"path":"/scim/v2/Users","response_code":500}
+{"path":"/scim/v2/Users","response_code":503}
+{"path":"/scim/v2/Users","response_code":500}
+{"path":"/scim/v2/Groups","response_code":200}
+{"path":"/scim/v2/Groups","response_code":404}
+{"path":"/scim/v2/Groups","response_code":200}
+{"path":"/health","response_code":200}
+{"path":"/health","response_code":500}
+{"path":"/health","response_code":200}
+{"path":"/scim/v2/Schemas","response_code":200}
+{"path":"/scim/v2/Schemas","response_code":304}

fixtures/metadata/session.rb

@@ -0,0 +1,10 @@
+class Session
+  def initialize(id, user_id)
+    @id = id
+    @user_id = user_id
+  end
+
+  def to_h
+    { "id" => @id, "user_id" => @user_id }
+  end
+end

fixtures/metadata/token.rb

@@ -0,0 +1,9 @@
+class Token
+  def initialize(claims)
+    @claims = claims
+  end
+
+  def to_h
+    @claims.merge("app_metadata" => @claims["app_metadata"] || {})
+  end
+end

fixtures/metadata/user.rb

@@ -0,0 +1,9 @@
+class User
+  def initialize(row)
+    @row = row
+  end
+
+  def app_metadata
+    @row.fetch("app_metadata", {})
+  end
+end

fixtures/roman/roman.rb

@@ -0,0 +1,3 @@
+def to_roman(number)
+  raise NotImplementedError
+end

fixtures/roman/roman_test.rb

@@ -0,0 +1,8 @@
+require_relative "roman"
+
+{ 1 => "I", 4 => "IV", 9 => "IX", 40 => "XL", 90 => "XC", 2024 => "MMXXIV" }.each do |number, want|
+  got = to_roman(number)
+  raise "to_roman(#{number}) = #{got.inspect}, want #{want.inspect}" unless got == want
+end
+
+puts "ok"

fixtures/sizes/data/report.csv

@@ -0,0 +1,1 @@
+id,widget,cents

fixtures/sizes/lib/app.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+require "csv"
+
+class App
+  def initialize(path = "data/report.csv")
+    @path = path
+  end
+
+  def rows
+    CSV.read(@path, headers: true)
+  end
+
+  def total
+    rows.sum { |row| row["cents"].to_i }
+  end
+end

fixtures/sizes/README.md

@@ -0,0 +1,3 @@
+# Widgets
+
+Inventory reports live in `data/`.

lib/elelem/evals/assertions.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+module Elelem
+  module Evals
+    class Assertions
+      def initialize(expect)
+        @expect = expect || {}
+      end
+
+      def failures(workspace:, response:)
+        [
+          verify_failure(workspace),
+          *file_failures(workspace),
+          *response_failures(response)
+        ].compact
+      end
+
+      private
+
+      def verify_failure(workspace)
+        command = @expect[:verify]
+        return unless command
+
+        result = workspace.sh(command)
+        return if result[:exit_status].zero?
+
+        "verify failed (exit #{result[:exit_status]}): #{command}"
+      end
+
+      def file_failures(workspace)
+        (@expect[:files] || {}).flat_map do |path, rules|
+          content = workspace.read(path)
+          next ["#{path} missing"] unless content
+
+          missing = Array(rules["contains"]).reject { |text| content.include?(text) }
+          present = Array(rules["not_contains"]).select { |text| content.include?(text) }
+
+          missing.map { |text| "#{path} missing #{text.inspect}" } +
+            present.map { |text| "#{path} still contains #{text.inspect}" }
+        end
+      end
+
+      def response_failures(response)
+        text = response.to_s
+
+        Array(@expect[:response_contains]).reject { |s| text.include?(s) }.map { |s| "response missing #{s.inspect}" } +
+          Array(@expect[:response_not_contains]).select { |s| text.include?(s) }.map { |s| "response contains #{s.inspect}" } +
+          Array(@expect[:response_matches]).reject { |p| Regexp.new(p).match?(text) }.map { |p| "response does not match /#{p}/" }
+      end
+    end
+  end
+end

lib/elelem/evals/bounded_client.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+module Elelem
+  module Evals
+    class BoundedClient
+      TurnLimitExceeded = Class.new(StandardError)
+
+      attr_reader :turns
+
+      def initialize(client, max_turns:)
+        @client = client
+        @max_turns = max_turns
+        @turns = 0
+      end
+
+      def fetch(messages, tools = [], &block)
+        @turns += 1
+        raise TurnLimitExceeded, "turn limit #{@max_turns} exceeded" if @turns > @max_turns
+
+        @client.fetch(messages, tools, &block)
+      end
+    end
+  end
+end

lib/elelem/evals/case.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+require "yaml"
+
+module Elelem
+  module Evals
+    class Case
+      DEFAULT_MAX_TURNS = 30
+
+      def self.load_all(dir)
+        Dir["#{dir}/*.yml"].sort.flat_map { |file| load_file(file) }
+      end
+
+      def self.load_file(file)
+        group = File.basename(file, ".yml")
+        YAML.safe_load_file(file).map { |attrs| new(group: group, **symbolize(attrs)) }
+      end
+
+      def self.symbolize(hash)
+        (hash || {}).transform_keys(&:to_sym)
+      end
+
+      attr_reader :id, :group, :fixture, :setup, :turns, :expect
+
+      def initialize(id:, fixture:, turns:, group: nil, setup: [], expect: {})
+        @id = id
+        @group = group
+        @fixture = fixture
+        @setup = Array(setup)
+        @turns = Array(turns)
+        @expect = self.class.symbolize(expect)
+      end
+
+      def max_turns
+        @expect.fetch(:max_turns, DEFAULT_MAX_TURNS)
+      end
+
+      def to_s
+        "#{group}/#{id}"
+      end
+    end
+  end
+end

lib/elelem/evals/cli.rb

@@ -0,0 +1,68 @@
+# frozen_string_literal: true
+
+require "optparse"
+
+module Elelem
+  module Evals
+    class CLI
+      ROOT = File.expand_path("../../..", __dir__)
+
+      def self.run(argv)
+        new.run(argv)
+      end
+
+      def run(argv)
+        options = parse(argv)
+        cases = Case.load_all(options[:cases]).select { |kase| match?(kase, options[:filter]) }
+
+        return list(cases) if options[:list]
+
+        client = -> { Elelem::Providers.build(options[:provider]) }
+        runner = Runner.new(fixtures: options[:fixtures], client: client, prompt: options[:prompt])
+        reporter = Reporter.new
+
+        cases.each do |kase|
+          reporter.record(runner.run(kase))
+        rescue StandardError => e
+          reporter.error(kase, e)
+        end
+
+        exit(reporter.summarize ? 0 : 1)
+      end
+
+      private
+
+      def list(cases)
+        cases.each { |kase| puts kase.to_s }
+      end
+
+      def match?(kase, filter)
+        filter.nil? || File.fnmatch(filter, kase.to_s)
+      end
+
+      def parse(argv)
+        options = {
+          provider: ENV["ELELEM_PROVIDER"] || "llama",
+          cases: File.join(ROOT, "cases"),
+          fixtures: File.join(ROOT, "fixtures"),
+          prompt: nil,
+          filter: nil,
+          list: false
+        }
+
+        OptionParser.new do |parser|
+          parser.banner = "Usage: elelem-evals [options]"
+
+          parser.on("-p", "--provider NAME", "Provider to run against (default: llama)") { |v| options[:provider] = v }
+          parser.on("-c", "--cases GLOB", "Filter cases by group/id glob, e.g. build/*") { |v| options[:filter] = v }
+          parser.on("--cases-dir DIR", "Directory of case yaml files") { |v| options[:cases] = v }
+          parser.on("--fixtures-dir DIR", "Directory of fixture directories") { |v| options[:fixtures] = v }
+          parser.on("-P", "--prompt FILE", "System prompt template file") { |v| options[:prompt] = File.read(v) }
+          parser.on("-l", "--list", "List matching cases without running them") { options[:list] = true }
+        end.parse!(argv)
+
+        options
+      end
+    end
+  end
+end

lib/elelem/evals/null_terminal.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module Elelem
+  module Evals
+    class NullTerminal < Elelem::Terminal
+      def initialize
+        super(quiet: true)
+      end
+
+      def interactive?
+        false
+      end
+    end
+  end
+end

lib/elelem/evals/reporter.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+module Elelem
+  module Evals
+    class Reporter
+      def initialize(out: $stdout)
+        @out = out
+        @results = []
+        @errors = 0
+      end
+
+      def record(result)
+        @results << result
+        @out.puts "#{result.passed? ? "PASS" : "FAIL"} #{result.group}/#{result.id}"
+        return if result.passed?
+
+        result.failures.each { |failure| @out.puts "  #{failure}" }
+      end
+
+      def error(kase, error)
+        @errors += 1
+        @out.puts "ERROR #{kase.group}/#{kase.id}: #{error.class}: #{error.message}"
+      end
+
+      def summarize
+        passed = @results.count(&:passed?)
+        total = @results.size + @errors
+        @out.puts
+        @out.puts "#{passed}/#{total} passed"
+        passed == total && @errors.zero?
+      end
+    end
+  end
+end

lib/elelem/evals/result.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+module Elelem
+  module Evals
+    Result = Data.define(:id, :group, :failures, :response) do
+      def passed?
+        failures.empty?
+      end
+    end
+  end
+end

lib/elelem/evals/runner.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+module Elelem
+  module Evals
+    class Runner
+      def initialize(fixtures:, client:, prompt: nil)
+        @fixtures = fixtures
+        @client = client
+        @prompt = prompt
+      end
+
+      def run(kase)
+        response = nil
+        bounded = BoundedClient.new(@client.call, max_turns: kase.max_turns)
+
+        Workspace.open(fixtures: @fixtures, fixture: kase.fixture, setup: kase.setup) do |workspace|
+          workspace.chdir do
+            agent = build_agent(bounded)
+            kase.turns.each { |turn| response = agent.turn(turn) }
+
+            failures = Assertions.new(kase.expect).failures(workspace: workspace, response: response)
+
+            Result.new(id: kase.id, group: kase.group, failures: failures, response: response.to_s)
+          end
+        end
+      end
+
+      private
+
+      def build_agent(client)
+        agent = Elelem::Agent.new(client, terminal: NullTerminal.new, system_prompt: @prompt)
+        Elelem::Plugins.setup!(agent)
+        agent
+      end
+    end
+  end
+end

lib/elelem/evals/version.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+module Elelem
+  module Evals
+    VERSION = "0.1.0"
+  end
+end

lib/elelem/evals/workspace.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+require "fileutils"
+require "tmpdir"
+
+module Elelem
+  module Evals
+    class Workspace
+      SetupFailed = Class.new(StandardError)
+
+      def self.open(fixtures:, fixture:, setup: [])
+        dir = Dir.mktmpdir("elelem-evals-")
+        workspace = new(dir, fixtures, fixture)
+        workspace.prepare(setup)
+        yield workspace
+      ensure
+        FileUtils.remove_entry(dir) if dir && File.directory?(dir)
+      end
+
+      attr_reader :path
+
+      def initialize(path, fixtures, fixture)
+        @path = path
+        @fixtures = fixtures
+        @fixture = fixture
+      end
+
+      def prepare(setup)
+        FileUtils.cp_r("#{@fixtures}/#{@fixture}/.", @path)
+        chdir { setup.each { |command| run_setup(command) } }
+      end
+
+      def chdir(&block)
+        Dir.chdir(@path) { Bundler.with_unbundled_env(&block) }
+      end
+
+      def sh(command)
+        Elelem.sh("bash", args: ["-c", command], cwd: @path)
+      end
+
+      def read(relative)
+        file = File.join(@path, relative)
+        File.exist?(file) ? File.read(file) : nil
+      end
+
+      private
+
+      def run_setup(command)
+        result = sh(command)
+        return if result[:exit_status].zero?
+
+        raise SetupFailed, "setup command failed (exit #{result[:exit_status]}): #{command}\n#{result[:content]}"
+      end
+    end
+  end
+end

lib/elelem/evals.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+require "elelem"
+
+require_relative "evals/version"
+require_relative "evals/case"
+require_relative "evals/bounded_client"
+require_relative "evals/assertions"
+require_relative "evals/workspace"
+require_relative "evals/null_terminal"
+require_relative "evals/result"
+require_relative "evals/runner"
+require_relative "evals/reporter"
+require_relative "evals/cli"