Commit 29572c7

@@ -6,7 +6,7 @@ module Elelem
       def initialize(
         model:,
         host: "localhost:11434",
-        think: "medium",
+        think: false,
         keep_alive: "5m",
         options: {},
         params: {},
@@ -104,7 +104,7 @@ module Elelem
       def build_request_body(messages, tools)
         {
           model: @model,
-          messages:,
+          messages: normalize(messages),
           stream: true,
           tools: presence(tools),
           think: @think,
@@ -117,6 +117,36 @@ module Elelem
         value unless value.nil? || value.empty?
       end
 
+      def normalize(messages)
+        pending = []
+
+        messages.map do |message|
+          case message[:role]
+          when "assistant" then normalize_calls(message, pending)
+          when "tool" then normalize_result(message, pending)
+          else message
+          end
+        end
+      end
+
+      def normalize_calls(message, pending)
+        calls = message[:tool_calls]
+        return message unless calls
+
+        pending.clear
+        message.merge(tool_calls: calls.map do |call|
+          pending << call[:name]
+          { function: { name: call[:name], arguments: call[:arguments] } }
+        end)
+      end
+
+      def normalize_result(message, pending)
+        name = pending.shift
+        return message unless name
+
+        message.except(:tool_call_id).merge(tool_name: name)
+      end
+
       def handle_event(event, tool_calls, &block)
         message = event["message"] || {}
 

@@ -6,9 +6,9 @@ Terminal coding agent. Be concise. Verify your work.
 - execute(command): shell command
 
 # Editing
-Use execute(`patch -p1`) for multi-line changes: `echo "DIFF" | patch -p1`
-Use execute(`sed`) for single-line changes: `sed -i'' 's/old/new/' file`
-Use write for new files or full rewrites
+Use write to change a file: read it, then write the full new contents.
+Use execute(`sed`) only for a trivial single-line substitution.
+Never shell out to patch, apply_patch, or heredocs to edit files.
 
 # Search
 Use execute(`rg`) for text search: `rg -n "pattern" .`

@@ -33,7 +33,7 @@ module Elelem
 
     def run(name, args)
       tool = tool_for(name)
-      return failure(error: "unknown tool: #{name}. Use 'execute' to run shell commands like rg, fd, git.", tools: to_a) unless tool
+      return failure(error: "unknown tool: #{name.inspect}. Available: #{tools.keys.join(", ")}.") unless tool
 
       errors = tool.validate(args)
       return failure(error: errors.join(", ")) if errors.any?

@@ -29,10 +29,60 @@ RSpec.describe Elelem::Net::Ollama do
   end
 
   describe "#fetch" do
-    it "sends only model, messages and stream by default" do
+    it "leaves thinking off by default" do
       client.fetch(messages) { }
 
-      expect(body).to eq(model: "gpt-oss:latest", messages:, stream: true, think: "medium", keep_alive: "5m")
+      expect(body).to eq(model: "gpt-oss:latest", messages:, stream: true, think: false, keep_alive: "5m")
+    end
+
+    it "nests assistant tool calls under function so ollama can name them" do
+      history = [
+        { role: "assistant", content: "", tool_calls: [{ id: "abc", name: "execute", arguments: { "command" => "ls" } }] },
+        { role: "tool", tool_call_id: "abc", content: "{}" }
+      ]
+
+      client.fetch(history) { }
+
+      expect(body[:messages].first[:tool_calls]).to eq([{ function: { name: "execute", arguments: { "command" => "ls" } } }])
+    end
+
+    it "attributes a tool result to the tool that produced it" do
+      history = [
+        { role: "assistant", content: "", tool_calls: [{ id: "abc", name: "execute", arguments: {} }] },
+        { role: "tool", tool_call_id: "abc", content: "{}" }
+      ]
+
+      client.fetch(history) { }
+
+      expect(body[:messages].last).to eq(role: "tool", tool_name: "execute", content: "{}")
+    end
+
+    it "attributes each result of a multi call turn to its own tool" do
+      history = [
+        {
+          role: "assistant",
+          content: "",
+          tool_calls: [
+            { id: nil, name: "read", arguments: { "path" => "a.rb" } },
+            { id: nil, name: "execute", arguments: { "command" => "ls" } }
+          ]
+        },
+        { role: "tool", tool_call_id: nil, content: "a" },
+        { role: "tool", tool_call_id: nil, content: "b" }
+      ]
+
+      client.fetch(history) { }
+
+      expect(body[:messages].last(2)).to eq([
+        { role: "tool", tool_name: "read", content: "a" },
+        { role: "tool", tool_name: "execute", content: "b" }
+      ])
+    end
+
+    it "leaves messages without tool calls untouched" do
+      client.fetch(messages) { }
+
+      expect(body[:messages]).to eq(messages)
     end
 
     it "sends tools when present" do

@@ -3,10 +3,12 @@
 RSpec.describe Elelem::Providers do
   before do
     described_class.registry.clear
+    Elelem::Plugins.registry.clear
   end
 
   after do
     described_class.registry.clear
+    Elelem::Plugins.registry.clear
   end
 
   describe ".register" do

@@ -0,0 +1,12 @@
+- id: multi-line-edit
+  fixture: hello
+  turns:
+    - 'change greet so it prints "Goodbye, #{name}." with a period instead of an
+      exclamation mark, and call it with "friend" instead of "world"'
+  expect:
+    verify: ruby hello.rb | grep -qxF "Goodbye, friend."
+    files:
+      hello.rb:
+        contains: ["Goodbye"]
+        not_contains: ["Hello"]
+    max_turns: 8

@@ -0,0 +1,11 @@
+- id: fix-failing-test
+  fixture: broken
+  turns:
+    - run test.rb and fix whatever is broken
+  expect:
+    verify: ruby test.rb
+    files:
+      lib/calc.rb:
+        contains: ["a + b"]
+    tools_used: ["execute"]
+    max_turns: 10

@@ -0,0 +1,15 @@
+- id: commit-50-72
+  fixture: hello
+  setup:
+    - git init -q
+    - git config user.email evals@example.com
+    - git config user.name evals
+    - git add -A
+    - git commit -q -m "initial commit"
+    - printf 'greet("again")\n' >> hello.rb
+  turns:
+    - commit the current changes with a message that follows the 50/72 rule
+  expect:
+    verify: test -z "$(git status --porcelain)" && test "$(git log -1 --format=%s | wc -c)" -le 51
+    tools_used: ["execute"]
+    max_turns: 8

@@ -0,0 +1,9 @@
+- id: summarize-json-log
+  fixture: logs
+  turns:
+    - which path returned the most 5xx responses in requests.jsonl, and how many?
+  expect:
+    response_contains: ["/scim/v2/Users"]
+    response_matches: ['/scim/v2/Users\D{0,40}3']
+    tools_not_used: ["write"]
+    max_turns: 6

@@ -0,0 +1,17 @@
+- id: locate-error-string
+  fixture: config
+  turns:
+    - where does the "Failed to load configuration" error come from?
+  expect:
+    response_contains: ["config/loader.rb"]
+    tools_not_used: ["write"]
+    max_turns: 6
+
+- id: list-references
+  fixture: metadata
+  turns:
+    - which files reference app_metadata?
+  expect:
+    response_contains: ["user.rb", "token.rb"]
+    tools_not_used: ["write"]
+    max_turns: 6

@@ -0,0 +1,22 @@
+- id: largest-tracked-file
+  fixture: sizes
+  setup:
+    - for i in $(seq 1 400); do printf '%d,widget-%d,%d\n' "$i" "$i" $((i * 7)) >> data/report.csv; done
+    - git init -q
+    - git config user.email evals@example.com
+    - git config user.name evals
+    - git add -A
+    - git commit -q -m "initial commit"
+    # .git/description is never read by git, so padding it is a safe stand-in
+    # for the pack files that dominate a real repository's byte counts.
+    - for i in $(seq 1 1200); do printf 'padding line %d\n' "$i" >> .git/description; done
+  turns:
+    - what is the biggest file in this repository?
+  expect:
+    response_contains: ["report.csv"]
+    response_not_contains: [".git"]
+    tools_not_used:
+      - write
+      - execute: { command: "-printf" }
+      - execute: { command: "du -b" }
+    max_turns: 6

@@ -0,0 +1,3 @@
+def add(a, b)
+  a - b
+end

@@ -0,0 +1,5 @@
+require_relative "lib/calc"
+
+raise "add(2, 3) should be 5, got #{add(2, 3)}" unless add(2, 3) == 5
+
+puts "ok"

@@ -0,0 +1,11 @@
+class Loader
+  def initialize(path)
+    @path = path
+  end
+
+  def call
+    raise "Failed to load configuration: #{@path}" unless File.exist?(@path)
+
+    Parser.new(File.read(@path)).call
+  end
+end

@@ -0,0 +1,9 @@
+class Parser
+  def initialize(text)
+    @text = text
+  end
+
+  def call
+    @text.lines.map { |line| line.split(":", 2) }.to_h
+  end
+end

@@ -0,0 +1,3 @@
+require_relative "config/loader"
+
+Loader.new("app.yml").call

@@ -0,0 +1,5 @@
+def greet(name)
+  puts "Hello, #{name}!"
+end
+
+greet("world")

@@ -0,0 +1,12 @@
+{"path":"/scim/v2/Users","response_code":200}
+{"path":"/scim/v2/Users","response_code":500}
+{"path":"/scim/v2/Users","response_code":503}
+{"path":"/scim/v2/Users","response_code":500}
+{"path":"/scim/v2/Groups","response_code":200}
+{"path":"/scim/v2/Groups","response_code":404}
+{"path":"/scim/v2/Groups","response_code":200}
+{"path":"/health","response_code":200}
+{"path":"/health","response_code":500}
+{"path":"/health","response_code":200}
+{"path":"/scim/v2/Schemas","response_code":200}
+{"path":"/scim/v2/Schemas","response_code":304}

@@ -0,0 +1,10 @@
+class Session
+  def initialize(id, user_id)
+    @id = id
+    @user_id = user_id
+  end
+
+  def to_h
+    { "id" => @id, "user_id" => @user_id }
+  end
+end

@@ -0,0 +1,9 @@
+class Token
+  def initialize(claims)
+    @claims = claims
+  end
+
+  def to_h
+    @claims.merge("app_metadata" => @claims["app_metadata"] || {})
+  end
+end

@@ -0,0 +1,9 @@
+class User
+  def initialize(row)
+    @row = row
+  end
+
+  def app_metadata
+    @row.fetch("app_metadata", {})
+  end
+end

@@ -0,0 +1,1 @@
+id,widget,cents

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+require "csv"
+
+class App
+  def initialize(path = "data/report.csv")
+    @path = path
+  end
+
+  def rows
+    CSV.read(@path, headers: true)
+  end
+
+  def total
+    rows.sum { |row| row["cents"].to_i }
+  end
+end

@@ -0,0 +1,3 @@
+# Widgets
+
+Inventory reports live in `data/`.

@@ -0,0 +1,111 @@
+# frozen_string_literal: true
+
+RSpec.describe Elelem::Evals::Assertions do
+  def failures(expect, workspace: nil, response: "", tools: [], turns: 1)
+    described_class.new(expect).failures(workspace: workspace, response: response, tools: tools, turns: turns)
+  end
+
+  describe "verify" do
+    it "passes when the command exits 0" do
+      Elelem::Evals::Workspace.open(fixture: "hello") do |workspace|
+        expect(failures({ verify: "true" }, workspace: workspace)).to be_empty
+      end
+    end
+
+    it "fails when the command exits non zero" do
+      Elelem::Evals::Workspace.open(fixture: "hello") do |workspace|
+        expect(failures({ verify: "false" }, workspace: workspace).first).to include("verify failed")
+      end
+    end
+  end
+
+  describe "files" do
+    it "fails when a required string is missing" do
+      Elelem::Evals::Workspace.open(fixture: "hello") do |workspace|
+        result = failures({ files: { "hello.rb" => { "contains" => ["Goodbye"] } } }, workspace: workspace)
+
+        expect(result.first).to include("hello.rb missing")
+      end
+    end
+
+    it "fails when a forbidden string is present" do
+      Elelem::Evals::Workspace.open(fixture: "hello") do |workspace|
+        result = failures({ files: { "hello.rb" => { "not_contains" => ["Hello"] } } }, workspace: workspace)
+
+        expect(result.first).to include("still contains")
+      end
+    end
+
+    it "fails when the file is missing" do
+      Elelem::Evals::Workspace.open(fixture: "hello") do |workspace|
+        result = failures({ files: { "nope.rb" => { "contains" => ["x"] } } }, workspace: workspace)
+
+        expect(result).to eq(["nope.rb missing"])
+      end
+    end
+  end
+
+  describe "response" do
+    it "passes when every string is present" do
+      expect(failures({ response_contains: %w[alpha beta] }, response: "alpha and beta")).to be_empty
+    end
+
+    it "fails when a string is absent" do
+      expect(failures({ response_contains: ["gamma"] }, response: "alpha").first).to include("response missing")
+    end
+
+    it "fails when a forbidden string is present" do
+      result = failures({ response_not_contains: [".git"] }, response: "the biggest file is .git/description")
+
+      expect(result.first).to include("response contains")
+    end
+
+    it "passes when no forbidden string is present" do
+      expect(failures({ response_not_contains: [".git"] }, response: "data/report.csv")).to be_empty
+    end
+
+    it "fails when a regex does not match" do
+      expect(failures({ response_matches: ['\d+'] }, response: "none").first).to include("does not match")
+    end
+  end
+
+  describe "tools" do
+    let(:tools) { [Elelem::Evals::ToolCall.new(name: "read", args: { "path" => "config/loader.rb" })] }
+
+    it "passes when a named tool was called" do
+      expect(failures({ tools_used: ["read"] }, tools: tools)).to be_empty
+    end
+
+    it "fails when a named tool was never called" do
+      expect(failures({ tools_used: ["write"] }, tools: tools).first).to include("never called")
+    end
+
+    it "passes when argument constraints match" do
+      expect(failures({ tools_used: [{ "read" => { "path" => "loader.rb" } }] }, tools: tools)).to be_empty
+    end
+
+    it "fails when argument constraints do not match" do
+      result = failures({ tools_used: [{ "read" => { "path" => "other.rb" } }] }, tools: tools)
+
+      expect(result.first).to include("never called")
+    end
+
+    it "fails when a forbidden tool was called" do
+      expect(failures({ tools_not_used: ["read"] }, tools: tools).first).to include("called read")
+    end
+  end
+
+  describe "max_turns" do
+    it "passes at the limit" do
+      expect(failures({ max_turns: 3 }, turns: 3)).to be_empty
+    end
+
+    it "fails over the limit" do
+      expect(failures({ max_turns: 3 }, turns: 4).first).to include("used 4 turns")
+    end
+  end
+
+  it "reports every failure at once" do
+    expect(failures({ response_contains: %w[a b], max_turns: 1 }, response: "", turns: 2).length).to eq(3)
+  end
+end

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+RSpec.describe Elelem::Evals::BoundedClient do
+  let(:inner) { double("client", fetch: []) }
+
+  subject(:client) { described_class.new(inner, max_turns: 2) }
+
+  it "delegates to the wrapped client" do
+    client.fetch([{ role: "user", content: "hi" }])
+
+    expect(inner).to have_received(:fetch)
+  end
+
+  it "counts turns" do
+    2.times { client.fetch([]) }
+
+    expect(client.turns).to eq(2)
+  end
+
+  it "raises once past the limit" do
+    2.times { client.fetch([]) }
+
+    expect { client.fetch([]) }.to raise_error(described_class::TurnLimitExceeded)
+  end
+
+  it "reports the overrun" do
+    3.times { client.fetch([]) rescue nil }
+
+    expect(client.turns).to eq(3)
+  end
+end

@@ -0,0 +1,61 @@
+# frozen_string_literal: true
+
+RSpec.describe Elelem::Evals::Case do
+  let(:yaml) do
+    <<~YAML
+      - id: patch-single-line
+        fixture: hello
+        setup:
+          - git init -q
+        turns:
+          - change the greeting to Goodbye
+        expect:
+          verify: ruby hello.rb
+          max_turns: 4
+      - id: no-expectations
+        fixture: hello
+        turns:
+          - say hello
+    YAML
+  end
+
+  let(:dir) { Dir.mktmpdir }
+  let(:file) do
+    path = File.join(dir, "editing.yml")
+    File.write(path, yaml)
+    path
+  end
+
+  after { FileUtils.remove_entry(dir) if File.directory?(dir) }
+
+  subject(:cases) { described_class.load_file(file) }
+
+  it "loads every case in the file" do
+    expect(cases.map(&:id)).to eq(%w[patch-single-line no-expectations])
+  end
+
+  it "takes the group from the file name" do
+    expect(cases.first.group).to eq("editing")
+  end
+
+  it "reads setup commands" do
+    expect(cases.first.setup).to eq(["git init -q"])
+  end
+
+  it "symbolizes the top level of expect" do
+    expect(cases.first.expect[:verify]).to eq("ruby hello.rb")
+  end
+
+  it "reads max_turns from expect" do
+    expect(cases.first.max_turns).to eq(4)
+  end
+
+  it "defaults max_turns to 10" do
+    expect(cases.last.max_turns).to eq(10)
+  end
+
+  it "defaults setup and expect to empty" do
+    expect(cases.last.setup).to eq([])
+    expect(cases.last.expect).to eq({})
+  end
+end

@@ -0,0 +1,127 @@
+# frozen_string_literal: true
+
+RSpec.describe Elelem::Evals::Improver do
+  # Replays a canned JSON body the way Ollama streams it.
+  class CannedClient
+    attr_reader :messages
+
+    def initialize(body)
+      @body = body
+    end
+
+    def fetch(messages, _tools = [], &block)
+      @messages = messages
+      block.call(type: "saying", text: @body)
+      []
+    end
+  end
+
+  let(:body) do
+    JSON.generate(
+      "analysis" => "the prompt never mentions git",
+      "changes" => [
+        { "old_text" => "# Policy", "new_text" => "# Git\nCommit with a 50 character subject.\n\n# Policy", "rationale" => "teach the 50/72 rule" }
+      ]
+    )
+  end
+
+  let(:client) { CannedClient.new(body) }
+  let(:prompt) { "Terminal coding agent.\n\n# Policy\n- Verify your work\n" }
+
+  subject(:improver) { described_class.new(client: client) }
+
+  describe "#plan" do
+    it "parses the model's JSON" do
+      plan = improver.plan(prompt: prompt, failures: [])
+
+      expect(plan["changes"].first["rationale"]).to eq("teach the 50/72 rule")
+    end
+
+    it "sends the prompt and the failures to the model" do
+      failure = Elelem::Evals::Result.new(
+        id: "commit-50-72", group: "git", status: "FAIL", failures: ["verify failed (exit 1)"],
+        turns: 3, duration: 1.0, tools: [], response: "committed"
+      )
+
+      improver.plan(prompt: prompt, failures: [failure])
+      sent = client.messages.map { |m| m[:content] }.join("\n")
+
+      expect(sent).to include("commit-50-72")
+      expect(sent).to include("verify failed")
+      expect(sent).to include("# Policy")
+    end
+
+    it "returns no changes when the model returns junk" do
+      plan = described_class.new(client: CannedClient.new("not json")).plan(prompt: prompt, failures: [])
+
+      expect(plan).to eq("analysis" => "", "changes" => [])
+    end
+
+    it "returns no changes when the model returns a JSON array" do
+      plan = described_class.new(client: CannedClient.new("[1, 2, 3]")).plan(prompt: prompt, failures: [])
+
+      expect(plan).to eq("analysis" => "", "changes" => [])
+    end
+
+    it "returns no changes when the model returns JSON null" do
+      plan = described_class.new(client: CannedClient.new("null")).plan(prompt: prompt, failures: [])
+
+      expect(plan).to eq("analysis" => "", "changes" => [])
+    end
+
+    it "returns no changes when changes is a JSON object" do
+      body = JSON.generate("analysis" => "x", "changes" => { "old_text" => "a", "new_text" => "b" })
+      plan = described_class.new(client: CannedClient.new(body)).plan(prompt: prompt, failures: [])
+
+      expect(plan["changes"]).to be_empty
+    end
+
+    it "does not share the changes array between fallback results" do
+      improver = described_class.new(client: CannedClient.new("not json"))
+
+      first = improver.plan(prompt: prompt, failures: [])
+      first["changes"] << "mutated"
+
+      expect(improver.plan(prompt: prompt, failures: [])["changes"]).to be_empty
+    end
+  end
+
+  describe "#apply" do
+    it "applies a change and reports the rationale" do
+      changes = [{ "old_text" => "# Policy", "new_text" => "# Git\n\n# Policy", "rationale" => "teach git" }]
+      updated, applied = improver.apply(prompt, changes)
+
+      expect(updated).to include("# Git")
+      expect(applied).to eq(["teach git"])
+    end
+
+    it "skips a change whose old_text is absent" do
+      changes = [{ "old_text" => "# Nowhere", "new_text" => "x", "rationale" => "nope" }]
+      updated, applied = improver.apply(prompt, changes)
+
+      expect(updated).to eq(prompt)
+      expect(applied).to be_empty
+    end
+
+    it "keeps regex escapes in new_text literal" do
+      changes = [{ "old_text" => "# Policy", "new_text" => "Use sed -i'' 's/a\\&b/c/' and \\0", "rationale" => "sed" }]
+      updated, = improver.apply(prompt, changes)
+
+      expect(updated).to include("s/a\\&b/c/' and \\0")
+    end
+
+    it "skips a change that is not an object" do
+      updated, applied = improver.apply(prompt, [["old_text", "# Policy"]])
+
+      expect(updated).to eq(prompt)
+      expect(applied).to be_empty
+    end
+
+    it "replaces only the first occurrence" do
+      changes = [{ "old_text" => "a", "new_text" => "b", "rationale" => "once" }]
+      updated, = improver.apply("a a a", changes)
+
+      expect(updated).to eq("b a a")
+    end
+  end
+end

@@ -0,0 +1,144 @@
+# frozen_string_literal: true
+
+RSpec.describe Elelem::Evals::Loop do
+  let(:dir) { Dir.mktmpdir }
+  let(:champion) { File.join(dir, "champion.erb") }
+  let(:cases) { [Elelem::Evals::Case.new(id: "a", group: "g", fixture: "hello", turns: ["hi"])] }
+  let(:out) { StringIO.new }
+  let(:no_preflight) { -> {} }
+  let(:changes) { [{ "old_text" => "original", "new_text" => "improved", "rationale" => "better" }] }
+
+  before { File.write(champion, "original prompt") }
+  after { FileUtils.remove_entry(dir) if File.directory?(dir) }
+
+  def result(id: "a", group: "g", status: "FAIL")
+    Elelem::Evals::Result.new(
+      id: id, group: group, status: status, failures: status == "PASS" ? [] : ["nope"],
+      turns: 1, duration: 0.1, tools: [], response: ""
+    )
+  end
+
+  def score(*results)
+    Elelem::Evals::Score.new(results: results)
+  end
+
+  def score_of(status)
+    score(result(status: status))
+  end
+
+  def scoring(by_prompt)
+    ->(prompt) { by_prompt.fetch(prompt) }
+  end
+
+  # Always proposes the same edit.
+  def improver_returning(changes)
+    instance_double(
+      Elelem::Evals::Improver,
+      plan: { "analysis" => "x", "changes" => changes },
+      apply: ["improved prompt", changes.map { |c| c["rationale"] }]
+    )
+  end
+
+  def build_loop(scorer_for:, improver: improver_returning(changes), preflight: no_preflight)
+    described_class.new(
+      cases: cases, champion: champion, workdir: dir, improver: improver,
+      scorer_for: scorer_for, out: out, preflight: preflight
+    )
+  end
+
+  it "promotes a challenger that fixes a case" do
+    build_loop(scorer_for: scoring("original prompt" => score_of("FAIL"), "improved prompt" => score_of("PASS"))).run(rounds: 1)
+
+    expect(File.read(champion)).to eq("improved prompt")
+  end
+
+  it "rejects a challenger that regresses a case" do
+    build_loop(scorer_for: scoring("original prompt" => score_of("PASS"), "improved prompt" => score_of("FAIL"))).run(rounds: 1)
+
+    expect(File.read(champion)).to eq("original prompt")
+  end
+
+  it "stops early when everything already passes" do
+    improver = improver_returning(changes)
+
+    build_loop(improver: improver, scorer_for: ->(_prompt) { score_of("PASS") }).run(rounds: 3)
+
+    expect(improver).not_to have_received(:plan)
+  end
+
+  it "writes each challenger to its own file" do
+    build_loop(scorer_for: scoring("original prompt" => score_of("FAIL"), "improved prompt" => score_of("PASS"))).run(rounds: 1)
+
+    expect(File).to exist(File.join(dir, "challenger-1.erb"))
+    expect(File.read(File.join(dir, "challenger-1.erb"))).to eq("improved prompt")
+  end
+
+  it "keeps the champion unchanged and consumes every round when no change can ever be applied" do
+    improver = instance_double(
+      Elelem::Evals::Improver,
+      plan: { "analysis" => "x", "changes" => changes },
+      apply: ["improved prompt", []]
+    )
+
+    result = build_loop(improver: improver, scorer_for: ->(_prompt) { score_of("FAIL") }).run(rounds: 3)
+
+    expect(result).to be(false)
+    expect(File.read(champion)).to eq("original prompt")
+    expect(File).not_to exist(File.join(dir, "challenger-1.erb"))
+    expect(improver).to have_received(:plan).exactly(3).times
+  end
+
+  it "appends a round to history" do
+    build_loop(scorer_for: scoring("original prompt" => score_of("FAIL"), "improved prompt" => score_of("PASS"))).run(rounds: 1)
+
+    entry = JSON.parse(File.read(File.join(dir, "history.jsonl")).lines.first)
+
+    expect(entry).to include("round" => 1, "promoted" => true)
+  end
+
+  it "withholds holdout failures from the improver" do
+    holdout = result(id: "h", group: "holdout")
+    visible = result
+    improver = improver_returning(changes)
+
+    build_loop(improver: improver, scorer_for: ->(_p) { score(holdout, visible) }).run(rounds: 1)
+
+    expect(improver).to have_received(:plan).with(prompt: anything, failures: [visible])
+  end
+
+  it "does not promote a challenger that only improves a holdout case" do
+    baseline = score(result, result(id: "h", group: "holdout"))
+    holdout_only = score(result, result(id: "h", group: "holdout", status: "PASS"))
+
+    build_loop(scorer_for: scoring("original prompt" => baseline, "improved prompt" => holdout_only)).run(rounds: 1)
+
+    expect(File.read(champion)).to eq("original prompt")
+  end
+
+  it "does not promote a challenger that regresses a holdout case" do
+    baseline = score(result, result(id: "h", group: "holdout", status: "PASS"))
+    regressed = score(result(status: "PASS"), result(id: "h", group: "holdout"))
+
+    build_loop(scorer_for: scoring("original prompt" => baseline, "improved prompt" => regressed)).run(rounds: 1)
+
+    expect(File.read(champion)).to eq("original prompt")
+  end
+
+  it "sends the improver only one failure per case even when every repeat fails" do
+    fails = Array.new(3) { result }
+    improver = improver_returning(changes)
+
+    build_loop(improver: improver, scorer_for: ->(_p) { score(*fails) }).run(rounds: 1)
+
+    expect(improver).to have_received(:plan).with(prompt: anything, failures: [fails.first])
+  end
+
+  it "raises before scoring anything when the preflight check fails" do
+    loop = build_loop(
+      scorer_for: ->(_prompt) { raise "scoring should not have run" },
+      preflight: -> { raise "connection refused" }
+    )
+
+    expect { loop.run(rounds: 1) }.to raise_error(/connection refused/)
+  end
+end

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+RSpec.describe Elelem::Evals::NullTerminal do
+  it "auto-allows tools the shipped rules would otherwise ask about" do
+    terminal = described_class.new
+
+    expect(terminal).not_to be_interactive
+    expect(Elelem::Permissions.new.check("execute", {}, terminal: terminal)).to be(true)
+  end
+end

@@ -0,0 +1,90 @@
+# frozen_string_literal: true
+
+RSpec.describe Elelem::Evals::Runner do
+  # Replies with a write tool call on the first fetch, then plain text.
+  class ScriptedClient
+    def initialize
+      @calls = 0
+    end
+
+    def fetch(_messages, _tools = [], &block)
+      @calls += 1
+      return first_turn(&block) if @calls == 1
+
+      block.call(type: "saying", text: "done, wrote goodbye.rb")
+      []
+    end
+
+    private
+
+    def first_turn(&block)
+      call = { id: "1", name: "write", arguments: { "path" => "goodbye.rb", "content" => "puts 'Goodbye'\n" } }
+      block.call(call.merge(type: "doing"))
+      [call]
+    end
+  end
+
+  let(:kase) do
+    Elelem::Evals::Case.new(
+      id: "writes-a-file",
+      group: "editing",
+      fixture: "hello",
+      turns: ["write goodbye.rb"],
+      expect: {
+        verify: "ruby goodbye.rb",
+        files: { "goodbye.rb" => { "contains" => ["Goodbye"] } },
+        response_contains: ["goodbye.rb"],
+        tools_used: [{ "write" => { "path" => "goodbye.rb" } }],
+        tools_not_used: ["read"],
+        max_turns: 5
+      }
+    )
+  end
+
+  subject(:runner) { described_class.new(prompt: "test prompt", client: -> { ScriptedClient.new }) }
+
+  it "passes a case the agent satisfies" do
+    result = runner.run(kase)
+
+    expect(result.failures).to be_empty
+    expect(result.status).to eq("PASS")
+  end
+
+  it "records the case identity" do
+    result = runner.run(kase)
+
+    expect(result.id).to eq("writes-a-file")
+    expect(result.group).to eq("editing")
+  end
+
+  it "records the tool calls" do
+    result = runner.run(kase)
+
+    expect(result.tools.map(&:name)).to eq(["write"])
+  end
+
+  it "counts turns" do
+    expect(runner.run(kase).turns).to eq(2)
+  end
+
+  it "fails a case whose assertions do not hold" do
+    failing = Elelem::Evals::Case.new(
+      id: "wants-more",
+      fixture: "hello",
+      turns: ["write goodbye.rb"],
+      expect: { response_contains: ["never said this"] }
+    )
+
+    result = runner.run(failing)
+
+    expect(result.status).to eq("FAIL")
+    expect(result.failures.first).to include("response missing")
+  end
+
+  it "leaves the working directory where it found it" do
+    before = Dir.pwd
+    runner.run(kase)
+
+    expect(Dir.pwd).to eq(before)
+  end
+end

@@ -0,0 +1,92 @@
+# frozen_string_literal: true
+
+RSpec.describe Elelem::Evals::Scorer do
+  def result(id, status)
+    Elelem::Evals::Result.new(
+      id: id, group: "g", status: status, failures: status == "PASS" ? [] : ["nope"],
+      turns: 1, duration: 0.1, tools: [], response: ""
+    )
+  end
+
+  # Returns the statuses it was given, in order, one per run call.
+  class FakeRunner
+    def initialize(statuses)
+      @statuses = statuses.dup
+    end
+
+    def run(kase)
+      status = @statuses.shift
+      Elelem::Evals::Result.new(
+        id: kase.id, group: kase.group, status: status, failures: status == "PASS" ? [] : ["nope"],
+        turns: 1, duration: 0.1, tools: [], response: ""
+      )
+    end
+  end
+
+  let(:kase) { Elelem::Evals::Case.new(id: "a", group: "g", fixture: "hello", turns: ["hi"]) }
+
+  it "passes a case only when every repeat passes" do
+    scorer = described_class.new(runner: FakeRunner.new(%w[PASS PASS PASS]), repeat: 3)
+
+    expect(scorer.call([kase]).passed).to eq(1)
+  end
+
+  it "fails a case when any repeat fails" do
+    scorer = described_class.new(runner: FakeRunner.new(%w[PASS FAIL PASS]), repeat: 3)
+    score = scorer.call([kase])
+
+    expect(score.passed).to eq(0)
+    expect(score.failed).to eq(1)
+  end
+
+  it "keeps every result" do
+    scorer = described_class.new(runner: FakeRunner.new(%w[PASS PASS PASS]), repeat: 3)
+
+    expect(scorer.call([kase]).results.length).to eq(3)
+  end
+
+  it "reports only failing results as failures" do
+    scorer = described_class.new(runner: FakeRunner.new(%w[PASS FAIL PASS]), repeat: 3)
+
+    expect(scorer.call([kase]).failures.length).to eq(1)
+  end
+
+  it "detects a regression against an earlier score" do
+    before = described_class.new(runner: FakeRunner.new(["PASS"]), repeat: 1).call([kase])
+    after = described_class.new(runner: FakeRunner.new(["FAIL"]), repeat: 1).call([kase])
+
+    expect(after.regressions_from(before)).to eq(["g/a"])
+  end
+
+  it "does not call an improvement a regression" do
+    before = described_class.new(runner: FakeRunner.new(["FAIL"]), repeat: 1).call([kase])
+    after = described_class.new(runner: FakeRunner.new(["PASS"]), repeat: 1).call([kase])
+
+    expect(after.regressions_from(before)).to be_empty
+  end
+
+  it "keeps cases with the same id in different groups apart" do
+    a = Elelem::Evals::Case.new(id: "x", group: "search", fixture: "hello", turns: ["hi"])
+    b = Elelem::Evals::Case.new(id: "x", group: "json", fixture: "hello", turns: ["hi"])
+    scorer = described_class.new(runner: FakeRunner.new(%w[PASS FAIL]), repeat: 1)
+
+    score = scorer.call([a, b])
+
+    expect(score.total).to eq(2)
+    expect(score.passed).to eq(1)
+  end
+
+  describe "#excluding" do
+    it "filters out every result from the given group" do
+      holdout = Elelem::Evals::Result.new(
+        id: "h", group: "holdout", status: "PASS", failures: [], turns: 1, duration: 0.1, tools: [], response: ""
+      )
+      score = Elelem::Evals::Score.new(results: [result("a", "PASS"), holdout])
+
+      excluded = score.excluding("holdout")
+
+      expect(excluded.total).to eq(1)
+      expect(excluded.results).to eq([result("a", "PASS")])
+    end
+  end
+end

@@ -0,0 +1,62 @@
+# frozen_string_literal: true
+
+RSpec.describe Elelem::Evals::Workspace do
+  it "copies the fixture into a fresh directory" do
+    described_class.open(fixture: "hello") do |workspace|
+      expect(workspace.read("hello.rb")).to include("greet")
+    end
+  end
+
+  it "leaves the fixture free of harness files" do
+    described_class.open(fixture: "hello") do |workspace|
+      expect(Dir.children(workspace.path)).to contain_exactly("hello.rb")
+    end
+  end
+
+  it "runs setup commands in the workspace" do
+    described_class.open(fixture: "hello", setup: ["echo hi > setup.txt"]) do |workspace|
+      expect(workspace.read("setup.txt")).to eq("hi\n")
+    end
+  end
+
+  it "removes the directory afterwards" do
+    path = described_class.open(fixture: "hello", &:path)
+
+    expect(File).not_to exist(path)
+  end
+
+  it "strips bundler variables inside chdir" do
+    saved = ENV["BUNDLE_GEMFILE"]
+    ENV["BUNDLE_GEMFILE"] = "/somewhere/Gemfile"
+
+    described_class.open(fixture: "hello") do |workspace|
+      workspace.chdir { expect(ENV["BUNDLE_GEMFILE"]).to be_nil }
+    end
+
+    expect(ENV["BUNDLE_GEMFILE"]).to eq("/somewhere/Gemfile")
+  ensure
+    saved ? (ENV["BUNDLE_GEMFILE"] = saved) : ENV.delete("BUNDLE_GEMFILE")
+  end
+
+  it "returns nil for a file that does not exist" do
+    described_class.open(fixture: "hello") do |workspace|
+      expect(workspace.read("nope.rb")).to be_nil
+    end
+  end
+
+  it "raises SetupFailed when a setup command fails" do
+    expect do
+      described_class.open(fixture: "hello", setup: ["exit 1"]) { |workspace| workspace }
+    end.to raise_error(described_class::SetupFailed)
+  end
+
+  it "still removes the directory when a setup command fails" do
+    before = Dir.glob(File.join(Dir.tmpdir, "elelem-evals-*"))
+
+    expect do
+      described_class.open(fixture: "hello", setup: ["exit 1"]) { |workspace| workspace }
+    end.to raise_error(described_class::SetupFailed)
+
+    expect(Dir.glob(File.join(Dir.tmpdir, "elelem-evals-*"))).to eq(before)
+  end
+end

@@ -0,0 +1,84 @@
+# frozen_string_literal: true
+
+module Elelem
+  module Evals
+    class Assertions
+      def initialize(expect)
+        @expect = expect || {}
+      end
+
+      def failures(workspace:, response:, tools:, turns:)
+        [
+          verify_failure(workspace),
+          *file_failures(workspace),
+          *response_failures(response),
+          *tool_failures(tools),
+          turns_failure(turns)
+        ].compact
+      end
+
+      private
+
+      def verify_failure(workspace)
+        command = @expect[:verify]
+        return unless command
+
+        result = workspace.sh(command)
+        return if result[:exit_status].zero?
+
+        "verify failed (exit #{result[:exit_status]}): #{command}"
+      end
+
+      def file_failures(workspace)
+        (@expect[:files] || {}).flat_map do |path, rules|
+          content = workspace.read(path)
+          next ["#{path} missing"] unless content
+
+          missing = Array(rules["contains"]).reject { |text| content.include?(text) }
+          present = Array(rules["not_contains"]).select { |text| content.include?(text) }
+
+          missing.map { |text| "#{path} missing #{text.inspect}" } +
+            present.map { |text| "#{path} still contains #{text.inspect}" }
+        end
+      end
+
+      def response_failures(response)
+        text = response.to_s
+
+        Array(@expect[:response_contains]).reject { |s| text.include?(s) }.map { |s| "response missing #{s.inspect}" } +
+          Array(@expect[:response_not_contains]).select { |s| text.include?(s) }.map { |s| "response contains #{s.inspect}" } +
+          Array(@expect[:response_matches]).reject { |p| Regexp.new(p).match?(text) }.map { |p| "response does not match /#{p}/" }
+      end
+
+      def tool_failures(tools)
+        Array(@expect[:tools_used]).reject { |entry| called?(tools, entry) }.map { |entry| "never called #{label(entry)}" } +
+          Array(@expect[:tools_not_used]).select { |name| called?(tools, name) }.map { |name| "called #{name}" }
+      end
+
+      def called?(tools, entry)
+        name, constraints = destructure(entry)
+
+        tools.any? do |call|
+          call.name == name.to_s && constraints.all? { |key, value| call.args[key.to_s].to_s.include?(value.to_s) }
+        end
+      end
+
+      def destructure(entry)
+        return entry.first.then { |name, constraints| [name, constraints || {}] } if entry.is_a?(Hash)
+
+        [entry, {}]
+      end
+
+      def label(entry)
+        entry.is_a?(Hash) ? entry.inspect : entry
+      end
+
+      def turns_failure(turns)
+        limit = @expect[:max_turns]
+        return unless limit && turns > limit
+
+        "used #{turns} turns, limit #{limit}"
+      end
+    end
+  end
+end

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+module Elelem
+  module Evals
+    class BoundedClient
+      TurnLimitExceeded = Class.new(StandardError)
+
+      attr_reader :turns
+
+      def initialize(client, max_turns:)
+        @client = client
+        @max_turns = max_turns
+        @turns = 0
+      end
+
+      def fetch(messages, tools = [], &block)
+        @turns += 1
+        raise TurnLimitExceeded, "turn limit #{@max_turns} exceeded" if @turns > @max_turns
+
+        @client.fetch(messages, tools, &block)
+      end
+    end
+  end
+end

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+require "yaml"
+
+module Elelem
+  module Evals
+    class Case
+      CASES = File.expand_path("../cases", __dir__)
+      DEFAULT_MAX_TURNS = 10
+
+      def self.load_all(dir = CASES)
+        Dir["#{dir}/*.yml"].sort.flat_map { |file| load_file(file) }
+      end
+
+      def self.load_file(file)
+        group = File.basename(file, ".yml")
+        YAML.safe_load_file(file).map { |attrs| new(group: group, **symbolize(attrs)) }
+      end
+
+      def self.symbolize(hash)
+        (hash || {}).transform_keys(&:to_sym)
+      end
+
+      attr_reader :id, :group, :fixture, :setup, :turns, :expect
+
+      def initialize(id:, fixture:, turns:, group: nil, setup: [], expect: {})
+        @id = id
+        @group = group
+        @fixture = fixture
+        @setup = Array(setup)
+        @turns = Array(turns)
+        @expect = self.class.symbolize(expect)
+      end
+
+      def max_turns
+        @expect.fetch(:max_turns, DEFAULT_MAX_TURNS)
+      end
+
+      def to_s
+        "#{group}/#{id}"
+      end
+    end
+  end
+end

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+module Elelem
+  module Evals
+    MODEL = ENV.fetch("EVAL_MODEL", "gpt-oss:latest")
+    IMPROVER_MODEL = ENV.fetch("EVAL_IMPROVER_MODEL", "gpt-oss:120b")
+
+    def self.ollama(model:, **params)
+      Elelem::Net::Ollama.new(model: model, keep_alive: "30m", **params)
+    end
+
+    def self.client(model: MODEL)
+      ollama(model: model, options: { temperature: 0, seed: 42 })
+    end
+  end
+end

@@ -0,0 +1,107 @@
+# frozen_string_literal: true
+
+module Elelem
+  module Evals
+    def self.improver_client(model: IMPROVER_MODEL)
+      ollama(model: model, options: { temperature: 0.3 }, params: { format: "json" })
+    end
+
+    class Improver
+      SYSTEM = <<~PROMPT
+        You are a prompt engineer improving the system prompt of elelem, a terminal
+        coding agent. The agent has three tools: read(path), write(path, content)
+        and execute(command).
+
+        You are shown the current prompt and the eval cases it failed. Suggest
+        targeted find/replace edits to the prompt that would fix those failures.
+
+        Rules:
+        - Each change must use an exact old_text string that appears in the prompt.
+        - Keep changes minimal. Do not rewrite whole sections.
+        - Do not remove or alter the ERB tags, they render the environment.
+        - Do not change what is already working.
+        - Focus on the highest impact failures first.
+
+        Respond with JSON in this exact format:
+        {
+          "analysis": "what is failing and why",
+          "changes": [
+            {"old_text": "exact text from the prompt", "new_text": "replacement", "rationale": "why this helps"}
+          ]
+        }
+
+        Return an empty changes array if no edit would help.
+      PROMPT
+
+      def initialize(client: Evals.improver_client)
+        @client = client
+      end
+
+      def plan(prompt:, failures:)
+        parsed = JSON.parse(complete(SYSTEM, user_message(prompt, failures)))
+        return empty unless parsed.is_a?(Hash)
+
+        { "analysis" => parsed["analysis"].to_s, "changes" => edits(parsed["changes"]) }
+      rescue JSON::ParserError
+        empty
+      end
+
+      def apply(prompt, changes)
+        applied = []
+
+        updated = edits(changes).reduce(prompt) do |text, change|
+          old_text = change["old_text"].to_s
+          next text if old_text.empty? || !text.include?(old_text)
+
+          applied << change["rationale"].to_s
+          # Block form, so \0 and \& in the model's text stay literal.
+          text.sub(old_text) { change["new_text"].to_s }
+        end
+
+        [updated, applied]
+      end
+
+      private
+
+      def edits(changes)
+        changes.is_a?(Array) ? changes.select { |change| change.is_a?(Hash) } : []
+      end
+
+      def user_message(prompt, failures)
+        <<~MESSAGE
+          ## Failing cases
+
+          #{failures.map { |result| summarize(result) }.join("\n")}
+
+          ## Current system prompt
+
+          ```erb
+          #{prompt}
+          ```
+        MESSAGE
+      end
+
+      def summarize(result)
+        JSON.generate(
+          id: result.id,
+          group: result.group,
+          failures: result.failures,
+          turns: result.turns,
+          response: result.response.to_s[0, 500]
+        )
+      end
+
+      def complete(system, user)
+        content = String.new
+        messages = [{ role: "system", content: system }, { role: "user", content: user }]
+
+        @client.fetch(messages, []) { |event| content << event[:text].to_s if event[:type] == "saying" }
+        content
+      end
+
+      def empty
+        { "analysis" => "", "changes" => [] }
+      end
+    end
+  end
+end

@@ -0,0 +1,132 @@
+# frozen_string_literal: true
+
+require "digest"
+require "time"
+
+module Elelem
+  module Evals
+    class Loop
+      HOLDOUT = "holdout"
+
+      DEFAULT_PREFLIGHT = lambda do
+        Evals.client.fetch([{ role: "user", content: "ping" }], []) { |_event| }
+      end
+
+      def initialize(cases: Case.load_all, champion: CHAMPION, workdir: WORKDIR, improver: Improver.new,
+        scorer_for: ->(prompt) { Scorer.new(runner: Runner.new(prompt: prompt)).call(cases) },
+        out: $stdout, preflight: DEFAULT_PREFLIGHT)
+        @cases = cases
+        @champion = champion
+        @workdir = workdir
+        @improver = improver
+        @scorer_for = scorer_for
+        @out = out
+        @preflight = preflight
+      end
+
+      def run(rounds: 3)
+        preflight!
+        prompt = File.read(@champion)
+        score = @scorer_for.call(prompt)
+        say "champion #{visible_summary(score)}"
+
+        rounds.times do |index|
+          return true if score.failed.zero?
+
+          outcome = round(index + 1, prompt, score)
+          prompt, score = outcome if outcome
+        end
+
+        score.failed.zero?
+      end
+
+      private
+
+      def preflight!
+        @preflight.call
+      rescue => e
+        raise "model preflight failed, is it reachable? (#{e.message})"
+      end
+
+      def round(number, prompt, score)
+        plan = @improver.plan(prompt: prompt, failures: visible(score))
+        say "round #{number}: #{plan["analysis"]}"
+
+        challenger, applied = @improver.apply(prompt, plan["changes"])
+
+        if applied.empty?
+          say("round #{number}: no change could be applied, skipping")
+          return nil
+        end
+
+        File.write(challenger_path(number), challenger)
+        new_score = @scorer_for.call(challenger)
+        regressions = new_score.regressions_from(score)
+
+        return promote(number, challenger, new_score, applied) if promote?(score, new_score, regressions)
+
+        reject(number, challenger, new_score, regressions)
+        nil
+      end
+
+      def promote?(score, new_score, regressions)
+        new_score.excluding(HOLDOUT).passed > score.excluding(HOLDOUT).passed && regressions.empty?
+      end
+
+      def promote(number, challenger, new_score, applied)
+        File.write(@champion, challenger)
+        say "round #{number}: promoted, #{visible_summary(new_score)}"
+        record(round: number, promoted: true, regressions: [], applied: applied, score: new_score, prompt: challenger)
+        [challenger, new_score]
+      end
+
+      def reject(number, challenger, new_score, regressions)
+        say "round #{number}: rejected, #{visible_summary(new_score)}, regressions #{regressions.join(", ")}"
+        record(round: number, promoted: false, regressions: regressions, applied: [], score: new_score, prompt: challenger)
+      end
+
+      def visible(score)
+        score.excluding(HOLDOUT).failures.uniq { |result| [result.group, result.id] }
+      end
+
+      def visible_summary(score)
+        shown = score.excluding(HOLDOUT)
+        return "#{shown.passed}/#{shown.total}" if shown.total == score.total
+
+        "#{shown.passed}/#{shown.total} visible, #{score.passed}/#{score.total} overall"
+      end
+
+      def challenger_path(number)
+        FileUtils.mkdir_p(@workdir)
+        File.join(@workdir, "challenger-#{number}.erb")
+      end
+
+      def record(round:, promoted:, regressions:, applied:, score:, prompt:)
+        shown = score.excluding(HOLDOUT)
+        entry = {
+          round: round,
+          promoted: promoted,
+          passed: shown.passed,
+          total: shown.total,
+          overall_passed: score.passed,
+          overall_total: score.total,
+          regressions: regressions,
+          applied: applied,
+          timestamp: Time.now.utc.iso8601,
+          model: MODEL,
+          improver_model: IMPROVER_MODEL,
+          prompt: Digest::SHA256.hexdigest(prompt)[0, 12]
+        }
+
+        FileUtils.mkdir_p(@workdir)
+        File.open(File.join(@workdir, "history.jsonl"), "a") do |file|
+          file.puts(JSON.generate(entry))
+        end
+      end
+
+      def say(message)
+        @out.puts(message)
+      end
+    end
+  end
+end

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module Elelem
+  module Evals
+    class NullTerminal < Elelem::Terminal
+      def initialize
+        super(quiet: true)
+      end
+
+      def interactive?
+        false
+      end
+    end
+  end
+end

@@ -0,0 +1,67 @@
+# frozen_string_literal: true
+
+module Elelem
+  module Evals
+    ToolCall = Data.define(:name, :args)
+
+    Result = Data.define(:id, :group, :status, :failures, :turns, :duration, :tools, :response) do
+      def passed?
+        status == "PASS"
+      end
+    end
+
+    # The prompt under test is the one elelem actually ships, not a copy that
+    # can drift from it. Challengers and history stay out of lib/.
+    CHAMPION = File.expand_path("../../../lib/elelem/prompts/default.erb", __dir__)
+    WORKDIR = File.expand_path("../prompts", __dir__)
+
+    class Runner
+      def initialize(prompt:, client: -> { Evals.client })
+        @prompt = prompt
+        @client = client
+      end
+
+      def run(kase)
+        started = now
+        tools = []
+        response = nil
+        bounded = BoundedClient.new(@client.call, max_turns: kase.max_turns)
+
+        Workspace.open(fixture: kase.fixture, setup: kase.setup) do |workspace|
+          workspace.chdir do
+            agent = build_agent(bounded, tools)
+            kase.turns.each { |turn| response = agent.turn(turn) }
+
+            failures = Assertions.new(kase.expect).failures(
+              workspace: workspace, response: response, tools: tools, turns: bounded.turns
+            )
+
+            Result.new(
+              id: kase.id,
+              group: kase.group,
+              status: failures.empty? ? "PASS" : "FAIL",
+              failures: failures,
+              turns: bounded.turns,
+              duration: (now - started).round(2),
+              tools: tools,
+              response: response.to_s
+            )
+          end
+        end
+      end
+
+      private
+
+      def build_agent(client, tools)
+        agent = Elelem::Agent.new(client, terminal: NullTerminal.new, system_prompt: @prompt)
+        Elelem::Plugins.setup!(agent)
+        agent.toolbox.after { |args, _result, tool_name:| tools << ToolCall.new(name: tool_name.to_s, args: args) }
+        agent
+      end
+
+      def now
+        Process.clock_gettime(Process::CLOCK_MONOTONIC)
+      end
+    end
+  end
+end

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+module Elelem
+  module Evals
+    Score = Data.define(:results) do
+      def status
+        results.group_by { |result| "#{result.group}/#{result.id}" }.transform_values { |runs| runs.all?(&:passed?) }
+      end
+
+      def passed
+        status.count { |_id, ok| ok }
+      end
+
+      def total
+        status.size
+      end
+
+      def failed
+        total - passed
+      end
+
+      def failures
+        results.reject(&:passed?)
+      end
+
+      def regressions_from(previous)
+        current = status
+        previous.status.select { |id, ok| ok && !current.fetch(id, false) }.keys
+      end
+
+      def excluding(group)
+        Score.new(results: results.reject { |result| result.group == group })
+      end
+    end
+
+    class Scorer
+      def initialize(runner:, repeat: 3)
+        @runner = runner
+        @repeat = repeat
+      end
+
+      def call(cases)
+        Score.new(results: cases.flat_map { |kase| Array.new(@repeat) { @runner.run(kase) } })
+      end
+    end
+  end
+end

@@ -0,0 +1,57 @@
+# frozen_string_literal: true
+
+require "fileutils"
+require "tmpdir"
+
+module Elelem
+  module Evals
+    class Workspace
+      SetupFailed = Class.new(StandardError)
+
+      FIXTURES = File.expand_path("../fixtures", __dir__)
+
+      def self.open(fixture:, setup: [])
+        dir = Dir.mktmpdir("elelem-evals-")
+        workspace = new(dir, fixture)
+        workspace.prepare(setup)
+        yield workspace
+      ensure
+        FileUtils.remove_entry(dir) if dir && File.directory?(dir)
+      end
+
+      attr_reader :path
+
+      def initialize(path, fixture)
+        @path = path
+        @fixture = fixture
+      end
+
+      def prepare(setup)
+        FileUtils.cp_r("#{FIXTURES}/#{@fixture}/.", @path)
+        chdir { setup.each { |command| run_setup(command) } }
+      end
+
+      def chdir(&block)
+        Dir.chdir(@path) { Bundler.with_unbundled_env(&block) }
+      end
+
+      def sh(command)
+        Elelem.sh("bash", args: ["-c", command], cwd: @path)
+      end
+
+      def read(relative)
+        file = File.join(@path, relative)
+        File.exist?(file) ? File.read(file) : nil
+      end
+
+      private
+
+      def run_setup(command)
+        result = sh(command)
+        return if result[:exit_status].zero?
+
+        raise SetupFailed, "setup command failed (exit #{result[:exit_status]}): #{command}\n#{result[:content]}"
+      end
+    end
+  end
+end

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+RSpec.describe "eval cases", :eval do
+  runner = Elelem::Evals::Runner.new(prompt: File.read(Elelem::Evals::CHAMPION))
+
+  Elelem::Evals::Case.load_all.each do |kase|
+    it kase.to_s do
+      result = runner.run(kase)
+
+      expect(result.failures).to be_empty, -> { "#{kase}\n  #{result.failures.join("\n  ")}\n\n#{result.response}" }
+    end
+  end
+end

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+require_relative "../../lib/elelem"
+
+Dir[File.expand_path("../evals/support/*.rb", __dir__)].sort.each { |file| require file }

@@ -10,4 +10,6 @@ RSpec.configure do |config|
   config.expect_with :rspec do |c|
     c.syntax = :expect
   end
+
+  config.filter_run_excluding(:eval) unless ENV["EVALS"]
 end

@@ -15,3 +15,5 @@ target/
 
 # rspec failure tracking
 .rspec_status
+
+/spec/evals/prompts/challenger-*.erb

@@ -5,4 +5,22 @@ require "rspec/core/rake_task"
 
 RSpec::Core::RakeTask.new(:spec)
 
+task :evals_env do
+  ENV["EVALS"] = "1"
+end
+
+desc "Score the champion system prompt against the eval cases"
+RSpec::Core::RakeTask.new({ evals: :evals_env }) do |t|
+  t.pattern = "spec/evals/cases_spec.rb"
+end
+
+namespace :evals do
+  desc "Tune the champion system prompt against the eval cases"
+  task :improve do
+    require_relative "spec/support/evals"
+
+    Elelem::Evals::Loop.new.run(rounds: Integer(ENV.fetch("ROUNDS", "3")))
+  end
+end
+
 task default: %i[spec]