Commit c5d46c2

@@ -43,6 +43,21 @@ RSpec.describe "Elelem.sh" do
     end
   end
 
+  describe "output truncation" do
+    it "passes short output through untouched" do
+      result = Elelem.sh("bash", args: ["-c", "echo short"])
+
+      expect(result[:content]).to eq("short\n")
+    end
+
+    it "elides the middle of output beyond the limit" do
+      result = Elelem.sh("bash", args: ["-c", "yes x | head -c 20000"])
+
+      expect(result[:content].length).to be < 20_000
+      expect(result[:content]).to include("characters elided")
+    end
+  end
+
   describe ".command_timeout" do
     around do |example|
       original = ENV["ELELEM_CMD_TIMEOUT"]

@@ -5,8 +5,6 @@
     - run test.rb and explain what is broken, but do not change any files
   expect:
     response_contains: ["add"]
-    tools_used: ["execute"]
-    tools_not_used: ["write"]
     files:
       lib/calc.rb:
         contains: ["a - b"]

@@ -9,4 +9,3 @@
     files:
       fizzbuzz.rb:
         contains: ["FizzBuzz"]
-    tools_used: ["write"]

@@ -8,10 +8,6 @@
     files:
       hello.rb:
         contains: ["farewell"]
-    tools_used: ["write"]
-    tools_not_used:
-      - execute: { command: "<<" }
-      - execute: { command: "patch" }
 
 # Defends: "Change a file with write: read it, then write the full new contents."
 # The read half is asserted directly; keeping app_metadata proves the whole file
@@ -25,7 +21,6 @@
     files:
       user.rb:
         contains: ["user_id", "app_metadata"]
-    tools_used: ["read", "write"]
 
 # Defends: "Use `sed` only for a trivial single-line substitution."
 - id: restructure-with-write
@@ -38,9 +33,6 @@
     files:
       user.rb:
         contains: ["module Auth"]
-    tools_used: ["write"]
-    tools_not_used:
-      - execute: { command: "sed -i" }
 
 - id: multi-line-edit
   fixture: hello

@@ -7,4 +7,3 @@
     files:
       lib/calc.rb:
         contains: ["a + b"]
-    tools_used: ["execute"]

@@ -11,7 +11,6 @@
     - commit the current changes with a message that follows the 50/72 rule
   expect:
     verify: test -z "$(git status --porcelain)" && test "$(git log -1 --format=%s | wc -c)" -le 51
-    tools_used: ["execute"]
 
 # Defends: query git history and report, without editing.
 - id: last-commit-subject
@@ -26,5 +25,3 @@
     - what is the subject line of the most recent git commit?
   expect:
     response_contains: ["Add greeting script"]
-    tools_used: ["execute"]
-    tools_not_used: ["write"]

@@ -10,7 +10,6 @@
       hello.rb:
         contains: ["hail"]
         not_contains: ["greet"]
-    tools_used: ["write"]
 
 # Paraphrased twin of edit/read-before-write, held out so the improver can't
 # overfit the read-then-write instruction to one fixture.
@@ -23,7 +22,6 @@
     files:
       token.rb:
         contains: ["issuer", "to_h"]
-    tools_used: ["read", "write"]
 
 # Paraphrased twin of voice/concise-answer: holds the conciseness bound out of
 # the improver's view so terseness can't overfit to one prompt.
@@ -34,7 +32,6 @@
   expect:
     response_contains: ["token.rb"]
     response_matches: ['\A[\s\S]{0,400}\z']
-    tools_not_used: ["write"]
 
 - id: locate-error-holdout
   fixture: config
@@ -42,7 +39,6 @@
     - which file raises the "Failed to load configuration" error?
   expect:
     response_contains: ["config/loader.rb"]
-    tools_not_used: ["write"]
 
 - id: locate-class
   fixture: config
@@ -50,7 +46,6 @@
     - which file defines the Loader class?
   expect:
     response_contains: ["config/loader.rb"]
-    tools_not_used: ["write"]
 
 - id: count-ruby-files
   fixture: metadata
@@ -58,7 +53,6 @@
     - how many ruby files are in this project?
   expect:
     response_matches: ['\b3\b']
-    tools_not_used: ["write"]
 
 - id: locate-parser
   fixture: config
@@ -66,4 +60,3 @@
     - which file defines the Parser class?
   expect:
     response_contains: ["config/parser.rb"]
-    tools_not_used: ["write"]

@@ -8,4 +8,3 @@
     files:
       roman.rb:
         not_contains: ["NotImplementedError"]
-    tools_used: ["write"]

@@ -5,4 +5,3 @@
   expect:
     response_contains: ["/scim/v2/Users"]
     response_matches: ['/scim/v2/Users[\s\S]{0,120}3']
-    tools_not_used: ["write"]

@@ -9,4 +9,3 @@
     verify: 'test "$(cat count.txt)" = "$(grep -c ''"response_code":500'' requests.jsonl)"'
     files:
       count.txt: {}
-    tools_used: ["execute"]

@@ -12,4 +12,3 @@
       main.rb:
         contains: ["salutation"]
         not_contains: ["greeting"]
-    tools_used: ["write"]

@@ -5,8 +5,6 @@
     - where does the "Failed to load configuration" error come from?
   expect:
     response_contains: ["config/loader.rb"]
-    tools_used: ["execute"]
-    tools_not_used: ["write"]
 
 # Defends: "- files: `fd -e rb .`"
 - id: count-ruby-files
@@ -15,9 +13,6 @@
     - how many ruby files are in this project?
   expect:
     response_matches: ['\b3\b']
-    tools_used:
-      - execute: { command: "fd" }
-    tools_not_used: ["write"]
 
 # Defends: "- text: `rg -n "pattern" .`"
 - id: grep-with-ripgrep
@@ -26,9 +21,6 @@
     - search the codebase for the text "Failed to load configuration" and tell me which file contains it
   expect:
     response_contains: ["config/loader.rb"]
-    tools_used:
-      - execute: { command: "rg " }
-    tools_not_used: ["write"]
 
 - id: list-references
   fixture: metadata
@@ -36,4 +28,3 @@
     - which files reference app_metadata?
   expect:
     response_contains: ["user.rb", "token.rb"]
-    tools_not_used: ["write"]

@@ -14,8 +14,4 @@
     - what is the biggest file in this repository?
   expect:
     response_contains: ["report.csv"]
-    response_not_contains: [".git"]
-    tools_not_used:
-      - write
-      - execute: { command: "-printf" }
-      - execute: { command: "du -b" }
+    response_not_contains: [".git/"]

@@ -5,7 +5,6 @@
     - write the number of ruby files that contain the text app_metadata to count.txt
   expect:
     verify: '[ "$(cat count.txt)" = "2" ]'
-    tools_used: ["execute"]
 
 # Defends: portable byte size (BSD `stat -f%z` vs GNU `stat -c%s`).
 - id: byte-size-portable
@@ -22,13 +21,3 @@
     - write the total number of lines across all ruby files in this project to total.txt
   expect:
     verify: '[ "$(cat total.txt)" = "$(cat *.rb | wc -l | tr -d " ")" ]'
-    tools_used: ["execute"]
-
-# Defends: answer a live-system-state question with the real command, not a guess.
-- id: current-time
-  fixture: blank
-  turns:
-    - what time is it?
-  expect:
-    tools_used:
-      - execute: { command: "date" }

@@ -9,4 +9,3 @@
   expect:
     response_contains: ["config/parser.rb"]
     response_matches: ['\A[\s\S]{0,400}\z']
-    tools_not_used: ["write"]

@@ -1,53 +0,0 @@
-# frozen_string_literal: true
-
-module Elelem
-  module Evals
-    class Ablator
-      Ablation = Data.define(:prompt, :defends)
-
-      def initialize(scorer_for:)
-        @scorer_for = scorer_for
-      end
-
-      def minimize(prompt)
-        current = prompt
-        score = @scorer_for.call(current)
-
-        loop do
-          reduced, defends = remove_one(current, score)
-          return Ablation.new(prompt: current, defends: defends) unless reduced
-
-          current, score = reduced
-        end
-      end
-
-      private
-
-      def remove_one(prompt, score)
-        lines = prompt.lines
-        defends = {}
-
-        lines.each_index do |index|
-          next unless ablatable?(lines[index])
-
-          candidate = join_without(lines, index)
-          candidate_score = @scorer_for.call(candidate)
-          regressions = candidate_score.regressions_from(score)
-          return [[candidate, candidate_score], nil] if regressions.empty?
-
-          defends[lines[index]] = regressions
-        end
-
-        [nil, defends]
-      end
-
-      def ablatable?(line)
-        !line.strip.empty? && !line.include?("<%")
-      end
-
-      def join_without(lines, index)
-        (lines[0...index] + lines[(index + 1)..]).join
-      end
-    end
-  end
-end

@@ -1,76 +0,0 @@
-# frozen_string_literal: true
-
-module Elelem
-  module Evals
-    class Assertions
-      def initialize(expect)
-        @expect = expect || {}
-      end
-
-      def failures(workspace:, response:, tools:)
-        [
-          verify_failure(workspace),
-          *file_failures(workspace),
-          *response_failures(response),
-          *tool_failures(tools)
-        ].compact
-      end
-
-      private
-
-      def verify_failure(workspace)
-        command = @expect[:verify]
-        return unless command
-
-        result = workspace.sh(command)
-        return if result[:exit_status].zero?
-
-        "verify failed (exit #{result[:exit_status]}): #{command}"
-      end
-
-      def file_failures(workspace)
-        (@expect[:files] || {}).flat_map do |path, rules|
-          content = workspace.read(path)
-          next ["#{path} missing"] unless content
-
-          missing = Array(rules["contains"]).reject { |text| content.include?(text) }
-          present = Array(rules["not_contains"]).select { |text| content.include?(text) }
-
-          missing.map { |text| "#{path} missing #{text.inspect}" } +
-            present.map { |text| "#{path} still contains #{text.inspect}" }
-        end
-      end
-
-      def response_failures(response)
-        text = response.to_s
-
-        Array(@expect[:response_contains]).reject { |s| text.include?(s) }.map { |s| "response missing #{s.inspect}" } +
-          Array(@expect[:response_not_contains]).select { |s| text.include?(s) }.map { |s| "response contains #{s.inspect}" } +
-          Array(@expect[:response_matches]).reject { |p| Regexp.new(p).match?(text) }.map { |p| "response does not match /#{p}/" }
-      end
-
-      def tool_failures(tools)
-        Array(@expect[:tools_used]).reject { |entry| called?(tools, entry) }.map { |entry| "never called #{label(entry)}" } +
-          Array(@expect[:tools_not_used]).select { |name| called?(tools, name) }.map { |name| "called #{name}" }
-      end
-
-      def called?(tools, entry)
-        name, constraints = destructure(entry)
-
-        tools.any? do |call|
-          call.name == name.to_s && constraints.all? { |key, value| call.args[key.to_s].to_s.include?(value.to_s) }
-        end
-      end
-
-      def destructure(entry)
-        return entry.first.then { |name, constraints| [name, constraints || {}] } if entry.is_a?(Hash)
-
-        [entry, {}]
-      end
-
-      def label(entry)
-        entry.is_a?(Hash) ? entry.inspect : entry
-      end
-    end
-  end
-end

@@ -1,24 +0,0 @@
-# frozen_string_literal: true
-
-module Elelem
-  module Evals
-    class BoundedClient
-      TurnLimitExceeded = Class.new(StandardError)
-
-      attr_reader :turns
-
-      def initialize(client, max_turns:)
-        @client = client
-        @max_turns = max_turns
-        @turns = 0
-      end
-
-      def fetch(messages, tools = [], &block)
-        @turns += 1
-        raise TurnLimitExceeded, "turn limit #{@max_turns} exceeded" if @turns > @max_turns
-
-        @client.fetch(messages, tools, &block)
-      end
-    end
-  end
-end

@@ -1,44 +0,0 @@
-# frozen_string_literal: true
-
-require "yaml"
-
-module Elelem
-  module Evals
-    class Case
-      CASES = File.expand_path("../cases", __dir__)
-      DEFAULT_MAX_TURNS = 30
-
-      def self.load_all(dir = CASES)
-        Dir["#{dir}/*.yml"].sort.flat_map { |file| load_file(file) }
-      end
-
-      def self.load_file(file)
-        group = File.basename(file, ".yml")
-        YAML.safe_load_file(file).map { |attrs| new(group: group, **symbolize(attrs)) }
-      end
-
-      def self.symbolize(hash)
-        (hash || {}).transform_keys(&:to_sym)
-      end
-
-      attr_reader :id, :group, :fixture, :setup, :turns, :expect
-
-      def initialize(id:, fixture:, turns:, group: nil, setup: [], expect: {})
-        @id = id
-        @group = group
-        @fixture = fixture
-        @setup = Array(setup)
-        @turns = Array(turns)
-        @expect = self.class.symbolize(expect)
-      end
-
-      def max_turns
-        @expect.fetch(:max_turns, DEFAULT_MAX_TURNS)
-      end
-
-      def to_s
-        "#{group}/#{id}"
-      end
-    end
-  end
-end

@@ -1,32 +0,0 @@
-# frozen_string_literal: true
-
-module Elelem
-  module Evals
-    MODEL = ENV.fetch("EVAL_MODEL", "gpt-oss:latest")
-    IMPROVER_MODEL = ENV.fetch("EVAL_IMPROVER_MODEL", "gpt-oss:120b")
-
-    def self.ollama(model:, **params)
-      host = ENV.fetch("OLLAMA_HOST", "localhost:11434")
-      Elelem::Net::Ollama.new(model: model, host: host, keep_alive: "30m", **params)
-    end
-
-    def self.gguf(model: ENV.fetch("GGUF_MODEL"))
-      @gguf ||= begin
-        gpu = %w[vulkan cuda metal].include?(Elelem::Net::GGUF.backend)
-        Elelem::Net::GGUF.new(
-          model: File.expand_path(model),
-          n_ctx: Integer(ENV.fetch("GGUF_N_CTX", "8192")),
-          n_gpu_layers: Integer(ENV.fetch("GGUF_N_GPU_LAYERS", gpu ? "999" : "0")),
-          temp: 0.0,
-          seed: 42
-        )
-      end
-    end
-
-    def self.client(model: MODEL)
-      return gguf if ENV["EVALS_PROVIDER"] == "gguf"
-
-      ollama(model: model, options: { temperature: 0, seed: 42 })
-    end
-  end
-end

@@ -1,110 +0,0 @@
-# frozen_string_literal: true
-
-module Elelem
-  module Evals
-    def self.improver_client(model: IMPROVER_MODEL)
-      ollama(model: model, options: { temperature: 0.3 }, params: { format: "json" })
-    end
-
-    class Improver
-      SYSTEM = <<~PROMPT
-        You are a prompt engineer improving the system prompt of a terminal coding agent.
-
-        You are shown the current prompt and the eval cases it failed. Suggest targeted find/replace edits to the prompt that would fix those failures.
-
-        Respond with JSON in this exact format:
-
-        ```json
-        {
-          "analysis": "what is failing and why",
-          "changes": [{ "old_text": "exact text from the prompt", "new_text": "replacement", "rationale": "why this helps" }]
-        }
-        ```
-
-        Return an empty changes array if no edit would help.
-      PROMPT
-
-      def initialize(client: Evals.improver_client)
-        @client = client
-      end
-
-      def plan(prompt:, failures:)
-        parsed = JSON.parse(complete(SYSTEM, user_message(prompt, failures)))
-        return empty unless parsed.is_a?(Hash)
-
-        { "analysis" => parsed["analysis"].to_s, "changes" => edits(parsed["changes"]) }
-      rescue JSON::ParserError
-        empty
-      end
-
-      def apply(prompt, changes)
-        applied = []
-
-        updated = edits(changes).reduce(prompt) do |text, change|
-          old_text = change["old_text"].to_s
-          new_text = change["new_text"].to_s
-
-          if old_text.empty?
-            next text if new_text.empty?
-
-            applied << change["rationale"].to_s
-            next append(text, new_text)
-          end
-
-          next text unless text.include?(old_text)
-
-          applied << change["rationale"].to_s
-          text.sub(old_text) { new_text }
-        end
-
-        [updated, applied]
-      end
-
-      private
-
-      def append(text, addition)
-        text.empty? || text.end_with?("\n") ? text + addition : "#{text}\n#{addition}"
-      end
-
-      def edits(changes)
-        changes.is_a?(Array) ? changes.select { |change| change.is_a?(Hash) } : []
-      end
-
-      def user_message(prompt, failures)
-        <<~MESSAGE
-          ## Failing cases
-
-          #{failures.map { |result| summarize(result) }.join("\n")}
-
-          ## Current system prompt
-
-          ```erb
-          #{prompt}
-          ```
-        MESSAGE
-      end
-
-      def summarize(result)
-        JSON.generate(
-          id: result.id,
-          group: result.group,
-          failures: result.failures,
-          turns: result.turns,
-          response: result.response.to_s[0, 500]
-        )
-      end
-
-      def complete(system, user)
-        content = String.new
-        messages = [{ role: "system", content: system }, { role: "user", content: user }]
-
-        @client.fetch(messages, []) { |event| content << event[:text].to_s if event[:type] == "saying" }
-        content
-      end
-
-      def empty
-        { "analysis" => "", "changes" => [] }
-      end
-    end
-  end
-end

@@ -1,165 +0,0 @@
-# frozen_string_literal: true
-
-require "digest"
-require "time"
-
-module Elelem
-  module Evals
-    class Loop
-      HOLDOUT = "holdout"
-      BUDGET = 1024
-
-      DEFAULT_PREFLIGHT = lambda do
-        Evals.client.fetch([{ role: "user", content: "ping" }], []) { |_event| }
-      end
-
-      def initialize(cases: Case.load_all, champion: CHAMPION, workdir: WORKDIR, improver: Improver.new,
-        scorer_for: Evals.scorer_for(cases: cases),
-        out: $stdout, preflight: DEFAULT_PREFLIGHT, budget: BUDGET)
-        @champion = champion
-        @workdir = workdir
-        @improver = improver
-        @scorer_for = scorer_for
-        @out = out
-        @preflight = preflight
-        @budget = budget
-      end
-
-      def run(rounds: 3)
-        preflight!
-        prompt = File.read(@champion)
-        score = @scorer_for.call(prompt)
-        say "champion #{visible_summary(score)}, #{score.turns} turns, #{score.duration}s, #{size_of(prompt)}"
-
-        rounds.times do |index|
-          return true if score.failed.zero?
-
-          outcome = round(index + 1, prompt, score)
-          prompt, score = outcome if outcome
-        end
-
-        score.failed.zero?
-      end
-
-      private
-
-      def preflight!
-        @preflight.call
-      rescue => e
-        raise "model preflight failed, is it reachable? (#{e.message})"
-      end
-
-      def round(number, prompt, score)
-        plan = @improver.plan(prompt: prompt, failures: visible(score))
-        say "round #{number}: #{plan["analysis"]}"
-
-        challenger, applied = @improver.apply(prompt, plan["changes"])
-
-        if applied.empty?
-          say("round #{number}: no change could be applied, skipping")
-          return nil
-        end
-
-        File.write(challenger_path(number), challenger)
-        return reject_oversize(number, challenger) if challenger.length > @budget
-
-        new_score = @scorer_for.call(challenger)
-        regressions = new_score.regressions_from(score)
-
-        return promote(number, prompt, challenger, new_score, applied) if promote?(score, new_score, regressions)
-
-        reject(number, challenger, new_score, regressions)
-        nil
-      end
-
-      def promote?(score, new_score, regressions)
-        optimizable(new_score).passed > optimizable(score).passed && regressions.empty?
-      end
-
-      def optimizable(score)
-        score.excluding(HOLDOUT)
-      end
-
-      def promote(number, prompt, challenger, new_score, applied)
-        File.write(@champion, challenger)
-        say "round #{number}: promoted, #{visible_summary(new_score)}, template #{prompt.length} -> #{challenger.length} chars (cap #{@budget})"
-        changed_lines(prompt, challenger).each { |line| say "  #{line}" }
-        record(round: number, promoted: true, regressions: [], applied: applied, score: new_score, prompt: challenger)
-        [challenger, new_score]
-      end
-
-      def reject(number, challenger, new_score, regressions)
-        say "round #{number}: rejected, #{visible_summary(new_score)}, regressions #{regressions.join(", ")}"
-        record(round: number, promoted: false, regressions: regressions, applied: [], score: new_score, prompt: challenger)
-      end
-
-      def reject_oversize(number, challenger)
-        say "round #{number}: rejected, template #{challenger.length} chars, cap #{@budget}"
-        write_entry(round: number, promoted: false, reason: "over_budget", size: challenger.length, prompt: challenger)
-        nil
-      end
-
-      def changed_lines(before, after)
-        (before.lines - after.lines).map { |line| "- #{line.strip}" } +
-          (after.lines - before.lines).map { |line| "+ #{line.strip}" }
-      end
-
-      def size_of(prompt)
-        over = prompt.length > @budget ? ", over cap" : ""
-        "template #{prompt.length} chars (cap #{@budget}#{over})"
-      end
-
-      def visible(score)
-        optimizable(score).failures.uniq { |result| [result.group, result.id] }
-      end
-
-      def visible_summary(score)
-        shown = optimizable(score)
-        return "#{shown.passed}/#{shown.total}" if shown.total == score.total
-
-        "#{shown.passed}/#{shown.total} visible, #{score.passed}/#{score.total} overall"
-      end
-
-      def challenger_path(number)
-        FileUtils.mkdir_p(@workdir)
-        File.join(@workdir, "challenger-#{number}.erb")
-      end
-
-      def record(round:, promoted:, regressions:, applied:, score:, prompt:)
-        shown = optimizable(score)
-        write_entry(
-          round: round,
-          promoted: promoted,
-          passed: shown.passed,
-          total: shown.total,
-          overall_passed: score.passed,
-          overall_total: score.total,
-          turns: score.turns,
-          duration: score.duration,
-          regressions: regressions,
-          applied: applied,
-          size: prompt.length,
-          prompt: prompt
-        )
-      end
-
-      def write_entry(prompt:, **entry)
-        entry = entry.merge(
-          timestamp: Time.now.utc.iso8601,
-          model: MODEL,
-          improver_model: IMPROVER_MODEL,
-          prompt: Digest::SHA256.hexdigest(prompt)[0, 12]
-        )
-
-        FileUtils.mkdir_p(@workdir)
-        File.open(File.join(@workdir, "history.jsonl"), "a") do |file|
-          file.puts(JSON.generate(entry))
-        end
-      end
-
-      def say(message)
-        @out.puts(message)
-      end
-    end
-  end
-end

@@ -1,15 +0,0 @@
-# frozen_string_literal: true
-
-module Elelem
-  module Evals
-    class NullTerminal < Elelem::Terminal
-      def initialize
-        super(quiet: true)
-      end
-
-      def interactive?
-        false
-      end
-    end
-  end
-end

@@ -1,65 +0,0 @@
-# frozen_string_literal: true
-
-module Elelem
-  module Evals
-    ToolCall = Data.define(:name, :args)
-
-    Result = Data.define(:id, :group, :status, :failures, :turns, :duration, :tools, :response) do
-      def passed?
-        status == "PASS"
-      end
-    end
-
-    CHAMPION = File.expand_path("../../../lib/elelem/prompts/default.erb", __dir__)
-    WORKDIR = File.expand_path("../prompts", __dir__)
-
-    class Runner
-      def initialize(prompt:, client: -> { Evals.client })
-        @prompt = prompt
-        @client = client
-      end
-
-      def run(kase)
-        started = now
-        tools = []
-        response = nil
-        bounded = BoundedClient.new(@client.call, max_turns: kase.max_turns)
-
-        Workspace.open(fixture: kase.fixture, setup: kase.setup) do |workspace|
-          workspace.chdir do
-            agent = build_agent(bounded, tools)
-            kase.turns.each { |turn| response = agent.turn(turn) }
-
-            failures = Assertions.new(kase.expect).failures(
-              workspace: workspace, response: response, tools: tools
-            )
-
-            Result.new(
-              id: kase.id,
-              group: kase.group,
-              status: failures.empty? ? "PASS" : "FAIL",
-              failures: failures,
-              turns: bounded.turns,
-              duration: (now - started).round(2),
-              tools: tools,
-              response: response.to_s
-            )
-          end
-        end
-      end
-
-      private
-
-      def build_agent(client, tools)
-        agent = Elelem::Agent.new(client, terminal: NullTerminal.new, system_prompt: @prompt)
-        Elelem::Plugins.setup!(agent)
-        agent.toolbox.after { |args, _result, tool_name:| tools << ToolCall.new(name: tool_name.to_s, args: args) }
-        agent
-      end
-
-      def now
-        Process.clock_gettime(Process::CLOCK_MONOTONIC)
-      end
-    end
-  end
-end

@@ -1,67 +0,0 @@
-# frozen_string_literal: true
-
-module Elelem
-  module Evals
-    Score = Data.define(:results) do
-      def status
-        results.group_by { |result| "#{result.group}/#{result.id}" }.transform_values { |runs| runs.all?(&:passed?) }
-      end
-
-      def passed
-        status.count { |_id, ok| ok }
-      end
-
-      def total
-        status.size
-      end
-
-      def failed
-        total - passed
-      end
-
-      def failures
-        results.reject(&:passed?)
-      end
-
-      def turns
-        results.sum(&:turns)
-      end
-
-      def duration
-        results.sum(&:duration).round(2)
-      end
-
-      def regressions_from(previous)
-        current = status
-        previous.status.select { |id, ok| ok && !current.fetch(id, false) }.keys
-      end
-
-      def excluding(*groups)
-        Score.new(results: results.reject { |result| groups.include?(result.group) })
-      end
-    end
-
-    class Scorer
-      def initialize(runner:, repeat: 3)
-        @runner = runner
-        @repeat = repeat
-      end
-
-      def call(cases)
-        Score.new(results: cases.flat_map { |kase| runs_for(kase) })
-      end
-
-      private
-
-      def runs_for(kase)
-        results = []
-        @repeat.times do
-          result = @runner.run(kase)
-          results << result
-          break unless result.passed?
-        end
-        results
-      end
-    end
-  end
-end

@@ -1,35 +0,0 @@
-# frozen_string_literal: true
-
-require "json"
-require "fileutils"
-
-module Elelem
-  module Evals
-    def self.scorer_for(cases: Case.load_all)
-      ->(prompt) { Scorer.new(runner: Runner.new(prompt: prompt)).call(cases) }
-    end
-
-    def self.minimize!(champion: CHAMPION, workdir: WORKDIR, out: $stdout, ablator: Ablator.new(scorer_for: scorer_for))
-      before = File.read(champion)
-      ablation = ablator.minimize(before)
-
-      FileUtils.mkdir_p(workdir)
-      File.write(File.join(workdir, "minimized.erb"), ablation.prompt)
-      File.write(File.join(workdir, "defends.json"), JSON.pretty_generate(ablation.defends))
-
-      out.puts "minimize: #{before.length} -> #{ablation.prompt.length} chars, wrote minimized.erb + defends.json to #{workdir}"
-      ablation
-    end
-
-    def self.regenerate(rounds:, seed:, workdir: WORKDIR, out: $stdout, loop_for: ->(champion) { Loop.new(champion: champion, out: out) })
-      FileUtils.mkdir_p(workdir)
-      candidate = File.join(workdir, "candidate.erb")
-      FileUtils.cp(seed, candidate)
-
-      loop_for.call(candidate).run(rounds: rounds)
-
-      out.puts "regenerate: grew #{candidate} from seed (champion untouched)"
-      candidate
-    end
-  end
-end

@@ -1,57 +0,0 @@
-# frozen_string_literal: true
-
-require "fileutils"
-require "tmpdir"
-
-module Elelem
-  module Evals
-    class Workspace
-      SetupFailed = Class.new(StandardError)
-
-      FIXTURES = File.expand_path("../fixtures", __dir__)
-
-      def self.open(fixture:, setup: [])
-        dir = Dir.mktmpdir("elelem-evals-")
-        workspace = new(dir, fixture)
-        workspace.prepare(setup)
-        yield workspace
-      ensure
-        FileUtils.remove_entry(dir) if dir && File.directory?(dir)
-      end
-
-      attr_reader :path
-
-      def initialize(path, fixture)
-        @path = path
-        @fixture = fixture
-      end
-
-      def prepare(setup)
-        FileUtils.cp_r("#{FIXTURES}/#{@fixture}/.", @path)
-        chdir { setup.each { |command| run_setup(command) } }
-      end
-
-      def chdir(&block)
-        Dir.chdir(@path) { Bundler.with_unbundled_env(&block) }
-      end
-
-      def sh(command)
-        Elelem.sh("bash", args: ["-c", command], cwd: @path)
-      end
-
-      def read(relative)
-        file = File.join(@path, relative)
-        File.exist?(file) ? File.read(file) : nil
-      end
-
-      private
-
-      def run_setup(command)
-        result = sh(command)
-        return if result[:exit_status].zero?
-
-        raise SetupFailed, "setup command failed (exit #{result[:exit_status]}): #{command}\n#{result[:content]}"
-      end
-    end
-  end
-end

@@ -1,5 +1,228 @@
 # frozen_string_literal: true
 
+module Elelem
+  module Evals
+    class NullTerminal < Elelem::Terminal
+      def initialize
+        super(quiet: true)
+      end
+
+      def interactive?
+        false
+      end
+    end
+
+    def self.gguf(model: ENV.fetch("GGUF_MODEL", File.expand_path("~/models/Qwen2.5-Coder-7B-Instruct-Q4_K_M.gguf")))
+      @gguf ||= begin
+        gpu = %w[vulkan cuda metal].include?(Elelem::Net::GGUF.backend)
+        Elelem::Net::GGUF.new(
+          model: File.expand_path(model),
+          n_ctx: Integer(ENV.fetch("GGUF_N_CTX", "8192")),
+          n_gpu_layers: Integer(ENV.fetch("GGUF_N_GPU_LAYERS", gpu ? "999" : "0")),
+          max_tokens: Integer(ENV.fetch("GGUF_MAX_TOKENS", "2048")),
+          temp: 0.0,
+          seed: 42
+        )
+      end
+    end
+
+    def self.client
+      gguf
+    end
+
+    class Case
+      CASES = File.expand_path("cases", __dir__)
+      DEFAULT_MAX_TURNS = 30
+
+      def self.load_all(dir = CASES)
+        Dir["#{dir}/*.yml"].sort.flat_map { |file| load_file(file) }
+      end
+
+      def self.load_file(file)
+        group = File.basename(file, ".yml")
+        YAML.safe_load_file(file).map { |attrs| new(group: group, **symbolize(attrs)) }
+      end
+
+      def self.symbolize(hash)
+        (hash || {}).transform_keys(&:to_sym)
+      end
+
+      attr_reader :id, :group, :fixture, :setup, :turns, :expect
+
+      def initialize(id:, fixture:, turns:, group: nil, setup: [], expect: {})
+        @id = id
+        @group = group
+        @fixture = fixture
+        @setup = Array(setup)
+        @turns = Array(turns)
+        @expect = self.class.symbolize(expect)
+      end
+
+      def max_turns
+        @expect.fetch(:max_turns, DEFAULT_MAX_TURNS)
+      end
+
+      def to_s
+        "#{group}/#{id}"
+      end
+    end
+
+    class BoundedClient
+      TurnLimitExceeded = Class.new(StandardError)
+
+      attr_reader :turns
+
+      def initialize(client, max_turns:)
+        @client = client
+        @max_turns = max_turns
+        @turns = 0
+      end
+
+      def fetch(messages, tools = [], &block)
+        @turns += 1
+        raise TurnLimitExceeded, "turn limit #{@max_turns} exceeded" if @turns > @max_turns
+
+        @client.fetch(messages, tools, &block)
+      end
+    end
+
+    class Assertions
+      def initialize(expect)
+        @expect = expect || {}
+      end
+
+      def failures(workspace:, response:)
+        [
+          verify_failure(workspace),
+          *file_failures(workspace),
+          *response_failures(response)
+        ].compact
+      end
+
+      private
+
+      def verify_failure(workspace)
+        command = @expect[:verify]
+        return unless command
+
+        result = workspace.sh(command)
+        return if result[:exit_status].zero?
+
+        "verify failed (exit #{result[:exit_status]}): #{command}"
+      end
+
+      def file_failures(workspace)
+        (@expect[:files] || {}).flat_map do |path, rules|
+          content = workspace.read(path)
+          next ["#{path} missing"] unless content
+
+          missing = Array(rules["contains"]).reject { |text| content.include?(text) }
+          present = Array(rules["not_contains"]).select { |text| content.include?(text) }
+
+          missing.map { |text| "#{path} missing #{text.inspect}" } +
+            present.map { |text| "#{path} still contains #{text.inspect}" }
+        end
+      end
+
+      def response_failures(response)
+        text = response.to_s
+
+        Array(@expect[:response_contains]).reject { |s| text.include?(s) }.map { |s| "response missing #{s.inspect}" } +
+          Array(@expect[:response_not_contains]).select { |s| text.include?(s) }.map { |s| "response contains #{s.inspect}" } +
+          Array(@expect[:response_matches]).reject { |p| Regexp.new(p).match?(text) }.map { |p| "response does not match /#{p}/" }
+      end
+    end
+
+    Result = Data.define(:id, :group, :failures, :response) do
+      def passed?
+        failures.empty?
+      end
+    end
+
+    CHAMPION = File.expand_path("../../lib/elelem/prompts/default.erb", __dir__)
+
+    class Runner
+      def initialize(prompt:, client: -> { Evals.client })
+        @prompt = prompt
+        @client = client
+      end
+
+      def run(kase)
+        response = nil
+        bounded = BoundedClient.new(@client.call, max_turns: kase.max_turns)
+
+        Workspace.open(fixture: kase.fixture, setup: kase.setup) do |workspace|
+          workspace.chdir do
+            agent = build_agent(bounded)
+            kase.turns.each { |turn| response = agent.turn(turn) }
+
+            failures = Assertions.new(kase.expect).failures(workspace: workspace, response: response)
+
+            Result.new(id: kase.id, group: kase.group, failures: failures, response: response.to_s)
+          end
+        end
+      end
+
+      private
+
+      def build_agent(client)
+        agent = Elelem::Agent.new(client, terminal: NullTerminal.new, system_prompt: @prompt)
+        Elelem::Plugins.setup!(agent)
+        agent
+      end
+    end
+
+    class Workspace
+      SetupFailed = Class.new(StandardError)
+
+      FIXTURES = File.expand_path("fixtures", __dir__)
+
+      def self.open(fixture:, setup: [])
+        dir = Dir.mktmpdir("elelem-evals-")
+        workspace = new(dir, fixture)
+        workspace.prepare(setup)
+        yield workspace
+      ensure
+        FileUtils.remove_entry(dir) if dir && File.directory?(dir)
+      end
+
+      attr_reader :path
+
+      def initialize(path, fixture)
+        @path = path
+        @fixture = fixture
+      end
+
+      def prepare(setup)
+        FileUtils.cp_r("#{FIXTURES}/#{@fixture}/.", @path)
+        chdir { setup.each { |command| run_setup(command) } }
+      end
+
+      def chdir(&block)
+        Dir.chdir(@path) { Bundler.with_unbundled_env(&block) }
+      end
+
+      def sh(command)
+        Elelem.sh("bash", args: ["-c", command], cwd: @path)
+      end
+
+      def read(relative)
+        file = File.join(@path, relative)
+        File.exist?(file) ? File.read(file) : nil
+      end
+
+      private
+
+      def run_setup(command)
+        result = sh(command)
+        return if result[:exit_status].zero?
+
+        raise SetupFailed, "setup command failed (exit #{result[:exit_status]}): #{command}\n#{result[:content]}"
+      end
+    end
+  end
+end
+
 RSpec.describe "eval cases" do
   runner = Elelem::Evals::Runner.new(prompt: File.read(Elelem::Evals::CHAMPION))
 

@@ -2,8 +2,10 @@
 
 require_relative "../lib/elelem"
 
+require "fileutils"
 require "shellwords"
 require "tmpdir"
+require "yaml"
 
 Dir[File.join(__dir__, "support/**/*.rb")].each { |f| require f }