Commit 0ca9b20

@@ -146,6 +146,14 @@ RSpec.describe Elelem::Evals::Loop do
     expect(entry).to include("round" => 1, "promoted" => true)
   end
 
+  it "records cost in history" do
+    build_loop(scorer_for: scoring("original prompt" => score_of("FAIL"), "improved prompt" => score_of("PASS"))).run(rounds: 1)
+
+    entry = JSON.parse(File.read(File.join(dir, "history.jsonl")).lines.first)
+
+    expect(entry).to include("turns" => 1, "duration" => 0.1)
+  end
+
   it "withholds holdout failures from the improver" do
     holdout = result(id: "h", group: "holdout")
     visible = result

@@ -110,6 +110,26 @@ RSpec.describe Elelem::Evals::Scorer do
     expect(score.passed).to eq(1)
   end
 
+  describe "cost" do
+    def counted(turns, duration)
+      Elelem::Evals::Result.new(
+        id: "a", group: "g", status: "PASS", failures: [], turns: turns, duration: duration, tools: [], response: ""
+      )
+    end
+
+    it "sums turns across results" do
+      score = Elelem::Evals::Score.new(results: [counted(2, 0.1), counted(3, 0.2)])
+
+      expect(score.turns).to eq(5)
+    end
+
+    it "sums duration across results" do
+      score = Elelem::Evals::Score.new(results: [counted(2, 0.1), counted(3, 0.25)])
+
+      expect(score.duration).to eq(0.35)
+    end
+  end
+
   describe "#excluding" do
     it "filters out every result from the given group" do
       holdout = Elelem::Evals::Result.new(

@@ -30,7 +30,7 @@ module Elelem
         preflight!
         prompt = File.read(@champion)
         score = @scorer_for.call(prompt)
-        say "champion #{visible_summary(score)}, #{size_of(prompt)}"
+        say "champion #{visible_summary(score)}, #{score.turns} turns, #{score.duration}s, #{size_of(prompt)}"
 
         rounds.times do |index|
           return true if score.failed.zero?
@@ -135,6 +135,8 @@ module Elelem
           total: shown.total,
           overall_passed: score.passed,
           overall_total: score.total,
+          turns: score.turns,
+          duration: score.duration,
           regressions: regressions,
           applied: applied,
           size: prompt.length,

@@ -23,6 +23,14 @@ module Elelem
         results.reject(&:passed?)
       end
 
+      def turns
+        results.sum(&:turns)
+      end
+
+      def duration
+        results.sum(&:duration).round(2)
+      end
+
       def regressions_from(previous)
         current = status
         previous.status.select { |id, ok| ok && !current.fetch(id, false) }.keys