Commit `4c99387`

2026-09-07 14:25:28

fix: adapt to elelem core's Config/Registry and Output/Input APIs

Elelem::Providers.build and Elelem::Plugins.setup! predate the Config/Registry refactor; NullTerminal subclassed the pre-split Terminal(quiet:) constructor. Replaced with Config.build/Config.apply and dropped NullTerminal entirely since Agent already defaults to NullOutput/NullInput. Bumped the elelem dependency to ~> 0.11 and added elelem-llama/ elelem-ollama to the Gemfile so `bundle exec` can actually resolve a provider at eval time - previously any real (non-stub) run failed outside bundler isolation. Verified: llama and ollama evals pass end-to-end against the current core, including tool-using cases (edit/*, search/*). Claude-Session: https://claude.ai/code/session_01DUuj4amvRrPxDnHPHBkvz5

main

1 parent 3f682f8

Changed files (7)

lib

elelem

evals

cli.rb

null_terminal.rb

runner.rb

evals.rb

elelem-evals.gemspec

Gemfile

Gemfile.lock

lib/elelem/evals/cli.rb

@@ -17,7 +17,7 @@ module Elelem
 
         return list(cases) if options[:list]
 
-        client = -> { Elelem::Providers.build(options[:provider]) }
+        client = -> { Elelem::Config.build(options[:provider]) }
         runner = Runner.new(fixtures: options[:fixtures], client: client, prompt: options[:prompt])
         reporter = Reporter.new

lib/elelem/evals/null_terminal.rb

@@ -1,15 +0,0 @@
-# frozen_string_literal: true
-
-module Elelem
-  module Evals
-    class NullTerminal < Elelem::Terminal
-      def initialize
-        super(quiet: true)
-      end
-
-      def interactive?
-        false
-      end
-    end
-  end
-end

lib/elelem/evals/runner.rb

@@ -28,8 +28,8 @@ module Elelem
       private
 
       def build_agent(client)
-        agent = Elelem::Agent.new(client, terminal: NullTerminal.new, system_prompt: @prompt)
-        Elelem::Plugins.setup!(agent)
+        agent = Elelem::Agent.new(client, system_prompt: @prompt)
+        Elelem::Config.apply(agent)
         agent
       end
     end

lib/elelem/evals.rb

@@ -7,7 +7,6 @@ require_relative "evals/case"
 require_relative "evals/bounded_client"
 require_relative "evals/assertions"
 require_relative "evals/workspace"
-require_relative "evals/null_terminal"
 require_relative "evals/result"
 require_relative "evals/runner"
 require_relative "evals/reporter"

elelem-evals.gemspec

@@ -27,6 +27,6 @@ Gem::Specification.new do |spec|
   spec.executables = spec.files.grep(%r{\Aexe/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
 
-  spec.add_dependency "elelem", "~> 0.10"
+  spec.add_dependency "elelem", "~> 0.11"
   spec.add_dependency "yaml", "~> 0.4"
 end

Gemfile

@@ -6,3 +6,5 @@ gemspec name: "elelem-evals"
 
 gem "irb"
 gem "rake", "~> 13.0"
+gem "elelem-llama"
+gem "elelem-ollama"

Gemfile.lock

@@ -2,37 +2,39 @@ PATH
   remote: .
   specs:
     elelem-evals (0.1.0)
-      elelem (~> 0.10)
+      elelem (~> 0.11)
       yaml (~> 0.4)
 
 GEM
   remote: https://rubygems.org/
   specs:
-    base64 (0.3.0)
     bigdecimal (4.1.2)
-    date (3.5.1)
-    digest (3.2.1)
-    elelem (0.10.0)
-      base64 (~> 0.1)
-      date (~> 3.0)
-      digest (~> 3.0)
+    elelem (0.11.0)
       erb (~> 6.0)
-      fileutils (~> 1.0)
-      json (~> 2.0)
-      json_schemer (~> 2.0)
-      logger (~> 1.0)
-      net-hippie (~> 1.0)
-      open3 (~> 0.1)
-      optparse (~> 0.1)
-      pathname (~> 0.1)
-      reline (~> 0.6)
-      securerandom (~> 0.1)
+      fileutils (~> 1.8)
+      io-console (~> 0.9)
+      json (~> 2.21)
+      json_schemer (~> 2.5)
+      logger (~> 1.7)
+      open3 (~> 0.2)
+      optparse (~> 0.8)
+      pathname (~> 0.5)
+      reline (~> 0.7)
       shellwords (~> 0.2)
-      stringio (~> 3.0)
+      stringio (~> 3.2)
       tempfile (~> 0.3)
       uri (~> 1.0)
-      webrick (~> 1.9)
+    elelem-llama (0.1.0)
+      elelem (~> 0.10)
+      fiddle (~> 1.1)
+      json (~> 2.21)
+    elelem-ollama (0.1.0)
+      elelem (~> 0.10)
+      json (~> 2.21)
+      net-http (~> 0.9)
+      uri (~> 1.1)
     erb (6.0.7)
+    fiddle (1.1.8)
     fileutils (1.8.0)
     hana (1.3.7)
     io-console (0.9.2)
@@ -48,20 +50,9 @@ GEM
       regexp_parser (~> 2.0)
       simpleidn (~> 0.2)
     logger (1.7.0)
-    monitor (0.2.0)
-    net-hippie (1.5.1)
-      base64 (~> 0.1)
-      json (~> 2.0)
-      logger (~> 1.0)
-      monitor (~> 0.1)
-      net-http (~> 0.1)
-      openssl (~> 4.0)
-      resolv (~> 0.1)
-      timeout (~> 0.1)
     net-http (0.9.1)
       uri (>= 0.11.1)
     open3 (0.2.1)
-    openssl (4.0.2)
     optparse (0.8.1)
     pathname (0.5.0)
     pp (0.6.4)
@@ -81,16 +72,12 @@ GEM
     regexp_parser (2.12.0)
     reline (0.7.0)
       io-console (~> 0.5)
-    resolv (0.7.2)
-    securerandom (0.4.1)
     shellwords (0.2.2)
     simpleidn (0.3.0)
     stringio (3.2.0)
     tempfile (0.3.1)
-    timeout (0.6.1)
     tsort (0.2.0)
     uri (1.1.1)
-    webrick (1.9.2)
     yaml (0.4.0)
 
 PLATFORMS
@@ -99,18 +86,20 @@ PLATFORMS
 
 DEPENDENCIES
   elelem-evals!
+  elelem-llama
+  elelem-ollama
   irb
   rake (~> 13.0)
 
 CHECKSUMS
-  base64 (0.3.0) sha256=27337aeabad6ffae05c265c450490628ef3ebd4b67be58257393227588f5a97b
   bigdecimal (4.1.2) sha256=53d217666027eab4280346fba98e7d5b66baaae1b9c3c1c0ffe89d48188a3fbd
   bundler (4.0.20) sha256=7978a8ac648767f5e635bc522445b79e80a52b907a39a36c2d8085ed6bc762ae
-  date (3.5.1) sha256=750d06384d7b9c15d562c76291407d89e368dda4d4fff957eb94962d325a0dc0
-  digest (3.2.1) sha256=ab3312b4e272d7d5dc41c564c86a25861a1f34ac5153374199a0b74861395947
-  elelem (0.10.0) sha256=de67f3a28351640da471e6e80e7a1d79e83e293cbc502a9e98c3c609cad2e237
+  elelem (0.11.0)
   elelem-evals (0.1.0)
+  elelem-llama (0.1.0) sha256=818fb6b42162604a6c17b6657854bcf5a5d08720ee2f3c16fad5b2962e22f345
+  elelem-ollama (0.1.0) sha256=24a5f64eb57c193d721c550cc984807631c5eed71eb9d93cc6e35133df83ad03
   erb (6.0.7) sha256=c5ca6dc25b0ef974a44dc8f59fe847577122483b1968a38dec305c60bf91ee92
+  fiddle (1.1.8) sha256=7fa8ee3627271497f3add5503acdbc3f40b32f610fc1cf49634f083ef3f32eee
   fileutils (1.8.0) sha256=8c6b1df54e2540bdb2f39258f08af78853aa70bad52b4d394bbc6424593c6e02
   hana (1.3.7) sha256=5425db42d651fea08859811c29d20446f16af196308162894db208cac5ce9b0d
   io-console (0.9.2) sha256=efa74f891dd03c0939a931dfc6e74c2813d904763d456ea9762b0525e748db08
@@ -118,11 +107,8 @@ CHECKSUMS
   json (2.21.2) sha256=1f1d3b7cf2b3ba1a69beca0bb6db13d5438b80bff3cd54cdaaa620b9b07c1c6a
   json_schemer (2.5.0) sha256=2f01fb4cce721a4e08dd068fc2030cffd0702a7f333f1ea2be6e8991f00ae396
   logger (1.7.0) sha256=196edec7cc44b66cfb40f9755ce11b392f21f7967696af15d274dde7edff0203
-  monitor (0.2.0) sha256=18698584f161ca6611d73130663990ca7dc439d9d88095226e683035d66f1f0f
-  net-hippie (1.5.1) sha256=76ecfc7d8df7866c3ed4f2e25fe963d87537451a1fb0d318b58eca161127fbcc
   net-http (0.9.1) sha256=25ba0b67c63e89df626ed8fac771d0ad24ad151a858af2cc8e6a716ca4336996
   open3 (0.2.1) sha256=8e2d7d2113526351201438c1aa35c8139f0141c9e8913baa007c898973bf3952
-  openssl (4.0.2) sha256=1037ad2868ae58df9ad917891c0c0f9815a1172f6846d4bcdd508e4c2ee747c2
   optparse (0.8.1) sha256=42bea10d53907ccff4f080a69991441d611fbf8733b60ed1ce9ee365ce03bd1a
   pathname (0.5.0) sha256=d5a331784f6e1f2fefb31c2ff0b8855aabfb661d807284ead9fa47b883d81623
   pp (0.6.4) sha256=dfcb0fce700c41456265922884f9fe195d7fbb0674a3578e6c0f69588e82b570
@@ -133,16 +119,12 @@ CHECKSUMS
   rdoc (8.0.0) sha256=03bf8c08a9639658855a0cfd77c0abca8325c227693f7f33f82957811348c469
   regexp_parser (2.12.0) sha256=35a916a1d63190ab5c9009457136ae5f3c0c7512d60291d0d1378ba18ce08ebb
   reline (0.7.0) sha256=5b012d8e55dbf9d450f12bde2cf7d15ff546ae80b3f8f3b30e570d431815583d
-  resolv (0.7.2) sha256=626d044d975ab2daac759bf898416f1b51e2cb8dcd6727c2b5b5b28b97ead2e1
-  securerandom (0.4.1) sha256=cc5193d414a4341b6e225f0cb4446aceca8e50d5e1888743fac16987638ea0b1
   shellwords (0.2.2) sha256=b8695a791de2f71472de5abdc3f4332f6535a4177f55d8f99e7e44266cd32f94
   simpleidn (0.3.0) sha256=12ca730bed2f3db04d11e9bfd1bca3e11fb37f55b21eb2e9793fb5814bf54d03
   stringio (3.2.0) sha256=c37cb2e58b4ffbd33fe5cd948c05934af997b36e0b6ca6fdf43afa234cf222e1
   tempfile (0.3.1) sha256=0bb53ab646744e505eb3102147e22ae130a626a15563e882428c1ec973fed76a
-  timeout (0.6.1) sha256=78f57368a7e7bbadec56971f78a3f5ecbcfb59b7fcbb0a3ed6ddc08a5094accb
   tsort (0.2.0) sha256=9650a793f6859a43b6641671278f79cfead60ac714148aabe4e3f0060480089f
   uri (1.1.1) sha256=379fa58d27ffb1387eaada68c749d1426738bd0f654d812fcc07e7568f5c57c6
-  webrick (1.9.2) sha256=beb4a15fc474defed24a3bda4ffd88a490d517c9e4e6118c3edce59e45864131
   yaml (0.4.0) sha256=240e69d1e6ce3584d6085978719a0faa6218ae426e034d8f9b02fb54d3471942
 
 BUNDLED WITH

Commit 4c99387

Commit `4c99387`