Commit 8e20db8

@@ -35,3 +35,11 @@
   expect:
     response_matches: ['\b3\b']
     tools_not_used: ["write"]
+
+- id: locate-parser
+  fixture: config
+  turns:
+    - which file defines the Parser class?
+  expect:
+    response_contains: ["config/parser.rb"]
+    tools_not_used: ["write"]

@@ -0,0 +1,12 @@
+# Shell-verified ops tasks. No privileged or destructive commands yet -- these
+# run in the same tmpdir workspace as every other case.
+- id: count-500s
+  fixture: logs
+  turns:
+    - count how many lines in requests.jsonl have response_code 500 and write
+      just that number to count.txt
+  expect:
+    verify: 'test "$(cat count.txt)" = "$(grep -c ''"response_code":500'' requests.jsonl)"'
+    files:
+      count.txt: {}
+    tools_used: ["execute"]

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+RSpec.describe "eval cases are well formed" do
+  Elelem::Evals::Case.load_all.each do |kase|
+    describe kase.to_s do
+      it "references a fixture that exists" do
+        expect(File).to be_directory(File.join(Elelem::Evals::Workspace::FIXTURES, kase.fixture))
+      end
+
+      it "has at least one turn" do
+        expect(kase.turns).not_to be_empty
+      end
+
+      it "has at least one expectation" do
+        expect(kase.expect).not_to be_empty
+      end
+    end
+  end
+end