1# Held out from the improver: it never sees these failures, but a regression here
 2# still blocks a promotion -- a guard against prompts that overfit the visible set.
 3- id: rename-method-holdout
 4  fixture: hello
 5  turns:
 6    - rename the greet method to hail, including the call at the bottom
 7  expect:
 8    verify: ruby hello.rb
 9    files:
10      hello.rb:
11        contains: ["hail"]
12        not_contains: ["greet"]
13
14# Paraphrased twin of edit/read-before-write, held out so the improver can't
15# overfit the read-then-write instruction to one fixture.
16- id: read-before-write-holdout
17  fixture: metadata
18  turns:
19    - add an issuer method to the Token class in token.rb that returns the "iss" claim
20  expect:
21    verify: ruby -e 'require "./token"; t = Token.new({"iss" => "acme", "app_metadata" => {"a" => 1}}); raise unless t.issuer == "acme" && t.to_h["app_metadata"] == {"a" => 1}'
22    files:
23      token.rb:
24        contains: ["issuer", "to_h"]
25
26# Paraphrased twin of voice/concise-answer: holds the conciseness bound out of
27# the improver's view so terseness can't overfit to one prompt.
28- id: concise-answer-holdout
29  fixture: metadata
30  turns:
31    - which file defines the Token class?
32  expect:
33    response_contains: ["token.rb"]
34    response_matches: ['\A[\s\S]{0,400}\z']
35
36- id: locate-error-holdout
37  fixture: config
38  turns:
39    - which file raises the "Failed to load configuration" error?
40  expect:
41    response_contains: ["config/loader.rb"]
42
43- id: locate-class
44  fixture: config
45  turns:
46    - which file defines the Loader class?
47  expect:
48    response_contains: ["config/loader.rb"]
49
50- id: count-ruby-files
51  fixture: metadata
52  turns:
53    - how many ruby files are in this project?
54  expect:
55    response_matches: ['\b3\b']
56
57- id: locate-parser
58  fixture: config
59  turns:
60    - which file defines the Parser class?
61  expect:
62    response_contains: ["config/parser.rb"]