internal/advpool at 91d433696a, reviewed by claude-code, verified by codex.
JailScorer.CompliantFailure builds its command from scoreWorkspace instead of authoredCmd, so under a test selection it runs the run's raw project command — which never collects the authored test — and returns that run's output (ignoring the pass/fail bool) as the explanation of why the authored test failed on unmutated code, which is then fed to the writer's repair prompt and parsed for deselect selectors by salvageByDeselect.
#!/bin/sh
set -e
cat > internal/advpool/zzrepro_compliantfailure_test.go <<'GOEOF'
// SPDX-License-Identifier: Elastic-2.0
package advpool
import (
"context"
"testing"
"github.com/pdbethke/corralai/internal/adequacy"
golang "github.com/pdbethke/corralai/internal/lang"
)
type reproCFJail struct {
cmds [][]string
files []map[string]string
}
func (j *reproCFJail) RunTest(ctx context.Context, files map[string]string, cmd []string) (bool, error) {
j.cmds = append(j.cmds, cmd)
j.files = append(j.files, files)
return true, nil
}
// RunTestVerbose reports a GREEN run: the dev suite passes.
func (j *reproCFJail) RunTestVerbose(ctx context.Context, files map[string]string, cmd []string) (bool, string, error) {
j.cmds = append(j.cmds, cmd)
j.files = append(j.files, files)
return true, "==== 1 passed in 0.01s ====", nil
}
func reproArgvHas(argv []string, want string) bool {
for _, a := range argv {
if a == want {
return true
}
}
return false
}
func reproKeys(m map[string]string) []string {
out := make([]string, 0, len(m))
for k := range m {
out = append(out, k)
}
return out
}
func TestReproCompliantFailureCommand(t *testing.T) {
base := map[string]string{
"calc.py": "def add(a, b):\n return a + b\n",
"tests/test_calc.py": "from calc import add\n\n\ndef test_add():\n assert add(1, 2) == 3\n",
}
sel := golang.Selection{
Method: "coverage-context",
Tests: []string{"tests/test_calc.py::test_add"},
Of: 1,
Cmd: []string{"python3", "-m", "pytest", "tests/test_calc.py::test_add"},
Base: []string{"python3", "-m", "pytest"},
}
j := &reproCFJail{}
s := JailScorer{
Jail: j,
BaseFiles: base,
Lang: "python",
Selection: sel,
DevTestPath: "tests/test_calc.py",
}
const runTestCmd = "python3 -m pytest tests/"
authoredTest := "def test_corral_kills_it():\n assert add(1, 2) == 99\n"
out := s.CompliantFailure(context.Background(), "calc.py", base["calc.py"], authoredTest, runTestCmd)
if len(j.cmds) != 1 {
t.Fatalf("expected exactly one jail run, got %d: %v", len(j.cmds), j.cmds)
}
ran := j.cmds[0]
// The authored test's own path is the one workspace key the repo did not
// already contain.
authoredPath := ""
for k := range j.files[0] {
if _, ok := base[k]; !ok {
authoredPath = k
}
}
if authoredPath == "" {
t.Fatalf("the authored test was not overlaid at all; workspace keys: %v", reproKeys(j.files[0]))
}
scoringCmd := s.AuthoredCommand("calc.py", adequacy.ShellSplit(runTestCmd))
t.Logf("authored test overlaid at: %s", authoredPath)
t.Logf("the authored PASS would run: %v", scoringCmd)
t.Logf("CompliantFailure actually ran: %v", ran)
t.Logf("CompliantFailure returned: %q", out)
if !reproArgvHas(scoringCmd, authoredPath) {
t.Fatalf("premise broken: the authored pass's own command does not name the authored test either: %v", scoringCmd)
}
if reproArgvHas(ran, authoredPath) {
t.Fatalf("no defect: CompliantFailure's command names the authored test: %v", ran)
}
if out == "" {
t.Fatalf("no defect: CompliantFailure returned nothing")
}
t.Logf("DEFECT: CompliantFailure ran %v, which never collects %s, and returned that run's output %q as the reason the authored test failed on the unmutated code -- even though that run PASSED (the bool from RunTestVerbose is discarded at gate.go:545). This string is fed verbatim to the writer's repair prompt (roles.go cleanFailureRepairBlock) and to lang.FailureDeselector.FailedTests inside salvageByDeselect (driver.go:1957).", ran, authoredPath, out)
}
GOEOF
go test ./internal/advpool/ -run TestReproCompliantFailureCommand -count=1 -v=== RUN TestReproCompliantFailureCommand
zzrepro_compliantfailure_test.go:92: authored test overlaid at: tests/test_calc_corral.py
zzrepro_compliantfailure_test.go:93: the authored PASS would run: [python3 -m pytest tests/test_calc.py::test_add tests/test_calc_corral.py]
zzrepro_compliantfailure_test.go:94: CompliantFailure actually ran: [python3 -m pytest tests/]
zzrepro_compliantfailure_test.go:95: CompliantFailure returned: "==== 1 passed in 0.01s ===="
zzrepro_compliantfailure_test.go:106: DEFECT: CompliantFailure ran [python3 -m pytest tests/], which never collects tests/test_calc_corral.py, and returned that run's output "==== 1 passed in 0.01s ====" as the reason the authored test failed on the unmutated code -- even though that run PASSED (the bool from RunTestVerbose is discarded at gate.go:545). This string is fed verbatim to the writer's repair prompt (roles.go cleanFailureRepairBlock) and to lang.FailureDeselector.FailedTests inside salvageByDeselect (driver.go:1957).
--- PASS: TestReproCompliantFailureCommand (0.00s)
PASS
ok github.com/pdbethke/corralai/internal/advpool 0.010sThe claim is narrower than stated. The supplied input runs `python3 -m pytest tests/`, and the authored workspace contains `tests/test_calc_corral.py`; that file matches pytest's test_*.py discovery convention, also documented at internal/lang/python.go:53. Directory discovery does not require each collected file to appear explicitly in argv. internal/advpool/gate.go:538-545 preserves this directory-wide command and overlays the authored file, so this input does not establish that the test is never collected. The reproduction's mock returns a passing result without executing or inspecting collection; its argv membership assertion cannot prove exclusion. The command mismatch with ScoreAuthoredReport and discarded pass/fail boolean are real, but the asserted noncollection and resulting diagnosis on this particular input are unsupported.
ref 096353ae9c5e1daf9e0edc73a41fc9ee010ac2456719caeed8919bd426ef342d#R1
For the languages with no lang.Preambler (javascript, typescript, ruby, php) PreambleFor returns the file's first 40 lines, so shardCode emits a <=40-line file WHOLE into every generator shard while still returning chunked=true — the shard prompt then asserts "The code below is NOT the whole file — only the functions you are attacking" over a prompt containing the other shards' bodies, and ShardIsChunked signs Verdict.PromptShape as "chunk".
#!/bin/sh
set -e
cat > internal/advpool/zzrepro_shardchunk_test.go <<'GOEOF'
// SPDX-License-Identifier: Elastic-2.0
package advpool
import (
"strings"
"testing"
"github.com/pdbethke/corralai/internal/repoindex"
)
func TestReproShardChunkClaimIsFalse(t *testing.T) {
// A javascript file: jsPlugin implements no lang.Preambler, so
// lang.PreambleFor falls back to "the first 40 lines" -- and this file
// is 8 lines, so the preamble IS the whole file.
code := "export function add(a, b) {\n return a + b;\n}\n\nexport function sub(a, b) {\n // SUBTRACT_MARKER\n return a - b;\n}\n"
sigs := []repoindex.Signature{
{Name: "add", Line: 1, Lines: 3, Complexity: 1},
{Name: "sub", Line: 5, Lines: 4, Complexity: 1},
}
// The shard is aimed at add() ONLY.
sh := Shard{Index: 0, Symbols: []string{"add"}, Complexity: 1, Lines: 3, Mutants: 2}
rs := RunSpec{Lang: "javascript", CodePath: "calc.js", Code: code, Goal: "adding must be correct", NMutants: 2}
shown, chunked := shardCode(langFor(rs), rs.Code, filterSignatures(sigs, sh.Symbols))
t.Logf("shardCode chunked=%v", chunked)
t.Logf("code shown to a shard aimed only at add():\n%s", shown)
if !chunked {
t.Fatalf("no defect: shardCode reported it fell back to the whole file")
}
if !ShardIsChunked(sigs, sh) {
t.Fatalf("no defect: ShardIsChunked already reports this run as unchunked, so PromptShape would read \"file\"")
}
if !strings.Contains(shown, "SUBTRACT_MARKER") {
t.Fatalf("no defect: the un-aimed symbol sub() was actually elided from the shard's view")
}
if strings.Contains(shown, "lines elided") {
t.Fatalf("no defect: an elision was recorded, so something really was cut")
}
// The downstream consequences, reported rather than asserted (they follow
// from chunked==true above).
prompt, pchunked := renderMutantGeneratorShard(rs, sigs, sh)
t.Logf("rendered shard prompt: chunked=%v, contains the un-aimed body=%v, contains the NOT-the-whole-file claim=%v",
pchunked,
strings.Contains(prompt, "SUBTRACT_MARKER"),
strings.Contains(prompt, "The code below is NOT the whole file"))
t.Logf("DEFECT: shardCode returned chunked=true while returning the ENTIRE file (sub()'s body included) because lang.PreambleFor's no-Preambler fallback -- the first 40 lines -- swallowed the whole file. renderMutantGeneratorShard therefore appends \"The code below is NOT the whole file - only the functions you are attacking\" to a prompt that is the whole file, and ShardIsChunked makes StartRun set the signed Verdict.PromptShape to \"chunk\", whose own doc says it means \"every shard of this run showed only its own symbols' bodies\".")
}
GOEOF
go test ./internal/advpool/ -run TestReproShardChunkClaimIsFalse -count=1 -v=== RUN TestReproShardChunkClaimIsFalse
zzrepro_shardchunk_test.go:26: shardCode chunked=true
zzrepro_shardchunk_test.go:27: code shown to a shard aimed only at add():
export function add(a, b) {
return a + b;
}
export function sub(a, b) {
// SUBTRACT_MARKER
return a - b;
}
zzrepro_shardchunk_test.go:45: rendered shard prompt: chunked=true, contains the un-aimed body=true, contains the NOT-the-whole-file claim=true
zzrepro_shardchunk_test.go:50: DEFECT: shardCode returned chunked=true while returning the ENTIRE file (sub()'s body included) because lang.PreambleFor's no-Preambler fallback -- the first 40 lines -- swallowed the whole file. renderMutantGeneratorShard therefore appends "The code below is NOT the whole file - only the functions you are attacking" to a prompt that is the whole file, and ShardIsChunked makes StartRun set the signed Verdict.PromptShape to "chunk", whose own doc says it means "every shard of this run showed only its own symbols' bodies".
--- PASS: TestReproShardChunkClaimIsFalse (0.00s)
PASS
ok github.com/pdbethke/corralai/internal/advpool 0.009sI checked whether preamble handling removes unrelated bodies or corrects the chunk flag. internal/lang/lang.go:320 returns the complete short file when the plugin lacks Preambler. internal/advpool/roles.go:510 onward writes that preamble, skips symbol spans already included, and returns true without checking whether anything was excluded. renderMutantGeneratorShard then adds the NOT-the-whole-file claim, while ShardIsChunked checks signature spans alone. The documented allowance for contextual preambles does not make the literal prompt and shape claims accurate for this input.
ref 096353ae9c5e1daf9e0edc73a41fc9ee010ac2456719caeed8919bd426ef342d#R2
matrixAdjudication auto-CONFIRMS a test-critic's "this test can never fail" finding whenever the test killed none of the run's mutants and any other test killed one, which confirms a vacuity claim for a perfectly good test that simply asserts behaviour the generator did not mutate — and that confirmation both persists to the signed-record-linked CriticFindings sink and earns the critic a leaderboard PASS.
I traced scope filtering, adjudication, persistence, and leaderboard gating. internal/advpool/critic_adjudicate.go:58 confirms a scored zero-kill row when catchable is true; internal/advpool/driver.go:3082 applies this to whole-test findings without evidence that the mutants exercise the behavior that test asserts. Another test killing a mutant does not establish this test can never fail. The observation reaches the record-linked sink at driver.go:2848 and can earn OutcomePass at driver.go:3297. Credit is conditional on a signed certified verdict at driver.go:2827, but that condition does not eliminate the unsound confirmation or its possible reward.
ref 096353ae9c5e1daf9e0edc73a41fc9ee010ac2456719caeed8919bd426ef342d#R3
Verdict.TestWriterFailed's doc asserts "aggregate forces needs-review whenever Survivors > 0 and ProvenMissed < Survivors — see aggregate", but aggregate implements no such rule (its only terms are blockingFindingOpen, devKillRate < threshold, testWriterFailed, poolTestUnsound and the Wilson-width check), so a reader relying on that comment believes an unproven-gap guard exists that does not.
I compared the comment at internal/advpool/driver.go:539-544 with aggregate's complete status logic and its caller. internal/advpool/aggregate.go:220 gates on the named failure flags and kill-rate threshold, followed by the interval-width check; it does not compare ProvenMissed against Survivors. The narrower statement that TestWriterFailed forces needs-review is correct, but the parenthetical explicitly describes a broader guard that is absent.
ref 096353ae9c5e1daf9e0edc73a41fc9ee010ac2456719caeed8919bd426ef342d#R4
aggregate never gates on BaselineFailed or SuiteIgnoresFile — the two could-not-grade states are carried onto the Verdict after aggregate returns, and the only thing keeping such a run out of StatusCertified is devKillRate(0) < threshold, which is enforced solely by NewDriver's threshold > 0 check; a Driver built as a struct literal (as this package's own tests do) with Threshold 0 signs StatusCertified over a run that graded nothing.
I checked constructor validation, the no-survivor transition, aggregation, and signing for an independent rejection. NewDriver rejects nonpositive thresholds, but the stipulated struct literal bypasses it. internal/advpool/driver.go:2309-2327 explicitly advances failed-baseline and surviving-canary runs to aggregation without writer failure flags. aggregate receives neither diagnosis, and tickAggregate attaches them afterward at driver.go:2776-2780. With Threshold and CertifyIntervalWidth both zero, the status can remain certified. CertSigner uses that status to choose the decision; withholding an unmeasured KillRate does not correct the certification status.
ref 096353ae9c5e1daf9e0edc73a41fc9ee010ac2456719caeed8919bd426ef342d#R5
Recorded so that "found nothing here" can be told apart from "never looked here".
The scope is heavily annotated and most of the arithmetic is careful — the budget split, the Wilson interval, the wire forms, the per-seat comparability filters all survived reading. The two defects I can demonstrate are both of the shape this package keeps producing: a rule applied at one door and not at the sibling door. Finding 1 is the sharper one. `JailScorer` has three ways to run the authored test and two of them (ScoreAuthoredReport, verifyAuthoredTestReaches) build the command through `authoredCmd`/`alone` so the authored test's own path is collected; `CompliantFailure` — the third — builds it from `scoreWorkspace` alone, so under a selection (the default) it runs the run's raw project command, which never collects the authored test, and then returns that run's output *discarding the pass/fail bool*. The string it returns is fed verbatim into the writer's repair prompt as "Run against the ORIGINAL source, your test reported: …" and into `lang.FailureDeselector.FailedTests` inside `salvageByDeselect`, so both the repair round and the salvage are driven by output from a command that never ran the file they are about. Finding 2 is a disclosure defect: for the four languages with no `lang.Preambler`, `PreambleFor` returns the file's first 40 lines, so `shardCode` emits them verbatim into every shard and — for a file of 40 lines or fewer — emits the whole file, while still returning `chunked=true`; the prompt then asserts "The code below is NOT the whole file" over a prompt containing the whole file, and `ShardIsChunked` signs `Verdict.PromptShape: "chunk"`. Findings 3 and 4 are read-only: `matrixAdjudication` auto-CONFIRMS a critic's vacuity claim on a floor that does not support it, and `Verdict.TestWriterFailed`'s doc asserts a guard `aggregate` does not implement. I did not execute any driver tick, any queue interaction, or anything in the sharded generator's retry/drop bookkeeping — absence of findings there is absence of looking.
R1 overstates a real command mismatch: its named directory-wide pytest command collects the authored file, and its mock never tests collection. R2, R3, R4, and R5 withstand refutation. R3's leaderboard consequence requires a certified run and configured sinks; R5 concerns construction that bypasses NewDriver's validation.