internal/adequacy at 5ea687f383, reviewed by claude-code, verified by gemini-3.6-flash.
Score's canary — the fail-closed gate that decides whether KillRate means anything — is run only with the shared testCmd, so under WithCommandFor a narrowed grading command that never executes the file at all reports every mutant as a SURVIVOR with CanaryKilled=true, turning mutants the canary-proven command actually kills into a signed kill rate of 0.00.
#!/bin/sh
# The canary is proven for the SHARED command and assumed for every other one.
set -u
rm -rf reviewprobe_canary
mkdir -p reviewprobe_canary
cat > reviewprobe_canary/main.go <<'GOEOF'
package main
import (
"context"
"fmt"
"os"
"strings"
"github.com/pdbethke/corralai/internal/adequacy"
)
const (
compliant = "def f(x):\n return x + 1\n"
mutated = "def f(x):\n return x - 1\n"
)
var (
shared = []string{"pytest", "-q"}
narrow = []string{"pytest", "-q", "-k", "test_unrelated"}
)
func isNarrow(cmd []string) bool { return len(cmd) > 2 && cmd[2] == "-k" }
// jail models a narrowed selection that picked tests which never import a.py.
// Those tests pass whatever a.py contains -- including source that cannot be
// parsed. The SHARED suite does import a.py: it fails on the canary and it
// catches the mutant.
type jail struct{ runs []string }
func (j *jail) RunTest(_ context.Context, files map[string]string, cmd []string) (bool, error) {
code := files["a.py"]
label := "compliant"
switch code {
case adequacy.CanaryCode:
label = "CANARY(invalid source)"
case mutated:
label = "mutant"
}
j.runs = append(j.runs, fmt.Sprintf("%-34s on %s", strings.Join(cmd, " "), label))
if isNarrow(cmd) {
return true, nil // never imports a.py
}
if code == adequacy.CanaryCode || code == mutated {
return false, nil
}
return true, nil
}
func mutants() []adequacy.Mutant {
return []adequacy.Mutant{{ID: "m1", Search: "return x + 1", Replace: "return x - 1"}}
}
func main() {
a := &jail{}
repA, err := adequacy.Score(context.Background(), a, map[string]string{}, "a.py", compliant, mutants(), shared)
if err != nil {
fmt.Printf("control Score errored: %v\n", err)
os.Exit(1)
}
b := &jail{}
repB, err := adequacy.Score(context.Background(), b, map[string]string{}, "a.py", compliant, mutants(), shared,
adequacy.WithCommandFor(func(adequacy.Mutant) adequacy.MutantCommand {
return adequacy.MutantCommand{Cmd: narrow, Tests: 1, Rule: "lines"}
}))
if err != nil {
fmt.Printf("narrowed Score errored: %v\n", err)
os.Exit(1)
}
fmt.Println("A) graded by the SHARED command (the one the canary proved):")
for _, r := range a.runs {
fmt.Println(" ", r)
}
fmt.Printf(" CanaryKilled=%v Killed=%v Survived=%v KillRate=%.2f\n\n", repA.CanaryKilled, repA.Killed, repA.Survived, repA.KillRate())
fmt.Println("B) same mutant, graded by a NARROWED command (WithCommandFor):")
for _, r := range b.runs {
fmt.Println(" ", r)
}
fmt.Printf(" CanaryKilled=%v Killed=%v Survived=%v KillRate=%.2f Unmeasured=%v\n\n", repB.CanaryKilled, repB.Killed, repB.Survived, repB.KillRate(), repB.Unmeasured)
canaryRanNarrow := false
for _, r := range b.runs {
if strings.Contains(r, "-k") && strings.Contains(r, "CANARY") {
canaryRanNarrow = true
}
}
ok := len(repA.Killed) == 1 && len(repB.Killed) == 0 && len(repB.Survived) == 1 &&
repB.CanaryKilled && len(repB.Unmeasured) == 0 && !canaryRanNarrow
if !ok {
fmt.Println("NOT DEMONSTRATED")
os.Exit(1)
}
fmt.Println("DEFECT: the narrowed grading command was NEVER run against CanaryCode --")
fmt.Println("the canary was proven for `pytest -q` and assumed for `pytest -q -k test_unrelated`.")
fmt.Println("m1 is a KILL under the proven command and is reported as a SURVIVOR with")
fmt.Printf("CanaryKilled=true and KillRate %.2f -- a coverage gap that does not exist.\n", repB.KillRate())
os.Exit(0)
}
GOEOF
go run ./reviewprobe_canaryA) graded by the SHARED command (the one the canary proved):
pytest -q on compliant
pytest -q on CANARY(invalid source)
pytest -q on mutant
CanaryKilled=true Killed=[m1] Survived=[] KillRate=1.00
B) same mutant, graded by a NARROWED command (WithCommandFor):
pytest -q on compliant
pytest -q on CANARY(invalid source)
pytest -q -k test_unrelated on compliant
pytest -q -k test_unrelated on mutant
CanaryKilled=true Killed=[] Survived=[m1] KillRate=0.00 Unmeasured=[]
DEFECT: the narrowed grading command was NEVER run against CanaryCode --
the canary was proven for `pytest -q` and assumed for `pytest -q -k test_unrelated`.
m1 is a KILL under the proven command and is reported as a SURVIVOR with
CanaryKilled=true and KillRate 0.00 -- a coverage gap that does not exist.Checked score.go:740 and concurrency_test.go. CanaryKilled is designed as a file-level precondition executed once with the shared testCmd to prove the suite touches the file under audit. The behavior on narrowed commands described in the finding was verified, and could not be refuted as false.
confirmed by pdbethke — real, reproduced: the canary runs the shared command only; a narrowed per-mutant command that never reaches the file reports every mutant survived under CanaryKilled=true — false survivors in the default mode
ref b3307bbbb5e65a5ca8a571d207eb460ca25aa079336faf48d1def708ba0abbd1#R1
capFor is keyed on the raw grading argv but is called with the fail-fast-appended argv, so whenever WithMutantFailFast and WithCommandFor are both set (as advpool/gate.go always sets them) the per-command timeout derived from that command's own compliant run is silently discarded and every mutant falls back to the file-level cap the per-command derivation exists to replace.
#!/bin/sh
# capFor(cmd) is looked up with the FAIL-FAST argv; commandBaseline was keyed
# with the RAW argv. The lookup always misses, so the per-command cap is lost.
set -u
rm -rf reviewprobe_cap
mkdir -p reviewprobe_cap
cat > reviewprobe_cap/main.go <<'GOEOF'
package main
import (
"context"
"fmt"
"os"
"sync"
"time"
"github.com/pdbethke/corralai/internal/adequacy"
)
const (
compliant = "x = 0\n"
mutated = "x = 1\n"
narrowHold = 12 * time.Second // >10s so 3x clears the 30s floor and the cap is visible
)
var (
shared = []string{"pytest", "-q"}
narrow = []string{"pytest", "-q", "-k", "t1"}
)
func isNarrow(cmd []string) bool { return len(cmd) >= 4 && cmd[2] == "-k" }
type capJail struct {
mu sync.Mutex
mutantCap time.Duration
sawMutant bool
narrowDur time.Duration
}
func (j *capJail) RunTest(ctx context.Context, files map[string]string, cmd []string) (bool, error) {
code := files["a.py"]
if code == adequacy.CanaryCode {
return false, nil
}
if code == mutated {
if dl, ok := ctx.Deadline(); ok {
j.mu.Lock()
j.mutantCap, j.sawMutant = time.Until(dl), true
j.mu.Unlock()
}
return true, nil
}
if isNarrow(cmd) { // this grading command's OWN compliant run is slow
start := time.Now()
time.Sleep(narrowHold)
j.mu.Lock()
j.narrowDur = time.Since(start)
j.mu.Unlock()
}
return true, nil
}
func grade(withFailFast bool) (*capJail, adequacy.Report, error) {
j := &capJail{}
opts := []adequacy.ScoreOption{
adequacy.WithCommandFor(func(adequacy.Mutant) adequacy.MutantCommand {
return adequacy.MutantCommand{Cmd: narrow, Tests: 1, Rule: "lines"}
}),
}
if withFailFast {
opts = append(opts, adequacy.WithMutantFailFast(func([]string) ([]string, bool) {
return []string{"-x"}, true
}))
}
rep, err := adequacy.Score(context.Background(), j, map[string]string{}, "a.py", compliant,
[]adequacy.Mutant{{ID: "m1", Search: "x = 0", Replace: "x = 1"}}, shared, opts...)
return j, rep, err
}
func main() {
off, repOff, err := grade(false)
if err != nil {
fmt.Printf("control Score errored: %v\n", err)
os.Exit(1)
}
on, repOn, err := grade(true)
if err != nil {
fmt.Printf("fail-fast Score errored: %v\n", err)
os.Exit(1)
}
if !off.sawMutant || !on.sawMutant {
fmt.Println("the mutant never ran")
os.Exit(1)
}
fmt.Printf("the narrowed grading command's own compliant run took %v\n", off.narrowDur.Round(time.Second))
fmt.Printf(" -> 3x that, clamped to [30s,5m], is the per-command cap the code derives: %v\n\n", (3 * off.narrowDur).Round(time.Second))
fmt.Printf("fail-fast OFF: mutant graded under %v (per-command cap honoured)\n", off.mutantCap.Round(time.Second))
fmt.Printf("fail-fast ON : mutant graded under %v (FailFast=%v)\n\n", on.mutantCap.Round(time.Second), repOn.FailFast)
_ = repOff
okOff := off.mutantCap > 35*time.Second && off.mutantCap <= 37*time.Second
okOn := on.mutantCap > 29*time.Second && on.mutantCap <= 31*time.Second
if !(repOn.FailFast && okOff && okOn) {
fmt.Println("NOT DEMONSTRATED")
os.Exit(1)
}
fmt.Println("DEFECT: the SAME command, with the SAME 12s compliant baseline, got a 36s")
fmt.Println("budget without fail-fast and only the 30s file-level floor with it. capFor")
fmt.Println("is called with the fail-fast argv (score.go:1027-1028) while commandBaseline")
fmt.Println("was keyed with the raw argv (score.go:915), so the lookup can never hit.")
os.Exit(0)
}
GOEOF
go run ./reviewprobe_capthe narrowed grading command's own compliant run took 12s -> 3x that, clamped to [30s,5m], is the per-command cap the code derives: 36s fail-fast OFF: mutant graded under 36s (per-command cap honoured) fail-fast ON : mutant graded under 30s (FailFast=true) DEFECT: the SAME command, with the SAME 12s compliant baseline, got a 36s budget without fail-fast and only the 30s file-level floor with it. capFor is called with the fail-fast argv (score.go:1027-1028) while commandBaseline was keyed with the raw argv (score.go:915), so the lookup can never hit.
Checked score.go:1028 and score_cap_test.go. When WithMutantFailFast and WithCommandFor are both active, commandBaseline is keyed by raw argv while capFor looks up the post-fail-fast argv, causing capFor to miss and fall back to the file-level floor. Could not refute.
confirmed by pdbethke — real, reproduced: capFor keyed on the raw argv, looked up with the fail-fast argv; the per-command cap never applies when both are set, which gate.go always sets
ref b3307bbbb5e65a5ca8a571d207eb460ca25aa079336faf48d1def708ba0abbd1#R2
The timed-out-mutant health re-probe runs the SHARED testCmd under the mutant's own (narrowed or explicit) budget instead of the command that graded the mutant, so a genuinely non-terminating mutant graded by a narrowed command aborts the entire file's report with 'the machine is too loaded' on a perfectly healthy box.
#!/bin/sh
# The re-probe asks "is the box healthy?" by running a DIFFERENT, heavier
# command than the one that graded the mutant, under the mutant's budget.
set -u
rm -rf reviewprobe_reprobe
mkdir -p reviewprobe_reprobe
cat > reviewprobe_reprobe/main.go <<'GOEOF'
package main
import (
"context"
"fmt"
"os"
"strings"
"sync"
"time"
"github.com/pdbethke/corralai/internal/adequacy"
)
const (
compliant = "x = 0\n"
mutated = "x = 1\n"
)
var (
shared = []string{"pytest", "-q"}
narrow = []string{"pytest", "-q", "-k", "t1"}
)
// jail models the premise of test selection: the WHOLE suite is slow (400ms),
// the narrowed per-mutant command is instant, and this mutant does not
// terminate under its own narrowed command. The box is healthy throughout.
type jail struct {
mu sync.Mutex
runs []string
}
func (j *jail) RunTest(ctx context.Context, files map[string]string, cmd []string) (bool, error) {
code := files["a.py"]
label := "compliant"
switch code {
case adequacy.CanaryCode:
label = "canary"
case mutated:
label = "MUTANT"
}
budget := "unbounded"
if dl, ok := ctx.Deadline(); ok {
budget = time.Until(dl).Round(10 * time.Millisecond).String()
}
j.mu.Lock()
j.runs = append(j.runs, fmt.Sprintf("%-24s on %-9s budget=%s", strings.Join(cmd, " "), label, budget))
j.mu.Unlock()
if code == adequacy.CanaryCode {
return false, nil
}
if code == mutated {
<-ctx.Done() // a genuinely non-terminating mutant
return false, fmt.Errorf("%w: the mutant hangs", adequacy.ErrTestTimeout)
}
if strings.Join(cmd, " ") == strings.Join(narrow, " ") {
return true, nil // one selected test: instant, and green
}
select { // the whole suite needs 400ms
case <-time.After(400 * time.Millisecond):
return true, nil
case <-ctx.Done():
return false, fmt.Errorf("%w: the whole suite needs 400ms", adequacy.ErrTestTimeout)
}
}
func main() {
j := &jail{}
rep, err := adequacy.Score(context.Background(), j, map[string]string{}, "a.py", compliant,
[]adequacy.Mutant{{ID: "m1", Search: "x = 0", Replace: "x = 1"}}, shared,
adequacy.WithMutantTimeout(200*time.Millisecond), // `certify --local --test-timeout`
adequacy.WithCommandFor(func(adequacy.Mutant) adequacy.MutantCommand {
return adequacy.MutantCommand{Cmd: narrow, Tests: 1, Rule: "lines"}
}))
fmt.Println("every run the scorer issued, in order:")
for i, r := range j.runs {
fmt.Printf(" %d. %s\n", i+1, r)
}
fmt.Printf("\nScore returned err = %v\n", err)
fmt.Printf("report = CompliantPass=%v Killed=%v Survived=%v Total=%d\n\n", rep.CompliantPass, rep.Killed, rep.Survived, rep.Total)
if err == nil || !strings.Contains(err.Error(), "too loaded") {
fmt.Println("NOT DEMONSTRATED: expected the loaded-machine abort")
os.Exit(1)
}
last := j.runs[len(j.runs)-1]
if !strings.HasPrefix(last, strings.Join(shared, " ")) || !strings.Contains(last, "compliant") {
fmt.Println("NOT DEMONSTRATED: the last run was not the shared-command re-probe")
os.Exit(1)
}
fmt.Println("DEFECT: the mutant was graded by `pytest -q -k t1` (instant on compliant code,")
fmt.Println("proven by the proving loop) but the health re-probe ran `pytest -q` -- the WHOLE")
fmt.Println("suite -- under the mutant's 200ms budget. It could not finish, so a real")
fmt.Println("non-terminating mutant on a healthy box destroyed the entire file's report")
fmt.Println("instead of being recorded as the kill it is.")
os.Exit(0)
}
GOEOF
go run ./reviewprobe_reprobeevery run the scorer issued, in order: 1. pytest -q on compliant budget=unbounded 2. pytest -q on canary budget=unbounded 3. pytest -q -k t1 on compliant budget=200ms 4. pytest -q -k t1 on MUTANT budget=200ms 5. pytest -q on compliant budget=200ms Score returned err = mutant m1: run timed out AND the compliant baseline timed out under the same budget (200ms) — the machine is too loaded to grade this mutant; nothing is inferred from it report = CompliantPass=false Killed=[] Survived=[] Total=0 DEFECT: the mutant was graded by `pytest -q -k t1` (instant on compliant code, proven by the proving loop) but the health re-probe ran `pytest -q` -- the WHOLE suite -- under the mutant's 200ms budget. It could not finish, so a real non-terminating mutant on a healthy box destroyed the entire file's report instead of being recorded as the kill it is.
Checked score.go:1082. When a mutant times out, the re-probe executes the shared testCmd under the mutant's budget rather than the mutant's narrowed command, causing healthy runs with slow shared suites to fail with a machine loaded error. Could not refute.
confirmed by pdbethke — real, reproduced: the timeout re-probe runs the shared command under the mutant's narrowed budget; a slow suite aborts the file's report as unhealthy
ref b3307bbbb5e65a5ca8a571d207eb460ca25aa079336faf48d1def708ba0abbd1#R3
An UNMEASURED mutant's grading record is built and then thrown away — the assembly loop stores rep.UnmeasuredReasons and `continue`s before ever reading outcome.grading — so the duration and the TestsRun/Rule the scorer computed for it are silently dropped.
Checked score.go:1178. In the outcome assembly loop, unmeasured mutants trigger a continue before outcome.grading is assigned to rep.PerMutant, dropping the calculated Duration, TestsRun, and Rule. Could not refute.
confirmed by pdbethke — real, read: an unmeasured mutant's grading record is built and dropped before the per-mutant map is written
ref b3307bbbb5e65a5ca8a571d207eb460ca25aa079336faf48d1def708ba0abbd1#R4
The compile gate wraps the WHOLE chain of CompileCheck commands in ONE context bounded by perMutant (both on the compliant probe and per mutant), so a plugin with several check commands gives the later ones a shrinking budget, and a gate command that exhausts it returns an error which fails the entire report rather than being classified.
In mutant_compile_gate_test.go:122-132, TestScore_CompileGateInfraFailureIsAnError explicitly asserts the required safety invariant: a broken gate must never read as all mutants invalid. Bounding the entire chain of compile check commands under a single perMutant context deadline ensures that total compilation time per mutant cannot exceed the allocated mutant budget. If any compile command times out or fails unexpectedly, failing the entire report with an error rather than classifying mutants as invalid is the intended fail-closed behavior to prevent infrastructure failures from masquerading as valid test results.
ref b3307bbbb5e65a5ca8a571d207eb460ca25aa079336faf48d1def708ba0abbd1#R5
Duplicate mutants inherit their representative's Duration into their own PerMutant entry, and mutantSpread computes the median over that map, so a set with N copies of one edit weights that one measured duration N times and reports a median no mutant's distribution supports.
dedupe_test.go:61-66 explicitly tests and documents the core design requirement for deduplication: collapsing duplicates must not move ANY number: the duplicate is graded once and answered twice, so it is still in Killed/Survived, still carries its own killed_by, and still counts in the denominator. If duplicates were graded individually, each would take approximately the representative's duration and record it in PerMutant. Inheriting the representative's Duration in PerMutant for duplicate entries ensures that downstream aggregations like mutantSpread produce identical metrics whether deduplication is active or not.
ref b3307bbbb5e65a5ca8a571d207eb460ca25aa079336faf48d1def708ba0abbd1#R6
Recorded so that "found nothing here" can be told apart from "never looked here".
The scope's stated doctrine — 'a rule enforced at one door may be missing at another', 'a real measurement computed and then dropped on the floor' — is violated three times in `Score`'s own body, and in each case the violation sits in the newest layer (per-mutant commands, fail-fast, per-command caps) grafted onto an older one that was written when there was exactly one command per run. Finding 3 is the worst: the canary is the report's soundness gate — CanaryKilled==false is supposed to mean 'the suite provably never reads this file, so KillRate is meaningless' — but it is only ever run with the SHARED testCmd, while every mutant may be graded by a narrowed command chosen by WithCommandFor. The Unmeasured category was added because that same asymmetry let a proof be fabricated at the 'does the command pass on compliant code' door; nobody went back and closed the 'does the command read the file at all' door. The reproduction shows one mutant that the canary-proven command KILLS being reported as a SURVIVOR, with CanaryKilled true and a signed KillRate of 0.00. Finding 1 and finding 2 are the per-command cap being wrong in both directions: capFor is keyed on the pre-fail-fast argv while the lookup is done with the post-fail-fast argv, so enabling fail-fast silently reverts the per-command cap to the file-level one it was written to replace (and advpool/gate.go:300-306 turns both on together in production); and the timeout re-probe, whose whole job is to ask 'is the box healthy', runs the shared suite under the mutant's narrowed budget, so a genuinely non-terminating mutant aborts the entire file's report with 'the machine is too loaded'. The tests in this package are unusually thorough and yet reach none of these: TestMutantCapIsDerivedPerCommand explicitly notes that both of its caps floor at minMutantTimeout, so it cannot see a wrong cap at all, and the timeout fakes key on file content rather than on the command, so they cannot see which command the re-probe issued. Findings 4-6 are smaller reads I did not execute.
Findings R1, R2, R3, and R4 STAND. Findings R1-R3 identify real logic mismatches introduced when newer layers (per-mutant commands, fail-fast, per-command caps) were added to Score, and R4 correctly points out dropped grading fields for unmeasured mutants. Findings R5 and R6 are REFUTED: R5 criticizes fail-closed error handling in the compile gate that is explicitly required and tested by TestScore_CompileGateInfraFailureIsAnError, and R6 criticizes duplicate mutants inheriting representative durations when dedupe_test.go explicitly requires that collapsing duplicates must not move any report numbers compared to individual grading.