{ "generated_from": "content/evidence.json + content/upstream.json + content/commons.json", "site": { "proposition": "Autonomous systems should not be trusted because they say they succeeded.", "supporting": "We investigate systems where consequential actions are explicitly authorised, independently observed, mechanically checked where possible, and replayable.", "issue": "2026-08-24", "version": "v1" }, "derived": { "findings": 18, "reproducible": 18, "proven": 4, "measured": 7, "observed": 7, "projects": 19, "upstream": 5, "upstreamOpen": 2, "controls": 15, "thirdParty": 7, "results": 3, "flaws": 4, "unknowns": 8, "props": 9, "covered": 4, "uncovered": 4, "contested": 1, "cases": 24, "matched": 24, "prereg": 13, "afterMeasurement": 1, "safetyExpectations": 23, "questions": 6, "mechanisms": 9 }, "grades": { "PROVEN": "follows from stated premises inside a named frame", "MEASURED": "empirical, with a stated sample and method", "OBSERVED": "witnessed in one instance, no frequency claim", "OPEN": "a question posed and not answered" }, "negative_control_states": { "PRESENT": "a known-bad input was shown to fail the check", "NONE": "no control is claimed", "NOT_APPLICABLE": "the result has no failing mode a control could exercise", "NOT_ESTABLISHED": "a control may exist and has not been shown to fire" }, "questions": [ { "id": "authorised-executed", "order": 1, "name": "Authorised is not executed", "plain": "A policy approves an operation. The operation that runs is not the one approved, and the record of approval reads exactly like a record of what ran.", "ask": "What binds the executed operation to the authorised one, and what would show the binding has failed?" }, { "id": "logged-observed", "order": 2, "name": "Logged is not observed", "plain": "A log records what a component intended. An observation records what changed. A monitor beside the action produces the first and is read as the second.", "ask": "What measures the effect from outside the thing that produced it?" }, { "id": "each-compliant-collectively-safe", "order": 3, "name": "Each compliant is not collectively safe", "plain": "Every agent stays inside its own limit. The shared quantity they draw on is breached anyway, and no per-agent check can see it.", "ask": "Which rule can bound the aggregate, under what preconditions, and where does it stop holding?" }, { "id": "passed-correct", "order": 4, "name": "Passed is not correct", "plain": "A judge, a scorer or a verifier says pass. The property it stands for has failed, and the pass carries no information about it.", "ask": "What does this checker actually distinguish, and against which oracle?" }, { "id": "spec-proved-property-proved", "order": 5, "name": "The specification proved is not the property intended", "plain": "Every property written from the specification holds. The requirement the design was meant to satisfy was never written down, so nothing checks it.", "ask": "Which requirements have anything checking them, and which formalism can even state them?" }, { "id": "replayed-reproduced", "order": 6, "name": "Replayed is not reproduced", "plain": "A record is intact and its integrity rechecks. Whether the recorded result follows from the recorded inputs is a different question with a different answer.", "ask": "Can the result be re-derived from the record alone, and where does re-derivation stop?" } ], "findings": [ { "slug": "n-1-permit-membership", "grade": "OBSERVED", "section": "constructible", "theme": "agent-control", "stage": "execute", "mechanism": "coverage-illusion", "plain": "A program passed all 24 of its recorded tests and still accepted a permission record that had never been issued.", "claim": "A start permit assembled entirely from public registry facts, with its reference number derived by the authorizer's own function and committed to no ledger, was accepted by the bounded executor and the registry-named transform was applied; the target's bytes changed.", "doesNotEstablish": "One defect in one program built for this study. Not a measurement of deployed systems, and no claim about how often authorisation records are treated as evidence of authorisation. Not a break-in: producing the forgery requires the ability to run code as the same user, and such an actor can edit the target directly without any record. The minimal fix establishes that a membership check closes this specific defect; it does not establish at-most-once execution, atomic consumption, or independent measurement of the effect.", "repo": "repowazdogz-droid/omega-n1-permit-membership", "verifiedAtCommit": "efba311", "reproduceCommand": "git clone https://github.com/repowazdogz-droid/omega-n1-permit-membership && cd omega-n1-permit-membership && python3 reproduce.py", "sourceUrl": "/findings/n-1-permit-membership/", "flagship": true, "question": "authorised-executed", "subject": "The frozen STARTAUTH-01 executor, attacked from outside; the minimal public reproduction re-creates the defect in a small program.", "frame": "One executor, one registry-named transform, one target file; forgery built from public registry facts.", "method": "Three forgeries of increasing strength handed to the executor; bytes hashed before and after.", "oracle": "The target bytes, hashed before and after each case.", "negativeControl": { "status": "PRESENT", "text": "S16 and S17, the two weaker forgeries, were refused. Their refusals are what make the third result legible." }, "denominator": "24 of 24 recorded cases matched their expectation; one of them (S18) expected the failure.", "independentReproduction": "None known.", "repoStatus": "active" }, { "slug": "authorized-limit-and-executed-write-disagreed", "grade": "OBSERVED", "section": "constructible", "theme": "agent-control", "stage": "authorise", "mechanism": "faithfulness", "plain": "An agent was limited to 4 KB. It wrote 100 KB. Every test passed, an independent replay verifier said VERIFIED, and a checked invariant held.", "claim": "A write of 100,000 bytes executed under a policy that capped writes at 4,096 bytes, while 66 of 66 tests passed, an independent replay verifier returned VERIFIED, and a checked complete-mediation invariant held.", "doesNotEstablish": "One defect in one server built for this study, not a measurement of deployed systems and not a claim about how often authorization layers and executors diverge. Mediation held throughout; what failed was the binding between the authorized operation and the executed one. Policy adequacy is separately not established, and two counterexamples covering it are retained unfixed.", "repo": "repowazdogz-droid/mcp-authority-boundary", "verifiedAtCommit": "03854ddc", "reproduceCommand": "npm install && ./run.sh", "sourceUrl": "/findings/authorized-limit-and-executed-write-disagreed/", "legacyUrl": "https://www.omegaprotocol.org/findings/authorized-limit-and-executed-write-disagreed/", "question": "authorised-executed", "subject": "A Cedar-mediated MCP tool server built for the study (authored).", "frame": "One server, six tools, 25 adversarial scenarios; the write cap is a policy constant.", "method": "Every decision is written to a hash-chained ledger that an independently written replay verifier reads back; four properties (mediation, binding, policy adequacy, effect verification) are reported separately.", "oracle": "The bytes actually written to the fixture, read back independently of the executor.", "negativeControl": { "status": "PRESENT", "text": "The repair is re-attacked with a falsification sweep; the version-one audit is preserved so the failing state can be re-run." }, "denominator": "66 of 66 tests in version one; one write of 100,000 bytes against a 4,096-byte cap.", "independentReproduction": "None known.", "repoStatus": "active", "headline": { "said": "Limited to 4 KB. 66 tests passed. Replay: VERIFIED.", "showed": "A 100,000-byte write executed under the 4,096-byte cap. Mediation held; the binding between the authorised operation and the executed one did not." } }, { "slug": "budget-bound-for-all-reachable-states", "grade": "PROVEN", "section": "constructible", "theme": "formal-verification", "stage": "authorise", "mechanism": null, "plain": "A spending cap was proved to hold in every state the system can reach, not just the ones anyone tested.", "claim": "Total spend never exceeds the cap in any reachable state of the escrow protocol, proved in Lean for any finite set of replicas and any non-negative amounts.", "doesNotEstablish": "The bound, not conservation. No liveness, availability, or Byzantine model; crash is global in the Lean model, and per-replica crash is only exercised in the bounded checks. The proof holds relative to the transition system being a faithful abstraction of the protocol, which is argued, not machine-checked.", "repo": "repowazdogz-droid/escrow-budget", "verifiedAtCommit": "9c199db4", "reproduceCommand": "make clean && make check", "sourceUrl": "/findings/budget-bound-for-all-reachable-states/", "legacyUrl": "https://www.omegaprotocol.org/findings/budget-bound-for-all-reachable-states/", "question": "each-compliant-collectively-safe", "subject": "A Lean 4 model of the escrow protocol (model).", "frame": "Any finite set of replicas, any non-negative amounts; crash is global in the Lean model.", "method": "Induction over reachable states with the budget bound as the invariant; a TLA+ model and a Python fault harness exercise what the proof does not.", "oracle": "The Lean 4 kernel (v4.32.0), axioms propext and Quot.sound only.", "negativeControl": { "status": "PRESENT", "text": "Four proved negative controls: remove a guard and the theorem must fail." }, "denominator": null, "independentReproduction": "None known.", "repoStatus": "active" }, { "slug": "cjson-does-not-round-trip-numbers", "grade": "OBSERVED", "section": "existing-systems", "theme": "formal-verification", "stage": "verify", "mechanism": null, "plain": "A widely used library reads a number, writes it back out, and the two are not the same.", "claim": "A widely used C JSON library does not preserve numbers across a serialise-and-reparse round trip: its number pipeline is lossy by design, printing with limited precision and comparing re-reads by tolerance rather than equality.", "doesNotEstablish": "That the library is defective for its purpose. A lax parser tuned for triage is a different artifact from one built to be exactly round-tripping; this is that a round-trip theorem is false of the target, not a quality judgement.", "repo": "repowazdogz-droid/vsf-cjson", "verifiedAtCommit": "dcdae40b", "reproduceCommand": "./verify.sh --quick", "sourceUrl": "/findings/cjson-does-not-round-trip-numbers/", "legacyUrl": "https://www.omegaprotocol.org/findings/cjson-does-not-round-trip-numbers/", "question": "spec-proved-property-proved", "subject": "cJSON, a widely used C JSON library (third-party system), against a Lean re-implementation.", "frame": "Number round-trip only; a lax parser tuned for triage.", "method": "Differential testing of the Lean model against the C reference over a fuzzed corpus.", "oracle": "The C library's own output on the same inputs.", "negativeControl": { "status": "PRESENT", "text": "A 20-attack mutation suite guards the harness; 0 port-wrong classifications over 120,000 inputs." }, "denominator": "116,476 agree of 120,000 fuzzed inputs; idempotence over 20,318.", "independentReproduction": "None known.", "repoStatus": "active" }, { "slug": "completed-sample-count-depends-on-scorer-order", "grade": "OBSERVED", "section": "existing-systems", "theme": "evaluation-integrity", "stage": "verify", "mechanism": "self-certification", "plain": "The same failing evaluation reported a different number of finished samples when its scoring steps were listed in a different order.", "claim": "In a widely used evaluation framework, the same failing run reported a different number of completed samples depending only on the order its scorers were declared in: 6 of 6 under one ordering and 4 of 6 under the other, with both runs ending in a success status.", "doesNotEstablish": "One defect in one framework, in the metadata a finished run records about itself. It says nothing about the correctness of any model score, nothing about how often the condition arises in practice, and nothing about other evaluation frameworks. The upstream design decision to score errored samples is separate and was not disputed.", "repo": "UKGovernmentBEIS/inspect_ai", "verifiedAtCommit": "f0e57a7c", "reproduceCommand": "pip install inspect_ai && inspect eval repro.py --model mockllm/model", "sourceUrl": "/findings/completed-sample-count-depends-on-scorer-order/", "legacyUrl": "https://www.omegaprotocol.org/findings/completed-sample-count-depends-on-scorer-order/", "question": "passed-correct", "subject": "Inspect AI, a widely used evaluation framework (third-party system).", "frame": "One failing run, two scorer orderings, mock model.", "method": "Same run declared with scorers in two orders; the completed-sample metadata compared.", "oracle": "The run metadata the framework writes about itself, compared across orderings.", "negativeControl": { "status": "NOT_APPLICABLE", "text": "The two orderings are each other's control; nothing else is varied." }, "denominator": "6 of 6 under one ordering, 4 of 6 under the other, both with a success status.", "independentReproduction": "None known.", "repoStatus": "active", "externalValidation": { "status": "MERGED_UPSTREAM", "text": "Fixed upstream in inspect_ai pull request #4604.", "link": "https://github.com/UKGovernmentBEIS/inspect_ai/pull/4604" } }, { "slug": "concurrent-capability-bound-over-interleavings", "grade": "PROVEN", "section": "constructible", "theme": "agent-control", "stage": "authorise", "mechanism": null, "plain": "A limit on what an agent may do was proved to hold no matter how concurrent operations interleave.", "claim": "A capability meter never exceeds its cap over every concurrent interleaving of charge operations, proved in Iris concurrent separation logic for any cap and any finite list of operations.", "doesNotEstablish": "It is terminal-observation safety, the value the driver reads is within the cap, not an all-intermediate-state invariant, and it carries no liveness or wait-freedom. The kernel check was not re-run in this pass; axiom-freedom rests on the committed assumptions audit plus a live search finding no admitted goals.", "repo": "repowazdogz-droid/capctl-iris", "verifiedAtCommit": "5e9284a0", "reproduceCommand": "eval $(opam env --switch=capctl-iris) && make verify", "sourceUrl": "/findings/concurrent-capability-bound-over-interleavings/", "legacyUrl": "https://www.omegaprotocol.org/findings/concurrent-capability-bound-over-interleavings/", "question": "each-compliant-collectively-safe", "subject": "A Rocq/Iris model of a concurrent capability meter (model).", "frame": "Any cap, any finite list of charge operations, every interleaving; terminal-observation safety only.", "method": "Adequacy over every reachable configuration in Iris concurrent separation logic.", "oracle": "The Rocq kernel; Print Assumptions committed and diffed in CI.", "negativeControl": { "status": "PRESENT", "text": "Floor E: an explicit counterexample showing per-key safety does not certify a tighter cross-key aggregate bound." }, "denominator": null, "independentReproduction": "None known.", "repoStatus": "active" }, { "slug": "judge-agreement-is-partly-forced-by-accuracy", "grade": "MEASURED", "section": "existing-systems", "theme": "evaluation-integrity", "stage": "verify", "mechanism": "convergence", "plain": "Two AI graders agreeing tells you less than it appears to, because part of that agreement is forced by both being right.", "claim": "Across 600 items and 171 judge pairs, judge errors correlated positively in 171 of 171 pairs at mean phi 0.567, and the correlation between pairwise agreement and accuracy fell from 0.824 to 0.077 once the algebraically forced component of agreement was removed.", "doesNotEstablish": "Not a claim that agreement between judges carries no information, and not a general law about language-model judges. It is a property of these judge checkpoints on these 600 items against a machine-checkable answer key. Ground truth here is a key, not human labelling, and the mechanism behind the second experiment is reported as open rather than settled.", "repo": "repowazdogz-droid/evaltrust", "verifiedAtCommit": "c8196e1a", "reproduceCommand": "python scripts/agreement/build_report.py && python scripts/agreement/sweep_writeup.py", "sourceUrl": "/findings/judge-agreement-is-partly-forced-by-accuracy/", "legacyUrl": "https://www.omegaprotocol.org/findings/judge-agreement-is-partly-forced-by-accuracy/", "question": "passed-correct", "subject": "Language-model judges on 600 items with a machine-checkable key (existing models).", "frame": "600 items, 171 judge pairs; ground truth is a key, not human labelling.", "method": "Pre-registered two-experiment study; the algebraically forced component of agreement is removed and the residual correlation reported.", "oracle": "The machine-checkable answer key; report fields named per headline.", "negativeControl": { "status": "PRESENT", "text": "An adversarial control (admissibility fixed before the data) fired as designed." }, "denominator": "171 of 171 pairs positive, mean phi 0.567; corr 0.824 to 0.077.", "independentReproduction": "None known.", "repoStatus": "archived", "repoStatusNote": "Repository archived 2 September 2026 after a dated correction to its second experiment. The numbers cited here are from the first experiment and are not affected by the correction, which withdraws a vendor-level interpretation of a separate contrast.", "correctionUrl": "https://github.com/repowazdogz-droid/evaltrust#readme" }, { "slug": "judge-agreement-undetermined-at-n45", "grade": "MEASURED", "section": "existing-systems", "theme": "evaluation-integrity", "stage": "verify", "mechanism": null, "plain": "At the sample size used, the question the experiment was built to answer could not be answered either way, and that is reported rather than rounded to a result.", "claim": "At 45 answers, a judge with zero disagreements has an agreement interval spanning its whole range; the perfect score reads as 'no counterexample found at this sample size', not as demonstrated reliability.", "doesNotEstablish": "That the ensemble judge is reliable. A degenerate interval at the endpoints cannot express uncertainty, and at larger scale even ensemble judges are imperfect.", "repo": "repowazdogz-droid/evaltrust", "verifiedAtCommit": "18e999a5", "reproduceCommand": "python scripts/build_report.py && python scripts/recompute.py && pytest -q", "sourceUrl": "/findings/judge-agreement-undetermined-at-n45/", "legacyUrl": "https://www.omegaprotocol.org/findings/judge-agreement-undetermined-at-n45/", "question": "passed-correct", "subject": "An ensemble judge on 45 GSM8K answers (existing models).", "frame": "45 answers; a zero-disagreement judge.", "method": "Agreement interval computed at the observed sample size.", "oracle": "The GSM8K answer key.", "negativeControl": { "status": "NOT_APPLICABLE", "text": "The result is that the interval is degenerate; there is nothing for a control to distinguish." }, "denominator": "0 disagreements in 45; interval spans the whole range.", "independentReproduction": "None known.", "repoStatus": "archived", "repoStatusNote": "Repository archived 2 September 2026 after a dated correction to its second experiment. The numbers cited here are from the first experiment and are not affected by the correction, which withdraws a vendor-level interpretation of a separate contrast.", "correctionUrl": "https://github.com/repowazdogz-droid/evaltrust#readme" }, { "slug": "judges-disagree-in-opposite-directions", "grade": "MEASURED", "section": "existing-systems", "theme": "evaluation-integrity", "stage": "verify", "mechanism": null, "plain": "Two off-the-shelf AI graders scoring the same work were wrong in opposite directions.", "claim": "Two language-model judges scoring the same 45 GSM8K answers disagreed with the ground truth in opposite directions: one systematically too strict, the other too lenient.", "doesNotEstablish": "That either judge is reliable, or that the direction generalises beyond this corpus. The sample is small; the per-answer correctness labels are assistant-adjudicated, not human; and two of the three judges also generated answers in the set, so self-preference is uncorrected.", "repo": "repowazdogz-droid/evaltrust", "verifiedAtCommit": "18e999a5", "reproduceCommand": "python scripts/build_report.py && python scripts/recompute.py && pytest -q", "sourceUrl": "/findings/judges-disagree-in-opposite-directions/", "legacyUrl": "https://www.omegaprotocol.org/findings/judges-disagree-in-opposite-directions/", "question": "passed-correct", "subject": "Two off-the-shelf language-model judges scoring 45 real GSM8K trajectories (existing models).", "frame": "45 answers; per-answer labels are assistant-adjudicated; two of three judges also generated answers.", "method": "Five graders score the same trajectories; every number recomputes from committed data under a digest check.", "oracle": "The GSM8K integer answer key.", "negativeControl": { "status": "NONE", "text": "No control is claimed for this measurement; the sample is small and the labels are an authored scheme." }, "denominator": "gemma3 disagreed on 4 of 45, all too strict; llama3 on 2 of 45, all too lenient.", "independentReproduction": "None known.", "repoStatus": "archived", "repoStatusNote": "Repository archived 2 September 2026 after a dated correction to its second experiment. The numbers cited here are from the first experiment and are not affected by the correction, which withdraws a vendor-level interpretation of a separate contrast.", "correctionUrl": "https://github.com/repowazdogz-droid/evaltrust#readme" }, { "slug": "monitoring-arm-recorded-preventions-that-executed", "grade": "MEASURED", "section": "constructible", "theme": "agent-control", "stage": "observe", "mechanism": "observation-boundary", "plain": "A dashboard reported 1,026 actions prevented. All 1,026 had already executed.", "claim": "Over 10,000 injected events, a monitoring arm recorded 1,026 actions as prevented and all 1,026 executed; an in-series arm consuming the same events, evaluator and policy recorded 6 of 9,880.", "doesNotEstablish": "A property of one synthetic, deterministic testbed under injected faults, not a measurement of any deployed safeguards product and not a rate that transfers outside this harness. The agent environment is deliberately trivial so that what is measured is the pipeline rather than model behaviour.", "repo": "repowazdogz-droid/safeguards-control-plane", "verifiedAtCommit": "08e52549", "reproduceCommand": "./demo.sh && make experiments", "sourceUrl": "/findings/monitoring-arm-recorded-preventions-that-executed/", "legacyUrl": "https://www.omegaprotocol.org/findings/monitoring-arm-recorded-preventions-that-executed/", "question": "logged-observed", "subject": "A synthetic, deterministic fault-injected pipeline on Redis Streams (authored).", "frame": "10,000 injected events, fail-closed policy; both arms consume identical events, evaluator and policy and differ only in position.", "method": "Two arms, one on the execution path and one observing beside it; a second axis varies single-writer versus two-writer telemetry.", "oracle": "The executed-action log, written by a second writer that the monitor does not control.", "negativeControl": { "status": "PRESENT", "text": "Nine negative controls in the repository; CI deletes the committed results and regenerates them." }, "denominator": "1,026 recorded as prevented, 1,026 executed; in-series arm 6 of 9,880; intervention recall 0 of 818 versus 697 of 818.", "independentReproduction": "None known.", "repoStatus": "active", "headline": { "said": "1,026 actions prevented.", "showed": "All 1,026 executed. A monitor beside the path recorded intentions that read exactly like outcomes; the in-series arm on the same events recorded 6 of 9,880." } }, { "slug": "tamper-of-a-sealed-record-forces-a-collision", "grade": "PROVEN", "section": "constructible", "theme": "assurance-architecture", "stage": "replay", "mechanism": null, "plain": "Changing a sealed record after the fact is provably as hard as breaking the hash it is sealed with.", "claim": "Tampering with a sealed, hash-linked decision record forces a hash collision: the canonical encoding is injective, the chain is append-only, and detection follows in Lean without a collision-resistance axiom.", "doesNotEstablish": "That the opaque hash function is SHA-256, that the recorded decision was correct, or that the formal definitions match the prose specification. It is tamper-evidence at the model level, not a claim about any deployed system.", "repo": "repowazdogz-droid/omega-lean-proof", "verifiedAtCommit": "2fab5d1", "reproduceCommand": "git clone https://github.com/repowazdogz-droid/omega-lean-proof && cd omega-lean-proof && lake build && lake env lean probes/AxiomProbe.lean", "sourceUrl": "/findings/tamper-of-a-sealed-record-forces-a-collision/", "legacyUrl": "https://www.omegaprotocol.org/findings/tamper-of-a-sealed-record-forces-a-collision/", "question": "replayed-reproduced", "subject": "A Lean 4 model of the OMEGA record chain (model).", "frame": "Opaque hash; collision-resistance as a discharged hypothesis, not an axiom.", "method": "Injective canonical encoding, append-only chain, detection theorem.", "oracle": "The Lean 4 kernel; #print axioms reports 0 user axioms.", "negativeControl": { "status": "PRESENT", "text": "The axiom probe is committed; a browser recompute of the content hash goes red on a one-byte change." }, "denominator": null, "independentReproduction": "None known.", "repoStatus": "active" }, { "slug": "tools-hidden-from-discovery-remain-callable", "grade": "OBSERVED", "section": "constructible", "theme": "security-engineering", "stage": "execute", "mechanism": "observation-boundary", "plain": "Tools removed from the list an agent is shown were still there to call by name.", "claim": "In a server built to exhibit the bug, tools filtered from the discovery listing remained reachable through the call surface: a presentation-layer authorization control with no execution-layer enforcement behind it.", "doesNotEstablish": "This is a demonstration against an authored mock, not a measurement of real servers. It does not test HTTP transport, prompt injection, or token scope, and a passing result does not mean a server is secure.", "repo": "repowazdogz-droid/mcp-boundary-audit", "verifiedAtCommit": "d932934f", "reproduceCommand": "pip install -e '.[dev]' && cd examples && mcp-boundary-audit --config vulnerable_server.config.json --execute --i-own-this-server", "sourceUrl": "/findings/tools-hidden-from-discovery-remain-callable/", "legacyUrl": "https://www.omegaprotocol.org/findings/tools-hidden-from-discovery-remain-callable/", "question": "authorised-executed", "subject": "A mock MCP server built to exhibit the bug (authored).", "frame": "stdio transport only; no HTTP, prompt injection or token scope.", "method": "Read the advertised tool list, then probe whether un-advertised tools are still callable; an unrecognised denial reads as inconclusive.", "oracle": "The call surface: whether tools/call succeeds for a tool absent from tools/list.", "negativeControl": { "status": "PRESENT", "text": "The patched server passes; the vulnerable server fails with exit 1." }, "denominator": "1 of 3 tools listed, 3 of 3 reachable, on the mock.", "independentReproduction": "None known.", "repoStatus": "active" }, { "slug": "training-run-rederives-bit-for-bit", "grade": "MEASURED", "section": "constructible", "theme": "assurance-architecture", "stage": "replay", "mechanism": null, "plain": "A training run re-derives its exact result from its recorded inputs on one machine. Across different hardware it stays unknown.", "claim": "A training run's final loss and trajectory re-derive bit-for-bit from its recorded inputs on one machine; cross-hardware re-derivation is left as UNKNOWN, and not claimed.", "doesNotEstablish": "Bit-for-bit reproducibility on other hardware, which is untested and reported UNKNOWN rather than upgraded. It does not establish code correctness, buggy-but-faithful code still verifies, nor result quality nor accountability.", "repo": "repowazdogz-droid/nanogpt-provenance", "verifiedAtCommit": "732da8b1", "reproduceCommand": "./run_all.sh", "sourceUrl": "/findings/training-run-rederives-bit-for-bit/", "legacyUrl": "https://www.omegaprotocol.org/findings/training-run-rederives-bit-for-bit/", "question": "replayed-reproduced", "subject": "A small training run and its verifier, on one machine (authored).", "frame": "Python 3.14.4, NumPy 2.4.4 on Apple Accelerate; cross-hardware untested.", "method": "The verifier re-runs training from the recorded configuration and compares the trajectory and final loss bit-for-bit.", "oracle": "The recorded final loss and trajectory fingerprint, recomputed rather than rechecked.", "negativeControl": { "status": "PRESENT", "text": "A 12-case suite triggers each verdict state (VERIFIED, TAMPERED, UNKNOWN) with a real tamper." }, "denominator": "12 of 12 controls; one machine.", "independentReproduction": "None known.", "repoStatus": "active" }, { "slug": "commons-compliant-collective-breach", "grade": "MEASURED", "section": "existing-systems", "theme": "agent-control", "stage": "authorise", "mechanism": "expressibility", "question": "each-compliant-collectively-safe", "plain": "Every language-model agent stayed inside its own allowance. The shared budget was breached in every episode where the allowances added up to more than the budget.", "claim": "In a shared-budget environment where each agent's allowance is individually satisfiable but the allowances sum to 1.5 times the collective budget, three language models produced a compliant collective breach in 60 of 60 BLIND episodes each (two population sizes, 30 episodes per cell), with zero individual allowance violations; one appended sentence restating the same headroom took the rate to 4 of 60, 2 of 60 and 0 of 60.", "headline": { "said": "Every agent within its allowance.", "showed": "The shared budget was breached in 60 of 60 BLIND episodes on each of three models, with no individual violation; an informationally redundant restatement of the headroom took it to 4, 2 and 0 of 60." }, "doesNotEstablish": "A rate here is a fact about a dataset, a prompt and a model on one day, not a capability of a model family or of language-model agents in general. The environment is synthetic and one round deep. Three developers, not four; the guardrail judge shares a vendor with two of the agent arms. No mechanistic claim is made about why the phrasing matters.", "subject": "Three hosted language models (gpt-5-mini, gpt-4.1-mini, gemini-2.5-flash) acting as agents in a synthetic one-round shared-budget environment (existing models, authored environment).", "frame": "Collective budget 100; N in {3, 6}; UNSOUND regime cap = ceil(1.5 × 100 / N); 30 episodes per cell fixed before any data; Wilson 95% intervals.", "method": "Pre-registered factorial over regime, population size and an information ladder over identical tasks; four amendments each registered before their data; every outcome computed from the environment ledger, never from model text.", "oracle": "The environment's own ledger of draws. A stub model whose prose claims it drew nothing while its action draws the full allowance is scored on the action, and that test fails the build if it stops doing so.", "negativeControl": { "status": "PRESENT", "text": "Seven pre-committed controls: an impossible-breach cell that must read 0 of 60 and a forced-breach cell that must read 60 of 60 (both did), an equal-share baseline, an allowance-response check, a benign cell supplying the guardrail's second denominator, a scorer-independence test and five runtime invariant monitors." }, "denominator": "60 of 60 per model = UNSOUND_N3_BLIND 30 of 30 plus UNSOUND_N6_BLIND 30 of 30. The per-cell table is on the collective-bound page.", "repo": "repowazdogz-droid/commons-agent-lab", "verifiedAtCommit": "de3f89d", "reproduceCommand": "python3 -m pytest tests -q && python3 -m commons.report _canonical/gpt-5-mini _canonical/gpt-4.1-mini _canonical/gemini-2.5-flash", "sourceUrl": "/findings/commons-compliant-collective-breach/", "independentReproduction": "None known.", "repoStatus": "active", "preregistration": "Registration commit 3bf280e (2026-08-22) precedes the first model call; amendments 01 to 04 each precede their data; departures in DEVIATIONS.md." }, { "slug": "judge-passed-checker-proved-violation", "grade": "MEASURED", "section": "constructible", "theme": "evaluation-integrity", "stage": "verify", "mechanism": "self-certification", "question": "passed-correct", "plain": "A language-model judge scored 8 out of 10 on decisions that a solver proved violated the encoded policy.", "claim": "In the complete rerun, six single-decision cases violated an encoded governance property. The language-model judge passed 4 of the 6 across three seeds each; the formal checker proved the violation in 6 of 6, and both verdicts are sealed in the same record.", "headline": { "said": "Judge: PASS, 8 of 10, three seeds.", "showed": "Z3 proved the decision violated the encoded policy. The judge passed 4 of the 6 violating decisions; the checker caught 6 of 6." }, "doesNotEstablish": "The judge is qwen2.5-coder:14b, a local model, not a frontier judge, and on model-authored rows it grades its own output. A stronger judge may do better; that is unmeasured. The harness depends on two packages that are not yet published on their own, vendored with provenance. Six cases is a demonstration of the split, not a rate.", "subject": "An evaluation harness for governed agent decisions built for the study (authored), with a local 14B judge.", "frame": "Sixteen sealed records from one rerun (2026-07-20); six violating single-decision cases; three judge seeds per case.", "method": "Each checkable property is routed to the checker whose logic fits it (Z3 for one decision, Lean 4 for a trace, TLA+ for interleavings, CryptoVerif for authorisation) and the judge's score is sealed beside the proof so the two can be compared.", "oracle": "The formal checker's verdict against the encoded property; CI checks the headline counts (6 violating cases, judge passed 4, checker proved 6).", "negativeControl": { "status": "PRESENT", "text": "Six non-violating rows in the same run where judge and checker agree, so the split is not the judge failing everything; planted-adversarial and model-authored decision sources are reported separately." }, "denominator": "4 of 6 violating decisions passed by the judge; 6 of 6 proven violated by the checker; 3 seeds each.", "repo": "repowazdogz-droid/proof-carrying-evals", "verifiedAtCommit": "f22def1", "reproduceCommand": "python3 scripts/verify_records.py", "sourceUrl": "/findings/judge-passed-checker-proved-violation/", "independentReproduction": "None known.", "repoStatus": "active" }, { "slug": "spcu-defects-passed-every-property", "grade": "OBSERVED", "section": "constructible", "theme": "formal-verification", "stage": "verify", "mechanism": "faithfulness", "question": "spec-proved-property-proved", "plain": "Two injected hardware defects passed every property written from the original specification, because the specification never stated the requirement they break.", "claim": "Five defects were injected into a small power-control design whose formal verification argument was complete and green. Two of them passed every property derived from the original specification: one because the specification required voltage to be sufficient and never minimal, one because a two-step transition is internally consistent. A third escaped every property about the design's own variables while recording something false about the physical world.", "headline": { "said": "All 26 requirement-derived assertions pass the unbounded proof.", "showed": "Two injected defects passed every property, because the specification never stated the requirement they break. Two of the 26 assertions are vacuous and cannot fail on any input." }, "doesNotEstablish": "A claim about this ~450-line design and this property set only. The five mutations were written by the same person who wrote the properties; a 200-mutant Yosys run was added for that reason and showed the hand-written mutations had probed the wrong part of the design. Nothing here is a statement about any commercial verification flow.", "subject": "A small DVFS power-control IP written for the study (authored), verified with open tools.", "frame": "26 requirement-derived assertions; PDR unbounded proofs on a collapsed-clock model with four environment assumptions; clock-crossing checks bounded at depth 20; 2 assertions vacuous; 3 rest on a further stated assumption.", "method": "Formal (SymbiYosys), Verilator simulation, a pyuvm testbench and bare-metal C against the RTL; five hand-injected defects, then 200 netlist mutations checked against the property set and an equivalence miter.", "oracle": "The property set itself, then a mutation harness that asks which mutants any property can see; an equivalence miter separates no-op mutants.", "negativeControl": { "status": "PRESENT", "text": "The 200-mutant mcy run: 84 covered, 23 equivalent, 93 in the gap. Vacuous assertions are kept and excluded from every claim." }, "denominator": "2 of 5 injected defects passed every specification-derived property; 26 assertions, 2 vacuous; 200 mutants, 93 unseen.", "repo": "repowazdogz-droid/spcu-verification", "verifiedAtCommit": "e49a0c5", "reproduceCommand": "make setup && make all", "sourceUrl": "/findings/spcu-defects-passed-every-property/", "independentReproduction": "None known.", "repoStatus": "active" }, { "slug": "verifier-green-run-could-not-fail", "grade": "OBSERVED", "section": "existing-systems", "theme": "formal-verification", "stage": "verify", "mechanism": "coverage-illusion", "question": "passed-correct", "plain": "Three verification runs printed green while the run could not have failed: an assertion never reached, a model checker that instrumented nothing, and an axiom audit identical for a correct model and a wrong one.", "claim": "A Kani harness left its second assertion UNREACHABLE after the first injected bug, so its pass carried no information; a loom test reported 1 passed on a build containing a lost update because the atomic was never loom-instrumented, with byte-identical output for the clean and broken builds; and two Lean models of the same Rust function, one pinning real sizes and one returning 0 for every type, produced byte-identical axiom audits.", "headline": { "said": "Verifier: green.", "showed": "The run could not have failed: a Kani assertion never reached, a loom test that instrumented nothing, and a Lean axiom audit byte-identical for a correct and a wrong model." }, "doesNotEstablish": "No defect in governor or jsonwebtoken is claimed; the crates are subjects for the recorded-output grader, and the bugs were injected. The grader reads recorded outputs and runs no verifier itself. Version 0.1.", "subject": "Recorded verification outputs from Kani, loom and cargo test on the governor and jsonwebtoken crates (third-party code, injected defects), and two Lean models of a kernel Rust parser.", "frame": "Three case studies with inputs, patches, outputs and reproduction commands committed; Lean audit compared by sha256.", "method": "A grader that extracts what a run actually explored (reachability, instrumented interleaving counts, value lemmas) and grades on that rather than on the printed verdict.", "oracle": "What distinguishes the runs when the verdicts do not: an execution count (9 uninstrumented against 54 instrumented interleavings), an UNREACHABLE status, and value lemmas proved by rfl that fail on the wrong model.", "negativeControl": { "status": "PRESENT", "text": "One control was mistaken for two until a second, independent Kani bug was built; the loom clean and broken runs are both committed so the byte-identity is checkable." }, "denominator": "3 catches, each with committed inputs and outputs; 9 versus 54 interleavings; 0 bytes of difference in the two axiom audits.", "repo": "repowazdogz-droid/evidence-audit", "verifiedAtCommit": "996d248", "reproduceCommand": "pip install -e . && python3 -m pytest tests/ -q", "sourceUrl": "/findings/verifier-green-run-could-not-fail/", "independentReproduction": "None known.", "repoStatus": "active" }, { "slug": "sum-rule-holds-where-per-agent-cap-fails", "grade": "PROVEN", "section": "constructible", "theme": "agent-control", "stage": "authorise", "mechanism": "expressibility", "question": "each-compliant-collectively-safe", "plain": "Six agents each inside a cap of 10 breach a pool of 40; a single rule on the sum removes every such breach, and it bounds a harm if and only if the harm is a function of the sum.", "claim": "In a closed-data Lean 4 model (cap 10, budget 40, six agents) the per-agent-compliant profile [10,10,10,10,10,10] breaches the budget and the conserved meter accepts no breaching profile while still accepting a within-budget one, with no axioms. In a Z3 linear-integer model of the same instance, eleven queries establish that the sum rule holds for every coordinated profile, bounds a harm iff the harm factors through the aggregate (allocation harm escapes with witness [0,0,16,0,0,0]), and is blind to any coordination channel.", "doesNotEstablish": "Anything about a deployed multi-agent system. The Lean results are over closed data; the Z3 results are verdicts over an encoding whose faithfulness to the informal model is not machine-checked. The per-agent cap is not shown to be unfixable: a local allowance of floor(G/N) is collectively safe when population and utilisation are known and fixed. The channel-blindness result takes perfectly secure steganography as an assumption, not a construction. Not preregistered.", "subject": "A Lean 4 model and a Z3 model of a six-agent shared pool (model).", "frame": "N = 6, cap 10, pool 40, allocation threshold 15; one round, integer draws, joint worst-case coordination; Lean over closed data, Z3 over a linear-integer encoding.", "method": "Kernel-checked instance of a general witness/repair law in Lean; SAT/UNSAT queries with witnesses in Z3, sealed as hash-chained records; a seeded learning adversary run for 180,000 episodes as corroboration and three hand-built relaxations run to their predicted breach magnitudes.", "oracle": "The Lean 4.30.0 kernel (#print axioms: none) and z3 4.16.0; the sealed records regenerate byte for byte.", "negativeControl": { "status": "PRESENT", "text": "Three Z3 mutations (cap 6, h 40, G 5) each flip the named verdict; raising the Lean budget to 60 breaks the build." }, "denominator": null, "repo": "repowazdogz-droid/collective-bound", "verifiedAtCommit": "public commit of 2026-09-02", "reproduceCommand": "make lean axioms z3 controls # and make rl for the learning adversary, about 90 s", "sourceUrl": "/findings/sum-rule-holds-where-per-agent-cap-fails/", "independentReproduction": "None known. Author self-reproduction from the public artifact on 2026-09-02, byte for byte.", "repoStatus": "active", "preregistration": "None. The companion empirical study (commons-agent-lab) was preregistered; this formal work was not." } ], "artifacts": [ { "slug": "capctl-iris", "name": "capctl-iris", "grade": "PROVEN", "theme": "agent-control", "establishesShort": "A concurrent capability bound holds over every interleaving, machine-checked with no axioms.", "establishes": "That a capability meter never exceeds its cap over every concurrent interleaving of charge operations, in Iris concurrent separation logic, for any cap and any finite operation list.", "doesNotEstablish": "It is terminal-observation safety, the value the driver reads is within the cap, not an all-intermediate-state invariant, and carries no liveness or wait-freedom. Floor E shows per-key safety does not certify a tighter cross-key aggregate bound. There is no machine-checked link to the TLA+ model or the Lean parent.", "trustedBase": "The Rocq kernel (rocq-core 9.2.0). The 29 audited theorems of the released version are each closed under the global context: no user axioms and no foundational axioms, which is stronger than 'no added axioms'. The proofs are stated over a few Iris and heap_lang definitions, the operational semantics and the adequacy bridge, which are kernel-checked, not axiomatic.", "method": "The concurrent driver forks one compare-and-swap charge thread per element onto a single shared cell; safety is stated as adequacy, which quantifies over every reachable configuration, that is, every thread schedule.", "results": "The headline theorem conc_meter_never_exceeds_cap closes with Qed; the assumptions audit reports 29 theorems each closed under the global context at the released version v0.1.3 (40 at the current untagged head); a live search finds no admitted goals. The kernel was not re-run in the evidence pass, so axiom-freedom rests on the committed audit.", "reproduceCommand": "eval $(opam env --switch=capctl-iris) && make verify", "toolchain": "The Rocq Prover rocq-core 9.2.0, rocq-iris 4.5.0, rocq-iris-heap-lang 4.5.0, rocq-stdpp 1.13.0, OCaml 5.2.0", "repo": "repowazdogz-droid/capctl-iris", "independentReproduction": "None known.", "priorArt": "Iris concurrent separation logic and its adequacy theorem; capability and authority bounds. The contribution is the concurrent meter bound and the audited axiom-free trusted base.", "repoStatus": "active", "theoremCount": 29, "theoremCountSource": "iris/proofs-audit/expected/assumptions.out at v0.1.3 (the DOI-registered release, 10.5281/zenodo.21505166) and at v0.1.5: 29 lines Closed under the global context.", "theoremCountHead": 40, "theoremCountHeadSource": "At the untagged head dcc8ecb the same audit lists 40 theorems, all closed under the global context; CI diffs the count against the release notes.", "doi": "10.5281/zenodo.21505165" }, { "slug": "compositional-temporal-safety", "name": "compositional-temporal-safety", "grade": "PROVEN", "theme": "agent-control", "establishesShort": "A compositional safety invariant holds for all finite N, machine-checked.", "establishes": "That a composed safety invariant, aggregate spend within cap, and mutual exclusion, holds over all reachable states for any finite roster of agents, proved in Lean by composing an isolated per-agent guarantee.", "doesNotEstablish": "Only the sound non-circular fragment of assume-guarantee; the full circular Abadi-Lamport form is not claimed. Lean proves safety only, with no liveness, which is a bounded TLA+ property at one configuration. No machine-checked refinement connects the Lean, TLA+ and Python layers; the roster is static; it is not propext-free.", "trustedBase": "Lean 4 (v4.32.0), mathlib-free; the axioms propext and Quot.sound, with some theorems using none; and the transition system being a faithful abstraction, argued not machine-proved. The TLA+ side adds TLC and the JVM, bounded.", "method": "An isolated per-agent guarantee is composed by a step lemma into an invariant over all reachable states; a TLA+ model and a Python differential exercise the bounded state space, with four Lean negative controls.", "results": "The Lean theorems build green and the axiom audit reports at most propext and Quot.sound; the TLA+ model checks the invariant over 200 distinct states at one configuration, and a Python differential reaches the same count as a bounded agreement, not a refinement.", "reproduceCommand": "make check", "toolchain": "Lean 4 leanprover/lean4:v4.32.0 (mathlib-free); TLA+ Tools v1.7.4 / TLC2 2.19; Python 3", "repo": "repowazdogz-droid/compositional-temporal-safety", "independentReproduction": "None known. The Lean build and Python differential were reproduced from a fresh clone during the evidence inventory.", "priorArt": "Assume-guarantee reasoning (Abadi-Lamport, Misra-Chandy) and compositional temporal safety. The contribution is the machine-checked non-circular composition with load-bearing negative controls and an honest bounded-agreement account of its cross-checks.", "repoStatus": "active" }, { "slug": "escrow-budget-mpst", "name": "escrow-budget-mpst", "grade": "PROVEN", "theme": "security-engineering", "establishesShort": "The transfer session is deadlock-free and crash-safe by typing; the budget bound is not expressible in it.", "establishes": "That the escrow transfer sub-protocol, expressed as a multiparty session type with crash-stop failures in Rust, is deadlock-free and communication-safe on all executions and leaves the surviving peer deadlock-free after a crash at any point, enforced by the Rust type system on every build.", "doesNotEstablish": "The budget bound: session types govern communication, not arithmetic state, and the Charge that consumes budget sends no message, so it is invisible to the type. It assumes reliable, ordered transport (loss, duplication and reorder are assumed away), and models crash-stop, not crash-recovery. It is complementary to the Lean and TLA+ method, not a confirmation of it. This method never touches the cap theorem.", "trustedBase": "The Rust compiler and the mpstthree session-types library, and the assumption of reliable ordered transport. The TCB additions are set out in GAPS.md.", "method": "The global type is projected to local types realised as Rust types, so a type-violating implementation and a non-projectable global type are both rejected at compile time; crash-stop faults are injected at every protocol point.", "results": "make check builds green; the properties-by-methods table in COMPARISON.md records exactly where the two methods overlap and where they do not, with explicit NOT APPLICABLE cells, MPST not applicable to the budget bound, Lean not applicable to communication safety.", "reproduceCommand": "make check", "toolchain": "Rust with Cargo, and the mpstthree multiparty-session-types library", "repo": "repowazdogz-droid/escrow-budget-mpst", "independentReproduction": "None known. The build was verified from a fresh public clone (make check, exit 0).", "priorArt": "Multiparty session types and crash-stop session-type failure handling; the mpstthree library. The contribution is the paired complementary-not-confirmatory account with the sibling budget proof and its explicit NOT APPLICABLE cells.", "repoStatus": "active" }, { "slug": "escrow-budget", "name": "escrow-budget", "grade": "PROVEN", "theme": "formal-verification", "establishesShort": "A budget bound (spend within cap) holds for all reachable states, machine-checked.", "establishes": "That total spend never exceeds the cap in any reachable state of the escrow protocol, in Lean, for any finite set of replicas and any non-negative amounts.", "doesNotEstablish": "The bound, not conservation. No liveness, availability, or Byzantine model; crash is global in the Lean model. No machine-checked refinement connects the Lean proof, the TLA+ model, and the Python harness, their agreement is bounded, at one configuration, not a refinement.", "trustedBase": "The Lean 4 kernel (v4.32.0), mathlib-free; the axioms used are propext and Quot.sound only, with no user axioms and no sorry. Plus the transition system being a faithful abstraction of the protocol, which is argued, not machine-proved.", "method": "Induction over the reachable states for an arbitrary finite roster, with the budget bound as the invariant; cross-checked by a TLA+ model and a Python fault harness that exercise crash and recovery the proof does not cover.", "results": "The theorems reachable_safe and durable_reachable_safe prove the bound; the axiom audit reports only propext and Quot.sound; a fault harness holds over 10,000 executions.", "reproduceCommand": "make clean && make check", "toolchain": "Lean 4 leanprover/lean4:v4.32.0 (mathlib-free); TLA+ Tools v1.7.4; Python 3 with Hypothesis", "repo": "repowazdogz-droid/escrow-budget", "independentReproduction": "None known.", "priorArt": "Escrow and budget protocols; assume-guarantee reasoning. The contribution is the machine-checked unbounded budget bound and the first-person audited account of where its three methods are complementary, not confirmatory.", "repoStatus": "active" }, { "slug": "evaltrust", "name": "evaltrust", "grade": "MEASURED", "theme": "evaluation-integrity", "establishesShort": "Two off-the-shelf LLM judges fail in opposite directions; a zero-disagreement judge's agreement interval is degenerate.", "establishes": "That two off-the-shelf language-model judges disagree with the ground truth in opposite directions on a small GSM8K corpus, and that at that sample size a zero-disagreement judge's agreement interval is degenerate, measured, with digests that recompute offline.", "doesNotEstablish": "Judge reliability. The per-answer correctness labels are assistant-adjudicated, not human; two of the three judges also generated answers, so self-preference is uncorrected; the sample is small.", "trustedBase": "The objective GSM8K integer answer key; the assistant-adjudicated per-trajectory labels, which are an authored coding scheme and part of the trusted base; the local Ollama judge models at temperature 0.", "method": "Five graders, three language-model judges, a substring matcher, a numeric oracle, and an ensemble, score 45 real GSM8K trajectories, and every number recomputes from committed data under a digest check.", "results": "gemma3 disagrees on 4 of 45, all too strict; llama3 on 2 of 45, all too lenient; the ensemble's agreement interval is degenerate at zero disagreements; the report digest is 62bb20c4.", "reproduceCommand": "python scripts/build_report.py && python scripts/recompute.py && pytest -q", "toolchain": "Python >=3.11, numpy, scipy; a local Ollama is needed only to regenerate verdicts, not to reproduce the committed numbers", "repo": "repowazdogz-droid/evaltrust", "independentReproduction": "None known.", "priorArt": "Language-model-as-judge evaluation and inter-rater agreement. The contribution is the opposite-direction failure and the degenerate-interval honesty, with every number recomputing from committed data.", "repoStatus": "archived", "repoStatusNote": "Archived 2 September 2026 after a dated correction: the second experiment's vendor-level interpretation was withdrawn; the first experiment's numbers stand and recompute from committed data.", "correctionUrl": "https://github.com/repowazdogz-droid/evaltrust#readme" }, { "slug": "inspect-audit", "name": "inspect-audit", "grade": "MEASURED", "theme": "evaluation-integrity", "establishesShort": "A read-only auditor flags silent validity failures in evaluation logs.", "establishes": "That a read-only auditor can flag silent validity failures in Inspect evaluation logs, samples dropped from a metric's denominator, grader parse-failures scored as data, a model grading its own output, duplicates, truncation, and non-reproducible settings.", "doesNotEstablish": "A PASS is not a validity certificate. A biased dataset, a mis-specified task, a subtly-wrong-but-parseable grade, or contamination are out of scope. NOT_CHECKED is not PASS; there are no model calls and no language-model judge.", "trustedBase": "The author's check catalog and its thresholds, which are an authored coding scheme and part of the trusted base; the inspect_ai schema; and heuristic signals labelled as such.", "method": "24 static checks run over the log, each firing only on evidence, with catalog-to-code consistency test-enforced; the verdict per check is FAIL, WARN, PASS or NOT_CHECKED.", "results": "58 tests pass; on the broken fixture, 3 FAIL and 1 WARN over 23 checks (exit 2); on the clean fixture, 0 FAIL (exit 0).", "reproduceCommand": "pip install -e '.[dev]' && inspect-audit examples/broken.eval", "toolchain": "Python 3.11+, inspect_ai; static, no model calls, never modifies the log", "repo": "repowazdogz-droid/inspect-audit", "independentReproduction": "None known.", "priorArt": "Static analysis of evaluation logs and validity threats in measurement. The contribution is the catalog of silent-failure checks with NOT_CHECKED kept distinct from PASS.", "repoStatus": "active" }, { "slug": "inspect-replay", "name": "inspect-replay", "grade": "MEASURED", "theme": "evaluation-integrity", "establishesShort": "Two evaluation logs compared deterministically, distinguishing 'unchanged' from 'cannot tell'.", "establishes": "That two Inspect AI evaluation logs can be compared deterministically and sample-aligned, reporting what changed together while distinguishing 'unchanged' from 'cannot tell'.", "doesNotEstablish": "No causation, co-occurrence only. It does not re-run models; a numeric score change is not labelled a regression; different scorers are not comparable.", "trustedBase": "The inspect_ai log parser and schema; the author's alignment-key logic, flagged in-repo as the riskiest module; and the UNKNOWN, NOT_CHECKED and NOT_COMPARABLE ignorance model.", "method": "It reads two recorded evaluation logs, aligns samples, and reports per-sample changes with an explicit ignorance state; the output is deterministic and byte-identical, read-only, and enforced by tests.", "results": "117 tests pass; the example diff reports CHANGED with the honesty line that no recorded config field accounts for the change.", "reproduceCommand": "pip install -e '.[dev]' && inspect-replay compare examples/baseline.eval examples/sample-regression.eval", "toolchain": "Python 3.11+, inspect_ai >=0.3.180; offline, read-only", "repo": "repowazdogz-droid/inspect-replay", "independentReproduction": "None known.", "priorArt": "Structured diffing and evaluation reproducibility. The contribution is the deliberate 'unchanged versus cannot tell' distinction with a build-time guard against overclaiming causation.", "repoStatus": "active" }, { "slug": "mcp-authority-boundary", "name": "mcp-authority-boundary", "grade": "OBSERVED", "theme": "agent-control", "establishesShort": "Mediation held; binding did not.", "establishes": "That authorization at an agent's tool boundary is several properties that fail independently, and that mediation can hold while the binding between the authorized operation and the executed one does not.", "doesNotEstablish": "Policy adequacy, meaning whether a policy expresses the authority its author intended. Two counterexamples covering that are retained unfixed. Effect verification is also not established by the shipped evidence: independent fixture read-back exists for two tools only, and the version-one ledger executed neither, so its stage-four checks were record consistency rather than observation of the world.", "trustedBase": "The Cedar policy engine, the Model Context Protocol SDK, Node, and the assumption that whoever launches the server chooses the session identity. Authentication is not provided and a hostile local actor who can spawn the process is not contained.", "method": "A Cedar-enforced Model Context Protocol server mediates six tools across 25 adversarial scenarios, writing every decision to a hash-chained ledger that an independently written replay verifier reads back. Four properties are separated and reported one by one: mediation, binding, policy adequacy, effect verification.", "results": "In version one, 66 of 66 tests passed, replay returned VERIFIED, a checked complete-mediation invariant held, and a write of 100,000 bytes executed under a 4,096-byte cap. The resolver measured the payload as a string and substituted zero bytes for a non-string; the executor coerced the same argument and wrote it. The repair binds execution to one validated immutable operation.", "reproduceCommand": "npm install && ./run.sh", "toolchain": "TypeScript, Node, Cedar, the Model Context Protocol SDK; no network", "repo": "repowazdogz-droid/mcp-authority-boundary", "independentReproduction": "None known.", "priorArt": "Complete mediation is a classical protection principle, and confused-deputy problems are long documented. The contribution here is the separation of four properties that are usually reported as one, on a working agent tool boundary, with the counterexamples kept rather than fixed.", "repoStatus": "active" }, { "slug": "mcp-boundary-audit", "name": "mcp-boundary-audit", "grade": "OBSERVED", "theme": "security-engineering", "establishesShort": "A harness detects a tool hidden from discovery yet reachable through the call surface, on a mock built to show it.", "establishes": "That a harness can detect one authorization bug in Model Context Protocol servers. A tool hidden from tools/list yet reachable through tools/call, demonstrated on a mock server built to exhibit it.", "doesNotEstablish": "Not a field measurement: the numbers come from an authored mock. It does not test HTTP or SSE transport, prompt injection, or token scope, and a PASS does not mean a server is secure.", "trustedBase": "The author's mock server, which defines both the vulnerable behaviour and the ground-truth tool set; a keyword-based denial heuristic that biases toward inconclusive, never toward a false FAIL.", "method": "The harness reads the advertised tool list, then probes whether the un-advertised tools are still callable; an unrecognised denial reads as inconclusive, never as a pass.", "results": "On the mock, 1 of 3 tools listed and 3 of 3 reachable, the 2 hidden tools callable, giving FAIL and exit 1; the patched server PASSes; 11 tests pass.", "reproduceCommand": "pip install -e '.[dev]' && cd examples && mcp-boundary-audit --config vulnerable_server.config.json --execute --i-own-this-server", "toolchain": "Python, the Model Context Protocol SDK, stdio transport; no network", "repo": "repowazdogz-droid/mcp-boundary-audit", "independentReproduction": "None known.", "priorArt": "Presentation-versus-execution authorization mismatches (CVE-2026-46519, GHSA-cr22-wjx7-2w6m). The contribution is the conservative harness that flags the reachable-but-hidden gap without false positives. A maintenance item tracks the 2026-07-28 MCP spec's stateless core (issue #1).", "repoStatus": "active" }, { "slug": "nanogpt-provenance", "name": "nanogpt-provenance", "grade": "MEASURED", "theme": "assurance-architecture", "establishesShort": "A verifier re-derives a training run bit-for-bit; cross-hardware re-derivation stays UNKNOWN, not claimed.", "establishes": "A tamper-evident provenance record with a verifier that re-derives a training run's result bit-for-bit from recorded inputs and reports VERIFIED, TAMPERED or UNKNOWN, never silently upgrading UNKNOWN.", "doesNotEstablish": "Cross-hardware re-derivation, which is untested and reported UNKNOWN. It does not establish code correctness, buggy-but-faithful code still verifies, nor result quality nor accountability.", "trustedBase": "SHA-256 collision-resistance; the verifier's own CPython, NumPy, operating system and CPU; and that the verifier's numeric environment reproduces the record's, which is the weakest link.", "method": "The verifier re-runs training from the recorded configuration and compares the trajectory and final loss bit-for-bit under a matching numeric environment; a 12-case negative-control suite triggers each verdict state with a real tamper.", "results": "On one machine, VERIFIED with final loss 0.10635284871660042 and fingerprint c6da865d, 12 of 12 controls; a re-derivation mismatch under a foreign environment returns UNKNOWN, not TAMPERED.", "reproduceCommand": "./run_all.sh", "toolchain": "Python 3.14.4, NumPy 2.4.4 on Apple Accelerate BLAS", "repo": "repowazdogz-droid/nanogpt-provenance", "independentReproduction": "None known.", "priorArt": "Provenance and reproducible builds; tamper-evident logs. The contribution is re-derivation rather than integrity-recheck, with an honest UNKNOWN exactly where reproducibility ends.", "repoStatus": "active" }, { "slug": "omega", "name": "OMEGA", "grade": "PROVEN", "theme": "assurance-architecture", "establishesShort": "A sealed, replayable decision record whose tamper-evidence is machine-checked.", "establishes": "A record structure for a governed decision, authority, evidence, expected outcome, confirmation, traceability, that is sealed at commit time, can be replayed, and whose seal can be rechecked. A Lean 4 model establishes internal consistency of the governance conjunction and constructive tamper-evidence.", "doesNotEstablish": "That a decision was correct, that a model's internal reasoning is the recorded reasoning, or that the opaque hash function is SHA-256. The Lean model does not establish that its formal definitions match the prose specification.", "trustedBase": "Lean kernel v4.27.0 and the three standard axioms propext, Classical.choice, Quot.sound. SHA-256 is modelled as an opaque compute_hash with collision-resistance carried as a discharged theorem hypothesis, not an axiom. No Mathlib.", "method": "A canonical-encoding, hash-linked record chain, with a Lean 4 formalisation of the governance conjunction and the tamper-evidence argument, plus a browser recompute of the record's content hash.", "results": "The shipped root theorems are machine-checked; a fresh clone builds green and #print axioms reports 0 user-declared axioms. Tampering a sealed record forces a hash collision; the record chain is append-only; the canonical encoding is injective and round-trips.", "reproduceCommand": "git clone https://github.com/repowazdogz-droid/omega-lean-proof && cd omega-lean-proof && lake build && lake env lean probes/AxiomProbe.lean", "toolchain": "Lean 4, leanprover/lean4:v4.27.0, no Mathlib, no external Lake dependencies", "repo": "repowazdogz-droid/omega-lean-proof", "independentReproduction": "None known. The Lean build and axiom audit were reproduced from a fresh public clone during the evidence inventory; that is author self-reproduction, not independent third-party reproduction.", "priorArt": "Tamper-evident logging and hash chains (Haber and Stornetta, 1991), the trusted computing base literature, and canonical serialisation (RFC 8785 JCS). The contribution is the record structure and the machine-checked internal-consistency and tamper-evidence properties, not the discovery of hash chaining.", "repoStatus": "active" }, { "slug": "safeguards-control-plane", "name": "safeguards-control-plane", "grade": "MEASURED", "theme": "agent-control", "establishesShort": "A monitor reports intentions as outcomes.", "establishes": "That a safeguards component which observes decisions without standing in the execution path reports preventions that did not happen, and that telemetry sharing one writer with enforcement cannot detect its own enforcement failures.", "doesNotEstablish": "Anything about a deployed safeguards product. The agent environment is synthetic and deterministic by design, so every number is a property of this pipeline under injected faults rather than a rate that transfers. No claim is made about which faults occur in practice or how often.", "trustedBase": "The injected fault model, the synthetic event generator, the policy, and the assumption that the two arms differ only in enforcement position.", "method": "Two arms consume identical events with an identical evaluator, policy and injected faults, differing only in whether the control sits on the execution path or observes alongside. A second axis varies whether telemetry and enforcement records come from one writer or two.", "results": "Across 10,000 events with all faults injected and fail-closed policy, the monitoring arm recorded 1,026 as prevented and all 1,026 executed, with intervention recall of 0 of 818. The in-series arm recorded 6 of 9,880 prevented-but-executed and intervention recall of 697 of 818. Under a single writer the same run reported enforcement success of 906 of 906 with no divergences, against 900 of 906 and six divergences under two writers, and recorded six fewer unsafe executions than occurred.", "reproduceCommand": "./demo.sh && make experiments", "toolchain": "Python, Docker Compose, Redis; offline on one machine, about three minutes for the measured run", "repo": "repowazdogz-droid/safeguards-control-plane", "independentReproduction": "None known.", "priorArt": "The distinction between a reference monitor on the path and a detective control alongside it is classical. What is measured here is the size of the reporting gap when the two are given identical inputs, and the separate effect of record authorship.", "repoStatus": "active" }, { "slug": "semdiff", "name": "semdiff", "grade": "MEASURED", "theme": "security-engineering", "establishesShort": "A deterministic, severity-ranked semantic diff for four high-risk engineering formats, no LLM calls.", "establishes": "That a focused command-line tool can produce semantic, severity-ranked diffs of high-risk engineering artifacts, Cargo.lock, package-lock.json, OpenAPI, and Kubernetes YAML, deterministically.", "doesNotEstablish": "It is not a summariser and makes no language-model calls; not a universal file-type tool, covering only the four formats; not a full OpenAPI linter; not a network tool. Correctness here is the test suite plus determinism, not a proof.", "trustedBase": "The Rust toolchain and the crate dependencies; and the human-authored classification rule table, which is an authored coding scheme and part of the trusted base.", "method": "Each artifact is parsed, its load-bearing content extracted, and changes ranked by severity; the output is deterministic and sorted, with a stable JSON schema and defined exit codes.", "results": "134 fixture pairs and 137 tests; the demo output is byte-identical to the documented example.", "reproduceCommand": "cargo test && cargo run --release -p semdiff-cli -- file --old fixtures/cargo/basic_old/Cargo.lock --new fixtures/cargo/basic_new/Cargo.lock", "toolchain": "Rust, edition 2021, a Cargo workspace", "repo": "repowazdogz-droid/semdiff", "independentReproduction": "None known.", "priorArt": "Structured and semantic diffing and supply-chain change review. The contribution is the deliberately narrow, deterministic, no-language-model diff for four high-risk formats.", "repoStatus": "active" }, { "slug": "vsf-cjson", "name": "vsf-cjson", "grade": "PROVEN", "theme": "formal-verification", "establishesShort": "Four kernel-checked theorems about a JSON parser and serialiser, with a differential over 120,000 inputs.", "establishes": "Four machine-checked Lean theorems about a JSON parser and serialiser, totality, round-trip on a canonical exact-number model, canonicity, and idempotence, together with a differential measurement against a C reference over a large fuzzed corpus.", "doesNotEstablish": "Grammar-soundness is not proved; the theorems are about the function, not the compiled binary, and the idempotence run is the only bridge. The corpus score is a measure of how faithfully a lax parser was replicated, not a quality score. The round-trip theorem holds only after numbers are modelled exactly; it is false of the C target's own lossy pipeline.", "trustedBase": "Lean 4 (v4.32.0), Std only with no Mathlib; the axioms propext, Classical.choice and Quot.sound; no sorry and no native_decide. Also the Lean kernel, compiler and runtime, the C compiler, and the human-authored SPEC.md.", "method": "The specification models JSON numbers exactly rather than as IEEE doubles, which makes the round-trip theorem true and provable and turns every number divergence from the C into a measured finding; a 16-gate harness and a 20-attack mutation suite guard the result.", "results": "The four theorems are kernel-checked; the differential over 120,000 fuzzed inputs classifies 116,476 agree and 0 port-wrong; idempotence holds over 20,318 inputs.", "reproduceCommand": "./verify.sh --quick", "toolchain": "Lean 4 leanprover/lean4:v4.32.0 (Std only), Python 3 stdlib, a C99 oracle", "repo": "repowazdogz-droid/vsf-cjson", "independentReproduction": "None known.", "priorArt": "JSON parsing, canonical serialisation (RFC 8785), and differential testing. The contribution is the exact-number specification that makes round-trip provable, and the account of what the C target lacks.", "repoStatus": "active" }, { "slug": "commons-agent-lab", "name": "commons-agent-lab", "grade": "MEASURED", "theme": "agent-control", "establishesShort": "A pre-registered study of whether LLM agents produce the collective failure that per-agent rules permit.", "establishes": "That three hosted language models, acting as agents in a shared-budget environment, produce a compliant collective breach in every BLIND episode where the allowances exceed the budget; that an aggregate-channel guardrail detects every pivotal action while a local-channel guardrail is analytically blind; and that one redundant sentence about the headroom moves the breach rate more than the information it carries.", "doesNotEstablish": "Anything about a deployment or a product. A rate is a fact about a dataset, a prompt and a model on one day. Three developers, not four. No mechanistic claim about why phrasing matters.", "trustedBase": "The environment and scorer (authored, with a scorer-independence test); the pre-registered design fingerprint; hosted model APIs on the run dates; the guardrail judge gpt-5-mini, which shares a vendor with two agent arms.", "method": "One-round shared-budget commons; factorial over regime, population and an information ladder; four amendments each registered before their data; seven pre-committed controls.", "results": "Compliant breach 60 of 60 in BLIND on each model; PEERSUM 0, 7 and 2 of 60; RESTATE 4, 2 and 0 of 60; aggregate-channel LLM judge 100 of 100 detections with 0 of 90 false blocks and 0 breaches in series; impossible-breach 0 of 60 and forced-breach 60 of 60 controls fired.", "reproduceCommand": "python3 -m pytest tests -q && python3 -m commons.report _canonical/gpt-5-mini _canonical/gpt-4.1-mini _canonical/gemini-2.5-flash", "toolchain": "Python 3; hosted model APIs only to regenerate; every number recomputes offline from results/_canonical", "repo": "repowazdogz-droid/commons-agent-lab", "independentReproduction": "None known.", "priorArt": "Commons and public-goods games; per-agent guardrails as the common deployment shape. The contribution is the pre-registered empirical counterpart to a formal result, with mechanical scoring and controls that fire.", "repoStatus": "active" }, { "slug": "proof-carrying-evals", "name": "proof-carrying-evals", "grade": "MEASURED", "theme": "evaluation-integrity", "establishesShort": "Governance properties routed to the checker whose logic fits, with the LLM judge's score sealed beside the proof.", "establishes": "That an evaluation harness can route each checkable governance property to a formal checker (Z3 for one decision, Lean 4 for a trace, TLA+ for interleavings, CryptoVerif for authorisation) and keep the language-model judge's opinion labelled as an opinion, so that on the six violating cases the judge's 4 of 6 passes and the checker's 6 of 6 proofs sit in one record.", "doesNotEstablish": "That a frontier judge would fail the same way; the judge is a local 14B model and sometimes grades its own output. Two dependencies are not yet published on their own and are vendored with provenance.", "trustedBase": "Z3, Lean 4, TLC and CryptoVerif and each encoding; the vendored omega_seal and omega_gate packages; the local judge model.", "method": "Sixteen sealed records from one complete rerun; model-authored and planted-adversarial decisions reported separately; CI recomputes the headline counts.", "results": "Judge passed 4 of 6 violating decisions; checker proved 6 of 6; on six non-violating rows both agree.", "reproduceCommand": "python3 scripts/verify_records.py", "toolchain": "Python; Z3, Lean 4, TLA+ Tools, CryptoVerif; a local Ollama judge", "repo": "repowazdogz-droid/proof-carrying-evals", "independentReproduction": "None known.", "priorArt": "LLM-as-judge evaluation and formal runtime verification. The contribution is the routing by property kind with both verdicts sealed together.", "repoStatus": "active" }, { "slug": "spcu-verification", "name": "spcu-verification", "grade": "OBSERVED", "theme": "formal-verification", "establishesShort": "A small power-control IP verified with open tools; two injected defects invisible to every specification-derived property.", "establishes": "That a complete, green formal argument over a small DVFS controller left two of five injected defects undetected because the specification never stated the requirement they violate, and that a 200-mutant run showed the hand-written mutations had probed the wrong part of the design.", "doesNotEstablish": "Anything about commercial flows or larger designs. The mutations and the properties share an author. Unbounded proofs are separated from bounded checks and vacuous assertions are excluded from every claim.", "trustedBase": "Yosys, SymbiYosys, ABC, Z3, Verilator, cocotb and pyuvm; the collapsed-clock model and its four environment assumptions; the hand-written specification.", "method": "Formal proofs (PDR), simulation, a UVM-architecture Python testbench and bare-metal C against the RTL; five injected defects; 200 netlist mutations checked against the property set and an equivalence miter.", "results": "26 requirement-derived assertions pass the prove task, 2 vacuous, 3 resting on a further stated assumption; four unseeded design bugs found by formal; 200 mutants: 84 covered, 23 equivalent, 93 in the gap.", "reproduceCommand": "make setup && make all", "toolchain": "OSS CAD Suite: Yosys 0.68, SymbiYosys, Verilator 5.051, mcy, Z3", "repo": "repowazdogz-droid/spcu-verification", "independentReproduction": "None known.", "priorArt": "Mutation analysis for formal property sets and specification adequacy. The contribution is the worked, open-tool instance with the vacuous assertions kept in view.", "repoStatus": "active" }, { "slug": "evidence-audit", "name": "evidence-audit", "grade": "OBSERVED", "theme": "formal-verification", "establishesShort": "A grader for recorded verification outputs that reports what a run explored, not the verdict it printed.", "establishes": "That recorded outputs from Kani, loom and cargo test, and recorded Lean artifacts, can be graded on what the run actually explored, and that three real green runs could not have failed.", "doesNotEstablish": "No defect in the subject crates is claimed. The tool runs no verifier and reads recorded outputs only. Version 0.1.", "trustedBase": "The recorded outputs themselves and the tool's parsers; the case-study patches.", "method": "Three committed case studies with inputs, patches, outputs and reproduction commands; grading by reachability, interleaving counts and value lemmas.", "results": "Kani assertion UNREACHABLE after the first injected bug; loom 9 uninstrumented against 54 instrumented interleavings with byte-identical verdicts; Lean axiom audits byte-identical for a correct and a wrong model.", "reproduceCommand": "pip install -e . && python3 -m pytest tests/ -q", "toolchain": "Rust, Kani, loom, Lean 4", "repo": "repowazdogz-droid/evidence-audit", "independentReproduction": "None known.", "priorArt": "Vacuity detection and coverage metrics for model checking. The contribution is grading recorded outputs across four tools with the catches committed as fixtures.", "repoStatus": "active" }, { "slug": "collective-bound", "name": "collective-bound", "grade": "PROVEN", "theme": "agent-control", "establishesShort": "The formal core behind the collective-bound result: Lean 4 (six theorems, no axioms), Z3 (eleven sealed verdicts), a learning adversary, negative controls.", "establishes": "That six agents each inside a cap of 10 breach a pool of 40 while a rule on the sum removes every such breach (Lean, closed data, no axioms); that the sum rule holds for every coordinated profile and bounds a harm iff the harm factors through the aggregate, and is channel-blind (Z3); and that a learning adversary with full sight of the meter never exceeded the bound over 180,000 episodes while each hand-built relaxation was exploitable to the predicted magnitude.", "doesNotEstablish": "Anything about a deployed multi-agent system. Lean over closed data only; Z3 encoding faithfulness argued, not proved; the per-agent cap is not shown to be unfixable; the channel-blindness result assumes perfectly secure steganography rather than constructing it. Not preregistered.", "trustedBase": "The Lean 4.30.0 kernel and toolchain; z3 4.16.0 and the faithfulness of the linear-integer encoding; the vendored omega_seal hashing (stdlib only); seeded numpy for the learning run.", "method": "Two Lean files extracted unmodified from a larger private development; five Python files and six sealed records extracted unmodified from a private Z3 development; runners that write only under out/; three Z3 mutation controls and one Lean mutation control.", "results": "lake build succeeds; #print axioms reports no axioms for all six theorems; records 0001 to 0003 and exp-0003 regenerate byte for byte (hashes 3fd7b869…, 95a0989f…, 34142856…, 23f5d078…); all four negative controls fail as required.", "reproduceCommand": "make lean axioms z3 controls", "toolchain": "Lean 4.30.0 via elan; Python 3.10+, z3-solver 4.16.0, numpy", "repo": "repowazdogz-droid/collective-bound", "independentReproduction": "None known. Author self-reproduction from a fresh clone on 2026-09-02.", "priorArt": "Aggregate and conservation caps as a design pattern; Motwani et al., Secret Collusion among AI Agents (NeurIPS 2024), whose undetectable-collusion setting is the adversary in the channel-blindness result, built on the perfect-security construction of Schroeder de Witt et al. (ICLR 2023). The contribution is the machine-checked interaction between the sum rule and undetectable coordination, and the exact boundary.", "repoStatus": "active" } ], "upstream": [ { "id": "inspect-ai-completed-samples", "subject": "Inspect AI (UK AI Security Institute evaluation framework)", "tested": "What the completed-sample count in a finished run's metadata actually counts when a scorer raises.", "found": "completed_samples was computed from scored samples, so the same failing run reported 6 of 6 or 4 of 6 depending only on scorer declaration order.", "reported": "Issue #4602, 23 July 2026", "fix": "Pull request #4604 by Anai-Guo, merged 29 July 2026; released in 0.3.252 (4 August 2026).", "status": "RELEASED", "link": "https://github.com/UKGovernmentBEIS/inspect_ai/pull/4604", "issue": "https://github.com/UKGovernmentBEIS/inspect_ai/issues/4602", "finding": "completed-sample-count-depends-on-scorer-order" }, { "id": "lemmascript-impure-extern", "subject": "LemmaScript (a verified-language toolchain emitting Dafny and Lean)", "tested": "Whether cross-file externs are modelled as deterministic functions, so two calls to a random source prove equal.", "found": "Every called cross-file symbol was emitted as an axiomatised pure function; dafny verify reported 1 verified, 0 errors on code whose rollDie() returns different values per call.", "reported": "Issue #206, 24 August 2026", "fix": "Pull request #207 (impure extern) by the maintainer, merged within six hours; tagged v0.6.1 the same day.", "status": "RELEASED", "link": "https://github.com/midspiral/LemmaScript/pull/207", "issue": "https://github.com/midspiral/LemmaScript/issues/206" }, { "id": "inspect-robots-judgement-source", "subject": "inspect-robots (robocurve embodied-evaluation harness)", "tested": "Whether the persisted log distinguishes an embodiment-adopted verdict from one sampled from a vision-language model.", "found": "Neither the final log nor the live snapshot recorded which path produced each operator judgement, so verdicts from different paths were indistinguishable on disk.", "reported": "Issue #413, 27 August 2026", "fix": "Pull request #422 by the maintainer, merged 31 August 2026; not yet in a release.", "status": "MERGED", "link": "https://github.com/robocurve/inspect-robots/pull/422", "issue": "https://github.com/robocurve/inspect-robots/issues/413" }, { "id": "labs-oo-agents-generator-span", "subject": "NVIDIA-NeMo labs-OO-Agents", "tested": "Where a tracing span for a generator-bodied agent method begins and ends.", "found": "Generator and async-generator methods were routed to the coroutine wrapper, so generation was skipped and the span ended at generator creation rather than at consumption.", "reported": "Issue #38, 27 July 2026", "fix": "Pull request #120 (fixes #38), merged 31 August 2026; not yet in a release.", "status": "MERGED", "link": "https://github.com/NVIDIA-NeMo/labs-OO-Agents/pull/120", "issue": "https://github.com/NVIDIA-NeMo/labs-OO-Agents/issues/38" }, { "id": "skillspector-anthropic-schema", "subject": "NVIDIA SkillSpector", "tested": "Whether the Anthropic provider's structured-output schema is accepted by the API it targets.", "found": "Every LLM-mode scan through the Anthropic provider failed: the schema carried minimum and maximum bounds the structured-output endpoint rejected.", "reported": "Issue #76, 15 June 2026", "fix": "Closed by the maintainers as resolved by the release snapshot in pull request #85 (native Anthropic provider), merged 16 June 2026. The snapshot does not touch the bounds directly; the maintainers' closing comment is the link between report and fix.", "status": "MERGED", "link": "https://github.com/NVIDIA/SkillSpector/pull/85", "issue": "https://github.com/NVIDIA/SkillSpector/issues/76" }, { "id": "cedar-spec-typed-expr-drt", "subject": "cedar-policy cedar-spec (Cedar's Lean model and differential tests)", "tested": "Whether the Rust and Lean typecheckers compute the same typed expression, not only the same pass/fail verdict.", "found": "Eight phantom-divergence classes surfaced while building the target; the pull request adds a differential target over the typed expression.", "reported": "Pull request #995 by Omega, 28 July 2026, against issue #840 (maintainer-filed)", "fix": "Open, unmerged.", "status": "OPEN", "link": "https://github.com/cedar-policy/cedar-spec/pull/995" }, { "id": "common-cells-ecc-proof", "subject": "pulp-platform common_cells (an open hardware IP library)", "tested": "Whether the ECC encoder and decoder pair satisfy four stated properties for every data width.", "found": "A SymbiYosys proof for cc_ecc_encode and cc_ecc_decode; widths 1, 2, 4, 5, 11 and 12 in the default target, and a sweep to width 64 proving all 128 properties.", "reported": "Pull request #355 by Omega, 11 August 2026, as a subtask of issue #288 (maintainer-filed)", "fix": "Open, unmerged.", "status": "OPEN", "link": "https://github.com/pulp-platform/common_cells/pull/355" } ] }