planarian_gym/
main.rs

1//! planarian-gym: runs catalog entries (SPEC §15).
2//!
3//!   planarian-gym list    [--catalog FILE]
4//!   planarian-gym run     [--catalog FILE] --harness fake [--id ID ...] [--all]
5//!   planarian-gym show-reconciliation --harness fake --id ID
6//!   planarian-gym verification [--write-baseline] [--gate]   d:a, trust and bypass report (§17a, §17b)
7//!
8//! The fake harness runs the real daemon core, executor, kernel and watchdog
9//! logic against the in-memory machine (`planarian-fake`). The VM harness is
10//! the flake's NixOS tests (`nix flake check`, `checks.vm-runtime`).
11
12mod catalog;
13mod check;
14mod conformance;
15mod evidence;
16mod live;
17mod metrics;
18mod serve;
19mod sim;
20mod site;
21#[cfg(test)]
22mod tests;
23mod verification;
24
25use sim::Sim;
26use toml::Value as T;
27
28const DEFAULT_CATALOG: &str = catalog::DIR;
29
30/// The catalog being run, for entries that run others (the conformance matrix).
31static CATALOG: std::sync::OnceLock<Vec<T>> = std::sync::OnceLock::new();
32
33#[derive(Debug, PartialEq, Eq, Clone, Copy, PartialOrd, Ord)]
34enum Result_ {
35    Pass,
36    Partial,
37    Fail,
38    Unsupported,
39    NotFake,
40}
41
42struct Run {
43    result: Result_,
44    notes: Vec<String>,
45    sim: Option<Sim>,
46}
47
48fn steps(entry: &T, key: &str) -> Vec<T> {
49    match entry.get(key) {
50        Some(T::Array(a)) => a.clone(),
51        Some(t @ T::Table(_)) => vec![t.clone()],
52        _ => vec![],
53    }
54}
55
56pub fn window(entry: &T) -> planarian_schema::Millis {
57    let s = entry.get("window").and_then(T::as_str).unwrap_or("10m");
58    let n: u64 = s.trim_end_matches(|c: char| c.is_alphabetic()).parse().unwrap_or(10);
59    planarian_schema::Millis(if s.ends_with('h') {
60        n * 3_600_000
61    } else if s.ends_with('s') && !s.ends_with("ms") {
62        n * 1000
63    } else {
64        n * 60_000
65    })
66}
67
68fn run_entry(entry: &T) -> Run {
69    let fake = entry.get("harness").and_then(T::as_array).is_some_and(|h| h.iter().any(|x| x.as_str() == Some("fake")));
70    if !fake {
71        return Run { result: Result_::NotFake, notes: vec!["VM-only entry".into()], sim: None };
72    }
73    // A matrix entry (fake-assumptions-conformance) runs the conformance suite.
74    if entry.get("matrix").is_some() {
75        let catalog = CATALOG.get().cloned().unwrap_or_default();
76        let (ok, notes) = conformance::run(&catalog);
77        return Run { result: if ok { Result_::Pass } else { Result_::Fail }, notes, sim: None };
78    }
79    let mut sim = Sim::new();
80    let setup = steps(entry, "setup");
81    let inject = steps(entry, "inject");
82    let after = steps(entry, "after");
83    let all: Vec<&T> = setup.iter().chain(inject.iter()).chain(after.iter()).collect();
84    let (consumed, crash) = sim.arm(&all);
85    // Let the machine settle into its healthy steady state first.
86    for _ in 0..3 {
87        sim.tick();
88    }
89    sim.begin();
90    for (i, s) in all.iter().enumerate() {
91        if consumed.contains(&i) {
92            continue;
93        }
94        match &crash {
95            Some((j, p)) if *j == i => sim.set_crash(Some(p.clone())),
96            _ => sim.set_crash(None),
97        }
98        if let Err(u) = sim.step(s) {
99            return Run { result: Result_::Unsupported, notes: vec![u.0], sim: Some(sim) };
100        }
101    }
102    sim.set_crash(None);
103    sim.run_window(window(entry));
104    let post = entry.get("required_post").cloned().unwrap_or(T::Table(Default::default()));
105    let v = check::judge(&sim.evidence(), &post);
106    let mut notes = v.failures.clone();
107    notes.extend(v.unchecked.iter().map(|u| format!("unchecked: {u}")));
108    let result = if !v.failures.is_empty() {
109        Result_::Fail
110    } else if !v.unchecked.is_empty() {
111        Result_::Partial
112    } else {
113        Result_::Pass
114    };
115    Run { result, notes, sim: Some(sim) }
116}
117
118/// The catalog's entries (a directory or one file), ids unique; exits 2 if not.
119fn load(path: &str) -> Vec<T> {
120    catalog::entries(std::path::Path::new(path)).unwrap_or_else(|e| {
121        eprintln!("{e}");
122        std::process::exit(2)
123    })
124}
125
126fn main() {
127    let args: Vec<String> = std::env::args().skip(1).collect();
128    if args.first().map(String::as_str) == Some("verification") {
129        let r = verification::run(args.iter().any(|a| a == "--write-baseline"), args.iter().any(|a| a == "--gate"));
130        print!("{}", r.text);
131        std::process::exit(if r.ok { 0 } else { 1 });
132    }
133    if args.first().map(String::as_str) == Some("site") {
134        std::process::exit(site::main(&args[1..]));
135    }
136    if args.first().map(String::as_str) == Some("vm") {
137        // The VM harness's in-VM half (live.rs).
138        let catalog = args
139            .iter()
140            .position(|a| a == "--catalog")
141            .and_then(|i| args.get(i + 1))
142            .cloned()
143            .or_else(|| std::env::var("PLANARIAN_GYM_CATALOG").ok())
144            .unwrap_or_else(|| DEFAULT_CATALOG.to_string());
145        std::process::exit(live::main(&args[1..], &load(&catalog)));
146    }
147    let mut catalog = DEFAULT_CATALOG.to_string();
148    let mut ids: Vec<String> = vec![];
149    let mut harness = "fake".to_string();
150    let mut verbose = false;
151    let mut show_notices = false;
152    let mut strict = false;
153    let mut i = 1;
154    while i < args.len() {
155        match args[i].as_str() {
156            "--catalog" => {
157                catalog = args.get(i + 1).cloned().unwrap_or_default();
158                i += 1;
159            }
160            "--id" => {
161                ids.push(args.get(i + 1).cloned().unwrap_or_default());
162                i += 1;
163            }
164            "--harness" => {
165                harness = args.get(i + 1).cloned().unwrap_or_default();
166                i += 1;
167            }
168            "--all" => {}
169            "--socket" => i += 1,
170            "-v" | "--verbose" => verbose = true,
171            "--notices" => show_notices = true,
172            "--strict" => strict = true,
173            other => {
174                eprintln!("unknown argument {other}");
175                std::process::exit(2);
176            }
177        }
178        i += 1;
179    }
180    let entries = load(&catalog);
181    let _ = CATALOG.set(entries.clone());
182    let cmd = args.first().map(String::as_str).unwrap_or("list");
183    if harness == "vm" && cmd != "list" {
184        eprintln!("The VM harness runs as NixOS tests: `nix build .#checks.x86_64-linux.vm-runtime` (needs KVM).");
185        std::process::exit(2);
186    }
187    let selected: Vec<&T> = entries
188        .iter()
189        .filter(|e| ids.is_empty() || ids.iter().any(|id| e.get("id").and_then(T::as_str) == Some(id)))
190        .collect();
191    if selected.is_empty() {
192        eprintln!("no catalog entries matched {ids:?}");
193        std::process::exit(2);
194    }
195    match cmd {
196        "list" => {
197            for e in selected {
198                println!(
199                    "{:<44} {:<36} {:<10} {}",
200                    e.get("id").and_then(T::as_str).unwrap_or(""),
201                    e.get("family").and_then(T::as_str).unwrap_or(""),
202                    e.get("scoring").and_then(T::as_str).unwrap_or(""),
203                    e.get("harness").map(|h| h.to_string()).unwrap_or_default()
204                );
205            }
206        }
207        "run" => {
208            let mut unwaived_unsupported = vec![];
209            let waived = harness_waivers(&std::fs::read_to_string("docs/deviations.md").unwrap_or_default());
210            let mut tally = std::collections::BTreeMap::new();
211            let mut metrics = metrics::Metrics::default();
212            for e in selected {
213                let id = e.get("id").and_then(T::as_str).unwrap_or("?");
214                let r = run_entry(e);
215                metrics.add(e, r.result == Result_::Pass, r.result == Result_::Partial, r.sim.as_ref());
216                *tally.entry(format!("{:?}", r.result)).or_insert(0) += 1;
217                // R15, the §15 harness contract: an UNSUPPORTED must-pass or
218                // acceptance entry fails a strict run unless docs/deviations.md
219                // waives it for the fake harness.
220                let gating = e.get("scoring").and_then(T::as_str) == Some("must-pass")
221                    || e.get("tags")
222                        .and_then(T::as_array)
223                        .is_some_and(|t| t.iter().any(|x| x.as_str() == Some("acceptance")));
224                if r.result == Result_::Unsupported && gating && !waived.contains(&(id.to_string(), "fake".to_string()))
225                {
226                    unwaived_unsupported.push(id.to_string());
227                }
228                println!("{:<12} {id}", format!("{:?}", r.result).to_uppercase());
229                if verbose || r.result == Result_::Fail || r.result == Result_::Unsupported {
230                    for n in &r.notes {
231                        println!("             {n}");
232                    }
233                }
234                if show_notices && let Some(sim) = &r.sim {
235                    for n in &sim.notices {
236                        println!("             notice: {n}");
237                    }
238                    for r in &sim.responses {
239                        println!("             response: {r:?}");
240                    }
241                }
242            }
243            println!("\n{tally:?}");
244            println!("\n{metrics}");
245            // A failure fails the run, and so does a key left unchecked (a
246            // pass that proves less than it says). Entries this harness
247            // cannot run are listed, and fail it with --strict unless the
248            // catalog waives them for it (R12, R15).
249            let failed = tally.contains_key("Fail") || tally.contains_key("Partial");
250            if strict && !unwaived_unsupported.is_empty() {
251                println!("unwaived UNSUPPORTED: {}", unwaived_unsupported.join(" "));
252            }
253            std::process::exit(if failed || (strict && !unwaived_unsupported.is_empty()) { 1 } else { 0 });
254        }
255        "serve" => {
256            let socket = args.iter().position(|a| a == "--socket").and_then(|i| args.get(i + 1)).cloned();
257            serve::serve(&socket.unwrap_or_else(|| "/tmp/planarian-gym.sock".into()));
258        }
259        "show-reconciliation" => {
260            let r = run_entry(selected[0]);
261            let Some(sim) = r.sim else { std::process::exit(1) };
262            match sim.ledger().last_reconciliation {
263                Some(rec) => {
264                    println!("by {:?}", rec.by);
265                    for a in rec.actions {
266                        println!("  {a}");
267                    }
268                }
269                None => {
270                    println!("no reconciliation record");
271                    std::process::exit(1);
272                }
273            }
274        }
275        other => {
276            eprintln!("unknown command {other}");
277            std::process::exit(2);
278        }
279    }
280}
281
282/// The "Harness waivers" table of docs/deviations.md (§15 harness
283/// contract): (entry, harness) pairs. nix/gym/harness-contract.py reads the
284/// same table for the VM runners.
285fn harness_waivers(deviations: &str) -> std::collections::BTreeSet<(String, String)> {
286    let mut out = std::collections::BTreeSet::new();
287    let mut table = false;
288    for line in deviations.lines() {
289        if line.starts_with("## ") {
290            table = line.trim().starts_with("## Harness waivers");
291            continue;
292        }
293        if !table {
294            continue;
295        }
296        let cells: Vec<&str> = line.split('|').map(str::trim).collect();
297        if let [_, entry, harness, ..] = cells.as_slice()
298            && let Some(id) = entry.strip_prefix('`').and_then(|x| x.strip_suffix('`'))
299        {
300            out.insert((id.to_string(), harness.to_string()));
301        }
302    }
303    out
304}
305
306#[cfg(test)]
307mod waiver_tests {
308    /// R15: only rows of the waiver table count, by entry and harness.
309    #[test]
310    fn harness_waivers_are_read_from_their_table() {
311        let doc = "# D\n| `elsewhere` | vm | no |\n## Harness waivers (§15)\n| Entry | Harness | Reason |\n| --- | --- | --- |\n| `a-b` | vm | why |\n## Next\n| `c` | fake | no |\n";
312        let w = super::harness_waivers(doc);
313        assert_eq!(w.len(), 1);
314        assert!(w.contains(&("a-b".to_string(), "vm".to_string())));
315    }
316}
planarian_gym/main.rs

planarian_gym/
main.rs
