1use crate::evidence::{Effect, Evidence};
6use crate::sim::{dur, unit};
7use planarian_exec::Submitted;
8use planarian_ipc::Response;
9use planarian_schema::*;
10use planarian_txn::ledger::{Entry, Record};
11use planarian_txn::machine::tag;
12use std::collections::{BTreeMap, BTreeSet};
13use toml::Value as T;
14
15#[derive(Debug, Default)]
16pub struct Verdict {
17 pub failures: Vec<String>,
18 pub unchecked: Vec<String>,
19}
20
21fn transition_name(t: &Transition) -> &'static str {
22 match t {
23 Transition::RestartUnit(_) => "RestartUnit",
24 Transition::StopUnit(_) => "StopUnit",
25 Transition::BindDriver { .. } => "BindDriver",
26 Transition::ApplyChange { .. } => "ApplyChange",
27 Transition::SwitchGeneration(_) => "SwitchGeneration",
28 Transition::Revert { .. } => "Revert",
29 Transition::Rollback => "Rollback",
30 Transition::LoadModel(_) => "LoadModel",
31 Transition::RestoreSnapshot(_) => "RestoreSnapshot",
32 Transition::CollectGarbage { .. } => "CollectGarbage",
33 Transition::SetRuntimeOption { .. } => "SetRuntimeOption",
34 Transition::SetOverride(_) => "SetOverride",
35 Transition::LiftOverride(_) => "LiftOverride",
36 Transition::StopAndSuppress { .. } => "StopAndSuppress",
37 Transition::Resolve { .. } => "Resolve",
38 }
39}
40
41fn target_of(t: &Transition) -> Option<UnitId> {
42 match t {
43 Transition::RestartUnit(u) | Transition::StopUnit(u) | Transition::LiftOverride(u) => Some(u.clone()),
44 Transition::BindDriver { device, .. } => Some(device.clone()),
45 Transition::StopAndSuppress { unit, .. } => Some(unit.clone()),
46 Transition::SetOverride(s) => Some(s.unit.clone()),
47 _ => None,
48 }
49}
50
51fn state_name(s: &TxnState) -> String {
52 match s {
53 TxnState::Rejected(r) => format!("Rejected({r:?})"),
54 s => tag(s).to_string(),
55 }
56}
57
58struct TxnView {
59 id: TxnId,
60 transition: Transition,
61 path: Vec<String>,
62 state: TxnState,
63 watchdog: bool,
64 epoch: u64,
65}
66
67fn txns(e: &Evidence) -> Vec<TxnView> {
68 let l = &e.ledger;
69 l.terminal
70 .iter()
71 .map(|(id, t)| TxnView {
72 id: *id,
73 transition: t.transition.clone(),
74 path: t.path.clone(),
75 state: t.state.clone(),
76 watchdog: t.owner.is_watchdog(),
77 epoch: t.epoch,
78 })
79 .chain(l.live.values().map(|t| TxnView {
80 id: t.txn.id,
81 transition: t.txn.transition.clone(),
82 path: t.path.clone(),
83 state: t.txn.state.clone(),
84 watchdog: t.txn.lease.owner.is_watchdog(),
85 epoch: t.txn.lease.epoch,
86 }))
87 .collect()
88}
89
90fn gen_named(e: &Evidence, name: &str) -> Option<GenerationId> {
91 match name {
92 "base" | "known_good" | "unchanged" => Some(e.base),
93 "candidate" => e.candidate,
94 "owner-built" => None,
95 _ => None,
96 }
97}
98
99pub fn same_unit(spec: &UnitId, seen: &UnitId) -> bool {
103 spec == seen
104 || (spec.scope == Scope::System
105 && spec.kind == seen.kind
106 && spec.name == seen.name
107 && matches!(seen.scope, Scope::User { session: Some(_), .. }))
108}
109
110fn subsequence(path: &[String], want: &[String]) -> bool {
111 let mut it = path.iter();
112 want.iter().all(|w| it.any(|p| p == w))
113}
114
115pub fn judge(e: &Evidence, post: &T) -> Verdict {
116 let mut v = Verdict::default();
117 let Some(post) = post.as_table() else { return v };
118 let current = e.current;
119 let boot = e.boot_default;
120 let overrides_empty = e.overrides_empty;
121 let ledger = &e.ledger;
122 let all = txns(e);
123 let finals = &e.finals;
124 let may_change: Vec<UnitId> =
125 post.get("may_change").and_then(T::as_array).into_iter().flatten().filter_map(T::as_str).map(unit).collect();
126
127 let known: BTreeSet<&UnitId> = e.pre_status.keys().chain(e.finals.keys()).chain(e.seen_status.keys()).collect();
130 let unknown = |u: &str| !known.is_empty() && !known.iter().any(|k| same_unit(&unit(u), k));
131 let declares_inferred = post
135 .get("overrides")
136 .and_then(T::as_array)
137 .is_some_and(|a| a.iter().any(|o| o.get("by").and_then(T::as_str) == Some("Unattributed")));
138 for o in &ledger.overrides {
139 let ok = match &o.by {
140 Actor::User { .. } => o.attribution == Attribution::Attested,
141 Actor::Unattributed => o.attribution == Attribution::Inferred && declares_inferred,
142 _ => false,
143 };
144 if !ok {
145 v.failures.push(format!("gamed success: {} overridden by {:?} ({:?})", o.unit, o.by, o.attribution));
146 }
147 }
148 for (key, val) in post {
149 let fail = |v: &mut Verdict, m: String| v.failures.push(format!("{key}: {m}"));
150 match key.as_str() {
151 "units" => {
152 for (u, want) in val.as_table().into_iter().flatten() {
153 let got = finals.get(&unit(u)).cloned().unwrap_or_else(|| "undefined".into());
154 if got != want.as_str().unwrap_or("") {
155 fail(&mut v, format!("{u} is {got}, want {want}"));
156 }
157 }
158 }
159 "may_change" | "settle" => {}
161 "others_unchanged" if val.as_bool() == Some(true) => {
162 let named: Vec<UnitId> =
163 post.get("units").and_then(T::as_table).into_iter().flatten().map(|(k, _)| unit(k)).collect();
164 for (u, before) in &e.pre_status {
165 if before != "Running" || may_change.contains(u) || named.contains(u) || e.touched.contains(u) {
166 continue;
167 }
168 let after = finals.get(u).cloned().unwrap_or_default();
169 if after != "Running" {
170 fail(&mut v, format!("{u} was Running and is now {after}"));
171 }
172 }
173 }
174 "others_unchanged" => {}
175 "invariants" | "journal_readable" => {
180 let i18 = val.as_array().is_some_and(|a| a.iter().any(|x| x.as_str() == Some("I18")));
181 if i18 && e.posture.chain_broken {
182 if !e.posture.finding.as_deref().is_some_and(|f| f.contains("journal")) {
183 fail(&mut v, "the chain is broken but planariad did not report it".into());
184 }
185 } else if !e.journal_replays {
186 fail(&mut v, "the journal does not replay".into());
187 }
188 }
189 "declaration_origin" => {
192 let name = unit(val.get("unit").and_then(T::as_str).unwrap_or("")).name;
193 let target = post
194 .get("incident")
195 .and_then(T::as_array)
196 .and_then(|a| a.first())
197 .and_then(|i| i.get("target"))
198 .and_then(T::as_str)
199 .unwrap_or("");
200 let decl = format!("service:{name} is new in generation");
201 let report = e.notices.iter().find(|n| n.starts_with(target) && n.contains(&decl));
202 let Some(report) = report else {
203 fail(&mut v, format!("no incident report on {target} names service:{name}'s declaration"));
204 continue;
205 };
206 let mut want: Vec<String> = vec![];
207 match val.get("who").and_then(T::as_str) {
208 Some("root") => want.push("by root".into()),
209 Some(u) => want.push(format!("by uid {}", u.trim_start_matches("user:"))),
210 None => {}
211 }
212 match val.get("path").and_then(T::as_str) {
213 Some(p) if p.starts_with("ApplyChange ") => {
214 want.push(format!("through apply {}", p.trim_start_matches("ApplyChange ")))
215 }
216 Some(p) => want.push(p.to_string()),
217 None => {}
218 }
219 match val.get("file").and_then(T::as_str) {
220 Some(f) if f.starts_with("owner:") => want.push("defined in /".into()),
222 Some(f) => want.push(f.to_string()),
223 None => {}
224 }
225 match val.get("when").and_then(T::as_str) {
226 Some("generation-created") => want.push("generation created at".into()),
227 Some("txn") => want.push("(transaction ".into()),
228 _ => {}
229 }
230 if val.get("lock_change").and_then(T::as_bool) == Some(true) && !report.contains(" → ") {
232 fail(&mut v, format!("the report does not give the input's lock change: {report}"));
233 }
234 for w in want.iter().filter(|w| !report.contains(w.as_str())) {
235 fail(&mut v, format!("the report does not say {w:?}: {report}"));
236 }
237 }
238 "not_started" | "unit_after_failure" => match &e.posture.responses {
241 None => v.unchecked.push(format!("{key} (systemd's; the VM harness observes it)")),
242 Some(seen) => {
243 if key == "not_started" && seen.contains("agents_running") {
244 fail(&mut v, "an agent ran while Planarian was failed".into());
245 }
246 if key == "unit_after_failure" && !seen.contains("disable_agents") {
247 fail(&mut v, "the agents were not stopped".into());
248 }
249 }
250 },
251 "posture" => {
254 let class = val.get("class").and_then(T::as_str).unwrap_or("").to_lowercase();
255 if !e.posture.classes.contains(&class) {
256 fail(&mut v, format!("class {class} not reached (reached: {:?})", e.posture.classes));
257 }
258 if let Some(t) = val.get("target_reached").and_then(T::as_str)
259 && t.to_lowercase() != format!("planarian-failed@{class}.target")
260 {
261 fail(&mut v, format!("{t} is not the target for class {class}"));
262 }
263 let want: Vec<&str> =
264 val.get("responses").and_then(T::as_array).into_iter().flatten().filter_map(T::as_str).collect();
265 let not: Vec<&str> =
266 val.get("not").and_then(T::as_array).into_iter().flatten().filter_map(T::as_str).collect();
267 match &e.posture.responses {
268 None if want.is_empty() && not.is_empty() => {}
269 None => v.unchecked.push("posture.responses (systemd's; the VM harness observes them)".into()),
270 Some(seen) => {
271 for r in want.iter().filter(|r| !seen.contains(**r)) {
272 fail(&mut v, format!("response {r} not applied"));
273 }
274 for r in not.iter().filter(|r| seen.contains(**r)) {
275 fail(&mut v, format!("response {r} applied, must not be"));
276 }
277 }
278 }
279 }
280 "generation" | "boot_default" => {
281 let have = if key == "generation" { current } else { boot };
282 match val.as_str() {
283 Some("owner-built") => {
284 if have == e.base {
285 fail(&mut v, "expected the owner's own generation".into());
286 }
287 }
288 Some(name) => match gen_named(e, name) {
289 Some(g) if g == have => {}
290 Some(g) => fail(&mut v, format!("is {}, want {name} ({})", have.0, g.0)),
291 None => fail(&mut v, format!("no {name} generation")),
292 },
293 None => v.unchecked.push(key.clone()),
294 }
295 }
296 "override_source" => {
297 let want_empty = matches!(val.as_str(), Some("base" | "known_good"));
298 if overrides_empty != want_empty {
299 fail(
300 &mut v,
301 format!("override file is {}", if overrides_empty { "the base" } else { "the candidate" }),
302 );
303 }
304 }
305 "known_good" => {
307 let want = if val.as_str() == Some("candidate") { e.candidate } else { Some(e.base) };
308 if ledger.known_good != want {
309 fail(&mut v, format!("{:?}, want {want:?}", ledger.known_good));
310 }
311 }
312 "live_tested" => {
315 let never = val.as_str() == Some("never");
316 let is = ledger.live_tested.is_some() && ledger.live_tested == e.candidate;
317 if is == never {
318 fail(&mut v, format!("{:?} vs candidate {:?}", ledger.live_tested, e.candidate));
319 }
320 }
321 "journal_agrees" => {
322 if current != boot {
323 fail(&mut v, format!("running {} but boot default {}", current.0, boot.0));
324 }
325 if ledger.live.values().any(|t| !t.txn.state.is_pre_mutation()) {
326 fail(&mut v, "a transaction is still in flight".into());
327 }
328 }
329 "security_floor" => {
331 let want = match val.as_str() {
332 Some("none" | "unchanged") => None,
333 Some(name) => gen_named(e, name),
334 None => None,
335 };
336 if ledger.security_floor != want {
337 fail(&mut v, format!("is {:?}, want {want:?}", ledger.security_floor));
338 }
339 }
340 "notified_sessions" => {
342 for sid in val.as_array().into_iter().flatten().filter_map(T::as_str) {
343 let told = e
344 .notices
345 .iter()
346 .any(|n| n.contains("other people's sessions") && n.contains(&format!("{sid} (uid")));
347 if !told {
348 fail(&mut v, format!("session {sid} was not notified"));
349 }
350 }
351 }
352 "notices_count" => {
355 let one = std::slice::from_ref(val);
356 for c in val.as_array().map(Vec::as_slice).unwrap_or(one) {
357 let m = c.get("match").and_then(T::as_str).unwrap_or("");
358 let want = c.get("count").and_then(T::as_integer).unwrap_or(0) as usize;
359 let got = e.notices.iter().filter(|n| n.contains(m)).count();
360 if got != want {
361 fail(&mut v, format!("{got} notices mention {m}, want {want}: {:?}", e.notices));
362 }
363 }
364 }
365 "escalation" => {
367 let want = val.as_str().unwrap_or("");
368 let found = e.seen_incidents.values().any(
369 |i| matches!(&i.state, IncidentState::Escalated { reason, .. } if format!("{reason:?}") == want),
370 );
371 if !found {
372 fail(&mut v, format!("no incident escalated with {want}"));
373 }
374 }
375 "interlock" => {
376 let set = ledger.interlocked();
377 if set != (val.as_str() == Some("set")) {
378 fail(&mut v, format!("interlock is {}", if set { "set" } else { "clear" }));
379 }
380 }
381 "inferred_listed" => {
384 let want: BTreeSet<String> =
385 val.as_array().into_iter().flatten().filter_map(T::as_str).map(String::from).collect();
386 let got: BTreeSet<String> = ledger
387 .overrides
388 .iter()
389 .filter(|o| o.attribution == Attribution::Inferred && !o.acknowledged)
390 .map(|o| o.unit.to_string())
391 .collect();
392 if got != want {
393 fail(&mut v, format!("inferred overrides listed: {got:?}, want {want:?}"));
394 }
395 }
396 "overrides" => {
397 let want: Vec<(String, String)> = val
398 .as_array()
399 .into_iter()
400 .flatten()
401 .map(|s| {
402 (
403 s.get("unit").and_then(T::as_str).unwrap_or("").to_string(),
404 s.get("until").and_then(T::as_str).unwrap_or("").to_string(),
405 )
406 })
407 .collect();
408 let have: Vec<(String, String)> = ledger
409 .overrides
410 .iter()
411 .map(|s| {
412 (
413 s.unit.to_string(),
414 match s.until {
415 Until::Explicit => "Explicit".into(),
416 Until::NextBoot => "NextBoot".into(),
417 Until::At(_) => "At".into(),
418 },
419 )
420 })
421 .collect();
422 if want != have {
423 fail(&mut v, format!("have {have:?}, want {want:?}"));
424 }
425 for (s, w) in ledger.overrides.iter().zip(val.as_array().into_iter().flatten()) {
426 if let Some(by) = w.get("by").and_then(T::as_str) {
427 let ok = match &s.by {
428 Actor::User { uid, session, .. } => {
429 format!("user:{uid}") == by
430 || session.as_ref().is_some_and(|x| format!("user:{uid}@{x}") == by)
431 }
432 Actor::Unattributed => by == "Unattributed",
433 _ => false,
434 };
435 if !ok {
436 fail(&mut v, format!("{} overridden by {:?}, want {by}", s.unit, s.by));
437 }
438 }
439 if let Some(a) = w.get("attribution").and_then(T::as_str)
440 && format!("{:?}", s.attribution) != a
441 {
442 fail(&mut v, format!("{} attribution {:?}, want {a}", s.unit, s.attribution));
443 }
444 }
445 }
446 "no_txn_executed" if val.as_bool() == Some(true) => {
449 for t in &all {
450 if state_name(&t.state) == "Committed" {
451 fail(&mut v, format!("txn {} ({}) committed", t.id.0, transition_name(&t.transition)));
452 }
453 }
454 match &e.effects {
455 None => v.unchecked.push(format!("{key} (backend effects)")),
456 Some(fx) => {
457 for f in fx {
458 fail(&mut v, format!("{} went through", describe(f)));
459 }
460 }
461 }
462 }
463 "agrees" => match &e.agreement {
467 Some(d) if !d.is_empty() => fail(&mut v, d.join("; ")),
468 Some(_) => {}
469 None => v.unchecked.push(key.clone()),
470 },
471 "override_file" => match (val.as_str(), &e.third_party_override) {
472 (Some("third-party"), Some(h)) => {
473 if e.override_hash != Some(*h) {
474 fail(&mut v, "the third party's override file was replaced".into());
475 }
476 }
477 (Some("third-party"), None) => fail(&mut v, "no third-party edit happened".into()),
478 _ => v.unchecked.push(key.clone()),
479 },
480 "incident_unchanged_by_refusal" => match &e.entries {
483 None => v.unchecked.push(key.clone()),
484 Some(en) => {
485 let u = unit(val.as_str().unwrap_or(""));
486 let refusals: Vec<Timestamp> = e
487 .log
488 .iter()
489 .filter(|r| target_of(&r.proposal.transition).as_ref() == Some(&u))
490 .filter(|r| matches!(r.result, Submitted::Rejected { .. }))
491 .map(|r| r.at)
492 .collect();
493 if refusals.is_empty() {
494 fail(&mut v, format!("no refused request on {u}"));
495 }
496 for at in refusals {
499 let mut prev: Option<&Incident> = None;
500 for x in en {
501 let Record::Incident(i) = &x.record else { continue };
502 if i.target != u {
503 continue;
504 }
505 if x.at == at
506 && let Some(p) = prev
507 && p.id == i.id
508 && (i.budget_left > p.budget_left
509 || i.next_attempt < p.next_attempt
510 || (i.state != p.state && i.state == IncidentState::Open))
511 {
512 fail(
513 &mut v,
514 format!("the incident on {u} was replenished when a request on it was refused"),
515 );
516 }
517 prev = Some(i);
518 }
519 }
520 }
521 },
522 "approval_recorded_by" => match &e.entries {
525 None => v.unchecked.push(key.clone()),
526 Some(en) => {
527 let want: u64 = match val.as_str() {
528 Some("root") => 0,
529 Some(u) => u.strip_prefix("user:").and_then(|n| n.parse().ok()).unwrap_or(u64::MAX),
530 None => u64::MAX,
531 };
532 let approved: Vec<TxnId> = all.iter().map(|t| t.id).collect();
533 let ok = en.iter().any(|x| {
534 matches!(&x.record, Record::TokenUsed { approver, txn, .. }
535 if *approver == want && approved.contains(&TxnId(*txn)))
536 });
537 if !ok {
538 fail(&mut v, format!("no approval of a transaction by {want} in the journal"));
539 }
540 }
541 },
542 "no_build_work" if val.as_bool() == Some(true) => match e.work {
544 None => v.unchecked.push(key.clone()),
545 Some((stages, builds)) => {
546 if stages + builds > 0 {
547 fail(&mut v, format!("{stages} source(s) staged and {builds} build(s) for a refused caller"));
548 }
549 }
550 },
551 "collected_none" if val.as_bool() == Some(true) => {
553 if !e.collected.is_empty() {
554 fail(&mut v, format!("collected {:?}", e.collected));
555 }
556 }
557 "nothing_left_open" if val.as_bool() == Some(true) => {
559 for t in e.ledger.live.values() {
560 fail(&mut v, format!("txn {} left in {}", t.txn.id.0, state_name(&t.txn.state)));
561 }
562 }
563 "no_txn_on" => {
564 for u in val.as_array().into_iter().flatten().filter_map(T::as_str) {
565 if unknown(u) {
566 fail(&mut v, format!("{u} is not a unit on this machine"));
567 }
568 let id = unit(u);
569 if all.iter().any(|t| target_of(&t.transition).as_ref() == Some(&id)) {
570 fail(&mut v, format!("a transaction targeted {u}"));
571 }
572 }
573 }
574 "no_autonomous_txn" => {
575 let auto = e
576 .log
577 .iter()
578 .filter(|r| !matches!(r.caller, Actor::User { .. }))
579 .filter(|r| matches!(r.result, Submitted::Finished { .. } | Submitted::Parked { .. }))
580 .count();
581 if auto > 0 {
582 fail(&mut v, format!("{auto} autonomous transaction(s)"));
583 }
584 }
585 "no_incident_on" => {
586 for u in val.as_array().into_iter().flatten().filter_map(T::as_str) {
587 let id = unit(u);
588 let per_session = id.kind == UnitKind::Target && id.name == "graphical-session";
591 if unknown(u) && !per_session {
592 fail(&mut v, format!("{u} is not a unit on this machine"));
593 }
594 if e.seen_incidents.values().any(|i| same_unit(&id, &i.target) && i.kind == IncidentKind::Repair) {
595 fail(&mut v, format!("an incident opened on {u}"));
596 }
597 }
598 }
599 "gc_protected" => {
600 let previous = e.boot_entries.iter().copied().filter(|g| *g < current).max();
602 for name in val.as_array().into_iter().flatten().filter_map(T::as_str) {
603 let gens: Vec<GenerationId> = match name {
604 "known_good" => ledger.known_good.into_iter().chain([e.boot_default]).collect(),
605 "base" => vec![current],
606 "previous" => previous.into_iter().collect(),
607 "candidate" => {
608 ledger.live.values().filter_map(|t| t.txn.candidate.as_ref().map(|(g, _)| *g)).collect()
609 }
610 other => {
611 v.unchecked.push(format!("gc_protected.{other}"));
612 continue;
613 }
614 };
615 if let Some(g) = gens.iter().find(|g| e.collected.contains(g)) {
616 fail(&mut v, format!("{name} generation {} was collected", g.0));
617 }
618 }
619 }
620 "status_seen" | "status_never" => {
621 for (u, names) in val.as_table().into_iter().flatten() {
622 let seen = e.seen_status.get(&unit(u)).cloned().unwrap_or_default();
623 for n in names.as_array().into_iter().flatten().filter_map(T::as_str) {
624 let has = seen.contains(n);
625 if key == "status_seen" && !has {
626 fail(&mut v, format!("{u} never showed {n} (saw {seen:?})"));
627 }
628 if key == "status_never" && has {
629 fail(&mut v, format!("{u} showed {n}"));
630 }
631 }
632 }
633 }
634 "metrics" => {
635 for (m, spec) in val.as_table().into_iter().flatten() {
636 match m.as_str() {
637 "restarts_max" => {
638 for (u, n) in spec.as_table().into_iter().flatten() {
639 let id = unit(u);
640 let count = e.restarts.get(&id).copied().unwrap_or(0);
641 if count > n.as_integer().unwrap_or(0) {
642 fail(&mut v, format!("{u} restarted {count} times (max {n})"));
643 }
644 }
645 }
646 "incidents_max" => {
647 let n = e.seen_incidents.values().filter(|i| i.kind == IncidentKind::Repair).count() as i64;
648 if n > spec.as_integer().unwrap_or(0) {
649 fail(&mut v, format!("{n} incidents"));
650 }
651 }
652 "attempt_gaps_min" => {
653 let mins: Vec<Millis> = spec.as_array().into_iter().flatten().filter_map(dur).collect();
660 let at: BTreeMap<TxnId, Timestamp> = e
661 .log
662 .iter()
663 .filter_map(|r| match &r.result {
664 Submitted::Finished { txn, .. } | Submitted::Parked { txn, .. } => {
665 Some((*txn, r.at))
666 }
667 _ => None,
668 })
669 .collect();
670 for i in e.seen_incidents.values().filter(|i| i.kind == IncidentKind::Repair) {
671 let times: Vec<Timestamp> =
672 i.attempts.iter().filter_map(|(t, _)| at.get(t).copied()).collect();
673 if times.len() < i.attempts.len() {
674 v.unchecked.push(format!("metrics.attempt_gaps_min ({}: attempt times)", i.target));
675 continue;
676 }
677 for (k, w) in times.windows(2).enumerate() {
678 let gap = w[1].0.saturating_sub(w[0].0);
679 if let Some(min) = mins.get(k).filter(|m| gap < m.0) {
680 fail(
681 &mut v,
682 format!(
683 "{}: attempts {} and {} {gap} ms apart (min {} ms)",
684 i.target,
685 k + 1,
686 k + 2,
687 min.0
688 ),
689 );
690 }
691 }
692 }
693 }
694 other => v.unchecked.push(format!("metrics.{other}")),
695 }
696 }
697 }
698 "reports" => {
699 for r in val.as_array().into_iter().flatten().filter_map(T::as_str) {
700 let notices = e.notices.join("\n");
701 let ok = match r {
702 "DiagnoserExhausted" => notices.contains("needs a person"),
703 "Flapping" | "Escalated" => {
704 notices.contains("needs a person") || notices.contains("keeps failing")
705 }
706 "Unauthorized" => {
707 e.responses
708 .iter()
709 .any(|x| matches!(x, Response::Rejected { reason, .. } if reason == "Unauthorized"))
710 || !e.posture.refused_acks.is_empty()
711 }
712 "JournalChainBroken" | "BootReport(JournalChainBroken)" => {
713 e.posture.finding.as_deref().is_some_and(|f| f.contains("journal"))
714 }
715 "PostureLifted" => e.posture.lifted,
716 r if r.starts_with("BootReport(UndeclaredEarly: ") => {
717 let u = r.trim_start_matches("BootReport(UndeclaredEarly: ").trim_end_matches(')');
718 let name = unit(u).name;
719 e.posture.finding.as_deref().is_some_and(|f| f.contains("undeclared") && f.contains(&name))
720 }
721 r if r.starts_with("AlarmAcknowledged(") => {
722 let who = r.trim_start_matches("AlarmAcknowledged(").trim_end_matches(')');
723 e.posture.acks.iter().any(|a| a == who)
724 }
725 "RecoveryFailed" | "alert" => {
726 ledger.terminal.values().any(|t| t.state == TxnState::RecoveryFailed)
727 }
728 "NoSpace" => e
729 .responses
730 .iter()
731 .any(|x| matches!(x, Response::Rejected { reason, .. } if reason == "NoSpace")),
732 "StopRewrittenNotice" => e.responses.iter().any(
733 |x| matches!(x, Response::Done { message, .. } if message.contains("until the next boot")),
734 ),
735 "AwaitingApproval" => notices.contains("needs approval"),
736 "AgentNameInUse" => e.responses.iter().any(|x| {
739 matches!(x, Response::Rejected { message, .. }
740 if message.contains("is already running, started by uid"))
741 }),
742 "NeedsApproval" => e.responses.iter().any(|x| matches!(x, Response::NeedsApproval { .. })),
744 "ApprovalInvalidated" => ledger.terminal.values().any(|t| {
745 matches!(&t.state, TxnState::Rejected(RejectReason::Other(w)) if w == "approval invalidated")
746 }),
747 "TokenReplayRefused" => e.responses.iter().any(|x| {
748 matches!(x, Response::Rejected { message, .. }
749 if message.contains("no pending change with that hash") || message.contains("already used"))
750 }),
751 "UntrackedExcluded" => notices.contains("git does not track them"),
752 "ExternalDependencyDown" => e.notices.iter().any(|n| {
754 n.starts_with("service:external:") && n.contains("nothing I know how to do would fix it")
755 }),
756 "ExternalDependencyUnreachable" => e.notices.iter().any(|n| {
758 n.starts_with("service:external:") && n.contains("can't reach it")
759 }),
760 "OutOfBandStopHint" => {
761 notices.contains("stopped, not crashed") && notices.contains("planarian override")
762 }
763 "DisruptionSessionLoss" => notices.contains("end your graphical session"),
764 "RestartToFinish" => notices.contains("restart to finish"),
765 "WaitsForWindow" => notices.contains("waits for a quiet moment"),
766 "InferredOverride" => notices.contains("stopped outside Planarian")
767 && notices.contains("take that as intended"),
768 "Undo" => notices.contains("To start it again: planarian release"),
769 "alarm" => e.notices.iter().any(|n| n.starts_with("ALARM: ")),
770 "BudgetUnsatisfiable" => notices.contains("BudgetUnsatisfiable"),
771 _ => {
772 v.unchecked.push(format!("reports.{r}"));
773 continue;
774 }
775 };
776 if !ok {
777 fail(&mut v, format!("{r} was not reported"));
778 }
779 }
780 }
781 "reconciliation" => {
782 let text = ledger.last_reconciliation.as_ref().map(|r| r.actions.join("\n")).unwrap_or_default();
783 if let Some(found) = val.get("found").and_then(T::as_str)
784 && !text.contains(&format!("finished it: {found}"))
785 {
786 fail(&mut v, format!("record does not say the watchdog finished it as {found}: {text:?}"));
787 }
788 for token in val.get("explains").and_then(T::as_array).into_iter().flatten().filter_map(T::as_str) {
789 let ok = if let Some(state) = token.strip_prefix("killed-during-") {
790 text.contains(state)
791 } else if token == "watchdog-reverted" {
792 text.contains("watchdog") && text.contains("Reverted")
793 } else if let Some(u) = token.strip_prefix("probe-failed:") {
794 text.contains(u)
795 } else if let Some(w) = token.strip_prefix("why:") {
796 text.contains(&format!("because recovery: {w}")) || text.contains(&format!("because {w}"))
797 } else {
798 v.unchecked.push(format!("reconciliation.explains {token}"));
799 continue;
800 };
801 if !ok {
802 fail(&mut v, format!("the record does not explain {token}: {text:?}"));
803 }
804 }
805 }
806 "txn" => {
807 let specs: Vec<&T> = match val {
808 T::Array(a) => a.iter().collect(),
809 one => vec![one],
810 };
811 for spec in specs {
812 check_txn(e, &all, spec, &mut v);
813 }
814 }
815 "one_of" => {
817 let alts: Vec<Verdict> = val.as_array().into_iter().flatten().map(|alt| judge(e, alt)).collect();
818 if !alts.iter().any(|a| a.failures.is_empty() && a.unchecked.is_empty()) {
819 if alts.iter().any(|a| a.failures.is_empty()) {
820 v.unchecked.push(format!("{key} (an alternative with unchecked keys)"));
821 } else {
822 let why: Vec<String> = alts.iter().map(|a| a.failures.join("; ")).collect();
823 fail(&mut v, format!("no alternative holds: {}", why.join(" | ")));
824 }
825 }
826 }
827 "canonical_probe_source" if val.as_str() == Some("core") => {
830 let canonical: BTreeMap<String, Vec<ProbeDef>> =
831 serde_json::from_str(include_str!("../../../share/canonical.json")).unwrap_or_default();
832 let ids: BTreeSet<String> = canonical.values().flatten().map(|p| p.id.0.clone()).collect();
833 let seen = e
834 .notices
835 .iter()
836 .any(|n| n.contains(": Degraded") && ids.iter().any(|id| n.contains(&format!("{id}: "))));
837 if !seen {
838 fail(&mut v, "no failure was reported by a canonical probe".into());
839 }
840 }
841 "incident" => {
842 for spec in val.as_array().into_iter().flatten() {
843 check_incident(e, spec, &mut v);
844 }
845 }
846 "bound_driver" => match &e.drivers {
847 None => v.unchecked.push(key.clone()),
848 Some(d) => {
849 for (dev, want) in val.as_table().into_iter().flatten() {
850 let got = d.get(&unit(dev)).map(String::as_str).unwrap_or("none");
851 if got != want.as_str().unwrap_or("") {
852 fail(&mut v, format!("{dev} is bound to {got}, want {want}"));
853 }
854 }
855 }
856 },
857 "evaluated_option" => match &e.options {
858 None => v.unchecked.push(key.clone()),
859 Some(o) => {
860 for (opt, want) in val.as_table().into_iter().flatten() {
861 let got = o.get(opt).map(value_str);
862 if got.as_deref() != Some(toml_str(want).as_str()) {
863 fail(&mut v, format!("{opt} is {got:?}, want {}", toml_str(want)));
864 }
865 }
866 }
867 },
868 "probe_definitions_unchanged" if val.as_bool() == Some(true) => match &e.probe_defs {
869 None => v.unchecked.push(key.clone()),
870 Some((before, after)) if before != after => fail(&mut v, "probe definitions changed".into()),
871 Some(_) => {}
872 },
873 "gc_root_released" if val.as_bool() == Some(true) => match &e.gc_roots {
874 None => v.unchecked.push(key.clone()),
875 Some((before, after)) => {
876 let left: Vec<&String> = after.difference(before).collect();
877 if !left.is_empty() {
878 fail(&mut v, format!("build roots left behind: {left:?}"));
879 }
880 }
881 },
882 "replay_equivalent" if val.as_bool() == Some(true) => match &e.full_replay {
883 None => v.unchecked.push(format!("{key} (no compaction seen)")),
884 Some(full) if full != ledger => {
885 let a = serde_json::to_value(full).unwrap_or_default();
886 let b = serde_json::to_value(ledger).unwrap_or_default();
887 let differ: Vec<&String> = a
888 .as_object()
889 .into_iter()
890 .flatten()
891 .filter(|(k, x)| b.get(k.as_str()) != Some(x))
892 .map(|(k, _)| k)
893 .collect();
894 fail(&mut v, format!("checkpoint + tail differs from the full history in {differ:?}"));
895 }
896 Some(_) => {}
897 },
898 "no_overlap" => match &e.effects {
899 None => v.unchecked.push(key.clone()),
900 Some(fx) => {
901 let names: Vec<&str> = val.as_array().into_iter().flatten().filter_map(T::as_str).collect();
902 let named: Vec<&Effect> = fx.iter().filter(|f| names.contains(&f.call.as_str())).collect();
903 for (i, a) in named.iter().enumerate() {
904 for b in &named[i + 1..] {
905 if a.call != b.call && overlap(a, b) {
906 fail(&mut v, format!("{} and {} overlapped", describe(a), describe(b)));
907 }
908 }
909 }
910 }
911 },
912 "quiescence_includes_tagged_jobs" if val.as_bool() == Some(true) => match &e.effects {
913 None => v.unchecked.push(key.clone()),
914 Some(fx) => {
915 let muts: Vec<&Effect> =
918 fx.iter().filter(|f| !f.call.starts_with("cancel") && !f.call.starts_with("protect")).collect();
919 for a in &muts {
920 for b in &muts {
921 if a.txn == b.txn && a.epoch < b.epoch && overlap(a, b) {
922 fail(&mut v, format!("{} ran during {}", describe(b), describe(a)));
923 }
924 }
925 }
926 }
927 },
928 "stale_owner_commit" if val.as_bool() == Some(false) => match &e.entries {
929 None => v.unchecked.push(key.clone()),
930 Some(en) => {
931 let mut epoch: BTreeMap<TxnId, u64> = BTreeMap::new();
932 for x in en {
933 match &x.record {
934 Record::Prepared(t) => {
935 epoch.insert(t.id, t.lease.epoch);
936 }
937 Record::Takeover { txn, lease, .. } => {
938 epoch.insert(*txn, lease.epoch);
939 }
940 Record::Advance { txn, epoch: by, to, .. }
941 if matches!(to, TxnState::Published | TxnState::Promoted | TxnState::Committed)
942 && epoch.get(txn).is_some_and(|cur| by < cur) =>
943 {
944 fail(&mut v, format!("txn {} advanced to {} by stale epoch {by}", txn.0, tag(to)));
945 }
946 _ => {}
947 }
948 }
949 if let Some(fx) = &e.effects {
952 for f in fx {
953 let begun = en.iter().position(|x| {
954 matches!(&x.record, Record::OpBegin { op, .. }
955 if op.txn == f.txn && op.epoch == f.epoch && op.seq == f.seq)
956 });
957 let stale = begun.is_some_and(|b| {
958 en[..b].iter().any(|x| {
959 matches!(&x.record, Record::Takeover { txn, lease, .. }
960 if *txn == f.txn && lease.epoch > f.epoch)
961 })
962 });
963 if stale {
964 fail(&mut v, format!("{} by a fenced owner went through", describe(f)));
965 }
966 }
967 }
968 }
969 },
970 "lease_epochs_strictly_increasing" | "owners_per_epoch_max" => match &e.entries {
971 None => v.unchecked.push(key.clone()),
972 Some(en) => {
973 let mut seq: BTreeMap<TxnId, Vec<u64>> = BTreeMap::new();
974 let mut owners: BTreeMap<(TxnId, u64), BTreeSet<String>> = BTreeMap::new();
975 for x in en {
976 let claim = match &x.record {
977 Record::Prepared(t) => Some(&t.lease),
978 Record::Takeover { lease, .. } => Some(lease),
979 _ => None,
980 };
981 if let Some(l) = claim {
982 seq.entry(l.txn).or_default().push(l.epoch);
983 owners.entry((l.txn, l.epoch)).or_default().insert(format!("{:?}", l.owner));
984 }
985 }
986 if key == "lease_epochs_strictly_increasing" && val.as_bool() == Some(true) {
987 for (t, s) in &seq {
988 if s.windows(2).any(|w| w[1] <= w[0]) {
989 fail(&mut v, format!("txn {} epochs {s:?}", t.0));
990 }
991 }
992 }
993 if key == "owners_per_epoch_max" {
994 let max = val.as_integer().unwrap_or(1) as usize;
995 for ((t, ep), o) in &owners {
996 if o.len() > max {
997 fail(&mut v, format!("txn {} epoch {ep} had {} owners", t.0, o.len()));
998 }
999 }
1000 }
1001 }
1002 },
1003 "journal_owner_after_takeover" => match &e.entries {
1006 None => v.unchecked.push(key.clone()),
1007 Some(en) => {
1008 let want = val.as_str().unwrap_or("");
1009 let mut held: BTreeMap<TxnId, u64> = BTreeMap::new();
1010 let mut took = false;
1011 for x in en {
1012 match &x.record {
1013 Record::Takeover { txn, lease, .. } => {
1014 took = true;
1015 if (want == "watchdog") != lease.owner.is_watchdog() {
1016 fail(&mut v, format!("txn {} taken over by {:?}", txn.0, lease.owner));
1017 }
1018 held.insert(*txn, lease.epoch);
1019 }
1020 Record::Advance { txn, epoch, to, .. } => {
1021 if let Some(&ep) = held.get(txn)
1022 && *epoch != ep
1023 {
1024 fail(
1025 &mut v,
1026 format!(
1027 "txn {} advanced to {to:?} at epoch {epoch} after takeover {ep}",
1028 txn.0
1029 ),
1030 );
1031 }
1032 }
1033 Record::OpBegin { op, call } => {
1034 if let Some(&ep) = held.get(&op.txn)
1035 && op.epoch != ep
1036 {
1037 fail(&mut v, format!("{call} journaled at epoch {} after takeover {ep}", op.epoch));
1038 }
1039 }
1040 _ => {}
1041 }
1042 }
1043 if !took {
1044 fail(&mut v, "no takeover in the journal".into());
1045 }
1046 }
1047 },
1048 "journal_order" => match &e.entries {
1049 None => v.unchecked.push(key.clone()),
1050 Some(en) => {
1051 let mut last = None;
1052 for token in val.as_array().into_iter().flatten().filter_map(T::as_str) {
1053 match position(en, token) {
1054 None => fail(&mut v, format!("no {token} in the journal")),
1055 Some(p) if last.is_some_and(|l| p <= l) => {
1056 fail(&mut v, format!("{token} is out of order"));
1057 }
1058 Some(p) => last = Some(p),
1059 }
1060 }
1061 }
1062 },
1063 "watchdog_probed_before_commit" if val.as_bool() == Some(true) => match &e.entries {
1064 None => v.unchecked.push(key.clone()),
1065 Some(en) => {
1066 let mut watchdog: BTreeSet<(TxnId, u64)> = BTreeSet::new();
1069 let mut ok = false;
1070 for x in en {
1071 match &x.record {
1072 Record::Takeover { txn, lease, .. } if lease.owner.is_watchdog() => {
1073 watchdog.insert((*txn, lease.epoch));
1074 }
1075 Record::Advance { txn, epoch, to: TxnState::Published, note, .. }
1076 if watchdog.contains(&(*txn, *epoch)) && note == "probes passed" =>
1077 {
1078 ok = true;
1079 }
1080 _ => {}
1081 }
1082 }
1083 if !ok {
1084 fail(&mut v, "the watchdog did not probe before publishing".into());
1085 }
1086 }
1087 },
1088 other => v.unchecked.push(other.to_string()),
1089 }
1090 }
1091 v
1092}
1093
1094fn value_str(v: &Value) -> String {
1095 match v {
1096 Value::Bool(b) => b.to_string(),
1097 Value::Int(i) => i.to_string(),
1098 Value::Str(s) => s.clone(),
1099 Value::List(l) => format!("[{}]", l.iter().map(value_str).collect::<Vec<_>>().join(", ")),
1100 }
1101}
1102
1103fn toml_str(t: &T) -> String {
1104 match t {
1105 T::String(s) => s.clone(),
1106 other => other.to_string(),
1107 }
1108}
1109
1110fn overlap(a: &Effect, b: &Effect) -> bool {
1113 let end = |f: &Effect| f.end.unwrap_or(Timestamp(u64::MAX));
1114 a.start < end(b) && b.start < end(a)
1115}
1116
1117fn describe(f: &Effect) -> String {
1118 format!("{} (txn {} epoch {}, {}–{:?} ms)", f.call, f.txn.0, f.epoch, f.start.0, f.end.map(|t| t.0))
1119}
1120
1121fn position(en: &[Entry], token: &str) -> Option<usize> {
1124 let (what, u) = token.strip_suffix(')')?.split_once('(')?;
1125 let id = unit(u);
1126 let mut on: BTreeMap<TxnId, Option<UnitId>> = BTreeMap::new();
1127 en.iter().position(|x| match &x.record {
1128 Record::Prepared(t) => {
1129 on.insert(t.id, target_of(&t.transition));
1130 false
1131 }
1132 Record::OverrideSet(s) => what == "Override" && s.unit == id,
1133 Record::OpBegin { op, call } => {
1134 what != "Override" && call.starts_with(what) && on.get(&op.txn).cloned().flatten() == Some(id.clone())
1135 }
1136 _ => false,
1137 })
1138}
1139
1140fn check_txn(e: &Evidence, all: &[TxnView], spec: &T, v: &mut Verdict) {
1141 let s = |k: &str| spec.get(k).and_then(T::as_str);
1142 let mut matching: Vec<&TxnView> = all.iter().collect();
1143 if let Some(t) = s("transition") {
1144 matching.retain(|x| transition_name(&x.transition) == t);
1145 }
1146 if let Some(u) = s("target") {
1147 let id = unit(u);
1148 matching.retain(|x| target_of(&x.transition).as_ref() == Some(&id));
1149 }
1150 if let Some(c) = s("change") {
1151 matching.retain(|x| matches!(&x.transition, Transition::ApplyChange { change, .. } if change.0 == c));
1152 }
1153 if let Some(want) = s("path_option") {
1154 matching.retain(|x| matches!(&x.transition, Transition::SetRuntimeOption { path, .. } if path == want));
1155 }
1156 if let Some(want) = spec.get("args").and_then(T::as_table) {
1159 matching.retain(|x| match &x.transition {
1160 Transition::ApplyChange { args, .. } => {
1161 want.iter().all(|(k, w)| args.get(k).map(value_str).as_deref() == Some(toml_str(w).as_str()))
1162 }
1163 _ => false,
1164 });
1165 }
1166 if let Some(want) = spec.get("emergency").and_then(T::as_bool) {
1169 matching.retain(|x| matches!(x.transition, Transition::CollectGarbage { emergency } if emergency == want));
1170 }
1171 let label = format!(
1172 "txn {}",
1173 spec.as_table()
1174 .map(|t| t.iter().map(|(k, v)| format!("{k}={v}")).collect::<Vec<_>>().join(" "))
1175 .unwrap_or_default()
1176 );
1177 if let Some(during) = spec.get("not_during").and_then(T::as_table) {
1183 match &e.entries {
1184 None => v.unchecked.push(format!("{label}: not_during (journal history)")),
1185 Some(en) => {
1186 let other = during.get("transition").and_then(T::as_str).unwrap_or("");
1187 let state = during.get("state").and_then(T::as_str).unwrap_or("");
1188 let others: BTreeSet<TxnId> =
1189 all.iter().filter(|x| transition_name(&x.transition) == other).map(|x| x.id).collect();
1190 let mine: BTreeSet<TxnId> = matching.iter().map(|x| x.id).collect();
1191 let mut inside: BTreeSet<TxnId> = BTreeSet::new();
1192 let mut boot_only: BTreeSet<TxnId> = BTreeSet::new();
1193 let mut saw_state = false;
1194 for x in en {
1195 match &x.record {
1196 Record::Advance { txn, to, .. } if others.contains(txn) => {
1197 if *to == TxnState::AwaitingBoot {
1198 boot_only.insert(*txn);
1199 }
1200 let held = if state == "exclusive" {
1201 to.is_exclusive() && !(*to == TxnState::Promoted && boot_only.contains(txn))
1202 } else {
1203 state_name(to) == state
1204 };
1205 if held {
1206 inside.insert(*txn);
1207 saw_state = true;
1208 } else {
1209 inside.remove(txn);
1210 }
1211 }
1212 Record::Prepared(p) if mine.contains(&p.id) && !inside.is_empty() => {
1213 v.failures.push(format!("{label}: txn {} prepared while {other} was {state}", p.id.0));
1214 }
1215 _ => {}
1216 }
1217 }
1218 if !saw_state {
1219 v.failures.push(format!("{label}: no {other} was ever {state}"));
1220 }
1221 }
1222 }
1223 if spec.get("terminal").is_none() {
1224 return;
1225 }
1226 }
1227 if let Some(why) = s("refused_with") {
1230 let refused = e.log.iter().any(|r| {
1231 s("transition").is_none_or(|t| transition_name(&r.proposal.transition) == t)
1232 && s("target").is_none_or(|u| target_of(&r.proposal.transition) == Some(unit(u)))
1233 && s("caller").is_none_or(|c| caller_is(r, c))
1234 && s("reject_detail")
1235 .is_none_or(|d| matches!(&r.result, Submitted::Rejected { detail, .. } if detail.contains(d)))
1236 && (matches!(&r.result, Submitted::Rejected { reason, .. } if format!("{reason:?}") == why)
1237 || matches!(&r.decision, Some(Decision::Reject(reason)) if format!("{reason:?}") == why))
1238 });
1239 if !refused {
1240 v.failures.push(format!("{label}: not refused with {why}"));
1241 }
1242 let by_caller = |id: TxnId| {
1244 s("caller").is_none_or(|c| {
1245 e.log.iter().any(|r| {
1246 matches!(&r.result, Submitted::Finished { txn, .. } | Submitted::Parked { txn, .. } if *txn == id)
1247 && caller_is(r, c)
1248 })
1249 })
1250 };
1251 for t in matching.iter().filter(|t| by_caller(t.id)) {
1252 if t.path.iter().any(|p| p == "Activating") {
1253 v.failures.push(format!("{label}: txn {} ran", t.id.0));
1254 }
1255 }
1256 return;
1257 }
1258 let count = spec.get("count").and_then(T::as_integer);
1259 if count == Some(0) {
1260 if spec.get("before_unit_running").is_some()
1261 || spec.get("while_interlock").is_some()
1262 || spec.get("while_incident").is_some()
1263 || spec.get("while_foreground").is_some()
1264 {
1265 conditional_none(e, spec, &label, v);
1266 } else if !matching.is_empty() {
1267 v.failures.push(format!("{label}: expected none, found {}", matching.len()));
1268 }
1269 return;
1270 }
1271 if matching.is_empty() {
1272 let wanted: Vec<String> = s("terminal")
1276 .map(|t| vec![t.to_string()])
1277 .or_else(|| {
1278 s("decision").filter(|d| d.starts_with("Reject")).map(|d| vec![d.replace("Reject(", "Rejected(")])
1279 })
1280 .or_else(|| {
1281 spec.get("decision_in")
1282 .and_then(T::as_array)
1283 .map(|a| a.iter().filter_map(T::as_str).map(|d| d.replace("Reject(", "Rejected(")).collect())
1284 })
1285 .unwrap_or_default();
1286 let refused = e.log.iter().any(|r| {
1287 s("transition").is_none_or(|t| transition_name(&r.proposal.transition) == t)
1288 && s("caller").is_none_or(|c| caller_is(r, c))
1289 && match &r.result {
1290 Submitted::Rejected { reason, .. } => wanted.iter().any(|w| *w == format!("Rejected({reason:?})")),
1291 _ => false,
1292 }
1293 });
1294 if !refused {
1295 v.failures.push(format!("{label}: no such transaction or refusal"));
1296 }
1297 return;
1298 }
1299 let t = matching.last().expect("non-empty");
1301 if let Some(c) = s("caller") {
1302 let by = e.log.iter().any(|r| {
1303 matches!(&r.result, Submitted::Finished { txn, .. } | Submitted::Parked { txn, .. } if *txn == t.id)
1304 && caller_is(r, c)
1305 });
1306 if !by {
1307 v.failures.push(format!("{label}: not submitted by {c}"));
1308 }
1309 }
1310 if let Some(want) = s("terminal") {
1311 let got = state_name(&t.state);
1312 let pending = want == "pending" && got == "Prepared";
1314 if got != want && !pending {
1315 v.failures.push(format!("{label}: ended {got}"));
1316 }
1317 }
1318 if let Some(want) = s("approver_class") {
1319 let class = e.log.iter().find_map(|r| match (&r.result, &r.decision) {
1320 (Submitted::Parked { txn, .. }, Some(Decision::Allow { approver, .. })) if *txn == t.id => Some(*approver),
1321 _ => None,
1322 });
1323 match class {
1324 Some(c) if format!("{c:?}") == want => {}
1325 Some(c) => v.failures.push(format!("{label}: approver class {c:?}")),
1326 None => v.failures.push(format!("{label}: never asked for approval")),
1327 }
1328 }
1329 if spec.get("while_admitted").and_then(T::as_bool) == Some(true) {
1332 match e.entries.as_ref() {
1333 None => v.unchecked.push(format!("{label}: while_admitted (journal history)")),
1334 Some(en) => {
1335 let mut awaited: BTreeSet<TxnId> = BTreeSet::new();
1336 let mut admitted: BTreeSet<TxnId> = BTreeSet::new();
1337 let mut inside = false;
1338 for x in en {
1339 let Record::Advance { txn, to, .. } = &x.record else { continue };
1340 match to {
1341 TxnState::AwaitingBoot => {
1342 awaited.insert(*txn);
1343 }
1344 TxnState::Promoted if awaited.contains(txn) => {
1345 admitted.insert(*txn);
1346 }
1347 _ if admitted.contains(txn) => {
1348 admitted.remove(txn);
1349 }
1350 TxnState::Committed if *txn == t.id => inside = !admitted.is_empty(),
1351 _ => {}
1352 }
1353 }
1354 if !inside {
1355 v.failures.push(format!("{label}: did not commit while a boot-only change was admitted"));
1356 }
1357 }
1358 }
1359 }
1360 if let Some(want) = s("terminal_prefix")
1361 && !state_name(&t.state).starts_with(want)
1362 {
1363 v.failures.push(format!("{label}: ended {}", state_name(&t.state)));
1364 }
1365 if let Some(path) = spec.get("path").and_then(T::as_array) {
1366 let want: Vec<String> = path.iter().filter_map(T::as_str).map(String::from).collect();
1367 if !subsequence(&t.path, &want) {
1368 v.failures.push(format!("{label}: path {:?} lacks {want:?}", t.path));
1369 }
1370 }
1371 if let Some(never) = spec.get("never").and_then(T::as_array) {
1372 for n in never.iter().filter_map(T::as_str) {
1373 if t.path.iter().any(|p| p == n) || state_name(&t.state) == n {
1374 v.failures.push(format!("{label}: went through {n}"));
1375 }
1376 }
1377 }
1378 if let Some(min) = spec.get("lease_epoch_min").and_then(T::as_integer)
1379 && (t.epoch as i64) < min
1380 {
1381 v.failures.push(format!("{label}: epoch {} < {min}", t.epoch));
1382 }
1383 if let Some(src) = s("source") {
1384 let target = target_of(&t.transition);
1385 let found = e.log.iter().any(|r| {
1386 transition_name(&r.proposal.transition) == transition_name(&t.transition)
1387 && target_of(&r.proposal.transition) == target
1388 && match &r.proposal.source {
1389 Source::Diagnoser { .. } => src == "diagnoser",
1390 Source::User => src == "user",
1391 Source::Intent { .. } => src == "intent",
1392 Source::Model(_) => src == "model",
1393 Source::Maintenance => src == "maintenance",
1394 }
1395 });
1396 if !found {
1397 v.failures.push(format!("{label}: not proposed by {src}"));
1398 }
1399 }
1400 if s("owner") == Some("watchdog") && !t.watchdog {
1401 v.failures.push(format!("{label}: not finished by the watchdog"));
1402 }
1403 if let Some(r) = s("recovery")
1404 && format!("{:?}", t.transition.recovery()) != r
1405 {
1406 v.failures.push(format!("{label}: recovery {:?}", t.transition.recovery()));
1407 }
1408 match s("decision") {
1409 Some("Allow(AskFirst)") => {
1410 let parked = e.log.iter().any(|r| {
1411 transition_name(&r.proposal.transition) == transition_name(&t.transition)
1412 && matches!(r.result, Submitted::Parked { .. })
1413 });
1414 if !parked {
1415 v.failures.push(format!("{label}: never asked for approval"));
1416 }
1417 }
1418 Some(d) if d.starts_with("Allow(") => {
1419 let parked = e.log.iter().any(|r| {
1420 transition_name(&r.proposal.transition) == transition_name(&t.transition)
1421 && matches!(r.result, Submitted::Parked { .. })
1422 });
1423 if parked {
1424 v.failures.push(format!("{label}: asked for approval, want {d}"));
1425 }
1426 }
1427 Some(d) => v.unchecked.push(format!("{label}: decision {d}")),
1428 None => {}
1429 }
1430 if let Some(want) = spec.get("facts").and_then(T::as_table) {
1431 let facts = e.log.iter().find_map(|r| match &r.result {
1432 Submitted::Finished { txn, .. } | Submitted::Parked { txn, .. } if *txn == t.id => r.facts.clone(),
1433 _ => None,
1434 });
1435 match facts {
1436 None => v.failures.push(format!("{label}: no derived facts in the executor log")),
1437 Some(f) => {
1438 let got = serde_json::to_value(&f).unwrap_or_default();
1439 for (k, w) in want {
1440 if k == "disruption" {
1443 let kinds: Option<Vec<Disruption>> = match w {
1444 T::String(_) => serde_json::to_value(w)
1445 .ok()
1446 .and_then(|x| serde_json::from_value(x).ok())
1447 .map(|d| vec![d]),
1448 T::Array(_) => serde_json::to_value(w).ok().and_then(|x| serde_json::from_value(x).ok()),
1449 _ => None,
1450 };
1451 let ok = match (&kinds, w) {
1452 (Some(ks), T::String(_)) => ks.iter().all(|d| f.disruption.contains(*d)),
1453 (Some(ks), _) => {
1454 f.disruption.members() == {
1455 let mut ks = ks.clone();
1456 ks.sort();
1457 ks.dedup();
1458 ks.retain(|d| *d != Disruption::None);
1459 ks
1460 }
1461 }
1462 (None, _) => false,
1463 };
1464 if !ok {
1465 v.failures
1466 .push(format!("{label}: facts.disruption is {:?}, want {w}", f.disruption.members()));
1467 }
1468 continue;
1469 }
1470 let w = serde_json::to_value(w).unwrap_or_default();
1471 if got.get(k) != Some(&w) {
1472 v.failures.push(format!("{label}: facts.{k} is {:?}, want {w}", got.get(k)));
1473 }
1474 }
1475 }
1476 }
1477 }
1478 if let Some(want) = spec.get("affected_includes").and_then(T::as_array) {
1481 let facts = e.log.iter().find_map(|r| match &r.result {
1482 Submitted::Finished { txn, .. } | Submitted::Parked { txn, .. } if *txn == t.id => r.facts.clone(),
1483 _ => None,
1484 });
1485 match facts {
1486 None => v.failures.push(format!("{label}: no derived facts in the executor log")),
1487 Some(f) => {
1488 for u in want.iter().filter_map(T::as_str) {
1489 if !f.affected_units.contains(&unit(u)) {
1490 v.failures.push(format!("{label}: {u} not among the affected units {:?}", f.affected_units));
1491 }
1492 }
1493 }
1494 }
1495 }
1496 if s("not_activated_before") == Some("approval") {
1498 let parked = e.log.iter().any(|r| matches!(&r.result, Submitted::Parked { txn, .. } if *txn == t.id));
1499 if !parked {
1500 v.failures.push(format!("{label}: never waited for approval"));
1501 }
1502 }
1503 if let Some(want) = s("tier") {
1506 let rec = e.log.iter().find(
1507 |r| matches!(&r.result, Submitted::Finished { txn, .. } | Submitted::Parked { txn, .. } if *txn == t.id),
1508 );
1509 let got = match rec {
1510 Some(r) if matches!(r.result, Submitted::Parked { .. }) => Some("AskFirst".to_string()),
1511 Some(r) => match &r.decision {
1512 Some(Decision::Allow { tier, .. }) => Some(format!("{tier:?}")),
1513 _ => None,
1514 },
1515 None => None,
1516 };
1517 if got.as_deref() != Some(want) {
1518 v.failures.push(format!("{label}: decided at {got:?}"));
1519 }
1520 }
1521 if let Some(after) = spec.get("started_after").and_then(T::as_table) {
1522 match &e.entries {
1523 None => v.unchecked.push(format!("{label}: started_after (journal history)")),
1524 Some(en) => {
1525 let other = after.get("transition").and_then(T::as_str).unwrap_or("");
1526 let left = after.get("left").and_then(T::as_str).unwrap_or("");
1527 let others: BTreeSet<TxnId> =
1528 all.iter().filter(|x| transition_name(&x.transition) == other).map(|x| x.id).collect();
1529 let mut inside = BTreeSet::new();
1531 let mut out = None;
1532 for (i, x) in en.iter().enumerate() {
1533 if let Record::Advance { txn, to, .. } = &x.record
1534 && others.contains(txn)
1535 {
1536 if state_name(to) == left {
1537 inside.insert(*txn);
1538 } else if inside.contains(txn) && out.is_none() {
1539 out = Some(i);
1540 }
1541 }
1542 }
1543 let began = en.iter().position(|x| matches!(&x.record, Record::Prepared(p) if p.id == t.id));
1544 match (out, began) {
1545 (None, _) => v.failures.push(format!("{label}: no {other} left {left}")),
1546 (_, None) => v.failures.push(format!("{label}: not in the journal")),
1547 (Some(o), Some(b)) if b < o => {
1548 v.failures.push(format!("{label}: prepared while {other} was still in {left}"))
1549 }
1550 _ => {}
1551 }
1552 }
1553 }
1554 }
1555 if spec.get("boot_entries_unchanged").and_then(T::as_bool) == Some(true) {
1557 match &e.entries {
1558 None => v.unchecked.push(format!("{label}: boot_entries_unchanged (journal history)")),
1559 Some(en) => {
1560 if en.iter().any(|x| {
1561 matches!(&x.record, Record::OpBegin { op, call } if op.txn == t.id && call.starts_with("publish_boot"))
1562 }) {
1563 v.failures.push(format!("{label}: wrote a boot entry"));
1564 }
1565 }
1566 }
1567 }
1568 for k in ["returns_current_digest", "reject_detail", "decision_in", "decision_not_below"] {
1569 if spec.get(k).is_some() {
1570 v.unchecked.push(format!("{label}: {k}"));
1571 }
1572 }
1573 for k in spec.as_table().into_iter().flat_map(|t| t.keys()) {
1575 if !TXN_KEYS.contains(&k.as_str()) {
1576 v.unchecked.push(format!("{label}: {k}"));
1577 }
1578 }
1579}
1580
1581const TXN_KEYS: &[&str] = &[
1582 "not_during",
1583 "affected_includes",
1584 "transition",
1585 "target",
1586 "change",
1587 "path_option",
1588 "args",
1589 "emergency",
1590 "count",
1591 "before_unit_running",
1592 "while_interlock",
1593 "while_incident",
1594 "terminal",
1595 "decision",
1596 "decision_in",
1597 "caller",
1598 "approver_class",
1599 "terminal_prefix",
1600 "path",
1601 "never",
1602 "lease_epoch_min",
1603 "source",
1604 "owner",
1605 "recovery",
1606 "facts",
1607 "not_activated_before",
1608 "returns_current_digest",
1609 "reject_detail",
1610 "decision_not_below",
1611 "refused_with",
1612 "while_admitted",
1613 "tier",
1614 "started_after",
1615 "boot_entries_unchanged",
1616];
1617
1618fn conditional_none(e: &Evidence, spec: &T, label: &str, v: &mut Verdict) {
1621 let s = |k: &str| spec.get(k).and_then(T::as_str);
1622 let incident_target: BTreeMap<IncidentId, UnitId> =
1623 e.seen_incidents.values().map(|i| (i.id, i.target.clone())).collect();
1624 let went: Vec<&planarian_daemon::LogRecord> = e
1625 .log
1626 .iter()
1627 .filter(|r| matches!(r.result, Submitted::Finished { .. } | Submitted::Parked { .. }))
1628 .filter(|r| s("transition").is_none_or(|t| transition_name(&r.proposal.transition) == t))
1629 .filter(|r| s("target").is_none_or(|u| target_of(&r.proposal.transition) == Some(unit(u))))
1630 .filter(|r| {
1631 s("source").is_none_or(|src| match &r.proposal.source {
1632 Source::Diagnoser { .. } => src == "diagnoser",
1633 Source::User => src == "user",
1634 Source::Intent { .. } => src == "intent",
1635 Source::Model(_) => src == "model",
1636 Source::Maintenance => src == "maintenance",
1637 })
1638 })
1639 .collect();
1640 if let Some(u) = s("before_unit_running") {
1641 let Some(line) = e.status_timeline.as_ref() else {
1642 v.unchecked.push(format!("{label} (status timeline)"));
1643 return;
1644 };
1645 let line = line.get(&unit(u)).cloned().unwrap_or_default();
1646 for r in &went {
1647 let at = line.iter().rev().find(|(t, _)| *t <= r.at).map(|(_, st)| st.as_str()).unwrap_or("unknown");
1648 if at != "Running" {
1649 v.failures.push(format!("{label}: proposed at {} ms while {u} was {at}", r.at.0));
1650 }
1651 }
1652 }
1653 if let Some(session) = s("while_foreground") {
1654 let Some(spans) = foreground_spans(e, session) else {
1655 v.unchecked.push(format!("{label} (foreground steps)"));
1656 return;
1657 };
1658 for r in &went {
1659 if spans.iter().any(|(a, b)| *a <= r.at && r.at < *b) {
1660 v.failures.push(format!("{label}: proposed at {} ms while Foreground held in {session}", r.at.0));
1661 }
1662 }
1663 }
1664 if s("while_interlock") == Some("set") || s("while_incident").is_some() {
1665 let Some(en) = e.entries.as_ref() else {
1666 v.unchecked.push(format!("{label} (journal history)"));
1667 return;
1668 };
1669 let mut spans: Vec<(Timestamp, Timestamp)> = vec![];
1671 let mut since: Option<Timestamp> = None;
1672 let target = s("target").map(unit);
1673 for x in en {
1674 let holds = match (&x.record, s("while_incident")) {
1675 (Record::Advance { to: TxnState::RecoveryFailed, .. }, None) => Some(true),
1676 (Record::Resolved { .. }, None) => Some(false),
1677 (Record::Incident(i), Some(state)) if Some(&i.target) == target.as_ref() => {
1678 Some(format!("{:?}", i.state).starts_with(state))
1679 }
1680 _ => None,
1681 };
1682 match (holds, since) {
1683 (Some(true), None) => since = Some(x.at),
1684 (Some(false), Some(t0)) => {
1685 spans.push((t0, x.at));
1686 since = None;
1687 }
1688 _ => {}
1689 }
1690 }
1691 if let Some(t0) = since {
1692 spans.push((t0, Timestamp(u64::MAX)));
1693 }
1694 for r in &went {
1695 if spans.iter().any(|(a, b)| *a <= r.at && r.at < *b) {
1696 let inc = match &r.proposal.source {
1697 Source::Diagnoser { incident } => incident_target.get(incident).map(|u| u.to_string()),
1698 _ => None,
1699 };
1700 v.failures.push(format!(
1701 "{label}: proposed at {} ms while the condition held{}",
1702 r.at.0,
1703 inc.map(|u| format!(" (incident on {u})")).unwrap_or_default()
1704 ));
1705 }
1706 }
1707 }
1708}
1709
1710fn foreground_spans(e: &Evidence, session: &str) -> Option<Vec<(Timestamp, Timestamp)>> {
1713 let steps = e.foreground.as_ref()?;
1714 let mut held: BTreeSet<&str> = BTreeSet::new();
1715 let mut spans = vec![];
1716 let mut since: Option<Timestamp> = None;
1717 for (at, s, on) in steps {
1718 if session != "any" && s != session {
1719 continue;
1720 }
1721 if *on {
1722 held.insert(s);
1723 } else {
1724 held.remove(s.as_str());
1725 }
1726 match (held.is_empty(), since) {
1727 (false, None) => since = Some(*at),
1728 (true, Some(t0)) => {
1729 spans.push((t0, *at));
1730 since = None;
1731 }
1732 _ => {}
1733 }
1734 }
1735 if let Some(t0) = since {
1736 spans.push((t0, Timestamp(u64::MAX)));
1737 }
1738 Some(spans)
1739}
1740
1741fn check_incident(e: &Evidence, spec: &T, v: &mut Verdict) {
1742 let Some(target) = spec.get("target").and_then(T::as_str) else { return };
1743 let id = unit(target);
1744 let label = format!("incident on {target}");
1745 let found: Vec<&Incident> = e.seen_incidents.values().filter(|i| same_unit(&id, &i.target)).collect();
1746 let Some(last) = found.last() else {
1747 v.failures.push(format!("{label}: none opened"));
1748 return;
1749 };
1750 if let Some(state) = spec.get("state").and_then(T::as_str) {
1751 let current = e.open_incidents.iter().find(|(k, _)| same_unit(&id, k)).map(|(_, i)| i.clone());
1752 let got = match ¤t {
1753 None => "Closed".to_string(),
1754 Some(i) => format!("{:?}", i.state).split([' ', '{', '(']).next().unwrap_or("").to_string(),
1755 };
1756 if got != state {
1757 v.failures.push(format!("{label}: is {got}, want {state}"));
1758 }
1759 }
1760 if let Some(want) = spec.get("escalation").and_then(T::as_str) {
1761 let ok = found
1762 .iter()
1763 .any(|i| matches!(&i.state, IncidentState::Escalated { reason, .. } if format!("{reason:?}") == want));
1764 if !ok {
1765 v.failures.push(format!("{label}: not escalated with {want}"));
1766 }
1767 }
1768 if let Some(n) = spec.get("regressed_count").and_then(T::as_integer)
1769 && found.iter().map(|i| i.regressions as i64).max().unwrap_or(0) < n
1770 {
1771 v.failures.push(format!("{label}: regressions below {n}"));
1772 }
1773 if let Some(n) = spec.get("budget_left").and_then(T::as_integer)
1774 && last.budget_left as i64 != n
1775 {
1776 v.failures.push(format!("{label}: budget {}", last.budget_left));
1777 }
1778 if let Some(session) = spec.get("budget_steady_while_foreground").and_then(T::as_str) {
1782 match (foreground_spans(e, session), &e.entries) {
1783 (Some(spans), Some(en)) => {
1784 for (a, b) in spans {
1785 let mut at_start: Option<u8> = None;
1786 for x in en {
1787 let Record::Incident(i) = &x.record else { continue };
1788 if !same_unit(&id, &i.target) {
1789 continue;
1790 }
1791 if x.at < a {
1792 at_start = Some(i.budget_left);
1793 } else if x.at < b {
1794 match at_start {
1795 None => at_start = Some(i.budget_left),
1796 Some(n) if n != i.budget_left => v.failures.push(format!(
1797 "{label}: budget {n} → {} at {} ms while Foreground held",
1798 i.budget_left, x.at.0
1799 )),
1800 _ => {}
1801 }
1802 }
1803 }
1804 }
1805 }
1806 _ => v.unchecked.push(format!("{label}: budget_steady_while_foreground (journal, foreground steps)")),
1807 }
1808 }
1809 if let Some(k) = spec.get("kind").and_then(T::as_str) {
1810 let want = if k == "diagnostic" { IncidentKind::Diagnostic } else { IncidentKind::Repair };
1811 if !found.iter().any(|i| i.kind == want) {
1812 v.failures.push(format!("{label}: no {k} incident"));
1813 }
1814 }
1815 if let Some(w) = spec.get("opened_within").and_then(T::as_str) {
1816 let limit = w.trim_end_matches('s').parse::<u64>().unwrap_or(0) * 1000;
1817 let at = e
1818 .first_incident_at
1819 .iter()
1820 .filter(|(k, _)| same_unit(&id, k))
1821 .map(|(_, t)| *t)
1822 .min()
1823 .unwrap_or(Timestamp(u64::MAX));
1824 if at.0.saturating_sub(e.start.0) > limit + 60_000 {
1825 v.unchecked.push(format!("{label}: opened_within (relative to inject)"));
1826 }
1827 }
1828 if let Some(of) = spec.get("root_of").and_then(T::as_array) {
1829 let by_incident: BTreeMap<IncidentId, UnitId> =
1832 e.seen_incidents.values().map(|i| (i.id, i.target.clone())).collect();
1833 let mut attributed: BTreeSet<UnitId> = BTreeSet::new();
1834 for r in &e.log {
1835 if let Source::Diagnoser { incident } = &r.proposal.source
1836 && let Some(t) = by_incident.get(incident)
1837 && *t != id
1838 && target_of(&r.proposal.transition).as_ref() == Some(&id)
1839 {
1840 attributed.insert(t.clone());
1841 }
1842 }
1843 let marker = format!("(root cause: {id})");
1844 for n in e.notices.iter().filter(|n| n.contains(&marker)) {
1845 if let Some((t, _)) = n.split_once(": ") {
1846 attributed.insert(unit(t));
1847 }
1848 }
1849 let want: BTreeSet<UnitId> = of.iter().filter_map(T::as_str).map(unit).collect();
1850 for u in want.difference(&attributed) {
1851 let seen =
1853 e.seen_status.get(u).map(|s| s.iter().cloned().collect::<Vec<_>>().join(", ")).unwrap_or_default();
1854 v.failures.push(format!("{label}: never named as the root cause of {u} (it was seen {seen})"));
1855 }
1856 for u in attributed.difference(&want) {
1857 v.failures.push(format!("{label}: named as the root cause of {u}, which it isn't"));
1858 }
1859 }
1860 if let Some(first) = spec.get("first_candidate").and_then(T::as_str) {
1861 match first.strip_prefix("reobserve:") {
1862 Some(probe) => {
1863 let said = e.notices.iter().any(|n| {
1864 n.starts_with(&format!("{id}: ")) && n.contains("re-observing through") && n.contains(probe)
1865 });
1866 let acted = e.log.iter().any(|r| {
1867 matches!(&r.proposal.source, Source::Diagnoser { incident } if found.iter().any(|i| i.id == *incident))
1868 });
1869 if !said {
1870 v.failures.push(format!("{label}: did not re-observe through {probe}"));
1871 }
1872 if acted {
1873 v.failures.push(format!("{label}: proposed a transition instead of re-observing"));
1874 }
1875 }
1876 None => v.unchecked.push(format!("{label}: first_candidate {first}")),
1877 }
1878 }
1879 if let Some(uid) = spec.get("session_bound").and_then(T::as_integer)
1882 && !e.notices.iter().any(|n| {
1883 found.iter().any(|i| n.starts_with(&format!("{}: ", i.target))) && n.contains(&format!("(uid {uid})"))
1884 })
1885 {
1886 v.failures.push(format!("{label}: not tied to the session of uid {uid}"));
1887 }
1888}
1889
1890fn caller_is(r: &planarian_daemon::LogRecord, want: &str) -> bool {
1897 match (want.strip_prefix("agent:"), &r.agent) {
1898 (Some(name), Some(a)) => a == name,
1899 (Some(name), None) => actor_is(&r.caller, &format!("agent:{name}")),
1900 (None, Some(_)) => false,
1901 (None, None) => actor_is(&r.caller, want),
1902 }
1903}
1904
1905fn actor_is(a: &Actor, want: &str) -> bool {
1906 match (a, want.split_once(':')) {
1907 (Actor::User { uid, .. }, None) => want == "root" && *uid == 0,
1908 (Actor::User { uid, session, .. }, Some(("user", n))) => match n.split_once('@') {
1909 Some((n, "-")) => n.parse() == Ok(*uid) && session.is_none(),
1910 Some((n, s)) => n.parse() == Ok(*uid) && session.as_deref() == Some(s),
1911 None => n.parse() == Ok(*uid),
1912 },
1913 (Actor::Agent(p), Some(("agent", name))) => *p == planarian_daemon::core::agent_principal(name),
1914 _ => false,
1915 }
1916}