planarian_gym/
metrics.rs

1//! The metrics §15 asks the gym to report, over a run of the catalog: repair
2//! rate, unnecessary interventions, disruption incurred, time to repair,
3//! restarts per target, and RecoveryFailed count.
4
5use crate::sim::Sim;
6use planarian_exec::Submitted;
7use planarian_schema::{Actor, Disruption, Transition, TxnState};
8use std::collections::BTreeMap;
9use std::fmt;
10use toml::Value as T;
11
12#[derive(Default)]
13pub struct Metrics {
14    runnable: usize,
15    passed: usize,
16    partial: usize,
17    repair_entries: usize,
18    repaired: usize,
19    intent_entries: usize,
20    unnecessary: usize,
21    disruption: BTreeMap<&'static str, usize>,
22    /// Seconds from injection to the last committed autonomous transaction,
23    /// for each repair entry that was cleared.
24    time_to_repair: Vec<u64>,
25    restarts: BTreeMap<String, usize>,
26    recovery_failed: usize,
27    recovery_failed_injected: usize,
28    recovery_failed_in: Vec<String>,
29}
30
31fn disruption_name(d: Disruption) -> &'static str {
32    match d {
33        Disruption::None => "none",
34        Disruption::Brief => "brief",
35        Disruption::SessionLoss => "session loss",
36        Disruption::DataInterruption => "data interruption",
37        Disruption::ConnectivityLoss => "connectivity loss",
38    }
39}
40
41impl Metrics {
42    pub fn add(&mut self, entry: &T, pass: bool, partial: bool, sim: Option<&Sim>) {
43        let Some(sim) = sim else { return };
44        self.runnable += 1;
45        self.passed += pass as usize;
46        self.partial += partial as usize;
47        let scoring = entry.get("scoring").and_then(T::as_str).unwrap_or("");
48        let cleared = pass || partial;
49        let autonomous = |r: &&planarian_daemon::LogRecord| !matches!(r.caller, Actor::User { .. });
50        let executed = |r: &&planarian_daemon::LogRecord| matches!(&r.result, Submitted::Finished { state, .. } if !matches!(state, TxnState::Rejected(_) | TxnState::Cancelled(_)));
51        if scoring == "repair" {
52            self.repair_entries += 1;
53            if cleared {
54                self.repaired += 1;
55                let last = sim
56                    .log
57                    .iter()
58                    .filter(autonomous)
59                    .filter(|r| matches!(r.result, Submitted::Finished { state: TxnState::Committed, .. }))
60                    .map(|r| r.at.0)
61                    .max();
62                if let Some(t) = last {
63                    self.time_to_repair.push(t.saturating_sub(sim.start.0) / 1000);
64                }
65            }
66        }
67        if scoring == "intent" {
68            self.intent_entries += 1;
69            // The user's own setup is not an intervention; anything the
70            // daemon did after it is.
71            self.unnecessary += sim.log.iter().filter(autonomous).filter(executed).count();
72        }
73        for r in sim.log.iter().filter(executed) {
74            if let Some(f) = &r.facts {
75                // Once per kind the change carries (C10), or as none.
76                let kinds = f.disruption.members();
77                if kinds.is_empty() {
78                    *self.disruption.entry(disruption_name(Disruption::None)).or_default() += 1;
79                }
80                for d in kinds {
81                    *self.disruption.entry(disruption_name(d)).or_default() += 1;
82                }
83            }
84            if let Transition::RestartUnit(u) = &r.proposal.transition {
85                *self.restarts.entry(u.to_string()).or_default() += 1;
86            }
87        }
88        let failed = sim.ledger().terminal.values().filter(|t| t.state == TxnState::RecoveryFailed).count();
89        // Expected where the entry itself names it (its setup waits for it,
90        // or its required_post asks for it).
91        let injected = entry.to_string().contains("RecoveryFailed");
92        if injected {
93            self.recovery_failed_injected += failed;
94        } else if failed > 0 {
95            self.recovery_failed += failed;
96            self.recovery_failed_in.push(entry.get("id").and_then(T::as_str).unwrap_or("?").to_string());
97        }
98    }
99}
100
101impl fmt::Display for Metrics {
102    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
103        writeln!(f, "metrics (fake harness, {} runnable entries):", self.runnable)?;
104        writeln!(
105            f,
106            "  cleared: {} fully checked, {} with every fake-checkable key holding ({:.0}% of runnable)",
107            self.passed,
108            self.partial,
109            100.0 * (self.passed + self.partial) as f64 / self.runnable.max(1) as f64
110        )?;
111        writeln!(f, "  repair rate: {}/{} repair entries cleared", self.repaired, self.repair_entries)?;
112        writeln!(
113            f,
114            "  unnecessary interventions: {} on {} intent entries (must be 0)",
115            self.unnecessary, self.intent_entries
116        )?;
117        let total: usize = self.disruption.values().sum();
118        let by: Vec<String> = self.disruption.iter().map(|(k, n)| format!("{k} {n}")).collect();
119        writeln!(f, "  disruption incurred: {total} executed transactions ({})", by.join(", "))?;
120        let mut t = self.time_to_repair.clone();
121        t.sort();
122        if !t.is_empty() {
123            writeln!(f, "  time to repair: median {} s, max {} s (simulated time)", t[t.len() / 2], t[t.len() - 1])?;
124        }
125        let max = self.restarts.iter().max_by_key(|(_, n)| **n);
126        if let Some((u, n)) = max {
127            writeln!(
128                f,
129                "  restarts: {} in total, at most {n} of {u} in one run set",
130                self.restarts.values().sum::<usize>()
131            )?;
132        }
133        write!(
134            f,
135            "  RecoveryFailed: {} unexpected (must be 0){}, {} where the entry injects it",
136            self.recovery_failed,
137            if self.recovery_failed_in.is_empty() {
138                String::new()
139            } else {
140                format!(" in {}", self.recovery_failed_in.join(", "))
141            },
142            self.recovery_failed_injected
143        )
144    }
145}
planarian_gym/metrics.rs

planarian_gym/
metrics.rs
