Skip to main content

planarian_daemon/
core.rs

1//! The daemon loop (SPEC §10, §13) and request handling (§14, §16).
2//!
3//! Each tick does the following, in order:
4//! 1. assemble state;
5//! 2. advance the incidents and journal any that changed;
6//! 3. re-probe the targets of diagnostic incidents;
7//! 4. unless the machine is busy or interlocked, ask the diagnoser for one
8//!    incident and submit its proposal through the executor;
9//! 5. expire approvals, compact the journal, and run the boot-confirmation
10//!    checkpoint.
11//!
12//! Every change goes through `Executor::submit`; nothing here calls a mutating
13//! backend method.
14
15use crate::render;
16use planarian_diagnose as diagnose;
17use planarian_exec::{Config as ExecConfig, Executor, PrebuildJob, Prebuilt, Submitted, derive::manifest_of};
18use planarian_incident::Manager;
19use planarian_intent::{self as intent, Interpretation};
20use planarian_ipc::{
21    Choice, ConsoleReport, GraphEdge, GraphNode, LogLine, Pending, Request, Response, StatusReport, TxnSummary, UnitRow,
22};
23use planarian_policy::{authorization_of, user_authorization};
24use planarian_schema::spec::predict;
25use planarian_schema::*;
26use planarian_state::{Assembler, Context};
27use planarian_txn::journal::{Journal, JournalError};
28use planarian_txn::ledger::{Ledger, Record};
29use planarian_txn::store::Store;
30use serde::{Deserialize, Serialize};
31use std::collections::{BTreeMap, BTreeSet, VecDeque};
32
33/// The 10-minute checkpoint for boot confirmation and initial enrollment (§10, §16).
34pub const CONFIRM_AFTER: Millis = Millis::secs(600);
35const NOTICES_KEPT: usize = 64;
36/// How long an external dependency's probes time out before it is said.
37const UNREACHABLE_AFTER: Millis = Millis::secs(120);
38/// The "restart to finish" notice repeats this often while a change waits
39/// for its boot (§12f).
40const RESTART_REMINDER: Millis = Millis::secs(24 * 3600);
41
42#[derive(Clone, Debug)]
43pub struct CoreConfig {
44    pub exec: ExecConfig,
45    /// Keywords and steps for "make X bigger" (§14).
46    pub runtime_options: Vec<intent::RuntimeOption>,
47    /// The first start of the daemon on this boot (NextBoot overrides end).
48    pub fresh_boot: bool,
49    /// The default startup grace (units may set their own).
50    pub startup_grace: Millis,
51    /// The failure posture in force (§12d, P16: the stronger of the booted
52    /// and the running generation's); Phase A ships Standard.
53    pub posture: Posture,
54    /// What each generation declares, for `planarian posture`.
55    pub posture_decl: crate::config::PostureDecl,
56    /// Limits on launched agents (§12d, R05).
57    pub agent_limits: planarian_schema::backend::AgentLimits,
58    /// Leave recoveries that may activate a generation until after ready
59    /// (`Core::finish_recovery`, #9): the daemon and the fake harness set it,
60    /// and call `finish_recovery` once ready.
61    pub defer_recovery: bool,
62}
63
64/// How long a generation's new declarations are named in incident reports (§15).
65pub const NEW_DECLARATIONS_FOR: Millis = Millis::secs(24 * 3600);
66
67/// How often the running daemon re-reads its journal's hash chain (I18).
68pub const CHAIN_CHECK_EVERY: Millis = Millis::secs(3600);
69/// §12d: how long a tick waits for the host to confirm a hold the daemon
70/// asks for on a network tool's stop outside Planarian; past it the hold
71/// counts as held (fail closed) and its job goes on in the host.
72pub const HOLD_WAIT: Millis = Millis::secs(5);
73/// §12d: how long a hold the host has not confirmed waits before it is
74/// asked for again (its job may still land meanwhile; the host is checked
75/// every tick).
76pub const HOLD_UNCONFIRMED_RETRY: Millis = Millis::secs(60);
77/// §12d: the first and the longest wait between re-applies of a hold whose
78/// firewall table is missing.
79pub const HOLD_RETRY_MIN: Millis = Millis::secs(10);
80pub const HOLD_RETRY_MAX: Millis = Millis::secs(300);
81
82/// The caller of a request, from `SO_PEERCRED`, group membership and logind.
83#[derive(Clone, Debug, PartialEq, Eq)]
84pub struct Peer {
85    pub uid: u32,
86    /// Root, or a member of `planarian-admin`.
87    pub admin: bool,
88    /// The logind session of the calling process (`sd_pid_get_session`), if any.
89    pub session: Option<String>,
90    /// The agent this process belongs to: its cgroup is the unit
91    /// `planarian-agent-<name>` (§11). An agent is its own principal, never
92    /// the uid it runs under, and is never an administrator or in a session.
93    pub agent: Option<String>,
94    /// The agent unit's instance the process runs in (its InvocationID,
95    /// from the cgroup), which authority is bound to (I24).
96    pub instance: Option<planarian_schema::InstanceId>,
97    /// The calling process has a controlling terminal, read by the daemon
98    /// from its pidfd (§11, B-3). Never taken from the request.
99    pub tty: bool,
100}
101
102impl Peer {
103    pub const fn new(uid: u32, admin: bool) -> Peer {
104        Peer { uid, admin, session: None, agent: None, instance: None, tty: false }
105    }
106
107    /// A process in agent `name`'s unit, in the given instance of it.
108    pub fn agent(uid: u32, name: &str, instance: Option<planarian_schema::InstanceId>) -> Peer {
109        Peer { uid, admin: false, session: None, agent: Some(name.into()), instance, tty: false }
110    }
111
112    pub const fn at_terminal(mut self) -> Peer {
113        self.tty = true;
114        self
115    }
116
117    pub fn in_session(mut self, session: &str) -> Peer {
118        self.session = Some(session.into());
119        self
120    }
121}
122
123/// An agent's principal id: in its own range, above every uid and the
124/// daemon's ids, stable across restarts (FNV-1a of the name).
125pub fn agent_principal(name: &str) -> PrincipalId {
126    let h = name.bytes().fold(0xcbf2_9ce4_8422_2325u64, |h, b| (h ^ b as u64).wrapping_mul(0x0100_0000_01b3));
127    PrincipalId((1 << 40) | (h & 0xffff_ffff))
128}
129
130/// The executor log (I7): one record per submitted proposal.
131#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
132pub struct LogRecord {
133    pub at: Timestamp,
134    pub proposal: Proposal,
135    pub caller: Actor,
136    pub pre_digest: Hash,
137    pub policy: PolicyVersion,
138    pub result: Submitted,
139    /// What the executor derived and decided, when it got that far.
140    #[serde(default)]
141    pub facts: Option<DerivedFacts>,
142    #[serde(default)]
143    pub decision: Option<Decision>,
144    /// The state after, and the transaction's journal path and last reason
145    /// (which carries the probe verdict).
146    #[serde(default)]
147    pub post_digest: Option<Hash>,
148    #[serde(default)]
149    pub path: Vec<String>,
150    #[serde(default)]
151    pub why: String,
152    /// The agent that sent the request, acting for `caller` (§11).
153    #[serde(default)]
154    pub agent: Option<String>,
155    /// §12 step 6 (C18): the transaction touches user data and no snapshot
156    /// was taken, because Phase A has no snapshot backend; it continued.
157    #[serde(default)]
158    pub no_snapshot: bool,
159    /// I7: an approved execution names who approved it and the `bound_to`
160    /// of the token used; a submission parked for approval carries the
161    /// `bound_to` it waits on.
162    #[serde(default)]
163    pub approver: Option<Actor>,
164    #[serde(default)]
165    pub bound_to: Option<Hash>,
166}
167
168/// §12 step 6 (C18): a transaction that got past step 5 (it ran, or waits
169/// for its activation window) with `UserData` among its sensitivities,
170/// where snapshots are unsupported (always, in Phase A: no snapshot backend
171/// is wired), logs `NoSnapshot` and continues.
172fn no_snapshot(result: &Submitted, facts: Option<&DerivedFacts>) -> bool {
173    matches!(result, Submitted::Finished { .. }) && facts.is_some_and(|f| f.sensitivity.contains(Sensitivity::UserData))
174}
175
176pub struct Core<B: Backends + Clone, S: Store> {
177    /// Whether a uid is an administrator (root or `planarian-admin`), for
178    /// the user an agent acts for; set by the daemon from the group file.
179    pub admin_of: std::sync::Arc<dyn Fn(u32) -> bool + Send + Sync>,
180    /// The agent behind the request being handled, for its log records.
181    acting_agent: Option<String>,
182    /// Recoveries left by start until after ready (`finish_recovery`).
183    after_ready: Vec<TxnId>,
184    pub ex: Executor<B, S>,
185    pub b: B,
186    pub asm: Assembler,
187    pub incidents: Manager,
188    pub notices: VecDeque<String>,
189    /// How many notices there have ever been (the deque keeps the last
190    /// `NOTICES_KEPT`), so the binary can print each new one once.
191    pub notices_total: u64,
192    /// Drained by the binary into the executor log file.
193    pub log: Vec<LogRecord>,
194    cfg: CoreConfig,
195    last: Option<(SystemState, UnitDefs)>,
196    choices: BTreeMap<u32, Vec<intent::Match>>,
197    incident_parked: BTreeMap<IncidentId, TxnId>,
198    /// Slow ticks warned (P9), and when the last warning was.
199    pub slow_tick_warnings: u64,
200    last_slow_warning: Option<Timestamp>,
201    slow_ticks_unwarned: u64,
202    /// §12a write-backs queued behind an active generation transaction,
203    /// with their caller (P7). Submitted once none is live.
204    writebacks: Vec<(Proposal, Authorization)>,
205    confirm: Option<(GenerationId, Timestamp)>,
206    manifest_gen: GenerationId,
207    /// The running generation's manifest, before the external source joins it.
208    gen_manifest: planarian_schema::Manifest,
209    /// The external desired-state document in force (§9 hook), verified by `main`.
210    pub external: Option<crate::hooks::DesiredDoc>,
211    /// Services the running generation declares that the one before it did
212    /// not, with where each came from, and since when (§15: a malicious
213    /// declaration is reported with its origin).
214    pub new_declarations: Vec<(String, String)>,
215    new_declarations_since: Timestamp,
216    interlock_noted: bool,
217    /// When each transaction in `AwaitingBoot` was last said to need a
218    /// restart (`CLOCK_BOOTTIME`); repeated daily (§12f).
219    restart_reminded: BTreeMap<TxnId, Timestamp>,
220    /// The attributed way to stop a unit has been taught once (§16, P23).
221    residue_taught: bool,
222    /// Canonical units repaired while an admitted boot-only change waits
223    /// for its confirmation; any one means it is not confirmed (§10).
224    repaired_before_confirmation: BTreeSet<UnitId>,
225    /// Incidents whose opening was already announced.
226    diag_noted: BTreeSet<IncidentId>,
227    /// External dependencies whose probes time out: since when, and
228    /// whether that was said (§10: Unknown, never Failed, so no incident).
229    unreachable: BTreeMap<UnitId, (Timestamp, bool)>,
230    /// When the journal's hash chain was last re-read from disk (I18).
231    chain_checked: Option<Timestamp>,
232    /// A broken chain found while running; `main` stops on it (§12d: the
233    /// response is in the posture units, not here).
234    pub integrity: Option<String>,
235    /// Each unit's status at the previous tick, to see transitions (§11 residue).
236    prev_status: BTreeMap<UnitId, UnitStatus>,
237    /// Security units' systemd restart counts at the previous tick (§11:
238    /// a kill systemd itself repaired is still alarmed).
239    prev_auto_restarts: BTreeMap<UnitId, u32>,
240    /// The boot report runs on the first tick after boot (§10).
241    boot_report_due: bool,
242    /// The last boot report this daemon ran, for `planarian report boot`.
243    last_boot_report: Option<(Timestamp, GenerationId, String)>,
244    /// Quiet mode (§12f) and the notices it holds back.
245    quiet: bool,
246    quiet_queue: Vec<String>,
247    /// Waiting generations interrupted by a restart, to prepare again (§12f).
248    reprepare: Vec<(TxnId, Transition)>,
249    /// §12d: the hold's firewall table was there at the last check (or no
250    /// hold was active). The alarm is raised once, on the change to missing
251    /// or uncheckable.
252    hold_table_seen: bool,
253    /// §12d: this episode's "removed" alarm and `HoldApplied` are written
254    /// (separately from the "could not be checked" one, so an episode that
255    /// starts uncheckable still journals the re-apply).
256    hold_missing_noted: bool,
257    /// §12d: while the table is missing, the next re-apply (boot clock) and
258    /// the wait after it, doubling to `HOLD_RETRY_MAX`.
259    hold_retry: Option<(Timestamp, Millis)>,
260    /// §12d: units the daemon itself asked the host to hold for a stop
261    /// outside Planarian (the residue path); their `HoldApplied` is by it.
262    daemon_held: BTreeSet<UnitId>,
263    /// §12d: units whose hold the host has not confirmed in time (alarmed
264    /// once), and when to ask again: a wedged hold job is not re-called on
265    /// every tick, where it would spend the whole hold budget and starve
266    /// other units' lifts.
267    hold_unconfirmed: BTreeMap<UnitId, Timestamp>,
268    /// §12d: units whose lift the host has not confirmed (`HoldLifted` is
269    /// journaled; the host may still hold them), and when to ask again. While
270    /// one stands, the unit is not journaled as held again, and its lift is
271    /// retried quietly at `HOLD_UNCONFIRMED_RETRY`'s cadence, so a hung lift
272    /// neither churns the journal nor spends every tick's budget.
273    lift_unconfirmed: BTreeMap<UnitId, Timestamp>,
274    /// §12d: the end of this tick's hold budget (`hold_wait`).
275    hold_left: Millis,
276    /// Generation transactions withdrawn from `Built` for a rollback about to
277    /// run (§12f, I6; owner decision, Nov 2), with when each began waiting,
278    /// to prepare again once no generation transaction is active.
279    withdrawn: Vec<(TxnId, Transition, Option<Timestamp>)>,
280    /// From the executor log of earlier runs: each transaction's proposal and
281    /// caller, so one prepared again keeps its source and caller (§12, C21).
282    pub earlier: BTreeMap<TxnId, (Proposal, Actor)>,
283    /// Units Planarian's own transactions touched since the previous tick, and
284    /// whether a generation changed: their stops are not someone else's.
285    touched: BTreeSet<UnitId>,
286    generation_touched: bool,
287    /// When to next check the §12b collection bounds.
288    next_gc_check: Timestamp,
289    /// The space report last noticed.
290    space_noted: Timestamp,
291    /// Executor log records waiting for their post-state (I7).
292    awaiting_post: Vec<LogRecord>,
293    /// Set by the daemon: a catalog change's stage, evaluate and build are
294    /// handed back as a `PrebuildJob` (`take_deferred`) to run on another
295    /// thread, instead of running inside `handle`. Off in tests and the gym.
296    pub defer: bool,
297    /// Set by the daemon while a deferred change builds elsewhere: what, and since when.
298    pub building: Option<(String, Timestamp)>,
299    deferred: Option<(PrebuildJob<B::Gens>, Resume)>,
300    /// The hash of the core-hash record the running generation installs
301    /// (`/etc/planarian/core.sha256`, I2), for `SystemState.core_hash` and
302    /// so for `digest()` (P19). Read when the daemon names the record
303    /// (`set_core_hash_record`) and again whenever the running generation
304    /// changes; zero where no record exists (tests, the fake harness).
305    core_hash: Hash,
306    core_hash_record: Option<std::path::PathBuf>,
307    /// The agent behind each change parked for approval, for the approved
308    /// execution's log record (I7).
309    parked_agents: BTreeMap<TxnId, Option<String>>,
310    /// §12d: held units whose hold an attested release lifted, until the
311    /// release ends; and held units whose failed repair administrators have
312    /// been told about (once per hold).
313    hold_released: BTreeSet<UnitId>,
314    hold_notified: BTreeSet<UnitId>,
315}
316
317/// How to finish a request whose slow part ran elsewhere.
318#[derive(Clone, Debug)]
319pub struct Resume {
320    peer: Peer,
321    kind: ResumeKind,
322}
323
324#[derive(Clone, Debug)]
325enum ResumeKind {
326    Submit { t: Transition, source: Source, reason: String },
327    Request(Request),
328}
329
330impl Resume {
331    /// A few words for "busy with …".
332    pub fn describe(&self) -> String {
333        match &self.kind {
334            ResumeKind::Submit { t, .. } => render::transition(t),
335            ResumeKind::Request(r) => crate::snapshot::describe(r),
336        }
337    }
338}
339
340/// P9: a slow tick is warned at most this often (the count rides along).
341pub const SLOW_TICK_WARN_EVERY: Millis = Millis::secs(60);
342
343/// How often the daemon checks the §12b collection bounds.
344pub const GC_CHECK: Millis = Millis(3600 * 1000);
345
346fn outcome_of(state: &TxnState) -> Outcome {
347    match state {
348        TxnState::Committed => Outcome::Succeeded,
349        TxnState::Reverted => Outcome::Failed("reverted".into()),
350        s => Outcome::Unknown(format!("{s:?}")),
351    }
352}
353
354impl<B: Backends + Clone, S: Store> Core<B, S> {
355    /// Start: adopt the running generation, reconcile the journal, restore
356    /// incidents, load the manifest, and schedule boot confirmation.
357    pub fn start(journal: Journal<S>, b: B, me: OwnerId, cfg: CoreConfig) -> Core<B, S> {
358        let fresh = cfg.fresh_boot;
359        let mut ex = Executor::new(journal, b.clone(), me, cfg.exec.clone());
360        ex.adopt_current();
361        let mut notices = VecDeque::new();
362        let terminal_before: BTreeSet<TxnId> = ex.kernel.journal.ledger().terminal.keys().copied().collect();
363        let mut deferred = vec![];
364        match ex.kernel.reconcile_deferring(cfg.defer_recovery) {
365            Ok((waiting, d)) => {
366                if !waiting.is_empty() {
367                    notices.push_back(format!("waiting for the recovery watchdog to finish transaction(s) {waiting:?}"))
368                }
369                deferred = d;
370            }
371            Err(e) => notices.push_back(format!("journal reconciliation failed: {e:?}")),
372        }
373        let ledger = ex.kernel.journal.ledger().clone();
374        // A generation waiting in Built for an activation window (§12f), or
375        // a change parked for approval (§12 step 4: tokens do not survive a
376        // restart), is Rejected(Interrupted) by the crash table and prepared
377        // again: the parked one asks for approval again.
378        let reprepare: Vec<(TxnId, Transition)> = ledger
379            .terminal
380            .iter()
381            .filter(|(id, t)| {
382                !terminal_before.contains(id)
383                    && t.state == TxnState::Rejected(RejectReason::Interrupted)
384                    && t.path.iter().rev().nth(1).is_some_and(|p| p == "Built" || p == "Prepared")
385            })
386            .map(|(id, t)| (*id, t.transition.clone()))
387            .collect();
388        // §12f: what a rollback withdrew and nothing has prepared again yet
389        // (no transaction names it in `prepared_again_from`), from the
390        // journal, so a restart between the two loses nothing and prepares
391        // nothing twice. Prepared again once the rollback is terminal.
392        let named: BTreeSet<TxnId> = ledger
393            .live
394            .values()
395            .filter_map(|t| t.txn.prepared_again_from)
396            .chain(ledger.terminal.values().filter_map(|t| t.prepared_again_from))
397            .collect();
398        let withdrawn: Vec<(TxnId, Transition, Option<Timestamp>)> = ledger
399            .terminal
400            .iter()
401            .filter(|(id, t)| {
402                t.withdrawn_for.is_some()
403                    && t.state == TxnState::Rejected(RejectReason::Interrupted)
404                    && !named.contains(id)
405                    && !reprepare.iter().any(|(r, _)| r == *id)
406            })
407            .map(|(id, t)| (*id, t.transition.clone(), None))
408            .collect();
409        let mut incidents = Manager::restore(ledger.incidents.values().cloned());
410        incidents.startup_grace = cfg.startup_grace;
411        let current = b.generations().current().unwrap_or_default();
412        let manifest = match b.generations().path_of(current).and_then(|p| b.generations().evaluate_built(&p)) {
413            Ok(e) => manifest_of(&e),
414            Err(o) => {
415                notices.push_back(format!(
416                    "the running generation has no Planarian manifest ({o:?}); nothing is watched until a generation built with services.planarian.enable is running"
417                ));
418                Default::default()
419            }
420        };
421        let asm = Assembler::new(manifest.clone(), cfg.exec.canonical.clone());
422        let now = b.clock().now();
423        let last_booted = b.generations().last_booted().ok();
424        // §10: a boot-only change admitted at this boot is confirmed at its
425        // 10-minute checkpoint.
426        let admitted = ledger.live.values().any(|t| {
427            t.txn.state == TxnState::Promoted
428                && t.path.iter().any(|p| p == "AwaitingBoot")
429                && t.txn.candidate.as_ref().map(|(g, _)| *g) == Some(current)
430        });
431        let confirm = match ledger.known_good {
432            _ if admitted => Some((current, now.plus(CONFIRM_AFTER))),
433            None => Some((current, now.plus(CONFIRM_AFTER))),
434            Some(_)
435                if ledger.live_tested.is_some()
436                    && ledger.live_tested == last_booted
437                    && last_booted == Some(current) =>
438            {
439                Some((current, now.plus(CONFIRM_AFTER)))
440            }
441            _ => None,
442        };
443        let mut core = Core {
444            core_hash: Hash::ZERO,
445            core_hash_record: None,
446            parked_agents: BTreeMap::new(),
447            hold_released: BTreeSet::new(),
448            hold_notified: BTreeSet::new(),
449            after_ready: deferred,
450            ex,
451            b,
452            asm,
453            incidents,
454            notices_total: notices.len() as u64,
455            notices,
456            log: vec![],
457            cfg,
458            last: None,
459            choices: BTreeMap::new(),
460            incident_parked: BTreeMap::new(),
461            slow_tick_warnings: 0,
462            last_slow_warning: None,
463            slow_ticks_unwarned: 0,
464            writebacks: vec![],
465            confirm,
466            manifest_gen: current,
467            gen_manifest: manifest,
468            external: None,
469            new_declarations: vec![],
470            new_declarations_since: now,
471            interlock_noted: false,
472            restart_reminded: BTreeMap::new(),
473            residue_taught: false,
474            repaired_before_confirmation: BTreeSet::new(),
475            diag_noted: BTreeSet::new(),
476            unreachable: BTreeMap::new(),
477            chain_checked: None,
478            integrity: None,
479            admin_of: std::sync::Arc::new(|uid| uid == 0),
480            acting_agent: None,
481            prev_status: BTreeMap::new(),
482            prev_auto_restarts: BTreeMap::new(),
483            boot_report_due: fresh,
484            last_boot_report: None,
485            quiet: false,
486            quiet_queue: vec![],
487            reprepare: vec![],
488            hold_table_seen: true,
489            hold_missing_noted: false,
490            hold_retry: None,
491            daemon_held: BTreeSet::new(),
492            hold_unconfirmed: BTreeMap::new(),
493            lift_unconfirmed: BTreeMap::new(),
494            hold_left: HOLD_WAIT,
495            withdrawn: vec![],
496            earlier: BTreeMap::new(),
497            touched: BTreeSet::new(),
498            generation_touched: false,
499            next_gc_check: Timestamp(0),
500            space_noted: Timestamp(0),
501            awaiting_post: Vec::new(),
502            defer: false,
503            building: None,
504            deferred: None,
505        };
506        if core.cfg.fresh_boot {
507            for s in ledger.overrides.iter().filter(|s| s.until == Until::NextBoot) {
508                core.journal(Record::OverrideLifted(s.unit.clone()));
509            }
510        }
511        if let Some(r) = &ledger.last_reconciliation {
512            for a in &r.actions {
513                core.notice(format!("reconciliation: {a}"));
514            }
515        }
516        core.reprepare = reprepare;
517        core.withdrawn = withdrawn;
518        core.note_declarations(None, current, false);
519        core
520    }
521
522    /// Records from the executor log of earlier runs (§12, C21): what each
523    /// transaction was proposed as, and by whom.
524    pub fn remember(&mut self, records: impl IntoIterator<Item = LogRecord>) {
525        for r in records {
526            if let Submitted::Finished { txn, .. } | Submitted::Parked { txn, .. } = &r.result {
527                self.earlier.insert(*txn, (r.proposal, r.caller));
528            }
529        }
530    }
531
532    /// The observation the last tick (or request) acted on.
533    pub fn last_observation(&self) -> Option<&(SystemState, UnitDefs)> {
534        self.last.as_ref()
535    }
536
537    pub fn notice(&mut self, s: String) {
538        // Quiet mode (§12f): non-urgent notices wait and come as one batch.
539        if self.quiet && !urgent(&s) {
540            self.quiet_queue.push(s);
541            return;
542        }
543        self.say(s);
544    }
545
546    fn say(&mut self, s: String) {
547        if self.notices.len() >= NOTICES_KEPT {
548            self.notices.pop_front();
549        }
550        self.notices.push_back(s);
551        self.notices_total += 1;
552    }
553
554    /// The notices made since `printed` of them (counted by
555    /// `notices_total`) were seen, oldest first: what to journal next.
556    pub fn notices_since(&self, printed: u64) -> impl Iterator<Item = &String> {
557        let new = self.notices_total.saturating_sub(printed).min(self.notices.len() as u64) as usize;
558        self.notices.iter().skip(self.notices.len() - new)
559    }
560
561    fn journal(&mut self, r: Record) {
562        let now = self.b.clock().now();
563        if let Err(e) = self.ex.kernel.journal.append(now, r) {
564            self.notice(format!("journal write failed: {e:?}"));
565        }
566    }
567
568    pub fn ledger(&mut self) -> Ledger {
569        self.ex.kernel.journal.sync_ledger().cloned().unwrap_or_default()
570    }
571
572    /// P19: the record the running generation installs, for the state's
573    /// `core_hash`; read now and on every generation change.
574    pub fn set_core_hash_record(&mut self, record: std::path::PathBuf) {
575        self.core_hash_record = Some(record);
576        self.read_core_hash();
577    }
578
579    /// The core hash the observed state carries (P19).
580    pub fn core_hash(&self) -> Hash {
581        self.core_hash
582    }
583
584    /// An unreadable record (not a missing one) keeps the value already
585    /// held and says so: nothing is derived from a failed read (rule 21).
586    fn read_core_hash(&mut self) {
587        let Some(record) = self.core_hash_record.clone() else { return };
588        match crate::config::core_hash(&record) {
589            Ok(h) => self.core_hash = h,
590            Err(e) => {
591                self.notice(format!("the core hash record could not be read ({e}); the state keeps the last one read"))
592            }
593        }
594    }
595
596    fn context(&self, l: &Ledger, current: GenerationId) -> Context {
597        let now = self.b.clock().now();
598        Context {
599            known_good: l.known_good.unwrap_or(current),
600            live_tested: l.live_tested,
601            accepted_baseline: l.accepted_baseline,
602            overrides: l.overrides.iter().filter(|s| s.holds_at(now)).cloned().collect(),
603            security_floor: l.security_floor,
604            core_hash: self.core_hash,
605        }
606    }
607
608    /// Generations GC must keep (I3): known-good, previous, current, boot
609    /// default, live-tested, accepted baseline, and every in-flight candidate.
610    fn protect(&self, l: &Ledger, current: GenerationId) -> Vec<GenerationId> {
611        let gens = self.b.generations();
612        let mut p = vec![current];
613        p.extend(l.known_good);
614        p.extend(l.live_tested);
615        p.extend(l.accepted_baseline);
616        p.extend(gens.boot_default().ok());
617        if let Ok(entries) = gens.boot_entries() {
618            p.extend(entries.into_iter().filter(|g| *g < current).max());
619        }
620        p.extend(l.live.values().filter_map(|t| t.txn.candidate.as_ref().map(|(g, _)| *g)));
621        p.sort();
622        p.dedup();
623        p
624    }
625
626    /// What `finish_recovery` will do, for "busy with …"; None if nothing.
627    pub fn pending_recovery(&self) -> Option<String> {
628        match self.after_ready.as_slice() {
629            [] => None,
630            [one] => Some(format!("recovering transaction {} after the restart", one.0)),
631            many => Some(format!("recovering {} transactions after the restart", many.len())),
632        }
633    }
634
635    /// After ready: run the recoveries start left (`CoreConfig::defer_recovery`).
636    pub fn finish_recovery(&mut self) {
637        let ids = std::mem::take(&mut self.after_ready);
638        if let Err(e) = self.ex.kernel.finish_deferred(&ids) {
639            self.notice(format!("journal reconciliation after ready failed: {e:?}"));
640        }
641    }
642
643    /// Observe without acting (also used to answer requests).
644    pub fn observe(&mut self) -> (SystemState, UnitDefs) {
645        let l = self.ledger();
646        let current = self.b.generations().current().unwrap_or(self.manifest_gen);
647        if current != self.manifest_gen {
648            if let Ok(e) = self.b.generations().path_of(current).and_then(|p| self.b.generations().evaluate_built(&p)) {
649                self.gen_manifest = manifest_of(&e);
650                self.asm.set_manifest(self.effective_manifest().0);
651            }
652            let prev = self.manifest_gen;
653            self.manifest_gen = current;
654            // A switch outside Planarian stops and starts units too: none of
655            // that is a person's stop (§11 residue).
656            self.generation_touched = true;
657            self.note_declarations(Some(prev), current, true);
658            // P19: a switch may install another core record.
659            self.read_core_hash();
660        }
661        let ctx = self.context(&l, current);
662        let protect = self.protect(&l, current);
663        self.ex.update_context(ctx.clone(), protect);
664        // No baseline yet: never above Runtime (§16 initial enrollment).
665        self.ex.cfg.settings.max_tier = if l.known_good.is_none() {
666            self.cfg.exec.settings.max_tier.min(Tier::Runtime)
667        } else {
668            self.cfg.exec.settings.max_tier
669        };
670        let mut s = self.asm.observe(&self.b, &ctx);
671        // §12d: what is held now, the journal's and the host's (fail closed:
672        // either one holding is held).
673        s.holds = l.holds();
674        if let Ok(h) = self.b.units().held() {
675            s.holds.extend(h);
676        }
677        let mut defs = self.asm.defs();
678        // §12d: each network-security unit's hold unit, known (so releasing
679        // it derives the held unit's sensitivity) but never expected.
680        let holds: Vec<UnitDef> = crate::hold::network_security(&defs).map(crate::hold::hold_def).collect();
681        for d in holds {
682            defs.entry(d.id.clone()).or_insert(d);
683        }
684        self.last = Some((s.clone(), defs.clone()));
685        if !self.awaiting_post.is_empty() {
686            let digest = s.digest();
687            for mut r in self.awaiting_post.drain(..) {
688                r.post_digest = Some(digest);
689                self.log.push(r);
690            }
691        }
692        (s, defs)
693    }
694
695    fn proposal(&self, t: Transition, s: &SystemState, defs: &UnitDefs, source: Source, reason: String) -> Proposal {
696        Proposal { observed: s.digest(), expected: predict(&t, s, defs), transition: t, reason, source }
697    }
698
699    fn submit(&mut self, p: Proposal, authz: Authorization, s: &SystemState, defs: &UnitDefs) -> Submitted {
700        self.submit_from(p, authz, s, defs, None)
701    }
702
703    /// `again`: the withdrawn transaction this prepares again (§12f), set
704    /// only by `prepare_again`.
705    fn submit_from(
706        &mut self,
707        p: Proposal,
708        authz: Authorization,
709        s: &SystemState,
710        defs: &UnitDefs,
711        again: Option<TxnId>,
712    ) -> Submitted {
713        let now = self.b.clock().now();
714        let pre = s.digest();
715        let policy = self.ex.policy_version();
716        let caller = authz.actor.clone();
717        let built = self.waiting_built();
718        let result = match again {
719            Some(from) => self.ex.submit_again(p.clone(), from, authz.clone(), s, defs),
720            None => self.ex.submit(p.clone(), authz.clone(), s, defs),
721        };
722        self.note_withdrawn(built);
723        if let Some((facts, _)) = self.ex.last.clone() {
724            self.touched.extend(facts.affected_units.iter().cloned());
725            // §11: the affected sessions are notified either way.
726            if facts.other_sessions_affected {
727                let others: Vec<String> = self
728                    .asm
729                    .sessions
730                    .iter()
731                    .filter(|x| {
732                        x.establishes() && Some(x.uid) != caller.uid() && facts.affected_sessions.contains(&x.id)
733                    })
734                    .map(|x| format!("{} (uid {})", x.id, x.uid))
735                    .collect();
736                self.notice(format!(
737                    "{}: {} would also affect other people's sessions: {}",
738                    render::transition(&p.transition),
739                    match caller.uid() {
740                        Some(u) => format!("asked for by uid {u}"),
741                        None => "proposed by Planarian".to_string(),
742                    },
743                    others.join(", ")
744                ));
745            }
746        }
747        self.generation_touched |= p.transition.changes_generation();
748        if let Submitted::Finished { txn, state, .. } = &result {
749            match state {
750                TxnState::Built => self.notice(format!(
751                    "transaction {}: {} is ready and waits for a quiet moment (it would interrupt what you are doing)",
752                    txn.0,
753                    render::transition(&p.transition)
754                )),
755                TxnState::AwaitingBoot => self.notice(format!(
756                    "transaction {}: {} takes effect when you restart; restart to finish",
757                    txn.0,
758                    render::transition(&p.transition)
759                )),
760                _ => {}
761            }
762        }
763        if let Some((g, fixes)) = self.ex.floor_raised.take() {
764            self.notice(format!(
765                "security floor is now generation {} (it fixes {}); going back below it needs an administrator",
766                g.0,
767                fixes.join(", ")
768            ));
769        }
770        if let (Transition::RestartUnit(u), Submitted::Finished { .. }) = (&p.transition, &result) {
771            self.incidents.started(u, now);
772        }
773        let (facts, decision) = self.ex.last.clone().unzip();
774        let excluded = std::mem::take(&mut self.ex.excluded);
775        if !excluded.is_empty() {
776            self.notice(format!(
777                "left out of the build because git does not track them (Nix cannot see them): {}",
778                excluded.join(", ")
779            ));
780        }
781        let txn = match &result {
782            Submitted::Finished { txn, .. } | Submitted::Parked { txn, .. } => Some(*txn),
783            _ => None,
784        };
785        let l = self.ledger();
786        let (path, why) = match txn {
787            Some(t) => (
788                l.path(t).map(<[String]>::to_vec).unwrap_or_default(),
789                l.live
790                    .get(&t)
791                    .map(|x| x.why.clone())
792                    .or_else(|| l.terminal.get(&t).map(|x| x.why.clone()))
793                    .unwrap_or_default(),
794            ),
795            None => (vec![], String::new()),
796        };
797        if let Submitted::Parked { txn, .. } = &result {
798            self.parked_agents.insert(*txn, self.acting_agent.clone());
799        }
800        let record = LogRecord {
801            at: now,
802            no_snapshot: no_snapshot(&result, facts.as_ref()),
803            approver: None,
804            bound_to: match &result {
805                Submitted::Parked { approve, .. } => Some(*approve),
806                _ => None,
807            },
808            proposal: p,
809            caller,
810            pre_digest: pre,
811            policy,
812            result: result.clone(),
813            facts,
814            decision,
815            post_digest: None,
816            path,
817            why,
818            agent: self.acting_agent.clone(),
819        };
820        // A finished transaction's record waits for the next observation for
821        // its post-state (observing here would run the probes again).
822        if matches!(result, Submitted::Finished { .. }) {
823            self.awaiting_post.push(record);
824        } else {
825            self.log.push(record);
826        }
827        // §12a: the runtime change is in effect; persist it in the background.
828        if let Submitted::Finished { followup: Some(wb), .. } = &result {
829            self.writebacks.push((wb.clone(), authz.clone()));
830            self.flush_writebacks();
831        }
832        result
833    }
834
835    /// §12d *Network security fails closed*, each tick. The journal's holds
836    /// (`Ledger::holds`) are the source of truth after a restart; they are
837    /// reconciled with the host's (`held`), and any disagreement keeps the
838    /// hold (fail closed).
839    /// Returns whether it applied a hold the observed state does not show.
840    /// What is left of this tick's hold budget: every hold call on the tick
841    /// (apply, lift, the table check, a re-apply) waits
842    /// at most this, and none is made once it is spent (the caller takes it
843    /// as "cannot say", the fail-closed side). So a stuck `systemctl` or
844    /// `nft` holds the tick up for HOLD_WAIT in all, not per call (reviewer,
845    /// #61). The budget is not counted from the tick's
846    /// start: only the time the hold calls take is spent (hold_spent), so
847    /// probes the hold itself slows (their traffic is dropped) never spend it
848    /// (catalog 41644729493, vm-netsec; reviewer, 7c2edfc); and lifts are
849    /// made first (network_holds), so other units' calls cannot keep a unit
850    /// that is back held (a livelock). A call skipped for want of budget is
851    /// not news: it is made next tick, and alarms nothing.
852    fn hold_wait(&self) -> Option<Millis> {
853        (self.hold_left.0 > 0).then_some(self.hold_left)
854    }
855
856    /// Spends what a hold call begun at `t0` took from the tick's budget.
857    fn hold_spent(&mut self, t0: Timestamp) {
858        let took = self.b.clock().boot_now().0.saturating_sub(t0.0);
859        self.hold_left = Millis(self.hold_left.0.saturating_sub(took));
860    }
861
862    /// Compact the journal and return the entries it discarded (the segment
863    /// it checkpointed), read in the same core call: no tick of this daemon
864    /// can journal between the two, so they are exactly what the checkpoint
865    /// replaced. The gym's forced compaction saves them for the judge's
866    /// full-history replay (`replay_equivalent`); reading the segment from
867    /// outside first left a gap a tick could journal into.
868    pub fn compact_saving(
869        &mut self,
870    ) -> Result<Vec<planarian_txn::ledger::Entry>, planarian_txn::journal::JournalError> {
871        let segment = self.ex.kernel.journal.segment()?;
872        self.ex.kernel.journal.compact()?;
873        Ok(segment)
874    }
875
876    fn network_holds(&mut self, s: &SystemState, defs: &UnitDefs) -> bool {
877        use crate::hold;
878        use planarian_txn::ledger::HoldBy;
879        let units: Vec<UnitId> = hold::network_security(defs).map(|d| d.id.clone()).collect();
880        if units.is_empty() {
881            return false;
882        }
883        let mut applied = false;
884        let journaled = self.ledger().holds();
885        let host = self.b.units().held();
886        // Anything else flushing the ruleset removes the hold's table while
887        // the units stay held: put it back (journal, alarm, re-apply).
888        let mut holding = journaled.clone();
889        if let Ok(h) = &host {
890            holding.extend(h.iter().cloned());
891        }
892        // Lifts first, before the table check and any unit's hold: a unit
893        // that is back is never kept held because the tick's hold budget
894        // went on other units' calls (catalog 41644729493). A unit whose
895        // release just ended, or that OnFailure= held unseen, takes the
896        // loop below (it has its own bookkeeping first).
897        let mut lifted = BTreeSet::new();
898        for u in &units {
899            if journaled.contains(u)
900                && !self.hold_released.contains(u)
901                && s.override_of(&hold::hold_unit(u)).is_none()
902                && !hold::failed(s, defs, u)
903                && hold::back(s, defs, u)
904                && self.lift_back(u)
905            {
906                lifted.insert(u.clone());
907            }
908        }
909        // A unit whose lift is pending is back and journaled as lifted, though
910        // the host still lists it: not a standing hold, so a flushed table
911        // is not re-applied for it (reviewer, #72).
912        holding.retain(|u| {
913            let pending = self.lift_unconfirmed.contains_key(u) && !hold::failed(s, defs, u) && hold::back(s, defs, u);
914            !(lifted.contains(u) || pending)
915        });
916        if holding.is_empty() {
917            self.hold_table_seen = true;
918            self.hold_missing_noted = false;
919            self.hold_retry = None;
920        } else {
921            self.hold_table(&holding, &units);
922        }
923        for u in units {
924            // Taken every tick, so an entry never outlives the tick that set it.
925            let asked = self.daemon_held.remove(&u);
926            let in_journal = journaled.contains(&u);
927            // None: the host cannot say; a failed unit is then held again
928            // (the call is idempotent).
929            let on_host = host.as_ref().ok().map(|h| h.contains(&u));
930            if on_host == Some(true) {
931                // The host has it: a later timeout is news again.
932                self.hold_unconfirmed.remove(&u);
933            }
934            if on_host == Some(false) && !hold::failed(s, defs, &u) {
935                // Neither failed nor held: the next episode's timeout alarms again.
936                self.hold_unconfirmed.remove(&u);
937            }
938            if lifted.contains(&u) {
939                continue;
940            }
941            // (`!in_journal` is defensive: a unit journaled as held has its entry
942            // dropped by the failed-again arm first.)
943            if self.lift_unconfirmed.contains_key(&u) && !in_journal {
944                match on_host {
945                    // The host cannot say: the entry stands, and the unit is
946                    // handled as below (the lift is asked again when it can).
947                    None => {}
948                    // The lift landed after all.
949                    Some(false) if !hold::failed(s, defs, &u) => {
950                        self.lift_unconfirmed.remove(&u);
951                        self.hold_notified.remove(&u);
952                        self.notice(format!("{u} is back; network traffic flows again"));
953                        continue;
954                    }
955                    // Still held on the host only because its lift is
956                    // pending: not a new hold. Retried when due.
957                    Some(true) if !hold::failed(s, defs, &u) && hold::back(s, defs, &u) => {
958                        let now = self.b.clock().boot_now();
959                        if self.lift_unconfirmed.get(&u).is_some_and(|next| now >= *next) && self.hold_wait().is_some()
960                        {
961                            self.lift_host(&u);
962                        }
963                        continue;
964                    }
965                    // Failed again, or not back: the hold is real again.
966                    Some(_) => {
967                        self.lift_unconfirmed.remove(&u);
968                    }
969                }
970            }
971            // An attested release (an override on the hold unit) lifts it
972            // until the release ends.
973            if s.override_of(&hold::hold_unit(&u)).is_some() {
974                if in_journal {
975                    self.journal(Record::HoldLifted {
976                        unit: u.clone(),
977                        reason: "released by an attested override".into(),
978                    });
979                }
980                self.hold_released.insert(u);
981                continue;
982            }
983            if self.hold_released.remove(&u) {
984                // The release ended: the hold script's marker goes; a unit
985                // still failed is held again below.
986                let _ = self.b.units().end_release(&u);
987            }
988            if hold::failed(s, defs, &u) {
989                if !in_journal {
990                    // systemd's OnFailure= already holds a crash or a kill;
991                    // a stop outside Planarian is the daemon's to hold.
992                    // Daemon when the daemon asked (residue): also when its
993                    // hold job merged with OnFailure='s for the same unit
994                    // (systemd merges the two starts into one job), which is
995                    // fine: the hold is the same either way.
996                    let by = if asked || on_host != Some(true) { HoldBy::Daemon } else { HoldBy::OnFailure };
997                    self.journal(Record::HoldApplied { unit: u.clone(), by });
998                    self.notice(hold::alarm(&u));
999                }
1000                if on_host != Some(true) {
1001                    let now = self.b.clock().boot_now();
1002                    // Asked already and not yet due again: its job may still
1003                    // land, and the host is checked every tick (above).
1004                    // A skip for want of budget waits too: asked next tick, quietly.
1005                    let waiting =
1006                        self.hold_unconfirmed.get(&u).is_some_and(|next| now < *next) || self.hold_wait().is_none();
1007                    let o = match self.hold_wait() {
1008                        _ if waiting => Outcome::Unknown("waiting for its hold job".into()),
1009                        Some(w) => {
1010                            let t0 = self.b.clock().boot_now();
1011                            let o = self.b.units().hold(&u, w);
1012                            self.hold_spent(t0);
1013                            o
1014                        }
1015                        None => Outcome::Unknown("no time left on this tick".into()),
1016                    };
1017                    match o {
1018                        // A failure alarms each time; a hold the host has not
1019                        // confirmed in time (its job may still land), once.
1020                        o @ Outcome::Failed(_) => {
1021                            self.notice(format!("ALARM: {u}: its traffic could not be held ({o:?})"))
1022                        }
1023                        _ if waiting => {}
1024                        o @ Outcome::Unknown(_) => {
1025                            let later = self.b.clock().boot_now().plus(HOLD_UNCONFIRMED_RETRY);
1026                            if self.hold_unconfirmed.insert(u.clone(), later).is_none() {
1027                                self.notice(format!("ALARM: {u}: its traffic could not be held ({o:?})"))
1028                            }
1029                        }
1030                        _ => applied |= !s.holds.contains(&u),
1031                    }
1032                }
1033                // Repair failed: the hold stays, and administrators hear of it once.
1034                let given_up = self.incidents.active_for(&u).is_some_and(|i| {
1035                    matches!(i.state, IncidentState::Cooldown { .. } | IncidentState::Escalated { .. })
1036                });
1037                if given_up && self.hold_notified.insert(u.clone()) {
1038                    self.notice(format!(
1039                        "ALARM: {u}: repair failed; its network traffic stays blocked. It needs a person: an administrator can fix it, or lift the hold with `planarian hold release {}`",
1040                        u.name
1041                    ));
1042                }
1043            } else if in_journal || on_host == Some(true) {
1044                if !in_journal {
1045                    // OnFailure= held it while the daemon was not looking.
1046                    self.journal(Record::HoldApplied { unit: u.clone(), by: HoldBy::OnFailure });
1047                }
1048                if hold::back(s, defs, &u) {
1049                    self.lift_back(&u);
1050                }
1051            }
1052        }
1053        applied
1054    }
1055
1056    /// Lifts a back unit's hold: journaled, then asked of the host. False
1057    /// when the tick has no budget left: nothing is done, and it is lifted
1058    /// on a later tick.
1059    fn lift_back(&mut self, u: &UnitId) -> bool {
1060        if self.hold_wait().is_none() {
1061            return false;
1062        }
1063        self.journal(Record::HoldLifted { unit: u.clone(), reason: "Running, and its Readiness probes pass".into() });
1064        self.lift_host(u);
1065        true
1066    }
1067
1068    /// Asks the host to lift a hold already journaled as lifted. A lift the
1069    /// host has not confirmed is noticed once, then retried quietly.
1070    fn lift_host(&mut self, u: &UnitId) {
1071        let o = match self.hold_wait() {
1072            Some(w) => {
1073                let t0 = self.b.clock().boot_now();
1074                let o = self.b.units().lift_hold(u, w);
1075                self.hold_spent(t0);
1076                o
1077            }
1078            None => Outcome::Unknown("no time left on this tick".into()),
1079        };
1080        match o {
1081            o @ (Outcome::Failed(_) | Outcome::Unknown(_)) => {
1082                let later = self.b.clock().boot_now().plus(HOLD_UNCONFIRMED_RETRY);
1083                if self.lift_unconfirmed.insert(u.clone(), later).is_none() {
1084                    self.notice(format!("{u}: the hold could not be lifted ({o:?}); its traffic stays blocked"))
1085                }
1086            }
1087            _ => {
1088                self.lift_unconfirmed.remove(u);
1089                self.hold_notified.remove(u);
1090                self.notice(format!("{u} is back; network traffic flows again"));
1091            }
1092        }
1093    }
1094
1095    /// §12d: the hold's table, while a hold stands. Missing: alarm once, then
1096    /// re-apply (a reload, which never lifts; `HoldApplied` journaled
1097    /// first), and retry with backoff until a check finds it. Uncheckable:
1098    /// alarm once and check again next tick; nothing is re-applied blind.
1099    fn hold_table(&mut self, holding: &BTreeSet<UnitId>, units: &[UnitId]) {
1100        use planarian_txn::ledger::HoldBy;
1101        if self.hold_wait().is_none() {
1102            // Not checked this tick, for want of budget: not "uncheckable".
1103            return;
1104        }
1105        let present = match self.hold_wait() {
1106            Some(w) => {
1107                let t0 = self.b.clock().boot_now();
1108                let o = self.b.units().hold_table_present(w);
1109                self.hold_spent(t0);
1110                o
1111            }
1112            None => Err(Outcome::Unknown("no time left on this tick".into())),
1113        };
1114        let held: Vec<UnitId> = holding.iter().filter(|u| units.contains(u)).cloned().collect();
1115        match &present {
1116            Ok(true) => {
1117                self.hold_table_seen = true;
1118                self.hold_missing_noted = false;
1119                self.hold_retry = None;
1120                return;
1121            }
1122            Err(o) => {
1123                if self.hold_table_seen {
1124                    for u in &held {
1125                        self.notice(format!(
1126                            "ALARM: {u}: the network hold's firewall table could not be checked ({o:?}); checking again"
1127                        ));
1128                    }
1129                }
1130                self.hold_table_seen = false;
1131                return;
1132            }
1133            Ok(false) => {}
1134        }
1135        let now = self.b.clock().boot_now();
1136        let first = !self.hold_missing_noted;
1137        if !first && self.hold_retry.is_some_and(|(next, _)| now < next) {
1138            return;
1139        }
1140        self.hold_missing_noted = true;
1141        for u in &held {
1142            if first {
1143                self.journal(Record::HoldApplied { unit: u.clone(), by: HoldBy::Daemon });
1144                self.notice(format!("ALARM: {u}: the network hold was removed from the firewall; re-applying it"));
1145            }
1146            if self.hold_wait().is_none() {
1147                continue;
1148            }
1149            let o = match self.hold_wait() {
1150                Some(w) => {
1151                    let t0 = self.b.clock().boot_now();
1152                    let o = self.b.units().reapply_hold(u, w);
1153                    self.hold_spent(t0);
1154                    o
1155                }
1156                None => Outcome::Unknown("no time left on this tick".into()),
1157            };
1158            if let o @ (Outcome::Failed(_) | Outcome::Unknown(_)) = o
1159                && first
1160            {
1161                self.notice(format!("ALARM: {u}: its traffic could not be held again ({o:?}); retrying"));
1162            }
1163        }
1164        self.hold_table_seen = false;
1165        // No budget left: the check is skipped, which is not "not found", and
1166        // leaves the backoff alone.
1167        let found = match self.hold_wait() {
1168            Some(w) => {
1169                let t0 = self.b.clock().boot_now();
1170                let r = matches!(self.b.units().hold_table_present(w), Ok(true));
1171                self.hold_spent(t0);
1172                Some(r)
1173            }
1174            None => None,
1175        };
1176        if found == Some(true) {
1177            self.hold_table_seen = true;
1178            self.hold_missing_noted = false;
1179            self.hold_retry = None;
1180        } else if found == Some(false) {
1181            let wait =
1182                self.hold_retry.map_or(HOLD_RETRY_MIN, |(_, w)| Millis(w.0.saturating_mul(2).min(HOLD_RETRY_MAX.0)));
1183            self.hold_retry = Some((Timestamp(now.0.saturating_add(wait.0)), wait));
1184        }
1185    }
1186
1187    /// Prepare again transactions interrupted before they mutated (by a
1188    /// restart, or withdrawn for a repair rollback).
1189    fn prepare_again(
1190        &mut self,
1191        list: Vec<(TxnId, Transition, Option<Timestamp>)>,
1192        why: &str,
1193        s: &SystemState,
1194        defs: &UnitDefs,
1195    ) {
1196        for (id, t, since) in list {
1197            // Its original source and caller (C21), so floors that depend on
1198            // who asked (the own-session exemption) still apply; the daemon's
1199            // own only when no record of it survives.
1200            let (source, authz) = match self.earlier.get(&id).cloned() {
1201                Some((p, caller @ Actor::User { .. })) => {
1202                    let admin = caller.uid().is_some_and(|u| (self.admin_of)(u));
1203                    (p.source, user_authorization(caller, admin))
1204                }
1205                Some((p, caller)) => (p.source, authorization_of(caller)),
1206                None => (Source::Maintenance, authorization_of(Actor::Daemon)),
1207            };
1208            let incident = match &source {
1209                Source::Diagnoser { incident } => Some(*incident),
1210                _ => None,
1211            };
1212            let p = self.proposal(t.clone(), s, defs, source, format!("prepared again {why}"));
1213            // Journaled with it: this is what prepares `id` again (§12f).
1214            let r = self.submit_from(p, authz, s, defs, Some(id));
1215            // A diagnoser's repair that waits again keeps its incident with it.
1216            if let (
1217                Some(inc),
1218                Submitted::Finished { txn, state: TxnState::Built, .. } | Submitted::Parked { txn, .. },
1219            ) = (incident, &r)
1220            {
1221                self.incident_parked.insert(inc, *txn);
1222            }
1223            // A withdrawal never extends a change's parking limit (I12).
1224            if let (Some(since), Submitted::Finished { txn, state: TxnState::Built, .. }) = (since, &r) {
1225                self.ex.keep_waiting_since(*txn, since);
1226            }
1227            self.notice(format!("{}: prepared again {why}: {}", render::transition(&t), render::submitted_short(&r)));
1228        }
1229    }
1230
1231    /// Generation transactions waiting in `Built`, with when each began
1232    /// waiting: what a rollback about to run would withdraw (§12f, I6).
1233    fn waiting_built(&mut self) -> Vec<(TxnId, Transition, Option<Timestamp>)> {
1234        let l = self.ledger();
1235        l.live
1236            .values()
1237            .filter(|t| t.txn.transition.changes_generation() && t.txn.state == TxnState::Built)
1238            .map(|t| (t.txn.id, t.txn.transition.clone(), self.ex.waiting_since(t.txn.id)))
1239            .collect()
1240    }
1241
1242    /// §12f, I6 (owner decision, Nov 2): a rollback of any source withdraws
1243    /// what waits in `Built` at the moment it is about to run (admitted, and
1244    /// approved if it asked; `exec::schedule`). Those of `before` the
1245    /// executor withdrew are remembered, with their proposal, caller and
1246    /// parking start, to prepare again once the rollback is terminal.
1247    fn note_withdrawn(&mut self, before: Vec<(TxnId, Transition, Option<Timestamp>)>) {
1248        if before.is_empty() {
1249            return;
1250        }
1251        let l = self.ledger();
1252        for (id, t, since) in before {
1253            if l.terminal.get(&id).is_none_or(|x| x.state != TxnState::Rejected(RejectReason::Interrupted)) {
1254                continue;
1255            }
1256            if !self.earlier.contains_key(&id) {
1257                let asked = self.log.iter().chain(&self.awaiting_post).find(|r| {
1258                    matches!(r.result, Submitted::Finished { txn, .. } | Submitted::Parked { txn, .. } if txn == id)
1259                });
1260                if let Some(r) = asked {
1261                    self.earlier.insert(id, (r.proposal.clone(), r.caller.clone()));
1262                }
1263            }
1264            self.notice(format!(
1265                "{}: set aside for a rollback (transaction {}); it will be prepared again once the rollback is done",
1266                render::transition(&t),
1267                id.0
1268            ));
1269            self.withdrawn.push((id, t, since));
1270        }
1271    }
1272
1273    /// The response to a submission; one parked for approval says, in one
1274    /// line, what it would change (§16 *Approvals show the change*).
1275    fn respond(&self, r: &Submitted) -> Response {
1276        let mut resp = render::response(r);
1277        if let Response::NeedsApproval { txn, why, .. } = &mut resp
1278            && let Some(p) = self.ex.parked.get(txn)
1279            && let Some(sum) = self.approval_diff(*txn, &p.proposal.transition, &p.facts).summary()
1280        {
1281            *why = if why.is_empty() { sum } else { format!("{why}; {sum}") };
1282        }
1283        resp
1284    }
1285
1286    /// §16 *Approvals show the change*: built from the derived facts (from
1287    /// the evaluated generations), never from the proposal.
1288    fn approval_diff(&self, txn: TxnId, t: &Transition, f: &DerivedFacts) -> planarian_ipc::ApprovalDiff {
1289        // The options, from the evaluated generations (§16): the one the
1290        // transaction runs on against its candidate. A generation built
1291        // outside Planarian has no recorded override, so the override diff
1292        // alone would show nothing it changes. If either evaluation cannot be
1293        // read, the core's override diff from the facts stands.
1294        let evaluated = self.evaluated_diff(txn);
1295        let options = evaluated.as_ref().or(f.config_diff.as_ref());
1296        // The units it would remove or stop: derive's `removes`, the same set
1297        // as `removes_sensitivity` (§11 Rollback row), each with its own.
1298        let (removed, each_known) = crate::approval::removed(f);
1299        crate::approval::diff(t, options, evaluated.is_some(), &removed, each_known, &self.ex.cfg.tables)
1300    }
1301
1302    /// Every option whose value differs between a live generation
1303    /// transaction's base and its candidate, as both evaluate.
1304    fn evaluated_diff(&self, txn: TxnId) -> Option<ConfigDiff> {
1305        let live = self.ex.kernel.journal.ledger().live.get(&txn)?;
1306        let (_, cand_path) = live.txn.candidate.as_ref()?;
1307        let gens = self.b.generations();
1308        let base = gens.path_of(live.txn.base).and_then(|p| gens.evaluate_built(&p)).ok()?;
1309        let cand = gens.evaluate_built(cand_path).ok()?;
1310        let mut d = ConfigDiff::default();
1311        for k in base.options.keys().chain(cand.options.keys()) {
1312            let (old, new) = (base.options.get(k), cand.options.get(k));
1313            if old != new {
1314                d.options.insert(k.clone(), (old.cloned(), new.cloned()));
1315            }
1316        }
1317        Some(d)
1318    }
1319
1320    /// A generation transaction is live, in any state (I6: one waiting in
1321    /// `Built` counts).
1322    fn generation_active(&mut self) -> bool {
1323        self.ledger().live.values().any(|t| t.txn.transition.changes_generation())
1324    }
1325
1326    /// §12a (P7): submit the queued write-backs, unless a generation
1327    /// transaction is active: then they stay queued, which is not a failure
1328    /// and gives no notice. One refused as busy goes back on the queue.
1329    fn flush_writebacks(&mut self) {
1330        while !self.writebacks.is_empty() && !self.generation_active() {
1331            let (mut wb, authz) = self.writebacks.remove(0);
1332            let (s2, d2) = self.observe();
1333            wb.observed = s2.digest();
1334            match self.ex.submit(wb.clone(), authz.clone(), &s2, &d2) {
1335                Submitted::Finished { state: TxnState::Committed, .. } => {}
1336                Submitted::Rejected { reason: RejectReason::Other(w), .. } if w == "busy" => {
1337                    self.writebacks.insert(0, (wb, authz));
1338                    return;
1339                }
1340                Submitted::Parked { approve, .. } => {
1341                    self.notice(format!("saving the setting needs approval: planarian approve {approve}"))
1342                }
1343                other => self.notice(format!(
1344                    "this setting will not survive reboot: saving it failed ({})",
1345                    render::submitted_short(&other)
1346                )),
1347            }
1348        }
1349    }
1350
1351    /// §13 (P22): for each unit, the catalog change last committed for it,
1352    /// if the override file no longer carries what it set.
1353    fn lost_changes(&self, l: &Ledger) -> BTreeMap<UnitId, (ChangeId, BTreeMap<String, Value>)> {
1354        let Ok((live, _)) = self.ex.b.generations().read_overrides() else { return BTreeMap::new() };
1355        let mut last: BTreeMap<UnitId, (ChangeId, BTreeMap<String, Value>)> = BTreeMap::new();
1356        // Terminal records in id order: the last commit for a unit wins.
1357        for t in l.terminal.values() {
1358            let (TxnState::Committed, Transition::ApplyChange { change, args }) = (&t.state, &t.transition) else {
1359                continue;
1360            };
1361            let (change, args) = if change.0 == planarian_changes::REAPPLY {
1362                match args.get("change") {
1363                    Some(Value::Str(c)) => {
1364                        let mut a = args.clone();
1365                        a.remove("change");
1366                        (ChangeId(c.clone()), a)
1367                    }
1368                    _ => continue,
1369                }
1370            } else {
1371                (change.clone(), args.clone())
1372            };
1373            if let Some(u) = planarian_changes::target_of(&change, &args) {
1374                last.insert(u, (change, args));
1375            }
1376        }
1377        let cat = &self.ex.cfg.catalog;
1378        last.retain(|_, (c, a)| {
1379            c.0 != "repair:reenable-predicted"
1380                && planarian_changes::handle(cat, c, a).is_ok_and(|set| !planarian_changes::carried(&live, &set))
1381        });
1382        last
1383    }
1384
1385    fn is_canonical(&self, u: &UnitId, defs: &UnitDefs) -> bool {
1386        defs.get(u).is_some_and(|d| {
1387            d.probes.iter().any(|p| p.origin == ProbeOrigin::Canonical && p.role == ProbeRole::Readiness)
1388        })
1389    }
1390
1391    fn canonical_healthy(&self, s: &SystemState, defs: &UnitDefs) -> bool {
1392        defs.iter()
1393            .filter(|(_, d)| {
1394                d.probes.iter().any(|p| p.origin == ProbeOrigin::Canonical && p.role == ProbeRole::Readiness)
1395            })
1396            .all(|(id, _)| !s.expected(id, defs) || s.status(id, defs) == UnitStatus::Running)
1397    }
1398
1399    /// One tick of the loop.
1400    /// Change the posture (the gym's `set_setting posture`; on a machine the
1401    /// generation declares it, §12d).
1402    pub fn set_posture(&mut self, p: Posture) {
1403        self.cfg.posture = p;
1404    }
1405
1406    /// §11, the unattributed residue: a unit seen running at the last tick and
1407    /// now ended by a termination signal (a `kill`, or a `systemctl stop`
1408    /// Planarian did not issue), which systemd will not restart and which is
1409    /// not Security-sensitive, is taken as intended under the Relaxed and
1410    /// Standard postures: an inferred `Override` until the next boot, recorded
1411    /// as an observation (it never passes through `decide`), with a notice
1412    /// and an undo. Units that went down with it are linked to it. A
1413    /// Security-sensitive unit's stop is always a fault, and alarmed; crash
1414    /// signals, exit statuses and OOM kills are faults. Owner decision Q5.
1415    /// Returns whether it recorded anything.
1416    fn residue(&mut self, s: &SystemState, defs: &UnitDefs) -> bool {
1417        let now = s.observed_at;
1418        let lenient = matches!(self.cfg.posture, Posture::Relaxed | Posture::Standard);
1419        let mut down: BTreeMap<UnitId, Ending> = BTreeMap::new();
1420        let mut current = BTreeMap::new();
1421        // Units that crashed this tick (up before, failed now): units that
1422        // require them and went down with them are part of that fault.
1423        let crashed: BTreeSet<UnitId> = defs
1424            .keys()
1425            .filter(|u| {
1426                matches!(self.prev_status.get(*u), Some(UnitStatus::Running | UnitStatus::Degraded(_)))
1427                    && matches!(s.status(u, defs), UnitStatus::Failed(_))
1428            })
1429            .cloned()
1430            .collect();
1431        use planarian_exec::residue::EndingFacts;
1432        let host_held = self.b.units().held();
1433        let mut tried_hold = false;
1434        for (u, def) in defs {
1435            let st = s.status(u, defs);
1436            // A Security unit killed by a termination signal that systemd
1437            // restarted between two ticks: Planarian never saw it down, but
1438            // it was still stopped outside Planarian (§11). Alarmed; systemd
1439            // has already restored it.
1440            if def.sensitivity.contains(Sensitivity::Security)
1441                && def.id.kind == UnitKind::Service
1442                && let Some(n) = self.b.units().auto_restarts(u)
1443            {
1444                let before = self.prev_auto_restarts.insert(u.clone(), n);
1445                // (Once systemd has restarted it, ExecMain* describe the new
1446                // process, so how the old one ended is no longer visible.)
1447                if before.is_some_and(|b| n > b) && !self.touched.contains(u) {
1448                    self.notice(format!(
1449                        "ALARM: {u}, which is security-sensitive, ended outside Planarian (killed, stopped or crashed); systemd started it again"
1450                    ));
1451                }
1452            }
1453            let was_up = matches!(self.prev_status.get(u), Some(UnitStatus::Running | UnitStatus::Degraded(_)));
1454            let now_down = matches!(st, UnitStatus::Failed(_) | UnitStatus::Stopped);
1455            current.insert(u.clone(), st.clone());
1456            // The decision is the pure classifier's (U3; Kani: exec::kani).
1457            let facts = EndingFacts {
1458                was_up,
1459                now_down,
1460                stopped_cleanly: st == UnitStatus::Stopped,
1461                ended: if was_up && now_down { self.b.units().ended_by(u) } else { None },
1462                service: def.id.kind == UnitKind::Service,
1463                touched_by_planarian: self.touched.contains(u) || self.generation_touched,
1464                expected: s.expected(u, defs),
1465                requirement_crashed: def
1466                    .edges
1467                    .iter()
1468                    .any(|e| e.from == *u && e.kind == EdgeKind::Requires && crashed.contains(&e.to)),
1469                security: def.sensitivity.contains(Sensitivity::Security),
1470                stops_are_faults: def.stops_are_faults,
1471                systemd_restarts: def.systemd_restart.as_ref().is_some_and(|p| p.restart),
1472                lenient,
1473                network_tool: def.holds,
1474                held: s.holds.contains(u) || host_held.as_ref().is_ok_and(|h| h.contains(u)),
1475                hold_tried: false,
1476            };
1477            use planarian_exec::residue::Residue;
1478            let mut r = planarian_exec::residue::classify(&facts);
1479            if let Residue::HoldFirst(_) = r {
1480                // §12d: "by planariad for a stop outside Planarian that is
1481                // not attributed": ask the host, whose condition (the
1482                // footprint, a release) decides, as OnFailure= does. A hold
1483                // the host cannot confirm counts as held (fail closed: the
1484                // unit is restarted, never inferred stopped). The next
1485                // network_holds journals it, by the daemon.
1486                // Deliberate and rare: the tick waits for the hold unit to
1487                // start (its condition waits about 3 s), at most HOLD_WAIT;
1488                // only here (a network tool's unattributed stop outside
1489                // Planarian), and P9's slow-tick warning says so by design.
1490                // Past HOLD_WAIT the hold counts as held (below) and its job
1491                // goes on in systemd: the socket, approvals and other repairs
1492                // are not stalled for `hold::apply`'s 30 s. Deciding before the residue is recorded is the
1493                // point: an Inferred stop must not be written for a unit
1494                // whose traffic is then held.
1495                let now = self.b.clock().boot_now();
1496                let waiting =
1497                    self.hold_unconfirmed.get(u).is_some_and(|next| now < *next) || self.hold_wait().is_none();
1498                let o = match self.hold_wait() {
1499                    _ if waiting => Outcome::Unknown("waiting for its hold job".into()),
1500                    Some(w) => {
1501                        let t0 = self.b.clock().boot_now();
1502                        let o = self.b.units().hold(u, w);
1503                        self.hold_spent(t0);
1504                        o
1505                    }
1506                    None => Outcome::Unknown("no time left on this tick".into()),
1507                };
1508                let held = match o {
1509                    Outcome::Unknown(_) if !waiting => {
1510                        // Not confirmed: alarmed once, and asked for again
1511                        // only after the retry (network_holds checks the host
1512                        // every tick meanwhile).
1513                        if self.hold_unconfirmed.insert(u.clone(), now.plus(HOLD_UNCONFIRMED_RETRY)).is_none() {
1514                            self.notice(format!("ALARM: {u}: its traffic could not be held ({o:?})"));
1515                        }
1516                        true
1517                    }
1518                    Outcome::Failed(_) | Outcome::Unknown(_) => true,
1519                    _ => self.b.units().held().map_or(true, |h| h.contains(u)),
1520                };
1521                if held {
1522                    self.daemon_held.insert(u.clone());
1523                }
1524                tried_hold = true;
1525                r = planarian_exec::residue::classify(&EndingFacts { held, hold_tried: true, ..facts });
1526            }
1527            match r {
1528                Residue::NotAStop | Residue::Fault => {}
1529                // The hold's own alarm (network_holds) is the alarm; the
1530                // incident manager restarts it, without asking
1531                // (`restarts_held_unit`).
1532                Residue::AlarmHeld(_) => {}
1533                Residue::HoldFirst(_) => unreachable!("classified again with hold_tried"),
1534                Residue::AlarmSecurity(e) => self.notice(format!(
1535                    "ALARM: {u}, which is security-sensitive, was stopped outside Planarian ({}); starting it again",
1536                    signal_name(e)
1537                )),
1538                // inferStops = false (B-11): never inferred as intent; a
1539                // fault, which the incident manager repairs.
1540                Residue::AlarmMustRun(e) => self.notice(format!(
1541                    "ALARM: {u}, which must keep running, was stopped outside Planarian ({}); starting it again",
1542                    signal_name(e)
1543                )),
1544                Residue::Intended(e) => {
1545                    down.insert(u.clone(), e);
1546                }
1547            }
1548        }
1549        self.prev_status = current;
1550        self.touched.clear();
1551        self.generation_touched = false;
1552        if down.is_empty() {
1553            return tried_hold;
1554        }
1555        // The units whose stop no other stop in this batch explains.
1556        let roots: Vec<UnitId> = down
1557            .keys()
1558            .filter(|u| !down.keys().any(|r| r != *u && planarian_schema::spec::dependents(r, defs).contains(*u)))
1559            .cloned()
1560            .collect();
1561        for r in roots {
1562            let with: Vec<UnitId> = planarian_schema::spec::dependents(&r, defs)
1563                .into_iter()
1564                .filter(|d| *d != r && down.contains_key(d))
1565                .collect();
1566            let how = signal_name(down[&r]);
1567            let mut record = |unit: UnitId, reason: String| {
1568                let o = Override {
1569                    unit,
1570                    desired: OverrideDesired::Stopped,
1571                    until: Until::NextBoot,
1572                    reason,
1573                    by: Actor::Unattributed,
1574                    attribution: Attribution::Inferred,
1575                    acknowledged: false,
1576                };
1577                self.journal(Record::OverrideSet(o));
1578            };
1579            record(r.clone(), format!("stopped outside Planarian ({how})"));
1580            for d in &with {
1581                record(d.clone(), format!("stopped together with {r}"));
1582            }
1583            let also = if with.is_empty() {
1584                String::new()
1585            } else {
1586                format!(
1587                    " (and {}, which went down with it)",
1588                    with.iter().map(|u| u.to_string()).collect::<Vec<_>>().join(", ")
1589                )
1590            };
1591            // §16 (P23): the first time, teach the attributed form.
1592            let teach = if std::mem::replace(&mut self.residue_taught, true) {
1593                String::new()
1594            } else {
1595                format!(" Next time, planarian stop {r} does the same with your name on it.")
1596            };
1597            self.notice(format!(
1598                "{r}: stopped outside Planarian ({how}); I take that as intended and leave it stopped until the next boot{also}. To start it again: planarian release {r}.{teach}"
1599            ));
1600        }
1601        let _ = now;
1602        true
1603    }
1604
1605    /// The services `cur` declares that `prev` (default: the boot entry
1606    /// before it) did not, and where each came from: the transaction that
1607    /// built `cur` (from the journal's Prepared record: its caller and
1608    /// transition, written to planarian/overrides.nix), or else a rebuild
1609    /// outside Planarian (the generation's creator and time, and the files
1610    /// that define the service).
1611    fn note_declarations(&mut self, prev: Option<GenerationId>, cur: GenerationId, announce: bool) {
1612        let gens = self.b.generations();
1613        let prev = prev.or_else(|| gens.boot_entries().ok()?.into_iter().filter(|g| *g < cur).max());
1614        let eval = |g: GenerationId| gens.path_of(g).and_then(|p| gens.evaluate_built(&p));
1615        let (Some(prev), Ok(now_e)) = (prev, eval(cur)) else { return };
1616        let Ok(before_e) = eval(prev) else { return };
1617        let (now_d, before) = (&now_e.declarations, &before_e.declarations);
1618        let added: Vec<(&String, &Vec<String>)> = now_d.iter().filter(|(k, _)| !before.contains_key(*k)).collect();
1619        if added.is_empty() {
1620            self.new_declarations.clear();
1621            return;
1622        }
1623        let made = self.ex.kernel.journal.segment().ok().and_then(|seg| {
1624            seg.into_iter().rev().find_map(|e| match e.record {
1625                Record::Prepared(t) if t.candidate.as_ref().is_some_and(|(g, _)| *g == cur) => Some(t),
1626                _ => None,
1627            })
1628        });
1629        let who = |a: &Actor| match a {
1630            Actor::User { uid: 0, .. } => "root".to_string(),
1631            Actor::User { uid, .. } => format!("uid {uid}"),
1632            a => format!("{a:?}"),
1633        };
1634        let created = gens.created_by(cur);
1635        self.new_declarations = added
1636            .into_iter()
1637            .map(|(svc, files)| {
1638                let how = match &made {
1639                    Some(t) => format!(
1640                        "by {} through {} (transaction {}), written to planarian/overrides.nix; defined in {}",
1641                        who(&t.facts.caller),
1642                        render::transition(&t.transition),
1643                        t.id.0,
1644                        if files.is_empty() { "an unknown file".to_string() } else { files.join(", ") }
1645                    ),
1646                    None => format!(
1647                        "by {} through {}, defined in {}; generation created at {}",
1648                        created.map(|(uid, _)| who(&Actor::user(uid))).unwrap_or_else(|| "an unknown user".into()),
1649                        // A file inside a flake input's source: that input,
1650                        // and its lock change since the previous generation.
1651                        match now_e.inputs.iter().find(|(_, (_, path))| {
1652                            !path.is_empty() && files.iter().any(|f| f.starts_with(path.as_str()))
1653                        }) {
1654                            Some((name, (rev, _))) => format!(
1655                                "flake input {name} ({} → {rev}) in an out-of-band nixos-rebuild",
1656                                before_e.inputs.get(name).map(|(r, _)| r.as_str()).unwrap_or("not an input before")
1657                            ),
1658                            None => "an out-of-band nixos-rebuild".into(),
1659                        },
1660                        if files.is_empty() { "an unknown file".to_string() } else { files.join(", ") },
1661                        created.map(|(_, at)| render::when(at)).unwrap_or_else(|| "an unknown time".into())
1662                    ),
1663                };
1664                (svc.clone(), format!("service:{svc} is new in generation {}, {how}", cur.0))
1665            })
1666            .collect();
1667        self.new_declarations_since = self.b.clock().now();
1668        if announce {
1669            let list: Vec<String> = self.new_declarations.iter().map(|(_, o)| o.clone()).collect();
1670            self.notice(format!("generation {} declares new services: {}", cur.0, list.join("; ")));
1671        }
1672    }
1673
1674    /// The generation's manifest with the external document's units joined
1675    /// (§9): a unit the generation declares stays the generation's. Returns
1676    /// the units the document declared that the generation overrode.
1677    fn effective_manifest(&self) -> (planarian_schema::Manifest, Vec<UnitId>) {
1678        let mut m = self.gen_manifest.clone();
1679        let mut shadowed = vec![];
1680        for (id, def) in self.external.iter().flat_map(|d| d.units.iter()) {
1681            if m.units.contains_key(id) {
1682                shadowed.push(id.clone());
1683            } else {
1684                m.units.insert(id.clone(), crate::hooks::sanitized(def));
1685            }
1686        }
1687        (m, shadowed)
1688    }
1689
1690    /// Where a unit's declaration came from, when not the generation (§9;
1691    /// future-work T5: an implant has a provenance trail).
1692    pub fn origin_of(&self, unit: &UnitId) -> Option<String> {
1693        let d = self.external.as_ref()?;
1694        (d.units.contains_key(unit) && !self.gen_manifest.units.contains_key(unit)).then(|| {
1695            format!(
1696                "the external desired-state source (issuer {:?}, sequence {})",
1697                if d.issuer.is_empty() { "unnamed" } else { &d.issuer },
1698                d.seq
1699            )
1700        })
1701    }
1702
1703    /// The health report (§10 hook), with the profile the core computes.
1704    pub fn health_report(&mut self) -> crate::hooks::HealthReport {
1705        let (s, defs) = self.observe();
1706        crate::hooks::report(crate::hooks::DisclosureProfile::Minimal, &s, &defs, self.incidents.incidents.values())
1707    }
1708
1709    /// A verified external document is now in force (or, with `None`, the
1710    /// configuration dropped the source).
1711    /// A verified external document accepted: journaled (its sequence and
1712    /// digest, B-16) when newer than the last one the journal accepted, then
1713    /// put in force.
1714    pub fn accept_external(&mut self, doc: crate::hooks::DesiredDoc, digest: Hash) {
1715        if self.ledger().external.is_none_or(|(s, _)| doc.seq > s) {
1716            self.journal(Record::ExternalAccepted { seq: doc.seq, digest });
1717        }
1718        self.set_external(Some(doc));
1719    }
1720
1721    pub fn set_external(&mut self, doc: Option<crate::hooks::DesiredDoc>) {
1722        let said = doc.as_ref().map(|d| {
1723            let names: Vec<String> = d.units.keys().map(|u| u.to_string()).collect();
1724            format!(
1725                "external desired state in force: sequence {} from {:?}, declaring {}",
1726                d.seq,
1727                if d.issuer.is_empty() { "an unnamed issuer" } else { &d.issuer },
1728                if names.is_empty() { "nothing".to_string() } else { names.join(", ") }
1729            )
1730        });
1731        self.external = doc;
1732        let (m, shadowed) = self.effective_manifest();
1733        self.asm.set_manifest(m);
1734        if let Some(s) = said {
1735            self.notice(s);
1736        }
1737        if !shadowed.is_empty() {
1738            let names: Vec<String> = shadowed.iter().map(|u| u.to_string()).collect();
1739            self.notice(format!(
1740                "the external desired state also declares {}, which this generation declares; the generation's declaration stands",
1741                names.join(", ")
1742            ));
1743        }
1744    }
1745
1746    /// The boot report (§10), once per boot: units from outside the
1747    /// generation. `main` runs it before signalling ready, so nothing that
1748    /// waits for planarian-ready.target starts before it is clean; the first
1749    /// tick runs it otherwise (the gym). The finding, if any.
1750    pub fn boot_report(&mut self) -> Option<String> {
1751        if !std::mem::take(&mut self.boot_report_due) {
1752            return None;
1753        }
1754        let foreign = self.b.units().foreign_units();
1755        for u in &foreign {
1756            self.notice(format!("INTEGRITY: boot report: {u} runs, but this generation does not declare it"));
1757        }
1758        let g = self.b.generations().current().unwrap_or_default();
1759        let text = if foreign.is_empty() {
1760            "clean: every running unit is declared by this generation".to_string()
1761        } else {
1762            format!("INTEGRITY: undeclared units: {}", foreign.join(", "))
1763        };
1764        self.last_boot_report = Some((self.b.clock().now(), g, text));
1765        (!foreign.is_empty()).then(|| format!("boot report: undeclared units: {}", foreign.join(", ")))
1766    }
1767
1768    pub fn tick(&mut self) {
1769        // §12d: all hold calls in this tick share one HOLD_WAIT, spent only
1770        // by the hold calls themselves (hold_wait).
1771        self.hold_left = HOLD_WAIT;
1772        // I7's agent for an approval is needed only while the change is
1773        // parked; a cancelled, expired, invalidated or refused one leaves.
1774        // An approved one leaves `ex.parked` only inside `Executor::approve`,
1775        // and the Approve handler writes its record in the same call, so
1776        // the record never misses its agent (a slow build is deferred before
1777        // `approve`, while the change is still parked).
1778        let parked = &self.ex.parked;
1779        self.parked_agents.retain(|t, _| parked.contains_key(t));
1780        let (s, defs) = self.observe();
1781        let now = s.observed_at;
1782        // P9: a tick over its 500 ms budget is warned, with the probes that
1783        // ran in it (they are the usual cause).
1784        // At most one warning a minute, counting the slow ticks between.
1785        if let Some((took, ran)) = self.asm.slow_tick.take() {
1786            self.slow_ticks_unwarned += 1;
1787            if self.last_slow_warning.is_none_or(|t| now.0 >= t.0 + SLOW_TICK_WARN_EVERY.0) {
1788                eprintln!(
1789                    "planariad: warning: slow tick: {} ms (budget {} ms; {} slow tick(s) since the last warning); probes run: {}",
1790                    took.0,
1791                    planarian_state::TICK_BUDGET.0,
1792                    self.slow_ticks_unwarned,
1793                    if ran.is_empty() { "none".to_string() } else { ran.join(", ") }
1794                );
1795                self.slow_tick_warnings += 1;
1796                self.slow_ticks_unwarned = 0;
1797                self.last_slow_warning = Some(now);
1798            }
1799        }
1800
1801        // §12f quiet mode: while foreground work holds in any session.
1802        let quiet = s.conditions.iter().any(|c| matches!(c, Condition::Foreground { .. }));
1803        if self.quiet && !quiet && !self.quiet_queue.is_empty() {
1804            let batch = std::mem::take(&mut self.quiet_queue);
1805            self.say(format!("while you were busy ({}): {}", batch.len(), batch.join(" · ")));
1806        }
1807        self.quiet = quiet;
1808        self.asm.quiet = quiet;
1809
1810        // §10 boot report, on the first tick after boot: units from outside the
1811        // generation are an Integrity failure (§12d); the journal chain was
1812        // checked when it was opened, and the core hash by planarian-recovery.
1813        if let Some(why) = self.boot_report() {
1814            self.integrity = Some(why);
1815            return;
1816        }
1817
1818        let again = std::mem::take(&mut self.reprepare).into_iter().map(|(id, t)| (id, t, None)).collect();
1819        self.prepare_again(again, "after a restart", &s, &defs);
1820        // §12f: what a rollback withdrew, once that rollback is terminal.
1821        if !self.withdrawn.is_empty() && !self.generation_active() {
1822            let again = std::mem::take(&mut self.withdrawn);
1823            self.prepare_again(again, "after the rollback", &s, &defs);
1824        }
1825
1826        // §11: stops nobody attributed, before anything treats them as faults.
1827        let (s, defs) = if self.residue(&s, &defs) { self.observe() } else { (s, defs) };
1828
1829        // I18: re-read the retained journal against the head this daemon holds.
1830        if self.chain_checked.is_none_or(|t| now.0.saturating_sub(t.0) >= CHAIN_CHECK_EVERY.0) {
1831            self.chain_checked = Some(now);
1832            if let Err(JournalError::Integrity(why)) = self.ex.kernel.journal.verify_chain() {
1833                self.integrity = Some(format!("journal: the hash chain is broken: {why}"));
1834                return;
1835            }
1836        }
1837
1838        // §12d: the hold is in place before the incident opens; a hold the
1839        // daemon just applied is in the state the repair is derived from.
1840        let (s, defs) = if self.network_holds(&s, &defs) { self.observe() } else { (s, defs) };
1841
1842        // §13: a hold opened here runs from the decision, after the probes.
1843        let decided = self.b.clock().now();
1844        for inc in self.incidents.tick_at(now, decided, &s, &defs) {
1845            if matches!(inc.state, IncidentState::Escalated { .. }) && inc.attempts.is_empty() {
1846                self.notice(format!(
1847                    "{}: escalated: it failed again soon after each of {} repairs; it needs a person (no automatic action for an hour)",
1848                    inc.target, inc.regressions
1849                ));
1850            }
1851            // A repair incident opens: say what failed, in the probes' words.
1852            // In quiet mode the repair's own notice is the one notice (§12f).
1853            if inc.kind == IncidentKind::Repair
1854                && inc.attempts.is_empty()
1855                && !self.quiet
1856                && self.diag_noted.insert(inc.id)
1857            {
1858                let failing: Vec<String> = s
1859                    .units
1860                    .get(&inc.target)
1861                    .map(|(_, ev)| {
1862                        ev.iter()
1863                            .filter_map(|e| match &e.result {
1864                                ProbeResult::Fail(m) => Some(format!("{}: {m}", e.probe.0)),
1865                                _ => None,
1866                            })
1867                            .collect()
1868                    })
1869                    .unwrap_or_default();
1870                // §1: name the deepest divergent unit below it, when it isn't itself.
1871                let none = BTreeMap::new();
1872                let inputs = diagnose::Inputs {
1873                    state: &s,
1874                    defs: &defs,
1875                    failed_since: &self.asm.failed_since,
1876                    known_drivers: &self.asm.known_drivers,
1877                    current_planarian_built: false,
1878                    alternatives: &none,
1879                    lost_changes: &BTreeMap::new(),
1880                };
1881                let root = diagnose::root_cause(&inc.target, &inputs);
1882                let because = if root == inc.target { String::new() } else { format!(" (root cause: {root})") };
1883                let origin = self.origin_of(&inc.target).map(|o| format!("; declared by {o}")).unwrap_or_default();
1884                // §15: what this generation added, with its origin, while recent.
1885                let origin = if !self.new_declarations.is_empty()
1886                    && now.0.saturating_sub(self.new_declarations_since.0) < NEW_DECLARATIONS_FOR.0
1887                {
1888                    let list: Vec<&str> = self.new_declarations.iter().take(3).map(|(_, o)| o.as_str()).collect();
1889                    format!("{origin}; new since the previous generation: {}", list.join("; "))
1890                } else {
1891                    origin
1892                };
1893                self.notice(format!(
1894                    "{}: {:?}{because}{}{origin}",
1895                    inc.target,
1896                    inc.divergence.1,
1897                    if failing.is_empty() { String::new() } else { format!(" ({})", failing.join("; ")) }
1898                ));
1899            }
1900            if inc.kind == IncidentKind::Diagnostic
1901                && inc.state != IncidentState::Closed
1902                && self.diag_noted.insert(inc.id)
1903            {
1904                // Re-observe through the Readiness probes that still answer;
1905                // never a restart (§10).
1906                let answering: Vec<String> = defs
1907                    .get(&inc.target)
1908                    .map(|d| {
1909                        let ev = s.units.get(&inc.target).map(|(_, e)| e.as_slice()).unwrap_or(&[]);
1910                        d.probes
1911                            .iter()
1912                            .filter(|p| p.role == ProbeRole::Readiness)
1913                            .filter(|p| {
1914                                !ev.iter().any(|e| {
1915                                    e.probe == p.id
1916                                        && matches!(
1917                                            e.result,
1918                                            ProbeResult::Timeout | ProbeResult::Abandoned | ProbeResult::Error(_)
1919                                        )
1920                                })
1921                            })
1922                            .map(|p| p.id.0.clone())
1923                            .collect()
1924                    })
1925                    .unwrap_or_default();
1926                self.notice(format!(
1927                    "{}: its health is unknown (checks time out); re-observing through {} rather than restarting it",
1928                    inc.target,
1929                    if answering.is_empty() { "its other checks".into() } else { answering.join(", ") }
1930                ));
1931            }
1932            self.journal(Record::Incident(inc));
1933        }
1934        let diag: Vec<UnitId> = self.incidents.diagnostic().iter().map(|i| i.target.clone()).collect();
1935        for t in diag {
1936            self.asm.reprobe(&t);
1937        }
1938
1939        let l = self.ledger();
1940        if l.interlocked() {
1941            if !self.interlock_noted {
1942                self.notice(
1943                    "recovery failed; autonomous repair is stopped until an administrator runs `planarian resolve`"
1944                        .into(),
1945                );
1946                self.interlock_noted = true;
1947            }
1948        } else {
1949            self.interlock_noted = false;
1950            // Unit repairs may run while a generation awaits its boot, and
1951            // once it is admitted at boot (I6).
1952            let busy = l.live.values().any(|t| {
1953                !t.txn.state.is_pre_mutation()
1954                    && t.txn.state != TxnState::AwaitingBoot
1955                    && !(t.txn.state == TxnState::Promoted && t.boot_only)
1956            });
1957            if !busy {
1958                self.repair_one(&s, &defs, &l);
1959            }
1960            // §12f: an activation window opened for a waiting generation.
1961            if !busy {
1962                for (id, r) in self.ex.activate_waiting(&s, &defs) {
1963                    // A repair that waited is an attempt once it runs.
1964                    let linked: Vec<IncidentId> =
1965                        self.incident_parked.iter().filter(|(_, t)| **t == id).map(|(i, _)| *i).collect();
1966                    for inc in linked {
1967                        self.incident_parked.remove(&inc);
1968                        if let Submitted::Finished { state, .. } = &r
1969                            && let Some(i) =
1970                                self.incidents.record_attempt(inc, id, outcome_of(state), self.b.clock().now())
1971                        {
1972                            self.journal(Record::Incident(i));
1973                        }
1974                    }
1975                    if matches!(r, Submitted::Finished { state: TxnState::Rejected(RejectReason::Stale), .. }) {
1976                        self.notice(format!(
1977                            "transaction {}: not activated: the configuration, the override file or what the change would do changed while it waited; ask for it again",
1978                            id.0
1979                        ));
1980                        continue;
1981                    }
1982                    self.notice(format!(
1983                        "transaction {}: activated now that nothing is in the way: {}",
1984                        id.0,
1985                        render::submitted_short(&r)
1986                    ));
1987                }
1988            }
1989            // Quiet mode defers garbage collection (§12f).
1990            if !busy && !self.quiet && now >= self.next_gc_check {
1991                self.next_gc_check = now.plus(GC_CHECK);
1992                self.collect_if_over(&s, &defs);
1993            }
1994        }
1995
1996        // §12a: write-backs that waited for a generation transaction.
1997        if !l.interlocked() {
1998            self.flush_writebacks();
1999        }
2000
2001        for t in self.ex.expire() {
2002            self.notice(format!("approval for transaction {} expired", t.0));
2003        }
2004        self.remind_restart(&l);
2005        for (t, change) in self.ex.expire_waiting() {
2006            // §12f: the incident, if any, reopens.
2007            self.incident_parked.retain(|_, w| *w != t);
2008            self.notice(format!(
2009                "transaction {} cancelled: it waited a day in Built without an activation window ({}); issue it again to retry",
2010                t.0,
2011                render::transition(&change)
2012            ));
2013        }
2014        if self.ex.kernel.journal.needs_compaction()
2015            && let Err(e) = self.ex.kernel.journal.compact()
2016        {
2017            self.notice(format!("journal compaction failed: {e:?}"));
2018        }
2019        self.confirm_boot(&s, &defs, now);
2020        self.note_space();
2021        self.note_unreachable(&s, &defs, now);
2022    }
2023
2024    /// §12f: a transaction in `AwaitingBoot` has no time limit; the "restart
2025    /// to finish" notice is repeated once a day until that restart.
2026    fn remind_restart(&mut self, l: &Ledger) {
2027        let now = self.b.clock().boot_now();
2028        let waiting: Vec<(TxnId, String)> = l
2029            .live
2030            .values()
2031            .filter(|t| t.txn.state == TxnState::AwaitingBoot)
2032            .map(|t| (t.txn.id, render::transition(&t.txn.transition)))
2033            .collect();
2034        self.restart_reminded.retain(|id, _| waiting.iter().any(|(w, _)| w == id));
2035        for (id, what) in waiting {
2036            let last = *self.restart_reminded.entry(id).or_insert(now);
2037            if now > last.plus(RESTART_REMINDER) {
2038                self.restart_reminded.insert(id, now);
2039                self.notice(format!("transaction {}: {what} is still waiting for a restart; restart to finish", id.0));
2040            }
2041        }
2042    }
2043
2044    /// §10: a probe that times out yields Unknown, never Failed, so an
2045    /// external dependency that stops answering (traffic dropped at a
2046    /// gateway) opens no incident and nothing local is touched. Still, say
2047    /// so once it has lasted, and again when it answers.
2048    fn note_unreachable(&mut self, s: &SystemState, defs: &UnitDefs, now: Timestamp) {
2049        for (id, def) in defs {
2050            if !def.sensitivity.contains(Sensitivity::External) {
2051                continue;
2052            }
2053            let timing_out: Vec<String> = match (s.status(id, defs), s.units.get(id)) {
2054                (UnitStatus::Unknown, Some((_, ev))) => ev
2055                    .iter()
2056                    .filter(|e| {
2057                        matches!(e.result, ProbeResult::Timeout | ProbeResult::Abandoned | ProbeResult::Error(_))
2058                    })
2059                    .map(|e| e.probe.0.clone())
2060                    .collect(),
2061                _ => vec![],
2062            };
2063            if timing_out.is_empty() {
2064                if let Some((_, true)) = self.unreachable.remove(id) {
2065                    self.notice(format!("{id}: answers again"));
2066                }
2067                continue;
2068            }
2069            let (since, said) = *self.unreachable.entry(id.clone()).or_insert((now, false));
2070            if !said && now >= since.plus(UNREACHABLE_AFTER) {
2071                self.unreachable.insert(id.clone(), (since, true));
2072                self.notice(format!(
2073                    "{id}: can't reach it: {} timing out for {} min; it may be the network on the way there, so nothing local is restarted",
2074                    timing_out.join(", "),
2075                    (now.0 - since.0) / 60_000
2076                ));
2077            }
2078        }
2079    }
2080
2081    /// §12b: a collection that could not meet the space budget says so.
2082    fn note_space(&mut self) {
2083        let Some(r) = self.b.generations().space_report() else { return };
2084        if r.at <= self.space_noted {
2085            return;
2086        }
2087        self.space_noted = r.at;
2088        if let Some(short) = r.shortfall {
2089            self.notice(format!(
2090                "BudgetUnsatisfiable: the generations Planarian must keep use {} over the space budget of {}; everything else was collected, so the budget is a target, not a guarantee",
2091                render::bytes(short),
2092                render::bytes(r.max_bytes)
2093            ));
2094        }
2095    }
2096
2097    /// §12b: when the generations exceed a bound and something can be
2098    /// collected, collect (Runtime, daemon-initiated).
2099    fn collect_if_over(&mut self, s: &SystemState, defs: &UnitDefs) {
2100        let l = self.ledger();
2101        let protect = self.protect(&l, s.generation);
2102        let Ok(plan) = self.b.generations().gc_plan(&protect, false) else { return };
2103        if plan.delete.is_empty() {
2104            return;
2105        }
2106        let reason = format!(
2107            "{} generation(s) over the collection bounds (keep {}, {} of {} bytes)",
2108            plan.delete.len(),
2109            plan.kept.len(),
2110            plan.kept_bytes,
2111            plan.max_bytes
2112        );
2113        let p = self.proposal(Transition::CollectGarbage { emergency: false }, s, defs, Source::Maintenance, reason);
2114        let _ = self.submit(p, authorization_of(Actor::Daemon), s, defs);
2115    }
2116
2117    fn repair_one(&mut self, s: &SystemState, defs: &UnitDefs, l: &Ledger) {
2118        let now = s.observed_at;
2119        // Drop parked links whose approval was decided or whose wait in
2120        // Built ended.
2121        let parked = self.ex.parked.clone();
2122        let ex = &self.ex;
2123        self.incident_parked.retain(|_, t| parked.contains_key(t) || ex.is_waiting(*t));
2124        // §12f (P8): in quiet mode a deferred repair is deferred in fact: the
2125        // loop skips it, so its budget and backoff do not advance.
2126        let quiet = self.quiet;
2127        let Some(inc) = self
2128            .incidents
2129            .actionable(now)
2130            .into_iter()
2131            .filter(|i| !quiet || foreground_depends_on(&i.target, defs))
2132            .find(|i| !self.incident_parked.contains_key(&i.id))
2133            .cloned()
2134        else {
2135            return;
2136        };
2137        let current = s.generation;
2138        let alternatives: BTreeMap<String, Vec<Value>> = defs
2139            .values()
2140            .filter_map(|d| d.runtime_repair.clone())
2141            .collect::<BTreeSet<String>>()
2142            .into_iter()
2143            .map(|opt| {
2144                let alts = self.ex.b.runtime().alternatives(&opt);
2145                (opt, alts)
2146            })
2147            .collect();
2148        let inputs = diagnose::Inputs {
2149            state: s,
2150            defs,
2151            failed_since: &self.asm.failed_since,
2152            known_drivers: &self.asm.known_drivers,
2153            current_planarian_built: l.live_tested == Some(current),
2154            alternatives: &alternatives,
2155            lost_changes: &self.lost_changes(l),
2156        };
2157        // The deepest divergent unit below the target (§13 step 1), named
2158        // whenever it isn't the target itself.
2159        let root = diagnose::root_cause(&inc.target, &inputs);
2160        let because = if root == inc.target { String::new() } else { format!(" (root cause: {root})") };
2161        // Every other unit in trouble now, with its root cause (named if this
2162        // repair turns out to be theirs). Not just open incidents: a unit can
2163        // diverge inside its startup grace, before it has one.
2164        let others_rooted: Vec<(UnitId, UnitId, UnitStatus)> = defs
2165            .keys()
2166            .filter(|u| **u != inc.target && s.expected(u, defs))
2167            .filter_map(|u| {
2168                let st = s.status(u, defs);
2169                matches!(st, UnitStatus::Degraded(_) | UnitStatus::Failed(_) | UnitStatus::Stopped)
2170                    .then(|| (u.clone(), diagnose::root_cause(u, &inputs), st))
2171            })
2172            .collect();
2173        // §13: a cooldown or hold set here runs from the decision, read
2174        // after the work before it, as an attempt's backoff does (not from
2175        // the observation's start).
2176        match diagnose::next(&inc, &inputs) {
2177            None => {
2178                if let Some(i) = self.incidents.exhausted(inc.id, self.b.clock().now()) {
2179                    self.journal(Record::Incident(i));
2180                }
2181                self.notice(format!(
2182                    "{}: {:?}{because} and nothing I know how to do would fix it; it needs a person (I'll look again in an hour)",
2183                    inc.target, inc.divergence.1
2184                ));
2185            }
2186            // §12e, I16: no automatic rollback below the security floor.
2187            Some(c)
2188                if c.transition == Transition::Rollback && s.security_floor.is_some_and(|f| s.known_good.0 < f.0) =>
2189            {
2190                if let Some(i) =
2191                    self.incidents.escalate(inc.id, EscalationReason::RevertBlockedByFloor, self.b.clock().now())
2192                {
2193                    self.journal(Record::Incident(i));
2194                }
2195                self.notice(format!(
2196                    "{}: {:?}{because}; rolling back would go below the security floor (generation {}), so it needs an administrator: the known-good generation {} lacks a security fix",
2197                    inc.target,
2198                    inc.divergence.1,
2199                    s.security_floor.map(|f| f.0).unwrap_or_default(),
2200                    s.known_good.0
2201                ));
2202            }
2203            Some(c) if self.ex.parked.values().any(|p| p.proposal.transition == c.transition) => {
2204                // Another incident with the same root cause is already waiting on
2205                // this exact change: wait with it rather than ask twice.
2206                let txn = self.ex.parked.values().find(|p| p.proposal.transition == c.transition).map(|p| p.txn);
2207                if let Some(t) = txn {
2208                    self.incident_parked.insert(inc.id, t);
2209                }
2210            }
2211            Some(c) => {
2212                // Stopped cleanly rather than crashed: likely someone outside
2213                // Planarian stopped it. Say how to make that stick.
2214                if inc.attempts.is_empty()
2215                    && matches!(c.transition, Transition::RestartUnit(_))
2216                    && c.target.kind == UnitKind::Service
2217                    && s.status(&c.target, defs) == UnitStatus::Stopped
2218                {
2219                    self.notice(format!(
2220                        "{}: it was stopped, not crashed, so I'm starting it again; to keep it stopped: planarian override {}",
2221                        c.target, c.target
2222                    ));
2223                }
2224                let p = self.proposal(
2225                    c.transition.clone(),
2226                    s,
2227                    defs,
2228                    Source::Diagnoser { incident: inc.id },
2229                    c.reason.clone(),
2230                );
2231                // §10: a canonical unit that needs repair before an admitted
2232                // boot-only change is confirmed counts against it.
2233                let admitted = l.live.values().any(|t| t.txn.state == TxnState::Promoted && t.boot_only);
2234                if admitted && self.is_canonical(&c.target, defs) {
2235                    self.repaired_before_confirmation.insert(c.target.clone());
2236                }
2237                match self.submit(p, authorization_of(Actor::Daemon), s, defs) {
2238                    // §12f: waiting in Built for an activation window is not an
2239                    // attempt; the incident waits with it, and reopens if the
2240                    // wait expires.
2241                    Submitted::Finished { txn, state: TxnState::Built, .. } => {
2242                        self.incident_parked.insert(inc.id, txn);
2243                    }
2244                    Submitted::Finished { txn, state, .. } => {
2245                        // §13: the backoff runs from the attempt itself, not
2246                        // from the observation that found it actionable
2247                        // (which may have taken a while): read after it ran,
2248                        // so the next attempt is never closer than the floor.
2249                        let at = self.b.clock().now();
2250                        if let Some(i) = self.incidents.record_attempt(inc.id, txn, outcome_of(&state), at) {
2251                            if matches!(i.state, IncidentState::Cooldown { .. }) {
2252                                self.notice(format!(
2253                                    "{}: tried {} time(s) and it keeps failing; it needs a person (I'll try once more in {} min)",
2254                                    inc.target,
2255                                    i.attempts.len(),
2256                                    i.backoff.0 / 60_000
2257                                ));
2258                            }
2259                            self.journal(Record::Incident(i));
2260                        }
2261                        self.notice(format!(
2262                            "{}: {} → {:?}{because}",
2263                            inc.target,
2264                            render::transition(&c.transition),
2265                            state
2266                        ));
2267                        // §1: other units in trouble because of the one just
2268                        // repaired are told so (their incidents close once
2269                        // they are well again).
2270                        if state == TxnState::Committed {
2271                            let dependents: Vec<(UnitId, UnitStatus)> = others_rooted
2272                                .iter()
2273                                .filter(|(t, root, _)| *root == c.target && *t != c.target)
2274                                .map(|(t, _, st)| (t.clone(), st.clone()))
2275                                .collect();
2276                            for (d, st) in dependents {
2277                                self.notice(format!(
2278                                    "{d}: {st:?} (root cause: {}); repaired with {}",
2279                                    c.target,
2280                                    render::transition(&c.transition)
2281                                ));
2282                            }
2283                        }
2284                    }
2285                    Submitted::Parked { txn, approve, facts } => {
2286                        self.incident_parked.insert(inc.id, txn);
2287                        let changes = self
2288                            .approval_diff(txn, &c.transition, &facts)
2289                            .summary()
2290                            .map(|s| format!("; {s}"))
2291                            .unwrap_or_default();
2292                        self.notice(format!(
2293                            "{}: {} needs approval ({}{changes}): planarian approve {approve}",
2294                            inc.target,
2295                            render::transition(&c.transition),
2296                            render::facts_summary(&facts)
2297                        ));
2298                    }
2299                    Submitted::Rejected { reason, detail, .. } => {
2300                        if let Some(i) = self.incidents.exhausted(inc.id, self.b.clock().now()) {
2301                            self.journal(Record::Incident(i));
2302                        }
2303                        self.notice(format!(
2304                            "{}: {} refused ({reason:?}: {detail})",
2305                            inc.target,
2306                            render::transition(&c.transition)
2307                        ));
2308                    }
2309                    Submitted::Error(e) => self.notice(format!("{}: {e}", inc.target)),
2310                }
2311            }
2312        }
2313    }
2314
2315    fn confirm_boot(&mut self, s: &SystemState, defs: &UnitDefs, now: Timestamp) {
2316        let Some((g, at)) = self.confirm else { return };
2317        if now < at {
2318            return;
2319        }
2320        self.confirm = None;
2321        let healthy = s.generation == g && self.canonical_healthy(s, defs);
2322        // §10, §12 step 7: the admitted boot-only change commits, or is
2323        // withdrawn, here and only here.
2324        let admitted: Vec<TxnId> = self
2325            .ledger()
2326            .live
2327            .values()
2328            .filter(|t| t.txn.state == TxnState::Promoted && t.path.iter().any(|p| p == "AwaitingBoot"))
2329            .map(|t| t.txn.id)
2330            .collect();
2331        // Repairs run meanwhile, but a canonical unit that needed one is
2332        // evidence against the change, even if it is up again now.
2333        let repaired = std::mem::take(&mut self.repaired_before_confirmation);
2334        for id in admitted {
2335            match self.ex.kernel.confirm_boot_only(id, healthy && repaired.is_empty()) {
2336                Ok(TxnState::Committed) => {}
2337                Ok(TxnState::Reverted) => {
2338                    let back = self.b.generations().boot_default().map(|d| d.0).unwrap_or_default();
2339                    let why = if repaired.is_empty() {
2340                        "canonical probes were failing at the 10-minute checkpoint".to_string()
2341                    } else {
2342                        let units: Vec<String> = repaired.iter().map(|u| u.to_string()).collect();
2343                        format!("{} needed a repair before the 10-minute checkpoint", units.join(", "))
2344                    };
2345                    self.notice(format!(
2346                        "transaction {}: not confirmed ({why}); generation {back} is the default again \
2347                         and a restart completes the withdrawal",
2348                        id.0
2349                    ))
2350                }
2351                Ok(other) => self.notice(format!("transaction {}: confirmation left it {other:?}", id.0)),
2352                Err(e) => self.notice(format!("transaction {}: confirmation failed: {e:?}", id.0)),
2353            }
2354        }
2355        if healthy {
2356            self.journal(Record::KnownGood(g));
2357            self.notice(format!("generation {} is now the boot-confirmed known-good baseline", g.0));
2358        } else {
2359            self.notice(format!(
2360                "generation {} was not confirmed: canonical probes were failing at the 10-minute checkpoint",
2361                g.0
2362            ));
2363        }
2364    }
2365
2366    // ---------- Requests ----------
2367
2368    /// §10: `Actor::User { uid, session, seat, active }` from the caller's
2369    /// logind session; a remote session has no seat and is never local.
2370    fn authz(&mut self, peer: &Peer) -> Authorization {
2371        if let Some(name) = &peer.agent {
2372            // §11 (owner decision, 2026-10-30): an agent has the permissions
2373            // of the user who started it, judged as that user with no login
2374            // session and no terminal. One the journal has no start for is
2375            // granted nothing.
2376            // Bound to the instance the peer runs in, never the name (I24).
2377            let instance = peer.instance.map(|i| i.0).unwrap_or_default();
2378            let Some(by) = self.ledger().agent_for(name, instance) else {
2379                let actor = Actor::Agent(agent_principal(name));
2380                return Authorization { principal: actor.principal(), actor, capabilities: Capabilities::default() };
2381            };
2382            let actor = Actor::User { uid: by, session: None, seat: None, active: false };
2383            return user_authorization(actor, (self.admin_of)(by));
2384        }
2385        let info =
2386            peer.session.as_ref().and_then(|id| self.asm.sessions.iter().find(|x| &x.id == id && x.uid == peer.uid));
2387        let actor = Actor::User {
2388            uid: peer.uid,
2389            session: info.map(|x| x.id.clone()),
2390            seat: info.filter(|x| !x.remote).and_then(|x| x.seat.clone()),
2391            active: info.is_some_and(|x| x.active),
2392        };
2393        user_authorization(actor, peer.admin)
2394    }
2395
2396    /// The slow part of the request just handled, if `handle` deferred it:
2397    /// run the job anywhere, then pass its result to `resume`. Until then the
2398    /// answer `handle` returned is a placeholder, not for the caller.
2399    pub fn take_deferred(&mut self) -> Option<(PrebuildJob<B::Gens>, Resume)> {
2400        self.deferred.take()
2401    }
2402
2403    /// Finish a deferred request with its prebuilt candidate: plan, decide
2404    /// and run it as `handle` would have, against a fresh observation.
2405    pub fn resume(&mut self, r: Resume, prebuilt: Prebuilt) -> Response {
2406        self.ex.prebuilt = Some(prebuilt);
2407        let defer = std::mem::replace(&mut self.defer, false);
2408        let resp = match r.kind {
2409            ResumeKind::Submit { t, source, reason } => self.user_submit(t, r.peer, source, reason),
2410            ResumeKind::Request(req) => self.handle(r.peer, req),
2411        };
2412        self.defer = defer;
2413        self.ex.prebuilt = None;
2414        resp
2415    }
2416
2417    fn user_submit(&mut self, t: Transition, peer: Peer, source: Source, reason: String) -> Response {
2418        // Admission before work (R09): a caller refused anyway queues no
2419        // staging or build; `submit` then refuses it, logged.
2420        let who = self.authz(&peer);
2421        let admitted = self.ex.admit(&t, &who).is_ok();
2422        if admitted
2423            && self.defer
2424            && self.deferred.is_none()
2425            && let Some(job) = self.ex.prebuild_job(&t)
2426        {
2427            self.deferred = Some((job, Resume { peer, kind: ResumeKind::Submit { t, source, reason } }));
2428            return Response::Ok { message: "deferred".into() };
2429        }
2430        let (s, defs) = self.observe();
2431        let p = self.proposal(t, &s, &defs, source, reason);
2432        let authz = self.authz(&peer);
2433        let t = p.transition.clone();
2434        let r = self.submit(p, authz, &s, &defs);
2435        // §16: the notice shows what a generation change waiting for
2436        // approval would change.
2437        if let Submitted::Parked { txn, approve, facts } = &r
2438            && let Some(changes) = self.approval_diff(*txn, &t, facts).summary()
2439        {
2440            self.notice(format!(
2441                "{} needs approval ({}; {changes}): planarian approve {approve}",
2442                render::transition(&t),
2443                render::facts_summary(facts)
2444            ));
2445        }
2446        let mut resp = self.respond(&r);
2447        // Who may approve it (§11): an administrator, when the class says so.
2448        if let (Response::NeedsApproval { txn, why, .. }, Submitted::Parked { .. }) = (&mut resp, &r)
2449            && self.ex.parked.get(txn).is_some_and(|p| p.class == ApproverClass::Administrator)
2450        {
2451            why.push_str("; an administrator must approve it");
2452        }
2453        resp
2454    }
2455
2456    fn status(&mut self) -> StatusReport {
2457        // The tick's observation (at most a tick old): answering a status
2458        // request must not run the probes again, or frequent readers (the
2459        // tray, the console) hold the core for a probe timeout each.
2460        let (s, defs) = self.last.clone().unwrap_or_else(|| self.observe());
2461        let l = self.ledger();
2462        let units = defs
2463            .keys()
2464            .map(|u| {
2465                let status = s.status(u, &defs);
2466                UnitRow { unit: u.clone(), detail: render::evidence(&s, u), expected: s.expected(u, &defs), status }
2467            })
2468            .collect();
2469        let pending = self
2470            .ex
2471            .parked
2472            .values()
2473            .map(|p| Pending {
2474                txn: p.txn,
2475                approve: p.bound_to,
2476                what: render::transition(&p.proposal.transition),
2477                facts: p.facts.clone(),
2478                diff: self.approval_diff(p.txn, &p.proposal.transition, &p.facts),
2479                // Shown to people: the boot-clock issue time on the wall clock.
2480                issued: Timestamp(
2481                    self.b.clock().now().0.saturating_sub(self.b.clock().boot_now().0.saturating_sub(p.issued.0)),
2482                ),
2483            })
2484            .collect();
2485        StatusReport {
2486            generation: s.generation,
2487            known_good: l.known_good,
2488            live_tested: l.live_tested,
2489            accepted_baseline: l.accepted_baseline,
2490            boot_default: self.b.generations().boot_default().ok(),
2491            baseline_note: l
2492                .known_good
2493                .is_none()
2494                .then(|| "no baseline yet: autonomy is limited to Runtime until this generation is confirmed".into()),
2495            units,
2496            incidents: self.incidents.incidents.values().cloned().collect(),
2497            pending,
2498            in_flight: l
2499                .live
2500                .values()
2501                .map(|t| (t.txn.id, t.txn.state.clone(), render::transition(&t.txn.transition)))
2502                .collect(),
2503            interlock: l
2504                .interlock
2505                .values()
2506                .map(|i| {
2507                    (i.txn, format!("{} failed from {}: {}", render::transition(&i.transition), i.failed_from, i.note))
2508                })
2509                .collect(),
2510            reconciliation: l.last_reconciliation.map(|r| r.actions),
2511            inferred: l
2512                .overrides
2513                .iter()
2514                .filter(|o| o.attribution == Attribution::Inferred && !o.acknowledged)
2515                .map(|o| (o.unit.clone(), o.reason.clone()))
2516                .collect(),
2517            notices: self
2518                .building
2519                .as_ref()
2520                .map(|(what, since)| {
2521                    let mins = self.b.clock().now().0.saturating_sub(since.0) / 60_000;
2522                    format!("building {what} ({mins} min so far); repairs go on meanwhile")
2523                })
2524                .into_iter()
2525                .chain(self.notices.iter().cloned())
2526                .collect(),
2527            space: self.b.generations().space_report(),
2528        }
2529    }
2530
2531    /// §16a: what the read-only console shows.
2532    fn console(&mut self) -> ConsoleReport {
2533        let status = self.status();
2534        let (s, defs) = self.last.clone().unwrap_or_else(|| self.observe());
2535        let l = self.ledger();
2536        let nodes = defs
2537            .iter()
2538            .map(|(u, def)| {
2539                let health = s.status(u, &defs);
2540                let observed = match s.units.get(u).map(|(o, _)| o) {
2541                    None => "not observed".to_string(),
2542                    Some(Observed::Failed(r)) => format!("Failed: {r:?}"),
2543                    Some(Observed::Unknown { reason, .. }) => format!("Unknown: {reason}"),
2544                    Some(Observed::Transitioning { .. }) => "Transitioning".to_string(),
2545                    // A loaded, inactive service is observed as `Bound`.
2546                    Some(Observed::Bound) if u.kind != UnitKind::Device => "Stopped".to_string(),
2547                    Some(o) => format!("{o:?}"),
2548                };
2549                GraphNode {
2550                    unit: u.clone(),
2551                    desired: format!("{:?}", def.desired.status),
2552                    expected: s.expected(u, &defs),
2553                    observed,
2554                    stale: health == UnitStatus::Unknown,
2555                    suppressed: matches!(health, UnitStatus::Overridden(_)),
2556                    health,
2557                }
2558            })
2559            .collect();
2560        let edges = defs
2561            .values()
2562            .flat_map(|d| d.edges.iter().map(|e| GraphEdge { from: e.from.clone(), to: e.to.clone(), kind: e.kind }))
2563            .collect();
2564        let mut timeline: Vec<TxnSummary> = l
2565            .live
2566            .values()
2567            .map(|t| TxnSummary {
2568                txn: t.txn.id,
2569                what: render::transition(&t.txn.transition),
2570                path: t.path.clone(),
2571                state: t.txn.state.clone(),
2572                live: true,
2573                by_watchdog: t.txn.lease.owner.is_watchdog(),
2574                why: t.why.clone(),
2575            })
2576            .chain(l.terminal.iter().map(|(id, t)| TxnSummary {
2577                txn: *id,
2578                what: render::transition(&t.transition),
2579                path: t.path.clone(),
2580                state: t.state.clone(),
2581                live: false,
2582                by_watchdog: t.owner.is_watchdog(),
2583                why: t.why.clone(),
2584            }))
2585            .collect();
2586        timeline.sort_by_key(|t| std::cmp::Reverse(t.txn));
2587        let gens = self.b.generations();
2588        let current = s.generation;
2589        let previous = gens.boot_entries().ok().and_then(|e| e.into_iter().filter(|g| *g < current).max());
2590        let last_change_reverted = l
2591            .terminal
2592            .values()
2593            .rev()
2594            .find(|t| t.transition.changes_generation())
2595            .is_some_and(|t| t.state == TxnState::Reverted);
2596        ConsoleReport {
2597            status,
2598            nodes,
2599            edges,
2600            timeline,
2601            previous,
2602            last_booted: gens.last_booted().ok(),
2603            candidates: l.live.values().filter_map(|t| t.txn.candidate.as_ref().map(|(g, _)| *g)).collect(),
2604            last_change_reverted,
2605            now: s.observed_at,
2606        }
2607    }
2608
2609    fn intent_index(&self, defs: &UnitDefs) -> intent::Index {
2610        let changes = planarian_changes::entries(&self.cfg.exec.catalog)
2611            .into_iter()
2612            .map(|e| intent::CatalogEntry { id: e.id, description: e.description })
2613            .collect();
2614        let options = self
2615            .cfg
2616            .runtime_options
2617            .iter()
2618            .map(|o| intent::RuntimeOption {
2619                current: self.b.runtime().get(&o.path).or(o.current.clone()),
2620                ..o.clone()
2621            })
2622            .collect();
2623        intent::Index::new(defs, changes, options)
2624    }
2625
2626    pub fn handle(&mut self, peer: Peer, req: Request) -> Response {
2627        self.acting_agent = peer.agent.clone();
2628        let r = self.handle_as(peer, req);
2629        self.acting_agent = None;
2630        r
2631    }
2632
2633    fn handle_as(&mut self, peer: Peer, req: Request) -> Response {
2634        match req {
2635            Request::Status => Response::Status(Box::new(self.status())),
2636            Request::Incidents => {
2637                Response::Incidents { incidents: self.incidents.incidents.values().cloned().collect() }
2638            }
2639            Request::Log { limit } => {
2640                let entries = self.ex.kernel.journal.segment().unwrap_or_default();
2641                let lines = entries
2642                    .iter()
2643                    .rev()
2644                    .take(limit)
2645                    .rev()
2646                    .map(|e| LogLine { seq: e.seq, at: e.at, text: render::record(&e.record) })
2647                    .collect();
2648                Response::Log { lines }
2649            }
2650            Request::Ask { phrase, choice } => {
2651                if let Some(i) = choice {
2652                    let Some(m) = self.choices.remove(&peer.uid).and_then(|c| c.into_iter().nth(i)) else {
2653                        return Response::Error { message: "no such choice; ask again".into() };
2654                    };
2655                    let src = Source::Intent { phrase: phrase.clone(), confidence: m.confidence };
2656                    return self.user_submit(m.transition, peer, src, format!("asked: \"{phrase}\""));
2657                }
2658                let (_, defs) = self.observe();
2659                let now = self.b.clock().now();
2660                match intent::interpret(&phrase, &self.intent_index(&defs), now) {
2661                    Interpretation::Propose(m) => {
2662                        let src = Source::Intent { phrase: phrase.clone(), confidence: m.confidence };
2663                        self.user_submit(m.transition, peer, src, format!("asked: \"{phrase}\" → {}", m.understood))
2664                    }
2665                    Interpretation::Confirm(ms) => {
2666                        let choices = ms
2667                            .iter()
2668                            .enumerate()
2669                            .map(|(index, m)| Choice { index, understood: m.understood.clone(), confidence: m.confidence.0 })
2670                            .collect();
2671                        self.choices.insert(peer.uid, ms);
2672                        Response::Confirm { choices }
2673                    }
2674                    Interpretation::Unmatched => Response::Unmatched {
2675                        message: "I don't know how to do that yet. Try `planarian status`, or phrases like \"restart wifi\", \"stop backups until I say\", \"make text bigger\".".into(),
2676                    },
2677                }
2678            }
2679            Request::Apply { change, args } => {
2680                self.user_submit(Transition::ApplyChange { change, args }, peer, Source::User, "from the CLI".into())
2681            }
2682            Request::SetOverride { unit, until, desired } => {
2683                // `by` is overwritten from the caller's credentials by the executor (§11).
2684                let s = Override::attested(unit, desired, until, "from the CLI".into(), Actor::user(peer.uid));
2685                self.user_submit(Transition::SetOverride(s), peer, Source::User, "from the CLI".into())
2686            }
2687            Request::HoldRelease { unit, until } => {
2688                // §12d: an attested override on the hold unit, through the
2689                // executor like any override, so the hold unit's sensitivity
2690                // (the held unit's) decides: administrator authority, or
2691                // AskFirst with an Administrator approver (§11).
2692                let hold = crate::hold::hold_unit(&unit);
2693                let (_, defs) = self.observe();
2694                if !defs.contains_key(&hold) {
2695                    return Response::Error {
2696                        message: format!("{unit} is not a network-security unit; there is no hold to release"),
2697                    };
2698                }
2699                let reason = format!("planarian hold release {}", unit.name);
2700                let o = Override::attested(
2701                    hold,
2702                    OverrideDesired::Stopped,
2703                    until.clone(),
2704                    reason.clone(),
2705                    Actor::user(peer.uid),
2706                );
2707                let r = self.user_submit(Transition::SetOverride(o), peer.clone(), Source::User, reason);
2708                if matches!(&r, Response::Done { state: TxnState::Committed, .. }) {
2709                    // The executor stopped the hold unit; the marker keeps a
2710                    // new failure from holding again until the release ends.
2711                    if self.ledger().holds().contains(&unit) {
2712                        self.journal(Record::HoldLifted {
2713                            unit: unit.clone(),
2714                            reason: format!("released by uid {}", peer.uid),
2715                        });
2716                    }
2717                    let _ = self.b.units().release_hold(&unit, crate::hold::release_until(&until), HOLD_WAIT);
2718                    self.hold_released.insert(unit);
2719                }
2720                r
2721            }
2722            Request::LiftOverride { unit } => {
2723                let r = self.user_submit(
2724                    Transition::LiftOverride(unit.clone()),
2725                    peer.clone(),
2726                    Source::User,
2727                    "from the CLI".into(),
2728                );
2729                // Incident control state changes only on an authorized,
2730                // successful human action (§13, R08): a refused or failed
2731                // release leaves the record, its budget and cooldown alone.
2732                let released = matches!(&r, Response::Done { state: TxnState::Committed, .. });
2733                if !released {
2734                    return r;
2735                }
2736                // Releasing an inferred stop also releases what went down with it.
2737                let with = format!("stopped together with {unit}");
2738                let linked: Vec<UnitId> = self
2739                    .ledger()
2740                    .overrides
2741                    .iter()
2742                    .filter(|o| o.attribution == Attribution::Inferred && o.reason == with)
2743                    .map(|o| o.unit.clone())
2744                    .collect();
2745                for u in linked {
2746                    let _ = self.user_submit(Transition::LiftOverride(u), peer.clone(), Source::User, with.clone());
2747                }
2748                let now = self.b.clock().now();
2749                if let Some(i) = self.incidents.human_acted(&unit, now) {
2750                    self.journal(Record::Incident(i));
2751                }
2752                r
2753            }
2754            Request::Stop { unit } => {
2755                // §14: a CLI StopUnit is rewritten so the repair loop can't race it.
2756                let t = Transition::StopAndSuppress {
2757                    unit: unit.clone(),
2758                    until: Until::NextBoot,
2759                    reason: "stopped from the CLI".into(),
2760                };
2761                match self.user_submit(t, peer, Source::User, "planarian stop".into()) {
2762                    Response::Done { txn, state, message } => Response::Done {
2763                        txn,
2764                        state,
2765                        message: format!(
2766                            "{message}. Note: `stop` means stop and leave stopped until the next boot; `planarian release {unit}` lets it run again."
2767                        ),
2768                    },
2769                    other => other,
2770                }
2771            }
2772            Request::Restart { unit } => {
2773                self.user_submit(Transition::RestartUnit(unit), peer, Source::User, "from the CLI".into())
2774            }
2775            Request::Gc => self.user_submit(
2776                Transition::CollectGarbage { emergency: false },
2777                peer,
2778                Source::User,
2779                "from the CLI".into(),
2780            ),
2781            Request::Acknowledge { unit } => {
2782                if peer.agent.is_some() {
2783                    return Response::Rejected {
2784                        reason: "Unauthorized".into(),
2785                        message: "an agent cannot acknowledge".into(),
2786                    };
2787                }
2788                let open: Vec<Override> = self
2789                    .ledger()
2790                    .overrides
2791                    .iter()
2792                    .filter(|o| o.attribution == Attribution::Inferred && !o.acknowledged)
2793                    .filter(|o| unit.as_ref().is_none_or(|u| *u == o.unit))
2794                    .cloned()
2795                    .collect();
2796                for o in &open {
2797                    self.journal(Record::OverrideSet(Override { acknowledged: true, ..o.clone() }));
2798                }
2799                if !open.is_empty() {
2800                    self.notice(format!(
2801                        "uid {} acknowledged: {}",
2802                        peer.uid,
2803                        open.iter().map(|o| format!("{} (inferred override)", o.unit)).collect::<Vec<_>>().join(", ")
2804                    ));
2805                }
2806                Response::Ok {
2807                    message: format!("{} inferred override(s) acknowledged; they hold until they end", open.len()),
2808                }
2809            }
2810            Request::AgentRun { name, argv } => {
2811                if peer.agent.is_some() {
2812                    return Response::Rejected {
2813                        reason: "Unauthorized".into(),
2814                        message: "an agent cannot start agents".into(),
2815                    };
2816                }
2817                if !planarian_schema::backend::valid_agent_name(&name) || argv.is_empty() {
2818                    return Response::Error {
2819                        message: "agent run <name> -- <command>: the name is 1-32 of a-z, 0-9 and -".into(),
2820                    };
2821                }
2822                // A name in use is refused, so a second start never rebinds
2823                // what a running agent may do (I24, B-20).
2824                if self.b.units().agent_running(&name) {
2825                    let owner = self.ledger().agent_owner(&name);
2826                    return Response::Rejected {
2827                        reason: "Unauthorized".into(),
2828                        message: match owner {
2829                            Some(u) => format!("agent {name} is already running, started by uid {u}; stop it first"),
2830                            None => format!("agent {name} is already running; stop it first"),
2831                        },
2832                    };
2833                }
2834                // §12d (R05): a per-uid limit on agents running at once.
2835                let limit = self.cfg.agent_limits.per_uid;
2836                let mine = self
2837                    .ledger()
2838                    .agents
2839                    .iter()
2840                    .filter(|(n, _, by)| *by == peer.uid && self.b.units().agent_running(n))
2841                    .map(|(n, _, _)| n.clone())
2842                    .collect::<BTreeSet<_>>();
2843                if mine.len() as u32 >= limit {
2844                    return Response::Rejected {
2845                        reason: "Unauthorized".into(),
2846                        message: format!(
2847                            "uid {} already runs {} agent(s), the limit (services.planarian.agents.limits.perUid = {limit})",
2848                            peer.uid,
2849                            mine.len()
2850                        ),
2851                    };
2852                }
2853                match self.b.units().start_agent(&name, &argv, &self.cfg.agent_limits) {
2854                    Ok(instance) => {
2855                        // Journaled once it has started, with the instance it
2856                        // started as: a failed start binds nothing (I24).
2857                        self.journal(Record::AgentStarted { name: name.clone(), instance: instance.0, by: peer.uid });
2858                        self.notice(format!(
2859                            "agent {name} started by uid {} ({}); it acts with that user's permissions",
2860                            peer.uid,
2861                            planarian_schema::backend::agent_unit(&name)
2862                        ));
2863                        Response::Ok { message: format!("agent {name} is running with your permissions") }
2864                    }
2865                    Err(other) => Response::Error { message: format!("could not start agent {name}: {other:?}") },
2866                }
2867            }
2868            Request::Undo => {
2869                let current = self.b.generations().current().unwrap_or_default();
2870                let prev =
2871                    self.b.generations().boot_entries().unwrap_or_default().into_iter().filter(|g| *g < current).max();
2872                match prev {
2873                    Some(to) => {
2874                        self.user_submit(Transition::Revert { to }, peer, Source::User, "planarian undo".into())
2875                    }
2876                    None => Response::Error { message: "there is no earlier generation to go back to".into() },
2877                }
2878            }
2879            Request::Approve { hash } => {
2880                if peer.uid != 0 || !peer.tty {
2881                    return Response::Error {
2882                        message: "approve requires root at an interactive terminal (Phase A)".into(),
2883                    };
2884                }
2885                if self.defer && self.deferred.is_none() {
2886                    let t = self.ex.parked.values().find(|p| p.bound_to == hash).map(|p| p.proposal.transition.clone());
2887                    if let Some(job) = t.and_then(|t| self.ex.prebuild_job(&t)) {
2888                        let req = Request::Approve { hash };
2889                        self.deferred = Some((job, Resume { peer, kind: ResumeKind::Request(req) }));
2890                        return Response::Ok { message: "deferred".into() };
2891                    }
2892                }
2893                let (s, defs) = self.observe();
2894                let approver = self.authz(&peer);
2895                let parked = self.ex.parked.values().find(|p| p.bound_to == hash).cloned();
2896                let (pre, policy) = (s.digest(), self.ex.policy_version());
2897                let built = self.waiting_built();
2898                let r = self.ex.approve(&hash, &approver, &s, &defs);
2899                self.note_withdrawn(built);
2900                // Re-bound and asked again under a new id (§12 step 4): the
2901                // proposer's agent goes with it.
2902                if let (Some(pk), Submitted::Parked { txn: again, .. }) = (&parked, &r)
2903                    && *again != pk.txn
2904                    && let Some(agent) = self.parked_agents.remove(&pk.txn)
2905                {
2906                    self.parked_agents.insert(*again, agent);
2907                }
2908                // I7: the approved execution writes its own log record; an
2909                // approval passes §12 step 6 here, not at submission (C18).
2910                if let (Some(pk), Submitted::Finished { txn, .. }) = (parked, &r) {
2911                    let l = self.ledger();
2912                    // The proposer's agent, not whoever is acting now.
2913                    let agent = self.parked_agents.remove(&pk.txn).flatten();
2914                    let rec = if *txn == pk.txn {
2915                        LogRecord {
2916                            at: self.b.clock().now(),
2917                            no_snapshot: no_snapshot(&r, Some(&pk.facts)),
2918                            approver: Some(approver.actor.clone()),
2919                            bound_to: Some(hash),
2920                            proposal: pk.proposal,
2921                            caller: pk.authz.actor,
2922                            pre_digest: pre,
2923                            policy,
2924                            result: r.clone(),
2925                            facts: Some(pk.facts),
2926                            decision: Some(Decision::Allow { tier: Tier::AskFirst, approver: pk.class }),
2927                            post_digest: None,
2928                            path: l.path(*txn).map(<[String]>::to_vec).unwrap_or_default(),
2929                            why: l.terminal.get(txn).map(|x| x.why.clone()).unwrap_or_default(),
2930                            agent,
2931                        }
2932                    } else {
2933                        // Re-bound (§12 step 4), and the fresh decision no
2934                        // longer asks: it ran as an ordinary submission under
2935                        // a new id, approved by nobody. Log it as one, with the
2936                        // new plan's facts and decision.
2937                        let (facts, decision) = self.ex.last.clone().unzip();
2938                        let mut proposal = pk.proposal;
2939                        proposal.observed = pre;
2940                        LogRecord {
2941                            at: self.b.clock().now(),
2942                            no_snapshot: no_snapshot(&r, facts.as_ref()),
2943                            approver: None,
2944                            bound_to: None,
2945                            proposal,
2946                            caller: pk.authz.actor,
2947                            pre_digest: pre,
2948                            policy,
2949                            result: r.clone(),
2950                            facts,
2951                            decision,
2952                            post_digest: None,
2953                            path: l.path(*txn).map(<[String]>::to_vec).unwrap_or_default(),
2954                            why: l.terminal.get(txn).map(|x| x.why.clone()).unwrap_or_default(),
2955                            agent,
2956                        }
2957                    };
2958                    self.awaiting_post.push(rec);
2959                }
2960                self.respond(&r)
2961            }
2962            Request::Cancel { txn } => {
2963                let Some(p) = self.ex.parked.get(&txn) else {
2964                    return Response::Error { message: "no such pending change".into() };
2965                };
2966                if !peer.admin && p.authz.actor.uid() != Some(peer.uid) {
2967                    return Response::Error {
2968                        message: "only whoever asked for it, or an administrator, can withdraw it".into(),
2969                    };
2970                }
2971                let r = self.ex.cancel(txn, Actor::user(peer.uid));
2972                self.respond(&r)
2973            }
2974            Request::Resolve { txn, resolution } => self.user_submit(
2975                Transition::Resolve { txn, resolution },
2976                peer,
2977                Source::User,
2978                "planarian resolve".into(),
2979            ),
2980            Request::MarkSecurityFloor => {
2981                if !peer.admin {
2982                    return Response::Error { message: "mark-security-floor requires administrator authority".into() };
2983                }
2984                let (s, _) = self.observe();
2985                let l = self.ledger();
2986                let g = s.generation;
2987                // §12e: the floor only moves forward, and never to a
2988                // generation that has not at least committed live.
2989                if l.known_good != Some(g) && l.live_tested != Some(g) {
2990                    return Response::Error {
2991                        message: format!("generation {} has not committed live or been boot-confirmed", g.0),
2992                    };
2993                }
2994                if l.security_floor.is_some_and(|f| f.0 >= g.0) {
2995                    return Response::Error {
2996                        message: format!(
2997                            "the security floor is already generation {}",
2998                            l.security_floor.unwrap_or(g).0
2999                        ),
3000                    };
3001                }
3002                self.journal(Record::SecurityFloor(Some(g)));
3003                Response::Ok {
3004                    message: format!(
3005                        "generation {} is now the security floor; going back below it needs an administrator",
3006                        g.0
3007                    ),
3008                }
3009            }
3010            Request::MarkGood { accept_unconfirmed } => {
3011                // §16 (K10): root at an interactive terminal, the terminal
3012                // established by the daemon from the peer, as for approve.
3013                if peer.uid != 0 || !peer.tty {
3014                    return Response::Error {
3015                        message: "mark-good requires root at an interactive terminal (Phase A)".into(),
3016                    };
3017                }
3018                let (s, defs) = self.observe();
3019                let failing: Vec<String> = defs
3020                    .keys()
3021                    .filter(|u| {
3022                        s.expected(u, &defs)
3023                            && matches!(
3024                                s.status(u, &defs),
3025                                UnitStatus::Degraded(_) | UnitStatus::Failed(_) | UnitStatus::Unknown
3026                            )
3027                    })
3028                    .map(|u| u.to_string())
3029                    .collect();
3030                if !failing.is_empty() {
3031                    return Response::Error {
3032                        message: format!("refused: Readiness probes are failing on {}", failing.join(", ")),
3033                    };
3034                }
3035                let booted = self.b.generations().last_booted().ok();
3036                if booted == Some(s.generation) {
3037                    self.journal(Record::KnownGood(s.generation));
3038                    Response::Ok { message: format!("generation {} is now known-good", s.generation.0) }
3039                } else if accept_unconfirmed {
3040                    self.journal(Record::AcceptedBaseline(Some(s.generation)));
3041                    Response::Ok {
3042                        message: format!(
3043                            "generation {} accepted as the recovery baseline; it is not boot-confirmed and does not replace the known-good boot entry",
3044                            s.generation.0
3045                        ),
3046                    }
3047                } else {
3048                    Response::Error {
3049                        message:
3050                            "this generation has not been booted; reboot into it first, or use --accept-unconfirmed"
3051                                .into(),
3052                    }
3053                }
3054            }
3055            Request::Console => Response::Console(Box::new(self.console())),
3056            // §16 (P17): the last boot report, and the posture in force.
3057            Request::ReportBoot => match &self.last_boot_report {
3058                Some((at, g, text)) => Response::Ok {
3059                    message: format!("boot report for generation {} at {}: {text}", g.0, render::when(*at)),
3060                },
3061                None => Response::Ok {
3062                    message: "no boot report since planariad started (it runs once, on the first start after boot)"
3063                        .into(),
3064                },
3065            },
3066            Request::Posture => {
3067                let d = &self.cfg.posture_decl;
3068                let mut message = format!(
3069                    "posture: {:?}, boot-ready timeout {} s (this generation declares {:?}, {} s)",
3070                    self.cfg.posture,
3071                    d.timeout_in_force(),
3072                    d.declared,
3073                    d.declared_timeout_secs
3074                );
3075                if d.weaker_waiting() {
3076                    message.push_str("; the weaker declaration takes effect at the next boot");
3077                }
3078                Response::Ok { message }
3079            }
3080            #[cfg(feature = "gym-hooks")]
3081            Request::GymPropose { transition, stale_digest, may_change } => {
3082                let (s, defs) = self.observe();
3083                let mut p = self.proposal(transition, &s, &defs, Source::User, "gym".into());
3084                if stale_digest {
3085                    p.observed = Hash::ZERO;
3086                }
3087                if let Some(m) = may_change {
3088                    p.expected.may_change = m.into_iter().collect();
3089                }
3090                let authz = self.authz(&peer);
3091                let r = self.submit(p, authz, &s, &defs);
3092                self.respond(&r)
3093            }
3094            #[cfg(feature = "gym-hooks")]
3095            Request::GymCompact => match self.compact_saving() {
3096                Ok(entries) => Response::Compacted {
3097                    entries: entries.iter().filter_map(|e| serde_json::to_value(e).ok()).collect(),
3098                },
3099                Err(e) => Response::Error { message: format!("{e:?}") },
3100            },
3101            #[cfg(not(feature = "gym-hooks"))]
3102            Request::GymPropose { .. } | Request::GymCompact => {
3103                Response::Error { message: "not available in this build (gym-hooks)".into() }
3104            }
3105            Request::Diagnose { unit, probe } => {
3106                let (_, defs) = self.observe();
3107                let Some(def) = defs.get(&unit) else {
3108                    return Response::Error { message: format!("no unit {unit}") };
3109                };
3110                // Only units the caller may act on: their own user scope, the
3111                // units users control, or any for an administrator (§11).
3112                let mine = matches!(unit.scope, Scope::User { uid, .. } if uid == peer.uid);
3113                let controllable = self.ex.cfg.settings.user_controllable_units.contains(&unit.name);
3114                if !(peer.admin || mine || controllable) {
3115                    return Response::Rejected {
3116                        reason: "Unauthorized".into(),
3117                        message: format!("you may not probe {unit}"),
3118                    };
3119                }
3120                let probes: Vec<ProbeId> = match probe {
3121                    Some(p) => vec![p],
3122                    None => def.probes.iter().map(|p| p.id.clone()).collect(),
3123                };
3124                for p in &probes {
3125                    let audible = def
3126                        .probes
3127                        .iter()
3128                        .any(|d| &d.id == p && matches!(d.side_effects, SideEffects::Audible | SideEffects::Visible));
3129                    // Sound or light only from the caller's own seated session.
3130                    let seated = peer.session.as_ref().is_some_and(|id| {
3131                        self.asm
3132                            .sessions
3133                            .iter()
3134                            .any(|x| &x.id == id && x.uid == peer.uid && x.seat.is_some() && !x.remote)
3135                    });
3136                    if audible && !seated {
3137                        return Response::Error {
3138                            message: format!("{} makes sound or light; run it from your own session", p.0),
3139                        };
3140                    }
3141                    self.asm.request(&unit, p);
3142                }
3143                Response::Ok { message: format!("running {} probe(s) on {unit} at the next tick", probes.len()) }
3144            }
3145        }
3146    }
3147}
3148
3149#[cfg(test)]
3150mod tests;
3151
3152fn signal_name(e: Ending) -> String {
3153    match e {
3154        Ending::Signal(1) => "SIGHUP".into(),
3155        Ending::Signal(2) => "SIGINT".into(),
3156        Ending::Signal(9) => "SIGKILL".into(),
3157        Ending::Signal(15) => "SIGTERM, as `systemctl stop` sends".into(),
3158        Ending::Signal(n) => format!("signal {n}"),
3159        Ending::Exited(n) => format!("exit status {n}"),
3160        Ending::OomKill => "the OOM killer".into(),
3161    }
3162}
3163
3164/// §12f: units foreground work plausibly depends on, whose repairs quiet
3165/// mode does not defer: a user's own units (audio), devices (sound, GPU,
3166/// input, network cards), and units whose restart drops a session or the
3167/// network (compositor, display manager, network, Bluetooth).
3168fn foreground_depends_on(u: &UnitId, defs: &UnitDefs) -> bool {
3169    matches!(u.scope, Scope::User { .. })
3170        || u.kind == UnitKind::Device
3171        || u.name.contains("bluetooth")
3172        || defs
3173            .get(u)
3174            .is_some_and(|d| matches!(d.restart_disruption, Disruption::SessionLoss | Disruption::ConnectivityLoss))
3175}
3176
3177/// Notices quiet mode never holds back (§12f): alarms, integrity findings,
3178/// what a repair did, and anything that needs a person.
3179fn urgent(s: &str) -> bool {
3180    s.starts_with("ALARM")
3181        || s.starts_with("INTEGRITY")
3182        || s.contains(" → ")
3183        || s.contains("needs a person")
3184        || s.contains("needs approval")
3185}