1use crate::render;
16use planarian_diagnose as diagnose;
17use planarian_exec::{Config as ExecConfig, Executor, PrebuildJob, Prebuilt, Submitted, derive::manifest_of};
18use planarian_incident::Manager;
19use planarian_intent::{self as intent, Interpretation};
20use planarian_ipc::{
21 Choice, ConsoleReport, GraphEdge, GraphNode, LogLine, Pending, Request, Response, StatusReport, TxnSummary, UnitRow,
22};
23use planarian_policy::{authorization_of, user_authorization};
24use planarian_schema::spec::predict;
25use planarian_schema::*;
26use planarian_state::{Assembler, Context};
27use planarian_txn::journal::{Journal, JournalError};
28use planarian_txn::ledger::{Ledger, Record};
29use planarian_txn::store::Store;
30use serde::{Deserialize, Serialize};
31use std::collections::{BTreeMap, BTreeSet, VecDeque};
32
33pub const CONFIRM_AFTER: Millis = Millis::secs(600);
35const NOTICES_KEPT: usize = 64;
36const UNREACHABLE_AFTER: Millis = Millis::secs(120);
38const RESTART_REMINDER: Millis = Millis::secs(24 * 3600);
41
42#[derive(Clone, Debug)]
43pub struct CoreConfig {
44 pub exec: ExecConfig,
45 pub runtime_options: Vec<intent::RuntimeOption>,
47 pub fresh_boot: bool,
49 pub startup_grace: Millis,
51 pub posture: Posture,
54 pub posture_decl: crate::config::PostureDecl,
56 pub agent_limits: planarian_schema::backend::AgentLimits,
58 pub defer_recovery: bool,
62}
63
64pub const NEW_DECLARATIONS_FOR: Millis = Millis::secs(24 * 3600);
66
67pub const CHAIN_CHECK_EVERY: Millis = Millis::secs(3600);
69pub const HOLD_WAIT: Millis = Millis::secs(5);
73pub const HOLD_UNCONFIRMED_RETRY: Millis = Millis::secs(60);
77pub const HOLD_RETRY_MIN: Millis = Millis::secs(10);
80pub const HOLD_RETRY_MAX: Millis = Millis::secs(300);
81
82#[derive(Clone, Debug, PartialEq, Eq)]
84pub struct Peer {
85 pub uid: u32,
86 pub admin: bool,
88 pub session: Option<String>,
90 pub agent: Option<String>,
94 pub instance: Option<planarian_schema::InstanceId>,
97 pub tty: bool,
100}
101
102impl Peer {
103 pub const fn new(uid: u32, admin: bool) -> Peer {
104 Peer { uid, admin, session: None, agent: None, instance: None, tty: false }
105 }
106
107 pub fn agent(uid: u32, name: &str, instance: Option<planarian_schema::InstanceId>) -> Peer {
109 Peer { uid, admin: false, session: None, agent: Some(name.into()), instance, tty: false }
110 }
111
112 pub const fn at_terminal(mut self) -> Peer {
113 self.tty = true;
114 self
115 }
116
117 pub fn in_session(mut self, session: &str) -> Peer {
118 self.session = Some(session.into());
119 self
120 }
121}
122
123pub fn agent_principal(name: &str) -> PrincipalId {
126 let h = name.bytes().fold(0xcbf2_9ce4_8422_2325u64, |h, b| (h ^ b as u64).wrapping_mul(0x0100_0000_01b3));
127 PrincipalId((1 << 40) | (h & 0xffff_ffff))
128}
129
130#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
132pub struct LogRecord {
133 pub at: Timestamp,
134 pub proposal: Proposal,
135 pub caller: Actor,
136 pub pre_digest: Hash,
137 pub policy: PolicyVersion,
138 pub result: Submitted,
139 #[serde(default)]
141 pub facts: Option<DerivedFacts>,
142 #[serde(default)]
143 pub decision: Option<Decision>,
144 #[serde(default)]
147 pub post_digest: Option<Hash>,
148 #[serde(default)]
149 pub path: Vec<String>,
150 #[serde(default)]
151 pub why: String,
152 #[serde(default)]
154 pub agent: Option<String>,
155 #[serde(default)]
158 pub no_snapshot: bool,
159 #[serde(default)]
163 pub approver: Option<Actor>,
164 #[serde(default)]
165 pub bound_to: Option<Hash>,
166}
167
168fn no_snapshot(result: &Submitted, facts: Option<&DerivedFacts>) -> bool {
173 matches!(result, Submitted::Finished { .. }) && facts.is_some_and(|f| f.sensitivity.contains(Sensitivity::UserData))
174}
175
176pub struct Core<B: Backends + Clone, S: Store> {
177 pub admin_of: std::sync::Arc<dyn Fn(u32) -> bool + Send + Sync>,
180 acting_agent: Option<String>,
182 after_ready: Vec<TxnId>,
184 pub ex: Executor<B, S>,
185 pub b: B,
186 pub asm: Assembler,
187 pub incidents: Manager,
188 pub notices: VecDeque<String>,
189 pub notices_total: u64,
192 pub log: Vec<LogRecord>,
194 cfg: CoreConfig,
195 last: Option<(SystemState, UnitDefs)>,
196 choices: BTreeMap<u32, Vec<intent::Match>>,
197 incident_parked: BTreeMap<IncidentId, TxnId>,
198 pub slow_tick_warnings: u64,
200 last_slow_warning: Option<Timestamp>,
201 slow_ticks_unwarned: u64,
202 writebacks: Vec<(Proposal, Authorization)>,
205 confirm: Option<(GenerationId, Timestamp)>,
206 manifest_gen: GenerationId,
207 gen_manifest: planarian_schema::Manifest,
209 pub external: Option<crate::hooks::DesiredDoc>,
211 pub new_declarations: Vec<(String, String)>,
215 new_declarations_since: Timestamp,
216 interlock_noted: bool,
217 restart_reminded: BTreeMap<TxnId, Timestamp>,
220 residue_taught: bool,
222 repaired_before_confirmation: BTreeSet<UnitId>,
225 diag_noted: BTreeSet<IncidentId>,
227 unreachable: BTreeMap<UnitId, (Timestamp, bool)>,
230 chain_checked: Option<Timestamp>,
232 pub integrity: Option<String>,
235 prev_status: BTreeMap<UnitId, UnitStatus>,
237 prev_auto_restarts: BTreeMap<UnitId, u32>,
240 boot_report_due: bool,
242 last_boot_report: Option<(Timestamp, GenerationId, String)>,
244 quiet: bool,
246 quiet_queue: Vec<String>,
247 reprepare: Vec<(TxnId, Transition)>,
249 hold_table_seen: bool,
253 hold_missing_noted: bool,
257 hold_retry: Option<(Timestamp, Millis)>,
260 daemon_held: BTreeSet<UnitId>,
263 hold_unconfirmed: BTreeMap<UnitId, Timestamp>,
268 lift_unconfirmed: BTreeMap<UnitId, Timestamp>,
274 hold_left: Millis,
276 withdrawn: Vec<(TxnId, Transition, Option<Timestamp>)>,
280 pub earlier: BTreeMap<TxnId, (Proposal, Actor)>,
283 touched: BTreeSet<UnitId>,
286 generation_touched: bool,
287 next_gc_check: Timestamp,
289 space_noted: Timestamp,
291 awaiting_post: Vec<LogRecord>,
293 pub defer: bool,
297 pub building: Option<(String, Timestamp)>,
299 deferred: Option<(PrebuildJob<B::Gens>, Resume)>,
300 core_hash: Hash,
306 core_hash_record: Option<std::path::PathBuf>,
307 parked_agents: BTreeMap<TxnId, Option<String>>,
310 hold_released: BTreeSet<UnitId>,
314 hold_notified: BTreeSet<UnitId>,
315}
316
317#[derive(Clone, Debug)]
319pub struct Resume {
320 peer: Peer,
321 kind: ResumeKind,
322}
323
324#[derive(Clone, Debug)]
325enum ResumeKind {
326 Submit { t: Transition, source: Source, reason: String },
327 Request(Request),
328}
329
330impl Resume {
331 pub fn describe(&self) -> String {
333 match &self.kind {
334 ResumeKind::Submit { t, .. } => render::transition(t),
335 ResumeKind::Request(r) => crate::snapshot::describe(r),
336 }
337 }
338}
339
340pub const SLOW_TICK_WARN_EVERY: Millis = Millis::secs(60);
342
343pub const GC_CHECK: Millis = Millis(3600 * 1000);
345
346fn outcome_of(state: &TxnState) -> Outcome {
347 match state {
348 TxnState::Committed => Outcome::Succeeded,
349 TxnState::Reverted => Outcome::Failed("reverted".into()),
350 s => Outcome::Unknown(format!("{s:?}")),
351 }
352}
353
354impl<B: Backends + Clone, S: Store> Core<B, S> {
355 pub fn start(journal: Journal<S>, b: B, me: OwnerId, cfg: CoreConfig) -> Core<B, S> {
358 let fresh = cfg.fresh_boot;
359 let mut ex = Executor::new(journal, b.clone(), me, cfg.exec.clone());
360 ex.adopt_current();
361 let mut notices = VecDeque::new();
362 let terminal_before: BTreeSet<TxnId> = ex.kernel.journal.ledger().terminal.keys().copied().collect();
363 let mut deferred = vec![];
364 match ex.kernel.reconcile_deferring(cfg.defer_recovery) {
365 Ok((waiting, d)) => {
366 if !waiting.is_empty() {
367 notices.push_back(format!("waiting for the recovery watchdog to finish transaction(s) {waiting:?}"))
368 }
369 deferred = d;
370 }
371 Err(e) => notices.push_back(format!("journal reconciliation failed: {e:?}")),
372 }
373 let ledger = ex.kernel.journal.ledger().clone();
374 let reprepare: Vec<(TxnId, Transition)> = ledger
379 .terminal
380 .iter()
381 .filter(|(id, t)| {
382 !terminal_before.contains(id)
383 && t.state == TxnState::Rejected(RejectReason::Interrupted)
384 && t.path.iter().rev().nth(1).is_some_and(|p| p == "Built" || p == "Prepared")
385 })
386 .map(|(id, t)| (*id, t.transition.clone()))
387 .collect();
388 let named: BTreeSet<TxnId> = ledger
393 .live
394 .values()
395 .filter_map(|t| t.txn.prepared_again_from)
396 .chain(ledger.terminal.values().filter_map(|t| t.prepared_again_from))
397 .collect();
398 let withdrawn: Vec<(TxnId, Transition, Option<Timestamp>)> = ledger
399 .terminal
400 .iter()
401 .filter(|(id, t)| {
402 t.withdrawn_for.is_some()
403 && t.state == TxnState::Rejected(RejectReason::Interrupted)
404 && !named.contains(id)
405 && !reprepare.iter().any(|(r, _)| r == *id)
406 })
407 .map(|(id, t)| (*id, t.transition.clone(), None))
408 .collect();
409 let mut incidents = Manager::restore(ledger.incidents.values().cloned());
410 incidents.startup_grace = cfg.startup_grace;
411 let current = b.generations().current().unwrap_or_default();
412 let manifest = match b.generations().path_of(current).and_then(|p| b.generations().evaluate_built(&p)) {
413 Ok(e) => manifest_of(&e),
414 Err(o) => {
415 notices.push_back(format!(
416 "the running generation has no Planarian manifest ({o:?}); nothing is watched until a generation built with services.planarian.enable is running"
417 ));
418 Default::default()
419 }
420 };
421 let asm = Assembler::new(manifest.clone(), cfg.exec.canonical.clone());
422 let now = b.clock().now();
423 let last_booted = b.generations().last_booted().ok();
424 let admitted = ledger.live.values().any(|t| {
427 t.txn.state == TxnState::Promoted
428 && t.path.iter().any(|p| p == "AwaitingBoot")
429 && t.txn.candidate.as_ref().map(|(g, _)| *g) == Some(current)
430 });
431 let confirm = match ledger.known_good {
432 _ if admitted => Some((current, now.plus(CONFIRM_AFTER))),
433 None => Some((current, now.plus(CONFIRM_AFTER))),
434 Some(_)
435 if ledger.live_tested.is_some()
436 && ledger.live_tested == last_booted
437 && last_booted == Some(current) =>
438 {
439 Some((current, now.plus(CONFIRM_AFTER)))
440 }
441 _ => None,
442 };
443 let mut core = Core {
444 core_hash: Hash::ZERO,
445 core_hash_record: None,
446 parked_agents: BTreeMap::new(),
447 hold_released: BTreeSet::new(),
448 hold_notified: BTreeSet::new(),
449 after_ready: deferred,
450 ex,
451 b,
452 asm,
453 incidents,
454 notices_total: notices.len() as u64,
455 notices,
456 log: vec![],
457 cfg,
458 last: None,
459 choices: BTreeMap::new(),
460 incident_parked: BTreeMap::new(),
461 slow_tick_warnings: 0,
462 last_slow_warning: None,
463 slow_ticks_unwarned: 0,
464 writebacks: vec![],
465 confirm,
466 manifest_gen: current,
467 gen_manifest: manifest,
468 external: None,
469 new_declarations: vec![],
470 new_declarations_since: now,
471 interlock_noted: false,
472 restart_reminded: BTreeMap::new(),
473 residue_taught: false,
474 repaired_before_confirmation: BTreeSet::new(),
475 diag_noted: BTreeSet::new(),
476 unreachable: BTreeMap::new(),
477 chain_checked: None,
478 integrity: None,
479 admin_of: std::sync::Arc::new(|uid| uid == 0),
480 acting_agent: None,
481 prev_status: BTreeMap::new(),
482 prev_auto_restarts: BTreeMap::new(),
483 boot_report_due: fresh,
484 last_boot_report: None,
485 quiet: false,
486 quiet_queue: vec![],
487 reprepare: vec![],
488 hold_table_seen: true,
489 hold_missing_noted: false,
490 hold_retry: None,
491 daemon_held: BTreeSet::new(),
492 hold_unconfirmed: BTreeMap::new(),
493 lift_unconfirmed: BTreeMap::new(),
494 hold_left: HOLD_WAIT,
495 withdrawn: vec![],
496 earlier: BTreeMap::new(),
497 touched: BTreeSet::new(),
498 generation_touched: false,
499 next_gc_check: Timestamp(0),
500 space_noted: Timestamp(0),
501 awaiting_post: Vec::new(),
502 defer: false,
503 building: None,
504 deferred: None,
505 };
506 if core.cfg.fresh_boot {
507 for s in ledger.overrides.iter().filter(|s| s.until == Until::NextBoot) {
508 core.journal(Record::OverrideLifted(s.unit.clone()));
509 }
510 }
511 if let Some(r) = &ledger.last_reconciliation {
512 for a in &r.actions {
513 core.notice(format!("reconciliation: {a}"));
514 }
515 }
516 core.reprepare = reprepare;
517 core.withdrawn = withdrawn;
518 core.note_declarations(None, current, false);
519 core
520 }
521
522 pub fn remember(&mut self, records: impl IntoIterator<Item = LogRecord>) {
525 for r in records {
526 if let Submitted::Finished { txn, .. } | Submitted::Parked { txn, .. } = &r.result {
527 self.earlier.insert(*txn, (r.proposal, r.caller));
528 }
529 }
530 }
531
532 pub fn last_observation(&self) -> Option<&(SystemState, UnitDefs)> {
534 self.last.as_ref()
535 }
536
537 pub fn notice(&mut self, s: String) {
538 if self.quiet && !urgent(&s) {
540 self.quiet_queue.push(s);
541 return;
542 }
543 self.say(s);
544 }
545
546 fn say(&mut self, s: String) {
547 if self.notices.len() >= NOTICES_KEPT {
548 self.notices.pop_front();
549 }
550 self.notices.push_back(s);
551 self.notices_total += 1;
552 }
553
554 pub fn notices_since(&self, printed: u64) -> impl Iterator<Item = &String> {
557 let new = self.notices_total.saturating_sub(printed).min(self.notices.len() as u64) as usize;
558 self.notices.iter().skip(self.notices.len() - new)
559 }
560
561 fn journal(&mut self, r: Record) {
562 let now = self.b.clock().now();
563 if let Err(e) = self.ex.kernel.journal.append(now, r) {
564 self.notice(format!("journal write failed: {e:?}"));
565 }
566 }
567
568 pub fn ledger(&mut self) -> Ledger {
569 self.ex.kernel.journal.sync_ledger().cloned().unwrap_or_default()
570 }
571
572 pub fn set_core_hash_record(&mut self, record: std::path::PathBuf) {
575 self.core_hash_record = Some(record);
576 self.read_core_hash();
577 }
578
579 pub fn core_hash(&self) -> Hash {
581 self.core_hash
582 }
583
584 fn read_core_hash(&mut self) {
587 let Some(record) = self.core_hash_record.clone() else { return };
588 match crate::config::core_hash(&record) {
589 Ok(h) => self.core_hash = h,
590 Err(e) => {
591 self.notice(format!("the core hash record could not be read ({e}); the state keeps the last one read"))
592 }
593 }
594 }
595
596 fn context(&self, l: &Ledger, current: GenerationId) -> Context {
597 let now = self.b.clock().now();
598 Context {
599 known_good: l.known_good.unwrap_or(current),
600 live_tested: l.live_tested,
601 accepted_baseline: l.accepted_baseline,
602 overrides: l.overrides.iter().filter(|s| s.holds_at(now)).cloned().collect(),
603 security_floor: l.security_floor,
604 core_hash: self.core_hash,
605 }
606 }
607
608 fn protect(&self, l: &Ledger, current: GenerationId) -> Vec<GenerationId> {
611 let gens = self.b.generations();
612 let mut p = vec![current];
613 p.extend(l.known_good);
614 p.extend(l.live_tested);
615 p.extend(l.accepted_baseline);
616 p.extend(gens.boot_default().ok());
617 if let Ok(entries) = gens.boot_entries() {
618 p.extend(entries.into_iter().filter(|g| *g < current).max());
619 }
620 p.extend(l.live.values().filter_map(|t| t.txn.candidate.as_ref().map(|(g, _)| *g)));
621 p.sort();
622 p.dedup();
623 p
624 }
625
626 pub fn pending_recovery(&self) -> Option<String> {
628 match self.after_ready.as_slice() {
629 [] => None,
630 [one] => Some(format!("recovering transaction {} after the restart", one.0)),
631 many => Some(format!("recovering {} transactions after the restart", many.len())),
632 }
633 }
634
635 pub fn finish_recovery(&mut self) {
637 let ids = std::mem::take(&mut self.after_ready);
638 if let Err(e) = self.ex.kernel.finish_deferred(&ids) {
639 self.notice(format!("journal reconciliation after ready failed: {e:?}"));
640 }
641 }
642
643 pub fn observe(&mut self) -> (SystemState, UnitDefs) {
645 let l = self.ledger();
646 let current = self.b.generations().current().unwrap_or(self.manifest_gen);
647 if current != self.manifest_gen {
648 if let Ok(e) = self.b.generations().path_of(current).and_then(|p| self.b.generations().evaluate_built(&p)) {
649 self.gen_manifest = manifest_of(&e);
650 self.asm.set_manifest(self.effective_manifest().0);
651 }
652 let prev = self.manifest_gen;
653 self.manifest_gen = current;
654 self.generation_touched = true;
657 self.note_declarations(Some(prev), current, true);
658 self.read_core_hash();
660 }
661 let ctx = self.context(&l, current);
662 let protect = self.protect(&l, current);
663 self.ex.update_context(ctx.clone(), protect);
664 self.ex.cfg.settings.max_tier = if l.known_good.is_none() {
666 self.cfg.exec.settings.max_tier.min(Tier::Runtime)
667 } else {
668 self.cfg.exec.settings.max_tier
669 };
670 let mut s = self.asm.observe(&self.b, &ctx);
671 s.holds = l.holds();
674 if let Ok(h) = self.b.units().held() {
675 s.holds.extend(h);
676 }
677 let mut defs = self.asm.defs();
678 let holds: Vec<UnitDef> = crate::hold::network_security(&defs).map(crate::hold::hold_def).collect();
681 for d in holds {
682 defs.entry(d.id.clone()).or_insert(d);
683 }
684 self.last = Some((s.clone(), defs.clone()));
685 if !self.awaiting_post.is_empty() {
686 let digest = s.digest();
687 for mut r in self.awaiting_post.drain(..) {
688 r.post_digest = Some(digest);
689 self.log.push(r);
690 }
691 }
692 (s, defs)
693 }
694
695 fn proposal(&self, t: Transition, s: &SystemState, defs: &UnitDefs, source: Source, reason: String) -> Proposal {
696 Proposal { observed: s.digest(), expected: predict(&t, s, defs), transition: t, reason, source }
697 }
698
699 fn submit(&mut self, p: Proposal, authz: Authorization, s: &SystemState, defs: &UnitDefs) -> Submitted {
700 self.submit_from(p, authz, s, defs, None)
701 }
702
703 fn submit_from(
706 &mut self,
707 p: Proposal,
708 authz: Authorization,
709 s: &SystemState,
710 defs: &UnitDefs,
711 again: Option<TxnId>,
712 ) -> Submitted {
713 let now = self.b.clock().now();
714 let pre = s.digest();
715 let policy = self.ex.policy_version();
716 let caller = authz.actor.clone();
717 let built = self.waiting_built();
718 let result = match again {
719 Some(from) => self.ex.submit_again(p.clone(), from, authz.clone(), s, defs),
720 None => self.ex.submit(p.clone(), authz.clone(), s, defs),
721 };
722 self.note_withdrawn(built);
723 if let Some((facts, _)) = self.ex.last.clone() {
724 self.touched.extend(facts.affected_units.iter().cloned());
725 if facts.other_sessions_affected {
727 let others: Vec<String> = self
728 .asm
729 .sessions
730 .iter()
731 .filter(|x| {
732 x.establishes() && Some(x.uid) != caller.uid() && facts.affected_sessions.contains(&x.id)
733 })
734 .map(|x| format!("{} (uid {})", x.id, x.uid))
735 .collect();
736 self.notice(format!(
737 "{}: {} would also affect other people's sessions: {}",
738 render::transition(&p.transition),
739 match caller.uid() {
740 Some(u) => format!("asked for by uid {u}"),
741 None => "proposed by Planarian".to_string(),
742 },
743 others.join(", ")
744 ));
745 }
746 }
747 self.generation_touched |= p.transition.changes_generation();
748 if let Submitted::Finished { txn, state, .. } = &result {
749 match state {
750 TxnState::Built => self.notice(format!(
751 "transaction {}: {} is ready and waits for a quiet moment (it would interrupt what you are doing)",
752 txn.0,
753 render::transition(&p.transition)
754 )),
755 TxnState::AwaitingBoot => self.notice(format!(
756 "transaction {}: {} takes effect when you restart; restart to finish",
757 txn.0,
758 render::transition(&p.transition)
759 )),
760 _ => {}
761 }
762 }
763 if let Some((g, fixes)) = self.ex.floor_raised.take() {
764 self.notice(format!(
765 "security floor is now generation {} (it fixes {}); going back below it needs an administrator",
766 g.0,
767 fixes.join(", ")
768 ));
769 }
770 if let (Transition::RestartUnit(u), Submitted::Finished { .. }) = (&p.transition, &result) {
771 self.incidents.started(u, now);
772 }
773 let (facts, decision) = self.ex.last.clone().unzip();
774 let excluded = std::mem::take(&mut self.ex.excluded);
775 if !excluded.is_empty() {
776 self.notice(format!(
777 "left out of the build because git does not track them (Nix cannot see them): {}",
778 excluded.join(", ")
779 ));
780 }
781 let txn = match &result {
782 Submitted::Finished { txn, .. } | Submitted::Parked { txn, .. } => Some(*txn),
783 _ => None,
784 };
785 let l = self.ledger();
786 let (path, why) = match txn {
787 Some(t) => (
788 l.path(t).map(<[String]>::to_vec).unwrap_or_default(),
789 l.live
790 .get(&t)
791 .map(|x| x.why.clone())
792 .or_else(|| l.terminal.get(&t).map(|x| x.why.clone()))
793 .unwrap_or_default(),
794 ),
795 None => (vec![], String::new()),
796 };
797 if let Submitted::Parked { txn, .. } = &result {
798 self.parked_agents.insert(*txn, self.acting_agent.clone());
799 }
800 let record = LogRecord {
801 at: now,
802 no_snapshot: no_snapshot(&result, facts.as_ref()),
803 approver: None,
804 bound_to: match &result {
805 Submitted::Parked { approve, .. } => Some(*approve),
806 _ => None,
807 },
808 proposal: p,
809 caller,
810 pre_digest: pre,
811 policy,
812 result: result.clone(),
813 facts,
814 decision,
815 post_digest: None,
816 path,
817 why,
818 agent: self.acting_agent.clone(),
819 };
820 if matches!(result, Submitted::Finished { .. }) {
823 self.awaiting_post.push(record);
824 } else {
825 self.log.push(record);
826 }
827 if let Submitted::Finished { followup: Some(wb), .. } = &result {
829 self.writebacks.push((wb.clone(), authz.clone()));
830 self.flush_writebacks();
831 }
832 result
833 }
834
835 fn hold_wait(&self) -> Option<Millis> {
853 (self.hold_left.0 > 0).then_some(self.hold_left)
854 }
855
856 fn hold_spent(&mut self, t0: Timestamp) {
858 let took = self.b.clock().boot_now().0.saturating_sub(t0.0);
859 self.hold_left = Millis(self.hold_left.0.saturating_sub(took));
860 }
861
862 pub fn compact_saving(
869 &mut self,
870 ) -> Result<Vec<planarian_txn::ledger::Entry>, planarian_txn::journal::JournalError> {
871 let segment = self.ex.kernel.journal.segment()?;
872 self.ex.kernel.journal.compact()?;
873 Ok(segment)
874 }
875
876 fn network_holds(&mut self, s: &SystemState, defs: &UnitDefs) -> bool {
877 use crate::hold;
878 use planarian_txn::ledger::HoldBy;
879 let units: Vec<UnitId> = hold::network_security(defs).map(|d| d.id.clone()).collect();
880 if units.is_empty() {
881 return false;
882 }
883 let mut applied = false;
884 let journaled = self.ledger().holds();
885 let host = self.b.units().held();
886 let mut holding = journaled.clone();
889 if let Ok(h) = &host {
890 holding.extend(h.iter().cloned());
891 }
892 let mut lifted = BTreeSet::new();
898 for u in &units {
899 if journaled.contains(u)
900 && !self.hold_released.contains(u)
901 && s.override_of(&hold::hold_unit(u)).is_none()
902 && !hold::failed(s, defs, u)
903 && hold::back(s, defs, u)
904 && self.lift_back(u)
905 {
906 lifted.insert(u.clone());
907 }
908 }
909 holding.retain(|u| {
913 let pending = self.lift_unconfirmed.contains_key(u) && !hold::failed(s, defs, u) && hold::back(s, defs, u);
914 !(lifted.contains(u) || pending)
915 });
916 if holding.is_empty() {
917 self.hold_table_seen = true;
918 self.hold_missing_noted = false;
919 self.hold_retry = None;
920 } else {
921 self.hold_table(&holding, &units);
922 }
923 for u in units {
924 let asked = self.daemon_held.remove(&u);
926 let in_journal = journaled.contains(&u);
927 let on_host = host.as_ref().ok().map(|h| h.contains(&u));
930 if on_host == Some(true) {
931 self.hold_unconfirmed.remove(&u);
933 }
934 if on_host == Some(false) && !hold::failed(s, defs, &u) {
935 self.hold_unconfirmed.remove(&u);
937 }
938 if lifted.contains(&u) {
939 continue;
940 }
941 if self.lift_unconfirmed.contains_key(&u) && !in_journal {
944 match on_host {
945 None => {}
948 Some(false) if !hold::failed(s, defs, &u) => {
950 self.lift_unconfirmed.remove(&u);
951 self.hold_notified.remove(&u);
952 self.notice(format!("{u} is back; network traffic flows again"));
953 continue;
954 }
955 Some(true) if !hold::failed(s, defs, &u) && hold::back(s, defs, &u) => {
958 let now = self.b.clock().boot_now();
959 if self.lift_unconfirmed.get(&u).is_some_and(|next| now >= *next) && self.hold_wait().is_some()
960 {
961 self.lift_host(&u);
962 }
963 continue;
964 }
965 Some(_) => {
967 self.lift_unconfirmed.remove(&u);
968 }
969 }
970 }
971 if s.override_of(&hold::hold_unit(&u)).is_some() {
974 if in_journal {
975 self.journal(Record::HoldLifted {
976 unit: u.clone(),
977 reason: "released by an attested override".into(),
978 });
979 }
980 self.hold_released.insert(u);
981 continue;
982 }
983 if self.hold_released.remove(&u) {
984 let _ = self.b.units().end_release(&u);
987 }
988 if hold::failed(s, defs, &u) {
989 if !in_journal {
990 let by = if asked || on_host != Some(true) { HoldBy::Daemon } else { HoldBy::OnFailure };
997 self.journal(Record::HoldApplied { unit: u.clone(), by });
998 self.notice(hold::alarm(&u));
999 }
1000 if on_host != Some(true) {
1001 let now = self.b.clock().boot_now();
1002 let waiting =
1006 self.hold_unconfirmed.get(&u).is_some_and(|next| now < *next) || self.hold_wait().is_none();
1007 let o = match self.hold_wait() {
1008 _ if waiting => Outcome::Unknown("waiting for its hold job".into()),
1009 Some(w) => {
1010 let t0 = self.b.clock().boot_now();
1011 let o = self.b.units().hold(&u, w);
1012 self.hold_spent(t0);
1013 o
1014 }
1015 None => Outcome::Unknown("no time left on this tick".into()),
1016 };
1017 match o {
1018 o @ Outcome::Failed(_) => {
1021 self.notice(format!("ALARM: {u}: its traffic could not be held ({o:?})"))
1022 }
1023 _ if waiting => {}
1024 o @ Outcome::Unknown(_) => {
1025 let later = self.b.clock().boot_now().plus(HOLD_UNCONFIRMED_RETRY);
1026 if self.hold_unconfirmed.insert(u.clone(), later).is_none() {
1027 self.notice(format!("ALARM: {u}: its traffic could not be held ({o:?})"))
1028 }
1029 }
1030 _ => applied |= !s.holds.contains(&u),
1031 }
1032 }
1033 let given_up = self.incidents.active_for(&u).is_some_and(|i| {
1035 matches!(i.state, IncidentState::Cooldown { .. } | IncidentState::Escalated { .. })
1036 });
1037 if given_up && self.hold_notified.insert(u.clone()) {
1038 self.notice(format!(
1039 "ALARM: {u}: repair failed; its network traffic stays blocked. It needs a person: an administrator can fix it, or lift the hold with `planarian hold release {}`",
1040 u.name
1041 ));
1042 }
1043 } else if in_journal || on_host == Some(true) {
1044 if !in_journal {
1045 self.journal(Record::HoldApplied { unit: u.clone(), by: HoldBy::OnFailure });
1047 }
1048 if hold::back(s, defs, &u) {
1049 self.lift_back(&u);
1050 }
1051 }
1052 }
1053 applied
1054 }
1055
1056 fn lift_back(&mut self, u: &UnitId) -> bool {
1060 if self.hold_wait().is_none() {
1061 return false;
1062 }
1063 self.journal(Record::HoldLifted { unit: u.clone(), reason: "Running, and its Readiness probes pass".into() });
1064 self.lift_host(u);
1065 true
1066 }
1067
1068 fn lift_host(&mut self, u: &UnitId) {
1071 let o = match self.hold_wait() {
1072 Some(w) => {
1073 let t0 = self.b.clock().boot_now();
1074 let o = self.b.units().lift_hold(u, w);
1075 self.hold_spent(t0);
1076 o
1077 }
1078 None => Outcome::Unknown("no time left on this tick".into()),
1079 };
1080 match o {
1081 o @ (Outcome::Failed(_) | Outcome::Unknown(_)) => {
1082 let later = self.b.clock().boot_now().plus(HOLD_UNCONFIRMED_RETRY);
1083 if self.lift_unconfirmed.insert(u.clone(), later).is_none() {
1084 self.notice(format!("{u}: the hold could not be lifted ({o:?}); its traffic stays blocked"))
1085 }
1086 }
1087 _ => {
1088 self.lift_unconfirmed.remove(u);
1089 self.hold_notified.remove(u);
1090 self.notice(format!("{u} is back; network traffic flows again"));
1091 }
1092 }
1093 }
1094
1095 fn hold_table(&mut self, holding: &BTreeSet<UnitId>, units: &[UnitId]) {
1100 use planarian_txn::ledger::HoldBy;
1101 if self.hold_wait().is_none() {
1102 return;
1104 }
1105 let present = match self.hold_wait() {
1106 Some(w) => {
1107 let t0 = self.b.clock().boot_now();
1108 let o = self.b.units().hold_table_present(w);
1109 self.hold_spent(t0);
1110 o
1111 }
1112 None => Err(Outcome::Unknown("no time left on this tick".into())),
1113 };
1114 let held: Vec<UnitId> = holding.iter().filter(|u| units.contains(u)).cloned().collect();
1115 match &present {
1116 Ok(true) => {
1117 self.hold_table_seen = true;
1118 self.hold_missing_noted = false;
1119 self.hold_retry = None;
1120 return;
1121 }
1122 Err(o) => {
1123 if self.hold_table_seen {
1124 for u in &held {
1125 self.notice(format!(
1126 "ALARM: {u}: the network hold's firewall table could not be checked ({o:?}); checking again"
1127 ));
1128 }
1129 }
1130 self.hold_table_seen = false;
1131 return;
1132 }
1133 Ok(false) => {}
1134 }
1135 let now = self.b.clock().boot_now();
1136 let first = !self.hold_missing_noted;
1137 if !first && self.hold_retry.is_some_and(|(next, _)| now < next) {
1138 return;
1139 }
1140 self.hold_missing_noted = true;
1141 for u in &held {
1142 if first {
1143 self.journal(Record::HoldApplied { unit: u.clone(), by: HoldBy::Daemon });
1144 self.notice(format!("ALARM: {u}: the network hold was removed from the firewall; re-applying it"));
1145 }
1146 if self.hold_wait().is_none() {
1147 continue;
1148 }
1149 let o = match self.hold_wait() {
1150 Some(w) => {
1151 let t0 = self.b.clock().boot_now();
1152 let o = self.b.units().reapply_hold(u, w);
1153 self.hold_spent(t0);
1154 o
1155 }
1156 None => Outcome::Unknown("no time left on this tick".into()),
1157 };
1158 if let o @ (Outcome::Failed(_) | Outcome::Unknown(_)) = o
1159 && first
1160 {
1161 self.notice(format!("ALARM: {u}: its traffic could not be held again ({o:?}); retrying"));
1162 }
1163 }
1164 self.hold_table_seen = false;
1165 let found = match self.hold_wait() {
1168 Some(w) => {
1169 let t0 = self.b.clock().boot_now();
1170 let r = matches!(self.b.units().hold_table_present(w), Ok(true));
1171 self.hold_spent(t0);
1172 Some(r)
1173 }
1174 None => None,
1175 };
1176 if found == Some(true) {
1177 self.hold_table_seen = true;
1178 self.hold_missing_noted = false;
1179 self.hold_retry = None;
1180 } else if found == Some(false) {
1181 let wait =
1182 self.hold_retry.map_or(HOLD_RETRY_MIN, |(_, w)| Millis(w.0.saturating_mul(2).min(HOLD_RETRY_MAX.0)));
1183 self.hold_retry = Some((Timestamp(now.0.saturating_add(wait.0)), wait));
1184 }
1185 }
1186
1187 fn prepare_again(
1190 &mut self,
1191 list: Vec<(TxnId, Transition, Option<Timestamp>)>,
1192 why: &str,
1193 s: &SystemState,
1194 defs: &UnitDefs,
1195 ) {
1196 for (id, t, since) in list {
1197 let (source, authz) = match self.earlier.get(&id).cloned() {
1201 Some((p, caller @ Actor::User { .. })) => {
1202 let admin = caller.uid().is_some_and(|u| (self.admin_of)(u));
1203 (p.source, user_authorization(caller, admin))
1204 }
1205 Some((p, caller)) => (p.source, authorization_of(caller)),
1206 None => (Source::Maintenance, authorization_of(Actor::Daemon)),
1207 };
1208 let incident = match &source {
1209 Source::Diagnoser { incident } => Some(*incident),
1210 _ => None,
1211 };
1212 let p = self.proposal(t.clone(), s, defs, source, format!("prepared again {why}"));
1213 let r = self.submit_from(p, authz, s, defs, Some(id));
1215 if let (
1217 Some(inc),
1218 Submitted::Finished { txn, state: TxnState::Built, .. } | Submitted::Parked { txn, .. },
1219 ) = (incident, &r)
1220 {
1221 self.incident_parked.insert(inc, *txn);
1222 }
1223 if let (Some(since), Submitted::Finished { txn, state: TxnState::Built, .. }) = (since, &r) {
1225 self.ex.keep_waiting_since(*txn, since);
1226 }
1227 self.notice(format!("{}: prepared again {why}: {}", render::transition(&t), render::submitted_short(&r)));
1228 }
1229 }
1230
1231 fn waiting_built(&mut self) -> Vec<(TxnId, Transition, Option<Timestamp>)> {
1234 let l = self.ledger();
1235 l.live
1236 .values()
1237 .filter(|t| t.txn.transition.changes_generation() && t.txn.state == TxnState::Built)
1238 .map(|t| (t.txn.id, t.txn.transition.clone(), self.ex.waiting_since(t.txn.id)))
1239 .collect()
1240 }
1241
1242 fn note_withdrawn(&mut self, before: Vec<(TxnId, Transition, Option<Timestamp>)>) {
1248 if before.is_empty() {
1249 return;
1250 }
1251 let l = self.ledger();
1252 for (id, t, since) in before {
1253 if l.terminal.get(&id).is_none_or(|x| x.state != TxnState::Rejected(RejectReason::Interrupted)) {
1254 continue;
1255 }
1256 if !self.earlier.contains_key(&id) {
1257 let asked = self.log.iter().chain(&self.awaiting_post).find(|r| {
1258 matches!(r.result, Submitted::Finished { txn, .. } | Submitted::Parked { txn, .. } if txn == id)
1259 });
1260 if let Some(r) = asked {
1261 self.earlier.insert(id, (r.proposal.clone(), r.caller.clone()));
1262 }
1263 }
1264 self.notice(format!(
1265 "{}: set aside for a rollback (transaction {}); it will be prepared again once the rollback is done",
1266 render::transition(&t),
1267 id.0
1268 ));
1269 self.withdrawn.push((id, t, since));
1270 }
1271 }
1272
1273 fn respond(&self, r: &Submitted) -> Response {
1276 let mut resp = render::response(r);
1277 if let Response::NeedsApproval { txn, why, .. } = &mut resp
1278 && let Some(p) = self.ex.parked.get(txn)
1279 && let Some(sum) = self.approval_diff(*txn, &p.proposal.transition, &p.facts).summary()
1280 {
1281 *why = if why.is_empty() { sum } else { format!("{why}; {sum}") };
1282 }
1283 resp
1284 }
1285
1286 fn approval_diff(&self, txn: TxnId, t: &Transition, f: &DerivedFacts) -> planarian_ipc::ApprovalDiff {
1289 let evaluated = self.evaluated_diff(txn);
1295 let options = evaluated.as_ref().or(f.config_diff.as_ref());
1296 let (removed, each_known) = crate::approval::removed(f);
1299 crate::approval::diff(t, options, evaluated.is_some(), &removed, each_known, &self.ex.cfg.tables)
1300 }
1301
1302 fn evaluated_diff(&self, txn: TxnId) -> Option<ConfigDiff> {
1305 let live = self.ex.kernel.journal.ledger().live.get(&txn)?;
1306 let (_, cand_path) = live.txn.candidate.as_ref()?;
1307 let gens = self.b.generations();
1308 let base = gens.path_of(live.txn.base).and_then(|p| gens.evaluate_built(&p)).ok()?;
1309 let cand = gens.evaluate_built(cand_path).ok()?;
1310 let mut d = ConfigDiff::default();
1311 for k in base.options.keys().chain(cand.options.keys()) {
1312 let (old, new) = (base.options.get(k), cand.options.get(k));
1313 if old != new {
1314 d.options.insert(k.clone(), (old.cloned(), new.cloned()));
1315 }
1316 }
1317 Some(d)
1318 }
1319
1320 fn generation_active(&mut self) -> bool {
1323 self.ledger().live.values().any(|t| t.txn.transition.changes_generation())
1324 }
1325
1326 fn flush_writebacks(&mut self) {
1330 while !self.writebacks.is_empty() && !self.generation_active() {
1331 let (mut wb, authz) = self.writebacks.remove(0);
1332 let (s2, d2) = self.observe();
1333 wb.observed = s2.digest();
1334 match self.ex.submit(wb.clone(), authz.clone(), &s2, &d2) {
1335 Submitted::Finished { state: TxnState::Committed, .. } => {}
1336 Submitted::Rejected { reason: RejectReason::Other(w), .. } if w == "busy" => {
1337 self.writebacks.insert(0, (wb, authz));
1338 return;
1339 }
1340 Submitted::Parked { approve, .. } => {
1341 self.notice(format!("saving the setting needs approval: planarian approve {approve}"))
1342 }
1343 other => self.notice(format!(
1344 "this setting will not survive reboot: saving it failed ({})",
1345 render::submitted_short(&other)
1346 )),
1347 }
1348 }
1349 }
1350
1351 fn lost_changes(&self, l: &Ledger) -> BTreeMap<UnitId, (ChangeId, BTreeMap<String, Value>)> {
1354 let Ok((live, _)) = self.ex.b.generations().read_overrides() else { return BTreeMap::new() };
1355 let mut last: BTreeMap<UnitId, (ChangeId, BTreeMap<String, Value>)> = BTreeMap::new();
1356 for t in l.terminal.values() {
1358 let (TxnState::Committed, Transition::ApplyChange { change, args }) = (&t.state, &t.transition) else {
1359 continue;
1360 };
1361 let (change, args) = if change.0 == planarian_changes::REAPPLY {
1362 match args.get("change") {
1363 Some(Value::Str(c)) => {
1364 let mut a = args.clone();
1365 a.remove("change");
1366 (ChangeId(c.clone()), a)
1367 }
1368 _ => continue,
1369 }
1370 } else {
1371 (change.clone(), args.clone())
1372 };
1373 if let Some(u) = planarian_changes::target_of(&change, &args) {
1374 last.insert(u, (change, args));
1375 }
1376 }
1377 let cat = &self.ex.cfg.catalog;
1378 last.retain(|_, (c, a)| {
1379 c.0 != "repair:reenable-predicted"
1380 && planarian_changes::handle(cat, c, a).is_ok_and(|set| !planarian_changes::carried(&live, &set))
1381 });
1382 last
1383 }
1384
1385 fn is_canonical(&self, u: &UnitId, defs: &UnitDefs) -> bool {
1386 defs.get(u).is_some_and(|d| {
1387 d.probes.iter().any(|p| p.origin == ProbeOrigin::Canonical && p.role == ProbeRole::Readiness)
1388 })
1389 }
1390
1391 fn canonical_healthy(&self, s: &SystemState, defs: &UnitDefs) -> bool {
1392 defs.iter()
1393 .filter(|(_, d)| {
1394 d.probes.iter().any(|p| p.origin == ProbeOrigin::Canonical && p.role == ProbeRole::Readiness)
1395 })
1396 .all(|(id, _)| !s.expected(id, defs) || s.status(id, defs) == UnitStatus::Running)
1397 }
1398
1399 pub fn set_posture(&mut self, p: Posture) {
1403 self.cfg.posture = p;
1404 }
1405
1406 fn residue(&mut self, s: &SystemState, defs: &UnitDefs) -> bool {
1417 let now = s.observed_at;
1418 let lenient = matches!(self.cfg.posture, Posture::Relaxed | Posture::Standard);
1419 let mut down: BTreeMap<UnitId, Ending> = BTreeMap::new();
1420 let mut current = BTreeMap::new();
1421 let crashed: BTreeSet<UnitId> = defs
1424 .keys()
1425 .filter(|u| {
1426 matches!(self.prev_status.get(*u), Some(UnitStatus::Running | UnitStatus::Degraded(_)))
1427 && matches!(s.status(u, defs), UnitStatus::Failed(_))
1428 })
1429 .cloned()
1430 .collect();
1431 use planarian_exec::residue::EndingFacts;
1432 let host_held = self.b.units().held();
1433 let mut tried_hold = false;
1434 for (u, def) in defs {
1435 let st = s.status(u, defs);
1436 if def.sensitivity.contains(Sensitivity::Security)
1441 && def.id.kind == UnitKind::Service
1442 && let Some(n) = self.b.units().auto_restarts(u)
1443 {
1444 let before = self.prev_auto_restarts.insert(u.clone(), n);
1445 if before.is_some_and(|b| n > b) && !self.touched.contains(u) {
1448 self.notice(format!(
1449 "ALARM: {u}, which is security-sensitive, ended outside Planarian (killed, stopped or crashed); systemd started it again"
1450 ));
1451 }
1452 }
1453 let was_up = matches!(self.prev_status.get(u), Some(UnitStatus::Running | UnitStatus::Degraded(_)));
1454 let now_down = matches!(st, UnitStatus::Failed(_) | UnitStatus::Stopped);
1455 current.insert(u.clone(), st.clone());
1456 let facts = EndingFacts {
1458 was_up,
1459 now_down,
1460 stopped_cleanly: st == UnitStatus::Stopped,
1461 ended: if was_up && now_down { self.b.units().ended_by(u) } else { None },
1462 service: def.id.kind == UnitKind::Service,
1463 touched_by_planarian: self.touched.contains(u) || self.generation_touched,
1464 expected: s.expected(u, defs),
1465 requirement_crashed: def
1466 .edges
1467 .iter()
1468 .any(|e| e.from == *u && e.kind == EdgeKind::Requires && crashed.contains(&e.to)),
1469 security: def.sensitivity.contains(Sensitivity::Security),
1470 stops_are_faults: def.stops_are_faults,
1471 systemd_restarts: def.systemd_restart.as_ref().is_some_and(|p| p.restart),
1472 lenient,
1473 network_tool: def.holds,
1474 held: s.holds.contains(u) || host_held.as_ref().is_ok_and(|h| h.contains(u)),
1475 hold_tried: false,
1476 };
1477 use planarian_exec::residue::Residue;
1478 let mut r = planarian_exec::residue::classify(&facts);
1479 if let Residue::HoldFirst(_) = r {
1480 let now = self.b.clock().boot_now();
1496 let waiting =
1497 self.hold_unconfirmed.get(u).is_some_and(|next| now < *next) || self.hold_wait().is_none();
1498 let o = match self.hold_wait() {
1499 _ if waiting => Outcome::Unknown("waiting for its hold job".into()),
1500 Some(w) => {
1501 let t0 = self.b.clock().boot_now();
1502 let o = self.b.units().hold(u, w);
1503 self.hold_spent(t0);
1504 o
1505 }
1506 None => Outcome::Unknown("no time left on this tick".into()),
1507 };
1508 let held = match o {
1509 Outcome::Unknown(_) if !waiting => {
1510 if self.hold_unconfirmed.insert(u.clone(), now.plus(HOLD_UNCONFIRMED_RETRY)).is_none() {
1514 self.notice(format!("ALARM: {u}: its traffic could not be held ({o:?})"));
1515 }
1516 true
1517 }
1518 Outcome::Failed(_) | Outcome::Unknown(_) => true,
1519 _ => self.b.units().held().map_or(true, |h| h.contains(u)),
1520 };
1521 if held {
1522 self.daemon_held.insert(u.clone());
1523 }
1524 tried_hold = true;
1525 r = planarian_exec::residue::classify(&EndingFacts { held, hold_tried: true, ..facts });
1526 }
1527 match r {
1528 Residue::NotAStop | Residue::Fault => {}
1529 Residue::AlarmHeld(_) => {}
1533 Residue::HoldFirst(_) => unreachable!("classified again with hold_tried"),
1534 Residue::AlarmSecurity(e) => self.notice(format!(
1535 "ALARM: {u}, which is security-sensitive, was stopped outside Planarian ({}); starting it again",
1536 signal_name(e)
1537 )),
1538 Residue::AlarmMustRun(e) => self.notice(format!(
1541 "ALARM: {u}, which must keep running, was stopped outside Planarian ({}); starting it again",
1542 signal_name(e)
1543 )),
1544 Residue::Intended(e) => {
1545 down.insert(u.clone(), e);
1546 }
1547 }
1548 }
1549 self.prev_status = current;
1550 self.touched.clear();
1551 self.generation_touched = false;
1552 if down.is_empty() {
1553 return tried_hold;
1554 }
1555 let roots: Vec<UnitId> = down
1557 .keys()
1558 .filter(|u| !down.keys().any(|r| r != *u && planarian_schema::spec::dependents(r, defs).contains(*u)))
1559 .cloned()
1560 .collect();
1561 for r in roots {
1562 let with: Vec<UnitId> = planarian_schema::spec::dependents(&r, defs)
1563 .into_iter()
1564 .filter(|d| *d != r && down.contains_key(d))
1565 .collect();
1566 let how = signal_name(down[&r]);
1567 let mut record = |unit: UnitId, reason: String| {
1568 let o = Override {
1569 unit,
1570 desired: OverrideDesired::Stopped,
1571 until: Until::NextBoot,
1572 reason,
1573 by: Actor::Unattributed,
1574 attribution: Attribution::Inferred,
1575 acknowledged: false,
1576 };
1577 self.journal(Record::OverrideSet(o));
1578 };
1579 record(r.clone(), format!("stopped outside Planarian ({how})"));
1580 for d in &with {
1581 record(d.clone(), format!("stopped together with {r}"));
1582 }
1583 let also = if with.is_empty() {
1584 String::new()
1585 } else {
1586 format!(
1587 " (and {}, which went down with it)",
1588 with.iter().map(|u| u.to_string()).collect::<Vec<_>>().join(", ")
1589 )
1590 };
1591 let teach = if std::mem::replace(&mut self.residue_taught, true) {
1593 String::new()
1594 } else {
1595 format!(" Next time, planarian stop {r} does the same with your name on it.")
1596 };
1597 self.notice(format!(
1598 "{r}: stopped outside Planarian ({how}); I take that as intended and leave it stopped until the next boot{also}. To start it again: planarian release {r}.{teach}"
1599 ));
1600 }
1601 let _ = now;
1602 true
1603 }
1604
1605 fn note_declarations(&mut self, prev: Option<GenerationId>, cur: GenerationId, announce: bool) {
1612 let gens = self.b.generations();
1613 let prev = prev.or_else(|| gens.boot_entries().ok()?.into_iter().filter(|g| *g < cur).max());
1614 let eval = |g: GenerationId| gens.path_of(g).and_then(|p| gens.evaluate_built(&p));
1615 let (Some(prev), Ok(now_e)) = (prev, eval(cur)) else { return };
1616 let Ok(before_e) = eval(prev) else { return };
1617 let (now_d, before) = (&now_e.declarations, &before_e.declarations);
1618 let added: Vec<(&String, &Vec<String>)> = now_d.iter().filter(|(k, _)| !before.contains_key(*k)).collect();
1619 if added.is_empty() {
1620 self.new_declarations.clear();
1621 return;
1622 }
1623 let made = self.ex.kernel.journal.segment().ok().and_then(|seg| {
1624 seg.into_iter().rev().find_map(|e| match e.record {
1625 Record::Prepared(t) if t.candidate.as_ref().is_some_and(|(g, _)| *g == cur) => Some(t),
1626 _ => None,
1627 })
1628 });
1629 let who = |a: &Actor| match a {
1630 Actor::User { uid: 0, .. } => "root".to_string(),
1631 Actor::User { uid, .. } => format!("uid {uid}"),
1632 a => format!("{a:?}"),
1633 };
1634 let created = gens.created_by(cur);
1635 self.new_declarations = added
1636 .into_iter()
1637 .map(|(svc, files)| {
1638 let how = match &made {
1639 Some(t) => format!(
1640 "by {} through {} (transaction {}), written to planarian/overrides.nix; defined in {}",
1641 who(&t.facts.caller),
1642 render::transition(&t.transition),
1643 t.id.0,
1644 if files.is_empty() { "an unknown file".to_string() } else { files.join(", ") }
1645 ),
1646 None => format!(
1647 "by {} through {}, defined in {}; generation created at {}",
1648 created.map(|(uid, _)| who(&Actor::user(uid))).unwrap_or_else(|| "an unknown user".into()),
1649 match now_e.inputs.iter().find(|(_, (_, path))| {
1652 !path.is_empty() && files.iter().any(|f| f.starts_with(path.as_str()))
1653 }) {
1654 Some((name, (rev, _))) => format!(
1655 "flake input {name} ({} → {rev}) in an out-of-band nixos-rebuild",
1656 before_e.inputs.get(name).map(|(r, _)| r.as_str()).unwrap_or("not an input before")
1657 ),
1658 None => "an out-of-band nixos-rebuild".into(),
1659 },
1660 if files.is_empty() { "an unknown file".to_string() } else { files.join(", ") },
1661 created.map(|(_, at)| render::when(at)).unwrap_or_else(|| "an unknown time".into())
1662 ),
1663 };
1664 (svc.clone(), format!("service:{svc} is new in generation {}, {how}", cur.0))
1665 })
1666 .collect();
1667 self.new_declarations_since = self.b.clock().now();
1668 if announce {
1669 let list: Vec<String> = self.new_declarations.iter().map(|(_, o)| o.clone()).collect();
1670 self.notice(format!("generation {} declares new services: {}", cur.0, list.join("; ")));
1671 }
1672 }
1673
1674 fn effective_manifest(&self) -> (planarian_schema::Manifest, Vec<UnitId>) {
1678 let mut m = self.gen_manifest.clone();
1679 let mut shadowed = vec![];
1680 for (id, def) in self.external.iter().flat_map(|d| d.units.iter()) {
1681 if m.units.contains_key(id) {
1682 shadowed.push(id.clone());
1683 } else {
1684 m.units.insert(id.clone(), crate::hooks::sanitized(def));
1685 }
1686 }
1687 (m, shadowed)
1688 }
1689
1690 pub fn origin_of(&self, unit: &UnitId) -> Option<String> {
1693 let d = self.external.as_ref()?;
1694 (d.units.contains_key(unit) && !self.gen_manifest.units.contains_key(unit)).then(|| {
1695 format!(
1696 "the external desired-state source (issuer {:?}, sequence {})",
1697 if d.issuer.is_empty() { "unnamed" } else { &d.issuer },
1698 d.seq
1699 )
1700 })
1701 }
1702
1703 pub fn health_report(&mut self) -> crate::hooks::HealthReport {
1705 let (s, defs) = self.observe();
1706 crate::hooks::report(crate::hooks::DisclosureProfile::Minimal, &s, &defs, self.incidents.incidents.values())
1707 }
1708
1709 pub fn accept_external(&mut self, doc: crate::hooks::DesiredDoc, digest: Hash) {
1715 if self.ledger().external.is_none_or(|(s, _)| doc.seq > s) {
1716 self.journal(Record::ExternalAccepted { seq: doc.seq, digest });
1717 }
1718 self.set_external(Some(doc));
1719 }
1720
1721 pub fn set_external(&mut self, doc: Option<crate::hooks::DesiredDoc>) {
1722 let said = doc.as_ref().map(|d| {
1723 let names: Vec<String> = d.units.keys().map(|u| u.to_string()).collect();
1724 format!(
1725 "external desired state in force: sequence {} from {:?}, declaring {}",
1726 d.seq,
1727 if d.issuer.is_empty() { "an unnamed issuer" } else { &d.issuer },
1728 if names.is_empty() { "nothing".to_string() } else { names.join(", ") }
1729 )
1730 });
1731 self.external = doc;
1732 let (m, shadowed) = self.effective_manifest();
1733 self.asm.set_manifest(m);
1734 if let Some(s) = said {
1735 self.notice(s);
1736 }
1737 if !shadowed.is_empty() {
1738 let names: Vec<String> = shadowed.iter().map(|u| u.to_string()).collect();
1739 self.notice(format!(
1740 "the external desired state also declares {}, which this generation declares; the generation's declaration stands",
1741 names.join(", ")
1742 ));
1743 }
1744 }
1745
1746 pub fn boot_report(&mut self) -> Option<String> {
1751 if !std::mem::take(&mut self.boot_report_due) {
1752 return None;
1753 }
1754 let foreign = self.b.units().foreign_units();
1755 for u in &foreign {
1756 self.notice(format!("INTEGRITY: boot report: {u} runs, but this generation does not declare it"));
1757 }
1758 let g = self.b.generations().current().unwrap_or_default();
1759 let text = if foreign.is_empty() {
1760 "clean: every running unit is declared by this generation".to_string()
1761 } else {
1762 format!("INTEGRITY: undeclared units: {}", foreign.join(", "))
1763 };
1764 self.last_boot_report = Some((self.b.clock().now(), g, text));
1765 (!foreign.is_empty()).then(|| format!("boot report: undeclared units: {}", foreign.join(", ")))
1766 }
1767
1768 pub fn tick(&mut self) {
1769 self.hold_left = HOLD_WAIT;
1772 let parked = &self.ex.parked;
1779 self.parked_agents.retain(|t, _| parked.contains_key(t));
1780 let (s, defs) = self.observe();
1781 let now = s.observed_at;
1782 if let Some((took, ran)) = self.asm.slow_tick.take() {
1786 self.slow_ticks_unwarned += 1;
1787 if self.last_slow_warning.is_none_or(|t| now.0 >= t.0 + SLOW_TICK_WARN_EVERY.0) {
1788 eprintln!(
1789 "planariad: warning: slow tick: {} ms (budget {} ms; {} slow tick(s) since the last warning); probes run: {}",
1790 took.0,
1791 planarian_state::TICK_BUDGET.0,
1792 self.slow_ticks_unwarned,
1793 if ran.is_empty() { "none".to_string() } else { ran.join(", ") }
1794 );
1795 self.slow_tick_warnings += 1;
1796 self.slow_ticks_unwarned = 0;
1797 self.last_slow_warning = Some(now);
1798 }
1799 }
1800
1801 let quiet = s.conditions.iter().any(|c| matches!(c, Condition::Foreground { .. }));
1803 if self.quiet && !quiet && !self.quiet_queue.is_empty() {
1804 let batch = std::mem::take(&mut self.quiet_queue);
1805 self.say(format!("while you were busy ({}): {}", batch.len(), batch.join(" · ")));
1806 }
1807 self.quiet = quiet;
1808 self.asm.quiet = quiet;
1809
1810 if let Some(why) = self.boot_report() {
1814 self.integrity = Some(why);
1815 return;
1816 }
1817
1818 let again = std::mem::take(&mut self.reprepare).into_iter().map(|(id, t)| (id, t, None)).collect();
1819 self.prepare_again(again, "after a restart", &s, &defs);
1820 if !self.withdrawn.is_empty() && !self.generation_active() {
1822 let again = std::mem::take(&mut self.withdrawn);
1823 self.prepare_again(again, "after the rollback", &s, &defs);
1824 }
1825
1826 let (s, defs) = if self.residue(&s, &defs) { self.observe() } else { (s, defs) };
1828
1829 if self.chain_checked.is_none_or(|t| now.0.saturating_sub(t.0) >= CHAIN_CHECK_EVERY.0) {
1831 self.chain_checked = Some(now);
1832 if let Err(JournalError::Integrity(why)) = self.ex.kernel.journal.verify_chain() {
1833 self.integrity = Some(format!("journal: the hash chain is broken: {why}"));
1834 return;
1835 }
1836 }
1837
1838 let (s, defs) = if self.network_holds(&s, &defs) { self.observe() } else { (s, defs) };
1841
1842 let decided = self.b.clock().now();
1844 for inc in self.incidents.tick_at(now, decided, &s, &defs) {
1845 if matches!(inc.state, IncidentState::Escalated { .. }) && inc.attempts.is_empty() {
1846 self.notice(format!(
1847 "{}: escalated: it failed again soon after each of {} repairs; it needs a person (no automatic action for an hour)",
1848 inc.target, inc.regressions
1849 ));
1850 }
1851 if inc.kind == IncidentKind::Repair
1854 && inc.attempts.is_empty()
1855 && !self.quiet
1856 && self.diag_noted.insert(inc.id)
1857 {
1858 let failing: Vec<String> = s
1859 .units
1860 .get(&inc.target)
1861 .map(|(_, ev)| {
1862 ev.iter()
1863 .filter_map(|e| match &e.result {
1864 ProbeResult::Fail(m) => Some(format!("{}: {m}", e.probe.0)),
1865 _ => None,
1866 })
1867 .collect()
1868 })
1869 .unwrap_or_default();
1870 let none = BTreeMap::new();
1872 let inputs = diagnose::Inputs {
1873 state: &s,
1874 defs: &defs,
1875 failed_since: &self.asm.failed_since,
1876 known_drivers: &self.asm.known_drivers,
1877 current_planarian_built: false,
1878 alternatives: &none,
1879 lost_changes: &BTreeMap::new(),
1880 };
1881 let root = diagnose::root_cause(&inc.target, &inputs);
1882 let because = if root == inc.target { String::new() } else { format!(" (root cause: {root})") };
1883 let origin = self.origin_of(&inc.target).map(|o| format!("; declared by {o}")).unwrap_or_default();
1884 let origin = if !self.new_declarations.is_empty()
1886 && now.0.saturating_sub(self.new_declarations_since.0) < NEW_DECLARATIONS_FOR.0
1887 {
1888 let list: Vec<&str> = self.new_declarations.iter().take(3).map(|(_, o)| o.as_str()).collect();
1889 format!("{origin}; new since the previous generation: {}", list.join("; "))
1890 } else {
1891 origin
1892 };
1893 self.notice(format!(
1894 "{}: {:?}{because}{}{origin}",
1895 inc.target,
1896 inc.divergence.1,
1897 if failing.is_empty() { String::new() } else { format!(" ({})", failing.join("; ")) }
1898 ));
1899 }
1900 if inc.kind == IncidentKind::Diagnostic
1901 && inc.state != IncidentState::Closed
1902 && self.diag_noted.insert(inc.id)
1903 {
1904 let answering: Vec<String> = defs
1907 .get(&inc.target)
1908 .map(|d| {
1909 let ev = s.units.get(&inc.target).map(|(_, e)| e.as_slice()).unwrap_or(&[]);
1910 d.probes
1911 .iter()
1912 .filter(|p| p.role == ProbeRole::Readiness)
1913 .filter(|p| {
1914 !ev.iter().any(|e| {
1915 e.probe == p.id
1916 && matches!(
1917 e.result,
1918 ProbeResult::Timeout | ProbeResult::Abandoned | ProbeResult::Error(_)
1919 )
1920 })
1921 })
1922 .map(|p| p.id.0.clone())
1923 .collect()
1924 })
1925 .unwrap_or_default();
1926 self.notice(format!(
1927 "{}: its health is unknown (checks time out); re-observing through {} rather than restarting it",
1928 inc.target,
1929 if answering.is_empty() { "its other checks".into() } else { answering.join(", ") }
1930 ));
1931 }
1932 self.journal(Record::Incident(inc));
1933 }
1934 let diag: Vec<UnitId> = self.incidents.diagnostic().iter().map(|i| i.target.clone()).collect();
1935 for t in diag {
1936 self.asm.reprobe(&t);
1937 }
1938
1939 let l = self.ledger();
1940 if l.interlocked() {
1941 if !self.interlock_noted {
1942 self.notice(
1943 "recovery failed; autonomous repair is stopped until an administrator runs `planarian resolve`"
1944 .into(),
1945 );
1946 self.interlock_noted = true;
1947 }
1948 } else {
1949 self.interlock_noted = false;
1950 let busy = l.live.values().any(|t| {
1953 !t.txn.state.is_pre_mutation()
1954 && t.txn.state != TxnState::AwaitingBoot
1955 && !(t.txn.state == TxnState::Promoted && t.boot_only)
1956 });
1957 if !busy {
1958 self.repair_one(&s, &defs, &l);
1959 }
1960 if !busy {
1962 for (id, r) in self.ex.activate_waiting(&s, &defs) {
1963 let linked: Vec<IncidentId> =
1965 self.incident_parked.iter().filter(|(_, t)| **t == id).map(|(i, _)| *i).collect();
1966 for inc in linked {
1967 self.incident_parked.remove(&inc);
1968 if let Submitted::Finished { state, .. } = &r
1969 && let Some(i) =
1970 self.incidents.record_attempt(inc, id, outcome_of(state), self.b.clock().now())
1971 {
1972 self.journal(Record::Incident(i));
1973 }
1974 }
1975 if matches!(r, Submitted::Finished { state: TxnState::Rejected(RejectReason::Stale), .. }) {
1976 self.notice(format!(
1977 "transaction {}: not activated: the configuration, the override file or what the change would do changed while it waited; ask for it again",
1978 id.0
1979 ));
1980 continue;
1981 }
1982 self.notice(format!(
1983 "transaction {}: activated now that nothing is in the way: {}",
1984 id.0,
1985 render::submitted_short(&r)
1986 ));
1987 }
1988 }
1989 if !busy && !self.quiet && now >= self.next_gc_check {
1991 self.next_gc_check = now.plus(GC_CHECK);
1992 self.collect_if_over(&s, &defs);
1993 }
1994 }
1995
1996 if !l.interlocked() {
1998 self.flush_writebacks();
1999 }
2000
2001 for t in self.ex.expire() {
2002 self.notice(format!("approval for transaction {} expired", t.0));
2003 }
2004 self.remind_restart(&l);
2005 for (t, change) in self.ex.expire_waiting() {
2006 self.incident_parked.retain(|_, w| *w != t);
2008 self.notice(format!(
2009 "transaction {} cancelled: it waited a day in Built without an activation window ({}); issue it again to retry",
2010 t.0,
2011 render::transition(&change)
2012 ));
2013 }
2014 if self.ex.kernel.journal.needs_compaction()
2015 && let Err(e) = self.ex.kernel.journal.compact()
2016 {
2017 self.notice(format!("journal compaction failed: {e:?}"));
2018 }
2019 self.confirm_boot(&s, &defs, now);
2020 self.note_space();
2021 self.note_unreachable(&s, &defs, now);
2022 }
2023
2024 fn remind_restart(&mut self, l: &Ledger) {
2027 let now = self.b.clock().boot_now();
2028 let waiting: Vec<(TxnId, String)> = l
2029 .live
2030 .values()
2031 .filter(|t| t.txn.state == TxnState::AwaitingBoot)
2032 .map(|t| (t.txn.id, render::transition(&t.txn.transition)))
2033 .collect();
2034 self.restart_reminded.retain(|id, _| waiting.iter().any(|(w, _)| w == id));
2035 for (id, what) in waiting {
2036 let last = *self.restart_reminded.entry(id).or_insert(now);
2037 if now > last.plus(RESTART_REMINDER) {
2038 self.restart_reminded.insert(id, now);
2039 self.notice(format!("transaction {}: {what} is still waiting for a restart; restart to finish", id.0));
2040 }
2041 }
2042 }
2043
2044 fn note_unreachable(&mut self, s: &SystemState, defs: &UnitDefs, now: Timestamp) {
2049 for (id, def) in defs {
2050 if !def.sensitivity.contains(Sensitivity::External) {
2051 continue;
2052 }
2053 let timing_out: Vec<String> = match (s.status(id, defs), s.units.get(id)) {
2054 (UnitStatus::Unknown, Some((_, ev))) => ev
2055 .iter()
2056 .filter(|e| {
2057 matches!(e.result, ProbeResult::Timeout | ProbeResult::Abandoned | ProbeResult::Error(_))
2058 })
2059 .map(|e| e.probe.0.clone())
2060 .collect(),
2061 _ => vec![],
2062 };
2063 if timing_out.is_empty() {
2064 if let Some((_, true)) = self.unreachable.remove(id) {
2065 self.notice(format!("{id}: answers again"));
2066 }
2067 continue;
2068 }
2069 let (since, said) = *self.unreachable.entry(id.clone()).or_insert((now, false));
2070 if !said && now >= since.plus(UNREACHABLE_AFTER) {
2071 self.unreachable.insert(id.clone(), (since, true));
2072 self.notice(format!(
2073 "{id}: can't reach it: {} timing out for {} min; it may be the network on the way there, so nothing local is restarted",
2074 timing_out.join(", "),
2075 (now.0 - since.0) / 60_000
2076 ));
2077 }
2078 }
2079 }
2080
2081 fn note_space(&mut self) {
2083 let Some(r) = self.b.generations().space_report() else { return };
2084 if r.at <= self.space_noted {
2085 return;
2086 }
2087 self.space_noted = r.at;
2088 if let Some(short) = r.shortfall {
2089 self.notice(format!(
2090 "BudgetUnsatisfiable: the generations Planarian must keep use {} over the space budget of {}; everything else was collected, so the budget is a target, not a guarantee",
2091 render::bytes(short),
2092 render::bytes(r.max_bytes)
2093 ));
2094 }
2095 }
2096
2097 fn collect_if_over(&mut self, s: &SystemState, defs: &UnitDefs) {
2100 let l = self.ledger();
2101 let protect = self.protect(&l, s.generation);
2102 let Ok(plan) = self.b.generations().gc_plan(&protect, false) else { return };
2103 if plan.delete.is_empty() {
2104 return;
2105 }
2106 let reason = format!(
2107 "{} generation(s) over the collection bounds (keep {}, {} of {} bytes)",
2108 plan.delete.len(),
2109 plan.kept.len(),
2110 plan.kept_bytes,
2111 plan.max_bytes
2112 );
2113 let p = self.proposal(Transition::CollectGarbage { emergency: false }, s, defs, Source::Maintenance, reason);
2114 let _ = self.submit(p, authorization_of(Actor::Daemon), s, defs);
2115 }
2116
2117 fn repair_one(&mut self, s: &SystemState, defs: &UnitDefs, l: &Ledger) {
2118 let now = s.observed_at;
2119 let parked = self.ex.parked.clone();
2122 let ex = &self.ex;
2123 self.incident_parked.retain(|_, t| parked.contains_key(t) || ex.is_waiting(*t));
2124 let quiet = self.quiet;
2127 let Some(inc) = self
2128 .incidents
2129 .actionable(now)
2130 .into_iter()
2131 .filter(|i| !quiet || foreground_depends_on(&i.target, defs))
2132 .find(|i| !self.incident_parked.contains_key(&i.id))
2133 .cloned()
2134 else {
2135 return;
2136 };
2137 let current = s.generation;
2138 let alternatives: BTreeMap<String, Vec<Value>> = defs
2139 .values()
2140 .filter_map(|d| d.runtime_repair.clone())
2141 .collect::<BTreeSet<String>>()
2142 .into_iter()
2143 .map(|opt| {
2144 let alts = self.ex.b.runtime().alternatives(&opt);
2145 (opt, alts)
2146 })
2147 .collect();
2148 let inputs = diagnose::Inputs {
2149 state: s,
2150 defs,
2151 failed_since: &self.asm.failed_since,
2152 known_drivers: &self.asm.known_drivers,
2153 current_planarian_built: l.live_tested == Some(current),
2154 alternatives: &alternatives,
2155 lost_changes: &self.lost_changes(l),
2156 };
2157 let root = diagnose::root_cause(&inc.target, &inputs);
2160 let because = if root == inc.target { String::new() } else { format!(" (root cause: {root})") };
2161 let others_rooted: Vec<(UnitId, UnitId, UnitStatus)> = defs
2165 .keys()
2166 .filter(|u| **u != inc.target && s.expected(u, defs))
2167 .filter_map(|u| {
2168 let st = s.status(u, defs);
2169 matches!(st, UnitStatus::Degraded(_) | UnitStatus::Failed(_) | UnitStatus::Stopped)
2170 .then(|| (u.clone(), diagnose::root_cause(u, &inputs), st))
2171 })
2172 .collect();
2173 match diagnose::next(&inc, &inputs) {
2177 None => {
2178 if let Some(i) = self.incidents.exhausted(inc.id, self.b.clock().now()) {
2179 self.journal(Record::Incident(i));
2180 }
2181 self.notice(format!(
2182 "{}: {:?}{because} and nothing I know how to do would fix it; it needs a person (I'll look again in an hour)",
2183 inc.target, inc.divergence.1
2184 ));
2185 }
2186 Some(c)
2188 if c.transition == Transition::Rollback && s.security_floor.is_some_and(|f| s.known_good.0 < f.0) =>
2189 {
2190 if let Some(i) =
2191 self.incidents.escalate(inc.id, EscalationReason::RevertBlockedByFloor, self.b.clock().now())
2192 {
2193 self.journal(Record::Incident(i));
2194 }
2195 self.notice(format!(
2196 "{}: {:?}{because}; rolling back would go below the security floor (generation {}), so it needs an administrator: the known-good generation {} lacks a security fix",
2197 inc.target,
2198 inc.divergence.1,
2199 s.security_floor.map(|f| f.0).unwrap_or_default(),
2200 s.known_good.0
2201 ));
2202 }
2203 Some(c) if self.ex.parked.values().any(|p| p.proposal.transition == c.transition) => {
2204 let txn = self.ex.parked.values().find(|p| p.proposal.transition == c.transition).map(|p| p.txn);
2207 if let Some(t) = txn {
2208 self.incident_parked.insert(inc.id, t);
2209 }
2210 }
2211 Some(c) => {
2212 if inc.attempts.is_empty()
2215 && matches!(c.transition, Transition::RestartUnit(_))
2216 && c.target.kind == UnitKind::Service
2217 && s.status(&c.target, defs) == UnitStatus::Stopped
2218 {
2219 self.notice(format!(
2220 "{}: it was stopped, not crashed, so I'm starting it again; to keep it stopped: planarian override {}",
2221 c.target, c.target
2222 ));
2223 }
2224 let p = self.proposal(
2225 c.transition.clone(),
2226 s,
2227 defs,
2228 Source::Diagnoser { incident: inc.id },
2229 c.reason.clone(),
2230 );
2231 let admitted = l.live.values().any(|t| t.txn.state == TxnState::Promoted && t.boot_only);
2234 if admitted && self.is_canonical(&c.target, defs) {
2235 self.repaired_before_confirmation.insert(c.target.clone());
2236 }
2237 match self.submit(p, authorization_of(Actor::Daemon), s, defs) {
2238 Submitted::Finished { txn, state: TxnState::Built, .. } => {
2242 self.incident_parked.insert(inc.id, txn);
2243 }
2244 Submitted::Finished { txn, state, .. } => {
2245 let at = self.b.clock().now();
2250 if let Some(i) = self.incidents.record_attempt(inc.id, txn, outcome_of(&state), at) {
2251 if matches!(i.state, IncidentState::Cooldown { .. }) {
2252 self.notice(format!(
2253 "{}: tried {} time(s) and it keeps failing; it needs a person (I'll try once more in {} min)",
2254 inc.target,
2255 i.attempts.len(),
2256 i.backoff.0 / 60_000
2257 ));
2258 }
2259 self.journal(Record::Incident(i));
2260 }
2261 self.notice(format!(
2262 "{}: {} → {:?}{because}",
2263 inc.target,
2264 render::transition(&c.transition),
2265 state
2266 ));
2267 if state == TxnState::Committed {
2271 let dependents: Vec<(UnitId, UnitStatus)> = others_rooted
2272 .iter()
2273 .filter(|(t, root, _)| *root == c.target && *t != c.target)
2274 .map(|(t, _, st)| (t.clone(), st.clone()))
2275 .collect();
2276 for (d, st) in dependents {
2277 self.notice(format!(
2278 "{d}: {st:?} (root cause: {}); repaired with {}",
2279 c.target,
2280 render::transition(&c.transition)
2281 ));
2282 }
2283 }
2284 }
2285 Submitted::Parked { txn, approve, facts } => {
2286 self.incident_parked.insert(inc.id, txn);
2287 let changes = self
2288 .approval_diff(txn, &c.transition, &facts)
2289 .summary()
2290 .map(|s| format!("; {s}"))
2291 .unwrap_or_default();
2292 self.notice(format!(
2293 "{}: {} needs approval ({}{changes}): planarian approve {approve}",
2294 inc.target,
2295 render::transition(&c.transition),
2296 render::facts_summary(&facts)
2297 ));
2298 }
2299 Submitted::Rejected { reason, detail, .. } => {
2300 if let Some(i) = self.incidents.exhausted(inc.id, self.b.clock().now()) {
2301 self.journal(Record::Incident(i));
2302 }
2303 self.notice(format!(
2304 "{}: {} refused ({reason:?}: {detail})",
2305 inc.target,
2306 render::transition(&c.transition)
2307 ));
2308 }
2309 Submitted::Error(e) => self.notice(format!("{}: {e}", inc.target)),
2310 }
2311 }
2312 }
2313 }
2314
2315 fn confirm_boot(&mut self, s: &SystemState, defs: &UnitDefs, now: Timestamp) {
2316 let Some((g, at)) = self.confirm else { return };
2317 if now < at {
2318 return;
2319 }
2320 self.confirm = None;
2321 let healthy = s.generation == g && self.canonical_healthy(s, defs);
2322 let admitted: Vec<TxnId> = self
2325 .ledger()
2326 .live
2327 .values()
2328 .filter(|t| t.txn.state == TxnState::Promoted && t.path.iter().any(|p| p == "AwaitingBoot"))
2329 .map(|t| t.txn.id)
2330 .collect();
2331 let repaired = std::mem::take(&mut self.repaired_before_confirmation);
2334 for id in admitted {
2335 match self.ex.kernel.confirm_boot_only(id, healthy && repaired.is_empty()) {
2336 Ok(TxnState::Committed) => {}
2337 Ok(TxnState::Reverted) => {
2338 let back = self.b.generations().boot_default().map(|d| d.0).unwrap_or_default();
2339 let why = if repaired.is_empty() {
2340 "canonical probes were failing at the 10-minute checkpoint".to_string()
2341 } else {
2342 let units: Vec<String> = repaired.iter().map(|u| u.to_string()).collect();
2343 format!("{} needed a repair before the 10-minute checkpoint", units.join(", "))
2344 };
2345 self.notice(format!(
2346 "transaction {}: not confirmed ({why}); generation {back} is the default again \
2347 and a restart completes the withdrawal",
2348 id.0
2349 ))
2350 }
2351 Ok(other) => self.notice(format!("transaction {}: confirmation left it {other:?}", id.0)),
2352 Err(e) => self.notice(format!("transaction {}: confirmation failed: {e:?}", id.0)),
2353 }
2354 }
2355 if healthy {
2356 self.journal(Record::KnownGood(g));
2357 self.notice(format!("generation {} is now the boot-confirmed known-good baseline", g.0));
2358 } else {
2359 self.notice(format!(
2360 "generation {} was not confirmed: canonical probes were failing at the 10-minute checkpoint",
2361 g.0
2362 ));
2363 }
2364 }
2365
2366 fn authz(&mut self, peer: &Peer) -> Authorization {
2371 if let Some(name) = &peer.agent {
2372 let instance = peer.instance.map(|i| i.0).unwrap_or_default();
2378 let Some(by) = self.ledger().agent_for(name, instance) else {
2379 let actor = Actor::Agent(agent_principal(name));
2380 return Authorization { principal: actor.principal(), actor, capabilities: Capabilities::default() };
2381 };
2382 let actor = Actor::User { uid: by, session: None, seat: None, active: false };
2383 return user_authorization(actor, (self.admin_of)(by));
2384 }
2385 let info =
2386 peer.session.as_ref().and_then(|id| self.asm.sessions.iter().find(|x| &x.id == id && x.uid == peer.uid));
2387 let actor = Actor::User {
2388 uid: peer.uid,
2389 session: info.map(|x| x.id.clone()),
2390 seat: info.filter(|x| !x.remote).and_then(|x| x.seat.clone()),
2391 active: info.is_some_and(|x| x.active),
2392 };
2393 user_authorization(actor, peer.admin)
2394 }
2395
2396 pub fn take_deferred(&mut self) -> Option<(PrebuildJob<B::Gens>, Resume)> {
2400 self.deferred.take()
2401 }
2402
2403 pub fn resume(&mut self, r: Resume, prebuilt: Prebuilt) -> Response {
2406 self.ex.prebuilt = Some(prebuilt);
2407 let defer = std::mem::replace(&mut self.defer, false);
2408 let resp = match r.kind {
2409 ResumeKind::Submit { t, source, reason } => self.user_submit(t, r.peer, source, reason),
2410 ResumeKind::Request(req) => self.handle(r.peer, req),
2411 };
2412 self.defer = defer;
2413 self.ex.prebuilt = None;
2414 resp
2415 }
2416
2417 fn user_submit(&mut self, t: Transition, peer: Peer, source: Source, reason: String) -> Response {
2418 let who = self.authz(&peer);
2421 let admitted = self.ex.admit(&t, &who).is_ok();
2422 if admitted
2423 && self.defer
2424 && self.deferred.is_none()
2425 && let Some(job) = self.ex.prebuild_job(&t)
2426 {
2427 self.deferred = Some((job, Resume { peer, kind: ResumeKind::Submit { t, source, reason } }));
2428 return Response::Ok { message: "deferred".into() };
2429 }
2430 let (s, defs) = self.observe();
2431 let p = self.proposal(t, &s, &defs, source, reason);
2432 let authz = self.authz(&peer);
2433 let t = p.transition.clone();
2434 let r = self.submit(p, authz, &s, &defs);
2435 if let Submitted::Parked { txn, approve, facts } = &r
2438 && let Some(changes) = self.approval_diff(*txn, &t, facts).summary()
2439 {
2440 self.notice(format!(
2441 "{} needs approval ({}; {changes}): planarian approve {approve}",
2442 render::transition(&t),
2443 render::facts_summary(facts)
2444 ));
2445 }
2446 let mut resp = self.respond(&r);
2447 if let (Response::NeedsApproval { txn, why, .. }, Submitted::Parked { .. }) = (&mut resp, &r)
2449 && self.ex.parked.get(txn).is_some_and(|p| p.class == ApproverClass::Administrator)
2450 {
2451 why.push_str("; an administrator must approve it");
2452 }
2453 resp
2454 }
2455
2456 fn status(&mut self) -> StatusReport {
2457 let (s, defs) = self.last.clone().unwrap_or_else(|| self.observe());
2461 let l = self.ledger();
2462 let units = defs
2463 .keys()
2464 .map(|u| {
2465 let status = s.status(u, &defs);
2466 UnitRow { unit: u.clone(), detail: render::evidence(&s, u), expected: s.expected(u, &defs), status }
2467 })
2468 .collect();
2469 let pending = self
2470 .ex
2471 .parked
2472 .values()
2473 .map(|p| Pending {
2474 txn: p.txn,
2475 approve: p.bound_to,
2476 what: render::transition(&p.proposal.transition),
2477 facts: p.facts.clone(),
2478 diff: self.approval_diff(p.txn, &p.proposal.transition, &p.facts),
2479 issued: Timestamp(
2481 self.b.clock().now().0.saturating_sub(self.b.clock().boot_now().0.saturating_sub(p.issued.0)),
2482 ),
2483 })
2484 .collect();
2485 StatusReport {
2486 generation: s.generation,
2487 known_good: l.known_good,
2488 live_tested: l.live_tested,
2489 accepted_baseline: l.accepted_baseline,
2490 boot_default: self.b.generations().boot_default().ok(),
2491 baseline_note: l
2492 .known_good
2493 .is_none()
2494 .then(|| "no baseline yet: autonomy is limited to Runtime until this generation is confirmed".into()),
2495 units,
2496 incidents: self.incidents.incidents.values().cloned().collect(),
2497 pending,
2498 in_flight: l
2499 .live
2500 .values()
2501 .map(|t| (t.txn.id, t.txn.state.clone(), render::transition(&t.txn.transition)))
2502 .collect(),
2503 interlock: l
2504 .interlock
2505 .values()
2506 .map(|i| {
2507 (i.txn, format!("{} failed from {}: {}", render::transition(&i.transition), i.failed_from, i.note))
2508 })
2509 .collect(),
2510 reconciliation: l.last_reconciliation.map(|r| r.actions),
2511 inferred: l
2512 .overrides
2513 .iter()
2514 .filter(|o| o.attribution == Attribution::Inferred && !o.acknowledged)
2515 .map(|o| (o.unit.clone(), o.reason.clone()))
2516 .collect(),
2517 notices: self
2518 .building
2519 .as_ref()
2520 .map(|(what, since)| {
2521 let mins = self.b.clock().now().0.saturating_sub(since.0) / 60_000;
2522 format!("building {what} ({mins} min so far); repairs go on meanwhile")
2523 })
2524 .into_iter()
2525 .chain(self.notices.iter().cloned())
2526 .collect(),
2527 space: self.b.generations().space_report(),
2528 }
2529 }
2530
2531 fn console(&mut self) -> ConsoleReport {
2533 let status = self.status();
2534 let (s, defs) = self.last.clone().unwrap_or_else(|| self.observe());
2535 let l = self.ledger();
2536 let nodes = defs
2537 .iter()
2538 .map(|(u, def)| {
2539 let health = s.status(u, &defs);
2540 let observed = match s.units.get(u).map(|(o, _)| o) {
2541 None => "not observed".to_string(),
2542 Some(Observed::Failed(r)) => format!("Failed: {r:?}"),
2543 Some(Observed::Unknown { reason, .. }) => format!("Unknown: {reason}"),
2544 Some(Observed::Transitioning { .. }) => "Transitioning".to_string(),
2545 Some(Observed::Bound) if u.kind != UnitKind::Device => "Stopped".to_string(),
2547 Some(o) => format!("{o:?}"),
2548 };
2549 GraphNode {
2550 unit: u.clone(),
2551 desired: format!("{:?}", def.desired.status),
2552 expected: s.expected(u, &defs),
2553 observed,
2554 stale: health == UnitStatus::Unknown,
2555 suppressed: matches!(health, UnitStatus::Overridden(_)),
2556 health,
2557 }
2558 })
2559 .collect();
2560 let edges = defs
2561 .values()
2562 .flat_map(|d| d.edges.iter().map(|e| GraphEdge { from: e.from.clone(), to: e.to.clone(), kind: e.kind }))
2563 .collect();
2564 let mut timeline: Vec<TxnSummary> = l
2565 .live
2566 .values()
2567 .map(|t| TxnSummary {
2568 txn: t.txn.id,
2569 what: render::transition(&t.txn.transition),
2570 path: t.path.clone(),
2571 state: t.txn.state.clone(),
2572 live: true,
2573 by_watchdog: t.txn.lease.owner.is_watchdog(),
2574 why: t.why.clone(),
2575 })
2576 .chain(l.terminal.iter().map(|(id, t)| TxnSummary {
2577 txn: *id,
2578 what: render::transition(&t.transition),
2579 path: t.path.clone(),
2580 state: t.state.clone(),
2581 live: false,
2582 by_watchdog: t.owner.is_watchdog(),
2583 why: t.why.clone(),
2584 }))
2585 .collect();
2586 timeline.sort_by_key(|t| std::cmp::Reverse(t.txn));
2587 let gens = self.b.generations();
2588 let current = s.generation;
2589 let previous = gens.boot_entries().ok().and_then(|e| e.into_iter().filter(|g| *g < current).max());
2590 let last_change_reverted = l
2591 .terminal
2592 .values()
2593 .rev()
2594 .find(|t| t.transition.changes_generation())
2595 .is_some_and(|t| t.state == TxnState::Reverted);
2596 ConsoleReport {
2597 status,
2598 nodes,
2599 edges,
2600 timeline,
2601 previous,
2602 last_booted: gens.last_booted().ok(),
2603 candidates: l.live.values().filter_map(|t| t.txn.candidate.as_ref().map(|(g, _)| *g)).collect(),
2604 last_change_reverted,
2605 now: s.observed_at,
2606 }
2607 }
2608
2609 fn intent_index(&self, defs: &UnitDefs) -> intent::Index {
2610 let changes = planarian_changes::entries(&self.cfg.exec.catalog)
2611 .into_iter()
2612 .map(|e| intent::CatalogEntry { id: e.id, description: e.description })
2613 .collect();
2614 let options = self
2615 .cfg
2616 .runtime_options
2617 .iter()
2618 .map(|o| intent::RuntimeOption {
2619 current: self.b.runtime().get(&o.path).or(o.current.clone()),
2620 ..o.clone()
2621 })
2622 .collect();
2623 intent::Index::new(defs, changes, options)
2624 }
2625
2626 pub fn handle(&mut self, peer: Peer, req: Request) -> Response {
2627 self.acting_agent = peer.agent.clone();
2628 let r = self.handle_as(peer, req);
2629 self.acting_agent = None;
2630 r
2631 }
2632
2633 fn handle_as(&mut self, peer: Peer, req: Request) -> Response {
2634 match req {
2635 Request::Status => Response::Status(Box::new(self.status())),
2636 Request::Incidents => {
2637 Response::Incidents { incidents: self.incidents.incidents.values().cloned().collect() }
2638 }
2639 Request::Log { limit } => {
2640 let entries = self.ex.kernel.journal.segment().unwrap_or_default();
2641 let lines = entries
2642 .iter()
2643 .rev()
2644 .take(limit)
2645 .rev()
2646 .map(|e| LogLine { seq: e.seq, at: e.at, text: render::record(&e.record) })
2647 .collect();
2648 Response::Log { lines }
2649 }
2650 Request::Ask { phrase, choice } => {
2651 if let Some(i) = choice {
2652 let Some(m) = self.choices.remove(&peer.uid).and_then(|c| c.into_iter().nth(i)) else {
2653 return Response::Error { message: "no such choice; ask again".into() };
2654 };
2655 let src = Source::Intent { phrase: phrase.clone(), confidence: m.confidence };
2656 return self.user_submit(m.transition, peer, src, format!("asked: \"{phrase}\""));
2657 }
2658 let (_, defs) = self.observe();
2659 let now = self.b.clock().now();
2660 match intent::interpret(&phrase, &self.intent_index(&defs), now) {
2661 Interpretation::Propose(m) => {
2662 let src = Source::Intent { phrase: phrase.clone(), confidence: m.confidence };
2663 self.user_submit(m.transition, peer, src, format!("asked: \"{phrase}\" → {}", m.understood))
2664 }
2665 Interpretation::Confirm(ms) => {
2666 let choices = ms
2667 .iter()
2668 .enumerate()
2669 .map(|(index, m)| Choice { index, understood: m.understood.clone(), confidence: m.confidence.0 })
2670 .collect();
2671 self.choices.insert(peer.uid, ms);
2672 Response::Confirm { choices }
2673 }
2674 Interpretation::Unmatched => Response::Unmatched {
2675 message: "I don't know how to do that yet. Try `planarian status`, or phrases like \"restart wifi\", \"stop backups until I say\", \"make text bigger\".".into(),
2676 },
2677 }
2678 }
2679 Request::Apply { change, args } => {
2680 self.user_submit(Transition::ApplyChange { change, args }, peer, Source::User, "from the CLI".into())
2681 }
2682 Request::SetOverride { unit, until, desired } => {
2683 let s = Override::attested(unit, desired, until, "from the CLI".into(), Actor::user(peer.uid));
2685 self.user_submit(Transition::SetOverride(s), peer, Source::User, "from the CLI".into())
2686 }
2687 Request::HoldRelease { unit, until } => {
2688 let hold = crate::hold::hold_unit(&unit);
2693 let (_, defs) = self.observe();
2694 if !defs.contains_key(&hold) {
2695 return Response::Error {
2696 message: format!("{unit} is not a network-security unit; there is no hold to release"),
2697 };
2698 }
2699 let reason = format!("planarian hold release {}", unit.name);
2700 let o = Override::attested(
2701 hold,
2702 OverrideDesired::Stopped,
2703 until.clone(),
2704 reason.clone(),
2705 Actor::user(peer.uid),
2706 );
2707 let r = self.user_submit(Transition::SetOverride(o), peer.clone(), Source::User, reason);
2708 if matches!(&r, Response::Done { state: TxnState::Committed, .. }) {
2709 if self.ledger().holds().contains(&unit) {
2712 self.journal(Record::HoldLifted {
2713 unit: unit.clone(),
2714 reason: format!("released by uid {}", peer.uid),
2715 });
2716 }
2717 let _ = self.b.units().release_hold(&unit, crate::hold::release_until(&until), HOLD_WAIT);
2718 self.hold_released.insert(unit);
2719 }
2720 r
2721 }
2722 Request::LiftOverride { unit } => {
2723 let r = self.user_submit(
2724 Transition::LiftOverride(unit.clone()),
2725 peer.clone(),
2726 Source::User,
2727 "from the CLI".into(),
2728 );
2729 let released = matches!(&r, Response::Done { state: TxnState::Committed, .. });
2733 if !released {
2734 return r;
2735 }
2736 let with = format!("stopped together with {unit}");
2738 let linked: Vec<UnitId> = self
2739 .ledger()
2740 .overrides
2741 .iter()
2742 .filter(|o| o.attribution == Attribution::Inferred && o.reason == with)
2743 .map(|o| o.unit.clone())
2744 .collect();
2745 for u in linked {
2746 let _ = self.user_submit(Transition::LiftOverride(u), peer.clone(), Source::User, with.clone());
2747 }
2748 let now = self.b.clock().now();
2749 if let Some(i) = self.incidents.human_acted(&unit, now) {
2750 self.journal(Record::Incident(i));
2751 }
2752 r
2753 }
2754 Request::Stop { unit } => {
2755 let t = Transition::StopAndSuppress {
2757 unit: unit.clone(),
2758 until: Until::NextBoot,
2759 reason: "stopped from the CLI".into(),
2760 };
2761 match self.user_submit(t, peer, Source::User, "planarian stop".into()) {
2762 Response::Done { txn, state, message } => Response::Done {
2763 txn,
2764 state,
2765 message: format!(
2766 "{message}. Note: `stop` means stop and leave stopped until the next boot; `planarian release {unit}` lets it run again."
2767 ),
2768 },
2769 other => other,
2770 }
2771 }
2772 Request::Restart { unit } => {
2773 self.user_submit(Transition::RestartUnit(unit), peer, Source::User, "from the CLI".into())
2774 }
2775 Request::Gc => self.user_submit(
2776 Transition::CollectGarbage { emergency: false },
2777 peer,
2778 Source::User,
2779 "from the CLI".into(),
2780 ),
2781 Request::Acknowledge { unit } => {
2782 if peer.agent.is_some() {
2783 return Response::Rejected {
2784 reason: "Unauthorized".into(),
2785 message: "an agent cannot acknowledge".into(),
2786 };
2787 }
2788 let open: Vec<Override> = self
2789 .ledger()
2790 .overrides
2791 .iter()
2792 .filter(|o| o.attribution == Attribution::Inferred && !o.acknowledged)
2793 .filter(|o| unit.as_ref().is_none_or(|u| *u == o.unit))
2794 .cloned()
2795 .collect();
2796 for o in &open {
2797 self.journal(Record::OverrideSet(Override { acknowledged: true, ..o.clone() }));
2798 }
2799 if !open.is_empty() {
2800 self.notice(format!(
2801 "uid {} acknowledged: {}",
2802 peer.uid,
2803 open.iter().map(|o| format!("{} (inferred override)", o.unit)).collect::<Vec<_>>().join(", ")
2804 ));
2805 }
2806 Response::Ok {
2807 message: format!("{} inferred override(s) acknowledged; they hold until they end", open.len()),
2808 }
2809 }
2810 Request::AgentRun { name, argv } => {
2811 if peer.agent.is_some() {
2812 return Response::Rejected {
2813 reason: "Unauthorized".into(),
2814 message: "an agent cannot start agents".into(),
2815 };
2816 }
2817 if !planarian_schema::backend::valid_agent_name(&name) || argv.is_empty() {
2818 return Response::Error {
2819 message: "agent run <name> -- <command>: the name is 1-32 of a-z, 0-9 and -".into(),
2820 };
2821 }
2822 if self.b.units().agent_running(&name) {
2825 let owner = self.ledger().agent_owner(&name);
2826 return Response::Rejected {
2827 reason: "Unauthorized".into(),
2828 message: match owner {
2829 Some(u) => format!("agent {name} is already running, started by uid {u}; stop it first"),
2830 None => format!("agent {name} is already running; stop it first"),
2831 },
2832 };
2833 }
2834 let limit = self.cfg.agent_limits.per_uid;
2836 let mine = self
2837 .ledger()
2838 .agents
2839 .iter()
2840 .filter(|(n, _, by)| *by == peer.uid && self.b.units().agent_running(n))
2841 .map(|(n, _, _)| n.clone())
2842 .collect::<BTreeSet<_>>();
2843 if mine.len() as u32 >= limit {
2844 return Response::Rejected {
2845 reason: "Unauthorized".into(),
2846 message: format!(
2847 "uid {} already runs {} agent(s), the limit (services.planarian.agents.limits.perUid = {limit})",
2848 peer.uid,
2849 mine.len()
2850 ),
2851 };
2852 }
2853 match self.b.units().start_agent(&name, &argv, &self.cfg.agent_limits) {
2854 Ok(instance) => {
2855 self.journal(Record::AgentStarted { name: name.clone(), instance: instance.0, by: peer.uid });
2858 self.notice(format!(
2859 "agent {name} started by uid {} ({}); it acts with that user's permissions",
2860 peer.uid,
2861 planarian_schema::backend::agent_unit(&name)
2862 ));
2863 Response::Ok { message: format!("agent {name} is running with your permissions") }
2864 }
2865 Err(other) => Response::Error { message: format!("could not start agent {name}: {other:?}") },
2866 }
2867 }
2868 Request::Undo => {
2869 let current = self.b.generations().current().unwrap_or_default();
2870 let prev =
2871 self.b.generations().boot_entries().unwrap_or_default().into_iter().filter(|g| *g < current).max();
2872 match prev {
2873 Some(to) => {
2874 self.user_submit(Transition::Revert { to }, peer, Source::User, "planarian undo".into())
2875 }
2876 None => Response::Error { message: "there is no earlier generation to go back to".into() },
2877 }
2878 }
2879 Request::Approve { hash } => {
2880 if peer.uid != 0 || !peer.tty {
2881 return Response::Error {
2882 message: "approve requires root at an interactive terminal (Phase A)".into(),
2883 };
2884 }
2885 if self.defer && self.deferred.is_none() {
2886 let t = self.ex.parked.values().find(|p| p.bound_to == hash).map(|p| p.proposal.transition.clone());
2887 if let Some(job) = t.and_then(|t| self.ex.prebuild_job(&t)) {
2888 let req = Request::Approve { hash };
2889 self.deferred = Some((job, Resume { peer, kind: ResumeKind::Request(req) }));
2890 return Response::Ok { message: "deferred".into() };
2891 }
2892 }
2893 let (s, defs) = self.observe();
2894 let approver = self.authz(&peer);
2895 let parked = self.ex.parked.values().find(|p| p.bound_to == hash).cloned();
2896 let (pre, policy) = (s.digest(), self.ex.policy_version());
2897 let built = self.waiting_built();
2898 let r = self.ex.approve(&hash, &approver, &s, &defs);
2899 self.note_withdrawn(built);
2900 if let (Some(pk), Submitted::Parked { txn: again, .. }) = (&parked, &r)
2903 && *again != pk.txn
2904 && let Some(agent) = self.parked_agents.remove(&pk.txn)
2905 {
2906 self.parked_agents.insert(*again, agent);
2907 }
2908 if let (Some(pk), Submitted::Finished { txn, .. }) = (parked, &r) {
2911 let l = self.ledger();
2912 let agent = self.parked_agents.remove(&pk.txn).flatten();
2914 let rec = if *txn == pk.txn {
2915 LogRecord {
2916 at: self.b.clock().now(),
2917 no_snapshot: no_snapshot(&r, Some(&pk.facts)),
2918 approver: Some(approver.actor.clone()),
2919 bound_to: Some(hash),
2920 proposal: pk.proposal,
2921 caller: pk.authz.actor,
2922 pre_digest: pre,
2923 policy,
2924 result: r.clone(),
2925 facts: Some(pk.facts),
2926 decision: Some(Decision::Allow { tier: Tier::AskFirst, approver: pk.class }),
2927 post_digest: None,
2928 path: l.path(*txn).map(<[String]>::to_vec).unwrap_or_default(),
2929 why: l.terminal.get(txn).map(|x| x.why.clone()).unwrap_or_default(),
2930 agent,
2931 }
2932 } else {
2933 let (facts, decision) = self.ex.last.clone().unzip();
2938 let mut proposal = pk.proposal;
2939 proposal.observed = pre;
2940 LogRecord {
2941 at: self.b.clock().now(),
2942 no_snapshot: no_snapshot(&r, facts.as_ref()),
2943 approver: None,
2944 bound_to: None,
2945 proposal,
2946 caller: pk.authz.actor,
2947 pre_digest: pre,
2948 policy,
2949 result: r.clone(),
2950 facts,
2951 decision,
2952 post_digest: None,
2953 path: l.path(*txn).map(<[String]>::to_vec).unwrap_or_default(),
2954 why: l.terminal.get(txn).map(|x| x.why.clone()).unwrap_or_default(),
2955 agent,
2956 }
2957 };
2958 self.awaiting_post.push(rec);
2959 }
2960 self.respond(&r)
2961 }
2962 Request::Cancel { txn } => {
2963 let Some(p) = self.ex.parked.get(&txn) else {
2964 return Response::Error { message: "no such pending change".into() };
2965 };
2966 if !peer.admin && p.authz.actor.uid() != Some(peer.uid) {
2967 return Response::Error {
2968 message: "only whoever asked for it, or an administrator, can withdraw it".into(),
2969 };
2970 }
2971 let r = self.ex.cancel(txn, Actor::user(peer.uid));
2972 self.respond(&r)
2973 }
2974 Request::Resolve { txn, resolution } => self.user_submit(
2975 Transition::Resolve { txn, resolution },
2976 peer,
2977 Source::User,
2978 "planarian resolve".into(),
2979 ),
2980 Request::MarkSecurityFloor => {
2981 if !peer.admin {
2982 return Response::Error { message: "mark-security-floor requires administrator authority".into() };
2983 }
2984 let (s, _) = self.observe();
2985 let l = self.ledger();
2986 let g = s.generation;
2987 if l.known_good != Some(g) && l.live_tested != Some(g) {
2990 return Response::Error {
2991 message: format!("generation {} has not committed live or been boot-confirmed", g.0),
2992 };
2993 }
2994 if l.security_floor.is_some_and(|f| f.0 >= g.0) {
2995 return Response::Error {
2996 message: format!(
2997 "the security floor is already generation {}",
2998 l.security_floor.unwrap_or(g).0
2999 ),
3000 };
3001 }
3002 self.journal(Record::SecurityFloor(Some(g)));
3003 Response::Ok {
3004 message: format!(
3005 "generation {} is now the security floor; going back below it needs an administrator",
3006 g.0
3007 ),
3008 }
3009 }
3010 Request::MarkGood { accept_unconfirmed } => {
3011 if peer.uid != 0 || !peer.tty {
3014 return Response::Error {
3015 message: "mark-good requires root at an interactive terminal (Phase A)".into(),
3016 };
3017 }
3018 let (s, defs) = self.observe();
3019 let failing: Vec<String> = defs
3020 .keys()
3021 .filter(|u| {
3022 s.expected(u, &defs)
3023 && matches!(
3024 s.status(u, &defs),
3025 UnitStatus::Degraded(_) | UnitStatus::Failed(_) | UnitStatus::Unknown
3026 )
3027 })
3028 .map(|u| u.to_string())
3029 .collect();
3030 if !failing.is_empty() {
3031 return Response::Error {
3032 message: format!("refused: Readiness probes are failing on {}", failing.join(", ")),
3033 };
3034 }
3035 let booted = self.b.generations().last_booted().ok();
3036 if booted == Some(s.generation) {
3037 self.journal(Record::KnownGood(s.generation));
3038 Response::Ok { message: format!("generation {} is now known-good", s.generation.0) }
3039 } else if accept_unconfirmed {
3040 self.journal(Record::AcceptedBaseline(Some(s.generation)));
3041 Response::Ok {
3042 message: format!(
3043 "generation {} accepted as the recovery baseline; it is not boot-confirmed and does not replace the known-good boot entry",
3044 s.generation.0
3045 ),
3046 }
3047 } else {
3048 Response::Error {
3049 message:
3050 "this generation has not been booted; reboot into it first, or use --accept-unconfirmed"
3051 .into(),
3052 }
3053 }
3054 }
3055 Request::Console => Response::Console(Box::new(self.console())),
3056 Request::ReportBoot => match &self.last_boot_report {
3058 Some((at, g, text)) => Response::Ok {
3059 message: format!("boot report for generation {} at {}: {text}", g.0, render::when(*at)),
3060 },
3061 None => Response::Ok {
3062 message: "no boot report since planariad started (it runs once, on the first start after boot)"
3063 .into(),
3064 },
3065 },
3066 Request::Posture => {
3067 let d = &self.cfg.posture_decl;
3068 let mut message = format!(
3069 "posture: {:?}, boot-ready timeout {} s (this generation declares {:?}, {} s)",
3070 self.cfg.posture,
3071 d.timeout_in_force(),
3072 d.declared,
3073 d.declared_timeout_secs
3074 );
3075 if d.weaker_waiting() {
3076 message.push_str("; the weaker declaration takes effect at the next boot");
3077 }
3078 Response::Ok { message }
3079 }
3080 #[cfg(feature = "gym-hooks")]
3081 Request::GymPropose { transition, stale_digest, may_change } => {
3082 let (s, defs) = self.observe();
3083 let mut p = self.proposal(transition, &s, &defs, Source::User, "gym".into());
3084 if stale_digest {
3085 p.observed = Hash::ZERO;
3086 }
3087 if let Some(m) = may_change {
3088 p.expected.may_change = m.into_iter().collect();
3089 }
3090 let authz = self.authz(&peer);
3091 let r = self.submit(p, authz, &s, &defs);
3092 self.respond(&r)
3093 }
3094 #[cfg(feature = "gym-hooks")]
3095 Request::GymCompact => match self.compact_saving() {
3096 Ok(entries) => Response::Compacted {
3097 entries: entries.iter().filter_map(|e| serde_json::to_value(e).ok()).collect(),
3098 },
3099 Err(e) => Response::Error { message: format!("{e:?}") },
3100 },
3101 #[cfg(not(feature = "gym-hooks"))]
3102 Request::GymPropose { .. } | Request::GymCompact => {
3103 Response::Error { message: "not available in this build (gym-hooks)".into() }
3104 }
3105 Request::Diagnose { unit, probe } => {
3106 let (_, defs) = self.observe();
3107 let Some(def) = defs.get(&unit) else {
3108 return Response::Error { message: format!("no unit {unit}") };
3109 };
3110 let mine = matches!(unit.scope, Scope::User { uid, .. } if uid == peer.uid);
3113 let controllable = self.ex.cfg.settings.user_controllable_units.contains(&unit.name);
3114 if !(peer.admin || mine || controllable) {
3115 return Response::Rejected {
3116 reason: "Unauthorized".into(),
3117 message: format!("you may not probe {unit}"),
3118 };
3119 }
3120 let probes: Vec<ProbeId> = match probe {
3121 Some(p) => vec![p],
3122 None => def.probes.iter().map(|p| p.id.clone()).collect(),
3123 };
3124 for p in &probes {
3125 let audible = def
3126 .probes
3127 .iter()
3128 .any(|d| &d.id == p && matches!(d.side_effects, SideEffects::Audible | SideEffects::Visible));
3129 let seated = peer.session.as_ref().is_some_and(|id| {
3131 self.asm
3132 .sessions
3133 .iter()
3134 .any(|x| &x.id == id && x.uid == peer.uid && x.seat.is_some() && !x.remote)
3135 });
3136 if audible && !seated {
3137 return Response::Error {
3138 message: format!("{} makes sound or light; run it from your own session", p.0),
3139 };
3140 }
3141 self.asm.request(&unit, p);
3142 }
3143 Response::Ok { message: format!("running {} probe(s) on {unit} at the next tick", probes.len()) }
3144 }
3145 }
3146 }
3147}
3148
3149#[cfg(test)]
3150mod tests;
3151
3152fn signal_name(e: Ending) -> String {
3153 match e {
3154 Ending::Signal(1) => "SIGHUP".into(),
3155 Ending::Signal(2) => "SIGINT".into(),
3156 Ending::Signal(9) => "SIGKILL".into(),
3157 Ending::Signal(15) => "SIGTERM, as `systemctl stop` sends".into(),
3158 Ending::Signal(n) => format!("signal {n}"),
3159 Ending::Exited(n) => format!("exit status {n}"),
3160 Ending::OomKill => "the OOM killer".into(),
3161 }
3162}
3163
3164fn foreground_depends_on(u: &UnitId, defs: &UnitDefs) -> bool {
3169 matches!(u.scope, Scope::User { .. })
3170 || u.kind == UnitKind::Device
3171 || u.name.contains("bluetooth")
3172 || defs
3173 .get(u)
3174 .is_some_and(|d| matches!(d.restart_disruption, Disruption::SessionLoss | Disruption::ConnectivityLoss))
3175}
3176
3177fn urgent(s: &str) -> bool {
3180 s.starts_with("ALARM")
3181 || s.starts_with("INTEGRITY")
3182 || s.contains(" → ")
3183 || s.contains("needs a person")
3184 || s.contains("needs approval")
3185}