Skip to main content

hopr_transport_session/supervision/
supervisor.rs

1//! Deterministic [`SessionPixSupervisor`] — the pure state machine for PIX
2//! session lifecycle.
3//!
4//! All methods take explicit [`std::time::Instant`] timestamps and a
5//! `served_total: u64` sample from the [`ServiceGate`](super::gate::ServiceGate).
6//! No method sleeps, spawns, or performs I/O.
7
8use std::time::{Duration, Instant};
9
10use hopr_api::{HoprBalance, OffchainPublicKey, types::internal::prelude::HoprPseudonym};
11use hopr_protocol_pix::{SsaId, SsaIndex, SsaRecoveryProgress};
12
13use super::{
14    PixParams, SessionPixAction, SessionPixCloseReason, SessionPixEvent, SupervisorConfig,
15    fill::{FillPlanner, FillTarget},
16    telemetry::{PixCycleOutcome, PixCyclePhase, PixCycleSummary, PixGateMode, PixSessionSnapshot, PixTurnEvents},
17};
18
19// ---------------------------------------------------------------------------
20// SsaPhase
21// ---------------------------------------------------------------------------
22
23#[derive(Debug, Clone, Copy, PartialEq, Eq)]
24enum SsaPhase {
25    /// Request has been sent; waiting for complete verifiable commitment.
26    AwaitingCommitment,
27    /// Commitment is verifiable; waiting for a sufficient deposit.
28    AwaitingDeposit,
29    /// Deposit confirmed; recovering shares.
30    Recovering,
31    /// SSA fully recovered; diagnostic tombstone until `tombstone_until`.
32    /// Any paid FIFO remainder lives separately in [`PaidRecoveryTail`].
33    Recovered { tombstone_until: Instant },
34    /// Phase that will produce a close action on next deadline check.
35    Closing,
36}
37
38/// Wei in one µHOPR, the unit `hopr_pix_deposits_*_uhopr_total` counts.
39const WEI_PER_UHOPR: u128 = 1_000_000_000_000;
40
41/// A deposit in µHOPR, saturating rather than wrapping.
42///
43/// Integer because the counters it feeds are, and µHOPR because wei overflows a `u64` at 18 HOPR
44/// while whole HOPR would round every realistic deposit to nothing.
45fn deposit_uhopr(amount: HoprBalance) -> u64 {
46    let wei = amount.amount();
47    let wei = if wei.bits() > 128 { u128::MAX } else { wei.low_u128() };
48    (wei / WEI_PER_UHOPR).min(u64::MAX as u128) as u64
49}
50
51// ---------------------------------------------------------------------------
52// PerSsaState
53// ---------------------------------------------------------------------------
54
55/// Internal state for one supervised SSA.
56struct PerSsaState {
57    ssa_id: SsaId<HoprPseudonym>,
58    /// First SSA index allocated in this request batch.
59    batch_id: u32,
60    phase: SsaPhase,
61
62    // Deadlines (None means not set for this phase).
63    commitment_deadline: Option<Instant>,
64    /// When to re-request the parts of this commitment that have not arrived, or `None` while there
65    /// is nothing to ask for.
66    ///
67    /// Set only once *some* of the commitment has arrived, and pushed out by every further part, so
68    /// it measures silence in the delivery rather than elapsed time. A cycle for which nothing was
69    /// ever delivered leaves it unset and dies on `commitment_deadline`, which is all the Exit can
70    /// do about one: it has no scope to name.
71    recommit_deadline: Option<Instant>,
72    /// Re-requests already made for this cycle, capped at
73    /// [`MAX_COMMITMENT_RETRANSMISSIONS`](hopr_protocol_pix::MAX_COMMITMENT_RETRANSMISSIONS).
74    ///
75    /// The cap is a count and not a duration on purpose. `commitment_deadline` bounds the asks in
76    /// *time*, which is what protects the Entry, but the packets they cost are this node's own: at a
77    /// misconfigured millisecond interval inside a 20-second deadline the time bound alone permits
78    /// twenty thousand of them. Matching the figure the Entry will answer means none of the asks is
79    /// wasted either.
80    recommit_attempts: u8,
81    deposit_deadline: Option<Instant>,
82    recovery_idle_deadline: Option<Instant>,
83    recovery_hard_deadline: Option<Instant>,
84
85    /// What the Entry deposited for this cycle, in µHOPR, or zero until the deposit confirms.
86    deposit_uhopr: u64,
87    /// When [`phase`](Self::phase) was entered, for `hopr_pix_cycle_phase_seconds`.
88    phase_entered_at: Instant,
89
90    // Progress tracking.
91    largest_useful_shares: u64,
92    /// Largest `shares_seen` observed for this cycle — the liveness counter, which moves for surplus
93    /// shares too. Also the staleness discriminator for snapshots, since it moves for every share
94    /// while `largest_useful_shares` does not.
95    largest_shares_seen: u64,
96    target_useful_shares: u64,
97    recovered_polynomials: u16,
98
99    // Overlap / deferred-request state.
100    next_request_pending_deposit: bool,
101    next_requested: bool,
102    /// Set on the last cycle allocated by [`SessionPixSupervisor::emit_request_next_ssa`], and only
103    /// on that one.
104    ///
105    /// The successor gate: a batch asks for its replacement once, when its *last* cycle is nearly
106    /// recovered. Every other flag here is per-cycle, so without this one each member of a batch
107    /// would answer its own `AlmostRecovered` with a whole batch of its own — `ssas_per_request`
108    /// batches per batch, compounding, each cycle a separate on-chain deposit for the Entry.
109    ///
110    /// The last cycle rather than the first because the batch is served in index order: gating on
111    /// the first would ask for the successor a whole batch too early, which is the exposure
112    /// `ssas_per_request` deliberately bounds.
113    ///
114    /// If the two-generation admission bound defers that request, `next_requested` remains set and
115    /// the supervisor-level `successor_request_deferred` flag owns the retry; the gate must not be
116    /// handed to another cycle and spend the request twice.
117    is_batch_last: bool,
118    /// Set when `Recovered` arrives before deposit confirms (i.e. during
119    /// `AwaitingCommitment` or `AwaitingDeposit`). Once deposit is confirmed
120    /// and the SSA enters `Recovering`, this flag triggers an immediate
121    /// tombstone transition — replaying the deferred `Recovered` event.
122    recovered_pending: bool,
123
124    // Service gating.
125    served_total_at_last_progress: u64,
126    /// The Session's served count when this cycle reached the front of the batch, or `None` while
127    /// it is still queued behind one.
128    ///
129    /// Subtracted from the served count at finalization to give the cycle's own egress. Baselined
130    /// at the front rather than at the request, because a queued cycle is not being served: the
131    /// Entry's emission window is clamped to one cycle, so charging a batch member for everything
132    /// its predecessors consumed would report the last cycle of a batch as having cost the whole
133    /// batch. `None` on a cycle that never reached the front is therefore the honest answer, and it
134    /// is reported as zero egress rather than as the Session's whole history.
135    served_total_at_front: Option<u64>,
136}
137
138impl PerSsaState {
139    fn new(ssa_id: SsaId<HoprPseudonym>, batch_id: u32, target_useful_shares: u64, now: Instant) -> Self {
140        Self {
141            ssa_id,
142            batch_id,
143            phase: SsaPhase::AwaitingCommitment,
144            phase_entered_at: now,
145            commitment_deadline: None,
146            recommit_deadline: None,
147            recommit_attempts: 0,
148            deposit_uhopr: 0,
149            deposit_deadline: None,
150            recovery_idle_deadline: None,
151            recovery_hard_deadline: None,
152            largest_useful_shares: 0,
153            largest_shares_seen: 0,
154            target_useful_shares,
155            recovered_polynomials: 0,
156            next_request_pending_deposit: false,
157            next_requested: false,
158            is_batch_last: false,
159            recovered_pending: false,
160            served_total_at_last_progress: 0,
161            served_total_at_front: None,
162        }
163    }
164
165    /// Moves to `phase`, returning how long the outgoing one lasted in milliseconds.
166    ///
167    /// `None` for an outgoing `Recovered` or `Closing`, which have no [`PixCyclePhase`] label — both
168    /// are retained state rather than live state, so neither is measured.
169    fn enter_phase(&mut self, phase: SsaPhase, now: Instant) -> Option<(PixCyclePhase, u64)> {
170        let left = match self.phase {
171            SsaPhase::AwaitingCommitment => Some(PixCyclePhase::AwaitingCommitment),
172            SsaPhase::AwaitingDeposit => Some(PixCyclePhase::AwaitingDeposit),
173            SsaPhase::Recovering => Some(PixCyclePhase::Recovering),
174            SsaPhase::Recovered { .. } | SsaPhase::Closing => None,
175        };
176        let elapsed = now.saturating_duration_since(self.phase_entered_at).as_millis() as u64;
177        self.phase = phase;
178        self.phase_entered_at = now;
179        left.map(|phase| (phase, elapsed))
180    }
181
182    /// This cycle's coverage, for the histograms observed once when it finalizes.
183    fn telemetry_summary(&self, outcome: PixCycleOutcome, served_total: u64) -> PixCycleSummary {
184        PixCycleSummary {
185            outcome,
186            // Saturating rather than assuming monotonicity: the baseline is taken from a
187            // `served_total` sample passed in by the worker, and a cycle retired on a deadline
188            // sweep can be summarized from an older sample than the one that baselined it.
189            egress_packets: self
190                .served_total_at_front
191                .map(|base| served_total.saturating_sub(base))
192                .unwrap_or(0),
193            accepted_shares: self.largest_shares_seen,
194            useful_shares: self.largest_useful_shares,
195            target_useful_shares: self.target_useful_shares,
196            deposit_uhopr: self.deposit_uhopr,
197        }
198    }
199
200    /// True if this SSA is past the recovery phase.
201    fn is_terminal(&self) -> bool {
202        matches!(self.phase, SsaPhase::Recovered { .. } | SsaPhase::Closing)
203    }
204}
205
206/// Bounded handoff for SURBs minted against the cycle that has just recovered.
207///
208/// Cryptographic recovery and transport-buffer exhaustion are different instants. The former frees
209/// the reconstructor's heavy state; this small receipt keeps the paid service front on the recovered
210/// predecessor until either its negotiated tail is exhausted or any progress proves the successor
211/// has reached the head of the FIFO. The latter need not be funded: observing an unfunded successor
212/// is precisely when the predecessor's receipt must stop authorizing service.
213struct PaidRecoveryTail {
214    ssa_id: SsaId<HoprPseudonym>,
215    /// When the tail took the paid front, for `hopr_pix_cycle_phase_seconds{phase="paid_tail"}`.
216    entered_at: Instant,
217    largest_shares_seen: u64,
218    served_total_at_last_progress: u64,
219    idle_deadline: Option<Instant>,
220    hard_deadline: Option<Instant>,
221}
222
223// ---------------------------------------------------------------------------
224// SessionPixSupervisor
225// ---------------------------------------------------------------------------
226
227/// Deterministic core of the PIX session supervisor.
228pub struct SessionPixSupervisor {
229    pub(crate) cfg: SupervisorConfig,
230    pub(crate) dims: PixParams,
231    pub(crate) pseudonym: HoprPseudonym,
232    pub(crate) closed: bool,
233    /// The Session is gone and only its funded front cycle is still being worked for.
234    ///
235    /// Set by [`on_session_closed`](Self::on_session_closed) and never cleared: a drain ends by
236    /// closing, one way or another. While it is set the supervisor asks for no successor, plans fill
237    /// at the ceiling, and stops service-gating the recovery idle deadline — see "Draining a closed
238    /// Session" in this module's documentation for why each of those follows from the Session having
239    /// left rather than being separate policy.
240    pub(crate) draining: bool,
241    next_ssa_index: u32,
242    /// The cycle the two share-order counters below are measured against — the earliest unrecovered
243    /// cycle, i.e. the one the Entry should currently be serving. `None` before the first batch exists.
244    share_order_front: Option<SsaIndex>,
245    /// Useful shares booked on [`Self::share_order_front`] since it took the front.
246    front_useful: u64,
247    /// Useful shares booked on any *other* live cycle over that same span.
248    ///
249    /// `off_front_useful / (front_useful + off_front_useful)` is the share-order ratio — see
250    /// [`SupervisorConfig::max_off_front_share_fraction`] for what it detects and why it is a fraction
251    /// rather than a count. Both counters reset when the front changes, which is what keeps the window
252    /// bounded to a single cycle's service.
253    off_front_useful: u64,
254    /// Whether the service gate currently follows a funded front cycle.
255    service_open: bool,
256    /// Front cycle for which the current gate mode was emitted.
257    ///
258    /// Tracking the identity as well as the mode lets a funded-to-funded handoff rebaseline the
259    /// served-without-progress ceiling for the newly promoted cycle.
260    service_front: Option<SsaIndex>,
261    /// A front cycle became paid and terminal in one event, so an unfunded successor still earns a
262    /// freshly restored predeposit allowance even though the gate never entered funded mode.
263    paid_front_handoff: bool,
264    /// The sole recovered predecessor authorized to report buffered liveness.
265    ///
266    /// This never becomes a set: accepting arbitrary recovered tombstones would let the Entry replay
267    /// old cycle tags to keep service open. Replacement on each in-order recovery is the supervisor's
268    /// independent bound behind the reconstructor's per-polynomial surplus counters.
269    paid_recovery_tail: Option<PaidRecoveryTail>,
270    /// A successor batch was earned but could not yet fit the two-generation reservation.
271    successor_request_deferred: bool,
272    /// Ordered SSAs (oldest first, newest last), including short-lived recovered tombstones.
273    ssas: Vec<PerSsaState>,
274    /// Tracks the first failure reason when multiple SSAs fail, so the
275    /// earliest cause is used for the final `Close` action rather than the last.
276    first_failure_reason: Option<SessionPixCloseReason>,
277    /// Cycles this Session has lost without recovering them, over its whole life.
278    ///
279    /// Cumulative, and that is the whole point: retiring a failed member leaves no trace on its
280    /// siblings, so without a counter that survives the retirement an Entry can lose one cycle per
281    /// batch indefinitely while a single funded sibling holds the Session open. Bounded by
282    /// [`SupervisorConfig::max_failed_cycles`].
283    failed_cycles: usize,
284    /// Plans the rate at which this Exit originates its own fill keep-alives.
285    ///
286    /// Kept here rather than in the worker because the rate is a function of the lifecycle state
287    /// this struct owns — which cycle is at the paid front and how far it has got — and because
288    /// every close path has to be able to stop it in the same action vector that carries the close.
289    fill: FillPlanner,
290    /// Greatest SSA index that has been retired (closed and removed).
291    ///
292    /// Prevents a stale `SsaRequestSent` from resurrecting a closed SSA. A high-watermark rather
293    /// than the set of every retired index: the set grew for the whole life of the Session and was
294    /// scanned linearly on every request, one entry per cycle, without bound.
295    ///
296    /// A watermark alone would be wrong if it were consulted first — retirement is not monotone,
297    /// since a later batch member can lose its deposit while an earlier one is still recovering, and
298    /// "at or below the greatest retired index" would then condemn that live earlier cycle. The
299    /// guard is therefore placed *after* the live-record lookup, so a cycle that still exists always
300    /// wins and the watermark only ever answers for indices that have none.
301    highest_retired_ssa_index: Option<SsaIndex>,
302    /// Cycle lifecycle transitions since the worker last drained this.
303    ///
304    /// Latched rather than emitted because this type is a pure function of its inputs and a metric
305    /// is not — the same arrangement as [`FillPlanner`]'s stall, which is latched here and drained
306    /// in the worker. Always compiled, including without the `telemetry` feature, so the default
307    /// test configuration exercises the counting rather than only the feature-enabled one.
308    telemetry: PixTurnEvents,
309}
310
311impl SessionPixSupervisor {
312    /// Create a new supervisor and emit the first `RequestSsa` action.
313    pub fn new(
314        cfg: SupervisorConfig,
315        dims: PixParams,
316        pseudonym: HoprPseudonym,
317        now: Instant,
318    ) -> (Self, Vec<SessionPixAction>) {
319        let mut s = Self {
320            fill: FillPlanner::new(&cfg, &dims, now),
321            cfg,
322            dims,
323            pseudonym,
324            next_ssa_index: 1,
325            share_order_front: None,
326            front_useful: 0,
327            off_front_useful: 0,
328            closed: false,
329            draining: false,
330            service_open: false,
331            service_front: None,
332            paid_front_handoff: false,
333            paid_recovery_tail: None,
334            successor_request_deferred: false,
335            ssas: Vec::with_capacity(2),
336            first_failure_reason: None,
337            failed_cycles: 0,
338            highest_retired_ssa_index: None,
339            telemetry: PixTurnEvents::default(),
340        };
341
342        let actions = s.emit_request_next_ssa(now);
343        s.refresh_share_order_front();
344        (s, actions)
345    }
346
347    /// Handle a lifecycle event.
348    pub fn handle_event(&mut self, ev: &SessionPixEvent, now: Instant, served_total: u64) -> Vec<SessionPixAction> {
349        if self.closed {
350            return Vec::new();
351        }
352
353        let actions = match ev {
354            SessionPixEvent::SsaRequestSent(ssa_id) => self.on_ssa_request_sent(ssa_id, now),
355            SessionPixEvent::CommitmentProgress(ssa_id) => self.on_commitment_progress(ssa_id, now),
356            SessionPixEvent::CommitmentVerified(ssa_id) => self.on_commitment_verified(ssa_id, now),
357            SessionPixEvent::DepositConfirmed { ssa_id, amount } => {
358                self.on_deposit_confirmed(ssa_id, *amount, now, served_total)
359            }
360            SessionPixEvent::DepositObserverClosed(ssa_id) => {
361                self.on_deposit_observer_closed(ssa_id, now, served_total)
362            }
363            SessionPixEvent::RecoveryProgress(progress) => self.on_recovery_progress(progress, now, served_total),
364            SessionPixEvent::AlmostRecovered(ssa_id) => self.on_almost_recovered(ssa_id, now),
365            SessionPixEvent::Recovered(ssa_id) => self.on_recovered(ssa_id, now, served_total),
366            SessionPixEvent::UnverifiableShares {
367                ssa_id,
368                observed_total,
369                peer,
370            } => self.on_unverifiable_shares(ssa_id, *observed_total, peer, now),
371            SessionPixEvent::SessionClosed { drainable_surbs } => self.on_session_closed(*drainable_surbs, now),
372        };
373
374        let mut lifecycle_actions = actions;
375        lifecycle_actions.extend(self.retry_deferred_successor_request(now));
376        self.consume_paid_tail_at_observed_boundary(now);
377        self.arm_recovery_clocks_for_earliest(now, served_total);
378        let mut actions = if lifecycle_actions
379            .iter()
380            .any(|action| matches!(action, SessionPixAction::Close(_)))
381        {
382            // A closing Session has no cycle left to fill for, and the fill stream outlives this
383            // state machine by however long it takes the driver to tear the Session down. Silencing
384            // it here rather than only there is what keeps a close from being followed by a burst of
385            // keep-alives for a cycle that no longer exists.
386            self.fill
387                .stop()
388                .map(SessionPixAction::SetFillRate)
389                .into_iter()
390                .collect()
391        } else {
392            self.sync_service_gate()
393        };
394        actions.extend(lifecycle_actions);
395        self.refresh_share_order_front();
396        actions
397    }
398
399    /// Raises the retired-index watermark.
400    ///
401    /// Takes the maximum rather than assuming the caller retires in order, because it does not: a
402    /// later batch member losing its deposit is retired before an earlier one that is still
403    /// recovering.
404    fn note_retired_index(&mut self, index: SsaIndex) {
405        self.highest_retired_ssa_index = Some(match self.highest_retired_ssa_index {
406            Some(highest) if highest >= index => highest,
407            _ => index,
408        });
409    }
410
411    /// Index of the earliest cycle that has not finished recovering — the one the Entry should be
412    /// serving, given that emission is clamped to a single cycle.
413    fn earliest_live_idx(&self) -> Option<usize> {
414        self.ssas
415            .iter()
416            .enumerate()
417            .filter(|(_, s)| !s.is_terminal())
418            .min_by_key(|(_, s)| s.ssa_id.ssa_index())
419            .map(|(i, _)| i)
420    }
421
422    /// Consumes the predecessor receipt once the current live front has reported any share.
423    ///
424    /// This check is centralized rather than tied only to `RecoveryProgress`: a previously queued
425    /// cycle can become the front because a sibling is retired or because its deposit arrives after
426    /// its first snapshot. In every case the observation proves the transport crossed the cycle
427    /// boundary. Keeping the old funded receipt beyond it would let an unfunded successor spend the
428    /// predecessor's remaining allowance.
429    /// Retires the paid tail, latching how long it held the front.
430    fn release_paid_tail(&mut self, now: Instant) {
431        if let Some(tail) = self.paid_recovery_tail.take() {
432            let elapsed = now.saturating_duration_since(tail.entered_at).as_millis() as u64;
433            self.telemetry.phase_durations.push((PixCyclePhase::PaidTail, elapsed));
434        }
435    }
436
437    fn consume_paid_tail_at_observed_boundary(&mut self, now: Instant) {
438        let Some(front_idx) = self.earliest_live_idx() else {
439            return;
440        };
441        let front_id = self.ssas[front_idx].ssa_id;
442        if self.ssas[front_idx].largest_shares_seen == 0 {
443            return;
444        }
445
446        if let Some(predecessor) = self
447            .paid_recovery_tail
448            .as_ref()
449            .map(|tail| tail.ssa_id)
450            .filter(|predecessor| predecessor.ssa_index() < front_id.ssa_index())
451        {
452            tracing::debug!(
453                %predecessor,
454                successor = %front_id,
455                "successor progress consumed the paid FIFO-tail handoff"
456            );
457            self.release_paid_tail(now);
458        }
459    }
460
461    /// Keeps the service gate aligned with the funding state of the earliest live cycle.
462    ///
463    /// A funded-to-unfunded handoff restores the configured predeposit allowance. A
464    /// funded-to-funded handoff emits `ReleaseService` too: the mode does not change, but the new
465    /// front must receive its own served-without-progress ceiling rather than inheriting the tail of
466    /// its predecessor's. An unfunded cycle that fails does not earn another allowance.
467    fn sync_service_gate(&mut self) -> Vec<SessionPixAction> {
468        if self.closed {
469            return Vec::new();
470        }
471
472        let front = self.earliest_live_idx();
473        let front_id = self
474            .paid_recovery_tail
475            .as_ref()
476            .map(|tail| tail.ssa_id.ssa_index())
477            .or_else(|| front.map(|idx| self.ssas[idx].ssa_id.ssa_index()));
478        let front_changed = front_id != self.service_front;
479        let front_funded =
480            self.paid_recovery_tail.is_some() || front.is_some_and(|idx| self.ssas[idx].phase == SsaPhase::Recovering);
481        let paid_front_handoff = std::mem::take(&mut self.paid_front_handoff);
482        self.service_front = front_id;
483
484        match (front_funded, self.service_open, front_changed) {
485            (true, false, _) | (true, true, true) => {
486                self.service_open = true;
487                tracing::debug!(?front_id, "aligning service gate with funded front cycle");
488                vec![SessionPixAction::ReleaseService]
489            }
490            (false, true, _) => {
491                self.service_open = false;
492                tracing::debug!(?front_id, "withholding service for unfunded front cycle");
493                vec![SessionPixAction::WithholdService]
494            }
495            (false, false, _) if paid_front_handoff => {
496                tracing::debug!(?front_id, "restoring predeposit service after paid front handoff");
497                vec![SessionPixAction::WithholdService]
498            }
499            _ => Vec::new(),
500        }
501    }
502
503    /// This Session's current contribution to the node-level live-set gauges.
504    ///
505    /// Recomputed from `ssas` and `paid_recovery_tail` rather than maintained incrementally, which
506    /// is what makes the gauges drain on every terminal path without any of those paths having to
507    /// know they exist. A Session holds at most `ssas_per_request × MAX_OVERLAPPING_BATCHES` cycles,
508    /// so the scan is over a handful of entries once per turn.
509    ///
510    /// `predeposit_exposure` comes from the gate, which the caller reads — this type deliberately
511    /// knows nothing about the gate's counters.
512    pub fn telemetry_snapshot(&self, predeposit_exposure: u64) -> PixSessionSnapshot {
513        let mut snapshot = PixSessionSnapshot {
514            // The mode the gate is actually in, which is what an operator reading this gauge next to
515            // `hopr_pix_predeposit_exposure_packets` is asking about. A paid recovered predecessor
516            // still holding the front counts as funded, because service continues to be authorized.
517            gate_mode: if self.service_open {
518                PixGateMode::Funded
519            } else {
520                PixGateMode::Predeposit
521            },
522            paid_tail: u32::from(self.paid_recovery_tail.is_some()),
523            predeposit_exposure_packets: predeposit_exposure,
524            stalled_share: u32::from(self.fill.is_stalled()),
525            ..Default::default()
526        };
527
528        for ssa in self.ssas.iter() {
529            // Recovered tombstones and `Closing` records are retained diagnostics, not live cycles.
530            // Counting them would leave `hopr_pix_cycles_active` unable to drain, since a tombstone
531            // outlives the cycle by `tombstone_retention_window`.
532            let bucket = match ssa.phase {
533                SsaPhase::AwaitingCommitment => &mut snapshot.awaiting_commitment,
534                SsaPhase::AwaitingDeposit => &mut snapshot.awaiting_deposit,
535                SsaPhase::Recovering => &mut snapshot.recovering,
536                SsaPhase::Recovered { .. } | SsaPhase::Closing => continue,
537            };
538            *bucket = bucket.saturating_add(1);
539        }
540
541        snapshot
542    }
543
544    /// Drains the cycle lifecycle transitions latched since the last call.
545    pub fn take_telemetry_events(&mut self) -> PixTurnEvents {
546        std::mem::take(&mut self.telemetry)
547    }
548
549    fn live_cycle_count(&self) -> usize {
550        self.ssas.iter().filter(|ssa| !ssa.is_terminal()).count()
551    }
552
553    fn live_batch_count(&self) -> usize {
554        let mut batches = Vec::with_capacity(crate::MAX_OVERLAPPING_BATCHES as usize);
555        for ssa in self.ssas.iter().filter(|ssa| !ssa.is_terminal()) {
556            if !batches.contains(&ssa.batch_id) {
557                batches.push(ssa.batch_id);
558            }
559        }
560        batches.len()
561    }
562
563    fn reserved_cycle_slots(&self) -> usize {
564        self.cfg
565            .ssas_per_request
566            .clamp(1, crate::MAX_SSA_BATCH_SIZE)
567            .saturating_mul(crate::MAX_OVERLAPPING_BATCHES as usize)
568    }
569
570    fn retry_deferred_successor_request(&mut self, now: Instant) -> Vec<SessionPixAction> {
571        if self.closed || !self.successor_request_deferred {
572            return Vec::new();
573        }
574
575        self.successor_request_deferred = false;
576        self.emit_request_next_ssa(now)
577    }
578
579    /// Re-points the share-order accounting at the current front of the batch, clearing it if the front
580    /// moved.
581    ///
582    /// Clearing on a change of front is what bounds the measurement window to one cycle's service, and
583    /// it is why an Entry cannot launder a spree by eventually finishing a cycle: the reset is only
584    /// reached by the front cycle *completing*, which is the thing the Exit wanted all along.
585    ///
586    /// Called after every event and every deadline sweep, so it covers both ways the front can move —
587    /// the front cycle recovering, or being retired.
588    fn refresh_share_order_front(&mut self) {
589        let front = self.earliest_live_idx().map(|i| self.ssas[i].ssa_id.ssa_index());
590        if front != self.share_order_front {
591            self.share_order_front = front;
592            self.front_useful = 0;
593            self.off_front_useful = 0;
594        }
595    }
596
597    /// Starts the recovery clocks of the earliest unrecovered cycle, if they are not running yet.
598    ///
599    /// A batch's cycles are served strictly in index order — the Entry's emission window is clamped to
600    /// one cycle (see [`hopr_protocol_pix::SHARE_EMISSION_WINDOW`]) — so a cycle behind the front of
601    /// the batch is *queued*, not stalled. Starting its clocks when its deposit confirmed, as this used
602    /// to, measured the queue wait rather than the recovery: at deployed dimensions one cycle takes
603    /// ~61 min of emission to exhaust, so every cycle after the first was retired by
604    /// `max_recovery_time` — or, worse, by `max_recovery_idle` inside a minute, since the idle gate
605    /// compares against *session-wide* service and a queued cycle sees plenty of that without any of it
606    /// being its own.
607    ///
608    /// So the clock starts when the cycle reaches the front, which is when it can actually make
609    /// progress, and [`SupervisorConfig::max_recovery_time`] then means what its name says: the ceiling
610    /// on recovering a single cycle. `served_total_at_last_progress` is re-baselined at the same moment
611    /// for the same reason — otherwise the idle gate would charge the queue wait against it.
612    ///
613    /// Called after every event and every deadline sweep, so it covers each way a cycle can reach the
614    /// front: its predecessor recovering, or its predecessor being retired.
615    fn arm_recovery_clocks_for_earliest(&mut self, now: Instant, served_total: u64) {
616        if self.closed {
617            return;
618        }
619        // The packets being served still carry the recovered predecessor's shares. Starting the
620        // successor now would charge that pipeline delay against both its idle and hard recovery
621        // budgets even though no successor share can possibly arrive yet.
622        if self.paid_recovery_tail.is_some() {
623            return;
624        }
625        let Some(idx) = self.earliest_live_idx() else {
626            return;
627        };
628
629        let ssa = &mut self.ssas[idx];
630        if ssa.phase != SsaPhase::Recovering || ssa.recovery_hard_deadline.is_some() {
631            return;
632        }
633
634        ssa.recovery_idle_deadline = now.checked_add(self.cfg.max_recovery_idle);
635        ssa.recovery_hard_deadline = now.checked_add(self.cfg.max_recovery_time);
636        ssa.served_total_at_last_progress = served_total;
637        // The same instant, for the same reason: this is when the cycle starts being served, so it
638        // is when its own egress begins. `get_or_insert` rather than a plain assignment because a
639        // cycle reaches the front once — the guard above returns early for one that already armed —
640        // but stating it here means a future caller cannot rebaseline a cycle mid-flight and make
641        // its recorded egress smaller than what it actually consumed.
642        ssa.served_total_at_front.get_or_insert(served_total);
643        tracing::debug!(ssa_id = %ssa.ssa_id, "recovery clocks started — cycle is at the front of the batch");
644    }
645
646    /// Check all deadlines and emit actions for any that have expired.
647    pub fn handle_deadline(&mut self, now: Instant, served_total: u64) -> Vec<SessionPixAction> {
648        if self.closed {
649            return Vec::new();
650        }
651
652        let mut actions = Vec::new();
653        let max_recovery_idle = self.cfg.max_recovery_idle;
654
655        // While the recovered predecessor still owns the service front, its original recovery
656        // clocks remain the resource bound. Progress refreshes only the idle clock; the hard clock
657        // is never extended. Thus a large but paid surplus may drain, while an Entry that declares
658        // one and then feeds neither tail nor successor cannot hold the Session forever.
659        if let Some(tail) = self.paid_recovery_tail.as_mut() {
660            let expired = if tail.hard_deadline.is_some_and(|deadline| now >= deadline) {
661                Some(SessionPixCloseReason::RecoveryDeadline)
662            } else if tail.idle_deadline.is_some_and(|deadline| now >= deadline) {
663                Some(SessionPixCloseReason::RecoveryIdle)
664            } else {
665                None
666            };
667
668            if let Some(reason) = expired {
669                if reason == SessionPixCloseReason::RecoveryIdle && served_total <= tail.served_total_at_last_progress {
670                    tail.idle_deadline = now.checked_add(max_recovery_idle);
671                } else {
672                    tracing::warn!(
673                        ssa_id = %tail.ssa_id,
674                        ?reason,
675                        largest_shares_seen = tail.largest_shares_seen,
676                        served_total_at_last_progress = tail.served_total_at_last_progress,
677                        "closing PIX Session while its paid recovery tail is stalled"
678                    );
679                    // The tail is ending here, not merely being abandoned with the Session, so it
680                    // owes its duration like every other phase that reaches a terminal edge.
681                    self.release_paid_tail(now);
682                    self.closed = true;
683                    return self.with_fill_stopped(vec![SessionPixAction::Close(reason)]);
684                }
685            }
686        }
687
688        let mut i = 0;
689        while i < self.ssas.len() {
690            if self.closed {
691                break;
692            }
693
694            let expired = {
695                let ssa = &self.ssas[i];
696                if ssa.is_terminal() {
697                    i += 1;
698                    continue;
699                }
700                ssa.check_deadlines(now)
701            };
702
703            // A stalled commitment is re-requested rather than given up on, and the timer re-arms so
704            // a repair that is itself lost is asked for again. This yields no close reason and is
705            // bounded by nothing of its own: the absolute `commitment_deadline` still ends the cycle,
706            // which is why it can repeat freely — and why it is skipped once that deadline has gone,
707            // rather than putting an ask on the wire for a Session about to close.
708            if expired.is_none()
709                && self.ssas[i].phase == SsaPhase::AwaitingCommitment
710                && self.ssas[i].recommit_deadline.is_some_and(|deadline| now >= deadline)
711            {
712                let ssa = &mut self.ssas[i];
713                ssa.recommit_attempts += 1;
714                // Disarmed rather than re-armed at the cap, so the worker stops waking for a timer
715                // that would produce nothing. Further arriving parts cannot re-arm it either.
716                ssa.recommit_deadline = (ssa.recommit_attempts < hopr_protocol_pix::MAX_COMMITMENT_RETRANSMISSIONS)
717                    .then(|| now.checked_add(self.cfg.commitment_recommit_interval))
718                    .flatten();
719                tracing::debug!(
720                    ssa_id = %ssa.ssa_id, attempt = ssa.recommit_attempts,
721                    "commitment delivery stalled — re-requesting the parts that did not arrive"
722                );
723                actions.push(SessionPixAction::RequestCommitmentRetransmission {
724                    ssa_id: ssa.ssa_id,
725                    params: self.dims,
726                });
727            }
728
729            if let Some(reason) = expired {
730                // Service-gated idle: if no service consumed since last progress,
731                // re-arm instead of closing.
732                //
733                // Not while draining, though. The re-arm asks "was any *gated* service consumed",
734                // and a drained Session has no application left to consume any — `served_total` is
735                // frozen for good, so the re-arm would fire for ever and the one bound on a drain
736                // admitted against an over-estimated buffer would be gone with it. A *productive*
737                // drain is unaffected: its own progress still refreshes `recovery_idle_deadline`
738                // through `on_recovery_progress`.
739                if reason == SessionPixCloseReason::RecoveryIdle
740                    && !self.draining
741                    && served_total <= self.ssas[i].served_total_at_last_progress
742                {
743                    self.ssas[i].recovery_idle_deadline = now.checked_add(max_recovery_idle);
744                    i += 1;
745                    continue;
746                }
747
748                actions.extend(self.close_ssa_and_collect(i, reason, served_total, now));
749                continue;
750            }
751            i += 1;
752        }
753
754        // If the session is already closing, skip tombstone retirement.
755        // Whole-session teardown retires everything via the retirement guard.
756        if self.closed {
757            return self.with_fill_stopped(actions);
758        }
759
760        // Remove tombstones that have expired and emit RetireSsa so the
761        // reconstructor and observer state is released mid-session.
762        let retired_ids: Vec<SsaId<_>> = self
763            .ssas
764            .iter()
765            .filter(|ssa| matches!(ssa.phase, SsaPhase::Recovered { tombstone_until } if now >= tombstone_until))
766            .map(|ssa| ssa.ssa_id)
767            .collect();
768        self.ssas
769            .retain(|ssa| !matches!(ssa.phase, SsaPhase::Recovered { tombstone_until } if now >= tombstone_until));
770        for id in retired_ids {
771            self.note_retired_index(id.ssa_index());
772            actions.push(SessionPixAction::RetireSsa(id));
773        }
774
775        // A terminal cycle may have released the older of two live batches. Keep retirement before
776        // the replacement request in the action stream so the reconstructor drops the old guard
777        // before allocating the successor.
778        actions.extend(self.retry_deferred_successor_request(now));
779        self.consume_paid_tail_at_observed_boundary(now);
780
781        // If no SSAs remain, close.
782        // Note: a successor request may still be in flight here (RequestSsa
783        // emitted, SsaRequestSent not yet observed) — intentionally not
784        // tracked. The tombstone retention window is the backstop; a request
785        // delayed beyond it (>30 s transport stall on an otherwise healthy
786        // session) is theoretical and does not warrant blocking normal
787        // tombstone expiry.
788        if self.ssas.is_empty() && !self.closed {
789            actions.push(SessionPixAction::Close(
790                self.first_failure_reason
791                    .unwrap_or(SessionPixCloseReason::NoSsaRemaining),
792            ));
793            self.closed = true;
794        }
795
796        // Retiring or tombstoning a cycle can promote the next one to the front of the batch — and if
797        // that one is already funded, promotion is the only moment left at which service can be
798        // released, since its deposit event is long past.
799        self.arm_recovery_clocks_for_earliest(now, served_total);
800        let mut gate_actions = self.sync_service_gate();
801        gate_actions.extend(actions);
802        self.refresh_share_order_front();
803
804        if self.closed {
805            return self.with_fill_stopped(gate_actions);
806        }
807        gate_actions
808    }
809
810    /// Runs the deadline sweep and, if one is due, the fill tick — in that order.
811    ///
812    /// The order is the point. The sweep is what arms a promoted cycle's recovery clocks, consumes a
813    /// spent FIFO tail and re-aligns the service gate, so running it first means the tick plans
814    /// against the lifecycle as it is *now* rather than as it was a second ago. A tick that ran first
815    /// would spend one whole sampling interval filling for a cycle that had already recovered.
816    ///
817    /// This is the entry point the worker uses; [`handle_deadline`](Self::handle_deadline) remains
818    /// separate because the deadline logic is meaningful — and tested — on its own.
819    pub fn handle_timers(&mut self, now: Instant, served_total: u64) -> Vec<SessionPixAction> {
820        let mut actions = self.handle_deadline(now, served_total);
821        if self.closed {
822            return actions;
823        }
824        if self.next_fill_tick().is_some_and(|tick| now >= tick) {
825            actions.extend(self.handle_fill_tick(now, served_total));
826        }
827        actions
828    }
829
830    /// Re-plans the fill rate, emitting an action only when the rate actually changes.
831    pub fn handle_fill_tick(&mut self, now: Instant, served_total: u64) -> Vec<SessionPixAction> {
832        if self.closed {
833            return Vec::new();
834        }
835        let target = self.fill_target();
836        self.fill
837            .plan(now, served_total, target)
838            .map(|rate| vec![SessionPixAction::SetFillRate(rate)])
839            .into_iter()
840            .flatten()
841            .collect()
842    }
843
844    /// Reports, once, that fill has fallen back because its target stopped progressing.
845    ///
846    /// Exposed for the worker, which is the layer that may have side effects; this state machine and
847    /// the planner behind it are pure, so neither can record the metric itself.
848    pub fn take_fill_stall(&mut self) -> bool {
849        self.fill.take_stall_onset()
850    }
851
852    /// When the fill planner next wants to run, or `None` if there is nothing for it to do.
853    ///
854    /// `None` while no cycle is being filled for *and* the stream is already silent, which is what
855    /// keeps an unfunded or non-PIX-idle Session from waking its worker once a second for nothing. A
856    /// running stream always yields a tick even without a target, because it is owed the zero that
857    /// stops it.
858    pub fn next_fill_tick(&self) -> Option<Instant> {
859        if self.closed || !self.fill.is_enabled() {
860            return None;
861        }
862        if self.fill_target().is_none() && self.fill.is_idle() {
863            return None;
864        }
865        Some(self.fill.next_tick())
866    }
867
868    /// The cycle the Exit should currently be filling for, if any.
869    ///
870    /// The recovered predecessor's FIFO tail wins whenever it exists, for two reasons that point the
871    /// same way: its SURBs sit ahead of the successor's in the Exit's own buffer, so packets sent now
872    /// carry *its* shares; and it holds the binding hard deadline, since the successor's clocks are
873    /// deliberately not armed until the tail is exhausted (see
874    /// [`arm_recovery_clocks_for_earliest`](Self::arm_recovery_clocks_for_earliest)).
875    ///
876    /// Otherwise it is the earliest live cycle, and only once it is both funded and clocked. An
877    /// unfunded cycle has nothing at stake to be filled for, and a funded one whose clocks have not
878    /// armed is queued behind a predecessor rather than being served — filling for it would spend
879    /// SURBs on shares the Entry cannot yet be emitting.
880    fn fill_target(&self) -> Option<FillTarget> {
881        if let Some(tail) = self.paid_recovery_tail.as_ref() {
882            return tail.hard_deadline.map(|hard_deadline| FillTarget {
883                ssa_id: tail.ssa_id,
884                largest_shares_seen: tail.largest_shares_seen,
885                hard_deadline,
886                drain: self.draining,
887            });
888        }
889
890        let ssa = &self.ssas[self.earliest_live_idx()?];
891        (ssa.phase == SsaPhase::Recovering)
892            .then_some(ssa)
893            .and_then(|ssa| ssa.recovery_hard_deadline.map(|hard_deadline| (ssa, hard_deadline)))
894            .map(|(ssa, hard_deadline)| FillTarget {
895                ssa_id: ssa.ssa_id,
896                largest_shares_seen: ssa.largest_shares_seen,
897                hard_deadline,
898                drain: self.draining,
899            })
900    }
901
902    /// Prepends the action that silences the fill stream, when there is one to silence.
903    ///
904    /// Returns `actions` untouched for a Session that never filled — a disabled one, or one that
905    /// closed before its first cycle funded — so the action stream of every non-filling Session is
906    /// exactly what it was before this existed.
907    fn with_fill_stopped(&mut self, actions: Vec<SessionPixAction>) -> Vec<SessionPixAction> {
908        match self.fill.stop() {
909            Some(rate) => {
910                let mut stopped = Vec::with_capacity(actions.len() + 1);
911                stopped.push(SessionPixAction::SetFillRate(rate));
912                stopped.extend(actions);
913                stopped
914            }
915            None => actions,
916        }
917    }
918
919    /// Returns the earliest deadline across all live SSAs, or `None`.
920    pub fn next_deadline(&self) -> Option<Instant> {
921        if self.closed || (self.ssas.is_empty() && self.paid_recovery_tail.is_none()) {
922            return None;
923        }
924
925        let ssa_deadline = self
926            .ssas
927            .iter()
928            .filter_map(|ssa| {
929                if ssa.is_terminal() {
930                    if let SsaPhase::Recovered { tombstone_until } = ssa.phase {
931                        return Some(tombstone_until);
932                    }
933                    return None;
934                }
935                // Return the earliest set deadline. The re-request timer belongs here like the rest:
936                // it is shorter than every other deadline of the phase it runs in, so leaving it out
937                // would mean the worker slept past it and only re-requested when something else woke
938                // it up.
939                ssa.commitment_deadline
940                    .into_iter()
941                    .chain(ssa.recommit_deadline)
942                    .chain(ssa.deposit_deadline)
943                    .chain(ssa.recovery_idle_deadline)
944                    .chain(ssa.recovery_hard_deadline)
945                    .min()
946            })
947            .min();
948        let tail_deadline = self
949            .paid_recovery_tail
950            .as_ref()
951            .and_then(|tail| tail.idle_deadline.into_iter().chain(tail.hard_deadline).min());
952
953        ssa_deadline.into_iter().chain(tail_deadline).min()
954    }
955
956    /// Feed back the result of executing an action.
957    pub fn action_result(&mut self, action: &SessionPixAction, ok: bool, _now: Instant) -> Vec<SessionPixAction> {
958        if self.closed {
959            return Vec::new();
960        }
961
962        match action {
963            SessionPixAction::RequestSsa { .. } if !ok => {
964                self.closed = true;
965                self.with_fill_stopped(vec![SessionPixAction::Close(
966                    SessionPixCloseReason::SupervisorUnavailable,
967                )])
968            }
969            SessionPixAction::Close(_) => {
970                self.closed = true;
971                Vec::new()
972            }
973            _ => Vec::new(),
974        }
975    }
976
977    // ------------------------------------------------------------------
978    // Internal event handlers
979    // ------------------------------------------------------------------
980
981    /// Arms the commitment deadline once the request this SSA was registered for has gone out.
982    ///
983    /// The record itself is created by [`emit_request_next_ssa`](Self::emit_request_next_ssa); this
984    /// only starts the clock, because the Entry cannot be late answering a request it has not been
985    /// sent yet.
986    fn on_ssa_request_sent(&mut self, ssa_id: &SsaId<HoprPseudonym>, now: Instant) -> Vec<SessionPixAction> {
987        // Validate pseudonym. Checked before the lookup: a foreign pseudonym never matches a record
988        // of ours, so looking first would answer a cross-session confusion with silence.
989        if ssa_id.pseudonym() != &self.pseudonym {
990            return vec![SessionPixAction::Close(SessionPixCloseReason::InvalidTransition)];
991        }
992
993        // The live-record lookup comes first, and the staleness guard only answers for indices that
994        // have no record. Retirement is not monotone — a later batch member can lose its deposit while
995        // an earlier one is still recovering — so a watermark consulted first would silently ignore a
996        // legitimate request for a cycle that is very much alive. Asking the records first removes the
997        // assumption instead of relying on it.
998        let Some(idx) = self.find_ssa_idx(ssa_id) else {
999            // No live record. Either this index was retired and the event is a straggler, which is
1000            // benign and ignored, or we were never asked for it at all, which is a protocol fault.
1001            if self
1002                .highest_retired_ssa_index
1003                .is_some_and(|highest| ssa_id.ssa_index() <= highest)
1004            {
1005                return Vec::new();
1006            }
1007            return vec![SessionPixAction::Close(SessionPixCloseReason::InvalidTransition)];
1008        };
1009
1010        // Idempotent: a repeated confirmation must not extend the deadline, and one arriving after
1011        // the SSA has moved on must not resurrect a phase it has left.
1012        if self.ssas[idx].phase == SsaPhase::AwaitingCommitment && self.ssas[idx].commitment_deadline.is_none() {
1013            self.ssas[idx].commitment_deadline = now.checked_add(crate::supervision::scaled_deadline(
1014                self.cfg.max_ssa_delivery_time,
1015                self.cfg.ssas_per_request,
1016            ));
1017        }
1018
1019        Vec::new()
1020    }
1021
1022    /// Arms — or re-arms — the re-request timer on a commitment that is under way but not complete.
1023    ///
1024    /// This is where the issue's rule lives: a cycle that received *nothing* never reaches here, so
1025    /// it never asks, which is correct because there is no scope for it to name. And because every
1026    /// arriving part pushes the timer out, the first ask cannot land in the middle of the burst —
1027    /// what the timer measures is a stall in delivery.
1028    ///
1029    /// Unscaled by the batch size, unlike the two absolute deadlines: this is a per-cycle stall
1030    /// detector, and a batch's commitment sets do not queue behind one another in a way that would
1031    /// make one cycle's silence expected.
1032    fn on_commitment_progress(&mut self, ssa_id: &SsaId<HoprPseudonym>, now: Instant) -> Vec<SessionPixAction> {
1033        if let Some(idx) = self.find_ssa_idx(ssa_id)
1034            && self.ssas[idx].phase == SsaPhase::AwaitingCommitment
1035            && self.ssas[idx].recommit_attempts < hopr_protocol_pix::MAX_COMMITMENT_RETRANSMISSIONS
1036        {
1037            self.ssas[idx].recommit_deadline = now.checked_add(self.cfg.commitment_recommit_interval);
1038        }
1039
1040        Vec::new()
1041    }
1042
1043    fn on_commitment_verified(&mut self, ssa_id: &SsaId<HoprPseudonym>, now: Instant) -> Vec<SessionPixAction> {
1044        let idx = match self.find_ssa_idx(ssa_id) {
1045            Some(i) => i,
1046            None => return Vec::new(),
1047        };
1048        let ssa = &mut self.ssas[idx];
1049        if ssa.phase != SsaPhase::AwaitingCommitment {
1050            return Vec::new();
1051        }
1052
1053        let deposit_deadline = now.checked_add(crate::supervision::scaled_deadline(
1054            self.cfg.max_deposit_wait,
1055            self.cfg.ssas_per_request,
1056        ));
1057        let phase_sample = ssa.enter_phase(SsaPhase::AwaitingDeposit, now);
1058        ssa.deposit_deadline = deposit_deadline;
1059        ssa.commitment_deadline = None;
1060        // Nothing is missing any more, so nothing is to be asked for.
1061        ssa.recommit_deadline = None;
1062        self.telemetry.phase_durations.extend(phase_sample);
1063
1064        // Behind the phase guard above, so a repeated `CommitmentVerified` for a cycle that has
1065        // already moved on counts nothing.
1066        self.telemetry.committed = self.telemetry.committed.saturating_add(1);
1067
1068        Vec::new()
1069    }
1070
1071    /// Funds the cycle on the deposit pool's verdict. The supervisor does not second-guess the amount.
1072    ///
1073    /// Sufficiency is priced and judged outside this workspace. The Exit's pool is handed the
1074    /// [`DepositUpdated`](hopr_api::node::DepositUpdated) sender together with the byte quota when the
1075    /// deposit address arrives, and it sends on that channel once the deposit clears the price it
1076    /// charges for that quota — so by the time this runs, the question has been answered by the only
1077    /// component that can answer it. Nothing in the Session layer prices a byte.
1078    ///
1079    /// This used to compare the running total against `max(expected_deposit, min_deposit)`. Both are
1080    /// gone, and not because they were merely unused: a configured floor here is not a backstop behind
1081    /// the pool but a second authority alongside it, and one set above the pool's price fails a cycle
1082    /// that has already been paid for — silently, on `max_deposit_wait`, with neither side able to
1083    /// break the tie. `expected_deposit` was the same idea arriving over the wire, and the commitment
1084    /// never carried an amount to fill it.
1085    ///
1086    /// What remains is the one check that needs no price: a zero balance asserts that nothing was
1087    /// deposited, so it is not a verdict and cannot release service. The deadline keeps running and a
1088    /// later confirmation can still fund the cycle.
1089    fn on_deposit_confirmed(
1090        &mut self,
1091        ssa_id: &SsaId<HoprPseudonym>,
1092        amount: HoprBalance,
1093        now: Instant,
1094        served_total: u64,
1095    ) -> Vec<SessionPixAction> {
1096        let idx = match self.find_ssa_idx(ssa_id) {
1097            Some(i) => i,
1098            None => return Vec::new(),
1099        };
1100        let was_front = self.earliest_live_idx() == Some(idx);
1101
1102        let ssa = &mut self.ssas[idx];
1103        if ssa.phase != SsaPhase::AwaitingDeposit {
1104            return Vec::new();
1105        }
1106
1107        if amount == HoprBalance::zero() {
1108            return Vec::new();
1109        }
1110
1111        // Transition to Recovering. The two recovery clocks are *not* started here — see
1112        // `arm_recovery_clocks_for_earliest`, which starts them when this cycle's turn comes.
1113        let phase_sample = ssa.enter_phase(SsaPhase::Recovering, now);
1114        ssa.deposit_deadline = None;
1115        ssa.served_total_at_last_progress = served_total;
1116        self.telemetry.phase_durations.extend(phase_sample);
1117
1118        // Behind both the phase guard and the zero-amount guard above, so a duplicate confirmation
1119        // and a zero balance — which is not a verdict — both count nothing.
1120        ssa.deposit_uhopr = deposit_uhopr(amount);
1121        self.telemetry.funded = self.telemetry.funded.saturating_add(1);
1122        self.telemetry.deposit_confirmed_uhopr =
1123            self.telemetry.deposit_confirmed_uhopr.saturating_add(ssa.deposit_uhopr);
1124
1125        // If recovery completed before the deposit arrived, immediately
1126        // tombstone the SSA — the Recovered event was deferred.
1127        let recovered_pending = ssa.recovered_pending;
1128
1129        let pending = ssa.next_request_pending_deposit;
1130        if pending {
1131            ssa.next_request_pending_deposit = false;
1132        }
1133        // End the mutable borrow on ssas[idx].
1134        let _ = ssa;
1135
1136        let mut actions = Vec::new();
1137
1138        if recovered_pending {
1139            if was_front {
1140                self.paid_front_handoff = true;
1141            }
1142            actions.extend(self.perform_recovered_transition(idx, now, served_total));
1143        }
1144
1145        if pending {
1146            // When recovered_pending is also set, perform_recovered_transition
1147            // won't emit RequestSsa (next_requested is already set by
1148            // on_recovered), so emit it here.
1149            actions.extend(self.emit_request_next_ssa(now));
1150        }
1151
1152        actions
1153    }
1154
1155    fn on_deposit_observer_closed(
1156        &mut self,
1157        ssa_id: &SsaId<HoprPseudonym>,
1158        now: Instant,
1159        served_total: u64,
1160    ) -> Vec<SessionPixAction> {
1161        let idx = match self.find_ssa_idx(ssa_id) {
1162            Some(i) => i,
1163            None => return Vec::new(),
1164        };
1165
1166        if self.ssas[idx].phase != SsaPhase::AwaitingDeposit {
1167            return Vec::new();
1168        }
1169
1170        self.close_ssa_and_collect(idx, SessionPixCloseReason::DepositObserverClosed, served_total, now)
1171    }
1172
1173    fn on_recovery_progress(
1174        &mut self,
1175        progress: &SsaRecoveryProgress<HoprPseudonym>,
1176        now: Instant,
1177        served_total: u64,
1178    ) -> Vec<SessionPixAction> {
1179        if self
1180            .paid_recovery_tail
1181            .as_ref()
1182            .is_some_and(|tail| tail.ssa_id == progress.ssa_id)
1183        {
1184            return self.on_recovered_tail_progress(progress, now, served_total);
1185        }
1186
1187        let idx = match self.find_ssa_idx(&progress.ssa_id) {
1188            Some(i) => i,
1189            None => return Vec::new(),
1190        };
1191
1192        // Absorb late progress on tombstones — the SSA is already fully
1193        // recovered and should not reset the session-wide gate watermark.
1194        if self.ssas[idx].is_terminal() {
1195            return Vec::new();
1196        }
1197
1198        // Validate every dimension-derived bound before mutating. Progress is produced locally, but
1199        // all of its source fields ultimately came from peer-tagged shares. Keeping the supervisor
1200        // as an independent backstop means a reconstructor regression cannot turn an oversized
1201        // counter into unbounded service.
1202        let target = self.dims.target_useful_shares();
1203        let max_shares_seen = self.dims.polys_per_ssa() as u64 * self.dims.emitted_shares_per_poly() as u64;
1204        if progress.target_useful_shares != target
1205            || progress.useful_shares > target
1206            || progress.shares_seen < progress.useful_shares
1207            || progress.shares_seen > max_shares_seen
1208            || progress.recovered_polynomials > self.dims.polys_per_ssa()
1209        {
1210            return vec![SessionPixAction::Close(SessionPixCloseReason::CounterRegression)];
1211        }
1212
1213        let ssa = &mut self.ssas[idx];
1214        let new_useful = progress.useful_shares;
1215        let new_seen = progress.shares_seen;
1216
1217        // Counter regression check, on the liveness counter because it is the one that moves for every
1218        // share — a snapshot can be newer than the last one while carrying the same `useful_shares`.
1219        //
1220        // The relay-as-Exit pipeline processes acknowledgement batches with
1221        // for_each_concurrent, so absolute progress snapshots from different
1222        // batches can arrive out of order. Treat a stale snapshot as benign
1223        // noise rather than a protocol violation.
1224        if new_seen <= ssa.largest_shares_seen {
1225            return Vec::new();
1226        }
1227
1228        // The accepted delta, taken here because this is the one point at which a snapshot is known
1229        // to be both validated and newer than everything seen before. The absolute/reordered
1230        // semantics the issue asks for fall out of the two high-water marks rather than needing
1231        // their own machinery: a repeated or stale snapshot returned at the guard above and adds
1232        // zero, and a snapshot arriving after a dropped one carries the whole missing delta,
1233        // because a mark is a maximum and never a running sum.
1234        let accepted_delta = new_seen - ssa.largest_shares_seen;
1235        // Clamped to the accepted delta, which is not defensive clutter. The two marks are
1236        // independent maxima over possibly different snapshots, so a reconstructor regression could
1237        // present `(seen 10, useful 2)` followed by `(seen 11, useful 11)` — a useful delta of nine
1238        // against an accepted delta of one — and mint a negative surplus. This file is an
1239        // independent backstop against exactly that class of fault everywhere else; it is one here
1240        // too.
1241        let useful_delta = new_useful.saturating_sub(ssa.largest_useful_shares).min(accepted_delta);
1242        self.telemetry.useful_shares = self.telemetry.useful_shares.saturating_add(useful_delta);
1243        self.telemetry.surplus_shares = self
1244            .telemetry
1245            .surplus_shares
1246            .saturating_add(accepted_delta - useful_delta);
1247
1248        ssa.largest_shares_seen = new_seen;
1249
1250        // Liveness tier. A share arrived, useful or not, so the Entry is serving this cycle — which is
1251        // the question the egress gate and the recovery-idle deadline are asking. Keying either of them
1252        // on `useful_shares` instead makes a conforming Entry's surplus run look like silence, and for
1253        // the gate that is self-inflicted: withholding service removes the only thing that could
1254        // produce the next useful share. See `SsaRecoveryProgress::shares_seen`.
1255        ssa.served_total_at_last_progress = served_total;
1256        // Refreshing is not arming: only `arm_recovery_clocks_for_earliest` starts a cycle's clocks,
1257        // and `recovery_hard_deadline` is how a cycle records that it has reached the front. Without
1258        // that second condition a stray reordered share *starts* the idle clock on a cycle still
1259        // queued behind the front — which the Entry cannot then feed, because emission is clamped to
1260        // one cycle — and 60 s later the idle gate sees session-wide service climbing and retires it.
1261        // That is the queue-wait failure `arm_recovery_clocks_for_earliest` documents, reached
1262        // through the refresh path instead of the deposit one. The clocks arm together or not at all.
1263        if ssa.phase == SsaPhase::Recovering && ssa.recovery_hard_deadline.is_some() {
1264            ssa.recovery_idle_deadline = now.checked_add(self.cfg.max_recovery_idle);
1265        }
1266
1267        // Payment tier. Only shares that advanced reconstruction are worth anything, so only they move
1268        // the recovery counters or count towards the share-order ratio.
1269        if new_useful > ssa.largest_useful_shares {
1270            let delta = new_useful - ssa.largest_useful_shares;
1271            ssa.largest_useful_shares = new_useful;
1272            ssa.recovered_polynomials = progress.recovered_polynomials;
1273
1274            if let Some(reason) = self.book_share_order_progress(&progress.ssa_id, delta) {
1275                return vec![SessionPixAction::Close(reason)];
1276            }
1277        }
1278
1279        let is_front = self.earliest_live_idx() == Some(idx);
1280
1281        // Only funded progress on the cycle service is currently charged against may reopen the
1282        // Session-wide ceiling. Unfunded progress is the H11 bypass; off-front progress belongs to a
1283        // queued cycle and must not buy service for the front either. The outer event transition
1284        // consumes any predecessor tail before synchronizing the gate, even for unfunded progress.
1285        if self.ssas[idx].phase == SsaPhase::Recovering && is_front {
1286            vec![SessionPixAction::ProgressNotification]
1287        } else {
1288            Vec::new()
1289        }
1290    }
1291
1292    /// Applies a progress snapshot from the sole recovered predecessor authorized to drain.
1293    ///
1294    /// The reconstructor enforces the strong per-polynomial budget. These aggregate checks are a
1295    /// second security boundary: even if that component regresses, the Entry cannot make a recovered
1296    /// cycle report more than `polys × (threshold + surplus)` packets, change its terminal useful
1297    /// count, or revive any older tombstone.
1298    fn on_recovered_tail_progress(
1299        &mut self,
1300        progress: &SsaRecoveryProgress<HoprPseudonym>,
1301        now: Instant,
1302        served_total: u64,
1303    ) -> Vec<SessionPixAction> {
1304        let target = self.dims.target_useful_shares();
1305        let expected_polynomials = self.dims.polys_per_ssa();
1306        let max_shares_seen = expected_polynomials as u64 * self.dims.emitted_shares_per_poly() as u64;
1307
1308        if progress.target_useful_shares != target
1309            || progress.useful_shares > target
1310            || progress.recovered_polynomials > expected_polynomials
1311            || progress.shares_seen < progress.useful_shares
1312            || progress.shares_seen > max_shares_seen
1313        {
1314            tracing::warn!(
1315                ssa_id = %progress.ssa_id,
1316                useful_shares = progress.useful_shares,
1317                shares_seen = progress.shares_seen,
1318                recovered_polynomials = progress.recovered_polynomials,
1319                "rejected impossible progress from a recovered PIX tail"
1320            );
1321            return vec![SessionPixAction::Close(SessionPixCloseReason::CounterRegression)];
1322        }
1323
1324        // Absolute snapshots from concurrent acknowledgement batches may arrive after the
1325        // `Recovered` event that overtook them. A non-terminal snapshot cannot buy liveness here,
1326        // but it is not evidence of peer misconduct either. Ignore it exactly as the live path
1327        // ignores an older absolute counter. Only a terminal snapshot can touch the paid receipt.
1328        if progress.useful_shares != target || progress.recovered_polynomials != expected_polynomials {
1329            return Vec::new();
1330        }
1331
1332        let (exhausted, accepted_delta) = {
1333            let tail = self
1334                .paid_recovery_tail
1335                .as_mut()
1336                .expect("caller matched the paid recovery tail");
1337            if progress.shares_seen <= tail.largest_shares_seen {
1338                return Vec::new();
1339            }
1340
1341            let accepted_delta = progress.shares_seen - tail.largest_shares_seen;
1342            tail.largest_shares_seen = progress.shares_seen;
1343            tail.served_total_at_last_progress = served_total;
1344            tail.idle_deadline = now.checked_add(self.cfg.max_recovery_idle);
1345            (progress.shares_seen == max_shares_seen, accepted_delta)
1346        };
1347
1348        // All surplus, by definition: this cycle has already reached its useful-share target, which
1349        // is what the guard above requires before a snapshot may touch the receipt at all. These
1350        // are the negotiated tail a conforming Entry keeps emitting while the FIFO drains — real
1351        // accepted traffic, and the evidence that keeps the paid front open, but no longer worth
1352        // anything towards recovery.
1353        self.telemetry.surplus_shares = self.telemetry.surplus_shares.saturating_add(accepted_delta);
1354
1355        // Keep the tombstone's diagnostics monotone while it is retained, but do not require the
1356        // record: tombstone cleanup is intentionally shorter than a Session's possible FIFO drain.
1357        if let Some(idx) = self.find_ssa_idx(&progress.ssa_id) {
1358            self.ssas[idx].largest_useful_shares = target;
1359            self.ssas[idx].largest_shares_seen = progress.shares_seen;
1360            self.ssas[idx].recovered_polynomials = expected_polynomials;
1361            self.ssas[idx].served_total_at_last_progress = served_total;
1362        }
1363
1364        if exhausted {
1365            self.release_paid_tail(now);
1366        }
1367        vec![SessionPixAction::ProgressNotification]
1368    }
1369
1370    /// Attributes `delta` useful shares to the front of the batch or behind it, and judges the ratio.
1371    ///
1372    /// The Entry serves a batch strictly in index order, so progress on anything but the front cycle is
1373    /// service the Exit cannot yet be paid for. See
1374    /// [`SupervisorConfig::max_off_front_share_fraction`] for the threat and for why this is measured
1375    /// as a fraction; the judgement waits for
1376    /// [`SupervisorConfig::min_share_order_sample`] shares of evidence.
1377    fn book_share_order_progress(
1378        &mut self,
1379        ssa_id: &SsaId<HoprPseudonym>,
1380        delta: u64,
1381    ) -> Option<SessionPixCloseReason> {
1382        let front = self.share_order_front?;
1383        if ssa_id.ssa_index() == front {
1384            self.front_useful = self.front_useful.saturating_add(delta);
1385        } else {
1386            self.off_front_useful = self.off_front_useful.saturating_add(delta);
1387        }
1388
1389        // Judged on every event, not only off-front ones: the sample can cross the floor on a share
1390        // that lands *on* the front cycle, and the ratio is already whatever it is by then. Waiting for
1391        // the next off-front share would only delay the same verdict.
1392        let total = self.front_useful.saturating_add(self.off_front_useful);
1393        if total < self.cfg.min_share_order_sample || total == 0 {
1394            return None;
1395        }
1396
1397        let fraction = self.off_front_useful as f64 / total as f64;
1398        if fraction > self.cfg.max_off_front_share_fraction {
1399            tracing::error!(
1400                %ssa_id, %front, fraction,
1401                limit = self.cfg.max_off_front_share_fraction,
1402                front_useful = self.front_useful,
1403                off_front_useful = self.off_front_useful,
1404                "entry is serving the batch out of order"
1405            );
1406            return Some(SessionPixCloseReason::BatchServedOutOfOrder);
1407        }
1408
1409        tracing::trace!(%ssa_id, %front, fraction, "progress behind the front of the batch");
1410        None
1411    }
1412
1413    fn on_almost_recovered(&mut self, ssa_id: &SsaId<HoprPseudonym>, now: Instant) -> Vec<SessionPixAction> {
1414        let idx = match self.find_ssa_idx(ssa_id) {
1415            Some(i) => i,
1416            None => return Vec::new(),
1417        };
1418
1419        // If already recovered (e.g. a concurrent batch delivered Recovered
1420        // before this AlmostRecovered), the next SSA is already requested —
1421        // no-op.
1422        if self.ssas[idx].is_terminal() {
1423            return Vec::new();
1424        }
1425
1426        // The successor gate. A cycle in the middle of a batch reaching its early-recovery threshold
1427        // says nothing about the batch as a whole: the ones behind it still have their full quota to
1428        // serve. Only the last cycle standing in for the batch may ask for a replacement.
1429        if !self.ssas[idx].is_batch_last {
1430            return Vec::new();
1431        }
1432
1433        let next_requested = self.ssas[idx].next_requested;
1434        let phase = self.ssas[idx].phase;
1435
1436        if next_requested {
1437            return Vec::new();
1438        }
1439
1440        match phase {
1441            SsaPhase::Recovering => {
1442                self.ssas[idx].next_requested = true;
1443                self.emit_request_next_ssa(now)
1444            }
1445            SsaPhase::AwaitingDeposit => {
1446                self.ssas[idx].next_requested = true;
1447                self.ssas[idx].next_request_pending_deposit = true;
1448                Vec::new()
1449            }
1450            // AwaitingCommitment or unknown phase: the next SSA request will be
1451            // triggered when the recovered transition fires (normal Recovering
1452            // path or deferred replay via recovered_pending). Set the deferred
1453            // flag so that commitment_verified can propagate it forward.
1454            _ => {
1455                self.ssas[idx].next_request_pending_deposit = true;
1456                self.ssas[idx].next_requested = true;
1457                Vec::new()
1458            }
1459        }
1460    }
1461
1462    fn on_recovered(
1463        &mut self,
1464        ssa_id: &SsaId<HoprPseudonym>,
1465        now: Instant,
1466        served_total: u64,
1467    ) -> Vec<SessionPixAction> {
1468        let idx = match self.find_ssa_idx(ssa_id) {
1469            Some(i) => i,
1470            None => return Vec::new(),
1471        };
1472
1473        // Guard against duplicate recovery events (possible with concurrent
1474        // batch processing). If already recovered, this is a no-op.
1475        if self.ssas[idx].is_terminal() {
1476            return Vec::new();
1477        }
1478
1479        match self.ssas[idx].phase {
1480            SsaPhase::Recovering => {
1481                // Normal path — transition to tombstone directly.
1482                self.perform_recovered_transition(idx, now, served_total)
1483            }
1484            SsaPhase::AwaitingDeposit => {
1485                // Recovery completed before deposit confirmed. Record the
1486                // pending flag so that when the deposit arrives we replay
1487                // the transition immediately after entering Recovering.
1488                self.ssas[idx].recovered_pending = true;
1489                // Also ensure the next SSA request is deferred — the
1490                // eventual tombstone will emit it. Subject to the same successor gate as
1491                // `on_almost_recovered`: a non-last cycle of a batch never asks, deferred or not.
1492                if !self.ssas[idx].next_requested && self.ssas[idx].is_batch_last {
1493                    self.ssas[idx].next_request_pending_deposit = true;
1494                    self.ssas[idx].next_requested = true;
1495                }
1496                Vec::new()
1497            }
1498            SsaPhase::AwaitingCommitment => {
1499                // Recovery outpaced commitment verification. Same semantics
1500                // as AwaitingDeposit: set pending flag so the transition
1501                // replays once commitment and deposit have both arrived.
1502                self.ssas[idx].recovered_pending = true;
1503                Vec::new()
1504            }
1505            // Closing / already Recovered: handled by the terminal guard above.
1506            _ => Vec::new(),
1507        }
1508    }
1509
1510    /// Books what a recovered cycle owes the counters and retires it into its tombstone.
1511    ///
1512    /// Split out because a drain returns before the rest of the transition, and would otherwise
1513    /// never book the deposit it just unlocked — making the one outcome the drain exists for read as
1514    /// stranded value in `hopr_pix_deposits_recovered_uhopr_total`.
1515    fn book_recovered_cycle(&mut self, idx: usize, now: Instant, served_total: u64) {
1516        // `Recovered` is a local cryptographic fact, so it supplies the lower bound even if its
1517        // final absolute Progress snapshot is delivered just afterwards by a concurrent worker.
1518        // Every later tail snapshot must keep these terminal fields exact.
1519        let target = self.dims.target_useful_shares();
1520        self.ssas[idx].largest_useful_shares = target;
1521        self.ssas[idx].largest_shares_seen = self.ssas[idx].largest_shares_seen.max(target);
1522        self.ssas[idx].recovered_polynomials = self.dims.polys_per_ssa();
1523
1524        // Counted here rather than on the `Recovered` event, because that event is not the
1525        // transition: `on_recovered` defers it when the cycle is still awaiting its commitment or
1526        // deposit, and replays it through this function once both arrive. This is the one place a
1527        // cycle becomes terminal-by-recovery, and its callers all guard against reaching it twice.
1528        self.telemetry.recovered = self.telemetry.recovered.saturating_add(1);
1529        // Summarized before the tombstone transition below, and before `paid_recovery_tail` starts
1530        // collecting the drain, which belongs to the FIFO rather than to this cycle's recovery. The
1531        // terminal fields were just pinned to the target above, so a recovered cycle's useful
1532        // fraction is exactly one by construction.
1533        let summary = self.ssas[idx].telemetry_summary(PixCycleOutcome::Recovered, served_total);
1534        self.telemetry.finalized.push(summary);
1535
1536        let phase_sample = self.ssas[idx].enter_phase(
1537            SsaPhase::Recovered {
1538                tombstone_until: now
1539                    .checked_add(self.cfg.tombstone_retention_window)
1540                    .unwrap_or_else(|| now + Duration::from_secs(86400 * 365)),
1541            },
1542            now,
1543        );
1544        self.telemetry.phase_durations.extend(phase_sample);
1545    }
1546
1547    /// Perform the terminal tombstone transition for a fully-recovered SSA.
1548    /// Called from `on_recovered` (normal Recovering path) and replayed from
1549    /// `on_deposit_confirmed`/`on_commitment_verified` when `recovered_pending`
1550    /// was set earlier.
1551    fn perform_recovered_transition(&mut self, idx: usize, now: Instant, served_total: u64) -> Vec<SessionPixAction> {
1552        let next_requested = self.ssas[idx].next_requested;
1553        // Read before the cycle is retired below, which would make it terminal and change the answer.
1554        let was_front = self.earliest_live_idx() == Some(idx);
1555        let max_shares_seen = self.dims.polys_per_ssa() as u64 * self.dims.emitted_shares_per_poly() as u64;
1556
1557        self.book_recovered_cycle(idx, now, served_total);
1558
1559        // The end of a drain, and the outcome it exists for: the deposit this cycle was paid is
1560        // released rather than stranded. None of the bookkeeping below is worth doing — the tail
1561        // receipt, the gate notification and the successor request all serve a Session that is about
1562        // to be torn down, and `closed` gates every entry point from here on. Returning before the
1563        // tail is created also guarantees a drain can never leave one behind.
1564        if self.draining {
1565            tracing::info!(
1566                ssa_id = %self.ssas[idx].ssa_id,
1567                "the funded cycle of a closed session recovered — the SURB drain is complete"
1568            );
1569            self.closed = true;
1570            return self.with_fill_stopped(vec![SessionPixAction::Close(SessionPixCloseReason::Drained)]);
1571        }
1572
1573        if was_front {
1574            // Replacing rather than accumulating is intentional. Only the immediate recovered
1575            // predecessor can occupy a FIFO immediately ahead of the live front; accepting an
1576            // arbitrary tombstone here would turn old-cycle replay into gate liveness.
1577            self.paid_recovery_tail =
1578                (self.ssas[idx].largest_shares_seen < max_shares_seen).then(|| PaidRecoveryTail {
1579                    ssa_id: self.ssas[idx].ssa_id,
1580                    entered_at: now,
1581                    largest_shares_seen: self.ssas[idx].largest_shares_seen,
1582                    served_total_at_last_progress: served_total,
1583                    // Recovery itself is fresh, cryptographic progress. Reusing the prior idle
1584                    // instant would let delayed/dropped progress snapshots make an honest tail
1585                    // expire immediately after recovery. Refresh this clock once; unlike the hard
1586                    // deadline below, it is the clock progress is meant to extend.
1587                    idle_deadline: now.checked_add(self.cfg.max_recovery_idle),
1588                    hard_deadline: self.ssas[idx]
1589                        .recovery_hard_deadline
1590                        .or_else(|| now.checked_add(self.cfg.max_recovery_time)),
1591                });
1592        }
1593
1594        self.ssas[idx].commitment_deadline = None;
1595        self.ssas[idx].deposit_deadline = None;
1596        self.ssas[idx].recovery_idle_deadline = None;
1597        self.ssas[idx].recovery_hard_deadline = None;
1598        self.ssas[idx].recovered_pending = false;
1599
1600        let mut actions = Vec::new();
1601        // A cycle that recovered with nothing buffered behind it leaves the Exit with nothing to fill
1602        // for until its successor reaches the front and arms its own clocks — which is at least a
1603        // deposit away. Stopping now rather than waiting for the successor keeps those SURBs for the
1604        // cycle that will actually be paid for; the next tick starts the ramp again from the
1605        // successor's true remainder.
1606        if was_front && self.paid_recovery_tail.is_none() {
1607            actions.extend(self.fill.stop().map(SessionPixAction::SetFillRate));
1608        }
1609        if was_front {
1610            // `Recovered` is stronger evidence than an ordinary progress snapshot. Emit this even
1611            // though the reconstructor normally sends the final snapshot first: progress delivery
1612            // is deliberately lossy under command-channel pressure, and without this reset a gate
1613            // already at its served-without-progress ceiling cannot spend the first buffered SURB
1614            // needed to produce tail liveness. If the successor is unfunded, `sync_service_gate`
1615            // prepends `WithholdService`; notifying afterwards changes only the dormant funded
1616            // watermark and cannot restore service.
1617            actions.push(SessionPixAction::ProgressNotification);
1618        }
1619        // Full recovery is the fallback trigger for the early signal, so it carries the same
1620        // successor gate: a non-last cycle of a batch is tombstoned without asking for anything.
1621        if !next_requested && self.ssas[idx].is_batch_last {
1622            self.ssas[idx].next_requested = true;
1623            actions.extend(self.emit_request_next_ssa(now));
1624        }
1625
1626        actions
1627    }
1628
1629    /// Closes the Session on the first unverifiable-share report. There is no tolerance to configure.
1630    ///
1631    /// Shares are not checked on arrival, so a report is not "a bad share" — it means a whole
1632    /// polynomial's share set failed to open its commitment, which surfaces only once `threshold` of
1633    /// them have been interpolated. `SsaPart::add_share` then marks that part failed and releases its
1634    /// shares, and nothing ever clears the flag; since the SSA is the sum of *every* polynomial's
1635    /// constant term, the cycle can no longer be reconstructed by any means and will never pay.
1636    ///
1637    /// So serving on is not tolerance, it is donation: every packet past this point is unpaid with no
1638    /// recovery to preserve at the end of it. That holds however the failure arose, which is why this
1639    /// is not a knob — a false positive from a verification bug leaves the part just as permanently
1640    /// failed, so a tolerance would buy unpaid service rather than the cycle it was raised to save.
1641    /// Closing here caps the exposure at the `threshold` packets already served rather than a multiple
1642    /// of it.
1643    ///
1644    /// `observed_total` is the reconstructor's cross-peer aggregate, and with no limit to compare it
1645    /// against it is logged rather than accumulated: it says how many polynomials the batch that
1646    /// triggered this took down, which is the difference between one bad relayer and a peer sending
1647    /// garbage wholesale.
1648    ///
1649    /// Note this closes the Session directly rather than through
1650    /// [`close_ssa_and_collect`](Self::close_ssa_and_collect), so a batch's surviving siblings do not
1651    /// keep it alive: unlike a lost deposit, this fault is evidence about the peer rather than about
1652    /// one cycle.
1653    fn on_unverifiable_shares(
1654        &mut self,
1655        ssa_id: &SsaId<HoprPseudonym>,
1656        observed_total: u64,
1657        peer: &OffchainPublicKey,
1658        _now: Instant,
1659    ) -> Vec<SessionPixAction> {
1660        let idx = match self.find_ssa_idx(ssa_id) {
1661            Some(i) => i,
1662            None => return Vec::new(),
1663        };
1664
1665        // Absorb late reports on a tombstoned or already-closing SSA: there is no service left to
1666        // stop, and the cycle they condemn has already left. Concurrent ack batches make these
1667        // ordinary rather than exceptional.
1668        if self.ssas[idx].is_terminal() {
1669            return Vec::new();
1670        }
1671
1672        tracing::warn!(
1673            %ssa_id,
1674            %peer,
1675            observed_total,
1676            phase = ?self.ssas[idx].phase,
1677            "closing PIX session: a polynomial's share set failed to open its commitment"
1678        );
1679
1680        self.with_fill_stopped(vec![SessionPixAction::Close(SessionPixCloseReason::UnverifiableShares)])
1681    }
1682
1683    /// Decides whether a Session closed at the Exit is worth draining its buffered SURBs into.
1684    ///
1685    /// `available` is the manager's estimate of the SURBs still spendable, already net of the fill
1686    /// reserve. The cycle it is measured against is the earliest live one, and only when it is both
1687    /// funded and clocked — i.e. exactly the cycle fill would have been working for. Two shapes have
1688    /// nothing a drain could buy and close at once: an unfunded front, where no deposit is at stake;
1689    /// and a recovered predecessor's paid FIFO tail, whose key is already recovered and behind which
1690    /// the successor is queued without clocks of its own.
1691    ///
1692    /// The comparison is against the *whole* remaining emission, with no loss margin. That is
1693    /// conservative in both directions on purpose: the estimate is an upper bound (see
1694    /// [`SessionPixEvent::SessionClosed`]), while in-flight packets are already subtracted from it
1695    /// and full recovery in practice needs fewer than `E` shares. A drain that runs out partway has
1696    /// spent the Session's SURBs down to the reserve and recovered nothing, which is strictly worse
1697    /// than never starting.
1698    ///
1699    /// Siblings are retired directly rather than through
1700    /// [`close_ssa_and_collect`](Self::close_ssa_and_collect), which would be wrong twice over: it
1701    /// charges [`failed_cycles`](Self::failed_cycles), and these cycles were retired by an operator
1702    /// closing a Session rather than lost by an Entry; and its `ssas.len() == 1` branch would close
1703    /// the Session outright. Recovered tombstones are left alone — they hold no reconstructor state
1704    /// and expire on their own timer with an idempotent `RetireSsa`.
1705    ///
1706    /// `_now` is taken for symmetry with every other handler; nothing here is timed. A drain inherits
1707    /// the target's existing clocks rather than starting any of its own, which is the point — the
1708    /// resource bound on the cycle must not be extended by the Session ending.
1709    fn on_session_closed(&mut self, available: u64, _now: Instant) -> Vec<SessionPixAction> {
1710        let target = self
1711            .paid_recovery_tail
1712            .is_none()
1713            .then(|| self.earliest_live_idx())
1714            .flatten()
1715            .filter(|&idx| {
1716                self.ssas[idx].phase == SsaPhase::Recovering && self.ssas[idx].recovery_hard_deadline.is_some()
1717            });
1718
1719        let cycle_shares = self.dims.polys_per_ssa() as u64 * self.dims.emitted_shares_per_poly() as u64;
1720        let remaining = target.map(|idx| cycle_shares.saturating_sub(self.ssas[idx].largest_shares_seen));
1721
1722        let Some((idx, remaining)) = target.zip(remaining).filter(|&(_, remaining)| available >= remaining) else {
1723            tracing::info!(
1724                available,
1725                ?remaining,
1726                target = ?target.map(|idx| self.ssas[idx].ssa_id),
1727                "closing PIX session: the exit-side close left nothing worth draining for"
1728            );
1729            self.closed = true;
1730            return self.with_fill_stopped(vec![SessionPixAction::Close(SessionPixCloseReason::SessionClosed)]);
1731        };
1732
1733        let target_id = self.ssas[idx].ssa_id;
1734        tracing::info!(
1735            %target_id,
1736            available,
1737            remaining,
1738            "draining buffered SURBs into the funded cycle of a session closed at the exit"
1739        );
1740
1741        self.draining = true;
1742        // A deferred request can never be retried now, and leaving the flag set would have
1743        // `retry_deferred_successor_request` call into the (now silent) request path every event.
1744        self.successor_request_deferred = false;
1745
1746        // Every sibling is behind the target in index order — the target *is* `earliest_live_idx` —
1747        // so removing them cannot move the front, and the tail of `handle_event` re-derives the same
1748        // gate state and share-order front it already had.
1749        let mut retired = Vec::new();
1750        self.ssas.retain(|ssa| {
1751            if ssa.ssa_id == target_id || ssa.is_terminal() {
1752                true
1753            } else {
1754                retired.push(ssa.ssa_id);
1755                false
1756            }
1757        });
1758
1759        retired
1760            .into_iter()
1761            .map(|ssa_id| {
1762                self.note_retired_index(ssa_id.ssa_index());
1763                SessionPixAction::RetireSsa(ssa_id)
1764            })
1765            .collect()
1766    }
1767
1768    // ------------------------------------------------------------------
1769    // Deadline helpers
1770    // ------------------------------------------------------------------
1771
1772    /// Close the SSA at `idx` and return close actions.
1773    fn close_ssa_and_collect(
1774        &mut self,
1775        idx: usize,
1776        reason: SessionPixCloseReason,
1777        served_total: u64,
1778        now: Instant,
1779    ) -> Vec<SessionPixAction> {
1780        if matches!(self.ssas[idx].phase, SsaPhase::Closing) {
1781            return Vec::new();
1782        }
1783
1784        // Track the first close reason so it isn't lost when multiple SSAs close.
1785        self.first_failure_reason.get_or_insert(reason);
1786        let close_reason = self.first_failure_reason.unwrap_or(reason);
1787
1788        // Charged before the branches below, because the whole point is a count that outlives the
1789        // record being retired. Every path into this function is a cycle lost without recovering —
1790        // a recovered cycle leaves through tombstone expiry in `handle_deadline`, not through here.
1791        self.failed_cycles += 1;
1792        // Same transition, so the same guard covers both: the `Closing` check at the top of this
1793        // function is what makes a repeated close of one cycle count once. The summary rides along
1794        // for the same reason — this is the one place a cycle is retired without recovering, and
1795        // the counters are still intact because nothing below has cleared them yet.
1796        self.telemetry.failed = self.telemetry.failed.saturating_add(1);
1797        let summary = self.ssas[idx].telemetry_summary(PixCycleOutcome::Failed, served_total);
1798        self.telemetry.finalized.push(summary);
1799
1800        // Warn-level diagnostic with full SSA state before closing.
1801        let ssa = &self.ssas[idx];
1802        tracing::warn!(
1803            ssa_id = %ssa.ssa_id,
1804            ?reason,
1805            phase = ?ssa.phase,
1806            largest_useful_shares = ssa.largest_useful_shares,
1807            target_useful_shares = ssa.target_useful_shares,
1808            recovered_polynomials = ssa.recovered_polynomials,
1809            served_total_at_last_progress = ssa.served_total_at_last_progress,
1810            ?ssa.commitment_deadline,
1811            ?ssa.deposit_deadline,
1812            ?ssa.recovery_idle_deadline,
1813            ?ssa.recovery_hard_deadline,
1814            "closing PIX SSA"
1815        );
1816
1817        let phase_sample = self.ssas[idx].enter_phase(SsaPhase::Closing, now);
1818        self.telemetry.phase_durations.extend(phase_sample);
1819
1820        if self.ssas.len() == 1 {
1821            self.closed = true;
1822            return self.with_fill_stopped(vec![SessionPixAction::Close(close_reason)]);
1823        }
1824
1825        // A batch has siblings to fall back on, which is what makes retiring one member survivable —
1826        // but only up to a point. Past the limit the Session goes, with the *first* failure's reason
1827        // rather than this one's, so the report names the cause instead of the last symptom.
1828        //
1829        // Strictly greater: the field is how many failures are *tolerated*, so the shipping value of
1830        // one keeps the existing "retire the member and carry on" behaviour for a batch's first loss
1831        // — including handing on the successor gate below — and closes on the second.
1832        if self.failed_cycles > self.cfg.max_failed_cycles.max(1) {
1833            tracing::warn!(
1834                failed_cycles = self.failed_cycles,
1835                max_failed_cycles = self.cfg.max_failed_cycles,
1836                ?close_reason,
1837                "closing PIX session: too many cycles lost without recovering"
1838            );
1839            self.closed = true;
1840            return self.with_fill_stopped(vec![SessionPixAction::Close(close_reason)]);
1841        }
1842
1843        // Clear deadlines on this SSA.
1844        self.ssas[idx].commitment_deadline = None;
1845        self.ssas[idx].deposit_deadline = None;
1846        self.ssas[idx].recovery_idle_deadline = None;
1847        self.ssas[idx].recovery_hard_deadline = None;
1848
1849        // If all SSAs are terminal, close the session.
1850        if self
1851            .ssas
1852            .iter()
1853            .all(|s| matches!(s.phase, SsaPhase::Closing | SsaPhase::Recovered { .. }))
1854        {
1855            self.closed = true;
1856            return self.with_fill_stopped(vec![SessionPixAction::Close(close_reason)]);
1857        }
1858
1859        // Remove this closing SSA and emit RetireSsa so the reconstructor
1860        // releases its builder/verifier/counter state mid-session.
1861        // Record the retired index to prevent stale SsaRequestSent events
1862        // from resurrecting it.
1863        let retired = self.ssas[idx].ssa_id;
1864        self.note_retired_index(retired.ssa_index());
1865        let carried_successor_gate = self.ssas[idx].is_batch_last && !self.ssas[idx].next_requested;
1866        self.ssas.remove(idx);
1867
1868        // Retiring the cycle that carried the successor gate would strand the batch: its surviving
1869        // siblings are barred from asking, so nothing would ever request a replacement and the
1870        // Session would die on a recovery timeout that names the timer rather than the cause. Hand
1871        // the gate to the newest cycle still standing, which is now the last of the batch that will
1872        // actually serve its quota — so the request still cannot come early. `next_requested` guards
1873        // it: a gate already spent must not be handed on and fire a second time.
1874        if carried_successor_gate
1875            && let Some(newest) = self
1876                .ssas
1877                .iter_mut()
1878                .filter(|s| !s.is_terminal())
1879                .max_by_key(|s| s.ssa_id.ssa_index())
1880        {
1881            tracing::debug!(
1882                %retired, promoted = %newest.ssa_id,
1883                "successor gate handed on from a retired cycle"
1884            );
1885            newest.is_batch_last = true;
1886        }
1887        vec![SessionPixAction::RetireSsa(retired)]
1888    }
1889
1890    /// Allocates the next batch of SSA indices and asks for all of them in one action.
1891    ///
1892    /// The batch size is [`SupervisorConfig::ssas_per_request`], clamped rather than trusted since a
1893    /// supervisor can be built from a config that never went through `validate_pix_supervision`.
1894    ///
1895    /// Exactly one cycle of the batch — the last one allocated — carries
1896    /// [`PerSsaState::is_batch_last`], and only that cycle may ask for the successor batch. Without
1897    /// it every member would ask for one of its own, since the request flags are per-cycle; see that
1898    /// field for why the last rather than the first.
1899    ///
1900    /// Before allocating, this enforces both halves of the admission reservation: no more than
1901    /// [`crate::MAX_OVERLAPPING_BATCHES`] live generations and no more than that many full batches'
1902    /// live cycles. A request earned while the reservation is full is retried after a generation
1903    /// releases its reconstructor state.
1904    ///
1905    /// What batching does change is the exposure *within* a batch: every cycle in it is unfunded at
1906    /// once, so the ceiling is `ssas_per_request` SSA quotas rather than one. That is the trade the
1907    /// knob exists to make, and it is why both deadlines are scaled by the same factor.
1908    fn emit_request_next_ssa(&mut self, now: Instant) -> Vec<SessionPixAction> {
1909        // A drain has no Session left to serve a successor's quota, so asking for one would commit
1910        // the Entry to a deposit against a cycle that could never be served. Guarded here rather than
1911        // at each of the four call sites — the early signal, the recovered transition, the deferred
1912        // retry and the deposit replay — so no route into a request can be missed.
1913        if self.draining {
1914            return Vec::new();
1915        }
1916
1917        let batch = self.cfg.ssas_per_request.clamp(1, crate::MAX_SSA_BATCH_SIZE);
1918        let live_cycles = self.live_cycle_count();
1919        let live_batches = self.live_batch_count();
1920        if live_batches >= crate::MAX_OVERLAPPING_BATCHES as usize
1921            || live_cycles.saturating_add(batch) > self.reserved_cycle_slots()
1922        {
1923            self.successor_request_deferred = true;
1924            tracing::debug!(
1925                live_cycles,
1926                live_batches,
1927                requested = batch,
1928                reserved = self.reserved_cycle_slots(),
1929                "deferring successor SSA batch until an older batch is released"
1930            );
1931            return Vec::new();
1932        }
1933
1934        self.successor_request_deferred = false;
1935        let batch_id = self.next_ssa_index;
1936        let mut ssa_ids = Vec::with_capacity(batch);
1937
1938        for _ in 0..batch {
1939            let index = self.next_ssa_index;
1940
1941            let Ok(ssa_index) = SsaIndex::try_from(index) else {
1942                break;
1943            };
1944            let Some(next) = index.checked_add(1) else {
1945                // Index space exhausted mid-batch. Stop here rather than wrapping: a reused index
1946                // would collide with a live cycle. Whatever was allocated still goes out.
1947                break;
1948            };
1949            self.next_ssa_index = next;
1950
1951            let ssa_id = SsaId::new(self.pseudonym, ssa_index);
1952
1953            // Register each SSA now rather than on `SsaRequestSent`. Carrying out the action registers
1954            // an Exit commitment, which is what makes shares for that SSA processable — so
1955            // observations about it can reach us before the confirmation that we asked for it does.
1956            // Every handler ignores an SSA it has no record of, and for `UnverifiableShares` that
1957            // would mean failing open on exactly the signal that must fail closed.
1958            //
1959            // No deadline yet: the request has not gone out, so there is nothing to be late for. That
1960            // is what `SsaRequestSent` adds, per index.
1961            self.ssas.push(PerSsaState::new(
1962                ssa_id,
1963                batch_id,
1964                self.dims.target_useful_shares(),
1965                now,
1966            ));
1967            // Counted at allocation rather than at `SsaRequestSent`, for the same reason the record
1968            // is created here: a cycle is registered before its request goes out, and observations
1969            // about it can arrive before the confirmation that it was asked for. A batch truncated
1970            // by index exhaustion counts only what it allocated.
1971            self.telemetry.requested = self.telemetry.requested.saturating_add(1);
1972            ssa_ids.push(ssa_id);
1973        }
1974
1975        // Nothing allocated means the index space is spent, and no further cycle can ever be funded.
1976        if ssa_ids.is_empty() {
1977            self.closed = true;
1978            return vec![SessionPixAction::Close(SessionPixCloseReason::InvalidTransition)];
1979        }
1980
1981        // The last cycle *actually* allocated carries the successor gate, which is what makes a
1982        // batch truncated by index exhaustion still able to ask for the next one.
1983        self.ssas
1984            .last_mut()
1985            .expect("a non-empty batch has just been pushed")
1986            .is_batch_last = true;
1987
1988        vec![SessionPixAction::RequestSsa {
1989            ssa_ids,
1990            params: self.dims,
1991        }]
1992    }
1993
1994    fn find_ssa_idx(&self, ssa_id: &SsaId<HoprPseudonym>) -> Option<usize> {
1995        self.ssas.iter().position(|s| s.ssa_id == *ssa_id)
1996    }
1997}
1998
1999// ---------------------------------------------------------------------------
2000// PerSsaState — deadline check (borrows immutably from self)
2001// ---------------------------------------------------------------------------
2002
2003impl PerSsaState {
2004    /// Check which deadline expired, if any.
2005    fn check_deadlines(&self, now: Instant) -> Option<SessionPixCloseReason> {
2006        match self.phase {
2007            SsaPhase::AwaitingCommitment => self
2008                .commitment_deadline
2009                .filter(|d| now >= *d)
2010                .map(|_| SessionPixCloseReason::CommitmentTimeout),
2011            SsaPhase::AwaitingDeposit => self
2012                .deposit_deadline
2013                .filter(|d| now >= *d)
2014                .map(|_| SessionPixCloseReason::DepositTimeout),
2015            SsaPhase::Recovering => {
2016                // Hard deadline is immutable.
2017                if let Some(d) = self.recovery_hard_deadline
2018                    && now >= d
2019                {
2020                    return Some(SessionPixCloseReason::RecoveryDeadline);
2021                }
2022                // Idle deadline — service gating happens in handle_deadline.
2023                if let Some(d) = self.recovery_idle_deadline
2024                    && now >= d
2025                {
2026                    return Some(SessionPixCloseReason::RecoveryIdle);
2027                }
2028                None
2029            }
2030            SsaPhase::Recovered { .. } | SsaPhase::Closing => None,
2031        }
2032    }
2033}
2034
2035// ---------------------------------------------------------------------------
2036// Tests
2037// ---------------------------------------------------------------------------
2038
2039#[cfg(test)]
2040mod tests {
2041    use std::time::{Duration, Instant};
2042
2043    use hopr_api::{
2044        HoprBalance,
2045        types::{crypto_random::Randomizable, internal::prelude::HoprPseudonym},
2046    };
2047    use hopr_protocol_pix::{SsaId, SsaIndex, SsaRecoveryProgress};
2048
2049    use super::{
2050        super::{FillRate, ORGANIC_WINDOW, PixFillConfig, SAMPLING_INTERVAL},
2051        *,
2052    };
2053
2054    /// A compact baseline for the state-machine tests — *not* [`SupervisorConfig::default`].
2055    ///
2056    /// Deadlines and budgets are shrunk to keep the tests short; the tolerances that remain are at
2057    /// their shipped values. Tests that care about a specific one say so by name.
2058    fn default_cfg() -> SupervisorConfig {
2059        SupervisorConfig {
2060            ssas_per_request: 1,
2061            allow_dynamic_ssa_batches: true,
2062            max_failed_cycles: 1,
2063            max_ssa_delivery_time: Duration::from_secs(20),
2064            commitment_recommit_interval: Duration::from_secs(3),
2065            max_deposit_wait: Duration::from_secs(60),
2066            max_recovery_idle: Duration::from_secs(60),
2067            max_recovery_time: Duration::from_secs(3600),
2068            max_off_front_share_fraction: 0.25,
2069            min_share_order_sample: 16384,
2070            max_predeposit_packets: 1024,
2071            max_served_without_progress: 256,
2072            tombstone_retention_window: Duration::from_secs(30),
2073            fill: PixFillConfig::default(),
2074        }
2075    }
2076
2077    /// Test dimensions, with a deliberately non-zero surplus.
2078    ///
2079    /// Surplus never enters the payment target, but it does bound the paid FIFO tail after recovery.
2080    /// A non-zero value makes both properties visible: `target_useful_shares` stays threshold-only,
2081    /// while accepted liveness cannot pass the emitted cycle volume.
2082    fn dims(polys: u16, threshold: u8) -> PixParams {
2083        PixParams::try_new(polys, threshold, 7, crate::types::LOCAL_PIX_SUITE).expect("test dimensions must be valid")
2084    }
2085
2086    fn pseudonym() -> HoprPseudonym {
2087        HoprPseudonym::random()
2088    }
2089
2090    fn ssa_id(p: HoprPseudonym, idx: u32) -> SsaId<HoprPseudonym> {
2091        SsaId::new(p, SsaIndex::new(idx).unwrap())
2092    }
2093
2094    /// A snapshot in which every share seen was useful — the shape of a cycle's first
2095    /// `threshold` shares per polynomial, and what every test predating the liveness split means.
2096    fn make_progress(
2097        ssa_id: SsaId<HoprPseudonym>,
2098        useful: u64,
2099        target: u64,
2100        recovered_polys: u16,
2101    ) -> SsaRecoveryProgress<HoprPseudonym> {
2102        make_progress_seen(ssa_id, useful, useful, target, recovered_polys)
2103    }
2104
2105    /// A snapshot where `seen` and `useful` can differ, i.e. the Entry is serving surplus.
2106    fn make_progress_seen(
2107        ssa_id: SsaId<HoprPseudonym>,
2108        useful: u64,
2109        seen: u64,
2110        target: u64,
2111        recovered_polys: u16,
2112    ) -> SsaRecoveryProgress<HoprPseudonym> {
2113        SsaRecoveryProgress {
2114            ssa_id,
2115            useful_shares: useful,
2116            shares_seen: seen,
2117            target_useful_shares: target,
2118            recovered_polynomials: recovered_polys,
2119        }
2120    }
2121
2122    fn make_recovered_tail_progress(
2123        sup: &SessionPixSupervisor,
2124        ssa_id: SsaId<HoprPseudonym>,
2125        shares_seen: u64,
2126    ) -> SsaRecoveryProgress<HoprPseudonym> {
2127        SsaRecoveryProgress {
2128            ssa_id,
2129            useful_shares: sup.dims.target_useful_shares(),
2130            shares_seen,
2131            target_useful_shares: sup.dims.target_useful_shares(),
2132            recovered_polynomials: sup.dims.polys_per_ssa(),
2133        }
2134    }
2135
2136    fn max_cycle_shares(sup: &SessionPixSupervisor) -> u64 {
2137        sup.dims.polys_per_ssa() as u64 * sup.dims.emitted_shares_per_poly() as u64
2138    }
2139
2140    fn sufficient_balance() -> HoprBalance {
2141        HoprBalance::new_base(1000)
2142    }
2143
2144    /// Drives every cycle of a batch to `Recovering`, so only the batch's own gates hold anything back.
2145    fn fund_batch(sup: &mut SessionPixSupervisor, p: HoprPseudonym, batch: u32, now: Instant) {
2146        for idx in 1..=batch {
2147            let id = ssa_id(p, idx);
2148            sup.handle_event(&SessionPixEvent::SsaRequestSent(id), now, 0);
2149            sup.handle_event(&SessionPixEvent::CommitmentVerified(id), now, 0);
2150            sup.handle_event(
2151                &SessionPixEvent::DepositConfirmed {
2152                    ssa_id: id,
2153                    amount: sufficient_balance(),
2154                },
2155                now,
2156                0,
2157            );
2158        }
2159    }
2160
2161    /// Drives a range of a batch's cycles to `AwaitingDeposit` and leaves them there.
2162    ///
2163    /// The complement of [`fund_batch`]: this is the state a cycle is *lost* from, and a funded one
2164    /// cannot be — `DepositObserverClosed` on a recovering cycle is a no-op.
2165    fn commit_unfunded(
2166        sup: &mut SessionPixSupervisor,
2167        p: HoprPseudonym,
2168        indices: std::ops::RangeInclusive<u32>,
2169        now: Instant,
2170    ) {
2171        for idx in indices {
2172            let id = ssa_id(p, idx);
2173            sup.handle_event(&SessionPixEvent::SsaRequestSent(id), now, 0);
2174            sup.handle_event(&SessionPixEvent::CommitmentVerified(id), now, 0);
2175        }
2176    }
2177
2178    fn _small_balance() -> HoprBalance {
2179        HoprBalance::new_base(1)
2180    }
2181
2182    /// Each phase is timed from when it was entered to when it was left, and only on leaving.
2183    #[test]
2184    fn a_cycle_is_timed_in_each_phase_it_leaves() {
2185        let p = pseudonym();
2186        let t0 = Instant::now();
2187        let (mut sup, _) = SessionPixSupervisor::new(default_cfg(), dims(10, 5), p, t0);
2188        let id = ssa_id(p, 1);
2189
2190        sup.handle_event(&SessionPixEvent::SsaRequestSent(id), t0, 0);
2191        assert!(
2192            sup.take_telemetry_events().phase_durations.is_empty(),
2193            "a cycle that has not left a phase has nothing to time"
2194        );
2195
2196        // Three seconds awaiting the commitment, then seven awaiting the deposit.
2197        sup.handle_event(&SessionPixEvent::CommitmentVerified(id), t0 + Duration::from_secs(3), 0);
2198        assert_eq!(
2199            vec![(PixCyclePhase::AwaitingCommitment, 3_000)],
2200            sup.take_telemetry_events().phase_durations
2201        );
2202
2203        sup.handle_event(
2204            &SessionPixEvent::DepositConfirmed {
2205                ssa_id: id,
2206                amount: sufficient_balance(),
2207            },
2208            t0 + Duration::from_secs(10),
2209            0,
2210        );
2211        assert_eq!(
2212            vec![(PixCyclePhase::AwaitingDeposit, 7_000)],
2213            sup.take_telemetry_events().phase_durations,
2214            "the clock restarts at each transition rather than measuring from the request"
2215        );
2216
2217        // Recovery runs for a minute. The tombstone it leaves is not a phase, so nothing further is
2218        // timed for this cycle.
2219        sup.handle_event(&SessionPixEvent::Recovered(id), t0 + Duration::from_secs(70), 0);
2220        assert_eq!(
2221            vec![(PixCyclePhase::Recovering, 60_000)],
2222            sup.take_telemetry_events().phase_durations
2223        );
2224    }
2225
2226    // ---------------------------------------------------------------
2227    // new / initial state
2228    // ---------------------------------------------------------------
2229
2230    #[test]
2231    fn new_emits_initial_request_for_index_one() {
2232        let p = pseudonym();
2233        let (sup, actions) = SessionPixSupervisor::new(default_cfg(), dims(10, 5), p, Instant::now());
2234
2235        assert_eq!(actions.len(), 1);
2236        match &actions[0] {
2237            SessionPixAction::RequestSsa { ssa_ids, params } => {
2238                assert_eq!(*params, dims(10, 5));
2239                assert_eq!(ssa_ids.len(), 1, "the default batch is a single SSA");
2240                assert_eq!(ssa_ids[0].ssa_index(), SsaIndex::new(1).unwrap());
2241            }
2242            other => panic!("expected RequestSsa, got {other:?}"),
2243        }
2244
2245        assert_eq!(sup.next_ssa_index, 2);
2246        assert!(!sup.closed);
2247
2248        // The SSA is tracked from the moment it is requested, so that an observation about it
2249        // cannot arrive before there is anywhere to record it.
2250        assert_eq!(sup.ssas.len(), 1);
2251        assert_eq!(sup.ssas[0].phase, SsaPhase::AwaitingCommitment);
2252        assert!(
2253            sup.ssas[0].commitment_deadline.is_none(),
2254            "the delivery clock starts when the request goes out, not when it is queued"
2255        );
2256    }
2257
2258    /// A batch is allocated and asked for as a unit: one action, `ssas_per_request` contiguous
2259    /// indices, one per-SSA record each, and the index counter advanced past all of them.
2260    ///
2261    /// One action rather than N is what puts the whole batch in a single `SsaRequest`, which is the
2262    /// point of the knob — and what makes the Entry's per-message cap the thing that has to accept it.
2263    #[test]
2264    fn a_batch_is_allocated_and_requested_as_one_action() {
2265        const BATCH: usize = 4;
2266
2267        let p = pseudonym();
2268        let cfg = SupervisorConfig {
2269            ssas_per_request: BATCH,
2270            ..default_cfg()
2271        };
2272        let (sup, actions) = SessionPixSupervisor::new(cfg, dims(10, 5), p, Instant::now());
2273
2274        assert_eq!(actions.len(), 1, "the batch must travel as a single action");
2275        match &actions[0] {
2276            SessionPixAction::RequestSsa { ssa_ids, .. } => {
2277                assert_eq!(
2278                    ssa_ids.iter().map(|i| i.ssa_index().get()).collect::<Vec<_>>(),
2279                    (1..=BATCH as u32).collect::<Vec<_>>(),
2280                    "the batch must cover contiguous indices from 1"
2281                );
2282            }
2283            other => panic!("expected RequestSsa, got {other:?}"),
2284        }
2285
2286        assert_eq!(
2287            sup.next_ssa_index,
2288            BATCH as u32 + 1,
2289            "the index counter must advance past the whole batch"
2290        );
2291        assert_eq!(
2292            sup.ssas.len(),
2293            BATCH,
2294            "every SSA in the batch needs its own record, or observations about it fail open"
2295        );
2296        assert!(sup.ssas.iter().all(|s| s.phase == SsaPhase::AwaitingCommitment));
2297        assert!(!sup.closed);
2298    }
2299
2300    /// Both per-cycle deadlines are multiplied by the batch size.
2301    ///
2302    /// The commitment clock must scale because a batch's clocks all start together while the Entry has
2303    /// that many commitment sets to produce; the deposit clock must scale because an Entry funding a
2304    /// batch in order finishes the last one that many deposits after the first. Without either, a peer
2305    /// answering correctly but in sequence is closed for being slow.
2306    #[test]
2307    fn batching_scales_both_per_cycle_deadlines() {
2308        const BATCH: usize = 3;
2309
2310        let p = pseudonym();
2311        let unit_commit = default_cfg().max_ssa_delivery_time;
2312        let unit_deposit = default_cfg().max_deposit_wait;
2313        let cfg = SupervisorConfig {
2314            ssas_per_request: BATCH,
2315            ..default_cfg()
2316        };
2317        let now = Instant::now();
2318        let (mut sup, actions) = SessionPixSupervisor::new(cfg, dims(10, 5), p, now);
2319        let SessionPixAction::RequestSsa { ssa_ids, .. } = &actions[0] else {
2320            panic!("expected RequestSsa");
2321        };
2322        let ssa_ids = ssa_ids.clone();
2323
2324        // Commitment clock: armed per index when the request goes out, at the scaled duration.
2325        for id in &ssa_ids {
2326            sup.handle_event(&SessionPixEvent::SsaRequestSent(*id), now, 0);
2327        }
2328        for (i, id) in ssa_ids.iter().enumerate() {
2329            let idx = sup.find_ssa_idx(id).expect("record must exist");
2330            assert_eq!(
2331                sup.ssas[idx].commitment_deadline,
2332                Some(now + BATCH as u32 * unit_commit),
2333                "cycle {i} must get the batch-scaled commitment deadline"
2334            );
2335        }
2336
2337        // Deposit clock: armed when that cycle's commitment verifies, also at the scaled duration.
2338        let later = now + Duration::from_secs(1);
2339        sup.handle_event(&SessionPixEvent::CommitmentVerified(ssa_ids[0]), later, 0);
2340        let idx = sup.find_ssa_idx(&ssa_ids[0]).expect("record must exist");
2341        assert_eq!(sup.ssas[idx].phase, SsaPhase::AwaitingDeposit);
2342        assert_eq!(
2343            sup.ssas[idx].deposit_deadline,
2344            Some(later + BATCH as u32 * unit_deposit),
2345            "the deposit deadline must be batch-scaled too"
2346        );
2347    }
2348
2349    /// An unvalidated `ssas_per_request` must not reach the deadline arithmetic or the allocator.
2350    ///
2351    /// A supervisor can be built from a config that never went through `validate_pix_supervision`, so
2352    /// zero must still request one SSA rather than none, and a value above the ceiling must be clamped
2353    /// rather than allocating an arbitrary batch or scaling a deadline past the duration cap.
2354    #[test]
2355    fn an_unvalidated_batch_size_is_clamped() {
2356        for (configured, expected) in [
2357            (0usize, 1usize),
2358            (crate::MAX_SSA_BATCH_SIZE + 5, crate::MAX_SSA_BATCH_SIZE),
2359        ] {
2360            let p = pseudonym();
2361            let cfg = SupervisorConfig {
2362                ssas_per_request: configured,
2363                ..default_cfg()
2364            };
2365            let (sup, actions) = SessionPixSupervisor::new(cfg, dims(10, 5), p, Instant::now());
2366            match &actions[0] {
2367                SessionPixAction::RequestSsa { ssa_ids, .. } => assert_eq!(
2368                    ssa_ids.len(),
2369                    expected,
2370                    "a configured batch of {configured} must be clamped to {expected}"
2371                ),
2372                other => panic!("expected RequestSsa, got {other:?}"),
2373            }
2374            assert_eq!(sup.ssas.len(), expected);
2375        }
2376    }
2377
2378    // ---------------------------------------------------------------
2379    // SsaRequestSent
2380    // ---------------------------------------------------------------
2381
2382    #[test]
2383    fn request_sent_starts_commitment_deadline() {
2384        let p = pseudonym();
2385        let (mut sup, _) = SessionPixSupervisor::new(default_cfg(), dims(10, 5), p, Instant::now());
2386        let now = Instant::now();
2387        let id = ssa_id(p, 1);
2388
2389        let actions = sup.handle_event(&SessionPixEvent::SsaRequestSent(id), now, 0);
2390        assert!(actions.is_empty());
2391
2392        let ssa = sup.ssas.iter().find(|s| s.ssa_id == id).unwrap();
2393        assert_eq!(ssa.phase, SsaPhase::AwaitingCommitment);
2394        assert!(ssa.commitment_deadline.is_some());
2395    }
2396
2397    #[test]
2398    fn request_sent_is_idempotent() {
2399        let p = pseudonym();
2400        let (mut sup, _) = SessionPixSupervisor::new(default_cfg(), dims(10, 5), p, Instant::now());
2401        let now = Instant::now();
2402        let id = ssa_id(p, 1);
2403
2404        sup.handle_event(&SessionPixEvent::SsaRequestSent(id), now, 0);
2405        let actions = sup.handle_event(&SessionPixEvent::SsaRequestSent(id), now, 0);
2406        assert!(actions.is_empty());
2407        assert_eq!(sup.ssas.len(), 1);
2408    }
2409
2410    #[test]
2411    fn request_sent_wrong_pseudonym_closes() {
2412        let p = pseudonym();
2413        let (mut sup, _) = SessionPixSupervisor::new(default_cfg(), dims(10, 5), p, Instant::now());
2414        let wrong_p = pseudonym();
2415        let now = Instant::now();
2416
2417        let actions = sup.handle_event(&SessionPixEvent::SsaRequestSent(ssa_id(wrong_p, 1)), now, 0);
2418        assert_eq!(actions.len(), 1);
2419        assert!(matches!(actions[0], SessionPixAction::Close(_)));
2420    }
2421
2422    // ---------------------------------------------------------------
2423    // CommitmentVerified
2424    // ---------------------------------------------------------------
2425
2426    #[test]
2427    fn commitment_verified_starts_the_deposit_deadline() {
2428        let p = pseudonym();
2429        let (mut sup, _) = SessionPixSupervisor::new(default_cfg(), dims(10, 5), p, Instant::now());
2430        let now = Instant::now();
2431        let id = ssa_id(p, 1);
2432
2433        sup.handle_event(&SessionPixEvent::SsaRequestSent(id), now, 0);
2434
2435        let actions = sup.handle_event(&SessionPixEvent::CommitmentVerified(id), now, 0);
2436        assert!(actions.is_empty());
2437
2438        let ssa = sup.ssas.iter().find(|s| s.ssa_id == id).unwrap();
2439        assert_eq!(ssa.phase, SsaPhase::AwaitingDeposit);
2440        assert!(ssa.deposit_deadline.is_some());
2441        assert!(ssa.commitment_deadline.is_none());
2442    }
2443
2444    #[test]
2445    fn commitment_verified_is_idempotent() {
2446        let p = pseudonym();
2447        let (mut sup, _) = SessionPixSupervisor::new(default_cfg(), dims(10, 5), p, Instant::now());
2448        let now = Instant::now();
2449        let id = ssa_id(p, 1);
2450
2451        sup.handle_event(&SessionPixEvent::SsaRequestSent(id), now, 0);
2452        sup.handle_event(&SessionPixEvent::CommitmentVerified(id), now, 0);
2453        let actions = sup.handle_event(&SessionPixEvent::CommitmentVerified(id), now, 0);
2454        assert!(actions.is_empty());
2455        assert_eq!(
2456            sup.ssas.iter().find(|s| s.ssa_id == id).unwrap().phase,
2457            SsaPhase::AwaitingDeposit
2458        );
2459    }
2460
2461    // ---------------------------------------------------------------
2462    // DepositConfirmed
2463    // ---------------------------------------------------------------
2464
2465    #[test]
2466    fn sufficient_deposit_enters_recovering_starts_idle_and_hard() {
2467        let p = pseudonym();
2468        let (mut sup, _) = SessionPixSupervisor::new(default_cfg(), dims(10, 5), p, Instant::now());
2469        let now = Instant::now();
2470        let id = ssa_id(p, 1);
2471
2472        sup.handle_event(&SessionPixEvent::SsaRequestSent(id), now, 0);
2473        sup.handle_event(&SessionPixEvent::CommitmentVerified(id), now, 0);
2474
2475        let actions = sup.handle_event(
2476            &SessionPixEvent::DepositConfirmed {
2477                ssa_id: id,
2478                amount: sufficient_balance(),
2479            },
2480            now,
2481            0,
2482        );
2483
2484        assert_eq!(actions.len(), 1);
2485        assert!(matches!(actions[0], SessionPixAction::ReleaseService));
2486
2487        let ssa = sup.ssas.iter().find(|s| s.ssa_id == id).unwrap();
2488        assert_eq!(ssa.phase, SsaPhase::Recovering);
2489        assert!(ssa.recovery_idle_deadline.is_some());
2490        assert!(ssa.recovery_hard_deadline.is_some());
2491        assert!(ssa.deposit_deadline.is_none());
2492    }
2493
2494    #[test]
2495    fn first_funding_emits_release_service_once() {
2496        let p = pseudonym();
2497        let (mut sup, _) = SessionPixSupervisor::new(default_cfg(), dims(10, 5), p, Instant::now());
2498        let now = Instant::now();
2499        let id1 = ssa_id(p, 1);
2500
2501        sup.handle_event(&SessionPixEvent::SsaRequestSent(id1), now, 0);
2502        sup.handle_event(&SessionPixEvent::CommitmentVerified(id1), now, 0);
2503
2504        let actions = sup.handle_event(
2505            &SessionPixEvent::DepositConfirmed {
2506                ssa_id: id1,
2507                amount: sufficient_balance(),
2508            },
2509            now,
2510            0,
2511        );
2512        assert_eq!(actions.len(), 1);
2513        assert!(matches!(actions[0], SessionPixAction::ReleaseService));
2514
2515        // Second deposit on a different SSA should not emit ReleaseService again.
2516        sup.handle_event(&SessionPixEvent::SsaRequestSent(ssa_id(p, 2)), now, 0);
2517        sup.handle_event(&SessionPixEvent::CommitmentVerified(ssa_id(p, 2)), now, 0);
2518        let actions = sup.handle_event(
2519            &SessionPixEvent::DepositConfirmed {
2520                ssa_id: ssa_id(p, 2),
2521                amount: sufficient_balance(),
2522            },
2523            now,
2524            0,
2525        );
2526        assert!(actions.iter().all(|a| !matches!(a, SessionPixAction::ReleaseService)));
2527    }
2528
2529    #[test]
2530    fn a_funded_successor_gets_a_fresh_service_ceiling_when_it_reaches_the_front() {
2531        let mut cfg = default_cfg();
2532        cfg.ssas_per_request = 2;
2533        cfg.max_served_without_progress = 2;
2534        let p = pseudonym();
2535        let now = Instant::now();
2536        let (mut sup, _) = SessionPixSupervisor::new(cfg, dims(10, 5), p, now);
2537
2538        fund_batch(&mut sup, p, 2, now);
2539        let actions = sup.handle_event(&SessionPixEvent::Recovered(ssa_id(p, 1)), now, 0);
2540        assert!(
2541            matches!(actions.as_slice(), [SessionPixAction::ProgressNotification]),
2542            "recovery resets the existing gate but is not the FIFO boundary: {actions:?}"
2543        );
2544
2545        let successor = sup.ssas.iter().find(|ssa| ssa.ssa_id == ssa_id(p, 2)).unwrap();
2546        assert_eq!(
2547            (successor.recovery_idle_deadline, successor.recovery_hard_deadline),
2548            (None, None),
2549            "successor clocks must not run while every served SURB still belongs to its predecessor"
2550        );
2551
2552        let target = sup.dims.target_useful_shares();
2553        let max_seen = max_cycle_shares(&sup);
2554        for (offset, seen) in ((target + 1)..=max_seen).enumerate() {
2555            let at = now + Duration::from_secs(offset as u64 + 1);
2556            let actions = sup.handle_event(
2557                &SessionPixEvent::RecoveryProgress(make_recovered_tail_progress(&sup, ssa_id(p, 1), seen)),
2558                at,
2559                offset as u64 + 1,
2560            );
2561            assert!(
2562                actions
2563                    .iter()
2564                    .any(|action| matches!(action, SessionPixAction::ProgressNotification)),
2565                "buffered paid share {seen} must refresh liveness: {actions:?}"
2566            );
2567            assert!(
2568                actions
2569                    .iter()
2570                    .all(|action| !matches!(action, SessionPixAction::Close(_))),
2571                "a tail longer than the gate ceiling must not close an honest Session: {actions:?}"
2572            );
2573            if seen == max_seen {
2574                assert!(
2575                    actions
2576                        .iter()
2577                        .any(|action| matches!(action, SessionPixAction::ReleaseService)),
2578                    "the funded successor must receive a fresh service ceiling at the exact FIFO boundary: {actions:?}"
2579                );
2580            }
2581        }
2582
2583        assert!(
2584            sup.paid_recovery_tail.is_none(),
2585            "the exact negotiated tail bound must finish the handoff"
2586        );
2587        let successor = sup.ssas.iter().find(|ssa| ssa.ssa_id == ssa_id(p, 2)).unwrap();
2588        assert!(
2589            successor.recovery_idle_deadline.is_some() && successor.recovery_hard_deadline.is_some(),
2590            "successor clocks start only after the final predecessor-bound SURB"
2591        );
2592    }
2593
2594    /// Any non-zero amount funds the cycle, because the amount is not the supervisor's to judge.
2595    ///
2596    /// The pool prices the quota and sends only once the deposit clears that price, so a confirmation
2597    /// is a verdict rather than evidence. A dust amount arriving here means the pool decided dust was
2598    /// enough — a pool bug, or an operator's deliberate pricing, and either way not something a floor
2599    /// in this crate could correct without overruling the component that knows the price.
2600    #[test]
2601    fn any_nonzero_deposit_funds_the_cycle() {
2602        for amount in [HoprBalance::new_base(1), sufficient_balance()] {
2603            let p = pseudonym();
2604            let (mut sup, _) = SessionPixSupervisor::new(default_cfg(), dims(10, 5), p, Instant::now());
2605            let now = Instant::now();
2606            let id = ssa_id(p, 1);
2607
2608            sup.handle_event(&SessionPixEvent::SsaRequestSent(id), now, 0);
2609            sup.handle_event(&SessionPixEvent::CommitmentVerified(id), now, 0);
2610
2611            let actions = sup.handle_event(&SessionPixEvent::DepositConfirmed { ssa_id: id, amount }, now, 0);
2612
2613            assert_eq!(
2614                sup.ssas.iter().find(|s| s.ssa_id == id).unwrap().phase,
2615                SsaPhase::Recovering,
2616                "a confirmation of {amount} is the pool's verdict and must fund the cycle"
2617            );
2618            assert!(!actions.is_empty());
2619        }
2620    }
2621
2622    /// A zero balance is the one confirmation that asserts nothing, and must not release service.
2623    ///
2624    /// It also must not consume the cycle's chance to be funded: the deadline keeps running, and a
2625    /// later confirmation still counts. That is what makes it safe for the observer to forward every
2626    /// message the pool sends rather than latching on the first.
2627    #[test]
2628    fn a_zero_deposit_is_not_a_verdict_and_leaves_the_cycle_fundable() {
2629        let p = pseudonym();
2630        let (mut sup, _) = SessionPixSupervisor::new(default_cfg(), dims(10, 5), p, Instant::now());
2631        let now = Instant::now();
2632        let id = ssa_id(p, 1);
2633
2634        sup.handle_event(&SessionPixEvent::SsaRequestSent(id), now, 0);
2635        sup.handle_event(&SessionPixEvent::CommitmentVerified(id), now, 0);
2636
2637        let deadline_before = sup.ssas.iter().find(|s| s.ssa_id == id).unwrap().deposit_deadline;
2638
2639        let actions = sup.handle_event(
2640            &SessionPixEvent::DepositConfirmed {
2641                ssa_id: id,
2642                amount: HoprBalance::zero(),
2643            },
2644            now,
2645            0,
2646        );
2647        assert!(actions.is_empty());
2648
2649        let ssa = sup.ssas.iter().find(|s| s.ssa_id == id).unwrap();
2650        assert_eq!(ssa.phase, SsaPhase::AwaitingDeposit);
2651        assert_eq!(
2652            ssa.deposit_deadline, deadline_before,
2653            "a zero confirmation must not extend or clear the deadline"
2654        );
2655
2656        // The next confirmation still funds it.
2657        let actions = sup.handle_event(
2658            &SessionPixEvent::DepositConfirmed {
2659                ssa_id: id,
2660                amount: sufficient_balance(),
2661            },
2662            now,
2663            0,
2664        );
2665        assert_eq!(
2666            sup.ssas.iter().find(|s| s.ssa_id == id).unwrap().phase,
2667            SsaPhase::Recovering
2668        );
2669        assert!(!actions.is_empty());
2670    }
2671
2672    /// Funding a queued cycle must not release service while an unfunded predecessor is live, but
2673    /// that funding must take effect if the predecessor is later retired. Otherwise the one-shot
2674    /// Session gate remains in predeposit mode even though the newly promoted front is funded, and
2675    /// no further deposit event exists to wake it.
2676    #[test]
2677    fn funded_successor_releases_service_when_unfunded_front_is_retired() {
2678        let mut cfg = default_cfg();
2679        cfg.ssas_per_request = 2;
2680        let p = pseudonym();
2681        let now = Instant::now();
2682        let (mut sup, _) = SessionPixSupervisor::new(cfg, dims(10, 5), p, now);
2683        let front = ssa_id(p, 1);
2684        let successor = ssa_id(p, 2);
2685
2686        for id in [front, successor] {
2687            sup.handle_event(&SessionPixEvent::SsaRequestSent(id), now, 0);
2688            sup.handle_event(&SessionPixEvent::CommitmentVerified(id), now, 0);
2689        }
2690
2691        let actions = sup.handle_event(
2692            &SessionPixEvent::DepositConfirmed {
2693                ssa_id: successor,
2694                amount: sufficient_balance(),
2695            },
2696            now,
2697            0,
2698        );
2699        assert!(
2700            actions.iter().all(|a| !matches!(a, SessionPixAction::ReleaseService)),
2701            "funding a cycle behind an unfunded front must not release service"
2702        );
2703
2704        let actions = sup.handle_event(&SessionPixEvent::DepositObserverClosed(front), now, 0);
2705        assert!(
2706            actions.iter().any(|a| matches!(a, SessionPixAction::ReleaseService)),
2707            "retiring the unfunded front must release service for its already-funded successor; got {actions:?}"
2708        );
2709    }
2710
2711    /// A cycle that funds and goes terminal in the same call must release only its own paid tail.
2712    ///
2713    /// `Recovered` can arrive while the cycle is still `AwaitingDeposit` — the Exit finishes
2714    /// reconstructing before the chain observer reports the deposit — and is deferred until the
2715    /// deposit lands. That one call then both funds the cycle and retires its heavy state, but the
2716    /// buffered SURBs were included in its paid quota. The gate stays funded only through that
2717    /// bounded tail, then restores the unfunded successor's predeposit allowance.
2718    #[test]
2719    fn a_cycle_that_funds_and_recovers_at_once_rearms_for_its_unfunded_successor() {
2720        let mut cfg = default_cfg();
2721        cfg.ssas_per_request = 2;
2722        let p = pseudonym();
2723        let now = Instant::now();
2724        let (mut sup, _) = SessionPixSupervisor::new(cfg, dims(10, 5), p, now);
2725        let front = ssa_id(p, 1);
2726        let successor = ssa_id(p, 2);
2727
2728        for id in [front, successor] {
2729            sup.handle_event(&SessionPixEvent::SsaRequestSent(id), now, 0);
2730            sup.handle_event(&SessionPixEvent::CommitmentVerified(id), now, 0);
2731        }
2732
2733        // Recovery completes before the deposit is observed, so the event is held.
2734        let actions = sup.handle_event(&SessionPixEvent::Recovered(front), now, 0);
2735        assert!(
2736            actions.iter().all(|a| !matches!(a, SessionPixAction::ReleaseService)),
2737            "recovering an unfunded cycle must not release service"
2738        );
2739
2740        // The deposit lands: the cycle funds and drops its heavy state in the same call. The compact
2741        // predecessor receipt may now release only the SURBs already paid by that deposit.
2742        let actions = sup.handle_event(
2743            &SessionPixEvent::DepositConfirmed {
2744                ssa_id: front,
2745                amount: sufficient_balance(),
2746            },
2747            now,
2748            0,
2749        );
2750        assert!(
2751            matches!(
2752                actions.as_slice(),
2753                [SessionPixAction::ReleaseService, SessionPixAction::ProgressNotification]
2754            ),
2755            "the paid handoff must open only the bounded predecessor tail, got {actions:?}"
2756        );
2757
2758        let max_seen = max_cycle_shares(&sup);
2759        let actions = sup.handle_event(
2760            &SessionPixEvent::RecoveryProgress(make_recovered_tail_progress(&sup, front, max_seen)),
2761            now,
2762            max_seen,
2763        );
2764        assert!(
2765            actions
2766                .iter()
2767                .any(|action| matches!(action, SessionPixAction::WithholdService)),
2768            "exhausting the paid tail must restore the unfunded successor's allowance: {actions:?}"
2769        );
2770
2771        let actions = sup.handle_event(
2772            &SessionPixEvent::DepositConfirmed {
2773                ssa_id: successor,
2774                amount: sufficient_balance(),
2775            },
2776            now,
2777            0,
2778        );
2779        assert!(
2780            actions.iter().any(|a| matches!(a, SessionPixAction::ReleaseService)),
2781            "the successor's own deposit must open service, got {actions:?}"
2782        );
2783    }
2784
2785    #[test]
2786    fn progress_on_an_unfunded_front_does_not_notify_the_service_gate() {
2787        let mut cfg = default_cfg();
2788        cfg.ssas_per_request = 2;
2789        cfg.max_off_front_share_fraction = 1.0;
2790        let p = pseudonym();
2791        let now = Instant::now();
2792        let (mut sup, _) = SessionPixSupervisor::new(cfg, dims(10, 5), p, now);
2793        let funded_front = ssa_id(p, 1);
2794        let unfunded_successor = ssa_id(p, 2);
2795
2796        for id in [funded_front, unfunded_successor] {
2797            sup.handle_event(&SessionPixEvent::SsaRequestSent(id), now, 0);
2798            sup.handle_event(&SessionPixEvent::CommitmentVerified(id), now, 0);
2799        }
2800        sup.handle_event(
2801            &SessionPixEvent::DepositConfirmed {
2802                ssa_id: funded_front,
2803                amount: sufficient_balance(),
2804            },
2805            now,
2806            0,
2807        );
2808        sup.handle_event(&SessionPixEvent::Recovered(funded_front), now, 0);
2809
2810        let actions = sup.handle_event(
2811            &SessionPixEvent::RecoveryProgress(make_progress(
2812                unfunded_successor,
2813                1,
2814                sup.dims.target_useful_shares(),
2815                0,
2816            )),
2817            now,
2818            1,
2819        );
2820
2821        assert!(
2822            actions
2823                .iter()
2824                .all(|action| !matches!(action, SessionPixAction::ProgressNotification)),
2825            "an unfunded front must not reopen the funded service ceiling, got {actions:?}"
2826        );
2827        assert!(
2828            actions
2829                .iter()
2830                .any(|action| matches!(action, SessionPixAction::WithholdService)),
2831            "the first successor-bound packet must close the predecessor's funded gate: {actions:?}"
2832        );
2833        assert!(
2834            sup.paid_recovery_tail.is_none(),
2835            "an unfunded successor must not trick the Exit into spending its predecessor's remaining allowance"
2836        );
2837    }
2838
2839    #[test]
2840    fn progress_behind_the_front_does_not_notify_the_service_gate() {
2841        let mut cfg = default_cfg();
2842        cfg.ssas_per_request = 2;
2843        cfg.max_off_front_share_fraction = 1.0;
2844        let p = pseudonym();
2845        let now = Instant::now();
2846        let (mut sup, _) = SessionPixSupervisor::new(cfg, dims(10, 5), p, now);
2847
2848        fund_batch(&mut sup, p, 2, now);
2849        let actions = sup.handle_event(
2850            &SessionPixEvent::RecoveryProgress(make_progress(ssa_id(p, 2), 1, sup.dims.target_useful_shares(), 0)),
2851            now,
2852            1,
2853        );
2854
2855        assert!(
2856            actions.is_empty(),
2857            "off-front progress must not reopen the gate, got {actions:?}"
2858        );
2859    }
2860
2861    /// A live cycle below the retired watermark must still accept its own request event.
2862    ///
2863    /// Retirement is not monotone: a later batch member can lose its deposit — timeout, observer
2864    /// closure — while an earlier one is still recovering, which puts the watermark *above* a live
2865    /// index. A staleness guard consulted before the record lookup would then silently drop the
2866    /// earlier cycle's `SsaRequestSent`, leaving its commitment deadline unarmed and the Entry free
2867    /// to never answer. Looking the record up first is what removes the assumption.
2868    #[test]
2869    fn a_live_cycle_below_the_retired_watermark_still_arms_its_deadline() {
2870        let mut cfg = default_cfg();
2871        cfg.ssas_per_request = 2;
2872        let p = pseudonym();
2873        let now = Instant::now();
2874        let (mut sup, _) = SessionPixSupervisor::new(cfg, dims(10, 5), p, now);
2875        let earlier = ssa_id(p, 1);
2876        let later = ssa_id(p, 2);
2877
2878        // The later member is retired first, which puts the watermark above the earlier one.
2879        sup.handle_event(&SessionPixEvent::SsaRequestSent(later), now, 0);
2880        sup.handle_event(&SessionPixEvent::CommitmentVerified(later), now, 0);
2881        sup.handle_event(&SessionPixEvent::DepositObserverClosed(later), now, 0);
2882        assert_eq!(
2883            Some(later.ssa_index()),
2884            sup.highest_retired_ssa_index,
2885            "the later member must have set the watermark"
2886        );
2887
2888        // The earlier member is still live, and its request event must still arm the clock.
2889        sup.handle_event(&SessionPixEvent::SsaRequestSent(earlier), now, 0);
2890        let armed = sup
2891            .ssas
2892            .iter()
2893            .find(|s| s.ssa_id == earlier)
2894            .expect("the earlier cycle must still be live")
2895            .commitment_deadline;
2896        assert!(
2897            armed.is_some(),
2898            "a live cycle below the watermark must still have its commitment deadline armed"
2899        );
2900    }
2901
2902    /// And a straggler for an index that really is gone stays benign rather than closing the Session.
2903    ///
2904    /// A batch of two, so retiring one member leaves the Session alive — retiring the last live cycle
2905    /// closes it outright, and a closed supervisor ignores everything, which would make the second
2906    /// half of this test vacuous.
2907    #[test]
2908    fn a_request_event_for_a_retired_index_is_ignored() {
2909        let mut cfg = default_cfg();
2910        cfg.ssas_per_request = 2;
2911        let p = pseudonym();
2912        let now = Instant::now();
2913        let (mut sup, _) = SessionPixSupervisor::new(cfg, dims(10, 5), p, now);
2914        let retired = ssa_id(p, 1);
2915
2916        sup.handle_event(&SessionPixEvent::SsaRequestSent(retired), now, 0);
2917        sup.handle_event(&SessionPixEvent::CommitmentVerified(retired), now, 0);
2918        sup.handle_event(&SessionPixEvent::DepositObserverClosed(retired), now, 0);
2919        assert!(!sup.closed, "the second batch member must keep the Session alive");
2920
2921        let actions = sup.handle_event(&SessionPixEvent::SsaRequestSent(retired), now, 0);
2922        assert!(
2923            actions.iter().all(|a| !matches!(a, SessionPixAction::Close(_))),
2924            "a straggling request for a retired index must be ignored, not fatal; got {actions:?}"
2925        );
2926
2927        // An index we were never asked for is a different matter — above the watermark and with no
2928        // record, it can only be an SSA this supervisor never registered.
2929        let actions = sup.handle_event(&SessionPixEvent::SsaRequestSent(ssa_id(p, 9)), now, 0);
2930        assert!(
2931            actions
2932                .iter()
2933                .any(|a| matches!(a, SessionPixAction::Close(SessionPixCloseReason::InvalidTransition))),
2934            "an index above the watermark with no record is a protocol fault; got {actions:?}"
2935        );
2936    }
2937
2938    #[test]
2939    fn duplicate_deposit_confirmation_is_idempotent() {
2940        let p = pseudonym();
2941        let (mut sup, _) = SessionPixSupervisor::new(default_cfg(), dims(10, 5), p, Instant::now());
2942        let now = Instant::now();
2943        let id = ssa_id(p, 1);
2944
2945        sup.handle_event(&SessionPixEvent::SsaRequestSent(id), now, 0);
2946        sup.handle_event(&SessionPixEvent::CommitmentVerified(id), now, 0);
2947
2948        sup.handle_event(
2949            &SessionPixEvent::DepositConfirmed {
2950                ssa_id: id,
2951                amount: sufficient_balance(),
2952            },
2953            now,
2954            0,
2955        );
2956
2957        let actions = sup.handle_event(
2958            &SessionPixEvent::DepositConfirmed {
2959                ssa_id: id,
2960                amount: sufficient_balance(),
2961            },
2962            now,
2963            0,
2964        );
2965        assert!(actions.is_empty());
2966    }
2967
2968    #[test]
2969    fn wrong_ssa_deposit_does_not_transition_any_ssa() {
2970        let p = pseudonym();
2971        let (mut sup, _) = SessionPixSupervisor::new(default_cfg(), dims(10, 5), p, Instant::now());
2972        let now = Instant::now();
2973
2974        sup.handle_event(&SessionPixEvent::SsaRequestSent(ssa_id(p, 1)), now, 0);
2975
2976        let actions = sup.handle_event(
2977            &SessionPixEvent::DepositConfirmed {
2978                ssa_id: ssa_id(p, 99),
2979                amount: sufficient_balance(),
2980            },
2981            now,
2982            0,
2983        );
2984        assert!(actions.is_empty());
2985    }
2986
2987    // ---------------------------------------------------------------
2988    // Deadlines
2989    // ---------------------------------------------------------------
2990
2991    #[test]
2992    fn commitment_deadline_expiry_closes_awaiting_commitment() {
2993        let p = pseudonym();
2994        let (mut sup, _) = SessionPixSupervisor::new(default_cfg(), dims(10, 5), p, Instant::now());
2995        let start = Instant::now();
2996        let id = ssa_id(p, 1);
2997
2998        sup.handle_event(&SessionPixEvent::SsaRequestSent(id), start, 0);
2999
3000        let actions = sup.handle_deadline(start + Duration::from_secs(10), 0);
3001        assert!(actions.is_empty());
3002
3003        let actions = sup.handle_deadline(start + Duration::from_secs(21), 0);
3004        assert_eq!(actions.len(), 1);
3005        assert!(matches!(
3006            actions[0],
3007            SessionPixAction::Close(SessionPixCloseReason::CommitmentTimeout)
3008        ));
3009    }
3010
3011    /// A commitment that is under way but has gone quiet is re-requested, repeatedly, without ever
3012    /// being closed for it — and the absolute deadline is still what ends the cycle.
3013    #[test]
3014    fn a_stalled_commitment_is_re_requested_until_the_delivery_deadline() {
3015        let p = pseudonym();
3016        let cfg = default_cfg();
3017        let (mut sup, _) = SessionPixSupervisor::new(cfg.clone(), dims(10, 5), p, Instant::now());
3018        let start = Instant::now();
3019        let id = ssa_id(p, 1);
3020
3021        sup.handle_event(&SessionPixEvent::SsaRequestSent(id), start, 0);
3022
3023        // Nothing has arrived, so there is no scope to ask for and no timer to expire.
3024        assert!(
3025            sup.handle_deadline(start + cfg.commitment_recommit_interval * 3, 0)
3026                .is_empty(),
3027            "a commitment nothing was delivered for must not be re-requested"
3028        );
3029
3030        // The first part arrives; the timer is armed from *that* instant.
3031        let first_part = start + Duration::from_secs(1);
3032        sup.handle_event(&SessionPixEvent::CommitmentProgress(id), first_part, 0);
3033        assert!(
3034            sup.handle_deadline(first_part + cfg.commitment_recommit_interval / 2, 0)
3035                .is_empty(),
3036            "the interval must be measured from the last part that arrived"
3037        );
3038
3039        // More of the burst lands, pushing the timer out again: the first ask must not land in the
3040        // middle of delivery.
3041        let last_part = first_part + cfg.commitment_recommit_interval;
3042        sup.handle_event(&SessionPixEvent::CommitmentProgress(id), last_part, 0);
3043        assert!(
3044            sup.handle_deadline(last_part + cfg.commitment_recommit_interval / 2, 0)
3045                .is_empty(),
3046            "an arriving part must re-arm the timer"
3047        );
3048
3049        // Delivery stalls. Every interval from here produces one ask, and no close, for as long as
3050        // the delivery deadline has room — at the shipped 3 s inside 20 s that is five of them, so
3051        // the count cap is not what binds at the defaults.
3052        let delivery_deadline =
3053            start + crate::supervision::scaled_deadline(cfg.max_ssa_delivery_time, cfg.ssas_per_request);
3054        let mut now = last_part;
3055        let mut asks = 0;
3056        while now + cfg.commitment_recommit_interval < delivery_deadline {
3057            now += cfg.commitment_recommit_interval;
3058            let actions = sup.handle_deadline(now, 0);
3059            assert!(
3060                matches!(
3061                    actions.as_slice(),
3062                    [SessionPixAction::RequestCommitmentRetransmission { ssa_id, params }]
3063                        if *ssa_id == id && *params == dims(10, 5)
3064                ),
3065                "ask {} must name the missing parts, got {actions:?}",
3066                asks + 1
3067            );
3068            asks += 1;
3069        }
3070        assert!(asks > 1, "the interval must fit several asks inside the deadline");
3071
3072        // The delivery deadline is untouched by any of it, and remains the only thing that ends the
3073        // cycle.
3074        let actions = sup.handle_deadline(delivery_deadline, 0);
3075        assert!(matches!(
3076            actions.as_slice(),
3077            [SessionPixAction::Close(SessionPixCloseReason::CommitmentTimeout)]
3078        ));
3079    }
3080
3081    /// Asks stop at [`MAX_COMMITMENT_RETRANSMISSIONS`], which is what the Entry will answer.
3082    ///
3083    /// The delivery deadline bounds the asks in *time*, and that is what protects the Entry — but the
3084    /// packets they cost are the Exit's own, and a short interval inside a long deadline leaves time
3085    /// for far more of them than any Entry answers. Hence a count.
3086    #[test]
3087    fn asks_for_a_stalled_commitment_are_capped_by_count() {
3088        let p = pseudonym();
3089        // Short enough that the whole budget is spent well inside the delivery deadline, so it is the
3090        // count and not the clock that stops it.
3091        let cfg = SupervisorConfig {
3092            commitment_recommit_interval: Duration::from_secs(1),
3093            ..default_cfg()
3094        };
3095        let (mut sup, _) = SessionPixSupervisor::new(cfg.clone(), dims(10, 5), p, Instant::now());
3096        let start = Instant::now();
3097        let id = ssa_id(p, 1);
3098
3099        sup.handle_event(&SessionPixEvent::SsaRequestSent(id), start, 0);
3100        sup.handle_event(&SessionPixEvent::CommitmentProgress(id), start, 0);
3101
3102        let mut now = start;
3103        for attempt in 1..=hopr_protocol_pix::MAX_COMMITMENT_RETRANSMISSIONS {
3104            now += cfg.commitment_recommit_interval;
3105            let actions = sup.handle_deadline(now, 0);
3106            assert!(
3107                matches!(
3108                    actions.as_slice(),
3109                    [SessionPixAction::RequestCommitmentRetransmission { ssa_id, .. }] if *ssa_id == id
3110                ),
3111                "attempt {attempt} must ask for the missing parts, got {actions:?}"
3112            );
3113        }
3114
3115        now += cfg.commitment_recommit_interval;
3116        assert!(
3117            sup.handle_deadline(now, 0).is_empty(),
3118            "the ask budget must be spent, with the delivery deadline still far off"
3119        );
3120        assert_eq!(
3121            Some(start + crate::supervision::scaled_deadline(cfg.max_ssa_delivery_time, cfg.ssas_per_request)),
3122            sup.next_deadline(),
3123            "a spent budget must leave no timer for the worker to wake on"
3124        );
3125
3126        // Further parts of the burst cannot buy more asks either.
3127        sup.handle_event(&SessionPixEvent::CommitmentProgress(id), now, 0);
3128        assert!(
3129            sup.handle_deadline(now + cfg.commitment_recommit_interval * 2, 0)
3130                .is_empty(),
3131            "an arriving part must not re-open a spent budget"
3132        );
3133    }
3134
3135    /// The re-request timer is the earliest deadline of the phase it runs in, so the worker has to be
3136    /// woken by it — and it must stop existing the moment the commitment completes.
3137    #[test]
3138    fn the_re_request_timer_leads_the_deadlines_and_ends_with_the_commitment() {
3139        let p = pseudonym();
3140        let cfg = default_cfg();
3141        let (mut sup, _) = SessionPixSupervisor::new(cfg.clone(), dims(10, 5), p, Instant::now());
3142        let start = Instant::now();
3143        let id = ssa_id(p, 1);
3144
3145        sup.handle_event(&SessionPixEvent::SsaRequestSent(id), start, 0);
3146        sup.handle_event(&SessionPixEvent::CommitmentProgress(id), start, 0);
3147        assert_eq!(
3148            Some(start + cfg.commitment_recommit_interval),
3149            sup.next_deadline(),
3150            "the worker must wake for the re-request before the delivery deadline"
3151        );
3152
3153        sup.handle_event(&SessionPixEvent::CommitmentVerified(id), start, 0);
3154        assert_eq!(
3155            Some(start + crate::supervision::scaled_deadline(cfg.max_deposit_wait, cfg.ssas_per_request)),
3156            sup.next_deadline(),
3157            "a completed commitment leaves only the deposit clock"
3158        );
3159        assert!(
3160            sup.handle_deadline(start + cfg.commitment_recommit_interval, 0)
3161                .is_empty(),
3162            "nothing may be re-requested once the commitment verified"
3163        );
3164
3165        // A straggling part of the burst arrives after completion — which retransmission makes
3166        // routine — and must not resurrect the timer.
3167        sup.handle_event(
3168            &SessionPixEvent::CommitmentProgress(id),
3169            start + cfg.commitment_recommit_interval,
3170            0,
3171        );
3172        assert!(
3173            sup.handle_deadline(start + cfg.commitment_recommit_interval * 3, 0)
3174                .is_empty(),
3175            "a late part must not re-arm a timer the phase has left"
3176        );
3177    }
3178
3179    #[test]
3180    fn deposit_deadline_expiry_closes_awaiting_deposit() {
3181        let p = pseudonym();
3182        let (mut sup, _) = SessionPixSupervisor::new(default_cfg(), dims(10, 5), p, Instant::now());
3183        let start = Instant::now();
3184        let id = ssa_id(p, 1);
3185
3186        sup.handle_event(&SessionPixEvent::SsaRequestSent(id), start, 0);
3187        sup.handle_event(&SessionPixEvent::CommitmentVerified(id), start, 0);
3188
3189        let actions = sup.handle_deadline(start + Duration::from_secs(61), 0);
3190        assert_eq!(actions.len(), 1);
3191        assert!(matches!(
3192            actions[0],
3193            SessionPixAction::Close(SessionPixCloseReason::DepositTimeout)
3194        ));
3195    }
3196
3197    // ---------------------------------------------------------------
3198    // DepositObserverClosed
3199    // ---------------------------------------------------------------
3200
3201    #[test]
3202    fn deposit_observer_closed_closes_with_distinct_reason() {
3203        let p = pseudonym();
3204        let (mut sup, _) = SessionPixSupervisor::new(default_cfg(), dims(10, 5), p, Instant::now());
3205        let now = Instant::now();
3206        let id = ssa_id(p, 1);
3207
3208        sup.handle_event(&SessionPixEvent::SsaRequestSent(id), now, 0);
3209        sup.handle_event(&SessionPixEvent::CommitmentVerified(id), now, 0);
3210
3211        let actions = sup.handle_event(&SessionPixEvent::DepositObserverClosed(id), now, 0);
3212        assert_eq!(actions.len(), 1);
3213        assert!(matches!(
3214            actions[0],
3215            SessionPixAction::Close(SessionPixCloseReason::DepositObserverClosed)
3216        ));
3217    }
3218
3219    // ---------------------------------------------------------------
3220    // Stale timer safety
3221    // ---------------------------------------------------------------
3222
3223    #[test]
3224    fn stale_timer_wake_after_transition_does_not_close() {
3225        let p = pseudonym();
3226        let (mut sup, _) = SessionPixSupervisor::new(default_cfg(), dims(10, 5), p, Instant::now());
3227        let start = Instant::now();
3228        let id = ssa_id(p, 1);
3229
3230        sup.handle_event(&SessionPixEvent::SsaRequestSent(id), start, 0);
3231        sup.handle_event(&SessionPixEvent::CommitmentVerified(id), start, 0);
3232
3233        // Deposit arrives before deadline.
3234        sup.handle_event(
3235            &SessionPixEvent::DepositConfirmed {
3236                ssa_id: id,
3237                amount: sufficient_balance(),
3238            },
3239            start,
3240            0,
3241        );
3242
3243        let actions = sup.handle_deadline(start + Duration::from_secs(120), 0);
3244        assert!(
3245            actions
3246                .iter()
3247                .all(|a| { !matches!(a, SessionPixAction::Close(SessionPixCloseReason::DepositTimeout)) })
3248        );
3249    }
3250
3251    // ---------------------------------------------------------------
3252    // Progress
3253    // ---------------------------------------------------------------
3254
3255    #[test]
3256    fn useful_progress_extends_idle_only_hard_never_moves() {
3257        let p = pseudonym();
3258        let mut cfg = default_cfg();
3259        cfg.max_recovery_idle = Duration::from_secs(60);
3260        let (mut sup, _) = SessionPixSupervisor::new(cfg, dims(10, 5), p, Instant::now());
3261        let start = Instant::now();
3262        let id = ssa_id(p, 1);
3263
3264        sup.handle_event(&SessionPixEvent::SsaRequestSent(id), start, 0);
3265        sup.handle_event(&SessionPixEvent::CommitmentVerified(id), start, 0);
3266        sup.handle_event(
3267            &SessionPixEvent::DepositConfirmed {
3268                ssa_id: id,
3269                amount: sufficient_balance(),
3270            },
3271            start,
3272            0,
3273        );
3274
3275        let hard_before = sup.ssas.iter().find(|s| s.ssa_id == id).unwrap().recovery_hard_deadline;
3276        let idle_before = sup.ssas.iter().find(|s| s.ssa_id == id).unwrap().recovery_idle_deadline;
3277
3278        sup.handle_event(
3279            &SessionPixEvent::RecoveryProgress(make_progress(id, 10, 50, 1)),
3280            start + Duration::from_secs(55),
3281            5,
3282        );
3283
3284        let ssa = sup.ssas.iter().find(|s| s.ssa_id == id).unwrap();
3285        assert_eq!(ssa.recovery_hard_deadline, hard_before);
3286        assert!(ssa.recovery_idle_deadline > idle_before);
3287    }
3288
3289    #[test]
3290    fn hard_deadline_immutable_under_trickle_progress() {
3291        let p = pseudonym();
3292        let mut cfg = default_cfg();
3293        cfg.max_recovery_idle = Duration::from_secs(10);
3294        cfg.max_recovery_time = Duration::from_secs(30);
3295        let (mut sup, _) = SessionPixSupervisor::new(cfg, dims(10, 5), p, Instant::now());
3296        let start = Instant::now();
3297        let id = ssa_id(p, 1);
3298
3299        sup.handle_event(&SessionPixEvent::SsaRequestSent(id), start, 0);
3300        sup.handle_event(&SessionPixEvent::CommitmentVerified(id), start, 0);
3301        sup.handle_event(
3302            &SessionPixEvent::DepositConfirmed {
3303                ssa_id: id,
3304                amount: sufficient_balance(),
3305            },
3306            start,
3307            0,
3308        );
3309
3310        for secs in [9u64, 19, 29] {
3311            sup.handle_event(
3312                &SessionPixEvent::RecoveryProgress(make_progress(id, secs + 1, 50, 1)),
3313                start + Duration::from_secs(secs),
3314                secs,
3315            );
3316        }
3317
3318        let actions = sup.handle_deadline(start + Duration::from_secs(31), 30);
3319        assert_eq!(actions.len(), 1);
3320        match &actions[0] {
3321            SessionPixAction::Close(SessionPixCloseReason::RecoveryDeadline) => {}
3322            other => panic!("expected RecoveryDeadline, got {other:?}"),
3323        }
3324    }
3325
3326    #[test]
3327    fn idle_expiry_without_service_since_progress_rearms() {
3328        let p = pseudonym();
3329        let mut cfg = default_cfg();
3330        cfg.max_recovery_idle = Duration::from_secs(10);
3331        let (mut sup, _) = SessionPixSupervisor::new(cfg, dims(10, 5), p, Instant::now());
3332        let start = Instant::now();
3333        let id = ssa_id(p, 1);
3334
3335        sup.handle_event(&SessionPixEvent::SsaRequestSent(id), start, 0);
3336        sup.handle_event(&SessionPixEvent::CommitmentVerified(id), start, 0);
3337        sup.handle_event(
3338            &SessionPixEvent::DepositConfirmed {
3339                ssa_id: id,
3340                amount: sufficient_balance(),
3341            },
3342            start,
3343            0,
3344        );
3345
3346        let actions = sup.handle_deadline(start + Duration::from_secs(11), 0);
3347        assert!(
3348            !actions
3349                .iter()
3350                .any(|a| matches!(a, SessionPixAction::Close(SessionPixCloseReason::RecoveryIdle)))
3351        );
3352
3353        // SSA should still be in Recovering.
3354        assert_eq!(
3355            sup.ssas.iter().find(|s| s.ssa_id == id).unwrap().phase,
3356            SsaPhase::Recovering
3357        );
3358    }
3359
3360    #[test]
3361    fn idle_expiry_with_service_and_no_progress_closes_recovery_idle() {
3362        let p = pseudonym();
3363        let mut cfg = default_cfg();
3364        cfg.max_recovery_idle = Duration::from_secs(10);
3365        let (mut sup, _) = SessionPixSupervisor::new(cfg, dims(10, 5), p, Instant::now());
3366        let start = Instant::now();
3367        let id = ssa_id(p, 1);
3368
3369        sup.handle_event(&SessionPixEvent::SsaRequestSent(id), start, 0);
3370        sup.handle_event(&SessionPixEvent::CommitmentVerified(id), start, 0);
3371        sup.handle_event(
3372            &SessionPixEvent::DepositConfirmed {
3373                ssa_id: id,
3374                amount: sufficient_balance(),
3375            },
3376            start,
3377            5,
3378        );
3379
3380        // Idle fires with served_total=5 (same as watermark) → re-arm.
3381        let actions = sup.handle_deadline(start + Duration::from_secs(11), 5);
3382        assert!(
3383            !actions
3384                .iter()
3385                .any(|a| matches!(a, SessionPixAction::Close(SessionPixCloseReason::RecoveryIdle)))
3386        );
3387
3388        // Now served_total increased to 10 (5 consumed since progress) → close.
3389        let actions = sup.handle_deadline(start + Duration::from_secs(22), 10);
3390        assert!(
3391            actions
3392                .iter()
3393                .any(|a| matches!(a, SessionPixAction::Close(SessionPixCloseReason::RecoveryIdle)))
3394        );
3395    }
3396
3397    #[test]
3398    fn progress_resamples_served_total_watermark() {
3399        let p = pseudonym();
3400        let mut cfg = default_cfg();
3401        cfg.max_recovery_idle = Duration::from_secs(10);
3402        let (mut sup, _) = SessionPixSupervisor::new(cfg, dims(10, 5), p, Instant::now());
3403        let start = Instant::now();
3404        let id = ssa_id(p, 1);
3405
3406        sup.handle_event(&SessionPixEvent::SsaRequestSent(id), start, 0);
3407        sup.handle_event(&SessionPixEvent::CommitmentVerified(id), start, 0);
3408        sup.handle_event(
3409            &SessionPixEvent::DepositConfirmed {
3410                ssa_id: id,
3411                amount: sufficient_balance(),
3412            },
3413            start,
3414            10,
3415        );
3416
3417        sup.handle_event(
3418            &SessionPixEvent::RecoveryProgress(make_progress(id, 5, 50, 1)),
3419            start + Duration::from_secs(5),
3420            15,
3421        );
3422
3423        let actions = sup.handle_deadline(start + Duration::from_secs(16), 15);
3424        assert!(
3425            !actions
3426                .iter()
3427                .any(|a| matches!(a, SessionPixAction::Close(SessionPixCloseReason::RecoveryIdle)))
3428        );
3429    }
3430
3431    /// A snapshot carrying only surplus must keep the cycle alive without paying it anything.
3432    ///
3433    /// This is the whole of C1 at the supervisor. A conforming Entry drains an emission window's
3434    /// surplus in one contiguous run — `surplus × min(polys, 256)` packets, 4096 at the shipped
3435    /// dimensions — during which `useful_shares` does not move at all. Every one of those packets is
3436    /// service the Exit is consuming, so a supervisor that keyed liveness on `useful_shares` would
3437    /// let the run exhaust `max_served_without_progress`, park the writer, and then close an honest
3438    /// Session with `RecoveryIdle` — the re-arm branch cannot save it, because service *was*
3439    /// consumed. Liveness must therefore move on `shares_seen`, and payment must not.
3440    #[test]
3441    fn a_surplus_only_snapshot_is_liveness_without_payment() {
3442        let p = pseudonym();
3443        let mut cfg = default_cfg();
3444        cfg.max_recovery_idle = Duration::from_secs(10);
3445        let (mut sup, _) = SessionPixSupervisor::new(cfg, dims(10, 5), p, Instant::now());
3446        let start = Instant::now();
3447        let id = ssa_id(p, 1);
3448
3449        sup.handle_event(&SessionPixEvent::SsaRequestSent(id), start, 0);
3450        sup.handle_event(&SessionPixEvent::CommitmentVerified(id), start, 0);
3451        sup.handle_event(
3452            &SessionPixEvent::DepositConfirmed {
3453                ssa_id: id,
3454                amount: sufficient_balance(),
3455            },
3456            start,
3457            0,
3458        );
3459
3460        // One useful share, to establish a baseline both counters agree on.
3461        sup.handle_event(
3462            &SessionPixEvent::RecoveryProgress(make_progress(id, 1, 50, 0)),
3463            start + Duration::from_secs(1),
3464            10,
3465        );
3466
3467        // Then a run of pure surplus: `shares_seen` climbs, `useful_shares` is pinned, and service is
3468        // being consumed throughout.
3469        for (i, seen) in (2..=6u64).enumerate() {
3470            let actions = sup.handle_event(
3471                &SessionPixEvent::RecoveryProgress(make_progress_seen(id, 1, seen, 50, 0)),
3472                start + Duration::from_secs(2 + i as u64),
3473                20 + 10 * i as u64,
3474            );
3475            assert!(
3476                actions
3477                    .iter()
3478                    .any(|a| matches!(a, SessionPixAction::ProgressNotification)),
3479                "a surplus share must reset the gate's served-without-progress ceiling"
3480            );
3481            assert!(
3482                !actions.iter().any(|a| matches!(a, SessionPixAction::Close(_))),
3483                "a surplus share must never close the Session"
3484            );
3485        }
3486
3487        // The idle deadline was refreshed by the surplus run, so it has not expired — even though
3488        // `served_total` has grown well past the watermark of the last *useful* share.
3489        let actions = sup.handle_deadline(start + Duration::from_secs(12), 70);
3490        assert!(
3491            !actions
3492                .iter()
3493                .any(|a| matches!(a, SessionPixAction::Close(SessionPixCloseReason::RecoveryIdle))),
3494            "an Entry serving its negotiated surplus is not idle"
3495        );
3496
3497        // Payment did not move: the surplus bought the Entry time, not credit.
3498        let ssa = sup.ssas.iter().find(|s| s.ssa_id == id).expect("cycle must be live");
3499        assert_eq!(1, ssa.largest_useful_shares, "surplus must not count as useful shares");
3500        assert_eq!(6, ssa.largest_shares_seen, "every share must count as liveness");
3501    }
3502
3503    /// Once the surplus run stops, the idle rule must still fire.
3504    ///
3505    /// The liveness split widens what counts as "the Entry is alive"; it must not make the anti-drip
3506    /// bound unreachable. With no snapshot of any kind and service still being consumed, the Session
3507    /// closes exactly as it did before.
3508    #[test]
3509    fn silence_after_a_surplus_run_still_closes_the_session() {
3510        let p = pseudonym();
3511        let mut cfg = default_cfg();
3512        cfg.max_recovery_idle = Duration::from_secs(10);
3513        let (mut sup, _) = SessionPixSupervisor::new(cfg, dims(10, 5), p, Instant::now());
3514        let start = Instant::now();
3515        let id = ssa_id(p, 1);
3516
3517        sup.handle_event(&SessionPixEvent::SsaRequestSent(id), start, 0);
3518        sup.handle_event(&SessionPixEvent::CommitmentVerified(id), start, 0);
3519        sup.handle_event(
3520            &SessionPixEvent::DepositConfirmed {
3521                ssa_id: id,
3522                amount: sufficient_balance(),
3523            },
3524            start,
3525            0,
3526        );
3527        sup.handle_event(
3528            &SessionPixEvent::RecoveryProgress(make_progress_seen(id, 1, 4, 50, 0)),
3529            start + Duration::from_secs(1),
3530            10,
3531        );
3532
3533        // Nothing further arrives, and service keeps being consumed.
3534        let actions = sup.handle_deadline(start + Duration::from_secs(12), 500);
3535        assert!(
3536            actions
3537                .iter()
3538                .any(|a| matches!(a, SessionPixAction::Close(SessionPixCloseReason::RecoveryIdle))),
3539            "a genuinely silent Entry must still be caught"
3540        );
3541    }
3542
3543    #[test]
3544    fn equal_snapshot_is_noop() {
3545        let p = pseudonym();
3546        let (mut sup, _) = SessionPixSupervisor::new(default_cfg(), dims(10, 5), p, Instant::now());
3547        let now = Instant::now();
3548        let id = ssa_id(p, 1);
3549
3550        sup.handle_event(&SessionPixEvent::SsaRequestSent(id), now, 0);
3551        sup.handle_event(&SessionPixEvent::CommitmentVerified(id), now, 0);
3552        sup.handle_event(
3553            &SessionPixEvent::DepositConfirmed {
3554                ssa_id: id,
3555                amount: sufficient_balance(),
3556            },
3557            now,
3558            0,
3559        );
3560
3561        let progress = make_progress(id, 10, 50, 1);
3562        sup.handle_event(&SessionPixEvent::RecoveryProgress(progress), now, 5);
3563
3564        let idle_before = sup.ssas.iter().find(|s| s.ssa_id == id).unwrap().recovery_idle_deadline;
3565
3566        let actions = sup.handle_event(&SessionPixEvent::RecoveryProgress(progress), now, 5);
3567        assert!(actions.is_empty());
3568
3569        assert_eq!(
3570            sup.ssas.iter().find(|s| s.ssa_id == id).unwrap().recovery_idle_deadline,
3571            idle_before
3572        );
3573    }
3574
3575    #[test]
3576    fn lower_snapshot_is_ignored_as_stale() {
3577        let p = pseudonym();
3578        let (mut sup, _) = SessionPixSupervisor::new(default_cfg(), dims(10, 5), p, Instant::now());
3579        let now = Instant::now();
3580        let id = ssa_id(p, 1);
3581
3582        sup.handle_event(&SessionPixEvent::SsaRequestSent(id), now, 0);
3583        sup.handle_event(&SessionPixEvent::RecoveryProgress(make_progress(id, 10, 50, 1)), now, 0);
3584
3585        // Stale snapshot from concurrent processing is silently ignored.
3586        // Close-on-regression was rejected because ack batches are processed
3587        // with for_each_concurrent, so out-of-order arrival is possible.
3588        let actions = sup.handle_event(&SessionPixEvent::RecoveryProgress(make_progress(id, 5, 50, 1)), now, 0);
3589        assert!(actions.is_empty(), "stale snapshot should be ignored, got: {actions:?}");
3590    }
3591
3592    #[test]
3593    fn inconsistent_target_closes_as_counter_regression() {
3594        let p = pseudonym();
3595        let (mut sup, _) = SessionPixSupervisor::new(default_cfg(), dims(10, 5), p, Instant::now());
3596        let now = Instant::now();
3597        let id = ssa_id(p, 1);
3598
3599        // Must register the SSA first.
3600        sup.handle_event(&SessionPixEvent::SsaRequestSent(id), now, 0);
3601        sup.handle_event(&SessionPixEvent::CommitmentVerified(id), now, 0);
3602        sup.handle_event(
3603            &SessionPixEvent::DepositConfirmed {
3604                ssa_id: id,
3605                amount: sufficient_balance(),
3606            },
3607            now,
3608            0,
3609        );
3610
3611        // RecoveryProgress with target != dims.target_useful_shares() = 50.
3612        let actions = sup.handle_event(&SessionPixEvent::RecoveryProgress(make_progress(id, 1, 99, 0)), now, 0);
3613        assert_eq!(actions.len(), 1);
3614        assert!(matches!(
3615            actions[0],
3616            SessionPixAction::Close(SessionPixCloseReason::CounterRegression)
3617        ));
3618    }
3619
3620    // ---------------------------------------------------------------
3621    // Fault tests
3622    // ---------------------------------------------------------------
3623
3624    /// A fault reported before the request confirmation still counts.
3625    ///
3626    /// Carrying out `RequestSsa` registers the Exit commitment, which is what makes shares for that
3627    /// SSA processable — so a share can fail verification and be reported before `SsaRequestSent`
3628    /// gets back. Every handler ignores an SSA it has no record of, which on this event would mean
3629    /// failing open on the one signal that has to fail closed.
3630    #[test]
3631    fn an_unverifiable_share_counts_even_before_the_request_is_confirmed() {
3632        let p = pseudonym();
3633        let (mut sup, actions) = SessionPixSupervisor::new(SupervisorConfig::default(), dims(10, 5), p, Instant::now());
3634        let now = Instant::now();
3635
3636        let id = match actions.as_slice() {
3637            [SessionPixAction::RequestSsa { ssa_ids, .. }] => ssa_ids[0],
3638            other => panic!("expected one RequestSsa, got {other:?}"),
3639        };
3640
3641        // No `SsaRequestSent` — the confirmation is still in flight.
3642        let actions = sup.handle_event(
3643            &SessionPixEvent::UnverifiableShares {
3644                ssa_id: id,
3645                observed_total: 1,
3646                peer: crate::supervision::test_peer(),
3647            },
3648            now,
3649            0,
3650        );
3651
3652        assert!(
3653            matches!(
3654                actions.as_slice(),
3655                [SessionPixAction::Close(SessionPixCloseReason::UnverifiableShares)]
3656            ),
3657            "expected a close, got {actions:?}"
3658        );
3659    }
3660
3661    /// The report closes whatever it says, because the count is not what makes it fatal.
3662    ///
3663    /// `observed_total` is a cross-peer aggregate assembled from concurrently processed ack batches,
3664    /// so it can arrive as any value and out of order. It used to be charged as a delta against a
3665    /// running maximum and compared to a tolerance; with no tolerance left it is a log field, and the
3666    /// close must not become conditional on it again — a total of one is a doomed cycle exactly as
3667    /// much as a total of a thousand.
3668    #[test]
3669    fn any_observed_total_closes_on_the_first_report() {
3670        for observed_total in [1, 2, u64::MAX] {
3671            let p = pseudonym();
3672            let (mut sup, _) = SessionPixSupervisor::new(default_cfg(), dims(10, 5), p, Instant::now());
3673            let now = Instant::now();
3674            let id = ssa_id(p, 1);
3675
3676            sup.handle_event(&SessionPixEvent::SsaRequestSent(id), now, 0);
3677
3678            let actions = sup.handle_event(
3679                &SessionPixEvent::UnverifiableShares {
3680                    ssa_id: id,
3681                    observed_total,
3682                    peer: crate::supervision::test_peer(),
3683                },
3684                now,
3685                0,
3686            );
3687
3688            assert!(
3689                matches!(
3690                    actions.as_slice(),
3691                    [SessionPixAction::Close(SessionPixCloseReason::UnverifiableShares)]
3692                ),
3693                "observed_total {observed_total} must close on the first report, got {actions:?}"
3694            );
3695        }
3696    }
3697
3698    /// A batch's healthy siblings must not keep the Session open.
3699    ///
3700    /// Every other per-cycle fault routes through `close_ssa_and_collect`, which retires the member
3701    /// and lets `max_failed_cycles` siblings carry on. This one does not, and deliberately: a
3702    /// polynomial that fails to open its commitment is evidence about the *peer*, and the same peer
3703    /// holds every other cycle in the batch.
3704    #[test]
3705    fn an_unverifiable_share_closes_the_session_even_with_live_siblings() {
3706        let p = pseudonym();
3707        let mut cfg = default_cfg();
3708        cfg.ssas_per_request = 3;
3709        cfg.max_failed_cycles = 10;
3710        let (mut sup, actions) = SessionPixSupervisor::new(cfg, dims(10, 5), p, Instant::now());
3711        let now = Instant::now();
3712
3713        let ids = match actions.as_slice() {
3714            [SessionPixAction::RequestSsa { ssa_ids, .. }] => ssa_ids.clone(),
3715            other => panic!("expected one RequestSsa, got {other:?}"),
3716        };
3717        assert_eq!(3, ids.len());
3718        for id in &ids {
3719            sup.handle_event(&SessionPixEvent::SsaRequestSent(*id), now, 0);
3720        }
3721
3722        // The fault lands on the *last* member, so two healthy cycles remain in front of it.
3723        let actions = sup.handle_event(
3724            &SessionPixEvent::UnverifiableShares {
3725                ssa_id: ids[2],
3726                observed_total: 1,
3727                peer: crate::supervision::test_peer(),
3728            },
3729            now,
3730            0,
3731        );
3732
3733        assert!(
3734            matches!(
3735                actions.as_slice(),
3736                [SessionPixAction::Close(SessionPixCloseReason::UnverifiableShares)]
3737            ),
3738            "a live batch must not absorb an unverifiable-share report, got {actions:?}"
3739        );
3740    }
3741
3742    // ---------------------------------------------------------------
3743    // AlmostRecovered / Recovered
3744    // ---------------------------------------------------------------
3745
3746    #[test]
3747    fn almost_recovered_while_recovering_requests_next_once() {
3748        let p = pseudonym();
3749        let (mut sup, _) = SessionPixSupervisor::new(default_cfg(), dims(10, 5), p, Instant::now());
3750        let now = Instant::now();
3751        let id1 = ssa_id(p, 1);
3752
3753        sup.handle_event(&SessionPixEvent::SsaRequestSent(id1), now, 0);
3754        sup.handle_event(&SessionPixEvent::CommitmentVerified(id1), now, 0);
3755        sup.handle_event(
3756            &SessionPixEvent::DepositConfirmed {
3757                ssa_id: id1,
3758                amount: sufficient_balance(),
3759            },
3760            now,
3761            0,
3762        );
3763
3764        let actions = sup.handle_event(&SessionPixEvent::AlmostRecovered(id1), now, 0);
3765        assert_eq!(actions.len(), 1);
3766        match &actions[0] {
3767            SessionPixAction::RequestSsa { ssa_ids, .. } => {
3768                assert_eq!(ssa_ids[0].ssa_index(), SsaIndex::new(2).unwrap());
3769            }
3770            other => panic!("expected RequestSsa, got {other:?}"),
3771        }
3772
3773        let actions = sup.handle_event(&SessionPixEvent::AlmostRecovered(id1), now, 0);
3774        assert!(actions.is_empty());
3775    }
3776
3777    /// Config for the share-order tests: a batch of three, generous dimensions so a cycle can carry a
3778    /// meaningful sample, and a small floor so the tests need few events.
3779    fn share_order_cfg(sample: u64, fraction: f64) -> SupervisorConfig {
3780        SupervisorConfig {
3781            ssas_per_request: 3,
3782            min_share_order_sample: sample,
3783            max_off_front_share_fraction: fraction,
3784            ..default_cfg()
3785        }
3786    }
3787
3788    /// Serving the front cycle, with only a trace of progress behind it, must never trip the ratio.
3789    ///
3790    /// The trace stands in for what the mixnet produces on its own: SURBs are permuted outbound and
3791    /// their acknowledgements again inbound, so a conforming Entry still shows *some* off-front
3792    /// progress at a cycle boundary. The point of measuring a fraction rather than a count is that a
3793    /// bounded permutation stays a vanishing share of a growing window however the mixer is tuned.
3794    #[test]
3795    fn conforming_service_never_trips_the_share_order_ratio() {
3796        let p = pseudonym();
3797        let (mut sup, _) = SessionPixSupervisor::new(share_order_cfg(1000, 0.25), dims(100, 10), p, Instant::now());
3798        let now = Instant::now();
3799        fund_batch(&mut sup, p, 3, now);
3800        let target = sup.dims.target_useful_shares();
3801
3802        for useful in [200, 400, 600, 800] {
3803            let actions = sup.handle_event(
3804                &SessionPixEvent::RecoveryProgress(make_progress(ssa_id(p, 1), useful, target, 10)),
3805                now,
3806                useful,
3807            );
3808            assert!(
3809                matches!(actions.as_slice(), [SessionPixAction::ProgressNotification]),
3810                "front-cycle service is what is supposed to happen, got {actions:?}"
3811            );
3812        }
3813
3814        // A boundary's worth of reordering: 50 shares against 800, far inside the ceiling.
3815        let actions = sup.handle_event(
3816            &SessionPixEvent::RecoveryProgress(make_progress(ssa_id(p, 2), 50, target, 1)),
3817            now,
3818            850,
3819        );
3820        assert!(
3821            actions.is_empty(),
3822            "off-front progress must neither convict nor reopen the gate"
3823        );
3824        assert!(!sup.closed);
3825    }
3826
3827    /// An Entry spreading a batch across all its cycles must be caught once the evidence is in.
3828    ///
3829    /// Three cycles served in parallel put two thirds of all progress behind the front — the signature
3830    /// the ratio exists for, and one that neither the service gate nor `max_recovery_idle` can see,
3831    /// since the front cycle keeps progressing and keeps refreshing its own idle timer.
3832    #[test]
3833    fn spreading_a_batch_across_its_cycles_trips_the_share_order_ratio() {
3834        const SAMPLE: u64 = 1000;
3835
3836        let p = pseudonym();
3837        let (mut sup, _) = SessionPixSupervisor::new(share_order_cfg(SAMPLE, 0.25), dims(100, 10), p, Instant::now());
3838        let now = Instant::now();
3839        fund_batch(&mut sup, p, 3, now);
3840        let target = sup.dims.target_useful_shares();
3841
3842        let mut closed_at = None;
3843        'outer: for round in 1..=5u64 {
3844            for cycle in 1..=3u32 {
3845                let actions = sup.handle_event(
3846                    &SessionPixEvent::RecoveryProgress(make_progress(ssa_id(p, cycle), round * 100, target, 10)),
3847                    now,
3848                    round * 300,
3849                );
3850                let sample = sup.front_useful + sup.off_front_useful;
3851                if let [SessionPixAction::Close(reason)] = actions.as_slice() {
3852                    assert_eq!(*reason, SessionPixCloseReason::BatchServedOutOfOrder);
3853                    closed_at = Some(sample);
3854                    break 'outer;
3855                }
3856                assert!(
3857                    sample < SAMPLE,
3858                    "at {sample} shares the evidence was in and the 2:1 split should have closed the Session"
3859                );
3860            }
3861        }
3862
3863        let closed_at = closed_at.expect("spreading across three cycles must close the Session");
3864        assert!(
3865            closed_at >= SAMPLE,
3866            "conviction at {closed_at} shares is below the evidence floor of {SAMPLE}"
3867        );
3868    }
3869
3870    /// The accounting is scoped to one cycle's service: completing the front cycle clears it.
3871    ///
3872    /// That bound is what stops an Entry laundering a spree of out-of-order service — the only way to
3873    /// reach the reset is to finish the front cycle, which is to say to earn the traffic it took.
3874    #[test]
3875    fn share_order_accounting_resets_when_the_front_cycle_completes() {
3876        let p = pseudonym();
3877        let (mut sup, _) = SessionPixSupervisor::new(share_order_cfg(1000, 0.25), dims(100, 10), p, Instant::now());
3878        let now = Instant::now();
3879        fund_batch(&mut sup, p, 3, now);
3880        let target = sup.dims.target_useful_shares();
3881
3882        sup.handle_event(
3883            &SessionPixEvent::RecoveryProgress(make_progress(ssa_id(p, 1), 500, target, 50)),
3884            now,
3885            500,
3886        );
3887        sup.handle_event(
3888            &SessionPixEvent::RecoveryProgress(make_progress(ssa_id(p, 2), 100, target, 10)),
3889            now,
3890            600,
3891        );
3892        assert_eq!(sup.share_order_front, Some(SsaIndex::new(1).unwrap()));
3893        assert_eq!((sup.front_useful, sup.off_front_useful), (500, 100));
3894
3895        sup.handle_event(&SessionPixEvent::Recovered(ssa_id(p, 1)), now, 600);
3896        assert_eq!(
3897            sup.share_order_front,
3898            Some(SsaIndex::new(2).unwrap()),
3899            "cycle 2 is the front now"
3900        );
3901        assert_eq!(
3902            (sup.front_useful, sup.off_front_useful),
3903            (0, 0),
3904            "the window is one cycle's service, so it starts empty on the new front"
3905        );
3906    }
3907
3908    /// Below the evidence floor nothing is judged, however lopsided the split.
3909    ///
3910    /// This is what keeps a cycle made unrecoverable by loss from convicting the Entry: it stops
3911    /// progressing while later cycles continue, so the fraction goes to 1.0 — and the floor outlasts
3912    /// the service-gated `max_recovery_idle` window that retires it, after which the front moves and
3913    /// the accounting resets. The ceiling here is 0.0, the strictest possible, so only the floor can be
3914    /// what holds the verdict back.
3915    #[test]
3916    fn no_verdict_is_reached_below_the_evidence_floor() {
3917        const SAMPLE: u64 = 1000;
3918
3919        let p = pseudonym();
3920        let (mut sup, _) = SessionPixSupervisor::new(share_order_cfg(SAMPLE, 0.0), dims(100, 10), p, Instant::now());
3921        let now = Instant::now();
3922        fund_batch(&mut sup, p, 3, now);
3923        let target = sup.dims.target_useful_shares();
3924
3925        // Every share lands behind the front, and still nothing is decided while the sample is thin.
3926        for useful in [300, 600, 900] {
3927            let actions = sup.handle_event(
3928                &SessionPixEvent::RecoveryProgress(make_progress(ssa_id(p, 3), useful, target, 10)),
3929                now,
3930                useful,
3931            );
3932            assert!(actions.is_empty(), "{useful} shares is below the floor and off-front");
3933        }
3934
3935        let actions = sup.handle_event(
3936            &SessionPixEvent::RecoveryProgress(make_progress(ssa_id(p, 3), SAMPLE, target, 10)),
3937            now,
3938            SAMPLE,
3939        );
3940        assert!(
3941            matches!(
3942                actions.as_slice(),
3943                [SessionPixAction::Close(SessionPixCloseReason::BatchServedOutOfOrder)]
3944            ),
3945            "at the floor the verdict must be reached, got {actions:?}"
3946        );
3947    }
3948
3949    /// A stray reordered share must not put a *queued* cycle on the idle clock.
3950    ///
3951    /// The sibling test below fixes the arming path: clocks start when a cycle reaches the front. But
3952    /// refreshing is not arming, and `on_recovery_progress` used to set `recovery_idle_deadline` for
3953    /// any cycle merely in `Recovering`. One share crossing a cycle boundary out of order — the exact
3954    /// thing [`SupervisorConfig::min_share_order_sample`] exists to tolerate — therefore *started* the
3955    /// idle clock on a cycle the Entry cannot serve, since emission is clamped to one cycle. A
3956    /// `max_recovery_idle` later the gate saw session-wide service climbing on the front cycle's
3957    /// behalf and retired the queued one, charging it for a queue wait it had no way to end.
3958    ///
3959    /// The two clocks arm together or not at all, so the hard deadline is the witness for "has been at
3960    /// the front", and a queued cycle must come out of this with neither.
3961    #[test]
3962    fn a_stray_share_on_a_queued_cycle_must_not_start_its_idle_clock() {
3963        const BATCH: u32 = 3;
3964
3965        let p = pseudonym();
3966        let cfg = SupervisorConfig {
3967            ssas_per_request: BATCH as usize,
3968            max_recovery_idle: Duration::from_secs(10),
3969            max_recovery_time: Duration::from_secs(3600),
3970            ..default_cfg()
3971        };
3972        let (mut sup, _) = SessionPixSupervisor::new(cfg, dims(10, 5), p, Instant::now());
3973        let start = Instant::now();
3974        fund_batch(&mut sup, p, BATCH, start);
3975        let target = sup.dims.target_useful_shares();
3976
3977        // One share for queued cycle 2 arrives while cycle 1 holds the front.
3978        sup.handle_event(
3979            &SessionPixEvent::RecoveryProgress(make_progress(ssa_id(p, 2), 1, target, 0)),
3980            start + Duration::from_secs(1),
3981            1_000,
3982        );
3983
3984        let queued = sup.ssas.iter().find(|s| s.ssa_id == ssa_id(p, 2)).expect("cycle 2");
3985        assert_eq!(
3986            (queued.recovery_hard_deadline, queued.recovery_idle_deadline),
3987            (None, None),
3988            "an off-front share must leave a queued cycle off both clocks, not just the hard one"
3989        );
3990
3991        // Cycle 1 is served normally, so session-wide service climbs past cycle 2's baseline — which is
3992        // what would convict it if the idle clock were running.
3993        for step in 1..=3u64 {
3994            sup.handle_event(
3995                &SessionPixEvent::RecoveryProgress(make_progress(ssa_id(p, 1), step, target, 0)),
3996                start + Duration::from_secs(1 + step * 3),
3997                1_000 + step * 1_000,
3998            );
3999        }
4000
4001        let actions = sup.handle_deadline(start + Duration::from_secs(12), 5_000);
4002        assert!(
4003            actions.is_empty(),
4004            "no cycle may be retired or closed while the front is being served, got {actions:?}"
4005        );
4006        assert_eq!(sup.ssas.len(), BATCH as usize, "the whole batch must still be live");
4007        assert_eq!(sup.failed_cycles, 0, "nothing may be charged as a failed cycle");
4008    }
4009
4010    /// A cycle queued behind the front of its batch must not be charged for the wait.
4011    ///
4012    /// The Entry serves a batch strictly in index order, so at deployed dimensions cycle 2 of a batch
4013    /// sits idle for the ~61 min it takes cycle 1's shares to be emitted. Starting its recovery clocks
4014    /// when its deposit confirmed measured that queue wait: `max_recovery_idle` would retire it inside
4015    /// a minute — the idle gate compares against *session-wide* service, which is plentiful while cycle
4016    /// 1 is served — and `max_recovery_time` would finish the job. Batching would have paid for
4017    /// `ssas_per_request` deposits and been able to use one.
4018    ///
4019    /// So the clocks start when the cycle reaches the front, and `max_recovery_time` bounds the
4020    /// recovery of a single cycle rather than its position in a queue.
4021    #[test]
4022    fn a_queued_cycle_starts_its_recovery_clocks_only_at_the_front_of_the_batch() {
4023        const BATCH: u32 = 3;
4024
4025        let p = pseudonym();
4026        let cfg = SupervisorConfig {
4027            ssas_per_request: BATCH as usize,
4028            max_recovery_idle: Duration::from_secs(10),
4029            max_recovery_time: Duration::from_secs(3600),
4030            ..default_cfg()
4031        };
4032        let (mut sup, _) = SessionPixSupervisor::new(cfg, dims(10, 5), p, Instant::now());
4033        let start = Instant::now();
4034        fund_batch(&mut sup, p, BATCH, start);
4035        let target = sup.dims.target_useful_shares();
4036
4037        let armed = |sup: &SessionPixSupervisor, idx: u32| {
4038            let ssa = sup
4039                .ssas
4040                .iter()
4041                .find(|s| s.ssa_id == ssa_id(p, idx))
4042                .expect("cycle must exist");
4043            (
4044                ssa.recovery_hard_deadline.is_some(),
4045                ssa.recovery_idle_deadline.is_some(),
4046            )
4047        };
4048
4049        assert_eq!(
4050            armed(&sup, 1),
4051            (true, true),
4052            "the front cycle's clocks run from funding"
4053        );
4054        for queued in 2..=BATCH {
4055            assert_eq!(
4056                armed(&sup, queued),
4057                (false, false),
4058                "cycle {queued} is queued behind the front and must not be on the clock yet"
4059            );
4060        }
4061
4062        // Cycle 1 is served for five times `max_recovery_idle`, with session-wide service climbing the
4063        // whole way. Nothing queued may be retired for that.
4064        for step in 1..=10u64 {
4065            let at = start + Duration::from_secs(step * 5);
4066            let served = step * 1000;
4067            sup.handle_event(
4068                &SessionPixEvent::RecoveryProgress(make_progress(ssa_id(p, 1), step, target, 1)),
4069                at,
4070                served,
4071            );
4072            let actions = sup.handle_deadline(at, served);
4073            assert!(
4074                !actions
4075                    .iter()
4076                    .any(|a| matches!(a, SessionPixAction::Close(_) | SessionPixAction::RetireSsa(_))),
4077                "no cycle may be closed or retired at +{}s, got {actions:?}",
4078                step * 5
4079            );
4080        }
4081        assert_eq!(sup.ssas.len(), BATCH as usize, "the whole batch must still be live");
4082
4083        // Cycle 1 recovers cryptographically, but its already-minted SURBs still own the FIFO and
4084        // cycle 2 must remain off the clock until that paid tail drains.
4085        let handover = start + Duration::from_secs(60);
4086        sup.handle_event(&SessionPixEvent::Recovered(ssa_id(p, 1)), handover, 10_000);
4087        assert_eq!(
4088            armed(&sup, 2),
4089            (false, false),
4090            "cycle 2 is still behind the paid FIFO tail"
4091        );
4092        assert_eq!(
4093            armed(&sup, 3),
4094            (false, false),
4095            "cycle 3 is still queued and must stay off the clock"
4096        );
4097
4098        let fifo_boundary = handover + Duration::from_secs(5);
4099        let max_seen = max_cycle_shares(&sup);
4100        sup.handle_event(
4101            &SessionPixEvent::RecoveryProgress(make_recovered_tail_progress(&sup, ssa_id(p, 1), max_seen)),
4102            fifo_boundary,
4103            10_100,
4104        );
4105        assert_eq!(armed(&sup, 2), (true, true), "cycle 2 starts only at the FIFO boundary");
4106
4107        let ssa2 = sup.ssas.iter().find(|s| s.ssa_id == ssa_id(p, 2)).expect("cycle 2");
4108        assert_eq!(
4109            ssa2.recovery_hard_deadline,
4110            fifo_boundary.checked_add(Duration::from_secs(3600)),
4111            "cycle 2's ceiling must be measured from its own turn, not from its deposit"
4112        );
4113        assert_eq!(
4114            ssa2.served_total_at_last_progress, 10_100,
4115            "the idle gate's baseline must be re-based at the handover, or the queue wait is charged to it"
4116        );
4117    }
4118
4119    /// At `ssas_per_request > 1`, only the batch's **last** SSA may ask for the successor batch.
4120    ///
4121    /// `next_requested` is per-`PerSsaState`, so without a batch-wide gate every member of a batch
4122    /// answers its own `AlmostRecovered` with a fresh batch of `ssas_per_request`: one batch of 3
4123    /// becomes three, then nine, each SSA a separate on-chain deposit demanded of the Entry. At
4124    /// `ssas_per_request == 1`, "one requesting cycle" and "one requesting batch" coincide, which is
4125    /// why every unbatched test misses the distinction.
4126    ///
4127    /// All three cycles are funded and `Recovering` before the first event, i.e. the case where the
4128    /// deferral in [`almost_recovered_while_awaiting_deposit_defers_request`] does *not* apply and
4129    /// only the batch gate can hold the request back.
4130    #[test]
4131    fn only_the_last_ssa_of_a_batch_asks_for_the_next_one() {
4132        const BATCH: usize = 3;
4133
4134        let p = pseudonym();
4135        let cfg = SupervisorConfig {
4136            ssas_per_request: BATCH,
4137            ..default_cfg()
4138        };
4139        let (mut sup, _) = SessionPixSupervisor::new(cfg, dims(10, 5), p, Instant::now());
4140        let now = Instant::now();
4141
4142        // Fund every cycle of the batch, so each one is in `Recovering`.
4143        for idx in 1..=BATCH as u32 {
4144            let id = ssa_id(p, idx);
4145            sup.handle_event(&SessionPixEvent::SsaRequestSent(id), now, 0);
4146            sup.handle_event(&SessionPixEvent::CommitmentVerified(id), now, 0);
4147            sup.handle_event(
4148                &SessionPixEvent::DepositConfirmed {
4149                    ssa_id: id,
4150                    amount: sufficient_balance(),
4151                },
4152                now,
4153                0,
4154            );
4155        }
4156        assert_eq!(sup.next_ssa_index, BATCH as u32 + 1, "one batch allocated so far");
4157
4158        // Every member but the last is silent, in the order the Exit actually reconstructs them.
4159        for idx in 1..BATCH as u32 {
4160            let actions = sup.handle_event(&SessionPixEvent::AlmostRecovered(ssa_id(p, idx)), now, 0);
4161            assert!(
4162                actions.is_empty(),
4163                "SSA {idx} is not the last of the batch and must not ask for a successor, got {actions:?}"
4164            );
4165            assert_eq!(
4166                sup.next_ssa_index,
4167                BATCH as u32 + 1,
4168                "SSA {idx} must not allocate any index"
4169            );
4170        }
4171
4172        // The last member is the one that pipelines the refill.
4173        let last = ssa_id(p, BATCH as u32);
4174        let actions = sup.handle_event(&SessionPixEvent::AlmostRecovered(last), now, 0);
4175        assert_eq!(actions.len(), 1, "the last SSA of the batch must ask exactly once");
4176        match &actions[0] {
4177            SessionPixAction::RequestSsa { ssa_ids, .. } => assert_eq!(
4178                ssa_ids.iter().map(|i| i.ssa_index().get()).collect::<Vec<_>>(),
4179                (BATCH as u32 + 1..=2 * BATCH as u32).collect::<Vec<_>>(),
4180                "the successor batch must be the next contiguous run"
4181            ),
4182            other => panic!("expected RequestSsa, got {other:?}"),
4183        }
4184
4185        // Once-only is preserved for the last member too.
4186        assert!(
4187            sup.handle_event(&SessionPixEvent::AlmostRecovered(last), now, 0)
4188                .is_empty(),
4189            "a repeated AlmostRecovered on the last member must not ask again"
4190        );
4191        assert_eq!(sup.next_ssa_index, 2 * BATCH as u32 + 1);
4192        assert_eq!(
4193            sup.live_cycle_count(),
4194            sup.reserved_cycle_slots(),
4195            "two full batches sit exactly on the admission reservation"
4196        );
4197    }
4198
4199    #[test]
4200    fn a_third_live_batch_is_deferred_even_when_share_order_policy_allows_it() {
4201        const BATCH: usize = 2;
4202
4203        let p = pseudonym();
4204        let cfg = SupervisorConfig {
4205            ssas_per_request: BATCH,
4206            max_off_front_share_fraction: 1.0,
4207            ..default_cfg()
4208        };
4209        let now = Instant::now();
4210        let (mut sup, _) = SessionPixSupervisor::new(cfg, dims(10, 5), p, now);
4211
4212        fund_batch(&mut sup, p, BATCH as u32, now);
4213        let actions = sup.handle_event(&SessionPixEvent::AlmostRecovered(ssa_id(p, 2)), now, 0);
4214        let second_batch = match actions.as_slice() {
4215            [SessionPixAction::RequestSsa { ssa_ids, .. }] => ssa_ids.clone(),
4216            other => panic!("expected the second batch, got {other:?}"),
4217        };
4218        for id in second_batch {
4219            sup.handle_event(&SessionPixEvent::SsaRequestSent(id), now, 0);
4220            sup.handle_event(&SessionPixEvent::CommitmentVerified(id), now, 0);
4221            sup.handle_event(
4222                &SessionPixEvent::DepositConfirmed {
4223                    ssa_id: id,
4224                    amount: sufficient_balance(),
4225                },
4226                now,
4227                0,
4228            );
4229        }
4230
4231        let actions = sup.handle_event(&SessionPixEvent::AlmostRecovered(ssa_id(p, 4)), now, 0);
4232        assert!(
4233            actions
4234                .iter()
4235                .all(|action| !matches!(action, SessionPixAction::RequestSsa { .. })),
4236            "two live batches already consume the reservation, got {actions:?}"
4237        );
4238        assert_eq!(sup.ssas.len(), 2 * BATCH, "a third batch must not be allocated");
4239    }
4240
4241    #[test]
4242    fn a_deferred_batch_retries_only_after_the_older_generation_is_released() {
4243        let p = pseudonym();
4244        let cfg = SupervisorConfig {
4245            ssas_per_request: 2,
4246            max_failed_cycles: 2,
4247            max_off_front_share_fraction: 1.0,
4248            ..default_cfg()
4249        };
4250        let now = Instant::now();
4251        let (mut sup, _) = SessionPixSupervisor::new(cfg, dims(10, 5), p, now);
4252
4253        // Leave cycle 1 unfunded and fund cycle 2, whose early signal earns batch 2.
4254        commit_unfunded(&mut sup, p, 1..=1, now);
4255        let second = ssa_id(p, 2);
4256        sup.handle_event(&SessionPixEvent::SsaRequestSent(second), now, 0);
4257        sup.handle_event(&SessionPixEvent::CommitmentVerified(second), now, 0);
4258        sup.handle_event(
4259            &SessionPixEvent::DepositConfirmed {
4260                ssa_id: second,
4261                amount: sufficient_balance(),
4262            },
4263            now,
4264            0,
4265        );
4266        assert!(
4267            sup.handle_event(&SessionPixEvent::AlmostRecovered(second), now, 0)
4268                .iter()
4269                .any(|action| matches!(action, SessionPixAction::RequestSsa { .. }))
4270        );
4271
4272        for idx in 3..=4 {
4273            let id = ssa_id(p, idx);
4274            sup.handle_event(&SessionPixEvent::SsaRequestSent(id), now, 0);
4275            sup.handle_event(&SessionPixEvent::CommitmentVerified(id), now, 0);
4276            sup.handle_event(
4277                &SessionPixEvent::DepositConfirmed {
4278                    ssa_id: id,
4279                    amount: sufficient_balance(),
4280                },
4281                now,
4282                0,
4283            );
4284        }
4285        assert!(
4286            sup.handle_event(&SessionPixEvent::AlmostRecovered(ssa_id(p, 4)), now, 0)
4287                .iter()
4288                .all(|action| !matches!(action, SessionPixAction::RequestSsa { .. }))
4289        );
4290
4291        // Releasing only one member of the old generation is insufficient.
4292        let actions = sup.handle_event(&SessionPixEvent::DepositObserverClosed(ssa_id(p, 1)), now, 0);
4293        assert!(
4294            actions
4295                .iter()
4296                .all(|action| !matches!(action, SessionPixAction::RequestSsa { .. }))
4297        );
4298
4299        // The remaining old member times out. Its RetireSsa must precede the now-admissible request,
4300        // otherwise the action driver would allocate the replacement before dropping the old guard.
4301        let actions = sup.handle_deadline(now + default_cfg().max_recovery_idle, 1);
4302        let retire_position = actions
4303            .iter()
4304            .position(|action| matches!(action, SessionPixAction::RetireSsa(id) if *id == second));
4305        let request_position = actions.iter().position(|action| {
4306            matches!(action, SessionPixAction::RequestSsa { ssa_ids, .. }
4307                if ssa_ids.iter().map(|id| id.ssa_index().get()).eq(5..=6))
4308        });
4309        assert!(
4310            matches!((retire_position, request_position), (Some(retire), Some(request)) if retire < request),
4311            "the old generation must retire before batch 3 is requested, got {actions:?}"
4312        );
4313        assert_eq!(sup.live_batch_count(), 2);
4314        assert_eq!(sup.live_cycle_count(), 4);
4315    }
4316
4317    /// Retiring the cycle that holds the successor gate must hand it to the newest survivor.
4318    ///
4319    /// Otherwise the batch is stranded: its siblings are barred from asking, so no replacement is
4320    /// ever requested and the Session dies on a recovery timeout instead of on its actual cause.
4321    /// The promoted cycle is the last of the batch that will really serve its quota, so the request
4322    /// still cannot come early.
4323    #[test]
4324    fn retiring_the_last_ssa_of_a_batch_hands_the_successor_gate_on() {
4325        const BATCH: usize = 3;
4326
4327        let p = pseudonym();
4328        let cfg = SupervisorConfig {
4329            ssas_per_request: BATCH,
4330            ..default_cfg()
4331        };
4332        let (mut sup, _) = SessionPixSupervisor::new(cfg, dims(10, 5), p, Instant::now());
4333        let now = Instant::now();
4334
4335        // Cycles 1 and 2 get funded; 3 — the gate holder — is left awaiting its deposit.
4336        for idx in 1..BATCH as u32 {
4337            let id = ssa_id(p, idx);
4338            sup.handle_event(&SessionPixEvent::SsaRequestSent(id), now, 0);
4339            sup.handle_event(&SessionPixEvent::CommitmentVerified(id), now, 0);
4340            sup.handle_event(
4341                &SessionPixEvent::DepositConfirmed {
4342                    ssa_id: id,
4343                    amount: sufficient_balance(),
4344                },
4345                now,
4346                0,
4347            );
4348        }
4349        let last = ssa_id(p, BATCH as u32);
4350        sup.handle_event(&SessionPixEvent::SsaRequestSent(last), now, 0);
4351        sup.handle_event(&SessionPixEvent::CommitmentVerified(last), now, 0);
4352
4353        // Its deposit observer gives up, so the gate holder is retired mid-batch.
4354        let actions = sup.handle_event(&SessionPixEvent::DepositObserverClosed(last), now, 0);
4355        assert!(
4356            actions
4357                .iter()
4358                .any(|a| matches!(a, SessionPixAction::RetireSsa(id) if *id == last)),
4359            "the unfunded last cycle must be retired, not close the Session, got {actions:?}"
4360        );
4361        assert!(!sup.closed, "surviving siblings must keep the Session alive");
4362
4363        // The gate is now cycle 2's, and cycle 1 is still barred.
4364        assert!(
4365            sup.handle_event(&SessionPixEvent::AlmostRecovered(ssa_id(p, 1)), now, 0)
4366                .is_empty(),
4367            "cycle 1 is still not the last survivor and must stay silent"
4368        );
4369        let actions = sup.handle_event(&SessionPixEvent::AlmostRecovered(ssa_id(p, 2)), now, 0);
4370        assert_eq!(
4371            actions.len(),
4372            1,
4373            "the newest survivor must have inherited the successor gate, got {actions:?}"
4374        );
4375        match &actions[0] {
4376            SessionPixAction::RequestSsa { ssa_ids, .. } => assert_eq!(
4377                ssa_ids.iter().map(|i| i.ssa_index().get()).collect::<Vec<_>>(),
4378                (BATCH as u32 + 1..=2 * BATCH as u32).collect::<Vec<_>>(),
4379                "the successor batch still starts past the whole retired batch"
4380            ),
4381            other => panic!("expected RequestSsa, got {other:?}"),
4382        }
4383    }
4384
4385    /// A second lost cycle closes a batched Session, however many funded siblings are still standing.
4386    ///
4387    /// Retiring a member and carrying on is what keeps one bad cycle from costing a whole Session,
4388    /// but it leaves no trace on the survivors — so without a count that outlives the retirement an
4389    /// Entry can lose one cycle per batch forever, paying for a fraction of what it is served. The
4390    /// counter is cumulative for exactly that reason: here the two losses are in *different* batches,
4391    /// and nothing in a per-batch view would connect them.
4392    ///
4393    /// The close carries the *first* failure's reason, so the report names the cause rather than the
4394    /// last symptom.
4395    #[test]
4396    fn the_second_lost_cycle_closes_a_batched_session() {
4397        const BATCH: usize = 4;
4398
4399        let p = pseudonym();
4400        let cfg = SupervisorConfig {
4401            ssas_per_request: BATCH,
4402            max_failed_cycles: 1,
4403            ..default_cfg()
4404        };
4405        let (mut sup, _) = SessionPixSupervisor::new(cfg, dims(10, 5), p, Instant::now());
4406        let now = Instant::now();
4407
4408        // Cycles 1 and 2 are funded and recovering; 3 and 4 are stuck awaiting their deposits, which
4409        // is the state an unpaid cycle is lost from.
4410        fund_batch(&mut sup, p, 2, now);
4411        commit_unfunded(&mut sup, p, 3..=BATCH as u32, now);
4412
4413        // First loss: survivable, and the batch carries on.
4414        let actions = sup.handle_event(&SessionPixEvent::DepositObserverClosed(ssa_id(p, 3)), now, 0);
4415        assert!(
4416            actions
4417                .iter()
4418                .any(|a| matches!(a, SessionPixAction::RetireSsa(id) if *id == ssa_id(p, 3))),
4419            "the first lost cycle must be retired rather than close the Session, got {actions:?}"
4420        );
4421        assert!(!sup.closed, "one loss is inside the limit");
4422
4423        // Second loss, with two funded siblings still recovering: over the limit, so the Session
4424        // goes anyway — which is the point, since those siblings are what would otherwise keep an
4425        // Entry losing one cycle per batch alive indefinitely.
4426        let actions = sup.handle_event(&SessionPixEvent::DepositObserverClosed(ssa_id(p, 4)), now, 0);
4427        assert!(
4428            actions
4429                .iter()
4430                .any(|a| matches!(a, SessionPixAction::Close(SessionPixCloseReason::DepositObserverClosed))),
4431            "the second lost cycle must close the Session, got {actions:?}"
4432        );
4433        assert!(sup.closed);
4434        assert_eq!(2, sup.failed_cycles);
4435    }
4436
4437    /// Raising the limit buys exactly the extra losses it names, and no more.
4438    ///
4439    /// The counter has to survive the retirement of the cycle that incremented it, so this drives
4440    /// three separate losses through a batch large enough that a survivor always remains — a
4441    /// per-batch or per-cycle count would never reach the limit.
4442    #[test]
4443    fn a_raised_failure_limit_tolerates_exactly_that_many_losses() {
4444        const BATCH: usize = 6;
4445
4446        let p = pseudonym();
4447        let cfg = SupervisorConfig {
4448            ssas_per_request: BATCH,
4449            max_failed_cycles: 3,
4450            ..default_cfg()
4451        };
4452        let (mut sup, _) = SessionPixSupervisor::new(cfg, dims(10, 5), p, Instant::now());
4453        let now = Instant::now();
4454
4455        // One funded cycle, so the Session always has a survivor and only the limit can close it.
4456        fund_batch(&mut sup, p, 1, now);
4457        commit_unfunded(&mut sup, p, 2..=BATCH as u32, now);
4458
4459        for idx in 2..=4 {
4460            sup.handle_event(&SessionPixEvent::DepositObserverClosed(ssa_id(p, idx)), now, 0);
4461            assert!(!sup.closed, "loss {} of 3 is inside the limit", idx - 1);
4462        }
4463
4464        sup.handle_event(&SessionPixEvent::DepositObserverClosed(ssa_id(p, 5)), now, 0);
4465        assert!(sup.closed, "the fourth loss is over a limit of three");
4466    }
4467
4468    #[test]
4469    fn almost_recovered_while_awaiting_deposit_defers_request() {
4470        let p = pseudonym();
4471        let (mut sup, _) = SessionPixSupervisor::new(default_cfg(), dims(10, 5), p, Instant::now());
4472        let now = Instant::now();
4473        let id1 = ssa_id(p, 1);
4474
4475        sup.handle_event(&SessionPixEvent::SsaRequestSent(id1), now, 0);
4476        sup.handle_event(&SessionPixEvent::CommitmentVerified(id1), now, 0);
4477
4478        let actions = sup.handle_event(&SessionPixEvent::AlmostRecovered(id1), now, 0);
4479        assert!(actions.is_empty());
4480
4481        let actions = sup.handle_event(
4482            &SessionPixEvent::DepositConfirmed {
4483                ssa_id: id1,
4484                amount: sufficient_balance(),
4485            },
4486            now,
4487            0,
4488        );
4489        assert_eq!(actions.len(), 2);
4490        assert!(matches!(actions[0], SessionPixAction::ReleaseService));
4491        match &actions[1] {
4492            SessionPixAction::RequestSsa { ssa_ids, .. } => {
4493                assert_eq!(ssa_ids[0].ssa_index(), SsaIndex::new(2).unwrap());
4494            }
4495            other => panic!("expected RequestSsa, got {other:?}"),
4496        }
4497    }
4498
4499    #[test]
4500    fn recovered_without_prior_early_event_falls_back_to_request() {
4501        let p = pseudonym();
4502        let (mut sup, _) = SessionPixSupervisor::new(default_cfg(), dims(10, 5), p, Instant::now());
4503        let now = Instant::now();
4504        let id1 = ssa_id(p, 1);
4505
4506        sup.handle_event(&SessionPixEvent::SsaRequestSent(id1), now, 0);
4507        sup.handle_event(&SessionPixEvent::CommitmentVerified(id1), now, 0);
4508        sup.handle_event(
4509            &SessionPixEvent::DepositConfirmed {
4510                ssa_id: id1,
4511                amount: sufficient_balance(),
4512            },
4513            now,
4514            0,
4515        );
4516
4517        let actions = sup.handle_event(&SessionPixEvent::Recovered(id1), now, 0);
4518        assert_eq!(actions.len(), 2, "expected gate reset followed by the fallback request");
4519        assert!(matches!(actions[0], SessionPixAction::ProgressNotification));
4520        match &actions[1] {
4521            SessionPixAction::RequestSsa { ssa_ids, .. } => {
4522                assert_eq!(ssa_ids[0].ssa_index(), SsaIndex::new(2).unwrap());
4523            }
4524            other => panic!("expected RequestSsa, got {other:?}"),
4525        }
4526    }
4527
4528    #[test]
4529    fn recovered_with_prior_early_event_does_not_fallback() {
4530        let p = pseudonym();
4531        let (mut sup, _) = SessionPixSupervisor::new(default_cfg(), dims(10, 5), p, Instant::now());
4532        let now = Instant::now();
4533        let id1 = ssa_id(p, 1);
4534
4535        sup.handle_event(&SessionPixEvent::SsaRequestSent(id1), now, 0);
4536        sup.handle_event(&SessionPixEvent::CommitmentVerified(id1), now, 0);
4537        sup.handle_event(
4538            &SessionPixEvent::DepositConfirmed {
4539                ssa_id: id1,
4540                amount: sufficient_balance(),
4541            },
4542            now,
4543            0,
4544        );
4545
4546        sup.handle_event(&SessionPixEvent::AlmostRecovered(id1), now, 0);
4547        let actions = sup.handle_event(&SessionPixEvent::Recovered(id1), now, 0);
4548        assert!(
4549            matches!(actions.as_slice(), [SessionPixAction::ProgressNotification]),
4550            "the existing gate is reset and now belongs to the predecessor's bounded tail: {actions:?}"
4551        );
4552    }
4553
4554    // ---------------------------------------------------------------
4555    // Close behavior
4556    // ---------------------------------------------------------------
4557
4558    #[test]
4559    fn close_action_emitted_at_most_once() {
4560        let p = pseudonym();
4561        let (mut sup, _) = SessionPixSupervisor::new(default_cfg(), dims(10, 5), p, Instant::now());
4562        let now = Instant::now();
4563        let id = ssa_id(p, 1);
4564
4565        sup.handle_event(&SessionPixEvent::SsaRequestSent(id), now, 0);
4566        sup.handle_deadline(now + Duration::from_secs(100), 0);
4567        assert!(sup.closed);
4568
4569        let empty = sup.handle_event(&SessionPixEvent::SsaRequestSent(ssa_id(p, 999)), now, 0);
4570        assert!(empty.is_empty());
4571        let empty = sup.handle_deadline(now + Duration::from_secs(200), 0);
4572        assert!(empty.is_empty());
4573    }
4574
4575    #[test]
4576    fn all_events_after_close_are_ignored() {
4577        let p = pseudonym();
4578        let (mut sup, _) = SessionPixSupervisor::new(default_cfg(), dims(10, 5), p, Instant::now());
4579        let now = Instant::now();
4580        let id = ssa_id(p, 1);
4581
4582        sup.handle_event(&SessionPixEvent::SsaRequestSent(id), now, 0);
4583        sup.handle_deadline(now + Duration::from_secs(100), 0);
4584        assert!(sup.closed);
4585
4586        for ev in &[
4587            SessionPixEvent::SsaRequestSent(ssa_id(p, 2)),
4588            SessionPixEvent::CommitmentVerified(ssa_id(p, 2)),
4589            SessionPixEvent::DepositConfirmed {
4590                ssa_id: ssa_id(p, 2),
4591                amount: sufficient_balance(),
4592            },
4593            SessionPixEvent::DepositObserverClosed(ssa_id(p, 2)),
4594            SessionPixEvent::RecoveryProgress(make_progress(ssa_id(p, 2), 1, 50, 0)),
4595            SessionPixEvent::AlmostRecovered(ssa_id(p, 2)),
4596            SessionPixEvent::Recovered(ssa_id(p, 2)),
4597            SessionPixEvent::UnverifiableShares {
4598                ssa_id: ssa_id(p, 2),
4599                observed_total: 1,
4600                peer: crate::supervision::test_peer(),
4601            },
4602        ] {
4603            assert!(
4604                sup.handle_event(ev, now, 0).is_empty(),
4605                "event should be ignored after close"
4606            );
4607        }
4608    }
4609
4610    // ---------------------------------------------------------------
4611    // next_deadline
4612    // ---------------------------------------------------------------
4613
4614    #[test]
4615    fn next_deadline_none_when_no_ssas() {
4616        let (sup, _) = SessionPixSupervisor::new(default_cfg(), dims(10, 5), pseudonym(), Instant::now());
4617        assert!(sup.next_deadline().is_none());
4618    }
4619
4620    #[test]
4621    fn next_deadline_returns_earliest() {
4622        let p = pseudonym();
4623        let (mut sup, _) = SessionPixSupervisor::new(default_cfg(), dims(10, 5), p, Instant::now());
4624        let now = Instant::now();
4625
4626        // Two SSAs in flight with deadlines 40 s apart: the later one is armed second, so returning
4627        // the earliest is a real choice rather than the only candidate.
4628        let id1 = ssa_id(p, 1);
4629        sup.handle_event(&SessionPixEvent::SsaRequestSent(id1), now, 0);
4630        let id2 = ssa_id(p, 2);
4631        sup.handle_event(&SessionPixEvent::SsaRequestSent(id2), now + Duration::from_secs(40), 0);
4632
4633        let dl = sup.next_deadline().unwrap();
4634
4635        let expected = now + Duration::from_secs(20);
4636        assert!((dl - expected).as_millis() < 10, "expected {expected:?}, got {dl:?}");
4637    }
4638
4639    /// Pipelining a second SSA must not disturb the first one's deadlines.
4640    ///
4641    /// This used to be a `SessionManager` test asserting that two per-index `PixKillSwitch` abort
4642    /// handles coexisted. The deadlines are the supervisor's now, so the invariant is stated here
4643    /// against the state it actually lives in — and it is stronger, because a shared timer would
4644    /// satisfy "both handles present" but not "both instants unchanged".
4645    #[test]
4646    fn pipelining_a_second_ssa_leaves_the_first_ones_deadlines_alone() {
4647        let p = pseudonym();
4648        let (mut sup, _) = SessionPixSupervisor::new(default_cfg(), dims(10, 5), p, Instant::now());
4649        let now = Instant::now();
4650
4651        let id1 = ssa_id(p, 1);
4652        sup.handle_event(&SessionPixEvent::SsaRequestSent(id1), now, 0);
4653        sup.handle_event(&SessionPixEvent::CommitmentVerified(id1), now, 0);
4654        sup.handle_event(
4655            &SessionPixEvent::DepositConfirmed {
4656                ssa_id: id1,
4657                amount: sufficient_balance(),
4658            },
4659            now,
4660            0,
4661        );
4662
4663        let before = sup.ssas.iter().find(|s| s.ssa_id == id1).unwrap();
4664        assert_eq!(before.phase, SsaPhase::Recovering);
4665        let (hard_before, idle_before) = (before.recovery_hard_deadline, before.recovery_idle_deadline);
4666
4667        // Early recovery on a funded SSA is what pipelines the next one.
4668        let later = now + Duration::from_secs(5);
4669        let actions = sup.handle_event(&SessionPixEvent::AlmostRecovered(id1), later, 0);
4670        let id2 = match actions.as_slice() {
4671            [SessionPixAction::RequestSsa { ssa_ids, .. }] => ssa_ids[0],
4672            other => panic!("expected exactly one RequestSsa, got {other:?}"),
4673        };
4674        assert_eq!(id2.ssa_index().get(), 2, "the pipelined SSA must take the next index");
4675
4676        sup.handle_event(&SessionPixEvent::SsaRequestSent(id2), later, 0);
4677        sup.handle_event(&SessionPixEvent::CommitmentVerified(id2), later, 0);
4678
4679        let after = sup.ssas.iter().find(|s| s.ssa_id == id1).unwrap();
4680        assert_eq!(
4681            after.phase,
4682            SsaPhase::Recovering,
4683            "pipelining moved the first SSA's phase"
4684        );
4685        assert_eq!(
4686            after.recovery_hard_deadline, hard_before,
4687            "pipelining moved the first SSA's hard recovery deadline"
4688        );
4689        assert_eq!(
4690            after.recovery_idle_deadline, idle_before,
4691            "pipelining moved the first SSA's idle recovery deadline"
4692        );
4693
4694        // ...and the second one is independently armed rather than sharing the first's timers.
4695        let second = sup.ssas.iter().find(|s| s.ssa_id == id2).unwrap();
4696        assert_eq!(second.phase, SsaPhase::AwaitingDeposit);
4697        assert!(second.deposit_deadline.is_some());
4698    }
4699
4700    // ---------------------------------------------------------------
4701    // SsaIndex overflow
4702    // ---------------------------------------------------------------
4703
4704    #[test]
4705    fn ssa_index_overflow_fails_closed() {
4706        let p = pseudonym();
4707        let (mut sup, _) = SessionPixSupervisor::new(default_cfg(), dims(10, 5), p, Instant::now());
4708        let now = Instant::now();
4709
4710        let id1 = ssa_id(p, 1);
4711        sup.handle_event(&SessionPixEvent::SsaRequestSent(id1), now, 0);
4712        sup.handle_event(&SessionPixEvent::CommitmentVerified(id1), now, 0);
4713        sup.handle_event(
4714            &SessionPixEvent::DepositConfirmed {
4715                ssa_id: id1,
4716                amount: sufficient_balance(),
4717            },
4718            now,
4719            0,
4720        );
4721
4722        // Wind the allocator to the last usable index, so pipelining the next one overflows.
4723        sup.next_ssa_index = u32::MAX;
4724
4725        let actions = sup.handle_event(&SessionPixEvent::AlmostRecovered(id1), now, 0);
4726        assert!(
4727            matches!(
4728                actions.as_slice(),
4729                [SessionPixAction::Close(SessionPixCloseReason::InvalidTransition)]
4730            ),
4731            "expected a single Close, got {actions:?}"
4732        );
4733        assert!(sup.closed);
4734        assert_eq!(
4735            sup.ssas.len(),
4736            1,
4737            "an index that could not be allocated must not leave a record behind"
4738        );
4739    }
4740
4741    // ---------------------------------------------------------------
4742    // action_result
4743    // ---------------------------------------------------------------
4744
4745    #[test]
4746    fn request_failure_result_closes() {
4747        let p = pseudonym();
4748        let (mut sup, _) = SessionPixSupervisor::new(default_cfg(), dims(10, 5), p, Instant::now());
4749        let now = Instant::now();
4750
4751        let actions = sup.action_result(
4752            &SessionPixAction::RequestSsa {
4753                ssa_ids: vec![ssa_id(p, 1)],
4754                params: dims(10, 5),
4755            },
4756            false,
4757            now,
4758        );
4759        assert_eq!(actions.len(), 1);
4760        assert!(matches!(
4761            actions[0],
4762            SessionPixAction::Close(SessionPixCloseReason::SupervisorUnavailable)
4763        ));
4764    }
4765
4766    // ---------------------------------------------------------------
4767    // Tombstone and multi-SSA lifecycle
4768    // ---------------------------------------------------------------
4769
4770    #[test]
4771    fn tombstone_expiry_clears_recovered_ssa() {
4772        let mut cfg = default_cfg();
4773        cfg.tombstone_retention_window = Duration::from_secs(10);
4774        let p = pseudonym();
4775        let (mut sup, _) = SessionPixSupervisor::new(cfg, dims(10, 5), p, Instant::now());
4776        let start = Instant::now();
4777        let id = ssa_id(p, 1);
4778
4779        sup.handle_event(&SessionPixEvent::SsaRequestSent(id), start, 0);
4780        sup.handle_event(&SessionPixEvent::CommitmentVerified(id), start, 0);
4781        sup.handle_event(
4782            &SessionPixEvent::DepositConfirmed {
4783                ssa_id: id,
4784                amount: sufficient_balance(),
4785            },
4786            start,
4787            0,
4788        );
4789        // Recovery pipelines the next SSA, which is tracked from the moment it is requested.
4790        sup.handle_event(&SessionPixEvent::Recovered(id), start, 0);
4791        assert_eq!(sup.ssas.len(), 2, "the recovered SSA's tombstone plus its successor");
4792
4793        // Before tombstone expires — still present.
4794        let actions = sup.handle_deadline(start + Duration::from_secs(5), 0);
4795        assert!(actions.is_empty());
4796        assert_eq!(sup.ssas.len(), 2);
4797
4798        // After tombstone expires — RetireSsa, and nothing else: the successor is still pending, so
4799        // the Session has not run out of SSAs. Closing here would kill a Session over a request
4800        // that is merely in flight.
4801        let actions = sup.handle_deadline(start + Duration::from_secs(11), 5);
4802        assert_eq!(actions.len(), 1, "expected RetireSsa alone, got {actions:?}");
4803        assert!(
4804            matches!(&actions[0], SessionPixAction::RetireSsa(rid) if *rid == id),
4805            "action should be RetireSsa({id}), got {:?}",
4806            actions[0]
4807        );
4808        assert_eq!(
4809            sup.ssas.len(),
4810            1,
4811            "the successor must survive its predecessor's retirement"
4812        );
4813
4814        let seen = sup.dims.target_useful_shares() + 1;
4815        let actions = sup.handle_event(
4816            &SessionPixEvent::RecoveryProgress(make_recovered_tail_progress(&sup, id, seen)),
4817            start + Duration::from_secs(12),
4818            6,
4819        );
4820        assert!(
4821            actions
4822                .iter()
4823                .any(|action| matches!(action, SessionPixAction::ProgressNotification)),
4824            "short tombstone cleanup must not truncate a longer paid FIFO drain: {actions:?}"
4825        );
4826    }
4827
4828    #[test]
4829    fn all_ssas_terminal_closes_session_after_multi_ssa_close() {
4830        let p = pseudonym();
4831        let (mut sup, _) = SessionPixSupervisor::new(default_cfg(), dims(10, 5), p, Instant::now());
4832        let start = Instant::now();
4833        let id1 = ssa_id(p, 1);
4834
4835        // Set up first SSA through recovery.
4836        sup.handle_event(&SessionPixEvent::SsaRequestSent(id1), start, 0);
4837        sup.handle_event(&SessionPixEvent::CommitmentVerified(id1), start, 0);
4838        sup.handle_event(
4839            &SessionPixEvent::DepositConfirmed {
4840                ssa_id: id1,
4841                amount: sufficient_balance(),
4842            },
4843            start,
4844            0,
4845        );
4846        // AlmostRecovered triggers next SSA request.
4847        let actions = sup.handle_event(&SessionPixEvent::AlmostRecovered(id1), start, 0);
4848        assert_eq!(actions.len(), 1);
4849        assert!(matches!(actions[0], SessionPixAction::RequestSsa { .. }));
4850        sup.handle_event(&SessionPixEvent::SsaRequestSent(ssa_id(p, 2)), start, 0);
4851
4852        // Both deadlines have expired by now (SSA 1 hard deadline at start + 3600s,
4853        // SSA 2 commitment deadline at start + 20s). The loop processes SSA 1 first
4854        // (RecoveryDeadline → removed), then SSA 2 (CommitmentTimeout → session close).
4855        let actions = sup.handle_deadline(start + Duration::from_secs(7200), 5);
4856        assert_eq!(actions.len(), 2, "expected RetireSsa(id1) + Close(RecoveryDeadline)");
4857        assert!(
4858            matches!(&actions[0], SessionPixAction::RetireSsa(rid) if *rid == id1),
4859            "first action should be RetireSsa({id1}), got {:?}",
4860            actions[0]
4861        );
4862        // SSA 1 fails with RecoveryDeadline first, so that reason is surfaced.
4863        assert!(matches!(
4864            actions[1],
4865            SessionPixAction::Close(SessionPixCloseReason::RecoveryDeadline)
4866        ));
4867        assert!(sup.closed);
4868    }
4869
4870    #[test]
4871    fn next_deadline_none_when_after_tombstone_expiry() {
4872        let mut cfg = default_cfg();
4873        cfg.tombstone_retention_window = Duration::from_secs(10);
4874        let p = pseudonym();
4875        let (mut sup, _) = SessionPixSupervisor::new(cfg, dims(10, 5), p, Instant::now());
4876        let start = Instant::now();
4877        let id = ssa_id(p, 1);
4878
4879        sup.handle_event(&SessionPixEvent::SsaRequestSent(id), start, 0);
4880        sup.handle_event(&SessionPixEvent::CommitmentVerified(id), start, 0);
4881        sup.handle_event(
4882            &SessionPixEvent::DepositConfirmed {
4883                ssa_id: id,
4884                amount: sufficient_balance(),
4885            },
4886            start,
4887            0,
4888        );
4889        sup.handle_event(&SessionPixEvent::Recovered(id), start, 0);
4890
4891        // Finish the paid FIFO tail; this test is about tombstone scheduling, not drain liveness.
4892        let max_seen = max_cycle_shares(&sup);
4893        sup.handle_event(
4894            &SessionPixEvent::RecoveryProgress(make_recovered_tail_progress(&sup, id, max_seen)),
4895            start,
4896            max_seen,
4897        );
4898
4899        // While tombstone is alive, next_deadline returns the tombstone_until.
4900        assert!(sup.next_deadline().is_some());
4901
4902        // After tombstone expires, handle_deadline removes it and closes.
4903        sup.handle_deadline(start + Duration::from_secs(11), 0);
4904
4905        assert!(sup.next_deadline().is_none());
4906    }
4907
4908    #[test]
4909    fn action_result_close_sets_closed() {
4910        let p = pseudonym();
4911        let (mut sup, _actions) = SessionPixSupervisor::new(default_cfg(), dims(10, 5), p, Instant::now());
4912        let now = Instant::now();
4913        assert!(!sup.closed);
4914
4915        // First fake-close via action_result.
4916        let _ = sup.action_result(&SessionPixAction::Close(SessionPixCloseReason::RecoveryIdle), true, now);
4917        assert!(sup.closed);
4918
4919        // After close, all subsequent calls are no-ops.
4920        let actions = sup.handle_deadline(now, 0);
4921        assert!(actions.is_empty());
4922        let actions = sup.handle_event(&SessionPixEvent::SsaRequestSent(ssa_id(p, 1)), now, 0);
4923        assert!(actions.is_empty());
4924    }
4925
4926    // -------------------------------------------------------------------
4927    // M-02: Event ordering guards
4928    // -------------------------------------------------------------------
4929
4930    #[test]
4931    fn recovered_before_commitment_is_ignored() {
4932        let p = pseudonym();
4933        let (mut sup, _) = SessionPixSupervisor::new(default_cfg(), dims(10, 5), p, Instant::now());
4934        let now = Instant::now();
4935        let id = ssa_id(p, 1);
4936
4937        sup.handle_event(&SessionPixEvent::SsaRequestSent(id), now, 0);
4938
4939        // Recovered arrives before CommitmentVerified — should be ignored.
4940        let actions = sup.handle_event(&SessionPixEvent::Recovered(id), now, 0);
4941        assert!(actions.is_empty(), "recovered before commitment should be ignored");
4942        assert!(!sup.closed);
4943    }
4944
4945    #[test]
4946    fn recovered_before_deposit_is_ignored() {
4947        let p = pseudonym();
4948        let (mut sup, _) = SessionPixSupervisor::new(default_cfg(), dims(10, 5), p, Instant::now());
4949        let now = Instant::now();
4950        let id = ssa_id(p, 1);
4951
4952        sup.handle_event(&SessionPixEvent::SsaRequestSent(id), now, 0);
4953        sup.handle_event(&SessionPixEvent::CommitmentVerified(id), now, 0);
4954
4955        // Recovered arrives before DepositConfirmed — should be ignored.
4956        let actions = sup.handle_event(&SessionPixEvent::Recovered(id), now, 0);
4957        assert!(actions.is_empty(), "recovered before deposit should be ignored");
4958    }
4959
4960    /// A fully recovered and subsequently funded SSA must not lead to the
4961    /// session being closed with `RecoveryIdle`. The `Recovered` event
4962    /// arriving during `AwaitingDeposit` is deferred via `recovered_pending`
4963    /// — once the deposit confirms and the SSA enters `Recovering`, the
4964    /// deferred tombstone transition fires immediately. Its recovery clocks then bound only the
4965    /// compact paid tail: with no service consumed, idle expiry re-arms rather than closing.
4966    #[test]
4967    fn recovered_before_deposit_then_funded_session_survives() {
4968        let p = pseudonym();
4969        let start = Instant::now();
4970        let (mut sup, _) = SessionPixSupervisor::new(default_cfg(), dims(2, 2), p, start);
4971        let id = ssa_id(p, 1);
4972
4973        sup.handle_event(&SessionPixEvent::SsaRequestSent(id), start, 0);
4974        sup.handle_event(&SessionPixEvent::CommitmentVerified(id), start, 0);
4975
4976        // Entry delivered all shares before the on-chain deposit confirmed.
4977        // Recovered arrives while still AwaitingDeposit — deferred.
4978        sup.handle_event(&SessionPixEvent::Recovered(id), start, 0);
4979
4980        // The deposit confirms — triggers deferred tombstone + RequestSsa.
4981        let actions = sup.handle_event(
4982            &SessionPixEvent::DepositConfirmed {
4983                ssa_id: id,
4984                amount: sufficient_balance(),
4985            },
4986            start,
4987            10,
4988        );
4989        assert!(!sup.closed, "session must be alive after funding");
4990
4991        // The first SSA should have been tombstoned (not stuck in Recovering).
4992        let ssa1_phase = sup.ssas.iter().find(|s| s.ssa_id == id).map(|s| &s.phase);
4993        assert!(
4994            matches!(ssa1_phase, Some(SsaPhase::Recovered { .. })),
4995            "SSA 1 should be tombstoned, got {ssa1_phase:?}"
4996        );
4997
4998        // A RequestSsa for the next SSA must have been emitted (so the
4999        // driver can start the successor).
5000        let next_id = ssa_id(p, 2);
5001        assert!(
5002            actions
5003                .iter()
5004                .any(|a| matches!(a, SessionPixAction::RequestSsa { ssa_ids, .. } if ssa_ids.contains(&next_id))),
5005            "expected RequestSsa for SSA 2, got actions: {actions:?}"
5006        );
5007
5008        // No packet was served after the handoff baseline, so the tail idle clock re-arms. Service
5009        // consumption without a corresponding old-tail or successor signal would close instead;
5010        // that is the bound preventing a declared surplus from becoming indefinite free service.
5011        let deadline_actions = sup.handle_deadline(start + Duration::from_secs(61), 10);
5012        assert!(
5013            !deadline_actions
5014                .iter()
5015                .any(|a| matches!(a, SessionPixAction::Close(SessionPixCloseReason::RecoveryIdle))),
5016            "must not close with RecoveryIdle, got: {deadline_actions:?}"
5017        );
5018    }
5019
5020    #[test]
5021    fn the_immediate_recovered_tail_reports_bounded_liveness() {
5022        let p = pseudonym();
5023        let (mut sup, _) = SessionPixSupervisor::new(default_cfg(), dims(10, 5), p, Instant::now());
5024        let start = Instant::now();
5025        let id = ssa_id(p, 1);
5026
5027        sup.handle_event(&SessionPixEvent::SsaRequestSent(id), start, 0);
5028        sup.handle_event(&SessionPixEvent::CommitmentVerified(id), start, 0);
5029        sup.handle_event(
5030            &SessionPixEvent::DepositConfirmed {
5031                ssa_id: id,
5032                amount: sufficient_balance(),
5033            },
5034            start,
5035            0,
5036        );
5037        sup.handle_event(&SessionPixEvent::Recovered(id), start, 0);
5038
5039        let seen = sup.dims.target_useful_shares() + 1;
5040        let actions = sup.handle_event(
5041            &SessionPixEvent::RecoveryProgress(make_recovered_tail_progress(&sup, id, seen)),
5042            start,
5043            100,
5044        );
5045        assert!(
5046            actions
5047                .iter()
5048                .any(|action| matches!(action, SessionPixAction::ProgressNotification)),
5049            "the authorized predecessor must keep the gate alive while its FIFO tail drains: {actions:?}"
5050        );
5051    }
5052
5053    /// A concurrent acknowledgement batch can publish an older absolute snapshot after the
5054    /// terminal event. It must not be accepted as tail liveness, but neither may it close an honest
5055    /// Session merely because `Recovered` overtook it.
5056    #[test]
5057    fn a_pre_recovery_snapshot_overtaken_by_recovered_is_absorbed() {
5058        let mut cfg = default_cfg();
5059        cfg.ssas_per_request = 2;
5060        let p = pseudonym();
5061        let start = Instant::now();
5062        let (mut sup, _) = SessionPixSupervisor::new(cfg, dims(10, 5), p, start);
5063        let recovered = ssa_id(p, 1);
5064        fund_batch(&mut sup, p, 2, start);
5065        sup.handle_event(&SessionPixEvent::Recovered(recovered), start, 0);
5066
5067        let target = sup.dims.target_useful_shares();
5068        let stale = SsaRecoveryProgress {
5069            ssa_id: recovered,
5070            useful_shares: target - 1,
5071            shares_seen: target + 1,
5072            target_useful_shares: target,
5073            recovered_polynomials: sup.dims.polys_per_ssa() - 1,
5074        };
5075        let actions = sup.handle_event(&SessionPixEvent::RecoveryProgress(stale), start, 1);
5076        assert!(
5077            actions.is_empty(),
5078            "an overtaken non-terminal snapshot must be stale noise, not liveness or a close: {actions:?}"
5079        );
5080        assert_eq!(
5081            sup.paid_recovery_tail.as_ref().map(|tail| tail.largest_shares_seen),
5082            Some(target),
5083            "the stale snapshot must not consume any paid tail"
5084        );
5085
5086        let actions = sup.handle_event(
5087            &SessionPixEvent::RecoveryProgress(make_recovered_tail_progress(&sup, recovered, target + 2)),
5088            start,
5089            2,
5090        );
5091        assert!(
5092            actions
5093                .iter()
5094                .any(|action| matches!(action, SessionPixAction::ProgressNotification)),
5095            "a later terminal snapshot must still consume the legitimate tail: {actions:?}"
5096        );
5097    }
5098
5099    /// `Recovered` is itself proof of fresh progress, even if its immediately preceding progress
5100    /// snapshot was dropped. The tail gets a new idle interval, while retaining the original hard
5101    /// deadline.
5102    #[test]
5103    fn recovery_refreshes_the_tail_idle_clock_once() {
5104        let mut cfg = default_cfg();
5105        cfg.ssas_per_request = 2;
5106        cfg.max_recovery_idle = Duration::from_secs(10);
5107        let p = pseudonym();
5108        let start = Instant::now();
5109        let (mut sup, _) = SessionPixSupervisor::new(cfg, dims(10, 5), p, start);
5110        let recovered = ssa_id(p, 1);
5111        fund_batch(&mut sup, p, 2, start);
5112
5113        let recovered_at = start + Duration::from_secs(9);
5114        sup.handle_event(&SessionPixEvent::Recovered(recovered), recovered_at, 5);
5115        let actions = sup.handle_deadline(start + Duration::from_secs(11), 6);
5116        assert!(
5117            actions
5118                .iter()
5119                .all(|action| !matches!(action, SessionPixAction::Close(SessionPixCloseReason::RecoveryIdle))),
5120            "the pre-recovery idle instant must not expire a freshly recovered tail: {actions:?}"
5121        );
5122
5123        let actions = sup.handle_deadline(start + Duration::from_secs(20), 6);
5124        assert!(
5125            matches!(
5126                actions.as_slice(),
5127                [SessionPixAction::Close(SessionPixCloseReason::RecoveryIdle)]
5128            ),
5129            "the one-time refresh must not weaken the ordinary stalled-tail bound: {actions:?}"
5130        );
5131    }
5132
5133    /// Defense in depth: even a forged local snapshot cannot exceed the negotiated cycle volume.
5134    #[test]
5135    fn a_recovered_tail_cannot_report_more_service_than_the_entry_paid_for() {
5136        let mut cfg = default_cfg();
5137        cfg.ssas_per_request = 2;
5138        let p = pseudonym();
5139        let start = Instant::now();
5140        let (mut sup, _) = SessionPixSupervisor::new(cfg, dims(10, 5), p, start);
5141        let recovered = ssa_id(p, 1);
5142        fund_batch(&mut sup, p, 2, start);
5143        sup.handle_event(&SessionPixEvent::Recovered(recovered), start, 0);
5144
5145        let impossible = max_cycle_shares(&sup) + 1;
5146        let actions = sup.handle_event(
5147            &SessionPixEvent::RecoveryProgress(make_recovered_tail_progress(&sup, recovered, impossible)),
5148            start,
5149            1,
5150        );
5151        assert!(
5152            matches!(
5153                actions.as_slice(),
5154                [SessionPixAction::Close(SessionPixCloseReason::CounterRegression)]
5155            ),
5156            "over-budget recovered progress must fail closed, never become gate liveness: {actions:?}"
5157        );
5158        assert_eq!(
5159            sup.paid_recovery_tail.as_ref().map(|tail| tail.largest_shares_seen),
5160            Some(sup.dims.target_useful_shares()),
5161            "the rejected snapshot must not mutate the authorized absolute watermark"
5162        );
5163    }
5164
5165    /// Declaring a paid remainder is not itself liveness: silence while consuming service closes.
5166    #[test]
5167    fn a_stalled_recovered_tail_cannot_hold_the_session_open() {
5168        let mut cfg = default_cfg();
5169        cfg.ssas_per_request = 2;
5170        cfg.max_recovery_idle = Duration::from_secs(10);
5171        let p = pseudonym();
5172        let start = Instant::now();
5173        let (mut sup, _) = SessionPixSupervisor::new(cfg, dims(10, 5), p, start);
5174        let recovered = ssa_id(p, 1);
5175        fund_batch(&mut sup, p, 2, start);
5176        sup.handle_event(&SessionPixEvent::Recovered(recovered), start, 0);
5177
5178        let actions = sup.handle_deadline(start + Duration::from_secs(11), 1);
5179        assert!(
5180            matches!(
5181                actions.as_slice(),
5182                [SessionPixAction::Close(SessionPixCloseReason::RecoveryIdle)]
5183            ),
5184            "the Entry cannot turn an unspent declared surplus into indefinite funded service: {actions:?}"
5185        );
5186    }
5187
5188    /// Valid tail traffic may refresh the idle timer, but it cannot stretch the paid predecessor's
5189    /// original absolute resource deadline.
5190    #[test]
5191    fn a_trickling_recovered_tail_cannot_extend_the_hard_deadline() {
5192        let mut cfg = default_cfg();
5193        cfg.ssas_per_request = 2;
5194        cfg.max_recovery_idle = Duration::from_secs(10);
5195        cfg.max_recovery_time = Duration::from_secs(20);
5196        let p = pseudonym();
5197        let start = Instant::now();
5198        let (mut sup, _) = SessionPixSupervisor::new(cfg, dims(10, 5), p, start);
5199        let recovered = ssa_id(p, 1);
5200        fund_batch(&mut sup, p, 2, start);
5201        sup.handle_event(
5202            &SessionPixEvent::Recovered(recovered),
5203            start + Duration::from_secs(1),
5204            0,
5205        );
5206
5207        let target = sup.dims.target_useful_shares();
5208        for (at, seen, served) in [(9, target + 1, 1), (18, target + 2, 2)] {
5209            let actions = sup.handle_event(
5210                &SessionPixEvent::RecoveryProgress(make_recovered_tail_progress(&sup, recovered, seen)),
5211                start + Duration::from_secs(at),
5212                served,
5213            );
5214            assert!(
5215                actions
5216                    .iter()
5217                    .all(|action| !matches!(action, SessionPixAction::Close(_))),
5218                "legitimate tail progress before the hard limit must remain live: {actions:?}"
5219            );
5220        }
5221
5222        let actions = sup.handle_deadline(start + Duration::from_secs(21), 3);
5223        assert!(
5224            matches!(
5225                actions.as_slice(),
5226                [SessionPixAction::Close(SessionPixCloseReason::RecoveryDeadline)]
5227            ),
5228            "tail trickle must not move the predecessor's absolute deadline: {actions:?}"
5229        );
5230        // The tail took the front at +1 s and ends here, so it owes its 20 s like any other phase
5231        // that reaches a terminal edge — being ended by a deadline is still ending.
5232        assert_eq!(
5233            Some(&(PixCyclePhase::PaidTail, 20_000)),
5234            sup.take_telemetry_events()
5235                .phase_durations
5236                .iter()
5237                .find(|(phase, _)| *phase == PixCyclePhase::PaidTail)
5238        );
5239    }
5240
5241    /// Recovered does not mean replayable: only the immediate predecessor owns the one tail slot.
5242    #[test]
5243    fn an_older_recovered_cycle_cannot_reopen_service() {
5244        let mut cfg = default_cfg();
5245        cfg.ssas_per_request = 2;
5246        cfg.max_off_front_share_fraction = 1.0;
5247        let p = pseudonym();
5248        let start = Instant::now();
5249        let (mut sup, _) = SessionPixSupervisor::new(cfg, dims(10, 5), p, start);
5250        let first = ssa_id(p, 1);
5251        let second = ssa_id(p, 2);
5252        fund_batch(&mut sup, p, 2, start);
5253
5254        sup.handle_event(&SessionPixEvent::Recovered(first), start, 0);
5255        sup.handle_event(
5256            &SessionPixEvent::RecoveryProgress(make_progress(second, 1, sup.dims.target_useful_shares(), 0)),
5257            start,
5258            1,
5259        );
5260        sup.handle_event(&SessionPixEvent::Recovered(second), start, 1);
5261
5262        let actions = sup.handle_event(
5263            &SessionPixEvent::RecoveryProgress(make_recovered_tail_progress(
5264                &sup,
5265                first,
5266                sup.dims.target_useful_shares() + 1,
5267            )),
5268            start,
5269            2,
5270        );
5271        assert!(
5272            actions.is_empty(),
5273            "an obsolete recovered-cycle tag must not refresh or reopen the Exit's service gate: {actions:?}"
5274        );
5275    }
5276
5277    #[test]
5278    fn late_tombstone_unverifiable_shares_are_absorbed() {
5279        let p = pseudonym();
5280        let (mut sup, _) = SessionPixSupervisor::new(default_cfg(), dims(10, 5), p, Instant::now());
5281        let start = Instant::now();
5282        let id = ssa_id(p, 1);
5283
5284        sup.handle_event(&SessionPixEvent::SsaRequestSent(id), start, 0);
5285        sup.handle_event(&SessionPixEvent::CommitmentVerified(id), start, 0);
5286        sup.handle_event(
5287            &SessionPixEvent::DepositConfirmed {
5288                ssa_id: id,
5289                amount: sufficient_balance(),
5290            },
5291            start,
5292            0,
5293        );
5294        sup.handle_event(&SessionPixEvent::Recovered(id), start, 0);
5295
5296        // Late unverifiable shares on tombstone — should be absorbed.
5297        let actions = sup.handle_event(
5298            &SessionPixEvent::UnverifiableShares {
5299                ssa_id: id,
5300                observed_total: 5,
5301                peer: crate::supervision::test_peer(),
5302            },
5303            start,
5304            100,
5305        );
5306        assert!(
5307            actions.is_empty(),
5308            "late tombstone unverifiable shares should be absorbed"
5309        );
5310    }
5311
5312    // ---------------------------------------------------------------
5313    // PIX fill
5314    // ---------------------------------------------------------------
5315
5316    /// Dimensions wide enough that the required fill rate is a whole number of packets per second.
5317    ///
5318    /// [`dims`]`(10, 5)` is 120 packets over an hour, which the planner correctly expresses as one
5319    /// packet every twenty-odd seconds — true, but not a rate any assertion about a *ramp* can be
5320    /// written against. These are the profiled dimensions of the worked example: 145 408 packets,
5321    /// which needs about 57 packets/s to clear the aim point.
5322    fn fill_dims() -> PixParams {
5323        dims(2048, 64)
5324    }
5325
5326    /// The rate carried by the last `SetFillRate` in `actions`, if any.
5327    fn planned_rate(actions: &[SessionPixAction]) -> Option<FillRate> {
5328        actions
5329            .iter()
5330            .filter_map(|action| match action {
5331                SessionPixAction::SetFillRate(rate) => Some(*rate),
5332                _ => None,
5333            })
5334            .next_back()
5335    }
5336
5337    /// A funded, clocked cycle at the front, with the fill planner untouched.
5338    ///
5339    /// Returns the supervisor and the cycle's id. `now` is both the construction instant and the
5340    /// moment the deposit confirms, so the cycle's hard deadline is exactly `max_recovery_time` out.
5341    fn funded_at_front(cfg: SupervisorConfig, dims: PixParams, now: Instant) -> (SessionPixSupervisor, HoprPseudonym) {
5342        let p = pseudonym();
5343        let (mut sup, _) = SessionPixSupervisor::new(cfg, dims, p, now);
5344        fund_batch(&mut sup, p, 1, now);
5345        (sup, p)
5346    }
5347
5348    /// Feeds `seen` shares of liveness to `id`, as the reconstructor would.
5349    fn feed_seen(sup: &mut SessionPixSupervisor, id: SsaId<HoprPseudonym>, seen: u64, now: Instant, served: u64) {
5350        let target = sup.dims.target_useful_shares();
5351        let progress = make_progress_seen(id, seen.min(target), seen, target, 0);
5352        sup.handle_event(&SessionPixEvent::RecoveryProgress(progress), now, served);
5353    }
5354
5355    /// Nothing is filled for a Session whose first cycle has not been paid for.
5356    ///
5357    /// Fill exists to protect a *deposit*, so an unfunded cycle has nothing at stake and must cost
5358    /// nothing. The stronger half of the property is the tick itself: `next_fill_tick` returning
5359    /// `None` is what keeps every unfunded PIX Session — including every one that will never be
5360    /// funded — from waking its worker once a second for the whole of its commitment and deposit
5361    /// deadlines.
5362    #[test]
5363    fn no_fill_is_planned_or_ticked_before_a_cycle_is_funded() {
5364        let t0 = Instant::now();
5365        let p = pseudonym();
5366        let (mut sup, _) = SessionPixSupervisor::new(default_cfg(), fill_dims(), p, t0);
5367
5368        assert_eq!(None, sup.next_fill_tick(), "a fresh Session has nothing to fill for");
5369
5370        commit_unfunded(&mut sup, p, 1..=1, t0);
5371        assert_eq!(
5372            None,
5373            sup.next_fill_tick(),
5374            "a committed but unfunded cycle still has nothing at stake"
5375        );
5376
5377        // Ticked by hand anyway, in case a caller drives the timers without consulting the schedule.
5378        let actions = sup.handle_timers(t0 + Duration::from_secs(2), 0);
5379        assert_eq!(None, planned_rate(&actions), "an unfunded cycle must not be filled");
5380    }
5381
5382    /// A cycle that recovers with nothing buffered behind it stops fill until its successor is paid.
5383    ///
5384    /// Recovery frees the reconstructor's state, but it does not put the successor at the front: the
5385    /// successor's clocks arm only once the predecessor's FIFO tail is exhausted, and until then there
5386    /// is no deadline to plan against. Filling on regardless would spend SURBs on a cycle nobody has
5387    /// deposited for yet — and `shares_seen` at the full emission is exactly the case where no tail is
5388    /// created, so this is the handoff with no cover at all.
5389    #[test]
5390    fn a_recovered_cycle_without_a_tail_stops_fill_and_plans_nothing_for_its_successor() {
5391        let t0 = Instant::now();
5392        let (mut sup, p) = funded_at_front(default_cfg(), fill_dims(), t0);
5393        let id = ssa_id(p, 1);
5394
5395        let running = planned_rate(&sup.handle_timers(t0 + SAMPLING_INTERVAL, 0))
5396            .expect("a funded cycle at the front must be filled");
5397        assert!(!running.is_zero());
5398
5399        // Fully emitted, so `perform_recovered_transition` leaves no paid tail behind.
5400        let whole_cycle = max_cycle_shares(&sup);
5401        feed_seen(&mut sup, id, whole_cycle, t0 + Duration::from_secs(2), 0);
5402        let actions = sup.handle_event(&SessionPixEvent::Recovered(id), t0 + Duration::from_secs(2), 0);
5403        assert_eq!(
5404            Some(FillRate::ZERO),
5405            planned_rate(&actions),
5406            "recovery without a tail must silence fill in the same action vector"
5407        );
5408
5409        assert_eq!(
5410            None,
5411            sup.next_fill_tick(),
5412            "an unfunded successor is not something to wake up for"
5413        );
5414        let actions = sup.handle_timers(t0 + Duration::from_secs(4), 0);
5415        assert_eq!(None, planned_rate(&actions), "and nothing is planned for it either");
5416    }
5417
5418    /// The whole point: an idle funded cycle is carried to completion inside its own deadline.
5419    ///
5420    /// The Session sends not one application packet — `served_total` never moves — so every share
5421    /// that comes back was earned by a fill keep-alive. The simulation closes the loop the deployment
5422    /// does: whatever fill emits in a second arrives as liveness in the next one, and the planner
5423    /// re-derives the rate from the remainder that leaves.
5424    ///
5425    /// Two properties, and the second is why `max_rate` is a configuration item rather than a
5426    /// constant: the cycle must finish *before the aim point*, and it must do so without the planner
5427    /// ever asking for more than the ceiling it was given.
5428    #[test]
5429    fn an_idle_funded_cycle_is_filled_to_completion_before_its_aim_point() {
5430        let cfg = default_cfg();
5431        let t0 = Instant::now();
5432        let (mut sup, p) = funded_at_front(cfg.clone(), fill_dims(), t0);
5433        let id = ssa_id(p, 1);
5434
5435        let cycle = max_cycle_shares(&sup);
5436        let aim_point = cfg.max_recovery_time.mul_f64(cfg.fill.finish_fraction);
5437
5438        let mut rate = FillRate::ZERO;
5439        let mut sent = 0.0_f64;
5440        let mut completed_at = None;
5441        for second in 1..=aim_point.as_secs() {
5442            let now = t0 + Duration::from_secs(second);
5443            if let Some(planned) = planned_rate(&sup.handle_timers(now, 0)) {
5444                assert!(
5445                    planned.as_packets_per_sec() <= cfg.fill.max_rate as f64,
5446                    "fill asked for {} packets/s against a ceiling of {}",
5447                    planned.as_packets_per_sec(),
5448                    cfg.fill.max_rate
5449                );
5450                rate = planned;
5451            }
5452
5453            // A lossless return path, so the packets sent in this second are the shares seen in it.
5454            sent += rate.as_packets_per_sec();
5455            if sent as u64 >= cycle {
5456                completed_at = Some(second);
5457                break;
5458            }
5459            feed_seen(&mut sup, id, sent as u64, now, 0);
5460        }
5461
5462        let completed_at = completed_at.unwrap_or_else(|| {
5463            panic!("an idle cycle of {cycle} packets was not filled to completion within {aim_point:?}")
5464        });
5465        assert!(
5466            Duration::from_secs(completed_at) < aim_point,
5467            "the cycle completed at {completed_at}s, which is not inside the {aim_point:?} aim point"
5468        );
5469    }
5470
5471    /// An application already sending faster than the requirement is not helped, only accompanied.
5472    ///
5473    /// Fill is `required − organic`, so a Session whose own traffic covers the cycle needs no help and
5474    /// must not be charged for any: what is left is the heartbeat, which exists for the Entry's idle
5475    /// timer and its SURB balancer rather than for the deadline. Getting this wrong is not a small
5476    /// waste — it doubles the return traffic of every busy PIX Session on the node.
5477    #[test]
5478    fn organic_traffic_above_the_requirement_holds_fill_at_the_heartbeat() {
5479        let cfg = default_cfg();
5480        let t0 = Instant::now();
5481        let (mut sup, p) = funded_at_front(cfg.clone(), fill_dims(), t0);
5482        let id = ssa_id(p, 1);
5483
5484        // Comfortably above the ~57 packets/s the cycle needs, and sustained past the averaging
5485        // window so the estimate is the application's rate rather than its first second.
5486        const ORGANIC: u64 = 200;
5487        let mut rate = None;
5488        for second in 1..=(4 * ORGANIC_WINDOW.as_secs()) {
5489            let now = t0 + Duration::from_secs(second);
5490            let served = ORGANIC * second;
5491            if let Some(planned) = planned_rate(&sup.handle_timers(now, served)) {
5492                rate = Some(planned);
5493            }
5494            feed_seen(&mut sup, id, served, now, served);
5495        }
5496
5497        assert_eq!(
5498            Some(FillRate::once_per(cfg.fill.heartbeat)),
5499            rate,
5500            "an application outrunning the requirement must leave fill at its heartbeat"
5501        );
5502    }
5503
5504    /// When the application stops, fill picks the cycle up from the remainder that is actually left.
5505    ///
5506    /// This is the case the deployment is full of — a VPN Session that transfers and then idles — and
5507    /// the trap in it is arithmetic rather than plumbing: the planner must re-derive the rate from
5508    /// `E − shares seen`, not from the rate it would have asked for at the start. A planner that
5509    /// replayed its opening figure would over-send by however much the application already
5510    /// contributed.
5511    #[test]
5512    fn fill_resumes_from_the_true_remainder_when_organic_traffic_stops() {
5513        let cfg = default_cfg();
5514        let t0 = Instant::now();
5515        let (mut sup, p) = funded_at_front(cfg.clone(), fill_dims(), t0);
5516        let id = ssa_id(p, 1);
5517
5518        const ORGANIC: u64 = 200;
5519        const BUSY_FOR: u64 = 300;
5520        let mut seen = 0;
5521        for second in 1..=BUSY_FOR {
5522            let now = t0 + Duration::from_secs(second);
5523            seen = ORGANIC * second;
5524            sup.handle_timers(now, seen);
5525            feed_seen(&mut sup, id, seen, now, seen);
5526        }
5527
5528        // The application stops dead. `served_total` is frozen from here on.
5529        let mut rate = None;
5530        for second in (BUSY_FOR + 1)..=(BUSY_FOR + 4 * ORGANIC_WINDOW.as_secs()) {
5531            let now = t0 + Duration::from_secs(second);
5532            if let Some(planned) = planned_rate(&sup.handle_timers(now, seen)) {
5533                rate = Some(planned);
5534            }
5535        }
5536
5537        let elapsed = Duration::from_secs(BUSY_FOR + 4 * ORGANIC_WINDOW.as_secs());
5538        let horizon = cfg.max_recovery_time.mul_f64(cfg.fill.finish_fraction) - elapsed;
5539        let expected = (max_cycle_shares(&sup) - seen) as f64 * (1.0 + cfg.fill.loss_margin) / horizon.as_secs_f64();
5540        let actual = rate
5541            .expect("fill must resume once the application stops")
5542            .as_packets_per_sec();
5543        assert!(
5544            (actual - expected).abs() <= 0.15 * expected,
5545            "fill resumed at {actual} packets/s against a remainder that needs {expected}"
5546        );
5547    }
5548
5549    /// The recovered predecessor's tail is filled against *its* clock, and the successor waits.
5550    ///
5551    /// The SURBs the Exit holds when a cycle recovers were minted for that cycle and sit ahead of the
5552    /// successor's in the same FIFO, so packets sent now carry the predecessor's shares and count
5553    /// against the predecessor's deadline — which is why the successor's clocks are deliberately not
5554    /// armed until the tail is spent. Planning against the successor here would aim at a deadline
5555    /// two hours further out and under-send the tail that is actually draining.
5556    #[test]
5557    fn the_paid_recovery_tail_is_filled_against_its_predecessors_deadline() {
5558        let cfg = default_cfg();
5559        let t0 = Instant::now();
5560        let (mut sup, p) = funded_at_front(cfg.clone(), fill_dims(), t0);
5561        let first = ssa_id(p, 1);
5562        let hard = sup.ssas[0]
5563            .recovery_hard_deadline
5564            .expect("a funded front cycle is clocked");
5565
5566        // Recovered with emission still outstanding, which is what leaves a paid tail behind.
5567        let partial = max_cycle_shares(&sup) / 2;
5568        feed_seen(&mut sup, first, partial, t0 + Duration::from_secs(1), 0);
5569        sup.handle_event(&SessionPixEvent::Recovered(first), t0 + Duration::from_secs(1), 0);
5570
5571        let target = sup.fill_target().expect("the paid tail is the thing being filled for");
5572        assert_eq!(first, target.ssa_id, "the tail, not the successor, is the target");
5573        assert_eq!(
5574            hard, target.hard_deadline,
5575            "the tail must be planned against the clock it inherited, not a fresh one"
5576        );
5577
5578        // The successor is funded and reaches the front only once the tail is consumed, which any
5579        // share on the successor proves.
5580        let second = ssa_id(p, 2);
5581        sup.handle_event(&SessionPixEvent::SsaRequestSent(second), t0 + Duration::from_secs(2), 0);
5582        sup.handle_event(
5583            &SessionPixEvent::CommitmentVerified(second),
5584            t0 + Duration::from_secs(2),
5585            0,
5586        );
5587        sup.handle_event(
5588            &SessionPixEvent::DepositConfirmed {
5589                ssa_id: second,
5590                amount: sufficient_balance(),
5591            },
5592            t0 + Duration::from_secs(2),
5593            0,
5594        );
5595        assert_eq!(
5596            Some(first),
5597            sup.fill_target().map(|target| target.ssa_id),
5598            "a funded successor does not take the front while the tail is still draining"
5599        );
5600
5601        feed_seen(&mut sup, second, 1, t0 + Duration::from_secs(3), 0);
5602        let target = sup.fill_target().expect("the successor is now the front");
5603        assert_eq!(second, target.ssa_id);
5604        assert!(
5605            target.hard_deadline > hard,
5606            "the successor's clock starts at the FIFO boundary, so it must be later than its predecessor's"
5607        );
5608    }
5609
5610    /// Within a batch, only the cycle at the front is filled for.
5611    ///
5612    /// A batch is served in index order because the Entry's emission window is clamped to one cycle,
5613    /// so its later members are queued rather than stalled — they have no clocks and can receive no
5614    /// shares. Filling for one of them would spend SURBs the Entry cannot answer.
5615    #[test]
5616    fn only_the_front_cycle_of_a_batch_is_filled_for() {
5617        let cfg = SupervisorConfig {
5618            ssas_per_request: 2,
5619            ..default_cfg()
5620        };
5621        let t0 = Instant::now();
5622        let p = pseudonym();
5623        let (mut sup, _) = SessionPixSupervisor::new(cfg, fill_dims(), p, t0);
5624        fund_batch(&mut sup, p, 2, t0);
5625
5626        assert_eq!(
5627            Some(ssa_id(p, 1)),
5628            sup.fill_target().map(|target| target.ssa_id),
5629            "the second cycle of a batch is queued, not being served"
5630        );
5631    }
5632
5633    /// A cycle that has stopped moving is filled at the heartbeat and no faster.
5634    ///
5635    /// The idle rule cannot reach this state: it re-arms whenever no *gated* service was consumed
5636    /// since the last progress, and fill is ungated, so a Session whose only traffic is fill looks
5637    /// perfectly quiet to it. Without the stall rule an Exit would spend `max_rate × max_recovery_time`
5638    /// — half a million packets at the shipped values — on a cycle whose polynomial has already failed
5639    /// or whose Entry is returning SURBs that carry nothing.
5640    ///
5641    /// The second half of the property matters just as much: a cycle that resumes must be filled
5642    /// again. A stall latch that never released would turn one slow minute into a lost deposit.
5643    #[test]
5644    fn a_stalled_cycle_falls_back_to_the_heartbeat_and_recovers_when_it_moves() {
5645        let cfg = default_cfg();
5646        let t0 = Instant::now();
5647        let (mut sup, p) = funded_at_front(cfg.clone(), fill_dims(), t0);
5648        let id = ssa_id(p, 1);
5649
5650        let running = planned_rate(&sup.handle_timers(t0 + SAMPLING_INTERVAL, 0)).expect("fill must start");
5651        assert!(
5652            running.as_packets_per_sec() > FillRate::once_per(cfg.fill.heartbeat).as_packets_per_sec(),
5653            "the fixture needs a rate above the heartbeat for the fallback to be observable"
5654        );
5655
5656        // Not one share arrives for the whole stall horizon.
5657        let mut rate = None;
5658        for second in 2..=(cfg.max_recovery_idle.as_secs() + 2) {
5659            if let Some(planned) = planned_rate(&sup.handle_timers(t0 + Duration::from_secs(second), 0)) {
5660                rate = Some(planned);
5661            }
5662        }
5663        assert_eq!(
5664            Some(FillRate::once_per(cfg.fill.heartbeat)),
5665            rate,
5666            "a motionless cycle must be held at the heartbeat"
5667        );
5668
5669        // And a single share is enough to put it back to work.
5670        let resumed_at = t0 + Duration::from_secs(cfg.max_recovery_idle.as_secs() + 3);
5671        feed_seen(&mut sup, id, 1, resumed_at, 0);
5672        let rate = planned_rate(&sup.handle_timers(resumed_at + SAMPLING_INTERVAL, 0))
5673            .expect("progress must lift the fallback");
5674        assert!(
5675            rate.as_packets_per_sec() > FillRate::once_per(cfg.fill.heartbeat).as_packets_per_sec(),
5676            "a cycle that resumed must be filled again, got {rate:?}"
5677        );
5678    }
5679
5680    /// A rate that has barely moved is not re-emitted.
5681    ///
5682    /// The planner runs every second for hours and the required rate drifts continuously, so without
5683    /// hysteresis this would put a per-second action on a channel sized for lifecycle transitions —
5684    /// and ask the driver to reconfigure a rate controller for a change no packet schedule can
5685    /// express.
5686    #[test]
5687    fn a_rate_that_barely_moved_is_not_re_emitted() {
5688        let t0 = Instant::now();
5689        let (mut sup, _) = funded_at_front(default_cfg(), fill_dims(), t0);
5690
5691        assert!(
5692            planned_rate(&sup.handle_timers(t0 + SAMPLING_INTERVAL, 0)).is_some(),
5693            "the first plan is always a change"
5694        );
5695        for second in 2..=10 {
5696            assert_eq!(
5697                None,
5698                planned_rate(&sup.handle_timers(t0 + Duration::from_secs(second), 0)),
5699                "second {second} re-emitted a rate that had not meaningfully changed"
5700            );
5701        }
5702    }
5703
5704    /// Closing the Session silences fill in the same action vector that carries the close.
5705    ///
5706    /// The fill stream outlives this state machine by however long the action driver takes to tear
5707    /// the Session down, and it is the one egress on the Exit with no application behind it to stop
5708    /// on its own. A close that did not carry the zero would be followed by keep-alives for a cycle
5709    /// that no longer exists.
5710    #[test]
5711    fn a_close_silences_fill_before_it_is_delivered() {
5712        let t0 = Instant::now();
5713        let (mut sup, p) = funded_at_front(default_cfg(), fill_dims(), t0);
5714        assert!(planned_rate(&sup.handle_timers(t0 + SAMPLING_INTERVAL, 0)).is_some());
5715
5716        let actions = sup.handle_event(
5717            &SessionPixEvent::UnverifiableShares {
5718                ssa_id: ssa_id(p, 1),
5719                observed_total: 1,
5720                peer: crate::supervision::test_peer(),
5721            },
5722            t0 + Duration::from_secs(2),
5723            0,
5724        );
5725        assert!(
5726            matches!(actions.first(), Some(SessionPixAction::SetFillRate(rate)) if rate.is_zero()),
5727            "the zero must lead the close, got {actions:?}"
5728        );
5729        assert!(
5730            actions.iter().any(|action| matches!(
5731                action,
5732                SessionPixAction::Close(SessionPixCloseReason::UnverifiableShares)
5733            )),
5734            "the close itself must still be there, got {actions:?}"
5735        );
5736    }
5737
5738    /// The same, on the path where a retired cycle exhausts the failure budget.
5739    ///
5740    /// A different close path with a different owner — `close_ssa_and_collect` rather than a fault
5741    /// handler — and the one an Entry can actually reach without misbehaving, by simply losing two
5742    /// deposits.
5743    #[test]
5744    fn retiring_past_the_failure_budget_silences_fill() {
5745        const BATCH: u32 = 4;
5746
5747        let cfg = SupervisorConfig {
5748            ssas_per_request: BATCH as usize,
5749            max_failed_cycles: 1,
5750            ..default_cfg()
5751        };
5752        let t0 = Instant::now();
5753        let p = pseudonym();
5754        let (mut sup, _) = SessionPixSupervisor::new(cfg, fill_dims(), p, t0);
5755
5756        // Cycles 1 and 2 funded and recovering, so the front is being filled for; 3 and 4 stuck
5757        // awaiting their deposits, which is the state an unpaid cycle is lost from.
5758        fund_batch(&mut sup, p, 2, t0);
5759        commit_unfunded(&mut sup, p, 3..=BATCH, t0);
5760        assert!(
5761            planned_rate(&sup.handle_timers(t0 + SAMPLING_INTERVAL, 0)).is_some(),
5762            "the funded front must be filling before the losses"
5763        );
5764
5765        // The first loss is survivable; the second is over the budget and takes the Session with it,
5766        // funded siblings and all.
5767        sup.handle_event(&SessionPixEvent::DepositObserverClosed(ssa_id(p, 3)), t0, 0);
5768        assert!(!sup.closed, "one loss is inside the limit");
5769        let actions = sup.handle_event(&SessionPixEvent::DepositObserverClosed(ssa_id(p, BATCH)), t0, 0);
5770
5771        assert!(
5772            actions
5773                .iter()
5774                .any(|action| matches!(action, SessionPixAction::Close(_))),
5775            "the second lost cycle must close the Session, got {actions:?}"
5776        );
5777        assert!(
5778            matches!(actions.first(), Some(SessionPixAction::SetFillRate(rate)) if rate.is_zero()),
5779            "the zero must lead the close, got {actions:?}"
5780        );
5781    }
5782
5783    // ---------------------------------------------------------------
5784    // Draining a closed Session
5785    // ---------------------------------------------------------------
5786
5787    /// A closed Session whose buffer covers the rest of the cycle keeps working for it.
5788    ///
5789    /// This is the whole feature in one test: the deposit both sides have already paid for is
5790    /// recoverable, the SURBs to recover it are already held, and the only thing that had been
5791    /// spending them was a Session that has now gone. Nothing about the close makes the cycle
5792    /// un-completable, so the supervisor neither closes nor asks for a successor it could never
5793    /// serve — it plans the remainder at the ceiling and works it off.
5794    #[test]
5795    fn a_closed_session_with_enough_surbs_drains_its_funded_cycle() {
5796        let cfg = default_cfg();
5797        let t0 = Instant::now();
5798        let (mut sup, _) = funded_at_front(cfg.clone(), fill_dims(), t0);
5799
5800        let actions = sup.handle_event(
5801            &SessionPixEvent::SessionClosed {
5802                drainable_surbs: max_cycle_shares(&sup),
5803            },
5804            t0 + Duration::from_secs(1),
5805            0,
5806        );
5807
5808        assert!(!sup.closed, "a drain leaves the state machine running");
5809        assert!(sup.draining);
5810        assert!(
5811            !actions
5812                .iter()
5813                .any(|action| matches!(action, SessionPixAction::Close(_) | SessionPixAction::RequestSsa { .. })),
5814            "a drain neither closes the Session nor asks for a successor, got {actions:?}"
5815        );
5816
5817        // The next tick asks for the whole remainder, which the ceiling then paces.
5818        let rate =
5819            planned_rate(&sup.handle_timers(t0 + Duration::from_secs(2), 0)).expect("a draining cycle must be filled");
5820        assert_eq!(FillRate::per_second(cfg.fill.max_rate), rate);
5821    }
5822
5823    /// A buffer one SURB short of the remainder is not worth draining.
5824    ///
5825    /// The estimate is an upper bound and the comparison is against the *whole* remaining emission,
5826    /// because a drain that runs out partway has spent the Session's SURBs down to the reserve and
5827    /// recovered nothing — strictly worse than closing at once, which is what it must do instead.
5828    #[test]
5829    fn a_closed_session_without_enough_surbs_closes_instead_of_draining() {
5830        let t0 = Instant::now();
5831        let (mut sup, p) = funded_at_front(default_cfg(), fill_dims(), t0);
5832        let id = ssa_id(p, 1);
5833
5834        // Filling before the close, so the refusal has a stream to silence.
5835        assert!(planned_rate(&sup.handle_timers(t0 + SAMPLING_INTERVAL, 0)).is_some());
5836
5837        let seen = max_cycle_shares(&sup) / 2;
5838        feed_seen(&mut sup, id, seen, t0 + Duration::from_secs(2), 0);
5839        let one_short = max_cycle_shares(&sup) - seen - 1;
5840
5841        let actions = sup.handle_event(
5842            &SessionPixEvent::SessionClosed {
5843                drainable_surbs: one_short,
5844            },
5845            t0 + Duration::from_secs(3),
5846            0,
5847        );
5848
5849        assert!(sup.closed);
5850        assert!(!sup.draining);
5851        assert!(
5852            matches!(actions.first(), Some(SessionPixAction::SetFillRate(rate)) if rate.is_zero()),
5853            "the zero must lead the close, got {actions:?}"
5854        );
5855        assert!(
5856            actions
5857                .iter()
5858                .any(|action| matches!(action, SessionPixAction::Close(SessionPixCloseReason::SessionClosed))),
5859            "got {actions:?}"
5860        );
5861    }
5862
5863    /// With no funded cycle to work for, a close is just a close.
5864    ///
5865    /// Two ways to have none. An unfunded front has nothing at stake — no deposit has been paid, so
5866    /// there is nothing to strand. A paid recovery tail means the front's key is *already* recovered,
5867    /// so draining it would buy nothing either; and its successor is queued behind the tail with no
5868    /// clocks of its own, which is precisely the state fill refuses to plan against.
5869    #[test]
5870    fn a_closed_session_with_no_funded_cycle_closes_instead_of_draining() {
5871        let plenty = u64::MAX;
5872
5873        let t0 = Instant::now();
5874        let p = pseudonym();
5875        let (mut sup, _) = SessionPixSupervisor::new(default_cfg(), fill_dims(), p, t0);
5876        commit_unfunded(&mut sup, p, 1..=1, t0);
5877
5878        let actions = sup.handle_event(
5879            &SessionPixEvent::SessionClosed {
5880                drainable_surbs: plenty,
5881            },
5882            t0 + Duration::from_secs(1),
5883            0,
5884        );
5885        assert!(
5886            sup.closed && !sup.draining,
5887            "an unfunded front has nothing to drain for"
5888        );
5889        assert!(
5890            actions
5891                .iter()
5892                .any(|action| matches!(action, SessionPixAction::Close(SessionPixCloseReason::SessionClosed))),
5893            "got {actions:?}"
5894        );
5895
5896        // And the same with a recovered predecessor's paid tail in front.
5897        let t0 = Instant::now();
5898        let (mut sup, p) = funded_at_front(default_cfg(), fill_dims(), t0);
5899        let first = ssa_id(p, 1);
5900        let half = max_cycle_shares(&sup) / 2;
5901        feed_seen(&mut sup, first, half, t0 + Duration::from_secs(1), 0);
5902        sup.handle_event(&SessionPixEvent::Recovered(first), t0 + Duration::from_secs(1), 0);
5903        assert!(
5904            sup.paid_recovery_tail.is_some(),
5905            "the fixture must leave a paid tail behind for this half to mean anything"
5906        );
5907
5908        let actions = sup.handle_event(
5909            &SessionPixEvent::SessionClosed {
5910                drainable_surbs: plenty,
5911            },
5912            t0 + Duration::from_secs(2),
5913            0,
5914        );
5915        assert!(sup.closed && !sup.draining, "a recovered key is not worth draining for");
5916        assert!(
5917            actions
5918                .iter()
5919                .any(|action| matches!(action, SessionPixAction::Close(SessionPixCloseReason::SessionClosed))),
5920            "got {actions:?}"
5921        );
5922    }
5923
5924    /// Everything but the cycle being drained is retired at once, and none of it counts as a failure.
5925    ///
5926    /// A drain works for exactly one cycle: the batch's later members are queued behind it and can
5927    /// receive no share while it drains, and there is no successor coming, so holding their
5928    /// reconstructor state open would cost memory for the length of the drain and buy nothing. The
5929    /// failure counter must not move either — these cycles were retired by an operator closing a
5930    /// Session, not lost by an Entry, and charging them would make a second drain on the same node
5931    /// look like an Entry over its budget.
5932    #[test]
5933    fn a_drain_retires_every_cycle_but_the_one_it_is_recovering() {
5934        const BATCH: u32 = 4;
5935
5936        let cfg = SupervisorConfig {
5937            ssas_per_request: BATCH as usize,
5938            ..default_cfg()
5939        };
5940        let t0 = Instant::now();
5941        let p = pseudonym();
5942        let (mut sup, _) = SessionPixSupervisor::new(cfg, fill_dims(), p, t0);
5943        fund_batch(&mut sup, p, 2, t0);
5944        commit_unfunded(&mut sup, p, 3..=BATCH, t0);
5945
5946        let actions = sup.handle_event(
5947            &SessionPixEvent::SessionClosed {
5948                drainable_surbs: max_cycle_shares(&sup),
5949            },
5950            t0 + Duration::from_secs(1),
5951            0,
5952        );
5953
5954        let retired = actions
5955            .iter()
5956            .filter_map(|action| match action {
5957                SessionPixAction::RetireSsa(id) => Some(id.ssa_index().get()),
5958                _ => None,
5959            })
5960            .collect::<Vec<_>>();
5961        assert_eq!(
5962            vec![2, 3, 4],
5963            retired,
5964            "every cycle but the one being drained must be retired, got {actions:?}"
5965        );
5966        assert!(!sup.closed && sup.draining);
5967        assert_eq!(0, sup.failed_cycles, "a drain retires siblings, it does not lose them");
5968        assert_eq!(1, sup.ssas.len());
5969        assert_eq!(ssa_id(p, 1), sup.ssas[0].ssa_id);
5970    }
5971
5972    /// A drain that finishes closes on its own reason and never asks for a successor.
5973    ///
5974    /// Both halves matter. `Drained` is what tells an operator the close was the good outcome — the
5975    /// deposit was recovered — rather than one more Session lost on a timer. And the successor
5976    /// request has to be suppressed on *every* route into it, the early-recovery signal included:
5977    /// there is no Session left to serve a successor's quota, so asking for one would commit the
5978    /// Entry to a deposit against a cycle that can never be served.
5979    #[test]
5980    fn a_completed_drain_closes_the_session_without_asking_for_a_successor() {
5981        let t0 = Instant::now();
5982        let (mut sup, p) = funded_at_front(default_cfg(), fill_dims(), t0);
5983        let id = ssa_id(p, 1);
5984        assert!(planned_rate(&sup.handle_timers(t0 + SAMPLING_INTERVAL, 0)).is_some());
5985
5986        sup.handle_event(
5987            &SessionPixEvent::SessionClosed {
5988                drainable_surbs: max_cycle_shares(&sup),
5989            },
5990            t0 + Duration::from_secs(2),
5991            0,
5992        );
5993        assert!(sup.draining);
5994
5995        let early = sup.handle_event(&SessionPixEvent::AlmostRecovered(id), t0 + Duration::from_secs(3), 0);
5996        assert!(
5997            early.is_empty(),
5998            "the early-recovery signal must not order a successor for a Session that is gone, got {early:?}"
5999        );
6000
6001        let actions = sup.handle_event(&SessionPixEvent::Recovered(id), t0 + Duration::from_secs(4), 0);
6002        assert!(sup.closed);
6003        assert!(
6004            matches!(actions.first(), Some(SessionPixAction::SetFillRate(rate)) if rate.is_zero()),
6005            "the zero must lead the close, got {actions:?}"
6006        );
6007        assert!(
6008            actions
6009                .iter()
6010                .any(|action| matches!(action, SessionPixAction::Close(SessionPixCloseReason::Drained))),
6011            "got {actions:?}"
6012        );
6013        assert!(
6014            !actions
6015                .iter()
6016                .any(|action| matches!(action, SessionPixAction::RequestSsa { .. })),
6017            "got {actions:?}"
6018        );
6019    }
6020
6021    /// A completed drain books its deposit as recovered, not as stranded.
6022    ///
6023    /// The drain exists to un-strand a deposit, so it is the one close path that must never leave
6024    /// its value out of `hopr_pix_deposits_recovered_uhopr_total`. It returns early from the
6025    /// recovered transition, which is what previously skipped the summary the counter reads.
6026    #[test]
6027    fn a_completed_drain_books_the_deposit_it_unlocked_as_recovered() {
6028        let t0 = Instant::now();
6029        let (mut sup, p) = funded_at_front(default_cfg(), fill_dims(), t0);
6030        let id = ssa_id(p, 1);
6031        // `funded_at_front` confirms `sufficient_balance()`, which is 1000 HOPR.
6032        assert_eq!(1_000_000_000, sup.take_telemetry_events().deposit_confirmed_uhopr);
6033
6034        sup.handle_event(
6035            &SessionPixEvent::SessionClosed {
6036                drainable_surbs: max_cycle_shares(&sup),
6037            },
6038            t0 + Duration::from_secs(2),
6039            0,
6040        );
6041        sup.handle_event(&SessionPixEvent::Recovered(id), t0 + Duration::from_secs(4), 0);
6042
6043        let events = sup.take_telemetry_events();
6044        assert_eq!(
6045            1, events.recovered,
6046            "a drained cycle recovered, so it counts as recovered"
6047        );
6048        let recovered: Vec<_> = events
6049            .finalized
6050            .iter()
6051            .filter(|summary| summary.outcome == PixCycleOutcome::Recovered)
6052            .collect();
6053        assert_eq!(
6054            1,
6055            recovered.len(),
6056            "the drain must summarize its cycle, or the value it unlocked reads as stranded"
6057        );
6058        assert_eq!(1_000_000_000, recovered[0].deposit_uhopr);
6059    }
6060
6061    /// A drain that stops making progress is closed by the idle deadline rather than re-armed.
6062    ///
6063    /// The idle rule normally re-arms whenever no *gated* service was consumed since the last
6064    /// progress, so that a merely quiet Entry is never disconnected. A drained Session has no
6065    /// application left to consume any, so `served_total` is frozen for good and the re-arm would
6066    /// never stop firing — the one bound on a drain admitted against an over-estimated buffer would
6067    /// be gone with it.
6068    #[test]
6069    fn a_stalled_drain_closes_on_the_recovery_idle_deadline() {
6070        let cfg = default_cfg();
6071        let t0 = Instant::now();
6072        let (mut sup, _) = funded_at_front(cfg.clone(), fill_dims(), t0);
6073
6074        sup.handle_event(
6075            &SessionPixEvent::SessionClosed {
6076                drainable_surbs: max_cycle_shares(&sup),
6077            },
6078            t0,
6079            0,
6080        );
6081        assert!(sup.draining);
6082
6083        // Not one share, and not one packet of gated service either — which is exactly the shape the
6084        // re-arm branch was written for, and the shape it must no longer answer to.
6085        let actions = sup.handle_deadline(t0 + cfg.max_recovery_idle, 0);
6086        assert!(sup.closed);
6087        assert!(
6088            actions
6089                .iter()
6090                .any(|action| matches!(action, SessionPixAction::Close(SessionPixCloseReason::RecoveryIdle))),
6091            "got {actions:?}"
6092        );
6093    }
6094
6095    /// And the hard deadline still ends a drain that is making progress but will not finish.
6096    ///
6097    /// The immutable per-cycle backstop is not relaxed for a drain: the reconstructor state it holds
6098    /// open is the resource the deadline exists to bound, and a closed Session is no reason to hold
6099    /// it longer. `check_deadlines` tests the hard clock before the idle one, which is why this
6100    /// reports the deadline rather than the silence that came with it.
6101    #[test]
6102    fn a_drain_still_ends_at_its_hard_recovery_deadline() {
6103        let cfg = default_cfg();
6104        let t0 = Instant::now();
6105        let (mut sup, _) = funded_at_front(cfg.clone(), fill_dims(), t0);
6106
6107        sup.handle_event(
6108            &SessionPixEvent::SessionClosed {
6109                drainable_surbs: max_cycle_shares(&sup),
6110            },
6111            t0,
6112            0,
6113        );
6114        assert!(sup.draining);
6115
6116        let actions = sup.handle_deadline(t0 + cfg.max_recovery_time, 0);
6117        assert!(sup.closed);
6118        assert!(
6119            actions
6120                .iter()
6121                .any(|action| matches!(action, SessionPixAction::Close(SessionPixCloseReason::RecoveryDeadline))),
6122            "got {actions:?}"
6123        );
6124    }
6125
6126    /// With fill disabled the supervisor behaves exactly as it did before fill existed.
6127    ///
6128    /// Not merely "sends no keep-alives": it must also never schedule a tick, or every PIX Session on
6129    /// a node that turned the feature off would still wake its worker once a second for the whole of
6130    /// its life.
6131    #[test]
6132    fn a_disabled_filler_plans_nothing_and_schedules_nothing() {
6133        let cfg = SupervisorConfig {
6134            fill: PixFillConfig {
6135                enabled: false,
6136                ..PixFillConfig::default()
6137            },
6138            ..default_cfg()
6139        };
6140        let t0 = Instant::now();
6141        let (mut sup, _) = funded_at_front(cfg, fill_dims(), t0);
6142
6143        assert_eq!(None, sup.next_fill_tick(), "a disabled filler has no schedule");
6144        for second in 1..=5 {
6145            let actions = sup.handle_timers(t0 + Duration::from_secs(second), 0);
6146            assert_eq!(None, planned_rate(&actions), "a disabled filler must plan nothing");
6147        }
6148    }
6149
6150    // -----------------------------------------------------------------------
6151    // Bounded aggregate telemetry — the census and the event latch
6152    // -----------------------------------------------------------------------
6153
6154    /// The census as the worker reads it, with no predeposit exposure to complicate the comparison.
6155    fn census(sup: &SessionPixSupervisor) -> PixSessionSnapshot {
6156        sup.telemetry_snapshot(0)
6157    }
6158
6159    /// A census naming the gate mode and the three live phase counts, for comparing against
6160    /// [`census`].
6161    ///
6162    /// The mode is a parameter rather than defaulted because it is half of what these tests are
6163    /// checking: an `hopr_pix_sessions_active` that stayed in the predeposit bucket after a deposit
6164    /// confirmed would tell an operator the opposite of the truth.
6165    fn phases(
6166        gate_mode: PixGateMode,
6167        awaiting_commitment: u32,
6168        awaiting_deposit: u32,
6169        recovering: u32,
6170    ) -> PixSessionSnapshot {
6171        PixSessionSnapshot {
6172            gate_mode,
6173            awaiting_commitment,
6174            awaiting_deposit,
6175            recovering,
6176            ..Default::default()
6177        }
6178    }
6179
6180    /// One Session walked end to end, asserting the census and the latch at every step.
6181    ///
6182    /// This is the test the whole census design exists for: a gauge that fails to come back down is
6183    /// invisible to an operator until they compare it against reality, and there is no reality to
6184    /// compare a metric against in production.
6185    #[test]
6186    fn a_session_walks_the_census_from_request_to_rollover() {
6187        let t0 = Instant::now();
6188        let p = pseudonym();
6189        let (mut sup, _) = SessionPixSupervisor::new(default_cfg(), dims(64, 8), p, t0);
6190        let first = ssa_id(p, 1);
6191
6192        // `new` allocates and requests the first cycle, so it is already awaiting its commitment.
6193        assert_eq!(phases(PixGateMode::Predeposit, 1, 0, 0), census(&sup));
6194        assert_eq!(
6195            PixTurnEvents {
6196                requested: 1,
6197                ..Default::default()
6198            },
6199            sup.take_telemetry_events(),
6200            "allocation is what counts as requested, not the confirmation that it was sent"
6201        );
6202        assert!(
6203            sup.take_telemetry_events().is_empty(),
6204            "the latch must drain rather than accumulate"
6205        );
6206
6207        sup.handle_event(&SessionPixEvent::SsaRequestSent(first), t0, 0);
6208        assert_eq!(phases(PixGateMode::Predeposit, 1, 0, 0), census(&sup));
6209        assert!(
6210            sup.take_telemetry_events().is_empty(),
6211            "confirming the send is not a lifecycle transition"
6212        );
6213
6214        sup.handle_event(&SessionPixEvent::CommitmentVerified(first), t0, 0);
6215        assert_eq!(phases(PixGateMode::Predeposit, 0, 1, 0), census(&sup));
6216        assert_eq!(
6217            PixTurnEvents {
6218                committed: 1,
6219                // Zero because this fixture drives every transition at `t0`; that the sample is
6220                // latched at all is the assertion. `a_cycle_is_timed_in_each_phase_it_leaves`
6221                // covers the durations themselves.
6222                phase_durations: vec![(PixCyclePhase::AwaitingCommitment, 0)],
6223                ..Default::default()
6224            },
6225            sup.take_telemetry_events()
6226        );
6227        assert_eq!(
6228            PixGateMode::Predeposit,
6229            census(&sup).gate_mode,
6230            "a committed but unfunded front is still spending its predeposit allowance"
6231        );
6232
6233        sup.handle_event(
6234            &SessionPixEvent::DepositConfirmed {
6235                ssa_id: first,
6236                amount: sufficient_balance(),
6237            },
6238            t0,
6239            0,
6240        );
6241        assert_eq!(phases(PixGateMode::Funded, 0, 0, 1), census(&sup));
6242        assert_eq!(
6243            PixTurnEvents {
6244                funded: 1,
6245                // `sufficient_balance()` is 1000 HOPR; confirming it books the value alongside the
6246                // count, in the same turn.
6247                deposit_confirmed_uhopr: 1_000_000_000,
6248                phase_durations: vec![(PixCyclePhase::AwaitingDeposit, 0)],
6249                ..Default::default()
6250            },
6251            sup.take_telemetry_events()
6252        );
6253        assert_eq!(PixGateMode::Funded, census(&sup).gate_mode);
6254
6255        // Recovery retires the cycle into a tombstone and asks for its successor, so the census
6256        // swaps one recovering cycle for one awaiting a commitment — never showing both.
6257        sup.handle_event(&SessionPixEvent::Recovered(first), t0, 0);
6258        let after = census(&sup);
6259        assert_eq!(
6260            1,
6261            after.awaiting_commitment + after.awaiting_deposit + after.recovering,
6262            "a tombstone is retained state, not live state: {after:?}"
6263        );
6264        assert_eq!(0, after.recovering, "the recovered cycle must leave the live census");
6265
6266        let events = sup.take_telemetry_events();
6267        assert_eq!(
6268            (1, 1),
6269            (events.recovered, events.requested),
6270            "the successor is requested in the same turn the predecessor recovers"
6271        );
6272        assert_eq!(
6273            vec![PixCycleSummary {
6274                outcome: PixCycleOutcome::Recovered,
6275                // This cycle was never served: the fixture drives the lifecycle directly, with a
6276                // served count of zero throughout.
6277                egress_packets: 0,
6278                accepted_shares: sup.dims.target_useful_shares(),
6279                useful_shares: sup.dims.target_useful_shares(),
6280                target_useful_shares: sup.dims.target_useful_shares(),
6281                // `sufficient_balance()` is 1000 HOPR, so the deposit the fixture confirmed reaches
6282                // the summary that decides how much value this recovery unlocked.
6283                deposit_uhopr: 1_000_000_000,
6284            }],
6285            events.finalized,
6286            "a recovered cycle is summarized exactly once, at full useful coverage"
6287        );
6288        assert_eq!(
6289            Some(1.0),
6290            events.finalized[0].useful_fraction(),
6291            "a recovered cycle's useful fraction is one by construction"
6292        );
6293    }
6294
6295    /// A cycle that recovers with its negotiated tail outstanding is reported as `paid_tail`.
6296    ///
6297    /// The distinction matters to an operator: the Session is still being served, and still funded,
6298    /// but by a receipt from a cycle that has already reconstructed rather than by a live one.
6299    #[test]
6300    fn a_paid_recovered_predecessor_is_censused_as_a_paid_tail() {
6301        let t0 = Instant::now();
6302        let p = pseudonym();
6303        let (mut sup, _) = SessionPixSupervisor::new(default_cfg(), dims(64, 8), p, t0);
6304        let first = ssa_id(p, 1);
6305
6306        fund_batch(&mut sup, p, 1, t0);
6307        // Some surplus is outstanding, so recovery leaves a receipt behind rather than nothing.
6308        sup.handle_event(
6309            &SessionPixEvent::RecoveryProgress(make_progress_seen(
6310                first,
6311                sup.dims.target_useful_shares(),
6312                sup.dims.target_useful_shares(),
6313                sup.dims.target_useful_shares(),
6314                sup.dims.polys_per_ssa(),
6315            )),
6316            t0,
6317            0,
6318        );
6319        sup.handle_event(&SessionPixEvent::Recovered(first), t0, 0);
6320
6321        assert_eq!(1, census(&sup).paid_tail, "the receipt must be visible");
6322        assert_eq!(
6323            PixGateMode::Funded,
6324            census(&sup).gate_mode,
6325            "a paid tail still authorizes service"
6326        );
6327
6328        // Draining the receipt to its emitted ceiling consumes it, and the census follows.
6329        sup.handle_event(
6330            &SessionPixEvent::RecoveryProgress(make_recovered_tail_progress(&sup, first, max_cycle_shares(&sup))),
6331            t0,
6332            0,
6333        );
6334        assert_eq!(0, census(&sup).paid_tail, "an exhausted receipt must leave the census");
6335    }
6336
6337    /// Every terminal path drains the live census, and each lost cycle is counted exactly once.
6338    ///
6339    /// Parameterised over the failure modes rather than written out per reason, because the property
6340    /// is the same for all of them and the risk is that one path forgets.
6341    #[test]
6342    fn every_terminal_path_drains_the_census_and_counts_its_loss_once() {
6343        let t0 = Instant::now();
6344
6345        // Each case drives a single-cycle Session into a terminal state a different way, and says
6346        // which of the two routes its cycle takes out. Stating it is what keeps a case honest: the
6347        // sum invariant below holds even for a case that has silently stopped reaching its
6348        // terminal path at all, because a cycle that stays live is absorbed by `still_live`.
6349        //
6350        // `Retired` — the supervisor retired the cycle itself, charging it as it went.
6351        // `LiveAtClose` — the Session closed outright with the cycle still live, so the charge is
6352        // `PixSessionTelemetry::release`'s to make from the census.
6353        #[derive(Clone, Copy, PartialEq, Eq, Debug)]
6354        enum Route {
6355            Retired,
6356            LiveAtClose,
6357        }
6358        type Kill = fn(&mut SessionPixSupervisor, HoprPseudonym, Instant);
6359        let cases: [(&str, Route, Kill); 5] = [
6360            ("commitment timeout", Route::Retired, |sup, p, t0| {
6361                // `SsaRequestSent` is what arms `commitment_deadline` — a cycle registered by
6362                // `emit_request_next_ssa` has none, because its request has not gone out and there
6363                // is nothing for it to be late for. Without this the sweep below expires nothing
6364                // and the case silently degenerates into "an unarmed cycle stays live".
6365                sup.handle_event(&SessionPixEvent::SsaRequestSent(ssa_id(p, 1)), t0, 0);
6366                sup.handle_timers(t0 + Duration::from_secs(3600), 0);
6367            }),
6368            ("deposit observer closed", Route::Retired, |sup, p, t0| {
6369                let id = ssa_id(p, 1);
6370                sup.handle_event(&SessionPixEvent::SsaRequestSent(id), t0, 0);
6371                sup.handle_event(&SessionPixEvent::CommitmentVerified(id), t0, 0);
6372                sup.handle_event(&SessionPixEvent::DepositObserverClosed(id), t0, 0);
6373            }),
6374            ("deposit timeout", Route::Retired, |sup, p, t0| {
6375                let id = ssa_id(p, 1);
6376                sup.handle_event(&SessionPixEvent::SsaRequestSent(id), t0, 0);
6377                sup.handle_event(&SessionPixEvent::CommitmentVerified(id), t0, 0);
6378                sup.handle_timers(t0 + Duration::from_secs(7200), 0);
6379            }),
6380            ("recovery hard deadline", Route::Retired, |sup, p, t0| {
6381                fund_batch(sup, p, 1, t0);
6382                sup.handle_timers(t0 + Duration::from_secs(100_000), 0);
6383            }),
6384            ("unverifiable shares", Route::LiveAtClose, |sup, p, t0| {
6385                fund_batch(sup, p, 1, t0);
6386                sup.handle_event(
6387                    &SessionPixEvent::UnverifiableShares {
6388                        ssa_id: ssa_id(p, 1),
6389                        observed_total: 1,
6390                        peer: crate::supervision::test_peer(),
6391                    },
6392                    t0,
6393                    0,
6394                );
6395            }),
6396        ];
6397
6398        for (name, route, kill) in cases {
6399            let p = pseudonym();
6400            let (mut sup, _) = SessionPixSupervisor::new(default_cfg(), dims(64, 8), p, t0);
6401            let requested = sup.take_telemetry_events().requested;
6402            assert_eq!(1, requested, "{name}: one cycle was allocated");
6403
6404            kill(&mut sup, p, t0);
6405
6406            let events = sup.take_telemetry_events();
6407            assert_eq!(0, events.recovered, "{name}: nothing recovered");
6408
6409            let charged_here = events.failed;
6410            let still_live = census(&sup).live_cycles();
6411
6412            // Asserted before the sum, because the sum alone cannot tell a case that reached its
6413            // terminal path from one that never got there.
6414            let (expected_charged, expected_live) = match route {
6415                Route::Retired => (1, 0),
6416                Route::LiveAtClose => (0, 1),
6417            };
6418            assert_eq!(
6419                (expected_charged, expected_live),
6420                (charged_here, still_live),
6421                "{name}: expected the {route:?} route"
6422            );
6423
6424            // And the invariant the aggregate actually reports: whichever route the cycle took, it
6425            // is accounted for exactly once. `UnverifiableShares` closes the Session without
6426            // retiring its cycles — the fault is evidence about the peer rather than about one
6427            // cycle — so its charge is `release`'s to make from the census.
6428            assert_eq!(
6429                requested,
6430                charged_here + still_live,
6431                "{name}: every requested cycle must be accounted for exactly once, got {charged_here} retired and \
6432                 {still_live} live"
6433            );
6434        }
6435    }
6436
6437    /// Tombstones and closing records never appear in the live census.
6438    ///
6439    /// Stated on its own because it is the failure the census design was chosen to avoid: a
6440    /// tombstone outlives its cycle by `tombstone_retention_window`, so counting one would leave
6441    /// `hopr_pix_cycles_active` reporting a cycle the Exit has already finished with.
6442    #[test]
6443    fn retained_but_finished_cycles_are_not_live() {
6444        let t0 = Instant::now();
6445        let p = pseudonym();
6446        let cfg = SupervisorConfig {
6447            ssas_per_request: 2,
6448            ..default_cfg()
6449        };
6450        let (mut sup, _) = SessionPixSupervisor::new(cfg, dims(64, 8), p, t0);
6451
6452        fund_batch(&mut sup, p, 2, t0);
6453        assert_eq!(2, census(&sup).recovering);
6454
6455        sup.handle_event(&SessionPixEvent::Recovered(ssa_id(p, 1)), t0, 0);
6456        assert!(
6457            sup.ssas.iter().any(|s| matches!(s.phase, SsaPhase::Recovered { .. })),
6458            "the tombstone must still be retained, or this test proves nothing"
6459        );
6460        assert_eq!(
6461            1,
6462            census(&sup).recovering,
6463            "only the surviving sibling is live; the tombstone is retained state"
6464        );
6465    }
6466
6467    /// A repeated lifecycle event counts its transition once.
6468    ///
6469    /// Concurrent acknowledgement batches make duplicates ordinary rather than exceptional, and the
6470    /// counters must not treat replay as progress.
6471    #[test]
6472    fn replayed_lifecycle_events_do_not_double_count() {
6473        let t0 = Instant::now();
6474        let p = pseudonym();
6475        let (mut sup, _) = SessionPixSupervisor::new(default_cfg(), dims(64, 8), p, t0);
6476        let first = ssa_id(p, 1);
6477        sup.take_telemetry_events();
6478
6479        for _ in 0..3 {
6480            sup.handle_event(&SessionPixEvent::SsaRequestSent(first), t0, 0);
6481            sup.handle_event(&SessionPixEvent::CommitmentVerified(first), t0, 0);
6482        }
6483        assert_eq!(
6484            1,
6485            sup.take_telemetry_events().committed,
6486            "a repeated CommitmentVerified must count one transition"
6487        );
6488
6489        for _ in 0..3 {
6490            sup.handle_event(
6491                &SessionPixEvent::DepositConfirmed {
6492                    ssa_id: first,
6493                    amount: sufficient_balance(),
6494                },
6495                t0,
6496                0,
6497            );
6498        }
6499        assert_eq!(
6500            1,
6501            sup.take_telemetry_events().funded,
6502            "a duplicate deposit confirmation must count one transition"
6503        );
6504
6505        for _ in 0..3 {
6506            sup.handle_event(&SessionPixEvent::Recovered(first), t0, 0);
6507        }
6508        assert_eq!(
6509            1,
6510            sup.take_telemetry_events().recovered,
6511            "a duplicate Recovered must count one transition"
6512        );
6513    }
6514
6515    /// A zero deposit is not a verdict, so it must not be counted as funding.
6516    #[test]
6517    fn a_zero_deposit_counts_no_funding() {
6518        let t0 = Instant::now();
6519        let p = pseudonym();
6520        let (mut sup, _) = SessionPixSupervisor::new(default_cfg(), dims(64, 8), p, t0);
6521        let first = ssa_id(p, 1);
6522
6523        sup.handle_event(&SessionPixEvent::SsaRequestSent(first), t0, 0);
6524        sup.handle_event(&SessionPixEvent::CommitmentVerified(first), t0, 0);
6525        sup.take_telemetry_events();
6526
6527        sup.handle_event(
6528            &SessionPixEvent::DepositConfirmed {
6529                ssa_id: first,
6530                amount: HoprBalance::zero(),
6531            },
6532            t0,
6533            0,
6534        );
6535
6536        assert_eq!(0, sup.take_telemetry_events().funded);
6537        assert_eq!(
6538            phases(PixGateMode::Predeposit, 0, 1, 0),
6539            census(&sup),
6540            "the cycle stays fundable, awaiting a real deposit"
6541        );
6542    }
6543
6544    // -----------------------------------------------------------------------
6545    // Share coverage — absolute snapshots into positive deltas
6546    // -----------------------------------------------------------------------
6547
6548    /// A funded, recovering single-cycle Session, ready to be fed progress snapshots.
6549    fn recovering_session(p: HoprPseudonym, t0: Instant) -> SessionPixSupervisor {
6550        let (mut sup, _) = SessionPixSupervisor::new(default_cfg(), dims(64, 8), p, t0);
6551        fund_batch(&mut sup, p, 1, t0);
6552        sup.take_telemetry_events();
6553        sup
6554    }
6555
6556    /// Feeds one snapshot and returns the `(useful, surplus)` it booked.
6557    fn feed(sup: &mut SessionPixSupervisor, id: SsaId<HoprPseudonym>, useful: u64, seen: u64) -> (u64, u64) {
6558        let target = sup.dims.target_useful_shares();
6559        let polys = sup.dims.polys_per_ssa();
6560        // `recovered_polynomials` is scaled with `useful` so the snapshot stays internally
6561        // consistent; the validator rejects one that is not, and this test is about deltas.
6562        let recovered = ((useful * polys as u64) / target.max(1)) as u16;
6563        sup.handle_event(
6564            &SessionPixEvent::RecoveryProgress(make_progress_seen(id, useful, seen, target, recovered)),
6565            Instant::now(),
6566            0,
6567        );
6568        let events = sup.take_telemetry_events();
6569        (events.useful_shares, events.surplus_shares)
6570    }
6571
6572    /// Absolute snapshots become positive deltas, and every degenerate ordering adds zero.
6573    ///
6574    /// The property the issue names: `shares_seen` is absolute and acknowledgement batches arrive
6575    /// out of order or coalesced, so adding it to a counter would over-count wildly. What is added
6576    /// is the difference against a high-water mark.
6577    #[test]
6578    fn absolute_snapshots_become_positive_deltas() {
6579        let t0 = Instant::now();
6580        let p = pseudonym();
6581        let mut sup = recovering_session(p, t0);
6582        let id = ssa_id(p, 1);
6583
6584        assert_eq!((10, 0), feed(&mut sup, id, 10, 10), "the first snapshot is all delta");
6585        assert_eq!((5, 0), feed(&mut sup, id, 15, 15), "and the next is the difference");
6586
6587        // Repeated: the same snapshot again.
6588        assert_eq!((0, 0), feed(&mut sup, id, 15, 15), "a repeated snapshot adds nothing");
6589        // Stale: reordered behind one already seen.
6590        assert_eq!((0, 0), feed(&mut sup, id, 12, 12), "a stale snapshot adds nothing");
6591        // Still stale on the liveness counter even though useful is unchanged.
6592        assert_eq!(
6593            (0, 0),
6594            feed(&mut sup, id, 15, 14),
6595            "an older liveness count adds nothing"
6596        );
6597
6598        // Coalesced: a jump standing in for snapshots that were dropped. The whole missing delta
6599        // arrives with it, because a high-water mark is a maximum and not a running sum.
6600        assert_eq!(
6601            (85, 0),
6602            feed(&mut sup, id, 100, 100),
6603            "a jump carries the missing delta"
6604        );
6605    }
6606
6607    /// Shares that arrive but advance nothing are surplus, and are counted as such.
6608    ///
6609    /// This is the tail of every conforming cycle — `surplus` shares per polynomial that arrive
6610    /// after its threshold is met — so an Exit whose `useful` and `surplus` rates are both healthy
6611    /// is being served correctly, and one with surplus but no useful progress is not.
6612    #[test]
6613    fn shares_that_advance_nothing_are_counted_as_surplus() {
6614        let t0 = Instant::now();
6615        let p = pseudonym();
6616        let mut sup = recovering_session(p, t0);
6617        let id = ssa_id(p, 1);
6618
6619        assert_eq!((20, 0), feed(&mut sup, id, 20, 20));
6620        // Liveness advances, payment does not: the conforming surplus run.
6621        assert_eq!(
6622            (0, 5),
6623            feed(&mut sup, id, 20, 25),
6624            "a surplus-only snapshot books surplus"
6625        );
6626        // A mixed snapshot splits.
6627        assert_eq!((10, 5), feed(&mut sup, id, 30, 40), "a mixed snapshot splits both ways");
6628    }
6629
6630    /// Useful and surplus always partition the accepted delta, over an arbitrary sequence.
6631    ///
6632    /// Stated as an invariant over a generated sequence rather than as another worked example,
6633    /// because the failure it guards against is an accounting slip in one of the branches rather
6634    /// than a wrong answer to one case.
6635    #[test]
6636    fn useful_and_surplus_always_partition_the_accepted_delta() {
6637        let t0 = Instant::now();
6638        let p = pseudonym();
6639        let mut sup = recovering_session(p, t0);
6640        let id = ssa_id(p, 1);
6641        let target = sup.dims.target_useful_shares();
6642        // The greatest liveness count these dimensions can produce: anything above it is a protocol
6643        // violation the validator closes on, so a walk must stay inside it to be booking deltas at
6644        // all rather than testing the rejection path.
6645        let max_seen = max_cycle_shares(&sup);
6646
6647        let mut booked_useful = 0;
6648        let mut booked_surplus = 0;
6649        let mut highest_seen = 0;
6650
6651        // A deterministic walk that revisits, regresses and jumps, which is what real delivery does.
6652        for step in [
6653            (5u64, 5u64),
6654            (5, 9),
6655            (3, 4),
6656            (9, 9),
6657            (9, 20),
6658            (9, 20),
6659            (30, 41),
6660            (12, 12),
6661            (target, max_seen),
6662            (target, max_seen),
6663        ] {
6664            let (useful, seen) = step;
6665            let (u, s) = feed(&mut sup, id, useful, seen);
6666            assert_eq!(
6667                seen.saturating_sub(highest_seen),
6668                u + s,
6669                "snapshot {step:?} must book exactly its accepted delta"
6670            );
6671            highest_seen = highest_seen.max(seen);
6672            booked_useful += u;
6673            booked_surplus += s;
6674        }
6675
6676        assert_eq!(
6677            highest_seen,
6678            booked_useful + booked_surplus,
6679            "over the whole sequence the two must sum to the greatest liveness count seen"
6680        );
6681        assert_eq!(
6682            target, booked_useful,
6683            "and the useful half must sum to the greatest useful count seen"
6684        );
6685    }
6686
6687    /// A recovered cycle's drain is all surplus.
6688    ///
6689    /// Its useful counter is pinned at the target by recovery, so every further share the FIFO
6690    /// delivers is accepted traffic that can no longer advance anything.
6691    #[test]
6692    fn a_recovered_tail_books_only_surplus() {
6693        let t0 = Instant::now();
6694        let p = pseudonym();
6695        let mut sup = recovering_session(p, t0);
6696        let id = ssa_id(p, 1);
6697        let target = sup.dims.target_useful_shares();
6698
6699        feed(&mut sup, id, target, target);
6700        sup.handle_event(&SessionPixEvent::Recovered(id), t0, 0);
6701        sup.take_telemetry_events();
6702        assert!(sup.paid_recovery_tail.is_some(), "the fixture must leave a receipt");
6703
6704        sup.handle_event(
6705            &SessionPixEvent::RecoveryProgress(make_recovered_tail_progress(&sup, id, target + 40)),
6706            t0,
6707            0,
6708        );
6709        let events = sup.take_telemetry_events();
6710        assert_eq!(0, events.useful_shares, "a recovered cycle can advance nothing further");
6711        assert_eq!(40, events.surplus_shares);
6712    }
6713
6714    // -----------------------------------------------------------------------
6715    // Cycle finalization
6716    // -----------------------------------------------------------------------
6717
6718    /// A cycle retired without recovering is summarized once, with what it actually got.
6719    #[test]
6720    fn a_failed_cycle_is_summarized_with_its_partial_coverage() {
6721        let t0 = Instant::now();
6722        let p = pseudonym();
6723        let mut sup = recovering_session(p, t0);
6724        let id = ssa_id(p, 1);
6725        let target = sup.dims.target_useful_shares();
6726
6727        // Reached the front and got a quarter of the way before stalling. `recovering_session`
6728        // funded it at a served count of zero, so that is where its egress is measured from.
6729        feed(&mut sup, id, target / 4, target / 2);
6730
6731        // Retire it on the hard recovery deadline, with the served count having moved on.
6732        let actions = sup.handle_timers(t0 + Duration::from_secs(100_000), 5_000);
6733        assert!(
6734            actions
6735                .iter()
6736                .any(|action| matches!(action, SessionPixAction::Close(_))),
6737            "the fixture must actually retire the cycle, got {actions:?}"
6738        );
6739
6740        let events = sup.take_telemetry_events();
6741        assert_eq!(1, events.failed);
6742        assert_eq!(1, events.finalized.len(), "summarized exactly once");
6743        let summary = events.finalized[0];
6744        assert_eq!(PixCycleOutcome::Failed, summary.outcome);
6745        assert_eq!(target / 4, summary.useful_shares);
6746        assert_eq!(target / 2, summary.accepted_shares);
6747        assert_eq!(Some(0.25), summary.useful_fraction());
6748        assert_eq!(Some(0.5), summary.accepted_fraction());
6749        assert_eq!(
6750            5_000, summary.egress_packets,
6751            "egress is the Session's served count measured from when this cycle reached the front"
6752        );
6753    }
6754
6755    /// Egress is measured from the front, not from the Session's start.
6756    ///
6757    /// The companion to the test above: there, the cycle reached the front at a served count of
6758    /// zero, so the two readings coincide and the property is invisible. Here a predecessor has
6759    /// already consumed 1 000 packets before this cycle is armed, and only the 4 000 after that
6760    /// belong to it.
6761    #[test]
6762    fn cycle_egress_is_measured_from_the_front_not_the_session_start() {
6763        let t0 = Instant::now();
6764        let p = pseudonym();
6765        let (mut sup, _) = SessionPixSupervisor::new(default_cfg(), dims(64, 8), p, t0);
6766        let id = ssa_id(p, 1);
6767
6768        // Funded at a served count of 1 000: the Session has already been passing data.
6769        sup.handle_event(&SessionPixEvent::SsaRequestSent(id), t0, 1_000);
6770        sup.handle_event(&SessionPixEvent::CommitmentVerified(id), t0, 1_000);
6771        sup.handle_event(
6772            &SessionPixEvent::DepositConfirmed {
6773                ssa_id: id,
6774                amount: sufficient_balance(),
6775            },
6776            t0,
6777            1_000,
6778        );
6779        sup.take_telemetry_events();
6780
6781        sup.handle_timers(t0 + Duration::from_secs(100_000), 5_000);
6782
6783        let events = sup.take_telemetry_events();
6784        assert_eq!(1, events.finalized.len());
6785        assert_eq!(
6786            4_000, events.finalized[0].egress_packets,
6787            "the 1 000 packets served before this cycle reached the front are not its cost"
6788        );
6789    }
6790
6791    /// A cycle that never reached the front is charged no egress.
6792    ///
6793    /// The Entry's emission window is clamped to one cycle, so a queued batch member is not being
6794    /// served at all. Charging it for its predecessors' traffic would report the last cycle of a
6795    /// batch as having cost the whole batch.
6796    #[test]
6797    fn a_queued_cycle_is_charged_no_egress() {
6798        let t0 = Instant::now();
6799        let p = pseudonym();
6800        let cfg = SupervisorConfig {
6801            ssas_per_request: 2,
6802            max_failed_cycles: 4,
6803            ..default_cfg()
6804        };
6805        let (mut sup, _) = SessionPixSupervisor::new(cfg, dims(64, 8), p, t0);
6806
6807        // Only the first cycle is funded, so the second never reaches the front.
6808        commit_unfunded(&mut sup, p, 1..=2, t0);
6809        sup.take_telemetry_events();
6810        sup.handle_event(&SessionPixEvent::DepositObserverClosed(ssa_id(p, 2)), t0, 9_999);
6811
6812        let events = sup.take_telemetry_events();
6813        assert_eq!(1, events.finalized.len());
6814        assert_eq!(
6815            0, events.finalized[0].egress_packets,
6816            "a cycle that never reached the front consumed none of the Session's egress"
6817        );
6818    }
6819
6820    /// An undefined coverage ratio is reported as absent rather than as zero.
6821    #[test]
6822    fn a_cycle_with_no_target_has_no_coverage_fraction() {
6823        let summary = PixCycleSummary {
6824            outcome: PixCycleOutcome::Failed,
6825            egress_packets: 10,
6826            accepted_shares: 0,
6827            useful_shares: 0,
6828            target_useful_shares: 0,
6829            deposit_uhopr: 0,
6830        };
6831        assert_eq!(None, summary.useful_fraction());
6832        assert_eq!(None, summary.accepted_fraction());
6833    }
6834
6835    /// Accepted coverage above one is the conforming case, not an error.
6836    #[test]
6837    fn accepted_coverage_may_exceed_one() {
6838        let summary = PixCycleSummary {
6839            outcome: PixCycleOutcome::Recovered,
6840            egress_packets: 4096,
6841            accepted_shares: 768,
6842            useful_shares: 512,
6843            target_useful_shares: 512,
6844            deposit_uhopr: 0,
6845        };
6846        assert_eq!(Some(1.0), summary.useful_fraction());
6847        assert_eq!(Some(1.5), summary.accepted_fraction());
6848    }
6849
6850    /// The snapshot reports the gate's exposure as given, without interpreting it.
6851    #[test]
6852    fn predeposit_exposure_is_carried_through_from_the_gate() {
6853        let t0 = Instant::now();
6854        let p = pseudonym();
6855        let (sup, _) = SessionPixSupervisor::new(default_cfg(), dims(64, 8), p, t0);
6856
6857        assert_eq!(0, sup.telemetry_snapshot(0).predeposit_exposure_packets);
6858        assert_eq!(917, sup.telemetry_snapshot(917).predeposit_exposure_packets);
6859    }
6860}