Skip to main content

hopr_protocol_session/socket/
loss_signal.rs

1//! A live signal of frame loss that a Session's owner can read.
2//!
3//! [`FrameLossReport`](super::loss_report) counts and logs loss per window. That is the right shape
4//! for a log line and the wrong one for a decision: the window closes every
5//! [`FRAME_LOSS_REPORT_INTERVAL`](super::loss_report::FRAME_LOSS_REPORT_INTERVAL), so a reader would
6//! learn about a lost frame up to half a minute after the fact, and a Session whose loss stopped
7//! would still look lossy for the rest of the window.
8//!
9//! This signal is that reader's view, and it exists because the Exit has a reason to act on loss
10//! within seconds: while it is not reassembling frames, the only replies that reach the Entry are its
11//! own SURB-level reports, and one sent every `surb_balance_notify_period` (15 s by default) puts the
12//! Entry's verdict on a dead relayer twelve to twenty-eight seconds after the kill instead of about
13//! eleven. Reporting faster while the Exit is losing frames is what closes that gap, and this is what
14//! tells it that it is.
15//!
16//! Unlike the log window, the signal is never reset: a loss is remembered until it ages past whatever
17//! window an observer asks about.
18//!
19//! # Stalls
20//!
21//! A loss is known late. A frame is lost when its reassembly times out, `frame_timeout` after its last
22//! segment arrived, and while segments keep arriving the reassembler looks for timed-out frames only
23//! once per timeout: three to six seconds at the Exit's 3 s default before the first loss of a forward
24//! path that went dark. The signal therefore also follows the frames as they arrive. Segments of two
25//! new frames since a frame last completed or was lost, with none completing for a while, is a forward
26//! path that delivers segments but no frames, and that shows within a second ([`stalled_for`]). A single
27//! frame arriving slowly is not a stall: the next frame starts only once its own segments are all sent.
28//! A sender that stops while frames are stalled does not end the stall, since a sender commonly stops
29//! exactly because nothing comes back; the frames' timeouts end it, as a loss.
30//!
31//! [`stalled_for`]: FrameLossSignal::stalled_for
32
33use std::{
34    sync::atomic::{AtomicU32, AtomicU64, Ordering},
35    time::{Duration, Instant},
36};
37
38/// Records frame loss as recent activity, for a reader that has to act on it within seconds.
39///
40/// Shared between the socket that observes the loss and whatever wants to react to it. Recording is
41/// two relaxed atomic operations on the loss path, so an observer that never asks costs nothing
42/// beyond them.
43///
44/// # One writer, any readers
45///
46/// The readers, [`lost_within`](Self::lost_within), [`lost_total`](Self::lost_total) and
47/// [`stalled_for`](Self::stalled_for), may be called from anywhere at any time. The writers,
48/// [`segment_received`](Self::segment_received), [`frame_completed`](Self::frame_completed) and
49/// [`record`](Self::record), must be called by one writer at a time. The stall state is three atomics
50/// that a writer updates one after another, and `segment_received` decides from one of them before it
51/// updates the next, so two writers interleaving could leave a stall that no frame matches, reported
52/// until the next completion or loss. A socket calls all three from its one sequentially polled stream
53/// chain, which is that single writer. A reader that catches a writer midway sees a stall begin or end
54/// one update late, no worse.
55#[derive(Debug, smart_default::SmartDefault)]
56pub struct FrameLossSignal {
57    /// The instant the millisecond stamps below count from.
58    #[default(Instant::now())]
59    origin: Instant,
60    /// When a frame was last lost, in milliseconds since [`origin`](Self::origin) plus one, so that
61    /// `0` means "never". A sentinel rather than an `Option` in an atomic.
62    last_loss_ms: AtomicU64,
63    /// Frames lost since the socket was built, never reset.
64    lost_total: AtomicU64,
65    /// The highest frame id a segment has carried so far.
66    highest_frame_id: AtomicU32,
67    /// Frames whose first segment arrived since a frame last completed or was lost.
68    frames_started: AtomicU32,
69    /// When the second of [`frames_started`](Self::frames_started) began, in milliseconds since
70    /// [`origin`](Self::origin) plus one; `0` while fewer than two have.
71    stalled_since_ms: AtomicU64,
72}
73
74impl FrameLossSignal {
75    /// Creates a signal that has observed no loss yet.
76    pub fn new() -> Self {
77        Self::default()
78    }
79
80    /// Milliseconds since [`origin`](Self::origin), plus one so that a stamp is never `0`.
81    fn stamp(&self) -> u64 {
82        u64::try_from(self.origin.elapsed().as_millis())
83            .unwrap_or(u64::MAX - 1)
84            .saturating_add(1)
85    }
86
87    /// Records that a frame was lost, which also ends a stall: its frames have timed out.
88    ///
89    /// A duplicate segment is not loss: the frame it belongs to was delivered.
90    pub fn record(&self) {
91        let elapsed = u64::try_from(self.origin.elapsed().as_millis()).unwrap_or(u64::MAX - 1);
92        // The previous stamp may be newer than this one when two loss events race; `fetch_max` keeps
93        // the most recent instant, which is the one an observer is asking about.
94        self.last_loss_ms
95            .fetch_max(elapsed.saturating_add(1), Ordering::Relaxed);
96        self.lost_total.fetch_add(1, Ordering::Relaxed);
97        self.end_stall();
98    }
99
100    /// Whether a frame was lost no longer than `window` ago.
101    ///
102    /// A loss recorded on the millisecond `window` back still counts; the granularity is a
103    /// millisecond, because that is all this signal keeps.
104    pub fn lost_within(&self, window: Duration) -> bool {
105        let last = self.last_loss_ms.load(Ordering::Relaxed);
106        if last == 0 {
107            return false;
108        }
109        let now = u64::try_from(self.origin.elapsed().as_millis()).unwrap_or(u64::MAX - 1);
110        // The stamp carries a one-millisecond offset, so the instant of the loss is `last - 1`.
111        now.saturating_sub(last - 1) <= u64::try_from(window.as_millis()).unwrap_or(u64::MAX)
112    }
113
114    /// Frames lost since this signal was created.
115    ///
116    /// Unlike the socket's per-window log summary, this count is never reset: it is the total for
117    /// the life of the socket.
118    pub fn lost_total(&self) -> u64 {
119        self.lost_total.load(Ordering::Relaxed)
120    }
121
122    /// Records that a segment of `frame_id` arrived (a duplicate of a delivered frame is not one).
123    ///
124    /// The first segment of a frame newer than any seen so far starts that frame; the second frame to
125    /// start since a frame last completed starts a stall (see [`stalled_for`](Self::stalled_for)).
126    pub fn segment_received(&self, frame_id: u32) {
127        if self.highest_frame_id.fetch_max(frame_id, Ordering::Relaxed) < frame_id
128            && self.frames_started.fetch_add(1, Ordering::Relaxed) == 1
129        {
130            self.stalled_since_ms.store(self.stamp(), Ordering::Relaxed);
131        }
132    }
133
134    /// Records that a frame completed its reassembly, which ends a stall.
135    pub fn frame_completed(&self) {
136        self.end_stall();
137    }
138
139    fn end_stall(&self) {
140        self.frames_started.store(0, Ordering::Relaxed);
141        self.stalled_since_ms.store(0, Ordering::Relaxed);
142    }
143
144    /// Whether frames have stalled for at least `after`: segments of two frames started since a frame
145    /// last completed or was lost, the second of them at least `after` ago, and none has completed or
146    /// been lost since.
147    ///
148    /// A stall lasts until a frame completes or one is lost, however long the segments pause: a sender
149    /// commonly stops because nothing comes back, and the frames' timeouts end the stall as a loss.
150    pub fn stalled_for(&self, after: Duration) -> bool {
151        let since = self.stalled_since_ms.load(Ordering::Relaxed);
152        since != 0 && self.stamp().saturating_sub(since) >= u64::try_from(after.as_millis()).unwrap_or(u64::MAX)
153    }
154}
155
156#[cfg(test)]
157mod tests {
158    use super::*;
159
160    #[test]
161    fn a_fresh_signal_has_no_loss() {
162        let signal = FrameLossSignal::new();
163        assert!(!signal.lost_within(Duration::from_secs(1)));
164        assert!(!signal.lost_within(Duration::from_secs(3600)));
165        assert_eq!(0, signal.lost_total());
166    }
167
168    #[test]
169    fn a_recorded_loss_is_seen_inside_its_window() {
170        let signal = FrameLossSignal::new();
171        signal.record();
172        assert!(
173            signal.lost_within(Duration::from_millis(500)),
174            "a loss recorded now is within a small window"
175        );
176        assert!(signal.lost_within(Duration::from_millis(1000)));
177        assert!(signal.lost_within(Duration::from_secs(3)));
178        assert_eq!(1, signal.lost_total());
179    }
180
181    #[test]
182    fn a_loss_ages_out_of_the_window() {
183        let window = Duration::from_millis(30);
184        let signal = FrameLossSignal::new();
185        signal.record();
186        std::thread::sleep(window * 2);
187        assert!(
188            !signal.lost_within(window),
189            "a loss older than the window must not be reported as recent"
190        );
191        assert_eq!(1, signal.lost_total(), "aging out does not forget the loss");
192    }
193
194    #[test]
195    fn frames_that_do_not_complete_while_segments_arrive_are_a_stall() {
196        let after = Duration::from_millis(30);
197        let signal = FrameLossSignal::new();
198        signal.segment_received(1);
199        signal.segment_received(2);
200        assert!(!signal.stalled_for(after), "not before `after` has passed");
201
202        std::thread::sleep(after + Duration::from_millis(5));
203        signal.segment_received(3);
204        assert!(
205            signal.stalled_for(after),
206            "two frames started, none completed for `after`"
207        );
208        assert_eq!(0, signal.lost_total(), "a stall is not a loss");
209    }
210
211    #[test]
212    fn a_completed_frame_ends_a_stall() {
213        let after = Duration::from_millis(30);
214        let signal = FrameLossSignal::new();
215        signal.segment_received(1);
216        signal.segment_received(2);
217        std::thread::sleep(after + Duration::from_millis(5));
218        signal.segment_received(2);
219        assert!(signal.stalled_for(after));
220
221        signal.frame_completed();
222        assert!(!signal.stalled_for(after));
223        signal.segment_received(3);
224        assert!(
225            !signal.stalled_for(after),
226            "one new frame since the completion is not a stall"
227        );
228    }
229
230    #[test]
231    fn a_single_slow_frame_is_not_a_stall() {
232        let after = Duration::from_millis(30);
233        let signal = FrameLossSignal::new();
234        signal.segment_received(1);
235        std::thread::sleep(after + Duration::from_millis(5));
236        signal.segment_received(1);
237        assert!(!signal.stalled_for(after), "segments of one frame only");
238    }
239
240    #[test]
241    fn a_stall_outlasts_a_pause_and_ends_with_a_loss() {
242        let after = Duration::from_millis(30);
243        let signal = FrameLossSignal::new();
244        signal.segment_received(1);
245        signal.segment_received(2);
246        std::thread::sleep(after * 3);
247        assert!(
248            signal.stalled_for(after),
249            "no segment since, but nothing completed either: the sender stopped because nothing came back"
250        );
251
252        signal.record();
253        assert!(!signal.stalled_for(after), "the stalled frames timed out, as a loss");
254        assert!(signal.lost_within(after));
255    }
256
257    #[test]
258    fn every_loss_counts_and_the_most_recent_wins() {
259        let signal = FrameLossSignal::new();
260        signal.record();
261        signal.record();
262        signal.record();
263        assert_eq!(3, signal.lost_total());
264        assert!(signal.lost_within(Duration::from_millis(500)));
265    }
266}