hopr_protocol_session/socket/loss_signal.rs
1//! A live signal of frame loss that a Session's owner can read.
2//!
3//! [`FrameLossReport`](super::loss_report) counts and logs loss per window. That is the right shape
4//! for a log line and the wrong one for a decision: the window closes every
5//! [`FRAME_LOSS_REPORT_INTERVAL`](super::loss_report::FRAME_LOSS_REPORT_INTERVAL), so a reader would
6//! learn about a lost frame up to half a minute after the fact, and a Session whose loss stopped
7//! would still look lossy for the rest of the window.
8//!
9//! This signal is that reader's view, and it exists because the Exit has a reason to act on loss
10//! within seconds: while it is not reassembling frames, the only replies that reach the Entry are its
11//! own SURB-level reports, and one sent every `surb_balance_notify_period` (15 s by default) puts the
12//! Entry's verdict on a dead relayer twelve to twenty-eight seconds after the kill instead of about
13//! eleven. Reporting faster while the Exit is losing frames is what closes that gap, and this is what
14//! tells it that it is.
15//!
16//! Unlike the log window, the signal is never reset: a loss is remembered until it ages past whatever
17//! window an observer asks about.
18//!
19//! # Stalls
20//!
21//! A loss is known late. A frame is lost when its reassembly times out, `frame_timeout` after its last
22//! segment arrived, and while segments keep arriving the reassembler looks for timed-out frames only
23//! once per timeout: three to six seconds at the Exit's 3 s default before the first loss of a forward
24//! path that went dark. The signal therefore also follows the frames as they arrive. Segments of two
25//! new frames since a frame last completed or was lost, with none completing for a while, is a forward
26//! path that delivers segments but no frames, and that shows within a second ([`stalled_for`]). A single
27//! frame arriving slowly is not a stall: the next frame starts only once its own segments are all sent.
28//! A sender that stops while frames are stalled does not end the stall, since a sender commonly stops
29//! exactly because nothing comes back; the frames' timeouts end it, as a loss.
30//!
31//! [`stalled_for`]: FrameLossSignal::stalled_for
32
33use std::{
34 sync::atomic::{AtomicU32, AtomicU64, Ordering},
35 time::{Duration, Instant},
36};
37
38/// Records frame loss as recent activity, for a reader that has to act on it within seconds.
39///
40/// Shared between the socket that observes the loss and whatever wants to react to it. Recording is
41/// two relaxed atomic operations on the loss path, so an observer that never asks costs nothing
42/// beyond them.
43///
44/// # One writer, any readers
45///
46/// The readers, [`lost_within`](Self::lost_within), [`lost_total`](Self::lost_total) and
47/// [`stalled_for`](Self::stalled_for), may be called from anywhere at any time. The writers,
48/// [`segment_received`](Self::segment_received), [`frame_completed`](Self::frame_completed) and
49/// [`record`](Self::record), must be called by one writer at a time. The stall state is three atomics
50/// that a writer updates one after another, and `segment_received` decides from one of them before it
51/// updates the next, so two writers interleaving could leave a stall that no frame matches, reported
52/// until the next completion or loss. A socket calls all three from its one sequentially polled stream
53/// chain, which is that single writer. A reader that catches a writer midway sees a stall begin or end
54/// one update late, no worse.
55#[derive(Debug, smart_default::SmartDefault)]
56pub struct FrameLossSignal {
57 /// The instant the millisecond stamps below count from.
58 #[default(Instant::now())]
59 origin: Instant,
60 /// When a frame was last lost, in milliseconds since [`origin`](Self::origin) plus one, so that
61 /// `0` means "never". A sentinel rather than an `Option` in an atomic.
62 last_loss_ms: AtomicU64,
63 /// Frames lost since the socket was built, never reset.
64 lost_total: AtomicU64,
65 /// The highest frame id a segment has carried so far.
66 highest_frame_id: AtomicU32,
67 /// Frames whose first segment arrived since a frame last completed or was lost.
68 frames_started: AtomicU32,
69 /// When the second of [`frames_started`](Self::frames_started) began, in milliseconds since
70 /// [`origin`](Self::origin) plus one; `0` while fewer than two have.
71 stalled_since_ms: AtomicU64,
72}
73
74impl FrameLossSignal {
75 /// Creates a signal that has observed no loss yet.
76 pub fn new() -> Self {
77 Self::default()
78 }
79
80 /// Milliseconds since [`origin`](Self::origin), plus one so that a stamp is never `0`.
81 fn stamp(&self) -> u64 {
82 u64::try_from(self.origin.elapsed().as_millis())
83 .unwrap_or(u64::MAX - 1)
84 .saturating_add(1)
85 }
86
87 /// Records that a frame was lost, which also ends a stall: its frames have timed out.
88 ///
89 /// A duplicate segment is not loss: the frame it belongs to was delivered.
90 pub fn record(&self) {
91 let elapsed = u64::try_from(self.origin.elapsed().as_millis()).unwrap_or(u64::MAX - 1);
92 // The previous stamp may be newer than this one when two loss events race; `fetch_max` keeps
93 // the most recent instant, which is the one an observer is asking about.
94 self.last_loss_ms
95 .fetch_max(elapsed.saturating_add(1), Ordering::Relaxed);
96 self.lost_total.fetch_add(1, Ordering::Relaxed);
97 self.end_stall();
98 }
99
100 /// Whether a frame was lost no longer than `window` ago.
101 ///
102 /// A loss recorded on the millisecond `window` back still counts; the granularity is a
103 /// millisecond, because that is all this signal keeps.
104 pub fn lost_within(&self, window: Duration) -> bool {
105 let last = self.last_loss_ms.load(Ordering::Relaxed);
106 if last == 0 {
107 return false;
108 }
109 let now = u64::try_from(self.origin.elapsed().as_millis()).unwrap_or(u64::MAX - 1);
110 // The stamp carries a one-millisecond offset, so the instant of the loss is `last - 1`.
111 now.saturating_sub(last - 1) <= u64::try_from(window.as_millis()).unwrap_or(u64::MAX)
112 }
113
114 /// Frames lost since this signal was created.
115 ///
116 /// Unlike the socket's per-window log summary, this count is never reset: it is the total for
117 /// the life of the socket.
118 pub fn lost_total(&self) -> u64 {
119 self.lost_total.load(Ordering::Relaxed)
120 }
121
122 /// Records that a segment of `frame_id` arrived (a duplicate of a delivered frame is not one).
123 ///
124 /// The first segment of a frame newer than any seen so far starts that frame; the second frame to
125 /// start since a frame last completed starts a stall (see [`stalled_for`](Self::stalled_for)).
126 pub fn segment_received(&self, frame_id: u32) {
127 if self.highest_frame_id.fetch_max(frame_id, Ordering::Relaxed) < frame_id
128 && self.frames_started.fetch_add(1, Ordering::Relaxed) == 1
129 {
130 self.stalled_since_ms.store(self.stamp(), Ordering::Relaxed);
131 }
132 }
133
134 /// Records that a frame completed its reassembly, which ends a stall.
135 pub fn frame_completed(&self) {
136 self.end_stall();
137 }
138
139 fn end_stall(&self) {
140 self.frames_started.store(0, Ordering::Relaxed);
141 self.stalled_since_ms.store(0, Ordering::Relaxed);
142 }
143
144 /// Whether frames have stalled for at least `after`: segments of two frames started since a frame
145 /// last completed or was lost, the second of them at least `after` ago, and none has completed or
146 /// been lost since.
147 ///
148 /// A stall lasts until a frame completes or one is lost, however long the segments pause: a sender
149 /// commonly stops because nothing comes back, and the frames' timeouts end the stall as a loss.
150 pub fn stalled_for(&self, after: Duration) -> bool {
151 let since = self.stalled_since_ms.load(Ordering::Relaxed);
152 since != 0 && self.stamp().saturating_sub(since) >= u64::try_from(after.as_millis()).unwrap_or(u64::MAX)
153 }
154}
155
156#[cfg(test)]
157mod tests {
158 use super::*;
159
160 #[test]
161 fn a_fresh_signal_has_no_loss() {
162 let signal = FrameLossSignal::new();
163 assert!(!signal.lost_within(Duration::from_secs(1)));
164 assert!(!signal.lost_within(Duration::from_secs(3600)));
165 assert_eq!(0, signal.lost_total());
166 }
167
168 #[test]
169 fn a_recorded_loss_is_seen_inside_its_window() {
170 let signal = FrameLossSignal::new();
171 signal.record();
172 assert!(
173 signal.lost_within(Duration::from_millis(500)),
174 "a loss recorded now is within a small window"
175 );
176 assert!(signal.lost_within(Duration::from_millis(1000)));
177 assert!(signal.lost_within(Duration::from_secs(3)));
178 assert_eq!(1, signal.lost_total());
179 }
180
181 #[test]
182 fn a_loss_ages_out_of_the_window() {
183 let window = Duration::from_millis(30);
184 let signal = FrameLossSignal::new();
185 signal.record();
186 std::thread::sleep(window * 2);
187 assert!(
188 !signal.lost_within(window),
189 "a loss older than the window must not be reported as recent"
190 );
191 assert_eq!(1, signal.lost_total(), "aging out does not forget the loss");
192 }
193
194 #[test]
195 fn frames_that_do_not_complete_while_segments_arrive_are_a_stall() {
196 let after = Duration::from_millis(30);
197 let signal = FrameLossSignal::new();
198 signal.segment_received(1);
199 signal.segment_received(2);
200 assert!(!signal.stalled_for(after), "not before `after` has passed");
201
202 std::thread::sleep(after + Duration::from_millis(5));
203 signal.segment_received(3);
204 assert!(
205 signal.stalled_for(after),
206 "two frames started, none completed for `after`"
207 );
208 assert_eq!(0, signal.lost_total(), "a stall is not a loss");
209 }
210
211 #[test]
212 fn a_completed_frame_ends_a_stall() {
213 let after = Duration::from_millis(30);
214 let signal = FrameLossSignal::new();
215 signal.segment_received(1);
216 signal.segment_received(2);
217 std::thread::sleep(after + Duration::from_millis(5));
218 signal.segment_received(2);
219 assert!(signal.stalled_for(after));
220
221 signal.frame_completed();
222 assert!(!signal.stalled_for(after));
223 signal.segment_received(3);
224 assert!(
225 !signal.stalled_for(after),
226 "one new frame since the completion is not a stall"
227 );
228 }
229
230 #[test]
231 fn a_single_slow_frame_is_not_a_stall() {
232 let after = Duration::from_millis(30);
233 let signal = FrameLossSignal::new();
234 signal.segment_received(1);
235 std::thread::sleep(after + Duration::from_millis(5));
236 signal.segment_received(1);
237 assert!(!signal.stalled_for(after), "segments of one frame only");
238 }
239
240 #[test]
241 fn a_stall_outlasts_a_pause_and_ends_with_a_loss() {
242 let after = Duration::from_millis(30);
243 let signal = FrameLossSignal::new();
244 signal.segment_received(1);
245 signal.segment_received(2);
246 std::thread::sleep(after * 3);
247 assert!(
248 signal.stalled_for(after),
249 "no segment since, but nothing completed either: the sender stopped because nothing came back"
250 );
251
252 signal.record();
253 assert!(!signal.stalled_for(after), "the stalled frames timed out, as a loss");
254 assert!(signal.lost_within(after));
255 }
256
257 #[test]
258 fn every_loss_counts_and_the_most_recent_wins() {
259 let signal = FrameLossSignal::new();
260 signal.record();
261 signal.record();
262 signal.record();
263 assert_eq!(3, signal.lost_total());
264 assert!(signal.lost_within(Duration::from_millis(500)));
265 }
266}