Skip to main content

hopr_protocol_session/socket/
mod.rs

1//! This module defines the socket-like interface for Session protocol.
2
3pub mod ack_state;
4mod loss_report;
5pub mod loss_signal;
6pub mod state;
7
8/// Contains socket statistics types.
9#[cfg(feature = "telemetry")]
10pub mod telemetry;
11
12use std::{
13    pin::Pin,
14    sync::{Arc, atomic::AtomicU32},
15    task::{Context, Poll},
16    time::Duration,
17};
18
19use futures::{FutureExt, SinkExt, StreamExt, TryStreamExt, future, future::AbortHandle};
20use futures_concurrency::stream::Merge;
21use loss_report::{FRAME_LOSS_REPORT_INTERVAL, FrameLossReport};
22use loss_signal::FrameLossSignal;
23use state::{SocketComponents, SocketState, Stateless};
24use tracing::{Instrument, instrument::WithSubscriber};
25#[cfg(feature = "telemetry")]
26use {
27    strum::IntoDiscriminant,
28    telemetry::{SessionMessageDiscriminants, SessionTelemetryTracker},
29};
30
31use crate::{
32    MAX_SESSION_MTU,
33    errors::SessionError,
34    processing::{ReassemblerExt, SegmenterExt, SequencerExt, types::FrameInspector},
35    protocol::{OrderedFrame, SegmentRequest, SeqIndicator, SessionCodec, SessionMessage, stop_on_invalid_header},
36    session_frame_size,
37};
38
39/// Configuration object for [`SessionSocket`].
40#[derive(Debug, Copy, Clone, Eq, PartialEq, smart_default::SmartDefault)]
41pub struct SessionSocketConfig {
42    /// The maximum size of a frame on the read/write interface of the [`SessionSocket`].
43    ///
44    /// Clamped by [`session_frame_size`] between one [`crate::session_socket_mtu`] payload and:
45    /// - `SeqIndicator::MAX` + 1 segments for stateless sockets, or
46    /// - min(`SeqIndicator::MAX` + 1, `SegmentRequest::MAX_MISSING_SEGMENTS_PER_FRAME`) segments for stateful sockets
47    ///
48    /// Default is [`MAX_SESSION_MTU`], i.e. exactly one segment per frame.
49    #[default(MAX_SESSION_MTU)]
50    pub frame_size: usize,
51    /// The maximum time to wait for a frame to be fully received.
52    ///
53    /// Default is 800 ms.
54    #[default(Duration::from_millis(800))]
55    pub frame_timeout: Duration,
56    /// Maximum number of segments to buffer in the downstream transport.
57    /// If 0 is given, the transport is unbuffered.
58    ///
59    /// Default is 0.
60    #[default(0)]
61    pub max_buffered_segments: usize,
62    /// Capacity of the frame reconstructor, the maximum number of incomplete frames, before
63    /// they are dropped.
64    ///
65    /// Default is 8192.
66    #[default(8192)]
67    pub capacity: usize,
68    /// Flushes data written to the socket immediately to the underlying transport.
69    ///
70    /// Default is false.
71    #[default(false)]
72    pub flush_immediately: bool,
73    /// Preserve datagram boundaries: each write to the socket is emitted as exactly one frame and
74    /// delivered to the peer as exactly one read, regardless of [`Self::frame_size`]. Use for
75    /// datagram-oriented targets (UDP/WireGuard). Only meaningful on stateless sockets.
76    ///
77    /// Default is false.
78    #[default(false)]
79    pub datagram: bool,
80    /// Capacity of the control channel, the maximum number of outstanding control messages.
81    ///
82    /// This option affects stateful sockets.
83    ///
84    /// Default is 2048.
85    #[default(2048)]
86    pub control_channel_capacity: usize,
87    /// Maximum time a fully-received frame may sit in the ordering buffer before being discarded
88    /// rather than delivered.
89    ///
90    /// Bounds how stale delivered data can be, so a stall surfaces as clean loss instead of a burst
91    /// of seconds-old frames. Distinct from [`Self::frame_timeout`], which bounds how long a
92    /// *missing* frame is waited for. Default `None` (no bound).
93    ///
94    /// A frame dropped here has already been acknowledged, since the acknowledgement is queued at
95    /// reassembly. That is deliberate: the ack states that the *path* delivered the frame, which it
96    /// did, and the drop is a local freshness policy applied afterwards. Withholding the ack would
97    /// make the sender retransmit data we discarded precisely for being stale, reintroducing the
98    /// latency tail this bound exists to remove.
99    #[default(None)]
100    pub max_frame_age: Option<Duration>,
101    /// Abandon the frame due next once this many later frames are already waiting behind it,
102    /// instead of holding them for [`Self::frame_timeout`].
103    ///
104    /// Head-of-line bound. `frame_timeout` waits for a frame that may still arrive; this bounds
105    /// how much already-received data is held hostage while that wait runs. On a session without
106    /// retransmission the missing frame is never coming, so the wait is pure cost: measured on a
107    /// cluster, 98.5% of bytes returned over the wire while 0.60% reached the application and the
108    /// application-side inter-arrival median sat exactly on the timeout.
109    ///
110    /// Counting frames rather than watching a clock decides on evidence -- a queue building up
111    /// behind a gap is a lost frame, where one or two frames ahead is ordinary reordering. The
112    /// right value tracks reordering depth, which is throughput x latency spread, so it is
113    /// deployment-specific and meant to be tuned. `None` (default) keeps the previous behaviour.
114    #[default(None)]
115    pub max_frames_behind_gap: Option<usize>,
116    /// Wait on a missing frame only as long as the reordering the Session has shown supports:
117    /// twice the deepest of the last minute, at least 250 ms, at most [`Self::frame_timeout`],
118    /// and all of it before any reordering is seen. Stateless sockets only.
119    ///
120    /// Head-of-line bound by time where [`Self::max_frames_behind_gap`] bounds by count. The frame
121    /// timeout is sized for the worst a deployment may see, and a Session without retransmission
122    /// that loses a frame holds everything behind it for that long, waiting for something that is
123    /// never coming. Measured on a cluster after a forward relayer died: the Exit's first
124    /// complete frames waited the whole 3 s behind frames lost before the path was replaced. A
125    /// frame that arrives after its gap was given up lengthens the wait again. A reliable socket
126    /// keeps the whole timeout, since a retransmission can still fill the gap.
127    ///
128    /// Default is true.
129    #[default(true)]
130    pub adapt_gap_wait: bool,
131}
132
133/// Logs a failed update of the socket state.
134///
135/// When the Session is closing, frames that are still in the pipeline can no longer update it.
136/// It's a normal situation and not an error.
137fn log_state_update_error(error: &SessionError, message: &'static str) {
138    if matches!(error, SessionError::StateNotRunning) {
139        tracing::debug!(%error, message);
140    } else {
141        tracing::error!(%error, message);
142    }
143}
144
145enum WriteState {
146    WriteOnly,
147    Writing,
148    Flushing(usize),
149}
150
151/// Socket-like object implementing the Session protocol that can operate on any transport that
152/// implements [`futures::io::AsyncRead`] and [`futures::io::AsyncWrite`].
153///
154/// The [`SocketState`] `S` given during instantiation can facilitate reliable or unreliable
155/// behavior (see [`AcknowledgementState`](ack_state::AcknowledgementState))
156///
157/// The constant argument `C` specifies the MTU in bytes of the underlying transport.
158#[pin_project::pin_project(PinnedDrop)]
159pub struct SessionSocket<const C: usize, S: SocketState<C>> {
160    state: S,
161    span: tracing::Span,
162    // This is where upstream writes the to-be-segmented frame data to
163    upstream_frames_in: Pin<Box<dyn futures::io::AsyncWrite + Send>>,
164    // This is where upstream reads the reconstructed frame data from
165    downstream_frames_out: Pin<Box<dyn futures::io::AsyncRead + Send>>,
166    write_state: WriteState,
167}
168
169// Ensure `state.stop()` runs even if the socket is dropped without
170// `poll_close` (e.g. its owning task was aborted). Without this, the
171// detached tasks spawned in `SocketState::run` keep their own channel
172// senders and never terminate, pinning the FrameInspector, ring-buffer,
173// and ack buffers per leaked session.
174#[pin_project::pinned_drop]
175impl<const C: usize, S: SocketState<C>> PinnedDrop for SessionSocket<C, S> {
176    fn drop(self: Pin<&mut Self>) {
177        let this = self.project();
178        let _entered = this.span.enter();
179        if let Err(error) = this.state.stop() {
180            tracing::debug!(%error, "state.stop on SessionSocket drop failed");
181        }
182    }
183}
184
185impl<const C: usize> SessionSocket<C, Stateless<C>> {
186    /// Creates a new stateless socket suitable for fast UDP-like communication.
187    ///
188    /// Note that this results in a faster socket than if created via [`SessionSocket::new`] with
189    /// [`Stateless`]. This is because the frame inspector does not need to be instantiated.
190    ///
191    /// `loss_signal`, when given, is stamped at every frame the socket loses (a frame that expired
192    /// incomplete or a sequencer discard, never a duplicate), for a reader that has to react within
193    /// seconds; the per-window log summary is too coarse for that. It also follows every segment and
194    /// every reassembled frame, so that it can tell frames that stall before any of them is lost.
195    /// `None` records nothing.
196    pub fn new_stateless<T, I>(
197        id: I,
198        transport: T,
199        cfg: SessionSocketConfig,
200        loss_signal: Option<Arc<FrameLossSignal>>,
201        #[cfg(feature = "telemetry")] stats: impl SessionTelemetryTracker + Clone + Send + 'static,
202    ) -> Result<Self, SessionError>
203    where
204        T: futures::io::AsyncRead + futures::io::AsyncWrite + Send + Unpin + 'static,
205        I: std::fmt::Display + Clone,
206    {
207        let span = tracing::debug_span!("session.socket");
208        let _entered = span.enter();
209        // Preserve frame boundaries within the number of segments SeqIndicator can enumerate.
210        let frame_size = session_frame_size::<C>(cfg.frame_size, (SeqIndicator::MAX + 1) as usize);
211
212        // Segment data incoming/outgoing using underlying transport
213        let mut framed = asynchronous_codec::Framed::new(transport, SessionCodec::<C>);
214
215        // Check if we allow sending multiple segments to downstream in a single write
216        // The HWM cannot be 0 bytes
217        framed.set_send_high_water_mark(1.max(cfg.max_buffered_segments * C));
218
219        // Downstream transport
220        let (packets_out, packets_in) = framed.split();
221        let packets_in = stop_on_invalid_header(packets_in);
222
223        // If needed, add also stats to individual stages.
224        #[cfg(feature = "telemetry")]
225        let (s0, s1, s2, s3) = { (stats.clone(), stats.clone(), stats.clone(), stats.clone()) };
226
227        // Pipeline IN: Data incoming from Upstream
228        let upstream_frames_in = packets_out
229            .with(move |segment| {
230                #[cfg(feature = "telemetry")]
231                s0.outgoing_message(SessionMessageDiscriminants::Segment);
232
233                future::ok::<_, SessionError>(SessionMessage::<C>::Segment(segment))
234            })
235            .segmenter_with_terminating_segment::<C>(frame_size, cfg.datagram);
236
237        let last_emitted_frame = Arc::new(AtomicU32::new(0));
238        let last_emitted_frame_clone = last_emitted_frame.clone();
239
240        // Dropped frames are counted and reported once per window, not logged one by one
241        let loss_report = Arc::new(FrameLossReport::with_signal(
242            &id,
243            FRAME_LOSS_REPORT_INTERVAL,
244            loss_signal,
245        ));
246        let (lr1, lr2, lr3) = (loss_report.clone(), loss_report.clone(), loss_report);
247
248        // The reordering the reassembler and the sequencer both see, which sizes the gap wait.
249        let reorder = cfg.adapt_gap_wait.then(crate::processing::SharedReorder::default);
250        let reorder_s1 = reorder.clone();
251
252        // Debug tracing spans for individual pipeline stages
253        let stage1_span = tracing::debug_span!("SessionSocket::packets_in::pre_reassembly");
254        let stage2_span = tracing::debug_span!("SessionSocket::packets_in::pre_sequencing");
255        let stage3_span = tracing::debug_span!("SessionSocket::packets_in::post_sequencing");
256
257        let (packets_in_abort_handle, packets_in_abort_reg) = AbortHandle::new_pair();
258
259        // Pipeline OUT: Packets incoming from Downstream
260        // Continue receiving packets from downstream, unless we received a terminating frame.
261        // Once the terminating frame is received, the `packets_in_abort_handle` is triggered, terminating the pipeline.
262        let downstream_frames_out = futures::stream::Abortable::new(packets_in, packets_in_abort_reg)
263            // Filter-out segments that we've seen already
264            .filter_map(move |packet| {
265                let _span = stage1_span.enter();
266                futures::future::ready(match packet {
267                    Ok(packet) => {
268                        packet.try_as_segment().filter(|s| {
269                            #[cfg(feature = "telemetry")]
270                            s1.incoming_message(SessionMessageDiscriminants::Segment);
271
272                            // Filter old frame ids to save space in the Reassembler
273                            let last_emitted_id = last_emitted_frame.load(std::sync::atomic::Ordering::Relaxed);
274                            if s.frame_id <= last_emitted_id {
275                                tracing::trace!(frame_id = s.frame_id, last_emitted_id, "frame already seen");
276                                // A segment of a frame whose gap was given up: it never completes, so this
277                                // is the only place to learn the wait was too short for it.
278                                if let Some(reorder) = reorder_s1.as_ref() {
279                                    let now = std::time::Instant::now();
280                                    if let Some(late) = reorder.late(s.frame_id, now) {
281                                        tracing::debug!(
282                                            frame_id = s.frame_id,
283                                            late_ms = late.as_millis() as u64,
284                                            "segment came after its frame's gap was given up"
285                                        );
286                                        reorder.record(now, late);
287                                    }
288                                }
289                                lr1.duplicate_segment();
290                                false
291                            } else {
292                                lr1.segment_received(s.frame_id);
293                                true
294                            }
295                        })
296                    }
297                    Err(error) => {
298                        if error.is_fatal_header() {
299                            tracing::error!(%error, "invalid session header; terminating session reader");
300                        } else {
301                            tracing::error!(%error, "unparseable packet");
302                        }
303                        #[cfg(feature = "telemetry")]
304                        s1.error();
305                        None
306                    }
307                })
308            })
309            // Reassemble the segments into frames
310            .reassembler_sharing(cfg.frame_timeout, cfg.capacity, reorder.clone())
311            // Discard frames that we could not reassemble
312            .filter_map(move |maybe_frame| {
313                let _span = stage2_span.enter();
314                futures::future::ready(match maybe_frame {
315                    Ok(frame) => {
316                        lr2.frame_reassembled();
317                        #[cfg(feature = "telemetry")]
318                        s2.frame_completed();
319                        Some(OrderedFrame(frame))
320                    }
321                    Err(error) => {
322                        tracing::debug!(%error, "failed to reassemble frame");
323                        lr2.frame_not_reassembled();
324                        #[cfg(feature = "telemetry")]
325                        s2.incomplete_frame();
326                        None
327                    }
328                })
329            })
330            // Put the frames into the correct sequence by Frame Ids
331            .sequencer_sharing(
332                crate::processing::SequencerConfig {
333                    max_wait: cfg.frame_timeout,
334                    capacity: cfg.capacity,
335                    max_item_age: cfg.max_frame_age,
336                    max_frames_behind_gap: cfg.max_frames_behind_gap,
337                    adapt_to_reordering: cfg.adapt_gap_wait,
338                },
339                reorder,
340            )
341            // Discard frames missing from the sequence
342            .filter_map(move |maybe_frame| {
343                let _span = stage3_span.enter();
344                future::ready(match maybe_frame {
345                    Ok(frame) => {
346                        last_emitted_frame_clone.store(frame.0.frame_id, std::sync::atomic::Ordering::Relaxed);
347                        if frame.0.is_terminating {
348                            tracing::debug!("terminating frame received");
349                            packets_in_abort_handle.abort();
350                        }
351                        lr3.frame_emitted();
352                        #[cfg(feature = "telemetry")]
353                        s3.frame_emitted();
354                        Some(Ok(frame.0))
355                    }
356                    // Downstream skips discarded frames
357                    Err(SessionError::FrameDiscarded(frame_id)) | Err(SessionError::IncompleteFrame(frame_id)) => {
358                        tracing::debug!(frame_id, "frame discarded");
359                        lr3.frame_discarded();
360                        #[cfg(feature = "telemetry")]
361                        s3.frame_discarded();
362                        None
363                    }
364                    Err(err) => {
365                        #[cfg(feature = "telemetry")]
366                        s3.error();
367                        Some(Err(std::io::Error::other(err)))
368                    }
369                })
370            })
371            .into_async_read();
372
373        Ok(Self {
374            state: Stateless::new(id),
375            span: span.clone(),
376            upstream_frames_in: Box::pin(upstream_frames_in),
377            downstream_frames_out: Box::pin(downstream_frames_out),
378            write_state: if cfg.flush_immediately {
379                WriteState::Writing
380            } else {
381                WriteState::WriteOnly
382            },
383        })
384    }
385}
386
387impl<const C: usize, S: SocketState<C> + Clone + 'static> SessionSocket<C, S> {
388    /// Creates a stateful socket with frame inspection capabilities - suitable for communication
389    /// requiring TCP-like delivery guarantees.
390    ///
391    /// `loss_signal`, when given, is stamped at every frame the socket loses (a frame that expired
392    /// incomplete or a sequencer discard, never a duplicate), for a reader that has to react within
393    /// seconds; the per-window log summary is too coarse for that. It also follows every segment and
394    /// every reassembled frame, so that it can tell frames that stall before any of them is lost.
395    /// `None` records nothing.
396    pub fn new<T>(
397        transport: T,
398        mut state: S,
399        cfg: SessionSocketConfig,
400        loss_signal: Option<Arc<FrameLossSignal>>,
401        #[cfg(feature = "telemetry")] stats: impl SessionTelemetryTracker + Clone + Send + 'static,
402    ) -> Result<Self, SessionError>
403    where
404        T: futures::io::AsyncRead + futures::io::AsyncWrite + Send + Unpin + 'static,
405    {
406        let span = tracing::debug_span!("session.socket");
407        let _entered = span.enter();
408        // The upper bound is additionally limited by the missing-segment bitmap in SegmentRequests.
409        let frame_size = session_frame_size::<C>(
410            cfg.frame_size,
411            SegmentRequest::<C>::MAX_MISSING_SEGMENTS_PER_FRAME.min((SeqIndicator::MAX + 1) as usize),
412        );
413
414        // Segment data incoming/outgoing using underlying transport
415        let mut framed = asynchronous_codec::Framed::new(transport, SessionCodec::<C>);
416
417        // Check if we allow sending multiple segments to downstream in a single write
418        // The HWM cannot be 0 bytes
419        framed.set_send_high_water_mark(1.max(cfg.max_buffered_segments * C));
420
421        // If needed, add also stats to individual stages.
422        #[cfg(feature = "telemetry")]
423        let (s0, s1, s2, s3) = { (stats.clone(), stats.clone(), stats.clone(), stats.clone()) };
424
425        // Downstream transport
426        let (packets_out, packets_in) = framed.split();
427        let packets_in = stop_on_invalid_header(packets_in);
428
429        let inspector = FrameInspector::new(cfg.capacity);
430
431        tracing::debug!(
432            capacity = cfg.control_channel_capacity,
433            "creating session control channel"
434        );
435        let (ctl_tx, ctl_rx) = hopr_utils::network_types::crossfire_sink::bounded_sink_channel::<SessionMessage<C>>(
436            cfg.control_channel_capacity.max(128),
437        );
438        state.run(SocketComponents {
439            inspector: Some(inspector.clone()),
440            ctl_tx,
441        })?;
442
443        // Pipeline IN: Data incoming from Upstream
444        let (segments_tx, segments_rx) = futures::channel::mpsc::channel(cfg.capacity);
445        let mut st_1 = state.clone();
446        let upstream_frames_in = segments_tx
447            .with(move |segment| {
448                let _span = tracing::debug_span!("SessionSocket::packets_out::segmenter").entered();
449                // The segment_sent event is raised only for segments coming from Upstream,
450                // not for the segments from the Control stream (= segment resends).
451                if let Err(error) = st_1.segment_sent(&segment) {
452                    tracing::debug!(%error, "outgoing segment state update failed");
453                }
454                future::ok::<_, futures::channel::mpsc::SendError>(SessionMessage::<C>::Segment(segment))
455            })
456            // Datagram mode is stateless-only (see `SessionSocketConfig::datagram`): a stateful
457            // socket's NACK missing-segment bitmap addresses only the first 8 segments of a frame,
458            // so an oversized datagram frame (9-63 segments) could not be recovered. Force it off
459            // here regardless of `cfg`, so the invariant holds even if a caller mis-sets the flag.
460            .segmenter_with_terminating_segment::<C>(frame_size, false);
461
462        // We have to merge the streams here and spawn a special task for it
463        // Since the control messages from the State can come independent of Upstream writes.
464        hopr_utils::runtime::prelude::spawn(
465            (ctl_rx, segments_rx)
466                .merge()
467                .map(move |msg| {
468                    #[cfg(feature = "telemetry")]
469                    s0.outgoing_message(msg.discriminant());
470                    Ok(msg)
471                })
472                .forward(packets_out)
473                .map(move |result| match result {
474                    Ok(_) => tracing::debug!("outgoing packet processing done"),
475                    Err(error) => {
476                        tracing::error!(%error, "error while processing outgoing packets")
477                    }
478                })
479                .instrument(tracing::debug_span!("SessionSocket::packets_out"))
480                .with_current_subscriber(),
481        );
482
483        let last_emitted_frame = Arc::new(AtomicU32::new(0));
484        let last_emitted_frame_clone = last_emitted_frame.clone();
485
486        let (packets_in_abort_handle, packets_in_abort_reg) = AbortHandle::new_pair();
487
488        // Dropped frames are counted and reported once per window, not logged one by one
489        let loss_report = Arc::new(FrameLossReport::with_signal(
490            state.session_id(),
491            FRAME_LOSS_REPORT_INTERVAL,
492            loss_signal,
493        ));
494        let (lr1, lr2, lr3) = (loss_report.clone(), loss_report.clone(), loss_report);
495
496        // Pipeline OUT: Packets incoming from Downstream
497        let mut st_1 = state.clone();
498        let mut st_2 = state.clone();
499        let mut st_3 = state.clone();
500
501        // Continue receiving packets from downstream, unless we received a terminating frame.
502        // Once the terminating frame is received, the `packets_in_abort_handle` is triggered, terminating the pipeline.
503        let downstream_frames_out = futures::stream::Abortable::new(packets_in, packets_in_abort_reg)
504            // Filter out Session control messages and update the State, pass only Segments onwards
505            .filter_map(move |packet| {
506                let _span = tracing::debug_span!("SessionSocket::packets_in::pre_reassembly").entered();
507                futures::future::ready(match packet {
508                    Ok(packet) => {
509                        if let Err(error) = st_1.incoming_message(&packet) {
510                            tracing::debug!(%error, "incoming message state update failed");
511                        }
512                        #[cfg(feature = "telemetry")]
513                        s1.incoming_message(packet.discriminant());
514
515                        // Filter old frame ids to save space in the Reassembler
516                        packet.try_as_segment().filter(|s| {
517                            let last_emitted_id = last_emitted_frame.load(std::sync::atomic::Ordering::Relaxed);
518                            if s.frame_id <= last_emitted_id {
519                                tracing::trace!(frame_id = s.frame_id, last_emitted_id, "frame already seen");
520                                lr1.duplicate_segment();
521                                false
522                            } else {
523                                lr1.segment_received(s.frame_id);
524                                true
525                            }
526                        })
527                    }
528                    Err(error) => {
529                        if error.is_fatal_header() {
530                            tracing::error!(%error, "invalid session header; terminating session reader");
531                        } else {
532                            tracing::error!(%error, "unparseable packet");
533                        }
534                        #[cfg(feature = "telemetry")]
535                        s1.error();
536                        None
537                    }
538                })
539            })
540            // Reassemble segments into frames
541            .reassembler_with_inspector(cfg.frame_timeout, cfg.capacity, inspector)
542            // Notify State once a frame has been reassembled, discard frames that we could not reassemble
543            .filter_map(move |maybe_frame| {
544                let _span = tracing::debug_span!("SessionSocket::packets_in::pre_sequencing").entered();
545                futures::future::ready(match maybe_frame {
546                    Ok(frame) => {
547                        lr2.frame_reassembled();
548                        if let Err(error) = st_2.frame_complete(frame.frame_id) {
549                            log_state_update_error(&error, "frame complete state update failed");
550                        }
551                        #[cfg(feature = "telemetry")]
552                        s2.frame_completed();
553                        Some(OrderedFrame(frame))
554                    }
555                    Err(error) => {
556                        tracing::debug!(%error, "failed to reassemble frame");
557                        lr2.frame_not_reassembled();
558                        #[cfg(feature = "telemetry")]
559                        s2.incomplete_frame();
560                        None
561                    }
562                })
563            })
564            // Put the frames into the correct sequence by Frame Ids
565            .sequencer_with(crate::processing::SequencerConfig {
566                max_wait: cfg.frame_timeout,
567                capacity: cfg.capacity,
568                max_item_age: cfg.max_frame_age,
569                max_frames_behind_gap: cfg.max_frames_behind_gap,
570                // A retransmission can still fill the gap, so it is waited out in full.
571                adapt_to_reordering: false,
572            })
573            // Discard frames missing from the sequence and
574            // notify the State about emitted or discarded frames
575            .filter_map(move |maybe_frame| {
576                let _span = tracing::debug_span!("SessionSocket::packets_in::post_sequencing").entered();
577                // Filter out discarded Frames and dispatch events to the State if needed
578                future::ready(match maybe_frame {
579                    Ok(frame) => {
580                        if let Err(error) = st_3.frame_emitted(frame.0.frame_id) {
581                            log_state_update_error(&error, "frame received state update failed");
582                        }
583                        last_emitted_frame_clone.store(frame.0.frame_id, std::sync::atomic::Ordering::Relaxed);
584                        if frame.0.is_terminating {
585                            tracing::debug!("terminating frame received");
586                            packets_in_abort_handle.abort();
587                        }
588                        lr3.frame_emitted();
589                        #[cfg(feature = "telemetry")]
590                        s3.frame_emitted();
591                        Some(Ok(frame.0))
592                    }
593                    Err(SessionError::FrameDiscarded(frame_id)) | Err(SessionError::IncompleteFrame(frame_id)) => {
594                        tracing::debug!(frame_id, "frame discarded");
595                        lr3.frame_discarded();
596                        if let Err(error) = st_3.frame_discarded(frame_id) {
597                            log_state_update_error(&error, "frame discarded state update failed");
598                        }
599                        #[cfg(feature = "telemetry")]
600                        s3.frame_discarded();
601                        None // Downstream skips discarded frames
602                    }
603                    Err(err) => {
604                        #[cfg(feature = "telemetry")]
605                        s3.error();
606                        Some(Err(std::io::Error::other(err)))
607                    }
608                })
609            })
610            .into_async_read();
611
612        Ok(Self {
613            state,
614            span: span.clone(),
615            upstream_frames_in: Box::pin(upstream_frames_in),
616            downstream_frames_out: Box::pin(downstream_frames_out),
617            write_state: if cfg.flush_immediately {
618                WriteState::Writing
619            } else {
620                WriteState::WriteOnly
621            },
622        })
623    }
624}
625
626impl<const C: usize, S: SocketState<C> + Clone + 'static> futures::io::AsyncRead for SessionSocket<C, S> {
627    fn poll_read(self: Pin<&mut Self>, cx: &mut Context<'_>, buf: &mut [u8]) -> Poll<std::io::Result<usize>> {
628        let this = self.project();
629        let _entered = this.span.enter();
630        let _poll = tracing::trace_span!("SessionSocket::poll_read", len = buf.len()).entered();
631        this.downstream_frames_out.as_mut().poll_read(cx, buf)
632    }
633}
634
635impl<const C: usize, S: SocketState<C> + Clone + 'static> futures::io::AsyncWrite for SessionSocket<C, S> {
636    fn poll_write(self: Pin<&mut Self>, cx: &mut Context<'_>, buf: &[u8]) -> Poll<std::io::Result<usize>> {
637        let this = self.project();
638        let _entered = this.span.enter();
639        let _poll = tracing::trace_span!("SessionSocket::poll_write", len = buf.len()).entered();
640        loop {
641            match this.write_state {
642                WriteState::WriteOnly => {
643                    return this.upstream_frames_in.as_mut().poll_write(cx, buf);
644                }
645                WriteState::Writing => {
646                    let len = futures::ready!(this.upstream_frames_in.as_mut().poll_write(cx, buf))?;
647                    *this.write_state = WriteState::Flushing(len);
648                }
649                WriteState::Flushing(len) => {
650                    let res = futures::ready!(this.upstream_frames_in.as_mut().poll_flush(cx)).map(|_| *len);
651                    *this.write_state = WriteState::Writing;
652                    return Poll::Ready(res);
653                }
654            }
655        }
656    }
657
658    fn poll_flush(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<std::io::Result<()>> {
659        let this = self.project();
660        let _entered = this.span.enter();
661        let _poll = tracing::trace_span!("SessionSocket::poll_flush").entered();
662        this.upstream_frames_in.as_mut().poll_flush(cx)
663    }
664
665    fn poll_close(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<std::io::Result<()>> {
666        let this = self.project();
667        let _entered = this.span.enter();
668        let _poll = tracing::trace_span!("SessionSocket::poll_close").entered();
669        if let Err(error) = this.state.stop() {
670            tracing::debug!(%error, "state.stop on SessionSocket close failed");
671        }
672        this.upstream_frames_in.as_mut().poll_close(cx)
673    }
674}
675
676#[cfg(feature = "runtime-tokio")]
677impl<const C: usize, S: SocketState<C> + Clone + 'static> tokio::io::AsyncRead for SessionSocket<C, S> {
678    fn poll_read(
679        mut self: Pin<&mut Self>,
680        cx: &mut Context<'_>,
681        buf: &mut tokio::io::ReadBuf<'_>,
682    ) -> Poll<std::io::Result<()>> {
683        let slice = buf.initialize_unfilled();
684        let n = std::task::ready!(futures::AsyncRead::poll_read(self.as_mut(), cx, slice))?;
685        buf.advance(n);
686        Poll::Ready(Ok(()))
687    }
688}
689
690#[cfg(feature = "runtime-tokio")]
691impl<const C: usize, S: SocketState<C> + Clone + 'static> tokio::io::AsyncWrite for SessionSocket<C, S> {
692    fn poll_write(mut self: Pin<&mut Self>, cx: &mut Context<'_>, buf: &[u8]) -> Poll<Result<usize, std::io::Error>> {
693        futures::AsyncWrite::poll_write(self.as_mut(), cx, buf)
694    }
695
696    fn poll_flush(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Result<(), std::io::Error>> {
697        futures::AsyncWrite::poll_flush(self.as_mut(), cx)
698    }
699
700    fn poll_shutdown(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Result<(), std::io::Error>> {
701        futures::AsyncWrite::poll_close(self.as_mut(), cx)
702    }
703}
704
705#[cfg(test)]
706mod tests {
707    use std::collections::HashSet;
708
709    use futures::{AsyncReadExt, AsyncWriteExt};
710    use futures_time::future::FutureExt;
711    use hopr_crypto_packet::prelude::HoprPacket;
712
713    use super::*;
714    #[cfg(feature = "telemetry")]
715    use crate::socket::telemetry::{NoopTracker, tests::TestTelemetryTracker};
716    use crate::{AcknowledgementState, AcknowledgementStateConfig, utils::test::*};
717
718    const MTU: usize = HoprPacket::PAYLOAD_SIZE;
719
720    /// The frame size production actually feeds a Session: [`MAX_SESSION_MTU`].
721    ///
722    /// `MTU` here is the real HOPR packet payload, so this is exactly one segment per frame and is
723    /// therefore also the real frame boundary — which is what the expectations below are about (how
724    /// much a lost frame costs).
725    const FRAME_SIZE: usize = MAX_SESSION_MTU;
726
727    const DATA_SIZE: usize = 17 * MTU + 271; // Use some size not directly divisible by the MTU
728
729    #[test]
730    fn state_update_errors_are_debug_only_when_the_session_is_closing() {
731        use test_log::tracing_subscriber::{Layer, layer::SubscriberExt};
732
733        struct LogLevels(std::sync::Arc<parking_lot::Mutex<Vec<tracing::Level>>>);
734
735        impl<S: tracing::Subscriber> Layer<S> for LogLevels {
736            fn on_event(&self, event: &tracing::Event<'_>, _: test_log::tracing_subscriber::layer::Context<'_, S>) {
737                self.0.lock().push(*event.metadata().level());
738            }
739        }
740
741        let levels = std::sync::Arc::new(parking_lot::Mutex::new(Vec::new()));
742        let subscriber = test_log::tracing_subscriber::registry().with(LogLevels(levels.clone()));
743        tracing::subscriber::with_default(subscriber, || {
744            log_state_update_error(&SessionError::StateNotRunning, "frame complete state update failed");
745            log_state_update_error(&SessionError::InvalidFrameId, "frame complete state update failed");
746        });
747        assert_eq!(*levels.lock(), vec![tracing::Level::DEBUG, tracing::Level::ERROR]);
748    }
749
750    #[tokio::test]
751    async fn malformed_messages_should_close_only_when_the_boundary_is_unknown() -> anyhow::Result<()> {
752        use tokio_util::compat::TokioAsyncReadCompatExt;
753
754        use crate::protocol::{Segment, SeqIndicator};
755
756        let acknowledgement = Vec::from(SessionMessage::<MTU>::Acknowledge(vec![1].try_into()?));
757        let expected = b"next valid segment";
758        let segment = Vec::from(SessionMessage::<MTU>::Segment(Segment {
759            frame_id: 1,
760            seq_idx: 0,
761            seq_flags: SeqIndicator::new_with_flags(1, false),
762            data: expected.to_vec().into_boxed_slice(),
763        }));
764        for stateless in [false, true] {
765            for (prefix, terminal) in [
766                (vec![0xee], true),
767                (vec![SessionMessage::<MTU>::VERSION, 0, 0xff, 0xff], true),
768                (vec![SessionMessage::<MTU>::VERSION, 0xff, 0, 2, 0xaa, 0xbb], false),
769                (vec![SessionMessage::<MTU>::VERSION, 0, 0, 1, 0xaa], false),
770            ] {
771                let (transport, mut remote) = tokio::io::duplex(8192);
772                let mut socket = if stateless {
773                    SessionSocket::new_stateless(
774                        "malformed",
775                        transport.compat(),
776                        SessionSocketConfig::default(),
777                        None,
778                        #[cfg(feature = "telemetry")]
779                        NoopTracker,
780                    )?
781                } else {
782                    SessionSocket::new(
783                        transport.compat(),
784                        Stateless::<MTU>::new("malformed"),
785                        SessionSocketConfig::default(),
786                        None,
787                        #[cfg(feature = "telemetry")]
788                        NoopTracker,
789                    )?
790                };
791                let mut data = prefix;
792                data.extend_from_slice(&acknowledgement);
793                data.extend_from_slice(&segment);
794                tokio::io::AsyncWriteExt::write_all(&mut remote, &data).await?;
795                let mut output = [0; 64];
796                let read = tokio::time::timeout(Duration::from_secs(2), socket.read(&mut output)).await??;
797                if terminal {
798                    assert_eq!(read, 0, "fatal header must terminate the session read direction");
799                    assert_eq!(socket.read(&mut output).await?, 0, "termination is permanent");
800                } else {
801                    assert_eq!(
802                        &output[..read],
803                        expected,
804                        "a known boundary must preserve the next message"
805                    );
806                }
807                // Keep `remote` alive until after the reads: transport EOF must not be required.
808                drop(remote);
809            }
810        }
811        Ok(())
812    }
813
814    #[test_log::test(tokio::test)]
815    async fn closing_the_socket_should_drain_outgoing_segments_and_close_the_transport() -> anyhow::Result<()> {
816        use crate::protocol::{Segment, SeqIndicator};
817
818        let (alice, bob) = setup_alice_bob::<MTU>(FaultyNetworkConfig::default(), None, None);
819        let mut socket = SessionSocket::new(
820            alice,
821            Stateless::<MTU>::new("alice"),
822            SessionSocketConfig::default(),
823            None,
824            #[cfg(feature = "telemetry")]
825            NoopTracker,
826        )?;
827        let expected = b"queued before closing";
828        socket.write_all(expected).await?;
829        socket.flush().await?;
830        socket.close().await?;
831
832        // Wait for transport EOF, not just the terminating segment: the detached outgoing
833        // task must finish forwarding and close its sink before this test's runtime exits.
834        let messages =
835            futures::TryStreamExt::try_collect::<Vec<_>>(asynchronous_codec::FramedRead::new(bob, SessionCodec::<MTU>))
836                .timeout(futures_time::time::Duration::from_secs(5))
837                .await??;
838        assert_eq!(
839            messages,
840            vec![
841                SessionMessage::Segment(Segment {
842                    frame_id: 1,
843                    seq_idx: 0,
844                    seq_flags: SeqIndicator::new_with_flags(1, false),
845                    data: expected.to_vec().into_boxed_slice(),
846                }),
847                SessionMessage::Segment(Segment::terminating(2)),
848            ],
849            "all queued data must precede the terminating segment, with no control messages"
850        );
851        Ok(())
852    }
853
854    #[test_log::test(tokio::test)]
855    async fn session_socket_should_preserve_datagram_frame_size() -> anyhow::Result<()> {
856        for (frame_size, datagram_size) in [
857            (SessionSocketConfig::default().frame_size, 1452),
858            (1456, 1456),
859            (1500, 1500),
860            (2800, 2800),
861        ] {
862            let (alice, bob) = setup_alice_bob::<MTU>(FaultyNetworkConfig::default(), None, None);
863            let cfg = SessionSocketConfig {
864                frame_size,
865                ..Default::default()
866            };
867            let mut sender = SessionSocket::<MTU, _>::new_stateless(
868                "alice",
869                alice,
870                cfg,
871                None,
872                #[cfg(feature = "telemetry")]
873                NoopTracker,
874            )?;
875            let mut receiver = SessionSocket::<MTU, _>::new_stateless(
876                "bob",
877                bob,
878                cfg,
879                None,
880                #[cfg(feature = "telemetry")]
881                NoopTracker,
882            )?;
883            let data = vec![7u8; datagram_size];
884            sender.write_all(&data).await?;
885            sender.flush().await?;
886
887            // UDP forwarding turns each read into a datagram: read_exact would hide a split frame.
888            let mut received = vec![0u8; datagram_size + 1];
889            let n = receiver
890                .read(&mut received)
891                .timeout(futures_time::time::Duration::from_secs(2))
892                .await??;
893            assert_eq!(n, datagram_size, "frame_size={frame_size}");
894            assert_eq!(&received[..n], data);
895        }
896        Ok(())
897    }
898
899    #[test_log::test(tokio::test)]
900    async fn stateless_socket_unidirectional_should_work() -> anyhow::Result<()> {
901        let (alice, bob) = setup_alice_bob::<MTU>(FaultyNetworkConfig::default(), None, None);
902
903        let sock_cfg = SessionSocketConfig {
904            frame_size: FRAME_SIZE,
905            ..Default::default()
906        };
907
908        #[cfg(feature = "telemetry")]
909        let (alice_tracker, bob_tracker) = (TestTelemetryTracker::default(), TestTelemetryTracker::default());
910
911        let mut alice_socket = SessionSocket::<MTU, _>::new_stateless(
912            "alice",
913            alice,
914            sock_cfg,
915            None,
916            #[cfg(feature = "telemetry")]
917            alice_tracker.clone(),
918        )?;
919        let mut bob_socket = SessionSocket::<MTU, _>::new_stateless(
920            "bob",
921            bob,
922            sock_cfg,
923            None,
924            #[cfg(feature = "telemetry")]
925            bob_tracker.clone(),
926        )?;
927
928        let data = hopr_types::crypto_random::random_bytes::<DATA_SIZE>();
929
930        alice_socket
931            .write_all(&data)
932            .timeout(futures_time::time::Duration::from_secs(2))
933            .await??;
934        alice_socket.flush().await?;
935
936        let mut bob_data = [0u8; DATA_SIZE];
937        bob_socket
938            .read_exact(&mut bob_data)
939            .timeout(futures_time::time::Duration::from_secs(2))
940            .await??;
941        assert_eq!(data, bob_data);
942
943        alice_socket.close().await?;
944        bob_socket.close().await?;
945
946        #[cfg(feature = "telemetry")]
947        {
948            insta::assert_yaml_snapshot!(alice_tracker);
949            insta::assert_yaml_snapshot!(bob_tracker);
950        }
951
952        Ok(())
953    }
954
955    #[test_log::test(tokio::test)]
956    async fn stateful_socket_unidirectional_should_work() -> anyhow::Result<()> {
957        let (alice, bob) = setup_alice_bob::<MTU>(FaultyNetworkConfig::default(), None, None);
958
959        let sock_cfg = SessionSocketConfig {
960            frame_size: FRAME_SIZE,
961            ..Default::default()
962        };
963
964        let ack_cfg = AcknowledgementStateConfig {
965            expected_packet_latency: Duration::from_millis(2),
966            acknowledgement_delay: Duration::from_millis(5),
967            ..Default::default()
968        };
969
970        let mut alice_socket = SessionSocket::<MTU, _>::new(
971            alice,
972            AcknowledgementState::new("alice", ack_cfg),
973            sock_cfg,
974            None,
975            #[cfg(feature = "telemetry")]
976            NoopTracker,
977        )?;
978        let mut bob_socket = SessionSocket::<MTU, _>::new(
979            bob,
980            AcknowledgementState::new("bob", ack_cfg),
981            sock_cfg,
982            None,
983            #[cfg(feature = "telemetry")]
984            NoopTracker,
985        )?;
986
987        let data = hopr_types::crypto_random::random_bytes::<DATA_SIZE>();
988
989        alice_socket
990            .write_all(&data)
991            .timeout(futures_time::time::Duration::from_secs(2))
992            .await??;
993        alice_socket.flush().await?;
994
995        let mut bob_data = [0u8; DATA_SIZE];
996        bob_socket
997            .read_exact(&mut bob_data)
998            .timeout(futures_time::time::Duration::from_secs(2))
999            .await??;
1000        assert_eq!(data, bob_data);
1001
1002        alice_socket.close().await?;
1003        bob_socket.close().await?;
1004
1005        Ok(())
1006    }
1007
1008    #[test_log::test(tokio::test)]
1009    async fn stateless_socket_bidirectional_should_work() -> anyhow::Result<()> {
1010        let (alice, bob) = setup_alice_bob::<MTU>(FaultyNetworkConfig::default(), None, None);
1011
1012        let sock_cfg = SessionSocketConfig {
1013            frame_size: FRAME_SIZE,
1014            ..Default::default()
1015        };
1016
1017        #[cfg(feature = "telemetry")]
1018        let (alice_tracker, bob_tracker) = (TestTelemetryTracker::default(), TestTelemetryTracker::default());
1019
1020        let mut alice_socket = SessionSocket::<MTU, _>::new_stateless(
1021            "alice",
1022            alice,
1023            sock_cfg,
1024            None,
1025            #[cfg(feature = "telemetry")]
1026            alice_tracker.clone(),
1027        )?;
1028        let mut bob_socket = SessionSocket::<MTU, _>::new_stateless(
1029            "bob",
1030            bob,
1031            sock_cfg,
1032            None,
1033            #[cfg(feature = "telemetry")]
1034            bob_tracker.clone(),
1035        )?;
1036
1037        let alice_sent_data = hopr_types::crypto_random::random_bytes::<DATA_SIZE>();
1038        alice_socket
1039            .write_all(&alice_sent_data)
1040            .timeout(futures_time::time::Duration::from_secs(2))
1041            .await??;
1042        alice_socket.flush().await?;
1043
1044        let bob_sent_data = hopr_types::crypto_random::random_bytes::<DATA_SIZE>();
1045        bob_socket
1046            .write_all(&bob_sent_data)
1047            .timeout(futures_time::time::Duration::from_secs(2))
1048            .await??;
1049        bob_socket.flush().await?;
1050
1051        let mut bob_recv_data = [0u8; DATA_SIZE];
1052        bob_socket
1053            .read_exact(&mut bob_recv_data)
1054            .timeout(futures_time::time::Duration::from_secs(2))
1055            .await??;
1056        assert_eq!(alice_sent_data, bob_recv_data);
1057
1058        let mut alice_recv_data = [0u8; DATA_SIZE];
1059        alice_socket
1060            .read_exact(&mut alice_recv_data)
1061            .timeout(futures_time::time::Duration::from_secs(2))
1062            .await??;
1063        assert_eq!(bob_sent_data, alice_recv_data);
1064
1065        #[cfg(feature = "telemetry")]
1066        {
1067            insta::assert_yaml_snapshot!(alice_tracker);
1068            insta::assert_yaml_snapshot!(bob_tracker);
1069        }
1070
1071        Ok(())
1072    }
1073
1074    #[test_log::test(tokio::test)]
1075    async fn stateful_socket_bidirectional_should_work() -> anyhow::Result<()> {
1076        let (alice, bob) = setup_alice_bob::<MTU>(FaultyNetworkConfig::default(), None, None);
1077
1078        // use hopr_network_types::capture::PcapIoExt;
1079        // let (alice, bob) = (alice.capture("alice.pcap"), bob.capture("bob.pcap"));
1080
1081        let sock_cfg = SessionSocketConfig {
1082            frame_size: FRAME_SIZE,
1083            ..Default::default()
1084        };
1085
1086        let ack_cfg = AcknowledgementStateConfig {
1087            expected_packet_latency: Duration::from_millis(2),
1088            acknowledgement_delay: Duration::from_millis(10),
1089            ..Default::default()
1090        };
1091
1092        let mut alice_socket = SessionSocket::<MTU, _>::new(
1093            alice,
1094            AcknowledgementState::new("alice", ack_cfg),
1095            sock_cfg,
1096            None,
1097            #[cfg(feature = "telemetry")]
1098            NoopTracker,
1099        )?;
1100        let mut bob_socket = SessionSocket::<MTU, _>::new(
1101            bob,
1102            AcknowledgementState::new("bob", ack_cfg),
1103            sock_cfg,
1104            None,
1105            #[cfg(feature = "telemetry")]
1106            NoopTracker,
1107        )?;
1108
1109        let alice_sent_data = hopr_types::crypto_random::random_bytes::<DATA_SIZE>();
1110        alice_socket
1111            .write_all(&alice_sent_data)
1112            .timeout(futures_time::time::Duration::from_secs(2))
1113            .await??;
1114        alice_socket.flush().await?;
1115
1116        let bob_sent_data = hopr_types::crypto_random::random_bytes::<DATA_SIZE>();
1117        bob_socket
1118            .write_all(&bob_sent_data)
1119            .timeout(futures_time::time::Duration::from_secs(2))
1120            .await??;
1121        bob_socket.flush().await?;
1122
1123        let mut bob_recv_data = [0u8; DATA_SIZE];
1124        bob_socket
1125            .read_exact(&mut bob_recv_data)
1126            .timeout(futures_time::time::Duration::from_secs(2))
1127            .await??;
1128        assert_eq!(alice_sent_data, bob_recv_data);
1129
1130        let mut alice_recv_data = [0u8; DATA_SIZE];
1131        alice_socket
1132            .read_exact(&mut alice_recv_data)
1133            .timeout(futures_time::time::Duration::from_secs(2))
1134            .await??;
1135        assert_eq!(bob_sent_data, alice_recv_data);
1136
1137        Ok(())
1138    }
1139
1140    #[test_log::test(tokio::test)]
1141    async fn stateless_socket_unidirectional_should_work_with_mixing() -> anyhow::Result<()> {
1142        let network_cfg = FaultyNetworkConfig {
1143            mixing_factor: 10,
1144            ..Default::default()
1145        };
1146
1147        let (alice, bob) = setup_alice_bob::<MTU>(network_cfg, None, None);
1148
1149        let sock_cfg = SessionSocketConfig {
1150            frame_size: FRAME_SIZE,
1151            ..Default::default()
1152        };
1153
1154        let mut alice_socket = SessionSocket::<MTU, _>::new_stateless(
1155            "alice",
1156            alice,
1157            sock_cfg,
1158            None,
1159            #[cfg(feature = "telemetry")]
1160            NoopTracker,
1161        )?;
1162        let mut bob_socket = SessionSocket::<MTU, _>::new_stateless(
1163            "bob",
1164            bob,
1165            sock_cfg,
1166            None,
1167            #[cfg(feature = "telemetry")]
1168            NoopTracker,
1169        )?;
1170
1171        let data = hopr_types::crypto_random::random_bytes::<DATA_SIZE>();
1172        alice_socket
1173            .write_all(&data)
1174            .timeout(futures_time::time::Duration::from_secs(2))
1175            .await??;
1176        alice_socket.flush().await?;
1177
1178        let mut bob_recv_data = [0u8; DATA_SIZE];
1179        bob_socket
1180            .read_exact(&mut bob_recv_data)
1181            .timeout(futures_time::time::Duration::from_secs(2))
1182            .await??;
1183        assert_eq!(data, bob_recv_data);
1184
1185        alice_socket.close().await?;
1186        bob_socket.close().await?;
1187
1188        Ok(())
1189    }
1190
1191    #[test_log::test(tokio::test)]
1192    async fn stateful_socket_unidirectional_should_work_with_mixing() -> anyhow::Result<()> {
1193        let network_cfg = FaultyNetworkConfig {
1194            mixing_factor: 10,
1195            ..Default::default()
1196        };
1197
1198        let (alice, bob) = setup_alice_bob::<MTU>(network_cfg, None, None);
1199
1200        let sock_cfg = SessionSocketConfig {
1201            frame_size: FRAME_SIZE,
1202            ..Default::default()
1203        };
1204
1205        let ack_cfg = AcknowledgementStateConfig {
1206            expected_packet_latency: Duration::from_millis(2),
1207            acknowledgement_delay: Duration::from_millis(5),
1208            ..Default::default()
1209        };
1210
1211        let mut alice_socket = SessionSocket::<MTU, _>::new(
1212            alice,
1213            AcknowledgementState::new("alice", ack_cfg),
1214            sock_cfg,
1215            None,
1216            #[cfg(feature = "telemetry")]
1217            NoopTracker,
1218        )?;
1219        let mut bob_socket = SessionSocket::<MTU, _>::new(
1220            bob,
1221            AcknowledgementState::new("bob", ack_cfg),
1222            sock_cfg,
1223            None,
1224            #[cfg(feature = "telemetry")]
1225            NoopTracker,
1226        )?;
1227
1228        let data = hopr_types::crypto_random::random_bytes::<DATA_SIZE>();
1229        alice_socket
1230            .write_all(&data)
1231            .timeout(futures_time::time::Duration::from_secs(2))
1232            .await??;
1233        alice_socket.flush().await?;
1234
1235        let mut bob_recv_data = [0u8; DATA_SIZE];
1236        bob_socket
1237            .read_exact(&mut bob_recv_data)
1238            .timeout(futures_time::time::Duration::from_secs(2))
1239            .await??;
1240        assert_eq!(data, bob_recv_data);
1241
1242        alice_socket.close().await?;
1243        bob_socket.close().await?;
1244
1245        Ok(())
1246    }
1247
1248    #[test_log::test(tokio::test)]
1249    async fn stateless_socket_bidirectional_should_work_with_mixing() -> anyhow::Result<()> {
1250        let network_cfg = FaultyNetworkConfig {
1251            mixing_factor: 10,
1252            ..Default::default()
1253        };
1254
1255        let (alice, bob) = setup_alice_bob::<MTU>(network_cfg, None, None);
1256
1257        let sock_cfg = SessionSocketConfig {
1258            frame_size: FRAME_SIZE,
1259            ..Default::default()
1260        };
1261
1262        let mut alice_socket = SessionSocket::<MTU, _>::new_stateless(
1263            "alice",
1264            alice,
1265            sock_cfg,
1266            None,
1267            #[cfg(feature = "telemetry")]
1268            NoopTracker,
1269        )?;
1270        let mut bob_socket = SessionSocket::<MTU, _>::new_stateless(
1271            "bob",
1272            bob,
1273            sock_cfg,
1274            None,
1275            #[cfg(feature = "telemetry")]
1276            NoopTracker,
1277        )?;
1278
1279        let alice_sent_data = hopr_types::crypto_random::random_bytes::<DATA_SIZE>();
1280        alice_socket
1281            .write_all(&alice_sent_data)
1282            .timeout(futures_time::time::Duration::from_secs(2))
1283            .await??;
1284        alice_socket.flush().await?;
1285
1286        let bob_sent_data = hopr_types::crypto_random::random_bytes::<DATA_SIZE>();
1287        bob_socket
1288            .write_all(&bob_sent_data)
1289            .timeout(futures_time::time::Duration::from_secs(2))
1290            .await??;
1291        bob_socket.flush().await?;
1292
1293        let mut bob_recv_data = [0u8; DATA_SIZE];
1294        bob_socket
1295            .read_exact(&mut bob_recv_data)
1296            .timeout(futures_time::time::Duration::from_secs(2))
1297            .await??;
1298        assert_eq!(alice_sent_data, bob_recv_data);
1299
1300        let mut alice_recv_data = [0u8; DATA_SIZE];
1301        alice_socket
1302            .read_exact(&mut alice_recv_data)
1303            .timeout(futures_time::time::Duration::from_secs(2))
1304            .await??;
1305        assert_eq!(bob_sent_data, alice_recv_data);
1306
1307        alice_socket.close().await?;
1308        bob_socket.close().await?;
1309
1310        Ok(())
1311    }
1312
1313    #[test_log::test(tokio::test)]
1314    async fn stateful_socket_bidirectional_should_work_with_mixing() -> anyhow::Result<()> {
1315        let network_cfg = FaultyNetworkConfig {
1316            mixing_factor: 10,
1317            ..Default::default()
1318        };
1319
1320        let (alice, bob) = setup_alice_bob::<MTU>(network_cfg, None, None);
1321
1322        let sock_cfg = SessionSocketConfig {
1323            frame_size: FRAME_SIZE,
1324            ..Default::default()
1325        };
1326
1327        let ack_cfg = AcknowledgementStateConfig {
1328            expected_packet_latency: Duration::from_millis(2),
1329            acknowledgement_delay: Duration::from_millis(5),
1330            ..Default::default()
1331        };
1332
1333        let mut alice_socket = SessionSocket::<MTU, _>::new(
1334            alice,
1335            AcknowledgementState::new("alice", ack_cfg),
1336            sock_cfg,
1337            None,
1338            #[cfg(feature = "telemetry")]
1339            NoopTracker,
1340        )?;
1341        let mut bob_socket = SessionSocket::<MTU, _>::new(
1342            bob,
1343            AcknowledgementState::new("bob", ack_cfg),
1344            sock_cfg,
1345            None,
1346            #[cfg(feature = "telemetry")]
1347            NoopTracker,
1348        )?;
1349
1350        let alice_sent_data = hopr_types::crypto_random::random_bytes::<DATA_SIZE>();
1351        alice_socket
1352            .write_all(&alice_sent_data)
1353            .timeout(futures_time::time::Duration::from_secs(2))
1354            .await??;
1355        alice_socket.flush().await?;
1356
1357        let bob_sent_data = hopr_types::crypto_random::random_bytes::<DATA_SIZE>();
1358        bob_socket
1359            .write_all(&bob_sent_data)
1360            .timeout(futures_time::time::Duration::from_secs(2))
1361            .await??;
1362        bob_socket.flush().await?;
1363
1364        let mut bob_recv_data = [0u8; DATA_SIZE];
1365        bob_socket
1366            .read_exact(&mut bob_recv_data)
1367            .timeout(futures_time::time::Duration::from_secs(2))
1368            .await??;
1369        assert_eq!(alice_sent_data, bob_recv_data);
1370
1371        let mut alice_recv_data = [0u8; DATA_SIZE];
1372        alice_socket
1373            .read_exact(&mut alice_recv_data)
1374            .timeout(futures_time::time::Duration::from_secs(2))
1375            .await??;
1376        assert_eq!(bob_sent_data, alice_recv_data);
1377
1378        alice_socket.close().await?;
1379        bob_socket.close().await?;
1380
1381        Ok(())
1382    }
1383
1384    #[test_log::test(tokio::test)]
1385    async fn stateless_socket_unidirectional_should_should_skip_missing_frames() -> anyhow::Result<()> {
1386        let (alice, bob) = setup_alice_bob::<MTU>(
1387            FaultyNetworkConfig {
1388                avg_delay: Duration::from_millis(10),
1389                ids_to_drop: HashSet::from_iter([0_usize]),
1390                ..Default::default()
1391            },
1392            None,
1393            None,
1394        );
1395
1396        let alice_cfg = SessionSocketConfig {
1397            frame_size: FRAME_SIZE,
1398            ..Default::default()
1399        };
1400
1401        let bob_cfg = SessionSocketConfig {
1402            frame_size: FRAME_SIZE,
1403            frame_timeout: Duration::from_millis(55),
1404            ..Default::default()
1405        };
1406
1407        #[cfg(feature = "telemetry")]
1408        let (alice_tracker, bob_tracker) = (TestTelemetryTracker::default(), TestTelemetryTracker::default());
1409
1410        let mut alice_socket = SessionSocket::<MTU, _>::new_stateless(
1411            "alice",
1412            alice,
1413            alice_cfg,
1414            None,
1415            #[cfg(feature = "telemetry")]
1416            alice_tracker.clone(),
1417        )?;
1418        let mut bob_socket = SessionSocket::<MTU, _>::new_stateless(
1419            "bob",
1420            bob,
1421            bob_cfg,
1422            None,
1423            #[cfg(feature = "telemetry")]
1424            bob_tracker.clone(),
1425        )?;
1426
1427        let data = hopr_types::crypto_random::random_bytes::<DATA_SIZE>();
1428        alice_socket
1429            .write_all(&data)
1430            .timeout(futures_time::time::Duration::from_secs(2))
1431            .await??;
1432        alice_socket.flush().await?;
1433        alice_socket.close().await?;
1434
1435        let mut bob_data = Vec::with_capacity(DATA_SIZE);
1436        bob_socket
1437            .read_to_end(&mut bob_data)
1438            .timeout(futures_time::time::Duration::from_secs(2))
1439            .await??;
1440
1441        // The whole first frame is discarded due to the missing first segment. A frame is a whole
1442        // number of segments, so the data lost is exactly the frame size.
1443        let segment_payload = FRAME_SIZE;
1444        assert_eq!(data.len() - segment_payload, bob_data.len());
1445        assert_eq!(&data[segment_payload..], &bob_data);
1446
1447        bob_socket.close().await?;
1448
1449        #[cfg(feature = "telemetry")]
1450        {
1451            insta::assert_yaml_snapshot!(alice_tracker);
1452            insta::assert_yaml_snapshot!(bob_tracker);
1453        }
1454
1455        Ok(())
1456    }
1457
1458    /// A sequencer discard is recorded into the loss signal, on both socket kinds.
1459    ///
1460    /// The network drops the one segment of frame 1, so that frame never arrives at all and the
1461    /// sequencer gives up on the gap once later frames have arrived behind it (`max_frames_behind_gap`).
1462    /// That is the discard event. Both kinds run with a state that does not retransmit (`Stateless`),
1463    /// so the lost frame is not recovered and there is no retransmitted duplicate here; the duplicate
1464    /// case has its own test, which feeds one in directly.
1465    #[test_log::test(tokio::test)]
1466    async fn a_sequencer_discard_is_recorded_by_both_socket_kinds() -> anyhow::Result<()> {
1467        for stateless in [true, false] {
1468            let (alice, bob) = setup_alice_bob::<MTU>(
1469                FaultyNetworkConfig {
1470                    avg_delay: Duration::from_millis(10),
1471                    ids_to_drop: HashSet::from_iter([0_usize]),
1472                    ..Default::default()
1473                },
1474                None,
1475                None,
1476            );
1477
1478            let signal = Arc::new(FrameLossSignal::new());
1479            let bob_cfg = SessionSocketConfig {
1480                frame_size: FRAME_SIZE,
1481                frame_timeout: Duration::from_millis(300),
1482                max_frames_behind_gap: Some(1),
1483                ..Default::default()
1484            };
1485            let alice_cfg = SessionSocketConfig {
1486                frame_size: FRAME_SIZE,
1487                ..Default::default()
1488            };
1489
1490            let mut alice_socket = if stateless {
1491                SessionSocket::<MTU, _>::new_stateless(
1492                    "alice",
1493                    alice,
1494                    alice_cfg,
1495                    None,
1496                    #[cfg(feature = "telemetry")]
1497                    NoopTracker,
1498                )?
1499            } else {
1500                SessionSocket::<MTU, _>::new(
1501                    alice,
1502                    Stateless::<MTU>::new("alice"),
1503                    alice_cfg,
1504                    None,
1505                    #[cfg(feature = "telemetry")]
1506                    NoopTracker,
1507                )?
1508            };
1509            let mut bob_socket = if stateless {
1510                SessionSocket::<MTU, _>::new_stateless(
1511                    "bob",
1512                    bob,
1513                    bob_cfg,
1514                    Some(signal.clone()),
1515                    #[cfg(feature = "telemetry")]
1516                    NoopTracker,
1517                )?
1518            } else {
1519                SessionSocket::<MTU, _>::new(
1520                    bob,
1521                    Stateless::<MTU>::new("bob"),
1522                    bob_cfg,
1523                    Some(signal.clone()),
1524                    #[cfg(feature = "telemetry")]
1525                    NoopTracker,
1526                )?
1527            };
1528
1529            assert_eq!(0, signal.lost_total(), "nothing was lost before a frame went out");
1530
1531            let data = hopr_types::crypto_random::random_bytes::<DATA_SIZE>();
1532            alice_socket
1533                .write_all(&data)
1534                .timeout(futures_time::time::Duration::from_secs(2))
1535                .await??;
1536            alice_socket.flush().await?;
1537
1538            // Everything except the lost first frame.
1539            let mut received = vec![0u8; DATA_SIZE - FRAME_SIZE];
1540            bob_socket
1541                .read_exact(&mut received)
1542                .timeout(futures_time::time::Duration::from_secs(20))
1543                .await??;
1544            assert_eq!(&data[FRAME_SIZE..], &received, "stateless={stateless}");
1545
1546            assert!(
1547                signal.lost_within(Duration::from_secs(30)),
1548                "the discarded frame must be recorded as recent loss (stateless={stateless})"
1549            );
1550            assert_eq!(
1551                1,
1552                signal.lost_total(),
1553                "one discarded frame is one loss (stateless={stateless})"
1554            );
1555
1556            alice_socket.close().await?;
1557            bob_socket.close().await?;
1558        }
1559        Ok(())
1560    }
1561
1562    /// One raw segment message, as it arrives off the wire.
1563    fn raw_segment(frame_id: u32, seq_idx: u8, segments: u8, data: &[u8]) -> Vec<u8> {
1564        use crate::protocol::{Segment, SeqIndicator};
1565
1566        Vec::from(SessionMessage::<MTU>::Segment(Segment {
1567            frame_id,
1568            seq_idx,
1569            seq_flags: SeqIndicator::new_with_flags(segments, false),
1570            data: data.to_vec().into_boxed_slice(),
1571        }))
1572    }
1573
1574    /// Opens the receive side of either socket kind over an in-memory transport, recording into `signal`.
1575    ///
1576    /// `reliable` is the real [`ReliableSocket`](crate::ReliableSocket) with an [`AcknowledgementState`], which
1577    /// requests the missing segments of an incomplete frame, not the stateless one.
1578    fn open_raw_receiver(
1579        reliable: bool,
1580        frame_timeout: Duration,
1581        signal: &Arc<FrameLossSignal>,
1582    ) -> anyhow::Result<(Box<dyn futures::io::AsyncRead + Send + Unpin>, tokio::io::DuplexStream)> {
1583        use tokio_util::compat::TokioAsyncReadCompatExt;
1584
1585        let (transport, remote) = tokio::io::duplex(1 << 20);
1586        let cfg = SessionSocketConfig {
1587            frame_size: FRAME_SIZE,
1588            frame_timeout,
1589            ..Default::default()
1590        };
1591        let socket: Box<dyn futures::io::AsyncRead + Send + Unpin> = if reliable {
1592            Box::new(SessionSocket::<MTU, _>::new(
1593                transport.compat(),
1594                AcknowledgementState::<MTU>::new("bob", AcknowledgementStateConfig::default()),
1595                cfg,
1596                Some(signal.clone()),
1597                #[cfg(feature = "telemetry")]
1598                NoopTracker,
1599            )?)
1600        } else {
1601            Box::new(SessionSocket::<MTU, _>::new_stateless(
1602                "bob",
1603                transport.compat(),
1604                cfg,
1605                Some(signal.clone()),
1606                #[cfg(feature = "telemetry")]
1607                NoopTracker,
1608            )?)
1609        };
1610        Ok((socket, remote))
1611    }
1612
1613    /// A frame that expires incomplete is recorded, on the stateless and on the reliable socket alike.
1614    ///
1615    /// Only the first of the frame's two segments ever arrives, so the reassembler gives up on the
1616    /// frame when its timeout runs out. The reliable socket asks for the missing segment meanwhile (into
1617    /// the transport nobody reads), which does not change that it is lost.
1618    #[test_log::test(tokio::test)]
1619    async fn a_frame_that_expires_incomplete_is_recorded_by_both_socket_kinds() -> anyhow::Result<()> {
1620        use tokio::io::AsyncWriteExt;
1621
1622        for reliable in [false, true] {
1623            let signal = Arc::new(FrameLossSignal::new());
1624            let (mut socket, mut remote) = open_raw_receiver(reliable, Duration::from_millis(100), &signal)?;
1625            assert_eq!(0, signal.lost_total(), "nothing lost yet (reliable={reliable})");
1626
1627            remote.write_all(&raw_segment(1, 0, 2, b"first half")).await?;
1628
1629            // Nothing is ever delivered; the read only gives the reassembler time to expire the frame.
1630            let mut buf = [0u8; 64];
1631            let _ = tokio::time::timeout(Duration::from_millis(800), socket.read(&mut buf)).await;
1632
1633            assert!(
1634                signal.lost_within(Duration::from_secs(30)),
1635                "the incomplete frame must be recorded as recent loss (reliable={reliable})"
1636            );
1637            assert_eq!(
1638                1,
1639                signal.lost_total(),
1640                "one incomplete frame is one loss (reliable={reliable})"
1641            );
1642        }
1643        Ok(())
1644    }
1645
1646    /// A duplicate segment is delivery, not loss, on both socket kinds.
1647    ///
1648    /// Frame 1 is delivered whole and read, which makes it the last emitted frame; its segment then
1649    /// arrives a second time, which is the duplicate the socket filters as already seen, and frame 2
1650    /// behind it still comes through. Nothing was lost, so the signal stays empty.
1651    #[test_log::test(tokio::test)]
1652    async fn a_duplicate_segment_is_not_recorded_by_either_socket_kind() -> anyhow::Result<()> {
1653        use tokio::io::AsyncWriteExt;
1654
1655        for reliable in [false, true] {
1656            let signal = Arc::new(FrameLossSignal::new());
1657            let (mut socket, mut remote) = open_raw_receiver(reliable, Duration::from_millis(100), &signal)?;
1658
1659            let first = raw_segment(1, 0, 1, b"frame one");
1660            remote.write_all(&first).await?;
1661            let mut buf = [0u8; 9];
1662            tokio::time::timeout(Duration::from_secs(2), socket.read_exact(&mut buf)).await??;
1663            assert_eq!(b"frame one", &buf, "reliable={reliable}");
1664
1665            // The duplicate: the same segment of a frame that has already been emitted.
1666            remote.write_all(&first).await?;
1667            remote.write_all(&raw_segment(2, 0, 1, b"frame two")).await?;
1668            tokio::time::timeout(Duration::from_secs(2), socket.read_exact(&mut buf)).await??;
1669            assert_eq!(b"frame two", &buf, "reliable={reliable}");
1670
1671            // Give any frame timeout the duplicate might have started a chance to run out.
1672            tokio::time::sleep(Duration::from_millis(300)).await;
1673            assert_eq!(
1674                0,
1675                signal.lost_total(),
1676                "a duplicate is not a loss (reliable={reliable})"
1677            );
1678            assert!(!signal.lost_within(Duration::from_secs(30)), "reliable={reliable}");
1679        }
1680        Ok(())
1681    }
1682
1683    /// Drives the head-of-line case end to end over the real socket pipeline.
1684    ///
1685    /// Segment 0 is dropped, so frame 1 can never be completed, and the sender is deliberately
1686    /// left **open**: closing it would drain the sequencer through `State::Done`, which discards
1687    /// missing frames immediately and would hide the very stall under test. Returns how long the
1688    /// surviving frames took to arrive, and asserts they arrived intact.
1689    async fn time_delivery_behind_a_lost_frame(max_frames_behind_gap: Option<usize>) -> anyhow::Result<Duration> {
1690        // hoprd's production value, chosen to clear the ~2 s SURB KeepAlive.
1691        const FRAME_TIMEOUT: Duration = Duration::from_secs(3);
1692
1693        let (alice, bob) = setup_alice_bob::<MTU>(
1694            FaultyNetworkConfig {
1695                avg_delay: Duration::from_millis(10),
1696                ids_to_drop: HashSet::from_iter([0_usize]),
1697                ..Default::default()
1698            },
1699            None,
1700            None,
1701        );
1702
1703        let mut alice_socket = SessionSocket::<MTU, _>::new_stateless(
1704            "alice",
1705            alice,
1706            SessionSocketConfig {
1707                frame_size: FRAME_SIZE,
1708                ..Default::default()
1709            },
1710            None,
1711            #[cfg(feature = "telemetry")]
1712            TestTelemetryTracker::default(),
1713        )?;
1714        let mut bob_socket = SessionSocket::<MTU, _>::new_stateless(
1715            "bob",
1716            bob,
1717            SessionSocketConfig {
1718                frame_size: FRAME_SIZE,
1719                frame_timeout: FRAME_TIMEOUT,
1720                max_frames_behind_gap,
1721                // The count bound alone: should the network ever reorder, the time bound would
1722                // take its place and the witness below would stop witnessing.
1723                adapt_gap_wait: false,
1724                ..Default::default()
1725            },
1726            None,
1727            #[cfg(feature = "telemetry")]
1728            TestTelemetryTracker::default(),
1729        )?;
1730
1731        let data = hopr_types::crypto_random::random_bytes::<DATA_SIZE>();
1732        alice_socket
1733            .write_all(&data)
1734            .timeout(futures_time::time::Duration::from_secs(2))
1735            .await??;
1736        alice_socket.flush().await?;
1737
1738        // Everything except the lost first frame.
1739        let mut received = vec![0u8; DATA_SIZE - FRAME_SIZE];
1740        let started = std::time::Instant::now();
1741        bob_socket
1742            .read_exact(&mut received)
1743            .timeout(futures_time::time::Duration::from_secs(20))
1744            .await??;
1745        let elapsed = started.elapsed();
1746
1747        assert_eq!(
1748            &data[FRAME_SIZE..],
1749            &received,
1750            "the frames that did arrive must be delivered intact, whenever they are released"
1751        );
1752
1753        alice_socket.close().await?;
1754        bob_socket.close().await?;
1755        Ok(elapsed)
1756    }
1757
1758    /// The fix, at the socket level: frames behind an unfillable gap are released on the evidence
1759    /// that later frames are queued, not after the frame timeout has run its course.
1760    #[test_log::test(tokio::test)]
1761    async fn stateless_socket_should_release_frames_behind_a_gap_without_waiting_for_the_timeout() -> anyhow::Result<()>
1762    {
1763        let elapsed = time_delivery_behind_a_lost_frame(Some(2)).await?;
1764        assert!(
1765            elapsed < Duration::from_secs(1),
1766            "frames already received must not wait on a frame that cannot arrive; took {elapsed:?}"
1767        );
1768        Ok(())
1769    }
1770
1771    /// The witness for the test above: with the bound disabled, the same loss on the same pipeline
1772    /// stalls for the whole frame timeout. Without this, a fast run could be crediting the fix for
1773    /// something the network or the harness was doing anyway.
1774    #[test_log::test(tokio::test)]
1775    async fn stateless_socket_without_the_gap_bound_should_stall_for_the_whole_frame_timeout() -> anyhow::Result<()> {
1776        let elapsed = time_delivery_behind_a_lost_frame(None).await?;
1777        assert!(
1778            elapsed >= Duration::from_secs(3),
1779            "the unbounded path is what the fix removes; it must still be observable here, took {elapsed:?}"
1780        );
1781        Ok(())
1782    }
1783
1784    /// Every 10-byte read of a stateless receiver, with when it completed.
1785    type StatelessReads = tokio::task::JoinHandle<Vec<(std::time::Instant, [u8; 10])>>;
1786
1787    /// Opens a stateless receiver over an in-memory transport, read continuously as a Session's
1788    /// reader does (so each segment is taken in when it arrives): every 10-byte read with when it
1789    /// completed, until the transport closes.
1790    fn stateless_reader(adapt_gap_wait: bool) -> anyhow::Result<(StatelessReads, tokio::io::DuplexStream)> {
1791        use futures::AsyncReadExt;
1792        use tokio_util::compat::TokioAsyncReadCompatExt;
1793
1794        let (transport, remote) = tokio::io::duplex(1 << 20);
1795        let mut socket = SessionSocket::<MTU, _>::new_stateless(
1796            "bob",
1797            transport.compat(),
1798            SessionSocketConfig {
1799                frame_size: FRAME_SIZE,
1800                frame_timeout: Duration::from_secs(3),
1801                adapt_gap_wait,
1802                ..Default::default()
1803            },
1804            None,
1805            #[cfg(feature = "telemetry")]
1806            NoopTracker,
1807        )?;
1808        let reader = tokio::spawn(async move {
1809            let mut buf = [0u8; 10];
1810            let mut reads = Vec::new();
1811            while socket.read_exact(&mut buf).await.is_ok() {
1812                reads.push((std::time::Instant::now(), buf));
1813            }
1814            reads
1815        });
1816        Ok((reader, remote))
1817    }
1818
1819    /// Sends frames `ids` whole, each as two segments `spread` apart, the frames 750 ms apart.
1820    async fn send_whole_frames(
1821        remote: &mut tokio::io::DuplexStream,
1822        ids: std::ops::RangeInclusive<u32>,
1823        spread: Duration,
1824    ) -> anyhow::Result<()> {
1825        use tokio::io::AsyncWriteExt;
1826        for id in ids {
1827            remote.write_all(&raw_segment(id, 0, 2, b"half-")).await?;
1828            tokio::time::sleep(spread).await;
1829            remote.write_all(&raw_segment(id, 1, 2, b"half-")).await?;
1830            tokio::time::sleep(Duration::from_millis(750).saturating_sub(spread)).await;
1831        }
1832        Ok(())
1833    }
1834
1835    /// How long frame `before + 2` waits behind lost frame `before + 1`, after `before` frames whose
1836    /// segments arrived 200 ms apart.
1837    async fn time_held_behind_a_lost_frame_of_several_segments(
1838        adapt_gap_wait: bool,
1839        before: u32,
1840    ) -> anyhow::Result<Duration> {
1841        use tokio::io::AsyncWriteExt;
1842
1843        let (reader, mut remote) = stateless_reader(adapt_gap_wait)?;
1844        send_whole_frames(&mut remote, 1..=before, Duration::from_millis(200)).await?;
1845
1846        let started = std::time::Instant::now();
1847        remote.write_all(&raw_segment(before + 2, 0, 2, b"after")).await?;
1848        remote.write_all(&raw_segment(before + 2, 1, 2, b"-lost")).await?;
1849        tokio::time::sleep(Duration::from_millis(3500)).await;
1850        drop(remote);
1851
1852        let reads = reader.await?;
1853        anyhow::ensure!(reads.len() == before as usize + 1, "{} reads", reads.len());
1854        let (at, last) = reads
1855            .last()
1856            .ok_or_else(|| anyhow::anyhow!("the frame after the lost one"))?;
1857        anyhow::ensure!(last == b"after-lost", "the frame after the lost one");
1858        Ok(at.duration_since(started))
1859    }
1860
1861    /// Frames of several segments rarely arrive out of order, so the sequencer alone would see no
1862    /// reordering to size its wait by. The spread of a frame's own segments is the spread of the
1863    /// paths, and bounds how far frames can arrive apart: 200 ms, so once eight frames over six
1864    /// seconds have shown it, a lost frame holds the next for about 400 ms instead of the whole 3 s
1865    /// frame timeout.
1866    #[test_log::test(tokio::test)]
1867    async fn stateless_socket_should_hold_frames_behind_a_lost_one_for_twice_the_spread_of_its_segments()
1868    -> anyhow::Result<()> {
1869        let held = time_held_behind_a_lost_frame_of_several_segments(true, 8).await?;
1870        assert!(held >= Duration::from_millis(380), "{held:?}");
1871        assert!(held < Duration::from_millis(1500), "{held:?}");
1872        Ok(())
1873    }
1874
1875    /// The witness: with the adaptation off, the same loss holds the next frame for the whole frame
1876    /// timeout.
1877    #[test_log::test(tokio::test)]
1878    async fn stateless_socket_without_the_adaptive_gap_wait_should_hold_frames_for_the_frame_timeout()
1879    -> anyhow::Result<()> {
1880        let held = time_held_behind_a_lost_frame_of_several_segments(false, 8).await?;
1881        assert!(held >= Duration::from_millis(2950), "{held:?}");
1882        Ok(())
1883    }
1884
1885    /// One frame says little about the paths a Session will take: a frame lost right after it still
1886    /// holds the next for the whole frame timeout.
1887    #[test_log::test(tokio::test)]
1888    async fn stateless_socket_should_wait_the_whole_timeout_before_it_has_seen_enough_reordering() -> anyhow::Result<()>
1889    {
1890        let held = time_held_behind_a_lost_frame_of_several_segments(true, 1).await?;
1891        assert!(held >= Duration::from_millis(2950), "{held:?}");
1892        Ok(())
1893    }
1894
1895    /// A frame given up while its last segment was still on the way never completes: the segment is
1896    /// dropped as already seen. Noticing it there is what lengthens a wait that was too short.
1897    #[test_log::test(tokio::test)]
1898    async fn a_segment_after_its_frames_gap_was_given_up_should_lengthen_the_wait() -> anyhow::Result<()> {
1899        use tokio::io::AsyncWriteExt;
1900
1901        let (reader, mut remote) = stateless_reader(true)?;
1902        // Eight frames over six seconds, 100 ms between their segments: a 250 ms wait.
1903        send_whole_frames(&mut remote, 1..=8, Duration::from_millis(100)).await?;
1904
1905        // Frame 9's second segment comes 700 ms after its first: frame 9 is given up after 250 ms ...
1906        remote.write_all(&raw_segment(9, 0, 2, b"late-")).await?;
1907        remote.write_all(&raw_segment(10, 0, 2, b"ten-h")).await?;
1908        remote.write_all(&raw_segment(10, 1, 2, b"alf-x")).await?;
1909        tokio::time::sleep(Duration::from_millis(700)).await;
1910        // ... and its segment, coming now, lengthens the wait to about 2 x 700 ms.
1911        remote.write_all(&raw_segment(9, 1, 2, b"frame")).await?;
1912        tokio::time::sleep(Duration::from_millis(50)).await;
1913
1914        // Frame 11 is lost; frame 12 waits the longer wait.
1915        let started = std::time::Instant::now();
1916        remote.write_all(&raw_segment(12, 0, 2, b"twelv")).await?;
1917        remote.write_all(&raw_segment(12, 1, 2, b"e-end")).await?;
1918        tokio::time::sleep(Duration::from_millis(3500)).await;
1919        drop(remote);
1920
1921        let reads = reader.await?;
1922        assert!(
1923            !reads.iter().any(|(_, buf)| buf == b"late-frame"),
1924            "a frame given up is not delivered late"
1925        );
1926        let (at, last) = reads.last().ok_or_else(|| anyhow::anyhow!("frames delivered"))?;
1927        assert_eq!(b"twelve-end", last);
1928        let held = at.duration_since(started);
1929        assert!(
1930            held >= Duration::from_millis(1200),
1931            "the late segment must lengthen the wait: {held:?}"
1932        );
1933        assert!(held < Duration::from_millis(2500), "{held:?}");
1934        Ok(())
1935    }
1936
1937    #[test_log::test(tokio::test)]
1938    async fn stateful_socket_unidirectional_should_should_not_skip_missing_frames() -> anyhow::Result<()> {
1939        let (alice, bob) = setup_alice_bob::<MTU>(
1940            FaultyNetworkConfig {
1941                avg_delay: Duration::from_millis(10),
1942                ids_to_drop: HashSet::from_iter([0_usize]),
1943                ..Default::default()
1944            },
1945            None,
1946            None,
1947        );
1948
1949        let alice_cfg = SessionSocketConfig {
1950            frame_size: FRAME_SIZE,
1951            ..Default::default()
1952        };
1953
1954        let bob_cfg = SessionSocketConfig {
1955            frame_size: FRAME_SIZE,
1956            frame_timeout: Duration::from_millis(1000),
1957            ..Default::default()
1958        };
1959
1960        let ack_cfg = AcknowledgementStateConfig {
1961            expected_packet_latency: Duration::from_millis(10),
1962            acknowledgement_delay: Duration::from_millis(40),
1963            ..Default::default()
1964        };
1965
1966        let mut alice_socket = SessionSocket::<MTU, _>::new(
1967            alice,
1968            AcknowledgementState::new("alice", ack_cfg),
1969            alice_cfg,
1970            None,
1971            #[cfg(feature = "telemetry")]
1972            NoopTracker,
1973        )?;
1974        let mut bob_socket = SessionSocket::<MTU, _>::new(
1975            bob,
1976            AcknowledgementState::new("bob", ack_cfg),
1977            bob_cfg,
1978            None,
1979            #[cfg(feature = "telemetry")]
1980            NoopTracker,
1981        )?;
1982
1983        let data = hopr_types::crypto_random::random_bytes::<DATA_SIZE>();
1984
1985        let alice_jh = tokio::spawn(async move {
1986            alice_socket
1987                .write_all(&data)
1988                .timeout(futures_time::time::Duration::from_secs(5))
1989                .await??;
1990
1991            alice_socket.flush().await?;
1992
1993            // Alice has to keep reading so that it is ready for retransmitting
1994            let mut vec = Vec::new();
1995            alice_socket.read_to_end(&mut vec).await?;
1996            alice_socket.close().await?;
1997
1998            Ok::<_, std::io::Error>(vec)
1999        });
2000
2001        let mut bob_data = [0u8; DATA_SIZE];
2002        bob_socket
2003            .read_exact(&mut bob_data)
2004            .timeout(futures_time::time::Duration::from_secs(5))
2005            .await??;
2006        assert_eq!(data, bob_data);
2007
2008        bob_socket.close().await?;
2009
2010        let alice_recv = alice_jh.await??;
2011        assert!(alice_recv.is_empty());
2012
2013        Ok(())
2014    }
2015
2016    #[test_log::test(tokio::test)]
2017    async fn stateless_socket_bidirectional_should_should_skip_missing_frames() -> anyhow::Result<()> {
2018        let (alice, bob) = setup_alice_bob::<MTU>(
2019            FaultyNetworkConfig {
2020                avg_delay: Duration::from_millis(10),
2021                ids_to_drop: HashSet::from_iter([0_usize]),
2022                ..Default::default()
2023            },
2024            None,
2025            None,
2026        );
2027
2028        let alice_cfg = SessionSocketConfig {
2029            frame_size: FRAME_SIZE,
2030            frame_timeout: Duration::from_millis(55),
2031            ..Default::default()
2032        };
2033
2034        let bob_cfg = SessionSocketConfig {
2035            frame_size: FRAME_SIZE,
2036            frame_timeout: Duration::from_millis(55),
2037            ..Default::default()
2038        };
2039
2040        #[cfg(feature = "telemetry")]
2041        let (alice_tracker, bob_tracker) = (TestTelemetryTracker::default(), TestTelemetryTracker::default());
2042
2043        let mut alice_socket = SessionSocket::<MTU, _>::new_stateless(
2044            "alice",
2045            alice,
2046            alice_cfg,
2047            None,
2048            #[cfg(feature = "telemetry")]
2049            alice_tracker.clone(),
2050        )?;
2051        let mut bob_socket = SessionSocket::<MTU, _>::new_stateless(
2052            "bob",
2053            bob,
2054            bob_cfg,
2055            None,
2056            #[cfg(feature = "telemetry")]
2057            bob_tracker.clone(),
2058        )?;
2059
2060        let alice_sent_data = hopr_types::crypto_random::random_bytes::<DATA_SIZE>();
2061        alice_socket
2062            .write_all(&alice_sent_data)
2063            .timeout(futures_time::time::Duration::from_secs(2))
2064            .await??;
2065        alice_socket.flush().await?;
2066
2067        let bob_sent_data = hopr_types::crypto_random::random_bytes::<DATA_SIZE>();
2068        bob_socket
2069            .write_all(&bob_sent_data)
2070            .timeout(futures_time::time::Duration::from_secs(2))
2071            .await??;
2072        bob_socket.flush().await?;
2073
2074        alice_socket.close().await?;
2075        bob_socket.close().await?;
2076
2077        let mut alice_recv_data = Vec::with_capacity(DATA_SIZE);
2078        alice_socket
2079            .read_to_end(&mut alice_recv_data)
2080            .timeout(futures_time::time::Duration::from_secs(2))
2081            .await??;
2082
2083        let mut bob_recv_data = Vec::with_capacity(DATA_SIZE);
2084        bob_socket
2085            .read_to_end(&mut bob_recv_data)
2086            .timeout(futures_time::time::Duration::from_secs(2))
2087            .await??;
2088
2089        // The whole first frame is discarded due to the missing first segment. A frame is a whole
2090        // number of segments, so the data lost is exactly the frame size.
2091        let segment_payload = FRAME_SIZE;
2092        assert_eq!(bob_sent_data.len() - segment_payload, alice_recv_data.len());
2093        assert_eq!(&bob_sent_data[segment_payload..], &alice_recv_data);
2094
2095        assert_eq!(alice_sent_data.len() - segment_payload, bob_recv_data.len());
2096        assert_eq!(&alice_sent_data[segment_payload..], &bob_recv_data);
2097
2098        #[cfg(feature = "telemetry")]
2099        {
2100            insta::assert_yaml_snapshot!(alice_tracker);
2101            insta::assert_yaml_snapshot!(bob_tracker);
2102        }
2103
2104        Ok(())
2105    }
2106
2107    //#[test_log::test(tokio::test)]
2108    #[test_log::test(tokio::test(flavor = "multi_thread"))]
2109    async fn stateful_socket_bidirectional_should_should_not_skip_missing_frames() -> anyhow::Result<()> {
2110        let (alice, bob) = setup_alice_bob::<MTU>(
2111            FaultyNetworkConfig {
2112                avg_delay: Duration::from_millis(10),
2113                ids_to_drop: HashSet::from_iter([0_usize]),
2114                ..Default::default()
2115            },
2116            None,
2117            None,
2118        );
2119
2120        // use hopr_network_types::capture::PcapIoExt;
2121        // let (alice, bob) = (alice.capture("alice.pcap"), bob.capture("bob.pcap"));
2122
2123        let alice_cfg = SessionSocketConfig {
2124            frame_size: FRAME_SIZE,
2125            frame_timeout: Duration::from_millis(1000),
2126            ..Default::default()
2127        };
2128
2129        let bob_cfg = SessionSocketConfig {
2130            frame_size: FRAME_SIZE,
2131            frame_timeout: Duration::from_millis(1000),
2132            ..Default::default()
2133        };
2134
2135        let ack_cfg = AcknowledgementStateConfig {
2136            expected_packet_latency: Duration::from_millis(10),
2137            acknowledgement_delay: Duration::from_millis(40),
2138            ..Default::default()
2139        };
2140
2141        let (mut alice_rx, mut alice_tx) = SessionSocket::<MTU, _>::new(
2142            alice,
2143            AcknowledgementState::new("alice", ack_cfg),
2144            alice_cfg,
2145            None,
2146            #[cfg(feature = "telemetry")]
2147            NoopTracker,
2148        )?
2149        .split();
2150
2151        let (mut bob_rx, mut bob_tx) = SessionSocket::<MTU, _>::new(
2152            bob,
2153            AcknowledgementState::new("bob", ack_cfg),
2154            bob_cfg,
2155            None,
2156            #[cfg(feature = "telemetry")]
2157            NoopTracker,
2158        )?
2159        .split();
2160
2161        let alice_sent_data = hopr_types::crypto_random::random_bytes::<DATA_SIZE>();
2162        let (alice_data_tx, alice_recv_data) = futures::channel::oneshot::channel();
2163        let alice_rx_jh = tokio::spawn(async move {
2164            let mut alice_recv_data = vec![0u8; DATA_SIZE];
2165            alice_rx.read_exact(&mut alice_recv_data).await?;
2166            alice_data_tx
2167                .send(alice_recv_data)
2168                .map_err(|_| std::io::Error::other("tx error"))?;
2169
2170            // Keep reading until the socket is closed
2171            alice_rx.read_to_end(&mut Vec::new()).await?;
2172            Ok::<_, std::io::Error>(())
2173        });
2174
2175        let bob_sent_data = hopr_types::crypto_random::random_bytes::<DATA_SIZE>();
2176        let (bob_data_tx, bob_recv_data) = futures::channel::oneshot::channel();
2177        let bob_rx_jh = tokio::spawn(async move {
2178            let mut bob_recv_data = vec![0u8; DATA_SIZE];
2179            bob_rx.read_exact(&mut bob_recv_data).await?;
2180            bob_data_tx
2181                .send(bob_recv_data)
2182                .map_err(|_| std::io::Error::other("tx error"))?;
2183
2184            // Keep reading until the socket is closed
2185            bob_rx.read_to_end(&mut Vec::new()).await?;
2186            Ok::<_, std::io::Error>(())
2187        });
2188
2189        let alice_tx_jh = tokio::spawn(async move {
2190            alice_tx
2191                .write_all(&alice_sent_data)
2192                .timeout(futures_time::time::Duration::from_secs(2))
2193                .await??;
2194            alice_tx.flush().await?;
2195
2196            // Once all data is sent, wait for the other side to receive it and close the socket
2197            let out = alice_recv_data.await.map_err(|_| std::io::Error::other("rx error"))?;
2198            alice_tx.close().await?;
2199            tracing::info!("alice closed");
2200            Ok::<_, std::io::Error>(out)
2201        });
2202
2203        let bob_tx_jh = tokio::spawn(async move {
2204            bob_tx
2205                .write_all(&bob_sent_data)
2206                .timeout(futures_time::time::Duration::from_secs(2))
2207                .await??;
2208            bob_tx.flush().await?;
2209
2210            // Once all data is sent, wait for the other side to receive it and close the socket
2211            let out = bob_recv_data.await.map_err(|_| std::io::Error::other("rx error"))?;
2212            bob_tx.close().await?;
2213            tracing::info!("bob closed");
2214            Ok::<_, std::io::Error>(out)
2215        });
2216
2217        let (alice_recv_data, bob_recv_data, a, b) =
2218            futures::future::try_join4(alice_tx_jh, bob_tx_jh, alice_rx_jh, bob_rx_jh)
2219                .timeout(futures_time::time::Duration::from_secs(4))
2220                .await??;
2221
2222        assert_eq!(&alice_sent_data, bob_recv_data?.as_slice());
2223        assert_eq!(&bob_sent_data, alice_recv_data?.as_slice());
2224        assert!(a.is_ok());
2225        assert!(b.is_ok());
2226
2227        Ok(())
2228    }
2229}