Skip to main content

hopr_transport/protocol/
stream.rs

1//! Infrastructure supporting converting a collection of `PeerId` split `libp2p_stream` managed
2//! individual peer-to-peer `libp2p::swarm::Stream`s.
3
4use std::{
5    pin::Pin,
6    sync::{
7        Arc,
8        atomic::{AtomicBool, AtomicUsize, Ordering},
9    },
10    task::{Context, Poll},
11    time::Duration,
12};
13
14use crossfire::mpsc;
15use futures::{
16    AsyncRead, AsyncReadExt, AsyncWrite, FutureExt, Sink, StreamExt,
17    channel::mpsc::{Receiver, Sender, channel},
18};
19use futures_timer::Delay;
20use hopr_api::network::NetworkStreamControl;
21use libp2p::PeerId;
22use tokio_util::{
23    codec::{Decoder, Encoder, FramedRead, FramedWrite},
24    compat::{FuturesAsyncReadCompatExt, FuturesAsyncWriteCompatExt},
25};
26
27#[cfg(all(feature = "telemetry", not(test)))]
28lazy_static::lazy_static! {
29    static ref METRIC_RING_BUFFER_DROPPED: hopr_api::types::telemetry::SimpleCounter =
30        hopr_api::types::telemetry::SimpleCounter::new(
31            "hopr_egress_ring_buffer_dropped",
32            "Number of packets dropped due to per-peer egress channel overflow (drop-newest)",
33        )
34        .unwrap();
35}
36
37/// Per-peer egress buffer: a bounded MPSC crossfire channel.
38///
39/// Only the sender is stored in the cache entry; the receiver is owned exclusively by
40/// the write pump spawned when the outgoing stream opens.  This is functionally SPSC
41/// (one drain-loop producer, one write-pump consumer) even though the sender type is
42/// Clone (required by `moka::sync::Cache`).
43///
44/// When the channel is full the drain loop drops the incoming packet (drop-newest)
45/// and yields via `yield_now()` so the write pump has a chance to drain items and
46/// free space before the next send attempt, preventing task starvation under high
47/// producer rates.
48///
49/// `token` is a unique identity for this sink instance. Pump and opener tasks that
50/// hold a clone compare it against the current cache entry before calling
51/// `cache.invalidate`, so that a stale task finishing after the sink was replaced
52/// (e.g. by an inbound stream arriving during an outgoing open) does not wipe the
53/// newer entry.
54#[derive(Clone)]
55struct PeerSink<T: Send + 'static> {
56    tx: crossfire::MAsyncTx<mpsc::Array<T>>,
57    token: Arc<()>,
58    /// `false` while the outgoing stream is still being opened, `true` once the write pump
59    /// is draining the channel. The egress drain only applies blocking backpressure on a full
60    /// channel when the stream is `ready`; while still opening it uses non-blocking drop-newest,
61    /// so a slow open for one peer never head-of-line-blocks other peers.
62    ready: Arc<AtomicBool>,
63}
64
65impl<T: Send + 'static> PeerSink<T> {
66    fn new(tx: crossfire::MAsyncTx<mpsc::Array<T>>) -> Self {
67        Self {
68            tx,
69            token: Arc::new(()),
70            ready: Arc::new(AtomicBool::new(false)),
71        }
72    }
73}
74
75type PeerStreamCache<T> = moka::sync::Cache<PeerId, PeerSink<T>>;
76
77/// Error produced by the per-peer write pump ([`StallGuardSink`]).
78#[derive(Debug)]
79enum EgressWriteError<E> {
80    /// The wrapped framed-writer sink returned an error of its own.
81    Sink(E),
82    /// The wrapped sink made no forward progress within the stall timeout.
83    Stalled { timeout: Duration },
84}
85
86impl<E: std::fmt::Display> std::fmt::Display for EgressWriteError<E> {
87    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
88        match self {
89            EgressWriteError::Sink(e) => write!(f, "egress sink error: {e}"),
90            EgressWriteError::Stalled { timeout } => {
91                write!(f, "egress write pump made no progress within {timeout:?}")
92            }
93        }
94    }
95}
96
97/// A [`Sink`] adapter that fails the per-peer write pump once the inner sink stops making progress.
98///
99/// The write pump is `rx.forward(frame_writer)`, where `frame_writer` writes to the peer's quinn
100/// substream. When a remote stops reading, quinn's `poll_write` parks (`Pending`) indefinitely
101/// under flow control, so the framed writer's `poll_ready`/`poll_flush` never resolve and the pump
102/// task parks forever — never completing, so the cache-eviction closure attached to it never runs.
103/// The stalled peer's channel then stays full and the shared egress drain burns one backpressure
104/// timeout per packet for that peer, head-of-line-blocking every other peer until the unbounded
105/// mixer queue exhausts memory (the 2026-08-27 `jura-dev` OOM).
106///
107/// This adapter arms an idle deadline the first time the inner sink returns `Pending` on a
108/// readiness/flush/close poll, and clears it the moment the inner sink returns `Ready`. If the
109/// deadline elapses while the inner sink is still `Pending`, the poll resolves to
110/// [`EgressWriteError::Stalled`], driving the pump to completion so its cache entry is evicted and
111/// the next send reopens the stream. A merely-slow peer that keeps making progress within the
112/// timeout is never killed — the deadline resets on every `Ready`.
113struct StallGuardSink<S> {
114    inner: S,
115    timeout: Duration,
116    /// Persistent idle timer, `reset` in place when a stall starts (the same persistent-`Delay`
117    /// pattern as the mixer's `MixerSink`) so a busy-but-healthy peer that oscillates
118    /// `Pending`/`Ready` does not churn a fresh `Delay` per flush cycle.
119    timer: Delay,
120    /// `true` while a readiness/flush/close poll is outstanding (inner last returned `Pending`).
121    /// While set, the `timer` is polled without resetting, so a continuous stall is measured from
122    /// its start rather than restarted on every poll.
123    armed: bool,
124}
125
126impl<S> StallGuardSink<S> {
127    fn new(inner: S, timeout: Duration) -> Self {
128        Self {
129            inner,
130            timeout,
131            timer: Delay::new(timeout),
132            armed: false,
133        }
134    }
135
136    /// Poll the inner sink, arming/clearing the idle timer and mapping a stall to an error.
137    fn poll_guarded<T, E>(
138        &mut self,
139        cx: &mut Context<'_>,
140        poll_inner: impl FnOnce(&mut S, &mut Context<'_>) -> Poll<Result<T, E>>,
141    ) -> Poll<Result<T, EgressWriteError<E>>> {
142        match poll_inner(&mut self.inner, cx) {
143            Poll::Ready(Ok(v)) => {
144                self.armed = false;
145                Poll::Ready(Ok(v))
146            }
147            Poll::Ready(Err(e)) => {
148                self.armed = false;
149                Poll::Ready(Err(EgressWriteError::Sink(e)))
150            }
151            Poll::Pending => {
152                if !self.armed {
153                    self.timer.reset(self.timeout);
154                    self.armed = true;
155                }
156                match self.timer.poll_unpin(cx) {
157                    Poll::Ready(()) => {
158                        self.armed = false;
159                        Poll::Ready(Err(EgressWriteError::Stalled { timeout: self.timeout }))
160                    }
161                    Poll::Pending => Poll::Pending,
162                }
163            }
164        }
165    }
166}
167
168impl<S, T> Sink<T> for StallGuardSink<S>
169where
170    S: Sink<T> + Unpin,
171{
172    type Error = EgressWriteError<S::Error>;
173
174    fn poll_ready(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Result<(), Self::Error>> {
175        self.get_mut().poll_guarded(cx, |s, cx| Pin::new(s).poll_ready(cx))
176    }
177
178    fn start_send(self: Pin<&mut Self>, item: T) -> Result<(), Self::Error> {
179        Pin::new(&mut self.get_mut().inner)
180            .start_send(item)
181            .map_err(EgressWriteError::Sink)
182    }
183
184    fn poll_flush(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Result<(), Self::Error>> {
185        self.get_mut().poll_guarded(cx, |s, cx| Pin::new(s).poll_flush(cx))
186    }
187
188    fn poll_close(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Result<(), Self::Error>> {
189        self.get_mut().poll_guarded(cx, |s, cx| Pin::new(s).poll_close(cx))
190    }
191}
192
193/// Spawn the write and read pump tasks for an open peer stream.
194///
195/// The write pump drains `rx` into the framed stream writer; the read pump
196/// forwards decoded frames to `ingress_from_peers`. Both tasks invalidate
197/// `cache[peer]` when they complete, but only if the cache entry still holds
198/// the same `token` — this prevents a stale task from wiping a newer sink.
199#[allow(clippy::too_many_arguments)]
200fn spawn_stream_pumps<S, C>(
201    peer: PeerId,
202    stream: S,
203    rx: crossfire::AsyncRx<mpsc::Array<<C as Decoder>::Item>>,
204    cache: PeerStreamCache<<C as Decoder>::Item>,
205    token: Arc<()>,
206    codec: C,
207    ingress_from_peers: Sender<(PeerId, <C as Decoder>::Item)>,
208    frame_writer_backpressure_bytes: usize,
209    write_stall_timeout: Duration,
210    ready: Arc<AtomicBool>,
211) where
212    S: AsyncRead + AsyncWrite + Send + 'static,
213    C: Encoder<<C as Decoder>::Item> + Decoder + Send + Sync + Clone + 'static,
214    <C as Encoder<<C as Decoder>::Item>>::Error: std::fmt::Debug + std::fmt::Display + Send + Sync + 'static,
215    <C as Decoder>::Error: std::fmt::Debug + std::fmt::Display + Send + Sync + 'static,
216    <C as Decoder>::Item: AsRef<[u8]> + Clone + Send + 'static,
217{
218    let (stream_rx, stream_tx) = stream.split();
219    let cache_for_write = cache.clone();
220    let cache_for_read = cache.clone();
221    let token_write = token.clone();
222
223    let mut frame_writer = FramedWrite::new(stream_tx.compat_write(), codec.clone());
224
225    // `set_backpressure_boundary` is a *byte* threshold on `FramedWrite`'s internal
226    // pending-write buffer: once the encoded frames exceed this many bytes, the next
227    // `poll_ready` call will issue a flush. A larger value lets adjacent small frames
228    // coalesce into a single quinn write, significantly reducing the number of
229    // connection-mutex acquisitions and driver wake-ups on the hot path.
230    frame_writer.set_backpressure_boundary(frame_writer_backpressure_bytes);
231
232    // Guard the framed writer so a remote that stops reading (quinn `poll_write` parked forever)
233    // fails the pump after `write_stall_timeout` instead of parking it indefinitely. On failure the
234    // pump completes and the eviction closure below removes the cache entry, so the next send reopens.
235    let frame_writer = StallGuardSink::new(frame_writer, write_stall_timeout);
236
237    // Write pump: drain the per-peer channel into the framed stream writer.
238    hopr_utils::runtime::prelude::spawn(
239        futures::future::lazy(move |_| {
240            // Flip `ready` only once this task is actually running and about to drain the
241            // channel, so the egress drain never applies blocking backpressure on a peer whose
242            // write pump has not started draining yet.
243            ready.store(true, Ordering::Relaxed);
244        })
245        .then(move |_| rx.into_stream().map(Ok).forward(frame_writer))
246        .inspect(move |res| {
247            tracing::debug!(%peer, ?res, component = "stream", "writing stream with peer finished");
248        })
249        .then(move |_| async move {
250            if cache_for_write
251                .get(&peer)
252                .is_some_and(|s| Arc::ptr_eq(&s.token, &token_write))
253            {
254                cache_for_write.invalidate(&peer);
255            }
256        }),
257    );
258
259    // Read pump: forward decoded frames to the ingress channel.
260    hopr_utils::runtime::prelude::spawn(
261        FramedRead::new(stream_rx.compat(), codec)
262            .filter_map(move |v| {
263                futures::future::ready(match v {
264                    Ok(v) => {
265                        tracing::trace!(%peer, "read message from peer stream");
266                        Some((peer, v))
267                    }
268                    Err(error) => {
269                        tracing::error!(%error, "Error decoding object from the underlying stream");
270                        None
271                    }
272                })
273            })
274            .map(Ok)
275            .forward(ingress_from_peers)
276            .inspect(move |res| match res {
277                Ok(_) => tracing::debug!(%peer, component = "stream", "incoming stream done reading"),
278                Err(error) => {
279                    tracing::error!(%peer, %error, component = "stream", "incoming stream failed on reading")
280                }
281            })
282            .then(move |_| async move {
283                if cache_for_read.get(&peer).is_some_and(|s| Arc::ptr_eq(&s.token, &token)) {
284                    cache_for_read.invalidate(&peer);
285                }
286            }),
287    );
288
289    tracing::trace!(%peer, "created new io for peer");
290}
291
292pub async fn process_stream_protocol<C, V>(
293    codec: C,
294    control: V,
295    stream_cfg: crate::config::StreamProtocolConfig,
296) -> super::errors::Result<(
297    Sender<(PeerId, <C as Decoder>::Item)>, // impl Sink<(PeerId, <C as Decoder>::Item)>,
298    Receiver<(PeerId, <C as Decoder>::Item)>, // impl Stream<Item = (PeerId, <C as Decoder>::Item)>,
299)>
300where
301    C: Encoder<<C as Decoder>::Item> + Decoder + Send + Sync + Clone + 'static,
302    <C as Encoder<<C as Decoder>::Item>>::Error: std::fmt::Debug + std::fmt::Display + Send + Sync + 'static,
303    <C as Decoder>::Error: std::fmt::Debug + std::fmt::Display + Send + Sync + 'static,
304    // `Unpin` is required so the egress drain can `await` a crossfire `send` (backpressure path).
305    <C as Decoder>::Item: AsRef<[u8]> + Clone + Send + Unpin + 'static,
306    V: NetworkStreamControl + Clone + Send + Sync + 'static,
307{
308    let (tx_out, mut rx_out) = channel::<(PeerId, <C as Decoder>::Item)>(100_000);
309    let (tx_in, rx_in) = channel::<(PeerId, <C as Decoder>::Item)>(100_000);
310
311    let cache_out: PeerStreamCache<<C as Decoder>::Item> = moka::sync::Cache::builder()
312        .max_capacity(2000)
313        .eviction_listener(|key: Arc<PeerId>, _, cause| {
314            tracing::trace!(peer = %key.as_ref(), ?cause, "evicting stream for peer");
315        })
316        .build();
317
318    // Bounds the number of in-flight stream-open tasks across all distinct peers.
319    const MAX_CONCURRENT_STREAM_OPENS: usize = 50;
320    let open_task_count = Arc::new(AtomicUsize::new(0));
321
322    let incoming = control
323        .clone()
324        .accept()
325        .map_err(|e| super::errors::ProtocolError::Logic(format!("failed to listen on protocol: {e}")))?;
326
327    let stream_open_timeout = stream_cfg.stream_open_timeout;
328    let frame_writer_backpressure_bytes = stream_cfg.frame_writer_backpressure_bytes;
329    let per_peer_channel_capacity = stream_cfg.per_peer_channel_capacity;
330    let egress_backpressure_timeout = stream_cfg.egress_backpressure_timeout;
331
332    let open_ctx = Arc::new((control, codec, tx_in));
333
334    let cache_ingress = cache_out.clone();
335    let open_ctx_ingress = open_ctx.clone();
336
337    // terminated when the incoming is dropped
338    let _ingress_process = hopr_utils::runtime::prelude::spawn(
339        incoming
340            .for_each(move |(peer, stream)| {
341                let cache = cache_ingress.clone();
342                let open_ctx = open_ctx_ingress.clone();
343
344                tracing::debug!(%peer, "received incoming peer-to-peer stream");
345                let (_control, codec, tx_in) = (&open_ctx.0, &open_ctx.1, &open_ctx.2);
346
347                let (tx, rx) = mpsc::bounded_async::<<C as Decoder>::Item>(per_peer_channel_capacity);
348                let sink = PeerSink::new(tx);
349                let token = sink.token.clone();
350                let ready = sink.ready.clone();
351                spawn_stream_pumps(
352                    peer,
353                    stream,
354                    rx,
355                    cache.clone(),
356                    token,
357                    codec.clone(),
358                    tx_in.clone(),
359                    frame_writer_backpressure_bytes,
360                    egress_backpressure_timeout,
361                    ready,
362                );
363                cache.insert(peer, sink);
364
365                futures::future::ready(())
366            })
367            .inspect(|_| {
368                tracing::info!(
369                    task = "ingress stream processing",
370                    "long-running background task finished"
371                )
372            }),
373    );
374
375    // Egress drain: reads outgoing packets from `rx_out` and enqueues them into
376    // the per-peer crossfire channel.
377    //
378    // The drain is non-blocking: `try_send` never awaits. On overflow the incoming
379    // packet is dropped (drop-newest) and `yield_now()` is called to let the write
380    // pump task drain the channel before the next send attempt, preventing task
381    // starvation under high producer rates.
382    //
383    // On cache miss, `get_with` atomically creates the per-peer channel and spawns
384    // exactly one opener task. In-flight opens are bounded by `open_task_count`
385    // (MAX_CONCURRENT_STREAM_OPENS = 50) to prevent resource exhaustion.
386    let _egress_process = hopr_utils::runtime::prelude::spawn(async move {
387        use futures::StreamExt as _;
388
389        while let Some((peer, msg)) = rx_out.next().await {
390            tracing::trace!(%peer, "trying to deliver message to peer");
391
392            let sink = if let Some(s) = cache_out.get(&peer) {
393                s
394            } else {
395                let cache2 = cache_out.clone();
396                let open_ctx2 = open_ctx.clone();
397                let open_count2 = open_task_count.clone();
398                cache_out.get_with(peer, move || {
399                    let (tx, rx) = mpsc::bounded_async::<<C as Decoder>::Item>(per_peer_channel_capacity);
400                    let sink = PeerSink::new(tx);
401                    let token = sink.token.clone();
402                    let ready = sink.ready.clone();
403
404                    if open_count2.fetch_add(1, Ordering::Relaxed) < MAX_CONCURRENT_STREAM_OPENS {
405                        hopr_utils::runtime::prelude::spawn(async move {
406                            tracing::trace!(%peer, "peer is not in cache, opening new stream");
407                            use futures_time::future::FutureExt as TimeExt;
408                            let (control, codec, tx_in) = (&open_ctx2.0, &open_ctx2.1, &open_ctx2.2);
409
410                            let stream = control
411                                .clone()
412                                .open(peer)
413                                .timeout(futures_time::time::Duration::from(stream_open_timeout))
414                                .await
415                                .map_err(|_| anyhow::anyhow!("timeout trying to open stream to {peer}"))
416                                .and_then(|s| {
417                                    s.map_err(|e| anyhow::anyhow!("could not open outgoing peer-to-peer stream: {e}"))
418                                });
419
420                            open_count2.fetch_sub(1, Ordering::Relaxed);
421
422                            match stream {
423                                Ok(stream) => {
424                                    tracing::debug!(%peer, "opening outgoing peer-to-peer stream");
425                                    spawn_stream_pumps(
426                                        peer,
427                                        stream,
428                                        rx,
429                                        cache2.clone(),
430                                        token,
431                                        codec.clone(),
432                                        tx_in.clone(),
433                                        frame_writer_backpressure_bytes,
434                                        egress_backpressure_timeout,
435                                        ready,
436                                    );
437                                }
438                                Err(error) => {
439                                    tracing::debug!(
440                                        %peer, %error,
441                                        "stream open failed/timed out; dropping buffered packets"
442                                    );
443                                    if cache2.get(&peer).is_some_and(|s| Arc::ptr_eq(&s.token, &token)) {
444                                        cache2.invalidate(&peer);
445                                    }
446                                }
447                            }
448                        });
449                    } else {
450                        open_count2.fetch_sub(1, Ordering::Relaxed);
451                        tracing::debug!(%peer, "stream-open concurrency limit reached; dropping buffered packets");
452                        hopr_utils::runtime::prelude::spawn(async move {
453                            if cache2.get(&peer).is_some_and(|s| Arc::ptr_eq(&s.token, &token)) {
454                                cache2.invalidate(&peer);
455                            }
456                        });
457                    }
458
459                    sink
460                })
461            };
462
463            match sink.tx.try_send(msg) {
464                Ok(()) => tracing::trace!(%peer, "message queued to peer channel"),
465                Err(crossfire::TrySendError::Full(_msg)) => {
466                    // Per-peer egress channel is full (capacity reached). The wire is
467                    // slower than the producer. Drop the newest packet — the mixer
468                    // bound (MixerConfig::capacity) provides upstream backpressure, so
469                    // this drop is a rate-limiting measure rather than the OOM
470                    // protection. A full channel on an open stream is handled identically
471                    // to a full channel during open: drop-newest, no blocking, so one
472                    // congested peer never head-of-line-blocks the shared drain loop.
473                    #[cfg(all(feature = "telemetry", not(test)))]
474                    METRIC_RING_BUFFER_DROPPED.increment();
475                    tracing::debug!(%peer, "per-peer egress channel full; dropping newest packet");
476                }
477                Err(crossfire::TrySendError::Disconnected(_)) => {
478                    // Receiver dropped (write pump died): invalidate and reopen on next send.
479                    // Guard with the token so we don't wipe a replacement sink that was
480                    // inserted (e.g. from an inbound stream) between our cache.get() above
481                    // and this branch.
482                    tracing::debug!(%peer, "peer sink disconnected; invalidating cache");
483                    if cache_out.get(&peer).is_some_and(|s| Arc::ptr_eq(&s.token, &sink.token)) {
484                        cache_out.invalidate(&peer);
485                    }
486                }
487            }
488        }
489
490        tracing::info!(
491            task = "egress stream processing",
492            "long-running background task finished"
493        );
494    });
495
496    Ok((tx_out, rx_in))
497}
498
499#[cfg(test)]
500mod tests {
501    use std::{
502        pin::Pin,
503        sync::{
504            Arc,
505            atomic::{AtomicUsize, Ordering},
506        },
507        task::{Context as TaskContext, Poll, Waker},
508    };
509
510    use anyhow::Context;
511    use async_trait::async_trait;
512    use futures::{SinkExt, Stream};
513    use parking_lot::Mutex;
514    use tokio_util::{bytes::BytesMut, codec::BytesCodec};
515
516    use super::*;
517
518    #[derive(Clone, Default, Debug)]
519    struct CountingControl {
520        open_calls: Arc<AtomicUsize>,
521    }
522
523    impl CountingControl {
524        fn open_calls(&self) -> usize {
525            self.open_calls.load(Ordering::Relaxed)
526        }
527    }
528
529    #[derive(Default)]
530    struct StalledWriteIo;
531
532    impl AsyncRead for StalledWriteIo {
533        fn poll_read(self: Pin<&mut Self>, _cx: &mut TaskContext<'_>, _buf: &mut [u8]) -> Poll<std::io::Result<usize>> {
534            Poll::Pending
535        }
536    }
537
538    impl AsyncWrite for StalledWriteIo {
539        fn poll_write(self: Pin<&mut Self>, _cx: &mut TaskContext<'_>, _buf: &[u8]) -> Poll<std::io::Result<usize>> {
540            Poll::Pending
541        }
542
543        fn poll_flush(self: Pin<&mut Self>, _cx: &mut TaskContext<'_>) -> Poll<std::io::Result<()>> {
544            Poll::Pending
545        }
546
547        fn poll_close(self: Pin<&mut Self>, _cx: &mut TaskContext<'_>) -> Poll<std::io::Result<()>> {
548            Poll::Ready(Ok(()))
549        }
550    }
551
552    #[async_trait]
553    impl hopr_api::network::traits::NetworkStreamControl for CountingControl {
554        fn accept(
555            self,
556        ) -> Result<impl Stream<Item = (PeerId, impl AsyncRead + AsyncWrite + Send)> + Send, impl std::error::Error>
557        {
558            Ok::<_, std::io::Error>(futures::stream::empty::<(PeerId, StalledWriteIo)>())
559        }
560
561        async fn open(self, _peer: PeerId) -> Result<impl AsyncRead + AsyncWrite + Send, impl std::error::Error> {
562            self.open_calls.fetch_add(1, Ordering::Relaxed);
563            Ok::<_, std::io::Error>(StalledWriteIo)
564        }
565    }
566
567    struct AsyncBinaryStreamChannel {
568        read: async_channel_io::ChannelReader,
569        write: async_channel_io::ChannelWriter,
570    }
571
572    impl AsyncBinaryStreamChannel {
573        pub fn new() -> Self {
574            let (write, read) = async_channel_io::pipe();
575            Self { read, write }
576        }
577    }
578
579    impl AsyncRead for AsyncBinaryStreamChannel {
580        fn poll_read(
581            self: std::pin::Pin<&mut Self>,
582            cx: &mut std::task::Context<'_>,
583            buf: &mut [u8],
584        ) -> std::task::Poll<std::io::Result<usize>> {
585            let mut pinned = std::pin::pin!(&mut self.get_mut().read);
586            pinned.as_mut().poll_read(cx, buf)
587        }
588    }
589
590    impl AsyncWrite for AsyncBinaryStreamChannel {
591        fn poll_write(
592            self: std::pin::Pin<&mut Self>,
593            cx: &mut std::task::Context<'_>,
594            buf: &[u8],
595        ) -> std::task::Poll<std::io::Result<usize>> {
596            let mut pinned = std::pin::pin!(&mut self.get_mut().write);
597            pinned.as_mut().poll_write(cx, buf)
598        }
599
600        fn poll_flush(
601            self: std::pin::Pin<&mut Self>,
602            cx: &mut std::task::Context<'_>,
603        ) -> std::task::Poll<std::io::Result<()>> {
604            let pinned = std::pin::pin!(&mut self.get_mut().write);
605            pinned.poll_flush(cx)
606        }
607
608        fn poll_close(
609            self: std::pin::Pin<&mut Self>,
610            cx: &mut std::task::Context<'_>,
611        ) -> std::task::Poll<std::io::Result<()>> {
612            let pinned = std::pin::pin!(&mut self.get_mut().write);
613            pinned.poll_close(cx)
614        }
615    }
616
617    #[tokio::test]
618    async fn split_codec_should_always_produce_correct_data() -> anyhow::Result<()> {
619        let stream = AsyncBinaryStreamChannel::new();
620        let codec = tokio_util::codec::BytesCodec::new();
621
622        let expected = [0u8, 1u8, 2u8, 3u8, 4u8, 5u8];
623        let value = tokio_util::bytes::BytesMut::from(expected.as_ref());
624
625        let (stream_rx, stream_tx) = stream.split();
626        let (mut tx, rx) = (
627            FramedWrite::new(stream_tx.compat_write(), codec),
628            FramedRead::new(stream_rx.compat(), codec),
629        );
630        tx.send(value)
631            .await
632            .map_err(|_| anyhow::anyhow!("should not fail on send"))?;
633
634        futures::pin_mut!(rx);
635
636        assert_eq!(
637            rx.next().await.context("Value must be present")??,
638            tokio_util::bytes::BytesMut::from(expected.as_ref())
639        );
640
641        Ok(())
642    }
643
644    // ---------------------------------------------------------------------------
645    // FlaggedStream — models a LivenessStream whose connection has been killed.
646    // ---------------------------------------------------------------------------
647
648    struct DeadSignal {
649        read_dead: std::sync::atomic::AtomicBool,
650        write_dead: std::sync::atomic::AtomicBool,
651        read_waker: Mutex<Option<Waker>>,
652        write_waker: Mutex<Option<Waker>>,
653    }
654
655    impl std::fmt::Debug for DeadSignal {
656        fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
657            f.debug_struct("DeadSignal")
658                .field("read_dead", &self.read_dead.load(Ordering::Relaxed))
659                .field("write_dead", &self.write_dead.load(Ordering::Relaxed))
660                .finish_non_exhaustive()
661        }
662    }
663
664    impl DeadSignal {
665        fn new() -> Arc<Self> {
666            Arc::new(Self {
667                read_dead: std::sync::atomic::AtomicBool::new(false),
668                write_dead: std::sync::atomic::AtomicBool::new(false),
669                read_waker: Mutex::new(None),
670                write_waker: Mutex::new(None),
671            })
672        }
673
674        fn kill_read(self: &Arc<Self>) {
675            self.read_dead.store(true, Ordering::Release);
676            let waker = self.read_waker.lock().take();
677            if let Some(w) = waker {
678                w.wake();
679            }
680        }
681
682        fn kill_write(self: &Arc<Self>) {
683            self.write_dead.store(true, Ordering::Release);
684            let waker = self.write_waker.lock().take();
685            if let Some(w) = waker {
686                w.wake();
687            }
688        }
689    }
690
691    struct FlaggedStream {
692        signal: Arc<DeadSignal>,
693    }
694
695    impl AsyncRead for FlaggedStream {
696        fn poll_read(self: Pin<&mut Self>, cx: &mut TaskContext<'_>, _buf: &mut [u8]) -> Poll<std::io::Result<usize>> {
697            *self.signal.read_waker.lock() = Some(cx.waker().clone());
698            if self.signal.read_dead.load(Ordering::Acquire) {
699                return Poll::Ready(Err(std::io::Error::from(std::io::ErrorKind::ConnectionAborted)));
700            }
701            Poll::Pending
702        }
703    }
704
705    impl AsyncWrite for FlaggedStream {
706        fn poll_write(self: Pin<&mut Self>, cx: &mut TaskContext<'_>, _buf: &[u8]) -> Poll<std::io::Result<usize>> {
707            *self.signal.write_waker.lock() = Some(cx.waker().clone());
708            if self.signal.write_dead.load(Ordering::Acquire) {
709                return Poll::Ready(Err(std::io::Error::from(std::io::ErrorKind::ConnectionAborted)));
710            }
711            Poll::Pending
712        }
713
714        fn poll_flush(self: Pin<&mut Self>, _cx: &mut TaskContext<'_>) -> Poll<std::io::Result<()>> {
715            Poll::Ready(Ok(()))
716        }
717
718        fn poll_close(self: Pin<&mut Self>, _cx: &mut TaskContext<'_>) -> Poll<std::io::Result<()>> {
719            Poll::Ready(Ok(()))
720        }
721    }
722
723    #[derive(Clone, Debug, Default)]
724    struct ScriptedControl {
725        open_calls: Arc<AtomicUsize>,
726        signals: Arc<Mutex<Vec<Arc<DeadSignal>>>>,
727    }
728
729    impl ScriptedControl {
730        fn open_calls(&self) -> usize {
731            self.open_calls.load(Ordering::Relaxed)
732        }
733
734        fn signal(&self, index: usize) -> Option<Arc<DeadSignal>> {
735            self.signals.lock().get(index).cloned()
736        }
737    }
738
739    #[async_trait]
740    impl hopr_api::network::traits::NetworkStreamControl for ScriptedControl {
741        fn accept(
742            self,
743        ) -> Result<impl Stream<Item = (PeerId, impl AsyncRead + AsyncWrite + Send)> + Send, impl std::error::Error>
744        {
745            Ok::<_, std::io::Error>(futures::stream::empty::<(PeerId, FlaggedStream)>())
746        }
747
748        async fn open(self, _peer: PeerId) -> Result<impl AsyncRead + AsyncWrite + Send, impl std::error::Error> {
749            let signal = DeadSignal::new();
750            self.signals.lock().push(signal.clone());
751            self.open_calls.fetch_add(1, Ordering::Relaxed);
752            Ok::<_, std::io::Error>(FlaggedStream { signal })
753        }
754    }
755
756    async fn wait_for(secs: u64, condition: impl Fn() -> bool) -> bool {
757        let deadline = tokio::time::Instant::now() + std::time::Duration::from_secs(secs);
758        while !condition() && tokio::time::Instant::now() < deadline {
759            tokio::time::sleep(std::time::Duration::from_millis(25)).await;
760        }
761        condition()
762    }
763
764    #[tokio::test]
765    async fn dead_stream_should_be_detected_on_read_path_and_allow_next_send_to_reopen() -> anyhow::Result<()> {
766        let control = ScriptedControl::default();
767
768        let (mut tx_out, _rx_in) = process_stream_protocol(
769            BytesCodec::new(),
770            control.clone(),
771            crate::config::StreamProtocolConfig {
772                per_peer_channel_capacity: 64,
773                ..Default::default()
774            },
775        )
776        .await?;
777
778        let peer = PeerId::random();
779        let msg = BytesMut::from(&b"probe"[..]);
780
781        tx_out
782            .send((peer, msg.clone()))
783            .await
784            .context("first send should succeed")?;
785
786        assert!(
787            wait_for(2, || control.open_calls() >= 1).await,
788            "stream was never opened"
789        );
790        let signal = control.signal(0).context("signal for stream #1 must exist")?;
791
792        signal.kill_read();
793
794        tx_out
795            .send((peer, msg.clone()))
796            .await
797            .context("second send into egress queue should succeed")?;
798
799        assert!(
800            wait_for(2, || control.open_calls() >= 2).await,
801            "stream was not reopened after connection kill (open_calls={})",
802            control.open_calls()
803        );
804
805        Ok(())
806    }
807
808    #[tokio::test]
809    async fn dead_stream_should_be_detected_on_write_path_and_allow_next_send_to_reopen() -> anyhow::Result<()> {
810        let control = ScriptedControl::default();
811
812        let (mut tx_out, _rx_in) = process_stream_protocol(
813            BytesCodec::new(),
814            control.clone(),
815            crate::config::StreamProtocolConfig {
816                per_peer_channel_capacity: 128,
817                ..Default::default()
818            },
819        )
820        .await?;
821
822        let peer = PeerId::random();
823        let msg = BytesMut::from(&b"payload"[..]);
824
825        tx_out.send((peer, msg.clone())).await.context("initial send")?;
826        assert!(wait_for(2, || control.open_calls() >= 1).await, "stream not opened");
827
828        let signal = control.signal(0).context("signal #1 must exist")?;
829
830        signal.kill_write();
831
832        let mut drained = 0usize;
833        while control.open_calls() < 2 && drained < 128 {
834            tx_out
835                .send((peer, msg.clone()))
836                .await
837                .with_context(|| format!("drain send {drained} into egress queue should succeed"))?;
838            drained += 1;
839        }
840
841        assert!(
842            wait_for(3, || control.open_calls() >= 2).await,
843            "stream was not reopened after writer kill (open_calls={})",
844            control.open_calls()
845        );
846
847        Ok(())
848    }
849
850    /// Reopening a permanently stalled peer must self-heal at the stall-timeout cadence — never as
851    /// tight per-send churn.
852    ///
853    /// `CountingControl` returns a `StalledWriteIo` whose `poll_write`/`poll_flush` always return
854    /// `Pending`. Before the sink-level stall mitigation the write pump parked forever and the entry
855    /// was never evicted (the previous version of this test asserted "must not reopen"). It now
856    /// terminates after the stall timeout so the entry is evicted and the next send reopens — but
857    /// the reopen cadence must stay bounded by the stall timeout, not spin once per send.
858    #[tokio::test]
859    async fn stalled_peer_reopen_cadence_should_be_bounded_by_the_stall_timeout() -> anyhow::Result<()> {
860        const STALL_TIMEOUT: std::time::Duration = std::time::Duration::from_millis(300);
861        const WINDOW: std::time::Duration = std::time::Duration::from_millis(1_500);
862
863        let control = CountingControl::default();
864        let (mut tx_out, _rx_in) = process_stream_protocol(
865            BytesCodec::new(),
866            control.clone(),
867            crate::config::StreamProtocolConfig {
868                per_peer_channel_capacity: 4,
869                frame_writer_backpressure_bytes: 1,
870                egress_backpressure_timeout: STALL_TIMEOUT,
871                ..Default::default()
872            },
873        )
874        .await?;
875
876        let peer = PeerId::random();
877        let msg = BytesMut::from(&b"x"[..]);
878
879        // Keep offering packets throughout the window so every eviction is promptly observable as a
880        // fresh open.
881        let deadline = tokio::time::Instant::now() + WINDOW;
882        while tokio::time::Instant::now() < deadline {
883            let _ = tx_out.send((peer, msg.clone())).await;
884            tokio::time::sleep(std::time::Duration::from_millis(25)).await;
885        }
886
887        let opens = control.open_calls();
888        assert!(
889            opens >= 2,
890            "a permanently stalled peer must self-heal by reopening (open_calls={opens})"
891        );
892
893        // Cadence bound: at most one reopen per stall timeout, plus generous slack for scheduling.
894        // A regression to per-send reopen churn (dozens/hundreds of opens) trips this.
895        let max_expected = (WINDOW.as_millis() / STALL_TIMEOUT.as_millis()) as usize + 3;
896        assert!(
897            opens <= max_expected,
898            "reopen churn detected: {opens} opens in {WINDOW:?} exceeds the ~{max_expected} bounded by the \
899             {STALL_TIMEOUT:?} stall timeout"
900        );
901
902        Ok(())
903    }
904
905    /// Reproduces the 2026-08-27 `jura-dev` OOM at the integration boundary.
906    ///
907    /// A peer whose remote stops reading makes the underlying `poll_write` park (`Pending`)
908    /// forever under flow control. The per-peer write pump (`rx.forward(frame_writer)`) then
909    /// parks with it and never completes, so its cache-eviction closure never runs: the sink is
910    /// never evicted, the peer's channel stays full, and the shared egress drain burns one
911    /// backpressure timeout per packet for that peer — head-of-line-blocking every other peer
912    /// until the unbounded mixer queue exhausts memory.
913    ///
914    /// The fix mitigates the stall *at the sink*: the write pump must fail once it makes no
915    /// progress for `egress_backpressure_timeout`, driving the pump to completion so the entry is
916    /// evicted and a later send reopens the stream. A merely-slow peer that keeps making progress
917    /// within the timeout is never killed.
918    ///
919    /// Under the unfixed code the pump parks forever, the entry is never evicted, and
920    /// `open_calls` stays pinned at 1 — this test fails.
921    #[tokio::test]
922    async fn stalled_write_pump_should_terminate_and_reopen_after_stall_timeout() -> anyhow::Result<()> {
923        const STALL_TIMEOUT: std::time::Duration = std::time::Duration::from_millis(500);
924
925        let control = CountingControl::default();
926        let (mut tx_out, _rx_in) = process_stream_protocol(
927            BytesCodec::new(),
928            control.clone(),
929            crate::config::StreamProtocolConfig {
930                per_peer_channel_capacity: 4,
931                // Flush every frame so the pump reaches the stalled writer immediately, rather than
932                // buffering frames inside `FramedWrite` until the byte threshold forces a flush.
933                frame_writer_backpressure_bytes: 1,
934                // Reused as the write-pump stall timeout.
935                egress_backpressure_timeout: STALL_TIMEOUT,
936                ..Default::default()
937            },
938        )
939        .await?;
940
941        let peer = PeerId::random();
942        let msg = BytesMut::from(&b"payload"[..]);
943
944        // First send opens the stream; its writer is permanently stalled (poll_write == Pending).
945        tx_out.send((peer, msg.clone())).await.context("first send")?;
946        assert!(
947            wait_for(2, || control.open_calls() >= 1).await,
948            "stream was never opened"
949        );
950
951        // A merely-slow peer must not be evicted: no reopen before the stall timeout elapses.
952        tokio::time::sleep(STALL_TIMEOUT / 2).await;
953        assert_eq!(
954            control.open_calls(),
955            1,
956            "stream reopened before the stall timeout elapsed — over-eager eviction would kill merely-slow peers"
957        );
958
959        // Past the stall timeout the pump must terminate, evict the cache entry, and a subsequent
960        // send must reopen the stream. Keep sending so an eviction is observable as a fresh open.
961        let mut sends = 0;
962        while control.open_calls() < 2 && sends < 200 {
963            let _ = tx_out.send((peer, msg.clone())).await;
964            sends += 1;
965            tokio::time::sleep(std::time::Duration::from_millis(25)).await;
966        }
967
968        assert!(
969            wait_for(3, || control.open_calls() >= 2).await,
970            "stalled write pump never terminated: the stream was not reopened after the stall timeout (open_calls={}) \
971             — the pump parked forever and the peer became a permanent silent black hole, exactly the field failure \
972             mode",
973            control.open_calls(),
974        );
975
976        Ok(())
977    }
978
979    // -----------------------------------------------------------------------
980    // StallGuardSink unit tests — isolate the sink-level stall mitigation from
981    // the surrounding stream-protocol machinery. Each covers one way the wrapped
982    // sink can stall (or make progress) and asserts the adapter's response.
983    // -----------------------------------------------------------------------
984
985    /// Outcome a [`ScriptedSink`] returns for a given poll.
986    #[derive(Clone, Copy)]
987    enum Op {
988        Ready,
989        Pending,
990        Err,
991    }
992
993    fn apply(op: Op) -> Poll<Result<(), std::io::Error>> {
994        match op {
995            Op::Ready => Poll::Ready(Ok(())),
996            Op::Pending => Poll::Pending,
997            Op::Err => Poll::Ready(Err(std::io::Error::other("scripted sink error"))),
998        }
999    }
1000
1001    /// A `Sink` whose `poll_ready`/`poll_flush`/`poll_close` outcomes are fixed per-op. A `Pending`
1002    /// op never registers a waker (modelling a quinn substream whose remote stopped reading) — the
1003    /// wake-up must come from `StallGuardSink`'s own timer.
1004    struct ScriptedSink {
1005        ready: Op,
1006        flush: Op,
1007        close: Op,
1008    }
1009
1010    impl Sink<u8> for ScriptedSink {
1011        type Error = std::io::Error;
1012
1013        fn poll_ready(self: Pin<&mut Self>, _cx: &mut TaskContext<'_>) -> Poll<Result<(), Self::Error>> {
1014            apply(self.ready)
1015        }
1016
1017        fn start_send(self: Pin<&mut Self>, _item: u8) -> Result<(), Self::Error> {
1018            Ok(())
1019        }
1020
1021        fn poll_flush(self: Pin<&mut Self>, _cx: &mut TaskContext<'_>) -> Poll<Result<(), Self::Error>> {
1022            apply(self.flush)
1023        }
1024
1025        fn poll_close(self: Pin<&mut Self>, _cx: &mut TaskContext<'_>) -> Poll<Result<(), Self::Error>> {
1026            apply(self.close)
1027        }
1028    }
1029
1030    /// A `Sink` whose first `poll_flush` parks once (waking itself so it is re-polled immediately),
1031    /// then succeeds — a peer that briefly stalls but recovers well within the timeout.
1032    #[derive(Default)]
1033    struct TransientFlushSink {
1034        flushed_once: bool,
1035    }
1036
1037    impl Sink<u8> for TransientFlushSink {
1038        type Error = std::io::Error;
1039
1040        fn poll_ready(self: Pin<&mut Self>, _cx: &mut TaskContext<'_>) -> Poll<Result<(), Self::Error>> {
1041            Poll::Ready(Ok(()))
1042        }
1043
1044        fn start_send(self: Pin<&mut Self>, _item: u8) -> Result<(), Self::Error> {
1045            Ok(())
1046        }
1047
1048        fn poll_flush(self: Pin<&mut Self>, cx: &mut TaskContext<'_>) -> Poll<Result<(), Self::Error>> {
1049            let this = self.get_mut();
1050            if this.flushed_once {
1051                Poll::Ready(Ok(()))
1052            } else {
1053                this.flushed_once = true;
1054                cx.waker().wake_by_ref();
1055                Poll::Pending
1056            }
1057        }
1058
1059        fn poll_close(self: Pin<&mut Self>, _cx: &mut TaskContext<'_>) -> Poll<Result<(), Self::Error>> {
1060            Poll::Ready(Ok(()))
1061        }
1062    }
1063
1064    #[tokio::test]
1065    async fn stall_guard_sink_should_error_when_poll_ready_stalls_past_timeout() {
1066        let mut sink = StallGuardSink::new(
1067            ScriptedSink {
1068                ready: Op::Pending,
1069                flush: Op::Ready,
1070                close: Op::Ready,
1071            },
1072            Duration::from_millis(150),
1073        );
1074        let res = tokio::time::timeout(Duration::from_secs(2), sink.send(1u8))
1075            .await
1076            .expect("StallGuardSink must resolve on its own timer, not hang");
1077        assert!(
1078            matches!(res, Err(EgressWriteError::Stalled { .. })),
1079            "a sink that never becomes ready must fail with Stalled, got {res:?}"
1080        );
1081    }
1082
1083    #[tokio::test]
1084    async fn stall_guard_sink_should_error_when_poll_flush_stalls_past_timeout() {
1085        let mut sink = StallGuardSink::new(
1086            ScriptedSink {
1087                ready: Op::Ready,
1088                flush: Op::Pending,
1089                close: Op::Ready,
1090            },
1091            Duration::from_millis(150),
1092        );
1093        let res = tokio::time::timeout(Duration::from_secs(2), sink.send(1u8))
1094            .await
1095            .expect("StallGuardSink must resolve on its own timer, not hang");
1096        assert!(
1097            matches!(res, Err(EgressWriteError::Stalled { .. })),
1098            "a sink that accepts but never flushes must fail with Stalled, got {res:?}"
1099        );
1100    }
1101
1102    #[tokio::test]
1103    async fn stall_guard_sink_should_error_when_poll_close_stalls_past_timeout() {
1104        let mut sink = StallGuardSink::new(
1105            ScriptedSink {
1106                ready: Op::Ready,
1107                flush: Op::Ready,
1108                close: Op::Pending,
1109            },
1110            Duration::from_millis(150),
1111        );
1112        let res = tokio::time::timeout(Duration::from_secs(2), sink.close())
1113            .await
1114            .expect("StallGuardSink must resolve on its own timer, not hang");
1115        assert!(
1116            matches!(res, Err(EgressWriteError::Stalled { .. })),
1117            "a sink that never closes must fail with Stalled, got {res:?}"
1118        );
1119    }
1120
1121    #[tokio::test]
1122    async fn stall_guard_sink_should_pass_through_a_healthy_sink_without_error() {
1123        let mut sink = StallGuardSink::new(
1124            ScriptedSink {
1125                ready: Op::Ready,
1126                flush: Op::Ready,
1127                close: Op::Ready,
1128            },
1129            Duration::from_millis(50),
1130        );
1131        for i in 0..100u8 {
1132            sink.send(i)
1133                .await
1134                .expect("a healthy sink must never be failed by the stall guard");
1135        }
1136        sink.close().await.expect("closing a healthy sink must succeed");
1137    }
1138
1139    #[tokio::test]
1140    async fn stall_guard_sink_should_surface_inner_errors_verbatim_not_as_a_stall() {
1141        let mut sink = StallGuardSink::new(
1142            ScriptedSink {
1143                ready: Op::Ready,
1144                flush: Op::Err,
1145                close: Op::Ready,
1146            },
1147            Duration::from_millis(150),
1148        );
1149        let res = sink.send(1u8).await;
1150        assert!(
1151            matches!(res, Err(EgressWriteError::Sink(_))),
1152            "an inner sink error must surface as Sink(_), not be masked as Stalled, got {res:?}"
1153        );
1154    }
1155
1156    #[tokio::test]
1157    async fn stall_guard_sink_should_not_error_on_a_transient_stall_that_recovers_in_time() {
1158        // Timeout far larger than the transient stall: the deadline is armed on the first (parked)
1159        // flush poll and must be cleared when the sink makes progress, so no Stalled error fires.
1160        let mut sink = StallGuardSink::new(TransientFlushSink::default(), Duration::from_secs(10));
1161        tokio::time::timeout(Duration::from_secs(2), sink.send(1u8))
1162            .await
1163            .expect("a sink that recovers before the timeout must not hang")
1164            .expect("a sink that recovers before the timeout must not be failed as Stalled");
1165    }
1166
1167    /// The stalled peer's stream opens exactly once (then its writer is gated shut by the test); any
1168    /// reopen fails. Every other peer opens a writer that accepts immediately.
1169    #[derive(Clone, Debug)]
1170    struct HeadOfLineControl {
1171        stalled_peer: PeerId,
1172        stalled_io: GatedWriteIo,
1173        healthy_io: GatedWriteIo,
1174        open_calls: Arc<AtomicUsize>,
1175        stalled_opens: Arc<AtomicUsize>,
1176    }
1177
1178    #[async_trait]
1179    impl hopr_api::network::traits::NetworkStreamControl for HeadOfLineControl {
1180        fn accept(
1181            self,
1182        ) -> Result<impl Stream<Item = (PeerId, impl AsyncRead + AsyncWrite + Send)> + Send, impl std::error::Error>
1183        {
1184            Ok::<_, std::io::Error>(futures::stream::empty::<(PeerId, GatedWriteIo)>())
1185        }
1186
1187        async fn open(self, peer: PeerId) -> Result<impl AsyncRead + AsyncWrite + Send, impl std::error::Error> {
1188            self.open_calls.fetch_add(1, Ordering::Relaxed);
1189            if peer == self.stalled_peer {
1190                // Only the first open establishes the stream. A permanently-shut peer that could
1191                // reopen would re-stall for another full timeout each cycle, making the burst take an
1192                // unbounded number of ~timeout cycles; failing the reopen bounds the fix's side to a
1193                // single eviction cycle, while the unfixed pump never evicts and so never reopens.
1194                if self.stalled_opens.fetch_add(1, Ordering::Relaxed) == 0 {
1195                    Ok::<GatedWriteIo, std::io::Error>(self.stalled_io.clone())
1196                } else {
1197                    Err(std::io::Error::other("stalled peer refuses reopen"))
1198                }
1199            } else {
1200                Ok(self.healthy_io.clone())
1201            }
1202        }
1203    }
1204
1205    /// End-to-end proof that the sink-level stall mitigation clears the head-of-line block that
1206    /// caused the 2026-08-27 `jura-dev` OOM: a peer whose write pump parks must not hold the shared
1207    /// drain loop hostage while a healthy peer waits behind it.
1208    ///
1209    /// The healthy packet is queued *after* the stalled burst, so it only arrives promptly if the
1210    /// stalled peer's sink is evicted and the loop stops serialising on it. Under the unfixed code
1211    /// the burst holds the loop for one backpressure timeout per packet forever, and the healthy
1212    /// peer is starved.
1213    #[tokio::test]
1214    async fn stalled_peer_must_not_head_of_line_block_a_healthy_peer() -> anyhow::Result<()> {
1215        const STALL_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(1);
1216        const OVERFLOW: usize = 8;
1217
1218        let stalled_peer = PeerId::random();
1219        let healthy_peer = PeerId::random();
1220
1221        // Both writers start open; the stalled peer is gated shut only after its stream is
1222        // established (below), modelling a remote that stops reading mid-stream.
1223        let stalled_io = GatedWriteIo {
1224            open: Arc::new(std::sync::atomic::AtomicBool::new(true)),
1225            ..Default::default()
1226        };
1227        let healthy_io = GatedWriteIo {
1228            open: Arc::new(std::sync::atomic::AtomicBool::new(true)),
1229            ..Default::default()
1230        };
1231
1232        let open_calls = Arc::new(AtomicUsize::new(0));
1233        let control = HeadOfLineControl {
1234            stalled_peer,
1235            stalled_io: stalled_io.clone(),
1236            healthy_io: healthy_io.clone(),
1237            open_calls: open_calls.clone(),
1238            stalled_opens: Arc::new(AtomicUsize::new(0)),
1239        };
1240
1241        let (mut tx_out, _rx_in) = process_stream_protocol(
1242            BytesCodec::new(),
1243            control,
1244            crate::config::StreamProtocolConfig {
1245                per_peer_channel_capacity: 2,
1246                // Flush every frame so the channel fills, not the `FramedWrite` buffer.
1247                frame_writer_backpressure_bytes: 1,
1248                egress_backpressure_timeout: STALL_TIMEOUT,
1249                ..Default::default()
1250            },
1251        )
1252        .await?;
1253
1254        let msg = BytesMut::from(&b"hello"[..]);
1255
1256        // Establish the ready precondition race-free: prime the stalled peer *while it is still
1257        // open* and wait until the byte is actually written to the wire. That positively proves the
1258        // stream opened and its write pump drained (`ready == true`), with no stall or eviction in
1259        // play yet — unlike waiting on a stalled poll, which could race the pump's own stall timeout.
1260        tx_out
1261            .send((stalled_peer, msg.clone()))
1262            .await
1263            .context("egress queue must accept priming stalled-peer packet")?;
1264        assert!(
1265            wait_for(2, || stalled_io.written() >= msg.len()).await,
1266            "priming packet was never written — stalled peer's stream/pump did not establish"
1267        );
1268        assert_eq!(
1269            open_calls.load(Ordering::Relaxed),
1270            1,
1271            "exactly one open expected during priming"
1272        );
1273
1274        // Now the remote stops reading: gate the writer shut so subsequent writes park. Only from
1275        // here does the stall (and its eviction timer) begin.
1276        stalled_io.gate_shut();
1277
1278        // Overflow the ready+full stalled channel. Under the old code each excess packet costs the
1279        // shared drain loop a full backpressure timeout, starving the healthy peer behind it; the fix
1280        // evicts the parked pump after one timeout so the loop moves on.
1281        for _ in 0..OVERFLOW {
1282            tx_out
1283                .send((stalled_peer, msg.clone()))
1284                .await
1285                .context("egress queue must accept stalled-peer packet")?;
1286        }
1287
1288        tx_out
1289            .send((healthy_peer, msg.clone()))
1290            .await
1291            .context("egress queue must accept healthy-peer packet")?;
1292
1293        let delivered = wait_for(3, || healthy_io.written() >= msg.len()).await;
1294
1295        assert_eq!(
1296            stalled_io.written(),
1297            msg.len(),
1298            "only the priming packet should reach the stalled writer; the post-shut burst must not"
1299        );
1300
1301        assert!(
1302            delivered,
1303            "a single stalled peer head-of-line-blocked the shared egress drain: the healthy peer got {} of {} bytes \
1304             within 3s, while {OVERFLOW} overflow packets to the stalled peer each held the loop for {STALL_TIMEOUT:?}",
1305            healthy_io.written(),
1306            msg.len(),
1307        );
1308
1309        Ok(())
1310    }
1311
1312    #[derive(Clone, Debug)]
1313    struct BimodalOpenControl {
1314        slow_peer: PeerId,
1315        open_delay: std::time::Duration,
1316        slow_open_calls: Arc<AtomicUsize>,
1317        fast_open_calls: Arc<AtomicUsize>,
1318    }
1319
1320    impl BimodalOpenControl {
1321        #[allow(dead_code)]
1322        fn slow_open_calls(&self) -> usize {
1323            self.slow_open_calls.load(Ordering::Relaxed)
1324        }
1325
1326        fn fast_open_calls(&self) -> usize {
1327            self.fast_open_calls.load(Ordering::Relaxed)
1328        }
1329    }
1330
1331    #[async_trait]
1332    impl hopr_api::network::traits::NetworkStreamControl for BimodalOpenControl {
1333        fn accept(
1334            self,
1335        ) -> Result<impl Stream<Item = (PeerId, impl AsyncRead + AsyncWrite + Send)> + Send, impl std::error::Error>
1336        {
1337            Ok::<_, std::io::Error>(futures::stream::empty::<(PeerId, StalledWriteIo)>())
1338        }
1339
1340        async fn open(self, peer: PeerId) -> Result<impl AsyncRead + AsyncWrite + Send, impl std::error::Error> {
1341            if peer == self.slow_peer {
1342                self.slow_open_calls.fetch_add(1, Ordering::Relaxed);
1343                tokio::time::sleep(self.open_delay).await;
1344                return Err::<AsyncBinaryStreamChannel, _>(std::io::Error::other("slow peer cannot connect"));
1345            }
1346            self.fast_open_calls.fetch_add(1, Ordering::Relaxed);
1347            Ok::<_, std::io::Error>(AsyncBinaryStreamChannel::new())
1348        }
1349    }
1350
1351    #[tokio::test]
1352    async fn egress_should_not_hol_block_fast_peer_behind_slow_opens() -> anyhow::Result<()> {
1353        let slow_peer = PeerId::random();
1354        let fast_peer = PeerId::random();
1355
1356        let control = BimodalOpenControl {
1357            slow_peer,
1358            open_delay: std::time::Duration::from_millis(5_000),
1359            slow_open_calls: Default::default(),
1360            fast_open_calls: Default::default(),
1361        };
1362
1363        let (mut tx_out, _rx_in) = process_stream_protocol(
1364            BytesCodec::new(),
1365            control.clone(),
1366            crate::config::StreamProtocolConfig {
1367                stream_open_timeout: std::time::Duration::from_millis(2_000),
1368                ..Default::default()
1369            },
1370        )
1371        .await?;
1372
1373        let msg = BytesMut::from(&b"x"[..]);
1374
1375        for _ in 0..3 {
1376            tx_out
1377                .send((slow_peer, msg.clone()))
1378                .await
1379                .context("egress queue must accept slow-peer packet")?;
1380        }
1381        tx_out
1382            .send((fast_peer, msg.clone()))
1383            .await
1384            .context("egress queue must accept fast-peer packet")?;
1385
1386        let deadline = tokio::time::Instant::now() + std::time::Duration::from_millis(1_000);
1387        while control.fast_open_calls() < 1 && tokio::time::Instant::now() < deadline {
1388            tokio::time::sleep(std::time::Duration::from_millis(10)).await;
1389        }
1390
1391        assert!(
1392            control.fast_open_calls() >= 1,
1393            "fast peer's stream open was not called within 1 s — egress drain is likely head-of-line blocked by \
1394             slow-peer opens"
1395        );
1396
1397        Ok(())
1398    }
1399
1400    #[derive(Clone, Debug)]
1401    struct DelayedControl {
1402        open_delay: std::time::Duration,
1403        open_calls: Arc<AtomicUsize>,
1404    }
1405
1406    #[async_trait]
1407    impl hopr_api::network::traits::NetworkStreamControl for DelayedControl {
1408        fn accept(
1409            self,
1410        ) -> Result<impl Stream<Item = (PeerId, impl AsyncRead + AsyncWrite + Send)> + Send, impl std::error::Error>
1411        {
1412            Ok::<_, std::io::Error>(futures::stream::empty::<(PeerId, AsyncBinaryStreamChannel)>())
1413        }
1414
1415        async fn open(self, _peer: PeerId) -> Result<impl AsyncRead + AsyncWrite + Send, impl std::error::Error> {
1416            self.open_calls.fetch_add(1, Ordering::Relaxed);
1417            tokio::time::sleep(self.open_delay).await;
1418            Ok::<_, std::io::Error>(AsyncBinaryStreamChannel::new())
1419        }
1420    }
1421
1422    /// Verifies that packets sent while the opener is in flight are buffered and
1423    /// all delivered once the stream opens — zero loss below channel capacity.
1424    #[tokio::test]
1425    async fn egress_buffers_during_slow_open_then_drains() -> anyhow::Result<()> {
1426        let open_calls = Arc::new(AtomicUsize::new(0));
1427        let control = DelayedControl {
1428            open_delay: std::time::Duration::from_millis(100),
1429            open_calls: open_calls.clone(),
1430        };
1431
1432        let (mut tx_out, mut rx_in) = process_stream_protocol(
1433            BytesCodec::new(),
1434            control,
1435            crate::config::StreamProtocolConfig {
1436                per_peer_channel_capacity: 64,
1437                ..Default::default()
1438            },
1439        )
1440        .await?;
1441
1442        let peer = PeerId::random();
1443        let msg = BytesMut::from(&b"hello"[..]);
1444
1445        let n = 10usize;
1446        let expected_bytes = n * msg.len();
1447        for _ in 0..n {
1448            tx_out
1449                .send((peer, msg.clone()))
1450                .await
1451                .context("send into egress queue should succeed")?;
1452        }
1453
1454        assert!(
1455            wait_for(2, || open_calls.load(Ordering::Relaxed) >= 1).await,
1456            "stream was never opened"
1457        );
1458
1459        let mut received_bytes = 0usize;
1460        let deadline = tokio::time::Instant::now() + std::time::Duration::from_secs(2);
1461        while received_bytes < expected_bytes && tokio::time::Instant::now() < deadline {
1462            if let Ok(Some((_, bytes))) =
1463                tokio::time::timeout(std::time::Duration::from_millis(100), rx_in.next()).await
1464            {
1465                received_bytes += bytes.len();
1466            }
1467        }
1468
1469        assert!(
1470            received_bytes >= expected_bytes,
1471            "expected at least {expected_bytes} bytes to be delivered after stream open; got {received_bytes}"
1472        );
1473
1474        Ok(())
1475    }
1476
1477    /// A writer whose `poll_write` accepts bytes while `open` and parks (`Pending`) while shut,
1478    /// counting the bytes it accepts. Gating it shut after it has drained a packet models a remote
1479    /// that stops reading only once the stream is established, so a burst can be forced onto the
1480    /// ready+full egress path with no dependence on scheduler/pipe-buffer timing.
1481    #[derive(Clone, Default, Debug)]
1482    struct GatedWriteIo {
1483        open: Arc<std::sync::atomic::AtomicBool>,
1484        written: Arc<AtomicUsize>,
1485        waker: Arc<Mutex<Option<Waker>>>,
1486    }
1487
1488    impl GatedWriteIo {
1489        fn release(&self) {
1490            self.open.store(true, Ordering::Relaxed);
1491            if let Some(waker) = self.waker.lock().take() {
1492                waker.wake();
1493            }
1494        }
1495
1496        /// Gate the writer shut so subsequent `poll_write`s park — models the remote ceasing to read.
1497        fn gate_shut(&self) {
1498            self.open.store(false, Ordering::Relaxed);
1499        }
1500
1501        fn written(&self) -> usize {
1502            self.written.load(Ordering::Relaxed)
1503        }
1504    }
1505
1506    impl AsyncRead for GatedWriteIo {
1507        fn poll_read(self: Pin<&mut Self>, _cx: &mut TaskContext<'_>, _buf: &mut [u8]) -> Poll<std::io::Result<usize>> {
1508            Poll::Pending
1509        }
1510    }
1511
1512    impl AsyncWrite for GatedWriteIo {
1513        fn poll_write(self: Pin<&mut Self>, cx: &mut TaskContext<'_>, buf: &[u8]) -> Poll<std::io::Result<usize>> {
1514            if self.open.load(Ordering::Relaxed) {
1515                self.written.fetch_add(buf.len(), Ordering::Relaxed);
1516                Poll::Ready(Ok(buf.len()))
1517            } else {
1518                *self.waker.lock() = Some(cx.waker().clone());
1519                Poll::Pending
1520            }
1521        }
1522
1523        fn poll_flush(self: Pin<&mut Self>, _cx: &mut TaskContext<'_>) -> Poll<std::io::Result<()>> {
1524            Poll::Ready(Ok(()))
1525        }
1526
1527        fn poll_close(self: Pin<&mut Self>, _cx: &mut TaskContext<'_>) -> Poll<std::io::Result<()>> {
1528            Poll::Ready(Ok(()))
1529        }
1530    }
1531
1532    #[derive(Clone, Debug)]
1533    struct GatedControl {
1534        io: GatedWriteIo,
1535        open_calls: Arc<AtomicUsize>,
1536    }
1537
1538    #[async_trait]
1539    impl hopr_api::network::traits::NetworkStreamControl for GatedControl {
1540        fn accept(
1541            self,
1542        ) -> Result<impl Stream<Item = (PeerId, impl AsyncRead + AsyncWrite + Send)> + Send, impl std::error::Error>
1543        {
1544            Ok::<_, std::io::Error>(futures::stream::empty::<(PeerId, GatedWriteIo)>())
1545        }
1546
1547        async fn open(self, _peer: PeerId) -> Result<impl AsyncRead + AsyncWrite + Send, impl std::error::Error> {
1548            self.open_calls.fetch_add(1, Ordering::Relaxed);
1549            Ok::<_, std::io::Error>(self.io.clone())
1550        }
1551    }
1552
1553    /// Regression guard for the egress backpressure fix.
1554    ///
1555    /// Once the stream is open and draining, a burst that overflows the per-peer channel must apply
1556    /// (bounded) backpressure so the producer is slowed to wire rate — no packet loss. Under the old
1557    /// `try_send` + drop-newest behavior, every packet that arrived while the channel was full was
1558    /// silently dropped.
1559    ///
1560    /// The [`GatedWriteIo`] writer is held shut so the per-peer channel is deterministically full while
1561    /// the stream is open (`ready == true`) — the exact ready+full path — regardless of scheduler
1562    /// timing. The writer is released *before* the backpressure timeout: with backpressure the queued
1563    /// overflow then drains to the wire (zero loss); with drop-newest the overflow was already gone.
1564    #[tokio::test]
1565    async fn egress_backpressures_when_open_channel_full() -> anyhow::Result<()> {
1566        let io = GatedWriteIo::default();
1567        let open_calls = Arc::new(AtomicUsize::new(0));
1568        let control = GatedControl {
1569            io: io.clone(),
1570            open_calls: open_calls.clone(),
1571        };
1572
1573        // Small per-peer channel so a modest burst overflows it while the stream is open.
1574        let (mut tx_out, _rx_in) = process_stream_protocol(
1575            BytesCodec::new(),
1576            control,
1577            crate::config::StreamProtocolConfig {
1578                per_peer_channel_capacity: 4,
1579                // Flush every frame so the per-peer channel (not the FramedWrite buffer) is the
1580                // bottleneck that fills — making the ready+full path unambiguous.
1581                frame_writer_backpressure_bytes: 1,
1582                ..Default::default()
1583            },
1584        )
1585        .await?;
1586
1587        let peer = PeerId::random();
1588        let msg = BytesMut::from(&b"hello"[..]);
1589        let n = 50usize;
1590
1591        // Burst far beyond the per-peer channel capacity while the writer is gated shut. The stream
1592        // opens and the write pump starts (ready == true) but cannot drain, so the channel fills and the
1593        // drain loop enters the ready+full path for the overflow. With the mixer bound providing upstream
1594        // backpressure, the egress drain drops the overflow packets (drop-newest on a full open channel)
1595        // rather than blocking — preventing head-of-line blocking from one congested peer.
1596        for _ in 0..n {
1597            tx_out
1598                .send((peer, msg.clone()))
1599                .await
1600                .context("send into egress queue should succeed")?;
1601        }
1602        assert!(
1603            wait_for(2, || open_calls.load(Ordering::Relaxed) >= 1).await,
1604            "stream was never opened"
1605        );
1606
1607        // Let the drain loop hit the full channel and drop the overflow.
1608        tokio::time::sleep(std::time::Duration::from_millis(200)).await;
1609
1610        // Release the writer. With drop-newest only the packets that fit the channel are kept.
1611        io.release();
1612
1613        // With per_peer_channel_capacity=4, at most 4 full frames reach the FramedWrite
1614        // (the channel holds 4 items before the write pump drains any). The fifth write is
1615        // always dropped. Allow one extra frame for the framing artifact (flushed on next cycle).
1616        let channel_fill_bytes = 4 * msg.len();
1617        let max_delivered = channel_fill_bytes + msg.len();
1618        assert!(
1619            wait_for(3, || io.written() >= 1).await,
1620            "at least something must reach the wire under drop-newest overflow (wrote {} bytes)",
1621            io.written()
1622        );
1623        assert!(
1624            io.written() <= max_delivered,
1625            "overflow with drop-newest on a full open channel must not exceed channel capacity + 1 frame (wrote {} \
1626             bytes, max {max_delivered})",
1627            io.written()
1628        );
1629
1630        Ok(())
1631    }
1632}