1use std::{
5 pin::Pin,
6 sync::{
7 Arc,
8 atomic::{AtomicBool, AtomicUsize, Ordering},
9 },
10 task::{Context, Poll},
11 time::Duration,
12};
13
14use crossfire::mpsc;
15use futures::{
16 AsyncRead, AsyncReadExt, AsyncWrite, FutureExt, Sink, StreamExt,
17 channel::mpsc::{Receiver, Sender, channel},
18};
19use futures_timer::Delay;
20use hopr_api::network::NetworkStreamControl;
21use libp2p::PeerId;
22use tokio_util::{
23 codec::{Decoder, Encoder, FramedRead, FramedWrite},
24 compat::{FuturesAsyncReadCompatExt, FuturesAsyncWriteCompatExt},
25};
26
27#[cfg(all(feature = "telemetry", not(test)))]
28lazy_static::lazy_static! {
29 static ref METRIC_RING_BUFFER_DROPPED: hopr_api::types::telemetry::SimpleCounter =
30 hopr_api::types::telemetry::SimpleCounter::new(
31 "hopr_egress_ring_buffer_dropped",
32 "Number of packets dropped due to per-peer egress channel overflow (drop-newest)",
33 )
34 .unwrap();
35}
36
37#[derive(Clone)]
55struct PeerSink<T: Send + 'static> {
56 tx: crossfire::MAsyncTx<mpsc::Array<T>>,
57 token: Arc<()>,
58 ready: Arc<AtomicBool>,
63}
64
65impl<T: Send + 'static> PeerSink<T> {
66 fn new(tx: crossfire::MAsyncTx<mpsc::Array<T>>) -> Self {
67 Self {
68 tx,
69 token: Arc::new(()),
70 ready: Arc::new(AtomicBool::new(false)),
71 }
72 }
73}
74
75type PeerStreamCache<T> = moka::sync::Cache<PeerId, PeerSink<T>>;
76
77#[derive(Debug)]
79enum EgressWriteError<E> {
80 Sink(E),
82 Stalled { timeout: Duration },
84}
85
86impl<E: std::fmt::Display> std::fmt::Display for EgressWriteError<E> {
87 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
88 match self {
89 EgressWriteError::Sink(e) => write!(f, "egress sink error: {e}"),
90 EgressWriteError::Stalled { timeout } => {
91 write!(f, "egress write pump made no progress within {timeout:?}")
92 }
93 }
94 }
95}
96
97struct StallGuardSink<S> {
114 inner: S,
115 timeout: Duration,
116 timer: Delay,
120 armed: bool,
124}
125
126impl<S> StallGuardSink<S> {
127 fn new(inner: S, timeout: Duration) -> Self {
128 Self {
129 inner,
130 timeout,
131 timer: Delay::new(timeout),
132 armed: false,
133 }
134 }
135
136 fn poll_guarded<T, E>(
138 &mut self,
139 cx: &mut Context<'_>,
140 poll_inner: impl FnOnce(&mut S, &mut Context<'_>) -> Poll<Result<T, E>>,
141 ) -> Poll<Result<T, EgressWriteError<E>>> {
142 match poll_inner(&mut self.inner, cx) {
143 Poll::Ready(Ok(v)) => {
144 self.armed = false;
145 Poll::Ready(Ok(v))
146 }
147 Poll::Ready(Err(e)) => {
148 self.armed = false;
149 Poll::Ready(Err(EgressWriteError::Sink(e)))
150 }
151 Poll::Pending => {
152 if !self.armed {
153 self.timer.reset(self.timeout);
154 self.armed = true;
155 }
156 match self.timer.poll_unpin(cx) {
157 Poll::Ready(()) => {
158 self.armed = false;
159 Poll::Ready(Err(EgressWriteError::Stalled { timeout: self.timeout }))
160 }
161 Poll::Pending => Poll::Pending,
162 }
163 }
164 }
165 }
166}
167
168impl<S, T> Sink<T> for StallGuardSink<S>
169where
170 S: Sink<T> + Unpin,
171{
172 type Error = EgressWriteError<S::Error>;
173
174 fn poll_ready(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Result<(), Self::Error>> {
175 self.get_mut().poll_guarded(cx, |s, cx| Pin::new(s).poll_ready(cx))
176 }
177
178 fn start_send(self: Pin<&mut Self>, item: T) -> Result<(), Self::Error> {
179 Pin::new(&mut self.get_mut().inner)
180 .start_send(item)
181 .map_err(EgressWriteError::Sink)
182 }
183
184 fn poll_flush(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Result<(), Self::Error>> {
185 self.get_mut().poll_guarded(cx, |s, cx| Pin::new(s).poll_flush(cx))
186 }
187
188 fn poll_close(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Result<(), Self::Error>> {
189 self.get_mut().poll_guarded(cx, |s, cx| Pin::new(s).poll_close(cx))
190 }
191}
192
193#[allow(clippy::too_many_arguments)]
200fn spawn_stream_pumps<S, C>(
201 peer: PeerId,
202 stream: S,
203 rx: crossfire::AsyncRx<mpsc::Array<<C as Decoder>::Item>>,
204 cache: PeerStreamCache<<C as Decoder>::Item>,
205 token: Arc<()>,
206 codec: C,
207 ingress_from_peers: Sender<(PeerId, <C as Decoder>::Item)>,
208 frame_writer_backpressure_bytes: usize,
209 write_stall_timeout: Duration,
210 ready: Arc<AtomicBool>,
211) where
212 S: AsyncRead + AsyncWrite + Send + 'static,
213 C: Encoder<<C as Decoder>::Item> + Decoder + Send + Sync + Clone + 'static,
214 <C as Encoder<<C as Decoder>::Item>>::Error: std::fmt::Debug + std::fmt::Display + Send + Sync + 'static,
215 <C as Decoder>::Error: std::fmt::Debug + std::fmt::Display + Send + Sync + 'static,
216 <C as Decoder>::Item: AsRef<[u8]> + Clone + Send + 'static,
217{
218 let (stream_rx, stream_tx) = stream.split();
219 let cache_for_write = cache.clone();
220 let cache_for_read = cache.clone();
221 let token_write = token.clone();
222
223 let mut frame_writer = FramedWrite::new(stream_tx.compat_write(), codec.clone());
224
225 frame_writer.set_backpressure_boundary(frame_writer_backpressure_bytes);
231
232 let frame_writer = StallGuardSink::new(frame_writer, write_stall_timeout);
236
237 hopr_utils::runtime::prelude::spawn(
239 futures::future::lazy(move |_| {
240 ready.store(true, Ordering::Relaxed);
244 })
245 .then(move |_| rx.into_stream().map(Ok).forward(frame_writer))
246 .inspect(move |res| {
247 tracing::debug!(%peer, ?res, component = "stream", "writing stream with peer finished");
248 })
249 .then(move |_| async move {
250 if cache_for_write
251 .get(&peer)
252 .is_some_and(|s| Arc::ptr_eq(&s.token, &token_write))
253 {
254 cache_for_write.invalidate(&peer);
255 }
256 }),
257 );
258
259 hopr_utils::runtime::prelude::spawn(
261 FramedRead::new(stream_rx.compat(), codec)
262 .filter_map(move |v| {
263 futures::future::ready(match v {
264 Ok(v) => {
265 tracing::trace!(%peer, "read message from peer stream");
266 Some((peer, v))
267 }
268 Err(error) => {
269 tracing::error!(%error, "Error decoding object from the underlying stream");
270 None
271 }
272 })
273 })
274 .map(Ok)
275 .forward(ingress_from_peers)
276 .inspect(move |res| match res {
277 Ok(_) => tracing::debug!(%peer, component = "stream", "incoming stream done reading"),
278 Err(error) => {
279 tracing::error!(%peer, %error, component = "stream", "incoming stream failed on reading")
280 }
281 })
282 .then(move |_| async move {
283 if cache_for_read.get(&peer).is_some_and(|s| Arc::ptr_eq(&s.token, &token)) {
284 cache_for_read.invalidate(&peer);
285 }
286 }),
287 );
288
289 tracing::trace!(%peer, "created new io for peer");
290}
291
292pub async fn process_stream_protocol<C, V>(
293 codec: C,
294 control: V,
295 stream_cfg: crate::config::StreamProtocolConfig,
296) -> super::errors::Result<(
297 Sender<(PeerId, <C as Decoder>::Item)>, Receiver<(PeerId, <C as Decoder>::Item)>, )>
300where
301 C: Encoder<<C as Decoder>::Item> + Decoder + Send + Sync + Clone + 'static,
302 <C as Encoder<<C as Decoder>::Item>>::Error: std::fmt::Debug + std::fmt::Display + Send + Sync + 'static,
303 <C as Decoder>::Error: std::fmt::Debug + std::fmt::Display + Send + Sync + 'static,
304 <C as Decoder>::Item: AsRef<[u8]> + Clone + Send + Unpin + 'static,
306 V: NetworkStreamControl + Clone + Send + Sync + 'static,
307{
308 let (tx_out, mut rx_out) = channel::<(PeerId, <C as Decoder>::Item)>(100_000);
309 let (tx_in, rx_in) = channel::<(PeerId, <C as Decoder>::Item)>(100_000);
310
311 let cache_out: PeerStreamCache<<C as Decoder>::Item> = moka::sync::Cache::builder()
312 .max_capacity(2000)
313 .eviction_listener(|key: Arc<PeerId>, _, cause| {
314 tracing::trace!(peer = %key.as_ref(), ?cause, "evicting stream for peer");
315 })
316 .build();
317
318 const MAX_CONCURRENT_STREAM_OPENS: usize = 50;
320 let open_task_count = Arc::new(AtomicUsize::new(0));
321
322 let incoming = control
323 .clone()
324 .accept()
325 .map_err(|e| super::errors::ProtocolError::Logic(format!("failed to listen on protocol: {e}")))?;
326
327 let stream_open_timeout = stream_cfg.stream_open_timeout;
328 let frame_writer_backpressure_bytes = stream_cfg.frame_writer_backpressure_bytes;
329 let per_peer_channel_capacity = stream_cfg.per_peer_channel_capacity;
330 let egress_backpressure_timeout = stream_cfg.egress_backpressure_timeout;
331
332 let open_ctx = Arc::new((control, codec, tx_in));
333
334 let cache_ingress = cache_out.clone();
335 let open_ctx_ingress = open_ctx.clone();
336
337 let _ingress_process = hopr_utils::runtime::prelude::spawn(
339 incoming
340 .for_each(move |(peer, stream)| {
341 let cache = cache_ingress.clone();
342 let open_ctx = open_ctx_ingress.clone();
343
344 tracing::debug!(%peer, "received incoming peer-to-peer stream");
345 let (_control, codec, tx_in) = (&open_ctx.0, &open_ctx.1, &open_ctx.2);
346
347 let (tx, rx) = mpsc::bounded_async::<<C as Decoder>::Item>(per_peer_channel_capacity);
348 let sink = PeerSink::new(tx);
349 let token = sink.token.clone();
350 let ready = sink.ready.clone();
351 spawn_stream_pumps(
352 peer,
353 stream,
354 rx,
355 cache.clone(),
356 token,
357 codec.clone(),
358 tx_in.clone(),
359 frame_writer_backpressure_bytes,
360 egress_backpressure_timeout,
361 ready,
362 );
363 cache.insert(peer, sink);
364
365 futures::future::ready(())
366 })
367 .inspect(|_| {
368 tracing::info!(
369 task = "ingress stream processing",
370 "long-running background task finished"
371 )
372 }),
373 );
374
375 let _egress_process = hopr_utils::runtime::prelude::spawn(async move {
387 use futures::StreamExt as _;
388
389 while let Some((peer, msg)) = rx_out.next().await {
390 tracing::trace!(%peer, "trying to deliver message to peer");
391
392 let sink = if let Some(s) = cache_out.get(&peer) {
393 s
394 } else {
395 let cache2 = cache_out.clone();
396 let open_ctx2 = open_ctx.clone();
397 let open_count2 = open_task_count.clone();
398 cache_out.get_with(peer, move || {
399 let (tx, rx) = mpsc::bounded_async::<<C as Decoder>::Item>(per_peer_channel_capacity);
400 let sink = PeerSink::new(tx);
401 let token = sink.token.clone();
402 let ready = sink.ready.clone();
403
404 if open_count2.fetch_add(1, Ordering::Relaxed) < MAX_CONCURRENT_STREAM_OPENS {
405 hopr_utils::runtime::prelude::spawn(async move {
406 tracing::trace!(%peer, "peer is not in cache, opening new stream");
407 use futures_time::future::FutureExt as TimeExt;
408 let (control, codec, tx_in) = (&open_ctx2.0, &open_ctx2.1, &open_ctx2.2);
409
410 let stream = control
411 .clone()
412 .open(peer)
413 .timeout(futures_time::time::Duration::from(stream_open_timeout))
414 .await
415 .map_err(|_| anyhow::anyhow!("timeout trying to open stream to {peer}"))
416 .and_then(|s| {
417 s.map_err(|e| anyhow::anyhow!("could not open outgoing peer-to-peer stream: {e}"))
418 });
419
420 open_count2.fetch_sub(1, Ordering::Relaxed);
421
422 match stream {
423 Ok(stream) => {
424 tracing::debug!(%peer, "opening outgoing peer-to-peer stream");
425 spawn_stream_pumps(
426 peer,
427 stream,
428 rx,
429 cache2.clone(),
430 token,
431 codec.clone(),
432 tx_in.clone(),
433 frame_writer_backpressure_bytes,
434 egress_backpressure_timeout,
435 ready,
436 );
437 }
438 Err(error) => {
439 tracing::debug!(
440 %peer, %error,
441 "stream open failed/timed out; dropping buffered packets"
442 );
443 if cache2.get(&peer).is_some_and(|s| Arc::ptr_eq(&s.token, &token)) {
444 cache2.invalidate(&peer);
445 }
446 }
447 }
448 });
449 } else {
450 open_count2.fetch_sub(1, Ordering::Relaxed);
451 tracing::debug!(%peer, "stream-open concurrency limit reached; dropping buffered packets");
452 hopr_utils::runtime::prelude::spawn(async move {
453 if cache2.get(&peer).is_some_and(|s| Arc::ptr_eq(&s.token, &token)) {
454 cache2.invalidate(&peer);
455 }
456 });
457 }
458
459 sink
460 })
461 };
462
463 match sink.tx.try_send(msg) {
464 Ok(()) => tracing::trace!(%peer, "message queued to peer channel"),
465 Err(crossfire::TrySendError::Full(_msg)) => {
466 #[cfg(all(feature = "telemetry", not(test)))]
474 METRIC_RING_BUFFER_DROPPED.increment();
475 tracing::debug!(%peer, "per-peer egress channel full; dropping newest packet");
476 }
477 Err(crossfire::TrySendError::Disconnected(_)) => {
478 tracing::debug!(%peer, "peer sink disconnected; invalidating cache");
483 if cache_out.get(&peer).is_some_and(|s| Arc::ptr_eq(&s.token, &sink.token)) {
484 cache_out.invalidate(&peer);
485 }
486 }
487 }
488 }
489
490 tracing::info!(
491 task = "egress stream processing",
492 "long-running background task finished"
493 );
494 });
495
496 Ok((tx_out, rx_in))
497}
498
499#[cfg(test)]
500mod tests {
501 use std::{
502 pin::Pin,
503 sync::{
504 Arc,
505 atomic::{AtomicUsize, Ordering},
506 },
507 task::{Context as TaskContext, Poll, Waker},
508 };
509
510 use anyhow::Context;
511 use async_trait::async_trait;
512 use futures::{SinkExt, Stream};
513 use parking_lot::Mutex;
514 use tokio_util::{bytes::BytesMut, codec::BytesCodec};
515
516 use super::*;
517
518 #[derive(Clone, Default, Debug)]
519 struct CountingControl {
520 open_calls: Arc<AtomicUsize>,
521 }
522
523 impl CountingControl {
524 fn open_calls(&self) -> usize {
525 self.open_calls.load(Ordering::Relaxed)
526 }
527 }
528
529 #[derive(Default)]
530 struct StalledWriteIo;
531
532 impl AsyncRead for StalledWriteIo {
533 fn poll_read(self: Pin<&mut Self>, _cx: &mut TaskContext<'_>, _buf: &mut [u8]) -> Poll<std::io::Result<usize>> {
534 Poll::Pending
535 }
536 }
537
538 impl AsyncWrite for StalledWriteIo {
539 fn poll_write(self: Pin<&mut Self>, _cx: &mut TaskContext<'_>, _buf: &[u8]) -> Poll<std::io::Result<usize>> {
540 Poll::Pending
541 }
542
543 fn poll_flush(self: Pin<&mut Self>, _cx: &mut TaskContext<'_>) -> Poll<std::io::Result<()>> {
544 Poll::Pending
545 }
546
547 fn poll_close(self: Pin<&mut Self>, _cx: &mut TaskContext<'_>) -> Poll<std::io::Result<()>> {
548 Poll::Ready(Ok(()))
549 }
550 }
551
552 #[async_trait]
553 impl hopr_api::network::traits::NetworkStreamControl for CountingControl {
554 fn accept(
555 self,
556 ) -> Result<impl Stream<Item = (PeerId, impl AsyncRead + AsyncWrite + Send)> + Send, impl std::error::Error>
557 {
558 Ok::<_, std::io::Error>(futures::stream::empty::<(PeerId, StalledWriteIo)>())
559 }
560
561 async fn open(self, _peer: PeerId) -> Result<impl AsyncRead + AsyncWrite + Send, impl std::error::Error> {
562 self.open_calls.fetch_add(1, Ordering::Relaxed);
563 Ok::<_, std::io::Error>(StalledWriteIo)
564 }
565 }
566
567 struct AsyncBinaryStreamChannel {
568 read: async_channel_io::ChannelReader,
569 write: async_channel_io::ChannelWriter,
570 }
571
572 impl AsyncBinaryStreamChannel {
573 pub fn new() -> Self {
574 let (write, read) = async_channel_io::pipe();
575 Self { read, write }
576 }
577 }
578
579 impl AsyncRead for AsyncBinaryStreamChannel {
580 fn poll_read(
581 self: std::pin::Pin<&mut Self>,
582 cx: &mut std::task::Context<'_>,
583 buf: &mut [u8],
584 ) -> std::task::Poll<std::io::Result<usize>> {
585 let mut pinned = std::pin::pin!(&mut self.get_mut().read);
586 pinned.as_mut().poll_read(cx, buf)
587 }
588 }
589
590 impl AsyncWrite for AsyncBinaryStreamChannel {
591 fn poll_write(
592 self: std::pin::Pin<&mut Self>,
593 cx: &mut std::task::Context<'_>,
594 buf: &[u8],
595 ) -> std::task::Poll<std::io::Result<usize>> {
596 let mut pinned = std::pin::pin!(&mut self.get_mut().write);
597 pinned.as_mut().poll_write(cx, buf)
598 }
599
600 fn poll_flush(
601 self: std::pin::Pin<&mut Self>,
602 cx: &mut std::task::Context<'_>,
603 ) -> std::task::Poll<std::io::Result<()>> {
604 let pinned = std::pin::pin!(&mut self.get_mut().write);
605 pinned.poll_flush(cx)
606 }
607
608 fn poll_close(
609 self: std::pin::Pin<&mut Self>,
610 cx: &mut std::task::Context<'_>,
611 ) -> std::task::Poll<std::io::Result<()>> {
612 let pinned = std::pin::pin!(&mut self.get_mut().write);
613 pinned.poll_close(cx)
614 }
615 }
616
617 #[tokio::test]
618 async fn split_codec_should_always_produce_correct_data() -> anyhow::Result<()> {
619 let stream = AsyncBinaryStreamChannel::new();
620 let codec = tokio_util::codec::BytesCodec::new();
621
622 let expected = [0u8, 1u8, 2u8, 3u8, 4u8, 5u8];
623 let value = tokio_util::bytes::BytesMut::from(expected.as_ref());
624
625 let (stream_rx, stream_tx) = stream.split();
626 let (mut tx, rx) = (
627 FramedWrite::new(stream_tx.compat_write(), codec),
628 FramedRead::new(stream_rx.compat(), codec),
629 );
630 tx.send(value)
631 .await
632 .map_err(|_| anyhow::anyhow!("should not fail on send"))?;
633
634 futures::pin_mut!(rx);
635
636 assert_eq!(
637 rx.next().await.context("Value must be present")??,
638 tokio_util::bytes::BytesMut::from(expected.as_ref())
639 );
640
641 Ok(())
642 }
643
644 struct DeadSignal {
649 read_dead: std::sync::atomic::AtomicBool,
650 write_dead: std::sync::atomic::AtomicBool,
651 read_waker: Mutex<Option<Waker>>,
652 write_waker: Mutex<Option<Waker>>,
653 }
654
655 impl std::fmt::Debug for DeadSignal {
656 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
657 f.debug_struct("DeadSignal")
658 .field("read_dead", &self.read_dead.load(Ordering::Relaxed))
659 .field("write_dead", &self.write_dead.load(Ordering::Relaxed))
660 .finish_non_exhaustive()
661 }
662 }
663
664 impl DeadSignal {
665 fn new() -> Arc<Self> {
666 Arc::new(Self {
667 read_dead: std::sync::atomic::AtomicBool::new(false),
668 write_dead: std::sync::atomic::AtomicBool::new(false),
669 read_waker: Mutex::new(None),
670 write_waker: Mutex::new(None),
671 })
672 }
673
674 fn kill_read(self: &Arc<Self>) {
675 self.read_dead.store(true, Ordering::Release);
676 let waker = self.read_waker.lock().take();
677 if let Some(w) = waker {
678 w.wake();
679 }
680 }
681
682 fn kill_write(self: &Arc<Self>) {
683 self.write_dead.store(true, Ordering::Release);
684 let waker = self.write_waker.lock().take();
685 if let Some(w) = waker {
686 w.wake();
687 }
688 }
689 }
690
691 struct FlaggedStream {
692 signal: Arc<DeadSignal>,
693 }
694
695 impl AsyncRead for FlaggedStream {
696 fn poll_read(self: Pin<&mut Self>, cx: &mut TaskContext<'_>, _buf: &mut [u8]) -> Poll<std::io::Result<usize>> {
697 *self.signal.read_waker.lock() = Some(cx.waker().clone());
698 if self.signal.read_dead.load(Ordering::Acquire) {
699 return Poll::Ready(Err(std::io::Error::from(std::io::ErrorKind::ConnectionAborted)));
700 }
701 Poll::Pending
702 }
703 }
704
705 impl AsyncWrite for FlaggedStream {
706 fn poll_write(self: Pin<&mut Self>, cx: &mut TaskContext<'_>, _buf: &[u8]) -> Poll<std::io::Result<usize>> {
707 *self.signal.write_waker.lock() = Some(cx.waker().clone());
708 if self.signal.write_dead.load(Ordering::Acquire) {
709 return Poll::Ready(Err(std::io::Error::from(std::io::ErrorKind::ConnectionAborted)));
710 }
711 Poll::Pending
712 }
713
714 fn poll_flush(self: Pin<&mut Self>, _cx: &mut TaskContext<'_>) -> Poll<std::io::Result<()>> {
715 Poll::Ready(Ok(()))
716 }
717
718 fn poll_close(self: Pin<&mut Self>, _cx: &mut TaskContext<'_>) -> Poll<std::io::Result<()>> {
719 Poll::Ready(Ok(()))
720 }
721 }
722
723 #[derive(Clone, Debug, Default)]
724 struct ScriptedControl {
725 open_calls: Arc<AtomicUsize>,
726 signals: Arc<Mutex<Vec<Arc<DeadSignal>>>>,
727 }
728
729 impl ScriptedControl {
730 fn open_calls(&self) -> usize {
731 self.open_calls.load(Ordering::Relaxed)
732 }
733
734 fn signal(&self, index: usize) -> Option<Arc<DeadSignal>> {
735 self.signals.lock().get(index).cloned()
736 }
737 }
738
739 #[async_trait]
740 impl hopr_api::network::traits::NetworkStreamControl for ScriptedControl {
741 fn accept(
742 self,
743 ) -> Result<impl Stream<Item = (PeerId, impl AsyncRead + AsyncWrite + Send)> + Send, impl std::error::Error>
744 {
745 Ok::<_, std::io::Error>(futures::stream::empty::<(PeerId, FlaggedStream)>())
746 }
747
748 async fn open(self, _peer: PeerId) -> Result<impl AsyncRead + AsyncWrite + Send, impl std::error::Error> {
749 let signal = DeadSignal::new();
750 self.signals.lock().push(signal.clone());
751 self.open_calls.fetch_add(1, Ordering::Relaxed);
752 Ok::<_, std::io::Error>(FlaggedStream { signal })
753 }
754 }
755
756 async fn wait_for(secs: u64, condition: impl Fn() -> bool) -> bool {
757 let deadline = tokio::time::Instant::now() + std::time::Duration::from_secs(secs);
758 while !condition() && tokio::time::Instant::now() < deadline {
759 tokio::time::sleep(std::time::Duration::from_millis(25)).await;
760 }
761 condition()
762 }
763
764 #[tokio::test]
765 async fn dead_stream_should_be_detected_on_read_path_and_allow_next_send_to_reopen() -> anyhow::Result<()> {
766 let control = ScriptedControl::default();
767
768 let (mut tx_out, _rx_in) = process_stream_protocol(
769 BytesCodec::new(),
770 control.clone(),
771 crate::config::StreamProtocolConfig {
772 per_peer_channel_capacity: 64,
773 ..Default::default()
774 },
775 )
776 .await?;
777
778 let peer = PeerId::random();
779 let msg = BytesMut::from(&b"probe"[..]);
780
781 tx_out
782 .send((peer, msg.clone()))
783 .await
784 .context("first send should succeed")?;
785
786 assert!(
787 wait_for(2, || control.open_calls() >= 1).await,
788 "stream was never opened"
789 );
790 let signal = control.signal(0).context("signal for stream #1 must exist")?;
791
792 signal.kill_read();
793
794 tx_out
795 .send((peer, msg.clone()))
796 .await
797 .context("second send into egress queue should succeed")?;
798
799 assert!(
800 wait_for(2, || control.open_calls() >= 2).await,
801 "stream was not reopened after connection kill (open_calls={})",
802 control.open_calls()
803 );
804
805 Ok(())
806 }
807
808 #[tokio::test]
809 async fn dead_stream_should_be_detected_on_write_path_and_allow_next_send_to_reopen() -> anyhow::Result<()> {
810 let control = ScriptedControl::default();
811
812 let (mut tx_out, _rx_in) = process_stream_protocol(
813 BytesCodec::new(),
814 control.clone(),
815 crate::config::StreamProtocolConfig {
816 per_peer_channel_capacity: 128,
817 ..Default::default()
818 },
819 )
820 .await?;
821
822 let peer = PeerId::random();
823 let msg = BytesMut::from(&b"payload"[..]);
824
825 tx_out.send((peer, msg.clone())).await.context("initial send")?;
826 assert!(wait_for(2, || control.open_calls() >= 1).await, "stream not opened");
827
828 let signal = control.signal(0).context("signal #1 must exist")?;
829
830 signal.kill_write();
831
832 let mut drained = 0usize;
833 while control.open_calls() < 2 && drained < 128 {
834 tx_out
835 .send((peer, msg.clone()))
836 .await
837 .with_context(|| format!("drain send {drained} into egress queue should succeed"))?;
838 drained += 1;
839 }
840
841 assert!(
842 wait_for(3, || control.open_calls() >= 2).await,
843 "stream was not reopened after writer kill (open_calls={})",
844 control.open_calls()
845 );
846
847 Ok(())
848 }
849
850 #[tokio::test]
859 async fn stalled_peer_reopen_cadence_should_be_bounded_by_the_stall_timeout() -> anyhow::Result<()> {
860 const STALL_TIMEOUT: std::time::Duration = std::time::Duration::from_millis(300);
861 const WINDOW: std::time::Duration = std::time::Duration::from_millis(1_500);
862
863 let control = CountingControl::default();
864 let (mut tx_out, _rx_in) = process_stream_protocol(
865 BytesCodec::new(),
866 control.clone(),
867 crate::config::StreamProtocolConfig {
868 per_peer_channel_capacity: 4,
869 frame_writer_backpressure_bytes: 1,
870 egress_backpressure_timeout: STALL_TIMEOUT,
871 ..Default::default()
872 },
873 )
874 .await?;
875
876 let peer = PeerId::random();
877 let msg = BytesMut::from(&b"x"[..]);
878
879 let deadline = tokio::time::Instant::now() + WINDOW;
882 while tokio::time::Instant::now() < deadline {
883 let _ = tx_out.send((peer, msg.clone())).await;
884 tokio::time::sleep(std::time::Duration::from_millis(25)).await;
885 }
886
887 let opens = control.open_calls();
888 assert!(
889 opens >= 2,
890 "a permanently stalled peer must self-heal by reopening (open_calls={opens})"
891 );
892
893 let max_expected = (WINDOW.as_millis() / STALL_TIMEOUT.as_millis()) as usize + 3;
896 assert!(
897 opens <= max_expected,
898 "reopen churn detected: {opens} opens in {WINDOW:?} exceeds the ~{max_expected} bounded by the \
899 {STALL_TIMEOUT:?} stall timeout"
900 );
901
902 Ok(())
903 }
904
905 #[tokio::test]
922 async fn stalled_write_pump_should_terminate_and_reopen_after_stall_timeout() -> anyhow::Result<()> {
923 const STALL_TIMEOUT: std::time::Duration = std::time::Duration::from_millis(500);
924
925 let control = CountingControl::default();
926 let (mut tx_out, _rx_in) = process_stream_protocol(
927 BytesCodec::new(),
928 control.clone(),
929 crate::config::StreamProtocolConfig {
930 per_peer_channel_capacity: 4,
931 frame_writer_backpressure_bytes: 1,
934 egress_backpressure_timeout: STALL_TIMEOUT,
936 ..Default::default()
937 },
938 )
939 .await?;
940
941 let peer = PeerId::random();
942 let msg = BytesMut::from(&b"payload"[..]);
943
944 tx_out.send((peer, msg.clone())).await.context("first send")?;
946 assert!(
947 wait_for(2, || control.open_calls() >= 1).await,
948 "stream was never opened"
949 );
950
951 tokio::time::sleep(STALL_TIMEOUT / 2).await;
953 assert_eq!(
954 control.open_calls(),
955 1,
956 "stream reopened before the stall timeout elapsed — over-eager eviction would kill merely-slow peers"
957 );
958
959 let mut sends = 0;
962 while control.open_calls() < 2 && sends < 200 {
963 let _ = tx_out.send((peer, msg.clone())).await;
964 sends += 1;
965 tokio::time::sleep(std::time::Duration::from_millis(25)).await;
966 }
967
968 assert!(
969 wait_for(3, || control.open_calls() >= 2).await,
970 "stalled write pump never terminated: the stream was not reopened after the stall timeout (open_calls={}) \
971 — the pump parked forever and the peer became a permanent silent black hole, exactly the field failure \
972 mode",
973 control.open_calls(),
974 );
975
976 Ok(())
977 }
978
979 #[derive(Clone, Copy)]
987 enum Op {
988 Ready,
989 Pending,
990 Err,
991 }
992
993 fn apply(op: Op) -> Poll<Result<(), std::io::Error>> {
994 match op {
995 Op::Ready => Poll::Ready(Ok(())),
996 Op::Pending => Poll::Pending,
997 Op::Err => Poll::Ready(Err(std::io::Error::other("scripted sink error"))),
998 }
999 }
1000
1001 struct ScriptedSink {
1005 ready: Op,
1006 flush: Op,
1007 close: Op,
1008 }
1009
1010 impl Sink<u8> for ScriptedSink {
1011 type Error = std::io::Error;
1012
1013 fn poll_ready(self: Pin<&mut Self>, _cx: &mut TaskContext<'_>) -> Poll<Result<(), Self::Error>> {
1014 apply(self.ready)
1015 }
1016
1017 fn start_send(self: Pin<&mut Self>, _item: u8) -> Result<(), Self::Error> {
1018 Ok(())
1019 }
1020
1021 fn poll_flush(self: Pin<&mut Self>, _cx: &mut TaskContext<'_>) -> Poll<Result<(), Self::Error>> {
1022 apply(self.flush)
1023 }
1024
1025 fn poll_close(self: Pin<&mut Self>, _cx: &mut TaskContext<'_>) -> Poll<Result<(), Self::Error>> {
1026 apply(self.close)
1027 }
1028 }
1029
1030 #[derive(Default)]
1033 struct TransientFlushSink {
1034 flushed_once: bool,
1035 }
1036
1037 impl Sink<u8> for TransientFlushSink {
1038 type Error = std::io::Error;
1039
1040 fn poll_ready(self: Pin<&mut Self>, _cx: &mut TaskContext<'_>) -> Poll<Result<(), Self::Error>> {
1041 Poll::Ready(Ok(()))
1042 }
1043
1044 fn start_send(self: Pin<&mut Self>, _item: u8) -> Result<(), Self::Error> {
1045 Ok(())
1046 }
1047
1048 fn poll_flush(self: Pin<&mut Self>, cx: &mut TaskContext<'_>) -> Poll<Result<(), Self::Error>> {
1049 let this = self.get_mut();
1050 if this.flushed_once {
1051 Poll::Ready(Ok(()))
1052 } else {
1053 this.flushed_once = true;
1054 cx.waker().wake_by_ref();
1055 Poll::Pending
1056 }
1057 }
1058
1059 fn poll_close(self: Pin<&mut Self>, _cx: &mut TaskContext<'_>) -> Poll<Result<(), Self::Error>> {
1060 Poll::Ready(Ok(()))
1061 }
1062 }
1063
1064 #[tokio::test]
1065 async fn stall_guard_sink_should_error_when_poll_ready_stalls_past_timeout() {
1066 let mut sink = StallGuardSink::new(
1067 ScriptedSink {
1068 ready: Op::Pending,
1069 flush: Op::Ready,
1070 close: Op::Ready,
1071 },
1072 Duration::from_millis(150),
1073 );
1074 let res = tokio::time::timeout(Duration::from_secs(2), sink.send(1u8))
1075 .await
1076 .expect("StallGuardSink must resolve on its own timer, not hang");
1077 assert!(
1078 matches!(res, Err(EgressWriteError::Stalled { .. })),
1079 "a sink that never becomes ready must fail with Stalled, got {res:?}"
1080 );
1081 }
1082
1083 #[tokio::test]
1084 async fn stall_guard_sink_should_error_when_poll_flush_stalls_past_timeout() {
1085 let mut sink = StallGuardSink::new(
1086 ScriptedSink {
1087 ready: Op::Ready,
1088 flush: Op::Pending,
1089 close: Op::Ready,
1090 },
1091 Duration::from_millis(150),
1092 );
1093 let res = tokio::time::timeout(Duration::from_secs(2), sink.send(1u8))
1094 .await
1095 .expect("StallGuardSink must resolve on its own timer, not hang");
1096 assert!(
1097 matches!(res, Err(EgressWriteError::Stalled { .. })),
1098 "a sink that accepts but never flushes must fail with Stalled, got {res:?}"
1099 );
1100 }
1101
1102 #[tokio::test]
1103 async fn stall_guard_sink_should_error_when_poll_close_stalls_past_timeout() {
1104 let mut sink = StallGuardSink::new(
1105 ScriptedSink {
1106 ready: Op::Ready,
1107 flush: Op::Ready,
1108 close: Op::Pending,
1109 },
1110 Duration::from_millis(150),
1111 );
1112 let res = tokio::time::timeout(Duration::from_secs(2), sink.close())
1113 .await
1114 .expect("StallGuardSink must resolve on its own timer, not hang");
1115 assert!(
1116 matches!(res, Err(EgressWriteError::Stalled { .. })),
1117 "a sink that never closes must fail with Stalled, got {res:?}"
1118 );
1119 }
1120
1121 #[tokio::test]
1122 async fn stall_guard_sink_should_pass_through_a_healthy_sink_without_error() {
1123 let mut sink = StallGuardSink::new(
1124 ScriptedSink {
1125 ready: Op::Ready,
1126 flush: Op::Ready,
1127 close: Op::Ready,
1128 },
1129 Duration::from_millis(50),
1130 );
1131 for i in 0..100u8 {
1132 sink.send(i)
1133 .await
1134 .expect("a healthy sink must never be failed by the stall guard");
1135 }
1136 sink.close().await.expect("closing a healthy sink must succeed");
1137 }
1138
1139 #[tokio::test]
1140 async fn stall_guard_sink_should_surface_inner_errors_verbatim_not_as_a_stall() {
1141 let mut sink = StallGuardSink::new(
1142 ScriptedSink {
1143 ready: Op::Ready,
1144 flush: Op::Err,
1145 close: Op::Ready,
1146 },
1147 Duration::from_millis(150),
1148 );
1149 let res = sink.send(1u8).await;
1150 assert!(
1151 matches!(res, Err(EgressWriteError::Sink(_))),
1152 "an inner sink error must surface as Sink(_), not be masked as Stalled, got {res:?}"
1153 );
1154 }
1155
1156 #[tokio::test]
1157 async fn stall_guard_sink_should_not_error_on_a_transient_stall_that_recovers_in_time() {
1158 let mut sink = StallGuardSink::new(TransientFlushSink::default(), Duration::from_secs(10));
1161 tokio::time::timeout(Duration::from_secs(2), sink.send(1u8))
1162 .await
1163 .expect("a sink that recovers before the timeout must not hang")
1164 .expect("a sink that recovers before the timeout must not be failed as Stalled");
1165 }
1166
1167 #[derive(Clone, Debug)]
1170 struct HeadOfLineControl {
1171 stalled_peer: PeerId,
1172 stalled_io: GatedWriteIo,
1173 healthy_io: GatedWriteIo,
1174 open_calls: Arc<AtomicUsize>,
1175 stalled_opens: Arc<AtomicUsize>,
1176 }
1177
1178 #[async_trait]
1179 impl hopr_api::network::traits::NetworkStreamControl for HeadOfLineControl {
1180 fn accept(
1181 self,
1182 ) -> Result<impl Stream<Item = (PeerId, impl AsyncRead + AsyncWrite + Send)> + Send, impl std::error::Error>
1183 {
1184 Ok::<_, std::io::Error>(futures::stream::empty::<(PeerId, GatedWriteIo)>())
1185 }
1186
1187 async fn open(self, peer: PeerId) -> Result<impl AsyncRead + AsyncWrite + Send, impl std::error::Error> {
1188 self.open_calls.fetch_add(1, Ordering::Relaxed);
1189 if peer == self.stalled_peer {
1190 if self.stalled_opens.fetch_add(1, Ordering::Relaxed) == 0 {
1195 Ok::<GatedWriteIo, std::io::Error>(self.stalled_io.clone())
1196 } else {
1197 Err(std::io::Error::other("stalled peer refuses reopen"))
1198 }
1199 } else {
1200 Ok(self.healthy_io.clone())
1201 }
1202 }
1203 }
1204
1205 #[tokio::test]
1214 async fn stalled_peer_must_not_head_of_line_block_a_healthy_peer() -> anyhow::Result<()> {
1215 const STALL_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(1);
1216 const OVERFLOW: usize = 8;
1217
1218 let stalled_peer = PeerId::random();
1219 let healthy_peer = PeerId::random();
1220
1221 let stalled_io = GatedWriteIo {
1224 open: Arc::new(std::sync::atomic::AtomicBool::new(true)),
1225 ..Default::default()
1226 };
1227 let healthy_io = GatedWriteIo {
1228 open: Arc::new(std::sync::atomic::AtomicBool::new(true)),
1229 ..Default::default()
1230 };
1231
1232 let open_calls = Arc::new(AtomicUsize::new(0));
1233 let control = HeadOfLineControl {
1234 stalled_peer,
1235 stalled_io: stalled_io.clone(),
1236 healthy_io: healthy_io.clone(),
1237 open_calls: open_calls.clone(),
1238 stalled_opens: Arc::new(AtomicUsize::new(0)),
1239 };
1240
1241 let (mut tx_out, _rx_in) = process_stream_protocol(
1242 BytesCodec::new(),
1243 control,
1244 crate::config::StreamProtocolConfig {
1245 per_peer_channel_capacity: 2,
1246 frame_writer_backpressure_bytes: 1,
1248 egress_backpressure_timeout: STALL_TIMEOUT,
1249 ..Default::default()
1250 },
1251 )
1252 .await?;
1253
1254 let msg = BytesMut::from(&b"hello"[..]);
1255
1256 tx_out
1261 .send((stalled_peer, msg.clone()))
1262 .await
1263 .context("egress queue must accept priming stalled-peer packet")?;
1264 assert!(
1265 wait_for(2, || stalled_io.written() >= msg.len()).await,
1266 "priming packet was never written — stalled peer's stream/pump did not establish"
1267 );
1268 assert_eq!(
1269 open_calls.load(Ordering::Relaxed),
1270 1,
1271 "exactly one open expected during priming"
1272 );
1273
1274 stalled_io.gate_shut();
1277
1278 for _ in 0..OVERFLOW {
1282 tx_out
1283 .send((stalled_peer, msg.clone()))
1284 .await
1285 .context("egress queue must accept stalled-peer packet")?;
1286 }
1287
1288 tx_out
1289 .send((healthy_peer, msg.clone()))
1290 .await
1291 .context("egress queue must accept healthy-peer packet")?;
1292
1293 let delivered = wait_for(3, || healthy_io.written() >= msg.len()).await;
1294
1295 assert_eq!(
1296 stalled_io.written(),
1297 msg.len(),
1298 "only the priming packet should reach the stalled writer; the post-shut burst must not"
1299 );
1300
1301 assert!(
1302 delivered,
1303 "a single stalled peer head-of-line-blocked the shared egress drain: the healthy peer got {} of {} bytes \
1304 within 3s, while {OVERFLOW} overflow packets to the stalled peer each held the loop for {STALL_TIMEOUT:?}",
1305 healthy_io.written(),
1306 msg.len(),
1307 );
1308
1309 Ok(())
1310 }
1311
1312 #[derive(Clone, Debug)]
1313 struct BimodalOpenControl {
1314 slow_peer: PeerId,
1315 open_delay: std::time::Duration,
1316 slow_open_calls: Arc<AtomicUsize>,
1317 fast_open_calls: Arc<AtomicUsize>,
1318 }
1319
1320 impl BimodalOpenControl {
1321 #[allow(dead_code)]
1322 fn slow_open_calls(&self) -> usize {
1323 self.slow_open_calls.load(Ordering::Relaxed)
1324 }
1325
1326 fn fast_open_calls(&self) -> usize {
1327 self.fast_open_calls.load(Ordering::Relaxed)
1328 }
1329 }
1330
1331 #[async_trait]
1332 impl hopr_api::network::traits::NetworkStreamControl for BimodalOpenControl {
1333 fn accept(
1334 self,
1335 ) -> Result<impl Stream<Item = (PeerId, impl AsyncRead + AsyncWrite + Send)> + Send, impl std::error::Error>
1336 {
1337 Ok::<_, std::io::Error>(futures::stream::empty::<(PeerId, StalledWriteIo)>())
1338 }
1339
1340 async fn open(self, peer: PeerId) -> Result<impl AsyncRead + AsyncWrite + Send, impl std::error::Error> {
1341 if peer == self.slow_peer {
1342 self.slow_open_calls.fetch_add(1, Ordering::Relaxed);
1343 tokio::time::sleep(self.open_delay).await;
1344 return Err::<AsyncBinaryStreamChannel, _>(std::io::Error::other("slow peer cannot connect"));
1345 }
1346 self.fast_open_calls.fetch_add(1, Ordering::Relaxed);
1347 Ok::<_, std::io::Error>(AsyncBinaryStreamChannel::new())
1348 }
1349 }
1350
1351 #[tokio::test]
1352 async fn egress_should_not_hol_block_fast_peer_behind_slow_opens() -> anyhow::Result<()> {
1353 let slow_peer = PeerId::random();
1354 let fast_peer = PeerId::random();
1355
1356 let control = BimodalOpenControl {
1357 slow_peer,
1358 open_delay: std::time::Duration::from_millis(5_000),
1359 slow_open_calls: Default::default(),
1360 fast_open_calls: Default::default(),
1361 };
1362
1363 let (mut tx_out, _rx_in) = process_stream_protocol(
1364 BytesCodec::new(),
1365 control.clone(),
1366 crate::config::StreamProtocolConfig {
1367 stream_open_timeout: std::time::Duration::from_millis(2_000),
1368 ..Default::default()
1369 },
1370 )
1371 .await?;
1372
1373 let msg = BytesMut::from(&b"x"[..]);
1374
1375 for _ in 0..3 {
1376 tx_out
1377 .send((slow_peer, msg.clone()))
1378 .await
1379 .context("egress queue must accept slow-peer packet")?;
1380 }
1381 tx_out
1382 .send((fast_peer, msg.clone()))
1383 .await
1384 .context("egress queue must accept fast-peer packet")?;
1385
1386 let deadline = tokio::time::Instant::now() + std::time::Duration::from_millis(1_000);
1387 while control.fast_open_calls() < 1 && tokio::time::Instant::now() < deadline {
1388 tokio::time::sleep(std::time::Duration::from_millis(10)).await;
1389 }
1390
1391 assert!(
1392 control.fast_open_calls() >= 1,
1393 "fast peer's stream open was not called within 1 s — egress drain is likely head-of-line blocked by \
1394 slow-peer opens"
1395 );
1396
1397 Ok(())
1398 }
1399
1400 #[derive(Clone, Debug)]
1401 struct DelayedControl {
1402 open_delay: std::time::Duration,
1403 open_calls: Arc<AtomicUsize>,
1404 }
1405
1406 #[async_trait]
1407 impl hopr_api::network::traits::NetworkStreamControl for DelayedControl {
1408 fn accept(
1409 self,
1410 ) -> Result<impl Stream<Item = (PeerId, impl AsyncRead + AsyncWrite + Send)> + Send, impl std::error::Error>
1411 {
1412 Ok::<_, std::io::Error>(futures::stream::empty::<(PeerId, AsyncBinaryStreamChannel)>())
1413 }
1414
1415 async fn open(self, _peer: PeerId) -> Result<impl AsyncRead + AsyncWrite + Send, impl std::error::Error> {
1416 self.open_calls.fetch_add(1, Ordering::Relaxed);
1417 tokio::time::sleep(self.open_delay).await;
1418 Ok::<_, std::io::Error>(AsyncBinaryStreamChannel::new())
1419 }
1420 }
1421
1422 #[tokio::test]
1425 async fn egress_buffers_during_slow_open_then_drains() -> anyhow::Result<()> {
1426 let open_calls = Arc::new(AtomicUsize::new(0));
1427 let control = DelayedControl {
1428 open_delay: std::time::Duration::from_millis(100),
1429 open_calls: open_calls.clone(),
1430 };
1431
1432 let (mut tx_out, mut rx_in) = process_stream_protocol(
1433 BytesCodec::new(),
1434 control,
1435 crate::config::StreamProtocolConfig {
1436 per_peer_channel_capacity: 64,
1437 ..Default::default()
1438 },
1439 )
1440 .await?;
1441
1442 let peer = PeerId::random();
1443 let msg = BytesMut::from(&b"hello"[..]);
1444
1445 let n = 10usize;
1446 let expected_bytes = n * msg.len();
1447 for _ in 0..n {
1448 tx_out
1449 .send((peer, msg.clone()))
1450 .await
1451 .context("send into egress queue should succeed")?;
1452 }
1453
1454 assert!(
1455 wait_for(2, || open_calls.load(Ordering::Relaxed) >= 1).await,
1456 "stream was never opened"
1457 );
1458
1459 let mut received_bytes = 0usize;
1460 let deadline = tokio::time::Instant::now() + std::time::Duration::from_secs(2);
1461 while received_bytes < expected_bytes && tokio::time::Instant::now() < deadline {
1462 if let Ok(Some((_, bytes))) =
1463 tokio::time::timeout(std::time::Duration::from_millis(100), rx_in.next()).await
1464 {
1465 received_bytes += bytes.len();
1466 }
1467 }
1468
1469 assert!(
1470 received_bytes >= expected_bytes,
1471 "expected at least {expected_bytes} bytes to be delivered after stream open; got {received_bytes}"
1472 );
1473
1474 Ok(())
1475 }
1476
1477 #[derive(Clone, Default, Debug)]
1482 struct GatedWriteIo {
1483 open: Arc<std::sync::atomic::AtomicBool>,
1484 written: Arc<AtomicUsize>,
1485 waker: Arc<Mutex<Option<Waker>>>,
1486 }
1487
1488 impl GatedWriteIo {
1489 fn release(&self) {
1490 self.open.store(true, Ordering::Relaxed);
1491 if let Some(waker) = self.waker.lock().take() {
1492 waker.wake();
1493 }
1494 }
1495
1496 fn gate_shut(&self) {
1498 self.open.store(false, Ordering::Relaxed);
1499 }
1500
1501 fn written(&self) -> usize {
1502 self.written.load(Ordering::Relaxed)
1503 }
1504 }
1505
1506 impl AsyncRead for GatedWriteIo {
1507 fn poll_read(self: Pin<&mut Self>, _cx: &mut TaskContext<'_>, _buf: &mut [u8]) -> Poll<std::io::Result<usize>> {
1508 Poll::Pending
1509 }
1510 }
1511
1512 impl AsyncWrite for GatedWriteIo {
1513 fn poll_write(self: Pin<&mut Self>, cx: &mut TaskContext<'_>, buf: &[u8]) -> Poll<std::io::Result<usize>> {
1514 if self.open.load(Ordering::Relaxed) {
1515 self.written.fetch_add(buf.len(), Ordering::Relaxed);
1516 Poll::Ready(Ok(buf.len()))
1517 } else {
1518 *self.waker.lock() = Some(cx.waker().clone());
1519 Poll::Pending
1520 }
1521 }
1522
1523 fn poll_flush(self: Pin<&mut Self>, _cx: &mut TaskContext<'_>) -> Poll<std::io::Result<()>> {
1524 Poll::Ready(Ok(()))
1525 }
1526
1527 fn poll_close(self: Pin<&mut Self>, _cx: &mut TaskContext<'_>) -> Poll<std::io::Result<()>> {
1528 Poll::Ready(Ok(()))
1529 }
1530 }
1531
1532 #[derive(Clone, Debug)]
1533 struct GatedControl {
1534 io: GatedWriteIo,
1535 open_calls: Arc<AtomicUsize>,
1536 }
1537
1538 #[async_trait]
1539 impl hopr_api::network::traits::NetworkStreamControl for GatedControl {
1540 fn accept(
1541 self,
1542 ) -> Result<impl Stream<Item = (PeerId, impl AsyncRead + AsyncWrite + Send)> + Send, impl std::error::Error>
1543 {
1544 Ok::<_, std::io::Error>(futures::stream::empty::<(PeerId, GatedWriteIo)>())
1545 }
1546
1547 async fn open(self, _peer: PeerId) -> Result<impl AsyncRead + AsyncWrite + Send, impl std::error::Error> {
1548 self.open_calls.fetch_add(1, Ordering::Relaxed);
1549 Ok::<_, std::io::Error>(self.io.clone())
1550 }
1551 }
1552
1553 #[tokio::test]
1565 async fn egress_backpressures_when_open_channel_full() -> anyhow::Result<()> {
1566 let io = GatedWriteIo::default();
1567 let open_calls = Arc::new(AtomicUsize::new(0));
1568 let control = GatedControl {
1569 io: io.clone(),
1570 open_calls: open_calls.clone(),
1571 };
1572
1573 let (mut tx_out, _rx_in) = process_stream_protocol(
1575 BytesCodec::new(),
1576 control,
1577 crate::config::StreamProtocolConfig {
1578 per_peer_channel_capacity: 4,
1579 frame_writer_backpressure_bytes: 1,
1582 ..Default::default()
1583 },
1584 )
1585 .await?;
1586
1587 let peer = PeerId::random();
1588 let msg = BytesMut::from(&b"hello"[..]);
1589 let n = 50usize;
1590
1591 for _ in 0..n {
1597 tx_out
1598 .send((peer, msg.clone()))
1599 .await
1600 .context("send into egress queue should succeed")?;
1601 }
1602 assert!(
1603 wait_for(2, || open_calls.load(Ordering::Relaxed) >= 1).await,
1604 "stream was never opened"
1605 );
1606
1607 tokio::time::sleep(std::time::Duration::from_millis(200)).await;
1609
1610 io.release();
1612
1613 let channel_fill_bytes = 4 * msg.len();
1617 let max_delivered = channel_fill_bytes + msg.len();
1618 assert!(
1619 wait_for(3, || io.written() >= 1).await,
1620 "at least something must reach the wire under drop-newest overflow (wrote {} bytes)",
1621 io.written()
1622 );
1623 assert!(
1624 io.written() <= max_delivered,
1625 "overflow with drop-newest on a full open channel must not exceed channel capacity + 1 frame (wrote {} \
1626 bytes, max {max_delivered})",
1627 io.written()
1628 );
1629
1630 Ok(())
1631 }
1632}