1mod builder;
4mod config;
5
6pub use builder::{PacketPipelineBuilder, Unset};
7use bytes::Bytes;
8pub use config::{AcknowledgementPipelineConfig, PacketPipelineConfig, PoolArbitrationConfig};
9use futures::{SinkExt, StreamExt, future::Either};
10use futures_time::{future::FutureExt as TimeExt, stream::StreamExt as TimeStreamExt};
11use hopr_api::{
12 PeerId,
13 node::TicketEvent,
14 types::{crypto::prelude::*, internal::prelude::*},
15};
16use hopr_crypto_packet::{HoprPixSpec, HoprShareResolution, HoprSsaCommitmentState, HoprSurb};
17use hopr_protocol_app::prelude::*;
18use hopr_protocol_hopr::prelude::*;
19use hopr_protocol_pix::{
20 CoefficientIndex, ExitAcknowledgementShareProcessor, PixGroup, PixGroupRepr, PixParams, PixSpec, PolynomialIndex,
21 ShareResolution, SsaCommitmentProof, SsaCommitmentState, SsaId, TaggedEncryptedPartialSsaShare,
22};
23use hopr_utils::{
24 network_types::timeout::{SinkTimeoutError, TimeoutSinkExt, TimeoutStreamExt},
25 runtime::AbortableList,
26};
27use tracing::{Instrument, instrument::WithSubscriber};
28
29use crate::{
30 PeerProtocolCounterRegistry,
31 trace_context::{ContextSender, ContextSink, WithContext},
32};
33
34const DEFAULT_ACK_INPUT_CONCURRENCY: usize = 10;
37const DEFAULT_ACK_OUTPUT_CONCURRENCY: usize = 10;
40const QUEUE_SEND_TIMEOUT: std::time::Duration = std::time::Duration::from_millis(50);
41const PACKET_DECODING_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(5);
54const PACKET_ENCODING_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(5);
55
56const PIPELINE_CONCURRENCY_PER_CPU: usize = 8;
58
59fn default_pipeline_concurrency(available_parallelism: usize) -> usize {
69 available_parallelism.max(1) * PIPELINE_CONCURRENCY_PER_CPU
70}
71
72#[cfg(any(feature = "telemetry", test))]
87fn unack_rejection_metric_label<T: UnacknowledgedTicketProcessor>(error: &T::Error) -> &'static str {
88 if T::is_duplicate_ticket_index_error(error) {
89 "duplicate_ticket_index"
90 } else {
91 "unack_processing_error"
92 }
93}
94
95#[cfg(any(feature = "telemetry", test))]
103fn replay_rejection_metric_label() -> &'static str {
104 "replay"
105}
106
107#[cfg(all(feature = "telemetry", not(test)))]
108lazy_static::lazy_static! {
109 static ref METRIC_PACKET_COUNT: hopr_api::types::telemetry::MultiCounter = hopr_api::types::telemetry::MultiCounter::new(
110 "hopr_packets_count",
111 "Number of processed packets of different types (sent, received, forwarded)",
112 &["type"]
113 ).unwrap();
114 static ref METRIC_PACKET_REJECTED_COUNT: hopr_api::types::telemetry::MultiCounter = hopr_api::types::telemetry::MultiCounter::new(
115 "hopr_packet_rejected_count",
116 "Number of incoming packets rejected due various reasons; reason=duplicate_ticket_index counts packets dropped before forwarding because their ticket index was already accepted on the same channel and epoch (replayed index attack); reason=replay counts packets dropped during decoding, as soon as their replay token is derived, because it had already been seen (these are not acknowledged)",
117 &["reason"]
118 ).unwrap();
119 static ref METRIC_PACKET_DECODE_TIMEOUTS: hopr_api::types::telemetry::SimpleCounter = hopr_api::types::telemetry::SimpleCounter::new(
124 "hopr_packet_decode_timeouts_total",
125 "Number of incoming packets dropped due to decode timeout (sustained rate indicates Rayon pool saturation)"
126 ).unwrap();
127 static ref METRIC_VALIDATION_ERRORS: hopr_api::types::telemetry::MultiCounter = hopr_api::types::telemetry::MultiCounter::new(
128 "hopr_packet_ticket_validation_errors",
129 "Number of different ticket validation errors encountered during packet processing",
130 &["type"]
131 ).unwrap();
132 static ref METRIC_RECEIVED_ACKS: hopr_api::types::telemetry::MultiCounter = hopr_api::types::telemetry::MultiCounter::new(
133 "hopr_protocol_ack_received_count",
134 "Number of received acknowledgements",
135 &["valid"]
136 ).unwrap();
137 static ref METRIC_SENT_ACKS: hopr_api::types::telemetry::SimpleCounter = hopr_api::types::telemetry::SimpleCounter::new(
138 "hopr_protocol_ack_sent_count",
139 "Number of sent message acknowledgements"
140 ).unwrap();
141 static ref METRIC_TICKETS_COUNT: hopr_api::types::telemetry::MultiCounter = hopr_api::types::telemetry::MultiCounter::new(
142 "hopr_tickets_count",
143 "Number of tickets by type (winning, losing, rejected)",
144 &["type"]
145 ).unwrap();
146 static ref METRIC_SHARE_INSERT_FAILURES: hopr_api::types::telemetry::SimpleCounter =
147 hopr_api::types::telemetry::SimpleCounter::new(
148 "hopr_pix_share_insert_failures",
149 "Number of failed encrypted share insertions into the exit acknowledgement processor"
150 ).unwrap();
151}
152
153#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash, strum::Display)]
154pub enum PacketPipelineProcesses {
155 #[strum(to_string = "HOPR [msg] - ingress")]
156 MsgIn,
157 #[strum(to_string = "HOPR [msg] - egress")]
158 MsgOut,
159 #[strum(to_string = "HOPR [ack] - egress")]
160 AckOut,
161 #[strum(to_string = "HOPR [ack] - ingress")]
162 AckIn,
163 #[strum(to_string = "HOPR [msg] - mixer")]
164 Mixer,
165}
166
167async fn start_outgoing_packet_pipeline<AppOut, A, E, WOut, WOutErr>(
169 app_outgoing: AppOut,
170 encoder: std::sync::Arc<E>,
171 exit_ack_proc: Option<A>,
172 wire_outgoing: WOut,
173 counters: PeerProtocolCounterRegistry,
174 concurrency: usize,
175) where
176 AppOut: futures::Stream + Send + 'static,
177 AppOut::Item: Into<WithContext<(ResolvedTransportRouting<HoprSurb>, ApplicationDataOut)>> + Send,
178 A: ExitAcknowledgementShareProcessor<HoprPixSpec> + Clone + Send + Sync + 'static,
179 E: PacketEncoder + Send + Sync + 'static,
180 WOut: futures::Sink<(PeerId, Bytes), Error = SinkTimeoutError<WOutErr>> + Clone + Unpin + Send + 'static,
181 WOutErr: std::error::Error,
182{
183 let res = app_outgoing
184 .map(|message| {
188 let ((routing, data), span) = message.into().into_parts();
189 let encode_span = tracing::debug_span!(parent: &span, "transport.encode", peer = tracing::field::Empty);
190 let span_fields = encode_span.clone();
191 let span = if encode_span.is_disabled() { span } else { encode_span };
192 let encoder = encoder.clone();
193 let exit_ack_proc = exit_ack_proc.clone();
194 let counters = counters.clone();
195 async move {
196 hopr_transport_session::counters::ENCODE_STAGE_ENTRIES
197 .fetch_add(1, std::sync::atomic::Ordering::Relaxed);
198 let generation = data.packet_info.and_then(|info| info.surb_generation);
201 match hopr_utils::parallelize::cpu::spawn_encode_blocking(
202 move || {
203 encoder.encode_packet(
204 data.data.to_bytes(),
205 routing,
206 data.packet_info
207 .map(|data| data.signals_to_destination)
208 .unwrap_or_default(),
209 generation,
210 )
211 },
212 "packet_encode",
213 )
214 .timeout(futures_time::time::Duration::from(PACKET_ENCODING_TIMEOUT))
215 .await
216 {
217 Ok(Ok(Ok(packet))) => {
218 if !span_fields.is_disabled() {
219 span_fields.record("peer", packet.next_hop.to_peerid_str());
220 }
221 #[cfg(all(feature = "telemetry", not(test)))]
222 METRIC_PACKET_COUNT.increment(&["sent"]);
223 counters.get_or_create(&packet.next_hop).record_message_sent();
224
225 if let Some(exit_ack_proc) = exit_ack_proc.as_ref()
229 && let Some(encrypted_pix_share) =
230 packet.encrypted_pix_share.filter(|s| !s.partial_share.is_empty())
231 {
233 let spi = encrypted_pix_share.ssa_polynomial_id();
234 if let Err(error) = exit_ack_proc.insert_encrypted_share(
235 &packet.next_hop,
236 packet.ack_challenge,
237 encrypted_pix_share,
238 ) {
239 #[cfg(all(feature = "telemetry", not(test)))]
240 METRIC_SHARE_INSERT_FAILURES.increment();
241 tracing::error!(
242 ack_challenge = %packet.ack_challenge,
243 ssa_poly_id = ?spi,
244 %error,
245 "failed to insert encrypted share into the exit acknowledgement processor"
246 );
247 } else {
248 tracing::trace!("inserted new encrypted share into the exit acknowledgement processor");
249 }
250 }
251
252 tracing::trace!("protocol message out");
253 Some((packet.next_hop.into(), packet.data))
254 }
255 Ok(Ok(Err(error))) => {
256 tracing::error!(%error, "outgoing packet could not be encoded");
257 None
258 }
259 Ok(Err(error)) => {
260 tracing::error!(%error, "parallel processing of the outgoing packet failed");
261 None
262 }
263 Err(error) => {
264 tracing::error!(%error, "timeout while processing the outgoing packet");
265 hopr_utils::parallelize::cpu::ENCODE_TIMEOUT_DROPS
266 .fetch_add(1, std::sync::atomic::Ordering::Relaxed);
267 None
268 }
269 }
270 }
271 .instrument(span)
272 })
273 .buffered(concurrency)
274 .filter_map(futures::future::ready)
275 .map(Ok)
276 .forward_to_timeout(wire_outgoing)
277 .in_current_span()
278 .await;
279
280 if let Err(error) = res {
281 tracing::error!(
282 task = "transport (protocol - msg egress)",
283 %error,
284 "long-running background task finished with error"
285 );
286 } else {
287 tracing::warn!(
288 task = "transport (protocol - msg egress)",
289 "long-running background task finished"
290 )
291 }
292}
293
294#[allow(clippy::too_many_arguments)]
301async fn start_incoming_packet_pipeline<WIn, WOut, D, T, TEvt, AckIn, AckOut, AppIn, AppInErr>(
302 (wire_outgoing, wire_incoming): (WOut, WIn),
303 decoder: std::sync::Arc<D>,
304 ticket_proc: std::sync::Arc<T>,
305 ticket_events: TEvt,
306 (ack_outgoing, ack_incoming): (AckOut, AckIn),
307 app_incoming: AppIn,
308 counters: super::counters::PeerProtocolCounterRegistry,
309 concurrency: usize,
310) where
311 WIn: futures::Stream<Item = (PeerId, Bytes)> + Send + 'static,
312 WOut: futures::Sink<(PeerId, Bytes)> + Clone + Unpin + Send + 'static,
313 WOut::Error: std::error::Error,
314 D: PacketDecoder + Sync + Send + 'static,
315 T: UnacknowledgedTicketProcessor + Send + 'static,
316 TEvt: futures::Sink<TicketEvent> + Clone + Unpin + Send + 'static,
317 TEvt::Error: std::error::Error,
318 AckIn: futures::Sink<(OffchainPublicKey, Vec<Acknowledgement>)> + Send + Unpin + Clone + 'static,
319 AckIn::Error: std::error::Error,
320 AckOut: futures::Sink<(OffchainPublicKey, Option<HalfKey>)> + Send + Unpin + Clone + 'static,
321 AckOut::Error: std::error::Error,
322 AppIn: futures::Sink<(HoprPseudonym, ApplicationDataIn), Error = SinkTimeoutError<AppInErr>> + Send + 'static,
323 AppInErr: std::error::Error,
324{
325 let ack_outgoing_success = ack_outgoing.clone();
326 let ack_outgoing_failure = ack_outgoing;
327 let ticket_proc_success = ticket_proc;
328 let ticket_events_unack = ticket_events.clone();
329 futures::pin_mut!(app_incoming);
330
331 let res = wire_incoming
332 .map(move |(peer, data)| {
337 let span = tracing::debug_span!(
338 parent: None,
339 "transport.receive",
340 %peer,
341 packet_id = crate::trace_context::next_packet_id(),
342 next_hop = tracing::field::Empty,
343 session_id = tracing::field::Empty
344 );
345 let output_span = span.clone();
346 let decoder = decoder.clone();
347 let mut ack_outgoing_failure = ack_outgoing_failure.clone();
348 let mut ticket_events_reject = ticket_events.clone();
349
350 async move {
351 tracing::trace!("protocol message in");
352 match hopr_utils::parallelize::cpu::spawn_decode_blocking(move || decoder.decode(peer, data), "packet_decode")
353 .timeout(futures_time::time::Duration::from(PACKET_DECODING_TIMEOUT))
354 .await {
355 Ok(Ok(Ok(packet))) => {
356 if let IncomingPacket::Final(packet) = &packet {
357 output_span.record("session_id", tracing::field::display(packet.sender));
358 } else if let IncomingPacket::Forwarded(packet) = &packet
359 && !output_span.is_disabled()
360 {
361 output_span.record("next_hop", packet.next_hop.to_peerid_str());
362 }
363 tracing::trace!(?packet, "successfully decoded incoming packet");
364 Some(WithContext::new(packet, output_span))
365 },
366 Ok(Ok(Err(IncomingPacketError::Undecodable(error)))) => {
367 tracing::trace!(%error, "not sending ack back on undecodable packet - possible adversarial behavior");
371
372 #[cfg(all(feature = "telemetry", not(test)))]
373 METRIC_PACKET_REJECTED_COUNT.increment(&["undecodable"]);
374
375 None
376 },
377 Ok(Ok(Err(IncomingPacketError::Replay(sender)))) => {
378 tracing::debug!(%peer, sender = sender.to_peerid_str(), "not sending ack back on replayed packet");
384
385 #[cfg(all(feature = "telemetry", not(test)))]
386 METRIC_PACKET_REJECTED_COUNT.increment(&[replay_rejection_metric_label()]);
387
388 None
389 },
390 Ok(Ok(Err(IncomingPacketError::ProcessingError(sender, error)))) => {
391 tracing::error!(%peer, %error, "failed to process the decoded packet");
392 ack_outgoing_failure
394 .send((*sender, None))
395 .await
396 .unwrap_or_else(|error| {
397 tracing::error!(%error, "failed to send ack to the egress queue");
398 });
399
400 #[cfg(all(feature = "telemetry", not(test)))]
401 METRIC_PACKET_REJECTED_COUNT.increment(&["processing_error"]);
402
403 None
404 },
405 Ok(Ok(Err(IncomingPacketError::InvalidTicket(sender, error)))) => {
406 tracing::error!(%peer, %error, "failed to validate ticket on the received packet");
407 if let Err(error) = ticket_events_reject
408 .send(TicketEvent::RejectedTicket(error.ticket, error.issuer))
409 .await {
410 tracing::error!(%error, "failed to notify invalid ticket rejection");
411 }
412 ack_outgoing_failure
414 .send((*sender, None))
415 .await
416 .unwrap_or_else(|error| {
417 tracing::error!(%error, "failed to send ack to the egress queue");
418 });
419
420 #[cfg(all(feature = "telemetry", not(test)))]
421 {
422 METRIC_VALIDATION_ERRORS.increment(&[error.kind.as_metric_label()]);
423 METRIC_PACKET_REJECTED_COUNT.increment(&["invalid_ticket"]);
424 METRIC_TICKETS_COUNT.increment(&["rejected"]);
425 }
426
427 None
428 }
429 Ok(Err(error)) => {
430 tracing::error!(%error, "parallel processing of the incoming packet failed");
431 None
432 },
433 Err(_) => {
434 tracing::error!(
436 %peer,
437 timeout_ms = PACKET_DECODING_TIMEOUT.as_millis() as u64,
438 "dropped incoming packet: decode timeout - check the 'hopr_rayon_queue_wait_seconds' metric for pool saturation"
439 );
440 hopr_utils::parallelize::cpu::DECODE_TIMEOUT_DROPS
441 .fetch_add(1, std::sync::atomic::Ordering::Relaxed);
442 #[cfg(all(feature = "telemetry", not(test)))]
443 {
444 METRIC_PACKET_DECODE_TIMEOUTS.increment();
445 METRIC_PACKET_REJECTED_COUNT.increment(&["timeout"]);
446 }
447
448 None
449 }
450 }
451 }.instrument(tracing::debug_span!(parent: &span, "incoming_packet_decode")).instrument(span)
452 })
453 .buffered(concurrency)
454 .filter_map(futures::future::ready)
455 .map(move |message| {
460 let (packet, span) = message.into_parts();
461 let output_span = span.clone();
462 let processing = match packet {
463 IncomingPacket::Acknowledgement(ack) => {
464 let mut ack_incoming = ack_incoming.clone();
465 let counters = counters.clone();
466 Either::Left(async move {
467 let IncomingAcknowledgementPacket { previous_hop, received_acks, .. } = *ack;
468 tracing::trace!(num_acks = received_acks.len(), "incoming acknowledgements");
469 counters.get_or_create(&previous_hop).record_acks_received(received_acks.len() as u64);
470
471 ack_incoming
472 .send((previous_hop, received_acks))
473 .await
474 .unwrap_or_else(|error| {
475 tracing::error!(%error, "failed dispatching received acknowledgement to the ticket ack queue");
476 });
477
478 None
480 })
481 }
482 IncomingPacket::Final(final_packet) => {
483 let mut ack_outgoing_success = ack_outgoing_success.clone();
484 Either::Right(Either::Left(async move {
485 let IncomingFinalPacket {
486 previous_hop,
487 sender,
488 plain_text,
489 ack_key,
490 info,
491 ..
492 } = *final_packet;
493 tracing::trace!("incoming final packet");
494
495 ack_outgoing_success
497 .send((previous_hop, Some(ack_key)))
498 .await
499 .unwrap_or_else(|error| {
500 tracing::error!(%error, "failed to send ack to the egress queue");
501 });
502
503 #[cfg(all(feature = "telemetry", not(test)))]
504 METRIC_PACKET_COUNT.increment(&["received"]);
505
506 Some(WithContext::new((sender, plain_text, info), output_span))
507 }))
508 }
509 IncomingPacket::Forwarded(fwd_packet) => {
510 let ticket_proc = ticket_proc_success.clone();
511 let mut wire_outgoing = wire_outgoing.clone();
512 let mut ack_outgoing_success = ack_outgoing_success.clone();
513 let mut ticket_events_reject = ticket_events_unack.clone();
514 let counters = counters.clone();
515 Either::Right(Either::Right(async move {
516 let IncomingForwardedPacket {
517 previous_hop,
518 next_hop,
519 data,
520 ack_key_prev_hop,
521 ack_challenge,
522 received_ticket,
523 ..
524 } = *fwd_packet;
525 if let Err(error) = ticket_proc.insert_unacknowledged_ticket(&next_hop, ack_challenge, received_ticket) {
527 if T::is_duplicate_ticket_index_error(&error) {
530 tracing::warn!(
531 previous_hop = previous_hop.to_peerid_str(),
532 next_hop = next_hop.to_peerid_str(),
533 %error,
534 "rejected packet with an already accepted ticket index"
535 );
536 } else {
537 tracing::error!(
538 previous_hop = previous_hop.to_peerid_str(),
539 next_hop = next_hop.to_peerid_str(),
540 %error,
541 "failed to insert unacknowledged ticket into the ticket processor"
542 );
543 }
544
545 if let Err(error) = ticket_events_reject
547 .send(TicketEvent::RejectedTicket(
548 Box::new(*received_ticket.verified_ticket()),
549 Some(*received_ticket.ticket.verified_issuer()),
550 ))
551 .await {
552 tracing::error!(%error, "failed to notify rejection of the ticket not accepted by the ticket processor");
553 }
554 ack_outgoing_success
556 .send((previous_hop, None))
557 .await
558 .unwrap_or_else(|error| {
559 tracing::error!(%error, "failed to send ack to the egress queue");
560 });
561
562 #[cfg(all(feature = "telemetry", not(test)))]
563 {
564 METRIC_PACKET_REJECTED_COUNT.increment(&[unack_rejection_metric_label::<T>(&error)]);
565 METRIC_TICKETS_COUNT.increment(&["rejected"]);
566 }
567
568 return None;
569 }
570
571 tracing::trace!("forwarding packet to the next hop");
573
574 match wire_outgoing.send((next_hop.into(), data)).await {
575 Ok(()) => {
576 counters.get_or_create(&next_hop).record_message_sent();
577
578 #[cfg(all(feature = "telemetry", not(test)))]
579 METRIC_PACKET_COUNT.increment(&["forwarded"]);
580 }
581 Err(error) => {
582 tracing::error!(%error, "failed to forward a packet to the transport layer");
583 return None;
584 }
585 }
586
587 tracing::trace!("acknowledging forwarded packet back");
589 ack_outgoing_success
590 .send((previous_hop, Some(ack_key_prev_hop)))
591 .await
592 .unwrap_or_else(|error| {
593 tracing::error!(%error, "failed to send ack to the egress queue");
594 });
595
596 None
597 }))
598 }
599 };
600 processing.instrument(tracing::debug_span!(parent: &span, "transport.process")).instrument(span)
601 })
602 .buffered(concurrency)
603 .filter_map(|maybe_data| futures::future::ready(
604 maybe_data
606 .and_then(|message| {
607 let ((sender, data, aux_info), span) = message.into_parts();
608 let _entered = span.enter();
609 ApplicationData::try_from(data)
610 .inspect_err(|error| tracing::error!(%sender, %error, "failed to decode application data"))
611 .ok()
612 .map(|data| WithContext::new((sender, ApplicationDataIn {
613 data,
614 packet_info: IncomingPacketInfo {
615 signals_from_sender: aux_info.packet_signals,
616 num_saved_surbs: aux_info.num_surbs,
617 num_evicted_surbs: aux_info.num_evicted_surbs,
618 min_return_path_hops: aux_info.min_return_path_hops,
619 }
620 }), span.clone()))
621 })
622 ))
623 .map(Ok)
624 .forward_to_timeout(ContextSink::new(app_incoming))
625 .in_current_span()
626 .await;
627
628 if let Err(error) = res {
629 tracing::error!(
630 task = "transport (protocol - msg ingress)",
631 %error,
632 "long-running background task finished with error"
633 );
634 } else {
635 tracing::warn!(
636 task = "transport (protocol - msg ingress)",
637 "long-running background task finished"
638 )
639 }
640}
641
642async fn start_outgoing_ack_pipeline<AckOut, E, WOut>(
643 ack_outgoing: AckOut,
644 encoder: std::sync::Arc<E>,
645 cfg: AcknowledgementPipelineConfig,
646 packet_key: OffchainKeypair,
647 wire_outgoing: WOut,
648) where
649 AckOut: futures::Stream + Send + 'static,
650 AckOut::Item: Into<WithContext<(OffchainPublicKey, Option<HalfKey>)>> + Send,
651 E: PacketEncoder + Sync + Send + 'static,
652 WOut: futures::Sink<(PeerId, Bytes)> + Clone + Unpin + Send + 'static,
653 WOut::Error: std::error::Error,
654{
655 ack_outgoing
656 .map(move |message| {
657 let ((destination, maybe_ack_key), span) = message.into().into_parts();
658 let _entered = span.enter();
659 let packet_key = packet_key.clone();
660 let ack = maybe_ack_key
662 .map(|ack_key| VerifiedAcknowledgement::new(ack_key, &packet_key))
663 .unwrap_or_else(|| VerifiedAcknowledgement::random(&packet_key));
664 (destination, WithContext::new(ack, span.clone()))
665 })
666 .buffer(futures_time::time::Duration::from(cfg.ack_buffer_interval))
667 .filter(|acks| futures::future::ready(!acks.is_empty()))
668 .scan(
676 halfbrown::HashMap::<OffchainPublicKey, Vec<WithContext<VerifiedAcknowledgement>>, ahash::RandomState>::with_capacity_and_hasher(
677 cfg.ack_grouping_capacity,
678 ahash::RandomState::default(),
679 ),
680 |groups, buffered_acks| {
681 for (dst, ack) in buffered_acks {
682 groups.entry(dst).or_default().push(ack);
683 }
684 tracing::trace!(
685 num_groups = groups.len(),
686 num_acks = groups.values().map(|v| v.len()).sum::<usize>(),
687 "acknowledgements grouped"
688 );
689 let drained: Vec<_> = groups.drain().collect();
690 futures::future::ready(Some(futures::stream::iter(drained)))
691 },
692 )
693 .flatten()
694 .for_each_concurrent(
695 cfg.ack_output_concurrency.filter(|&n| n > 0).unwrap_or(DEFAULT_ACK_OUTPUT_CONCURRENCY),
696 move |(destination, acks)| {
697 let span = tracing::debug_span!(parent: None, "outgoing_ack_batch", peer = destination.to_peerid_str());
698 for ack in &acks {
699 span.follows_from(ack.span());
700 }
701 let acks = acks.into_iter().map(|ack| ack.value).collect::<Vec<_>>();
702 let encoder = encoder.clone();
703 let mut wire_outgoing = wire_outgoing.clone();
704 async move {
705 let c = acks.chunks(MAX_ACKNOWLEDGEMENTS_BATCH_SIZE).map(|c| c.to_vec()).collect::<Vec<_>>();
708 for ack_chunk in c {
709 let encoder = encoder.clone();
710 #[cfg(all(feature = "telemetry", not(test)))]
711 let ack_chunk_len = ack_chunk.len() as u64;
712 match hopr_utils::parallelize::cpu::spawn_fifo_blocking(move || encoder.encode_acknowledgements(&ack_chunk, &destination), "ack_encode").await {
713 Ok(Ok(ack_packet)) => {
714 wire_outgoing
715 .feed((ack_packet.next_hop.into(), ack_packet.data))
716 .await
717 .unwrap_or_else(|error| {
718 tracing::error!(%error, "failed to forward an acknowledgement to the transport layer");
719 });
720
721 #[cfg(all(feature = "telemetry", not(test)))]
722 METRIC_SENT_ACKS.increment_by(ack_chunk_len);
723 }
724 Ok(Err(error)) => tracing::error!(%error, "failed to encode acknowledgements"),
725 Err(error) => tracing::error!(%error, "parallel processing of the outgoing acknowledgements failed"),
726 }
727 }
728 if let Err(error) = wire_outgoing.flush().await {
729 tracing::error!(%error, "failed to flush acknowledgements batch to the transport layer");
730 }
731 tracing::trace!("acknowledgements out");
732 }.instrument(span)
733 }
734 )
735 .in_current_span()
736 .await;
737
738 tracing::warn!(
739 task = "transport (protocol - ack outgoing)",
740 "long-running background task finished"
741 );
742}
743
744async fn start_exit_incoming_ack_pipeline<S, AckIn, A, SEvt>(
750 ack_incoming: AckIn,
751 exit_proc: A,
752 recovered_ssa: SEvt,
753 concurrency: usize,
754) where
755 S: PixSpec,
756 AckIn: futures::Stream + Send + 'static,
757 AckIn::Item: Into<WithContext<(OffchainPublicKey, Vec<Acknowledgement>)>> + Send,
758 A: ExitAcknowledgementShareProcessor<S> + Clone + Send + Sync + 'static,
759 SEvt: futures::Sink<ShareResolution<S::Pseudonym, S::AddressPrivateKey>> + Clone + Unpin + Send + 'static,
760 SEvt::Error: std::error::Error,
761{
762 ack_incoming
766 .for_each_concurrent(concurrency, move |message| {
767 let ((peer, acks), span) = message.into().into_parts();
768 let exit_proc = exit_proc.clone();
769 let exit_proc_check = exit_proc.clone();
770 let mut recovered_ssa = recovered_ssa.clone();
771 async move {
772 tracing::trace!(num = acks.len(), "received acknowledgements");
773
774 if !exit_proc_check.has_pending_shares(&peer) {
775 tracing::trace!("no pending pix shares, skipping acknowledgement");
776 return;
777 }
778
779 match hopr_utils::parallelize::cpu::spawn_fifo_blocking(
780 move || exit_proc.acknowledge_shares(peer, acks),
781 "exit_ack_decode",
782 )
783 .await
784 {
785 Ok(Ok(ssa_priv_keys)) => {
786 if let Err(error) = recovered_ssa
787 .send_all(&mut futures::stream::iter(ssa_priv_keys.into_iter().map(Ok)))
788 .await
789 {
790 tracing::error!(%peer, %error, "failed to send pix resolution");
791 }
792 }
793 Ok(Err(ref error)) if exit_proc_check.is_expected_error(error) => {
794 tracing::debug!(%peer, %error, "expected error while acknowledging pix share")
795 }
796 Ok(Err(error)) => {
797 tracing::error!(%peer, %error, "failed to acknowledge pix share")
798 }
799 Err(error) => {
800 tracing::error!(%peer, %error, "failed to spawn pix share acknowledgement")
801 }
802 }
803 }
804 .instrument(tracing::debug_span!(parent: &span, "incoming_ack_batch"))
805 .instrument(span)
806 })
807 .in_current_span()
808 .await;
809
810 tracing::warn!(
811 task = "transport (protocol - pix share acknowledgement)",
812 "long-running background task finished"
813 );
814}
815
816async fn start_drain_incoming_ack_pipeline<AckIn>(ack_incoming: AckIn)
824where
825 AckIn: futures::Stream + Send + 'static,
826 AckIn::Item: Into<WithContext<(OffchainPublicKey, Vec<Acknowledgement>)>> + Send,
827{
828 ack_incoming
829 .for_each(move |message| {
830 let ((_, acks), span) = message.into().into_parts();
831 let _entered = span.enter();
832 tracing::trace!(num = acks.len(), "received acknowledgements (drained, not processed)");
833 futures::future::ready(())
834 })
835 .in_current_span()
836 .await;
837
838 tracing::warn!(
839 task = "transport (protocol - ticket acknowledgement drain)",
840 "long-running background task finished"
841 );
842}
843
844async fn start_relay_incoming_ack_pipeline<AckIn, T, TEvt, A, SEvt>(
845 ack_incoming: AckIn,
846 ticket_events: TEvt,
847 ticket_proc: std::sync::Arc<T>,
848 exit_ack_proc: std::sync::Arc<A>,
849 recovered_ssa: SEvt,
850 concurrency: usize,
851) where
852 AckIn: futures::Stream + Send + 'static,
853 AckIn::Item: Into<WithContext<(OffchainPublicKey, Vec<Acknowledgement>)>> + Send,
854 T: UnacknowledgedTicketProcessor + Sync + Send + 'static,
855 TEvt: futures::Sink<TicketEvent> + Clone + Unpin + Send + 'static,
856 TEvt::Error: std::error::Error,
857 A: ExitAcknowledgementShareProcessor<HoprPixSpec> + Send + Sync + 'static,
858 SEvt: futures::Sink<HoprShareResolution> + Clone + Unpin + Send + 'static,
859 SEvt::Error: std::error::Error,
860{
861 ack_incoming
862 .for_each_concurrent(concurrency, move |message| {
863 let ((peer, acks), span) = message.into().into_parts();
864 let ticket_proc = ticket_proc.clone();
865 let mut ticket_evt = ticket_events.clone();
866 let exit_proc = exit_ack_proc.clone();
867 let mut ssa_evt = recovered_ssa.clone();
868 async move {
869 tracing::trace!(num = acks.len(), "received acknowledgements");
870 if acks.is_empty() {
871 return;
872 }
873
874 let exit_proc_for_spawn = exit_proc.clone();
887 let (ticket_result, exit_result) = if exit_proc.has_pending_shares(&peer) {
888 let ack_clone = acks.clone();
889 let ticket_fut = hopr_utils::parallelize::cpu::spawn_fifo_blocking(
890 move || ticket_proc.acknowledge_tickets(peer, ack_clone),
891 "ack_decode",
892 );
893 let exit_fut = hopr_utils::parallelize::cpu::spawn_fifo_blocking(
894 move || exit_proc_for_spawn.acknowledge_shares(peer, acks),
895 "exit_ack_decode",
896 );
897 let (t_r, e_r) = futures::future::join(ticket_fut, exit_fut).await;
898 (t_r, Some(e_r))
899 } else {
900 let ticket_fut = hopr_utils::parallelize::cpu::spawn_fifo_blocking(
901 move || ticket_proc.acknowledge_tickets(peer, acks),
902 "ack_decode",
903 );
904 let t_r = ticket_fut.await;
905 (t_r, None)
906 };
907
908 match ticket_result {
909 Ok(Ok(resolutions)) if !resolutions.is_empty() => {
910 let resolutions_iter = resolutions.into_iter().filter_map(|resolution| match resolution {
911 ResolvedAcknowledgement::RelayingWin(redeemable_ticket) => {
912 tracing::trace!("received ack for a winning ticket");
913 #[cfg(all(feature = "telemetry", not(test)))]
914 {
915 METRIC_RECEIVED_ACKS.increment(&["true"]);
916 METRIC_TICKETS_COUNT.increment(&["winning"]);
917 }
918 Some(Ok(TicketEvent::WinningTicket(redeemable_ticket)))
919 }
920 ResolvedAcknowledgement::RelayingLoss(_) => {
921 tracing::trace!("received ack for a losing ticket");
923 #[cfg(all(feature = "telemetry", not(test)))]
924 {
925 METRIC_RECEIVED_ACKS.increment(&["true"]);
926 METRIC_TICKETS_COUNT.increment(&["losing"]);
927 }
928 None
929 }
930 });
931
932 if let Err(error) = ticket_evt.send_all(&mut futures::stream::iter(resolutions_iter)).await {
934 tracing::error!(%error, "failed to notify ticket resolutions");
935 }
936 }
937 Ok(Ok(_)) => {
938 tracing::debug!("acknowledgement batch could not acknowledge any ticket");
939 }
940 Ok(Err(TicketAcknowledgementError::UnexpectedAcknowledgement)) => {
941 tracing::trace!("received unexpected acknowledgement");
944 }
945 Ok(Err(error)) => {
946 tracing::error!(%error, "failed to acknowledge ticket");
947 }
948 Err(error) => {
949 tracing::error!(%error, "parallel processing of the incoming acknowledgements failed")
950 }
951 }
952
953 if let Some(result) = exit_result {
955 match result {
956 Ok(Ok(ssa_priv_keys)) if !ssa_priv_keys.is_empty() => {
957 if let Err(error) = ssa_evt
958 .send_all(&mut futures::stream::iter(ssa_priv_keys.into_iter().map(Ok)))
959 .await
960 {
961 tracing::error!(%peer, %error, "failed to send pix resolution");
962 }
963 }
964 Ok(Ok(_)) => {
965 tracing::trace!(%peer, "empty pix share resolution batch");
966 }
967 Ok(Err(ref error)) if exit_proc.is_expected_error(error) => {
968 tracing::trace!(%peer, %error, "pix share acknowledgement skipped");
969 }
970 Ok(Err(error)) => {
971 tracing::error!(%peer, %error, "pix share acknowledgement failed");
972 }
973 Err(error) => {
974 tracing::error!(%peer, %error, "failed to spawn pix share acknowledgement")
975 }
976 }
977 }
978 }
979 .instrument(tracing::debug_span!(parent: &span, "incoming_ack_batch"))
980 .instrument(span)
981 })
982 .in_current_span()
983 .await;
984
985 tracing::warn!(
986 task = "transport (protocol - ticket acknowledgement)",
987 "long-running background task finished"
988 );
989}
990#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)]
999pub enum NodeType {
1000 Relay,
1001 Entry,
1002 Exit,
1003}
1004
1005#[derive(Debug, Default, Copy, Clone)]
1010#[doc(hidden)]
1011pub struct NoopTicketProcessor;
1012
1013impl UnacknowledgedTicketProcessor for NoopTicketProcessor {
1014 type Error = std::convert::Infallible;
1015
1016 #[inline]
1017 fn is_duplicate_ticket_index_error(_: &Self::Error) -> bool {
1018 false
1019 }
1020
1021 #[inline]
1022 fn insert_unacknowledged_ticket(
1023 &self,
1024 _: &OffchainPublicKey,
1025 _: HalfKeyChallenge,
1026 _: UnacknowledgedTicket,
1027 ) -> Result<(), Self::Error> {
1028 Ok(())
1029 }
1030
1031 #[inline]
1032 fn acknowledge_tickets(
1033 &self,
1034 _: OffchainPublicKey,
1035 _: Vec<Acknowledgement>,
1036 ) -> Result<Vec<ResolvedAcknowledgement>, TicketAcknowledgementError<Self::Error>> {
1037 Ok(Vec::with_capacity(0))
1038 }
1039}
1040
1041#[derive(Debug, Default, Copy, Clone)]
1042#[doc(hidden)]
1043pub struct NopExitAcknowledgementShareProcessor;
1044
1045impl ExitAcknowledgementShareProcessor<HoprPixSpec> for NopExitAcknowledgementShareProcessor {
1046 type Error = std::convert::Infallible;
1047
1048 fn has_pending_shares(&self, _: &OffchainPublicKey) -> bool {
1049 false
1050 }
1051
1052 fn new_exit_commitment(&self, _: SsaId<HoprPseudonym>, _: PixParams) -> Result<PixGroup<HoprPixSpec>, Self::Error> {
1064 Ok(PixGroup::<HoprPixSpec>::default())
1065 }
1066
1067 #[inline]
1068 fn insert_coefficient_commitments(
1069 &self,
1070 ssa_id: SsaId<SimplePseudonym>,
1071 _: CoefficientIndex,
1072 _: Option<SsaCommitmentProof<HoprPixSpec>>,
1073 _: impl Iterator<Item = (PolynomialIndex, PixGroupRepr<HoprPixSpec>)>,
1074 ) -> Result<HoprSsaCommitmentState, Self::Error> {
1075 Ok(SsaCommitmentState::new(ssa_id))
1076 }
1077
1078 #[inline]
1079 fn retire_ssa(&self, _: SsaId<HoprPseudonym>) {}
1080
1081 #[inline]
1082 fn insert_encrypted_share(
1083 &self,
1084 _: &OffchainPublicKey,
1085 _: HalfKeyChallenge,
1086 _: TaggedEncryptedPartialSsaShare<HoprPixSpec>,
1087 ) -> Result<(), Self::Error> {
1088 Ok(())
1089 }
1090
1091 #[inline]
1092 fn acknowledge_shares(
1093 &self,
1094 _: OffchainPublicKey,
1095 _: Vec<Acknowledgement>,
1096 ) -> Result<Vec<HoprShareResolution>, Self::Error> {
1097 Ok(Vec::with_capacity(0))
1098 }
1099}
1100
1101#[allow(clippy::too_many_arguments)]
1110#[tracing::instrument(skip_all, level = "trace", fields(me = packet_key.public().to_peerid_str()))]
1111pub(super) fn run_packet_pipeline_inner<WIn, WOut, A, C, D, T, TEvt, SEvt, AppOut, AppIn>(
1112 node_type: NodeType,
1113 packet_key: OffchainKeypair,
1114 wire_msg: (WOut, WIn),
1115 codec: (C, D),
1116 ticket_proc: T,
1117 exit_ack_proc: A,
1118 ticket_events: TEvt,
1119 ssa_events: SEvt,
1120 cfg: PacketPipelineConfig,
1121 api: (AppOut, AppIn),
1122 counters: PeerProtocolCounterRegistry,
1123) -> AbortableList<PacketPipelineProcesses>
1124where
1125 WOut: futures::Sink<(PeerId, Bytes)> + Clone + Unpin + Send + 'static,
1126 WOut::Error: std::error::Error,
1127 WIn: futures::Stream<Item = (PeerId, Bytes)> + Send + 'static,
1128 C: PacketEncoder + Sync + Send + 'static,
1129 D: PacketDecoder + Sync + Send + 'static,
1130 A: ExitAcknowledgementShareProcessor<HoprPixSpec> + Send + Sync + 'static,
1131 T: UnacknowledgedTicketProcessor + Sync + Send + 'static,
1132 SEvt: futures::Sink<HoprShareResolution> + Clone + Unpin + Send + 'static,
1133 SEvt::Error: std::error::Error,
1134 TEvt: futures::Sink<TicketEvent> + Clone + Unpin + Send + 'static,
1135 TEvt::Error: std::error::Error,
1136 AppOut: futures::Sink<(HoprPseudonym, ApplicationDataIn)> + Send + 'static,
1137 AppOut::Error: std::error::Error,
1138 AppIn: futures::Stream + Send + 'static,
1139 AppIn::Item: Into<WithContext<(ResolvedTransportRouting<HoprSurb>, ApplicationDataOut)>> + Send,
1140{
1141 let mut processes = AbortableList::default();
1142
1143 #[cfg(all(feature = "telemetry", not(test)))]
1144 {
1145 lazy_static::initialize(&METRIC_PACKET_COUNT);
1147 lazy_static::initialize(&METRIC_PACKET_DECODE_TIMEOUTS);
1148 lazy_static::initialize(&METRIC_PACKET_REJECTED_COUNT);
1149 lazy_static::initialize(&METRIC_VALIDATION_ERRORS);
1150 }
1151
1152 let (outgoing_ack_tx, outgoing_ack_rx) = hopr_utils::network_types::crossfire_sink::bounded_sink_channel::<
1153 WithContext<(OffchainPublicKey, Option<HalfKey>)>,
1154 >(cfg.ack_config.ack_out_buffer_size);
1155
1156 let (incoming_ack_tx, incoming_ack_rx) = hopr_utils::network_types::crossfire_sink::bounded_sink_channel::<
1157 WithContext<(OffchainPublicKey, Vec<Acknowledgement>)>,
1158 >(cfg.ack_config.ticket_ack_buffer_size);
1159
1160 let (wire_out, wire_in) = (wire_msg.0.with_timeout(QUEUE_SEND_TIMEOUT), wire_msg.1);
1163 let (app_out, app_in) = (api.0.with_timeout(QUEUE_SEND_TIMEOUT), api.1);
1164 let incoming_ack_tx = ContextSender::new(incoming_ack_tx).with_timeout(QUEUE_SEND_TIMEOUT);
1165 let outgoing_ack_tx = ContextSender::new(outgoing_ack_tx).with_timeout(QUEUE_SEND_TIMEOUT);
1166 let ticket_events = ticket_events.with_timeout(QUEUE_SEND_TIMEOUT);
1167 let ssa_events = ssa_events.with_timeout(QUEUE_SEND_TIMEOUT);
1168
1169 let encoder = std::sync::Arc::new(codec.0);
1170 let decoder = std::sync::Arc::new(codec.1);
1171 let ticket_proc = std::sync::Arc::new(ticket_proc);
1172 let exit_ack_proc = std::sync::Arc::new(exit_ack_proc);
1173
1174 let available_parallelism = std::thread::available_parallelism().ok().map(|n| n.get()).unwrap_or(1);
1178 let default_concurrency = default_pipeline_concurrency(available_parallelism);
1179
1180 let output_concurrency = cfg.output_concurrency.filter(|&n| n > 0).unwrap_or(default_concurrency);
1181 let input_concurrency = cfg.input_concurrency.filter(|&n| n > 0).unwrap_or(default_concurrency);
1182
1183 hopr_utils::parallelize::cpu::with_arbitration_once(cfg.arbitration.to_arbitration());
1189
1190 processes.insert(
1191 PacketPipelineProcesses::MsgOut,
1192 hopr_utils::spawn_as_abortable!(
1193 start_outgoing_packet_pipeline(
1194 app_in,
1195 encoder.clone(),
1196 (node_type != NodeType::Entry).then(|| exit_ack_proc.clone()),
1201 wire_out.clone(),
1202 counters.clone(),
1203 output_concurrency
1204 )
1205 .in_current_span()
1206 .with_current_subscriber()
1207 ),
1208 );
1209
1210 processes.insert(
1211 PacketPipelineProcesses::MsgIn,
1212 hopr_utils::spawn_as_abortable!(
1213 start_incoming_packet_pipeline(
1214 (wire_out.clone(), wire_in),
1215 decoder,
1216 ticket_proc.clone(),
1217 ticket_events.clone(),
1218 (outgoing_ack_tx, incoming_ack_tx),
1219 app_out,
1220 counters.clone(),
1221 input_concurrency,
1222 )
1223 .in_current_span()
1224 .with_current_subscriber()
1225 ),
1226 );
1227
1228 processes.insert(
1229 PacketPipelineProcesses::AckOut,
1230 hopr_utils::spawn_as_abortable!(
1231 start_outgoing_ack_pipeline(outgoing_ack_rx, encoder, cfg.ack_config, packet_key.clone(), wire_out,)
1232 .in_current_span()
1233 .with_current_subscriber()
1234 ),
1235 );
1236
1237 let ack_input_concurrency = cfg
1238 .ack_config
1239 .ack_input_concurrency
1240 .filter(|&n| n > 0)
1241 .unwrap_or(DEFAULT_ACK_INPUT_CONCURRENCY);
1242
1243 match node_type {
1244 NodeType::Relay => {
1245 processes.insert(
1246 PacketPipelineProcesses::AckIn,
1247 hopr_utils::spawn_as_abortable!(
1248 start_relay_incoming_ack_pipeline(
1249 incoming_ack_rx,
1250 ticket_events,
1251 ticket_proc,
1252 exit_ack_proc.clone(),
1253 ssa_events.clone(),
1254 ack_input_concurrency,
1255 )
1256 .in_current_span()
1257 .with_current_subscriber()
1258 ),
1259 );
1260 }
1261 NodeType::Exit => {
1262 let _ = (ticket_events, ticket_proc);
1266 processes.insert(
1267 PacketPipelineProcesses::AckIn,
1268 hopr_utils::spawn_as_abortable!(
1269 start_exit_incoming_ack_pipeline(
1270 incoming_ack_rx,
1271 exit_ack_proc,
1272 ssa_events,
1273 ack_input_concurrency,
1274 )
1275 .in_current_span()
1276 .with_current_subscriber()
1277 ),
1278 );
1279 }
1280 NodeType::Entry => {
1281 let _ = (ticket_events, ticket_proc, ack_input_concurrency);
1285 processes.insert(
1286 PacketPipelineProcesses::AckIn,
1287 hopr_utils::spawn_as_abortable!(
1288 start_drain_incoming_ack_pipeline(incoming_ack_rx)
1289 .in_current_span()
1290 .with_current_subscriber()
1291 ),
1292 );
1293 }
1294 }
1295
1296 processes
1297}
1298
1299#[cfg(test)]
1300mod tests {
1301 use futures::channel::mpsc;
1302 use hopr_api::types::crypto_random::Randomizable;
1303 use hopr_crypto_packet::prelude::PacketSignals;
1304
1305 use super::*;
1306
1307 struct ContextTestEncoder {
1309 dispatch: tracing::Dispatch,
1310 next_hop: Option<OffchainPublicKey>,
1311 }
1312
1313 #[cfg(feature = "tracing-context")]
1314 struct ContextTestDecoder {
1315 dispatch: tracing::Dispatch,
1316 peer: OffchainPublicKey,
1317 sessions: [HoprPseudonym; 2],
1318 }
1319
1320 #[cfg(feature = "tracing-context")]
1321 impl PacketDecoder for ContextTestDecoder {
1322 type Error = std::io::Error;
1323
1324 fn decode(&self, _: PeerId, data: Bytes) -> Result<IncomingPacket, IncomingPacketError<Self::Error>> {
1325 let marker = data.first().copied().ok_or_else(|| {
1326 IncomingPacketError::Undecodable(std::io::Error::other("missing context-test marker"))
1327 })?;
1328 tracing::dispatcher::with_default(&self.dispatch, || {
1329 tracing::debug!(marker, "actual decoder context");
1330 });
1331 if marker < 2 {
1332 let data = ApplicationData::new(1234u64, vec![marker])
1333 .map_err(|error| IncomingPacketError::Undecodable(std::io::Error::other(error)))?;
1334 Ok(IncomingPacket::Final(Box::new(IncomingFinalPacket {
1335 packet_tag: Default::default(),
1336 previous_hop: self.peer,
1337 sender: self.sessions[usize::from(marker)],
1338 replied_on_surb: None,
1339 plain_text: data.to_bytes().into(),
1340 ack_key: HalfKey::default(),
1341 info: Default::default(),
1342 })))
1343 } else if marker == 2 {
1344 Ok(IncomingPacket::Acknowledgement(Box::new(
1345 IncomingAcknowledgementPacket {
1346 packet_tag: Default::default(),
1347 previous_hop: self.peer,
1348 received_acks: vec![VerifiedAcknowledgement::random(&OffchainKeypair::random()).leak()],
1349 },
1350 )))
1351 } else {
1352 Err(IncomingPacketError::Undecodable(std::io::Error::other(
1353 "context-test decode failure",
1354 )))
1355 }
1356 }
1357 }
1358
1359 #[cfg(feature = "tracing-context")]
1360 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
1361 #[serial_test::serial(routing_logs)]
1362 async fn incoming_packets_should_keep_context_through_decode_dispatch_and_ack_queues() -> anyhow::Result<()> {
1363 use anyhow::Context;
1364
1365 let logs = crate::tests::support::Logs::default();
1366 logs.with_cpu_subscriber(logs.subscriber(tracing::Level::DEBUG), async {
1367 let peer = *OffchainKeypair::random().public();
1368 let sessions = [HoprPseudonym::random(), HoprPseudonym::random()];
1369 let dispatch = tracing::dispatcher::get_default(Clone::clone);
1370 let decoder = std::sync::Arc::new(ContextTestDecoder {
1371 dispatch: dispatch.clone(),
1372 peer,
1373 sessions,
1374 });
1375 let (app_tx, app_rx) = hopr_utils::network_types::crossfire_sink::bounded_sink_channel(4);
1376 let (ack_tx, ack_rx) = hopr_utils::network_types::crossfire_sink::bounded_sink_channel(4);
1377 start_incoming_packet_pipeline(
1378 (
1379 futures::sink::drain(),
1380 futures::stream::iter((0u8..4).map(move |marker| (PeerId::from(peer), Bytes::from(vec![marker])))),
1381 ),
1382 decoder,
1383 std::sync::Arc::new(ScriptedTicketProc::resolving_nothing()),
1384 futures::sink::drain(),
1385 (futures::sink::drain(), ContextSender::new(ack_tx)),
1386 ContextSender::new(app_tx).with_timeout(QUEUE_SEND_TIMEOUT),
1387 PeerProtocolCounterRegistry::default(),
1388 2,
1389 )
1390 .await;
1391 let messages = app_rx.collect::<Vec<_>>().await;
1392 assert_eq!(
1393 messages.len(),
1394 2,
1395 "failed decode and acknowledgements must not reach the app"
1396 );
1397 for (index, message) in messages.into_iter().enumerate() {
1398 assert_eq!(message.value.0, sessions[index]);
1399 assert_eq!(message.value.1.data.plain_text.as_ref(), [index as u8]);
1400 async {
1401 tracing::debug!(index, "received session dispatch");
1402 }
1403 .instrument(message.into_parts().1)
1404 .await;
1405 }
1406 let acknowledgements = ack_rx.collect::<Vec<_>>().await;
1407 assert_eq!(acknowledgements.len(), 1);
1408 let tickets = ScriptedTicketProc {
1409 outcome: std::sync::Arc::new(move || {
1410 tracing::dispatcher::with_default(&dispatch, || {
1411 tracing::debug!("actual acknowledgement worker context");
1412 });
1413 Ok(Vec::new())
1414 }),
1415 calls: Default::default(),
1416 };
1417 start_relay_incoming_ack_pipeline(
1418 futures::stream::iter(acknowledgements),
1419 futures::sink::drain(),
1420 std::sync::Arc::new(tickets),
1421 std::sync::Arc::new(ScriptedExitProc::new(|| Ok(vec![])).never_pending()),
1422 futures::sink::drain(),
1423 2,
1424 )
1425 .await;
1426
1427 let decoded = logs.records("actual decoder context");
1428 assert_eq!(decoded.len(), 4);
1429 assert!(
1430 decoded
1431 .iter()
1432 .all(|record| record.ancestry == ["transport.receive", "incoming_packet_decode"])
1433 );
1434 let workers = logs.records("actual acknowledgement worker context");
1435 assert_eq!(workers.len(), 1);
1436 assert_eq!(
1437 workers[0].ancestry,
1438 ["transport.receive", "transport.process", "incoming_ack_batch"]
1439 );
1440 assert!(!workers[0].fields.contains_key("peer"));
1441 let output = logs.output();
1442 let worker = output
1443 .lines()
1444 .find(|line| line.contains("actual acknowledgement worker context"))
1445 .context("missing acknowledgement worker output")?;
1446 assert!(
1447 worker.contains("packet_id=") && worker.contains(&peer.to_peerid_str()),
1448 "{worker}"
1449 );
1450 for (index, id) in sessions.iter().enumerate() {
1451 let line = output
1452 .lines()
1453 .find(|line| line.contains("received session dispatch") && line.contains(&format!("index={index}")))
1454 .context("missing session dispatch output")?;
1455 assert!(line.contains(&id.to_string()) && line.contains("packet_id="), "{line}");
1456 assert!(!line.contains(&sessions[1 - index].to_string()), "{line}");
1457 }
1458 anyhow::Ok(())
1459 })?
1460 }
1461
1462 impl PacketEncoder for ContextTestEncoder {
1463 type Error = std::io::Error;
1464
1465 fn encode_packet<T: AsRef<[u8]> + Send + 'static, S: Into<PacketSignals> + Send + 'static>(
1466 &self,
1467 data: T,
1468 _: ResolvedTransportRouting<HoprSurb>,
1469 _: S,
1470 _: Option<u8>,
1471 ) -> Result<OutgoingPacket, Self::Error> {
1472 tracing::dispatcher::with_default(&self.dispatch, || {
1474 let marker = data.as_ref().last().copied();
1475 tracing::debug!(?marker, "actual encoder context");
1476 });
1477 if let Some(next_hop) = self.next_hop {
1478 Ok(OutgoingPacket {
1479 next_hop,
1480 ack_challenge: HalfKeyChallenge::default(),
1481 encrypted_pix_share: None,
1482 data: Bytes::copy_from_slice(data.as_ref()),
1483 minted_surbs: Vec::new(),
1484 })
1485 } else {
1486 Err(std::io::Error::other("context test stops after encoding entry"))
1487 }
1488 }
1489
1490 fn encode_acknowledgements(
1491 &self,
1492 acks: &[VerifiedAcknowledgement],
1493 _: &OffchainPublicKey,
1494 ) -> Result<OutgoingPacket, Self::Error> {
1495 tracing::dispatcher::with_default(&self.dispatch, || {
1496 tracing::debug!(count = acks.len(), "actual acknowledgement encoder context");
1497 });
1498 Ok(OutgoingPacket {
1499 next_hop: self
1500 .next_hop
1501 .ok_or_else(|| std::io::Error::other("missing acknowledgement destination"))?,
1502 ack_challenge: HalfKeyChallenge::default(),
1503 encrypted_pix_share: None,
1504 data: Bytes::new(),
1505 minted_surbs: Vec::new(),
1506 })
1507 }
1508 }
1509
1510 #[cfg(feature = "tracing-context")]
1512 #[rstest::rstest]
1513 #[case::entry(NodeType::Entry)]
1514 #[case::exit(NodeType::Exit)]
1515 #[case::relay(NodeType::Relay)]
1516 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
1517 #[serial_test::serial(routing_logs)]
1518 async fn packet_workers_deliver_messages_and_process_acks_for_their_role(
1519 #[case] role: NodeType,
1520 ) -> anyhow::Result<()> {
1521 use anyhow::Context;
1522
1523 let logs = crate::tests::support::Logs::default();
1524 logs.with_cpu_subscriber(logs.subscriber(tracing::Level::DEBUG), async {
1525 let key = OffchainKeypair::random();
1526 let peer = *key.public();
1527 let session = HoprPseudonym::random();
1528 let dispatch = tracing::dispatcher::get_default(Clone::clone);
1529 let (wire_tx, wire_rx) = mpsc::channel(8);
1530 let (app_tx, app_rx) = mpsc::channel(8);
1531 let (ssa_tx, ssa_rx) = mpsc::channel(8);
1532 let tickets = ScriptedTicketProc::resolving_nothing();
1533 let ssa_id = SsaId::new(session, 1.try_into()?);
1534 let shares = ScriptedExitProc::new(move || Ok(vec![HoprShareResolution::AlmostRecoveredSsa(ssa_id)]));
1535 let outgoing = ApplicationDataOut::with_no_packet_info(ApplicationData::new(1234u64, vec![9])?);
1536 let expected_outgoing = outgoing.data.to_bytes();
1537 let path = hopr_api::types::internal::path::ValidatedPath::direct(
1538 peer,
1539 ChainKeypair::random().public().to_address(),
1540 );
1541 let processes = run_packet_pipeline_inner(
1542 role,
1543 key,
1544 (
1545 wire_tx,
1546 futures::stream::iter([0u8, 2].map(|marker| (peer.into(), Bytes::from(vec![marker])))),
1547 ),
1548 (
1549 ContextTestEncoder {
1550 dispatch: dispatch.clone(),
1551 next_hop: Some(peer),
1552 },
1553 ContextTestDecoder {
1554 dispatch,
1555 peer,
1556 sessions: [session, HoprPseudonym::random()],
1557 },
1558 ),
1559 tickets.clone(),
1560 shares.clone(),
1561 futures::sink::drain(),
1562 ssa_tx,
1563 PacketPipelineConfig::default(),
1564 (
1565 app_tx,
1566 futures::stream::iter([(ResolvedTransportRouting::forward_only(path), outgoing)]),
1567 ),
1568 PeerProtocolCounterRegistry::default(),
1569 );
1570 for worker in [
1571 PacketPipelineProcesses::MsgIn,
1572 PacketPipelineProcesses::MsgOut,
1573 PacketPipelineProcesses::AckIn,
1574 PacketPipelineProcesses::AckOut,
1575 ] {
1576 assert!(processes.contains(&worker), "missing worker {worker:?} for {role:?}");
1577 }
1578 let (app, wire, resolutions) = tokio::time::timeout(std::time::Duration::from_secs(5), async {
1579 futures::join!(
1580 app_rx.collect::<Vec<_>>(),
1581 wire_rx.collect::<Vec<_>>(),
1582 ssa_rx.collect::<Vec<_>>()
1583 )
1584 })
1585 .await
1586 .context("all workers must drain finite input and close their outputs")?;
1587 assert_eq!(app.len(), 1);
1588 assert_eq!(app[0].0, session);
1589 assert_eq!(app[0].1.data.plain_text.as_ref(), [0]);
1590 assert!(
1591 wire.iter().any(
1592 |(target, bytes)| *target == PeerId::from(peer) && bytes.as_ref() == expected_outgoing.as_ref()
1593 )
1594 );
1595 assert_eq!(tickets.calls(), usize::from(role == NodeType::Relay));
1596 assert_eq!(shares.calls(), usize::from(role != NodeType::Entry));
1597 assert_eq!(resolutions.len(), usize::from(role != NodeType::Entry));
1598 if role != NodeType::Entry {
1599 assert_eq!(resolutions[0], HoprShareResolution::AlmostRecoveredSsa(ssa_id));
1600 }
1601 anyhow::Ok(())
1602 })?
1603 }
1604
1605 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
1606 #[serial_test::serial(routing_logs)]
1607 async fn acknowledgement_batches_should_link_all_packets_without_changing_grouping() -> anyhow::Result<()> {
1608 let logs = crate::tests::support::Logs::default();
1609 logs.with_cpu_subscriber(logs.subscriber(tracing::Level::DEBUG), async {
1610 let key = OffchainKeypair::random();
1611 let peer = *key.public();
1612 let messages = ["packet.one", "packet.two"];
1613 let spans = [
1614 tracing::debug_span!("packet.one", packet_id = 1),
1615 tracing::debug_span!("packet.two", packet_id = 2),
1616 ];
1617 let input = spans
1618 .into_iter()
1619 .map(move |span| WithContext::new((peer, Some(HalfKey::default())), span))
1620 .collect::<Vec<_>>();
1621 let (wire_tx, wire_rx) = mpsc::channel(4);
1622 start_outgoing_ack_pipeline(
1623 futures::stream::iter(input),
1624 std::sync::Arc::new(ContextTestEncoder {
1625 dispatch: tracing::dispatcher::get_default(Clone::clone),
1626 next_hop: Some(peer),
1627 }),
1628 AcknowledgementPipelineConfig::default(),
1629 key,
1630 wire_tx,
1631 )
1632 .await;
1633 assert_eq!(
1634 wire_rx.collect::<Vec<_>>().await.len(),
1635 1,
1636 "same-peer acknowledgements must stay batched"
1637 );
1638 let encoded = logs.records("actual acknowledgement encoder context");
1639 assert_eq!(encoded.len(), 1);
1640 assert_eq!(encoded[0].fields["count"], "2");
1641 assert_eq!(encoded[0].ancestry, ["outgoing_ack_batch"]);
1642 for cause in messages {
1643 assert!(logs.causes().contains(&("outgoing_ack_batch", cause)));
1644 }
1645 anyhow::Ok(())
1646 })?
1647 }
1648
1649 #[rstest::rstest]
1650 #[case(true, false, false)]
1651 #[case(false, false, false)]
1652 #[case(true, true, false)]
1653 #[case(true, false, true)]
1654 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
1655 #[serial_test::serial(routing_logs)]
1656 async fn outgoing_encoder_should_enter_each_queued_messages_session(
1657 #[case] succeeds: bool,
1658 #[case] wire_closed: bool,
1659 #[case] filtered: bool,
1660 ) -> anyhow::Result<()> {
1661 use anyhow::Context as _;
1662
1663 use crate::{tests::support::Logs, trace_context::WithContext};
1664
1665 let logs = Logs::default();
1666 let subscriber = logs.subscriber_disabling_spans(
1667 tracing::Level::DEBUG,
1668 if filtered { &["transport.encode"] } else { &[] },
1669 );
1670 logs.with_cpu_subscriber(subscriber, async {
1671 let mut input = Vec::new();
1672 let markers: &[u8] = if wire_closed { &[11] } else { &[11, 22] };
1674 for &marker in markers {
1675 let span = tracing::debug_span!("session", session_id = marker);
1676 let data = ApplicationDataOut::with_no_packet_info(ApplicationData::new(1234u64, vec![marker])?);
1677 let path = hopr_api::types::internal::path::ValidatedPath::direct(
1678 *OffchainKeypair::random().public(),
1679 ChainKeypair::random().public().to_address(),
1680 );
1681 input.push(WithContext::new(
1682 (ResolvedTransportRouting::forward_only(path), data),
1683 span,
1684 ));
1685 }
1686 let next_hop = *OffchainKeypair::random().public();
1687 let encoder = std::sync::Arc::new(ContextTestEncoder {
1688 dispatch: tracing::dispatcher::get_default(Clone::clone),
1689 next_hop: succeeds.then_some(next_hop),
1690 });
1691 let counters = PeerProtocolCounterRegistry::default();
1692 let (wire_tx, wire_rx) = mpsc::channel(2);
1693 let wire_rx = (!wire_closed).then_some(wire_rx);
1694 start_outgoing_packet_pipeline(
1695 futures::stream::iter(input),
1696 encoder,
1697 None::<NopExitAcknowledgementShareProcessor>,
1698 wire_tx.with_timeout(QUEUE_SEND_TIMEOUT),
1699 counters.clone(),
1700 2,
1701 )
1702 .await;
1703 let delivered = if let Some(wire_rx) = wire_rx {
1704 wire_rx.collect::<Vec<_>>().await
1705 } else {
1706 Vec::new()
1707 };
1708 assert_eq!(
1709 delivered.len(),
1710 if succeeds && !wire_closed { markers.len() } else { 0 }
1711 );
1712 assert_eq!(
1713 counters.get_or_create(&next_hop).take_sent(),
1714 if succeeds { markers.len() as u64 } else { 0 }
1715 );
1716 for ((peer, data), marker) in delivered.iter().zip([11u8, 22]) {
1717 assert_eq!(*peer, PeerId::from(next_hop));
1718 assert_eq!(
1719 data.last(),
1720 Some(&marker),
1721 "encoded messages must retain submission order"
1722 );
1723 }
1724 let events = logs.records("actual encoder context");
1725 assert_eq!(events.len(), markers.len());
1726 for event in events {
1727 assert_eq!(
1728 event.ancestry,
1729 if filtered {
1730 vec!["session"]
1731 } else {
1732 vec!["session", "transport.encode"]
1733 }
1734 );
1735 }
1736 let output = logs.output();
1737 for marker in markers {
1738 let line = output
1739 .lines()
1740 .find(|line| {
1741 line.contains("actual encoder context") && line.contains(&format!("marker=Some({marker})"))
1742 })
1743 .context("missing encoding record")?;
1744 assert!(line.contains(&format!("session_id={marker}")), "wrong session: {line}");
1745 }
1746 assert_eq!(
1747 logs.records("long-running background task finished with error").len(),
1748 usize::from(wire_closed),
1749 "wire failures must terminate the pipeline with an error: {output}"
1750 );
1751 anyhow::Ok(())
1752 })?
1753 }
1754
1755 struct LabelTestTicketProc;
1759
1760 impl UnacknowledgedTicketProcessor for LabelTestTicketProc {
1761 type Error = HoprProtocolError;
1762
1763 fn insert_unacknowledged_ticket(
1764 &self,
1765 _: &OffchainPublicKey,
1766 _: HalfKeyChallenge,
1767 _: UnacknowledgedTicket,
1768 ) -> Result<(), Self::Error> {
1769 Ok(())
1770 }
1771
1772 fn is_duplicate_ticket_index_error(error: &Self::Error) -> bool {
1773 error.is_duplicate_ticket_index()
1774 }
1775
1776 fn acknowledge_tickets(
1777 &self,
1778 _: OffchainPublicKey,
1779 _: Vec<Acknowledgement>,
1780 ) -> Result<Vec<ResolvedAcknowledgement>, TicketAcknowledgementError<Self::Error>> {
1781 Ok(Vec::with_capacity(0))
1782 }
1783 }
1784
1785 #[test]
1793 fn duplicate_ticket_index_gets_its_own_rejection_metric_label() {
1794 let duplicate = HoprProtocolError::DuplicateTicketIndex(Default::default(), 1, 5);
1795 assert_eq!(
1796 unack_rejection_metric_label::<LabelTestTicketProc>(&duplicate),
1797 "duplicate_ticket_index",
1798 "a replayed index must be reported under its own label"
1799 );
1800
1801 for other in [HoprProtocolError::InvalidSender, HoprProtocolError::KeyNotFound] {
1802 assert_eq!(
1803 unack_rejection_metric_label::<LabelTestTicketProc>(&other),
1804 "unack_processing_error",
1805 "{other} must keep the generic label"
1806 );
1807 }
1808 }
1809
1810 #[test]
1813 fn a_replay_has_its_own_rejection_metric_label() {
1814 assert_eq!(replay_rejection_metric_label(), "replay");
1815 assert_ne!(
1816 replay_rejection_metric_label(),
1817 "processing_error",
1818 "sharing the generic label would hide replays among ordinary drops"
1819 );
1820 }
1821
1822 #[test]
1830 fn default_pipeline_concurrency_is_cpu_scaled() {
1831 assert_eq!(
1833 default_pipeline_concurrency(4),
1834 4 * PIPELINE_CONCURRENCY_PER_CPU,
1835 "per-stage concurrency default must be the CPU-scaled deep queue",
1836 );
1837 for cpus in 1..=16 {
1839 assert!(
1840 default_pipeline_concurrency(cpus) >= cpus,
1841 "concurrency default collapsed below CPU count for cpus={cpus}",
1842 );
1843 }
1844 assert!(default_pipeline_concurrency(0) >= 1);
1846 }
1847
1848 #[test]
1849 fn noop_exit_proc_has_no_pending_shares() {
1850 let noop = NopExitAcknowledgementShareProcessor;
1851 let dummy = *OffchainKeypair::random().public();
1852 assert!(
1853 !noop.has_pending_shares(&dummy),
1854 "NopExitAcknowledgementShareProcessor must always report no pending shares"
1855 );
1856 }
1857
1858 #[test]
1862 fn noop_exit_proc_succeeds_on_every_method() -> anyhow::Result<()> {
1863 let noop = NopExitAcknowledgementShareProcessor;
1864 let peer = *OffchainKeypair::random().public();
1865 let ssa_id = SsaId::new(HoprPseudonym::random(), 1.try_into()?);
1866
1867 assert_eq!(
1868 noop.new_exit_commitment(ssa_id, PixParams::try_new_for::<HoprPixSpec>(2, 2, 0)?)?,
1869 PixGroup::<HoprPixSpec>::default()
1870 );
1871
1872 let state = noop.insert_coefficient_commitments(ssa_id, 0, None, std::iter::empty())?;
1873 assert_eq!(state, SsaCommitmentState::new(ssa_id));
1874
1875 noop.retire_ssa(ssa_id);
1876
1877 noop.insert_encrypted_share(
1878 &peer,
1879 HalfKey::random().to_challenge()?,
1880 TaggedEncryptedPartialSsaShare {
1881 pseudonym: HoprPseudonym::random(),
1882 nonce: Default::default(),
1883 partial_share: Default::default(),
1884 },
1885 )?;
1886
1887 assert!(noop.acknowledge_shares(peer, vec![])?.is_empty());
1888
1889 Ok(())
1893 }
1894
1895 #[derive(Clone)]
1898 struct ScriptedExitProc {
1899 outcome: std::sync::Arc<dyn Fn() -> Result<Vec<HoprShareResolution>, ScriptedError> + Send + Sync>,
1900 calls: std::sync::Arc<std::sync::atomic::AtomicUsize>,
1901 pending: bool,
1905 }
1906
1907 #[derive(Debug, Clone, Copy, PartialEq, Eq, thiserror::Error)]
1908 enum ScriptedError {
1909 #[error("nothing was expected from this peer")]
1910 Expected,
1911 #[error("the reconstructor is broken")]
1912 Unexpected,
1913 }
1914
1915 impl ScriptedExitProc {
1916 fn new(outcome: impl Fn() -> Result<Vec<HoprShareResolution>, ScriptedError> + Send + Sync + 'static) -> Self {
1917 Self {
1918 outcome: std::sync::Arc::new(outcome),
1919 calls: Default::default(),
1920 pending: true,
1921 }
1922 }
1923
1924 fn never_pending(mut self) -> Self {
1926 self.pending = false;
1927 self
1928 }
1929
1930 fn calls(&self) -> usize {
1931 self.calls.load(std::sync::atomic::Ordering::SeqCst)
1932 }
1933 }
1934
1935 impl ExitAcknowledgementShareProcessor<HoprPixSpec> for ScriptedExitProc {
1936 type Error = ScriptedError;
1937
1938 fn has_pending_shares(&self, _: &OffchainPublicKey) -> bool {
1939 self.pending
1940 }
1941
1942 fn is_expected_error(&self, error: &Self::Error) -> bool {
1943 matches!(error, ScriptedError::Expected)
1944 }
1945
1946 fn new_exit_commitment(
1947 &self,
1948 _: SsaId<HoprPseudonym>,
1949 _: PixParams,
1950 ) -> Result<PixGroup<HoprPixSpec>, Self::Error> {
1951 Ok(PixGroup::<HoprPixSpec>::default())
1952 }
1953
1954 fn insert_coefficient_commitments(
1955 &self,
1956 ssa_id: SsaId<SimplePseudonym>,
1957 _: CoefficientIndex,
1958 _: Option<SsaCommitmentProof<HoprPixSpec>>,
1959 _: impl Iterator<Item = (PolynomialIndex, PixGroupRepr<HoprPixSpec>)>,
1960 ) -> Result<HoprSsaCommitmentState, Self::Error> {
1961 Ok(SsaCommitmentState::new(ssa_id))
1962 }
1963
1964 fn retire_ssa(&self, _: SsaId<HoprPseudonym>) {}
1965
1966 fn insert_encrypted_share(
1967 &self,
1968 _: &OffchainPublicKey,
1969 _: HalfKeyChallenge,
1970 _: TaggedEncryptedPartialSsaShare<HoprPixSpec>,
1971 ) -> Result<(), Self::Error> {
1972 Ok(())
1973 }
1974
1975 fn acknowledge_shares(
1976 &self,
1977 _: OffchainPublicKey,
1978 _: Vec<Acknowledgement>,
1979 ) -> Result<Vec<HoprShareResolution>, Self::Error> {
1980 self.calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
1981 (self.outcome)()
1982 }
1983 }
1984
1985 fn one_batch() -> Vec<(OffchainPublicKey, Vec<Acknowledgement>)> {
1986 vec![(*OffchainKeypair::random().public(), vec![])]
1987 }
1988
1989 async fn run_exit_pipeline(proc: ScriptedExitProc, evt: mpsc::Sender<HoprShareResolution>) -> ScriptedExitProc {
1990 start_exit_incoming_ack_pipeline(futures::stream::iter(one_batch()), proc.clone(), evt, 4).await;
1991 proc
1992 }
1993
1994 #[tokio::test]
1997 async fn exit_ack_pipeline_survives_a_dead_resolution_sink() {
1998 let ssa_id = SsaId::new(HoprPseudonym::random(), 1.try_into().expect("non-zero"));
1999 let proc = ScriptedExitProc::new(move || Ok(vec![HoprShareResolution::AlmostRecoveredSsa(ssa_id)]));
2000
2001 let (tx, rx) = mpsc::channel::<HoprShareResolution>(4);
2002 drop(rx);
2003
2004 let proc = run_exit_pipeline(proc, tx).await;
2005 assert_eq!(proc.calls(), 1, "the batch must still have been processed");
2006 }
2007
2008 #[tokio::test]
2009 async fn exit_ack_pipeline_forwards_resolutions_to_the_sink() -> anyhow::Result<()> {
2010 let ssa_id = SsaId::new(HoprPseudonym::random(), 1.try_into()?);
2011 let proc = ScriptedExitProc::new(move || Ok(vec![HoprShareResolution::AlmostRecoveredSsa(ssa_id)]));
2012
2013 let (tx, mut rx) = mpsc::channel::<HoprShareResolution>(4);
2014 run_exit_pipeline(proc, tx).await;
2015
2016 let received = rx
2017 .next()
2018 .await
2019 .ok_or_else(|| anyhow::anyhow!("no resolution forwarded"))?;
2020 assert_eq!(received, HoprShareResolution::AlmostRecoveredSsa(ssa_id));
2021 Ok(())
2022 }
2023
2024 #[tokio::test]
2027 async fn exit_ack_pipeline_survives_both_error_kinds() {
2028 for error in [ScriptedError::Expected, ScriptedError::Unexpected] {
2029 let proc = ScriptedExitProc::new(move || Err(error));
2030 let (tx, _rx) = mpsc::channel::<HoprShareResolution>(4);
2031 let proc = run_exit_pipeline(proc, tx).await;
2032 assert_eq!(proc.calls(), 1, "{error:?} must not abort the pipeline");
2033 }
2034 }
2035
2036 #[tokio::test]
2039 async fn exit_ack_pipeline_skips_peers_with_no_pending_shares() {
2040 let proc = ScriptedExitProc::new(|| Ok(vec![])).never_pending();
2041 let (tx, _rx) = mpsc::channel::<HoprShareResolution>(4);
2042 let proc = run_exit_pipeline(proc, tx).await;
2043
2044 assert_eq!(
2045 proc.calls(),
2046 0,
2047 "a peer with no pending shares must not reach the thread pool"
2048 );
2049 }
2050
2051 #[derive(Clone)]
2054 struct ScriptedTicketProc {
2055 outcome: std::sync::Arc<
2056 dyn Fn() -> Result<Vec<ResolvedAcknowledgement>, TicketAcknowledgementError<ScriptedError>> + Send + Sync,
2057 >,
2058 calls: std::sync::Arc<std::sync::atomic::AtomicUsize>,
2059 }
2060
2061 impl ScriptedTicketProc {
2062 fn resolving_nothing() -> Self {
2065 Self {
2066 outcome: std::sync::Arc::new(|| Ok(Vec::new())),
2067 calls: Default::default(),
2068 }
2069 }
2070
2071 fn calls(&self) -> usize {
2072 self.calls.load(std::sync::atomic::Ordering::SeqCst)
2073 }
2074 }
2075
2076 impl UnacknowledgedTicketProcessor for ScriptedTicketProc {
2077 type Error = ScriptedError;
2078
2079 fn is_duplicate_ticket_index_error(_: &Self::Error) -> bool {
2080 false
2081 }
2082
2083 fn insert_unacknowledged_ticket(
2084 &self,
2085 _: &OffchainPublicKey,
2086 _: HalfKeyChallenge,
2087 _: UnacknowledgedTicket,
2088 ) -> Result<(), Self::Error> {
2089 Ok(())
2090 }
2091
2092 fn acknowledge_tickets(
2093 &self,
2094 _: OffchainPublicKey,
2095 _: Vec<Acknowledgement>,
2096 ) -> Result<Vec<ResolvedAcknowledgement>, TicketAcknowledgementError<Self::Error>> {
2097 self.calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
2098 (self.outcome)()
2099 }
2100 }
2101
2102 fn one_ack_batch() -> Vec<(OffchainPublicKey, Vec<Acknowledgement>)> {
2105 vec![(
2106 *OffchainKeypair::random().public(),
2107 vec![VerifiedAcknowledgement::random(&OffchainKeypair::random()).leak()],
2108 )]
2109 }
2110
2111 async fn run_relay_pipeline(
2114 tickets: ScriptedTicketProc,
2115 exit: ScriptedExitProc,
2116 ssa_evt: mpsc::Sender<HoprShareResolution>,
2117 batch: Vec<(OffchainPublicKey, Vec<Acknowledgement>)>,
2118 ) -> (ScriptedTicketProc, ScriptedExitProc) {
2119 let (ticket_evt, _ticket_rx) = mpsc::channel::<TicketEvent>(4);
2122 start_relay_incoming_ack_pipeline(
2123 futures::stream::iter(batch),
2124 ticket_evt,
2125 std::sync::Arc::new(tickets.clone()),
2126 std::sync::Arc::new(exit.clone()),
2127 ssa_evt,
2128 4,
2129 )
2130 .await;
2131 (tickets, exit)
2132 }
2133
2134 #[tokio::test]
2137 async fn relay_ack_pipeline_acknowledges_tickets_and_shares_for_the_same_batch() -> anyhow::Result<()> {
2138 let ssa_id = SsaId::new(HoprPseudonym::random(), 1.try_into()?);
2139 let exit = ScriptedExitProc::new(move || Ok(vec![HoprShareResolution::AlmostRecoveredSsa(ssa_id)]));
2140
2141 let (tx, mut rx) = mpsc::channel::<HoprShareResolution>(4);
2142 let (tickets, exit) =
2143 run_relay_pipeline(ScriptedTicketProc::resolving_nothing(), exit, tx, one_ack_batch()).await;
2144
2145 assert_eq!(tickets.calls(), 1, "the ticket arm must run for every batch");
2146 assert_eq!(exit.calls(), 1, "a peer with pending shares must reach the share arm");
2147 assert_eq!(
2148 rx.next().await,
2149 Some(HoprShareResolution::AlmostRecoveredSsa(ssa_id)),
2150 "the recovered SSA must be forwarded to the sink"
2151 );
2152 Ok(())
2153 }
2154
2155 #[tokio::test]
2158 async fn relay_ack_pipeline_skips_share_acknowledgement_without_pending_shares() {
2159 let exit = ScriptedExitProc::new(|| Ok(vec![])).never_pending();
2160
2161 let (tx, mut rx) = mpsc::channel::<HoprShareResolution>(4);
2162 let (tickets, exit) =
2163 run_relay_pipeline(ScriptedTicketProc::resolving_nothing(), exit, tx, one_ack_batch()).await;
2164
2165 assert_eq!(tickets.calls(), 1, "the ticket arm runs regardless of pending shares");
2166 assert_eq!(
2167 exit.calls(),
2168 0,
2169 "a peer with no pending shares must not reach the thread pool"
2170 );
2171 assert_eq!(rx.next().await, None, "a skipped share arm can resolve nothing");
2172 }
2173
2174 #[tokio::test]
2177 async fn relay_ack_pipeline_ignores_empty_acknowledgement_batches() {
2178 let (tx, _rx) = mpsc::channel::<HoprShareResolution>(4);
2179 let (tickets, exit) = run_relay_pipeline(
2180 ScriptedTicketProc::resolving_nothing(),
2181 ScriptedExitProc::new(|| Ok(vec![])),
2182 tx,
2183 vec![(*OffchainKeypair::random().public(), vec![])],
2184 )
2185 .await;
2186
2187 assert_eq!(tickets.calls(), 0);
2188 assert_eq!(exit.calls(), 0);
2189 }
2190
2191 #[tokio::test]
2194 async fn relay_ack_pipeline_survives_both_pix_error_kinds() {
2195 for error in [ScriptedError::Expected, ScriptedError::Unexpected] {
2196 let (tx, _rx) = mpsc::channel::<HoprShareResolution>(4);
2197 let (tickets, exit) = run_relay_pipeline(
2198 ScriptedTicketProc::resolving_nothing(),
2199 ScriptedExitProc::new(move || Err(error)),
2200 tx,
2201 one_ack_batch(),
2202 )
2203 .await;
2204
2205 assert_eq!(exit.calls(), 1, "{error:?} must not abort the pipeline");
2206 assert_eq!(tickets.calls(), 1, "{error:?} must not cost the ticket arm its batch");
2207 }
2208 }
2209
2210 #[tokio::test]
2213 async fn relay_ack_pipeline_forwards_nothing_for_an_empty_resolution_batch() {
2214 let (tx, mut rx) = mpsc::channel::<HoprShareResolution>(4);
2215 let (_, exit) = run_relay_pipeline(
2216 ScriptedTicketProc::resolving_nothing(),
2217 ScriptedExitProc::new(|| Ok(vec![])),
2218 tx,
2219 one_ack_batch(),
2220 )
2221 .await;
2222
2223 assert_eq!(exit.calls(), 1);
2224 assert_eq!(rx.next().await, None, "an empty batch must produce no event");
2225 }
2226
2227 #[tokio::test]
2230 async fn relay_ack_pipeline_survives_a_dead_pix_resolution_sink() -> anyhow::Result<()> {
2231 let ssa_id = SsaId::new(HoprPseudonym::random(), 1.try_into()?);
2232 let (tx, rx) = mpsc::channel::<HoprShareResolution>(4);
2233 drop(rx);
2234
2235 let (tickets, exit) = run_relay_pipeline(
2236 ScriptedTicketProc::resolving_nothing(),
2237 ScriptedExitProc::new(move || Ok(vec![HoprShareResolution::AlmostRecoveredSsa(ssa_id)])),
2238 tx,
2239 one_ack_batch(),
2240 )
2241 .await;
2242
2243 assert_eq!(exit.calls(), 1, "the batch must still have been processed");
2244 assert_eq!(tickets.calls(), 1);
2245 Ok(())
2246 }
2247
2248 #[tokio::test]
2258 async fn drain_pipeline_keeps_receiver_alive() {
2259 let (tx, rx) = mpsc::channel::<(OffchainPublicKey, Vec<Acknowledgement>)>(16);
2260 let drain = tokio::spawn(start_drain_incoming_ack_pipeline(rx));
2261
2262 tokio::task::yield_now().await;
2264
2265 assert!(!tx.is_closed(), "drain task must hold the receiver alive");
2267
2268 drop(tx);
2270 drain
2271 .await
2272 .expect("drain task must finish cleanly after sender is dropped");
2273 }
2274
2275 #[test_log::test(tokio::test)]
2276 async fn drain_pipeline_consumes_nonempty_and_empty_batches_under_backpressure() -> anyhow::Result<()> {
2277 let key = OffchainKeypair::random();
2278 let (mut tx, rx) = mpsc::channel(0);
2279 let drain = start_drain_incoming_ack_pipeline(rx);
2280 let send = async move {
2281 tx.send((*key.public(), vec![VerifiedAcknowledgement::random(&key).leak()]))
2282 .await?;
2283 tx.send((*key.public(), vec![])).await?;
2284 drop(tx);
2285 anyhow::Ok(())
2286 };
2287
2288 let (sent, ()) =
2289 tokio::time::timeout(std::time::Duration::from_secs(5), futures::future::join(send, drain)).await?;
2290 sent?;
2291 Ok(())
2292 }
2293
2294 #[tokio::test]
2296 async fn drain_pipeline_completes_on_empty_stream() {
2297 let (tx, rx) = mpsc::channel::<(OffchainPublicKey, Vec<Acknowledgement>)>(32);
2298 let drain = tokio::spawn(start_drain_incoming_ack_pipeline(rx));
2299 drop(tx);
2300 drain
2301 .await
2302 .expect("drain task must finish cleanly after sender is dropped");
2303 }
2304
2305 #[tokio::test]
2307 async fn disconnected_sender_returns_err_not_panic() {
2308 let (tx, rx) = mpsc::channel::<u8>(4);
2309 drop(rx);
2310 let mut tx2 = tx.clone();
2311 let result = tx2.send(42u8).await;
2312 assert!(result.is_err(), "send to disconnected receiver must return Err");
2313 }
2314
2315 #[tokio::test]
2318 async fn session_management_dispatcher_survives_disconnected_sink() -> anyhow::Result<()> {
2319 use anyhow::Context;
2320 use futures::SinkExt;
2321
2322 let (data_tx, data_rx) = mpsc::channel::<u8>(4);
2323
2324 drop(data_rx);
2326
2327 let upstream = futures::stream::iter(0u8..10);
2329
2330 let dispatcher = tokio::spawn(async move {
2332 upstream
2333 .for_each(move |item| {
2334 let mut tx = data_tx.clone();
2335 async move {
2336 let _ = tx.send(item).await;
2338 }
2339 })
2340 .await;
2341 });
2342
2343 dispatcher
2345 .await
2346 .context("dispatcher must complete cleanly even with a disconnected sink")?;
2347 Ok(())
2348 }
2349
2350 #[tokio::test]
2353 async fn session_management_dispatcher_does_not_drop_upstream_on_disconnected_sink() -> anyhow::Result<()> {
2354 use anyhow::Context;
2355 use futures::SinkExt;
2356
2357 let (data_tx, data_rx) = mpsc::channel::<u8>(1);
2358 drop(data_rx); let (upstream_tx, upstream_rx) = mpsc::channel::<u8>(16);
2361 let upstream = upstream_rx;
2362
2363 let dispatcher = tokio::spawn(async move {
2364 upstream
2365 .for_each(move |item| {
2366 let mut tx = data_tx.clone();
2367 async move {
2368 let _ = tx.send(item).await;
2369 }
2370 })
2371 .await;
2372 });
2373
2374 let mut sender = upstream_tx;
2376 for i in 0u8..20 {
2377 sender.send(i).await.context("upstream send must succeed")?;
2378 }
2379 drop(sender); dispatcher
2382 .await
2383 .context("dispatcher must complete when upstream closes, even with disconnected sink")?;
2384 Ok(())
2385 }
2386
2387 #[tokio::test]
2404 async fn buffered_decode_preserves_arrival_order_under_inverted_latency() {
2405 use std::time::Duration;
2406
2407 const N: usize = 16;
2408 let concurrency = N; let output: Vec<usize> = futures::stream::iter(0..N)
2411 .map(|i| async move {
2412 tokio::time::sleep(Duration::from_millis(((N - i) * 4) as u64)).await;
2414 if i == 3 { None } else { Some(i) }
2416 })
2417 .buffered(concurrency)
2418 .filter_map(futures::future::ready)
2419 .collect()
2420 .await;
2421
2422 let expected: Vec<usize> = (0..N).filter(|&i| i != 3).collect();
2423 assert_eq!(
2424 output, expected,
2425 "buffered(concurrency) must emit decoded packets in arrival order regardless of decode latency; got \
2426 {output:?}. A regression to then_concurrent()/FuturesUnordered would emit completion order and fail here."
2427 );
2428 }
2429
2430 #[tokio::test]
2433 async fn a_replayed_packet_should_not_be_acknowledged() -> anyhow::Result<()> {
2434 struct AlwaysReplay(std::sync::atomic::AtomicUsize);
2435
2436 impl PacketDecoder for AlwaysReplay {
2437 type Error = HoprProtocolError;
2438
2439 fn decode(
2440 &self,
2441 sender: PeerId,
2442 _: bytes::Bytes,
2443 ) -> Result<IncomingPacket, IncomingPacketError<Self::Error>> {
2444 self.0.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
2445 Err(IncomingPacketError::Replay(
2446 OffchainPublicKey::from_peerid(&sender)
2447 .map_err(|_| IncomingPacketError::Undecodable(HoprProtocolError::InvalidSender))?
2448 .into(),
2449 ))
2450 }
2451 }
2452
2453 let sender = *OffchainKeypair::random().public();
2454 let (packet_tx, packet_rx) = futures::channel::mpsc::unbounded::<(PeerId, bytes::Bytes)>();
2457 let (wire_out, _wire_out_rx) = futures::channel::mpsc::unbounded::<(PeerId, bytes::Bytes)>();
2458 let (app_out, _app_rx) = futures::channel::mpsc::unbounded::<(HoprPseudonym, ApplicationDataIn)>();
2459 let (ack_out_tx, mut ack_out_rx) = futures::channel::mpsc::unbounded::<(OffchainPublicKey, Option<HalfKey>)>();
2460 let (ack_in_tx, _ack_in_rx) = futures::channel::mpsc::unbounded::<(OffchainPublicKey, Vec<Acknowledgement>)>();
2461 let app_out = app_out.with_timeout(QUEUE_SEND_TIMEOUT);
2463 let ack_out_tx = ack_out_tx.with_timeout(QUEUE_SEND_TIMEOUT);
2464 let ack_in_tx = ack_in_tx.with_timeout(QUEUE_SEND_TIMEOUT);
2465 let (ticket_evt, _ticket_rx) = mpsc::channel::<TicketEvent>(4);
2466
2467 let decoder = std::sync::Arc::new(AlwaysReplay(Default::default()));
2468 let _process = hopr_utils::spawn_as_abortable!(start_incoming_packet_pipeline(
2469 (wire_out, packet_rx),
2470 decoder.clone(),
2471 std::sync::Arc::new(NoopTicketProcessor),
2472 ticket_evt,
2473 (ack_out_tx, ack_in_tx),
2474 app_out,
2475 PeerProtocolCounterRegistry::default(),
2476 1,
2477 ));
2478
2479 packet_tx.unbounded_send((sender.into(), bytes::Bytes::from_static(b"replayed")))?;
2481
2482 tokio::time::timeout(std::time::Duration::from_secs(5), async {
2485 while decoder.0.load(std::sync::atomic::Ordering::SeqCst) == 0 {
2486 tokio::time::sleep(std::time::Duration::from_millis(5)).await;
2487 }
2488 })
2489 .await
2490 .map_err(|_| anyhow::anyhow!("the packet must reach the decoder"))?;
2491
2492 let acked = tokio::time::timeout(std::time::Duration::from_millis(300), ack_out_rx.next())
2493 .await
2494 .ok()
2495 .flatten();
2496 assert!(
2497 acked.is_none(),
2498 "a replayed packet must not be acknowledged, got {acked:?}"
2499 );
2500
2501 Ok(())
2502 }
2503}