1use std::sync::Arc;
2
3use futures::{FutureExt, SinkExt, StreamExt};
4use futures_concurrency::stream::StreamExt as _;
5use hopr_api::{
6 ct::{ProbeRouting, ProbingTrafficGeneration},
7 graph::{EdgeTransportTelemetry, NetworkGraphError, NetworkGraphUpdate, NetworkGraphView},
8 types::{
9 crypto::types::OffchainPublicKey, crypto_random::Randomizable, internal::prelude::*,
10 primitive::traits::AsUnixTimestamp,
11 },
12};
13use hopr_protocol_app::{
14 prelude::{ApplicationDataIn, ApplicationDataOut, OutgoingPacketInfo, ReservedTag},
15 v1::Tag,
16};
17use hopr_transport_tag_allocator::{AllocatedTag, TagAllocator};
18use hopr_utils::{platform::time::native::current_time, runtime::AbortableList};
19
20use crate::{
21 HoprProbeProcess,
22 config::ProbeConfig,
23 content::Message,
24 ping::PingQueryReplier,
25 types::{NeighborProbe, NeighborTelemetry, PathTelemetry},
26};
27
28type CacheNeighborKey = (HoprPseudonym, NeighborProbe);
29type CacheNeighborValue = (Box<NodeId>, std::time::Duration, Option<PingQueryReplier>);
30
31pub enum ProbeDispatch {
33 Consumed,
35 Passthrough(HoprPseudonym, ApplicationDataIn),
37}
38
39#[derive(Clone)]
45pub struct ProbeClassifierState<G> {
46 active_neighbor_probes: moka::future::Cache<CacheNeighborKey, CacheNeighborValue>,
47 active_path_probes: moka::future::Cache<Tag, (PathTelemetry, Arc<AllocatedTag>)>,
48 network_graph: G,
49}
50
51impl<G> ProbeClassifierState<G>
52where
53 G: NetworkGraphUpdate + Clone + Send + Sync + 'static,
54{
55 pub async fn classify<T>(
60 &self,
61 mut push_to_network: T,
62 pseudonym: HoprPseudonym,
63 in_data: ApplicationDataIn,
64 ) -> ProbeDispatch
65 where
66 T: futures::Sink<(DestinationRouting, ApplicationDataOut)> + Unpin + Send + 'static,
67 T::Error: Send,
68 {
69 let tag: Tag = in_data.data.application_tag;
70
71 if let Some((path_telemetry, _allocated_tag)) = self.active_path_probes.remove(&tag).await {
72 let span = tracing::trace_span!("loopback_probe_reply", %pseudonym, %tag);
73 let _entered = span.enter();
74 tracing::trace!(%tag, "loopback probe successfully received");
75 self.network_graph
76 .record_edge::<NeighborTelemetry, PathTelemetry>(hopr_api::graph::MeasurableEdge::Probe(Ok(
77 EdgeTransportTelemetry::Loopback(path_telemetry),
78 )));
79 } else if tag == ReservedTag::Ping.into() {
80 let message: anyhow::Result<Message> = in_data
81 .data
82 .try_into()
83 .map_err(|e| anyhow::anyhow!("failed to convert data into message: {e}"));
84
85 match message {
86 Ok(message) => match message {
87 Message::Telemetry(_) => {
88 tracing::warn!(%pseudonym, "received telemetry on reserved ping tag, ignoring");
89 }
90 Message::Probe(NeighborProbe::Ping(ping)) => {
91 tracing::trace!(%pseudonym, nonce = const_hex::encode(ping), "received ping");
92 tracing::trace!(%pseudonym, nonce = const_hex::encode(ping), "wrapping a pong in the found SURB");
93
94 let message = Message::Probe(NeighborProbe::Pong(ping));
95 if let Ok(data) = message.try_into() {
96 let routing = DestinationRouting::Return(pseudonym.into());
97 let data = ApplicationDataOut::with_no_packet_info(data);
98 if let Err(_error) = push_to_network.send((routing, data)).await {
99 tracing::error!(%pseudonym, "failed to send back a pong");
100 }
101 } else {
102 tracing::error!(%pseudonym, "failed to convert pong message into data");
103 }
104 }
105 Message::Probe(NeighborProbe::Pong(pong)) => {
106 tracing::trace!(%pseudonym, nonce = const_hex::encode(pong), "received pong");
107 if let Some((peer, start, replier)) = self
108 .active_neighbor_probes
109 .remove(&(pseudonym, NeighborProbe::Ping(pong)))
110 .await
111 {
112 let span = tracing::trace_span!("neighbor_probe_reply", %peer, %pseudonym, nonce = const_hex::encode(pong));
113 let _entered = span.enter();
114 let latency = current_time().as_unix_timestamp().saturating_sub(start);
115
116 if let NodeId::Offchain(opk) = peer.as_ref() {
117 tracing::trace!(%pseudonym, nonce = const_hex::encode(pong), latency_ms = latency.as_millis(), "probe successful");
118 self.network_graph.record_edge::<NeighborTelemetry, PathTelemetry>(
119 hopr_api::graph::MeasurableEdge::Probe(Ok(EdgeTransportTelemetry::Neighbor(
120 NeighborTelemetry {
121 peer: *opk,
122 rtt: latency,
123 },
124 ))),
125 )
126 } else {
127 tracing::warn!(%pseudonym, nonce = const_hex::encode(pong), latency_ms = latency.as_millis(), "probe successful to non-offchain peer");
128 }
129
130 if let Some(replier) = replier {
131 replier.notify(Ok(latency));
132 }
133 } else {
134 tracing::trace!(%pseudonym, nonce = const_hex::encode(pong), possible_reasons = "[timeout, adversary]", "received pong for unknown probe");
135 }
136 }
137 },
138 Err(error) => tracing::error!(%pseudonym, %error, "cannot deserialize message"),
139 }
140 } else {
141 return ProbeDispatch::Passthrough(pseudonym, in_data);
142 }
143
144 ProbeDispatch::Consumed
145 }
146
147 pub fn filter_stream<T, S>(
150 self,
151 push_to_network: T,
152 stream: S,
153 ) -> impl futures::Stream<Item = (HoprPseudonym, ApplicationDataIn)>
154 where
155 T: futures::Sink<(DestinationRouting, ApplicationDataOut)> + Clone + Unpin + Send + Sync + 'static,
156 T::Error: Send,
157 S: futures::Stream<Item = (HoprPseudonym, ApplicationDataIn)>,
158 {
159 use futures::StreamExt;
160 stream.filter_map(move |(pseudonym, data)| {
161 let state = self.clone();
162 let push = push_to_network.clone();
163 async move {
164 match state.classify(push, pseudonym, data).await {
165 ProbeDispatch::Consumed => None,
166 ProbeDispatch::Passthrough(ps, d) => Some((ps, d)),
167 }
168 }
169 })
170 }
171}
172
173pub struct Probe {
178 cfg: ProbeConfig,
180 tag_allocator: Arc<dyn TagAllocator + Send + Sync>,
182}
183
184impl Probe {
185 pub fn new(cfg: ProbeConfig, tag_allocator: Arc<dyn TagAllocator + Send + Sync>) -> Self {
186 Self { cfg, tag_allocator }
187 }
188
189 pub async fn continuously_scan<T, V, Tr, G>(
195 self,
196 api_out: T, manual_events: V, probing_traffic_generator: Tr,
199 network_graph: G,
200 ) -> (AbortableList<HoprProbeProcess>, ProbeClassifierState<G>)
201 where
202 T: futures::Sink<(DestinationRouting, ApplicationDataOut)> + Clone + Send + Sync + Unpin + 'static,
203 T::Error: Send,
204 V: futures::Stream<Item = (OffchainPublicKey, PingQueryReplier)> + Send + 'static,
205 Tr: ProbingTrafficGeneration + Send + Sync + 'static,
206 G: NetworkGraphView + NetworkGraphUpdate + Clone + Send + Sync + 'static,
207 {
208 let max_parallel_probes = self.cfg.max_parallel_probes;
209
210 let probing_routes = probing_traffic_generator.build();
211
212 let network_graph_internal_neighbor = network_graph.clone();
214 let network_graph_internal_path = network_graph.clone();
215 let timeout = self.cfg.timeout;
216 let active_neighbor_probes: moka::future::Cache<CacheNeighborKey, CacheNeighborValue> =
217 moka::future::Cache::builder()
218 .time_to_live(timeout)
219 .max_capacity(100_000)
220 .async_eviction_listener(
221 move |k: Arc<CacheNeighborKey>,
222 v: CacheNeighborValue,
223 cause|
224 -> moka::notification::ListenerFuture {
225 if matches!(cause, moka::notification::RemovalCause::Expired) {
226 let store = network_graph_internal_neighbor.clone();
228 let (peer, _start, notifier) = v;
229 let span = tracing::trace_span!("neighbor_probe_expiry", %peer, pseudonym = %k.0);
230 let _entered = span.enter();
231
232 tracing::trace!(%peer, pseudonym = %k.0, probe = %k.1, reason = "timeout", "neighbor probe failed");
233 if let Some(replier) = notifier {
234 if matches!(peer.as_ref(), NodeId::Offchain(_)) {
235 replier.notify(Err(()));
236 } else {
237 tracing::warn!(
238 reason = "non-offchain peer",
239 "cannot notify timeout for non-offchain peer"
240 );
241 }
242 };
243
244 if let NodeId::Offchain(opk) = peer.as_ref() {
245 let opk: OffchainPublicKey = *opk;
246 store
247 .record_edge::<NeighborTelemetry, PathTelemetry>(
248 hopr_api::graph::MeasurableEdge::Probe(Err(
249 NetworkGraphError::ProbeNeighborTimeout(Box::new(opk)),
250 )),
251 );
252 futures::FutureExt::boxed(futures::future::ready(()))
253
254 } else {
255 futures::FutureExt::boxed(futures::future::ready(()))
256 }
257 } else {
258 futures::FutureExt::boxed(futures::future::ready(()))
260 }
261 },
262 )
263 .build();
264
265 let active_path_probes: moka::future::Cache<Tag, (PathTelemetry, Arc<AllocatedTag>)> =
266 moka::future::Cache::builder()
267 .time_to_live(timeout)
268 .max_capacity(100_000)
269 .async_eviction_listener(
270 move |tag: Arc<Tag>,
271 (path, _allocated_tag): (PathTelemetry, Arc<AllocatedTag>),
272 cause|
273 -> moka::notification::ListenerFuture {
274 if matches!(cause, moka::notification::RemovalCause::Expired) {
275 let span = tracing::trace_span!("loopback_probe_expiry", %tag);
277 let _entered = span.enter();
278 let store = network_graph_internal_path.clone();
279
280 tracing::trace!(%tag, reason = "timeout", "loopback probe failed");
281
282 store.record_edge::<NeighborTelemetry, PathTelemetry>(
283 hopr_api::graph::MeasurableEdge::Probe(Err(NetworkGraphError::ProbeLoopbackTimeout(
284 path,
285 ))),
286 );
287 futures::FutureExt::boxed(futures::future::ready(()))
288 } else {
289 futures::FutureExt::boxed(futures::future::ready(()))
291 }
292 },
293 )
294 .build();
295
296 let push_to_network = api_out.clone();
297
298 let mut processes = AbortableList::default();
299
300 let direct_neighbors =
302 probing_routes
303 .map(|peer| (peer, None))
304 .merge(manual_events.filter_map(|(peer, notifier)| async move {
305 let routing = DestinationRouting::Forward {
306 destination: Box::new(peer.into()),
307 pseudonym: Some(HoprPseudonym::random()),
308 forward_options: RoutingOptions::Hops(0.try_into().expect("0 is a valid u8")),
309 return_options: Some(RoutingOptions::Hops(0.try_into().expect("0 is a valid u8"))),
310 };
311 Some((ProbeRouting::Neighbor(routing), Some(notifier)))
312 }));
313
314 let tag_allocator = self.tag_allocator.clone();
315 let classifier_neighbor_probes = active_neighbor_probes.clone();
316 let classifier_path_probes = active_path_probes.clone();
317 let emit_diag = hopr_utils::runtime::diagnostics::ConcurrentDiagnostics::new(
318 "probe_emit_for_each_concurrent",
319 module_path!(),
320 file!(),
321 line!(),
322 );
323 processes.insert(
324 HoprProbeProcess::Emit,
325 hopr_utils::spawn_as_abortable_named!("probe_emit", async move {
326 direct_neighbors
327 .for_each_concurrent(max_parallel_probes, move |(peer, notifier)| {
328 let active_neighbor_probes = active_neighbor_probes.clone();
329 let active_path_probes = active_path_probes.clone();
330 let push_to_network = push_to_network.clone();
331 let tag_allocator = tag_allocator.clone();
332 let emit_diag = emit_diag.clone();
333
334 emit_diag.wrap(|| async move {
335 match peer {
336 ProbeRouting::Neighbor(DestinationRouting::Forward {
337 destination,
338 pseudonym,
339 forward_options,
340 return_options,
341 }) => {
342 let nonce = NeighborProbe::random_nonce();
343
344 let message = Message::Probe(nonce);
345
346 if let Ok(data) = message.try_into() {
347 let routing = DestinationRouting::Forward {
348 destination: destination.clone(),
349 pseudonym,
350 forward_options,
351 return_options,
352 };
353 let data = ApplicationDataOut {
356 data,
357 packet_info: Some(OutgoingPacketInfo {
358 max_surbs_in_packet: 1,
359 ..Default::default()
360 }),
361 surb_refund: Default::default(),
362 };
363 let mut push_to_network = push_to_network.clone();
364
365 if let Err(_error) = push_to_network.send((routing, data)).await {
366 tracing::error!("failed to send out a ping");
367 } else {
368 active_neighbor_probes
369 .insert(
370 (
371 pseudonym
372 .expect("the pseudonym must be present in Forward routing"),
373 nonce,
374 ),
375 (destination, current_time().as_unix_timestamp(), notifier),
376 )
377 .await;
378 }
379 } else {
380 tracing::error!("failed to convert ping message into data");
381 }
382 }
383 ProbeRouting::Neighbor(DestinationRouting::Return(_surb_matcher)) => tracing::error!(
384 error = "logical error",
385 "resolved transport routing is not forward"
386 ),
387 ProbeRouting::Looping((routing, path_id)) => {
388 let message = Message::Telemetry(PathTelemetry {
389 id: hopr_api::types::crypto_random::random_bytes(),
390 path: std::array::from_fn(|i| path_id[i / 8].to_le_bytes()[i % 8]),
391 timestamp: std::time::SystemTime::now()
392 .duration_since(std::time::UNIX_EPOCH)
393 .unwrap_or_default()
394 .as_millis(),
395 });
396
397 if let Some(allocated_tag) = tag_allocator.allocate() {
398 let tag_value = allocated_tag.value();
399
400 if let Ok(packet) = hopr_protocol_app::prelude::ApplicationData::new(
401 tag_value,
402 message.to_bytes().as_ref(),
403 ) {
404 let mut push_to_network = push_to_network.clone();
405
406 if let Err(_error) = push_to_network
409 .send((
410 routing,
411 ApplicationDataOut {
412 data: packet,
413 packet_info: Some(OutgoingPacketInfo {
414 max_surbs_in_packet: 0,
415 ..Default::default()
416 }),
417 surb_refund: Default::default(),
418 },
419 ))
420 .await
421 {
422 tracing::error!("failed to send out a ping");
423 } else {
424 if let Message::Telemetry(telemetry) = message {
426 active_path_probes
427 .insert(tag_value.into(), (telemetry, Arc::new(allocated_tag)))
428 .await;
429 }
430 }
431 } else {
432 tracing::error!("failed to construct data for path telemetry")
433 }
434 } else {
435 tracing::warn!("probing telemetry tag pool exhausted, skipping loopback probe");
436 }
437 }
438 }
439 })
440 })
441 .inspect(|_| {
442 tracing::warn!(
443 task = "transport (probe - generate outgoing)",
444 "long-running background task finished"
445 )
446 })
447 .await;
448 }),
449 );
450
451 let classifier = ProbeClassifierState {
452 active_neighbor_probes: classifier_neighbor_probes,
453 active_path_probes: classifier_path_probes,
454 network_graph,
455 };
456
457 (processes, classifier)
458 }
459}
460
461#[cfg(test)]
462mod tests {
463 use std::{collections::VecDeque, sync::RwLock, time::Duration};
464
465 use async_trait::async_trait;
466 use futures::future::BoxFuture;
467 use hopr_api::{
468 graph::{
469 EdgeLinkObservable, MeasurableEdge, NetworkGraphError,
470 traits::{EdgeNetworkObservableRead, EdgeObservableRead, EdgeObservableWrite, EdgeProtocolObservable},
471 },
472 types::crypto::keypairs::{ChainKeypair, Keypair, OffchainKeypair},
473 };
474 use hopr_protocol_app::prelude::{ApplicationData, ReservedTag, Tag};
475
476 use super::*;
477 use crate::errors::ProbeError;
478
479 lazy_static::lazy_static!(
480 static ref OFFCHAIN_KEYPAIR: OffchainKeypair = OffchainKeypair::random();
481 static ref ONCHAIN_KEYPAIR: ChainKeypair = ChainKeypair::random();
482 static ref NEIGHBOURS: Vec<OffchainPublicKey> = vec![
483 *OffchainKeypair::random().public(),
484 *OffchainKeypair::random().public(),
485 *OffchainKeypair::random().public(),
486 *OffchainKeypair::random().public(),
487 ];
488 );
489
490 #[derive(Debug, Clone, Copy, Default)]
492 pub struct TestEdgeTransportObservations;
493
494 impl EdgeLinkObservable for TestEdgeTransportObservations {
495 fn record(&mut self, _latency: std::result::Result<Duration, ()>) {}
496
497 fn average_latency(&self) -> Option<Duration> {
498 None
499 }
500
501 fn average_probe_rate(&self) -> Option<f64> {
504 Some(1.0)
505 }
506
507 fn score(&self) -> Option<f64> {
508 Some(1.0)
509 }
510 }
511
512 impl EdgeNetworkObservableRead for TestEdgeTransportObservations {
513 fn is_connected(&self) -> Option<bool> {
514 Some(true)
515 }
516 }
517
518 impl EdgeProtocolObservable for TestEdgeTransportObservations {
519 fn balance(&self) -> Option<hopr_api::graph::traits::Balance> {
520 None
521 }
522 }
523
524 impl hopr_api::graph::EdgeImmediateProtocolObservable for TestEdgeTransportObservations {
525 fn ack_rate(&self) -> Option<f64> {
526 None
527 }
528 }
529
530 #[derive(Debug, Clone, Copy, Default)]
531 pub struct TestEdgeObservations;
532
533 impl EdgeObservableWrite for TestEdgeObservations {
534 fn record(&mut self, _measurement: hopr_api::graph::traits::EdgeWeightType) {}
535 }
536
537 impl EdgeObservableRead for TestEdgeObservations {
538 type ImmediateMeasurement = TestEdgeTransportObservations;
539 type IntermediateMeasurement = TestEdgeTransportObservations;
540
541 fn last_update(&self) -> std::time::Duration {
542 std::time::SystemTime::now()
543 .duration_since(std::time::UNIX_EPOCH)
544 .unwrap_or_default()
545 }
546
547 fn immediate_qos(&self) -> Option<&Self::ImmediateMeasurement> {
548 None
549 }
550
551 fn intermediate_qos(&self) -> Option<&Self::IntermediateMeasurement> {
552 None
553 }
554
555 fn score(&self) -> Option<f64> {
556 Some(1.0)
557 }
558 }
559
560 #[derive(Debug, Clone)]
561 pub struct PeerStore {
562 me: OffchainPublicKey,
563 get_peers: Arc<RwLock<VecDeque<Vec<OffchainPublicKey>>>>,
564 #[allow(clippy::type_complexity)]
565 on_finished: Arc<RwLock<Vec<(OffchainPublicKey, crate::errors::Result<Duration>)>>>,
566 }
567
568 impl NetworkGraphUpdate for PeerStore {
569 fn record_edge<N, P>(&self, telemetry: MeasurableEdge<N, P>)
571 where
572 N: hopr_api::graph::MeasurablePeer + Send + Clone,
573 P: hopr_api::graph::MeasurablePath + Send + Clone,
574 {
575 let mut on_finished = self.on_finished.write().unwrap();
576
577 match telemetry {
578 hopr_api::graph::MeasurableEdge::Probe(Ok(EdgeTransportTelemetry::Neighbor(neighbor_telemetry))) => {
579 let peer: OffchainPublicKey = *neighbor_telemetry.peer();
580 let duration = neighbor_telemetry.rtt();
581 on_finished.push((peer, Ok(duration)));
582 }
583 hopr_api::graph::MeasurableEdge::Probe(Err(NetworkGraphError::ProbeNeighborTimeout(peer))) => {
584 on_finished.push((
585 *peer.as_ref(),
586 Err(ProbeError::TrafficError(NetworkGraphError::ProbeNeighborTimeout(peer))),
587 ));
588 }
589 MeasurableEdge::Probe(Ok(EdgeTransportTelemetry::Loopback(_)))
590 | MeasurableEdge::Probe(Err(NetworkGraphError::ProbeLoopbackTimeout(_))) => {}
591 _ => panic!("unexpected telemetry type, unimplemented"),
592 }
593 }
594
595 fn record_node<N>(&self, _node: N)
596 where
597 N: hopr_api::graph::MeasurableNode + Send + Clone,
598 {
599 unimplemented!()
600 }
601
602 fn set_ticket_face_value(&self, _ticket_face_value: hopr_api::graph::traits::Balance) {}
603 }
604
605 #[async_trait]
606 impl NetworkGraphView for PeerStore {
607 type NodeId = OffchainPublicKey;
608 type Observed = TestEdgeObservations;
609
610 fn identity(&self) -> &OffchainPublicKey {
611 &self.me
612 }
613
614 fn ticket_face_value(&self) -> Option<hopr_api::graph::traits::Balance> {
615 None
616 }
617
618 fn node_count(&self) -> usize {
619 self.get_peers.read().unwrap().front().map_or(0, |v| v.len())
620 }
621
622 fn contains_node(&self, _key: &OffchainPublicKey) -> bool {
623 false
624 }
625
626 fn nodes(&self) -> futures::stream::BoxStream<'static, OffchainPublicKey> {
628 let mut get_peers = self.get_peers.write().unwrap();
629 Box::pin(futures::stream::iter(get_peers.pop_front().unwrap_or_default()))
630 }
631
632 fn edge(&self, _src: &OffchainPublicKey, _dest: &OffchainPublicKey) -> Option<TestEdgeObservations> {
633 Some(TestEdgeObservations)
634 }
635
636 fn path_slot(&self, _key: &OffchainPublicKey) -> Option<u64> {
638 None
639 }
640 }
641
642 type TestClassifier = ProbeClassifierState<PeerStore>;
643
644 #[derive(Clone, Default)]
646 struct ProbeLogs {
647 spans: Arc<parking_lot::Mutex<Vec<&'static str>>>,
648 messages: Arc<parking_lot::Mutex<Vec<(String, usize)>>>,
649 depth: Arc<std::sync::atomic::AtomicUsize>,
650 }
651
652 struct MessageVisitor(String);
653
654 impl tracing::field::Visit for MessageVisitor {
655 fn record_debug(&mut self, field: &tracing::field::Field, value: &dyn std::fmt::Debug) {
657 if field.name() == "message" {
658 self.0 = format!("{value:?}");
659 }
660 }
661 }
662
663 impl tracing::Subscriber for ProbeLogs {
664 fn enabled(&self, metadata: &tracing::Metadata<'_>) -> bool {
665 metadata.target().starts_with("hopr_transport_probe")
668 }
669
670 fn new_span(&self, attrs: &tracing::span::Attributes<'_>) -> tracing::span::Id {
671 let mut spans = self.spans.lock();
672 spans.push(attrs.metadata().name());
673 tracing::span::Id::from_u64(spans.len() as u64)
674 }
675
676 fn record(&self, _: &tracing::span::Id, _: &tracing::span::Record<'_>) {}
677
678 fn record_follows_from(&self, _: &tracing::span::Id, _: &tracing::span::Id) {}
679
680 fn event(&self, event: &tracing::Event<'_>) {
681 let mut message = MessageVisitor(String::new());
682 event.record(&mut message);
683 self.messages
684 .lock()
685 .push((message.0, self.depth.load(std::sync::atomic::Ordering::Relaxed)));
686 }
687
688 fn enter(&self, _: &tracing::span::Id) {
689 self.depth.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
690 }
691
692 fn exit(&self, _: &tracing::span::Id) {
693 self.depth.fetch_sub(1, std::sync::atomic::Ordering::Relaxed);
694 }
695 }
696
697 #[test]
698 fn probe_reply_logs_stay_in_the_producer_span() -> anyhow::Result<()> {
700 let logs = ProbeLogs::default();
701 tracing::subscriber::with_default(logs.clone(), || {
702 tokio::runtime::Builder::new_current_thread()
703 .enable_all()
704 .build()?
705 .block_on(async {
706 let store = PeerStore {
707 me: *OFFCHAIN_KEYPAIR.public(),
708 get_peers: Arc::new(RwLock::new(VecDeque::new())),
709 on_finished: Arc::new(RwLock::new(Vec::new())),
710 };
711 let classifier = ProbeClassifierState {
712 active_neighbor_probes: moka::future::Cache::new(1),
713 active_path_probes: moka::future::Cache::new(1),
714 network_graph: store.clone(),
715 };
716 let pseudonym = HoprPseudonym::random();
717 let ordinary = ApplicationDataIn {
718 data: ApplicationData::new(Tag::MAX, b"ordinary")?,
719 packet_info: Default::default(),
720 };
721 assert!(matches!(
722 classifier.classify(futures::sink::drain(), pseudonym, ordinary).await,
723 ProbeDispatch::Passthrough(..)
724 ));
725 assert!(
726 logs.spans.lock().is_empty(),
727 "ordinary packets must not create probe spans: {:?}",
728 *logs.spans.lock()
729 );
730
731 let nonce = [7; NeighborProbe::NONCE_SIZE];
732 classifier
733 .active_neighbor_probes
734 .insert(
735 (pseudonym, NeighborProbe::Ping(nonce)),
736 (
737 Box::new(NodeId::Offchain(NEIGHBOURS[0])),
738 current_time().as_unix_timestamp(),
739 None,
740 ),
741 )
742 .await;
743 classifier
744 .classify(
745 futures::sink::drain(),
746 pseudonym,
747 ApplicationDataIn {
748 data: Message::Probe(NeighborProbe::Pong(nonce)).try_into()?,
749 packet_info: Default::default(),
750 },
751 )
752 .await;
753 assert_eq!(&*logs.spans.lock(), &["neighbor_probe_reply"]);
754 assert!(
755 logs.messages
756 .lock()
757 .iter()
758 .any(|(message, depth)| { message == "probe successful" && *depth == 1 })
759 );
760 assert_eq!(
761 store
762 .on_finished
763 .read()
764 .map_err(|_| anyhow::anyhow!("probe graph lock poisoned"))?
765 .len(),
766 1
767 );
768 anyhow::Ok(())
769 })
770 })
771 }
772
773 struct TestInterface {
774 probe_classifier: TestClassifier,
775 from_probing_to_network_rx: futures::channel::mpsc::Receiver<(DestinationRouting, ApplicationDataOut)>,
776 from_probing_to_network_tx: futures::channel::mpsc::Sender<(DestinationRouting, ApplicationDataOut)>,
777 manual_probe_tx: futures::channel::mpsc::Sender<(OffchainPublicKey, PingQueryReplier)>,
778 }
779
780 async fn test_with_probing<F, Fut>(cfg: ProbeConfig, store: PeerStore, test: F) -> anyhow::Result<()>
782 where
783 Fut: std::future::Future<Output = anyhow::Result<()>>,
784 F: Fn(TestInterface) -> Fut + Send + Sync + 'static,
785 {
786 let tag_allocators = hopr_transport_tag_allocator::create_allocators(
787 ReservedTag::range().end..u16::MAX as u64 + 1,
788 [
789 (hopr_transport_tag_allocator::Usage::Session, 2048),
790 (hopr_transport_tag_allocator::Usage::SessionTerminalTelemetry, 4000),
791 (hopr_transport_tag_allocator::Usage::ProvingTelemetry, 10000),
792 ],
793 )
794 .expect("tag allocators should be created");
795 let probing_allocator = tag_allocators
796 .into_iter()
797 .find_map(|(u, alloc)| matches!(u, hopr_transport_tag_allocator::Usage::ProvingTelemetry).then_some(alloc))
798 .expect("probing allocator should exist");
799
800 let probe = Probe::new(cfg, probing_allocator);
801
802 let (from_probing_to_network_tx, from_probing_to_network_rx) =
803 futures::channel::mpsc::channel::<(DestinationRouting, ApplicationDataOut)>(100);
804
805 let (manual_probe_tx, manual_probe_rx) =
806 futures::channel::mpsc::channel::<(OffchainPublicKey, PingQueryReplier)>(100);
807
808 let (jhs, probe_classifier) = probe
809 .continuously_scan(
810 from_probing_to_network_tx.clone(),
811 manual_probe_rx,
812 TestProbeStrategy::ImmediateNeighbor { store: store.clone() },
813 store,
814 )
815 .await;
816
817 let interface = TestInterface {
818 probe_classifier,
819 from_probing_to_network_rx,
820 from_probing_to_network_tx,
821 manual_probe_tx,
822 };
823
824 let result = test(interface).await;
825
826 jhs.abort_all();
827
828 result
829 }
830
831 const NO_PROBE_PASSES: f64 = 0.0;
832 const ALL_PROBES_PASS: f64 = 1.0;
833
834 fn concurrent_classify(
837 delay: Option<std::time::Duration>,
838 pass_rate: f64,
839 classifier: TestClassifier,
840 push_to_network: futures::channel::mpsc::Sender<(DestinationRouting, ApplicationDataOut)>,
841 ) -> impl Fn((DestinationRouting, ApplicationDataOut)) -> BoxFuture<'static, ()> {
842 debug_assert!(
843 (NO_PROBE_PASSES..=ALL_PROBES_PASS).contains(&pass_rate),
844 "Pass rate must be between {NO_PROBE_PASSES} and {ALL_PROBES_PASS}"
845 );
846
847 move |(path, data_out): (DestinationRouting, ApplicationDataOut)| -> BoxFuture<'static, ()> {
848 let classifier = classifier.clone();
849 let push_to_network = push_to_network.clone();
850
851 Box::pin(async move {
852 if let DestinationRouting::Forward { pseudonym, .. } = path {
853 let message: Message = data_out.data.try_into().expect("failed to convert data into message");
854 if let Message::Probe(NeighborProbe::Ping(ping)) = message {
855 let pong_message = Message::Probe(NeighborProbe::Pong(ping));
856
857 if let Some(delay) = delay {
858 tokio::time::sleep(delay).await;
859 }
860
861 if rand::random_range(NO_PROBE_PASSES..=ALL_PROBES_PASS) < pass_rate {
862 let pseudonym = pseudonym.expect("the pseudonym is always known from cache");
863 classifier
864 .classify(
865 push_to_network,
866 pseudonym,
867 ApplicationDataIn {
868 data: pong_message
869 .try_into()
870 .expect("failed to convert pong message into data"),
871 packet_info: Default::default(),
872 },
873 )
874 .await;
875 }
876 }
877 };
878 })
879 }
880 }
881
882 #[tokio::test]
883 async fn probe_should_record_value_for_manual_neighbor_probe() -> anyhow::Result<()> {
885 let cfg = ProbeConfig {
886 timeout: std::time::Duration::from_millis(5),
887 interval: std::time::Duration::from_secs(0),
888 ..Default::default()
889 };
890
891 let store = PeerStore {
892 me: *OFFCHAIN_KEYPAIR.public(),
893 get_peers: Arc::new(RwLock::new(VecDeque::new())),
894 on_finished: Arc::new(RwLock::new(Vec::new())),
895 };
896
897 test_with_probing(cfg, store, move |iface: TestInterface| async move {
898 let mut manual_probe_tx = iface.manual_probe_tx;
899 let from_probing_to_network_rx = iface.from_probing_to_network_rx;
900 let from_probing_to_network_tx = iface.from_probing_to_network_tx;
901 let probe_classifier = iface.probe_classifier;
902
903 let (tx, mut rx) = futures::channel::mpsc::channel::<std::result::Result<Duration, ()>>(128);
904 manual_probe_tx.send((NEIGHBOURS[0], PingQueryReplier::new(tx))).await?;
905
906 let _jh: hopr_utils::runtime::prelude::JoinHandle<()> = tokio::spawn(async move {
907 from_probing_to_network_rx
908 .for_each_concurrent(
909 cfg.max_parallel_probes + 1,
910 concurrent_classify(None, ALL_PROBES_PASS, probe_classifier, from_probing_to_network_tx),
911 )
912 .await;
913 });
914
915 let _duration = tokio::time::timeout(std::time::Duration::from_secs(1), rx.next())
916 .await?
917 .ok_or_else(|| anyhow::anyhow!("Probe did not return a result in time"))?
918 .map_err(|_| anyhow::anyhow!("Probe failed"))?;
919
920 Ok(())
921 })
922 .await
923 }
924
925 #[tokio::test]
926 async fn probe_should_record_failure_on_manual_fail() -> anyhow::Result<()> {
928 let cfg = ProbeConfig {
929 timeout: std::time::Duration::from_millis(5),
930 interval: std::time::Duration::from_secs(0),
931 ..Default::default()
932 };
933
934 let store = PeerStore {
935 me: *OFFCHAIN_KEYPAIR.public(),
936 get_peers: Arc::new(RwLock::new(VecDeque::new())),
937 on_finished: Arc::new(RwLock::new(Vec::new())),
938 };
939
940 test_with_probing(cfg, store, move |iface: TestInterface| async move {
941 let mut manual_probe_tx = iface.manual_probe_tx;
942 let from_probing_to_network_rx = iface.from_probing_to_network_rx;
943 let from_probing_to_network_tx = iface.from_probing_to_network_tx;
944 let probe_classifier = iface.probe_classifier;
945
946 let (tx, mut rx) = futures::channel::mpsc::channel::<std::result::Result<Duration, ()>>(128);
947 manual_probe_tx.send((NEIGHBOURS[0], PingQueryReplier::new(tx))).await?;
948
949 let _jh: hopr_utils::runtime::prelude::JoinHandle<()> = tokio::spawn(async move {
950 from_probing_to_network_rx
951 .for_each_concurrent(
952 cfg.max_parallel_probes + 1,
953 concurrent_classify(None, NO_PROBE_PASSES, probe_classifier, from_probing_to_network_tx),
954 )
955 .await;
956 });
957
958 assert!(tokio::time::timeout(cfg.timeout * 2, rx.next()).await.is_err());
959
960 Ok(())
961 })
962 .await
963 }
964
965 #[tokio::test]
966 async fn probe_should_record_results_of_successful_automatically_generated_probes() -> anyhow::Result<()> {
968 let cfg = ProbeConfig {
969 timeout: std::time::Duration::from_millis(20),
970 max_parallel_probes: NEIGHBOURS.len(),
971 interval: std::time::Duration::from_secs(0),
972 ..Default::default()
973 };
974
975 let store = PeerStore {
976 me: *OFFCHAIN_KEYPAIR.public(),
977 get_peers: Arc::new(RwLock::new({
978 let mut neighbors = VecDeque::new();
979 neighbors.push_back(NEIGHBOURS.clone());
980 neighbors
981 })),
982 on_finished: Arc::new(RwLock::new(Vec::new())),
983 };
984
985 test_with_probing(cfg, store.clone(), move |iface: TestInterface| async move {
986 let from_probing_to_network_rx = iface.from_probing_to_network_rx;
987 let from_probing_to_network_tx = iface.from_probing_to_network_tx;
988 let probe_classifier = iface.probe_classifier;
989
990 let _jh: hopr_utils::runtime::prelude::JoinHandle<()> = tokio::spawn(async move {
991 from_probing_to_network_rx
992 .for_each_concurrent(
993 cfg.max_parallel_probes + 1,
994 concurrent_classify(None, ALL_PROBES_PASS, probe_classifier, from_probing_to_network_tx),
995 )
996 .await;
997 });
998
999 tokio::time::sleep(cfg.timeout * 3).await;
1003
1004 Ok(())
1005 })
1006 .await?;
1007
1008 assert_eq!(
1009 store
1010 .on_finished
1011 .read()
1012 .expect("should be lockable")
1013 .iter()
1014 .filter(|(_peer, result)| result.is_ok())
1015 .count(),
1016 NEIGHBOURS.len()
1017 );
1018
1019 Ok(())
1020 }
1021
1022 #[tokio::test]
1023 async fn probe_should_record_results_of_timed_out_automatically_generated_probes() -> anyhow::Result<()> {
1025 let cfg = ProbeConfig {
1026 timeout: std::time::Duration::from_millis(10),
1027 max_parallel_probes: NEIGHBOURS.len(),
1028 interval: std::time::Duration::from_secs(0),
1029 ..Default::default()
1030 };
1031
1032 let store = PeerStore {
1033 me: *OFFCHAIN_KEYPAIR.public(),
1034 get_peers: Arc::new(RwLock::new({
1035 let mut neighbors = VecDeque::new();
1036 neighbors.push_back(NEIGHBOURS.clone());
1037 neighbors
1038 })),
1039 on_finished: Arc::new(RwLock::new(Vec::new())),
1040 };
1041
1042 let timeout = cfg.timeout * 2;
1043
1044 test_with_probing(cfg, store.clone(), move |iface: TestInterface| async move {
1045 let from_probing_to_network_rx = iface.from_probing_to_network_rx;
1046 let from_probing_to_network_tx = iface.from_probing_to_network_tx;
1047 let probe_classifier = iface.probe_classifier;
1048
1049 let _jh: hopr_utils::runtime::prelude::JoinHandle<()> = tokio::spawn(async move {
1050 from_probing_to_network_rx
1051 .for_each_concurrent(
1052 cfg.max_parallel_probes + 1,
1053 concurrent_classify(
1054 Some(timeout),
1055 ALL_PROBES_PASS,
1056 probe_classifier,
1057 from_probing_to_network_tx,
1058 ),
1059 )
1060 .await;
1061 });
1062
1063 tokio::time::sleep(timeout * 2).await;
1065
1066 Ok(())
1067 })
1068 .await?;
1069
1070 assert_eq!(
1071 store
1072 .on_finished
1073 .read()
1074 .expect("should be lockable")
1075 .iter()
1076 .filter(|(_peer, result)| result.is_err())
1077 .count(),
1078 NEIGHBOURS.len()
1079 );
1080
1081 Ok(())
1082 }
1083
1084 #[tokio::test]
1085 async fn probe_should_reply_with_pong_when_receiving_ping() -> anyhow::Result<()> {
1086 use anyhow::Context;
1087
1088 let cfg = ProbeConfig {
1089 timeout: std::time::Duration::from_millis(100),
1090 interval: std::time::Duration::from_secs(10),
1091 ..Default::default()
1092 };
1093
1094 let store = PeerStore {
1095 me: *OFFCHAIN_KEYPAIR.public(),
1096 get_peers: Arc::new(RwLock::new(VecDeque::new())),
1097 on_finished: Arc::new(RwLock::new(Vec::new())),
1098 };
1099
1100 test_with_probing(cfg, store, move |iface: TestInterface| async move {
1101 let probe_classifier = iface.probe_classifier;
1102 let from_probing_to_network_tx = iface.from_probing_to_network_tx;
1103 let mut from_probing_to_network_rx = iface.from_probing_to_network_rx;
1104
1105 let ping = NeighborProbe::random_nonce();
1107 let ping_nonce = match ping {
1108 NeighborProbe::Ping(n) => n,
1109 _ => unreachable!(),
1110 };
1111 let ping_msg = Message::Probe(ping);
1112 let app_data: ApplicationData = ping_msg.try_into().context("converting ping to ApplicationData")?;
1113
1114 let result = probe_classifier
1116 .classify(
1117 from_probing_to_network_tx,
1118 HoprPseudonym::random(),
1119 ApplicationDataIn {
1120 data: app_data,
1121 packet_info: Default::default(),
1122 },
1123 )
1124 .await;
1125 anyhow::ensure!(matches!(result, ProbeDispatch::Consumed), "ping should be consumed");
1126
1127 let (routing, data_out) = tokio::time::timeout(Duration::from_secs(2), from_probing_to_network_rx.next())
1129 .await
1130 .context("timeout waiting for pong")?
1131 .context("probe should send pong reply")?;
1132
1133 anyhow::ensure!(
1135 matches!(routing, DestinationRouting::Return(_)),
1136 "pong should use Return routing, got: {routing:?}"
1137 );
1138
1139 let response_msg: Message = data_out.data.try_into().context("converting response to Message")?;
1141 anyhow::ensure!(
1142 matches!(response_msg, Message::Probe(NeighborProbe::Pong(n)) if n == ping_nonce),
1143 "response should be Pong with matching nonce"
1144 );
1145
1146 Ok(())
1147 })
1148 .await
1149 }
1150
1151 #[tokio::test]
1152 async fn probe_should_pass_through_non_associated_tags() -> anyhow::Result<()> {
1154 let cfg = ProbeConfig {
1155 timeout: std::time::Duration::from_millis(20),
1156 interval: std::time::Duration::from_secs(0),
1157 ..Default::default()
1158 };
1159
1160 let store = PeerStore {
1161 me: *OFFCHAIN_KEYPAIR.public(),
1162 get_peers: Arc::new(RwLock::new({
1163 let mut neighbors = VecDeque::new();
1164 neighbors.push_back(NEIGHBOURS.clone());
1165 neighbors
1166 })),
1167 on_finished: Arc::new(RwLock::new(Vec::new())),
1168 };
1169
1170 test_with_probing(cfg, store.clone(), move |iface: TestInterface| async move {
1171 let probe_classifier = iface.probe_classifier;
1172 let from_probing_to_network_tx = iface.from_probing_to_network_tx;
1173
1174 let expected_data = ApplicationData::new(Tag::MAX, b"Hello, this is a test message!")?;
1175
1176 let result = probe_classifier
1177 .classify(
1178 from_probing_to_network_tx,
1179 HoprPseudonym::random(),
1180 ApplicationDataIn {
1181 data: expected_data.clone(),
1182 packet_info: Default::default(),
1183 },
1184 )
1185 .await;
1186
1187 match result {
1188 ProbeDispatch::Passthrough(_, actual) => assert_eq!(actual.data, expected_data),
1189 ProbeDispatch::Consumed => anyhow::bail!("expected Passthrough, got Consumed"),
1190 }
1191
1192 Ok(())
1193 })
1194 .await
1195 }
1196
1197 #[derive(Clone)]
1201 enum TestProbeStrategy {
1202 ManualNeighbor,
1203 ImmediateNeighbor {
1204 store: PeerStore,
1205 },
1206 OneShotLoopback {
1207 routing: DestinationRouting,
1208 path_id: hopr_api::types::internal::routing::PathId,
1209 },
1210 }
1211
1212 impl hopr_api::ct::ProbingTrafficGeneration for TestProbeStrategy {
1213 fn build(&self) -> futures::stream::BoxStream<'static, hopr_api::ct::ProbeRouting> {
1214 match self {
1215 Self::ManualNeighbor => Box::pin(futures::stream::pending()),
1216 Self::ImmediateNeighbor { store } => {
1217 let peers: Vec<OffchainPublicKey> =
1218 store.get_peers.write().unwrap().pop_front().unwrap_or_default();
1219 Box::pin(futures::StreamExt::chain(
1220 futures::stream::iter(peers.into_iter().map(|peer| {
1221 ProbeRouting::Neighbor(DestinationRouting::Forward {
1222 destination: Box::new(peer.into()),
1223 pseudonym: Some(HoprPseudonym::random()),
1224 forward_options: RoutingOptions::Hops(0.try_into().expect("0 is a valid u8")),
1225 return_options: Some(RoutingOptions::Hops(0.try_into().expect("0 is a valid u8"))),
1226 })
1227 })),
1228 futures::stream::pending(),
1229 ))
1230 }
1231 Self::OneShotLoopback { routing, path_id } => {
1232 let probe = hopr_api::ct::ProbeRouting::Looping((routing.clone(), *path_id));
1233 Box::pin(futures::StreamExt::chain(
1234 futures::stream::iter(std::iter::once(probe)),
1235 futures::stream::pending(),
1236 ))
1237 }
1238 }
1239 }
1240 }
1241
1242 #[rstest::rstest]
1249 #[case::neighbor_probe_requests_one_surb(TestProbeStrategy::ManualNeighbor, 1)]
1250 #[case::loopback_probe_requests_zero_surbs(
1251 TestProbeStrategy::OneShotLoopback {
1252 routing: DestinationRouting::Forward {
1253 destination: Box::new((*OFFCHAIN_KEYPAIR.public()).into()),
1254 pseudonym: Some(HoprPseudonym::random()),
1255 forward_options: RoutingOptions::Hops(1.try_into().expect("1 is a valid u8")),
1256 return_options: None,
1257 },
1258 path_id: [1, 2, 3, 4, 5],
1259 },
1260 0,
1261 )]
1262 #[tokio::test]
1263 async fn probe_should_emit_with_expected_surb_count(
1265 #[case] strategy: TestProbeStrategy,
1266 #[case] expected_max_surbs: usize,
1267 #[values(true, false)] reply: bool,
1268 ) -> anyhow::Result<()> {
1269 use anyhow::Context;
1270
1271 let cfg = ProbeConfig {
1272 timeout: std::time::Duration::from_millis(200),
1273 interval: std::time::Duration::from_secs(0),
1274 ..Default::default()
1275 };
1276
1277 let tag_allocators = hopr_transport_tag_allocator::create_allocators(
1280 ReservedTag::range().end..u16::MAX as u64 + 1,
1281 [
1282 (hopr_transport_tag_allocator::Usage::Session, 2048),
1283 (hopr_transport_tag_allocator::Usage::SessionTerminalTelemetry, 4000),
1284 (hopr_transport_tag_allocator::Usage::ProvingTelemetry, 10000),
1285 ],
1286 )
1287 .expect("tag allocators should be created");
1288 let probing_allocator = tag_allocators
1289 .into_iter()
1290 .find_map(|(u, alloc)| matches!(u, hopr_transport_tag_allocator::Usage::ProvingTelemetry).then_some(alloc))
1291 .expect("probing allocator should exist");
1292
1293 let probe = Probe::new(cfg, probing_allocator);
1294
1295 let (from_probing_to_network_tx, mut from_probing_to_network_rx) =
1296 futures::channel::mpsc::channel::<(DestinationRouting, ApplicationDataOut)>(100);
1297 let (mut manual_probe_tx, manual_probe_rx) =
1298 futures::channel::mpsc::channel::<(OffchainPublicKey, PingQueryReplier)>(100);
1299
1300 let store = PeerStore {
1301 me: *OFFCHAIN_KEYPAIR.public(),
1302 get_peers: Arc::new(RwLock::new(VecDeque::new())),
1303 on_finished: Arc::new(RwLock::new(Vec::new())),
1304 };
1305
1306 let is_manual = matches!(strategy, TestProbeStrategy::ManualNeighbor);
1309 let (jhs, classifier) = probe
1310 .continuously_scan(from_probing_to_network_tx, manual_probe_rx, strategy, store.clone())
1311 .await;
1312
1313 if is_manual {
1314 let (tx, _rx) = futures::channel::mpsc::channel::<std::result::Result<Duration, ()>>(128);
1315 manual_probe_tx.send((NEIGHBOURS[0], PingQueryReplier::new(tx))).await?;
1316 }
1317
1318 let (routing, data_out) =
1319 tokio::time::timeout(std::time::Duration::from_secs(1), from_probing_to_network_rx.next())
1320 .await?
1321 .ok_or_else(|| anyhow::anyhow!("no probe emitted"))?;
1322
1323 let packet_info = data_out
1324 .packet_info
1325 .ok_or_else(|| anyhow::anyhow!("probe must carry explicit OutgoingPacketInfo"))?;
1326 assert_eq!(
1327 packet_info.max_surbs_in_packet, expected_max_surbs,
1328 "probe must request exactly {expected_max_surbs} SURB(s)"
1329 );
1330
1331 let pseudonym = match routing {
1332 DestinationRouting::Forward { pseudonym, .. } => pseudonym.context("probe pseudonym")?,
1333 _ => anyhow::bail!("probe must have forward routing"),
1334 };
1335 let tag = data_out.data.application_tag;
1336 let nonce = if is_manual {
1337 match Message::try_from(data_out.data.clone())? {
1338 Message::Probe(NeighborProbe::Ping(nonce)) => Some(nonce),
1339 _ => anyhow::bail!("neighbor probe must be a ping"),
1340 }
1341 } else {
1342 None
1343 };
1344 tokio::time::timeout(Duration::from_secs(2), async {
1346 loop {
1347 let registered = if let Some(nonce) = nonce {
1348 classifier
1349 .active_neighbor_probes
1350 .get(&(pseudonym, NeighborProbe::Ping(nonce)))
1351 .await
1352 .is_some()
1353 } else {
1354 classifier.active_path_probes.get(&tag).await.is_some()
1355 };
1356 if registered {
1357 break;
1358 }
1359 tokio::task::yield_now().await;
1360 }
1361 })
1362 .await?;
1363 if reply {
1364 let data = if let Some(nonce) = nonce {
1365 Message::Probe(NeighborProbe::Pong(nonce)).try_into()?
1366 } else {
1367 data_out.data.clone()
1368 };
1369 for _ in 0..2 {
1370 classifier
1371 .classify(
1372 futures::sink::drain(),
1373 pseudonym,
1374 ApplicationDataIn {
1375 data: data.clone(),
1376 packet_info: Default::default(),
1377 },
1378 )
1379 .await;
1380 }
1381 }
1382 tokio::time::sleep(cfg.timeout * 2).await;
1383 classifier.active_neighbor_probes.run_pending_tasks().await;
1384 classifier.active_path_probes.run_pending_tasks().await;
1385 jhs.abort_all();
1386 if is_manual {
1387 let outcomes = store
1388 .on_finished
1389 .read()
1390 .map_err(|_| anyhow::anyhow!("probe graph lock poisoned"))?;
1391 assert_eq!(outcomes.len(), 1, "duplicate replies must not update the graph twice");
1392 assert_eq!(outcomes[0].0, NEIGHBOURS[0]);
1393 assert_eq!(outcomes[0].1.is_ok(), reply);
1394 }
1395 assert_eq!(classifier.active_neighbor_probes.entry_count(), 0);
1396 assert_eq!(classifier.active_path_probes.entry_count(), 0);
1397 Ok(())
1398 }
1399 #[tokio::test]
1400 async fn neighbor_expiry_updates_the_graph_once() -> anyhow::Result<()> {
1402 let cfg = ProbeConfig {
1403 timeout: std::time::Duration::from_millis(10),
1404 interval: std::time::Duration::ZERO,
1405 ..Default::default()
1406 };
1407 let store = PeerStore {
1408 me: *OFFCHAIN_KEYPAIR.public(),
1409 get_peers: Arc::new(RwLock::new(VecDeque::new())),
1410 on_finished: Arc::new(RwLock::new(Vec::new())),
1411 };
1412 test_with_probing(cfg, store.clone(), move |iface: TestInterface| async move {
1413 let mut manual = iface.manual_probe_tx;
1414 let (tx, mut rx) = futures::channel::mpsc::channel(8);
1415 manual.send((NEIGHBOURS[0], PingQueryReplier::new(tx))).await?;
1416 let mut outbound = iface.from_probing_to_network_rx;
1418 tokio::time::timeout(std::time::Duration::from_secs(2), outbound.next()).await?;
1419 tokio::time::sleep(cfg.timeout * 2).await;
1420 iface.probe_classifier.active_neighbor_probes.run_pending_tasks().await;
1421 let result = tokio::time::timeout(std::time::Duration::from_secs(2), rx.next()).await?;
1422 assert!(matches!(result, Some(Err(()))));
1423 Ok(())
1424 })
1425 .await?;
1426 let outcomes = store
1427 .on_finished
1428 .read()
1429 .map_err(|_| anyhow::anyhow!("probe graph lock poisoned"))?;
1430 assert_eq!(outcomes.len(), 1);
1431 assert_eq!(outcomes[0].0, NEIGHBOURS[0]);
1432 assert!(outcomes[0].1.is_err());
1433 Ok(())
1434 }
1435}