Skip to main content

hopr_transport_probe/
probe.rs

1use std::sync::Arc;
2
3use futures::{FutureExt, SinkExt, StreamExt};
4use futures_concurrency::stream::StreamExt as _;
5use hopr_api::{
6    ct::{ProbeRouting, ProbingTrafficGeneration},
7    graph::{EdgeTransportTelemetry, NetworkGraphError, NetworkGraphUpdate, NetworkGraphView},
8    types::{
9        crypto::types::OffchainPublicKey, crypto_random::Randomizable, internal::prelude::*,
10        primitive::traits::AsUnixTimestamp,
11    },
12};
13use hopr_protocol_app::{
14    prelude::{ApplicationDataIn, ApplicationDataOut, OutgoingPacketInfo, ReservedTag},
15    v1::Tag,
16};
17use hopr_transport_tag_allocator::{AllocatedTag, TagAllocator};
18use hopr_utils::{platform::time::native::current_time, runtime::AbortableList};
19
20use crate::{
21    HoprProbeProcess,
22    config::ProbeConfig,
23    content::Message,
24    ping::PingQueryReplier,
25    types::{NeighborProbe, NeighborTelemetry, PathTelemetry},
26};
27
28type CacheNeighborKey = (HoprPseudonym, NeighborProbe);
29type CacheNeighborValue = (Box<NodeId>, std::time::Duration, Option<PingQueryReplier>);
30
31/// Result of classifying one incoming message through the probe layer.
32pub enum ProbeDispatch {
33    /// The message was a probe message and has been consumed internally.
34    Consumed,
35    /// The message was not related to probing; caller should route it further.
36    Passthrough(HoprPseudonym, ApplicationDataIn),
37}
38
39/// Shared state used to classify incoming messages as probe or non-probe.
40///
41/// Obtained from [`Probe::continuously_scan`]. Use [`filter_stream`](ProbeClassifierState::filter_stream)
42/// to wrap an incoming stream so that probe messages are consumed internally and non-probe messages
43/// are yielded to the caller.
44#[derive(Clone)]
45pub struct ProbeClassifierState<G> {
46    active_neighbor_probes: moka::future::Cache<CacheNeighborKey, CacheNeighborValue>,
47    active_path_probes: moka::future::Cache<Tag, (PathTelemetry, Arc<AllocatedTag>)>,
48    network_graph: G,
49}
50
51impl<G> ProbeClassifierState<G>
52where
53    G: NetworkGraphUpdate + Clone + Send + Sync + 'static,
54{
55    /// Classify one incoming `(pseudonym, data)` pair.
56    ///
57    /// The `push_to_network` sink is used to send pong replies when the message is a Ping.
58    /// Returns `Consumed` if the message was a probe, or `Passthrough` for all other messages.
59    pub async fn classify<T>(
60        &self,
61        mut push_to_network: T,
62        pseudonym: HoprPseudonym,
63        in_data: ApplicationDataIn,
64    ) -> ProbeDispatch
65    where
66        T: futures::Sink<(DestinationRouting, ApplicationDataOut)> + Unpin + Send + 'static,
67        T::Error: Send,
68    {
69        let tag: Tag = in_data.data.application_tag;
70
71        if let Some((path_telemetry, _allocated_tag)) = self.active_path_probes.remove(&tag).await {
72            let span = tracing::trace_span!("loopback_probe_reply", %pseudonym, %tag);
73            let _entered = span.enter();
74            tracing::trace!(%tag, "loopback probe successfully received");
75            self.network_graph
76                .record_edge::<NeighborTelemetry, PathTelemetry>(hopr_api::graph::MeasurableEdge::Probe(Ok(
77                    EdgeTransportTelemetry::Loopback(path_telemetry),
78                )));
79        } else if tag == ReservedTag::Ping.into() {
80            let message: anyhow::Result<Message> = in_data
81                .data
82                .try_into()
83                .map_err(|e| anyhow::anyhow!("failed to convert data into message: {e}"));
84
85            match message {
86                Ok(message) => match message {
87                    Message::Telemetry(_) => {
88                        tracing::warn!(%pseudonym, "received telemetry on reserved ping tag, ignoring");
89                    }
90                    Message::Probe(NeighborProbe::Ping(ping)) => {
91                        tracing::trace!(%pseudonym, nonce = const_hex::encode(ping), "received ping");
92                        tracing::trace!(%pseudonym, nonce = const_hex::encode(ping), "wrapping a pong in the found SURB");
93
94                        let message = Message::Probe(NeighborProbe::Pong(ping));
95                        if let Ok(data) = message.try_into() {
96                            let routing = DestinationRouting::Return(pseudonym.into());
97                            let data = ApplicationDataOut::with_no_packet_info(data);
98                            if let Err(_error) = push_to_network.send((routing, data)).await {
99                                tracing::error!(%pseudonym, "failed to send back a pong");
100                            }
101                        } else {
102                            tracing::error!(%pseudonym, "failed to convert pong message into data");
103                        }
104                    }
105                    Message::Probe(NeighborProbe::Pong(pong)) => {
106                        tracing::trace!(%pseudonym, nonce = const_hex::encode(pong), "received pong");
107                        if let Some((peer, start, replier)) = self
108                            .active_neighbor_probes
109                            .remove(&(pseudonym, NeighborProbe::Ping(pong)))
110                            .await
111                        {
112                            let span = tracing::trace_span!("neighbor_probe_reply", %peer, %pseudonym, nonce = const_hex::encode(pong));
113                            let _entered = span.enter();
114                            let latency = current_time().as_unix_timestamp().saturating_sub(start);
115
116                            if let NodeId::Offchain(opk) = peer.as_ref() {
117                                tracing::trace!(%pseudonym, nonce = const_hex::encode(pong), latency_ms = latency.as_millis(), "probe successful");
118                                self.network_graph.record_edge::<NeighborTelemetry, PathTelemetry>(
119                                    hopr_api::graph::MeasurableEdge::Probe(Ok(EdgeTransportTelemetry::Neighbor(
120                                        NeighborTelemetry {
121                                            peer: *opk,
122                                            rtt: latency,
123                                        },
124                                    ))),
125                                )
126                            } else {
127                                tracing::warn!(%pseudonym, nonce = const_hex::encode(pong), latency_ms = latency.as_millis(), "probe successful to non-offchain peer");
128                            }
129
130                            if let Some(replier) = replier {
131                                replier.notify(Ok(latency));
132                            }
133                        } else {
134                            tracing::trace!(%pseudonym, nonce = const_hex::encode(pong), possible_reasons = "[timeout, adversary]", "received pong for unknown probe");
135                        }
136                    }
137                },
138                Err(error) => tracing::error!(%pseudonym, %error, "cannot deserialize message"),
139            }
140        } else {
141            return ProbeDispatch::Passthrough(pseudonym, in_data);
142        }
143
144        ProbeDispatch::Consumed
145    }
146
147    /// Wraps `stream` as an in-place filter: probe messages are handled internally (telemetry,
148    /// pong replies via `push_to_network`), non-probe messages are yielded.
149    pub fn filter_stream<T, S>(
150        self,
151        push_to_network: T,
152        stream: S,
153    ) -> impl futures::Stream<Item = (HoprPseudonym, ApplicationDataIn)>
154    where
155        T: futures::Sink<(DestinationRouting, ApplicationDataOut)> + Clone + Unpin + Send + Sync + 'static,
156        T::Error: Send,
157        S: futures::Stream<Item = (HoprPseudonym, ApplicationDataIn)>,
158    {
159        use futures::StreamExt;
160        stream.filter_map(move |(pseudonym, data)| {
161            let state = self.clone();
162            let push = push_to_network.clone();
163            async move {
164                match state.classify(push, pseudonym, data).await {
165                    ProbeDispatch::Consumed => None,
166                    ProbeDispatch::Passthrough(ps, d) => Some((ps, d)),
167                }
168            }
169        })
170    }
171}
172
173/// Probe functionality builder.
174///
175/// The builder holds information about this node's own addresses and the configuration for the probing process. It is
176/// then used to construct the probing process itself.
177pub struct Probe {
178    /// Probe configuration.
179    cfg: ProbeConfig,
180    /// Tag allocator for probing telemetry tags.
181    tag_allocator: Arc<dyn TagAllocator + Send + Sync>,
182}
183
184impl Probe {
185    pub fn new(cfg: ProbeConfig, tag_allocator: Arc<dyn TagAllocator + Send + Sync>) -> Self {
186        Self { cfg, tag_allocator }
187    }
188
189    /// The main function that assembles and starts the probing process.
190    ///
191    /// Returns the abortable list of background tasks (probe emission) and a
192    /// [`ProbeClassifierState`] for inline classification of incoming messages.
193    /// Use [`ProbeClassifierState::filter_stream`] to wrap the incoming stream.
194    pub async fn continuously_scan<T, V, Tr, G>(
195        self,
196        api_out: T,       // lower tx channel for sending outgoing probes and pong replies
197        manual_events: V, // explicit requests from the API
198        probing_traffic_generator: Tr,
199        network_graph: G,
200    ) -> (AbortableList<HoprProbeProcess>, ProbeClassifierState<G>)
201    where
202        T: futures::Sink<(DestinationRouting, ApplicationDataOut)> + Clone + Send + Sync + Unpin + 'static,
203        T::Error: Send,
204        V: futures::Stream<Item = (OffchainPublicKey, PingQueryReplier)> + Send + 'static,
205        Tr: ProbingTrafficGeneration + Send + Sync + 'static,
206        G: NetworkGraphView + NetworkGraphUpdate + Clone + Send + Sync + 'static,
207    {
208        let max_parallel_probes = self.cfg.max_parallel_probes;
209
210        let probing_routes = probing_traffic_generator.build();
211
212        // Currently active probes
213        let network_graph_internal_neighbor = network_graph.clone();
214        let network_graph_internal_path = network_graph.clone();
215        let timeout = self.cfg.timeout;
216        let active_neighbor_probes: moka::future::Cache<CacheNeighborKey, CacheNeighborValue> =
217            moka::future::Cache::builder()
218                .time_to_live(timeout)
219                .max_capacity(100_000)
220                .async_eviction_listener(
221                    move |k: Arc<CacheNeighborKey>,
222                          v: CacheNeighborValue,
223                          cause|
224                          -> moka::notification::ListenerFuture {
225                        if matches!(cause, moka::notification::RemovalCause::Expired) {
226                            // If the eviction cause is expiration => record as a failed probe
227                            let store = network_graph_internal_neighbor.clone();
228                            let (peer, _start, notifier) = v;
229                            let span = tracing::trace_span!("neighbor_probe_expiry", %peer, pseudonym = %k.0);
230                            let _entered = span.enter();
231
232                            tracing::trace!(%peer, pseudonym = %k.0, probe = %k.1, reason = "timeout", "neighbor probe failed");
233                            if let Some(replier) = notifier {
234                                if matches!(peer.as_ref(), NodeId::Offchain(_)) {
235                                    replier.notify(Err(()));
236                                } else {
237                                    tracing::warn!(
238                                        reason = "non-offchain peer",
239                                        "cannot notify timeout for non-offchain peer"
240                                    );
241                                }
242                            };
243
244                            if let NodeId::Offchain(opk) = peer.as_ref() {
245                                let opk: OffchainPublicKey = *opk;
246                                store
247                                    .record_edge::<NeighborTelemetry, PathTelemetry>(
248                                        hopr_api::graph::MeasurableEdge::Probe(Err(
249                                            NetworkGraphError::ProbeNeighborTimeout(Box::new(opk)),
250                                        )),
251                                    );
252                                futures::FutureExt::boxed(futures::future::ready(()))
253
254                            } else {
255                                futures::FutureExt::boxed(futures::future::ready(()))
256                            }
257                        } else {
258                            // If the eviction cause is not expiration, nothing needs to be done
259                            futures::FutureExt::boxed(futures::future::ready(()))
260                        }
261                    },
262                )
263                .build();
264
265        let active_path_probes: moka::future::Cache<Tag, (PathTelemetry, Arc<AllocatedTag>)> =
266            moka::future::Cache::builder()
267                .time_to_live(timeout)
268                .max_capacity(100_000)
269                .async_eviction_listener(
270                    move |tag: Arc<Tag>,
271                          (path, _allocated_tag): (PathTelemetry, Arc<AllocatedTag>),
272                          cause|
273                          -> moka::notification::ListenerFuture {
274                        if matches!(cause, moka::notification::RemovalCause::Expired) {
275                            // If the eviction cause is expiration => record as a failed probe
276                            let span = tracing::trace_span!("loopback_probe_expiry", %tag);
277                            let _entered = span.enter();
278                            let store = network_graph_internal_path.clone();
279
280                            tracing::trace!(%tag, reason = "timeout", "loopback probe failed");
281
282                            store.record_edge::<NeighborTelemetry, PathTelemetry>(
283                                hopr_api::graph::MeasurableEdge::Probe(Err(NetworkGraphError::ProbeLoopbackTimeout(
284                                    path,
285                                ))),
286                            );
287                            futures::FutureExt::boxed(futures::future::ready(()))
288                        } else {
289                            // If the eviction cause is not expiration, nothing needs to be done
290                            futures::FutureExt::boxed(futures::future::ready(()))
291                        }
292                    },
293                )
294                .build();
295
296        let push_to_network = api_out.clone();
297
298        let mut processes = AbortableList::default();
299
300        // -- Emit probes --
301        let direct_neighbors =
302            probing_routes
303                .map(|peer| (peer, None))
304                .merge(manual_events.filter_map(|(peer, notifier)| async move {
305                    let routing = DestinationRouting::Forward {
306                        destination: Box::new(peer.into()),
307                        pseudonym: Some(HoprPseudonym::random()),
308                        forward_options: RoutingOptions::Hops(0.try_into().expect("0 is a valid u8")),
309                        return_options: Some(RoutingOptions::Hops(0.try_into().expect("0 is a valid u8"))),
310                    };
311                    Some((ProbeRouting::Neighbor(routing), Some(notifier)))
312                }));
313
314        let tag_allocator = self.tag_allocator.clone();
315        let classifier_neighbor_probes = active_neighbor_probes.clone();
316        let classifier_path_probes = active_path_probes.clone();
317        let emit_diag = hopr_utils::runtime::diagnostics::ConcurrentDiagnostics::new(
318            "probe_emit_for_each_concurrent",
319            module_path!(),
320            file!(),
321            line!(),
322        );
323        processes.insert(
324            HoprProbeProcess::Emit,
325            hopr_utils::spawn_as_abortable_named!("probe_emit", async move {
326                direct_neighbors
327                    .for_each_concurrent(max_parallel_probes, move |(peer, notifier)| {
328                        let active_neighbor_probes = active_neighbor_probes.clone();
329                        let active_path_probes = active_path_probes.clone();
330                        let push_to_network = push_to_network.clone();
331                        let tag_allocator = tag_allocator.clone();
332                        let emit_diag = emit_diag.clone();
333
334                        emit_diag.wrap(|| async move {
335                            match peer {
336                                ProbeRouting::Neighbor(DestinationRouting::Forward {
337                                    destination,
338                                    pseudonym,
339                                    forward_options,
340                                    return_options,
341                                }) => {
342                                    let nonce = NeighborProbe::random_nonce();
343
344                                    let message = Message::Probe(nonce);
345
346                                    if let Ok(data) = message.try_into() {
347                                        let routing = DestinationRouting::Forward {
348                                            destination: destination.clone(),
349                                            pseudonym,
350                                            forward_options,
351                                            return_options,
352                                        };
353                                        // Neighbor probes are sent to a direct neighbor and returned via a zero-hop
354                                        // return path: only 1 SURB is ever consumed. See hoprnet/hoprnet#7972.
355                                        let data = ApplicationDataOut {
356                                            data,
357                                            packet_info: Some(OutgoingPacketInfo {
358                                                max_surbs_in_packet: 1,
359                                                ..Default::default()
360                                            }),
361                                            surb_refund: Default::default(),
362                                        };
363                                        let mut push_to_network = push_to_network.clone();
364
365                                        if let Err(_error) = push_to_network.send((routing, data)).await {
366                                            tracing::error!("failed to send out a ping");
367                                        } else {
368                                            active_neighbor_probes
369                                                .insert(
370                                                    (
371                                                        pseudonym
372                                                            .expect("the pseudonym must be present in Forward routing"),
373                                                        nonce,
374                                                    ),
375                                                    (destination, current_time().as_unix_timestamp(), notifier),
376                                                )
377                                                .await;
378                                        }
379                                    } else {
380                                        tracing::error!("failed to convert ping message into data");
381                                    }
382                                }
383                                ProbeRouting::Neighbor(DestinationRouting::Return(_surb_matcher)) => tracing::error!(
384                                    error = "logical error",
385                                    "resolved transport routing is not forward"
386                                ),
387                                ProbeRouting::Looping((routing, path_id)) => {
388                                    let message = Message::Telemetry(PathTelemetry {
389                                        id: hopr_api::types::crypto_random::random_bytes(),
390                                        path: std::array::from_fn(|i| path_id[i / 8].to_le_bytes()[i % 8]),
391                                        timestamp: std::time::SystemTime::now()
392                                            .duration_since(std::time::UNIX_EPOCH)
393                                            .unwrap_or_default()
394                                            .as_millis(),
395                                    });
396
397                                    if let Some(allocated_tag) = tag_allocator.allocate() {
398                                        let tag_value = allocated_tag.value();
399
400                                        if let Ok(packet) = hopr_protocol_app::prelude::ApplicationData::new(
401                                            tag_value,
402                                            message.to_bytes().as_ref(),
403                                        ) {
404                                            let mut push_to_network = push_to_network.clone();
405
406                                            // Loopback telemetry probes are self-routed and never replied to via
407                                            // SURB, so no SURBs should be bundled. See hoprnet/hoprnet#7972.
408                                            if let Err(_error) = push_to_network
409                                                .send((
410                                                    routing,
411                                                    ApplicationDataOut {
412                                                        data: packet,
413                                                        packet_info: Some(OutgoingPacketInfo {
414                                                            max_surbs_in_packet: 0,
415                                                            ..Default::default()
416                                                        }),
417                                                        surb_refund: Default::default(),
418                                                    },
419                                                ))
420                                                .await
421                                            {
422                                                tracing::error!("failed to send out a ping");
423                                            } else {
424                                                // the object is constructed above, so will always match
425                                                if let Message::Telemetry(telemetry) = message {
426                                                    active_path_probes
427                                                        .insert(tag_value.into(), (telemetry, Arc::new(allocated_tag)))
428                                                        .await;
429                                                }
430                                            }
431                                        } else {
432                                            tracing::error!("failed to construct data for path telemetry")
433                                        }
434                                    } else {
435                                        tracing::warn!("probing telemetry tag pool exhausted, skipping loopback probe");
436                                    }
437                                }
438                            }
439                        })
440                    })
441                    .inspect(|_| {
442                        tracing::warn!(
443                            task = "transport (probe - generate outgoing)",
444                            "long-running background task finished"
445                        )
446                    })
447                    .await;
448            }),
449        );
450
451        let classifier = ProbeClassifierState {
452            active_neighbor_probes: classifier_neighbor_probes,
453            active_path_probes: classifier_path_probes,
454            network_graph,
455        };
456
457        (processes, classifier)
458    }
459}
460
461#[cfg(test)]
462mod tests {
463    use std::{collections::VecDeque, sync::RwLock, time::Duration};
464
465    use async_trait::async_trait;
466    use futures::future::BoxFuture;
467    use hopr_api::{
468        graph::{
469            EdgeLinkObservable, MeasurableEdge, NetworkGraphError,
470            traits::{EdgeNetworkObservableRead, EdgeObservableRead, EdgeObservableWrite, EdgeProtocolObservable},
471        },
472        types::crypto::keypairs::{ChainKeypair, Keypair, OffchainKeypair},
473    };
474    use hopr_protocol_app::prelude::{ApplicationData, ReservedTag, Tag};
475
476    use super::*;
477    use crate::errors::ProbeError;
478
479    lazy_static::lazy_static!(
480        static ref OFFCHAIN_KEYPAIR: OffchainKeypair = OffchainKeypair::random();
481        static ref ONCHAIN_KEYPAIR: ChainKeypair = ChainKeypair::random();
482        static ref NEIGHBOURS: Vec<OffchainPublicKey> = vec![
483            *OffchainKeypair::random().public(),
484            *OffchainKeypair::random().public(),
485            *OffchainKeypair::random().public(),
486            *OffchainKeypair::random().public(),
487        ];
488    );
489
490    /// Test stub implementation of Observable.
491    #[derive(Debug, Clone, Copy, Default)]
492    pub struct TestEdgeTransportObservations;
493
494    impl EdgeLinkObservable for TestEdgeTransportObservations {
495        fn record(&mut self, _latency: std::result::Result<Duration, ()>) {}
496
497        fn average_latency(&self) -> Option<Duration> {
498            None
499        }
500
501        // A measured, perfect link — so the probe tests see the same edge they did before the
502        // trait gained the presence distinction. `has_observations` derives from this.
503        fn average_probe_rate(&self) -> Option<f64> {
504            Some(1.0)
505        }
506
507        fn score(&self) -> Option<f64> {
508            Some(1.0)
509        }
510    }
511
512    impl EdgeNetworkObservableRead for TestEdgeTransportObservations {
513        fn is_connected(&self) -> Option<bool> {
514            Some(true)
515        }
516    }
517
518    impl EdgeProtocolObservable for TestEdgeTransportObservations {
519        fn balance(&self) -> Option<hopr_api::graph::traits::Balance> {
520            None
521        }
522    }
523
524    impl hopr_api::graph::EdgeImmediateProtocolObservable for TestEdgeTransportObservations {
525        fn ack_rate(&self) -> Option<f64> {
526            None
527        }
528    }
529
530    #[derive(Debug, Clone, Copy, Default)]
531    pub struct TestEdgeObservations;
532
533    impl EdgeObservableWrite for TestEdgeObservations {
534        fn record(&mut self, _measurement: hopr_api::graph::traits::EdgeWeightType) {}
535    }
536
537    impl EdgeObservableRead for TestEdgeObservations {
538        type ImmediateMeasurement = TestEdgeTransportObservations;
539        type IntermediateMeasurement = TestEdgeTransportObservations;
540
541        fn last_update(&self) -> std::time::Duration {
542            std::time::SystemTime::now()
543                .duration_since(std::time::UNIX_EPOCH)
544                .unwrap_or_default()
545        }
546
547        fn immediate_qos(&self) -> Option<&Self::ImmediateMeasurement> {
548            None
549        }
550
551        fn intermediate_qos(&self) -> Option<&Self::IntermediateMeasurement> {
552            None
553        }
554
555        fn score(&self) -> Option<f64> {
556            Some(1.0)
557        }
558    }
559
560    #[derive(Debug, Clone)]
561    pub struct PeerStore {
562        me: OffchainPublicKey,
563        get_peers: Arc<RwLock<VecDeque<Vec<OffchainPublicKey>>>>,
564        #[allow(clippy::type_complexity)]
565        on_finished: Arc<RwLock<Vec<(OffchainPublicKey, crate::errors::Result<Duration>)>>>,
566    }
567
568    impl NetworkGraphUpdate for PeerStore {
569        /// Store neighbor successes and timeouts in the test harness, ignoring loopback measurements.
570        fn record_edge<N, P>(&self, telemetry: MeasurableEdge<N, P>)
571        where
572            N: hopr_api::graph::MeasurablePeer + Send + Clone,
573            P: hopr_api::graph::MeasurablePath + Send + Clone,
574        {
575            let mut on_finished = self.on_finished.write().unwrap();
576
577            match telemetry {
578                hopr_api::graph::MeasurableEdge::Probe(Ok(EdgeTransportTelemetry::Neighbor(neighbor_telemetry))) => {
579                    let peer: OffchainPublicKey = *neighbor_telemetry.peer();
580                    let duration = neighbor_telemetry.rtt();
581                    on_finished.push((peer, Ok(duration)));
582                }
583                hopr_api::graph::MeasurableEdge::Probe(Err(NetworkGraphError::ProbeNeighborTimeout(peer))) => {
584                    on_finished.push((
585                        *peer.as_ref(),
586                        Err(ProbeError::TrafficError(NetworkGraphError::ProbeNeighborTimeout(peer))),
587                    ));
588                }
589                MeasurableEdge::Probe(Ok(EdgeTransportTelemetry::Loopback(_)))
590                | MeasurableEdge::Probe(Err(NetworkGraphError::ProbeLoopbackTimeout(_))) => {}
591                _ => panic!("unexpected telemetry type, unimplemented"),
592            }
593        }
594
595        fn record_node<N>(&self, _node: N)
596        where
597            N: hopr_api::graph::MeasurableNode + Send + Clone,
598        {
599            unimplemented!()
600        }
601
602        fn set_ticket_face_value(&self, _ticket_face_value: hopr_api::graph::traits::Balance) {}
603    }
604
605    #[async_trait]
606    impl NetworkGraphView for PeerStore {
607        type NodeId = OffchainPublicKey;
608        type Observed = TestEdgeObservations;
609
610        fn identity(&self) -> &OffchainPublicKey {
611            &self.me
612        }
613
614        fn ticket_face_value(&self) -> Option<hopr_api::graph::traits::Balance> {
615            None
616        }
617
618        fn node_count(&self) -> usize {
619            self.get_peers.read().unwrap().front().map_or(0, |v| v.len())
620        }
621
622        fn contains_node(&self, _key: &OffchainPublicKey) -> bool {
623            false
624        }
625
626        /// Returns a stream of all known nodes in the network graph.
627        fn nodes(&self) -> futures::stream::BoxStream<'static, OffchainPublicKey> {
628            let mut get_peers = self.get_peers.write().unwrap();
629            Box::pin(futures::stream::iter(get_peers.pop_front().unwrap_or_default()))
630        }
631
632        fn edge(&self, _src: &OffchainPublicKey, _dest: &OffchainPublicKey) -> Option<TestEdgeObservations> {
633            Some(TestEdgeObservations)
634        }
635
636        /// Probing never identifies a path by slot, so nothing here depends on this mapping.
637        fn path_slot(&self, _key: &OffchainPublicKey) -> Option<u64> {
638            None
639        }
640    }
641
642    type TestClassifier = ProbeClassifierState<PeerStore>;
643
644    /// Capture producer spans and the span depth at which each log message is emitted.
645    #[derive(Clone, Default)]
646    struct ProbeLogs {
647        spans: Arc<parking_lot::Mutex<Vec<&'static str>>>,
648        messages: Arc<parking_lot::Mutex<Vec<(String, usize)>>>,
649        depth: Arc<std::sync::atomic::AtomicUsize>,
650    }
651
652    struct MessageVisitor(String);
653
654    impl tracing::field::Visit for MessageVisitor {
655        /// Extract the message field without coupling the assertion to logger formatting.
656        fn record_debug(&mut self, field: &tracing::field::Field, value: &dyn std::fmt::Debug) {
657            if field.name() == "message" {
658                self.0 = format!("{value:?}");
659            }
660        }
661    }
662
663    impl tracing::Subscriber for ProbeLogs {
664        fn enabled(&self, metadata: &tracing::Metadata<'_>) -> bool {
665            // Tokio's runtime instrumentation is enabled by workspace features; only probe
666            // spans and events belong to the producer-context assertions below.
667            metadata.target().starts_with("hopr_transport_probe")
668        }
669
670        fn new_span(&self, attrs: &tracing::span::Attributes<'_>) -> tracing::span::Id {
671            let mut spans = self.spans.lock();
672            spans.push(attrs.metadata().name());
673            tracing::span::Id::from_u64(spans.len() as u64)
674        }
675
676        fn record(&self, _: &tracing::span::Id, _: &tracing::span::Record<'_>) {}
677
678        fn record_follows_from(&self, _: &tracing::span::Id, _: &tracing::span::Id) {}
679
680        fn event(&self, event: &tracing::Event<'_>) {
681            let mut message = MessageVisitor(String::new());
682            event.record(&mut message);
683            self.messages
684                .lock()
685                .push((message.0, self.depth.load(std::sync::atomic::Ordering::Relaxed)));
686        }
687
688        fn enter(&self, _: &tracing::span::Id) {
689            self.depth.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
690        }
691
692        fn exit(&self, _: &tracing::span::Id) {
693            self.depth.fetch_sub(1, std::sync::atomic::Ordering::Relaxed);
694        }
695    }
696
697    #[test]
698    /// Ordinary packets create no probe span; matched replies log inside their producer span.
699    fn probe_reply_logs_stay_in_the_producer_span() -> anyhow::Result<()> {
700        let logs = ProbeLogs::default();
701        tracing::subscriber::with_default(logs.clone(), || {
702            tokio::runtime::Builder::new_current_thread()
703                .enable_all()
704                .build()?
705                .block_on(async {
706                    let store = PeerStore {
707                        me: *OFFCHAIN_KEYPAIR.public(),
708                        get_peers: Arc::new(RwLock::new(VecDeque::new())),
709                        on_finished: Arc::new(RwLock::new(Vec::new())),
710                    };
711                    let classifier = ProbeClassifierState {
712                        active_neighbor_probes: moka::future::Cache::new(1),
713                        active_path_probes: moka::future::Cache::new(1),
714                        network_graph: store.clone(),
715                    };
716                    let pseudonym = HoprPseudonym::random();
717                    let ordinary = ApplicationDataIn {
718                        data: ApplicationData::new(Tag::MAX, b"ordinary")?,
719                        packet_info: Default::default(),
720                    };
721                    assert!(matches!(
722                        classifier.classify(futures::sink::drain(), pseudonym, ordinary).await,
723                        ProbeDispatch::Passthrough(..)
724                    ));
725                    assert!(
726                        logs.spans.lock().is_empty(),
727                        "ordinary packets must not create probe spans: {:?}",
728                        *logs.spans.lock()
729                    );
730
731                    let nonce = [7; NeighborProbe::NONCE_SIZE];
732                    classifier
733                        .active_neighbor_probes
734                        .insert(
735                            (pseudonym, NeighborProbe::Ping(nonce)),
736                            (
737                                Box::new(NodeId::Offchain(NEIGHBOURS[0])),
738                                current_time().as_unix_timestamp(),
739                                None,
740                            ),
741                        )
742                        .await;
743                    classifier
744                        .classify(
745                            futures::sink::drain(),
746                            pseudonym,
747                            ApplicationDataIn {
748                                data: Message::Probe(NeighborProbe::Pong(nonce)).try_into()?,
749                                packet_info: Default::default(),
750                            },
751                        )
752                        .await;
753                    assert_eq!(&*logs.spans.lock(), &["neighbor_probe_reply"]);
754                    assert!(
755                        logs.messages
756                            .lock()
757                            .iter()
758                            .any(|(message, depth)| { message == "probe successful" && *depth == 1 })
759                    );
760                    assert_eq!(
761                        store
762                            .on_finished
763                            .read()
764                            .map_err(|_| anyhow::anyhow!("probe graph lock poisoned"))?
765                            .len(),
766                        1
767                    );
768                    anyhow::Ok(())
769                })
770        })
771    }
772
773    struct TestInterface {
774        probe_classifier: TestClassifier,
775        from_probing_to_network_rx: futures::channel::mpsc::Receiver<(DestinationRouting, ApplicationDataOut)>,
776        from_probing_to_network_tx: futures::channel::mpsc::Sender<(DestinationRouting, ApplicationDataOut)>,
777        manual_probe_tx: futures::channel::mpsc::Sender<(OffchainPublicKey, PingQueryReplier)>,
778    }
779
780    /// Run probe producers and classifiers with bounded test channels.
781    async fn test_with_probing<F, Fut>(cfg: ProbeConfig, store: PeerStore, test: F) -> anyhow::Result<()>
782    where
783        Fut: std::future::Future<Output = anyhow::Result<()>>,
784        F: Fn(TestInterface) -> Fut + Send + Sync + 'static,
785    {
786        let tag_allocators = hopr_transport_tag_allocator::create_allocators(
787            ReservedTag::range().end..u16::MAX as u64 + 1,
788            [
789                (hopr_transport_tag_allocator::Usage::Session, 2048),
790                (hopr_transport_tag_allocator::Usage::SessionTerminalTelemetry, 4000),
791                (hopr_transport_tag_allocator::Usage::ProvingTelemetry, 10000),
792            ],
793        )
794        .expect("tag allocators should be created");
795        let probing_allocator = tag_allocators
796            .into_iter()
797            .find_map(|(u, alloc)| matches!(u, hopr_transport_tag_allocator::Usage::ProvingTelemetry).then_some(alloc))
798            .expect("probing allocator should exist");
799
800        let probe = Probe::new(cfg, probing_allocator);
801
802        let (from_probing_to_network_tx, from_probing_to_network_rx) =
803            futures::channel::mpsc::channel::<(DestinationRouting, ApplicationDataOut)>(100);
804
805        let (manual_probe_tx, manual_probe_rx) =
806            futures::channel::mpsc::channel::<(OffchainPublicKey, PingQueryReplier)>(100);
807
808        let (jhs, probe_classifier) = probe
809            .continuously_scan(
810                from_probing_to_network_tx.clone(),
811                manual_probe_rx,
812                TestProbeStrategy::ImmediateNeighbor { store: store.clone() },
813                store,
814            )
815            .await;
816
817        let interface = TestInterface {
818            probe_classifier,
819            from_probing_to_network_rx,
820            from_probing_to_network_tx,
821            manual_probe_tx,
822        };
823
824        let result = test(interface).await;
825
826        jhs.abort_all();
827
828        result
829    }
830
831    const NO_PROBE_PASSES: f64 = 0.0;
832    const ALL_PROBES_PASS: f64 = 1.0;
833
834    /// Simulates the network: receives outgoing probe packets and feeds pong responses back
835    /// through the classifier (mirroring what the remote peer + packet pipeline would do).
836    fn concurrent_classify(
837        delay: Option<std::time::Duration>,
838        pass_rate: f64,
839        classifier: TestClassifier,
840        push_to_network: futures::channel::mpsc::Sender<(DestinationRouting, ApplicationDataOut)>,
841    ) -> impl Fn((DestinationRouting, ApplicationDataOut)) -> BoxFuture<'static, ()> {
842        debug_assert!(
843            (NO_PROBE_PASSES..=ALL_PROBES_PASS).contains(&pass_rate),
844            "Pass rate must be between {NO_PROBE_PASSES} and {ALL_PROBES_PASS}"
845        );
846
847        move |(path, data_out): (DestinationRouting, ApplicationDataOut)| -> BoxFuture<'static, ()> {
848            let classifier = classifier.clone();
849            let push_to_network = push_to_network.clone();
850
851            Box::pin(async move {
852                if let DestinationRouting::Forward { pseudonym, .. } = path {
853                    let message: Message = data_out.data.try_into().expect("failed to convert data into message");
854                    if let Message::Probe(NeighborProbe::Ping(ping)) = message {
855                        let pong_message = Message::Probe(NeighborProbe::Pong(ping));
856
857                        if let Some(delay) = delay {
858                            tokio::time::sleep(delay).await;
859                        }
860
861                        if rand::random_range(NO_PROBE_PASSES..=ALL_PROBES_PASS) < pass_rate {
862                            let pseudonym = pseudonym.expect("the pseudonym is always known from cache");
863                            classifier
864                                .classify(
865                                    push_to_network,
866                                    pseudonym,
867                                    ApplicationDataIn {
868                                        data: pong_message
869                                            .try_into()
870                                            .expect("failed to convert pong message into data"),
871                                        packet_info: Default::default(),
872                                    },
873                                )
874                                .await;
875                        }
876                    }
877                };
878            })
879        }
880    }
881
882    #[tokio::test]
883    // #[tracing_test::traced_test]
884    async fn probe_should_record_value_for_manual_neighbor_probe() -> anyhow::Result<()> {
885        let cfg = ProbeConfig {
886            timeout: std::time::Duration::from_millis(5),
887            interval: std::time::Duration::from_secs(0),
888            ..Default::default()
889        };
890
891        let store = PeerStore {
892            me: *OFFCHAIN_KEYPAIR.public(),
893            get_peers: Arc::new(RwLock::new(VecDeque::new())),
894            on_finished: Arc::new(RwLock::new(Vec::new())),
895        };
896
897        test_with_probing(cfg, store, move |iface: TestInterface| async move {
898            let mut manual_probe_tx = iface.manual_probe_tx;
899            let from_probing_to_network_rx = iface.from_probing_to_network_rx;
900            let from_probing_to_network_tx = iface.from_probing_to_network_tx;
901            let probe_classifier = iface.probe_classifier;
902
903            let (tx, mut rx) = futures::channel::mpsc::channel::<std::result::Result<Duration, ()>>(128);
904            manual_probe_tx.send((NEIGHBOURS[0], PingQueryReplier::new(tx))).await?;
905
906            let _jh: hopr_utils::runtime::prelude::JoinHandle<()> = tokio::spawn(async move {
907                from_probing_to_network_rx
908                    .for_each_concurrent(
909                        cfg.max_parallel_probes + 1,
910                        concurrent_classify(None, ALL_PROBES_PASS, probe_classifier, from_probing_to_network_tx),
911                    )
912                    .await;
913            });
914
915            let _duration = tokio::time::timeout(std::time::Duration::from_secs(1), rx.next())
916                .await?
917                .ok_or_else(|| anyhow::anyhow!("Probe did not return a result in time"))?
918                .map_err(|_| anyhow::anyhow!("Probe failed"))?;
919
920            Ok(())
921        })
922        .await
923    }
924
925    #[tokio::test]
926    // #[tracing_test::traced_test]
927    async fn probe_should_record_failure_on_manual_fail() -> anyhow::Result<()> {
928        let cfg = ProbeConfig {
929            timeout: std::time::Duration::from_millis(5),
930            interval: std::time::Duration::from_secs(0),
931            ..Default::default()
932        };
933
934        let store = PeerStore {
935            me: *OFFCHAIN_KEYPAIR.public(),
936            get_peers: Arc::new(RwLock::new(VecDeque::new())),
937            on_finished: Arc::new(RwLock::new(Vec::new())),
938        };
939
940        test_with_probing(cfg, store, move |iface: TestInterface| async move {
941            let mut manual_probe_tx = iface.manual_probe_tx;
942            let from_probing_to_network_rx = iface.from_probing_to_network_rx;
943            let from_probing_to_network_tx = iface.from_probing_to_network_tx;
944            let probe_classifier = iface.probe_classifier;
945
946            let (tx, mut rx) = futures::channel::mpsc::channel::<std::result::Result<Duration, ()>>(128);
947            manual_probe_tx.send((NEIGHBOURS[0], PingQueryReplier::new(tx))).await?;
948
949            let _jh: hopr_utils::runtime::prelude::JoinHandle<()> = tokio::spawn(async move {
950                from_probing_to_network_rx
951                    .for_each_concurrent(
952                        cfg.max_parallel_probes + 1,
953                        concurrent_classify(None, NO_PROBE_PASSES, probe_classifier, from_probing_to_network_tx),
954                    )
955                    .await;
956            });
957
958            assert!(tokio::time::timeout(cfg.timeout * 2, rx.next()).await.is_err());
959
960            Ok(())
961        })
962        .await
963    }
964
965    #[tokio::test]
966    // #[tracing_test::traced_test]
967    async fn probe_should_record_results_of_successful_automatically_generated_probes() -> anyhow::Result<()> {
968        let cfg = ProbeConfig {
969            timeout: std::time::Duration::from_millis(20),
970            max_parallel_probes: NEIGHBOURS.len(),
971            interval: std::time::Duration::from_secs(0),
972            ..Default::default()
973        };
974
975        let store = PeerStore {
976            me: *OFFCHAIN_KEYPAIR.public(),
977            get_peers: Arc::new(RwLock::new({
978                let mut neighbors = VecDeque::new();
979                neighbors.push_back(NEIGHBOURS.clone());
980                neighbors
981            })),
982            on_finished: Arc::new(RwLock::new(Vec::new())),
983        };
984
985        test_with_probing(cfg, store.clone(), move |iface: TestInterface| async move {
986            let from_probing_to_network_rx = iface.from_probing_to_network_rx;
987            let from_probing_to_network_tx = iface.from_probing_to_network_tx;
988            let probe_classifier = iface.probe_classifier;
989
990            let _jh: hopr_utils::runtime::prelude::JoinHandle<()> = tokio::spawn(async move {
991                from_probing_to_network_rx
992                    .for_each_concurrent(
993                        cfg.max_parallel_probes + 1,
994                        concurrent_classify(None, ALL_PROBES_PASS, probe_classifier, from_probing_to_network_tx),
995                    )
996                    .await;
997            });
998
999            // Wait for the full probe lifecycle: emit → network round trip → process.
1000            // Must exceed cfg.timeout (the cache TTL) to avoid probes being evicted
1001            // as timeouts before the pong response is processed.
1002            tokio::time::sleep(cfg.timeout * 3).await;
1003
1004            Ok(())
1005        })
1006        .await?;
1007
1008        assert_eq!(
1009            store
1010                .on_finished
1011                .read()
1012                .expect("should be lockable")
1013                .iter()
1014                .filter(|(_peer, result)| result.is_ok())
1015                .count(),
1016            NEIGHBOURS.len()
1017        );
1018
1019        Ok(())
1020    }
1021
1022    #[tokio::test]
1023    // #[tracing_test::traced_test]
1024    async fn probe_should_record_results_of_timed_out_automatically_generated_probes() -> anyhow::Result<()> {
1025        let cfg = ProbeConfig {
1026            timeout: std::time::Duration::from_millis(10),
1027            max_parallel_probes: NEIGHBOURS.len(),
1028            interval: std::time::Duration::from_secs(0),
1029            ..Default::default()
1030        };
1031
1032        let store = PeerStore {
1033            me: *OFFCHAIN_KEYPAIR.public(),
1034            get_peers: Arc::new(RwLock::new({
1035                let mut neighbors = VecDeque::new();
1036                neighbors.push_back(NEIGHBOURS.clone());
1037                neighbors
1038            })),
1039            on_finished: Arc::new(RwLock::new(Vec::new())),
1040        };
1041
1042        let timeout = cfg.timeout * 2;
1043
1044        test_with_probing(cfg, store.clone(), move |iface: TestInterface| async move {
1045            let from_probing_to_network_rx = iface.from_probing_to_network_rx;
1046            let from_probing_to_network_tx = iface.from_probing_to_network_tx;
1047            let probe_classifier = iface.probe_classifier;
1048
1049            let _jh: hopr_utils::runtime::prelude::JoinHandle<()> = tokio::spawn(async move {
1050                from_probing_to_network_rx
1051                    .for_each_concurrent(
1052                        cfg.max_parallel_probes + 1,
1053                        concurrent_classify(
1054                            Some(timeout),
1055                            ALL_PROBES_PASS,
1056                            probe_classifier,
1057                            from_probing_to_network_tx,
1058                        ),
1059                    )
1060                    .await;
1061            });
1062
1063            // wait for the probes to start and finish
1064            tokio::time::sleep(timeout * 2).await;
1065
1066            Ok(())
1067        })
1068        .await?;
1069
1070        assert_eq!(
1071            store
1072                .on_finished
1073                .read()
1074                .expect("should be lockable")
1075                .iter()
1076                .filter(|(_peer, result)| result.is_err())
1077                .count(),
1078            NEIGHBOURS.len()
1079        );
1080
1081        Ok(())
1082    }
1083
1084    #[tokio::test]
1085    async fn probe_should_reply_with_pong_when_receiving_ping() -> anyhow::Result<()> {
1086        use anyhow::Context;
1087
1088        let cfg = ProbeConfig {
1089            timeout: std::time::Duration::from_millis(100),
1090            interval: std::time::Duration::from_secs(10),
1091            ..Default::default()
1092        };
1093
1094        let store = PeerStore {
1095            me: *OFFCHAIN_KEYPAIR.public(),
1096            get_peers: Arc::new(RwLock::new(VecDeque::new())),
1097            on_finished: Arc::new(RwLock::new(Vec::new())),
1098        };
1099
1100        test_with_probing(cfg, store, move |iface: TestInterface| async move {
1101            let probe_classifier = iface.probe_classifier;
1102            let from_probing_to_network_tx = iface.from_probing_to_network_tx;
1103            let mut from_probing_to_network_rx = iface.from_probing_to_network_rx;
1104
1105            // Build a Ping message with the reserved Ping tag
1106            let ping = NeighborProbe::random_nonce();
1107            let ping_nonce = match ping {
1108                NeighborProbe::Ping(n) => n,
1109                _ => unreachable!(),
1110            };
1111            let ping_msg = Message::Probe(ping);
1112            let app_data: ApplicationData = ping_msg.try_into().context("converting ping to ApplicationData")?;
1113
1114            // Classify the ping directly — the classifier sends the pong reply to push_to_network
1115            let result = probe_classifier
1116                .classify(
1117                    from_probing_to_network_tx,
1118                    HoprPseudonym::random(),
1119                    ApplicationDataIn {
1120                        data: app_data,
1121                        packet_info: Default::default(),
1122                    },
1123                )
1124                .await;
1125            anyhow::ensure!(matches!(result, ProbeDispatch::Consumed), "ping should be consumed");
1126
1127            // The probe should reply with a Pong on the network channel
1128            let (routing, data_out) = tokio::time::timeout(Duration::from_secs(2), from_probing_to_network_rx.next())
1129                .await
1130                .context("timeout waiting for pong")?
1131                .context("probe should send pong reply")?;
1132
1133            // Verify it's a Return routing (SURB-based reply)
1134            anyhow::ensure!(
1135                matches!(routing, DestinationRouting::Return(_)),
1136                "pong should use Return routing, got: {routing:?}"
1137            );
1138
1139            // Verify the payload is a Pong with the same nonce
1140            let response_msg: Message = data_out.data.try_into().context("converting response to Message")?;
1141            anyhow::ensure!(
1142                matches!(response_msg, Message::Probe(NeighborProbe::Pong(n)) if n == ping_nonce),
1143                "response should be Pong with matching nonce"
1144            );
1145
1146            Ok(())
1147        })
1148        .await
1149    }
1150
1151    #[tokio::test]
1152    // #[tracing_test::traced_test]
1153    async fn probe_should_pass_through_non_associated_tags() -> anyhow::Result<()> {
1154        let cfg = ProbeConfig {
1155            timeout: std::time::Duration::from_millis(20),
1156            interval: std::time::Duration::from_secs(0),
1157            ..Default::default()
1158        };
1159
1160        let store = PeerStore {
1161            me: *OFFCHAIN_KEYPAIR.public(),
1162            get_peers: Arc::new(RwLock::new({
1163                let mut neighbors = VecDeque::new();
1164                neighbors.push_back(NEIGHBOURS.clone());
1165                neighbors
1166            })),
1167            on_finished: Arc::new(RwLock::new(Vec::new())),
1168        };
1169
1170        test_with_probing(cfg, store.clone(), move |iface: TestInterface| async move {
1171            let probe_classifier = iface.probe_classifier;
1172            let from_probing_to_network_tx = iface.from_probing_to_network_tx;
1173
1174            let expected_data = ApplicationData::new(Tag::MAX, b"Hello, this is a test message!")?;
1175
1176            let result = probe_classifier
1177                .classify(
1178                    from_probing_to_network_tx,
1179                    HoprPseudonym::random(),
1180                    ApplicationDataIn {
1181                        data: expected_data.clone(),
1182                        packet_info: Default::default(),
1183                    },
1184                )
1185                .await;
1186
1187            match result {
1188                ProbeDispatch::Passthrough(_, actual) => assert_eq!(actual.data, expected_data),
1189                ProbeDispatch::Consumed => anyhow::bail!("expected Passthrough, got Consumed"),
1190            }
1191
1192            Ok(())
1193        })
1194        .await
1195    }
1196
1197    /// How a probe gets triggered inside the test: either via the manual ping channel
1198    /// (for neighbor probes) or by injecting a single [`ProbeRouting::Looping`] into the
1199    /// probing traffic generator (for loopback probes).
1200    #[derive(Clone)]
1201    enum TestProbeStrategy {
1202        ManualNeighbor,
1203        ImmediateNeighbor {
1204            store: PeerStore,
1205        },
1206        OneShotLoopback {
1207            routing: DestinationRouting,
1208            path_id: hopr_api::types::internal::routing::PathId,
1209        },
1210    }
1211
1212    impl hopr_api::ct::ProbingTrafficGeneration for TestProbeStrategy {
1213        fn build(&self) -> futures::stream::BoxStream<'static, hopr_api::ct::ProbeRouting> {
1214            match self {
1215                Self::ManualNeighbor => Box::pin(futures::stream::pending()),
1216                Self::ImmediateNeighbor { store } => {
1217                    let peers: Vec<OffchainPublicKey> =
1218                        store.get_peers.write().unwrap().pop_front().unwrap_or_default();
1219                    Box::pin(futures::StreamExt::chain(
1220                        futures::stream::iter(peers.into_iter().map(|peer| {
1221                            ProbeRouting::Neighbor(DestinationRouting::Forward {
1222                                destination: Box::new(peer.into()),
1223                                pseudonym: Some(HoprPseudonym::random()),
1224                                forward_options: RoutingOptions::Hops(0.try_into().expect("0 is a valid u8")),
1225                                return_options: Some(RoutingOptions::Hops(0.try_into().expect("0 is a valid u8"))),
1226                            })
1227                        })),
1228                        futures::stream::pending(),
1229                    ))
1230                }
1231                Self::OneShotLoopback { routing, path_id } => {
1232                    let probe = hopr_api::ct::ProbeRouting::Looping((routing.clone(), *path_id));
1233                    Box::pin(futures::StreamExt::chain(
1234                        futures::stream::iter(std::iter::once(probe)),
1235                        futures::stream::pending(),
1236                    ))
1237                }
1238            }
1239        }
1240    }
1241
1242    /// Regression test for hoprnet/hoprnet#7972: each probe type must be emitted with the
1243    /// exact SURB count it actually consumes — no more, no less — so the path planner does
1244    /// not resolve (and the SURB store does not accumulate) unused return paths.
1245    ///
1246    /// - Neighbor probe: 1 SURB (zero-hop pong return).
1247    /// - Loopback probe: 0 SURBs (self-routed, never replied to).
1248    #[rstest::rstest]
1249    #[case::neighbor_probe_requests_one_surb(TestProbeStrategy::ManualNeighbor, 1)]
1250    #[case::loopback_probe_requests_zero_surbs(
1251        TestProbeStrategy::OneShotLoopback {
1252            routing: DestinationRouting::Forward {
1253                destination: Box::new((*OFFCHAIN_KEYPAIR.public()).into()),
1254                pseudonym: Some(HoprPseudonym::random()),
1255                forward_options: RoutingOptions::Hops(1.try_into().expect("1 is a valid u8")),
1256                return_options: None,
1257            },
1258            path_id: [1, 2, 3, 4, 5],
1259        },
1260        0,
1261    )]
1262    #[tokio::test]
1263    /// Check emitted probe SURB counts for configured path lengths and enabled probe modes.
1264    async fn probe_should_emit_with_expected_surb_count(
1265        #[case] strategy: TestProbeStrategy,
1266        #[case] expected_max_surbs: usize,
1267        #[values(true, false)] reply: bool,
1268    ) -> anyhow::Result<()> {
1269        use anyhow::Context;
1270
1271        let cfg = ProbeConfig {
1272            timeout: std::time::Duration::from_millis(200),
1273            interval: std::time::Duration::from_secs(0),
1274            ..Default::default()
1275        };
1276
1277        // Wire up a full probing harness with the parameterized strategy injected as the
1278        // traffic generator. Mirrors `test_with_probing` but allows arbitrary `ProbingTrafficGeneration`.
1279        let tag_allocators = hopr_transport_tag_allocator::create_allocators(
1280            ReservedTag::range().end..u16::MAX as u64 + 1,
1281            [
1282                (hopr_transport_tag_allocator::Usage::Session, 2048),
1283                (hopr_transport_tag_allocator::Usage::SessionTerminalTelemetry, 4000),
1284                (hopr_transport_tag_allocator::Usage::ProvingTelemetry, 10000),
1285            ],
1286        )
1287        .expect("tag allocators should be created");
1288        let probing_allocator = tag_allocators
1289            .into_iter()
1290            .find_map(|(u, alloc)| matches!(u, hopr_transport_tag_allocator::Usage::ProvingTelemetry).then_some(alloc))
1291            .expect("probing allocator should exist");
1292
1293        let probe = Probe::new(cfg, probing_allocator);
1294
1295        let (from_probing_to_network_tx, mut from_probing_to_network_rx) =
1296            futures::channel::mpsc::channel::<(DestinationRouting, ApplicationDataOut)>(100);
1297        let (mut manual_probe_tx, manual_probe_rx) =
1298            futures::channel::mpsc::channel::<(OffchainPublicKey, PingQueryReplier)>(100);
1299
1300        let store = PeerStore {
1301            me: *OFFCHAIN_KEYPAIR.public(),
1302            get_peers: Arc::new(RwLock::new(VecDeque::new())),
1303            on_finished: Arc::new(RwLock::new(Vec::new())),
1304        };
1305
1306        // Kick off the probing process before triggering — the strategy's stream drives
1307        // the loopback case, and we push to `manual_probe_tx` below for the neighbor case.
1308        let is_manual = matches!(strategy, TestProbeStrategy::ManualNeighbor);
1309        let (jhs, classifier) = probe
1310            .continuously_scan(from_probing_to_network_tx, manual_probe_rx, strategy, store.clone())
1311            .await;
1312
1313        if is_manual {
1314            let (tx, _rx) = futures::channel::mpsc::channel::<std::result::Result<Duration, ()>>(128);
1315            manual_probe_tx.send((NEIGHBOURS[0], PingQueryReplier::new(tx))).await?;
1316        }
1317
1318        let (routing, data_out) =
1319            tokio::time::timeout(std::time::Duration::from_secs(1), from_probing_to_network_rx.next())
1320                .await?
1321                .ok_or_else(|| anyhow::anyhow!("no probe emitted"))?;
1322
1323        let packet_info = data_out
1324            .packet_info
1325            .ok_or_else(|| anyhow::anyhow!("probe must carry explicit OutgoingPacketInfo"))?;
1326        assert_eq!(
1327            packet_info.max_surbs_in_packet, expected_max_surbs,
1328            "probe must request exactly {expected_max_surbs} SURB(s)"
1329        );
1330
1331        let pseudonym = match routing {
1332            DestinationRouting::Forward { pseudonym, .. } => pseudonym.context("probe pseudonym")?,
1333            _ => anyhow::bail!("probe must have forward routing"),
1334        };
1335        let tag = data_out.data.application_tag;
1336        let nonce = if is_manual {
1337            match Message::try_from(data_out.data.clone())? {
1338                Message::Probe(NeighborProbe::Ping(nonce)) => Some(nonce),
1339                _ => anyhow::bail!("neighbor probe must be a ping"),
1340            }
1341        } else {
1342            None
1343        };
1344        // The receiver can wake before the producer finishes registering the outbound probe.
1345        tokio::time::timeout(Duration::from_secs(2), async {
1346            loop {
1347                let registered = if let Some(nonce) = nonce {
1348                    classifier
1349                        .active_neighbor_probes
1350                        .get(&(pseudonym, NeighborProbe::Ping(nonce)))
1351                        .await
1352                        .is_some()
1353                } else {
1354                    classifier.active_path_probes.get(&tag).await.is_some()
1355                };
1356                if registered {
1357                    break;
1358                }
1359                tokio::task::yield_now().await;
1360            }
1361        })
1362        .await?;
1363        if reply {
1364            let data = if let Some(nonce) = nonce {
1365                Message::Probe(NeighborProbe::Pong(nonce)).try_into()?
1366            } else {
1367                data_out.data.clone()
1368            };
1369            for _ in 0..2 {
1370                classifier
1371                    .classify(
1372                        futures::sink::drain(),
1373                        pseudonym,
1374                        ApplicationDataIn {
1375                            data: data.clone(),
1376                            packet_info: Default::default(),
1377                        },
1378                    )
1379                    .await;
1380            }
1381        }
1382        tokio::time::sleep(cfg.timeout * 2).await;
1383        classifier.active_neighbor_probes.run_pending_tasks().await;
1384        classifier.active_path_probes.run_pending_tasks().await;
1385        jhs.abort_all();
1386        if is_manual {
1387            let outcomes = store
1388                .on_finished
1389                .read()
1390                .map_err(|_| anyhow::anyhow!("probe graph lock poisoned"))?;
1391            assert_eq!(outcomes.len(), 1, "duplicate replies must not update the graph twice");
1392            assert_eq!(outcomes[0].0, NEIGHBOURS[0]);
1393            assert_eq!(outcomes[0].1.is_ok(), reply);
1394        }
1395        assert_eq!(classifier.active_neighbor_probes.entry_count(), 0);
1396        assert_eq!(classifier.active_path_probes.entry_count(), 0);
1397        Ok(())
1398    }
1399    #[tokio::test]
1400    /// A timed-out neighbor probe notifies its caller and updates the graph exactly once.
1401    async fn neighbor_expiry_updates_the_graph_once() -> anyhow::Result<()> {
1402        let cfg = ProbeConfig {
1403            timeout: std::time::Duration::from_millis(10),
1404            interval: std::time::Duration::ZERO,
1405            ..Default::default()
1406        };
1407        let store = PeerStore {
1408            me: *OFFCHAIN_KEYPAIR.public(),
1409            get_peers: Arc::new(RwLock::new(VecDeque::new())),
1410            on_finished: Arc::new(RwLock::new(Vec::new())),
1411        };
1412        test_with_probing(cfg, store.clone(), move |iface: TestInterface| async move {
1413            let mut manual = iface.manual_probe_tx;
1414            let (tx, mut rx) = futures::channel::mpsc::channel(8);
1415            manual.send((NEIGHBOURS[0], PingQueryReplier::new(tx))).await?;
1416            // Consume the outbound packet without returning a pong.
1417            let mut outbound = iface.from_probing_to_network_rx;
1418            tokio::time::timeout(std::time::Duration::from_secs(2), outbound.next()).await?;
1419            tokio::time::sleep(cfg.timeout * 2).await;
1420            iface.probe_classifier.active_neighbor_probes.run_pending_tasks().await;
1421            let result = tokio::time::timeout(std::time::Duration::from_secs(2), rx.next()).await?;
1422            assert!(matches!(result, Some(Err(()))));
1423            Ok(())
1424        })
1425        .await?;
1426        let outcomes = store
1427            .on_finished
1428            .read()
1429            .map_err(|_| anyhow::anyhow!("probe graph lock poisoned"))?;
1430        assert_eq!(outcomes.len(), 1);
1431        assert_eq!(outcomes[0].0, NEIGHBOURS[0]);
1432        assert!(outcomes[0].1.is_err());
1433        Ok(())
1434    }
1435}