1use std::{
74 collections::{HashMap, VecDeque},
75 sync::{
76 Arc,
77 atomic::{AtomicBool, AtomicU64, Ordering},
78 },
79 time::{Duration, Instant},
80};
81
82use bytes::Bytes;
83use dashmap::{DashMap, DashSet};
84use hopr_api::{
85 graph::{ForwardAndReturnPath, MeasurableEdge, MeasurablePath, MeasurablePeer, NetworkGraphView, SurbTelemetry},
86 types::{
87 crypto::{prelude::*, types::OffchainPublicKey},
88 internal::prelude::*,
89 },
90};
91use hopr_crypto_packet::{HoprSurb, prelude::PacketSignals};
92use hopr_protocol_hopr::{
93 IncomingPacket, IncomingPacketError, MAXIMUM_PENDING_SURB_MAX_BYTES, MINIMUM_PENDING_SURB_MAX_BYTES,
94 OutgoingPacket, PacketDecoder, PacketEncoder,
95};
96
97use crate::{
98 observer::{TransportEvent, TransportObserver},
99 protocol::surb_map::RandomEvictionMap,
100};
101
102pub(crate) const PATH_ID_SLOTS: usize = 5;
104
105const PENDING_SURB_TTL: Duration = Duration::from_secs(60);
111
112const PENDING_SURB_HORIZON: Duration = Duration::from_secs(15 * 60);
123const _: () = assert!(PENDING_SURB_HORIZON.as_secs() >= 10 * PENDING_SURB_TTL.as_secs());
124const WRITE_OFF_HORIZON: Duration = Duration::from_secs(2 * 60);
141
142const REBUMP_COOLDOWN: Duration = Duration::from_secs(2 * 60);
153const _: () = assert!(REBUMP_COOLDOWN.as_secs() == 2 * PENDING_SURB_TTL.as_secs());
154
155const PROBATION_HEALTH: f64 = 1e-3;
166
167const VERDICT_CONFIRMATION: Duration = Duration::from_millis(900);
181
182#[derive(Debug, Default)]
187pub struct SurbRoundTripCounters {
188 expected: AtomicU64,
189 observed: AtomicU64,
190}
191
192impl SurbRoundTripCounters {
193 fn record_expected(&self, count: u64) {
194 self.expected.fetch_add(count, Ordering::Relaxed);
195 }
196
197 fn record_observed(&self, count: u64) {
198 self.observed.fetch_add(count, Ordering::Relaxed);
199 }
200
201 fn take(&self) -> (u64, u64) {
203 (
204 self.expected.swap(0, Ordering::Relaxed),
205 self.observed.swap(0, Ordering::Relaxed),
206 )
207 }
208}
209
210#[derive(Clone)]
216pub struct SurbRoundTripRegistry {
217 destinations: Arc<DashMap<ForwardAndReturnPath, OffchainPublicKey>>,
218 observer: Option<TransportObserver>,
219 inner: Arc<DashMap<ForwardAndReturnPath, Arc<SurbRoundTripCounters>>>,
220 relayer_health: Arc<RelayerHealth>,
222 pending: Arc<RandomEvictionMap<PendingSurb>>,
228 written_off: Arc<DashMap<HoprPseudonym, HashMap<OffchainPublicKey, VecDeque<Instant>>>>,
236 bumped: Arc<DashMap<(HoprPseudonym, OffchainPublicKey), Instant>>,
241 written_off_legs: Arc<RandomEvictionMap<(ReturnRelayers, Instant)>>,
258 pending_lost: Arc<AtomicU64>,
264}
265
266impl Default for SurbRoundTripRegistry {
267 fn default() -> Self {
268 Self::with_budget(MINIMUM_PENDING_SURB_MAX_BYTES)
269 }
270}
271
272impl std::fmt::Debug for SurbRoundTripRegistry {
273 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
274 f.debug_struct("SurbRoundTripRegistry")
275 .field("pairs", &self.inner.len())
276 .finish_non_exhaustive()
277 }
278}
279
280impl SurbRoundTripRegistry {
281 pub fn with_observer(mut self, observer: TransportObserver) -> Self {
283 self.observer = Some(observer);
284 self
285 }
286
287 fn record_minted(&self, paths: ForwardAndReturnPath, count: u64, destination: OffchainPublicKey) {
289 self.destinations.insert(paths, destination);
290 self.record_expected(paths, count);
291 if let Some(observer) = &self.observer {
292 observer(TransportEvent::Surb {
293 destination,
294 paths,
295 minted: count,
296 returned: 0,
297 });
298 }
299 }
300
301 pub fn with_budget(max_bytes: u64) -> Self {
309 let budget = max_bytes.clamp(MINIMUM_PENDING_SURB_MAX_BYTES, MAXIMUM_PENDING_SURB_MAX_BYTES);
310 Self::from_maps(RandomEvictionMap::new(budget), RandomEvictionMap::new(budget))
311 }
312
313 #[cfg(test)]
316 pub(crate) fn with_records(records: usize, seed: u64) -> Self {
317 Self::from_maps(
318 RandomEvictionMap::with_seed(records, seed),
319 RandomEvictionMap::with_seed(records, seed ^ !0),
320 )
321 }
322
323 fn from_maps(
324 pending: RandomEvictionMap<PendingSurb>,
325 written_off_legs: RandomEvictionMap<(ReturnRelayers, Instant)>,
326 ) -> Self {
327 Self {
328 inner: Default::default(),
329 destinations: Default::default(),
330 observer: None,
331 pending: Arc::new(pending),
332 relayer_health: Default::default(),
333 written_off: Default::default(),
334 bumped: Default::default(),
335 written_off_legs: Arc::new(written_off_legs),
336 pending_lost: Default::default(),
337 }
338 }
339
340 pub fn take_pending_lost(&self) -> u64 {
348 self.pending_lost.swap(0, Ordering::Relaxed)
349 }
350
351 fn remember(&self, surb_id: HoprSurbId, pending: PendingSurb) {
360 if let Some((evicted_id, evicted)) = self.pending.insert(surb_id, pending) {
361 self.pending_lost.fetch_add(1, Ordering::Relaxed);
362 if evicted.partner_replied.is_some() {
363 self.relayer_health.discard_race(&evicted.return_relayers, &evicted_id);
364 }
365 }
366 }
367
368 fn remember_written_off(&self, surb_id: HoprSurbId, leg: ReturnRelayers, minted_at: Instant) {
373 if let Some((evicted_id, (evicted, _))) = self.written_off_legs.insert(surb_id, (leg, minted_at)) {
374 self.relayer_health.discard_race(&evicted, &evicted_id);
375 }
376 }
377
378 pub fn expire_step(&self) {
383 self.pending.expire_step(PENDING_SURB_HORIZON);
384 self.written_off_legs.expire_step(PENDING_SURB_HORIZON);
385 }
386
387 pub fn has_write_offs(&self) -> bool {
390 !self.written_off.is_empty()
391 }
392
393 fn judges(&self, pair_index: usize) -> bool {
404 pair_index == FIRST_SPENT_INDEX
405 }
406
407 fn open_race(&self, waiting: &ReturnRelayers, partner: HoprSurbId, home: &ReturnRelayers, round_trip: Duration) {
418 self.relayer_health.open_race(waiting, partner, home, round_trip);
419 if !self.pending.contains_key(&partner) && !self.written_off_legs.contains_key(&partner) {
420 self.relayer_health.discard_race(waiting, &partner);
421 }
422 }
423
424 fn counters(&self, paths: ForwardAndReturnPath) -> Arc<SurbRoundTripCounters> {
425 self.inner.entry(paths).or_default().value().clone()
426 }
427
428 pub fn record_expected(&self, paths: ForwardAndReturnPath, count: u64) {
430 self.counters(paths).record_expected(count);
431 }
432
433 pub fn relayer_health(&self) -> Arc<RelayerHealth> {
435 self.relayer_health.clone()
436 }
437
438 pub fn age_write_offs(&self) {
442 let health = &self.relayer_health;
443 for mut session in self.written_off.iter_mut() {
444 session.retain(|relayer, written_off_at| {
445 while written_off_at
447 .front()
448 .is_some_and(|at| at.elapsed() > WRITE_OFF_HORIZON)
449 {
450 written_off_at.pop_front();
451 }
452 health.is_known_dead(relayer) && !written_off_at.is_empty()
453 });
454 }
455 self.written_off.retain(|_, session| !session.is_empty());
456 let pass = Instant::now();
459 self.bumped.retain(|(_, relayer), at| {
460 if health.is_known_dead(relayer) {
461 *at = pass;
462 true
463 } else {
464 at.elapsed() < REBUMP_COOLDOWN
465 }
466 });
467 }
468
469 pub(crate) fn dead_stock(&self) -> Vec<StockThroughDead> {
482 let health = &self.relayer_health;
483 let dead: Vec<OffchainPublicKey> = health
486 .probation
487 .iter()
488 .map(|relayer| *relayer)
489 .chain(
490 health
491 .confirmed
492 .iter()
493 .map(|relayer| *relayer)
494 .filter(|relayer| health.dead.contains(relayer)),
495 )
496 .collect();
497 let mut stock = Vec::new();
498 if dead.is_empty() {
499 return stock;
500 }
501 self.pending.for_each(|id, pending| {
502 if let Some(relayer) = relayers_of(&pending.return_relayers).find(|relayer| dead.contains(relayer)) {
503 stock.push(StockThroughDead {
504 id: *id,
505 pseudonym: pending.pseudonym,
506 dead: relayer,
507 return_relayers: pending.return_relayers,
508 minted_at: pending.minted_at,
509 });
510 }
511 });
512 stock
513 }
514
515 pub fn write_off_dead(&self) -> Vec<HoprPseudonym> {
528 self.age_write_offs();
529 self.write_off(self.dead_stock())
530 }
531
532 pub(crate) fn write_off(&self, stock: Vec<StockThroughDead>) -> Vec<HoprPseudonym> {
560 let health = &self.relayer_health;
561 let mut reached: Vec<HoprPseudonym> = Vec::new();
562 for StockThroughDead {
563 id,
564 pseudonym,
565 dead,
566 return_relayers,
567 minted_at,
568 } in stock
569 {
570 let since = match health.known_dead_since(&dead) {
575 Some(since) if health.is_confirmed_dead(&dead) => since,
576 _ => continue,
578 };
579 self.remember_written_off(id, return_relayers, minted_at);
582 if self.pending.remove(&id).is_none() {
583 self.written_off_legs.remove(&id);
586 continue;
587 }
588 let mut session = self.written_off.entry(pseudonym).or_default();
589 let first_for_pair = !session.contains_key(&dead);
590 session.entry(dead).or_default().push_back(Instant::now());
591 drop(session);
592 if minted_at >= since {
595 tracing::trace!(
596 %pseudonym,
597 relayer = %dead,
598 "probe through a relayer known dead, written off without a new SURB generation"
599 );
600 continue;
601 }
602 if self.bumped.contains_key(&(pseudonym, dead)) {
603 if first_for_pair {
604 tracing::debug!(
605 %pseudonym,
606 relayer = %dead,
607 "SURB through a relayer already bumped for, written off without a new SURB generation"
608 );
609 }
610 continue;
611 }
612 self.bumped.insert((pseudonym, dead), Instant::now());
613 if !reached.contains(&pseudonym) {
614 reached.push(pseudonym);
615 }
616 }
617 for pseudonym in &reached {
619 self.written_off.remove(pseudonym);
620 }
621 if !reached.is_empty() {
622 let now = Instant::now();
631 let known_dead: Vec<OffchainPublicKey> = health
632 .dead
633 .iter()
634 .map(|relayer| *relayer)
635 .chain(health.probation.iter().map(|relayer| *relayer))
636 .collect();
637 for pseudonym in &reached {
638 for relayer in &known_dead {
639 self.bumped.entry((*pseudonym, *relayer)).or_insert(now);
640 }
641 }
642
643 let mut wiped: Vec<(HoprSurbId, ReturnRelayers, Instant)> = Vec::new();
651 self.pending.for_each(|id, pending| {
652 if reached.contains(&pending.pseudonym) {
653 wiped.push((*id, pending.return_relayers, pending.minted_at));
654 }
655 });
656 for (id, return_relayers, minted_at) in wiped {
657 self.remember_written_off(id, return_relayers, minted_at);
658 if self.pending.remove(&id).is_none() {
659 self.written_off_legs.remove(&id);
661 }
662 }
663 }
664 reached
665 }
666
667 pub fn invalid(&self) -> HashMap<HoprPseudonym, (u64, f64)> {
677 if self.written_off.is_empty() {
678 return HashMap::new();
679 }
680 let mut outstanding: HashMap<HoprPseudonym, u64> = HashMap::new();
681 self.pending.for_each(|_, pending| {
682 *outstanding.entry(pending.pseudonym).or_default() += 1;
683 });
684 self.written_off
685 .iter()
686 .map(|session| {
687 let count: u64 = session.values().map(|written_off_at| written_off_at.len() as u64).sum();
688 let total = count + outstanding.get(session.key()).copied().unwrap_or_default();
689 (*session.key(), (count, count as f64 / total.max(1) as f64))
690 })
691 .collect()
692 }
693
694 pub fn forget_session(&self, pseudonym: &HoprPseudonym) {
696 self.written_off.remove(pseudonym);
697 self.bumped.retain(|(session, _), _| session != pseudonym);
698 }
699
700 pub fn record_observed(&self, paths: ForwardAndReturnPath, count: u64) {
702 self.counters(paths).record_observed(count);
703 if let Some(observer) = &self.observer {
704 let destination = self.destinations.get(&paths).map(|destination| *destination);
705 if let Some(destination) = destination {
706 observer(TransportEvent::Surb {
707 destination,
708 paths,
709 minted: 0,
710 returned: count,
711 });
712 }
713 }
714 }
715
716 pub fn drain(&self) -> Vec<(ForwardAndReturnPath, u64, u64)> {
718 self.inner
719 .iter()
720 .filter_map(|entry| {
721 let (expected, observed) = entry.value().take();
722 (expected > 0 || observed > 0).then(|| (*entry.key(), expected, observed))
723 })
724 .collect()
725 }
726}
727
728pub type PathSlotResolver = Arc<dyn Fn(&OffchainPublicKey) -> Option<u64> + Send + Sync>;
734
735pub fn no_path_slots() -> PathSlotResolver {
740 Arc::new(|_| None)
741}
742
743pub(crate) type ReturnRelayers = [Option<OffchainPublicKey>; PATH_ID_SLOTS];
745
746#[derive(Debug, Clone, Copy)]
748pub(crate) struct PendingSurb {
749 paths: ForwardAndReturnPath,
751 pseudonym: HoprPseudonym,
753 return_relayers: ReturnRelayers,
755 minted_at: Instant,
757 partner: Option<HoprSurbId>,
759 pair_index: usize,
761 partner_replied: Option<(ReturnRelayers, Instant)>,
763}
764
765#[derive(Debug, Clone, Copy)]
768pub(crate) struct StockThroughDead {
769 id: HoprSurbId,
770 pseudonym: HoprPseudonym,
771 dead: OffchainPublicKey,
773 return_relayers: ReturnRelayers,
774 minted_at: Instant,
775}
776
777#[derive(Debug, Default)]
780pub struct RelayerHealth {
781 relayers: DashMap<OffchainPublicKey, RaceHistory>,
782 lost_neighbours: DashSet<OffchainPublicKey>,
784 neighbours_changed: AtomicBool,
785 dead: DashSet<OffchainPublicKey>,
787 dead_since: DashMap<OffchainPublicKey, Instant>,
789 confirmed: DashSet<OffchainPublicKey>,
792 probation: DashSet<OffchainPublicKey>,
796 known_dead_since: DashMap<OffchainPublicKey, Instant>,
801 dead_over: DashMap<OffchainPublicKey, Hop>,
806 suspected: DashMap<OffchainPublicKey, (Hop, Instant)>,
810 published: DashMap<OffchainPublicKey, f64>,
812 last_reply: LastReply,
814}
815
816#[derive(Debug)]
820struct LastReply {
821 epoch: Instant,
822 since_epoch_ms_plus_one: AtomicU64,
823}
824
825impl Default for LastReply {
826 fn default() -> Self {
827 Self {
828 epoch: Instant::now(),
829 since_epoch_ms_plus_one: AtomicU64::new(0),
830 }
831 }
832}
833
834const FIRST_SPENT_INDEX: usize = 0;
845
846#[derive(Debug, Clone, Copy)]
848struct Race {
849 at: Instant,
850 margin: f64,
852 winning_round_trip: f64,
854}
855
856impl Race {
857 fn lost(&self) -> bool {
859 self.margin >= PENDING_SURB_TTL.as_secs_f64()
860 }
861}
862
863#[derive(Debug, Default)]
864struct RaceHistory {
865 races: VecDeque<Race>,
866 open: HashMap<HoprSurbId, Race>,
869 open_legs: HashMap<HoprSurbId, (ReturnRelayers, usize)>,
873 memory: Duration,
875 last_home: HashMap<Hop, Instant>,
879}
880
881const MIN_RACES: usize = 3;
883
884const MIN_RACES_OVER_ONE_HOP: usize = 4;
896
897impl RaceHistory {
898 fn forget_older_than(&mut self, memory: Duration) {
899 while self.races.len() > MIN_RACES && self.races.front().is_some_and(|race| race.at.elapsed() > memory) {
900 self.races.pop_front();
901 }
902 }
903
904 fn finished(&self) -> impl Iterator<Item = &Race> {
906 self.races.iter().filter(|race| !race.lost())
907 }
908
909 fn bar(&self, fallback_usual: Option<f64>) -> Option<f64> {
913 match (self.usual_margin(), fallback_usual) {
916 (Some(usual), _) => Some(usual),
917 (None, Some(fallback)) => Some(
918 self.finished()
919 .map(|race| race.margin.abs())
920 .fold(0.0, f64::max)
921 .max(fallback),
922 ),
923 (None, None) => None,
924 }
925 }
926
927 fn usual_margin(&self) -> Option<f64> {
930 (self.finished().count() >= MIN_RACES)
931 .then(|| self.finished().map(|race| race.margin.abs()).fold(0.0, f64::max))
932 }
933
934 fn unanswered(
978 &self,
979 last_reply: Option<Instant>,
980 fallback_usual: Option<f64>,
981 bars: &HashMap<OffchainPublicKey, f64>,
982 ) -> Vec<(HoprSurbId, Instant)> {
983 let Some(own) = self.bar(fallback_usual) else {
984 return Vec::new();
985 };
986 let now = Instant::now();
987 self.open
988 .iter()
989 .filter_map(|(surb, race)| {
990 let bar = self
992 .open_legs
993 .get(surb)
994 .into_iter()
995 .flat_map(|(leg, _)| relayers_of(leg))
996 .filter_map(|relayer| bars.get(&relayer))
997 .fold(own, |bar, other| bar.max(*other));
998 race.at
999 .checked_add(Duration::from_secs_f64(bar))
1000 .map(|due| (*surb, due))
1001 })
1002 .filter(|(_, due)| *due < now && last_reply.is_some_and(|reply| reply > *due))
1004 .filter(|(surb, due)| self.home_over_the_hop_of(surb).is_none_or(|home| home < *due))
1006 .collect()
1007 }
1008
1009 fn unanswered_at_all(&self, unanswered: &[(HoprSurbId, Instant)]) -> Vec<HoprSurbId> {
1018 let home = self.last_home.values().max().copied();
1019 unanswered
1020 .iter()
1021 .filter(|(_, due)| home.is_none_or(|home| home < *due))
1022 .map(|(surb, _)| *surb)
1023 .collect()
1024 }
1025
1026 fn hop_at_fault(&self, unanswered: &[(HoprSurbId, Instant)]) -> Option<Hop> {
1031 let mut hops = unanswered.iter().filter_map(|(surb, _)| self.hop_of(surb));
1032 let hop = hops.next()?;
1033 if hops.any(|other| other != hop) {
1034 return None;
1035 }
1036 let oldest = unanswered
1037 .iter()
1038 .filter_map(|(surb, _)| self.open.get(surb))
1039 .map(|race| race.at)
1040 .min()?;
1041 self.last_home
1042 .iter()
1043 .any(|(other, home)| *other != hop && *home >= oldest)
1044 .then_some(hop)
1045 }
1046
1047 fn home_over_the_hop_of(&self, surb: &HoprSurbId) -> Option<Instant> {
1052 self.hop_of(surb).and_then(|hop| self.last_home.get(&hop)).copied()
1053 }
1054
1055 fn hop_of(&self, surb: &HoprSurbId) -> Option<Hop> {
1057 self.open_legs.get(surb).map(|(_, position)| Hop::at(*position))
1058 }
1059}
1060
1061#[derive(Debug)]
1063struct Suspect {
1064 relayer: OffchainPublicKey,
1065 unanswered: Vec<HoprSurbId>,
1067 open: HashMap<HoprSurbId, usize>,
1070 needed: usize,
1073}
1074
1075impl Suspect {
1076 fn note_legs(&mut self, legs: &HashMap<HoprSurbId, ReturnRelayers>) {
1081 for (surb, leg) in legs {
1082 let on_the_leg =
1083 hops_of(leg).find_map(|(relayer, _, position)| (relayer == self.relayer).then_some(position));
1084 if let Some(position) = on_the_leg {
1085 self.open.entry(*surb).or_insert(position);
1086 }
1087 }
1088 }
1089
1090 fn explains(&self, unanswered_for_any: &std::collections::HashSet<HoprSurbId>) -> (usize, usize) {
1093 self.open
1094 .iter()
1095 .filter(|(surb, _)| unanswered_for_any.contains(*surb))
1096 .fold((0, 0), |(explains, depth), (_, position)| {
1097 (explains + 1, depth + position)
1098 })
1099 }
1100
1101 fn is_tied_with(&self, other: &Self, unanswered_for_any: &std::collections::HashSet<HoprSurbId>) -> bool {
1105 self.explains(unanswered_for_any).0 == other.explains(unanswered_for_any).0
1106 && self.unanswered.iter().all(|surb| other.open.contains_key(surb))
1107 && other.unanswered.iter().all(|surb| self.open.contains_key(surb))
1108 && !self
1109 .open
1110 .iter()
1111 .filter(|(surb, _)| unanswered_for_any.contains(*surb))
1112 .filter_map(|(surb, position)| other.open.get(surb).map(|behind| (position, behind)))
1113 .all(|(position, behind)| position < behind)
1114 }
1115}
1116
1117fn explain_away(
1169 mut suspects: Vec<Suspect>,
1170 known_dead: impl Fn(&OffchainPublicKey) -> bool,
1171) -> std::collections::HashSet<OffchainPublicKey> {
1172 let unanswered_for_any: std::collections::HashSet<HoprSurbId> = suspects
1173 .iter()
1174 .flat_map(|suspect| suspect.unanswered.iter().copied())
1175 .collect();
1176 suspects.sort_by_cached_key(|suspect| {
1177 let (explains, depth) = suspect.explains(&unanswered_for_any);
1178 (std::cmp::Reverse(explains), depth, suspect.relayer.to_string())
1179 });
1180 let mut explained: std::collections::HashSet<HoprSurbId> = std::collections::HashSet::new();
1181 let mut black_holes = std::collections::HashSet::new();
1182 let mut set_aside = std::collections::HashSet::new();
1183 for (rank, suspect) in suspects.iter().enumerate() {
1184 let relayer = suspect.relayer;
1185 if set_aside.contains(&relayer) {
1186 continue;
1187 }
1188 let own = suspect
1189 .unanswered
1190 .iter()
1191 .filter(|surb| !explained.contains(*surb))
1192 .count();
1193 if own < suspect.needed {
1194 tracing::debug!(
1195 %relayer,
1196 unanswered = suspect.unanswered.len(),
1197 own,
1198 "overdue replies explained by a relayer judged dead on their legs"
1199 );
1200 continue;
1201 }
1202 let tied = suspects[rank + 1..]
1203 .iter()
1204 .filter(|_| !known_dead(&relayer))
1205 .find(|other| !set_aside.contains(&other.relayer) && suspect.is_tied_with(other, &unanswered_for_any));
1206 let Some(other) = tied else {
1207 black_holes.insert(relayer);
1208 explained.extend(suspect.open.keys().copied());
1209 continue;
1210 };
1211 set_aside.insert(other.relayer);
1212 explained.extend(suspect.open.keys().chain(other.open.keys()).copied());
1213 if known_dead(&other.relayer) {
1214 black_holes.insert(other.relayer);
1215 } else {
1216 tracing::debug!(%relayer, other = %other.relayer, "overdue replies do not tell two relayers apart yet");
1217 }
1218 }
1219 black_holes
1220}
1221
1222#[derive(Debug, Clone, Copy)]
1225struct Judgement {
1226 margin: f64,
1227 uncertainty: f64,
1228 fastest: f64,
1229}
1230
1231impl Judgement {
1232 fn health(&self, unit: f64) -> f64 {
1235 unit / unit.max(self.margin - self.uncertainty)
1236 }
1237
1238 fn indistinct(&self, unit: f64) -> std::ops::RangeInclusive<f64> {
1241 let slack = self.uncertainty.max(unit);
1242 unit / unit.max(self.margin + slack)..=unit / unit.max(self.margin - slack)
1243 }
1244}
1245
1246fn relayers_of(leg: &ReturnRelayers) -> impl Iterator<Item = OffchainPublicKey> + '_ {
1247 leg.iter().flatten().copied()
1248}
1249
1250#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
1266enum Hop {
1267 FromReplier,
1268 FromRelayer,
1269}
1270
1271impl Hop {
1272 fn at(position: usize) -> Self {
1274 if position == 0 {
1275 Self::FromReplier
1276 } else {
1277 Self::FromRelayer
1278 }
1279 }
1280}
1281
1282fn hops_of(leg: &ReturnRelayers) -> impl Iterator<Item = (OffchainPublicKey, Hop, usize)> + '_ {
1284 relayers_of(leg)
1285 .enumerate()
1286 .map(|(position, relayer)| (relayer, Hop::at(position), position))
1287}
1288
1289impl RelayerHealth {
1290 pub(crate) fn record_race(
1293 &self,
1294 later: &ReturnRelayers,
1295 later_surb: HoprSurbId,
1296 earlier: &ReturnRelayers,
1297 winning_round_trip: Duration,
1298 by: Duration,
1299 ) {
1300 let (at, winning_round_trip) = (Instant::now(), winning_round_trip.as_secs_f64());
1301 let race = |margin| Race {
1302 at,
1303 margin,
1304 winning_round_trip,
1305 };
1306 for relayer in relayers_of(later) {
1307 if let Some(mut history) = self.relayers.get_mut(&relayer) {
1308 history.open.remove(&later_surb);
1309 history.open_legs.remove(&later_surb);
1310 }
1311 if !earlier.contains(&Some(relayer)) {
1312 self.record(relayer, race(by.as_secs_f64()));
1313 }
1314 }
1315 for relayer in relayers_of(earlier).filter(|relayer| !later.contains(&Some(*relayer))) {
1316 self.record(relayer, race(-by.as_secs_f64()));
1317 }
1318 }
1319
1320 pub(crate) fn open_race(
1323 &self,
1324 waiting: &ReturnRelayers,
1325 surb: HoprSurbId,
1326 home: &ReturnRelayers,
1327 winning_round_trip: Duration,
1328 ) {
1329 let race = Race {
1330 at: Instant::now(),
1331 margin: 0.0,
1332 winning_round_trip: winning_round_trip.as_secs_f64(),
1333 };
1334 let on_both = |relayer: &OffchainPublicKey, hop: &Hop| {
1337 hops_of(home).any(|(other, other_hop, _)| other == *relayer && other_hop == *hop)
1338 };
1339 for (relayer, _, position) in hops_of(waiting).filter(|(relayer, hop, _)| !on_both(relayer, hop)) {
1340 let mut history = self.relayers.entry(relayer).or_default();
1341 history.open.insert(surb, race);
1342 history.open_legs.insert(surb, (*waiting, position));
1343 }
1344 }
1345
1346 fn discard_race(&self, waiting: &ReturnRelayers, surb: &HoprSurbId) {
1349 for relayer in relayers_of(waiting) {
1350 if let Some(mut history) = self.relayers.get_mut(&relayer) {
1351 history.open.remove(surb);
1352 history.open_legs.remove(surb);
1353 }
1354 }
1355 }
1356
1357 fn discard_race_if_fresh(&self, waiting: &ReturnRelayers, surb: &HoprSurbId, minted_at: Instant) {
1360 for relayer in relayers_of(waiting).filter(|relayer| self.is_fresh_for(relayer, minted_at)) {
1361 if let Some(mut history) = self.relayers.get_mut(&relayer) {
1362 history.open.remove(surb);
1363 history.open_legs.remove(surb);
1364 }
1365 }
1366 }
1367
1368 fn record(&self, relayer: OffchainPublicKey, race: Race) {
1369 let mut history = self.relayers.entry(relayer).or_default();
1370 history.races.push_back(race);
1371 let memory = history.memory;
1372 if !memory.is_zero() {
1373 history.forget_older_than(memory);
1374 }
1375 }
1376
1377 pub fn set_neighbour_connected(&self, neighbour: OffchainPublicKey, connected: bool) {
1379 let changed = if connected {
1380 self.lost_neighbours.remove(&neighbour).is_some()
1381 } else {
1382 self.lost_neighbours.insert(neighbour)
1383 };
1384 if changed {
1385 self.neighbours_changed.store(true, Ordering::Relaxed);
1386 }
1387 }
1388
1389 pub fn take_neighbour_change(&self) -> bool {
1391 self.neighbours_changed.swap(false, Ordering::Relaxed)
1392 }
1393
1394 pub(crate) fn record_reply(&self) {
1396 self.record_reply_at(Instant::now());
1397 }
1398
1399 fn record_reply_at(&self, at: Instant) {
1400 let ms = u64::try_from(at.saturating_duration_since(self.last_reply.epoch).as_millis()).unwrap_or(u64::MAX - 1);
1401 self.last_reply
1402 .since_epoch_ms_plus_one
1403 .fetch_max(ms + 1, Ordering::Relaxed);
1404 }
1405
1406 fn last_reply(&self) -> Option<Instant> {
1408 let stored = self.last_reply.since_epoch_ms_plus_one.load(Ordering::Relaxed);
1409 stored
1410 .checked_sub(1)
1411 .and_then(|ms| self.last_reply.epoch.checked_add(Duration::from_millis(ms)))
1412 }
1413
1414 pub fn any_known_dead(&self) -> bool {
1416 !self.dead.is_empty() || !self.probation.is_empty()
1417 }
1418
1419 fn is_known_dead(&self, relayer: &OffchainPublicKey) -> bool {
1422 self.dead.contains(relayer) || self.probation.contains(relayer)
1423 }
1424
1425 fn is_confirmed_dead(&self, relayer: &OffchainPublicKey) -> bool {
1429 self.probation.contains(relayer) || (self.dead.contains(relayer) && self.confirmed.contains(relayer))
1430 }
1431
1432 fn note_home(&self, leg: &ReturnRelayers, minted_at: Instant) {
1437 let now = Instant::now();
1438 for (relayer, hop, _) in hops_of(leg).filter(|(relayer, ..)| self.is_fresh_for(relayer, minted_at)) {
1439 if let Some(mut history) = self.relayers.get_mut(&relayer) {
1441 history.last_home.insert(hop, now);
1442 }
1443 }
1444 }
1445
1446 fn mark_dead(&self, relayer: OffchainPublicKey) {
1448 let now = Instant::now();
1449 self.dead.insert(relayer);
1450 self.dead_since.insert(relayer, now);
1451 self.known_dead_since.insert(relayer, now);
1452 }
1453
1454 fn has_stood(&self, relayer: &OffchainPublicKey, hop: Hop) -> bool {
1468 let now = Instant::now();
1469 let (suspected_over, since) = *self.suspected.entry(*relayer).or_insert((hop, now));
1470 if suspected_over != hop {
1471 self.suspected.insert(*relayer, (hop, now));
1474 return false;
1475 }
1476 let delivered = self
1477 .relayers
1478 .get(relayer)
1479 .is_some_and(|history| history.last_home.get(&hop).is_some_and(|home| *home >= since));
1480 if delivered {
1481 self.suspected.insert(*relayer, (hop, now));
1483 return false;
1484 }
1485 since.elapsed() >= VERDICT_CONFIRMATION
1486 }
1487
1488 fn known_dead_since(&self, relayer: &OffchainPublicKey) -> Option<Instant> {
1490 self.known_dead_since.get(relayer).map(|since| *since)
1491 }
1492
1493 fn is_fresh_for(&self, relayer: &OffchainPublicKey, minted_at: Instant) -> bool {
1501 !self.is_confirmed_dead(relayer) || self.known_dead_since(relayer).is_none_or(|since| minted_at >= since)
1502 }
1503
1504 fn revive_by_reply(&self, leg: &ReturnRelayers, minted_at: Instant) {
1510 for (relayer, hop, _) in hops_of(leg) {
1511 if !self.is_known_dead(&relayer) || !self.is_fresh_for(&relayer, minted_at) {
1514 continue;
1515 }
1516 if self.dead_over.get(&relayer).is_some_and(|dead_over| *dead_over != hop) {
1519 continue;
1520 }
1521 let was_dead = self.dead.remove(&relayer).is_some();
1522 self.dead_since.remove(&relayer);
1523 self.known_dead_since.remove(&relayer);
1524 self.dead_over.remove(&relayer);
1525 self.confirmed.remove(&relayer);
1526 let was_on_probation = self.probation.remove(&relayer).is_some();
1527 if was_dead || was_on_probation {
1528 tracing::info!(%relayer, on_probation = was_on_probation, "relayer revived by a reply");
1529 }
1530 }
1531 }
1532
1533 #[cfg(test)]
1534 fn is_dead(&self, relayer: &OffchainPublicKey) -> bool {
1535 self.dead.contains(relayer)
1536 }
1537
1538 #[cfg(test)]
1539 fn is_on_probation(&self, relayer: &OffchainPublicKey) -> bool {
1540 self.probation.contains(relayer)
1541 }
1542
1543 pub fn refresh(&self, memory: u32) -> Vec<(OffchainPublicKey, f64, f64)> {
1553 let last_reply = self.last_reply();
1554 let fallback_usual = self
1556 .relayers
1557 .iter()
1558 .filter_map(|entry| entry.usual_margin())
1559 .max_by(f64::total_cmp);
1560 let bars: HashMap<OffchainPublicKey, f64> = self
1564 .relayers
1565 .iter()
1566 .filter_map(|entry| entry.bar(fallback_usual).map(|bar| (*entry.key(), bar)))
1567 .collect();
1568 let mut suspects: Vec<Suspect> = Vec::new();
1569 let mut unanswered_legs: HashMap<HoprSurbId, ReturnRelayers> = HashMap::new();
1570 let mut hop_at_fault: HashMap<OffchainPublicKey, Hop> = HashMap::new();
1571 for mut entry in self.relayers.iter_mut() {
1572 let mut lost = Vec::new();
1575 entry.open.retain(|_, race| {
1576 let open = race.at.elapsed() <= PENDING_SURB_TTL;
1577 if !open {
1578 lost.push(Race {
1579 at: Instant::now(),
1580 margin: PENDING_SURB_TTL.as_secs_f64(),
1581 ..*race
1582 });
1583 }
1584 open
1585 });
1586 entry.races.extend(lost);
1587 let RaceHistory {
1588 open,
1589 open_legs,
1590 last_home,
1591 ..
1592 } = &mut *entry;
1593 open_legs.retain(|surb, _| open.contains_key(surb));
1594 last_home.retain(|_, home| home.elapsed() <= PENDING_SURB_TTL);
1596 let unanswered = entry.unanswered(last_reply, fallback_usual, &bars);
1597 let (unanswered, needed): (Vec<HoprSurbId>, usize) = match entry.hop_at_fault(&unanswered) {
1598 Some(hop) if unanswered.len() >= MIN_RACES_OVER_ONE_HOP => {
1600 hop_at_fault.insert(*entry.key(), hop);
1601 (
1602 unanswered.iter().map(|(surb, _)| *surb).collect(),
1603 MIN_RACES_OVER_ONE_HOP,
1604 )
1605 }
1606 _ => (entry.unanswered_at_all(&unanswered), MIN_RACES),
1608 };
1609 if unanswered.len() >= needed {
1610 let open = entry
1611 .open
1612 .keys()
1613 .map(|surb| (*surb, entry.open_legs.get(surb).map_or(0, |(_, position)| *position)))
1614 .collect();
1615 unanswered_legs.extend(
1616 unanswered
1617 .iter()
1618 .filter_map(|surb| entry.open_legs.get(surb).map(|(leg, _)| (*surb, *leg))),
1619 );
1620 suspects.push(Suspect {
1621 relayer: *entry.key(),
1622 unanswered,
1623 open,
1624 needed,
1625 });
1626 }
1627 }
1628 for suspect in &mut suspects {
1629 suspect.note_legs(&unanswered_legs);
1630 }
1631 let black_holes = explain_away(suspects, |relayer| self.is_known_dead(relayer));
1632 self.dead.retain(|relayer| {
1633 let still = black_holes.contains(relayer);
1634 if still {
1635 if self
1637 .dead_since
1638 .get(relayer)
1639 .is_some_and(|since| since.elapsed() >= VERDICT_CONFIRMATION)
1640 {
1641 self.confirmed.insert(*relayer);
1642 }
1643 } else if self.confirmed.remove(relayer).is_some() {
1644 tracing::info!(%relayer, "dead verdict lapsed into probation");
1645 self.dead_since.remove(relayer);
1646 self.probation.insert(*relayer);
1647 } else {
1648 tracing::info!(%relayer, "dead verdict withdrawn before it was confirmed");
1651 self.dead_since.remove(relayer);
1652 self.known_dead_since.remove(relayer);
1653 self.dead_over.remove(relayer);
1654 }
1655 still
1656 });
1657 let back_from_probation: Vec<OffchainPublicKey> = self
1658 .probation
1659 .iter()
1660 .filter(|relayer| black_holes.contains(relayer))
1661 .map(|relayer| *relayer)
1662 .collect();
1663 for relayer in back_from_probation {
1664 tracing::info!(%relayer, "relayer on probation judged dead again");
1665 self.probation.remove(&relayer);
1666 self.dead.insert(relayer);
1668 self.confirmed.insert(relayer);
1669 self.dead_since.insert(
1670 relayer,
1671 Instant::now()
1672 .checked_sub(VERDICT_CONFIRMATION)
1673 .unwrap_or_else(Instant::now),
1674 );
1675 }
1676 for relayer in &black_holes {
1677 if self.is_known_dead(relayer) {
1678 continue;
1679 }
1680 match hop_at_fault.remove(relayer) {
1681 None => {
1684 tracing::info!(%relayer, "relayer judged dead");
1685 self.mark_dead(*relayer);
1686 }
1687 Some(hop) if self.has_stood(relayer, hop) => {
1690 tracing::info!(%relayer, over = ?hop, "relayer judged dead");
1691 self.mark_dead(*relayer);
1692 self.confirmed.insert(*relayer);
1693 self.dead_over.insert(*relayer, hop);
1694 }
1695 Some(_) => {}
1696 }
1697 }
1698 self.suspected
1700 .retain(|relayer, _| black_holes.contains(relayer) && !self.is_known_dead(relayer));
1701
1702 let judged: HashMap<OffchainPublicKey, Judgement> = self
1703 .relayers
1704 .iter()
1705 .map(|entry| *entry.key())
1706 .collect::<Vec<_>>()
1707 .into_iter()
1708 .filter_map(|relayer| self.judge(&relayer, memory).map(|judgement| (relayer, judgement)))
1709 .collect();
1710 self.relayers.retain(|_, history| {
1712 !history.open.is_empty()
1713 || history
1714 .races
1715 .back()
1716 .is_some_and(|race| race.at.elapsed() <= history.memory.max(PENDING_SURB_TTL))
1717 });
1718 let unit = judged
1719 .values()
1720 .map(|judgement| judgement.fastest)
1721 .min_by(f64::total_cmp);
1722
1723 let relayers: std::collections::HashSet<OffchainPublicKey> = judged
1724 .keys()
1725 .copied()
1726 .chain(self.published.iter().map(|entry| *entry.key()))
1727 .chain(self.dead.iter().map(|relayer| *relayer))
1728 .chain(self.probation.iter().map(|relayer| *relayer))
1729 .collect();
1730
1731 let mut changed = Vec::new();
1732 for relayer in relayers {
1733 let previous = self.published.get(&relayer).map_or(1.0, |health| *health);
1734 let current = match (judged.get(&relayer), unit) {
1735 _ if self.dead.contains(&relayer) => 0.0,
1736 _ if self.probation.contains(&relayer) => PROBATION_HEALTH,
1737 (Some(judgement), Some(unit)) if memory > 0 => {
1738 if previous > 0.0 && judgement.indistinct(unit).contains(&previous) {
1739 continue;
1740 }
1741 judgement.health(unit)
1742 }
1743 _ => 1.0,
1744 };
1745 if current == previous {
1746 continue;
1747 }
1748 if current < 1.0 {
1749 self.published.insert(relayer, current);
1750 } else {
1751 self.published.remove(&relayer);
1752 }
1753 changed.push((relayer, previous, current));
1754 }
1755 changed
1756 }
1757
1758 fn judge(&self, relayer: &OffchainPublicKey, memory: u32) -> Option<Judgement> {
1761 let mut history = self.relayers.get_mut(relayer)?;
1762 let remembered = history.memory;
1763 let recent = |race: &Race| remembered.is_zero() || race.at.elapsed() <= remembered;
1764 let still_out = history.open.values().filter(|race| recent(race)).map(|race| Race {
1765 margin: race.at.elapsed().as_secs_f64(),
1766 ..*race
1767 });
1768 let races: Vec<Race> = history
1769 .races
1770 .iter()
1771 .rev()
1772 .enumerate()
1773 .take_while(|(i, race)| *i < MIN_RACES || recent(race))
1774 .map(|(_, race)| *race)
1775 .chain(still_out)
1776 .collect();
1777 if races.len() < MIN_RACES {
1778 return None;
1779 }
1780 let mut margins: Vec<f64> = races.iter().map(|race| race.margin).collect();
1781 let margin = median(&mut margins);
1782 let spread = median(&mut margins.iter().map(|m| (m - margin).abs()).collect::<Vec<_>>());
1783 history.memory = Duration::from_secs_f64(f64::from(memory) * (margin.abs() + spread));
1784 let memory = history.memory;
1785 history.forget_older_than(memory);
1786
1787 let fastest = races
1788 .iter()
1789 .map(|race| race.winning_round_trip)
1790 .fold(f64::INFINITY, f64::min);
1791 Some(Judgement {
1792 margin,
1793 uncertainty: spread / (races.len() as f64).sqrt(),
1794 fastest,
1795 })
1796 }
1797}
1798
1799fn median(values: &mut [f64]) -> f64 {
1801 values.sort_unstable_by(f64::total_cmp);
1802 values[(values.len() - 1) / 2]
1803}
1804
1805pub fn path_slots_of<G>(graph: G) -> PathSlotResolver
1807where
1808 G: NetworkGraphView<NodeId = OffchainPublicKey> + Send + Sync + 'static,
1809{
1810 Arc::new(move |key| graph.path_slot(key))
1811}
1812
1813fn path_id(slots: &PathSlotResolver, nodes: impl IntoIterator<Item = OffchainPublicKey>) -> Option<PathId> {
1819 let mut id = [0u64; PATH_ID_SLOTS];
1820 let mut len = 0;
1821
1822 for node in nodes {
1823 if len == PATH_ID_SLOTS {
1824 return None;
1825 }
1826 id[len] = slots(&node)?;
1827 len += 1;
1828 }
1829
1830 (len > 1).then_some(id)
1831}
1832
1833fn round_trip_paths(
1839 slots: &PathSlotResolver,
1840 me: &OffchainPublicKey,
1841 forward: &[OffchainPublicKey],
1842 reply: &[OffchainPublicKey],
1843) -> Option<ForwardAndReturnPath> {
1844 let destination = *forward.last()?;
1845
1846 Some(ForwardAndReturnPath {
1847 forward: path_id(slots, std::iter::once(*me).chain(forward.iter().copied()))?,
1848 reply: path_id(slots, std::iter::once(destination).chain(reply.iter().copied()))?,
1849 })
1850}
1851
1852#[derive(Debug, Clone)]
1857pub enum NoPeerTelemetry {}
1858
1859impl MeasurablePeer for NoPeerTelemetry {
1860 fn peer(&self) -> &OffchainPublicKey {
1861 match *self {}
1862 }
1863
1864 fn rtt(&self) -> Duration {
1865 match *self {}
1866 }
1867}
1868
1869#[derive(Debug, Clone)]
1871pub enum NoPathTelemetry {}
1872
1873impl MeasurablePath for NoPathTelemetry {
1874 fn id(&self) -> &[u8] {
1875 match *self {}
1876 }
1877
1878 fn path(&self) -> &[u8] {
1879 match *self {}
1880 }
1881
1882 fn timestamp(&self) -> u128 {
1883 match *self {}
1884 }
1885}
1886
1887pub fn flush_into<G>(registry: &SurbRoundTripRegistry, graph: &G, timestamp: u128)
1891where
1892 G: hopr_api::graph::NetworkGraphUpdate,
1893{
1894 let (mut legs, mut total_expected, mut total_observed) = (0usize, 0u64, 0u64);
1895
1896 for (paths, expected, observed) in registry.drain() {
1897 legs += 1;
1898 total_expected += expected;
1899 total_observed += observed;
1900 tracing::trace!(
1903 forward = ?paths.forward,
1904 reply = ?paths.reply,
1905 expected,
1906 observed,
1907 "surb round-trip pair"
1908 );
1909 graph.record_edge::<NoPeerTelemetry, NoPathTelemetry>(MeasurableEdge::Surb(SurbTelemetry {
1910 paths,
1911 timestamp,
1912 expected,
1913 observed,
1914 }));
1915 }
1916
1917 if legs == 0 {
1918 tracing::trace!("surb round-trip flush tick: nothing to flush");
1920 return;
1921 }
1922
1923 {
1924 tracing::info!(
1926 legs,
1927 expected = total_expected,
1928 observed = total_observed,
1929 "surb round-trip flush tick"
1930 );
1931 }
1932}
1933
1934pub async fn refresh_relayer_health<G, F>(
1941 registry: &SurbRoundTripRegistry,
1942 graph: &G,
1943 memory: u32,
1944 weight: f64,
1945 replan: impl FnOnce() -> F,
1946) -> Vec<HoprPseudonym>
1947where
1948 G: hopr_api::graph::NetworkGraphUpdate,
1949 F: std::future::Future<Output = ()>,
1950{
1951 let health = ®istry.relayer_health;
1952 let changed = health.refresh(memory);
1953 for (relayer, previous, current) in &changed {
1955 tracing::debug!(%relayer, previous, current, "relayer health changed");
1956 }
1957 let healths: Vec<(OffchainPublicKey, f64)> = health
1960 .published
1961 .iter()
1962 .map(|entry| (*entry.key(), *entry.value()))
1963 .chain(
1964 changed
1965 .iter()
1966 .filter(|(.., current)| *current >= 1.0)
1967 .map(|(relayer, _, current)| (*relayer, *current)),
1968 )
1969 .map(|(relayer, health)| (relayer, health.powf(weight)))
1970 .collect();
1971 if !healths.is_empty() {
1972 graph.record_nodes_health(&healths);
1973 }
1974 if health.take_neighbour_change() || !changed.is_empty() {
1975 replan().await;
1976 }
1977 if weight == 0.0 {
1980 return Vec::new();
1981 }
1982 registry.age_write_offs();
1988 if !health.any_known_dead() {
1989 return Vec::new();
1990 }
1991 let registry = registry.clone();
1992 hopr_utils::parallelize::cpu::spawn_blocking(
1993 move || registry.write_off(registry.dead_stock()),
1994 "surb_write_off_dead",
1995 )
1996 .await
1997 .unwrap_or_else(|error| {
1998 tracing::warn!(%error, "SURB write-off deferred to the next flush");
1999 Vec::new()
2000 })
2001}
2002
2003pub fn invalid_updates(
2006 previous: &HashMap<HoprPseudonym, (u64, f64)>,
2007 current: &HashMap<HoprPseudonym, (u64, f64)>,
2008) -> Vec<(HoprPseudonym, (u64, f64))> {
2009 current
2010 .iter()
2011 .map(|(session, invalid)| (*session, *invalid))
2012 .chain(
2013 previous
2014 .keys()
2015 .filter(|session| !current.contains_key(session))
2016 .map(|session| (*session, (0, 0.0))),
2017 )
2018 .collect()
2019}
2020
2021#[derive(Clone)]
2026pub struct SurbTelemetryCodec<C> {
2027 inner: C,
2028 me: OffchainPublicKey,
2029 slots: PathSlotResolver,
2030 registry: SurbRoundTripRegistry,
2031}
2032
2033impl<C: std::fmt::Debug> std::fmt::Debug for SurbTelemetryCodec<C> {
2034 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
2035 f.debug_struct("SurbTelemetryCodec")
2037 .field("inner", &self.inner)
2038 .field("me", &self.me)
2039 .finish_non_exhaustive()
2040 }
2041}
2042
2043impl<C> SurbTelemetryCodec<C> {
2044 pub fn new(inner: C, me: OffchainPublicKey, slots: PathSlotResolver, registry: SurbRoundTripRegistry) -> Self {
2046 Self {
2047 inner,
2048 me,
2049 slots,
2050 registry,
2051 }
2052 }
2053
2054 fn on_minted(&self, routing: &ResolvedTransportRouting<HoprSurb>, minted: &[HoprSurbId]) {
2059 let ResolvedTransportRouting::Forward {
2060 pseudonym,
2061 forward_path,
2062 return_paths,
2063 } = routing
2064 else {
2065 return;
2066 };
2067
2068 let forward: Vec<_> = forward_path.transport_path().iter().copied().collect();
2069 let Some(destination) = forward.last().copied() else {
2070 return;
2071 };
2072
2073 tracing::trace!(
2074 minted = minted.len(),
2075 return_paths = return_paths.len(),
2076 forward_hops = forward.len(),
2077 "surb mint seen by telemetry"
2078 );
2079
2080 let replies: Vec<Vec<OffchainPublicKey>> = return_paths
2081 .iter()
2082 .map(|return_path| return_path.transport_path().iter().copied().collect())
2083 .collect();
2084 let return_relayers: Vec<ReturnRelayers> = replies
2087 .iter()
2088 .map(|reply| {
2089 let mut relayers = [None; PATH_ID_SLOTS];
2090 for (slot, relayer) in relayers.iter_mut().zip(&reply[..reply.len().saturating_sub(1)]) {
2091 *slot = Some(*relayer);
2092 }
2093 relayers
2094 })
2095 .collect();
2096 let partners = race_partners(minted, &return_relayers);
2097 let minted_at = Instant::now();
2098
2099 for (i, surb_id) in minted.iter().enumerate().take(replies.len()) {
2100 let Some(paths) = round_trip_paths(&self.slots, &self.me, &forward, &replies[i]) else {
2101 tracing::trace!(reply_hops = replies[i].len(), "surb round-trip legs did not resolve");
2102 continue;
2103 };
2104
2105 self.registry.record_minted(paths, 1, destination);
2106 self.registry.remember(
2107 *surb_id,
2108 PendingSurb {
2109 paths,
2110 pseudonym: *pseudonym,
2111 return_relayers: return_relayers[i],
2112 minted_at,
2113 partner: partners[i],
2114 pair_index: i % 2,
2115 partner_replied: None,
2116 },
2117 );
2118 }
2119 }
2120
2121 fn on_replied(&self, surb_id: &HoprSurbId) {
2123 let health = &self.registry.relayer_health;
2124 health.record_reply();
2127 let Some(pending) = self.registry.pending.remove(surb_id) else {
2129 if let Some((leg, minted_at)) = self.registry.written_off_legs.remove(surb_id) {
2135 health.note_home(&leg, minted_at);
2136 health.discard_race_if_fresh(&leg, surb_id, minted_at);
2144 health.revive_by_reply(&leg, minted_at);
2145 } else {
2146 tracing::trace!("reply on a surb with no pending legs");
2147 }
2148 return;
2149 };
2150 self.registry.record_observed(pending.paths, 1);
2151
2152 health.note_home(&pending.return_relayers, pending.minted_at);
2155 health.revive_by_reply(&pending.return_relayers, pending.minted_at);
2156 if let Some((partner_relayers, partner_home)) = pending.partner_replied {
2159 if !self.registry.judges(pending.pair_index) {
2160 return;
2161 }
2162 health.record_race(
2163 &pending.return_relayers,
2164 *surb_id,
2165 &partner_relayers,
2166 partner_home.duration_since(pending.minted_at),
2167 partner_home.elapsed(),
2168 );
2169 } else if let Some(partner) = pending.partner {
2170 let home = Instant::now();
2175 let waiting = self.registry.pending.update(&partner, |waiting| {
2176 waiting.partner_replied = Some((pending.return_relayers, home));
2177 *waiting
2178 });
2179 if let Some(waiting) = waiting {
2180 if !self.registry.judges(waiting.pair_index) {
2181 return;
2182 }
2183 self.registry.open_race(
2184 &waiting.return_relayers,
2185 partner,
2186 &pending.return_relayers,
2187 home.duration_since(pending.minted_at),
2188 );
2189 }
2190 }
2191 }
2192}
2193
2194fn race_partners(minted: &[HoprSurbId], return_relayers: &[ReturnRelayers]) -> Vec<Option<HoprSurbId>> {
2197 (0..minted.len())
2198 .map(|i| {
2199 let neighbour = i ^ 1;
2200 match (return_relayers.get(i), return_relayers.get(neighbour)) {
2201 (Some(own), Some(other)) if own[0].is_some() && other[0].is_some() && own != other => {
2202 minted.get(neighbour).copied()
2203 }
2204 _ => None,
2205 }
2206 })
2207 .collect()
2208}
2209
2210impl<C> PacketEncoder for SurbTelemetryCodec<C>
2211where
2212 C: PacketEncoder,
2213{
2214 type Error = C::Error;
2215
2216 fn encode_packet<T: AsRef<[u8]> + Send + 'static, S: Into<PacketSignals> + Send + 'static>(
2217 &self,
2218 data: T,
2219 routing: ResolvedTransportRouting<HoprSurb>,
2220 signals: S,
2221 generation: Option<u8>,
2222 ) -> Result<OutgoingPacket, <Self as PacketEncoder>::Error> {
2223 let packet = self.inner.encode_packet(data, routing.clone(), signals, generation)?;
2224 self.on_minted(&routing, &packet.minted_surbs);
2225 Ok(packet)
2226 }
2227
2228 fn encode_acknowledgements(
2229 &self,
2230 acks: &[VerifiedAcknowledgement],
2231 destination: &OffchainPublicKey,
2232 ) -> Result<OutgoingPacket, <Self as PacketEncoder>::Error> {
2233 self.inner.encode_acknowledgements(acks, destination)
2234 }
2235}
2236
2237impl<C> PacketDecoder for SurbTelemetryCodec<C>
2238where
2239 C: PacketDecoder,
2240{
2241 type Error = C::Error;
2242
2243 fn decode(
2245 &self,
2246 sender: PeerId,
2247 data: Bytes,
2248 ) -> Result<IncomingPacket, IncomingPacketError<<Self as PacketDecoder>::Error>> {
2249 let packet = self.inner.decode(sender, data)?;
2250
2251 if let IncomingPacket::Final(f) = &packet {
2252 tracing::trace!(on_surb = f.replied_on_surb.is_some(), "final packet decoded");
2253 }
2254
2255 if let IncomingPacket::Final(final_packet) = &packet
2256 && let Some(surb_id) = final_packet.replied_on_surb
2257 {
2258 self.on_replied(&surb_id);
2259 }
2260
2261 Ok(packet)
2262 }
2263}
2264
2265#[cfg(test)]
2266mod tests {
2267 use anyhow::Context;
2268 use hopr_api::{
2269 graph::{EdgeObservableRead, NetworkGraphWrite},
2270 types::{
2271 crypto::{
2272 crypto_traits::Randomizable,
2273 prelude::{ChainKeypair, Keypair, OffchainKeypair},
2274 },
2275 primitive::primitives::Address,
2276 },
2277 };
2278 use hopr_network_graph::petgraph::ChannelGraph;
2279
2280 use super::*;
2281
2282 fn recorder(graph: ChannelGraph) -> SurbTelemetryCodec<()> {
2284 let me = *graph.identity();
2285 SurbTelemetryCodec::new((), me, path_slots_of(graph), SurbRoundTripRegistry::default())
2286 }
2287
2288 fn surb_id(seed: u8) -> HoprSurbId {
2289 [seed; SURB_ID_SIZE]
2290 }
2291
2292 fn address() -> Address {
2294 ChainKeypair::random().public().to_address()
2295 }
2296
2297 fn graph_with(count: usize) -> (ChannelGraph, OffchainPublicKey, Vec<OffchainPublicKey>) {
2298 let me = *OffchainKeypair::random().public();
2299 let graph = ChannelGraph::new(me);
2300
2301 let peers = (0..count)
2302 .map(|_| {
2303 let key = *OffchainKeypair::random().public();
2304 graph.add_node(key);
2305 key
2306 })
2307 .collect();
2308
2309 (graph, me, peers)
2310 }
2311
2312 #[test]
2313 fn drain_should_report_nothing_before_anything_is_recorded() {
2314 assert!(SurbRoundTripRegistry::default().drain().is_empty());
2315 }
2316
2317 #[test]
2318 fn registry_debug_output_stays_bounded_without_consuming_telemetry() {
2319 let registry = SurbRoundTripRegistry::default().with_observer(Arc::new(|_| {}));
2320 assert_eq!(format!("{registry:?}"), "SurbRoundTripRegistry { pairs: 0, .. }");
2321
2322 let destination = key();
2323 for peer in 1..=64 {
2324 registry.record_minted(
2325 ForwardAndReturnPath {
2326 forward: [0, peer, 0, 0, 0],
2327 reply: [peer, 0, 0, 0, 0],
2328 },
2329 1,
2330 destination,
2331 );
2332 }
2333 assert_eq!(format!("{registry:?}"), "SurbRoundTripRegistry { pairs: 64, .. }");
2335 let counts = registry.drain();
2336 assert_eq!(counts.len(), 64);
2337 assert!(
2338 counts
2339 .iter()
2340 .all(|(_, minted, returned)| (*minted, *returned) == (1, 0))
2341 );
2342 }
2343
2344 #[test]
2345 fn drain_should_take_the_counts_and_leave_the_entry_empty() {
2346 let registry = SurbRoundTripRegistry::default();
2347 let paths = ForwardAndReturnPath {
2348 forward: [0, 1, 0, 0, 0],
2349 reply: [1, 0, 0, 0, 0],
2350 };
2351
2352 registry.record_expected(paths, 3);
2353 registry.record_observed(paths, 2);
2354
2355 assert_eq!(vec![(paths, 3, 2)], registry.drain());
2356 assert!(registry.drain().is_empty());
2358 }
2359
2360 fn jitter(i: u64) -> Duration {
2362 Duration::from_millis(5 + i * 7 % 40)
2363 }
2364
2365 const ROUND_TRIP: Duration = Duration::from_millis(200);
2367
2368 fn key() -> OffchainPublicKey {
2369 *OffchainKeypair::random().public()
2370 }
2371
2372 fn leg(relayers: &[OffchainPublicKey]) -> ReturnRelayers {
2373 let mut leg = [None; PATH_ID_SLOTS];
2374 for (slot, relayer) in leg.iter_mut().zip(relayers) {
2375 *slot = Some(*relayer);
2376 }
2377 leg
2378 }
2379
2380 fn margins_of(health: &RelayerHealth, relayer: &OffchainPublicKey) -> Vec<f64> {
2381 health
2382 .relayers
2383 .get(relayer)
2384 .map(|history| history.races.iter().map(|race| race.margin).collect())
2385 .unwrap_or_default()
2386 }
2387
2388 fn open_races_of(health: &RelayerHealth, relayer: &OffchainPublicKey) -> usize {
2389 health.relayers.get(relayer).map_or(0, |history| history.open.len())
2390 }
2391
2392 fn ago(by: Duration) -> anyhow::Result<Instant> {
2393 Instant::now().checked_sub(by).context("monotonic clock too young")
2394 }
2395
2396 fn settled_since(health: &RelayerHealth, relayer: OffchainPublicKey, at: Instant, margin: Duration) {
2397 for _ in 0..MIN_RACES {
2398 health.record(
2399 relayer,
2400 Race {
2401 at,
2402 margin: margin.as_secs_f64(),
2403 winning_round_trip: ROUND_TRIP.as_secs_f64(),
2404 },
2405 );
2406 }
2407 }
2408
2409 fn backdate_open(health: &RelayerHealth, relayer: OffchainPublicKey, at: Instant) {
2410 if let Some(mut history) = health.relayers.get_mut(&relayer) {
2411 history.open.values_mut().for_each(|race| race.at = at);
2412 }
2413 }
2414
2415 fn health_after_flush(health: &RelayerHealth, relayers: &[OffchainPublicKey]) -> Vec<f64> {
2417 health.refresh(4);
2418 relayers
2419 .iter()
2420 .map(|relayer| health.published.get(relayer).map_or(1.0, |health| *health))
2421 .collect()
2422 }
2423
2424 fn let_suspicions_stand(health: &RelayerHealth) {
2426 for mut suspicion in health.suspected.iter_mut() {
2427 suspicion.value_mut().1 = Instant::now()
2428 .checked_sub(VERDICT_CONFIRMATION + Duration::from_millis(1))
2429 .expect("monotonic clock too young");
2430 }
2431 }
2432
2433 fn judged_dead_by_the_second_flush(health: &RelayerHealth) -> Vec<OffchainPublicKey> {
2436 let mut judged = judged_dead(&health.refresh(4));
2437 let_suspicions_stand(health);
2438 judged.extend(judged_dead(&health.refresh(4)));
2439 judged
2440 }
2441
2442 fn judged_dead(changes: &[(OffchainPublicKey, f64, f64)]) -> Vec<OffchainPublicKey> {
2443 changes
2444 .iter()
2445 .filter(|(_, previous, current)| *current == 0.0 && *previous > 0.0)
2446 .map(|(relayer, ..)| *relayer)
2447 .collect()
2448 }
2449
2450 fn pending_to(relayers: &[OffchainPublicKey], partner: Option<HoprSurbId>, pair_index: usize) -> PendingSurb {
2451 PendingSurb {
2452 paths: ForwardAndReturnPath {
2453 forward: [0; PATH_ID_SLOTS],
2454 reply: [0; PATH_ID_SLOTS],
2455 },
2456 pseudonym: *SESSION,
2457 return_relayers: leg(relayers),
2458 minted_at: Instant::now(),
2459 partner,
2460 pair_index,
2461 partner_replied: None,
2462 }
2463 }
2464
2465 static SESSION: std::sync::LazyLock<HoprPseudonym> = std::sync::LazyLock::new(HoprPseudonym::random);
2466
2467 fn judge_dead(health: &RelayerHealth, relayer: OffchainPublicKey) {
2470 health.mark_dead(relayer);
2473 health.dead_since.insert(
2474 relayer,
2475 Instant::now()
2476 .checked_sub(VERDICT_CONFIRMATION + Duration::from_millis(1))
2477 .expect("monotonic clock too young"),
2478 );
2479 health.confirmed.insert(relayer);
2480 }
2481
2482 fn confirm_verdicts(health: &RelayerHealth) {
2485 for mut since in health.dead_since.iter_mut() {
2486 *since.value_mut() = Instant::now()
2487 .checked_sub(VERDICT_CONFIRMATION + Duration::from_millis(1))
2488 .expect("monotonic clock too young");
2489 }
2490 for relayer in health.dead.iter() {
2491 health.confirmed.insert(*relayer);
2492 }
2493 }
2494
2495 fn open_since(health: &RelayerHealth, relayer: OffchainPublicKey, count: u8, since: Instant) {
2497 for seed in 1..=count {
2498 let mut history = health.relayers.entry(relayer).or_default();
2499 history.open.insert(
2500 surb_id(seed),
2501 Race {
2502 at: since,
2503 margin: 0.0,
2504 winning_round_trip: ROUND_TRIP.as_secs_f64(),
2505 },
2506 );
2507 history.open_legs.insert(surb_id(seed), (leg(&[relayer]), 0));
2508 }
2509 }
2510
2511 #[test]
2512 fn a_relayer_losing_its_races_by_seconds_should_lose_its_weight() {
2513 let health = RelayerHealth::default();
2514 let [fast, also_fast, stalled] = [(); 3].map(|_| key());
2515 for i in 0..40 {
2516 let (winner, loser) = if i % 2 == 0 {
2517 (fast, also_fast)
2518 } else {
2519 (also_fast, fast)
2520 };
2521 health.record_race(&leg(&[loser]), surb_id(0), &leg(&[winner]), ROUND_TRIP, jitter(i));
2522 health.record_race(
2523 &leg(&[stalled]),
2524 surb_id(0),
2525 &leg(&[winner]),
2526 ROUND_TRIP,
2527 Duration::from_secs(3) + jitter(i),
2528 );
2529 }
2530
2531 let factors = health_after_flush(&health, &[fast, also_fast, stalled]);
2532 assert_eq!(
2533 &[1.0, 1.0],
2534 &factors[..2],
2535 "trading wins by milliseconds is not a stall"
2536 );
2537 assert!(
2538 factors[2] < 0.1,
2539 "a relayer fifteen round-trips late must lose its weight, got {}",
2540 factors[2]
2541 );
2542 }
2543
2544 #[test]
2546 fn a_stalled_middle_relayer_should_take_the_blame_not_its_neighbours() {
2547 let health = RelayerHealth::default();
2548 let [entry, stalled, healthy, exit, other_entry, other_exit] = [(); 6].map(|_| key());
2549 for i in 0..40 {
2550 if i % 2 == 0 {
2551 health.record_race(
2552 &leg(&[entry, stalled, exit]),
2553 surb_id(0),
2554 &leg(&[entry, healthy, exit]),
2555 ROUND_TRIP,
2556 Duration::from_secs(3) + jitter(i),
2557 );
2558 } else {
2559 health.record_race(
2560 &leg(&[other_entry, healthy, other_exit]),
2561 surb_id(0),
2562 &leg(&[entry, healthy, exit]),
2563 ROUND_TRIP,
2564 jitter(i),
2565 );
2566 }
2567 }
2568
2569 let factors = health_after_flush(&health, &[stalled, entry, healthy, exit, other_entry, other_exit]);
2570 assert!(
2571 factors[0] < 0.1,
2572 "the stalled relayer must lose its weight, got {}",
2573 factors[0]
2574 );
2575 assert_eq!(&[1.0; 5], &factors[1..], "its neighbours on that path are not to blame");
2576 }
2577
2578 #[test]
2579 fn a_relayer_lagging_by_less_than_a_round_trip_should_keep_its_weight() {
2580 let health = RelayerHealth::default();
2581 let [near, far] = [(); 2].map(|_| key());
2582 for i in 0..40 {
2583 health.record_race(
2584 &leg(&[far]),
2585 surb_id(0),
2586 &leg(&[near]),
2587 ROUND_TRIP,
2588 Duration::from_millis(150) + jitter(i) / 10,
2589 );
2590 }
2591
2592 assert_eq!(vec![1.0, 1.0], health_after_flush(&health, &[near, far]));
2593 }
2594
2595 #[test]
2596 fn a_relayer_with_too_few_races_should_stay_neutral() {
2597 let health = RelayerHealth::default();
2598 let [fast, young] = [(); 2].map(|_| key());
2599 for _ in 1..MIN_RACES {
2600 health.record_race(
2601 &leg(&[young]),
2602 surb_id(0),
2603 &leg(&[fast]),
2604 ROUND_TRIP,
2605 Duration::from_secs(3),
2606 );
2607 }
2608
2609 assert_eq!(vec![1.0, 1.0], health_after_flush(&health, &[fast, young]));
2610 }
2611
2612 #[test]
2613 fn a_recovered_relayer_should_regain_its_weight_once_its_late_races_are_forgotten() -> anyhow::Result<()> {
2614 let health = RelayerHealth::default();
2615 let [fast, recovered] = [(); 2].map(|_| key());
2616 let long_ago = Instant::now()
2617 .checked_sub(Duration::from_secs(60))
2618 .context("monotonic clock younger than a minute")?;
2619 for i in 0..20 {
2620 health.record(
2621 recovered,
2622 Race {
2623 at: long_ago,
2624 margin: 3.0 + jitter(i).as_secs_f64(),
2625 winning_round_trip: ROUND_TRIP.as_secs_f64(),
2626 },
2627 );
2628 }
2629 assert!(
2630 health_after_flush(&health, &[fast, recovered])[1] < 0.1,
2631 "precondition: judged stalled"
2632 );
2633
2634 for i in 0..20 {
2635 let (later, earlier) = if i % 2 == 0 {
2636 (fast, recovered)
2637 } else {
2638 (recovered, fast)
2639 };
2640 health.record_race(&leg(&[later]), surb_id(0), &leg(&[earlier]), ROUND_TRIP, jitter(i));
2641 }
2642 assert_eq!(vec![1.0, 1.0], health_after_flush(&health, &[fast, recovered]));
2643 Ok(())
2644 }
2645
2646 #[test]
2649 fn a_reply_still_out_should_count_as_lost_by_at_least_its_wait() -> anyhow::Result<()> {
2650 let health = RelayerHealth::default();
2651 let [fast, also_fast, stalled] = [(); 3].map(|_| key());
2652 for i in 0..20 {
2653 health.record_race(&leg(&[also_fast]), surb_id(0), &leg(&[fast]), ROUND_TRIP, jitter(i));
2654 }
2655 health.record_race(&leg(&[stalled]), surb_id(0), &leg(&[fast]), ROUND_TRIP, jitter(0));
2656 let partners_home = Instant::now()
2657 .checked_sub(Duration::from_secs(2))
2658 .context("monotonic clock younger than two seconds")?;
2659 open_since(&health, stalled, 5, partners_home);
2660
2661 let factors = health_after_flush(&health, &[fast, also_fast, stalled]);
2662 assert_eq!(&[1.0, 1.0], &factors[..2]);
2663 assert!(
2664 factors[2] < 0.2,
2665 "replies two seconds overdue must cost the relayer its weight, got {}",
2666 factors[2]
2667 );
2668 Ok(())
2669 }
2670
2671 #[test]
2672 fn a_relayer_whose_replies_stop_coming_home_should_be_judged_dead_everywhere() -> anyhow::Result<()> {
2673 let health = RelayerHealth::default();
2674 let [entry, black_hole, exit, healthy] = [(); 4].map(|_| key());
2675 settled_since(&health, black_hole, ago(Duration::from_secs(10))?, jitter(0));
2676 for seed in 1..=MIN_RACES as u8 {
2677 health.open_race(
2678 &leg(&[entry, black_hole, exit]),
2679 surb_id(seed),
2680 &leg(&[entry, healthy, exit]),
2681 ROUND_TRIP,
2682 );
2683 }
2684 backdate_open(&health, black_hole, ago(ROUND_TRIP * 2)?);
2685 health.record_reply();
2687
2688 assert_eq!(vec![black_hole], judged_dead(&health.refresh(4)));
2689 assert!(health.is_dead(&black_hole));
2690 assert_eq!(
2691 0,
2692 open_races_of(&health, &entry),
2693 "relayers on both legs are not to blame"
2694 );
2695 assert!(
2696 judged_dead(&health.refresh(4)).is_empty(),
2697 "only new verdicts are reported"
2698 );
2699 Ok(())
2700 }
2701
2702 fn overdue_legs(health: &RelayerHealth, legs: &[&[OffchainPublicKey]], first_seed: u8, healthy: OffchainPublicKey) {
2704 for (i, relayers) in legs.iter().enumerate() {
2705 health.open_race(
2706 &leg(relayers),
2707 surb_id(first_seed + i as u8),
2708 &leg(&[healthy]),
2709 ROUND_TRIP,
2710 );
2711 }
2712 }
2713
2714 fn backdate_all_open(health: &RelayerHealth, at: Instant) {
2715 for mut history in health.relayers.iter_mut() {
2716 history.open.values_mut().for_each(|race| race.at = at);
2717 }
2718 }
2719
2720 fn sorted(mut relayers: Vec<OffchainPublicKey>) -> Vec<OffchainPublicKey> {
2721 relayers.sort_by_key(|relayer| relayer.to_string());
2722 relayers
2723 }
2724
2725 #[test]
2728 fn a_relayer_on_the_legs_of_a_dead_one_should_not_be_judged_dead_with_it() -> anyhow::Result<()> {
2729 let health = RelayerHealth::default();
2730 let [dead, innocent, other, healthy] = [(); 4].map(|_| key());
2731 let [l1, l2, l3, l4] = [(); 4].map(|_| key());
2732 settled_since(&health, dead, ago(Duration::from_secs(10))?, jitter(0));
2733 overdue_legs(
2734 &health,
2735 &[
2736 &[innocent, dead, l1],
2737 &[innocent, dead, l2],
2738 &[innocent, dead, l3],
2739 &[other, dead, l4],
2740 ],
2741 1,
2742 healthy,
2743 );
2744 backdate_all_open(&health, ago(ROUND_TRIP * 2)?);
2745 health.record_reply();
2746
2747 assert_eq!(vec![dead], judged_dead(&health.refresh(4)));
2748 assert!(
2749 !health.is_dead(&innocent),
2750 "three overdue replies, all on legs through the dead one"
2751 );
2752 Ok(())
2753 }
2754
2755 #[test]
2760 fn a_relayer_cut_off_from_the_replier_should_be_judged_not_the_relayer_behind_it() -> anyhow::Result<()> {
2761 let health = RelayerHealth::default();
2762 let [cut, behind, ahead, healthy] = [(); 4].map(|_| key());
2763 settled_since(&health, cut, ago(Duration::from_secs(10))?, jitter(0));
2764 let cut_off: &[OffchainPublicKey] = &[cut, behind];
2765 overdue_legs(&health, &[cut_off; MIN_RACES_OVER_ONE_HOP], 1, healthy);
2766 backdate_all_open(&health, ago(ROUND_TRIP * 2)?);
2767 health.record_reply();
2768 health.note_home(&leg(&[ahead, cut]), Instant::now());
2770
2771 assert!(
2772 judged_dead(&health.refresh(4)).is_empty(),
2773 "delivering over its other hop, it is only suspected at the first flush"
2774 );
2775 assert!(
2776 health.published.iter().all(|health| *health > 0.0),
2777 "and the planner is not told to steer around it"
2778 );
2779 let_suspicions_stand(&health);
2780 assert_eq!(vec![cut], judged_dead(&health.refresh(4)));
2781 assert!(
2782 health.is_confirmed_dead(&cut),
2783 "the second flush is the second look: its stock is written off now"
2784 );
2785 assert!(
2786 !health.is_dead(&behind),
2787 "its overdue replies are all on legs through the one cut off"
2788 );
2789
2790 health.revive_by_reply(&leg(&[ahead, cut]), Instant::now());
2793 assert!(health.is_dead(&cut), "a reply over a hop that always worked is no news");
2794 health.revive_by_reply(&leg(&[cut, behind]), Instant::now());
2795 assert!(
2796 !health.is_dead(&cut),
2797 "a reply over the hop the verdict is about lifts it"
2798 );
2799 Ok(())
2800 }
2801
2802 #[test]
2806 fn a_relayer_on_both_legs_over_different_hops_should_still_race() {
2807 let health = RelayerHealth::default();
2808 let [cut, behind, ahead] = [(); 3].map(|_| key());
2809
2810 health.open_race(&leg(&[cut, behind]), surb_id(1), &leg(&[ahead, cut]), ROUND_TRIP);
2811 assert_eq!(
2812 1,
2813 open_races_of(&health, &cut),
2814 "out as the first relayer, home as the second"
2815 );
2816 assert_eq!(1, open_races_of(&health, &behind));
2817
2818 health.open_race(&leg(&[cut, behind]), surb_id(2), &leg(&[cut, ahead]), ROUND_TRIP);
2819 assert_eq!(
2820 1,
2821 open_races_of(&health, &cut),
2822 "first on both legs: the same hop, it cancels out"
2823 );
2824 assert_eq!(2, open_races_of(&health, &behind));
2825
2826 let [relayer, one, another] = [(); 3].map(|_| key());
2827 health.open_race(&leg(&[one, relayer]), surb_id(3), &leg(&[another, relayer]), ROUND_TRIP);
2828 assert_eq!(
2829 0,
2830 open_races_of(&health, &relayer),
2831 "behind another relayer on both legs: the same hop, whichever relayer that is"
2832 );
2833 assert_eq!(1, open_races_of(&health, &one));
2834 }
2835
2836 #[test]
2839 fn a_relayer_suspected_over_one_hop_should_be_cleared_by_a_reply_over_it_before_the_next_flush()
2840 -> anyhow::Result<()> {
2841 let health = RelayerHealth::default();
2842 let [relayer, behind, other, healthy] = [(); 4].map(|_| key());
2843 settled_since(&health, relayer, ago(Duration::from_secs(10))?, jitter(0));
2844 settled_since(&health, behind, ago(Duration::from_secs(10))?, jitter(0));
2845 let late: &[OffchainPublicKey] = &[relayer, behind];
2846 overdue_legs(&health, &[late; MIN_RACES_OVER_ONE_HOP], 1, healthy);
2847 backdate_all_open(&health, ago(ROUND_TRIP * 2)?);
2848 health.record_reply();
2849 health.note_home(&leg(&[other, relayer]), Instant::now());
2851 health.note_home(&leg(&[other, behind]), Instant::now());
2852
2853 assert!(judged_dead(&health.refresh(4)).is_empty());
2854 assert!(
2855 health.suspected.contains_key(&relayer),
2856 "suspected over the hop from the replier"
2857 );
2858 assert!(
2859 health.published.iter().all(|health| *health > 0.0),
2860 "no relayer is published as dead for a suspicion"
2861 );
2862
2863 health.note_home(&leg(&[relayer, behind]), Instant::now());
2865 let_suspicions_stand(&health);
2866 assert!(judged_dead(&health.refresh(4)).is_empty());
2867 assert!(!health.is_dead(&relayer));
2868 assert!(health.suspected.is_empty(), "a suspicion that did not stand is dropped");
2869 Ok(())
2870 }
2871
2872 #[test]
2874 fn a_reply_older_than_any_open_race_should_be_forgotten() -> anyhow::Result<()> {
2875 let health = RelayerHealth::default();
2876 let [relayer, ahead] = [(); 2].map(|_| key());
2877 settled_since(&health, relayer, ago(Duration::from_secs(10))?, jitter(0));
2878 health.note_home(&leg(&[ahead, relayer]), Instant::now());
2879 health
2880 .relayers
2881 .get_mut(&relayer)
2882 .expect("has races")
2883 .last_home
2884 .insert(Hop::FromReplier, ago(PENDING_SURB_TTL + Duration::from_secs(1))?);
2885
2886 health.refresh(4);
2887
2888 let history = health.relayers.get(&relayer).expect("still racing");
2889 assert_eq!(
2890 vec![Hop::FromRelayer],
2891 history.last_home.keys().copied().collect::<Vec<_>>(),
2892 "the hop used a minute ago is forgotten, the one used now is kept"
2893 );
2894 Ok(())
2895 }
2896
2897 #[test]
2901 fn a_reply_over_the_suspected_hop_should_start_the_suspicion_over() -> anyhow::Result<()> {
2902 let health = RelayerHealth::default();
2903 let [relayer, behind, other, healthy] = [(); 4].map(|_| key());
2904 settled_since(&health, relayer, ago(Duration::from_secs(20))?, jitter(0));
2905 settled_since(&health, behind, ago(Duration::from_secs(20))?, jitter(0));
2906 let late: &[OffchainPublicKey] = &[relayer, behind];
2907 overdue_legs(&health, &[late; MIN_RACES_OVER_ONE_HOP], 1, healthy);
2908 backdate_all_open(&health, ago(ROUND_TRIP * 2)?);
2909 health.record_reply();
2910 health.note_home(&leg(&[other, relayer]), Instant::now());
2911 health.note_home(&leg(&[other, behind]), Instant::now());
2912 health
2915 .suspected
2916 .insert(relayer, (Hop::FromReplier, ago(Duration::from_secs(10))?));
2917 health
2918 .relayers
2919 .get_mut(&relayer)
2920 .expect("has races")
2921 .last_home
2922 .insert(Hop::FromReplier, ago(Duration::from_secs(8))?);
2923
2924 assert!(
2925 judged_dead(&health.refresh(4)).is_empty(),
2926 "ten seconds, but not ten seconds of silence over the hop"
2927 );
2928 assert!(health.suspected.contains_key(&relayer), "suspected anew");
2929
2930 let_suspicions_stand(&health);
2931 assert_eq!(vec![relayer], judged_dead(&health.refresh(4)));
2932 Ok(())
2933 }
2934
2935 #[test]
2938 fn a_suspicion_over_the_other_hop_should_have_a_flush_of_its_own_to_stand() -> anyhow::Result<()> {
2939 let health = RelayerHealth::default();
2940 let [relayer, behind, other, healthy] = [(); 4].map(|_| key());
2941 settled_since(&health, relayer, ago(Duration::from_secs(20))?, jitter(0));
2942 settled_since(&health, behind, ago(Duration::from_secs(20))?, jitter(0));
2943 settled_since(&health, other, ago(Duration::from_secs(20))?, jitter(0));
2944 let first: &[OffchainPublicKey] = &[relayer, behind];
2946 overdue_legs(&health, &[first; MIN_RACES_OVER_ONE_HOP], 1, healthy);
2947 backdate_all_open(&health, ago(ROUND_TRIP * 2)?);
2948 health.record_reply();
2949 health.note_home(&leg(&[other, relayer]), Instant::now());
2950 health.note_home(&leg(&[other, behind]), Instant::now());
2951 assert!(judged_dead(&health.refresh(4)).is_empty());
2952 assert_eq!(Some(Hop::FromReplier), health.suspected.get(&relayer).map(|s| s.0));
2953
2954 health.note_home(&leg(&[relayer, behind]), Instant::now());
2957 let second: &[OffchainPublicKey] = &[other, relayer];
2958 overdue_legs(&health, &[second; MIN_RACES_OVER_ONE_HOP], 20, healthy);
2959 backdate_all_open(&health, ago(ROUND_TRIP * 2)?);
2960 health
2961 .relayers
2962 .get_mut(&relayer)
2963 .expect("has races")
2964 .last_home
2965 .insert(Hop::FromRelayer, ago(Duration::from_secs(30))?);
2966
2967 let_suspicions_stand(&health);
2968 assert!(
2969 judged_dead(&health.refresh(4)).is_empty(),
2970 "the first suspicion stood, but it was over the other hop"
2971 );
2972 assert_eq!(Some(Hop::FromRelayer), health.suspected.get(&relayer).map(|s| s.0));
2973
2974 let_suspicions_stand(&health);
2975 assert_eq!(vec![relayer], judged_dead(&health.refresh(4)));
2976 assert_eq!(Some(Hop::FromRelayer), health.dead_over.get(&relayer).map(|hop| *hop));
2977 Ok(())
2978 }
2979
2980 #[test]
2984 fn an_overdue_reply_should_be_answered_only_by_a_reply_over_the_same_hop() -> anyhow::Result<()> {
2985 let [first, second, other, healthy] = [(); 4].map(|_| key());
2986 let judged_after = |answers: &[&[OffchainPublicKey]]| -> anyhow::Result<Vec<OffchainPublicKey>> {
2987 let health = RelayerHealth::default();
2988 settled_since(&health, first, ago(Duration::from_secs(10))?, jitter(0));
2989 settled_since(&health, second, ago(Duration::from_secs(10))?, jitter(0));
2990 let overdue: &[OffchainPublicKey] = &[first, second];
2991 overdue_legs(&health, &[overdue; MIN_RACES_OVER_ONE_HOP], 1, healthy);
2992 backdate_all_open(&health, ago(ROUND_TRIP * 2)?);
2993 health.record_reply();
2994 for answer in answers {
2995 health.note_home(&leg(answer), Instant::now());
2996 }
2997 Ok(judged_dead_by_the_second_flush(&health))
2998 };
2999
3000 assert!(
3001 judged_after(&[&[first, second]])?.is_empty(),
3002 "a reply over the same leg answers both relayers"
3003 );
3004 assert!(
3005 judged_after(&[&[first, other], &[other, second]])?.is_empty(),
3006 "one from the replier, the other from another relayer, as on the overdue legs: both are answered"
3007 );
3008 assert_eq!(
3009 vec![first],
3010 judged_after(&[&[other, first], &[other, second]])?,
3011 "the first one delivers behind another relayer only: what the replier hands it is what is lost"
3012 );
3013 assert_eq!(
3014 vec![second],
3015 judged_after(&[&[first, other], &[second, other]])?,
3016 "the second one delivers as a first relayer only: what a relayer hands it is what is lost"
3017 );
3018 assert_eq!(
3019 vec![first],
3020 judged_after(&[&[other, second]])?,
3021 "with nothing home through the first one at all, it explains the replies overdue behind it"
3022 );
3023 Ok(())
3024 }
3025
3026 #[test]
3029 fn a_relayer_overdue_over_both_hops_should_be_judged_as_a_whole() -> anyhow::Result<()> {
3030 let health = RelayerHealth::default();
3031 let [relayer, ahead, behind, healthy] = [(); 4].map(|_| key());
3032 settled_since(
3033 &health,
3034 relayer,
3035 ago(Duration::from_secs(60))?,
3036 Duration::from_millis(500),
3037 );
3038 let from_the_replier: &[OffchainPublicKey] = &[relayer, behind];
3041 overdue_legs(&health, &[from_the_replier; MIN_RACES_OVER_ONE_HOP - 1], 1, healthy);
3042 backdate_all_open(&health, ago(Duration::from_secs(3))?);
3043 let from_a_relayer = MIN_RACES_OVER_ONE_HOP as u8;
3044 overdue_legs(&health, &[&[ahead, relayer]], from_a_relayer, healthy);
3045 for mut history in health.relayers.iter_mut() {
3046 if let Some(race) = history.open.get_mut(&surb_id(from_a_relayer)) {
3047 race.at = ago(Duration::from_secs(2))?;
3048 }
3049 }
3050 health.record_reply();
3051 if let Some(mut history) = health.relayers.get_mut(&relayer) {
3053 history
3054 .last_home
3055 .insert(Hop::FromReplier, ago(Duration::from_millis(2800))?);
3056 history
3057 .last_home
3058 .insert(Hop::FromRelayer, ago(Duration::from_millis(2900))?);
3059 }
3060 health.note_home(&leg(&[ahead, healthy]), Instant::now());
3062 health.note_home(&leg(&[healthy, behind]), Instant::now());
3063
3064 assert_eq!(vec![relayer], judged_dead(&health.refresh(4)));
3065 assert!(health.suspected.is_empty(), "judged, not suspected over one hop");
3066 assert!(
3067 !health.dead_over.contains_key(&relayer),
3068 "any reply through it lifts the verdict"
3069 );
3070 Ok(())
3071 }
3072
3073 #[test]
3077 fn overdue_replies_over_both_hops_should_be_answered_by_a_reply_over_either() -> anyhow::Result<()> {
3078 let health = RelayerHealth::default();
3079 let [relayer, ahead, behind, healthy] = [(); 4].map(|_| key());
3080 settled_since(
3081 &health,
3082 relayer,
3083 ago(Duration::from_secs(60))?,
3084 Duration::from_millis(500),
3085 );
3086 overdue_legs(&health, &[&[relayer, behind], &[relayer, behind]], 1, healthy);
3088 backdate_all_open(&health, ago(Duration::from_secs(10))?);
3089 overdue_legs(&health, &[&[ahead, relayer]], 3, healthy);
3090 for mut history in health.relayers.iter_mut() {
3091 if let Some(race) = history.open.get_mut(&surb_id(3)) {
3092 race.at = ago(Duration::from_secs(2))?;
3093 }
3094 }
3095 health.record_reply();
3096 if let Some(mut history) = health.relayers.get_mut(&relayer) {
3098 history.last_home.insert(Hop::FromRelayer, ago(Duration::from_secs(5))?);
3099 }
3100 health.note_home(&leg(&[ahead, healthy]), Instant::now());
3102 health.note_home(&leg(&[healthy, behind]), Instant::now());
3103
3104 assert!(
3105 judged_dead_by_the_second_flush(&health).is_empty(),
3106 "one reply unanswered, not three"
3107 );
3108 assert!(health.suspected.is_empty());
3109 Ok(())
3110 }
3111
3112 #[test]
3116 fn a_relayer_delivering_over_its_other_hop_should_take_more_overdue_replies_to_be_suspected() -> anyhow::Result<()>
3117 {
3118 let [relayer, behind, other, healthy] = [(); 4].map(|_| key());
3119 let judged_with = |overdue: usize, delivers_behind_another: bool| -> anyhow::Result<Vec<OffchainPublicKey>> {
3120 let health = RelayerHealth::default();
3121 settled_since(&health, relayer, ago(Duration::from_secs(10))?, jitter(0));
3122 settled_since(&health, behind, ago(Duration::from_secs(10))?, jitter(0));
3123 let late: &[OffchainPublicKey] = &[relayer, behind];
3124 overdue_legs(&health, &vec![late; overdue], 1, healthy);
3125 backdate_all_open(&health, ago(ROUND_TRIP * 2)?);
3126 health.record_reply();
3127 health.note_home(&leg(&[other, behind]), Instant::now());
3128 if delivers_behind_another {
3129 health.note_home(&leg(&[other, relayer]), Instant::now());
3130 }
3131 let judged = judged_dead(&health.refresh(4));
3132 assert_eq!(
3133 delivers_behind_another && overdue >= MIN_RACES_OVER_ONE_HOP,
3134 health.suspected.contains_key(&relayer),
3135 "suspected over one hop only from {MIN_RACES_OVER_ONE_HOP} overdue replies on ({overdue})"
3136 );
3137 let_suspicions_stand(&health);
3138 Ok(judged.into_iter().chain(judged_dead(&health.refresh(4))).collect())
3139 };
3140
3141 assert!(judged_with(MIN_RACES_OVER_ONE_HOP - 1, true)?.is_empty());
3142 assert_eq!(vec![relayer], judged_with(MIN_RACES_OVER_ONE_HOP, true)?);
3143 assert_eq!(
3144 vec![relayer],
3145 judged_with(MIN_RACES, false)?,
3146 "delivering nothing, it is judged as a whole"
3147 );
3148 Ok(())
3149 }
3150
3151 #[test]
3154 fn a_relayer_delivering_over_its_other_hop_should_keep_its_bar_after_a_dead_one_explains_replies()
3155 -> anyhow::Result<()> {
3156 let [dead, behind, helper, ahead, healthy] = [(); 5].map(|_| key());
3157 let health = RelayerHealth::default();
3158 for relayer in [dead, behind, helper, ahead] {
3159 settled_since(&health, relayer, ago(Duration::from_secs(10))?, jitter(0));
3160 }
3161 let through_helper: &[OffchainPublicKey] = &[dead, helper];
3163 overdue_legs(&health, &[through_helper; MIN_RACES], 1, healthy);
3164 overdue_legs(&health, &[&[dead, behind]], 10, healthy);
3165 let behind_ahead: &[OffchainPublicKey] = &[ahead, behind];
3167 overdue_legs(&health, &[behind_ahead; MIN_RACES_OVER_ONE_HOP - 1], 20, healthy);
3168 backdate_all_open(&health, ago(ROUND_TRIP * 2)?);
3169 health.record_reply();
3170 health.note_home(&leg(&[healthy, helper]), Instant::now());
3171 health.note_home(&leg(&[ahead, healthy]), Instant::now());
3172 health.note_home(&leg(&[behind, healthy]), Instant::now());
3173
3174 assert_eq!(vec![dead], judged_dead_by_the_second_flush(&health));
3175 assert!(health.suspected.is_empty(), "`behind` is not even suspected");
3176 Ok(())
3177 }
3178
3179 #[test]
3185 fn replies_lost_behind_different_relayers_should_be_answered_by_a_reply_behind_any_relayer() -> anyhow::Result<()> {
3186 let health = RelayerHealth::default();
3187 let [relayer, one, two, three, fourth, healthy] = [(); 6].map(|_| key());
3188 for settled in [relayer, one, two, three] {
3189 settled_since(&health, settled, ago(Duration::from_secs(10))?, jitter(0));
3190 }
3191 overdue_legs(
3192 &health,
3193 &[&[one, relayer], &[two, relayer], &[three, relayer]],
3194 1,
3195 healthy,
3196 );
3197 backdate_all_open(&health, ago(ROUND_TRIP * 2)?);
3198 health.record_reply();
3199 for ahead in [one, two, three] {
3201 health.note_home(&leg(&[ahead, healthy]), Instant::now());
3202 }
3203 health.note_home(&leg(&[fourth, relayer]), Instant::now());
3204
3205 assert!(judged_dead_by_the_second_flush(&health).is_empty());
3206 assert!(health.suspected.is_empty(), "answered, so not even suspected");
3207 Ok(())
3208 }
3209
3210 #[test]
3213 fn of_two_relayers_with_the_same_overdue_replies_the_one_upstream_should_be_judged() -> anyhow::Result<()> {
3214 for swap in [false, true] {
3216 let health = RelayerHealth::default();
3217 let (mut dead, mut behind) = (key(), key());
3218 if (dead.to_string() < behind.to_string()) == swap {
3219 std::mem::swap(&mut dead, &mut behind);
3220 }
3221 let healthy = key();
3222 settled_since(&health, dead, ago(Duration::from_secs(10))?, jitter(0));
3223 overdue_legs(
3224 &health,
3225 &[&[dead, behind], &[dead, behind], &[dead, behind]],
3226 1,
3227 healthy,
3228 );
3229 backdate_all_open(&health, ago(ROUND_TRIP * 2)?);
3230 health.record_reply();
3231
3232 assert_eq!(vec![dead], judged_dead(&health.refresh(4)), "swap = {swap}");
3233 assert!(!health.is_dead(&behind));
3234 }
3235 Ok(())
3236 }
3237
3238 #[test]
3242 fn a_relayer_with_fewer_unanswered_replies_should_be_judged_if_it_explains_as_many_from_further_up()
3243 -> anyhow::Result<()> {
3244 let health = RelayerHealth::default();
3245 let [dead, behind, healthy] = [(); 3].map(|_| key());
3246 settled_since(&health, dead, ago(Duration::from_secs(60))?, jitter(0));
3247 settled_since(&health, behind, ago(Duration::from_secs(60))?, jitter(0));
3248 let on_both: &[OffchainPublicKey] = &[dead, behind];
3249 overdue_legs(&health, &[on_both; 2], 1, healthy);
3251 backdate_all_open(&health, ago(Duration::from_secs(20))?);
3252 overdue_legs(&health, &[on_both; 3], 3, healthy);
3253 for mut history in health.relayers.iter_mut() {
3254 for seed in 3..6 {
3255 if let Some(race) = history.open.get_mut(&surb_id(seed)) {
3256 race.at = ago(Duration::from_secs(10))?;
3257 }
3258 }
3259 }
3260 health.record_reply();
3261 health
3263 .relayers
3264 .get_mut(&dead)
3265 .expect("has races")
3266 .last_home
3267 .insert(Hop::FromReplier, ago(Duration::from_secs(15))?);
3268
3269 assert_eq!(vec![dead], judged_dead_by_the_second_flush(&health));
3270 assert!(
3271 !health.is_dead(&behind),
3272 "five unanswered replies to three, all of them on legs through the dead one"
3273 );
3274 Ok(())
3275 }
3276
3277 #[test]
3281 fn a_relayer_behind_a_slower_one_should_be_held_to_the_slower_ones_bar() -> anyhow::Result<()> {
3282 let health = RelayerHealth::default();
3283 let [slow, behind, healthy] = [(); 3].map(|_| key());
3284 settled_since(&health, slow, ago(Duration::from_secs(60))?, Duration::from_secs(5));
3285 settled_since(
3286 &health,
3287 behind,
3288 ago(Duration::from_secs(60))?,
3289 Duration::from_millis(100),
3290 );
3291 let on_both: &[OffchainPublicKey] = &[slow, behind];
3292 overdue_legs(&health, &[on_both; 3], 1, healthy);
3293 backdate_all_open(&health, ago(Duration::from_secs(2))?);
3294 health.record_reply();
3295
3296 assert!(
3297 judged_dead_by_the_second_flush(&health).is_empty(),
3298 "two seconds is late for the relayer behind, not for a leg through one that takes five"
3299 );
3300
3301 backdate_all_open(&health, ago(Duration::from_secs(20))?);
3303 health.record_reply();
3304 assert_eq!(vec![slow], judged_dead_by_the_second_flush(&health));
3305 assert!(!health.is_dead(&behind));
3306 Ok(())
3307 }
3308
3309 #[test]
3314 fn a_relayer_behind_a_stalling_one_should_not_be_judged_for_replies_the_stalling_one_held() -> anyhow::Result<()> {
3315 for swap in [false, true] {
3317 let health = RelayerHealth::default();
3318 let (mut stalling, mut behind) = (key(), key());
3319 if (stalling.to_string() < behind.to_string()) == swap {
3320 std::mem::swap(&mut stalling, &mut behind);
3321 }
3322 let [ahead, elsewhere, healthy] = [(); 3].map(|_| key());
3323 for relayer in [stalling, behind, ahead] {
3324 settled_since(&health, relayer, ago(Duration::from_secs(60))?, jitter(0));
3325 }
3326 for seed in 1..4 {
3329 health.open_race(
3330 &leg(&[stalling, behind]),
3331 surb_id(seed),
3332 &leg(&[stalling, elsewhere]),
3333 ROUND_TRIP,
3334 );
3335 }
3336 assert!(
3337 !health
3338 .relayers
3339 .get(&stalling)
3340 .is_some_and(|history| !history.open.is_empty()),
3341 "on both legs over the same hop: no race (swap = {swap})"
3342 );
3343 let reached_over_another: &[OffchainPublicKey] = &[ahead, stalling];
3345 overdue_legs(&health, &[reached_over_another; 3], 4, healthy);
3346 backdate_all_open(&health, ago(ROUND_TRIP * 2)?);
3347 health.record_reply();
3348 health
3349 .relayers
3350 .get_mut(&ahead)
3351 .expect("has races")
3352 .last_home
3353 .insert(Hop::FromReplier, Instant::now());
3354
3355 assert_eq!(
3356 vec![stalling],
3357 judged_dead_by_the_second_flush(&health),
3358 "swap = {swap}"
3359 );
3360 assert!(
3361 !health.is_dead(&behind),
3362 "its three replies are on legs through the stalling one (swap = {swap})"
3363 );
3364 }
3365 Ok(())
3366 }
3367
3368 #[test]
3371 fn a_relayer_with_nothing_unanswered_of_its_own_should_not_explain_the_one_behind_it() -> anyhow::Result<()> {
3372 let health = RelayerHealth::default();
3373 let [ahead, cut_off, elsewhere] = [(); 3].map(|_| key());
3374 for relayer in [ahead, cut_off] {
3375 settled_since(&health, relayer, ago(Duration::from_secs(60))?, jitter(0));
3376 }
3377 for seed in 1..4 {
3378 health.open_race(
3379 &leg(&[ahead, cut_off]),
3380 surb_id(seed),
3381 &leg(&[ahead, elsewhere]),
3382 ROUND_TRIP,
3383 );
3384 }
3385 backdate_all_open(&health, ago(ROUND_TRIP * 2)?);
3386 health.record_reply();
3387
3388 assert_eq!(vec![cut_off], judged_dead_by_the_second_flush(&health));
3389 assert!(!health.is_dead(&ahead));
3390 Ok(())
3391 }
3392
3393 #[test]
3397 fn two_relayers_the_overdue_replies_do_not_tell_apart_should_wait_for_a_reply_that_does() -> anyhow::Result<()> {
3398 for swap in [false, true] {
3400 let health = RelayerHealth::default();
3401 let (mut dead, mut other) = (key(), key());
3402 if (dead.to_string() < other.to_string()) == swap {
3403 std::mem::swap(&mut dead, &mut other);
3404 }
3405 let [third, healthy] = [(); 2].map(|_| key());
3406 for relayer in [dead, other, third] {
3407 settled_since(&health, relayer, ago(Duration::from_secs(60))?, jitter(0));
3408 }
3409 let (ahead, behind): (&[OffchainPublicKey], &[OffchainPublicKey]) = (&[dead, other], &[other, dead]);
3410 overdue_legs(&health, &[ahead, ahead, behind, behind], 1, healthy);
3411 backdate_all_open(&health, ago(ROUND_TRIP * 2)?);
3412 health.record_reply();
3413
3414 assert!(
3415 judged_dead_by_the_second_flush(&health).is_empty(),
3416 "four replies each, on the same legs, each ahead on two (swap = {swap})"
3417 );
3418
3419 overdue_legs(&health, &[&[dead, third]], 5, healthy);
3421 backdate_all_open(&health, ago(ROUND_TRIP * 2)?);
3422 health.record_reply();
3423 assert_eq!(vec![dead], judged_dead_by_the_second_flush(&health), "swap = {swap}");
3424 assert!(!health.is_dead(&other));
3425 assert!(!health.is_dead(&third));
3426 }
3427 Ok(())
3428 }
3429
3430 #[test]
3432 fn a_relayer_already_judged_should_keep_its_verdict_in_a_tie() -> anyhow::Result<()> {
3433 for swap in [false, true] {
3434 let health = RelayerHealth::default();
3435 let (mut dead, mut other) = (key(), key());
3436 if (dead.to_string() < other.to_string()) == swap {
3437 std::mem::swap(&mut dead, &mut other);
3438 }
3439 let healthy = key();
3440 for relayer in [dead, other] {
3441 settled_since(&health, relayer, ago(Duration::from_secs(60))?, jitter(0));
3442 }
3443 let (ahead, behind): (&[OffchainPublicKey], &[OffchainPublicKey]) = (&[dead, other], &[other, dead]);
3444 overdue_legs(&health, &[ahead, ahead, behind, behind], 1, healthy);
3445 backdate_all_open(&health, ago(ROUND_TRIP * 2)?);
3446 health.record_reply();
3447 judge_dead(&health, dead);
3448
3449 health.refresh(4);
3450
3451 assert!(health.is_dead(&dead), "swap = {swap}");
3452 assert!(
3453 !health.is_on_probation(&dead),
3454 "its verdict did not lapse (swap = {swap})"
3455 );
3456 assert!(!health.is_dead(&other), "swap = {swap}");
3457 }
3458 Ok(())
3459 }
3460
3461 #[test]
3463 fn two_relayers_dead_on_separate_legs_should_both_be_judged_dead() -> anyhow::Result<()> {
3464 let health = RelayerHealth::default();
3465 let [first, second, healthy] = [(); 3].map(|_| key());
3466 let [a1, a2, a3, a4, b1, b2, b3] = [(); 7].map(|_| key());
3467 settled_since(&health, first, ago(Duration::from_secs(10))?, jitter(0));
3468 overdue_legs(
3469 &health,
3470 &[&[first, a1], &[first, a2], &[first, a3], &[first, a4]],
3471 1,
3472 healthy,
3473 );
3474 overdue_legs(&health, &[&[second, b1], &[second, b2], &[second, b3]], 10, healthy);
3475 backdate_all_open(&health, ago(ROUND_TRIP * 2)?);
3476 health.record_reply();
3477
3478 assert_eq!(sorted(vec![first, second]), sorted(judged_dead(&health.refresh(4))));
3479 Ok(())
3480 }
3481
3482 #[test]
3485 fn a_relayer_on_a_dead_ones_legs_should_still_be_judged_on_its_own_overdue_replies() -> anyhow::Result<()> {
3486 let health = RelayerHealth::default();
3487 let [dead, also, healthy] = [(); 3].map(|_| key());
3488 let [v, w, x, y, z1, z2, z3] = [(); 7].map(|_| key());
3489 settled_since(&health, dead, ago(Duration::from_secs(10))?, jitter(0));
3490 overdue_legs(
3491 &health,
3492 &[
3493 &[dead, also],
3494 &[dead, also],
3495 &[dead, v],
3496 &[dead, w],
3497 &[dead, x],
3498 &[dead, y],
3499 &[also, z1],
3500 &[also, z2],
3501 &[also, z3],
3502 ],
3503 1,
3504 healthy,
3505 );
3506 backdate_all_open(&health, ago(ROUND_TRIP * 2)?);
3507 health.record_reply();
3508
3509 assert_eq!(sorted(vec![dead, also]), sorted(judged_dead(&health.refresh(4))));
3510 Ok(())
3511 }
3512
3513 fn judged_behind_a_shared_relayer(
3517 reply_through_shared: bool,
3518 ) -> anyhow::Result<(Vec<OffchainPublicKey>, [OffchainPublicKey; 4], RelayerHealth)> {
3519 let [shared, first, second, healthy] = [(); 4].map(|_| key());
3520 let [a1, a2, a3, b1, b2, b3] = [(); 6].map(|_| key());
3521 let legs: [&[OffchainPublicKey]; 6] = [
3522 &[shared, first, a1],
3523 &[shared, first, a2],
3524 &[shared, first, a3],
3525 &[shared, second, b1],
3526 &[shared, second, b2],
3527 &[shared, second, b3],
3528 ];
3529 let health = RelayerHealth::default();
3530 settled_since(&health, first, ago(Duration::from_secs(10))?, jitter(0));
3531 overdue_legs(&health, &legs, 1, healthy);
3532 backdate_all_open(&health, ago(ROUND_TRIP * 2)?);
3533 if reply_through_shared {
3534 health.note_home(&leg(&[shared, healthy]), Instant::now());
3535 }
3536 health.record_reply();
3537 Ok((
3538 sorted(judged_dead(&health.refresh(4))),
3539 [shared, first, second, healthy],
3540 health,
3541 ))
3542 }
3543
3544 #[test]
3547 fn a_relayer_shared_by_two_dead_ones_should_be_cleared_by_a_reply_through_it() -> anyhow::Result<()> {
3548 let (judged, [_, first, second, _], _) = judged_behind_a_shared_relayer(true)?;
3549 assert_eq!(sorted(vec![first, second]), judged);
3550 Ok(())
3551 }
3552
3553 #[test]
3559 fn a_relayer_shared_by_two_dead_ones_is_judged_in_their_place_as_a_known_limitation() -> anyhow::Result<()> {
3560 let (judged, [shared, ..], _) = judged_behind_a_shared_relayer(false)?;
3561 assert_eq!(vec![shared], judged);
3562 Ok(())
3563 }
3564
3565 #[rstest::rstest]
3570 #[case::before_confirmation(false)]
3571 #[case::once_confirmed(true)]
3572 fn a_relayer_judged_in_place_of_dead_ones_should_be_cleared_by_replies_as_its_verdict_allows(
3573 #[case] confirmed: bool,
3574 ) -> anyhow::Result<()> {
3575 let (judged, [shared, _, _, healthy], health) = judged_behind_a_shared_relayer(false)?;
3576 assert_eq!(vec![shared], judged);
3577 if confirmed {
3578 confirm_verdicts(&health);
3579 }
3580 let older = ago(Duration::from_secs(1))?;
3581 health.note_home(&leg(&[shared, healthy]), older);
3582 health.revive_by_reply(&leg(&[shared, healthy]), older);
3583 assert_eq!(
3584 !confirmed,
3585 !health.is_known_dead(&shared),
3586 "a reply on a SURB older than the verdict clears it only before the verdict is confirmed"
3587 );
3588 health.revive_by_reply(&leg(&[shared, healthy]), Instant::now());
3589 assert!(
3590 !health.is_known_dead(&shared),
3591 "a reply on a SURB minted since clears it"
3592 );
3593 Ok(())
3594 }
3595
3596 #[test]
3599 fn replies_out_no_longer_than_usual_should_not_make_a_black_hole() -> anyhow::Result<()> {
3600 let health = RelayerHealth::default();
3601 let slow = key();
3602 settled_since(&health, slow, ago(Duration::from_secs(5))?, Duration::from_millis(500));
3603 let just_now = Instant::now()
3604 .checked_sub(Duration::from_millis(200))
3605 .context("monotonic clock younger than 200 ms")?;
3606 open_since(&health, slow, MIN_RACES as u8, just_now);
3607 assert!(
3608 judged_dead(&health.refresh(4)).is_empty(),
3609 "out for less than its usual 500 ms"
3610 );
3611
3612 let overdue = Instant::now()
3613 .checked_sub(Duration::from_secs(2))
3614 .context("monotonic clock younger than two seconds")?;
3615 open_since(&health, slow, MIN_RACES as u8, overdue);
3616 health.record_reply();
3618 assert_eq!(vec![slow], judged_dead(&health.refresh(4)));
3619 Ok(())
3620 }
3621
3622 #[test]
3625 fn a_dead_relayer_should_not_be_revived_to_full_health_by_expiry() -> anyhow::Result<()> {
3626 let health = RelayerHealth::default();
3627 let [black_hole, healthy] = [(); 2].map(|_| key());
3628 let expired = Instant::now()
3629 .checked_sub(PENDING_SURB_TTL + Duration::from_secs(1))
3630 .context("monotonic clock younger than a SURB's lifetime")?;
3631 settled_since(&health, healthy, Instant::now(), jitter(0));
3632 settled_since(&health, black_hole, expired - Duration::from_secs(1), jitter(0));
3633 open_since(&health, black_hole, 2 * MIN_RACES as u8, expired);
3634
3635 let [revived, _] = health_after_flush(&health, &[black_hole, healthy])[..] else {
3636 anyhow::bail!("two relayers asked for");
3637 };
3638 assert!(!health.is_dead(&black_hole), "no reply is out any more");
3639 assert!(revived < 0.01, "revived to a sliver only, got {revived}");
3640 Ok(())
3641 }
3642
3643 fn put_on_probation(health: &RelayerHealth, relayer: OffchainPublicKey) -> anyhow::Result<()> {
3646 settled_since(health, relayer, ago(Duration::from_secs(10))?, jitter(0));
3647 open_since(health, relayer, MIN_RACES as u8, ago(Duration::from_secs(5))?);
3648 health.record_reply();
3649 health.refresh(4);
3650 anyhow::ensure!(health.is_dead(&relayer), "a black hole is judged dead");
3651 confirm_verdicts(health);
3652 backdate_open(health, relayer, ago(PENDING_SURB_TTL + Duration::from_secs(1))?);
3653 health.refresh(4);
3654 anyhow::ensure!(
3655 health.is_on_probation(&relayer) && !health.is_dead(&relayer),
3656 "the lapsed verdict leaves the relayer on probation"
3657 );
3658 Ok(())
3659 }
3660
3661 fn published_health(health: &RelayerHealth, relayer: &OffchainPublicKey) -> f64 {
3662 health.published.get(relayer).map_or(1.0, |health| *health)
3663 }
3664
3665 #[test]
3670 fn a_dead_relayer_should_stay_dead_until_a_reply_comes_home_through_it() -> anyhow::Result<()> {
3671 let (graph, _me, _peers) = graph_with(0);
3672 let recorder = recorder(graph);
3673 let health = recorder.registry.relayer_health();
3674 let [black_hole, healthy] = [(); 2].map(|_| key());
3675 put_on_probation(&health, black_hole)?;
3676
3677 for _ in 0..5 {
3678 health.refresh(4);
3679 assert!(health.is_known_dead(&black_hole));
3680 assert_eq!(PROBATION_HEALTH, published_health(&health, &black_hole));
3681 }
3682
3683 recorder
3684 .registry
3685 .pending
3686 .insert(surb_id(1), pending_to(&[healthy], None, 0));
3687 recorder.on_replied(&surb_id(1));
3688 health.refresh(4);
3689 assert!(
3690 health.is_known_dead(&black_hole),
3691 "a reply through another relayer proves nothing"
3692 );
3693
3694 recorder
3695 .registry
3696 .pending
3697 .insert(surb_id(2), pending_to(&[healthy, black_hole], None, 0));
3698 recorder.on_replied(&surb_id(2));
3699 assert!(
3700 !health.is_known_dead(&black_hole),
3701 "a reply home through it ends the verdict, whatever else is on the leg"
3702 );
3703 Ok(())
3704 }
3705
3706 #[test]
3709 fn a_lapsed_verdict_should_put_the_relayer_on_probation_not_restore_its_health() -> anyhow::Result<()> {
3710 let health = RelayerHealth::default();
3711 let relayer = key();
3712 settled_since(&health, relayer, ago(Duration::from_secs(10))?, jitter(0));
3713 open_since(&health, relayer, MIN_RACES as u8, ago(Duration::from_secs(5))?);
3714 health.record_reply();
3715 assert_eq!(vec![relayer], judged_dead(&health.refresh(4)));
3716 assert_eq!(0.0, published_health(&health, &relayer));
3717 confirm_verdicts(&health);
3718
3719 backdate_open(&health, relayer, ago(PENDING_SURB_TTL + Duration::from_secs(1))?);
3720 let changes = health.refresh(4);
3721 assert_eq!(
3722 vec![(relayer, 0.0, PROBATION_HEALTH)],
3723 changes,
3724 "from dead to probation, in one step and by that much"
3725 );
3726 assert!(!health.is_dead(&relayer));
3727 assert!(health.is_on_probation(&relayer));
3728 assert_eq!(PROBATION_HEALTH, published_health(&health, &relayer));
3729
3730 assert!(health.refresh(4).is_empty(), "nothing moves it from there");
3731 assert_eq!(PROBATION_HEALTH, published_health(&health, &relayer));
3732 Ok(())
3733 }
3734
3735 #[rstest::rstest]
3739 #[case::probe_written_off_already(true)]
3740 #[case::probe_still_pending(false)]
3741 fn a_probe_reply_should_revive_a_relayer(#[case] written_off_first: bool) -> anyhow::Result<()> {
3742 let (graph, _me, _peers) = graph_with(0);
3743 let recorder = recorder(graph);
3744 let health = recorder.registry.relayer_health();
3745 let relayer = key();
3746 put_on_probation(&health, relayer)?;
3747 if let Some(mut history) = health.relayers.get_mut(&relayer) {
3749 history.races.retain(Race::lost);
3750 }
3751
3752 recorder.registry.bumped.insert((*SESSION, relayer), Instant::now());
3754 recorder
3755 .registry
3756 .pending
3757 .insert(surb_id(7), pending_to(&[relayer], None, 0));
3758 if written_off_first {
3759 recorder.registry.write_off_dead();
3760 assert!(
3761 recorder.registry.pending.get(&surb_id(7)).is_none(),
3762 "the probe is written off, and out of the pending map"
3763 );
3764 assert_eq!(HashMap::from([(*SESSION, (1, 1.0))]), recorder.registry.invalid());
3765 }
3766
3767 recorder.on_replied(&surb_id(7));
3768 assert!(!health.is_known_dead(&relayer), "alive again");
3769
3770 let changes = health.refresh(4);
3771 let [(revived, previous, current)] = changes[..] else {
3772 anyhow::bail!("one relayer's health moves, got {changes:?}");
3773 };
3774 assert_eq!((relayer, PROBATION_HEALTH), (revived, previous));
3775 assert!(
3776 current > PROBATION_HEALTH && current < 0.01,
3777 "its lost races still hold it down: {current}"
3778 );
3779 recorder.registry.write_off_dead();
3780 assert!(recorder.registry.invalid().is_empty(), "its SURBs work again");
3781 Ok(())
3782 }
3783
3784 #[test]
3788 fn a_relayer_dying_again_on_probation_should_not_bump_the_session_again() -> anyhow::Result<()> {
3789 let registry = SurbRoundTripRegistry::default();
3790 let health = registry.relayer_health();
3791 let [relayer, measured] = [(); 2].map(|_| key());
3792 settled_since(&health, measured, ago(Duration::from_secs(10))?, jitter(0));
3795 settled_since(&health, relayer, ago(Duration::from_secs(10))?, jitter(0));
3796 open_since(&health, relayer, MIN_RACES as u8, ago(Duration::from_secs(5))?);
3797 health.record_reply();
3798 registry.pending.insert(surb_id(1), pending_to(&[relayer], None, 0));
3799 health.refresh(4);
3800 assert!(health.is_dead(&relayer));
3801 confirm_verdicts(&health);
3802 assert_eq!(vec![*SESSION], registry.write_off_dead(), "the first verdict bumps");
3803
3804 backdate_open(&health, relayer, ago(PENDING_SURB_TTL + Duration::from_secs(1))?);
3805 health.refresh(4);
3806 assert!(health.is_on_probation(&relayer));
3807
3808 registry.pending.insert(surb_id(2), pending_to(&[relayer], None, 0));
3809 assert!(
3810 registry.write_off_dead().is_empty(),
3811 "a probe on probation bumps nothing"
3812 );
3813 assert_eq!(
3814 HashMap::from([(*SESSION, (1, 1.0))]),
3815 registry.invalid(),
3816 "the first verdict's count went with the wipe, only the probe is counted"
3817 );
3818
3819 for mut entry in registry.bumped.iter_mut() {
3820 *entry.value_mut() = ago(REBUMP_COOLDOWN + Duration::from_secs(1))?;
3821 }
3822 open_since(&health, relayer, MIN_RACES as u8, ago(Duration::from_secs(1))?);
3823 health.record_reply();
3824 health.refresh(4);
3825 assert!(health.is_dead(&relayer), "a black hole again, and so dead again");
3826 assert!(!health.is_on_probation(&relayer));
3827
3828 registry.pending.insert(surb_id(3), pending_to(&[relayer], None, 0));
3829 assert!(
3830 registry.write_off_dead().is_empty(),
3831 "the pair stayed bumped through the lapse, so this is no first verdict, cooldown or not"
3832 );
3833 assert_eq!(HashMap::from([(*SESSION, (2, 1.0))]), registry.invalid());
3834 Ok(())
3835 }
3836
3837 #[test]
3841 fn a_bumped_session_should_drop_the_written_off_count_of_the_wiped_stock() {
3842 let registry = SurbRoundTripRegistry::default();
3843 let dead = key();
3844 for seed in 1..=3 {
3845 registry.pending.insert(surb_id(seed), pending_to(&[dead], None, 0));
3846 }
3847 judge_dead(®istry.relayer_health, dead);
3848
3849 assert_eq!(
3850 vec![*SESSION],
3851 registry.write_off_dead(),
3852 "the verdict bumps the session"
3853 );
3854 assert!(
3855 registry.invalid().is_empty(),
3856 "the three SURBs counted are gone with the stock they were in"
3857 );
3858
3859 registry.pending.insert(surb_id(4), pending_to(&[dead], None, 0));
3860 assert!(
3861 registry.write_off_dead().is_empty(),
3862 "the relayer is still dead, nothing more is bumped"
3863 );
3864 assert_eq!(
3865 HashMap::from([(*SESSION, (1, 1.0))]),
3866 registry.invalid(),
3867 "only the SURB minted through it since is written off"
3868 );
3869 }
3870
3871 #[test]
3874 fn a_pair_bumped_once_stays_bumped_while_the_relayer_is_known_dead() -> anyhow::Result<()> {
3875 let registry = SurbRoundTripRegistry::default();
3876 let health = registry.relayer_health();
3877 let dead = key();
3878 registry.pending.insert(surb_id(1), pending_to(&[dead], None, 0));
3879 judge_dead(&health, dead);
3880 assert_eq!(vec![*SESSION], registry.write_off_dead(), "the verdict bumps");
3881
3882 health.dead.remove(&dead);
3883 health.probation.insert(dead);
3884 for mut entry in registry.bumped.iter_mut() {
3885 *entry.value_mut() = ago(REBUMP_COOLDOWN + Duration::from_secs(1))?;
3886 }
3887 registry.pending.insert(surb_id(2), pending_to(&[dead], None, 0));
3888 assert!(
3889 registry.write_off_dead().is_empty(),
3890 "on probation past the cooldown, the pair is still bumped"
3891 );
3892 assert_eq!(
3893 1,
3894 registry.bumped.len(),
3895 "the mark outlives the cooldown while the relayer is known dead"
3896 );
3897
3898 health.probation.remove(&dead);
3899 judge_dead(&health, dead);
3900 registry.pending.insert(surb_id(3), pending_to(&[dead], None, 0));
3901 assert!(
3902 registry.write_off_dead().is_empty(),
3903 "and so it is when it dies again there"
3904 );
3905 Ok(())
3906 }
3907
3908 #[test]
3911 fn a_revived_relayer_that_dies_again_after_the_cooldown_is_bumped_again() -> anyhow::Result<()> {
3912 let (graph, _me, _peers) = graph_with(0);
3913 let recorder = recorder(graph);
3914 let registry = &recorder.registry;
3915 let health = registry.relayer_health();
3916 let dead = key();
3917 registry.pending.insert(surb_id(1), pending_to(&[dead], None, 0));
3918 judge_dead(&health, dead);
3919 assert_eq!(vec![*SESSION], registry.write_off_dead(), "the verdict bumps");
3920
3921 registry.pending.insert(surb_id(2), pending_to(&[dead], None, 0));
3922 recorder.on_replied(&surb_id(2));
3923 assert!(!health.is_known_dead(&dead), "revived by a reply");
3924 registry.write_off_dead();
3925 assert_eq!(
3926 1,
3927 registry.bumped.len(),
3928 "the mark is kept for the cooldown after the revival"
3929 );
3930
3931 judge_dead(&health, dead);
3932 registry.pending.insert(surb_id(3), pending_to(&[dead], None, 0));
3933 assert!(
3934 registry.write_off_dead().is_empty(),
3935 "a quick re-death is inside the cooldown"
3936 );
3937
3938 health.dead.remove(&dead);
3939 for mut entry in registry.bumped.iter_mut() {
3940 *entry.value_mut() = ago(REBUMP_COOLDOWN + Duration::from_secs(1))?;
3941 }
3942 registry.write_off_dead();
3943 assert!(
3944 registry.bumped.is_empty(),
3945 "revived and past the cooldown, the mark is dropped"
3946 );
3947
3948 registry.pending.insert(surb_id(4), pending_to(&[dead], None, 0));
3950 judge_dead(&health, dead);
3951 assert_eq!(
3952 vec![*SESSION],
3953 registry.write_off_dead(),
3954 "a death after the cooldown is a new verdict"
3955 );
3956 Ok(())
3957 }
3958
3959 #[test]
3962 fn a_relayer_revived_after_a_long_death_should_be_guarded_from_its_revival() -> anyhow::Result<()> {
3963 let (graph, _me, _peers) = graph_with(0);
3964 let recorder = recorder(graph);
3965 let registry = &recorder.registry;
3966 let health = registry.relayer_health();
3967 let dead = key();
3968 registry.pending.insert(surb_id(1), pending_to(&[dead], None, 0));
3969 judge_dead(&health, dead);
3970 assert_eq!(vec![*SESSION], registry.write_off_dead(), "the verdict bumps");
3971
3972 for mut entry in registry.bumped.iter_mut() {
3974 *entry.value_mut() = ago(REBUMP_COOLDOWN + Duration::from_secs(1))?;
3975 }
3976 assert!(
3977 registry.write_off_dead().is_empty(),
3978 "still dead, nothing more is bumped"
3979 );
3980
3981 registry.pending.insert(surb_id(2), pending_to(&[dead], None, 0));
3982 recorder.on_replied(&surb_id(2));
3983 assert!(!health.is_known_dead(&dead), "revived by a reply");
3984 registry.write_off_dead();
3985 assert_eq!(
3986 1,
3987 registry.bumped.len(),
3988 "the mark is kept for the cooldown after the revival, however long the death before it"
3989 );
3990
3991 judge_dead(&health, dead);
3992 registry.pending.insert(surb_id(3), pending_to(&[dead], None, 0));
3993 assert!(
3994 registry.write_off_dead().is_empty(),
3995 "a quick re-death after a long death is inside the cooldown"
3996 );
3997 Ok(())
3998 }
3999
4000 #[test]
4005 fn a_probe_should_not_bump_a_session_the_wipe_never_reached() -> anyhow::Result<()> {
4006 let registry = SurbRoundTripRegistry::default();
4007 let health = registry.relayer_health();
4008 let dead = key();
4009 let unreached = HoprPseudonym::random();
4010 let probe_for = |pseudonym| PendingSurb {
4011 pseudonym,
4012 ..pending_to(&[dead], None, 0)
4013 };
4014
4015 judge_dead(&health, dead);
4016 health.known_dead_since.insert(dead, ago(Duration::from_secs(2))?);
4017 registry.pending.insert(
4018 surb_id(1),
4019 PendingSurb {
4020 minted_at: ago(Duration::from_secs(5))?,
4021 ..pending_to(&[dead], None, 0)
4022 },
4023 );
4024 registry.pending.insert(surb_id(2), probe_for(*SESSION));
4025 registry.pending.insert(surb_id(3), probe_for(unreached));
4026 assert_eq!(
4027 vec![*SESSION],
4028 registry.write_off_dead(),
4029 "the stock minted before the death bumps its session; the probes bump nothing"
4030 );
4031 assert!(
4032 registry.invalid().contains_key(&unreached),
4033 "the probe is written off all the same"
4034 );
4035
4036 health.dead.remove(&dead);
4037 health.confirmed.remove(&dead);
4038 health.probation.insert(dead);
4039 registry.pending.insert(surb_id(4), probe_for(unreached));
4040 assert!(
4041 registry.write_off_dead().is_empty(),
4042 "a probe through a relayer on probation"
4043 );
4044
4045 health.probation.remove(&dead);
4046 judge_dead(&health, dead);
4047 registry.pending.insert(surb_id(5), probe_for(unreached));
4048 assert!(
4049 registry.write_off_dead().is_empty(),
4050 "nor once the relayer is dead again: the death began before the probe"
4051 );
4052 Ok(())
4053 }
4054
4055 #[test]
4058 fn a_lost_neighbour_should_not_write_off_its_surbs() {
4059 let registry = SurbRoundTripRegistry::default();
4060 let [neighbour, middle] = [(); 2].map(|_| key());
4061 registry
4062 .pending
4063 .insert(surb_id(1), pending_to(&[middle, neighbour], None, 0));
4064 registry
4065 .pending
4066 .insert(surb_id(2), pending_to(&[neighbour, middle], None, 0));
4067 let health = ®istry.relayer_health;
4068
4069 health.set_neighbour_connected(neighbour, false);
4070 assert!(registry.invalid().is_empty());
4071 assert!(!health.is_dead(&neighbour), "no verdict on the relayer itself");
4072 assert!(health.take_neighbour_change());
4073 assert!(!health.take_neighbour_change(), "a change is reported once");
4074
4075 health.set_neighbour_connected(neighbour, true);
4076 assert!(registry.invalid().is_empty());
4077 assert!(health.take_neighbour_change());
4078 }
4079
4080 #[test]
4081 fn settling_a_race_should_close_it() {
4082 let health = RelayerHealth::default();
4083 let [early, late] = [(); 2].map(|_| key());
4084 health.open_race(&leg(&[late]), surb_id(1), &leg(&[early]), ROUND_TRIP);
4085 health.record_race(&leg(&[late]), surb_id(1), &leg(&[early]), ROUND_TRIP, jitter(0));
4086
4087 assert_eq!(0, open_races_of(&health, &late));
4088 assert_eq!(1, margins_of(&health, &late).len());
4089 }
4090
4091 #[test]
4094 fn relayers_finishing_late_should_be_ordered_by_how_late() {
4095 let health = RelayerHealth::default();
4096 let [outsider, late, later] = [(); 3].map(|_| key());
4097 for i in 0..20 {
4098 health.record_race(
4099 &leg(&[late]),
4100 surb_id(0),
4101 &leg(&[outsider]),
4102 ROUND_TRIP,
4103 Duration::from_secs(1) + jitter(i),
4104 );
4105 health.record_race(
4106 &leg(&[later]),
4107 surb_id(0),
4108 &leg(&[outsider]),
4109 ROUND_TRIP,
4110 Duration::from_secs(3) + jitter(i),
4111 );
4112 }
4113
4114 let factors = health_after_flush(&health, &[late, later]);
4115 assert!(1.0 > factors[0] && factors[0] > factors[1], "{factors:?}");
4116 }
4117
4118 #[test]
4121 fn a_health_within_its_own_uncertainty_should_not_be_reported_again() {
4122 let health = RelayerHealth::default();
4123 let [fast, stalled] = [(); 2].map(|_| key());
4124 let mut race = |i| {
4125 health.record_race(
4126 &leg(&[stalled]),
4127 surb_id(0),
4128 &leg(&[fast]),
4129 ROUND_TRIP,
4130 Duration::from_secs(3) + jitter(i),
4131 )
4132 };
4133 (0..20).for_each(&mut race);
4134 assert_eq!(1, health.refresh(4).len(), "precondition: reported once stalled");
4135
4136 (20..24).for_each(&mut race);
4137 assert!(health.refresh(4).is_empty());
4138 }
4139
4140 #[test]
4144 fn a_race_should_be_settled_by_the_gap_between_the_two_replies() -> anyhow::Result<()> {
4145 let (graph, _me, _peers) = graph_with(0);
4146 let recorder = recorder(graph);
4147 let [early, late] = [(); 2].map(|_| key());
4148 let minted_at = Instant::now();
4149 let first_spent = FIRST_SPENT_INDEX;
4150 let pending_via = |relayer, partner| PendingSurb {
4151 minted_at,
4152 ..pending_to(
4153 &[relayer],
4154 Some(partner),
4155 if relayer == late { first_spent } else { 1 - first_spent },
4156 )
4157 };
4158 recorder
4159 .registry
4160 .pending
4161 .insert(surb_id(1), pending_via(early, surb_id(2)));
4162 recorder
4163 .registry
4164 .pending
4165 .insert(surb_id(2), pending_via(late, surb_id(1)));
4166 let health = &recorder.registry.relayer_health;
4167
4168 recorder.on_replied(&surb_id(1));
4169 assert!(
4170 margins_of(health, &early).is_empty(),
4171 "no race settled until both are home"
4172 );
4173 assert_eq!(
4174 1,
4175 open_races_of(health, &late),
4176 "the reply still out is a race already being lost"
4177 );
4178 let mut waiting = recorder
4179 .registry
4180 .pending
4181 .get(&surb_id(2))
4182 .context("the late SURB is out")?;
4183 let (relayers, home) = waiting.partner_replied.context("its partner is home")?;
4184 waiting.partner_replied = Some((relayers, home.checked_sub(Duration::from_millis(20)).context("clock")?));
4185 recorder.registry.pending.insert(surb_id(2), waiting);
4186 recorder.on_replied(&surb_id(2));
4187 assert_eq!(0, open_races_of(health, &late));
4188
4189 let lost_by = margins_of(health, &late);
4190 assert_eq!(1, lost_by.len());
4191 assert!(
4192 lost_by[0] >= 0.02,
4193 "the later reply loses by the gap, got {}",
4194 lost_by[0]
4195 );
4196 assert_eq!(vec![-lost_by[0]], margins_of(health, &early));
4197 Ok(())
4198 }
4199
4200 #[test]
4201 fn invalid_surbs_should_be_counted_per_session_and_relative_to_what_is_outstanding() {
4202 let registry = SurbRoundTripRegistry::default();
4203 let [dead, alive] = [(); 2].map(|_| key());
4204 let other = HoprPseudonym::random();
4205 for (seed, relayer) in [(1, dead), (2, alive), (3, alive), (4, alive)] {
4206 registry.pending.insert(surb_id(seed), pending_to(&[relayer], None, 0));
4207 }
4208 registry.pending.insert(
4209 surb_id(5),
4210 PendingSurb {
4211 pseudonym: other,
4212 ..pending_to(&[dead], None, 0)
4213 },
4214 );
4215 registry.write_off_dead();
4216 assert!(registry.invalid().is_empty(), "nothing is dead yet");
4217
4218 registry.bumped.insert((*SESSION, dead), Instant::now());
4220 registry.bumped.insert((other, dead), Instant::now());
4221 judge_dead(®istry.relayer_health, dead);
4222 registry.write_off_dead();
4223 assert_eq!(
4224 HashMap::from([(*SESSION, (1, 0.25)), (other, (1, 1.0))]),
4225 registry.invalid()
4226 );
4227 }
4228
4229 #[test]
4232 fn written_off_surbs_should_stay_invalid_until_a_reply_proves_their_relayer() {
4233 let (graph, ..) = graph_with(1);
4234 let recorder = recorder(graph);
4235 let registry = &recorder.registry;
4236 let [dead, alive] = [(); 2].map(|_| key());
4237 for (seed, relayer) in [(1, dead), (2, alive)] {
4238 registry.pending.insert(surb_id(seed), pending_to(&[relayer], None, 0));
4239 }
4240 registry.bumped.insert((*SESSION, dead), Instant::now());
4242 judge_dead(®istry.relayer_health, dead);
4243 registry.write_off_dead();
4244
4245 for seed in [1, 2] {
4246 registry.pending.remove(&surb_id(seed));
4247 }
4248 assert_eq!(
4249 HashMap::from([(*SESSION, (1, 1.0))]),
4250 registry.invalid(),
4251 "pending SURBs gone, the written-off one still counts"
4252 );
4253
4254 registry.relayer_health.dead.remove(&dead);
4256 registry.relayer_health.probation.insert(dead);
4257 registry.write_off_dead();
4258 assert_eq!(
4259 HashMap::from([(*SESSION, (1, 1.0))]),
4260 registry.invalid(),
4261 "a lapsed verdict is no evidence of recovery"
4262 );
4263
4264 registry.pending.insert(surb_id(4), pending_to(&[alive], None, 0));
4265 recorder.on_replied(&surb_id(4));
4266 registry.write_off_dead();
4267 assert!(
4268 !registry.invalid().is_empty(),
4269 "a reply over another relayer proves nothing"
4270 );
4271
4272 registry.pending.insert(surb_id(5), pending_to(&[dead], None, 0));
4273 recorder.on_replied(&surb_id(5));
4274 assert!(
4275 !registry.relayer_health.is_known_dead(&dead),
4276 "a reply through it ends the probation"
4277 );
4278 registry.write_off_dead();
4279 assert!(
4280 registry.invalid().is_empty(),
4281 "a reply through it proves the relayer works again, and the next flush drops the write-off"
4282 );
4283
4284 judge_dead(®istry.relayer_health, dead);
4285 registry.pending.insert(surb_id(3), pending_to(&[dead], None, 0));
4286 registry.write_off_dead();
4287 registry.forget_session(&SESSION);
4288 assert!(registry.invalid().is_empty(), "a closed session is forgotten");
4289 }
4290
4291 #[test]
4296 fn a_dead_verdict_should_bump_a_session_once_and_write_off_only_its_surbs_through_the_relayer() {
4297 let registry = SurbRoundTripRegistry::default();
4298 let [dead, alive] = [(); 2].map(|_| key());
4299 registry.pending.insert(surb_id(1), pending_to(&[dead], None, 0));
4300 registry.pending.insert(surb_id(2), pending_to(&[alive], None, 0));
4301 registry.pending.insert(surb_id(3), pending_to(&[alive], None, 0));
4302 judge_dead(®istry.relayer_health, dead);
4303
4304 assert_eq!(
4305 vec![*SESSION],
4306 registry.write_off_dead(),
4307 "the verdict reaches the session once"
4308 );
4309 assert!(
4310 registry.pending.get(&surb_id(1)).is_none(),
4311 "the SURB through the dead relayer is written off"
4312 );
4313 assert!(
4314 registry.pending.get(&surb_id(2)).is_none() && registry.pending.get(&surb_id(3)).is_none(),
4315 "the healthy two are not written off, but the wipe takes them all the same: they will never be spent, so \
4316 they are no longer expected back"
4317 );
4318 assert!(
4319 registry.invalid().is_empty(),
4320 "the wipe takes the SURB that was counted, and the count goes with it"
4321 );
4322
4323 registry.pending.insert(surb_id(4), pending_to(&[dead], None, 0));
4324 registry.pending.insert(surb_id(5), pending_to(&[alive], None, 0));
4325 assert!(
4326 registry.write_off_dead().is_empty(),
4327 "a standing verdict writes off what still routes through the relayer, but bumps nothing more"
4328 );
4329 assert_eq!(
4330 HashMap::from([(*SESSION, (1, 1.0 / 2.0))]),
4331 registry.invalid(),
4332 "one SURB of the two minted since the wipe runs through the dead relayer"
4333 );
4334
4335 registry.relayer_health.dead.remove(&dead);
4336 assert!(registry.write_off_dead().is_empty());
4337 assert!(registry.invalid().is_empty(), "the write-off lapses with the verdict");
4338 }
4339
4340 #[test]
4342 fn two_dead_relayers_reaching_a_session_together_should_bump_it_once() {
4343 let registry = SurbRoundTripRegistry::default();
4344 let [dead, also_dead] = [(); 2].map(|_| key());
4345 registry.pending.insert(surb_id(1), pending_to(&[dead], None, 0));
4346 registry.pending.insert(surb_id(2), pending_to(&[also_dead], None, 0));
4347 judge_dead(®istry.relayer_health, dead);
4348 judge_dead(®istry.relayer_health, also_dead);
4349
4350 assert_eq!(vec![*SESSION], registry.write_off_dead());
4351 assert!(
4352 registry.invalid().is_empty(),
4353 "the bump took both counts with the stock"
4354 );
4355 assert_eq!(2, registry.bumped.len(), "both pairs are marked");
4356 }
4357
4358 #[test]
4363 fn a_verdict_refiring_within_the_cooldown_should_write_off_but_not_bump() {
4364 let registry = SurbRoundTripRegistry::default();
4365 let dead = key();
4366 registry.pending.insert(surb_id(1), pending_to(&[dead], None, 0));
4367 judge_dead(®istry.relayer_health, dead);
4368 assert_eq!(vec![*SESSION], registry.write_off_dead(), "the first verdict bumps");
4369
4370 registry.relayer_health.dead.remove(&dead);
4371 assert!(registry.write_off_dead().is_empty());
4372 assert!(
4373 registry.invalid().is_empty(),
4374 "nothing is written off for a relayer that recovered"
4375 );
4376
4377 judge_dead(®istry.relayer_health, dead);
4378 registry.pending.insert(surb_id(2), pending_to(&[dead], None, 0));
4379 assert!(
4380 registry.write_off_dead().is_empty(),
4381 "the same pair, re-reached inside the cooldown, is not bumped again"
4382 );
4383 assert_eq!(
4384 HashMap::from([(*SESSION, (1, 1.0))]),
4385 registry.invalid(),
4386 "but the SURB through the relayer is written off all the same"
4387 );
4388 }
4389
4390 #[test]
4392 fn the_cooldown_should_hold_per_relayer() {
4393 let registry = SurbRoundTripRegistry::default();
4394 let [first, second] = [(); 2].map(|_| key());
4395 registry.pending.insert(surb_id(1), pending_to(&[first], None, 0));
4396 judge_dead(®istry.relayer_health, first);
4397 assert_eq!(vec![*SESSION], registry.write_off_dead());
4398
4399 registry.pending.insert(surb_id(2), pending_to(&[second], None, 0));
4400 judge_dead(®istry.relayer_health, second);
4401 assert_eq!(
4402 vec![*SESSION],
4403 registry.write_off_dead(),
4404 "a different relayer is not covered by the first one's cooldown"
4405 );
4406 }
4407
4408 #[test]
4409 fn forgetting_a_session_should_drop_its_cooldown() {
4410 let registry = SurbRoundTripRegistry::default();
4411 let dead = key();
4412 let other = HoprPseudonym::random();
4413 registry.pending.insert(surb_id(1), pending_to(&[dead], None, 0));
4414 registry.pending.insert(
4415 surb_id(2),
4416 PendingSurb {
4417 pseudonym: other,
4418 ..pending_to(&[dead], None, 0)
4419 },
4420 );
4421 judge_dead(®istry.relayer_health, dead);
4422 let reached = registry.write_off_dead();
4423 assert_eq!(2, reached.len());
4424 assert_eq!(2, registry.bumped.len());
4425
4426 registry.forget_session(&SESSION);
4427 assert_eq!(
4428 vec![(other, dead)],
4429 registry.bumped.iter().map(|entry| *entry.key()).collect::<Vec<_>>(),
4430 "only the closed session's cooldown goes"
4431 );
4432
4433 registry.relayer_health.dead.remove(&dead);
4435 registry.write_off_dead();
4436 registry.pending.insert(surb_id(3), pending_to(&[dead], None, 0));
4438 judge_dead(®istry.relayer_health, dead);
4439 assert_eq!(vec![*SESSION], registry.write_off_dead());
4440 }
4441
4442 #[test]
4443 fn invalid_updates_should_reapply_every_mark_and_clear_the_vanished() {
4444 let [kept, gone, new] = [(); 3].map(|_| HoprPseudonym::random());
4445 let previous = HashMap::from([(kept, (5, 0.5)), (gone, (2, 0.1))]);
4446 let current = HashMap::from([(kept, (5, 0.5)), (new, (1, 1.0))]);
4447
4448 let mut updates = invalid_updates(&previous, ¤t);
4449 updates.sort_by_key(|(session, _)| *session);
4450 let mut expected = vec![(kept, (5, 0.5)), (new, (1, 1.0)), (gone, (0, 0.0))];
4451 expected.sort_by_key(|(session, _)| *session);
4452 assert_eq!(expected, updates, "unchanged marks too, for sessions opened since");
4453 }
4454
4455 #[tokio::test]
4458 async fn a_dead_verdict_should_reach_the_graph_before_the_replan_and_the_write_off_after() -> anyhow::Result<()> {
4459 let (graph, me, peers) = graph_with(1);
4460 let [dead] = peers[..] else {
4461 anyhow::bail!("one peer asked for");
4462 };
4463 graph.add_edge(&me, &dead)?;
4464 let registry = SurbRoundTripRegistry::default();
4465 registry.pending.insert(surb_id(1), pending_to(&[dead], None, 0));
4466 let health = registry.relayer_health();
4467 settled_since(&health, dead, ago(Duration::from_secs(10))?, jitter(0));
4468 open_since(&health, dead, MIN_RACES as u8, ago(Duration::from_secs(5))?);
4469 health.record_reply();
4470
4471 let seen_at_replan = std::cell::Cell::new(None);
4472 let first_flush = refresh_relayer_health(®istry, &graph, 4, 1.0, || async {
4473 let edge_health = graph.edge(&me, &dead).and_then(|edge| edge.health());
4474 seen_at_replan.set(Some((edge_health, registry.pending.get(&surb_id(1)).is_some())));
4475 })
4476 .await;
4477
4478 assert_eq!(
4479 Some((Some(0.0), true)),
4480 seen_at_replan.get(),
4481 "the graph has the verdict, and the SURB is not written off yet"
4482 );
4483 assert!(
4484 first_flush.is_empty() && registry.pending.get(&surb_id(1)).is_some(),
4485 "the verdict acts on the graph at once, but on the stock only once the next flush confirms it"
4486 );
4487
4488 confirm_verdicts(&health);
4490 let bumped = refresh_relayer_health(®istry, &graph, 4, 1.0, || async {}).await;
4491 assert!(
4492 registry.pending.get(&surb_id(1)).is_none(),
4493 "written off once confirmed: out of the pending map"
4494 );
4495 assert!(
4496 registry.invalid().is_empty(),
4497 "and the bump it led to took the count with the stock"
4498 );
4499 assert_eq!(
4500 vec![*SESSION],
4501 bumped,
4502 "the session the verdict reached is handed back for its SURB generation to be advanced"
4503 );
4504 Ok(())
4505 }
4506
4507 #[tokio::test]
4509 async fn a_confirmed_dead_verdict_should_write_off_nothing_at_weight_zero() -> anyhow::Result<()> {
4510 let (graph, me, peers) = graph_with(1);
4511 let [dead] = peers[..] else {
4512 anyhow::bail!("one peer asked for");
4513 };
4514 graph.add_edge(&me, &dead)?;
4515 let registry = SurbRoundTripRegistry::default();
4516 registry.pending.insert(surb_id(1), pending_to(&[dead], None, 0));
4517 let health = registry.relayer_health();
4518 settled_since(&health, dead, ago(Duration::from_secs(10))?, jitter(0));
4519 open_since(&health, dead, MIN_RACES as u8, ago(Duration::from_secs(5))?);
4520 health.record_reply();
4521
4522 refresh_relayer_health(®istry, &graph, 4, 0.0, || async {}).await;
4523 confirm_verdicts(&health);
4524 let bumped = refresh_relayer_health(®istry, &graph, 4, 0.0, || async {}).await;
4525
4526 assert!(bumped.is_empty(), "no session is handed back for a generation advance");
4527 assert!(!registry.has_write_offs(), "nothing is written off");
4528 Ok(())
4529 }
4530
4531 #[tokio::test]
4533 async fn an_edge_added_after_a_verdict_should_carry_it_from_the_next_flush() -> anyhow::Result<()> {
4534 let (graph, me, peers) = graph_with(1);
4535 let [dead] = peers[..] else {
4536 anyhow::bail!("one peer asked for");
4537 };
4538 let registry = SurbRoundTripRegistry::default();
4539 let health = registry.relayer_health();
4540 settled_since(&health, dead, ago(Duration::from_secs(10))?, jitter(0));
4541 open_since(&health, dead, MIN_RACES as u8, ago(Duration::from_secs(5))?);
4542 health.record_reply();
4543 refresh_relayer_health(®istry, &graph, 4, 1.0, || async {}).await;
4544
4545 graph.add_edge(&me, &dead)?;
4546 refresh_relayer_health(®istry, &graph, 4, 1.0, || async {}).await;
4547 assert_eq!(Some(0.0), graph.edge(&me, &dead).and_then(|edge| edge.health()));
4548 Ok(())
4549 }
4550
4551 #[derive(Clone, Copy)]
4553 enum Standing {
4554 Judged,
4555 Dead,
4556 OnProbation,
4557 }
4558
4559 #[rstest::rstest]
4562 #[case::as_judged(1.0, Standing::Judged, 0.1)]
4563 #[case::harsher(2.0, Standing::Judged, 0.01)]
4564 #[case::switched_off(0.0, Standing::Judged, 1.0)]
4565 #[case::dead(1.0, Standing::Dead, 0.0)]
4566 #[case::switched_off_for_the_dead(0.0, Standing::Dead, 1.0)]
4567 #[case::on_probation(1.0, Standing::OnProbation, PROBATION_HEALTH)]
4568 #[case::on_probation_tempered(0.5, Standing::OnProbation, 0.031_622_776_601_683_79)]
4569 #[case::switched_off_for_probation(0.0, Standing::OnProbation, 1.0)]
4570 #[tokio::test]
4571 async fn the_health_weight_should_shape_what_the_graph_gets(
4572 #[case] weight: f64,
4573 #[case] standing: Standing,
4574 #[case] recorded: f64,
4575 ) -> anyhow::Result<()> {
4576 let (graph, me, peers) = graph_with(1);
4577 let [relayer] = peers[..] else {
4578 anyhow::bail!("one peer asked for");
4579 };
4580 graph.add_edge(&me, &relayer)?;
4581 let registry = SurbRoundTripRegistry::default();
4582 let health = registry.relayer_health();
4583 match standing {
4584 Standing::Judged => settled_since(&health, relayer, ago(Duration::from_secs(1))?, Duration::from_secs(2)),
4585 Standing::Dead => {
4586 settled_since(&health, relayer, ago(Duration::from_secs(10))?, jitter(0));
4587 open_since(&health, relayer, MIN_RACES as u8, ago(Duration::from_secs(5))?);
4588 health.record_reply();
4589 }
4590 Standing::OnProbation => put_on_probation(&health, relayer)?,
4591 }
4592 refresh_relayer_health(®istry, &graph, 4, weight, || async {}).await;
4593
4594 let on_edge = graph.edge(&me, &relayer).and_then(|edge| edge.health()).unwrap_or(1.0);
4595 assert!((on_edge - recorded).abs() < 1e-9, "got {on_edge}");
4596 Ok(())
4597 }
4598
4599 #[test]
4603 fn a_first_surb_still_out_while_the_second_is_home_should_count_against_its_relayer() {
4604 let (graph, _me, _peers) = graph_with(0);
4605 let recorder = recorder(graph);
4606 let [late, home] = [(); 2].map(|_| key());
4607 let pending_via = |relayer, partner, pair_index| pending_to(&[relayer], Some(partner), pair_index);
4608 recorder
4609 .registry
4610 .pending
4611 .insert(surb_id(1), pending_via(late, surb_id(2), 0));
4612 recorder
4613 .registry
4614 .pending
4615 .insert(surb_id(2), pending_via(home, surb_id(1), 1));
4616
4617 recorder.on_replied(&surb_id(2));
4618 assert_eq!(1, open_races_of(&recorder.registry.relayer_health, &late));
4619
4620 recorder.on_replied(&surb_id(1));
4621 assert!(
4622 recorder
4623 .registry
4624 .relayer_health
4625 .relayers
4626 .get(&late)
4627 .is_some_and(|history| !history.races.is_empty()),
4628 "a late reply settles a race against its relayer"
4629 );
4630 }
4631
4632 #[test]
4637 fn under_fifo_the_first_surb_home_opens_no_race_and_the_second_home_first_does() {
4638 let (graph, _me, _peers) = graph_with(0);
4639 let recorder = recorder(graph);
4640 let health = &recorder.registry.relayer_health;
4641 let pending_via = |relayer, partner, pair_index| pending_to(&[relayer], Some(partner), pair_index);
4642
4643 let [first, second] = [(); 2].map(|_| key());
4645 recorder
4646 .registry
4647 .pending
4648 .insert(surb_id(1), pending_via(first, surb_id(2), 0));
4649 recorder
4650 .registry
4651 .pending
4652 .insert(surb_id(2), pending_via(second, surb_id(1), 1));
4653 recorder.on_replied(&surb_id(1));
4654 assert_eq!(0, open_races_of(health, &first));
4655 assert_eq!(
4656 0,
4657 open_races_of(health, &second),
4658 "the first coming home first opens no race on the second"
4659 );
4660 recorder.on_replied(&surb_id(2));
4661 assert!(margins_of(health, &first).is_empty() && margins_of(health, &second).is_empty());
4662
4663 let [spent_first, spent_second] = [(); 2].map(|_| key());
4665 recorder
4666 .registry
4667 .pending
4668 .insert(surb_id(3), pending_via(spent_first, surb_id(4), 0));
4669 recorder
4670 .registry
4671 .pending
4672 .insert(surb_id(4), pending_via(spent_second, surb_id(3), 1));
4673 recorder.on_replied(&surb_id(4));
4674 assert_eq!(
4675 1,
4676 open_races_of(health, &spent_first),
4677 "the first is out while the second is home"
4678 );
4679 assert_eq!(0, open_races_of(health, &spent_second));
4680 recorder.on_replied(&surb_id(3));
4681 assert_eq!(
4682 0,
4683 open_races_of(health, &spent_first),
4684 "and settled once it is home too"
4685 );
4686 assert_eq!(1, margins_of(health, &spent_first).len());
4687 assert_eq!(1, margins_of(health, &spent_second).len());
4688 }
4689
4690 #[test]
4696 fn a_pause_in_the_repliers_stream_must_not_inflate_the_overdue_bar_under_fifo() -> anyhow::Result<()> {
4697 let (graph, _me, _peers) = graph_with(0);
4698 let recorder = recorder(graph);
4699 let health = &recorder.registry.relayer_health;
4700 let [slow, other] = [(); 2].map(|_| key());
4701 let pause = Duration::from_secs(10);
4702
4703 for pair in 0..MIN_RACES as u8 {
4705 let (first, second) = (surb_id(1 + 2 * pair), surb_id(2 + 2 * pair));
4706 recorder
4707 .registry
4708 .pending
4709 .insert(first, pending_to(&[slow], Some(second), 0));
4710 recorder
4711 .registry
4712 .pending
4713 .insert(second, pending_to(&[other], Some(first), 1));
4714 recorder.on_replied(&first);
4715 let mut waiting = recorder
4716 .registry
4717 .pending
4718 .get(&second)
4719 .context("the second SURB is out")?;
4720 let (relayers, home) = waiting.partner_replied.context("its partner is home")?;
4721 waiting.partner_replied = Some((relayers, home.checked_sub(pause).context("clock")?));
4722 recorder.registry.pending.insert(second, waiting);
4723 recorder.on_replied(&second);
4724 }
4725 assert!(
4726 margins_of(health, &slow).is_empty() && margins_of(health, &other).is_empty(),
4727 "a pause between the replies of a pair settles no race"
4728 );
4729
4730 settled_since(health, slow, ago(Duration::from_secs(10))?, Duration::from_millis(50));
4733 open_since(health, slow, MIN_RACES as u8, ago(Duration::from_secs(1))?);
4734 assert_eq!(
4735 vec![slow],
4736 judged_dead(&health.refresh(4)),
4737 "overdue after its normal margin, not after the pause"
4738 );
4739 Ok(())
4740 }
4741
4742 #[test]
4745 fn time_waited_in_the_repliers_store_should_not_raise_the_bar_for_a_black_hole() -> anyhow::Result<()> {
4746 let health = RelayerHealth::default();
4747 let relayer = key();
4748 for _ in 0..MIN_RACES {
4749 health.record(
4750 relayer,
4751 Race {
4752 at: ago(Duration::from_secs(10))?,
4753 margin: jitter(0).as_secs_f64(),
4754 winning_round_trip: 54.0,
4755 },
4756 );
4757 }
4758 open_since(&health, relayer, MIN_RACES as u8, ago(Duration::from_secs(1))?);
4759 health.record_reply();
4760
4761 assert_eq!(vec![relayer], judged_dead(&health.refresh(4)));
4762 Ok(())
4763 }
4764
4765 #[test]
4769 fn three_lost_replies_and_a_pause_must_not_make_a_dead_verdict() -> anyhow::Result<()> {
4770 let health = RelayerHealth::default();
4771 let victim = key();
4772 settled_since(
4773 &health,
4774 victim,
4775 ago(Duration::from_secs(10))?,
4776 Duration::from_millis(50),
4777 );
4778 open_since(&health, victim, MIN_RACES as u8, ago(Duration::from_secs(5))?);
4779
4780 assert!(
4781 judged_dead(&health.refresh(4)).is_empty() && !health.is_dead(&victim),
4782 "five seconds of silence with nothing home anywhere is a pause, not a verdict"
4783 );
4784
4785 health.record_reply();
4786 assert_eq!(
4787 vec![victim],
4788 judged_dead(&health.refresh(4)),
4789 "a reply home through another relayer after they fell due makes them overdue"
4790 );
4791 Ok(())
4792 }
4793
4794 #[test]
4798 fn a_relayer_with_a_clean_history_can_still_be_judged_dead() -> anyhow::Result<()> {
4799 let health = RelayerHealth::default();
4800 let [measured, newcomer] = [(); 2].map(|_| key());
4801 settled_since(
4802 &health,
4803 measured,
4804 ago(Duration::from_secs(10))?,
4805 Duration::from_millis(100),
4806 );
4807 open_since(&health, newcomer, MIN_RACES as u8, ago(Duration::from_secs(1))?);
4808 health.record_reply();
4809
4810 assert_eq!(vec![newcomer], judged_dead(&health.refresh(4)));
4811 Ok(())
4812 }
4813
4814 #[test]
4818 fn expired_open_races_must_not_mask_a_black_hole() -> anyhow::Result<()> {
4819 let health = RelayerHealth::default();
4820 let [relayer, measured] = [(); 2].map(|_| key());
4821 settled_since(
4822 &health,
4823 relayer,
4824 ago(Duration::from_secs(70))?,
4825 Duration::from_millis(50),
4826 );
4827 settled_since(
4831 &health,
4832 measured,
4833 ago(Duration::from_secs(10))?,
4834 Duration::from_millis(50),
4835 );
4836 open_since(
4837 &health,
4838 relayer,
4839 MIN_RACES as u8,
4840 ago(PENDING_SURB_TTL + Duration::from_secs(5))?,
4841 );
4842 health.record_reply();
4843 assert!(
4844 judged_dead(&health.refresh(4)).is_empty(),
4845 "the races that expired are booked as lost, none is open any more"
4846 );
4847 assert_eq!(0, open_races_of(&health, &relayer));
4848
4849 open_since(&health, relayer, MIN_RACES as u8, ago(Duration::from_secs(1))?);
4850 health.record_reply();
4851 assert_eq!(
4852 vec![relayer],
4853 judged_dead(&health.refresh(4)),
4854 "three more overdue replies, with the network delivering, are a verdict however the earlier ones ended"
4855 );
4856 Ok(())
4857 }
4858
4859 #[test]
4861 fn without_any_margin_history_anywhere_nothing_is_judged_dead() -> anyhow::Result<()> {
4862 let health = RelayerHealth::default();
4863 let relayer = key();
4864 open_since(&health, relayer, MIN_RACES as u8, ago(Duration::from_secs(5))?);
4865 health.record_reply();
4866
4867 assert!(judged_dead(&health.refresh(4)).is_empty());
4868 assert!(!health.is_dead(&relayer));
4869 Ok(())
4870 }
4871
4872 #[test]
4874 fn a_total_return_outage_judges_no_relayer_dead() -> anyhow::Result<()> {
4875 let health = RelayerHealth::default();
4876 let relayers = [(); 3].map(|_| key());
4877 for relayer in relayers {
4878 settled_since(
4879 &health,
4880 relayer,
4881 ago(Duration::from_secs(10))?,
4882 Duration::from_millis(50),
4883 );
4884 open_since(&health, relayer, MIN_RACES as u8, ago(Duration::from_secs(5))?);
4885 }
4886
4887 assert!(judged_dead(&health.refresh(4)).is_empty());
4888 assert!(relayers.iter().all(|relayer| !health.is_dead(relayer)));
4889 Ok(())
4890 }
4891
4892 #[test]
4895 fn a_reply_home_before_a_race_fell_due_does_not_corroborate_it() -> anyhow::Result<()> {
4896 let mut health = RelayerHealth::default();
4897 health.last_reply.epoch = ago(Duration::from_secs(30))?;
4898 let victim = key();
4899 settled_since(
4900 &health,
4901 victim,
4902 ago(Duration::from_secs(10))?,
4903 Duration::from_millis(50),
4904 );
4905 open_since(&health, victim, MIN_RACES as u8, ago(Duration::from_secs(5))?);
4906
4907 health.record_reply_at(ago(Duration::from_secs(7))?);
4908 assert!(
4909 judged_dead(&health.refresh(4)).is_empty(),
4910 "the last reply home predates the races, so nothing was delivering while they were out"
4911 );
4912
4913 health.record_reply_at(ago(Duration::from_secs(2))?);
4914 assert_eq!(vec![victim], judged_dead(&health.refresh(4)));
4915 Ok(())
4916 }
4917
4918 #[test]
4920 fn symmetric_jitter_should_cost_neither_relayer() {
4921 let health = RelayerHealth::default();
4922 let [a, b] = [(); 2].map(|_| key());
4923 for i in 0..20 {
4924 let (later, earlier) = if i % 2 == 0 { (a, b) } else { (b, a) };
4925 health.record_race(
4926 &leg(&[later]),
4927 surb_id(i as u8),
4928 &leg(&[earlier]),
4929 ROUND_TRIP,
4930 jitter(i),
4931 );
4932 }
4933 assert_eq!(vec![1.0, 1.0], health_after_flush(&health, &[a, b]));
4934 }
4935
4936 #[test]
4937 fn replies_out_without_settled_races_should_not_make_a_black_hole() {
4938 let health = RelayerHealth::default();
4939 let relayer = key();
4940 open_since(&health, relayer, 5, Instant::now() - Duration::from_secs(5));
4941
4942 for _ in 0..2 {
4943 assert!(judged_dead(&health.refresh(4)).is_empty());
4944 }
4945 assert!(!health.is_dead(&relayer));
4946 }
4947
4948 #[test]
4949 fn surbs_should_race_their_neighbour_only_over_another_return_path() {
4950 let [a, b] = [(); 2].map(|_| key());
4951 let ids = [surb_id(1), surb_id(2), surb_id(3)];
4952
4953 assert_eq!(
4954 vec![Some(ids[1]), Some(ids[0]), None],
4955 race_partners(&ids, &[leg(&[a]), leg(&[b]), leg(&[a])]),
4956 "the odd one out has no partner"
4957 );
4958 assert_eq!(
4959 vec![None, None],
4960 race_partners(&ids[..2], &[leg(&[a, b]), leg(&[a, b])]),
4961 "a path cannot race itself"
4962 );
4963 assert_eq!(
4964 vec![None, None],
4965 race_partners(&ids[..2], &[leg(&[]), leg(&[b])]),
4966 "a direct reply has no relayer to blame"
4967 );
4968 }
4969
4970 #[test]
4972 fn five_surbs_in_one_packet_should_race_in_adjacent_pairs() {
4973 let [a, b] = [(); 2].map(|_| key());
4974 let ids = [1, 2, 3, 4, 5].map(surb_id);
4975 assert_eq!(
4976 vec![Some(ids[1]), Some(ids[0]), Some(ids[3]), Some(ids[2]), None],
4977 race_partners(&ids, &[leg(&[a]), leg(&[b]), leg(&[b]), leg(&[a]), leg(&[a])])
4978 );
4979 }
4980
4981 #[test]
4982 fn round_trip_paths_should_join_the_legs_at_the_destination() -> anyhow::Result<()> {
4983 let (graph, me, peers) = graph_with(1);
4984 let destination = peers[0];
4985
4986 let paths = round_trip_paths(&path_slots_of(graph.clone()), &me, &[destination], &[me])
4987 .context("both nodes are in the graph")?;
4988
4989 let me_slot = graph.path_slot(&me).context("self is in the graph")?;
4990 let dest_slot = graph.path_slot(&destination).context("destination is in the graph")?;
4991
4992 assert_eq!([me_slot, dest_slot, 0, 0, 0], paths.forward);
4995 assert_eq!([dest_slot, me_slot, 0, 0, 0], paths.reply);
4996 Ok(())
4997 }
4998
4999 #[test]
5000 fn round_trip_paths_should_be_none_when_a_node_is_unknown_to_the_graph() {
5001 let (graph, me, _) = graph_with(0);
5002 let stranger = *OffchainKeypair::random().public();
5003
5004 assert!(round_trip_paths(&path_slots_of(graph), &me, &[stranger], &[me]).is_none());
5007 }
5008
5009 #[test]
5010 fn round_trip_paths_should_be_none_for_a_leg_too_long_to_identify() {
5011 let (graph, me, peers) = graph_with(5);
5012 let forward: Vec<_> = peers.clone();
5013
5014 assert!(round_trip_paths(&path_slots_of(graph), &me, &forward, &[me]).is_none());
5016 }
5017
5018 #[test]
5019 fn minting_a_surb_should_raise_an_expectation_over_the_legs_it_was_minted_on() -> anyhow::Result<()> {
5020 let (graph, me, peers) = graph_with(1);
5021 let destination = peers[0];
5022 let recorder = recorder(graph);
5023
5024 let routing = ResolvedTransportRouting::Forward {
5025 pseudonym: HoprPseudonym::random(),
5026 forward_path: ValidatedPath::direct(destination, address()),
5027 return_paths: vec![ValidatedPath::direct(me, address())],
5028 };
5029
5030 recorder.on_minted(&routing, &[surb_id(1)]);
5031
5032 let drained = recorder.registry.drain();
5033 assert_eq!(1, drained.len());
5034 let (_, expected, observed) = drained[0];
5035 assert_eq!(1, expected);
5036 assert_eq!(0, observed, "nothing has come back yet");
5037 Ok(())
5038 }
5039
5040 #[test]
5041 fn a_reply_should_be_credited_to_the_legs_its_surb_was_minted_on() -> anyhow::Result<()> {
5042 let (graph, me, peers) = graph_with(1);
5043 let destination = peers[0];
5044 let recorder = recorder(graph);
5045
5046 let routing = ResolvedTransportRouting::Forward {
5047 pseudonym: HoprPseudonym::random(),
5048 forward_path: ValidatedPath::direct(destination, address()),
5049 return_paths: vec![ValidatedPath::direct(me, address())],
5050 };
5051
5052 recorder.on_minted(&routing, &[surb_id(1)]);
5053 recorder.on_replied(&surb_id(1));
5054
5055 let drained = recorder.registry.drain();
5056 assert_eq!(1, drained.len());
5057 let (_, expected, observed) = drained[0];
5058 assert_eq!((1, 1), (expected, observed));
5059 Ok(())
5060 }
5061
5062 #[test]
5063 fn a_surb_should_only_be_credited_once() -> anyhow::Result<()> {
5064 let (graph, me, peers) = graph_with(1);
5065 let destination = peers[0];
5066 let recorder = recorder(graph);
5067
5068 let routing = ResolvedTransportRouting::Forward {
5069 pseudonym: HoprPseudonym::random(),
5070 forward_path: ValidatedPath::direct(destination, address()),
5071 return_paths: vec![ValidatedPath::direct(me, address())],
5072 };
5073
5074 recorder.on_minted(&routing, &[surb_id(1)]);
5075 recorder.on_replied(&surb_id(1));
5076 recorder.on_replied(&surb_id(1));
5079
5080 let (_, expected, observed) = recorder.registry.drain()[0];
5081 assert_eq!((1, 1), (expected, observed));
5082 Ok(())
5083 }
5084
5085 #[test]
5086 fn flushing_should_leave_nothing_behind_to_report_twice() -> anyhow::Result<()> {
5087 let (graph, me, peers) = graph_with(1);
5088 let destination = peers[0];
5089 let recorder = recorder(graph.clone());
5090
5091 let routing = ResolvedTransportRouting::Forward {
5092 pseudonym: HoprPseudonym::random(),
5093 forward_path: ValidatedPath::direct(destination, address()),
5094 return_paths: vec![ValidatedPath::direct(me, address())],
5095 };
5096 recorder.on_minted(&routing, &[surb_id(1)]);
5097 recorder.on_replied(&surb_id(1));
5098
5099 flush_into(&recorder.registry, &graph, 0);
5100
5101 assert!(recorder.registry.drain().is_empty());
5104 Ok(())
5105 }
5106
5107 #[test]
5108 fn an_idle_flush_should_log_nothing_at_info() -> anyhow::Result<()> {
5109 let (graph, me, peers) = graph_with(1);
5110 let destination = peers[0];
5111
5112 let idle = Arc::new(InfoCounter::default());
5113 tracing::subscriber::with_default(idle.clone(), || {
5114 flush_into(&SurbRoundTripRegistry::default(), &graph, 0)
5115 });
5116 assert_eq!(0, idle.infos.load(Ordering::Relaxed), "nothing was minted or replied");
5117
5118 let recorder = recorder(graph.clone());
5119 recorder.on_minted(
5120 &ResolvedTransportRouting::Forward {
5121 pseudonym: HoprPseudonym::random(),
5122 forward_path: ValidatedPath::direct(destination, address()),
5123 return_paths: vec![ValidatedPath::direct(me, address())],
5124 },
5125 &[surb_id(1)],
5126 );
5127 let busy = Arc::new(InfoCounter::default());
5128 tracing::subscriber::with_default(busy.clone(), || flush_into(&recorder.registry, &graph, 0));
5129 assert_eq!(
5130 1,
5131 busy.infos.load(Ordering::Relaxed),
5132 "one summary line for the interval"
5133 );
5134 Ok(())
5135 }
5136
5137 #[test]
5138 fn routine_surb_traffic_should_not_emit_debug_per_mint_or_pair() -> anyhow::Result<()> {
5140 let (graph, me, peers) = graph_with(1);
5141 let recorder = recorder(graph.clone());
5142 let logs = Arc::new(InfoCounter::default());
5143 tracing::subscriber::with_default(logs.clone(), || {
5144 for _ in 0..1000 {
5145 recorder.on_minted(
5146 &ResolvedTransportRouting::Forward {
5147 pseudonym: HoprPseudonym::random(),
5148 forward_path: ValidatedPath::direct(peers[0], address()),
5149 return_paths: vec![ValidatedPath::direct(me, address())],
5150 },
5151 &[surb_id(1)],
5152 );
5153 recorder.on_replied(&surb_id(1));
5154 }
5155 flush_into(&recorder.registry, &graph, 0);
5156 });
5157 assert_eq!(logs.debugs.load(Ordering::Relaxed), 0);
5158 assert_eq!(logs.infos.load(Ordering::Relaxed), 1);
5159 Ok(())
5160 }
5161
5162 #[derive(Default)]
5164 struct InfoCounter {
5165 infos: AtomicU64,
5166 debugs: AtomicU64,
5167 }
5168
5169 impl tracing::Subscriber for InfoCounter {
5170 fn enabled(&self, _: &tracing::Metadata<'_>) -> bool {
5171 true
5172 }
5173
5174 fn new_span(&self, _: &tracing::span::Attributes<'_>) -> tracing::span::Id {
5175 tracing::span::Id::from_u64(1)
5176 }
5177
5178 fn record(&self, _: &tracing::span::Id, _: &tracing::span::Record<'_>) {}
5179
5180 fn record_follows_from(&self, _: &tracing::span::Id, _: &tracing::span::Id) {}
5181
5182 fn event(&self, event: &tracing::Event<'_>) {
5184 if *event.metadata().level() == tracing::Level::INFO {
5185 self.infos.fetch_add(1, Ordering::Relaxed);
5186 }
5187 if *event.metadata().level() == tracing::Level::DEBUG
5188 && event.metadata().target() == "hopr_transport::protocol::surb_telemetry"
5189 {
5190 self.debugs.fetch_add(1, Ordering::Relaxed);
5191 }
5192 }
5193
5194 fn enter(&self, _: &tracing::span::Id) {}
5195
5196 fn exit(&self, _: &tracing::span::Id) {}
5197 }
5198
5199 #[test]
5200 fn a_reply_should_be_credited_when_the_halves_are_separate_instances() -> anyhow::Result<()> {
5201 let (graph, me, peers) = graph_with(1);
5205 let destination = peers[0];
5206 let registry = SurbRoundTripRegistry::default();
5207
5208 let minting = SurbTelemetryCodec::new((), me, path_slots_of(graph.clone()), registry.clone());
5209 let observing = SurbTelemetryCodec::new((), me, path_slots_of(graph), registry.clone());
5210
5211 minting.on_minted(
5212 &ResolvedTransportRouting::Forward {
5213 pseudonym: HoprPseudonym::random(),
5214 forward_path: ValidatedPath::direct(destination, address()),
5215 return_paths: vec![ValidatedPath::direct(me, address())],
5216 },
5217 &[surb_id(1)],
5218 );
5219 observing.on_replied(&surb_id(1));
5220
5221 let (_, expected, observed) = registry.drain()[0];
5222 assert_eq!(
5223 (1, 1),
5224 (expected, observed),
5225 "the reply must reach the legs the mint recorded"
5226 );
5227 Ok(())
5228 }
5229
5230 #[test]
5231 fn a_reply_on_a_surb_we_never_minted_should_be_ignored() {
5232 let (graph, ..) = graph_with(1);
5233 let recorder = recorder(graph);
5234
5235 recorder.on_replied(&surb_id(9));
5236
5237 assert!(recorder.registry.drain().is_empty());
5238 }
5239
5240 #[test]
5243 fn a_verdict_must_hold_for_a_second_flush_before_the_stock_is_touched() -> anyhow::Result<()> {
5244 let registry = SurbRoundTripRegistry::default();
5245 let health = registry.relayer_health();
5246 let [dead, other] = [(); 2].map(|_| key());
5247 registry.pending.insert(surb_id(1), pending_to(&[dead], None, 0));
5248 settled_since(
5249 &health,
5250 other,
5251 ago(Duration::from_secs(10))?,
5252 Duration::from_millis(100),
5253 );
5254 open_since(&health, dead, MIN_RACES as u8, ago(Duration::from_secs(5))?);
5255 health.record_reply();
5256
5257 assert_eq!(vec![dead], judged_dead(&health.refresh(4)), "the verdict is made");
5258 assert!(health.is_dead(&dead));
5259 assert!(
5260 registry.write_off_dead().is_empty() && registry.pending.get(&surb_id(1)).is_some(),
5261 "nothing is written off or bumped in the flush of the verdict"
5262 );
5263
5264 confirm_verdicts(&health);
5265 assert_eq!(
5266 vec![*SESSION],
5267 registry.write_off_dead(),
5268 "the next flush finds the relayer still dead and acts"
5269 );
5270 assert!(registry.pending.get(&surb_id(1)).is_none());
5271 Ok(())
5272 }
5273
5274 #[test]
5277 fn a_verdict_revived_before_the_second_flush_costs_no_wipe() -> anyhow::Result<()> {
5278 let registry = SurbRoundTripRegistry::default();
5279 let health = registry.relayer_health();
5280 let [slow, fast] = [(); 2].map(|_| key());
5281 registry.pending.insert(surb_id(1), pending_to(&[slow], None, 0));
5282 settled_since(&health, fast, ago(Duration::from_secs(10))?, Duration::from_millis(50));
5283 open_since(&health, slow, MIN_RACES as u8, ago(Duration::from_millis(500))?);
5284 health.record_reply();
5285
5286 assert_eq!(vec![slow], judged_dead(&health.refresh(4)));
5287 assert!(registry.write_off_dead().is_empty(), "not acted on yet");
5288
5289 health.revive_by_reply(&[Some(slow), None, None, None, None], ago(Duration::from_millis(500))?);
5292 assert!(!health.is_known_dead(&slow));
5293 confirm_verdicts(&health);
5294 assert!(
5295 registry.write_off_dead().is_empty(),
5296 "revived in time: no write-off, no bump"
5297 );
5298 assert!(registry.pending.get(&surb_id(1)).is_some());
5299 assert!(registry.invalid().is_empty());
5300 Ok(())
5301 }
5302
5303 #[test]
5307 fn a_relayer_revived_by_a_reply_should_stay_alive_at_the_next_flush() -> anyhow::Result<()> {
5308 let (graph, _me, _peers) = graph_with(0);
5309 let recorder = recorder(graph);
5310 let registry = &recorder.registry;
5311 let health = registry.relayer_health();
5312 let [slow, fast] = [(); 2].map(|_| key());
5313 settled_since(&health, fast, ago(Duration::from_secs(10))?, Duration::from_millis(50));
5314 for seed in 1..=3u8 {
5315 registry
5316 .pending
5317 .insert(surb_id(seed), pending_to(&[slow], Some(surb_id(100 + seed)), 0));
5318 health.open_race(&leg(&[slow]), surb_id(seed), &leg(&[fast]), ROUND_TRIP);
5319 }
5320 backdate_open(&health, slow, ago(Duration::from_millis(500))?);
5321 health.record_reply();
5322 health.refresh(4);
5323 anyhow::ensure!(health.is_dead(&slow), "a black hole is judged dead");
5324
5325 registry.pending.insert(surb_id(50), pending_to(&[slow], None, 0));
5327 recorder.on_replied(&surb_id(50));
5328 assert!(!health.is_known_dead(&slow), "revived by the reply");
5329 health.refresh(4);
5330 assert!(
5331 !health.is_known_dead(&slow),
5332 "the races left open before the reply are not a black hole after it"
5333 );
5334 assert!(
5335 registry.write_off_dead().is_empty(),
5336 "and nothing is written off or advanced"
5337 );
5338 Ok(())
5339 }
5340
5341 #[test]
5346 fn a_stale_overdue_race_should_not_hold_back_the_verdict_on_a_relayer_gone_silent() -> anyhow::Result<()> {
5347 let health = RelayerHealth::default();
5348 let silent = key();
5349 settled_since(&health, silent, ago(Duration::from_secs(40))?, jitter(0));
5350 let stale = ago(Duration::from_secs(50))?;
5352 if let Some(mut history) = health.relayers.get_mut(&silent) {
5353 history.open.insert(
5354 surb_id(100),
5355 Race {
5356 at: stale,
5357 margin: 0.0,
5358 winning_round_trip: ROUND_TRIP.as_secs_f64(),
5359 },
5360 );
5361 history.open_legs.insert(surb_id(100), (leg(&[silent]), 0));
5362 history
5364 .last_home
5365 .insert(Hop::FromReplier, ago(Duration::from_secs(20))?);
5366 }
5367 open_since(&health, silent, MIN_RACES as u8, ago(Duration::from_secs(5))?);
5368 health.record_reply();
5370
5371 assert_eq!(vec![silent], judged_dead(&health.refresh(4)));
5372 Ok(())
5373 }
5374
5375 #[test]
5378 fn a_verdict_that_lapses_before_it_is_confirmed_should_be_withdrawn() -> anyhow::Result<()> {
5379 let registry = SurbRoundTripRegistry::default();
5380 let health = registry.relayer_health();
5381 let [slow, fast] = [(); 2].map(|_| key());
5382 registry.pending.insert(surb_id(1), pending_to(&[slow], None, 0));
5383 settled_since(&health, fast, ago(Duration::from_secs(10))?, Duration::from_millis(50));
5384 open_since(&health, slow, MIN_RACES as u8, ago(Duration::from_millis(500))?);
5385 health.record_reply();
5386 assert_eq!(vec![slow], judged_dead(&health.refresh(4)));
5387 assert!(
5388 registry.write_off_dead().is_empty(),
5389 "not acted on in the flush of the verdict"
5390 );
5391
5392 settled_since(&health, fast, Instant::now(), Duration::from_secs(3));
5394 health.refresh(4);
5395 assert!(!health.is_known_dead(&slow), "withdrawn, neither dead nor on probation");
5396 assert!(
5397 registry.write_off_dead().is_empty(),
5398 "and nothing is written off or advanced"
5399 );
5400 assert!(
5401 registry.pending.get(&surb_id(1)).is_some(),
5402 "its SURB is still expected back"
5403 );
5404 Ok(())
5405 }
5406
5407 #[test]
5411 fn a_second_relayer_confirmed_after_a_bump_should_not_bump_for_the_wiped_stock() {
5412 let registry = SurbRoundTripRegistry::default();
5413 let health = registry.relayer_health();
5414 let [first, second] = [(); 2].map(|_| key());
5415 registry.pending.insert(surb_id(1), pending_to(&[first], None, 0));
5416 registry.pending.insert(surb_id(2), pending_to(&[second], None, 0));
5417 judge_dead(&health, first);
5418 health.mark_dead(second);
5419 assert_eq!(vec![*SESSION], registry.write_off_dead(), "the first bump");
5420
5421 confirm_verdicts(&health);
5422 assert!(
5423 registry.write_off_dead().is_empty(),
5424 "the second relayer's SURB went with the first wipe"
5425 );
5426
5427 registry.pending.insert(surb_id(3), pending_to(&[second], None, 0));
5428 assert!(
5429 registry.write_off_dead().is_empty(),
5430 "the wipe settled the second relayer too: a SURB through it minted since is written off, not a reason to \
5431 drop the counterparty's fresh stock again"
5432 );
5433 assert_eq!(HashMap::from([(*SESSION, (1, 1.0))]), registry.invalid());
5434 }
5435
5436 #[test]
5441 fn a_reply_on_a_wiped_surb_should_revive_a_relayer_judged_in_the_same_flush() {
5442 let (graph, _me, _peers) = graph_with(0);
5443 let recorder = recorder(graph);
5444 let registry = &recorder.registry;
5445 let health = registry.relayer_health();
5446 let [dead, straggler] = [(); 2].map(|_| key());
5447 registry.pending.insert(surb_id(1), pending_to(&[dead], None, 0));
5448 registry.pending.insert(surb_id(2), pending_to(&[straggler], None, 0));
5449 judge_dead(&health, dead);
5450 health.mark_dead(straggler);
5451 assert_eq!(
5452 vec![*SESSION],
5453 registry.write_off_dead(),
5454 "the confirmed verdict wipes the stock"
5455 );
5456 assert!(
5457 registry.pending.get(&surb_id(2)).is_none(),
5458 "the straggler's SURB went with the wipe"
5459 );
5460
5461 recorder.on_replied(&surb_id(2));
5462 assert!(
5463 !health.is_known_dead(&straggler),
5464 "a reply on a wiped SURB revives the relayer it came home through"
5465 );
5466 assert!(health.is_known_dead(&dead), "and only that one");
5467 }
5468
5469 #[rstest::rstest]
5476 #[case::still_pending(false)]
5477 #[case::written_off(true)]
5478 fn a_late_reply_should_not_revive_a_relayer_once_its_verdict_is_confirmed(
5479 #[case] written_off_first: bool,
5480 ) -> anyhow::Result<()> {
5481 let (graph, _me, _peers) = graph_with(0);
5482 let recorder = recorder(graph);
5483 let registry = &recorder.registry;
5484 let health = registry.relayer_health();
5485 let [stalled, reference] = [(); 2].map(|_| key());
5486 settled_since(&health, reference, ago(Duration::from_secs(10))?, jitter(0));
5488 open_since(&health, stalled, MIN_RACES as u8, ago(Duration::from_secs(2))?);
5489 let mut late = pending_to(&[stalled], None, 0);
5490 late.minted_at = ago(Duration::from_secs(2))?;
5491 registry.pending.insert(surb_id(1), late);
5492 health.mark_dead(stalled);
5493 confirm_verdicts(&health);
5494 if written_off_first {
5495 registry.write_off_dead();
5496 assert!(
5497 registry.pending.get(&surb_id(1)).is_none(),
5498 "written off with the verdict"
5499 );
5500 }
5501
5502 recorder.on_replied(&surb_id(1));
5503 assert!(
5504 health.is_dead(&stalled),
5505 "a reply on a SURB minted before the confirmed verdict does not revive it"
5506 );
5507 assert!(
5508 health
5509 .relayers
5510 .get(&stalled)
5511 .is_none_or(|history| history.last_home.is_empty()),
5512 "nor does it answer the races the verdict was about"
5513 );
5514 health.record_reply();
5515 health.refresh(4);
5516 assert!(
5517 health.is_dead(&stalled),
5518 "so the verdict still stands at the next flush"
5519 );
5520
5521 registry.pending.insert(surb_id(2), pending_to(&[stalled], None, 0));
5522 recorder.on_replied(&surb_id(2));
5523 assert!(
5524 !health.is_known_dead(&stalled),
5525 "a reply on a SURB minted after the verdict revives it"
5526 );
5527 Ok(())
5528 }
5529
5530 #[test]
5537 fn a_probe_through_a_relayer_the_wipe_covered_should_not_bump_again_after_it_lapses() {
5538 let registry = SurbRoundTripRegistry::default();
5539 let health = registry.relayer_health();
5540 let [first, second, third] = [(); 3].map(|_| key());
5541 registry.pending.insert(surb_id(1), pending_to(&[first], None, 0));
5542 judge_dead(&health, first);
5543 health.mark_dead(second);
5544 health.mark_dead(third);
5545 assert_eq!(vec![*SESSION], registry.write_off_dead(), "the one bump");
5546
5547 confirm_verdicts(&health);
5548 for relayer in [first, second, third] {
5550 health.dead.remove(&relayer);
5551 health.probation.insert(relayer);
5552 }
5553 registry.pending.insert(surb_id(2), pending_to(&[third], None, 0));
5554 assert!(
5555 registry.write_off_dead().is_empty(),
5556 "a probe through a relayer the wipe covered must not advance the generation again"
5557 );
5558 assert_eq!(
5559 HashMap::from([(*SESSION, (1, 1.0))]),
5560 registry.invalid(),
5561 "it is written off"
5562 );
5563 }
5564
5565 #[test]
5568 fn write_offs_should_age_out_once_the_counterparty_has_spent_them() -> anyhow::Result<()> {
5569 let registry = SurbRoundTripRegistry::default();
5570 let health = registry.relayer_health();
5571 let dead = key();
5572 health.known_dead_since.insert(dead, Instant::now());
5574 health.probation.insert(dead);
5575 registry.bumped.insert((*SESSION, dead), Instant::now());
5577 registry.pending.insert(surb_id(1), pending_to(&[dead], None, 0));
5578 assert!(registry.write_off_dead().is_empty());
5579 assert_eq!(HashMap::from([(*SESSION, (1, 1.0))]), registry.invalid());
5580
5581 for mut session in registry.written_off.iter_mut() {
5582 for written_off_at in session.values_mut() {
5583 for at in written_off_at.iter_mut() {
5584 *at = ago(WRITE_OFF_HORIZON + Duration::from_secs(1))?;
5585 }
5586 }
5587 }
5588 assert!(registry.write_off_dead().is_empty());
5589 assert!(
5590 registry.invalid().is_empty(),
5591 "spent by the counterparty since, it is no longer counted"
5592 );
5593 Ok(())
5594 }
5595 #[test]
5596 fn observed_callback_can_access_the_destination_without_a_held_guard() {
5598 let (_, _, peers) = graph_with(1);
5599 let destination = peers[0];
5600 let paths = ForwardAndReturnPath {
5601 forward: [0; 5],
5602 reply: [0; 5],
5603 };
5604 let registry = SurbRoundTripRegistry::default();
5605 registry.record_minted(paths, 3, destination);
5606 let destinations = registry.destinations.clone();
5607 let calls = Arc::new(std::sync::atomic::AtomicU64::new(0));
5608 let registry = registry.with_observer(Arc::new({
5609 let calls = calls.clone();
5610 move |event| {
5611 assert!(matches!(
5613 destinations.try_get_mut(&paths),
5614 dashmap::try_result::TryResult::Present(_)
5615 ));
5616 assert!(matches!(
5617 event,
5618 TransportEvent::Surb {
5619 minted: 0,
5620 returned: 2,
5621 ..
5622 }
5623 ));
5624 calls.fetch_add(1, Ordering::Relaxed);
5625 }
5626 }));
5627 registry.record_observed(paths, 2);
5628 assert_eq!(calls.load(Ordering::Relaxed), 1);
5629 assert_eq!(registry.drain(), vec![(paths, 3, 2)]);
5630 }
5631
5632 #[test]
5633 fn observing_an_unknown_destination_keeps_counts_without_notifying() {
5635 let paths = ForwardAndReturnPath {
5636 forward: [0; 5],
5637 reply: [0; 5],
5638 };
5639 let registry = SurbRoundTripRegistry::default().with_observer(Arc::new(|_| {
5640 panic!("unknown destinations must not notify the observer");
5641 }));
5642 registry.record_observed(paths, 2);
5643 assert_eq!(registry.drain(), vec![(paths, 0, 2)]);
5644 }
5645
5646 #[test]
5647 fn observed_surb_counts_are_independent_of_graph_drains_and_duplicate_replies() -> anyhow::Result<()> {
5649 let (graph, me, peers) = graph_with(1);
5650 let events = Arc::new(parking_lot::Mutex::new(Vec::new()));
5651 let mut recorder = recorder(graph);
5652 recorder.registry = recorder.registry.with_observer(Arc::new({
5653 let events = events.clone();
5654 move |event| events.lock().push(event)
5655 }));
5656 let surbs = |events: &mut Vec<TransportEvent>| -> (u64, u64, Vec<OffchainPublicKey>) {
5657 events
5658 .drain(..)
5659 .fold((0, 0, Vec::new()), |(m, r, mut d), event| match event {
5660 TransportEvent::Surb {
5661 destination,
5662 minted,
5663 returned,
5664 ..
5665 } => {
5666 d.push(destination);
5667 (m + minted, r + returned, d)
5668 }
5669 _ => (m, r, d),
5670 })
5671 };
5672 recorder.on_minted(
5673 &ResolvedTransportRouting::Forward {
5674 pseudonym: HoprPseudonym::random(),
5675 forward_path: ValidatedPath::direct(peers[0], address()),
5676 return_paths: vec![ValidatedPath::direct(me, address())],
5677 },
5678 &[surb_id(1)],
5679 );
5680 let (minted, returned, _) = surbs(&mut events.lock());
5681 assert_eq!((minted, returned), (1, 0));
5682 assert_eq!(recorder.registry.drain().len(), 1);
5683 recorder.on_replied(&surb_id(1));
5684 recorder.on_replied(&surb_id(1));
5685 assert_eq!(recorder.registry.drain().len(), 1);
5686 let (minted, returned, destinations) = surbs(&mut events.lock());
5687 assert_eq!((minted, returned), (0, 1), "a duplicate reply is not counted twice");
5688 assert_eq!(destinations, vec![peers[0]]);
5689 Ok(())
5690 }
5691
5692 fn wide_id(n: u32) -> HoprSurbId {
5694 let mut id = [0u8; SURB_ID_SIZE];
5695 id[..size_of::<u32>()].copy_from_slice(&n.to_le_bytes());
5696 id
5697 }
5698
5699 const SEED: u64 = 0x5EED;
5700
5701 #[test]
5705 fn a_full_pending_map_should_keep_the_newest_record() {
5706 let registry = SurbRoundTripRegistry::with_records(64, SEED);
5707 for seed in 0..=64u8 {
5711 registry.remember(surb_id(seed), pending_to(&[], None, 0));
5712 }
5713
5714 assert!(
5715 registry.pending.contains_key(&surb_id(64)),
5716 "the newest record must be admitted, not refused as TinyLFU would"
5717 );
5718 assert_eq!(64, registry.pending.len(), "the map stays at capacity");
5719 assert_eq!(
5720 1,
5721 registry.take_pending_lost(),
5722 "the 65th mint into a 64-record map evicts exactly one record, and it is counted"
5723 );
5724 }
5725
5726 #[test]
5740 fn under_overload_no_session_should_lose_all_its_replies() {
5741 const CAPACITY: u32 = 2_000;
5742 const QUIET_WAIT_FACTOR: u32 = 3;
5744 const QUIET_EVERY: u32 = 8;
5746
5747 for (overload, min_busy, min_quiet) in [(1.05, 0.7, 0.5), (2.0, 0.1, 0.01)] {
5748 let registry = SurbRoundTripRegistry::with_records(CAPACITY as usize, SEED);
5749 let [busy, quiet] = [(); 2].map(|_| HoprPseudonym::random());
5750 let busy_share = f64::from(QUIET_EVERY - 1) / f64::from(QUIET_EVERY);
5752 let quiet_share = 1.0 / f64::from(QUIET_EVERY);
5753 let busy_wait =
5754 (overload * f64::from(CAPACITY) / (busy_share + quiet_share * f64::from(QUIET_WAIT_FACTOR))) as u32;
5755 let quiet_wait = QUIET_WAIT_FACTOR * busy_wait;
5756 let is_quiet = |tick: u32| tick.is_multiple_of(QUIET_EVERY);
5757
5758 let [mut busy_matched, mut busy_total, mut quiet_matched, mut quiet_total] = [0u32; 4];
5759 for tick in 0..20 * quiet_wait {
5761 let session = if is_quiet(tick) { quiet } else { busy };
5762 registry.remember(
5763 wide_id(tick),
5764 PendingSurb {
5765 pseudonym: session,
5766 ..pending_to(&[], None, 0)
5767 },
5768 );
5769 for (wait, quiet_session) in [(busy_wait, false), (quiet_wait, true)] {
5771 let Some(minted) = tick
5772 .checked_sub(wait)
5773 .filter(|minted| is_quiet(*minted) == quiet_session)
5774 else {
5775 continue;
5776 };
5777 let matched = registry.pending.remove(&wide_id(minted)).is_some();
5778 if tick >= 4 * quiet_wait {
5780 let (matches, total) = if quiet_session {
5781 (&mut quiet_matched, &mut quiet_total)
5782 } else {
5783 (&mut busy_matched, &mut busy_total)
5784 };
5785 *matches += u32::from(matched);
5786 *total += 1;
5787 }
5788 }
5789 }
5790
5791 let (busy_rate, quiet_rate) = (
5792 f64::from(busy_matched) / f64::from(busy_total),
5793 f64::from(quiet_matched) / f64::from(quiet_total),
5794 );
5795 assert!(
5796 busy_rate >= min_busy,
5797 "at {overload}x the busy Session matched {busy_rate:.3} of its replies"
5798 );
5799 assert!(
5800 quiet_rate >= min_quiet,
5801 "at {overload}x the Session with the long wait matched {quiet_rate:.3} of its replies, which an \
5802 eviction by age would take to zero"
5803 );
5804 }
5805 }
5806
5807 #[test]
5812 fn a_quiet_session_holding_surbs_through_a_dead_relayer_should_be_reached_under_overload() {
5813 const CAPACITY: u32 = 2_000;
5814 const QUIET_STOCK: u32 = 500;
5815 const OVERLOAD: u32 = 2;
5817
5818 let registry = SurbRoundTripRegistry::with_records(CAPACITY as usize, SEED);
5819 let [quiet, busy] = [(); 2].map(|_| HoprPseudonym::random());
5820 let [dead, alive] = [(); 2].map(|_| key());
5821
5822 for n in 0..QUIET_STOCK {
5825 registry.remember(
5826 wide_id(n),
5827 PendingSurb {
5828 pseudonym: quiet,
5829 ..pending_to(&[dead], None, 0)
5830 },
5831 );
5832 }
5833 let wait = OVERLOAD * CAPACITY;
5836 for tick in 0..4 * CAPACITY {
5837 registry.remember(
5838 wide_id(QUIET_STOCK + tick),
5839 PendingSurb {
5840 pseudonym: busy,
5841 ..pending_to(&[alive], None, 0)
5842 },
5843 );
5844 if let Some(minted) = tick.checked_sub(wait) {
5845 registry.pending.remove(&wide_id(QUIET_STOCK + minted));
5846 }
5847 }
5848 let mut held = 0;
5849 registry
5850 .pending
5851 .for_each(|_, pending| held += u32::from(pending.pseudonym == quiet));
5852 assert!(held > 0, "some of the quiet Session's stock must still be recorded");
5853
5854 judge_dead(®istry.relayer_health, dead);
5855 assert_eq!(
5856 vec![quiet],
5857 registry.write_off_dead(),
5858 "the dead relayer reaches the quiet Session, and only it"
5859 );
5860 }
5861
5862 #[test]
5865 fn a_pending_record_should_not_be_refused_with_the_map_below_capacity() {
5866 let registry = SurbRoundTripRegistry::with_records(128, SEED);
5867 for seed in 0..100u8 {
5868 registry.remember(surb_id(seed), pending_to(&[], None, 0));
5869 }
5870
5871 assert!(registry.pending.contains_key(&surb_id(0)), "the oldest is still there");
5872 assert!(registry.pending.contains_key(&surb_id(99)), "and so is the newest");
5873 assert_eq!(100, registry.pending.len());
5874 assert_eq!(0, registry.take_pending_lost(), "nothing was evicted to make room");
5875 }
5876
5877 #[test]
5881 fn records_the_pending_map_cannot_hold_should_be_counted_and_taken() {
5882 let registry = SurbRoundTripRegistry::with_records(8, SEED);
5883 assert_eq!(0, registry.take_pending_lost());
5884 for seed in 0..32u8 {
5885 registry.remember(surb_id(seed), pending_to(&[], None, 0));
5886 }
5887
5888 assert_eq!(
5889 24,
5890 registry.take_pending_lost(),
5891 "32 mints into an 8-record map leaves 24 records it could not hold"
5892 );
5893 assert_eq!(0, registry.take_pending_lost(), "and they are reported once");
5894 }
5895
5896 #[test]
5900 fn a_written_off_leg_the_map_cannot_hold_should_not_count_as_lost() {
5901 let registry = SurbRoundTripRegistry::with_records(8, SEED);
5902 for seed in 0..32u8 {
5903 registry.remember_written_off(surb_id(seed), leg(&[]), Instant::now());
5904 }
5905
5906 assert_eq!(8, registry.written_off_legs.len(), "the map holds what it can");
5907 assert_eq!(0, registry.take_pending_lost());
5908 }
5909
5910 #[test]
5913 fn an_evicted_written_off_leg_should_not_leave_its_race_open() {
5914 let registry = SurbRoundTripRegistry::with_records(1, SEED);
5915 let health = registry.relayer_health();
5916 let [late, home] = [(); 2].map(|_| key());
5917
5918 registry.remember_written_off(surb_id(1), leg(&[late]), Instant::now());
5919 registry.open_race(&leg(&[late]), surb_id(1), &leg(&[home]), ROUND_TRIP);
5920 assert_eq!(1, open_races_of(&health, &late));
5921
5922 registry.remember_written_off(surb_id(2), leg(&[]), Instant::now());
5923 assert!(!registry.written_off_legs.contains_key(&surb_id(1)));
5924 assert_eq!(0, open_races_of(&health, &late));
5925 }
5926
5927 #[test]
5930 fn a_budget_outside_the_accepted_range_should_be_clamped() {
5931 let capacity_of = |budget| {
5932 let registry = SurbRoundTripRegistry::with_budget(budget);
5933 (registry.pending.capacity(), registry.written_off_legs.capacity())
5934 };
5935 let floor = capacity_of(MINIMUM_PENDING_SURB_MAX_BYTES);
5936 assert_eq!(floor, capacity_of(0), "no budget is the floor");
5937 assert_eq!(
5938 floor,
5939 capacity_of(100_000),
5940 "a record count passed as bytes is the floor"
5941 );
5942 assert_eq!(floor, capacity_of(MINIMUM_PENDING_SURB_MAX_BYTES - 1));
5943 assert_eq!(
5944 floor,
5945 capacity_of(MINIMUM_PENDING_SURB_MAX_BYTES),
5946 "and the floor itself is honoured"
5947 );
5948 let ceiling = capacity_of(MAXIMUM_PENDING_SURB_MAX_BYTES);
5949 assert_eq!(ceiling, capacity_of(u64::MAX), "no budget is above the ceiling");
5950 assert!(ceiling.0 > floor.0 && ceiling.1 > floor.1);
5951 }
5952
5953 #[test]
5957 fn each_map_should_honour_the_byte_budget_by_its_own_record_cost() {
5958 use crate::protocol::surb_map::record_bytes;
5959
5960 let budget = 64 * MINIMUM_PENDING_SURB_MAX_BYTES;
5961 let registry = SurbRoundTripRegistry::with_budget(budget);
5962 assert!(registry.pending.capacity() as u64 * record_bytes::<PendingSurb>() as u64 <= budget);
5963 assert!(
5964 registry.written_off_legs.capacity() as u64 * record_bytes::<(ReturnRelayers, Instant)>() as u64 <= budget
5965 );
5966 assert!(
5967 registry.written_off_legs.capacity() > 2 * registry.pending.capacity(),
5968 "a written-off leg is a fraction of a pending record, and is charged as one"
5969 );
5970
5971 let floor = SurbRoundTripRegistry::with_budget(MINIMUM_PENDING_SURB_MAX_BYTES);
5972 assert!(
5973 floor.pending.capacity() >= 100_000,
5974 "the floor holds at least the old fixed cap: {}",
5975 floor.pending.capacity()
5976 );
5977 assert!((140_000..=148_000).contains(&floor.pending.capacity()));
5979 }
5980
5981 #[test]
5984 fn the_default_budget_should_hold_many_deployed_size_stocks() {
5985 use hopr_protocol_hopr::{DEFAULT_PENDING_SURB_MAX_BYTES, SurbStoreConfig};
5986
5987 assert_eq!(
5989 DEFAULT_PENDING_SURB_MAX_BYTES,
5990 SurbStoreConfig::default().pending_surb_max_bytes
5991 );
5992 const DEPLOYED_STOCK: usize = 19_264;
5994 let registry = SurbRoundTripRegistry::with_budget(DEFAULT_PENDING_SURB_MAX_BYTES);
5995 let records = registry.pending.capacity();
5997 assert!(
5998 (450_000..=458_000).contains(&records),
5999 "the default holds {records} records, not the ~454 000 documented"
6000 );
6001 assert_eq!(23, records / DEPLOYED_STOCK);
6002 }
6003
6004 #[test]
6007 fn the_validated_minimum_budget_should_be_accepted() {
6008 use hopr_protocol_hopr::SurbStoreConfig;
6009 use validator::Validate;
6010
6011 let cfg = SurbStoreConfig {
6012 pending_surb_max_bytes: MINIMUM_PENDING_SURB_MAX_BYTES,
6013 ..Default::default()
6014 };
6015 assert!(cfg.validate().is_ok(), "the floor is a valid configuration");
6016 }
6017
6018 #[test]
6023 fn an_evicted_surb_should_not_leave_its_race_open() -> anyhow::Result<()> {
6024 const CAPACITY: usize = 8;
6025 let registry = SurbRoundTripRegistry::with_records(CAPACITY, SEED);
6026 let health = registry.relayer_health();
6027 let recorder = SurbTelemetryCodec::new((), key(), no_path_slots(), registry);
6028 let [late, home] = [(); 2].map(|_| key());
6029
6030 for (first, second) in [(1, 2), (3, 4)] {
6033 recorder
6034 .registry
6035 .remember(surb_id(first), pending_to(&[late], Some(surb_id(second)), 0));
6036 recorder
6037 .registry
6038 .remember(surb_id(second), pending_to(&[home], Some(surb_id(first)), 1));
6039 recorder.on_replied(&surb_id(second));
6040 }
6041 assert_eq!(2, open_races_of(&health, &late));
6042
6043 let mut seed = 100u8;
6045 while recorder.registry.pending.contains_key(&surb_id(1)) && recorder.registry.pending.contains_key(&surb_id(3))
6046 {
6047 assert!(seed < 250, "no record of the pair was evicted");
6048 recorder.registry.remember(surb_id(seed), pending_to(&[], None, 0));
6049 seed += 1;
6050 }
6051 let evicted = [1, 3]
6052 .into_iter()
6053 .find(|first| !recorder.registry.pending.contains_key(&surb_id(*first)))
6054 .context("a record of the pair is gone")?;
6055 let kept: Vec<u8> = [1, 3]
6056 .into_iter()
6057 .filter(|first| recorder.registry.pending.contains_key(&surb_id(*first)))
6058 .collect();
6059 assert!(recorder.registry.take_pending_lost() > 0);
6060 assert_eq!(
6061 kept.len(),
6062 open_races_of(&health, &late),
6063 "the evicted record's race is discarded, the other stays open while its record does"
6064 );
6065
6066 recorder.on_replied(&surb_id(evicted));
6068 assert!(margins_of(&health, &late).is_empty());
6069 assert_eq!(kept.len(), open_races_of(&health, &late));
6070 Ok(())
6071 }
6072
6073 #[test]
6078 fn a_race_opened_on_a_surb_already_gone_should_not_stay_open() {
6079 let registry = SurbRoundTripRegistry::default();
6080 let health = registry.relayer_health();
6081 let [late, home] = [(); 2].map(|_| key());
6082
6083 registry.open_race(&leg(&[late]), surb_id(1), &leg(&[home]), ROUND_TRIP);
6084 assert_eq!(
6085 0,
6086 open_races_of(&health, &late),
6087 "gone from both maps: nothing can close it"
6088 );
6089
6090 registry
6091 .pending
6092 .insert(surb_id(2), pending_to(&[late], Some(surb_id(3)), 0));
6093 registry.open_race(&leg(&[late]), surb_id(2), &leg(&[home]), ROUND_TRIP);
6094 assert_eq!(1, open_races_of(&health, &late), "still pending: its reply settles it");
6095
6096 registry
6097 .written_off_legs
6098 .insert(surb_id(4), (leg(&[late]), Instant::now()));
6099 registry.open_race(&leg(&[late]), surb_id(4), &leg(&[home]), ROUND_TRIP);
6100 assert_eq!(
6101 2,
6102 open_races_of(&health, &late),
6103 "written off: kept, as when the write-off comes after"
6104 );
6105 }
6106
6107 #[rstest::rstest]
6116 #[case::minted_after_the_verdict(true)]
6117 #[case::minted_before_the_verdict(false)]
6118 fn the_reply_of_a_written_off_surb_should_close_the_open_races_it_is_news_about(
6119 #[case] minted_after_verdict: bool,
6120 ) -> anyhow::Result<()> {
6121 let registry = SurbRoundTripRegistry::default();
6122 let health = registry.relayer_health();
6123 let recorder = SurbTelemetryCodec::new((), key(), no_path_slots(), registry);
6124 let [healthy, dead, home] = [(); 3].map(|_| key());
6125
6126 recorder
6127 .registry
6128 .pending
6129 .insert(surb_id(1), pending_to(&[healthy, dead], Some(surb_id(2)), 0));
6130 recorder
6131 .registry
6132 .pending
6133 .insert(surb_id(2), pending_to(&[home], Some(surb_id(1)), 1));
6134 recorder.on_replied(&surb_id(2));
6135 assert_eq!(1, open_races_of(&health, &healthy));
6136 assert_eq!(1, open_races_of(&health, &dead));
6137
6138 judge_dead(&health, dead);
6140 let minted_at = if minted_after_verdict {
6141 Instant::now()
6142 } else {
6143 ago(Duration::from_secs(5))?
6144 };
6145 recorder
6146 .registry
6147 .remember_written_off(surb_id(1), leg(&[healthy, dead]), minted_at);
6148 recorder.registry.pending.remove(&surb_id(1));
6149 assert_eq!(1, open_races_of(&health, &healthy), "a written-off SURB keeps its race");
6150
6151 recorder.on_replied(&surb_id(1));
6152 assert_eq!(
6153 0,
6154 open_races_of(&health, &healthy),
6155 "the healthy relayer delivered: its race is closed"
6156 );
6157 assert_eq!(
6158 usize::from(!minted_after_verdict),
6159 open_races_of(&health, &dead),
6160 "a stale reply is no news about the confirmed-dead relayer, so its race stays open"
6161 );
6162 assert!(margins_of(&health, &healthy).is_empty() && margins_of(&health, &dead).is_empty());
6163 Ok(())
6164 }
6165
6166 #[test]
6171 fn a_reply_after_its_partner_was_consumed_should_find_nothing_to_update() {
6172 let registry = SurbRoundTripRegistry::default();
6173 let health = registry.relayer_health();
6174 let recorder = SurbTelemetryCodec::new((), key(), no_path_slots(), registry);
6175 let [late, home] = [(); 2].map(|_| key());
6176
6177 recorder
6178 .registry
6179 .pending
6180 .insert(surb_id(1), pending_to(&[late], Some(surb_id(2)), 0));
6181 recorder
6182 .registry
6183 .pending
6184 .insert(surb_id(2), pending_to(&[home], Some(surb_id(1)), 1));
6185 recorder.on_replied(&surb_id(1));
6186 recorder.on_replied(&surb_id(2));
6187
6188 assert!(
6189 !recorder.registry.pending.contains_key(&surb_id(1)),
6190 "consumed stays consumed"
6191 );
6192 assert!(recorder.registry.pending.is_empty());
6193 assert_eq!(0, open_races_of(&health, &late));
6194 }
6195
6196 #[rstest::rstest]
6205 #[case::fresh_reply_revives(true)]
6206 #[case::stale_reply_does_not(false)]
6207 fn a_relayer_revived_during_the_walk_should_reach_no_session(#[case] fresh: bool) -> anyhow::Result<()> {
6208 let registry = SurbRoundTripRegistry::default();
6209 let health = registry.relayer_health();
6210 let (stocked, unreached) = (*SESSION, HoprPseudonym::random());
6211 let dead = key();
6212 judge_dead(&health, dead);
6213 health.known_dead_since.insert(dead, ago(Duration::from_secs(2))?);
6214 registry.pending.insert(
6215 surb_id(1),
6216 PendingSurb {
6217 minted_at: ago(Duration::from_secs(5))?,
6218 ..pending_to(&[dead], None, 0)
6219 },
6220 );
6221 registry.pending.insert(
6222 surb_id(2),
6223 PendingSurb {
6224 pseudonym: unreached,
6225 ..pending_to(&[dead], None, 0)
6226 },
6227 );
6228
6229 let stock = registry.dead_stock();
6230 assert_eq!(2, stock.len(), "both are found while the relayer is dead");
6231 let reply_minted_at = if fresh {
6232 Instant::now()
6233 } else {
6234 ago(Duration::from_secs(5))?
6235 };
6236 health.revive_by_reply(&leg(&[dead]), reply_minted_at);
6237 assert_eq!(fresh, !health.is_known_dead(&dead));
6238 let reached = registry.write_off(stock);
6239 if fresh {
6240 assert_eq!(Vec::<HoprPseudonym>::new(), reached, "revived: no session is reached");
6241 assert!(registry.pending.contains_key(&surb_id(1)) && registry.pending.contains_key(&surb_id(2)));
6242 assert!(registry.invalid().is_empty(), "and nothing is written off");
6243 assert!(!registry.bumped.contains_key(&(stocked, dead)));
6244 } else {
6245 assert_eq!(
6246 vec![stocked],
6247 reached,
6248 "still dead: the stock reaches its session, the probe none"
6249 );
6250 assert!(!registry.pending.contains_key(&surb_id(1)));
6251 }
6252 Ok(())
6253 }
6254
6255 #[rstest::rstest]
6261 #[case::confirmed_dead(false)]
6262 #[case::on_probation(true)]
6263 fn a_relayer_caught_half_way_through_its_revival_should_reach_no_session(#[case] on_probation: bool) {
6264 let registry = SurbRoundTripRegistry::default();
6265 let health = registry.relayer_health();
6266 let unreached = HoprPseudonym::random();
6267 let dead = key();
6268 judge_dead(&health, dead);
6269 if on_probation {
6270 health.dead.remove(&dead);
6272 health.dead_since.remove(&dead);
6273 health.confirmed.remove(&dead);
6274 health.probation.insert(dead);
6275 }
6276 registry.pending.insert(
6278 surb_id(1),
6279 PendingSurb {
6280 pseudonym: unreached,
6281 ..pending_to(&[dead], None, 0)
6282 },
6283 );
6284 let stock = registry.dead_stock();
6285 assert_eq!(1, stock.len(), "found while the relayer is dead");
6286
6287 health.known_dead_since.remove(&dead);
6289 assert!(health.is_confirmed_dead(&dead));
6290 assert_eq!(
6291 Vec::<HoprPseudonym>::new(),
6292 registry.write_off(stock),
6293 "no session is reached"
6294 );
6295 assert!(registry.pending.contains_key(&surb_id(1)), "the probe stays pending");
6296 assert!(registry.invalid().is_empty(), "and nothing is written off");
6297 assert!(!registry.bumped.contains_key(&(unreached, dead)));
6298 }
6299
6300 #[test]
6303 fn a_surb_spent_during_the_walk_should_not_be_written_off() {
6304 let registry = SurbRoundTripRegistry::default();
6305 let dead = key();
6306 judge_dead(®istry.relayer_health, dead);
6307 registry.pending.insert(surb_id(1), pending_to(&[dead], None, 0));
6308
6309 let stock = registry.dead_stock();
6310 registry.pending.remove(&surb_id(1));
6311 assert!(registry.write_off(stock).is_empty());
6312 assert!(registry.invalid().is_empty(), "it is not counted as held");
6313 assert!(!registry.written_off_legs.contains_key(&surb_id(1)), "nor remembered");
6314 }
6315
6316 #[test]
6318 fn the_walks_should_be_gated_on_a_relayer_being_known_dead_and_a_write_off_standing() {
6319 let registry = SurbRoundTripRegistry::default();
6320 let health = registry.relayer_health();
6321 let dead = key();
6322 assert!(!health.any_known_dead() && !registry.has_write_offs());
6323 assert!(registry.dead_stock().is_empty());
6324
6325 registry.pending.insert(surb_id(1), pending_to(&[dead], None, 0));
6326 registry.bumped.insert((*SESSION, dead), Instant::now());
6328 judge_dead(&health, dead);
6329 assert!(health.any_known_dead());
6330 registry.write_off_dead();
6331 assert!(registry.has_write_offs());
6332
6333 health.dead.remove(&dead);
6334 assert!(!health.any_known_dead());
6335 health.probation.insert(dead);
6336 assert!(health.any_known_dead(), "probation is known dead");
6337 }
6338
6339 #[cfg(target_os = "linux")]
6341 const PAGE_BYTES: usize = 4096;
6342
6343 #[cfg(target_os = "linux")]
6345 fn resident_bytes() -> anyhow::Result<usize> {
6346 let statm = std::fs::read_to_string("/proc/self/statm").context("reading /proc/self/statm")?;
6347 let pages: usize = statm
6348 .split_whitespace()
6349 .nth(1)
6350 .context("no resident figure")?
6351 .parse()
6352 .context("resident pages")?;
6353 Ok(pages * PAGE_BYTES)
6354 }
6355
6356 #[cfg(target_os = "linux")]
6361 fn assert_budget_covers_resident_memory<V: Copy>(value: V) -> anyhow::Result<()> {
6362 use crate::protocol::surb_map::record_bytes;
6363
6364 const RECORDS: u64 = 1_000_000;
6365 const SLACK: f64 = 1.10;
6367 let id = |n: u64| n.wrapping_mul(0x9E37_79B9_7F4A_7C15).to_le_bytes();
6368
6369 let before = resident_bytes()?;
6370 let map = RandomEvictionMap::<V>::with_seed(RECORDS as usize, SEED);
6371 for n in 0..RECORDS {
6372 map.insert(id(n), value);
6373 }
6374 for n in RECORDS..4 * RECORDS {
6375 map.insert(id(n), value);
6376 }
6377 for n in (RECORDS..3 * RECORDS).step_by(3) {
6378 map.remove(&id(n));
6379 }
6380 for n in 10 * RECORDS..11 * RECORDS {
6381 map.insert(id(n), value);
6382 }
6383 let per_record = (resident_bytes()? - before) as f64 / map.len() as f64;
6384 let accounted = record_bytes::<V>() as f64;
6385 eprintln!("{} B per record resident, {accounted} accounted", per_record.round());
6386 assert!(
6387 accounted * SLACK >= per_record,
6388 "a record costs {per_record:.0} B, more than the {accounted} B accounted"
6389 );
6390 Ok(())
6391 }
6392
6393 #[test]
6394 #[ignore = "reads the resident set of the process: run it alone"]
6395 #[cfg(target_os = "linux")]
6396 fn a_pending_record_should_not_cost_more_than_its_accounted_bytes() -> anyhow::Result<()> {
6397 assert_budget_covers_resident_memory(pending_to(&[], None, 0))
6398 }
6399
6400 #[test]
6401 #[ignore = "reads the resident set of the process: run it alone"]
6402 #[cfg(target_os = "linux")]
6403 fn a_written_off_leg_should_not_cost_more_than_its_accounted_bytes() -> anyhow::Result<()> {
6404 assert_budget_covers_resident_memory((leg(&[]), Instant::now()))
6405 }
6406}