Skip to main content

hopr_transport/protocol/
counters.rs

1use std::{
2    sync::{
3        Arc,
4        atomic::{AtomicU64, Ordering},
5    },
6    time::Duration,
7};
8
9use dashmap::DashMap;
10use hopr_api::types::crypto::types::OffchainPublicKey;
11
12/// Minimal atomic counters for per-peer protocol conformance tracking.
13///
14/// Tracks the number of messages sent and acknowledgments received for a
15/// single peer. All operations are lock-free using relaxed atomic ordering.
16#[derive(Debug, Default)]
17pub struct PeerProtocolCounters {
18    messages_sent: AtomicU64,
19    acks_received: AtomicU64,
20}
21
22impl PeerProtocolCounters {
23    /// Record that a message was sent to this peer.
24    #[inline]
25    pub fn record_message_sent(&self) {
26        self.messages_sent.fetch_add(1, Ordering::Relaxed);
27    }
28
29    /// Record that an acknowledgment was received from this peer.
30    #[inline]
31    pub fn record_ack_received(&self) {
32        self.acks_received.fetch_add(1, Ordering::Relaxed);
33    }
34
35    /// Record that `count` acknowledgments were received from this peer in a batch.
36    #[inline]
37    pub fn record_acks_received(&self, count: u64) {
38        self.acks_received.fetch_add(count, Ordering::Relaxed);
39    }
40
41    /// Swap the sent-message counter to 0, returning the accumulated value.
42    #[inline]
43    pub fn take_sent(&self) -> u64 {
44        self.messages_sent.swap(0, Ordering::Relaxed)
45    }
46
47    /// Swap the acknowledgment counter to 0, returning the accumulated value.
48    #[inline]
49    pub fn take_acks(&self) -> u64 {
50        self.acks_received.swap(0, Ordering::Relaxed)
51    }
52
53    /// Swap both counters to 0, returning accumulated values.
54    ///
55    /// Each counter is swapped independently via its own atomic operation;
56    /// there is no single atomic snapshot of the pair.
57    pub fn take(&self) -> (u64, u64) {
58        (self.take_sent(), self.take_acks())
59    }
60}
61
62/// Thread-safe registry of per-peer protocol conformance counters.
63///
64/// Keyed by [`OffchainPublicKey`] — no PeerId conversion needed since the
65/// protocol pipeline already operates on offchain keys.
66#[derive(Debug, Default, Clone)]
67pub struct PeerProtocolCounterRegistry {
68    inner: Arc<DashMap<OffchainPublicKey, Arc<PeerProtocolCounters>>>,
69}
70
71impl PeerProtocolCounterRegistry {
72    /// Get or create counters for the given peer.
73    pub fn get_or_create(&self, peer: &OffchainPublicKey) -> Arc<PeerProtocolCounters> {
74        self.inner
75            .entry(*peer)
76            .or_insert_with(|| Arc::new(PeerProtocolCounters::default()))
77            .value()
78            .clone()
79    }
80
81    /// Drains the counters for one conformance report, giving every packet in it `ack_grace` to be
82    /// acknowledged.
83    ///
84    /// [`Self::drain`] reads both counters at the same instant, so a packet still in flight then is
85    /// reported as sent but never acknowledged. In steady traffic that is a sliver of the window. A
86    /// burst that starts just before the flush, though, is in flight almost entirely: the report
87    /// reads as an acknowledgment rate near zero, the edge falls below the path planner's
88    /// `min_ack_rate`, and it stays out of path selection until the next report (hoprnet#8484).
89    ///
90    /// So the sent counts are taken first, and the acknowledgments only `ack_grace` later. Packets
91    /// sent during the grace stay counted for the next report. Their acknowledgments that arrive
92    /// within it are credited to this one, a report early. The decayed totals the graph keeps absorb
93    /// that: a report later, a burst's acknowledgments still weigh 0.9 against its packets.
94    ///
95    /// A peer first seen during the grace is left for the next report.
96    ///
97    /// Not cancellation safe: the sent counts are taken before the grace, so dropping the future
98    /// while it waits loses them from every report. The acknowledgments are only taken after the
99    /// grace and stay counted. The flush task is only dropped when the transport stops.
100    ///
101    /// Returns `(peer, msgs_sent, acks_received)` for non-zero entries, like [`Self::drain`].
102    pub async fn drain_settled(&self, ack_grace: Duration) -> Vec<(OffchainPublicKey, u64, u64)> {
103        // Each peer's counters are held through the grace, so its acks are taken without a second
104        // lookup. Collected before the await: no map guard may be held across it.
105        let pending = self
106            .inner
107            .iter()
108            .map(|entry| (*entry.key(), entry.value().clone(), entry.value().take_sent()))
109            .collect::<Vec<_>>();
110
111        if !ack_grace.is_zero() {
112            futures_timer::Delay::new(ack_grace).await;
113        }
114
115        pending
116            .into_iter()
117            .filter_map(|(peer, counters, sent)| {
118                let received = counters.take_acks();
119                (sent > 0 || received > 0).then_some((peer, sent, received))
120            })
121            .collect()
122    }
123
124    /// Swap all counters to 0, returning `(peer, msgs_sent, acks_received)` for non-zero entries.
125    pub fn drain(&self) -> Vec<(OffchainPublicKey, u64, u64)> {
126        self.inner
127            .iter()
128            .filter_map(|entry| {
129                let (sent, received) = entry.value().take();
130                if sent > 0 || received > 0 {
131                    Some((*entry.key(), sent, received))
132                } else {
133                    None
134                }
135            })
136            .collect()
137    }
138}
139
140#[cfg(test)]
141mod tests {
142    use std::time::Duration;
143
144    use anyhow::Context;
145    use hopr_api::types::crypto::prelude::{Keypair, OffchainKeypair};
146
147    use super::*;
148
149    #[test]
150    fn counters_should_start_at_zero() {
151        let counters = PeerProtocolCounters::default();
152        let (sent, received) = counters.take();
153
154        assert_eq!(sent, 0);
155        assert_eq!(received, 0);
156    }
157
158    #[test]
159    fn counters_should_record_messages_sent() {
160        let counters = PeerProtocolCounters::default();
161
162        counters.record_message_sent();
163        counters.record_message_sent();
164        counters.record_message_sent();
165
166        let (sent, received) = counters.take();
167        assert_eq!(sent, 3);
168        assert_eq!(received, 0);
169    }
170
171    #[test]
172    fn counters_should_record_acks_received() {
173        let counters = PeerProtocolCounters::default();
174
175        counters.record_ack_received();
176        counters.record_ack_received();
177
178        let (sent, received) = counters.take();
179        assert_eq!(sent, 0);
180        assert_eq!(received, 2);
181    }
182
183    #[test]
184    fn take_should_reset_counters_to_zero() {
185        let counters = PeerProtocolCounters::default();
186
187        counters.record_message_sent();
188        counters.record_ack_received();
189
190        let (sent1, received1) = counters.take();
191        assert_eq!(sent1, 1);
192        assert_eq!(received1, 1);
193
194        let (sent2, received2) = counters.take();
195        assert_eq!(sent2, 0);
196        assert_eq!(received2, 0);
197    }
198
199    #[test]
200    fn registry_should_create_and_retrieve_counters() -> anyhow::Result<()> {
201        let registry = PeerProtocolCounterRegistry::default();
202        let peer = *OffchainKeypair::random().public();
203
204        let counters = registry.get_or_create(&peer);
205        counters.record_message_sent();
206
207        let same_counters = registry.get_or_create(&peer);
208        same_counters.record_message_sent();
209
210        let (sent, _) = counters.take();
211        assert_eq!(sent, 2, "both calls should share the same counter instance");
212        Ok(())
213    }
214
215    #[test]
216    fn drain_should_return_only_nonzero_entries() -> anyhow::Result<()> {
217        let registry = PeerProtocolCounterRegistry::default();
218        let peer_a = *OffchainKeypair::random().public();
219        let peer_b = *OffchainKeypair::random().public();
220        let peer_c = *OffchainKeypair::random().public();
221
222        registry.get_or_create(&peer_a).record_message_sent();
223        registry.get_or_create(&peer_b); // no activity
224        registry.get_or_create(&peer_c).record_ack_received();
225
226        let drained = registry.drain();
227        assert_eq!(drained.len(), 2, "only peers with non-zero counters should be drained");
228
229        let a_entry = drained
230            .iter()
231            .find(|(p, ..)| *p == peer_a)
232            .context("peer_a should be in drain results")?;
233        assert_eq!(a_entry.1, 1);
234        assert_eq!(a_entry.2, 0);
235
236        let c_entry = drained
237            .iter()
238            .find(|(p, ..)| *p == peer_c)
239            .context("peer_c should be in drain results")?;
240        assert_eq!(c_entry.1, 0);
241        assert_eq!(c_entry.2, 1);
242
243        Ok(())
244    }
245
246    #[test]
247    fn drain_should_reset_counters() {
248        let registry = PeerProtocolCounterRegistry::default();
249        let peer = *OffchainKeypair::random().public();
250
251        registry.get_or_create(&peer).record_message_sent();
252
253        let first_drain = registry.drain();
254        assert_eq!(first_drain.len(), 1);
255
256        let second_drain = registry.drain();
257        assert!(second_drain.is_empty(), "counters should be zero after drain");
258    }
259
260    #[tokio::test]
261    async fn drain_settled_should_credit_acks_that_arrive_within_the_grace() -> anyhow::Result<()> {
262        let registry = PeerProtocolCounterRegistry::default();
263        let peer = *OffchainKeypair::random().public();
264        let counters = registry.get_or_create(&peer);
265
266        // A burst whose acknowledgments are all still in flight when the flush starts.
267        for _ in 0..10 {
268            counters.record_message_sent();
269        }
270
271        // Drive the drain into its grace deterministically rather than racing a timer: the first
272        // poll takes the `sent` snapshot and starts the grace, so the acks recorded next are
273        // guaranteed to land inside the window before `take_acks` reads them. A starved executor
274        // could otherwise let the grace timer and an ack timer fire together, with the drain polled
275        // first and reading zero acks.
276        let mut drained = std::pin::pin!(registry.drain_settled(Duration::from_millis(50)));
277        anyhow::ensure!(
278            futures::poll!(drained.as_mut()).is_pending(),
279            "the drain must be waiting out its grace before the acks are recorded"
280        );
281        counters.record_acks_received(10);
282        let drained = drained.await;
283
284        let (_, sent, received) = drained.first().context("the peer must be reported")?;
285        assert_eq!(
286            (*sent, *received),
287            (10, 10),
288            "acknowledgments arriving within the grace must be credited"
289        );
290        Ok(())
291    }
292
293    #[tokio::test]
294    async fn drain_settled_should_carry_packets_sent_during_the_grace_into_the_next_report() -> anyhow::Result<()> {
295        let registry = PeerProtocolCounterRegistry::default();
296        let peer = *OffchainKeypair::random().public();
297        let counters = registry.get_or_create(&peer);
298
299        counters.record_message_sent();
300
301        // First poll takes the `sent` snapshot (one packet) and starts the grace; the packets
302        // recorded next therefore land after the snapshot, inside the grace, with no dependence on
303        // timer order.
304        let mut first = std::pin::pin!(registry.drain_settled(Duration::from_millis(50)));
305        anyhow::ensure!(
306            futures::poll!(first.as_mut()).is_pending(),
307            "the drain must be waiting out its grace before the in-grace packets are recorded"
308        );
309        counters.record_message_sent();
310        counters.record_message_sent();
311        let first = first.await;
312        let (_, sent, _) = first.first().context("the peer must be reported")?;
313        assert_eq!(*sent, 1, "only the packet sent before the grace belongs to this report");
314
315        let second = registry.drain_settled(Duration::ZERO).await;
316        let (_, sent, _) = second.first().context("the carried packets must be reported")?;
317        assert_eq!(*sent, 2, "packets sent during the grace must be reported next time");
318        Ok(())
319    }
320
321    #[tokio::test]
322    async fn drain_settled_should_report_only_active_peers() {
323        let registry = PeerProtocolCounterRegistry::default();
324        registry.get_or_create(OffchainKeypair::random().public());
325
326        assert!(
327            registry.drain_settled(Duration::ZERO).await.is_empty(),
328            "a peer with nothing sent or received must not be reported"
329        );
330    }
331}