Skip to main content

hopr_transport/protocol/pipeline/
config.rs

1//! Configuration structures for the HOPR packet processing pipeline.
2
3use validator::{Validate, ValidationError, ValidationErrors};
4
5fn default_ack_buffer_interval() -> std::time::Duration {
6    std::time::Duration::from_millis(200)
7}
8
9fn default_ack_grouping_capacity() -> usize {
10    5
11}
12
13fn default_ticket_ack_buffer_size() -> usize {
14    50_000
15}
16
17fn default_ack_out_buffer_size() -> usize {
18    50_000
19}
20
21/// Configuration for the acknowledgement processing pipeline.
22#[derive(Debug, Copy, Clone, smart_default::SmartDefault, Eq, PartialEq)]
23#[cfg_attr(feature = "serde", derive(serde::Deserialize, serde::Serialize))]
24pub struct AcknowledgementPipelineConfig {
25    /// Interval for which to wait to buffer acknowledgements before sending them out.
26    ///
27    /// Default is 200 ms.
28    #[default(default_ack_buffer_interval())]
29    #[cfg_attr(
30        feature = "serde",
31        serde(default = "default_ack_buffer_interval", with = "humantime_serde")
32    )]
33    pub ack_buffer_interval: std::time::Duration,
34    /// Initial capacity when grouping outgoing acknowledgements.
35    ///
36    /// If set too low, it causes additional reallocations in the outgoing acknowledgement processing pipeline.
37    /// The value should grow if `ack_buffer_interval` grows.
38    ///
39    /// Default is 5.
40    #[default(default_ack_grouping_capacity())]
41    #[cfg_attr(feature = "serde", serde(default = "default_ack_grouping_capacity"))]
42    pub ack_grouping_capacity: usize,
43    /// Capacity of the `incoming_ack` MPSC channel carrying received acknowledgements
44    /// to the ticket-ack processing pipeline.
45    ///
46    /// The previous hardcoded value of 1_000_000 pre-allocated ~MBs of ring buffer per node even
47    /// though real-world throughput rarely saturates more than a few thousand entries. Let the
48    /// 50 ms sink timeouts (`QUEUE_SEND_TIMEOUT`) propagate backpressure instead.
49    ///
50    /// The default is 50 000.
51    #[default(default_ticket_ack_buffer_size())]
52    #[cfg_attr(feature = "serde", serde(default = "default_ticket_ack_buffer_size"))]
53    pub ticket_ack_buffer_size: usize,
54    /// Capacity of the `outgoing_ack` MPSC channel carrying acknowledgements to be sent back
55    /// to the previous hop.
56    ///
57    /// The default is 50 000. See [`ticket_ack_buffer_size`](Self::ticket_ack_buffer_size) for the
58    /// rationale on why this is smaller than the original hardcoded 1_000_000.
59    #[default(default_ack_out_buffer_size())]
60    #[cfg_attr(feature = "serde", serde(default = "default_ack_out_buffer_size"))]
61    pub ack_out_buffer_size: usize,
62    /// Maximum concurrency when processing incoming (received) acknowledgements.
63    ///
64    /// `None` or `Some(0)` both fall back to a default of 10.
65    pub ack_input_concurrency: Option<usize>,
66    /// Maximum concurrency when processing outgoing (sent-back) acknowledgements.
67    ///
68    /// `None` or `Some(0)` both fall back to a default of 10.
69    pub ack_output_concurrency: Option<usize>,
70}
71
72// Requires manual implementation due to https://github.com/Keats/validator/issues/285
73impl Validate for AcknowledgementPipelineConfig {
74    fn validate(&self) -> Result<(), ValidationErrors> {
75        let mut errors = ValidationErrors::new();
76        if self.ack_grouping_capacity == 0 {
77            errors.add("ack_grouping_capacity", ValidationError::new("must be greater than 0"));
78        }
79        if self.ack_buffer_interval < std::time::Duration::from_millis(10) {
80            errors.add("ack_buffer_interval", ValidationError::new("must be at least 10 ms"));
81        }
82        if self.ticket_ack_buffer_size == 0 {
83            errors.add("ticket_ack_buffer_size", ValidationError::new("must be greater than 0"));
84        }
85        if self.ack_out_buffer_size == 0 {
86            errors.add("ack_out_buffer_size", ValidationError::new("must be greater than 0"));
87        }
88        if errors.is_empty() { Ok(()) } else { Err(errors) }
89    }
90}
91
92fn default_arbitration_enabled() -> bool {
93    true
94}
95fn default_arbitration_occupancy_pct() -> u32 {
96    75
97}
98fn default_arbitration_encode_reserve_pct() -> u32 {
99    50
100}
101
102/// Arbitration of the shared Rayon pool, protecting the encode path (SPHINX wrap + SURB generation)
103/// from decode floods (SPHINX peel — relay forwarding + exit termination).
104///
105/// The arbiter is asymmetric and occupancy-gated: only decode is ever throttled, and only when the
106/// pool is genuinely saturated *and* encode work is present, so pure forwarding and unsaturated
107/// nodes are untouched. Enforced inside `spawn_decode_blocking`
108/// (see `hopr_utils::parallelize::cpu::configure_arbitration`).
109#[derive(Clone, Copy, Debug, PartialEq, Eq, smart_default::SmartDefault, Validate)]
110#[cfg_attr(
111    feature = "serde",
112    derive(serde::Serialize, serde::Deserialize),
113    serde(deny_unknown_fields)
114)]
115pub struct PoolArbitrationConfig {
116    /// When `false`, decode is never throttled (the pool is shared first-come-first-served).
117    #[default(default_arbitration_enabled())]
118    #[cfg_attr(feature = "serde", serde(default = "default_arbitration_enabled"))]
119    pub enabled: bool,
120    /// Pool occupancy (percent of threads actually running) at or above which decode admission may
121    /// engage. Below it, decode is never throttled.
122    #[default(default_arbitration_occupancy_pct())]
123    #[validate(range(min = 1, max = 100))]
124    #[cfg_attr(feature = "serde", serde(default = "default_arbitration_occupancy_pct"))]
125    pub occupancy_pct: u32,
126    /// Share of the pool (percent) that decode yields to encode when both contend under saturation.
127    #[default(default_arbitration_encode_reserve_pct())]
128    #[validate(range(min = 1, max = 100))]
129    #[cfg_attr(feature = "serde", serde(default = "default_arbitration_encode_reserve_pct"))]
130    pub encode_reserve_pct: u32,
131}
132
133impl PoolArbitrationConfig {
134    /// Maps this flat (serde-friendly) config onto the pool arbiter's
135    /// [`ArbitrationConfig`](hopr_utils::parallelize::cpu::ArbitrationConfig) enum, where the disabled
136    /// state carries no tuning percentages.
137    pub fn to_arbitration(&self) -> hopr_utils::parallelize::cpu::ArbitrationConfig {
138        use hopr_utils::parallelize::cpu::ArbitrationConfig;
139        if self.enabled {
140            ArbitrationConfig::Enabled {
141                occupancy_pct: self.occupancy_pct,
142                encode_reserve_pct: self.encode_reserve_pct,
143            }
144        } else {
145            ArbitrationConfig::Disabled
146        }
147    }
148}
149
150/// Overall configuration of the input/output packet processing pipeline.
151#[derive(Clone, Copy, Debug, Default, PartialEq, Validate)]
152#[cfg_attr(
153    feature = "serde",
154    derive(serde::Serialize, serde::Deserialize),
155    serde(deny_unknown_fields)
156)]
157pub struct PacketPipelineConfig {
158    /// Maximum concurrency (ready-queue depth) when processing outgoing packets (SPHINX encode).
159    ///
160    /// `None` or `Some(0)` both fall back to the CPU-derived default (`available_parallelism * 8`).
161    pub output_concurrency: Option<usize>,
162    /// Maximum concurrency (ready-queue depth) when processing incoming packets (SPHINX decode).
163    ///
164    /// `None` or `Some(0)` both fall back to the CPU-derived default (`available_parallelism * 8`).
165    ///
166    /// This is a queue depth feeding the shared Rayon pool, not a thread count, so it is derived
167    /// from the CPU count and is independent of the (smaller) pool size — tying it to the pool
168    /// collapsed decode to a near-serial rate on small hosts and halved relay forwarding throughput.
169    pub input_concurrency: Option<usize>,
170    /// How long routing resolution keeps waiting for a return path's SURBs before giving up on the
171    /// packet.
172    ///
173    /// `None` falls back to the default of 6 s. `Some(0)` disables the wait entirely, dropping a
174    /// return packet the first time its SURBs are missing.
175    ///
176    /// The right value trades two failures against each other. Too short loses data on a session
177    /// whose SURB pool is only momentarily empty, which is why the wait exists at all. Too long
178    /// stalls **every** packet the node originates, not just this one: resolution preserves
179    /// submission order, so an unresolvable packet withholds everything behind it, and a
180    /// counterparty that has gone away never sends another SURB. An unbounded wait here took a
181    /// production exit's entire egress down for 1h44m while it still forwarded and acknowledged
182    /// normally.
183    ///
184    /// **Set this if you know your session's frame timeout.** The default errs long, because a
185    /// library cannot know it; a packet held past that timeout is discarded by the receiver anyway,
186    /// so the wait is pure stall from then on. hoprd, whose sessions time frames out at 3 s,
187    /// configures 1 s.
188    #[cfg_attr(feature = "serde", serde(default, with = "humantime_serde"))]
189    pub surb_resolution_wait: Option<std::time::Duration>,
190    /// Configuration of the packet acknowledgement processing
191    #[validate(nested)]
192    pub ack_config: AcknowledgementPipelineConfig,
193    /// Fair-share arbitration of the shared Rayon pool between encode and decode.
194    #[validate(nested)]
195    #[cfg_attr(feature = "serde", serde(default))]
196    pub arbitration: PoolArbitrationConfig,
197}
198
199#[cfg(all(test, feature = "serde"))]
200mod tests {
201    use super::*;
202
203    /// Everything this struct requires in a document, minus the wait under test.
204    ///
205    /// Only `surb_resolution_wait` carries `serde(default)`, so the surrounding fields have to be
206    /// written out; `deny_unknown_fields` means the document must otherwise be exact.
207    const REQUIRED_FIELDS: &str = "output_concurrency: null\ninput_concurrency: null\nack_config:\n  \
208                                   ack_input_concurrency: null\n  ack_output_concurrency: null\n";
209
210    fn parse(wait: Option<&str>) -> PacketPipelineConfig {
211        let doc = match wait {
212            Some(value) => format!("{REQUIRED_FIELDS}surb_resolution_wait: {value}\n"),
213            None => REQUIRED_FIELDS.to_string(),
214        };
215        serde_saphyr::from_str(&doc).unwrap_or_else(|e| panic!("config must parse:\n{doc}\n{e}"))
216    }
217
218    /// The wait has to survive a config file, in the human-readable form the rest of this config
219    /// uses. `Option<Duration>` through `humantime_serde` is easy to get wrong in a way that only
220    /// shows up when someone's YAML is silently ignored.
221    #[test]
222    fn the_surb_resolution_wait_should_round_trip_through_yaml() {
223        assert_eq!(
224            Some(std::time::Duration::from_secs(2)),
225            parse(Some("2s")).surb_resolution_wait,
226            "a duration string must reach the field"
227        );
228        assert_eq!(
229            None,
230            parse(None).surb_resolution_wait,
231            "an omitted wait must stay unset so the default applies"
232        );
233        assert_eq!(
234            Some(std::time::Duration::ZERO),
235            parse(Some("0s")).surb_resolution_wait,
236            "zero must reach the code as zero, not as unset"
237        );
238    }
239
240    /// The `SmartDefault` derive and the serde field defaults share the `default_arbitration_*` fns,
241    /// so they can't diverge — pin the values the arbiter ships with, and that a default config
242    /// validates (the percentages sit inside the `1..=100` range).
243    #[test]
244    fn pool_arbitration_config_defaults_are_enabled_75_50_and_valid() {
245        let cfg = PoolArbitrationConfig::default();
246        assert!(cfg.enabled);
247        assert_eq!(cfg.occupancy_pct, 75);
248        assert_eq!(cfg.encode_reserve_pct, 50);
249        assert!(
250            cfg.validate().is_ok(),
251            "the default arbitration config must pass validation"
252        );
253    }
254
255    /// `to_arbitration` maps the flat config onto the arbiter enum: enabled → `Enabled` with the
256    /// percentages, disabled → `Disabled` (no percentages).
257    #[test]
258    fn pool_arbitration_config_maps_onto_the_arbiter_enum() {
259        use hopr_utils::parallelize::cpu::ArbitrationConfig;
260        let enabled = PoolArbitrationConfig {
261            enabled: true,
262            occupancy_pct: 80,
263            encode_reserve_pct: 40,
264        };
265        assert_eq!(
266            enabled.to_arbitration(),
267            ArbitrationConfig::Enabled {
268                occupancy_pct: 80,
269                encode_reserve_pct: 40
270            }
271        );
272        let disabled = PoolArbitrationConfig {
273            enabled: false,
274            ..PoolArbitrationConfig::default()
275        };
276        assert_eq!(disabled.to_arbitration(), ArbitrationConfig::Disabled);
277    }
278}