Fix SYN limiter lifecycle and default burst

Co-Authored-By: brekotis <93345790+brekotis@users.noreply.github.com>
This commit is contained in:
Alexey
2026-06-12 14:40:26 +03:00
parent c4954f745f
commit 2675779915
6 changed files with 96 additions and 34 deletions
+7 -7
View File
@@ -2219,10 +2219,10 @@ Note: This section also accepts the legacy alias `[server.admin_api]` (same sche
| [`ip`](#ip) | `IpAddr` | — | `` | | [`ip`](#ip) | `IpAddr` | — | `` |
| [`port`](#port-serverlisteners) | `u16` | `server.port` | `` | | [`port`](#port-serverlisteners) | `u16` | `server.port` | `` |
| [`client_mss`](#client_mss-serverlisteners) | `String` | `[server].client_mss` | `` | | [`client_mss`](#client_mss-serverlisteners) | `String` | `[server].client_mss` | `` |
| [`synlimit`](#synlimit-serverlisteners) | `false`, `"iptables"`, or `"nftables"` | `false` | `` | | [`synlimit`](#synlimit-serverlisteners) | `false`, `"iptables"`, or `"nftables"` | `false` | `` |
| [`synlimit_seconds`](#synlimit_seconds-serverlisteners) | `u32` | `1` | `` | | [`synlimit_seconds`](#synlimit_seconds-serverlisteners) | `u32` | `1` | `` |
| [`synlimit_hitcount`](#synlimit_hitcount-serverlisteners) | `u32` | `1` | `` | | [`synlimit_hitcount`](#synlimit_hitcount-serverlisteners) | `u32` | `1` | `` |
| [`synlimit_burst`](#synlimit_burst-serverlisteners) | `u32` | `3` | `` | | [`synlimit_burst`](#synlimit_burst-serverlisteners) | `u32` | `2` | `` |
| [`announce`](#announce) | `String` | — | `` | | [`announce`](#announce) | `String` | — | `` |
| [`announce_ip`](#announce_ip) | `IpAddr` | — | `` | | [`announce_ip`](#announce_ip) | `IpAddr` | — | `` |
| [`proxy_protocol`](#proxy_protocol) | `bool` | — | `` | | [`proxy_protocol`](#proxy_protocol) | `bool` | — | `` |
@@ -2260,7 +2260,7 @@ Note: This section also accepts the legacy alias `[server.admin_api]` (same sche
``` ```
## synlimit (server.listeners) ## synlimit (server.listeners)
- **Constraints / validation**: `false`, `"iptables"`, or `"nftables"`. Omitted or `false` disables SYN limiting for this listener. - **Constraints / validation**: `false`, `"iptables"`, or `"nftables"`. Omitted or `false` disables SYN limiting for this listener.
- **Description**: Installs per-listener Linux netfilter SYN limiter rules for the listener port. `"iptables"` uses `iptables`/`ip6tables` filter rules with the `hashlimit` match as a per-source token bucket. `"nftables"` uses per-source `meter` rules with `limit rate over` and auto-detects whether the host already uses `inet`, `ip`, or `ip6` table families before creating Telemt-owned tables. The token-bucket rate is `synlimit_hitcount / synlimit_seconds`; `synlimit_burst` controls the burst size. Rules are reconciled at runtime and removed during graceful Telemt shutdown; `SIGKILL` cannot be cleaned up by the process. Requires CAP_NET_ADMIN and listener restart/rebind for config changes. - **Description**: Installs per-listener Linux netfilter SYN limiter rules for the listener port. `"iptables"` uses `iptables`/`ip6tables` filter rules with the `hashlimit` match as a per-source token bucket. `"nftables"` uses per-source `meter` rules with `limit rate over` and auto-detects whether the host already uses `inet`, `ip`, or `ip6` table families before creating Telemt-owned tables. The token-bucket rate is `synlimit_hitcount / synlimit_seconds`; `synlimit_burst` controls the burst size. Rules are reconciled at runtime and removed during graceful Telemt shutdown; `SIGKILL` cannot be cleaned up by the process. Requires CAP_NET_ADMIN. `synlimit*` changes hot-reload for existing listener endpoints; changing listener `ip` or `port` still requires restart/rebind.
- **Example**: - **Example**:
```toml ```toml
@@ -2299,7 +2299,7 @@ Note: This section also accepts the legacy alias `[server.admin_api]` (same sche
synlimit_hitcount = 1 synlimit_hitcount = 1
``` ```
## synlimit_burst (server.listeners) ## synlimit_burst (server.listeners)
- **Constraints / validation**: `u32`, must be `> 0`. Default is `3`. - **Constraints / validation**: `u32`, must be `> 0`. Default is `2`.
- **Description**: Token-bucket burst size for both SYN limiter backends. Higher values allow short connection bursts from the same source IP before the steady-state `synlimit_hitcount / synlimit_seconds` rate is enforced. - **Description**: Token-bucket burst size for both SYN limiter backends. Higher values allow short connection bursts from the same source IP before the steady-state `synlimit_hitcount / synlimit_seconds` rate is enforced.
- **Example**: - **Example**:
@@ -2308,7 +2308,7 @@ Note: This section also accepts the legacy alias `[server.admin_api]` (same sche
ip = "0.0.0.0" ip = "0.0.0.0"
port = 443 port = 443
synlimit = "iptables" synlimit = "iptables"
synlimit_burst = 3 synlimit_burst = 2
``` ```
## announce ## announce
- **Constraints / validation**: `String` (optional). Must not be empty when set. - **Constraints / validation**: `String` (optional). Must not be empty when set.
+7 -7
View File
@@ -2225,10 +2225,10 @@
| [`ip`](#ip) | `IpAddr` | — | `` | | [`ip`](#ip) | `IpAddr` | — | `` |
| [`port`](#port-serverlisteners) | `u16` | `server.port` | `` | | [`port`](#port-serverlisteners) | `u16` | `server.port` | `` |
| [`client_mss`](#client_mss-serverlisteners) | `String` | `[server].client_mss` | `` | | [`client_mss`](#client_mss-serverlisteners) | `String` | `[server].client_mss` | `` |
| [`synlimit`](#synlimit-serverlisteners) | `false`, `"iptables"` или `"nftables"` | `false` | `` | | [`synlimit`](#synlimit-serverlisteners) | `false`, `"iptables"` или `"nftables"` | `false` | `` |
| [`synlimit_seconds`](#synlimit_seconds-serverlisteners) | `u32` | `1` | `` | | [`synlimit_seconds`](#synlimit_seconds-serverlisteners) | `u32` | `1` | `` |
| [`synlimit_hitcount`](#synlimit_hitcount-serverlisteners) | `u32` | `1` | `` | | [`synlimit_hitcount`](#synlimit_hitcount-serverlisteners) | `u32` | `1` | `` |
| [`synlimit_burst`](#synlimit_burst-serverlisteners) | `u32` | `3` | `` | | [`synlimit_burst`](#synlimit_burst-serverlisteners) | `u32` | `2` | `` |
| [`announce`](#announce) | `String` | — | `` | | [`announce`](#announce) | `String` | — | `` |
| [`announce_ip`](#announce_ip) | `IpAddr` | — | `` | | [`announce_ip`](#announce_ip) | `IpAddr` | — | `` |
| [`proxy_protocol`](#proxy_protocol) | `bool` | — | `` | | [`proxy_protocol`](#proxy_protocol) | `bool` | — | `` |
@@ -2266,7 +2266,7 @@
``` ```
## synlimit (server.listeners) ## synlimit (server.listeners)
- **Ограничения / валидация**: `false`, `"iptables"` или `"nftables"`. Если параметр не задан или задан как `false`, SYN limiter для этого listener’а выключен. - **Ограничения / валидация**: `false`, `"iptables"` или `"nftables"`. Если параметр не задан или задан как `false`, SYN limiter для этого listener’а выключен.
- **Описание**: Устанавливает per-listener Linux netfilter SYN limiter rules для порта listener’а. `"iptables"` использует `iptables`/`ip6tables` filter rules с `hashlimit` match как per-source token bucket. `"nftables"` использует per-source `meter` rules с `limit rate over` и автоматически определяет, какие table families уже используются на хосте (`inet`, `ip`, `ip6`), перед созданием Telemt-owned tables. Token-bucket rate равен `synlimit_hitcount / synlimit_seconds`; `synlimit_burst` управляет burst size. Rules reconciled at runtime и удаляются при graceful shutdown Telemt; `SIGKILL` процессом не очищается. Требует CAP_NET_ADMIN и restart/rebind listener’а для изменений конфигурации. - **Описание**: Устанавливает per-listener Linux netfilter SYN limiter rules для порта listener’а. `"iptables"` использует `iptables`/`ip6tables` filter rules с `hashlimit` match как per-source token bucket. `"nftables"` использует per-source `meter` rules с `limit rate over` и автоматически определяет, какие table families уже используются на хосте (`inet`, `ip`, `ip6`), перед созданием Telemt-owned tables. Token-bucket rate равен `synlimit_hitcount / synlimit_seconds`; `synlimit_burst` управляет burst size. Rules reconciled at runtime и удаляются при graceful shutdown Telemt; `SIGKILL` процессом не очищается. Требует CAP_NET_ADMIN. Изменения `synlimit*` hot-reload’ятся для существующих listener endpoints; изменение listener `ip` или `port` по-прежнему требует restart/rebind.
- **Пример**: - **Пример**:
```toml ```toml
@@ -2305,7 +2305,7 @@
synlimit_hitcount = 1 synlimit_hitcount = 1
``` ```
## synlimit_burst (server.listeners) ## synlimit_burst (server.listeners)
- **Ограничения / валидация**: `u32`, должно быть `> 0`. Значение по умолчанию: `3`. - **Ограничения / валидация**: `u32`, должно быть `> 0`. Значение по умолчанию: `2`.
- **Описание**: Token-bucket burst size для обоих SYN limiter backends. Более высокие значения разрешают short connection bursts с одного source IP перед применением steady-state rate `synlimit_hitcount / synlimit_seconds`. - **Описание**: Token-bucket burst size для обоих SYN limiter backends. Более высокие значения разрешают short connection bursts с одного source IP перед применением steady-state rate `synlimit_hitcount / synlimit_seconds`.
- **Пример**: - **Пример**:
@@ -2314,7 +2314,7 @@
ip = "0.0.0.0" ip = "0.0.0.0"
port = 443 port = 443
synlimit = "iptables" synlimit = "iptables"
synlimit_burst = 3 synlimit_burst = 2
``` ```
## announce ## announce
- **Ограничения / валидация**: `String` (необязательный параметр). Не должен быть пустым, если задан. - **Ограничения / валидация**: `String` (необязательный параметр). Не должен быть пустым, если задан.
+1 -1
View File
@@ -56,7 +56,7 @@ const DEFAULT_CONNTRACK_PRESSURE_LOW_WATERMARK_PCT: u8 = 70;
const DEFAULT_CONNTRACK_DELETE_BUDGET_PER_SEC: u64 = 4096; const DEFAULT_CONNTRACK_DELETE_BUDGET_PER_SEC: u64 = 4096;
const DEFAULT_SYNLIMIT_SECONDS: u32 = 1; const DEFAULT_SYNLIMIT_SECONDS: u32 = 1;
const DEFAULT_SYNLIMIT_HITCOUNT: u32 = 1; const DEFAULT_SYNLIMIT_HITCOUNT: u32 = 1;
const DEFAULT_SYNLIMIT_BURST: u32 = 3; const DEFAULT_SYNLIMIT_BURST: u32 = 2;
const DEFAULT_UPSTREAM_CONNECT_RETRY_ATTEMPTS: u32 = 2; const DEFAULT_UPSTREAM_CONNECT_RETRY_ATTEMPTS: u32 = 2;
const DEFAULT_UPSTREAM_UNHEALTHY_FAIL_THRESHOLD: u32 = 5; const DEFAULT_UPSTREAM_UNHEALTHY_FAIL_THRESHOLD: u32 = 5;
const DEFAULT_UPSTREAM_CONNECT_BUDGET_MS: u64 = 3000; const DEFAULT_UPSTREAM_CONNECT_BUDGET_MS: u64 = 3000;
+3 -3
View File
@@ -907,12 +907,12 @@ async fn run_telemt_core(
std::process::exit(1); std::process::exit(1);
} }
synlimit_control::reconcile_synlimit_rules(&config).await;
synlimit_control::spawn_synlimit_controller(config_rx.clone());
// On Unix, caller supplies privilege drop after bind (may require root for port < 1024). // On Unix, caller supplies privilege drop after bind (may require root for port < 1024).
drop_after_bind(); drop_after_bind();
synlimit_control::reconcile_synlimit_rules(&config).await;
synlimit_control::spawn_synlimit_controller(config_rx.clone());
runtime_tasks::apply_runtime_log_filter( runtime_tasks::apply_runtime_log_filter(
has_rust_log, has_rust_log,
&effective_log_level, &effective_log_level,
+3 -1
View File
@@ -103,7 +103,9 @@ async fn perform_shutdown(
let uptime_secs = process_started_at.elapsed().as_secs(); let uptime_secs = process_started_at.elapsed().as_secs();
info!("Uptime: {}", format_uptime(uptime_secs)); info!("Uptime: {}", format_uptime(uptime_secs));
synlimit_control::clear_synlimit_rules_all_backends().await; if let Err(error) = synlimit_control::clear_synlimit_rules_all_backends().await {
warn!(error = %error, "Failed to clear SYN limiter rules during shutdown");
}
// Graceful ME pool shutdown // Graceful ME pool shutdown
if let Some(pool) = &me_pool { if let Some(pool) = &me_pool {
+74 -14
View File
@@ -80,7 +80,9 @@ pub(crate) fn spawn_synlimit_controller(config_rx: watch::Receiver<Arc<ProxyConf
tokio::spawn(async move { tokio::spawn(async move {
wait_for_config_channel_close_and_reconcile(config_rx).await; wait_for_config_channel_close_and_reconcile(config_rx).await;
clear_synlimit_rules_all_backends().await; if let Err(error) = clear_synlimit_rules_all_backends().await {
warn!(error = %error, "Failed to clear SYN limiter rules after config channel close");
}
}); });
} }
@@ -94,7 +96,9 @@ async fn wait_for_config_channel_close_and_reconcile(
} }
pub(crate) async fn reconcile_synlimit_rules(cfg: &ProxyConfig) { pub(crate) async fn reconcile_synlimit_rules(cfg: &ProxyConfig) {
clear_synlimit_rules_all_backends().await; if let Err(error) = clear_synlimit_rules_all_backends().await {
warn!(error = %error, "Failed to clear existing SYN limiter rules before reconcile");
}
let targets = synlimit_targets(cfg); let targets = synlimit_targets(cfg);
if targets.is_empty() { if targets.is_empty() {
@@ -119,10 +123,23 @@ pub(crate) async fn reconcile_synlimit_rules(cfg: &ProxyConfig) {
} }
} }
pub(crate) async fn clear_synlimit_rules_all_backends() { pub(crate) async fn clear_synlimit_rules_all_backends() -> Result<(), String> {
clear_nft_synlimit_rules_all_families().await; let mut errors = Vec::new();
clear_iptables_synlimit_rules_for_binary("iptables").await; if let Err(error) = clear_nft_synlimit_rules_all_families().await {
clear_iptables_synlimit_rules_for_binary("ip6tables").await; errors.push(error);
}
if let Err(error) = clear_iptables_synlimit_rules_for_binary("iptables").await {
errors.push(error);
}
if let Err(error) = clear_iptables_synlimit_rules_for_binary("ip6tables").await {
errors.push(error);
}
if errors.is_empty() {
Ok(())
} else {
Err(errors.join("; "))
}
} }
fn has_synlimit_config(cfg: &ProxyConfig) -> bool { fn has_synlimit_config(cfg: &ProxyConfig) -> bool {
@@ -314,21 +331,40 @@ fn synlimit_rate_arg(seconds: u32, hitcount: u32) -> String {
format!("{}/day", amount.max(1)) format!("{}/day", amount.max(1))
} }
async fn clear_iptables_synlimit_rules_for_binary(binary: &str) { async fn clear_iptables_synlimit_rules_for_binary(binary: &str) -> Result<(), String> {
let mut errors = Vec::new();
for _ in 0..8 { for _ in 0..8 {
if run_command( match run_command(
binary, binary,
&["-t", "filter", "-D", "INPUT", "-j", IPTABLES_CHAIN], &["-t", "filter", "-D", "INPUT", "-j", IPTABLES_CHAIN],
None, None,
) )
.await .await
.is_err()
{ {
Ok(()) => {}
Err(error) if is_missing_command_or_iptables_rule(&error) => break,
Err(error) => {
errors.push(format!("{binary} delete INPUT jump failed: {error}"));
break; break;
} }
} }
let _ = run_command(binary, &["-t", "filter", "-F", IPTABLES_CHAIN], None).await; }
let _ = run_command(binary, &["-t", "filter", "-X", IPTABLES_CHAIN], None).await; if let Err(error) = run_command(binary, &["-t", "filter", "-F", IPTABLES_CHAIN], None).await
&& !is_missing_command_or_iptables_rule(&error)
{
errors.push(format!("{binary} flush chain failed: {error}"));
}
if let Err(error) = run_command(binary, &["-t", "filter", "-X", IPTABLES_CHAIN], None).await
&& !is_missing_command_or_iptables_rule(&error)
{
errors.push(format!("{binary} delete chain failed: {error}"));
}
if errors.is_empty() {
Ok(())
} else {
Err(errors.join(", "))
}
} }
async fn apply_nft_synlimit_rules(targets: &SynLimitTargets) -> Result<(), String> { async fn apply_nft_synlimit_rules(targets: &SynLimitTargets) -> Result<(), String> {
@@ -439,17 +475,41 @@ fn nft_synlimit_script(plan: NftApplyPlan<'_>) -> String {
script script
} }
async fn clear_nft_synlimit_rules_all_families() { async fn clear_nft_synlimit_rules_all_families() -> Result<(), String> {
let mut errors = Vec::new();
for family in [NftFamily::Inet, NftFamily::Ip, NftFamily::Ip6] { for family in [NftFamily::Inet, NftFamily::Ip, NftFamily::Ip6] {
let _ = run_command( if let Err(error) = run_command(
"nft", "nft",
&["delete", "table", family.as_str(), NFT_TABLE], &["delete", "table", family.as_str(), NFT_TABLE],
None, None,
) )
.await; .await
&& !is_missing_command_or_nft_table(&error)
{
errors.push(format!(
"nft delete table {} {NFT_TABLE} failed: {error}",
family.as_str()
));
} }
} }
if errors.is_empty() {
Ok(())
} else {
Err(errors.join(", "))
}
}
fn is_missing_command_or_iptables_rule(error: &str) -> bool {
error.contains("is not available")
|| error.contains("No chain/target/match by that name")
|| error.contains("does not exist")
}
fn is_missing_command_or_nft_table(error: &str) -> bool {
error.contains("is not available") || error.contains("No such file or directory")
}
async fn run_command(binary: &str, args: &[&str], stdin: Option<String>) -> Result<(), String> { async fn run_command(binary: &str, args: &[&str], stdin: Option<String>) -> Result<(), String> {
let Some(command_path) = resolve_command(binary) else { let Some(command_path) = resolve_command(binary) else {
return Err(format!("{binary} is not available")); return Err(format!("{binary} is not available"));