diff --git a/Cargo.lock b/Cargo.lock index 71423e6..fb0f718 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2790,7 +2790,7 @@ checksum = "7b2093cf4c8eb1e67749a6762251bc9cd836b6fc171623bd0a9d324d37af2417" [[package]] name = "telemt" -version = "3.4.13" +version = "3.4.14" dependencies = [ "aes", "anyhow", diff --git a/Cargo.toml b/Cargo.toml index 99570db..64094e1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "telemt" -version = "3.4.13" +version = "3.4.14" edition = "2024" [features] diff --git a/docs/Advanced_settings/TUNING.de.md b/docs/Advanced_settings/TUNING.de.md index 3b0f31d..4ae113e 100644 --- a/docs/Advanced_settings/TUNING.de.md +++ b/docs/Advanced_settings/TUNING.de.md @@ -86,6 +86,9 @@ Die unten angegebenen `Default`-Werte sind Code-Defaults (bei fehlendem Schlüss | `[[upstreams]].weight` | alle Upstreams | `u16` | nein | `1` | Basisgewicht für weighted-random Auswahl. | | `[[upstreams]].enabled` | alle Upstreams | `bool` | nein | `true` | Deaktivierte Einträge werden beim Start ignoriert. | | `[[upstreams]].scopes` | alle Upstreams | `String` | nein | `""` | Komma-separierte Scope-Tags für Request-Routing. | +| `[[upstreams]].ipv4` | alle Upstreams | `Option` | nein | `auto` | Erlaubt IPv4-DC-Ziele für diesen Upstream. | +| `[[upstreams]].ipv6` | alle Upstreams | `Option` | nein | `auto` | Erlaubt IPv6-DC-Ziele für diesen Upstream, inklusive Proxy-Egress unabhängig vom Host-IPv6. | +| `[[upstreams]].prefer` | alle Upstreams | `Option<4 \| 6>` | nein | effective `[network].prefer` | Pro-Upstream-Präferenz für die DC-Ziel-Adressfamilie. | | `interface` | `direct` | `Option` | nein | `null` | Interface-Name (z. B. `eth0`) oder lokale Literal-IP. | | `bind_addresses` | `direct` | `Option>` | nein | `null` | Explizite Source-IP-Kandidaten (strikter Vorrang vor `interface`). | | `address` | `socks4` | `String` | ja | n/a | SOCKS4-Server (`ip:port` oder `host:port`). | diff --git a/docs/Advanced_settings/TUNING.en.md b/docs/Advanced_settings/TUNING.en.md index 6a6a320..e9d705a 100644 --- a/docs/Advanced_settings/TUNING.en.md +++ b/docs/Advanced_settings/TUNING.en.md @@ -86,6 +86,9 @@ Defaults below are code defaults (used when a key is omitted), not necessarily v | `[[upstreams]].weight` | all upstreams | `u16` | no | `1` | Base weight for weighted-random selection. | | `[[upstreams]].enabled` | all upstreams | `bool` | no | `true` | Disabled entries are ignored at startup. | | `[[upstreams]].scopes` | all upstreams | `String` | no | `""` | Comma-separated scope tags for request-level routing. | +| `[[upstreams]].ipv4` | all upstreams | `Option` | no | `auto` | Allow IPv4 DC targets for this upstream. | +| `[[upstreams]].ipv6` | all upstreams | `Option` | no | `auto` | Allow IPv6 DC targets for this upstream, including proxy egress independent of host IPv6. | +| `[[upstreams]].prefer` | all upstreams | `Option<4 \| 6>` | no | effective `[network].prefer` | Per-upstream DC target family preference. | | `interface` | `direct` | `Option` | no | `null` | Interface name (e.g. `eth0`) or literal local IP for bind selection. | | `bind_addresses` | `direct` | `Option>` | no | `null` | Explicit local source IP candidates (strict priority over `interface`). | | `address` | `socks4` | `String` | yes | n/a | SOCKS4 server endpoint (`ip:port` or `host:port`). | diff --git a/docs/Advanced_settings/TUNING.ru.md b/docs/Advanced_settings/TUNING.ru.md index bae8fdd..0649bca 100644 --- a/docs/Advanced_settings/TUNING.ru.md +++ b/docs/Advanced_settings/TUNING.ru.md @@ -86,6 +86,9 @@ | `[[upstreams]].weight` | все upstream | `u16` | нет | `1` | Базовый вес в weighted-random выборе. | | `[[upstreams]].enabled` | все upstream | `bool` | нет | `true` | Выключенные записи игнорируются на старте. | | `[[upstreams]].scopes` | все upstream | `String` | нет | `""` | Список scope-токенов через запятую для маршрутизации. | +| `[[upstreams]].ipv4` | все upstream | `Option` | нет | `auto` | Разрешает IPv4 DC-targets для этого upstream. | +| `[[upstreams]].ipv6` | все upstream | `Option` | нет | `auto` | Разрешает IPv6 DC-targets для этого upstream, включая proxy egress независимо от IPv6 на хосте. | +| `[[upstreams]].prefer` | все upstream | `Option<4 \| 6>` | нет | эффективный `[network].prefer` | Предпочтительное семейство DC-target для конкретного upstream. | | `interface` | `direct` | `Option` | нет | `null` | Имя интерфейса (например `eth0`) или literal локальный IP. | | `bind_addresses` | `direct` | `Option>` | нет | `null` | Явные кандидаты source IP (имеют приоритет над `interface`). | | `address` | `socks4` | `String` | да | n/a | Адрес SOCKS4 сервера (`ip:port` или `host:port`). | diff --git a/docs/Architecture/API/API.md b/docs/Architecture/API/API.md index f037fc7..9266aad 100644 --- a/docs/Architecture/API/API.md +++ b/docs/Architecture/API/API.md @@ -103,6 +103,7 @@ Notes: | `GET` | `/v1/runtime/me-selftest` | none | `200` | `RuntimeMeSelftestData` | | `GET` | `/v1/runtime/connections/summary` | none | `200` | `RuntimeEdgeConnectionsSummaryData` | | `GET` | `/v1/runtime/events/recent` | none | `200` | `RuntimeEdgeEventsData` | +| `GET` | `/v1/runtime/tls-fingerprints` | optional `limit=1..1000` | `200` | `RuntimeEdgeTlsFingerprintsData` | | `GET` | `/v1/stats/users/active-ips` | none | `200` | `UserActiveIps[]` | | `GET` | `/v1/stats/users` | none | `200` | `UserInfo[]` | | `GET` | `/v1/users` | none | `200` | `UserInfo[]` | @@ -111,6 +112,8 @@ Notes: | `PATCH` | `/v1/users/{username}` | `PatchUserRequest` | `200` or `202` | `UserInfo` | | `DELETE` | `/v1/users/{username}` | none | `200` or `202` | `DeleteUserResponse` | | `POST` | `/v1/users/{username}/rotate-secret` | `RotateSecretRequest` or empty body | `200` or `202` | `CreateUserResponse` | +| `POST` | `/v1/users/{username}/enable` | empty body | `200` or `202` | `UserInfo` | +| `POST` | `/v1/users/{username}/disable` | empty body | `200` or `202` | `UserInfo` | | `POST` | `/v1/users/{username}/reset-quota` | empty body | `200` | `ResetUserQuotaResponse` | ## Endpoint Behavior @@ -146,6 +149,8 @@ Notes: | `PATCH /v1/users/{username}` | Updates selected per-user fields with JSON Merge Patch semantics. | | `DELETE /v1/users/{username}` | Deletes one user and related per-user access-map entries. | | `POST /v1/users/{username}/rotate-secret` | Rotates one user's secret and returns the effective secret. | +| `POST /v1/users/{username}/enable` | Enables one user, removing any disabled override from config. | +| `POST /v1/users/{username}/disable` | Disables one user and closes active runtime sessions for that user. | | `POST /v1/users/{username}/reset-quota` | Resets one user's runtime quota counter and persists quota state. | ## Common Error Codes @@ -175,6 +180,8 @@ Notes: | `PUT /v1/users/{username}` | `405 method_not_allowed`. | | `POST /v1/users/{username}` | `404 not_found`. | | `POST /v1/users/{username}/rotate-secret/` | Trailing slash is trimmed and the route matches `rotate-secret`. | +| `POST /v1/users/{username}/enable/` | Trailing slash is trimmed and the route matches `enable`. | +| `POST /v1/users/{username}/disable/` | Trailing slash is trimmed and the route matches `disable`. | | `POST /v1/users/{username}/reset-quota/` | Trailing slash is trimmed and the route matches `reset-quota`. | ## Body and JSON Semantics @@ -208,6 +215,7 @@ Notes: | `rate_limit_up_bps` | `u64` | no | Per-user upload rate limit in bytes per second. | | `rate_limit_down_bps` | `u64` | no | Per-user download rate limit in bytes per second. | | `max_unique_ips` | `usize` | no | Per-user unique source IP limit. | +| `enabled` | `bool` | no | User enable flag. Missing means enabled. `false` persists a disabled override. | ### `PatchUserRequest` | Field | Type | Required | Description | @@ -220,6 +228,7 @@ Notes: | `rate_limit_up_bps` | `u64|null` | no | Per-user upload rate limit in bytes per second; `null` removes the upload direction limit. | | `rate_limit_down_bps` | `u64|null` | no | Per-user download rate limit in bytes per second; `null` removes the download direction limit. | | `max_unique_ips` | `usize|null` | no | Per-user unique source IP limit; `null` removes the per-user override. | +| `enabled` | `bool|null` | no | `false` disables the user. `true` or `null` removes the disabled override, so the user is enabled. | ### `access.user_source_deny` via API - In current API surface, per-user deny-list is **not** exposed as a dedicated field in `CreateUserRequest` / `PatchUserRequest`. @@ -807,6 +816,43 @@ An empty request body is accepted and generates a new secret automatically. | `event_type` | `string` | Event kind identifier. | | `context` | `string` | Context text (truncated to implementation-defined max length). | +### `RuntimeEdgeTlsFingerprintsData` +| Field | Type | Description | +| --- | --- | --- | +| `enabled` | `bool` | Endpoint availability under `runtime_edge_enabled`. | +| `reason` | `string?` | `feature_disabled` when endpoint is disabled. | +| `generated_at_epoch_secs` | `u64` | Snapshot generation timestamp. | +| `data` | `RuntimeEdgeTlsFingerprintsPayload?` | Null when unavailable. | + +#### `RuntimeEdgeTlsFingerprintsPayload` +| Field | Type | Description | +| --- | --- | --- | +| `limit` | `usize` | Effective Top-N row count. | +| `retention_secs` | `u64` | In-memory retention window, derived from `general.beobachten_minutes`. | +| `capacity` | `usize` | Maximum retained fingerprint buckets. | +| `dropped_total` | `u64` | Buckets dropped because the collector was full. | +| `parse_error_total` | `u64` | Complete ClientHello records that could not be fingerprinted. | +| `by_fingerprint` | `RuntimeEdgeTlsFingerprintRow[]` | Global JA3/JA4 leaderboard. | +| `by_ip` | `RuntimeEdgeTlsFingerprintRow[]` | Source-IP scoped leaderboard. | +| `by_cidr` | `RuntimeEdgeTlsFingerprintRow[]` | Source CIDR scoped leaderboard (`/24` for IPv4, `/56` for IPv6). | +| `by_user` | `RuntimeEdgeTlsFingerprintRow[]` | Authenticated user scoped leaderboard. | + +#### `RuntimeEdgeTlsFingerprintRow` +| Field | Type | Description | +| --- | --- | --- | +| `scope` | `string?` | IP, CIDR, or username; absent in `by_fingerprint`. | +| `ja3` | `string` | JA3 MD5 hash. | +| `ja3_raw` | `string` | Raw JA3 field string. | +| `ja4` | `string` | JA4 TLS client fingerprint. | +| `ja4_raw` | `string` | Raw JA4 material used for the hashed parts. | +| `total` | `u64` | Complete ClientHello observations for this bucket. | +| `auth_success` | `u64` | TLS-authenticated observations for this bucket. | +| `bad_or_probe` | `u64` | Complete ClientHello observations later classified as bad/probe. | +| `first_seen_epoch_secs` | `u64` | First observation timestamp. | +| `last_seen_epoch_secs` | `u64` | Last observation timestamp. | + +JA3 follows the Salesforce ClientHello field order. JA4 follows the FoxIO TLS-client `a_b_c` format; GREASE values are excluded and no high-cardinality Prometheus labels are emitted for fingerprints. + ### `ZeroAllData` | Field | Type | Description | | --- | --- | --- | @@ -1165,6 +1211,7 @@ An empty request body is accepted and generates a new secret automatically. | Field | Type | Description | | --- | --- | --- | | `username` | `string` | Username. | +| `enabled` | `bool` | Effective user enable flag. Missing config entry is reported as `true`. | | `in_runtime` | `bool` | Whether current runtime config already contains this user. | | `user_ad_tag` | `string?` | Optional ad tag (32 hex chars). | | `max_tcp_conns` | `usize?` | Optional max concurrent TCP limit. | @@ -1239,6 +1286,8 @@ Link generation uses active config and enabled modes: | `POST /v1/users` | Creates user, validates config, then atomically updates only affected `access.*` TOML tables (`access.users` always, plus optional per-user tables present in request). | | `PATCH /v1/users/{username}` | Partial update of provided fields only. Missing fields remain unchanged; explicit `null` removes optional per-user entries. The write path updates only affected `access.*` TOML tables. | | `POST /v1/users/{username}/rotate-secret` | Replaces the user's secret with a provided valid 32-hex value or a generated value, then returns the effective secret in `CreateUserResponse`. | +| `POST /v1/users/{username}/enable` | Enables the user idempotently by removing the `access.user_enabled[username]` override and updating the runtime admission state immediately. | +| `POST /v1/users/{username}/disable` | Disables the user idempotently by writing `access.user_enabled[username] = false`, updating runtime admission immediately, and cancelling active sessions for that username. | | `POST /v1/users/{username}/reset-quota` | Resets the runtime quota counter for the route username, persists quota state to `general.quota_state_path`, and does not modify user config. | | `DELETE /v1/users/{username}` | Deletes only specified user, removes this user from related optional `access.user_*` maps, blocks last-user deletion, and atomically updates only related `access.*` TOML tables. | @@ -1282,6 +1331,7 @@ Additional runtime endpoint behavior: | `/v1/runtime/me-selftest` | No | ME pool unavailable => `enabled=false`, `reason=source_unavailable` | `enabled=true`, full payload | | `/v1/runtime/connections/summary` | `runtime_edge_enabled=false` => `enabled=false`, `reason=feature_disabled` | Recompute lock contention with no cache entry => `enabled=true`, `reason=source_unavailable` | `enabled=true`, full payload | | `/v1/runtime/events/recent` | `runtime_edge_enabled=false` => `enabled=false`, `reason=feature_disabled` | Not used in current implementation | `enabled=true`, full payload | +| `/v1/runtime/tls-fingerprints` | `runtime_edge_enabled=false` => `enabled=false`, `reason=feature_disabled` | Not used in current implementation | `enabled=true`, full payload | ## ME Fallback Behavior Exposed Via API diff --git a/docs/Architecture/Fronting-splitting/TLS_JA3_JA4_ANALYSIS.ru.md b/docs/Architecture/Fronting-splitting/TLS_JA3_JA4_ANALYSIS.ru.md new file mode 100644 index 0000000..ebb62ae --- /dev/null +++ b/docs/Architecture/Fronting-splitting/TLS_JA3_JA4_ANALYSIS.ru.md @@ -0,0 +1,507 @@ +# JA3 и JA4 анализ в Telemt + +Этот документ описывает, как использовать JA3/JA4 telemetry в Telemt для диагностики блокировок, которые происходят на основе TLS ClientHello, особенно JA4 TLS client fingerprint. + +Цель документа практическая: помочь оператору понять, какой клиентский TLS-отпечаток реально доходит до Telemt, как он распределён по IP/CIDR/пользователям, и как отделить JA4-based фильтрацию от блокировки по IP, SNI, домену, server flight или активному сканированию. + +## Коротко + +JA3 и JA4 описывают форму TLS ClientHello. ClientHello отправляет клиент, поэтому JA3/JA4 в этом контексте являются fingerprint'ами клиентской TLS-реализации, а не Telemt как сервера. + +Telemt собирает JA3/JA4 только из уже прочитанного полного ClientHello: + +- без packet capture; +- без MITM; +- без расшифровки TLS; +- без дополнительных сетевых чтений; +- без Prometheus labels с высокой кардинальностью; +- с ограниченным in-memory TTL/cap collector. + +Собранные данные доступны: + +- через API: `GET /v1/runtime/tls-fingerprints`; +- через `/beobachten`, если `general.beobachten=true`. + +Основная польза: + +- увидеть, какие JA4 реально используют клиенты; +- понять, один ли fingerprint страдает у всех пользователей; +- отделить проблему клиента от проблемы IP/ASN/домена; +- увидеть, доходят ли проблемные соединения до Telemt вообще; +- сравнить successful TLS-auth и bad/probe поток для одного fingerprint; +- собрать evidence для последующего изменения клиента, маршрута или deployment-профиля. + +## Что такое JA3 + +JA3 - старый и широко совместимый способ получить hash от TLS ClientHello. + +JA3 строится из ClientHello fields: + +```text +SSLVersion,Cipher,SSLExtension,EllipticCurve,EllipticCurvePointFormat +``` + +Значения внутри полей записываются в порядке, в котором они пришли в ClientHello. GREASE values исключаются. Итоговая строка хэшируется MD5, поэтому в API есть два поля: + +- `ja3` - MD5 hash; +- `ja3_raw` - исходная строка, из которой получен hash. + +Практическое значение JA3 в 2026 году ограничено тем, что современные TLS-клиенты и браузерные стеки могут менять порядок extensions. Поэтому JA3 полезен как совместимый исторический сигнал, но для диагностики современных блокировок обычно важнее JA4. + +## Что такое JA4 + +JA4 TLS client fingerprint - более структурированный fingerprint ClientHello. + +JA4 в Telemt считается для TLS-over-TCP ClientHello и имеет форму: + +```text +t__ +``` + +Пример: + +```text +t13d1516h2_8daaf6152771_e5627efa2ab1 +``` + +Части JA4: + +| Часть | Смысл | +| --- | --- | +| `t` | TLS over TCP. Telemt сейчас не считает JA4 для QUIC/DTLS. | +| `13`, `12`, `11`, `10` | TLS version, предпочтительно из `supported_versions`. | +| `d` / `i` | Есть SNI domain (`d`) или SNI отсутствует (`i`). | +| `15` | Количество cipher suites без GREASE, capped до `99`. | +| `16` | Количество extensions без GREASE, capped до `99`. | +| `h2`, `h1`, `00` | ALPN marker: первый и последний символ первого ALPN value или `00`. | +| `cipher_hash` | SHA256 от отсортированного списка ciphers, первые 12 hex chars. | +| `extension_hash` | SHA256 от отсортированных extensions плюс signature algorithms, первые 12 hex chars. | + +Важное отличие JA4 от JA3: JA4 нормализует часть полей, поэтому он устойчивее к простому изменению порядка extensions. Это делает JA4 удобным для фильтров и одновременно полезным для диагностики таких фильтров. + +## Где Telemt видит ClientHello + +В TLS/FakeTLS режиме Telemt получает первые bytes соединения и определяет, похоже ли оно на TLS handshake. Если record является полным ClientHello и проходит bounds checks, Telemt один раз парсит его для JA3/JA4. + +Дальше возможны три исхода: + +1. **Успешный MTProxy/FakeTLS клиент** + - Telemt принимает TLS-auth; + - fingerprint записывается в global/IP/CIDR scopes; + - после успешной TLS-auth Telemt добавляет user scope. + +2. **Bad client или probe** + - ClientHello полный, но auth не проходит; + - fingerprint записывается в global/IP/CIDR scopes; + - user scope не записывается; + - `bad_or_probe` увеличивается. + +3. **Неполный или обрезанный ClientHello** + - fingerprint не считается; + - такие случаи остаются в существующих bad-class counters. + +Если фильтр режет трафик до того, как TCP connection или ClientHello дошли до процесса Telemt, Telemt не увидит этот fingerprint. Это важнейшее диагностическое отличие: отсутствие fingerprint'а во время жалобы пользователя часто означает блокировку до приложения, а не проблему внутри Telemt. + +## Включение сбора + +Collector включается, когда включён хотя бы один потребитель: + +```toml +[general] +beobachten = true +beobachten_minutes = 10 +``` + +или: + +```toml +[server.api] +runtime_edge_enabled = true +runtime_edge_top_n = 50 +``` + +Практически: + +- для файлового/metrics endpoint анализа достаточно `general.beobachten=true`; +- для API snapshot нужен `server.api.runtime_edge_enabled=true`; +- `general.beobachten_minutes` задаёт retention window для fingerprint buckets; +- `server.api.runtime_edge_top_n` задаёт default Top-N размер API snapshot. + +## API snapshot + +Endpoint: + +```bash +curl -s http://127.0.0.1:9091/v1/runtime/tls-fingerprints +``` + +С явным лимитом: + +```bash +curl -s 'http://127.0.0.1:9091/v1/runtime/tls-fingerprints?limit=100' +``` + +Если API защищён header'ом: + +```bash +curl -s \ + -H 'Authorization: Bearer YOUR_TOKEN' \ + 'http://127.0.0.1:9091/v1/runtime/tls-fingerprints?limit=100' +``` + +Если `runtime_edge_enabled=false`, endpoint возвращает payload с: + +```json +{ + "enabled": false, + "reason": "feature_disabled" +} +``` + +### Структура payload + +Основные поля: + +| Поле | Смысл | +| --- | --- | +| `retention_secs` | Текущее TTL окно collector'а. | +| `capacity` | Максимум retained buckets. | +| `dropped_total` | Сколько новых buckets отброшено из-за cap. | +| `parse_error_total` | Сколько полных ClientHello не удалось распарсить. | +| `by_fingerprint` | Top fingerprints глобально. | +| `by_ip` | Top fingerprints по exact source IP. | +| `by_cidr` | Top fingerprints по source prefix: IPv4 `/24`, IPv6 `/56`. | +| `by_user` | Top fingerprints по authenticated user. | + +Строка snapshot: + +| Поле | Смысл | +| --- | --- | +| `scope` | IP, CIDR или username. В `by_fingerprint` отсутствует. | +| `ja3` | JA3 hash. | +| `ja3_raw` | Raw JA3 string. | +| `ja4` | JA4 TLS client fingerprint. | +| `ja4_raw` | Raw JA4 material. | +| `total` | Сколько полных ClientHello попало в этот bucket. | +| `auth_success` | Сколько из них успешно прошли TLS-auth. | +| `bad_or_probe` | Сколько были bad/probe после полного ClientHello. | +| `first_seen_epoch_secs` | Первый timestamp bucket'а. | +| `last_seen_epoch_secs` | Последний timestamp bucket'а. | + +### Быстрый просмотр через jq + +Top JA4 глобально: + +```bash +curl -s http://127.0.0.1:9091/v1/runtime/tls-fingerprints \ + | jq -r '.data.data.by_fingerprint[] | [.ja4, .total, .auth_success, .bad_or_probe] | @tsv' +``` + +Top JA4 по пользователям: + +```bash +curl -s http://127.0.0.1:9091/v1/runtime/tls-fingerprints?limit=100 \ + | jq -r '.data.data.by_user[] | [.scope, .ja4, .total, .auth_success] | @tsv' +``` + +Top JA4 по CIDR: + +```bash +curl -s http://127.0.0.1:9091/v1/runtime/tls-fingerprints?limit=100 \ + | jq -r '.data.data.by_cidr[] | [.scope, .ja4, .total, .auth_success, .bad_or_probe] | @tsv' +``` + +Ошибки парсинга и drops: + +```bash +curl -s http://127.0.0.1:9091/v1/runtime/tls-fingerprints \ + | jq '.data.data | {retention_secs, capacity, dropped_total, parse_error_total}' +``` + +## Beobachten output + +Если включён endpoint metrics, `/beobachten` содержит обычные forensic buckets и, когда есть данные, append-only секцию TLS fingerprints: + +```bash +curl -s http://127.0.0.1:9090/beobachten +``` + +Фрагмент: + +```text +[tls_fingerprints] +retention_secs=600 capacity=65536 dropped_total=0 parse_error_total=0 +[tls_fingerprints.by_fingerprint] +ja4=t13d1516h2_8daaf6152771_e5627efa2ab1 ja3=... total=42 auth_success=41 bad_or_probe=1 first_seen=... last_seen=... +[tls_fingerprints.by_cidr] +scope=203.0.113.0/24 ja4=t13d1516h2_8daaf6152771_e5627efa2ab1 ja3=... total=10 auth_success=10 bad_or_probe=0 first_seen=... last_seen=... +``` + +`/beobachten` удобен для быстрой операторской диагностики без API client. API удобнее для автоматической корреляции. + +## Как анализировать JA4-based блокировку + +### 1. Зафиксировать симптом + +Перед анализом нужно записать: + +- какие пользователи жалуются; +- какая версия Telegram client используется; +- какая платформа: Desktop, Android, iOS; +- какой источник сети: mobile ISP, home ISP, corporate network, country/region; +- работает ли тот же пользователь через другой network path; +- работает ли другой пользователь с того же IP/CIDR; +- видит ли Telemt новые ClientHello от проблемного пользователя в момент попытки. + +JA4 без контекста почти всегда недостаточен. Фильтры часто используют сочетание: + +- JA4; +- destination IP; +- SNI; +- порт; +- ASN/source network; +- rate или connection pattern; +- reputation домена/IP; +- active probing result. + +### 2. Проверить, доходит ли ClientHello до Telemt + +Во время попытки подключения проблемного пользователя смотрите: + +```bash +curl -s 'http://127.0.0.1:9091/v1/runtime/tls-fingerprints?limit=200' \ + | jq '.data.data.by_user, .data.data.by_ip, .data.data.by_cidr' +``` + +Интерпретация: + +| Наблюдение | Вероятный вывод | +| --- | --- | +| Нет новых rows для IP/CIDR пользователя | Блокировка до Telemt: routing, firewall, ISP/DPI drop, IP block, SYN/TCP reset, UDP/TCP path issue. | +| Есть `by_ip`/`by_cidr`, но нет `by_user` | ClientHello дошёл, но TLS-auth/MTProxy layer не дошёл до успешного пользователя. Возможны bad key, probe, wrong client, active scanner, обрыв после ClientHello. | +| Есть `by_user.auth_success` | Клиентский JA4 дошёл и был принят Telemt. Если пользователь всё равно видит проблему, искать нужно дальше: relay path, Telegram upstream, quota, route mode, session cancellation, ME/direct routing. | +| Резко растёт `bad_or_probe` для одного JA4 | Вероятны сканеры или неправильные клиенты с тем же fingerprint family. | + +### 3. Сравнить working и blocked случаи + +Снимите snapshot во время working case и blocked case: + +```bash +curl -s 'http://127.0.0.1:9091/v1/runtime/tls-fingerprints?limit=500' > tls-fp-working.json +curl -s 'http://127.0.0.1:9091/v1/runtime/tls-fingerprints?limit=500' > tls-fp-blocked.json +``` + +Сравните: + +- появился ли тот же `ja4` в blocked сети; +- меняется ли `ja4` между версиями клиента; +- меняется ли только IP/CIDR при том же `ja4`; +- есть ли `auth_success` для того же `ja4` из других сетей; +- отличается ли `bad_or_probe` между сетями. + +Ключевая матрица: + +| Working JA4 | Blocked JA4 | Вывод | +| --- | --- | --- | +| Same | Same, но blocked network не доходит до Telemt | Вероятна фильтрация по JA4 + destination/IP/SNI/network до приложения. | +| Same | Same, доходит и `auth_success>0` | JA4 ClientHello не является точкой отказа; искать post-auth проблему. | +| Different | Blocked только один JA4 | Вероятен client-version/platform-specific fingerprint block. | +| Same | `bad_or_probe` растёт, `auth_success=0` | Возможно, доходит не тот клиент/secret или фильтр/прокси ломает поток после ClientHello. | + +### 4. Разделить client JA4 и server fingerprint + +JA4 ClientHello - это клиентская сторона. Настройки Telemt вроде TLS-front server flight, `mask_host`, ticket-tail или CCS replay не меняют ClientHello, который отправляет Telegram client. + +Если фильтр принимает решение строго после ClientHello, то серверные улучшения могут не помочь. В этом случае полезные действия: + +- проверить обновление Telegram client; +- сравнить платформы и версии клиента; +- проверить, меняется ли JA4 на другой версии; +- проверить, блокируется ли тот же JA4 к другому destination; +- проверить, блокируется ли другой JA4 к тому же Telemt IP/SNI; +- собрать evidence для client-side fingerprint fix. + +Если ClientHello проходит, а блокировка возникает после server response, тогда уже важны: + +- форма FakeTLS server flight; +- TLS front profile fidelity; +- `mask_host` поведение для non-auth clients; +- certificate/provenance fallback для сканеров; +- TCP relay behavior; +- upstream route к Telegram. + +### 5. Коррелировать с packet capture + +Telemt collector показывает только то, что процесс увидел. Для подтверждения фильтрации до Telemt нужен внешний capture. + +На сервере: + +```bash +sudo tcpdump -i any -w telemt-clienthello.pcap host CLIENT_IP and port 443 +``` + +Быстрый tshark вывод ClientHello fields: + +```bash +tshark -r telemt-clienthello.pcap -Y "tls.handshake.type == 1" -T fields \ + -e frame.time_epoch \ + -e ip.src \ + -e ip.dst \ + -e tcp.srcport \ + -e tcp.dstport \ + -e tls.handshake.extensions_server_name \ + -e tls.handshake.extensions_alpn_str +``` + +Если на клиентской стороне capture видит ClientHello, а серверный capture не видит, проблема в сети между клиентом и сервером. Если серверный capture видит ClientHello, но Telemt API не видит fingerprint, проверьте порт, listener, PROXY protocol, TLS record fragmentation и bounds/errors. + +## Практические сценарии + +### Сценарий A: один JA4 перестал работать у многих пользователей + +Признаки: + +- один `ja4` доминирует в жалобах; +- у разных source CIDR нет `auth_success`; +- working пользователи используют другой JA4; +- обновление клиента меняет поведение. + +Вероятный вывод: фильтр на стороне сети научился распознавать конкретный ClientHello family. + +Действия: + +- сравнить Telegram client versions; +- проверить, не используют ли пользователи старые клиенты; +- собрать `ja4`, `ja4_raw`, platform/version, source network; +- проверить тот же client через другую сеть; +- проверить другой client version через ту же сеть. + +### Сценарий B: один CIDR не работает, JA4 обычный + +Признаки: + +- тот же `ja4` успешно работает из других сетей; +- проблемный `/24` или `/56` не доходит до Telemt или не получает `auth_success`; +- нет общей корреляции по версии клиента. + +Вероятный вывод: проблема не в JA4 alone, а в source network policy или destination reputation. + +Действия: + +- сменить route/VPS/IP; +- проверить port; +- проверить SNI/domain reputation; +- сравнить с другим Telemt endpoint; +- смотреть server-side packet capture. + +### Сценарий C: много `bad_or_probe` на одном JA4 + +Признаки: + +- `bad_or_probe` высокий; +- `by_user` пустой или слабый; +- source IP/CIDR разнообразные; +- попытки не соответствуют реальным пользователям. + +Вероятный вывод: активное сканирование или нерелевантный TLS traffic с похожим ClientHello. + +Действия: + +- смотреть `/beobachten` по IP classes; +- проверить `unknown_tls_sni` и bad-client counters; +- убедиться, что fallback `mask_host` отвечает правдоподобно; +- не делать вывод о блокировке пользователей только по global `bad_or_probe`. + +### Сценарий D: `auth_success` есть, но пользователь жалуется + +Признаки: + +- fingerprint присутствует в `by_user`; +- `auth_success` растёт; +- соединение проходит TLS-auth. + +Вероятный вывод: JA4 ClientHello не является причиной отказа в этом случае. + +Действия: + +- проверить user enabled/disabled status; +- проверить quota; +- проверить direct/ME route; +- проверить upstream health; +- проверить runtime events; +- смотреть relay/session logs. + +## Что нельзя вывести из JA3/JA4 + +JA3/JA4 не говорят: + +- почему сеть приняла решение о блокировке; +- какой именно vendor DPI используется; +- был ли block только по JA4 или по связке JA4+IP+SNI; +- что произошло с соединением после TLS-auth; +- как выглядит server-side TLS fingerprint; +- как ведёт себя HTTP layer после TLS. + +JA3/JA4 также не являются уникальной идентичностью человека. Это fingerprint клиентской TLS-реализации и её настроек. Один fingerprint может быть у большого числа пользователей. + +## Ограничения collector'а Telemt + +- Считается только TLS ClientHello, который полностью дошёл до Telemt. +- QUIC/DTLS/HTTP JA4 variants не собираются. +- Truncated ClientHello не fingerprint'ится. +- User scope появляется только после успешной TLS-auth. +- `by_ip` и `by_cidr` отражают source address после нормализации/PROXY protocol path, если он используется. +- Collector bounded: при большом количестве уникальных buckets возможен рост `dropped_total`. +- Retention зависит от `general.beobachten_minutes`. +- Данные runtime in-memory; это snapshot для диагностики, а не долговременное хранилище. + +## Рекомендованный workflow расследования + +1. Включить `runtime_edge_enabled=true` и разумный `runtime_edge_top_n`, например `100`. +2. Зафиксировать baseline в период нормальной работы. +3. Во время жалобы снять API snapshot и `/beobachten`. +4. Сравнить `by_user`, `by_ip`, `by_cidr`, `by_fingerprint`. +5. Проверить, появляется ли problematic source в Telemt вообще. +6. Если не появляется, снять packet capture на сервере и клиенте. +7. Если появляется без `auth_success`, проверить secret/client/proxy link и bad/probe counters. +8. Если появляется с `auth_success`, исключить JA4 ClientHello как primary cause и перейти к relay/upstream/runtime диагностике. +9. Если один JA4 стабильно коррелирует с block, собрать client version/platform evidence. +10. Проверить, меняет ли обновление клиента JA4 и результат подключения. + +## Минимальный incident report + +Для полезного отчёта по JA4-based блокировке соберите: + +```text +time_window: +telemt_version: +server_ip: +server_port: +tls_domain: +mask_host: +client_platform: +client_version: +source_network: +source_ip_or_cidr: +ja4: +ja4_raw: +ja3: +total: +auth_success: +bad_or_probe: +seen_in_by_user: yes/no +seen_in_by_ip: yes/no +seen_in_by_cidr: yes/no +server_tcpdump_seen_clienthello: yes/no +client_tcpdump_sent_clienthello: yes/no +works_from_other_network: yes/no +works_with_other_client_version: yes/no +``` + +Этот набор обычно достаточен, чтобы отличить client fingerprint block от IP/SNI/reputation block и от post-auth проблем Telemt. + +## Источники форматов + +- JA3 reference: https://github.com/salesforce/ja3 +- JA4 technical details: https://github.com/FoxIO-LLC/ja4/blob/main/technical_details/JA4.md + diff --git a/docs/Config_params/CONFIG_PARAMS.en.md b/docs/Config_params/CONFIG_PARAMS.en.md index f28e338..03fcc4f 100644 --- a/docs/Config_params/CONFIG_PARAMS.en.md +++ b/docs/Config_params/CONFIG_PARAMS.en.md @@ -632,7 +632,7 @@ This document lists all configuration keys accepted by `config.toml`. ``` ## beobachten - **Constraints / validation**: `bool`. - - **Description**: Enables per-IP forensic observation buckets. + - **Description**: Enables per-IP forensic observation buckets and appends TLS JA3/JA4 fingerprint snapshots to Beobachten output when available. - **Example**: ```toml @@ -641,7 +641,7 @@ This document lists all configuration keys accepted by `config.toml`. ``` ## beobachten_minutes - **Constraints / validation**: Must be `> 0` (minutes). - - **Description**: Retention window (minutes) for per-IP observation buckets. + - **Description**: Retention window (minutes) for per-IP observation buckets and in-memory TLS fingerprint buckets. - **Example**: ```toml @@ -2173,7 +2173,7 @@ Note: This section also accepts the legacy alias `[server.admin_api]` (same sche ``` ## runtime_edge_top_n - **Constraints / validation**: `1..=1000`. - - **Description**: Top-N size for edge connection leaderboard. + - **Description**: Top-N size for edge connection and TLS fingerprint leaderboard snapshots. - **Example**: ```toml @@ -2934,6 +2934,7 @@ If your backend or network is very bandwidth-constrained, reduce cap first. If p | Key | Type | Default | Hot-Reload | | --- | ---- | ------- | ---------- | | [`users`](#users) | `Map` | `{"default": "000…000"}` | `✔` | +| [`user_enabled`](#user_enabled-1) | `Map` | `{}` | `✔` | | [`user_ad_tags`](#user_ad_tags) | `Map` | `{}` | `✔` | | [`user_max_tcp_conns`](#user_max_tcp_conns) | `Map` | `{}` | `✔` | | [`user_max_tcp_conns_global_each`](#user_max_tcp_conns_global_each) | `usize` | `0` | `✔` | @@ -2960,6 +2961,16 @@ If your backend or network is very bandwidth-constrained, reduce cap first. If p alice = "00112233445566778899aabbccddeeff" bob = "0123456789abcdef0123456789abcdef" ``` +## user_enabled + - **Constraints / validation**: `Map`. + - **Description**: Optional per-user enable overrides. Missing users are enabled by default. A value of `false` disables new sessions for that user; setting the value to `true` is accepted but equivalent to removing the override. API enable operations remove the override, while disable operations write `false`. + - **Runtime behavior**: Hot reload applies this map immediately. Users disabled through API or config reload are rejected after successful authentication and active runtime sessions for that username are cancelled. + - **Example**: + + ```toml + [access.user_enabled] + alice = false + ``` ## user_ad_tags - **Constraints / validation**: Each value must be **exactly 32 hex characters** (same format as `general.ad_tag`). An all-zero tag is allowed but logs a warning. - **Description**: Per-user sponsored-channel ad tag override. When a user has an entry here, it takes precedence over `general.ad_tag`. @@ -3120,6 +3131,7 @@ If your backend or network is very bandwidth-constrained, reduce cap first. If p | [`scopes`](#scopes) | `String` | `""` | `✘` | | [`ipv4`](#ipv4-upstreams) | `bool` | — (auto) | `✘` | | [`ipv6`](#ipv6-upstreams) | `bool` | — (auto) | `✘` | +| [`prefer`](#prefer-upstreams) | `4` or `6` | effective `[network].prefer` | `✘` | | [`interface`](#interface) | `String` | — | `✘` | | [`bind_addresses`](#bind_addresses) | `String[]` | — | `✘` | | [`bindtodevice`](#bindtodevice) | `String` | — | `✘` | @@ -3191,7 +3203,7 @@ If your backend or network is very bandwidth-constrained, reduce cap first. If p ``` ## ipv6 (upstreams) - **Constraints / validation**: `bool` (optional). - - **Description**: Allows IPv6 DC targets for this upstream. When omitted, Telemt auto-detects support from runtime connectivity state. + - **Description**: Allows IPv6 DC targets for this upstream. When omitted, Telemt auto-detects support from runtime connectivity state. Set this to `true` when the upstream proxy is reachable from the local host over IPv4 but the proxy itself can connect to Telegram DCs over IPv6. - **Example**: ```toml @@ -3199,6 +3211,18 @@ If your backend or network is very bandwidth-constrained, reduce cap first. If p type = "direct" ipv6 = false ``` +## prefer (upstreams) + - **Constraints / validation**: Optional integer. Must be `4` or `6`. + - **Description**: Overrides the IP family preference for Telegram DC targets selected through this upstream. When omitted, the upstream inherits the effective global `[network].prefer` decision. Use `prefer = 6` together with `ipv6 = true` for a SOCKS or Shadowsocks upstream that can egress over IPv6 even when the local Telemt host is IPv4-only. + - **Example**: + + ```toml + [[upstreams]] + type = "socks5" + address = "192.0.2.10:1080" + ipv6 = true + prefer = 6 + ``` ## interface - **Constraints / validation**: `String` (optional). - For `"direct"`: may be an IP address (used as explicit local bind) or an OS interface name (resolved to an IP at runtime; Unix only). diff --git a/docs/Config_params/CONFIG_PARAMS.ru.md b/docs/Config_params/CONFIG_PARAMS.ru.md index 75c4cb0..3e99623 100644 --- a/docs/Config_params/CONFIG_PARAMS.ru.md +++ b/docs/Config_params/CONFIG_PARAMS.ru.md @@ -632,7 +632,7 @@ ``` ## beobachten - **Ограничения / валидация**: `bool`. - - **Описание**: Включает "криминалистическое" наблюдения для каждого IP-адреса. Анализирует поведение всех подключений и записывает возможные типы клиентов, которые посылают active-probing запросы. + - **Описание**: Включает "криминалистическое" наблюдения для каждого IP-адреса. Анализирует поведение всех подключений, записывает возможные типы клиентов, которые посылают active-probing запросы, и добавляет snapshot’ы TLS JA3/JA4 fingerprint’ов в Beobachten output, когда есть данные. - **Пример**: ```toml @@ -641,7 +641,7 @@ ``` ## beobachten_minutes - **Ограничения / валидация**: Должно быть `> 0` (минут). - - **Описание**: Время хранения (минуты) для сегментов наблюдения по каждому IP-адресу. + - **Описание**: Время хранения (минуты) для сегментов наблюдения по каждому IP-адресу и in-memory bucket’ов TLS fingerprint’ов. - **Пример**: ```toml @@ -2179,7 +2179,7 @@ ``` ## runtime_edge_top_n - **Ограничения / валидация**: `1..=1000`. - - **Описание**: Размер выборки Top-N для рейтинга (leaderboard) edge-соединений. + - **Описание**: Размер выборки Top-N для snapshot’ов рейтинга edge-соединений и TLS fingerprint’ов. - **Пример**: ```toml @@ -3127,6 +3127,7 @@ | [`scopes`](#scopes) | `String` | `""` | `✘` | | [`ipv4`](#ipv4-upstreams) | `bool` | — (auto) | `✘` | | [`ipv6`](#ipv6-upstreams) | `bool` | — (auto) | `✘` | +| [`prefer`](#prefer-upstreams) | `4` или `6` | эффективный `[network].prefer` | `✘` | | [`interface`](#interface) | `String` | — | `✘` | | [`bind_addresses`](#bind_addresses) | `String[]` | — | `✘` | | [`bindtodevice`](#bindtodevice) | `String` | — | `✘` | @@ -3198,7 +3199,7 @@ ``` ## ipv6 (upstreams) - **Ограничения / валидация**: `bool` (необязательный параметр). - - **Описание**: Разрешает IPv6 DC-targets для этого upstream. Если не задан, Telemt определяет поддержку автоматически по runtime-состоянию connectivity. + - **Описание**: Разрешает IPv6 DC-targets для этого upstream. Если не задан, Telemt определяет поддержку автоматически по runtime-состоянию connectivity. Установите `true`, если upstream proxy доступен с локального хоста по IPv4, но сам proxy умеет подключаться к Telegram DC по IPv6. - **Пример**: ```toml @@ -3206,6 +3207,18 @@ type = "direct" ipv6 = false ``` +## prefer (upstreams) + - **Ограничения / валидация**: Необязательное число. Должно быть `4` или `6`. + - **Описание**: Переопределяет предпочтительное IP-семейство для Telegram DC-targets, выбранных через этот upstream. Если параметр не задан, upstream наследует эффективное глобальное решение `[network].prefer`. Используйте `prefer = 6` вместе с `ipv6 = true` для SOCKS или Shadowsocks upstream, который умеет выходить в IPv6, даже если локальный хост с Telemt работает только по IPv4. + - **Пример**: + + ```toml + [[upstreams]] + type = "socks5" + address = "192.0.2.10:1080" + ipv6 = true + prefer = 6 + ``` ## interface - **Ограничения / валидация**: `String` (необязательный параметр). - для `"direct"`: может быть IP-адресом (используется как явный local bind) или именем сетевого интерфейса ОС (резолвится в IP во время выполнения; только Unix). diff --git a/docs/FAQ.ru.md b/docs/FAQ.ru.md index 0deac1f..d636bdb 100644 --- a/docs/FAQ.ru.md +++ b/docs/FAQ.ru.md @@ -40,6 +40,8 @@ hello2 = "ad_tag2" > Проблема с TLS отпечатком исправлена в последних версиях клиентов Telegram для Desktop / Android / iOS. > Обновите свой клиент для корректной работы с MTProxy Fake-TLS! +- Для расследования блокировок на базе JA4 ClientHello используйте отдельную инструкцию: [`JA3 и JA4 анализ в Telemt`](Architecture/Fronting-splitting/TLS_JA3_JA4_ANALYSIS.ru.md). + - Мы считаем это прорывом, которому на сегодняшний день нет стабильных аналогов; - Исходя из этого: если `telemt` настроен правильно, **режим TLS полностью идентичен реальному «рукопожатию» + обмену данными** с указанным хостом; - Вот наши доказательства: diff --git a/src/api/config_store.rs b/src/api/config_store.rs index 1416667..6be4040 100644 --- a/src/api/config_store.rs +++ b/src/api/config_store.rs @@ -14,6 +14,7 @@ use super::model::ApiFailure; #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub(super) enum AccessSection { Users, + UserEnabled, UserAdTags, UserMaxTcpConns, UserExpirations, @@ -26,6 +27,7 @@ impl AccessSection { fn table_name(self) -> &'static str { match self { Self::Users => "access.users", + Self::UserEnabled => "access.user_enabled", Self::UserAdTags => "access.user_ad_tags", Self::UserMaxTcpConns => "access.user_max_tcp_conns", Self::UserExpirations => "access.user_expirations", @@ -135,6 +137,15 @@ fn render_access_section(cfg: &ProxyConfig, section: AccessSection) -> Result { + let rows: BTreeMap = cfg + .access + .user_enabled + .iter() + .map(|(key, value)| (key.clone(), *value)) + .collect(); + serialize_table_body(&rows)? + } AccessSection::UserAdTags => { let rows: BTreeMap = cfg .access @@ -204,6 +215,7 @@ fn render_access_section(cfg: &ProxyConfig, section: AccessSection) -> Result bool { match section { AccessSection::Users => cfg.access.users.is_empty(), + AccessSection::UserEnabled => cfg.access.user_enabled.is_empty(), AccessSection::UserAdTags => cfg.access.user_ad_tags.is_empty(), AccessSection::UserMaxTcpConns => cfg.access.user_max_tcp_conns.is_empty(), AccessSection::UserExpirations => cfg.access.user_expirations.is_empty(), diff --git a/src/api/mod.rs b/src/api/mod.rs index 2e2ef6f..4239b59 100644 --- a/src/api/mod.rs +++ b/src/api/mod.rs @@ -22,6 +22,7 @@ use tracing::{debug, info, warn}; use crate::config::{ApiGrayAction, ProxyConfig}; use crate::ip_tracker::UserIpTracker; use crate::proxy::route_mode::RouteRuntimeController; +use crate::proxy::shared_state::ProxySharedState; use crate::startup::StartupTracker; use crate::stats::Stats; use crate::transport::UpstreamManager; @@ -51,9 +52,10 @@ use model::{ PatchUserRequest, ResetUserQuotaResponse, RotateSecretRequest, SummaryData, UserActiveIps, is_valid_username, }; +use patch::Patch; use runtime_edge::{ EdgeConnectionsCacheEntry, build_runtime_connections_summary_data, - build_runtime_events_recent_data, + build_runtime_events_recent_data, build_runtime_tls_fingerprints_data, }; use runtime_init::build_runtime_initialization_data; use runtime_min::{ @@ -71,7 +73,8 @@ use runtime_zero::{ build_system_info_data, }; use users::{ - build_user_quota_list, create_user, delete_user, patch_user, rotate_secret, users_from_config, + build_user_quota_list, create_user, delete_user, patch_user, rotate_secret, set_user_enabled, + users_from_config, }; const API_MAX_CONTROL_CONNECTIONS: usize = 1024; @@ -107,6 +110,7 @@ pub(super) struct ApiShared { pub(super) runtime_state: Arc, pub(super) startup_tracker: Arc, pub(super) route_runtime: Arc, + pub(super) proxy_shared: Arc, } impl ApiShared { @@ -165,12 +169,15 @@ fn allowed_methods_for_path(path: &str) -> Option<&'static str> { | "/v1/runtime/me-selftest" | "/v1/runtime/connections/summary" | "/v1/runtime/events/recent" + | "/v1/runtime/tls-fingerprints" | "/v1/stats/users/active-ips" | "/v1/stats/users/quota" | "/v1/stats/users" => Some(ALLOW_GET), "/v1/users" => Some(ALLOW_GET_POST), _ if user_action_route_matches(path, "/reset-quota") => Some(ALLOW_POST), _ if user_action_route_matches(path, "/rotate-secret") => Some(ALLOW_POST), + _ if user_action_route_matches(path, "/enable") => Some(ALLOW_POST), + _ if user_action_route_matches(path, "/disable") => Some(ALLOW_POST), _ if path .strip_prefix("/v1/users/") .map(|user| !user.is_empty() && !user.contains('/')) @@ -188,6 +195,7 @@ pub async fn serve( ip_tracker: Arc, me_pool: Arc>>>, route_runtime: Arc, + proxy_shared: Arc, upstream_manager: Arc, config_rx: watch::Receiver>, admission_rx: watch::Receiver, @@ -237,6 +245,7 @@ pub async fn serve( runtime_state: runtime_state.clone(), startup_tracker, route_runtime, + proxy_shared, }); spawn_runtime_watchers( @@ -532,6 +541,15 @@ async fn handle( ); Ok(success_response(StatusCode::OK, data, revision)) } + ("GET", "/v1/runtime/tls-fingerprints") => { + let revision = current_revision(&shared.config_path).await?; + let data = build_runtime_tls_fingerprints_data( + shared.as_ref(), + cfg.as_ref(), + query.as_deref(), + ); + Ok(success_response(StatusCode::OK, data, revision)) + } ("GET", "/v1/stats/users/active-ips") => { let revision = current_revision(&shared.config_path).await?; let usernames: Vec<_> = cfg.access.users.keys().cloned().collect(); @@ -582,6 +600,7 @@ async fn handle( } let expected_revision = parse_if_match(req.headers()); let body = read_json::(req.into_body(), body_limit).await?; + let requested_enabled = body.enabled; let result = create_user(body, expected_revision, &shared).await; let (mut data, revision) = match result { Ok(ok) => ok, @@ -594,6 +613,25 @@ async fn handle( }; let runtime_cfg = config_rx.borrow().clone(); data.user.in_runtime = runtime_cfg.access.users.contains_key(&data.user.username); + if let Some(enabled) = requested_enabled { + shared + .proxy_shared + .set_user_enabled(&data.user.username, enabled); + if !enabled { + let cancelled = shared + .proxy_shared + .cancel_user_sessions(&data.user.username); + if cancelled > 0 { + shared.runtime_events.record( + "api.user.disable.runtime", + format!( + "username={} cancelled_sessions={}", + data.user.username, cancelled + ), + ); + } + } + } shared.runtime_events.record( "api.user.create.ok", format!("username={}", data.user.username), @@ -606,6 +644,99 @@ async fn handle( Ok(success_response(status, data, revision)) } _ => { + if method == Method::POST + && let Some(base_user) = normalized_path + .strip_prefix("/v1/users/") + .and_then(|path| path.strip_suffix("/enable")) + && !base_user.is_empty() + && !base_user.contains('/') + { + let base_user = parse_route_username(base_user)?; + if api_cfg.read_only { + return Ok(error_response( + request_id, + ApiFailure::new( + StatusCode::FORBIDDEN, + "read_only", + "API runs in read-only mode", + ), + )); + } + let expected_revision = parse_if_match(req.headers()); + let result = + set_user_enabled(base_user, true, expected_revision, &shared).await; + let (mut data, revision) = match result { + Ok(ok) => ok, + Err(error) => { + shared.runtime_events.record( + "api.user.enable.failed", + format!("username={} code={}", base_user, error.code), + ); + return Err(error); + } + }; + let runtime_cfg = config_rx.borrow().clone(); + data.in_runtime = runtime_cfg.access.users.contains_key(&data.username); + shared.proxy_shared.set_user_enabled(base_user, true); + shared + .runtime_events + .record("api.user.enable.ok", format!("username={}", base_user)); + let status = if data.in_runtime { + StatusCode::OK + } else { + StatusCode::ACCEPTED + }; + return Ok(success_response(status, data, revision)); + } + if method == Method::POST + && let Some(base_user) = normalized_path + .strip_prefix("/v1/users/") + .and_then(|path| path.strip_suffix("/disable")) + && !base_user.is_empty() + && !base_user.contains('/') + { + let base_user = parse_route_username(base_user)?; + if api_cfg.read_only { + return Ok(error_response( + request_id, + ApiFailure::new( + StatusCode::FORBIDDEN, + "read_only", + "API runs in read-only mode", + ), + )); + } + let expected_revision = parse_if_match(req.headers()); + let result = + set_user_enabled(base_user, false, expected_revision, &shared).await; + let (mut data, revision) = match result { + Ok(ok) => ok, + Err(error) => { + shared.runtime_events.record( + "api.user.disable.failed", + format!("username={} code={}", base_user, error.code), + ); + return Err(error); + } + }; + let runtime_cfg = config_rx.borrow().clone(); + data.in_runtime = runtime_cfg.access.users.contains_key(&data.username); + let newly_disabled = shared.proxy_shared.set_user_enabled(base_user, false); + let cancelled = shared.proxy_shared.cancel_user_sessions(base_user); + shared.runtime_events.record( + "api.user.disable.ok", + format!( + "username={} newly_disabled={} cancelled_sessions={}", + base_user, newly_disabled, cancelled + ), + ); + let status = if data.in_runtime { + StatusCode::OK + } else { + StatusCode::ACCEPTED + }; + return Ok(success_response(status, data, revision)); + } if method == Method::POST && let Some(user) = normalized_path .strip_prefix("/v1/users/") @@ -763,6 +894,11 @@ async fn handle( let expected_revision = parse_if_match(req.headers()); let body = read_json::(req.into_body(), body_limit).await?; + let enabled_update = match &body.enabled { + Patch::Unchanged => None, + Patch::Remove => Some(true), + Patch::Set(enabled) => Some(*enabled), + }; let result = patch_user(user, body, expected_revision, &shared).await; let (mut data, revision) = match result { Ok(ok) => ok, @@ -776,6 +912,22 @@ async fn handle( }; let runtime_cfg = config_rx.borrow().clone(); data.in_runtime = runtime_cfg.access.users.contains_key(&data.username); + if let Some(enabled) = enabled_update { + shared + .proxy_shared + .set_user_enabled(&data.username, enabled); + if !enabled { + let cancelled = + shared.proxy_shared.cancel_user_sessions(&data.username); + shared.runtime_events.record( + "api.user.disable.runtime", + format!( + "username={} cancelled_sessions={}", + data.username, cancelled + ), + ); + } + } shared .runtime_events .record("api.user.patch.ok", format!("username={}", data.username)); @@ -809,9 +961,12 @@ async fn handle( return Err(error); } }; - shared - .runtime_events - .record("api.user.delete.ok", format!("username={}", deleted_user)); + shared.proxy_shared.set_user_enabled(&deleted_user, true); + let cancelled = shared.proxy_shared.cancel_user_sessions(&deleted_user); + shared.runtime_events.record( + "api.user.delete.ok", + format!("username={} cancelled_sessions={}", deleted_user, cancelled), + ); let runtime_cfg = config_rx.borrow().clone(); let in_runtime = runtime_cfg.access.users.contains_key(&deleted_user); let response = DeleteUserResponse { diff --git a/src/api/model.rs b/src/api/model.rs index 56e8fea..5a183d5 100644 --- a/src/api/model.rs +++ b/src/api/model.rs @@ -479,6 +479,7 @@ pub(super) struct TlsDomainLink { #[derive(Serialize)] pub(super) struct UserInfo { pub(super) username: String, + pub(super) enabled: bool, pub(super) in_runtime: bool, pub(super) user_ad_tag: Option, pub(super) max_tcp_conns: Option, @@ -545,6 +546,7 @@ pub(super) struct CreateUserRequest { pub(super) rate_limit_up_bps: Option, pub(super) rate_limit_down_bps: Option, pub(super) max_unique_ips: Option, + pub(super) enabled: Option, } #[derive(Deserialize)] @@ -564,6 +566,8 @@ pub(super) struct PatchUserRequest { pub(super) rate_limit_down_bps: Patch, #[serde(default, deserialize_with = "patch_field")] pub(super) max_unique_ips: Patch, + #[serde(default, deserialize_with = "patch_field")] + pub(super) enabled: Patch, } #[derive(Default, Deserialize)] diff --git a/src/api/runtime_edge.rs b/src/api/runtime_edge.rs index b61f504..639dbe0 100644 --- a/src/api/runtime_edge.rs +++ b/src/api/runtime_edge.rs @@ -12,6 +12,8 @@ const FEATURE_DISABLED_REASON: &str = "feature_disabled"; const SOURCE_UNAVAILABLE_REASON: &str = "source_unavailable"; const EVENTS_DEFAULT_LIMIT: usize = 50; const EVENTS_MAX_LIMIT: usize = 1000; +const TLS_FINGERPRINTS_MAX_LIMIT: usize = 1000; +const RUNTIME_EDGE_RETENTION_MAX_MINUTES: u64 = 24 * 60; #[derive(Clone, Serialize)] pub(super) struct RuntimeEdgeConnectionUserData { @@ -90,6 +92,44 @@ pub(super) struct RuntimeEdgeEventsData { pub(super) data: Option, } +#[derive(Serialize)] +pub(super) struct RuntimeEdgeTlsFingerprintRow { + #[serde(skip_serializing_if = "Option::is_none")] + pub(super) scope: Option, + pub(super) ja3: String, + pub(super) ja3_raw: String, + pub(super) ja4: String, + pub(super) ja4_raw: String, + pub(super) total: u64, + pub(super) auth_success: u64, + pub(super) bad_or_probe: u64, + pub(super) first_seen_epoch_secs: u64, + pub(super) last_seen_epoch_secs: u64, +} + +#[derive(Serialize)] +pub(super) struct RuntimeEdgeTlsFingerprintsPayload { + pub(super) limit: usize, + pub(super) retention_secs: u64, + pub(super) capacity: usize, + pub(super) dropped_total: u64, + pub(super) parse_error_total: u64, + pub(super) by_fingerprint: Vec, + pub(super) by_ip: Vec, + pub(super) by_cidr: Vec, + pub(super) by_user: Vec, +} + +#[derive(Serialize)] +pub(super) struct RuntimeEdgeTlsFingerprintsData { + pub(super) enabled: bool, + #[serde(skip_serializing_if = "Option::is_none")] + pub(super) reason: Option<&'static str>, + pub(super) generated_at_epoch_secs: u64, + #[serde(skip_serializing_if = "Option::is_none")] + pub(super) data: Option, +} + pub(super) async fn build_runtime_connections_summary_data( shared: &ApiShared, cfg: &ProxyConfig, @@ -162,6 +202,65 @@ pub(super) fn build_runtime_events_recent_data( } } +pub(super) fn build_runtime_tls_fingerprints_data( + shared: &ApiShared, + cfg: &ProxyConfig, + query: Option<&str>, +) -> RuntimeEdgeTlsFingerprintsData { + let now_epoch_secs = now_epoch_secs(); + let api_cfg = &cfg.server.api; + if !api_cfg.runtime_edge_enabled { + return RuntimeEdgeTlsFingerprintsData { + enabled: false, + reason: Some(FEATURE_DISABLED_REASON), + generated_at_epoch_secs: now_epoch_secs, + data: None, + }; + } + + let limit = parse_recent_events_limit( + query, + api_cfg.runtime_edge_top_n.max(1), + TLS_FINGERPRINTS_MAX_LIMIT, + ); + let snapshot = shared + .stats + .tls_fingerprint_snapshot(runtime_edge_retention(cfg), limit); + + RuntimeEdgeTlsFingerprintsData { + enabled: true, + reason: None, + generated_at_epoch_secs: now_epoch_secs, + data: Some(RuntimeEdgeTlsFingerprintsPayload { + limit, + retention_secs: snapshot.retention_secs, + capacity: snapshot.capacity, + dropped_total: snapshot.dropped_total, + parse_error_total: snapshot.parse_error_total, + by_fingerprint: snapshot + .by_fingerprint + .into_iter() + .map(runtime_tls_fingerprint_row) + .collect(), + by_ip: snapshot + .by_ip + .into_iter() + .map(runtime_tls_fingerprint_row) + .collect(), + by_cidr: snapshot + .by_cidr + .into_iter() + .map(runtime_tls_fingerprint_row) + .collect(), + by_user: snapshot + .by_user + .into_iter() + .map(runtime_tls_fingerprint_row) + .collect(), + }), + } +} + async fn get_connections_payload_cached( shared: &ApiShared, cache_ttl_ms: u64, @@ -286,6 +385,35 @@ fn parse_recent_events_limit(query: Option<&str>, default_limit: usize, max_limi default_limit } +fn runtime_edge_retention(cfg: &ProxyConfig) -> Duration { + let minutes = cfg + .general + .beobachten_minutes + .clamp(1, RUNTIME_EDGE_RETENTION_MAX_MINUTES); + Duration::from_secs(minutes.saturating_mul(60)) +} + +fn runtime_tls_fingerprint_row( + row: crate::stats::TlsFingerprintSnapshotRow, +) -> RuntimeEdgeTlsFingerprintRow { + RuntimeEdgeTlsFingerprintRow { + scope: if row.scope_key.is_empty() { + None + } else { + Some(row.scope_key) + }, + ja3: row.ja3, + ja3_raw: row.ja3_raw, + ja4: row.ja4, + ja4_raw: row.ja4_raw, + total: row.total, + auth_success: row.auth_success, + bad_or_probe: row.bad_or_probe, + first_seen_epoch_secs: row.first_seen_epoch_secs, + last_seen_epoch_secs: row.last_seen_epoch_secs, + } +} + fn now_epoch_secs() -> u64 { SystemTime::now() .duration_since(UNIX_EPOCH) diff --git a/src/api/users.rs b/src/api/users.rs index 24815fc..48acc25 100644 --- a/src/api/users.rs +++ b/src/api/users.rs @@ -32,6 +32,7 @@ pub(super) async fn create_user( let touches_user_rate_limits = body.rate_limit_up_bps.is_some() || body.rate_limit_down_bps.is_some(); let touches_user_max_unique_ips = body.max_unique_ips.is_some(); + let touches_user_enabled = matches!(body.enabled, Some(false)); if !is_valid_username(&body.username) { return Err(ApiFailure::bad_request( @@ -111,6 +112,9 @@ pub(super) async fn create_user( .user_max_unique_ips .insert(body.username.clone(), limit); } + if matches!(body.enabled, Some(false)) { + cfg.access.user_enabled.insert(body.username.clone(), false); + } cfg.validate() .map_err(|e| ApiFailure::bad_request(format!("config validation failed: {}", e)))?; @@ -134,6 +138,9 @@ pub(super) async fn create_user( if touches_user_max_unique_ips { touched_sections.push(AccessSection::UserMaxUniqueIps); } + if touches_user_enabled { + touched_sections.push(AccessSection::UserEnabled); + } let revision = save_access_sections_to_disk(&shared.config_path, &cfg, &touched_sections).await?; @@ -161,6 +168,7 @@ pub(super) async fn create_user( .find(|entry| entry.username == body.username) .unwrap_or(UserInfo { username: body.username.clone(), + enabled: cfg.access.is_user_enabled(&body.username), in_runtime: false, user_ad_tag: None, max_tcp_conns: cfg @@ -202,6 +210,7 @@ pub(super) async fn patch_user( let touches_user_rate_limits = !matches!(&body.rate_limit_up_bps, Patch::Unchanged) || !matches!(&body.rate_limit_down_bps, Patch::Unchanged); let touches_user_max_unique_ips = !matches!(&body.max_unique_ips, Patch::Unchanged); + let touches_user_enabled = !matches!(&body.enabled, Patch::Unchanged); if let Some(secret) = body.secret.as_ref() && !is_valid_user_secret(secret) @@ -313,6 +322,15 @@ pub(super) async fn patch_user( Some(Some(limit)) } }; + match body.enabled { + Patch::Unchanged => {} + Patch::Remove | Patch::Set(true) => { + cfg.access.user_enabled.remove(user); + } + Patch::Set(false) => { + cfg.access.user_enabled.insert(user.to_string(), false); + } + } cfg.validate() .map_err(|e| ApiFailure::bad_request(format!("config validation failed: {}", e)))?; @@ -339,6 +357,9 @@ pub(super) async fn patch_user( if touches_user_max_unique_ips { touched_sections.push(AccessSection::UserMaxUniqueIps); } + if touches_user_enabled { + touched_sections.push(AccessSection::UserEnabled); + } let revision = if touched_sections.is_empty() { current_revision(&shared.config_path).await? @@ -399,6 +420,7 @@ pub(super) async fn rotate_secret( .map_err(|e| ApiFailure::bad_request(format!("config validation failed: {}", e)))?; let touched_sections = [ AccessSection::Users, + AccessSection::UserEnabled, AccessSection::UserAdTags, AccessSection::UserMaxTcpConns, AccessSection::UserExpirations, @@ -434,6 +456,55 @@ pub(super) async fn rotate_secret( )) } +pub(super) async fn set_user_enabled( + user: &str, + enabled: bool, + expected_revision: Option, + shared: &ApiShared, +) -> Result<(UserInfo, String), ApiFailure> { + let _guard = shared.mutation_lock.lock().await; + let mut cfg = load_config_from_disk(&shared.config_path).await?; + ensure_expected_revision(&shared.config_path, expected_revision.as_deref()).await?; + + if !cfg.access.users.contains_key(user) { + return Err(ApiFailure::new( + StatusCode::NOT_FOUND, + "not_found", + "User not found", + )); + } + + if enabled { + cfg.access.user_enabled.remove(user); + } else { + cfg.access.user_enabled.insert(user.to_string(), false); + } + + cfg.validate() + .map_err(|e| ApiFailure::bad_request(format!("config validation failed: {}", e)))?; + let revision = + save_access_sections_to_disk(&shared.config_path, &cfg, &[AccessSection::UserEnabled]) + .await?; + drop(_guard); + + let (detected_ip_v4, detected_ip_v6) = shared.detected_link_ips(); + let users = users_from_config( + &cfg, + &shared.stats, + &shared.ip_tracker, + detected_ip_v4, + detected_ip_v6, + None, + ) + .await; + let user_info = users + .into_iter() + .find(|entry| entry.username == user) + .ok_or_else(|| ApiFailure::internal("failed to build updated user view"))?; + + Ok((user_info, revision)) +} + pub(super) async fn delete_user( user: &str, expected_revision: Option, @@ -459,6 +530,7 @@ pub(super) async fn delete_user( } cfg.access.users.remove(user); + cfg.access.user_enabled.remove(user); cfg.access.user_ad_tags.remove(user); cfg.access.user_max_tcp_conns.remove(user); cfg.access.user_expirations.remove(user); @@ -470,6 +542,7 @@ pub(super) async fn delete_user( .map_err(|e| ApiFailure::bad_request(format!("config validation failed: {}", e)))?; let touched_sections = [ AccessSection::Users, + AccessSection::UserEnabled, AccessSection::UserAdTags, AccessSection::UserMaxTcpConns, AccessSection::UserExpirations, @@ -518,6 +591,7 @@ pub(super) async fn users_from_config( }) .unwrap_or_else(empty_user_links); users.push(UserInfo { + enabled: cfg.access.is_user_enabled(&username), in_runtime: runtime_cfg .map(|runtime| runtime.access.users.contains_key(&username)) .unwrap_or(false), @@ -876,6 +950,43 @@ mod tests { assert_eq!(alice.rate_limit_down_bps, None); } + #[tokio::test] + async fn users_from_config_reports_user_enabled_default_and_override() { + let mut cfg = ProxyConfig::default(); + cfg.access.users.insert( + "alice".to_string(), + "0123456789abcdef0123456789abcdef".to_string(), + ); + cfg.access.users.insert( + "bob".to_string(), + "fedcba9876543210fedcba9876543210".to_string(), + ); + cfg.access.user_enabled.insert("bob".to_string(), false); + + let stats = Stats::new(); + let tracker = UserIpTracker::new(); + let users = users_from_config(&cfg, &stats, &tracker, None, None, None).await; + let alice = users + .iter() + .find(|entry| entry.username == "alice") + .expect("alice must be present"); + let bob = users + .iter() + .find(|entry| entry.username == "bob") + .expect("bob must be present"); + + assert!(alice.enabled); + assert!(!bob.enabled); + + cfg.access.user_enabled.insert("bob".to_string(), true); + let users = users_from_config(&cfg, &stats, &tracker, None, None, None).await; + let bob = users + .iter() + .find(|entry| entry.username == "bob") + .expect("bob must be present"); + assert!(bob.enabled); + } + #[tokio::test] async fn users_from_config_marks_runtime_membership_when_snapshot_is_provided() { let mut disk_cfg = ProxyConfig::default(); diff --git a/src/cli.rs b/src/cli.rs index bda7d92..e740b2b 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -705,6 +705,9 @@ ignore_time_skew = false type = "direct" enabled = true weight = 10 +# Optional per-upstream DC family policy: +# ipv6 = true +# prefer = 6 "#, username = username, secret = secret, diff --git a/src/config/hot_reload.rs b/src/config/hot_reload.rs index 8135d31..4faef9b 100644 --- a/src/config/hot_reload.rs +++ b/src/config/hot_reload.rs @@ -118,6 +118,7 @@ pub struct HotFields { pub me_admission_poll_ms: u64, pub me_warn_rate_limit_ms: u64, pub users: std::collections::HashMap, + pub user_enabled: std::collections::HashMap, pub user_ad_tags: std::collections::HashMap, pub user_max_tcp_conns: std::collections::HashMap, pub user_max_tcp_conns_global_each: usize, @@ -247,6 +248,7 @@ impl HotFields { me_admission_poll_ms: cfg.general.me_admission_poll_ms, me_warn_rate_limit_ms: cfg.general.me_warn_rate_limit_ms, users: cfg.access.users.clone(), + user_enabled: cfg.access.user_enabled.clone(), user_ad_tags: cfg.access.user_ad_tags.clone(), user_max_tcp_conns: cfg.access.user_max_tcp_conns.clone(), user_max_tcp_conns_global_each: cfg.access.user_max_tcp_conns_global_each, @@ -551,6 +553,7 @@ fn overlay_hot_fields(old: &ProxyConfig, new: &ProxyConfig) -> ProxyConfig { cfg.general.me_warn_rate_limit_ms = new.general.me_warn_rate_limit_ms; cfg.access.users = new.access.users.clone(); + cfg.access.user_enabled = new.access.user_enabled.clone(); cfg.access.user_ad_tags = new.access.user_ad_tags.clone(); cfg.access.user_max_tcp_conns = new.access.user_max_tcp_conns.clone(); cfg.access.user_max_tcp_conns_global_each = new.access.user_max_tcp_conns_global_each; @@ -1178,6 +1181,16 @@ fn log_changes( } } + if old_hot.user_enabled != new_hot.user_enabled { + info!( + "config reload: user_enabled updated ({} disabled overrides)", + new_hot + .user_enabled + .values() + .filter(|enabled| !**enabled) + .count() + ); + } if old_hot.user_max_tcp_conns != new_hot.user_max_tcp_conns { info!( "config reload: user_max_tcp_conns updated ({} entries)", diff --git a/src/config/load.rs b/src/config/load.rs index d14510f..231b164 100644 --- a/src/config/load.rs +++ b/src/config/load.rs @@ -411,6 +411,7 @@ const TLS_FETCH_CONFIG_KEYS: &[&str] = &[ const ACCESS_CONFIG_KEYS: &[&str] = &[ "users", + "user_enabled", "user_ad_tags", "user_max_tcp_conns", "user_max_tcp_conns_global_each", @@ -1006,6 +1007,14 @@ fn validate_upstreams(config: &ProxyConfig) -> Result<()> { "upstream.ipv4 and upstream.ipv6 cannot both be false".to_string(), )); } + if let Some(prefer) = upstream.prefer + && prefer != 4 + && prefer != 6 + { + return Err(ProxyError::Config( + "upstream.prefer must be 4 or 6".to_string(), + )); + } if let UpstreamType::Shadowsocks { url, .. } = &upstream.upstream_type { let parsed = ShadowsocksServerConfig::from_url(url) @@ -1021,6 +1030,26 @@ fn validate_upstreams(config: &ProxyConfig) -> Result<()> { Ok(()) } +fn normalize_upstream_family_policy(config: &mut ProxyConfig) { + for (idx, upstream) in config.upstreams.iter_mut().enumerate() { + if matches!(upstream.ipv4, Some(false)) && upstream.prefer == Some(4) { + warn!( + upstream = idx, + "upstream.prefer=4 but upstream.ipv4=false; forcing prefer=6" + ); + upstream.prefer = Some(6); + } + + if matches!(upstream.ipv6, Some(false)) && upstream.prefer == Some(6) { + warn!( + upstream = idx, + "upstream.prefer=6 but upstream.ipv6=false; forcing prefer=4" + ); + upstream.prefer = Some(4); + } + } +} + // ============= Main Config ============= #[derive(Debug, Clone, Serialize, Deserialize, Default)] @@ -2199,8 +2228,10 @@ impl ProxyConfig { selected_scope: String::new(), ipv4: None, ipv6: None, + prefer: None, }); } + normalize_upstream_family_policy(&mut config); // Ensure default DC203 override is present. config diff --git a/src/config/tests/load_mask_shape_security_tests.rs b/src/config/tests/load_mask_shape_security_tests.rs index 41b1f94..b213978 100644 --- a/src/config/tests/load_mask_shape_security_tests.rs +++ b/src/config/tests/load_mask_shape_security_tests.rs @@ -1,14 +1,21 @@ use super::*; use std::fs; use std::path::PathBuf; +use std::sync::atomic::{AtomicU64, Ordering}; use std::time::{SystemTime, UNIX_EPOCH}; +static TEMP_CONFIG_COUNTER: AtomicU64 = AtomicU64::new(0); + fn write_temp_config(contents: &str) -> PathBuf { let nonce = SystemTime::now() .duration_since(UNIX_EPOCH) .expect("system time must be after unix epoch") .as_nanos(); - let path = std::env::temp_dir().join(format!("telemt-load-mask-shape-security-{nonce}.toml")); + let seq = TEMP_CONFIG_COUNTER.fetch_add(1, Ordering::Relaxed); + let pid = std::process::id(); + let path = std::env::temp_dir().join(format!( + "telemt-load-mask-shape-security-{pid}-{seq}-{nonce}.toml" + )); fs::write(&path, contents).expect("temp config write must succeed"); path } diff --git a/src/config/types.rs b/src/config/types.rs index b707dff..4f9d568 100644 --- a/src/config/types.rs +++ b/src/config/types.rs @@ -1892,6 +1892,9 @@ pub struct AccessConfig { #[serde(default = "default_access_users")] pub users: HashMap, + #[serde(default)] + pub user_enabled: HashMap, + /// Per-user ad_tag (32 hex chars from @MTProxybot). #[serde(default)] pub user_ad_tags: HashMap, @@ -1963,6 +1966,7 @@ impl Default for AccessConfig { fn default() -> Self { Self { users: default_access_users(), + user_enabled: HashMap::new(), user_ad_tags: HashMap::new(), user_max_tcp_conns: HashMap::new(), user_max_tcp_conns_global_each: default_user_max_tcp_conns_global_each(), @@ -1983,6 +1987,10 @@ impl Default for AccessConfig { } impl AccessConfig { + pub fn is_user_enabled(&self, username: &str) -> bool { + self.user_enabled.get(username).copied().unwrap_or(true) + } + /// Returns true if `ip` is contained in any CIDR listed for `username` under `user_source_deny`. pub fn is_user_source_ip_denied(&self, username: &str, ip: IpAddr) -> bool { self.user_source_deny @@ -2057,6 +2065,20 @@ pub struct UpstreamConfig { /// `None` means auto-detect from runtime connectivity state. #[serde(default)] pub ipv6: Option, + /// Per-upstream IP family preference for Telegram DC targets. + /// `None` inherits the effective global `[network].prefer` decision. + #[serde(default)] + pub prefer: Option, +} + +impl UpstreamConfig { + pub fn prefer_ipv6(&self, default_prefer_ipv6: bool) -> bool { + match self.prefer { + Some(6) => true, + Some(4) => false, + _ => default_prefer_ipv6, + } + } } #[derive(Debug, Clone, Serialize, Deserialize)] diff --git a/src/conntrack_control.rs b/src/conntrack_control.rs index 33a9174..dfbf5fc 100644 --- a/src/conntrack_control.rs +++ b/src/conntrack_control.rs @@ -705,7 +705,7 @@ fn nofile_soft_limit() -> Option { if rc != 0 { return None; } - return Some(lim.rlim_cur); + return Some(lim.rlim_cur.into()); } #[cfg(not(target_os = "linux"))] { diff --git a/src/error.rs b/src/error.rs index ff58f4e..1cefe97 100644 --- a/src/error.rs +++ b/src/error.rs @@ -245,6 +245,9 @@ pub enum ProxyError { InvalidSecret { user: String, reason: String }, // ============= User Errors ============= + #[error("User {user} disabled")] + UserDisabled { user: String }, + #[error("User {user} expired")] UserExpired { user: String }, diff --git a/src/maestro/connectivity.rs b/src/maestro/connectivity.rs index 0cb561d..54f837b 100644 --- a/src/maestro/connectivity.rs +++ b/src/maestro/connectivity.rs @@ -147,7 +147,7 @@ pub(crate) async fn run_startup_connectivity( .any(|r| r.rtt_ms.is_some()); if upstream_result.both_available { - if prefer_ipv6 { + if upstream_result.prefer_ipv6 { info!(" IPv6 in use / IPv4 is fallback"); } else { info!(" IPv4 in use / IPv6 is fallback"); diff --git a/src/maestro/mod.rs b/src/maestro/mod.rs index e711ab4..2d4fb54 100644 --- a/src/maestro/mod.rs +++ b/src/maestro/mod.rs @@ -464,6 +464,12 @@ async fn run_telemt_core( config.network.dns_overrides.len() ); } + let shared_state = ProxySharedState::new(); + shared_state.apply_user_enabled_config(&config.access.user_enabled); + shared_state.traffic_limiter.apply_policy( + config.access.user_rate_limits.clone(), + config.access.cidr_rate_limits.clone(), + ); let (api_config_tx, api_config_rx) = watch::channel(Arc::new(config.clone())); let (detected_ips_tx, detected_ips_rx) = watch::channel((None::, None::)); @@ -502,6 +508,7 @@ async fn run_telemt_core( let me_pool_api = api_me_pool.clone(); let upstream_manager_api = upstream_manager.clone(); let route_runtime_api = route_runtime.clone(); + let proxy_shared_api = shared_state.clone(); let config_rx_api = api_config_rx.clone(); let admission_rx_api = admission_rx.clone(); let config_path_api = config_path.clone(); @@ -515,6 +522,7 @@ async fn run_telemt_core( ip_tracker_api, me_pool_api, route_runtime_api, + proxy_shared_api, upstream_manager_api, config_rx_api, admission_rx_api, @@ -732,11 +740,6 @@ async fn run_telemt_core( )); let buffer_pool = Arc::new(BufferPool::with_config(64 * 1024, 4096)); - let shared_state = ProxySharedState::new(); - shared_state.traffic_limiter.apply_policy( - config.access.user_rate_limits.clone(), - config.access.cidr_rate_limits.clone(), - ); if direct_first_startup { startup_tracker diff --git a/src/maestro/runtime_tasks.rs b/src/maestro/runtime_tasks.rs index 8b9a9aa..6099014 100644 --- a/src/maestro/runtime_tasks.rs +++ b/src/maestro/runtime_tasks.rs @@ -3,7 +3,7 @@ use std::path::Path; use std::sync::Arc; use tokio::sync::{mpsc, watch}; -use tracing::{debug, warn}; +use tracing::{debug, info, warn}; use tracing_subscriber::EnvFilter; use tracing_subscriber::reload; @@ -234,6 +234,27 @@ pub(crate) async fn spawn_runtime_tasks( } }); + let shared_user_enabled = shared_state.clone(); + let mut config_rx_user_enabled = config_rx.clone(); + tokio::spawn(async move { + loop { + if config_rx_user_enabled.changed().await.is_err() { + break; + } + let cfg = config_rx_user_enabled.borrow_and_update().clone(); + for user in shared_user_enabled.apply_user_enabled_config(&cfg.access.user_enabled) { + let cancelled = shared_user_enabled.cancel_user_sessions(&user); + if cancelled > 0 { + info!( + user = %user, + cancelled, + "Disabled user sessions cancelled after config reload" + ); + } + } + } + }); + let beobachten_writer = beobachten.clone(); let config_rx_beobachten = config_rx.clone(); tokio::spawn(async move { diff --git a/src/metrics.rs b/src/metrics.rs index 61a26c5..b4baad0 100644 --- a/src/metrics.rs +++ b/src/metrics.rs @@ -55,8 +55,10 @@ pub async fn serve( return; } }; - let is_ipv6 = addr.is_ipv6(); - match bind_metrics_listener(addr, is_ipv6, listen_backlog) { + // Match `server.api.listen`: `[::]:port` is a dual-stack wildcard + // on Linux when `net.ipv6.bindv6only=0`. + let ipv6_only = addr.is_ipv6() && !addr.ip().is_unspecified(); + match bind_metrics_listener(addr, ipv6_only, listen_backlog) { Ok(listener) => { info!("Metrics endpoint: http://{}/metrics and /beobachten", addr); serve_listener( @@ -286,7 +288,7 @@ async fn handle( } if req.uri().path() == "/beobachten" { - let body = render_beobachten(beobachten, config); + let body = render_beobachten(stats, beobachten, config); let resp = Response::builder() .status(StatusCode::OK) .header("content-type", "text/plain; charset=utf-8") @@ -302,13 +304,22 @@ async fn handle( Ok(resp) } -fn render_beobachten(beobachten: &BeobachtenStore, config: &ProxyConfig) -> String { +fn render_beobachten(stats: &Stats, beobachten: &BeobachtenStore, config: &ProxyConfig) -> String { if !config.general.beobachten { return "beobachten disabled\n".to_string(); } let ttl = Duration::from_secs(config.general.beobachten_minutes.saturating_mul(60)); - beobachten.snapshot_text(ttl) + let mut body = beobachten.snapshot_text(ttl); + let tls_text = stats.tls_fingerprint_snapshot_text(ttl, 20); + if !tls_text.is_empty() { + if !body.ends_with('\n') { + body.push('\n'); + } + body.push('\n'); + body.push_str(&tls_text); + } + body } fn tls_front_domains(config: &ProxyConfig) -> Vec { diff --git a/src/protocol/mod.rs b/src/protocol/mod.rs index f0b3a1a..9ffff7c 100644 --- a/src/protocol/mod.rs +++ b/src/protocol/mod.rs @@ -4,6 +4,7 @@ pub mod constants; pub mod frame; pub mod obfuscation; pub mod tls; +pub mod tls_fingerprint; #[allow(unused_imports)] pub use constants::*; @@ -13,3 +14,5 @@ pub use frame::*; pub use obfuscation::*; #[allow(unused_imports)] pub use tls::*; +#[allow(unused_imports)] +pub use tls_fingerprint::*; diff --git a/src/protocol/tls_fingerprint.rs b/src/protocol/tls_fingerprint.rs new file mode 100644 index 0000000..4598d45 --- /dev/null +++ b/src/protocol/tls_fingerprint.rs @@ -0,0 +1,450 @@ +//! Passive JA3 / JA4 TLS ClientHello fingerprinting. + +use crate::crypto::hash::md5; +use crate::crypto::sha256; +use crate::protocol::constants::TLS_RECORD_HANDSHAKE; + +const EXT_SNI: u16 = 0x0000; +const EXT_SUPPORTED_GROUPS: u16 = 0x000a; +const EXT_EC_POINT_FORMATS: u16 = 0x000b; +const EXT_SIGNATURE_ALGORITHMS: u16 = 0x000d; +const EXT_ALPN: u16 = 0x0010; +const EXT_SUPPORTED_VERSIONS: u16 = 0x002b; + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct TlsClientFingerprint { + pub ja3: String, + pub ja3_raw: String, + pub ja4: String, + pub ja4_raw: String, +} + +#[derive(Default)] +struct ParsedClientHello { + legacy_version: u16, + ciphers: Vec, + extensions: Vec, + supported_groups: Vec, + ec_point_formats: Vec, + signature_algorithms: Vec, + supported_versions: Vec, + alpn_first: Option>, + sni_present: bool, +} + +pub fn fingerprint_client_hello(handshake: &[u8]) -> Option { + let parsed = parse_client_hello(handshake)?; + let ja3_raw = ja3_raw(&parsed); + let ja3 = hex::encode(md5(ja3_raw.as_bytes())); + let (ja4, ja4_raw) = ja4(&parsed); + + Some(TlsClientFingerprint { + ja3, + ja3_raw, + ja4, + ja4_raw, + }) +} + +fn parse_client_hello(handshake: &[u8]) -> Option { + if handshake.len() < 5 || handshake[0] != TLS_RECORD_HANDSHAKE { + return None; + } + + let record_len = read_u16_at(handshake, 3)? as usize; + let record_end = 5usize.checked_add(record_len)?; + if record_end > handshake.len() { + return None; + } + + let mut pos = 5usize; + if *handshake.get(pos)? != 0x01 { + return None; + } + pos = pos.checked_add(1)?; + + if pos + 3 > record_end { + return None; + } + let handshake_len = ((usize::from(handshake[pos])) << 16) + | ((usize::from(handshake[pos + 1])) << 8) + | usize::from(handshake[pos + 2]); + pos = pos.checked_add(3)?; + let handshake_end = pos.checked_add(handshake_len)?; + if handshake_end > record_end { + return None; + } + + if pos + 2 + 32 > handshake_end { + return None; + } + let legacy_version = read_u16_at(handshake, pos)?; + pos = pos.checked_add(2 + 32)?; + + let session_id_len = usize::from(*handshake.get(pos)?); + pos = pos.checked_add(1)?.checked_add(session_id_len)?; + if pos + 2 > handshake_end { + return None; + } + + let cipher_len = read_u16_at(handshake, pos)? as usize; + pos = pos.checked_add(2)?; + let cipher_end = pos.checked_add(cipher_len)?; + if cipher_end > handshake_end || cipher_len % 2 != 0 { + return None; + } + let mut ciphers = Vec::with_capacity(cipher_len / 2); + while pos + 1 < cipher_end { + let value = read_u16_at(handshake, pos)?; + if !is_grease(value) { + ciphers.push(value); + } + pos = pos.checked_add(2)?; + } + + let comp_len = usize::from(*handshake.get(pos)?); + pos = pos.checked_add(1)?.checked_add(comp_len)?; + if pos > handshake_end { + return None; + } + + let mut parsed = ParsedClientHello { + legacy_version, + ciphers, + ..ParsedClientHello::default() + }; + + if pos == handshake_end { + return Some(parsed); + } + if pos + 2 > handshake_end { + return None; + } + + let ext_len = read_u16_at(handshake, pos)? as usize; + pos = pos.checked_add(2)?; + let ext_end = pos.checked_add(ext_len)?; + if ext_end > handshake_end { + return None; + } + + while pos + 4 <= ext_end { + let etype = read_u16_at(handshake, pos)?; + let elen = read_u16_at(handshake, pos + 2)? as usize; + pos = pos.checked_add(4)?; + let data_end = pos.checked_add(elen)?; + if data_end > ext_end { + return None; + } + let data = handshake.get(pos..data_end)?; + + if !is_grease(etype) { + parsed.extensions.push(etype); + match etype { + EXT_SNI => parsed.sni_present = true, + EXT_SUPPORTED_GROUPS => { + parsed.supported_groups = parse_u16_vector(data, 2)?; + } + EXT_EC_POINT_FORMATS => { + parsed.ec_point_formats = parse_u8_vector(data)?; + } + EXT_SIGNATURE_ALGORITHMS => { + parsed.signature_algorithms = parse_u16_vector(data, 2)?; + } + EXT_ALPN => { + parsed.alpn_first = parse_alpn_first(data)?; + } + EXT_SUPPORTED_VERSIONS => { + parsed.supported_versions = parse_u16_vector(data, 1)?; + } + _ => {} + } + } + + pos = data_end; + } + + if pos != ext_end { + return None; + } + + Some(parsed) +} + +fn parse_u16_vector(data: &[u8], len_prefix_len: usize) -> Option> { + let (list_len, mut pos) = match len_prefix_len { + 1 => (usize::from(*data.first()?), 1usize), + 2 => (read_u16_at(data, 0)? as usize, 2usize), + _ => return None, + }; + let list_end = pos.checked_add(list_len)?; + if list_end > data.len() || list_len % 2 != 0 { + return None; + } + + let mut out = Vec::with_capacity(list_len / 2); + while pos + 1 < list_end { + let value = read_u16_at(data, pos)?; + if !is_grease(value) { + out.push(value); + } + pos = pos.checked_add(2)?; + } + Some(out) +} + +fn parse_u8_vector(data: &[u8]) -> Option> { + let list_len = usize::from(*data.first()?); + let list_start = 1usize; + let list_end = list_start.checked_add(list_len)?; + if list_end > data.len() { + return None; + } + Some(data.get(list_start..list_end)?.to_vec()) +} + +fn parse_alpn_first(data: &[u8]) -> Option>> { + if data.len() < 2 { + return None; + } + let list_len = read_u16_at(data, 0)? as usize; + let mut pos = 2usize; + let list_end = pos.checked_add(list_len)?; + if list_end > data.len() { + return None; + } + if pos == list_end { + return Some(None); + } + + let protocol_len = usize::from(*data.get(pos)?); + pos = pos.checked_add(1)?; + let protocol_end = pos.checked_add(protocol_len)?; + if protocol_end > list_end { + return None; + } + if protocol_len == 0 { + return Some(None); + } + Some(Some(data.get(pos..protocol_end)?.to_vec())) +} + +fn ja3_raw(parsed: &ParsedClientHello) -> String { + format!( + "{},{},{},{},{}", + parsed.legacy_version, + join_decimal_u16(&parsed.ciphers), + join_decimal_u16(&parsed.extensions), + join_decimal_u16(&parsed.supported_groups), + join_decimal_u8(&parsed.ec_point_formats) + ) +} + +fn ja4(parsed: &ParsedClientHello) -> (String, String) { + let a = format!( + "t{}{}{:02}{:02}{}", + ja4_version_code(parsed), + if parsed.sni_present { "d" } else { "i" }, + count_ja4(parsed.ciphers.len()), + count_ja4(parsed.extensions.len()), + ja4_alpn_marker(parsed.alpn_first.as_deref()) + ); + + let mut ciphers = parsed.ciphers.clone(); + ciphers.sort_unstable(); + let cipher_raw = join_hex_u16(&ciphers); + let cipher_hash = if ciphers.is_empty() { + "000000000000".to_string() + } else { + sha256_truncated_12(&cipher_raw) + }; + + let mut extensions_for_hash = parsed + .extensions + .iter() + .copied() + .filter(|value| *value != EXT_SNI && *value != EXT_ALPN) + .collect::>(); + extensions_for_hash.sort_unstable(); + let extension_raw = join_hex_u16(&extensions_for_hash); + let signature_raw = join_hex_u16(&parsed.signature_algorithms); + let extension_hash_input = if signature_raw.is_empty() { + extension_raw.clone() + } else { + format!("{extension_raw}_{signature_raw}") + }; + let extension_hash = if extensions_for_hash.is_empty() { + "000000000000".to_string() + } else { + sha256_truncated_12(&extension_hash_input) + }; + + ( + format!("{a}_{cipher_hash}_{extension_hash}"), + format!("{a}_{cipher_raw}_{extension_hash_input}"), + ) +} + +fn ja4_version_code(parsed: &ParsedClientHello) -> &'static str { + let version = parsed + .supported_versions + .iter() + .copied() + .max() + .unwrap_or(parsed.legacy_version); + match version { + 0x0304 => "13", + 0x0303 => "12", + 0x0302 => "11", + 0x0301 => "10", + 0x0300 => "s3", + 0x0002 => "s2", + 0xfeff => "d1", + 0xfefd => "d2", + 0xfefc => "d3", + _ => "00", + } +} + +fn ja4_alpn_marker(alpn_first: Option<&[u8]>) -> String { + let Some(value) = alpn_first else { + return "00".to_string(); + }; + let Some(first) = value.first().copied() else { + return "00".to_string(); + }; + let last = value.last().copied().unwrap_or(first); + if first.is_ascii_alphanumeric() && last.is_ascii_alphanumeric() { + return format!("{}{}", first as char, last as char); + } + + let encoded = hex::encode(value); + if encoded.is_empty() { + return "00".to_string(); + } + let first_hex = encoded.as_bytes()[0] as char; + let last_hex = encoded.as_bytes()[encoded.len().saturating_sub(1)] as char; + format!("{first_hex}{last_hex}") +} + +fn count_ja4(count: usize) -> usize { + count.min(99) +} + +fn sha256_truncated_12(input: &str) -> String { + let mut encoded = hex::encode(sha256(input.as_bytes())); + encoded.truncate(12); + encoded +} + +fn join_decimal_u16(values: &[u16]) -> String { + values + .iter() + .map(u16::to_string) + .collect::>() + .join("-") +} + +fn join_decimal_u8(values: &[u8]) -> String { + values + .iter() + .map(u8::to_string) + .collect::>() + .join("-") +} + +fn join_hex_u16(values: &[u16]) -> String { + values + .iter() + .map(|value| format!("{value:04x}")) + .collect::>() + .join(",") +} + +fn read_u16_at(buf: &[u8], pos: usize) -> Option { + Some(u16::from_be_bytes([ + *buf.get(pos)?, + *buf.get(pos.checked_add(1)?)?, + ])) +} + +fn is_grease(value: u16) -> bool { + let high = (value >> 8) as u8; + let low = value as u8; + high == low && (high & 0x0f) == 0x0a +} + +#[cfg(test)] +mod tests { + use super::*; + + fn sample_client_hello() -> Vec { + let mut body = Vec::new(); + body.extend_from_slice(&[0x03, 0x03]); + body.extend_from_slice(&[0x11; 32]); + body.push(0); + body.extend_from_slice(&10u16.to_be_bytes()); + body.extend_from_slice(&[0x0a, 0x0a, 0x13, 0x01, 0x13, 0x02, 0xc0, 0x2f, 0x00, 0xff]); + body.push(1); + body.push(0); + + let mut extensions = Vec::new(); + append_ext(&mut extensions, EXT_SNI, &[0, 0]); + append_ext(&mut extensions, EXT_ALPN, &[0, 3, 2, b'h', b'2']); + append_ext( + &mut extensions, + EXT_SUPPORTED_GROUPS, + &[0, 6, 0x0a, 0x0a, 0x00, 0x17, 0x00, 0x1d], + ); + append_ext(&mut extensions, EXT_EC_POINT_FORMATS, &[1, 0]); + append_ext( + &mut extensions, + EXT_SIGNATURE_ALGORITHMS, + &[0, 4, 0x04, 0x03, 0x08, 0x04], + ); + append_ext( + &mut extensions, + EXT_SUPPORTED_VERSIONS, + &[4, 0x03, 0x04, 0x03, 0x03], + ); + body.extend_from_slice(&(extensions.len() as u16).to_be_bytes()); + body.extend_from_slice(&extensions); + + let mut record = Vec::new(); + record.push(TLS_RECORD_HANDSHAKE); + record.extend_from_slice(&[0x03, 0x01]); + record.extend_from_slice(&((body.len() + 4) as u16).to_be_bytes()); + record.push(0x01); + record.extend_from_slice(&[ + ((body.len() >> 16) & 0xff) as u8, + ((body.len() >> 8) & 0xff) as u8, + (body.len() & 0xff) as u8, + ]); + record.extend_from_slice(&body); + record + } + + fn append_ext(out: &mut Vec, etype: u16, data: &[u8]) { + out.extend_from_slice(&etype.to_be_bytes()); + out.extend_from_slice(&(data.len() as u16).to_be_bytes()); + out.extend_from_slice(data); + } + + #[test] + fn ja3_and_ja4_ignore_grease_and_remain_stable() { + let fp = fingerprint_client_hello(&sample_client_hello()) + .expect("sample ClientHello must fingerprint"); + assert_eq!( + fp.ja3_raw, + "771,4865-4866-49199-255,0-16-10-11-13-43,23-29,0" + ); + assert!(fp.ja4.starts_with("t13d0406h2_")); + } + + #[test] + fn malformed_client_hello_returns_none() { + let mut hello = sample_client_hello(); + hello.truncate(12); + assert!(fingerprint_client_hello(&hello).is_none()); + } +} diff --git a/src/proxy/client.rs b/src/proxy/client.rs index a8357b7..34b540b 100644 --- a/src/proxy/client.rs +++ b/src/proxy/client.rs @@ -98,6 +98,7 @@ use crate::error::{HandshakeResult, ProxyError, Result, StreamError}; use crate::ip_tracker::UserIpTracker; use crate::protocol::constants::*; use crate::protocol::tls; +use crate::protocol::tls_fingerprint::{self, TlsClientFingerprint}; use crate::stats::beobachten::BeobachtenStore; use crate::stats::{ReplayChecker, Stats}; use crate::stream::{BufferPool, CryptoReader, CryptoWriter}; @@ -350,6 +351,60 @@ fn record_beobachten_class( beobachten.record(class, peer_ip, beobachten_ttl(config)); } +fn tls_fingerprint_collection_enabled(config: &ProxyConfig) -> bool { + config.general.beobachten || config.server.api.runtime_edge_enabled +} + +fn observe_tls_client_fingerprint( + stats: &Stats, + config: &ProxyConfig, + peer_ip: IpAddr, + handshake: &[u8], +) -> Option { + if !tls_fingerprint_collection_enabled(config) { + return None; + } + + match tls_fingerprint::fingerprint_client_hello(handshake) { + Some(fingerprint) => { + stats.record_tls_fingerprint_observed(&fingerprint, peer_ip, beobachten_ttl(config)); + Some(fingerprint) + } + None => { + stats.increment_tls_fingerprint_parse_error(); + None + } + } +} + +fn record_tls_fingerprint_auth_success( + stats: &Stats, + config: &ProxyConfig, + peer_ip: IpAddr, + fingerprint: Option<&TlsClientFingerprint>, + user: &str, +) { + if let Some(fingerprint) = fingerprint { + stats.record_tls_fingerprint_auth_success( + fingerprint, + peer_ip, + user, + beobachten_ttl(config), + ); + } +} + +fn record_tls_fingerprint_bad_or_probe( + stats: &Stats, + config: &ProxyConfig, + peer_ip: IpAddr, + fingerprint: Option<&TlsClientFingerprint>, +) { + if let Some(fingerprint) = fingerprint { + stats.record_tls_fingerprint_bad_or_probe(fingerprint, peer_ip, beobachten_ttl(config)); + } +} + fn classify_expected_64_got_0(kind: std::io::ErrorKind) -> Option<&'static str> { match kind { std::io::ErrorKind::UnexpectedEof => Some("expected_64_got_0_unexpected_eof"), @@ -705,6 +760,9 @@ where )); } + let tls_fingerprint = + observe_tls_client_fingerprint(stats.as_ref(), &config, real_peer.ip(), &handshake); + let (read_half, write_half) = tokio::io::split(stream); let (mut tls_reader, tls_writer, tls_user) = match handle_tls_handshake_with_shared( @@ -715,6 +773,12 @@ where HandshakeResult::Success(result) => result, HandshakeResult::BadClient { reader, writer } => { stats.increment_connects_bad_with_class("tls_handshake_bad_client"); + record_tls_fingerprint_bad_or_probe( + stats.as_ref(), + &config, + real_peer.ip(), + tls_fingerprint.as_ref(), + ); return Ok(masking_outcome( reader, writer, @@ -726,10 +790,23 @@ where )); } HandshakeResult::Error(e) => { + record_tls_fingerprint_bad_or_probe( + stats.as_ref(), + &config, + real_peer.ip(), + tls_fingerprint.as_ref(), + ); increment_bad_on_unknown_tls_sni(stats.as_ref(), &e); return Err(e); } }; + record_tls_fingerprint_auth_success( + stats.as_ref(), + &config, + real_peer.ip(), + tls_fingerprint.as_ref(), + tls_user.as_str(), + ); debug!(peer = %peer, "Reading MTProto handshake through TLS"); let mtproto_data = tls_reader.read_exact(HANDSHAKE_LEN).await?; @@ -1295,6 +1372,13 @@ impl RunningClientHandler { )); } + let tls_fingerprint = observe_tls_client_fingerprint( + self.stats.as_ref(), + &self.config, + peer.ip(), + &handshake, + ); + let config = self.config.clone(); let replay_checker = self.replay_checker.clone(); let stats = self.stats.clone(); @@ -1318,6 +1402,12 @@ impl RunningClientHandler { HandshakeResult::Success(result) => result, HandshakeResult::BadClient { reader, writer } => { stats.increment_connects_bad_with_class("tls_handshake_bad_client"); + record_tls_fingerprint_bad_or_probe( + stats.as_ref(), + &config, + peer.ip(), + tls_fingerprint.as_ref(), + ); return Ok(masking_outcome( reader, writer, @@ -1329,10 +1419,23 @@ impl RunningClientHandler { )); } HandshakeResult::Error(e) => { + record_tls_fingerprint_bad_or_probe( + stats.as_ref(), + &config, + peer.ip(), + tls_fingerprint.as_ref(), + ); increment_bad_on_unknown_tls_sni(stats.as_ref(), &e); return Err(e); } }; + record_tls_fingerprint_auth_success( + stats.as_ref(), + &config, + peer.ip(), + tls_fingerprint.as_ref(), + tls_user.as_str(), + ); debug!(peer = %peer, "Reading MTProto handshake through TLS"); let mtproto_data = tls_reader.read_exact(HANDSHAKE_LEN).await?; @@ -1558,6 +1661,11 @@ impl RunningClientHandler { { let user = success.user.clone(); + if !shared.is_user_enabled(&user) { + warn!(user = %user, "Disabled user rejected"); + return Err(ProxyError::UserDisabled { user }); + } + let user_limit_reservation = match Self::acquire_user_connection_reservation_static( &user, &config, @@ -1576,6 +1684,8 @@ impl RunningClientHandler { let route_snapshot = route_runtime.snapshot(); let session_id = rng.u64(); + let _user_session = shared.register_user_session(&user, session_id); + let session_cancel = _user_session.token(); let selected_me_pool = if config.general.use_middle_proxy && matches!(route_snapshot.mode, RelayRouteMode::Middle) { @@ -1607,6 +1717,7 @@ impl RunningClientHandler { route_runtime.subscribe(), route_snapshot, session_id, + session_cancel.clone(), shared.clone(), ) .await @@ -1625,6 +1736,7 @@ impl RunningClientHandler { route_snapshot, session_id, local_addr, + session_cancel.clone(), shared.clone(), ) .await @@ -1644,6 +1756,7 @@ impl RunningClientHandler { route_snapshot, session_id, local_addr, + session_cancel, shared.clone(), ) .await diff --git a/src/proxy/direct_relay.rs b/src/proxy/direct_relay.rs index 2fea54a..4f7cb20 100644 --- a/src/proxy/direct_relay.rs +++ b/src/proxy/direct_relay.rs @@ -10,6 +10,7 @@ use std::time::Duration; use tokio::io::{AsyncRead, AsyncWrite, AsyncWriteExt, ReadHalf, WriteHalf, split}; use tokio::sync::watch; +use tokio_util::sync::CancellationToken; use tracing::{debug, info, warn}; use crate::config::ProxyConfig; @@ -258,6 +259,7 @@ where route_snapshot, session_id, SocketAddr::from(([0, 0, 0, 0], config.server.port)), + CancellationToken::new(), ProxySharedState::new(), ) .await @@ -276,6 +278,7 @@ pub(crate) async fn handle_via_direct_with_shared( route_snapshot: RouteCutoverState, session_id: u64, local_addr: SocketAddr, + session_cancel: CancellationToken, shared: Arc, ) -> Result<()> where @@ -302,14 +305,25 @@ where "Ignoring invalid scope hint and falling back to default upstream selection" ); } - let tg_stream = upstream_manager - .connect(dc_addr, Some(success.dc_idx), scope_hint) - .await?; + let tg_stream = tokio::select! { + result = upstream_manager.connect(dc_addr, Some(success.dc_idx), scope_hint) => result?, + _ = session_cancel.cancelled() => { + return Err(ProxyError::UserDisabled { + user: user.to_string(), + }); + } + }; debug!(peer = %success.peer, dc_addr = %dc_addr, "Connected, performing TG handshake"); - let (tg_reader, tg_writer) = - do_tg_handshake_static(tg_stream, &success, &config, rng.as_ref()).await?; + let (tg_reader, tg_writer) = tokio::select! { + result = do_tg_handshake_static(tg_stream, &success, &config, rng.as_ref()) => result?, + _ = session_cancel.cancelled() => { + return Err(ProxyError::UserDisabled { + user: user.to_string(), + }); + } + }; debug!(peer = %success.peer, "TG handshake complete, starting relay"); @@ -331,20 +345,22 @@ where } else { Duration::from_secs(1800) }; - let relay_result = crate::proxy::relay::relay_bidirectional_with_activity_timeout_and_lease( - client_reader, - client_writer, - tg_reader, - tg_writer, - config.general.direct_relay_copy_buf_c2s_bytes, - config.general.direct_relay_copy_buf_s2c_bytes, - user, - Arc::clone(&stats), - config.access.user_data_quota.get(user).copied(), - buffer_pool, - traffic_lease, - relay_activity_timeout, - ); + let relay_result = + crate::proxy::relay::relay_bidirectional_with_activity_timeout_lease_and_cancel( + client_reader, + client_writer, + tg_reader, + tg_writer, + config.general.direct_relay_copy_buf_c2s_bytes, + config.general.direct_relay_copy_buf_s2c_bytes, + user, + Arc::clone(&stats), + config.access.user_data_quota.get(user).copied(), + buffer_pool, + traffic_lease, + relay_activity_timeout, + session_cancel.clone(), + ); tokio::pin!(relay_result); let relay_result = loop { if let Some(cutover) = @@ -371,6 +387,11 @@ where break relay_result.await; } } + _ = session_cancel.cancelled() => { + break Err(ProxyError::UserDisabled { + user: user.to_string(), + }); + } } }; diff --git a/src/proxy/middle_relay/session.rs b/src/proxy/middle_relay/session.rs index 81cf297..4865993 100644 --- a/src/proxy/middle_relay/session.rs +++ b/src/proxy/middle_relay/session.rs @@ -13,6 +13,7 @@ pub(crate) async fn handle_via_middle_proxy( mut route_rx: watch::Receiver, route_snapshot: RouteCutoverState, session_id: u64, + session_cancel: CancellationToken, shared: Arc, ) -> Result<()> where @@ -20,6 +21,10 @@ where W: AsyncWrite + Unpin + Send + 'static, { let user = success.user.clone(); + if session_cancel.is_cancelled() { + return Err(ProxyError::UserDisabled { user }); + } + let quota_limit = config.access.user_data_quota.get(&user).copied(); let quota_user_stats = quota_limit.map(|_| stats.get_or_create_user_stats_handle(&user)); let peer = success.peer; @@ -590,6 +595,25 @@ where } tokio::select! { + _ = session_cancel.cancelled() => { + warn!( + user = %user, + conn_id, + "Disabled user middle session cancelled" + ); + let _ = enqueue_c2me_command_in( + shared.as_ref(), + &c2me_tx, + C2MeCommand::Close, + c2me_send_timeout, + stats.as_ref(), + ) + .await; + main_result = Err(ProxyError::UserDisabled { + user: user.clone(), + }); + break; + } changed = route_rx.changed(), if route_watch_open => { if changed.is_err() { route_watch_open = false; diff --git a/src/proxy/relay.rs b/src/proxy/relay.rs index 5ea9e87..36af33a 100644 --- a/src/proxy/relay.rs +++ b/src/proxy/relay.rs @@ -55,11 +55,13 @@ use crate::error::{ProxyError, Result}; use crate::proxy::traffic_limiter::TrafficLease; use crate::stats::Stats; use crate::stream::BufferPool; +use std::future::pending; use std::sync::Arc; use std::sync::atomic::{AtomicBool, Ordering}; use std::time::Duration; use tokio::io::{AsyncRead, AsyncWrite, AsyncWriteExt, copy_bidirectional_with_sizes}; use tokio::time::Instant; +use tokio_util::sync::CancellationToken; use tracing::{debug, warn}; // ============= Constants ============= @@ -191,6 +193,84 @@ pub async fn relay_bidirectional_with_activity_timeout_and_lease traffic_lease: Option>, activity_timeout: Duration, ) -> Result<()> +where + CR: AsyncRead + Unpin + Send + 'static, + CW: AsyncWrite + Unpin + Send + 'static, + SR: AsyncRead + Unpin + Send + 'static, + SW: AsyncWrite + Unpin + Send + 'static, +{ + relay_bidirectional_with_activity_timeout_lease_cancel_inner( + client_reader, + client_writer, + server_reader, + server_writer, + c2s_buf_size, + s2c_buf_size, + user, + stats, + quota_limit, + _buffer_pool, + traffic_lease, + activity_timeout, + None, + ) + .await +} + +pub async fn relay_bidirectional_with_activity_timeout_lease_and_cancel( + client_reader: CR, + client_writer: CW, + server_reader: SR, + server_writer: SW, + c2s_buf_size: usize, + s2c_buf_size: usize, + user: &str, + stats: Arc, + quota_limit: Option, + _buffer_pool: Arc, + traffic_lease: Option>, + activity_timeout: Duration, + session_cancel: CancellationToken, +) -> Result<()> +where + CR: AsyncRead + Unpin + Send + 'static, + CW: AsyncWrite + Unpin + Send + 'static, + SR: AsyncRead + Unpin + Send + 'static, + SW: AsyncWrite + Unpin + Send + 'static, +{ + relay_bidirectional_with_activity_timeout_lease_cancel_inner( + client_reader, + client_writer, + server_reader, + server_writer, + c2s_buf_size, + s2c_buf_size, + user, + stats, + quota_limit, + _buffer_pool, + traffic_lease, + activity_timeout, + Some(session_cancel), + ) + .await +} + +async fn relay_bidirectional_with_activity_timeout_lease_cancel_inner( + client_reader: CR, + client_writer: CW, + server_reader: SR, + server_writer: SW, + c2s_buf_size: usize, + s2c_buf_size: usize, + user: &str, + stats: Arc, + quota_limit: Option, + _buffer_pool: Arc, + traffic_lease: Option>, + activity_timeout: Duration, + session_cancel: Option, +) -> Result<()> where CR: AsyncRead + Unpin + Send + 'static, CW: AsyncWrite + Unpin + Send + 'static, @@ -287,14 +367,29 @@ where // // When the watchdog fires, select! drops the copy future, // releasing the &mut borrows on client and server. - let copy_result = tokio::select! { + enum RelayOutcome { + Copy(std::io::Result<(u64, u64)>), + ActivityTimeout, + UserDisabled, + } + + let cancel_wait = async move { + match session_cancel { + Some(token) => token.cancelled().await, + None => pending::<()>().await, + } + }; + tokio::pin!(cancel_wait); + + let relay_outcome = tokio::select! { result = copy_bidirectional_with_sizes( &mut client, &mut server, c2s_buf_size.max(1), s2c_buf_size.max(1), - ) => Some(result), - _ = watchdog => None, // Activity timeout — cancel relay + ) => RelayOutcome::Copy(result), + _ = watchdog => RelayOutcome::ActivityTimeout, + _ = &mut cancel_wait => RelayOutcome::UserDisabled, }; // ── Clean shutdown ────────────────────────────────────────────── @@ -308,8 +403,8 @@ where let s2c_ops = counters.s2c_ops.load(Ordering::Relaxed); let duration = epoch.elapsed(); - match copy_result { - Some(Ok((c2s, s2c))) => { + match relay_outcome { + RelayOutcome::Copy(Ok((c2s, s2c))) => { // Normal completion — one side closed the connection debug!( user = %user_owned, @@ -322,7 +417,7 @@ where ); Ok(()) } - Some(Err(e)) if is_quota_io_error(&e) => { + RelayOutcome::Copy(Err(e)) if is_quota_io_error(&e) => { let c2s = counters.c2s_bytes.load(Ordering::Relaxed); let s2c = counters.s2c_bytes.load(Ordering::Relaxed); warn!( @@ -338,7 +433,7 @@ where user: user_owned.clone(), }) } - Some(Err(e)) => { + RelayOutcome::Copy(Err(e)) => { // I/O error in one of the directions let c2s = counters.c2s_bytes.load(Ordering::Relaxed); let s2c = counters.s2c_bytes.load(Ordering::Relaxed); @@ -354,7 +449,7 @@ where ); Err(e.into()) } - None => { + RelayOutcome::ActivityTimeout => { // Activity timeout (watchdog fired) let c2s = counters.c2s_bytes.load(Ordering::Relaxed); let s2c = counters.s2c_bytes.load(Ordering::Relaxed); @@ -369,6 +464,22 @@ where ); Ok(()) } + RelayOutcome::UserDisabled => { + let c2s = counters.c2s_bytes.load(Ordering::Relaxed); + let s2c = counters.s2c_bytes.load(Ordering::Relaxed); + debug!( + user = %user_owned, + c2s_bytes = c2s, + s2c_bytes = s2c, + c2s_msgs = c2s_ops, + s2c_msgs = s2c_ops, + duration_secs = duration.as_secs(), + "Relay finished (user disabled)" + ); + Err(ProxyError::UserDisabled { + user: user_owned.clone(), + }) + } } } diff --git a/src/proxy/shared_state.rs b/src/proxy/shared_state.rs index 11e390e..9ed319b 100644 --- a/src/proxy/shared_state.rs +++ b/src/proxy/shared_state.rs @@ -1,5 +1,5 @@ -use std::collections::HashSet; use std::collections::hash_map::RandomState; +use std::collections::{HashMap, HashSet}; use std::net::{IpAddr, SocketAddr}; use std::sync::atomic::{AtomicBool, AtomicU32, AtomicU64, Ordering}; use std::sync::{Arc, Mutex}; @@ -7,6 +7,7 @@ use std::time::Instant; use dashmap::DashMap; use tokio::sync::mpsc; +use tokio_util::sync::CancellationToken; use crate::proxy::handshake::{AuthProbeSaturationState, AuthProbeState}; use crate::proxy::middle_relay::{DesyncDedupRotationState, RelayIdleCandidateRegistry}; @@ -67,10 +68,35 @@ pub(crate) struct ProxySharedState { pub(crate) handshake: HandshakeSharedState, pub(crate) middle_relay: MiddleRelaySharedState, pub(crate) traffic_limiter: Arc, + disabled_users: DashMap, + active_user_sessions: DashMap<(String, u64), CancellationToken>, pub(crate) conntrack_pressure_active: AtomicBool, pub(crate) conntrack_close_tx: Mutex>>, } +#[must_use = "registered user sessions must be kept alive until relay completion"] +pub(crate) struct UserSessionRegistration { + token: CancellationToken, + _guard: UserSessionGuard, +} + +impl UserSessionRegistration { + pub(crate) fn token(&self) -> CancellationToken { + self.token.clone() + } +} + +struct UserSessionGuard { + shared: Arc, + key: (String, u64), +} + +impl Drop for UserSessionGuard { + fn drop(&mut self) { + self.shared.active_user_sessions.remove(&self.key); + } +} + impl ProxySharedState { pub(crate) fn new() -> Arc { Arc::new(Self { @@ -101,11 +127,82 @@ impl ProxySharedState { relay_idle_mark_seq: AtomicU64::new(0), }, traffic_limiter: TrafficLimiter::new(), + disabled_users: DashMap::new(), + active_user_sessions: DashMap::new(), conntrack_pressure_active: AtomicBool::new(false), conntrack_close_tx: Mutex::new(None), }) } + pub(crate) fn is_user_enabled(&self, user: &str) -> bool { + !self.disabled_users.contains_key(user) + } + + pub(crate) fn set_user_enabled(&self, user: &str, enabled: bool) -> bool { + if enabled { + self.disabled_users.remove(user); + false + } else { + self.disabled_users.insert(user.to_string(), ()).is_none() + } + } + + pub(crate) fn apply_user_enabled_config( + &self, + user_enabled: &HashMap, + ) -> Vec { + let desired_disabled = user_enabled + .iter() + .filter_map(|(user, enabled)| (!*enabled).then_some(user.clone())) + .collect::>(); + let current_disabled = self + .disabled_users + .iter() + .map(|entry| entry.key().clone()) + .collect::>(); + + for user in current_disabled.difference(&desired_disabled) { + self.disabled_users.remove(user); + } + let newly_disabled = desired_disabled + .difference(¤t_disabled) + .cloned() + .collect::>(); + for user in desired_disabled { + self.disabled_users.insert(user, ()); + } + newly_disabled + } + + pub(crate) fn register_user_session( + self: &Arc, + user: &str, + session_id: u64, + ) -> UserSessionRegistration { + let token = CancellationToken::new(); + let key = (user.to_string(), session_id); + self.active_user_sessions.insert(key.clone(), token.clone()); + UserSessionRegistration { + token, + _guard: UserSessionGuard { + shared: Arc::clone(self), + key, + }, + } + } + + pub(crate) fn cancel_user_sessions(&self, user: &str) -> usize { + let tokens = self + .active_user_sessions + .iter() + .filter_map(|entry| (entry.key().0 == user).then(|| entry.value().clone())) + .collect::>(); + for token in &tokens { + token.cancel(); + } + tokens.len() + } + pub(crate) fn set_conntrack_close_sender(&self, tx: mpsc::Sender) { match self.conntrack_close_tx.lock() { Ok(mut guard) => { @@ -166,3 +263,48 @@ impl ProxySharedState { self.conntrack_pressure_active.load(Ordering::Relaxed) } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn user_enabled_config_sync_tracks_disabled_overrides() { + let shared = ProxySharedState::new(); + assert!(shared.is_user_enabled("alice")); + + let mut user_enabled = HashMap::new(); + user_enabled.insert("alice".to_string(), false); + user_enabled.insert("bob".to_string(), true); + + let mut newly_disabled = shared.apply_user_enabled_config(&user_enabled); + newly_disabled.sort(); + assert_eq!(newly_disabled, vec!["alice".to_string()]); + assert!(!shared.is_user_enabled("alice")); + assert!(shared.is_user_enabled("bob")); + + assert!(shared.apply_user_enabled_config(&user_enabled).is_empty()); + + user_enabled.clear(); + assert!(shared.apply_user_enabled_config(&user_enabled).is_empty()); + assert!(shared.is_user_enabled("alice")); + } + + #[test] + fn cancel_user_sessions_cancels_only_registered_matching_user() { + let shared = ProxySharedState::new(); + let alice_1 = shared.register_user_session("alice", 1); + let alice_2 = shared.register_user_session("alice", 2); + let bob = shared.register_user_session("bob", 1); + let alice_1_token = alice_1.token(); + let alice_2_token = alice_2.token(); + let bob_token = bob.token(); + + drop(alice_1); + + assert_eq!(shared.cancel_user_sessions("alice"), 1); + assert!(!alice_1_token.is_cancelled()); + assert!(alice_2_token.is_cancelled()); + assert!(!bob_token.is_cancelled()); + } +} diff --git a/src/proxy/tests/client_masking_blackhat_campaign_tests.rs b/src/proxy/tests/client_masking_blackhat_campaign_tests.rs index c48caa0..4f1437b 100644 --- a/src/proxy/tests/client_masking_blackhat_campaign_tests.rs +++ b/src/proxy/tests/client_masking_blackhat_campaign_tests.rs @@ -39,6 +39,7 @@ fn new_upstream_manager(stats: Arc) -> Arc { selected_scope: String::new(), ipv4: None, ipv6: None, + prefer: None, }], 1, 1, diff --git a/src/proxy/tests/client_masking_budget_security_tests.rs b/src/proxy/tests/client_masking_budget_security_tests.rs index 11a72a0..e6ac8a8 100644 --- a/src/proxy/tests/client_masking_budget_security_tests.rs +++ b/src/proxy/tests/client_masking_budget_security_tests.rs @@ -35,6 +35,7 @@ fn build_harness(config: ProxyConfig) -> PipelineHarness { selected_scope: String::new(), ipv4: None, ipv6: None, + prefer: None, }], 1, 1, diff --git a/src/proxy/tests/client_masking_diagnostics_security_tests.rs b/src/proxy/tests/client_masking_diagnostics_security_tests.rs index a55bb79..5acb1c0 100644 --- a/src/proxy/tests/client_masking_diagnostics_security_tests.rs +++ b/src/proxy/tests/client_masking_diagnostics_security_tests.rs @@ -19,6 +19,7 @@ fn new_upstream_manager(stats: Arc) -> Arc { selected_scope: String::new(), ipv4: None, ipv6: None, + prefer: None, }], 1, 1, diff --git a/src/proxy/tests/client_masking_fragmented_classifier_security_tests.rs b/src/proxy/tests/client_masking_fragmented_classifier_security_tests.rs index 5817f24..4fef022 100644 --- a/src/proxy/tests/client_masking_fragmented_classifier_security_tests.rs +++ b/src/proxy/tests/client_masking_fragmented_classifier_security_tests.rs @@ -19,6 +19,7 @@ fn new_upstream_manager(stats: Arc) -> Arc { selected_scope: String::new(), ipv4: None, ipv6: None, + prefer: None, }], 1, 1, diff --git a/src/proxy/tests/client_masking_hard_adversarial_tests.rs b/src/proxy/tests/client_masking_hard_adversarial_tests.rs index 709ff49..86bd4fe 100644 --- a/src/proxy/tests/client_masking_hard_adversarial_tests.rs +++ b/src/proxy/tests/client_masking_hard_adversarial_tests.rs @@ -33,6 +33,7 @@ fn new_upstream_manager(stats: Arc) -> Arc { selected_scope: String::new(), ipv4: None, ipv6: None, + prefer: None, }], 1, 1, diff --git a/src/proxy/tests/client_masking_http2_fragmented_preface_security_tests.rs b/src/proxy/tests/client_masking_http2_fragmented_preface_security_tests.rs index 49c9aa6..0506671 100644 --- a/src/proxy/tests/client_masking_http2_fragmented_preface_security_tests.rs +++ b/src/proxy/tests/client_masking_http2_fragmented_preface_security_tests.rs @@ -19,6 +19,7 @@ fn new_upstream_manager(stats: Arc) -> Arc { selected_scope: String::new(), ipv4: None, ipv6: None, + prefer: None, }], 1, 1, diff --git a/src/proxy/tests/client_masking_prefetch_config_pipeline_integration_security_tests.rs b/src/proxy/tests/client_masking_prefetch_config_pipeline_integration_security_tests.rs index 6ebaa5a..94630ec 100644 --- a/src/proxy/tests/client_masking_prefetch_config_pipeline_integration_security_tests.rs +++ b/src/proxy/tests/client_masking_prefetch_config_pipeline_integration_security_tests.rs @@ -19,6 +19,7 @@ fn new_upstream_manager(stats: Arc) -> Arc { selected_scope: String::new(), ipv4: None, ipv6: None, + prefer: None, }], 1, 1, diff --git a/src/proxy/tests/client_masking_prefetch_invariant_security_tests.rs b/src/proxy/tests/client_masking_prefetch_invariant_security_tests.rs index 9491e3f..44efa54 100644 --- a/src/proxy/tests/client_masking_prefetch_invariant_security_tests.rs +++ b/src/proxy/tests/client_masking_prefetch_invariant_security_tests.rs @@ -46,6 +46,7 @@ fn build_harness(secret_hex: &str, mask_port: u16) -> PipelineHarness { selected_scope: String::new(), ipv4: None, ipv6: None, + prefer: None, }], 1, 1, diff --git a/src/proxy/tests/client_masking_probe_evasion_blackhat_tests.rs b/src/proxy/tests/client_masking_probe_evasion_blackhat_tests.rs index 62a2ef8..22b6f28 100644 --- a/src/proxy/tests/client_masking_probe_evasion_blackhat_tests.rs +++ b/src/proxy/tests/client_masking_probe_evasion_blackhat_tests.rs @@ -24,6 +24,7 @@ fn make_test_upstream_manager(stats: Arc) -> Arc { selected_scope: String::new(), ipv4: None, ipv6: None, + prefer: None, }], 1, 1, diff --git a/src/proxy/tests/client_masking_redteam_expected_fail_tests.rs b/src/proxy/tests/client_masking_redteam_expected_fail_tests.rs index 09ec626..3243bdd 100644 --- a/src/proxy/tests/client_masking_redteam_expected_fail_tests.rs +++ b/src/proxy/tests/client_masking_redteam_expected_fail_tests.rs @@ -47,6 +47,7 @@ fn build_harness(secret_hex: &str, mask_port: u16) -> RedTeamHarness { selected_scope: String::new(), ipv4: None, ipv6: None, + prefer: None, }], 1, 1, @@ -240,6 +241,7 @@ async fn redteam_03_masking_duration_must_be_less_than_1ms_when_backend_down() { selected_scope: String::new(), ipv4: None, ipv6: None, + prefer: None, }], 1, 1, @@ -484,6 +486,7 @@ async fn measure_invalid_probe_duration_ms(delay_ms: u64, tls_len: u16, body_sen selected_scope: String::new(), ipv4: None, ipv6: None, + prefer: None, }], 1, 1, @@ -561,6 +564,7 @@ async fn capture_forwarded_probe_len(tls_len: u16, body_sent: usize) -> usize { selected_scope: String::new(), ipv4: None, ipv6: None, + prefer: None, }], 1, 1, diff --git a/src/proxy/tests/client_masking_replay_timing_security_tests.rs b/src/proxy/tests/client_masking_replay_timing_security_tests.rs index 788ce80..6ee205f 100644 --- a/src/proxy/tests/client_masking_replay_timing_security_tests.rs +++ b/src/proxy/tests/client_masking_replay_timing_security_tests.rs @@ -21,6 +21,7 @@ fn new_upstream_manager(stats: Arc) -> Arc { selected_scope: String::new(), ipv4: None, ipv6: None, + prefer: None, }], 1, 1, diff --git a/src/proxy/tests/client_masking_shape_classifier_fuzz_redteam_expected_fail_tests.rs b/src/proxy/tests/client_masking_shape_classifier_fuzz_redteam_expected_fail_tests.rs index ed1ac8d..5a36cbd 100644 --- a/src/proxy/tests/client_masking_shape_classifier_fuzz_redteam_expected_fail_tests.rs +++ b/src/proxy/tests/client_masking_shape_classifier_fuzz_redteam_expected_fail_tests.rs @@ -19,6 +19,7 @@ fn new_upstream_manager(stats: Arc) -> Arc { selected_scope: String::new(), ipv4: None, ipv6: None, + prefer: None, }], 1, 1, diff --git a/src/proxy/tests/client_masking_shape_hardening_adversarial_tests.rs b/src/proxy/tests/client_masking_shape_hardening_adversarial_tests.rs index 45ce014..158e2cc 100644 --- a/src/proxy/tests/client_masking_shape_hardening_adversarial_tests.rs +++ b/src/proxy/tests/client_masking_shape_hardening_adversarial_tests.rs @@ -19,6 +19,7 @@ fn new_upstream_manager(stats: Arc) -> Arc { selected_scope: String::new(), ipv4: None, ipv6: None, + prefer: None, }], 1, 1, diff --git a/src/proxy/tests/client_masking_shape_hardening_redteam_expected_fail_tests.rs b/src/proxy/tests/client_masking_shape_hardening_redteam_expected_fail_tests.rs index f160b01..8ed2e7c 100644 --- a/src/proxy/tests/client_masking_shape_hardening_redteam_expected_fail_tests.rs +++ b/src/proxy/tests/client_masking_shape_hardening_redteam_expected_fail_tests.rs @@ -19,6 +19,7 @@ fn new_upstream_manager(stats: Arc) -> Arc { selected_scope: String::new(), ipv4: None, ipv6: None, + prefer: None, }], 1, 1, diff --git a/src/proxy/tests/client_masking_shape_hardening_security_tests.rs b/src/proxy/tests/client_masking_shape_hardening_security_tests.rs index 9948e60..65b7e67 100644 --- a/src/proxy/tests/client_masking_shape_hardening_security_tests.rs +++ b/src/proxy/tests/client_masking_shape_hardening_security_tests.rs @@ -19,6 +19,7 @@ fn new_upstream_manager(stats: Arc) -> Arc { selected_scope: String::new(), ipv4: None, ipv6: None, + prefer: None, }], 1, 1, diff --git a/src/proxy/tests/client_masking_stress_adversarial_tests.rs b/src/proxy/tests/client_masking_stress_adversarial_tests.rs index 575bfb5..a6f734c 100644 --- a/src/proxy/tests/client_masking_stress_adversarial_tests.rs +++ b/src/proxy/tests/client_masking_stress_adversarial_tests.rs @@ -33,6 +33,7 @@ fn new_upstream_manager(stats: Arc) -> Arc { selected_scope: String::new(), ipv4: None, ipv6: None, + prefer: None, }], 1, 1, diff --git a/src/proxy/tests/client_security_tests.rs b/src/proxy/tests/client_security_tests.rs index e819e4f..506e230 100644 --- a/src/proxy/tests/client_security_tests.rs +++ b/src/proxy/tests/client_security_tests.rs @@ -341,6 +341,7 @@ async fn relay_task_abort_releases_user_gate_and_ip_reservation() { selected_scope: String::new(), ipv4: None, ipv6: None, + prefer: None, }], 1, 1, @@ -459,6 +460,7 @@ async fn relay_cutover_releases_user_gate_and_ip_reservation() { selected_scope: String::new(), ipv4: None, ipv6: None, + prefer: None, }], 1, 1, @@ -586,6 +588,7 @@ async fn integration_route_cutover_and_quota_overlap_fails_closed_and_releases_s selected_scope: String::new(), ipv4: None, ipv6: None, + prefer: None, }], 1, 1, @@ -759,6 +762,7 @@ async fn proxy_protocol_header_is_rejected_when_trust_list_is_empty() { selected_scope: String::new(), ipv4: None, ipv6: None, + prefer: None, }], 1, 1, @@ -839,6 +843,7 @@ async fn proxy_protocol_header_from_untrusted_peer_range_is_rejected_under_load( selected_scope: String::new(), ipv4: None, ipv6: None, + prefer: None, }], 1, 1, @@ -1032,6 +1037,7 @@ async fn short_tls_probe_is_masked_through_client_pipeline() { selected_scope: String::new(), ipv4: None, ipv6: None, + prefer: None, }], 1, 1, @@ -1123,6 +1129,7 @@ async fn tls12_record_probe_is_masked_through_client_pipeline() { selected_scope: String::new(), ipv4: None, ipv6: None, + prefer: None, }], 1, 1, @@ -1212,6 +1219,7 @@ async fn handle_client_stream_increments_connects_all_exactly_once() { selected_scope: String::new(), ipv4: None, ipv6: None, + prefer: None, }], 1, 1, @@ -1308,6 +1316,7 @@ async fn running_client_handler_increments_connects_all_exactly_once() { selected_scope: String::new(), ipv4: None, ipv6: None, + prefer: None, }], 1, 1, @@ -1401,6 +1410,7 @@ async fn idle_pooled_connection_closes_cleanly_in_generic_stream_path() { selected_scope: String::new(), ipv4: None, ipv6: None, + prefer: None, }], 1, 1, @@ -1475,6 +1485,7 @@ async fn idle_pooled_connection_closes_cleanly_in_client_handler_path() { selected_scope: String::new(), ipv4: None, ipv6: None, + prefer: None, }], 1, 1, @@ -1564,6 +1575,7 @@ async fn partial_tls_header_stall_triggers_handshake_timeout() { selected_scope: String::new(), ipv4: None, ipv6: None, + prefer: None, }], 1, 1, @@ -1892,6 +1904,7 @@ async fn valid_tls_path_does_not_fall_back_to_mask_backend() { selected_scope: String::new(), ipv4: None, ipv6: None, + prefer: None, }], 1, 1, @@ -2004,6 +2017,7 @@ async fn valid_tls_with_invalid_mtproto_falls_back_to_mask_backend() { selected_scope: String::new(), ipv4: None, ipv6: None, + prefer: None, }], 1, 1, @@ -2114,6 +2128,7 @@ async fn client_handler_tls_bad_mtproto_is_forwarded_to_mask_backend() { selected_scope: String::new(), ipv4: None, ipv6: None, + prefer: None, }], 1, 1, @@ -2239,6 +2254,7 @@ async fn alpn_mismatch_tls_probe_is_masked_through_client_pipeline() { selected_scope: String::new(), ipv4: None, ipv6: None, + prefer: None, }], 1, 1, @@ -2335,6 +2351,7 @@ async fn invalid_hmac_tls_probe_is_masked_through_client_pipeline() { selected_scope: String::new(), ipv4: None, ipv6: None, + prefer: None, }], 1, 1, @@ -2437,6 +2454,7 @@ async fn burst_invalid_tls_probes_are_masked_verbatim() { selected_scope: String::new(), ipv4: None, ipv6: None, + prefer: None, }], 1, 1, @@ -3395,6 +3413,7 @@ async fn relay_connect_error_releases_user_and_ip_before_return() { selected_scope: String::new(), ipv4: None, ipv6: None, + prefer: None, }], 1, 1, @@ -3963,6 +3982,7 @@ async fn untrusted_proxy_header_source_is_rejected() { selected_scope: String::new(), ipv4: None, ipv6: None, + prefer: None, }], 1, 1, @@ -4036,6 +4056,7 @@ async fn empty_proxy_trusted_cidrs_rejects_proxy_header_by_default() { selected_scope: String::new(), ipv4: None, ipv6: None, + prefer: None, }], 1, 1, @@ -4136,6 +4157,7 @@ async fn oversized_tls_record_is_masked_in_generic_stream_pipeline() { selected_scope: String::new(), ipv4: None, ipv6: None, + prefer: None, }], 1, 1, @@ -4242,6 +4264,7 @@ async fn oversized_tls_record_is_masked_in_client_handler_pipeline() { selected_scope: String::new(), ipv4: None, ipv6: None, + prefer: None, }], 1, 1, @@ -4362,6 +4385,7 @@ async fn tls_record_len_min_minus_1_is_rejected_in_generic_stream_pipeline() { selected_scope: String::new(), ipv4: None, ipv6: None, + prefer: None, }], 1, 1, @@ -4468,6 +4492,7 @@ async fn tls_record_len_min_minus_1_is_rejected_in_client_handler_pipeline() { selected_scope: String::new(), ipv4: None, ipv6: None, + prefer: None, }], 1, 1, @@ -4577,6 +4602,7 @@ async fn tls_record_len_16384_is_accepted_in_generic_stream_pipeline() { selected_scope: String::new(), ipv4: None, ipv6: None, + prefer: None, }], 1, 1, @@ -4681,6 +4707,7 @@ async fn tls_record_len_16384_is_accepted_in_client_handler_pipeline() { selected_scope: String::new(), ipv4: None, ipv6: None, + prefer: None, }], 1, 1, diff --git a/src/proxy/tests/client_timing_profile_adversarial_tests.rs b/src/proxy/tests/client_timing_profile_adversarial_tests.rs index bc452a8..44e194b 100644 --- a/src/proxy/tests/client_timing_profile_adversarial_tests.rs +++ b/src/proxy/tests/client_timing_profile_adversarial_tests.rs @@ -32,6 +32,7 @@ fn make_test_upstream_manager(stats: Arc) -> Arc { selected_scope: String::new(), ipv4: None, ipv6: None, + prefer: None, }], 1, 1, @@ -74,12 +75,17 @@ async fn run_generic_once(class: ProbeClass) -> u128 { let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); let backend_addr = listener.local_addr().unwrap(); let backend_reply = REPLY_404.to_vec(); + let probe = match class { + ProbeClass::MalformedTlsTruncation => malformed_tls_probe(), + ProbeClass::PlainWebBaseline => plain_web_probe(), + }; let accept_task = tokio::spawn({ let backend_reply = backend_reply.clone(); + let expected_probe_len = probe.len(); async move { let (mut stream, _) = listener.accept().await.unwrap(); - let mut buf = [0u8; 5]; + let mut buf = vec![0u8; expected_probe_len]; stream.read_exact(&mut buf).await.unwrap(); stream.write_all(&backend_reply).await.unwrap(); } @@ -93,6 +99,7 @@ async fn run_generic_once(class: ProbeClass) -> u128 { cfg.censorship.mask_host = Some("127.0.0.1".to_string()); cfg.censorship.mask_port = backend_addr.port(); cfg.censorship.mask_proxy_protocol = 0; + cfg.censorship.mask_shape_hardening = false; if matches!(class, ProbeClass::PlainWebBaseline) { cfg.general.modes.classic = false; @@ -129,11 +136,6 @@ async fn run_generic_once(class: ProbeClass) -> u128 { false, )); - let probe = match class { - ProbeClass::MalformedTlsTruncation => malformed_tls_probe(), - ProbeClass::PlainWebBaseline => plain_web_probe(), - }; - let started = Instant::now(); client_side.write_all(&probe).await.unwrap(); client_side.shutdown().await.unwrap(); @@ -169,11 +171,16 @@ async fn run_client_handler_once(class: ProbeClass) -> u128 { let front_addr = front_listener.local_addr().unwrap(); let backend_reply = REPLY_404.to_vec(); + let probe = match class { + ProbeClass::MalformedTlsTruncation => malformed_tls_probe(), + ProbeClass::PlainWebBaseline => plain_web_probe(), + }; let mask_accept_task = tokio::spawn({ let backend_reply = backend_reply.clone(); + let expected_probe_len = probe.len(); async move { let (mut stream, _) = mask_listener.accept().await.unwrap(); - let mut buf = [0u8; 5]; + let mut buf = vec![0u8; expected_probe_len]; stream.read_exact(&mut buf).await.unwrap(); stream.write_all(&backend_reply).await.unwrap(); } @@ -187,6 +194,7 @@ async fn run_client_handler_once(class: ProbeClass) -> u128 { cfg.censorship.mask_host = Some("127.0.0.1".to_string()); cfg.censorship.mask_port = backend_addr.port(); cfg.censorship.mask_proxy_protocol = 0; + cfg.censorship.mask_shape_hardening = false; if matches!(class, ProbeClass::PlainWebBaseline) { cfg.general.modes.classic = false; @@ -239,11 +247,6 @@ async fn run_client_handler_once(class: ProbeClass) -> u128 { }) }; - let probe = match class { - ProbeClass::MalformedTlsTruncation => malformed_tls_probe(), - ProbeClass::PlainWebBaseline => plain_web_probe(), - }; - let mut client = TcpStream::connect(front_addr).await.unwrap(); let started = Instant::now(); client.write_all(&probe).await.unwrap(); diff --git a/src/proxy/tests/client_tls_clienthello_size_security_tests.rs b/src/proxy/tests/client_tls_clienthello_size_security_tests.rs index a779c92..5fdb6d6 100644 --- a/src/proxy/tests/client_tls_clienthello_size_security_tests.rs +++ b/src/proxy/tests/client_tls_clienthello_size_security_tests.rs @@ -34,6 +34,7 @@ fn make_test_upstream_manager(stats: Arc) -> Arc { selected_scope: String::new(), ipv4: None, ipv6: None, + prefer: None, }], 1, 1, diff --git a/src/proxy/tests/client_tls_clienthello_truncation_adversarial_tests.rs b/src/proxy/tests/client_tls_clienthello_truncation_adversarial_tests.rs index aa0b925..2716f23 100644 --- a/src/proxy/tests/client_tls_clienthello_truncation_adversarial_tests.rs +++ b/src/proxy/tests/client_tls_clienthello_truncation_adversarial_tests.rs @@ -35,6 +35,7 @@ fn make_test_upstream_manager(stats: Arc) -> Arc { selected_scope: String::new(), ipv4: None, ipv6: None, + prefer: None, }], 1, 1, diff --git a/src/proxy/tests/client_tls_mtproto_fallback_security_tests.rs b/src/proxy/tests/client_tls_mtproto_fallback_security_tests.rs index edea451..838cd45 100644 --- a/src/proxy/tests/client_tls_mtproto_fallback_security_tests.rs +++ b/src/proxy/tests/client_tls_mtproto_fallback_security_tests.rs @@ -49,6 +49,7 @@ fn build_harness(secret_hex: &str, mask_port: u16) -> PipelineHarness { selected_scope: String::new(), ipv4: None, ipv6: None, + prefer: None, }], 1, 1, diff --git a/src/proxy/tests/direct_relay_security_tests.rs b/src/proxy/tests/direct_relay_security_tests.rs index 67d1eee..f66397a 100644 --- a/src/proxy/tests/direct_relay_security_tests.rs +++ b/src/proxy/tests/direct_relay_security_tests.rs @@ -1338,6 +1338,7 @@ async fn direct_relay_abort_midflight_releases_route_gauge() { selected_scope: String::new(), ipv4: None, ipv6: None, + prefer: None, }], 1, 1, @@ -1448,6 +1449,7 @@ async fn direct_relay_cutover_midflight_releases_route_gauge() { selected_scope: String::new(), ipv4: None, ipv6: None, + prefer: None, }], 1, 1, @@ -1570,6 +1572,7 @@ async fn direct_relay_cutover_storm_multi_session_keeps_generic_errors_and_relea selected_scope: String::new(), ipv4: None, ipv6: None, + prefer: None, }], 1, 1, @@ -1803,6 +1806,7 @@ async fn negative_direct_relay_dc_connection_refused_fails_fast() { selected_scope: String::new(), ipv4: None, ipv6: None, + prefer: None, }], 1, 100, @@ -1897,6 +1901,7 @@ async fn adversarial_direct_relay_cutover_integrity() { selected_scope: String::new(), ipv4: None, ipv6: None, + prefer: None, }], 1, 100, diff --git a/src/proxy/tests/masking_timing_budget_coupling_security_tests.rs b/src/proxy/tests/masking_timing_budget_coupling_security_tests.rs index fda6de7..05fc80f 100644 --- a/src/proxy/tests/masking_timing_budget_coupling_security_tests.rs +++ b/src/proxy/tests/masking_timing_budget_coupling_security_tests.rs @@ -22,6 +22,7 @@ async fn adversarial_delayed_interface_lookup_does_not_consume_outcome_floor_bud let refresh_lock = LOCAL_INTERFACE_REFRESH_LOCK.get_or_init(|| AsyncMutex::new(())); let held_refresh_guard = refresh_lock.lock().await; + reset_local_interface_enumerations_for_tests(); let (mut client, server) = duplex(1024); let started = Instant::now(); diff --git a/src/proxy/tests/proxy_shared_state_isolation_tests.rs b/src/proxy/tests/proxy_shared_state_isolation_tests.rs index faa045f..6879a1f 100644 --- a/src/proxy/tests/proxy_shared_state_isolation_tests.rs +++ b/src/proxy/tests/proxy_shared_state_isolation_tests.rs @@ -61,6 +61,7 @@ fn new_client_harness() -> ClientHarness { selected_scope: String::new(), ipv4: None, ipv6: None, + prefer: None, }], 1, 1, diff --git a/src/stats/mod.rs b/src/stats/mod.rs index 7360fc1..7f05bbf 100644 --- a/src/stats/mod.rs +++ b/src/stats/mod.rs @@ -10,6 +10,7 @@ mod me_counters; mod me_getters; mod replay; pub mod telemetry; +pub mod tls_fingerprints; mod users; mod writer_counters; @@ -22,6 +23,7 @@ use std::time::Instant; #[allow(unused_imports)] pub use self::replay::{ReplayChecker, ReplayStats}; use self::telemetry::TelemetryPolicy; +pub use self::tls_fingerprints::TlsFingerprintSnapshotRow; use crate::config::MeWriterPickMode; #[derive(Clone, Copy)] @@ -333,6 +335,7 @@ pub struct Stats { telemetry_user_enabled: AtomicBool, telemetry_me_level: AtomicU8, cached_epoch_secs: AtomicU64, + tls_fingerprints: tls_fingerprints::TlsFingerprintCollector, user_stats: DashMap>, user_stats_last_cleanup_epoch_secs: AtomicU64, start_time: parking_lot::RwLock>, diff --git a/src/stats/tls_fingerprints.rs b/src/stats/tls_fingerprints.rs new file mode 100644 index 0000000..424aa4e --- /dev/null +++ b/src/stats/tls_fingerprints.rs @@ -0,0 +1,556 @@ +//! Bounded TLS JA3/JA4 fingerprint aggregation. + +use std::cmp::Reverse; +use std::hash::Hash; +use std::net::{IpAddr, Ipv6Addr}; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::time::Duration; + +use dashmap::DashMap; +use dashmap::mapref::entry::Entry; + +use crate::protocol::tls_fingerprint::TlsClientFingerprint; + +use super::Stats; + +const CLEANUP_INTERVAL_SECS: u64 = 30; +const MAX_TLS_FINGERPRINT_BUCKETS: usize = 65_536; + +#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)] +pub enum TlsFingerprintScopeKind { + Fingerprint, + Ip, + Cidr, + User, +} + +#[derive(Clone, Debug)] +pub struct TlsFingerprintSnapshotRow { + pub scope_key: String, + pub ja3: String, + pub ja3_raw: String, + pub ja4: String, + pub ja4_raw: String, + pub total: u64, + pub auth_success: u64, + pub bad_or_probe: u64, + pub first_seen_epoch_secs: u64, + pub last_seen_epoch_secs: u64, +} + +#[derive(Clone, Debug)] +pub struct TlsFingerprintSnapshot { + pub retention_secs: u64, + pub capacity: usize, + pub dropped_total: u64, + pub parse_error_total: u64, + pub by_fingerprint: Vec, + pub by_ip: Vec, + pub by_cidr: Vec, + pub by_user: Vec, +} + +#[derive(Clone, Debug, Eq, Hash, PartialEq)] +struct TlsFingerprintKey { + scope_kind: TlsFingerprintScopeKind, + scope_key: String, + ja3: String, + ja3_raw: String, + ja4: String, + ja4_raw: String, +} + +struct TlsFingerprintEntry { + first_seen_epoch_secs: AtomicU64, + last_seen_epoch_secs: AtomicU64, + total: AtomicU64, + auth_success: AtomicU64, + bad_or_probe: AtomicU64, +} + +#[derive(Default)] +pub struct TlsFingerprintCollector { + entries: DashMap, + dropped_total: AtomicU64, + parse_error_total: AtomicU64, + last_cleanup_epoch_secs: AtomicU64, +} + +impl TlsFingerprintCollector { + pub fn record_observed( + &self, + fingerprint: &TlsClientFingerprint, + peer_ip: IpAddr, + ttl: Duration, + ) { + if ttl.is_zero() { + return; + } + let now = now_epoch_secs(); + self.cleanup_if_needed(now, ttl.as_secs()); + self.record_scoped( + scope_key(TlsFingerprintScopeKind::Fingerprint, ""), + fingerprint, + now, + true, + false, + false, + ); + self.record_scoped( + scope_key(TlsFingerprintScopeKind::Ip, &peer_ip.to_string()), + fingerprint, + now, + true, + false, + false, + ); + self.record_scoped( + scope_key(TlsFingerprintScopeKind::Cidr, &cidr_bucket(peer_ip)), + fingerprint, + now, + true, + false, + false, + ); + } + + pub fn record_auth_success( + &self, + fingerprint: &TlsClientFingerprint, + peer_ip: IpAddr, + user: &str, + ttl: Duration, + ) { + if ttl.is_zero() || user.is_empty() { + return; + } + let now = now_epoch_secs(); + self.cleanup_if_needed(now, ttl.as_secs()); + self.record_scoped( + scope_key(TlsFingerprintScopeKind::Fingerprint, ""), + fingerprint, + now, + false, + true, + false, + ); + self.record_scoped( + scope_key(TlsFingerprintScopeKind::Ip, &peer_ip.to_string()), + fingerprint, + now, + false, + true, + false, + ); + self.record_scoped( + scope_key(TlsFingerprintScopeKind::Cidr, &cidr_bucket(peer_ip)), + fingerprint, + now, + false, + true, + false, + ); + self.record_scoped( + scope_key(TlsFingerprintScopeKind::User, user), + fingerprint, + now, + true, + true, + false, + ); + } + + pub fn record_bad_or_probe( + &self, + fingerprint: &TlsClientFingerprint, + peer_ip: IpAddr, + ttl: Duration, + ) { + if ttl.is_zero() { + return; + } + let now = now_epoch_secs(); + self.cleanup_if_needed(now, ttl.as_secs()); + self.record_scoped( + scope_key(TlsFingerprintScopeKind::Fingerprint, ""), + fingerprint, + now, + false, + false, + true, + ); + self.record_scoped( + scope_key(TlsFingerprintScopeKind::Ip, &peer_ip.to_string()), + fingerprint, + now, + false, + false, + true, + ); + self.record_scoped( + scope_key(TlsFingerprintScopeKind::Cidr, &cidr_bucket(peer_ip)), + fingerprint, + now, + false, + false, + true, + ); + } + + pub fn increment_parse_error(&self) { + self.parse_error_total.fetch_add(1, Ordering::Relaxed); + } + + pub fn snapshot(&self, ttl: Duration, limit: usize) -> TlsFingerprintSnapshot { + let now = now_epoch_secs(); + self.cleanup(now, ttl.as_secs()); + + let limit = limit.clamp(1, 1000); + let mut by_fingerprint = Vec::new(); + let mut by_ip = Vec::new(); + let mut by_cidr = Vec::new(); + let mut by_user = Vec::new(); + + for entry in self.entries.iter() { + let row = snapshot_row(entry.key(), entry.value()); + match entry.key().scope_kind { + TlsFingerprintScopeKind::Fingerprint => by_fingerprint.push(row), + TlsFingerprintScopeKind::Ip => by_ip.push(row), + TlsFingerprintScopeKind::Cidr => by_cidr.push(row), + TlsFingerprintScopeKind::User => by_user.push(row), + } + } + + sort_and_truncate(&mut by_fingerprint, limit); + sort_and_truncate(&mut by_ip, limit); + sort_and_truncate(&mut by_cidr, limit); + sort_and_truncate(&mut by_user, limit); + + TlsFingerprintSnapshot { + retention_secs: ttl.as_secs(), + capacity: MAX_TLS_FINGERPRINT_BUCKETS, + dropped_total: self.dropped_total.load(Ordering::Relaxed), + parse_error_total: self.parse_error_total.load(Ordering::Relaxed), + by_fingerprint, + by_ip, + by_cidr, + by_user, + } + } + + pub fn snapshot_text(&self, ttl: Duration, limit: usize) -> String { + let snapshot = self.snapshot(ttl, limit); + if snapshot.by_fingerprint.is_empty() + && snapshot.by_ip.is_empty() + && snapshot.by_cidr.is_empty() + && snapshot.by_user.is_empty() + { + return String::new(); + } + + let mut out = String::new(); + out.push_str("[tls_fingerprints]\n"); + out.push_str(&format!( + "retention_secs={} capacity={} dropped_total={} parse_error_total={}\n", + snapshot.retention_secs, + snapshot.capacity, + snapshot.dropped_total, + snapshot.parse_error_total + )); + append_rows( + &mut out, + "tls_fingerprints.by_fingerprint", + &snapshot.by_fingerprint, + ); + append_rows(&mut out, "tls_fingerprints.by_ip", &snapshot.by_ip); + append_rows(&mut out, "tls_fingerprints.by_cidr", &snapshot.by_cidr); + append_rows(&mut out, "tls_fingerprints.by_user", &snapshot.by_user); + out + } + + fn record_scoped( + &self, + scope: (TlsFingerprintScopeKind, String), + fingerprint: &TlsClientFingerprint, + now_epoch_secs: u64, + count_total: bool, + count_auth_success: bool, + count_bad_or_probe: bool, + ) { + let key = TlsFingerprintKey { + scope_kind: scope.0, + scope_key: scope.1, + ja3: fingerprint.ja3.clone(), + ja3_raw: fingerprint.ja3_raw.clone(), + ja4: fingerprint.ja4.clone(), + ja4_raw: fingerprint.ja4_raw.clone(), + }; + + if let Some(entry) = self.entries.get(&key) { + update_entry( + entry.value(), + now_epoch_secs, + count_total, + count_auth_success, + count_bad_or_probe, + ); + return; + } + + if self.entries.len() >= MAX_TLS_FINGERPRINT_BUCKETS { + self.dropped_total.fetch_add(1, Ordering::Relaxed); + return; + } + + match self.entries.entry(key) { + Entry::Occupied(entry) => { + update_entry( + entry.get(), + now_epoch_secs, + count_total, + count_auth_success, + count_bad_or_probe, + ); + } + Entry::Vacant(entry) => { + entry.insert(TlsFingerprintEntry::new( + now_epoch_secs, + if count_total { 1 } else { 0 }, + if count_auth_success { 1 } else { 0 }, + if count_bad_or_probe { 1 } else { 0 }, + )); + } + } + } + + fn cleanup_if_needed(&self, now_epoch_secs: u64, ttl_secs: u64) { + let last = self.last_cleanup_epoch_secs.load(Ordering::Relaxed); + if now_epoch_secs.saturating_sub(last) < CLEANUP_INTERVAL_SECS { + return; + } + if self + .last_cleanup_epoch_secs + .compare_exchange(last, now_epoch_secs, Ordering::AcqRel, Ordering::Relaxed) + .is_err() + { + return; + } + self.cleanup(now_epoch_secs, ttl_secs); + } + + fn cleanup(&self, now_epoch_secs: u64, ttl_secs: u64) { + if ttl_secs == 0 { + self.entries.clear(); + return; + } + self.entries.retain(|_, entry| { + let last_seen = entry.last_seen_epoch_secs.load(Ordering::Relaxed); + now_epoch_secs.saturating_sub(last_seen) <= ttl_secs + }); + } +} + +impl TlsFingerprintEntry { + fn new(now_epoch_secs: u64, total: u64, auth_success: u64, bad_or_probe: u64) -> Self { + Self { + first_seen_epoch_secs: AtomicU64::new(now_epoch_secs), + last_seen_epoch_secs: AtomicU64::new(now_epoch_secs), + total: AtomicU64::new(total), + auth_success: AtomicU64::new(auth_success), + bad_or_probe: AtomicU64::new(bad_or_probe), + } + } +} + +fn update_entry( + entry: &TlsFingerprintEntry, + now_epoch_secs: u64, + count_total: bool, + count_auth_success: bool, + count_bad_or_probe: bool, +) { + entry + .last_seen_epoch_secs + .store(now_epoch_secs, Ordering::Relaxed); + if count_total { + entry.total.fetch_add(1, Ordering::Relaxed); + } + if count_auth_success { + entry.auth_success.fetch_add(1, Ordering::Relaxed); + } + if count_bad_or_probe { + entry.bad_or_probe.fetch_add(1, Ordering::Relaxed); + } +} + +fn snapshot_row(key: &TlsFingerprintKey, entry: &TlsFingerprintEntry) -> TlsFingerprintSnapshotRow { + TlsFingerprintSnapshotRow { + scope_key: key.scope_key.clone(), + ja3: key.ja3.clone(), + ja3_raw: key.ja3_raw.clone(), + ja4: key.ja4.clone(), + ja4_raw: key.ja4_raw.clone(), + total: entry.total.load(Ordering::Relaxed), + auth_success: entry.auth_success.load(Ordering::Relaxed), + bad_or_probe: entry.bad_or_probe.load(Ordering::Relaxed), + first_seen_epoch_secs: entry.first_seen_epoch_secs.load(Ordering::Relaxed), + last_seen_epoch_secs: entry.last_seen_epoch_secs.load(Ordering::Relaxed), + } +} + +fn sort_and_truncate(rows: &mut Vec, limit: usize) { + rows.sort_by_key(|row| { + ( + Reverse(row.total), + row.scope_key.clone(), + row.ja4.clone(), + row.ja3.clone(), + ) + }); + rows.truncate(limit); +} + +fn append_rows(out: &mut String, section: &str, rows: &[TlsFingerprintSnapshotRow]) { + if rows.is_empty() { + return; + } + out.push('['); + out.push_str(section); + out.push_str("]\n"); + for row in rows { + if row.scope_key.is_empty() { + out.push_str(&format!( + "ja4={} ja3={} total={} auth_success={} bad_or_probe={} first_seen={} last_seen={}\n", + row.ja4, + row.ja3, + row.total, + row.auth_success, + row.bad_or_probe, + row.first_seen_epoch_secs, + row.last_seen_epoch_secs + )); + } else { + out.push_str(&format!( + "scope={} ja4={} ja3={} total={} auth_success={} bad_or_probe={} first_seen={} last_seen={}\n", + row.scope_key, + row.ja4, + row.ja3, + row.total, + row.auth_success, + row.bad_or_probe, + row.first_seen_epoch_secs, + row.last_seen_epoch_secs + )); + } + } +} + +fn scope_key(kind: TlsFingerprintScopeKind, key: &str) -> (TlsFingerprintScopeKind, String) { + (kind, key.to_string()) +} + +fn cidr_bucket(ip: IpAddr) -> String { + match ip { + IpAddr::V4(ip) => { + let [a, b, c, _] = ip.octets(); + format!("{a}.{b}.{c}.0/24") + } + IpAddr::V6(ip) => { + let mut octets = ip.octets(); + for byte in &mut octets[7..] { + *byte = 0; + } + format!("{}/56", Ipv6Addr::from(octets)) + } + } +} + +fn now_epoch_secs() -> u64 { + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap_or_default() + .as_secs() +} + +impl Stats { + pub fn record_tls_fingerprint_observed( + &self, + fingerprint: &TlsClientFingerprint, + peer_ip: IpAddr, + ttl: Duration, + ) { + if self.telemetry_core_enabled() { + self.tls_fingerprints + .record_observed(fingerprint, peer_ip, ttl); + } + } + + pub fn record_tls_fingerprint_auth_success( + &self, + fingerprint: &TlsClientFingerprint, + peer_ip: IpAddr, + user: &str, + ttl: Duration, + ) { + if self.telemetry_core_enabled() { + self.tls_fingerprints + .record_auth_success(fingerprint, peer_ip, user, ttl); + } + } + + pub fn record_tls_fingerprint_bad_or_probe( + &self, + fingerprint: &TlsClientFingerprint, + peer_ip: IpAddr, + ttl: Duration, + ) { + if self.telemetry_core_enabled() { + self.tls_fingerprints + .record_bad_or_probe(fingerprint, peer_ip, ttl); + } + } + + pub fn increment_tls_fingerprint_parse_error(&self) { + if self.telemetry_core_enabled() { + self.tls_fingerprints.increment_parse_error(); + } + } + + pub fn tls_fingerprint_snapshot(&self, ttl: Duration, limit: usize) -> TlsFingerprintSnapshot { + self.tls_fingerprints.snapshot(ttl, limit) + } + + pub fn tls_fingerprint_snapshot_text(&self, ttl: Duration, limit: usize) -> String { + self.tls_fingerprints.snapshot_text(ttl, limit) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn fp() -> TlsClientFingerprint { + TlsClientFingerprint { + ja3: "ja3".to_string(), + ja3_raw: "771,4865,,,0".to_string(), + ja4: "t13d010100_hash_hash".to_string(), + ja4_raw: "raw".to_string(), + } + } + + #[test] + fn aggregates_ip_cidr_and_user_scopes() { + let collector = TlsFingerprintCollector::default(); + let ip: IpAddr = "192.0.2.15".parse().expect("test IP parses"); + collector.record_observed(&fp(), ip, Duration::from_secs(60)); + collector.record_auth_success(&fp(), ip, "alice", Duration::from_secs(60)); + let snapshot = collector.snapshot(Duration::from_secs(60), 10); + + assert_eq!(snapshot.by_fingerprint[0].total, 1); + assert_eq!(snapshot.by_fingerprint[0].auth_success, 1); + assert_eq!(snapshot.by_ip[0].scope_key, "192.0.2.15"); + assert_eq!(snapshot.by_cidr[0].scope_key, "192.0.2.0/24"); + assert_eq!(snapshot.by_user[0].scope_key, "alice"); + assert_eq!(snapshot.by_user[0].total, 1); + } +} diff --git a/src/transport/upstream.rs b/src/transport/upstream.rs index 229dfbf..f741d02 100644 --- a/src/transport/upstream.rs +++ b/src/transport/upstream.rs @@ -169,6 +169,7 @@ pub struct StartupPingResult { pub v6_results: Vec, pub v4_results: Vec, pub upstream_name: String, + pub prefer_ipv6: bool, /// True if both IPv6 and IPv4 have at least one working DC pub both_available: bool, } @@ -313,8 +314,8 @@ pub struct UpstreamEgressInfo { #[derive(Debug, Clone)] struct HealthCheckGroup { dc_idx: i16, - primary: Vec, - fallback: Vec, + v4_endpoints: Vec, + v6_endpoints: Vec, } // ============= Upstream Manager ============= @@ -532,6 +533,31 @@ impl UpstreamManager { dc_preference: IpPreference, ) -> Result { let (allow_ipv4, allow_ipv6) = Self::resolve_runtime_dc_families(upstream, dc_preference); + let preferred_ipv6 = match dc_preference { + IpPreference::PreferV6 => Some(true), + IpPreference::PreferV4 => Some(false), + IpPreference::BothWork | IpPreference::Unknown | IpPreference::Unavailable => { + upstream.prefer.map(|prefer| prefer == 6) + } + }; + if let Some(preferred_ipv6) = preferred_ipv6 + && target.is_ipv6() != preferred_ipv6 + { + let preferred_allowed = if preferred_ipv6 { + allow_ipv6 + } else { + allow_ipv4 + }; + if preferred_allowed { + if let Some(dc_idx) = dc_idx + && let Some(remapped) = + Self::dc_table_addr(dc_idx, preferred_ipv6, target.port()) + { + return Ok(remapped); + } + } + } + if (target.is_ipv4() && allow_ipv4) || (target.is_ipv6() && allow_ipv6) { return Ok(target); } @@ -1327,7 +1353,7 @@ impl UpstreamManager { /// Tests BOTH IPv6 and IPv4, returns separate results for each. pub async fn ping_all_dcs( &self, - _prefer_ipv6: bool, + prefer_ipv6: bool, dc_overrides: &HashMap>, ipv4_enabled: bool, ipv6_enabled: bool, @@ -1355,6 +1381,7 @@ impl UpstreamManager { let (upstream_ipv4_enabled, upstream_ipv6_enabled) = Self::resolve_probe_dc_families(upstream_config, ipv4_enabled, ipv6_enabled); + let upstream_prefer_ipv6 = upstream_config.prefer_ipv6(prefer_ipv6); let upstream_name = match &upstream_config.upstream_type { UpstreamType::Direct { interface, @@ -1600,6 +1627,7 @@ impl UpstreamManager { v6_results, v4_results, upstream_name, + prefer_ipv6: upstream_prefer_ipv6, both_available, }); } @@ -1636,7 +1664,6 @@ impl UpstreamManager { } fn build_health_check_groups( - prefer_ipv6: bool, ipv4_enabled: bool, ipv6_enabled: bool, dc_overrides: &HashMap>, @@ -1713,26 +1740,32 @@ impl UpstreamManager { for dc_idx in all_dcs { let v4_endpoints = v4_by_dc.remove(&dc_idx).unwrap_or_default(); let v6_endpoints = v6_by_dc.remove(&dc_idx).unwrap_or_default(); - let (primary, fallback) = if prefer_ipv6 { - (v6_endpoints, v4_endpoints) - } else { - (v4_endpoints, v6_endpoints) - }; - if primary.is_empty() && fallback.is_empty() { + if v4_endpoints.is_empty() && v6_endpoints.is_empty() { continue; } groups.push(HealthCheckGroup { dc_idx, - primary, - fallback, + v4_endpoints, + v6_endpoints, }); } groups } + fn health_check_endpoint_order( + group: &HealthCheckGroup, + prefer_ipv6: bool, + ) -> [(bool, &[SocketAddr]); 2] { + if prefer_ipv6 { + [(true, &group.v6_endpoints), (false, &group.v4_endpoints)] + } else { + [(true, &group.v4_endpoints), (false, &group.v6_endpoints)] + } + } + // ============= Health Checks ============= /// Background health check based on reachable DC groups through each upstream. @@ -1744,8 +1777,24 @@ impl UpstreamManager { ipv6_enabled: bool, dc_overrides: HashMap>, ) { - let groups = - Self::build_health_check_groups(prefer_ipv6, ipv4_enabled, ipv6_enabled, &dc_overrides); + let (health_ipv4_enabled, health_ipv6_enabled) = { + let guard = self.upstreams.read().await; + ( + ipv4_enabled + || guard + .iter() + .any(|upstream| upstream.config.ipv4 == Some(true)), + ipv6_enabled + || guard + .iter() + .any(|upstream| upstream.config.ipv6 == Some(true)), + ) + }; + let groups = Self::build_health_check_groups( + health_ipv4_enabled, + health_ipv6_enabled, + &dc_overrides, + ); let required_healthy_groups = Self::required_healthy_group_count(groups.len()); let mut endpoint_rotation: HashMap<(usize, i16, bool), usize> = HashMap::new(); @@ -1786,6 +1835,7 @@ impl UpstreamManager { }; let (upstream_ipv4_enabled, upstream_ipv6_enabled) = Self::resolve_probe_dc_families(&config, ipv4_enabled, ipv6_enabled); + let upstream_prefer_ipv6 = config.prefer_ipv6(prefer_ipv6); let mut healthy_groups = 0usize; let mut latency_updates: Vec<(usize, f64)> = Vec::new(); @@ -1795,7 +1845,7 @@ impl UpstreamManager { let mut group_rtt_ms = None; for (is_primary, endpoints) in - [(true, &group.primary), (false, &group.fallback)] + Self::health_check_endpoint_order(group, upstream_prefer_ipv6) { if endpoints.is_empty() { continue; @@ -1990,26 +2040,30 @@ mod tests { ], ); - let groups = UpstreamManager::build_health_check_groups(true, true, true, &overrides); + let groups = UpstreamManager::build_health_check_groups(true, true, &overrides); let dc2 = groups .iter() .find(|g| g.dc_idx == 2) .expect("dc2 must be present"); - assert!(dc2.primary.iter().all(|addr| addr.is_ipv6())); - assert!(dc2.fallback.iter().all(|addr| addr.is_ipv4())); + assert!(dc2.v6_endpoints.iter().all(|addr| addr.is_ipv6())); + assert!(dc2.v4_endpoints.iter().all(|addr| addr.is_ipv4())); assert!( - dc2.primary + dc2.v6_endpoints .contains(&"[2001:db8::10]:443".parse::().unwrap()) ); assert!( - dc2.fallback + dc2.v4_endpoints .contains(&"203.0.113.10:443".parse::().unwrap()) ); assert!( - dc2.fallback + dc2.v4_endpoints .contains(&"203.0.113.11:443".parse::().unwrap()) ); + + let ordered = UpstreamManager::health_check_endpoint_order(dc2, true); + assert!(ordered[0].1.iter().all(|addr| addr.is_ipv6())); + assert!(ordered[1].1.iter().all(|addr| addr.is_ipv4())); } #[test] @@ -2024,22 +2078,22 @@ mod tests { ], ); - let groups = UpstreamManager::build_health_check_groups(false, true, false, &overrides); + let groups = UpstreamManager::build_health_check_groups(true, false, &overrides); let dc9 = groups .iter() .find(|g| g.dc_idx == 9) .expect("override-only dc group must be present"); - assert_eq!(dc9.primary.len(), 2); + assert_eq!(dc9.v4_endpoints.len(), 2); assert!( - dc9.primary + dc9.v4_endpoints .contains(&"198.51.100.1:443".parse::().unwrap()) ); assert!( - dc9.primary + dc9.v4_endpoints .contains(&"198.51.100.2:443".parse::().unwrap()) ); - assert!(dc9.fallback.is_empty()); + assert!(dc9.v6_endpoints.is_empty()); } #[test] @@ -2072,6 +2126,7 @@ mod tests { selected_scope: String::new(), ipv4: None, ipv6: None, + prefer: None, }; assert!(UpstreamManager::is_unscoped_upstream(&upstream)); @@ -2127,6 +2182,7 @@ mod tests { selected_scope: String::new(), ipv4: None, ipv6: None, + prefer: None, }], 1, 100,