Dual-stack fixes for Upstreams by #798

Co-Authored-By: brekotis <93345790+brekotis@users.noreply.github.com>
This commit is contained in:
Alexey
2026-06-01 19:50:26 +03:00
parent 2264980926
commit 462215b53c
28 changed files with 186 additions and 27 deletions
+3
View File
@@ -705,6 +705,9 @@ ignore_time_skew = false
type = "direct"
enabled = true
weight = 10
# Optional per-upstream DC family policy:
# ipv6 = true
# prefer = 6
"#,
username = username,
secret = secret,
+30
View File
@@ -1007,6 +1007,14 @@ fn validate_upstreams(config: &ProxyConfig) -> Result<()> {
"upstream.ipv4 and upstream.ipv6 cannot both be false".to_string(),
));
}
if let Some(prefer) = upstream.prefer
&& prefer != 4
&& prefer != 6
{
return Err(ProxyError::Config(
"upstream.prefer must be 4 or 6".to_string(),
));
}
if let UpstreamType::Shadowsocks { url, .. } = &upstream.upstream_type {
let parsed = ShadowsocksServerConfig::from_url(url)
@@ -1022,6 +1030,26 @@ fn validate_upstreams(config: &ProxyConfig) -> Result<()> {
Ok(())
}
fn normalize_upstream_family_policy(config: &mut ProxyConfig) {
for (idx, upstream) in config.upstreams.iter_mut().enumerate() {
if matches!(upstream.ipv4, Some(false)) && upstream.prefer == Some(4) {
warn!(
upstream = idx,
"upstream.prefer=4 but upstream.ipv4=false; forcing prefer=6"
);
upstream.prefer = Some(6);
}
if matches!(upstream.ipv6, Some(false)) && upstream.prefer == Some(6) {
warn!(
upstream = idx,
"upstream.prefer=6 but upstream.ipv6=false; forcing prefer=4"
);
upstream.prefer = Some(4);
}
}
}
// ============= Main Config =============
#[derive(Debug, Clone, Serialize, Deserialize, Default)]
@@ -2200,8 +2228,10 @@ impl ProxyConfig {
selected_scope: String::new(),
ipv4: None,
ipv6: None,
prefer: None,
});
}
normalize_upstream_family_policy(&mut config);
// Ensure default DC203 override is present.
config
+14
View File
@@ -2065,6 +2065,20 @@ pub struct UpstreamConfig {
/// `None` means auto-detect from runtime connectivity state.
#[serde(default)]
pub ipv6: Option<bool>,
/// Per-upstream IP family preference for Telegram DC targets.
/// `None` inherits the effective global `[network].prefer` decision.
#[serde(default)]
pub prefer: Option<u8>,
}
impl UpstreamConfig {
pub fn prefer_ipv6(&self, default_prefer_ipv6: bool) -> bool {
match self.prefer {
Some(6) => true,
Some(4) => false,
_ => default_prefer_ipv6,
}
}
}
#[derive(Debug, Clone, Serialize, Deserialize)]
+1 -1
View File
@@ -147,7 +147,7 @@ pub(crate) async fn run_startup_connectivity(
.any(|r| r.rtt_ms.is_some());
if upstream_result.both_available {
if prefer_ipv6 {
if upstream_result.prefer_ipv6 {
info!(" IPv6 in use / IPv4 is fallback");
} else {
info!(" IPv4 in use / IPv6 is fallback");
@@ -39,6 +39,7 @@ fn new_upstream_manager(stats: Arc<Stats>) -> Arc<UpstreamManager> {
selected_scope: String::new(),
ipv4: None,
ipv6: None,
prefer: None,
}],
1,
1,
@@ -35,6 +35,7 @@ fn build_harness(config: ProxyConfig) -> PipelineHarness {
selected_scope: String::new(),
ipv4: None,
ipv6: None,
prefer: None,
}],
1,
1,
@@ -19,6 +19,7 @@ fn new_upstream_manager(stats: Arc<Stats>) -> Arc<UpstreamManager> {
selected_scope: String::new(),
ipv4: None,
ipv6: None,
prefer: None,
}],
1,
1,
@@ -19,6 +19,7 @@ fn new_upstream_manager(stats: Arc<Stats>) -> Arc<UpstreamManager> {
selected_scope: String::new(),
ipv4: None,
ipv6: None,
prefer: None,
}],
1,
1,
@@ -33,6 +33,7 @@ fn new_upstream_manager(stats: Arc<Stats>) -> Arc<UpstreamManager> {
selected_scope: String::new(),
ipv4: None,
ipv6: None,
prefer: None,
}],
1,
1,
@@ -19,6 +19,7 @@ fn new_upstream_manager(stats: Arc<Stats>) -> Arc<UpstreamManager> {
selected_scope: String::new(),
ipv4: None,
ipv6: None,
prefer: None,
}],
1,
1,
@@ -19,6 +19,7 @@ fn new_upstream_manager(stats: Arc<Stats>) -> Arc<UpstreamManager> {
selected_scope: String::new(),
ipv4: None,
ipv6: None,
prefer: None,
}],
1,
1,
@@ -46,6 +46,7 @@ fn build_harness(secret_hex: &str, mask_port: u16) -> PipelineHarness {
selected_scope: String::new(),
ipv4: None,
ipv6: None,
prefer: None,
}],
1,
1,
@@ -24,6 +24,7 @@ fn make_test_upstream_manager(stats: Arc<Stats>) -> Arc<UpstreamManager> {
selected_scope: String::new(),
ipv4: None,
ipv6: None,
prefer: None,
}],
1,
1,
@@ -47,6 +47,7 @@ fn build_harness(secret_hex: &str, mask_port: u16) -> RedTeamHarness {
selected_scope: String::new(),
ipv4: None,
ipv6: None,
prefer: None,
}],
1,
1,
@@ -240,6 +241,7 @@ async fn redteam_03_masking_duration_must_be_less_than_1ms_when_backend_down() {
selected_scope: String::new(),
ipv4: None,
ipv6: None,
prefer: None,
}],
1,
1,
@@ -484,6 +486,7 @@ async fn measure_invalid_probe_duration_ms(delay_ms: u64, tls_len: u16, body_sen
selected_scope: String::new(),
ipv4: None,
ipv6: None,
prefer: None,
}],
1,
1,
@@ -561,6 +564,7 @@ async fn capture_forwarded_probe_len(tls_len: u16, body_sent: usize) -> usize {
selected_scope: String::new(),
ipv4: None,
ipv6: None,
prefer: None,
}],
1,
1,
@@ -21,6 +21,7 @@ fn new_upstream_manager(stats: Arc<Stats>) -> Arc<UpstreamManager> {
selected_scope: String::new(),
ipv4: None,
ipv6: None,
prefer: None,
}],
1,
1,
@@ -19,6 +19,7 @@ fn new_upstream_manager(stats: Arc<Stats>) -> Arc<UpstreamManager> {
selected_scope: String::new(),
ipv4: None,
ipv6: None,
prefer: None,
}],
1,
1,
@@ -19,6 +19,7 @@ fn new_upstream_manager(stats: Arc<Stats>) -> Arc<UpstreamManager> {
selected_scope: String::new(),
ipv4: None,
ipv6: None,
prefer: None,
}],
1,
1,
@@ -19,6 +19,7 @@ fn new_upstream_manager(stats: Arc<Stats>) -> Arc<UpstreamManager> {
selected_scope: String::new(),
ipv4: None,
ipv6: None,
prefer: None,
}],
1,
1,
@@ -19,6 +19,7 @@ fn new_upstream_manager(stats: Arc<Stats>) -> Arc<UpstreamManager> {
selected_scope: String::new(),
ipv4: None,
ipv6: None,
prefer: None,
}],
1,
1,
@@ -33,6 +33,7 @@ fn new_upstream_manager(stats: Arc<Stats>) -> Arc<UpstreamManager> {
selected_scope: String::new(),
ipv4: None,
ipv6: None,
prefer: None,
}],
1,
1,
+27
View File
@@ -341,6 +341,7 @@ async fn relay_task_abort_releases_user_gate_and_ip_reservation() {
selected_scope: String::new(),
ipv4: None,
ipv6: None,
prefer: None,
}],
1,
1,
@@ -459,6 +460,7 @@ async fn relay_cutover_releases_user_gate_and_ip_reservation() {
selected_scope: String::new(),
ipv4: None,
ipv6: None,
prefer: None,
}],
1,
1,
@@ -586,6 +588,7 @@ async fn integration_route_cutover_and_quota_overlap_fails_closed_and_releases_s
selected_scope: String::new(),
ipv4: None,
ipv6: None,
prefer: None,
}],
1,
1,
@@ -759,6 +762,7 @@ async fn proxy_protocol_header_is_rejected_when_trust_list_is_empty() {
selected_scope: String::new(),
ipv4: None,
ipv6: None,
prefer: None,
}],
1,
1,
@@ -839,6 +843,7 @@ async fn proxy_protocol_header_from_untrusted_peer_range_is_rejected_under_load(
selected_scope: String::new(),
ipv4: None,
ipv6: None,
prefer: None,
}],
1,
1,
@@ -1032,6 +1037,7 @@ async fn short_tls_probe_is_masked_through_client_pipeline() {
selected_scope: String::new(),
ipv4: None,
ipv6: None,
prefer: None,
}],
1,
1,
@@ -1123,6 +1129,7 @@ async fn tls12_record_probe_is_masked_through_client_pipeline() {
selected_scope: String::new(),
ipv4: None,
ipv6: None,
prefer: None,
}],
1,
1,
@@ -1212,6 +1219,7 @@ async fn handle_client_stream_increments_connects_all_exactly_once() {
selected_scope: String::new(),
ipv4: None,
ipv6: None,
prefer: None,
}],
1,
1,
@@ -1308,6 +1316,7 @@ async fn running_client_handler_increments_connects_all_exactly_once() {
selected_scope: String::new(),
ipv4: None,
ipv6: None,
prefer: None,
}],
1,
1,
@@ -1401,6 +1410,7 @@ async fn idle_pooled_connection_closes_cleanly_in_generic_stream_path() {
selected_scope: String::new(),
ipv4: None,
ipv6: None,
prefer: None,
}],
1,
1,
@@ -1475,6 +1485,7 @@ async fn idle_pooled_connection_closes_cleanly_in_client_handler_path() {
selected_scope: String::new(),
ipv4: None,
ipv6: None,
prefer: None,
}],
1,
1,
@@ -1564,6 +1575,7 @@ async fn partial_tls_header_stall_triggers_handshake_timeout() {
selected_scope: String::new(),
ipv4: None,
ipv6: None,
prefer: None,
}],
1,
1,
@@ -1892,6 +1904,7 @@ async fn valid_tls_path_does_not_fall_back_to_mask_backend() {
selected_scope: String::new(),
ipv4: None,
ipv6: None,
prefer: None,
}],
1,
1,
@@ -2004,6 +2017,7 @@ async fn valid_tls_with_invalid_mtproto_falls_back_to_mask_backend() {
selected_scope: String::new(),
ipv4: None,
ipv6: None,
prefer: None,
}],
1,
1,
@@ -2114,6 +2128,7 @@ async fn client_handler_tls_bad_mtproto_is_forwarded_to_mask_backend() {
selected_scope: String::new(),
ipv4: None,
ipv6: None,
prefer: None,
}],
1,
1,
@@ -2239,6 +2254,7 @@ async fn alpn_mismatch_tls_probe_is_masked_through_client_pipeline() {
selected_scope: String::new(),
ipv4: None,
ipv6: None,
prefer: None,
}],
1,
1,
@@ -2335,6 +2351,7 @@ async fn invalid_hmac_tls_probe_is_masked_through_client_pipeline() {
selected_scope: String::new(),
ipv4: None,
ipv6: None,
prefer: None,
}],
1,
1,
@@ -2437,6 +2454,7 @@ async fn burst_invalid_tls_probes_are_masked_verbatim() {
selected_scope: String::new(),
ipv4: None,
ipv6: None,
prefer: None,
}],
1,
1,
@@ -3395,6 +3413,7 @@ async fn relay_connect_error_releases_user_and_ip_before_return() {
selected_scope: String::new(),
ipv4: None,
ipv6: None,
prefer: None,
}],
1,
1,
@@ -3963,6 +3982,7 @@ async fn untrusted_proxy_header_source_is_rejected() {
selected_scope: String::new(),
ipv4: None,
ipv6: None,
prefer: None,
}],
1,
1,
@@ -4036,6 +4056,7 @@ async fn empty_proxy_trusted_cidrs_rejects_proxy_header_by_default() {
selected_scope: String::new(),
ipv4: None,
ipv6: None,
prefer: None,
}],
1,
1,
@@ -4136,6 +4157,7 @@ async fn oversized_tls_record_is_masked_in_generic_stream_pipeline() {
selected_scope: String::new(),
ipv4: None,
ipv6: None,
prefer: None,
}],
1,
1,
@@ -4242,6 +4264,7 @@ async fn oversized_tls_record_is_masked_in_client_handler_pipeline() {
selected_scope: String::new(),
ipv4: None,
ipv6: None,
prefer: None,
}],
1,
1,
@@ -4362,6 +4385,7 @@ async fn tls_record_len_min_minus_1_is_rejected_in_generic_stream_pipeline() {
selected_scope: String::new(),
ipv4: None,
ipv6: None,
prefer: None,
}],
1,
1,
@@ -4468,6 +4492,7 @@ async fn tls_record_len_min_minus_1_is_rejected_in_client_handler_pipeline() {
selected_scope: String::new(),
ipv4: None,
ipv6: None,
prefer: None,
}],
1,
1,
@@ -4577,6 +4602,7 @@ async fn tls_record_len_16384_is_accepted_in_generic_stream_pipeline() {
selected_scope: String::new(),
ipv4: None,
ipv6: None,
prefer: None,
}],
1,
1,
@@ -4681,6 +4707,7 @@ async fn tls_record_len_16384_is_accepted_in_client_handler_pipeline() {
selected_scope: String::new(),
ipv4: None,
ipv6: None,
prefer: None,
}],
1,
1,
@@ -32,6 +32,7 @@ fn make_test_upstream_manager(stats: Arc<Stats>) -> Arc<UpstreamManager> {
selected_scope: String::new(),
ipv4: None,
ipv6: None,
prefer: None,
}],
1,
1,
@@ -34,6 +34,7 @@ fn make_test_upstream_manager(stats: Arc<Stats>) -> Arc<UpstreamManager> {
selected_scope: String::new(),
ipv4: None,
ipv6: None,
prefer: None,
}],
1,
1,
@@ -35,6 +35,7 @@ fn make_test_upstream_manager(stats: Arc<Stats>) -> Arc<UpstreamManager> {
selected_scope: String::new(),
ipv4: None,
ipv6: None,
prefer: None,
}],
1,
1,
@@ -49,6 +49,7 @@ fn build_harness(secret_hex: &str, mask_port: u16) -> PipelineHarness {
selected_scope: String::new(),
ipv4: None,
ipv6: None,
prefer: None,
}],
1,
1,
@@ -1338,6 +1338,7 @@ async fn direct_relay_abort_midflight_releases_route_gauge() {
selected_scope: String::new(),
ipv4: None,
ipv6: None,
prefer: None,
}],
1,
1,
@@ -1448,6 +1449,7 @@ async fn direct_relay_cutover_midflight_releases_route_gauge() {
selected_scope: String::new(),
ipv4: None,
ipv6: None,
prefer: None,
}],
1,
1,
@@ -1570,6 +1572,7 @@ async fn direct_relay_cutover_storm_multi_session_keeps_generic_errors_and_relea
selected_scope: String::new(),
ipv4: None,
ipv6: None,
prefer: None,
}],
1,
1,
@@ -1803,6 +1806,7 @@ async fn negative_direct_relay_dc_connection_refused_fails_fast() {
selected_scope: String::new(),
ipv4: None,
ipv6: None,
prefer: None,
}],
1,
100,
@@ -1897,6 +1901,7 @@ async fn adversarial_direct_relay_cutover_integrity() {
selected_scope: String::new(),
ipv4: None,
ipv6: None,
prefer: None,
}],
1,
100,
@@ -61,6 +61,7 @@ fn new_client_harness() -> ClientHarness {
selected_scope: String::new(),
ipv4: None,
ipv6: None,
prefer: None,
}],
1,
1,
+82 -26
View File
@@ -169,6 +169,7 @@ pub struct StartupPingResult {
pub v6_results: Vec<DcPingResult>,
pub v4_results: Vec<DcPingResult>,
pub upstream_name: String,
pub prefer_ipv6: bool,
/// True if both IPv6 and IPv4 have at least one working DC
pub both_available: bool,
}
@@ -313,8 +314,8 @@ pub struct UpstreamEgressInfo {
#[derive(Debug, Clone)]
struct HealthCheckGroup {
dc_idx: i16,
primary: Vec<SocketAddr>,
fallback: Vec<SocketAddr>,
v4_endpoints: Vec<SocketAddr>,
v6_endpoints: Vec<SocketAddr>,
}
// ============= Upstream Manager =============
@@ -532,6 +533,31 @@ impl UpstreamManager {
dc_preference: IpPreference,
) -> Result<SocketAddr> {
let (allow_ipv4, allow_ipv6) = Self::resolve_runtime_dc_families(upstream, dc_preference);
let preferred_ipv6 = match dc_preference {
IpPreference::PreferV6 => Some(true),
IpPreference::PreferV4 => Some(false),
IpPreference::BothWork | IpPreference::Unknown | IpPreference::Unavailable => {
upstream.prefer.map(|prefer| prefer == 6)
}
};
if let Some(preferred_ipv6) = preferred_ipv6
&& target.is_ipv6() != preferred_ipv6
{
let preferred_allowed = if preferred_ipv6 {
allow_ipv6
} else {
allow_ipv4
};
if preferred_allowed {
if let Some(dc_idx) = dc_idx
&& let Some(remapped) =
Self::dc_table_addr(dc_idx, preferred_ipv6, target.port())
{
return Ok(remapped);
}
}
}
if (target.is_ipv4() && allow_ipv4) || (target.is_ipv6() && allow_ipv6) {
return Ok(target);
}
@@ -1327,7 +1353,7 @@ impl UpstreamManager {
/// Tests BOTH IPv6 and IPv4, returns separate results for each.
pub async fn ping_all_dcs(
&self,
_prefer_ipv6: bool,
prefer_ipv6: bool,
dc_overrides: &HashMap<String, Vec<String>>,
ipv4_enabled: bool,
ipv6_enabled: bool,
@@ -1355,6 +1381,7 @@ impl UpstreamManager {
let (upstream_ipv4_enabled, upstream_ipv6_enabled) =
Self::resolve_probe_dc_families(upstream_config, ipv4_enabled, ipv6_enabled);
let upstream_prefer_ipv6 = upstream_config.prefer_ipv6(prefer_ipv6);
let upstream_name = match &upstream_config.upstream_type {
UpstreamType::Direct {
interface,
@@ -1600,6 +1627,7 @@ impl UpstreamManager {
v6_results,
v4_results,
upstream_name,
prefer_ipv6: upstream_prefer_ipv6,
both_available,
});
}
@@ -1636,7 +1664,6 @@ impl UpstreamManager {
}
fn build_health_check_groups(
prefer_ipv6: bool,
ipv4_enabled: bool,
ipv6_enabled: bool,
dc_overrides: &HashMap<String, Vec<String>>,
@@ -1713,26 +1740,32 @@ impl UpstreamManager {
for dc_idx in all_dcs {
let v4_endpoints = v4_by_dc.remove(&dc_idx).unwrap_or_default();
let v6_endpoints = v6_by_dc.remove(&dc_idx).unwrap_or_default();
let (primary, fallback) = if prefer_ipv6 {
(v6_endpoints, v4_endpoints)
} else {
(v4_endpoints, v6_endpoints)
};
if primary.is_empty() && fallback.is_empty() {
if v4_endpoints.is_empty() && v6_endpoints.is_empty() {
continue;
}
groups.push(HealthCheckGroup {
dc_idx,
primary,
fallback,
v4_endpoints,
v6_endpoints,
});
}
groups
}
fn health_check_endpoint_order(
group: &HealthCheckGroup,
prefer_ipv6: bool,
) -> [(bool, &[SocketAddr]); 2] {
if prefer_ipv6 {
[(true, &group.v6_endpoints), (false, &group.v4_endpoints)]
} else {
[(true, &group.v4_endpoints), (false, &group.v6_endpoints)]
}
}
// ============= Health Checks =============
/// Background health check based on reachable DC groups through each upstream.
@@ -1744,8 +1777,24 @@ impl UpstreamManager {
ipv6_enabled: bool,
dc_overrides: HashMap<String, Vec<String>>,
) {
let groups =
Self::build_health_check_groups(prefer_ipv6, ipv4_enabled, ipv6_enabled, &dc_overrides);
let (health_ipv4_enabled, health_ipv6_enabled) = {
let guard = self.upstreams.read().await;
(
ipv4_enabled
|| guard
.iter()
.any(|upstream| upstream.config.ipv4 == Some(true)),
ipv6_enabled
|| guard
.iter()
.any(|upstream| upstream.config.ipv6 == Some(true)),
)
};
let groups = Self::build_health_check_groups(
health_ipv4_enabled,
health_ipv6_enabled,
&dc_overrides,
);
let required_healthy_groups = Self::required_healthy_group_count(groups.len());
let mut endpoint_rotation: HashMap<(usize, i16, bool), usize> = HashMap::new();
@@ -1786,6 +1835,7 @@ impl UpstreamManager {
};
let (upstream_ipv4_enabled, upstream_ipv6_enabled) =
Self::resolve_probe_dc_families(&config, ipv4_enabled, ipv6_enabled);
let upstream_prefer_ipv6 = config.prefer_ipv6(prefer_ipv6);
let mut healthy_groups = 0usize;
let mut latency_updates: Vec<(usize, f64)> = Vec::new();
@@ -1795,7 +1845,7 @@ impl UpstreamManager {
let mut group_rtt_ms = None;
for (is_primary, endpoints) in
[(true, &group.primary), (false, &group.fallback)]
Self::health_check_endpoint_order(group, upstream_prefer_ipv6)
{
if endpoints.is_empty() {
continue;
@@ -1990,26 +2040,30 @@ mod tests {
],
);
let groups = UpstreamManager::build_health_check_groups(true, true, true, &overrides);
let groups = UpstreamManager::build_health_check_groups(true, true, &overrides);
let dc2 = groups
.iter()
.find(|g| g.dc_idx == 2)
.expect("dc2 must be present");
assert!(dc2.primary.iter().all(|addr| addr.is_ipv6()));
assert!(dc2.fallback.iter().all(|addr| addr.is_ipv4()));
assert!(dc2.v6_endpoints.iter().all(|addr| addr.is_ipv6()));
assert!(dc2.v4_endpoints.iter().all(|addr| addr.is_ipv4()));
assert!(
dc2.primary
dc2.v6_endpoints
.contains(&"[2001:db8::10]:443".parse::<SocketAddr>().unwrap())
);
assert!(
dc2.fallback
dc2.v4_endpoints
.contains(&"203.0.113.10:443".parse::<SocketAddr>().unwrap())
);
assert!(
dc2.fallback
dc2.v4_endpoints
.contains(&"203.0.113.11:443".parse::<SocketAddr>().unwrap())
);
let ordered = UpstreamManager::health_check_endpoint_order(dc2, true);
assert!(ordered[0].1.iter().all(|addr| addr.is_ipv6()));
assert!(ordered[1].1.iter().all(|addr| addr.is_ipv4()));
}
#[test]
@@ -2024,22 +2078,22 @@ mod tests {
],
);
let groups = UpstreamManager::build_health_check_groups(false, true, false, &overrides);
let groups = UpstreamManager::build_health_check_groups(true, false, &overrides);
let dc9 = groups
.iter()
.find(|g| g.dc_idx == 9)
.expect("override-only dc group must be present");
assert_eq!(dc9.primary.len(), 2);
assert_eq!(dc9.v4_endpoints.len(), 2);
assert!(
dc9.primary
dc9.v4_endpoints
.contains(&"198.51.100.1:443".parse::<SocketAddr>().unwrap())
);
assert!(
dc9.primary
dc9.v4_endpoints
.contains(&"198.51.100.2:443".parse::<SocketAddr>().unwrap())
);
assert!(dc9.fallback.is_empty());
assert!(dc9.v6_endpoints.is_empty());
}
#[test]
@@ -2072,6 +2126,7 @@ mod tests {
selected_scope: String::new(),
ipv4: None,
ipv6: None,
prefer: None,
};
assert!(UpstreamManager::is_unscoped_upstream(&upstream));
@@ -2127,6 +2182,7 @@ mod tests {
selected_scope: String::new(),
ipv4: None,
ipv6: None,
prefer: None,
}],
1,
100,