{ let websocket_upstream_h2 = !websocket_legacy_support && handshake.as_ref().is_some_and(|handshake| handshake.h2()) && matches!(backend.protocol, crate::config::UpstreamProtocol::Http2 | crate::config::UpstreamProtocol::H2); if let Some(handshake) = &handshake { request.headers_mut().remove("content-length"); request.headers_mut().remove("transfer-encoding"); if websocket_upstream_h2 { request.headers_mut().remove("connection"); request.headers_mut().remove("upgrade"); request.headers_mut().remove("sec-websocket-key"); request.headers_mut().remove("sec-websocket-accept"); } else { request .headers_mut() .insert("connection", HeaderValue::from_static("Upgrade")); request .headers_mut() .insert("upgrade", HeaderValue::from_static("websocket")); if !handshake.h1() { *request.method_mut() = Method::GET; request.headers_mut().remove("sec-websocket-key"); request.headers_mut().remove("sec-websocket-accept"); if let Some(key) = handshake.upstream_key() { if let Ok(key) = HeaderValue::from_str(key) { request.headers_mut().insert("sec-websocket-key", key); } else { return error( StatusCode::INTERNAL_SERVER_ERROR, "RustProxy: failed to build WebSocket handshake\n", ); } } } } } if let Some(auth) = frontend.and_then(|f| f.basic_auth.as_ref()) { auth.apply_upstream(request.headers_mut(), basic_identity); } if range_requested || transform_settings.is_some_and(|settings| settings.enabled) { request.headers_mut().remove("accept-encoding"); } let mut revalidation_sent = false; if pending_revalidation { if let Some(cached) = &cached_revalidation { revalidation_sent = cached.apply_validators(request.headers_mut()); } } if request.method() == Method::HEAD && cache_ticket.is_some() { *request.method_mut() = Method::GET; } let retry_method = request.method().clone(); let retry_policy = route.pool.as_ref().map(|pool| &pool.settings.retry); let pool_retry_possible = route.backend_id.is_none() && !admin_control_plane && handshake.is_none() && retry_safe_method(&retry_method) && route.pool.as_ref().is_some_and(|pool| pool.members.len() > 1) && retry_policy.is_some_and(|policy| policy.max_retries > 0); let h2_replay_possible = !admin_control_plane && handshake.is_none() && retry_safe_method(&retry_method) && matches!(backend.protocol, crate::config::UpstreamProtocol::Http2 | crate::config::UpstreamProtocol::H2 | crate::config::UpstreamProtocol::Auto); let replay_headers = ((pool_retry_possible || h2_replay_possible) && (request.body().size_hint().exact() == Some(0) || request.version() == Version::HTTP_3)) .then(|| request.headers().clone()); let expected_backend_host = if backend.preserve_host { &original_host } else { &backend.host_header }; let retry_host_follows_backend = replay_headers.as_ref() .and_then(|headers| headers.get(HOST)) .is_some_and(|value| value == expected_backend_host); let Some(upstream_target) = upstream_request_target( uri, request.headers_mut(), &backend, websocket_legacy_support && !websocket_upstream_h2, websocket_upstream_h2, ) else { return error(StatusCode::BAD_GATEWAY, "RustProxy: invalid upstream authority\n"); }; *request.uri_mut() = upstream_target.clone(); let upstream_version = if websocket_legacy_support && !websocket_upstream_h2 { Version::HTTP_11 } else { upstream_request_version(&backend, websocket_upstream_h2) }; if normalize_h3_h1_unknown_body(&mut request, upstream_version) .await .is_err() { return error( StatusCode::BAD_REQUEST, "RustProxy: invalid or incomplete request body\n", ); } *request.version_mut() = upstream_version; let mut h2_direct_session = None; if websocket_upstream_h2 { let Some(exchange) = exchange.clone() else { return error( StatusCode::INTERNAL_SERVER_ERROR, "RustProxy: WebSocket exchange state unavailable\n", ); }; let stop_token = stop .as_ref() .map(|value| value.0.clone()) .unwrap_or_else(|| state.shutdown.clone()); let session = state.tunnel_registry.register_pending( exchange.clone(), backend.origin.clone(), "h2-rfc8441-h2-upstream", stop_token, lease.clone(), ); websocket::spawn_h2_idle_watch(state.clone(), &session); let body = std::mem::replace(request.body_mut(), Body::empty()); *request.body_mut() = websocket::wrap_h2_request_body(body, session.clone(), exchange); h2_direct_session = Some(session); } let request_body_empty = request.body().size_hint().exact() == Some(0); let proactive_base_headers = if cache_ticket.is_some() && cache_settings.proactive_refresh_enabled { request.headers().clone() } else { HeaderMap::new() }; let response_header_timeout = frontend .and_then(|frontend| frontend.proxy_timeouts.response_header_timeout()) .unwrap_or(state.header_timeout); let retry_enabled = pool_retry_possible && request_body_empty && replay_headers.is_some(); let upstream_started = Instant::now(); let mut attempt_started = Instant::now(); let mut active_backend = backend.clone(); let mut active_lease = backend_lease.clone(); drop(backend_lease); let mut active_affinity_cookie = affinity_cookie.clone(); let mut active_uri = upstream_target; let mut request_attempt = Some(request); let mut attempted_backends = Vec::::new(); let mut retries = 0u8; let mut graceful_retry_done = false; let upstream_result = loop { let client = if handshake.is_some() && !websocket_upstream_h2 { &active_backend.websocket_client } else { &active_backend.client }; let result = tokio::time::timeout( response_header_timeout, client.request(request_attempt.take().expect("upstream request attempt missing")), ) .await; let attempt_elapsed = attempt_started.elapsed(); if let Ok(Err(cause)) = &result { if !graceful_retry_done && !admin_control_plane && handshake.is_none() && request_body_empty && retry_safe_method(&retry_method) && (is_graceful_h2_goaway(cause) || is_related_h2_close(cause, &active_backend)) { if let Some(next_request) = replay_headers.as_ref().and_then(|headers| retry_request( &retry_method, upstream_request_version(&active_backend, false), headers, &active_backend, &original_host, &origin_suffix, retry_host_follows_backend, )) { graceful_retry_done = true; if let Some(lease) = &active_lease { lease.retry_attempted(); } tracing::debug!(backend = %active_backend.id, "retrying safe empty request after graceful upstream H2 shutdown"); request_attempt = Some(next_request); attempt_started = Instant::now(); continue; } } } let Some(policy) = retry_policy.as_ref() else { break result; }; let retry_reason = match &result { Ok(Ok(response)) if policy.retries_status(response.status().as_u16()) => { Some("http_status") } Ok(Err(cause)) if cause.is_connect() && policy.retry_connect_errors => Some("connect"), Err(_) if policy.retry_timeouts => Some("timeout"), _ => None, }; if !retry_enabled || retries >= policy.max_retries || retry_reason.is_none() { break result; } attempted_backends.push(active_backend.id.clone()); let Some(retry_headers) = replay_headers.as_ref() else { break result; }; let Some(pool) = route.pool.as_ref() else { break result; }; let selection = match pool .select_retry_wait( context.client_ip, &retry_headers, tls, &attempted_backends, ) .await { Ok(selection) => selection, Err(crate::balance::SelectionFailure::Capacity) => { tracing::debug!( vhost = %pool.id, attempt = retries, "retry stopped because all remaining backends are at concurrency capacity" ); break result; } Err(crate::balance::SelectionFailure::Unavailable) => break result, }; let next_backend = selection.lease.member.backend.clone(); let Some(next_uri) = upstream_uri(&next_backend, &origin_suffix) else { break result; }; let Some(next_request) = retry_request( &retry_method, upstream_request_version(&next_backend, false), &retry_headers, &next_backend, &original_host, &origin_suffix, retry_host_follows_backend, ) else { break result; }; if let Some(lease) = &active_lease { match &result { Ok(Ok(response)) => { lease.response(response.status().as_u16(), attempt_elapsed, true) } Ok(Err(cause)) => { let connect_failure = cause.is_connect(); let detail = upstream_failure_detail( cause, &active_backend, attempt_elapsed, connect_failure, ); lease.error_detail( upstream_health_error_class(cause, connect_failure, &active_backend), &detail, attempt_elapsed, ); } Err(_) => lease.error_detail( "response_header_timeout", "response_header_timeout", attempt_elapsed, ), } lease.retry_attempted(); } retries = retries.saturating_add(1); selection.lease.started(); if let Some(exchange) = &exchange { *exchange.backend.lock().unwrap_or_else(|e| e.into_inner()) = Some(selection.lease.clone()); } tracing::info!( previous_backend = %active_backend.id, backend = %next_backend.id, attempt = retries, reason = retry_reason.unwrap_or("unknown"), "retrying replay-safe request on another backend" ); active_backend = next_backend; active_lease = Some(selection.lease); active_affinity_cookie = selection.cookie; active_uri = next_uri; request_attempt = Some(next_request); attempt_started = Instant::now(); }; let final_attempt_elapsed = attempt_started.elapsed(); ( websocket_upstream_h2, revalidation_sent, h2_direct_session, request_body_empty, proactive_base_headers, upstream_started, active_backend, active_lease, active_affinity_cookie, active_uri, retries, upstream_result, final_attempt_elapsed, ) }